13 #if defined(ARMNN_SERIALIZER) 16 #if defined(ARMNN_CAFFE_PARSER) 19 #if defined(ARMNN_TF_PARSER) 22 #if defined(ARMNN_TF_LITE_PARSER) 25 #if defined(ARMNN_ONNX_PARSER) 28 #if defined(ARMNN_TFLITE_DELEGATE) 32 #include <tensorflow/lite/builtin_ops.h> 33 #include <tensorflow/lite/c/builtin_op_data.h> 34 #include <tensorflow/lite/c/common.h> 35 #include <tensorflow/lite/optional_debug_tools.h> 36 #include <tensorflow/lite/kernels/builtin_op_kernels.h> 37 #include <tensorflow/lite/interpreter.h> 38 #include <tensorflow/lite/kernels/register.h> 42 #if defined(ARMNN_TFLITE_DELEGATE) 44 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
48 std::unique_ptr<tflite::FlatBufferModel> model = tflite::FlatBufferModel::BuildFromFile(params.
m_ModelPath.c_str());
50 auto tfLiteInterpreter = std::make_unique<Interpreter>();
51 tflite::ops::builtin::BuiltinOpResolver resolver;
53 tflite::InterpreterBuilder builder(*model, resolver);
54 builder(&tfLiteInterpreter);
55 tfLiteInterpreter->AllocateTensors();
59 std::unique_ptr<TfLiteDelegate, decltype(&armnnDelegate::TfLiteArmnnDelegateDelete)>
63 int status = tfLiteInterpreter->ModifyGraphWithDelegate(std::move(theArmnnDelegate));
64 if (status == kTfLiteError)
66 ARMNN_LOG(fatal) <<
"Could not register ArmNN TfLite Delegate to TfLiteInterpreter!";
70 std::vector<std::string> inputBindings;
73 inputBindings.push_back(inputName);
80 const size_t numInputs = inputBindings.size();
82 for(
unsigned int inputIndex = 0; inputIndex < numInputs; ++inputIndex)
84 int input = tfLiteInterpreter->inputs()[inputIndex];
85 TfLiteIntArray* inputDims = tfLiteInterpreter->tensor(input)->dims;
88 for (
unsigned int dim = 0; dim < static_cast<unsigned int>(inputDims->size); ++dim)
90 inputSize *= inputDims->data[dim];
93 if (params.
m_InputTypes[inputIndex].compare(
"float") == 0)
95 auto inputData = tfLiteInterpreter->typed_tensor<
float>(input);
99 ARMNN_LOG(fatal) <<
"Input tensor is null, input type: " 100 "\"" << params.
m_InputTypes[inputIndex] <<
"\" may be incorrect.";
104 std::vector<float> tensorData;
105 PopulateTensorWithDataGeneric<float>(tensorData,
108 [](
const std::string& s)
109 {
return std::stof(s); });
111 std::copy(tensorData.begin(), tensorData.end(), inputData);
113 else if (params.
m_InputTypes[inputIndex].compare(
"int8") == 0)
115 auto inputData = tfLiteInterpreter->typed_tensor<int8_t>(input);
117 if(inputData == NULL)
119 ARMNN_LOG(fatal) <<
"Input tensor is null, input type: " 120 "\"" << params.
m_InputTypes[inputIndex] <<
"\" may be incorrect.";
124 std::vector<int8_t> tensorData;
125 PopulateTensorWithDataGeneric<int8_t>(tensorData,
128 [](
const std::string& s)
131 std::copy(tensorData.begin(), tensorData.end(), inputData);
133 else if (params.
m_InputTypes[inputIndex].compare(
"int") == 0)
135 auto inputData = tfLiteInterpreter->typed_tensor<int32_t>(input);
137 if(inputData == NULL)
139 ARMNN_LOG(fatal) <<
"Input tensor is null, input type: " 140 "\"" << params.
m_InputTypes[inputIndex] <<
"\" may be incorrect.";
144 std::vector<int32_t> tensorData;
145 PopulateTensorWithDataGeneric<int32_t>(tensorData,
148 [](
const std::string& s)
149 {
return std::stoi(s); });
151 std::copy(tensorData.begin(), tensorData.end(), inputData);
153 else if (params.
m_InputTypes[inputIndex].compare(
"qasymm8") == 0)
155 auto inputData = tfLiteInterpreter->typed_tensor<uint8_t>(input);
157 if(inputData == NULL)
159 ARMNN_LOG(fatal) <<
"Input tensor is null, input type: " 160 "\"" << params.
m_InputTypes[inputIndex] <<
"\" may be incorrect.";
164 std::vector<uint8_t> tensorData;
165 PopulateTensorWithDataGeneric<uint8_t>(tensorData,
168 [](
const std::string& s)
171 std::copy(tensorData.begin(), tensorData.end(), inputData);
175 ARMNN_LOG(fatal) <<
"Unsupported input tensor data type \"" << params.
m_InputTypes[inputIndex] <<
"\". ";
183 tfLiteInterpreter->Invoke();
186 for (
unsigned int outputIndex = 0; outputIndex < params.
m_OutputNames.size(); ++outputIndex)
188 auto tfLiteDelegateOutputId = tfLiteInterpreter->outputs()[outputIndex];
189 TfLiteIntArray* outputDims = tfLiteInterpreter->tensor(tfLiteDelegateOutputId)->dims;
192 for (
unsigned int dim = 0; dim < static_cast<unsigned int>(outputDims->size); ++dim)
194 outputSize *= outputDims->data[dim];
200 auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<
float>(tfLiteDelegateOutputId);
201 if(tfLiteDelageOutputData == NULL)
203 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 204 "\"" << params.
m_OutputTypes[outputIndex] <<
"\" may be incorrect.";
208 for (
int i = 0; i < outputSize; ++i)
210 std::cout << tfLiteDelageOutputData[i] <<
", ";
213 std::cout << std::endl;
217 else if (params.
m_OutputTypes[outputIndex].compare(
"int") == 0)
219 auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<int32_t>(tfLiteDelegateOutputId);
220 if(tfLiteDelageOutputData == NULL)
222 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 223 "\"" << params.
m_OutputTypes[outputIndex] <<
"\" may be incorrect.";
227 for (
int i = 0; i < outputSize; ++i)
229 std::cout << tfLiteDelageOutputData[i] <<
", ";
232 std::cout << std::endl;
236 else if (params.
m_OutputTypes[outputIndex].compare(
"int8") == 0)
238 auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<int8_t>(tfLiteDelegateOutputId);
239 if(tfLiteDelageOutputData == NULL)
241 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 242 "\"" << params.
m_OutputTypes[outputIndex] <<
"\" may be incorrect.";
246 for (
int i = 0; i < outputSize; ++i)
248 std::cout << signed(tfLiteDelageOutputData[i]) <<
", ";
251 std::cout << std::endl;
255 else if (params.
m_OutputTypes[outputIndex].compare(
"qasymm8") == 0)
257 auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<uint8_t>(tfLiteDelegateOutputId);
258 if(tfLiteDelageOutputData == NULL)
260 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 261 "\"" << params.
m_OutputTypes[outputIndex] <<
"\" may be incorrect.";
265 for (
int i = 0; i < outputSize; ++i)
267 std::cout << unsigned(tfLiteDelageOutputData[i]) <<
", ";
270 std::cout << std::endl;
276 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 278 "\" may be incorrect. Output type can be specified with -z argument";
281 std::cout << std::endl;
288 template<
typename TParser,
typename TDataType>
290 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
292 using TContainer = mapbox::util::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;
294 std::vector<TContainer> inputDataContainers;
339 for(
unsigned int i = 0; i < numInputs; ++i)
342 armnn::MakeOptional<QuantizationParams>(
348 armnn::MakeOptional<std::string>(
366 inputDataContainers.push_back(tensorData);
370 std::vector<TContainer> outputDataContainers;
372 for (
unsigned int i = 0; i < numOutputs; ++i)
376 outputDataContainers.push_back(std::vector<float>(model.
GetOutputSize(i)));
380 outputDataContainers.push_back(std::vector<int>(model.
GetOutputSize(i)));
384 outputDataContainers.push_back(std::vector<uint8_t>(model.
GetOutputSize(i)));
396 auto inference_duration = model.
Run(inputDataContainers, outputDataContainers);
400 ARMNN_LOG(warning) <<
"The input data was generated, note that the output will not be useful";
405 for (
size_t i = 0; i < numOutputs; i++)
414 mapbox::util::apply_visitor(printer, outputDataContainers[i]);
417 ARMNN_LOG(info) <<
"\nInference time: " << std::setprecision(2)
418 << std::fixed << inference_duration.count() <<
" ms\n";
423 ARMNN_LOG(info) <<
"Threshold time: " << std::setprecision(2)
425 auto thresholdMinusInference = params.
m_ThresholdTime - inference_duration.count();
426 ARMNN_LOG(info) <<
"Threshold time - Inference time: " << std::setprecision(2)
427 << std::fixed << thresholdMinusInference <<
" ms" <<
"\n";
429 if (thresholdMinusInference < 0)
431 std::string errorMessage =
"Elapsed inference time is greater than provided threshold time.";
448 int main(
int argc,
const char* argv[])
468 if (modelFormat.find(
"armnn") != std::string::npos)
470 #if defined(ARMNN_SERIALIZER) 471 return MainImpl<armnnDeserializer::IDeserializer, float>(ProgramOptions.
m_ExNetParams, runtime);
473 ARMNN_LOG(fatal) <<
"Not built with serialization support.";
477 else if (modelFormat.find(
"caffe") != std::string::npos)
479 #if defined(ARMNN_CAFFE_PARSER) 480 return MainImpl<armnnCaffeParser::ICaffeParser, float>(ProgramOptions.
m_ExNetParams, runtime);
482 ARMNN_LOG(fatal) <<
"Not built with Caffe parser support.";
486 else if (modelFormat.find(
"onnx") != std::string::npos)
488 #if defined(ARMNN_ONNX_PARSER) 489 return MainImpl<armnnOnnxParser::IOnnxParser, float>(ProgramOptions.
m_ExNetParams, runtime);
491 ARMNN_LOG(fatal) <<
"Not built with Onnx parser support.";
495 else if (modelFormat.find(
"tensorflow") != std::string::npos)
497 #if defined(ARMNN_TF_PARSER) 498 return MainImpl<armnnTfParser::ITfParser, float>(ProgramOptions.
m_ExNetParams, runtime);
500 ARMNN_LOG(fatal) <<
"Not built with Tensorflow parser support.";
504 else if(modelFormat.find(
"tflite") != std::string::npos)
509 #if defined(ARMNN_TF_LITE_DELEGATE) 510 return TfLiteDelegateMainImpl(ProgramOptions.
m_ExNetParams, runtime);
512 ARMNN_LOG(fatal) <<
"Not built with Arm NN Tensorflow-Lite delegate support.";
516 #if defined(ARMNN_TF_LITE_PARSER) 517 return MainImpl<armnnTfLiteParser::ITfLiteParser, float>(ProgramOptions.
m_ExNetParams, runtime);
519 ARMNN_LOG(fatal) <<
"Not built with Tensorflow-Lite parser support.";
525 ARMNN_LOG(fatal) <<
"Unknown model format: '" << modelFormat
526 <<
"'. Please include 'caffe', 'tensorflow', 'tflite' or 'onnx'";
ExecuteNetworkParams m_ExNetParams
std::vector< std::string > m_InputTypes
static IRuntimePtr Create(const CreationOptions &options)
std::string m_MLGOTuningFilePath
std::vector< TensorShapePtr > m_InputTensorShapes
QuantizationParams GetInputQuantizationParams(unsigned int inputIndex=0u) const
const std::vector< armnn::BindingPointInfo > & GetOutputBindingInfos() const
void ConfigureLogging(bool printToStandardOutput, bool printToDebugOutput, LogSeverity severity)
Configures the logging behaviour of the ARMNN library.
bool m_EnableFp16TurboMode
std::string m_DynamicBackendsPath
mapbox::util::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > > TContainer
std::string m_DynamicBackendsPath
virtual const char * what() const noexcept override
armnn::IRuntime::CreationOptions m_RuntimeOptions
#define ARMNN_LOG(severity)
bool m_EnableFp16TurboMode
void PopulateTensorWithData(TContainer &tensorData, unsigned int numElements, const std::string &dataTypeStr, const armnn::Optional< QuantizationParams > &qParams, const armnn::Optional< std::string > &dataFile)
std::vector< std::string > m_OutputNames
Copyright (c) 2021 ARM Limited and Contributors.
std::vector< std::string > m_OutputTensorFiles
bool m_VisualizePostOptimizationModel
std::string m_CachedNetworkFilePath
bool m_EnableBf16TurboMode
unsigned int GetOutputSize(unsigned int outputIndex=0u) const
std::vector< std::string > m_InputBindings
std::vector< armnn::BackendId > m_ComputeDevices
std::vector< std::string > m_OutputTypes
std::vector< armnn::TensorShape > m_InputShapes
bool m_GenerateTensorData
Holds all parameters necessary to execute a network Check ExecuteNetworkProgramOptions.cpp for a description of each parameter.
std::vector< std::string > m_OutputBindings
std::vector< armnn::BackendId > m_ComputeDevices
unsigned int m_NumberOfThreads
std::vector< std::string > m_InputNames
bool m_EnableBf16TurboMode
std::vector< std::string > m_InputTensorDataFilePaths
Holds and parses program options for the ExecuteNetwork application.
TfLiteDelegate * TfLiteArmnnDelegateCreate(armnnDelegate::DelegateOptions options)
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
bool m_PrintIntermediateLayers
std::string m_CachedNetworkFilePath
std::chrono::duration< double, std::milli > Run(const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
bool m_EnableLayerDetails
int main(int argc, const char *argv[])
Base class for all ArmNN exceptions so that users can filter to just those.
unsigned int GetInputSize(unsigned int inputIndex=0u) const
std::string m_MLGOTuningFilePath
void TfLiteArmnnDelegateDelete(TfLiteDelegate *tfLiteDelegate)
Optional< T > MakeOptional(Args &&... args)
Utility template that constructs an object of type T in-place and wraps it inside an Optional<T> obje...
int MainImpl(const ExecuteNetworkParams ¶ms, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
unsigned int m_NumberOfThreads
std::enable_if_t< std::is_unsigned< Source >::value &&std::is_unsigned< Dest >::value, Dest > numeric_cast(Source source)
std::string m_ModelFormat