13 #if defined(ARMNN_SERIALIZER) 16 #if defined(ARMNN_CAFFE_PARSER) 19 #if defined(ARMNN_TF_PARSER) 22 #if defined(ARMNN_TF_LITE_PARSER) 25 #if defined(ARMNN_ONNX_PARSER) 28 #if defined(ARMNN_TFLITE_DELEGATE) 29 #include <armnn_delegate.hpp> 30 #include <DelegateOptions.hpp> 32 #include <tensorflow/lite/builtin_ops.h> 33 #include <tensorflow/lite/c/builtin_op_data.h> 34 #include <tensorflow/lite/c/common.h> 35 #include <tensorflow/lite/optional_debug_tools.h> 36 #include <tensorflow/lite/kernels/builtin_op_kernels.h> 37 #include <tensorflow/lite/interpreter.h> 38 #include <tensorflow/lite/kernels/register.h> 42 #if defined(ARMNN_TFLITE_DELEGATE) 44 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
46 using namespace tflite;
48 std::unique_ptr<tflite::FlatBufferModel> model = tflite::FlatBufferModel::BuildFromFile(params.
m_ModelPath.c_str());
50 auto tfLiteInterpreter = std::make_unique<Interpreter>();
51 tflite::ops::builtin::BuiltinOpResolver resolver;
53 tflite::InterpreterBuilder builder(*model, resolver);
54 builder(&tfLiteInterpreter);
55 tfLiteInterpreter->AllocateTensors();
59 std::unique_ptr<TfLiteDelegate, decltype(&armnnDelegate::TfLiteArmnnDelegateDelete)>
60 theArmnnDelegate(armnnDelegate::TfLiteArmnnDelegateCreate(delegateOptions),
61 armnnDelegate::TfLiteArmnnDelegateDelete);
63 int status = tfLiteInterpreter->ModifyGraphWithDelegate(std::move(theArmnnDelegate));
65 std::vector<std::string> inputBindings;
68 inputBindings.push_back(inputName);
75 const size_t numInputs = inputBindings.size();
77 for(
unsigned int inputIndex = 0; inputIndex < numInputs; ++inputIndex)
79 int input = tfLiteInterpreter->inputs()[inputIndex];
80 TfLiteIntArray* inputDims = tfLiteInterpreter->tensor(input)->dims;
83 for (
unsigned int dim = 0; dim < static_cast<unsigned int>(inputDims->size); ++dim)
85 inputSize *= inputDims->data[dim];
88 if (params.
m_InputTypes[inputIndex].compare(
"float") == 0)
90 auto inputData = tfLiteInterpreter->typed_tensor<
float>(input);
91 std::vector<float> tensorData;
92 PopulateTensorWithDataGeneric<float>(tensorData,
95 [](
const std::string& s)
96 {
return std::stof(s); });
98 std::copy(tensorData.begin(), tensorData.end(), inputData);
100 else if (params.
m_InputTypes[inputIndex].compare(
"int8") == 0)
102 auto inputData = tfLiteInterpreter->typed_tensor<int8_t>(input);
103 std::vector<int8_t> tensorData;
104 PopulateTensorWithDataGeneric<int8_t>(tensorData,
107 [](
const std::string& s)
110 std::copy(tensorData.begin(), tensorData.end(), inputData);
112 else if (params.
m_InputTypes[inputIndex].compare(
"int") == 0)
114 auto inputData = tfLiteInterpreter->typed_tensor<int32_t>(input);
115 std::vector<int32_t> tensorData;
116 PopulateTensorWithDataGeneric<int32_t>(tensorData,
119 [](
const std::string& s)
120 {
return std::stoi(s); });
122 std::copy(tensorData.begin(), tensorData.end(), inputData);
124 else if (params.
m_InputTypes[inputIndex].compare(
"qasymm8") == 0)
126 auto inputData = tfLiteInterpreter->typed_tensor<uint8_t>(input);
127 std::vector<uint8_t> tensorData;
128 PopulateTensorWithDataGeneric<uint8_t>(tensorData,
131 [](
const std::string& s)
134 std::copy(tensorData.begin(), tensorData.end(), inputData);
138 ARMNN_LOG(fatal) <<
"Unsupported input tensor data type \"" << params.
m_InputTypes[inputIndex] <<
"\". ";
146 tfLiteInterpreter->Invoke();
149 for (
unsigned int outputIndex = 0; outputIndex < params.
m_OutputNames.size(); ++outputIndex)
151 auto tfLiteDelegateOutputId = tfLiteInterpreter->outputs()[outputIndex];
152 TfLiteIntArray* outputDims = tfLiteInterpreter->tensor(tfLiteDelegateOutputId)->dims;
155 for (
unsigned int dim = 0; dim < static_cast<unsigned int>(outputDims->size); ++dim)
157 outputSize *= outputDims->data[dim];
163 auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<
float>(tfLiteDelegateOutputId);
164 if(tfLiteDelageOutputData == NULL)
166 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 167 "\"" << params.
m_OutputTypes[outputIndex] <<
"\" may be incorrect.";
171 for (
int i = 0; i < outputSize; ++i)
173 std::cout << tfLiteDelageOutputData[i] <<
", ";
176 std::cout << std::endl;
180 else if (params.
m_OutputTypes[outputIndex].compare(
"int") == 0)
182 auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<int32_t>(tfLiteDelegateOutputId);
183 if(tfLiteDelageOutputData == NULL)
185 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 186 "\"" << params.
m_OutputTypes[outputIndex] <<
"\" may be incorrect.";
190 for (
int i = 0; i < outputSize; ++i)
192 std::cout << tfLiteDelageOutputData[i] <<
", ";
195 std::cout << std::endl;
199 else if (params.
m_OutputTypes[outputIndex].compare(
"int8") == 0)
201 auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<int8_t>(tfLiteDelegateOutputId);
202 if(tfLiteDelageOutputData == NULL)
204 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 205 "\"" << params.
m_OutputTypes[outputIndex] <<
"\" may be incorrect.";
209 for (
int i = 0; i < outputSize; ++i)
211 std::cout << signed(tfLiteDelageOutputData[i]) <<
", ";
214 std::cout << std::endl;
218 else if (params.
m_OutputTypes[outputIndex].compare(
"qasymm8") == 0)
220 auto tfLiteDelageOutputData = tfLiteInterpreter->typed_tensor<uint8_t>(tfLiteDelegateOutputId);
221 if(tfLiteDelageOutputData == NULL)
223 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 224 "\"" << params.
m_OutputTypes[outputIndex] <<
"\" may be incorrect.";
228 for (
int i = 0; i < outputSize; ++i)
230 std::cout << unsigned(tfLiteDelageOutputData[i]) <<
", ";
233 std::cout << std::endl;
239 ARMNN_LOG(fatal) <<
"Output tensor is null, output type: " 241 "\" may be incorrect. Output type can be specified with -z argument";
244 std::cout << std::endl;
251 template<
typename TParser,
typename TDataType>
253 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
255 using TContainer = mapbox::util::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;
257 std::vector<TContainer> inputDataContainers;
298 for(
unsigned int i = 0; i < numInputs; ++i)
301 armnn::MakeOptional<QuantizationParams>(
307 armnn::MakeOptional<std::string>(
325 inputDataContainers.push_back(tensorData);
329 std::vector<TContainer> outputDataContainers;
331 for (
unsigned int i = 0; i < numOutputs; ++i)
335 outputDataContainers.push_back(std::vector<float>(model.
GetOutputSize(i)));
339 outputDataContainers.push_back(std::vector<int>(model.
GetOutputSize(i)));
343 outputDataContainers.push_back(std::vector<uint8_t>(model.
GetOutputSize(i)));
355 auto inference_duration = model.
Run(inputDataContainers, outputDataContainers);
359 ARMNN_LOG(warning) <<
"The input data was generated, note that the output will not be useful";
364 for (
size_t i = 0; i < numOutputs; i++)
373 mapbox::util::apply_visitor(printer, outputDataContainers[i]);
376 ARMNN_LOG(info) <<
"\nInference time: " << std::setprecision(2)
377 << std::fixed << inference_duration.count() <<
" ms\n";
382 ARMNN_LOG(info) <<
"Threshold time: " << std::setprecision(2)
384 auto thresholdMinusInference = params.
m_ThresholdTime - inference_duration.count();
385 ARMNN_LOG(info) <<
"Threshold time - Inference time: " << std::setprecision(2)
386 << std::fixed << thresholdMinusInference <<
" ms" <<
"\n";
388 if (thresholdMinusInference < 0)
390 std::string errorMessage =
"Elapsed inference time is greater than provided threshold time.";
407 int main(
int argc,
const char* argv[])
427 if (modelFormat.find(
"armnn") != std::string::npos)
429 #if defined(ARMNN_SERIALIZER) 430 return MainImpl<armnnDeserializer::IDeserializer, float>(ProgramOptions.
m_ExNetParams, runtime);
432 ARMNN_LOG(fatal) <<
"Not built with serialization support.";
436 else if (modelFormat.find(
"caffe") != std::string::npos)
438 #if defined(ARMNN_CAFFE_PARSER) 439 return MainImpl<armnnCaffeParser::ICaffeParser, float>(ProgramOptions.
m_ExNetParams, runtime);
441 ARMNN_LOG(fatal) <<
"Not built with Caffe parser support.";
445 else if (modelFormat.find(
"onnx") != std::string::npos)
447 #if defined(ARMNN_ONNX_PARSER) 448 return MainImpl<armnnOnnxParser::IOnnxParser, float>(ProgramOptions.
m_ExNetParams, runtime);
450 ARMNN_LOG(fatal) <<
"Not built with Onnx parser support.";
454 else if (modelFormat.find(
"tensorflow") != std::string::npos)
456 #if defined(ARMNN_TF_PARSER) 457 return MainImpl<armnnTfParser::ITfParser, float>(ProgramOptions.
m_ExNetParams, runtime);
459 ARMNN_LOG(fatal) <<
"Not built with Tensorflow parser support.";
463 else if(modelFormat.find(
"tflite") != std::string::npos)
468 #if defined(ARMNN_TF_LITE_DELEGATE) 469 return TfLiteDelegateMainImpl(ProgramOptions.
m_ExNetParams, runtime);
471 ARMNN_LOG(fatal) <<
"Not built with Tensorflow-Lite parser support.";
475 #if defined(ARMNN_TF_LITE_PARSER) 476 return MainImpl<armnnTfLiteParser::ITfLiteParser, float>(ProgramOptions.
m_ExNetParams, runtime);
478 ARMNN_LOG(fatal) <<
"Not built with Tensorflow-Lite parser support.";
484 ARMNN_LOG(fatal) <<
"Unknown model format: '" << modelFormat
485 <<
"'. Please include 'caffe', 'tensorflow', 'tflite' or 'onnx'";
ExecuteNetworkParams m_ExNetParams
std::vector< std::string > m_InputTypes
static IRuntimePtr Create(const CreationOptions &options)
std::vector< TensorShapePtr > m_InputTensorShapes
QuantizationParams GetInputQuantizationParams(unsigned int inputIndex=0u) const
const std::vector< armnn::BindingPointInfo > & GetOutputBindingInfos() const
void ConfigureLogging(bool printToStandardOutput, bool printToDebugOutput, LogSeverity severity)
Configures the logging behaviour of the ARMNN library.
bool m_EnableFp16TurboMode
std::string m_DynamicBackendsPath
mapbox::util::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > > TContainer
std::string m_DynamicBackendsPath
virtual const char * what() const noexcept override
armnn::IRuntime::CreationOptions m_RuntimeOptions
#define ARMNN_LOG(severity)
bool m_EnableFp16TurboMode
void PopulateTensorWithData(TContainer &tensorData, unsigned int numElements, const std::string &dataTypeStr, const armnn::Optional< QuantizationParams > &qParams, const armnn::Optional< std::string > &dataFile)
std::vector< std::string > m_OutputNames
Copyright (c) 2020 ARM Limited.
std::vector< std::string > m_OutputTensorFiles
bool m_VisualizePostOptimizationModel
bool m_EnableBf16TurboMode
unsigned int GetOutputSize(unsigned int outputIndex=0u) const
std::vector< std::string > m_InputBindings
std::vector< armnn::BackendId > m_ComputeDevices
std::vector< std::string > m_OutputTypes
std::vector< armnn::TensorShape > m_InputShapes
bool m_GenerateTensorData
Holds all parameters necessary to execute a network Check ExecuteNetworkProgramOptions.cpp for a description of each parameter.
std::vector< std::string > m_OutputBindings
std::vector< armnn::BackendId > m_ComputeDevices
std::vector< std::string > m_InputNames
bool m_EnableBf16TurboMode
std::vector< std::string > m_InputTensorDataFilePaths
Holds and parses program options for the ExecuteNetwork application.
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
bool m_PrintIntermediateLayers
std::chrono::duration< double, std::milli > Run(const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
bool m_EnableLayerDetails
int main(int argc, const char *argv[])
Base class for all ArmNN exceptions so that users can filter to just those.
unsigned int GetInputSize(unsigned int inputIndex=0u) const
Optional< T > MakeOptional(Args &&... args)
Utility template that constructs an object of type T in-place and wraps it inside an Optional<T> obje...
int MainImpl(const ExecuteNetworkParams ¶ms, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
std::enable_if_t< std::is_unsigned< Source >::value &&std::is_unsigned< Dest >::value, Dest > numeric_cast(Source source)
std::string m_ModelFormat