8 #if defined(ARMNN_SERIALIZER) 11 #if defined(ARMNN_CAFFE_PARSER) 14 #if defined(ARMNN_TF_PARSER) 17 #if defined(ARMNN_TF_LITE_PARSER) 20 #if defined(ARMNN_ONNX_PARSER) 24 #include "../InferenceTest.hpp" 29 #include <boost/algorithm/string/trim.hpp> 30 #include <boost/algorithm/string/split.hpp> 31 #include <boost/algorithm/string/classification.hpp> 32 #include <boost/program_options.hpp> 33 #include <boost/variant.hpp> 46 namespace po = boost::program_options;
48 template<
typename T,
typename TParseElementFunc>
49 std::vector<T> ParseArrayImpl(std::istream& stream, TParseElementFunc parseElementFunc,
const char * chars =
"\t ,:")
51 std::vector<T> result;
54 while (std::getline(stream, line))
56 std::vector<std::string> tokens;
60 boost::split(tokens, line, boost::algorithm::is_any_of(chars), boost::token_compress_on);
62 catch (
const std::exception& e)
64 ARMNN_LOG(error) <<
"An error occurred when splitting tokens: " << e.what();
67 for (
const std::string& token : tokens)
73 result.push_back(parseElementFunc(token));
75 catch (
const std::exception&)
77 ARMNN_LOG(error) <<
"'" << token <<
"' is not a valid number. It has been ignored.";
86 bool CheckOption(
const po::variables_map& vm,
90 if (option ==
nullptr)
96 return vm.find(option) != vm.end();
99 void CheckOptionDependency(
const po::variables_map& vm,
101 const char* required)
104 if (option ==
nullptr || required ==
nullptr)
106 throw po::error(
"Invalid option to check dependency for");
110 if (CheckOption(vm, option) && !vm[option].defaulted())
112 if (CheckOption(vm, required) == 0 || vm[required].defaulted())
114 throw po::error(std::string(
"Option '") + option +
"' requires option '" + required +
"'.");
119 void CheckOptionDependencies(
const po::variables_map& vm)
121 CheckOptionDependency(vm,
"model-path",
"model-format");
122 CheckOptionDependency(vm,
"model-path",
"input-name");
123 CheckOptionDependency(vm,
"model-path",
"output-name");
124 CheckOptionDependency(vm,
"input-tensor-shape",
"model-path");
127 template<armnn::DataType NonQuantizedType>
128 auto ParseDataArray(std::istream & stream);
130 template<armnn::DataType QuantizedType>
131 auto ParseDataArray(std::istream& stream,
132 const float& quantizationScale,
133 const int32_t& quantizationOffset);
136 auto ParseDataArray<armnn::DataType::Float32>(std::istream & stream)
138 return ParseArrayImpl<float>(stream, [](
const std::string& s) {
return std::stof(s); });
142 auto ParseDataArray<armnn::DataType::Signed32>(std::istream & stream)
144 return ParseArrayImpl<int>(stream, [](
const std::string & s) {
return std::stoi(s); });
148 auto ParseDataArray<armnn::DataType::QAsymmU8>(std::istream& stream)
150 return ParseArrayImpl<uint8_t>(stream,
151 [](
const std::string& s) {
return boost::numeric_cast<uint8_t>(std::stoi(s)); });
155 auto ParseDataArray<armnn::DataType::QAsymmU8>(std::istream& stream,
156 const float& quantizationScale,
157 const int32_t& quantizationOffset)
159 return ParseArrayImpl<uint8_t>(stream,
160 [&quantizationScale, &quantizationOffset](
const std::string & s)
162 return boost::numeric_cast<uint8_t>(
163 armnn::Quantize<uint8_t>(std::stof(s),
165 quantizationOffset));
168 std::vector<unsigned int> ParseArray(std::istream& stream)
170 return ParseArrayImpl<unsigned int>(stream,
171 [](
const std::string& s) {
return boost::numeric_cast<
unsigned int>(std::stoi(s)); });
174 std::vector<std::string> ParseStringList(
const std::string & inputString,
const char * delimiter)
176 std::stringstream stream(inputString);
177 return ParseArrayImpl<std::string>(stream, [](
const std::string& s) {
return boost::trim_copy(s); }, delimiter);
180 void RemoveDuplicateDevices(std::vector<armnn::BackendId>& computeDevices)
183 for (
auto i = computeDevices.begin(); i != computeDevices.end(); ++i)
185 for (
auto j = std::next(i); j != computeDevices.end(); ++j)
196 computeDevices.end());
199 struct TensorPrinter :
public boost::static_visitor<>
201 TensorPrinter(
const std::string& binding,
const armnn::TensorInfo& info,
const std::string& outputTensorFile)
202 : m_OutputBinding(binding)
203 , m_Scale(info.GetQuantizationScale())
204 , m_Offset(info.GetQuantizationOffset())
205 , m_OutputTensorFile(outputTensorFile)
208 void operator()(
const std::vector<float>& values)
210 ForEachValue(values, [](
float value)
212 printf(
"%f ", value);
217 void operator()(
const std::vector<uint8_t>& values)
219 auto& scale = m_Scale;
220 auto& offset = m_Offset;
221 std::vector<float> dequantizedValues;
222 ForEachValue(values, [&scale, &offset, &dequantizedValues](uint8_t value)
225 printf(
"%f ", dequantizedValue);
226 dequantizedValues.push_back(dequantizedValue);
228 WriteToFile(dequantizedValues);
231 void operator()(
const std::vector<int>& values)
233 ForEachValue(values, [](
int value)
235 printf(
"%d ", value);
241 template<
typename Container,
typename Delegate>
242 void ForEachValue(
const Container& c, Delegate delegate)
244 std::cout << m_OutputBinding <<
": ";
245 for (
const auto& value : c)
253 void WriteToFile(
const std::vector<T>& values)
255 if (!m_OutputTensorFile.empty())
257 std::ofstream outputTensorFile;
258 outputTensorFile.open(m_OutputTensorFile, std::ofstream::out | std::ofstream::trunc);
259 if (outputTensorFile.is_open())
261 outputTensorFile << m_OutputBinding <<
": ";
262 std::copy(values.begin(), values.end(), std::ostream_iterator<T>(outputTensorFile,
" "));
266 ARMNN_LOG(info) <<
"Output Tensor File: " << m_OutputTensorFile <<
" could not be opened!";
268 outputTensorFile.close();
272 std::string m_OutputBinding;
275 std::string m_OutputTensorFile;
280 template<armnn::DataType ArmnnType,
typename T = armnn::ResolveType<ArmnnType>>
281 std::vector<T> GenerateDummyTensorData(
unsigned int numElements)
283 return std::vector<T>(numElements,
static_cast<T
>(0));
286 using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;
289 void PopulateTensorWithData(
TContainer& tensorData,
290 unsigned int numElements,
291 const std::string& dataTypeStr,
295 const bool readFromFile = dataFile.
has_value() && !dataFile.
value().empty();
296 const bool quantizeData = qParams.
has_value();
298 std::ifstream inputTensorFile;
301 inputTensorFile = std::ifstream(dataFile.
value());
304 if (dataTypeStr.compare(
"float") == 0)
308 const float qScale = qParams.
value().first;
309 const int qOffset = qParams.
value().second;
311 tensorData = readFromFile ?
312 ParseDataArray<armnn::DataType::QAsymmU8>(inputTensorFile, qScale, qOffset) :
313 GenerateDummyTensorData<armnn::DataType::QAsymmU8>(numElements);
317 tensorData = readFromFile ?
318 ParseDataArray<armnn::DataType::Float32>(inputTensorFile) :
319 GenerateDummyTensorData<armnn::DataType::Float32>(numElements);
322 else if (dataTypeStr.compare(
"int") == 0)
324 tensorData = readFromFile ?
325 ParseDataArray<armnn::DataType::Signed32>(inputTensorFile) :
326 GenerateDummyTensorData<armnn::DataType::Signed32>(numElements);
328 else if (dataTypeStr.compare(
"qasymm8") == 0)
330 tensorData = readFromFile ?
331 ParseDataArray<armnn::DataType::QAsymmU8>(inputTensorFile) :
332 GenerateDummyTensorData<armnn::DataType::QAsymmU8>(numElements);
336 std::string errorMessage =
"Unsupported tensor data type " + dataTypeStr;
339 inputTensorFile.close();
343 inputTensorFile.close();
371 bool m_EnableLayerDetails =
false;
373 bool m_ParseUnsupported =
false;
376 template<
typename TParser,
typename TDataType>
378 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
380 using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;
382 std::vector<TContainer> inputDataContainers;
420 for(
unsigned int i = 0; i < numInputs; ++i)
439 PopulateTensorWithData(tensorData,
445 inputDataContainers.push_back(tensorData);
449 std::vector<TContainer> outputDataContainers;
451 for (
unsigned int i = 0; i < numOutputs; ++i)
455 outputDataContainers.push_back(std::vector<float>(model.
GetOutputSize(i)));
459 outputDataContainers.push_back(std::vector<int>(model.
GetOutputSize(i)));
463 outputDataContainers.push_back(std::vector<uint8_t>(model.
GetOutputSize(i)));
473 auto inference_duration = model.
Run(inputDataContainers, outputDataContainers);
477 ARMNN_LOG(warning) <<
"The input data was generated, note that the output will not be useful";
482 for (
size_t i = 0; i < numOutputs; i++)
487 TensorPrinter printer(inferenceModelParams.
m_OutputBindings[i], infoOut, outputTensorFile);
488 boost::apply_visitor(printer, outputDataContainers[i]);
491 ARMNN_LOG(info) <<
"\nInference time: " << std::setprecision(2)
492 << std::fixed << inference_duration.count() <<
" ms";
497 ARMNN_LOG(info) <<
"Threshold time: " << std::setprecision(2)
499 auto thresholdMinusInference = params.
m_ThresholdTime - inference_duration.count();
500 ARMNN_LOG(info) <<
"Threshold time - Inference time: " << std::setprecision(2)
501 << std::fixed << thresholdMinusInference <<
" ms" <<
"\n";
503 if (thresholdMinusInference < 0)
505 std::string errorMessage =
"Elapsed inference time is greater than provided threshold time.";
521 const std::string& inputTensorShapesStr,
522 const vector<armnn::BackendId>& computeDevices,
523 const std::string& dynamicBackendsPath,
524 const std::string& path,
525 const std::string& inputNames,
526 const std::string& inputTensorDataFilePaths,
527 const std::string& inputTypes,
529 const std::string& outputTypes,
530 const std::string& outputNames,
531 const std::string& outputTensorFiles,
532 bool enableProfiling,
533 bool enableFp16TurboMode,
534 const double& thresholdTime,
535 bool printIntermediate,
536 const size_t subgraphId,
537 bool enableLayerDetails =
false,
538 bool parseUnsupported =
false,
539 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
541 std::string modelFormat = boost::trim_copy(format);
542 std::string modelPath = boost::trim_copy(path);
543 std::vector<std::string> inputNamesVector = ParseStringList(inputNames,
",");
544 std::vector<std::string> inputTensorShapesVector = ParseStringList(inputTensorShapesStr,
":");
545 std::vector<std::string> inputTensorDataFilePathsVector = ParseStringList(
546 inputTensorDataFilePaths,
",");
547 std::vector<std::string> outputNamesVector = ParseStringList(outputNames,
",");
548 std::vector<std::string> inputTypesVector = ParseStringList(inputTypes,
",");
549 std::vector<std::string> outputTypesVector = ParseStringList(outputTypes,
",");
550 std::vector<std::string> outputTensorFilesVector = ParseStringList(outputTensorFiles,
",");
554 if (modelFormat.find(
"bin") != std::string::npos)
556 isModelBinary =
true;
558 else if (modelFormat.find(
"txt") != std::string::npos || modelFormat.find(
"text") != std::string::npos)
560 isModelBinary =
false;
564 ARMNN_LOG(fatal) <<
"Unknown model format: '" << modelFormat <<
"'. Please include 'binary' or 'text'";
568 if ((inputTensorShapesVector.size() != 0) && (inputTensorShapesVector.size() != inputNamesVector.size()))
570 ARMNN_LOG(fatal) <<
"input-name and input-tensor-shape must have the same amount of elements.";
574 if ((inputTensorDataFilePathsVector.size() != 0) &&
575 (inputTensorDataFilePathsVector.size() != inputNamesVector.size()))
577 ARMNN_LOG(fatal) <<
"input-name and input-tensor-data must have the same amount of elements.";
581 if ((outputTensorFilesVector.size() != 0) &&
582 (outputTensorFilesVector.size() != outputNamesVector.size()))
584 ARMNN_LOG(fatal) <<
"output-name and write-outputs-to-file must have the same amount of elements.";
588 if (inputTypesVector.size() == 0)
591 inputTypesVector.assign(inputNamesVector.size(),
"float");
593 else if ((inputTypesVector.size() != 0) && (inputTypesVector.size() != inputNamesVector.size()))
595 ARMNN_LOG(fatal) <<
"input-name and input-type must have the same amount of elements.";
599 if (outputTypesVector.size() == 0)
602 outputTypesVector.assign(outputNamesVector.size(),
"float");
604 else if ((outputTypesVector.size() != 0) && (outputTypesVector.size() != outputNamesVector.size()))
606 ARMNN_LOG(fatal) <<
"output-name and output-type must have the same amount of elements.";
611 std::vector<std::unique_ptr<armnn::TensorShape>> inputTensorShapes;
613 if (!inputTensorShapesVector.empty())
615 inputTensorShapes.reserve(inputTensorShapesVector.size());
617 for(
const std::string& shape : inputTensorShapesVector)
619 std::stringstream ss(shape);
620 std::vector<unsigned int> dims = ParseArray(ss);
625 inputTensorShapes.push_back(std::make_unique<armnn::TensorShape>(dims.size(), dims.data()));
629 ARMNN_LOG(fatal) <<
"Cannot create tensor shape: " << e.
what();
636 if (thresholdTime < 0)
638 ARMNN_LOG(fatal) <<
"Threshold time supplied as a command line argument is less than zero.";
667 ARMNN_LOG(warning) <<
"No input files provided, input tensors will be filled with 0s.";
671 if (modelFormat.find(
"armnn") != std::string::npos)
673 #if defined(ARMNN_SERIALIZER) 674 return MainImpl<armnnDeserializer::IDeserializer, float>(params, runtime);
676 ARMNN_LOG(fatal) <<
"Not built with serialization support.";
680 else if (modelFormat.find(
"caffe") != std::string::npos)
682 #if defined(ARMNN_CAFFE_PARSER) 683 return MainImpl<armnnCaffeParser::ICaffeParser, float>(params, runtime);
685 ARMNN_LOG(fatal) <<
"Not built with Caffe parser support.";
689 else if (modelFormat.find(
"onnx") != std::string::npos)
691 #if defined(ARMNN_ONNX_PARSER) 692 return MainImpl<armnnOnnxParser::IOnnxParser, float>(params, runtime);
694 ARMNN_LOG(fatal) <<
"Not built with Onnx parser support.";
698 else if (modelFormat.find(
"tensorflow") != std::string::npos)
700 #if defined(ARMNN_TF_PARSER) 701 return MainImpl<armnnTfParser::ITfParser, float>(params, runtime);
703 ARMNN_LOG(fatal) <<
"Not built with Tensorflow parser support.";
707 else if(modelFormat.find(
"tflite") != std::string::npos)
709 #if defined(ARMNN_TF_LITE_PARSER) 712 ARMNN_LOG(fatal) <<
"Unknown model format: '" << modelFormat <<
"'. Only 'binary' format supported \ 716 return MainImpl<armnnTfLiteParser::ITfLiteParser, float>(params, runtime);
718 ARMNN_LOG(fatal) <<
"Unknown model format: '" << modelFormat <<
719 "'. Please include 'caffe', 'tensorflow', 'tflite' or 'onnx'";
725 ARMNN_LOG(fatal) <<
"Unknown model format: '" << modelFormat <<
726 "'. Please include 'caffe', 'tensorflow', 'tflite' or 'onnx'";
732 const bool enableProfiling,
const bool enableFp16TurboMode,
const double& thresholdTime,
733 const bool printIntermediate,
bool enableLayerDetails =
false,
bool parseUnuspported =
false)
735 boost::ignore_unused(runtime);
736 std::string modelFormat;
737 std::string modelPath;
738 std::string inputNames;
739 std::string inputTensorShapes;
740 std::string inputTensorDataFilePaths;
741 std::string outputNames;
742 std::string inputTypes;
743 std::string outputTypes;
744 std::string dynamicBackendsPath;
745 std::string outputTensorFiles;
747 size_t subgraphId = 0;
749 const std::string backendsMessage = std::string(
"The preferred order of devices to run layers on by default. ")
750 + std::string(
"Possible choices: ")
753 po::options_description desc(
"Options");
757 (
"model-format,f", po::value(&modelFormat),
758 "armnn-binary, caffe-binary, caffe-text, tflite-binary, onnx-binary, onnx-text, tensorflow-binary or " 760 (
"model-path,m", po::value(&modelPath),
"Path to model file, e.g. .armnn, .caffemodel, .prototxt, " 762 (
"compute,c", po::value<std::vector<armnn::BackendId>>()->multitoken(),
763 backendsMessage.c_str())
764 (
"dynamic-backends-path,b", po::value(&dynamicBackendsPath),
765 "Path where to load any available dynamic backend from. " 766 "If left empty (the default), dynamic backends will not be used.")
767 (
"input-name,i", po::value(&inputNames),
"Identifier of the input tensors in the network separated by comma.")
768 (
"subgraph-number,n", po::value<size_t>(&subgraphId)->default_value(0),
"Id of the subgraph to be " 769 "executed. Defaults to 0.")
770 (
"input-tensor-shape,s", po::value(&inputTensorShapes),
771 "The shape of the input tensors in the network as a flat array of integers separated by comma. " 772 "Several shapes can be passed separating them by semicolon. " 773 "This parameter is optional, depending on the network.")
774 (
"input-tensor-data,d", po::value(&inputTensorDataFilePaths)->default_value(
""),
775 "Path to files containing the input data as a flat array separated by whitespace. " 776 "Several paths can be passed separating them by comma. If not specified, the network will be run with dummy " 777 "data (useful for profiling).")
778 (
"input-type,y",po::value(&inputTypes),
"The type of the input tensors in the network separated by comma. " 779 "If unset, defaults to \"float\" for all defined inputs. " 780 "Accepted values (float, int or qasymm8).")
781 (
"quantize-input,q",po::bool_switch()->default_value(
false),
782 "If this option is enabled, all float inputs will be quantized to qasymm8. " 783 "If unset, default to not quantized. " 784 "Accepted values (true or false)")
785 (
"output-type,z",po::value(&outputTypes),
"The type of the output tensors in the network separated by comma. " 786 "If unset, defaults to \"float\" for all defined outputs. " 787 "Accepted values (float, int or qasymm8).")
788 (
"output-name,o", po::value(&outputNames),
789 "Identifier of the output tensors in the network separated by comma.")
790 (
"write-outputs-to-file,w", po::value(&outputTensorFiles),
791 "Comma-separated list of output file paths keyed with the binding-id of the output slot. " 792 "If left empty (the default), the output tensors will not be written to a file.");
794 catch (
const std::exception& e)
799 BOOST_ASSERT_MSG(
false,
"Caught unexpected exception");
800 ARMNN_LOG(fatal) <<
"Fatal internal error: " << e.what();
804 std::vector<const char*> clOptions;
805 clOptions.reserve(csvRow.
values.size());
806 for (
const std::string& value : csvRow.
values)
808 clOptions.push_back(value.c_str());
811 po::variables_map vm;
814 po::store(po::parse_command_line(static_cast<int>(clOptions.size()), clOptions.data(), desc), vm);
818 CheckOptionDependencies(vm);
820 catch (
const po::error& e)
822 std::cerr << e.what() << std::endl << std::endl;
823 std::cerr << desc << std::endl;
828 bool quantizeInput = vm[
"quantize-input"].as<
bool>();
831 std::vector<armnn::BackendId> computeDevices = vm[
"compute"].as<std::vector<armnn::BackendId>>();
834 RemoveDuplicateDevices(computeDevices);
837 std::string invalidBackends;
840 ARMNN_LOG(fatal) <<
"The list of preferred devices contains invalid backend IDs: " 845 return RunTest(modelFormat, inputTensorShapes, computeDevices, dynamicBackendsPath, modelPath, inputNames,
846 inputTensorDataFilePaths, inputTypes, quantizeInput, outputTypes, outputNames, outputTensorFiles,
847 enableProfiling, enableFp16TurboMode, thresholdTime, printIntermediate, subgraphId,
848 enableLayerDetails, parseUnuspported);
std::vector< TensorShapePtr > m_InputTensorShapes
std::vector< std::string > values
std::string GetBackendIdsAsString() const
bool m_VisualizePostOptimizationModel
unsigned int GetInputSize(unsigned int inputIndex=0u) const
int RunCsvTest(const armnnUtils::CsvRow &csvRow, const std::shared_ptr< armnn::IRuntime > &runtime, const bool enableProfiling, const bool enableFp16TurboMode, const double &thresholdTime, const bool printIntermediate, bool enableLayerDetails=false, bool parseUnuspported=false)
int RunTest(const std::string &format, const std::string &inputTensorShapesStr, const vector< armnn::BackendId > &computeDevices, const std::string &dynamicBackendsPath, const std::string &path, const std::string &inputNames, const std::string &inputTensorDataFilePaths, const std::string &inputTypes, bool quantizeInput, const std::string &outputTypes, const std::string &outputNames, const std::string &outputTensorFiles, bool enableProfiling, bool enableFp16TurboMode, const double &thresholdTime, bool printIntermediate, const size_t subgraphId, bool enableLayerDetails=false, bool parseUnsupported=false, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
float Dequantize(QuantizedType value, float scale, int32_t offset)
std::pair< float, int32_t > QuantizationParams
bool m_EnableLayerDetails
#define ARMNN_LOG(severity)
std::chrono::duration< double, std::milli > Run(const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
std::vector< std::string > m_InputBindings
std::string m_DynamicBackendsPath
virtual const char * what() const noexcept override
const std::vector< armnn::BindingPointInfo > & GetOutputBindingInfos() const
BackendRegistry & BackendRegistryInstance()
std::vector< string > m_OutputTypes
bool m_PrintIntermediateLayers
std::vector< armnn::BackendId > m_ComputeDevices
std::vector< string > m_OutputNames
bool m_EnableFp16TurboMode
int MainImpl(const ExecuteNetworkParams ¶ms, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
std::vector< armnn::BackendId > m_ComputeDevices
QuantizationParams GetInputQuantizationParams(unsigned int inputIndex=0u) const
Base class for all ArmNN exceptions so that users can filter to just those.
boost::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > > TContainer
std::string m_DynamicBackendsPath
std::vector< string > m_InputNames
std::vector< string > m_InputTypes
std::vector< string > m_InputTensorDataFilePaths
bool m_EnableFp16TurboMode
std::vector< string > m_OutputTensorFiles
unsigned int GetOutputSize(unsigned int outputIndex=0u) const
std::vector< std::string > m_OutputBindings
std::unique_ptr< armnn::TensorShape > TensorShapePtr
std::vector< armnn::TensorShape > m_InputShapes
bool has_value() const noexcept
bool m_GenerateTensorData