8 #if defined(ARMNN_SERIALIZER) 11 #if defined(ARMNN_CAFFE_PARSER) 14 #if defined(ARMNN_TF_PARSER) 17 #if defined(ARMNN_TF_LITE_PARSER) 20 #if defined(ARMNN_ONNX_PARSER) 24 #include "../InferenceTest.hpp" 29 #include <boost/algorithm/string/trim.hpp> 30 #include <boost/algorithm/string/split.hpp> 31 #include <boost/algorithm/string/classification.hpp> 32 #include <boost/program_options.hpp> 33 #include <boost/variant.hpp> 46 namespace po = boost::program_options;
48 template<
typename T,
typename TParseElementFunc>
49 std::vector<T> ParseArrayImpl(std::istream& stream, TParseElementFunc parseElementFunc,
const char * chars =
"\t ,:")
51 std::vector<T> result;
54 while (std::getline(stream, line))
56 std::vector<std::string> tokens;
60 boost::split(tokens, line, boost::algorithm::is_any_of(chars), boost::token_compress_on);
62 catch (
const std::exception& e)
64 ARMNN_LOG(error) <<
"An error occurred when splitting tokens: " << e.what();
67 for (
const std::string& token : tokens)
73 result.push_back(parseElementFunc(token));
75 catch (
const std::exception&)
77 ARMNN_LOG(error) <<
"'" << token <<
"' is not a valid number. It has been ignored.";
86 bool CheckOption(
const po::variables_map& vm,
90 if (option ==
nullptr)
96 return vm.find(option) != vm.end();
99 void CheckOptionDependency(
const po::variables_map& vm,
101 const char* required)
104 if (option ==
nullptr || required ==
nullptr)
106 throw po::error(
"Invalid option to check dependency for");
110 if (CheckOption(vm, option) && !vm[option].defaulted())
112 if (CheckOption(vm, required) == 0 || vm[required].defaulted())
114 throw po::error(std::string(
"Option '") + option +
"' requires option '" + required +
"'.");
119 void CheckOptionDependencies(
const po::variables_map& vm)
121 CheckOptionDependency(vm,
"model-path",
"model-format");
122 CheckOptionDependency(vm,
"model-path",
"input-name");
123 CheckOptionDependency(vm,
"model-path",
"output-name");
124 CheckOptionDependency(vm,
"input-tensor-shape",
"model-path");
127 template<armnn::DataType NonQuantizedType>
128 auto ParseDataArray(std::istream & stream);
130 template<armnn::DataType QuantizedType>
131 auto ParseDataArray(std::istream& stream,
132 const float& quantizationScale,
133 const int32_t& quantizationOffset);
136 auto ParseDataArray<armnn::DataType::Float32>(std::istream & stream)
138 return ParseArrayImpl<float>(stream, [](
const std::string& s) {
return std::stof(s); });
142 auto ParseDataArray<armnn::DataType::Signed32>(std::istream & stream)
144 return ParseArrayImpl<int>(stream, [](
const std::string & s) {
return std::stoi(s); });
148 auto ParseDataArray<armnn::DataType::QAsymmU8>(std::istream& stream)
150 return ParseArrayImpl<uint8_t>(stream,
155 auto ParseDataArray<armnn::DataType::QAsymmU8>(std::istream& stream,
156 const float& quantizationScale,
157 const int32_t& quantizationOffset)
159 return ParseArrayImpl<uint8_t>(stream,
160 [&quantizationScale, &quantizationOffset](
const std::string & s)
163 armnn::Quantize<uint8_t>(std::stof(s),
165 quantizationOffset));
168 std::vector<unsigned int> ParseArray(std::istream& stream)
170 return ParseArrayImpl<unsigned int>(stream,
174 std::vector<std::string> ParseStringList(
const std::string & inputString,
const char * delimiter)
176 std::stringstream stream(inputString);
177 return ParseArrayImpl<std::string>(stream, [](
const std::string& s) {
return boost::trim_copy(s); }, delimiter);
180 void RemoveDuplicateDevices(std::vector<armnn::BackendId>& computeDevices)
183 for (
auto i = computeDevices.begin(); i != computeDevices.end(); ++i)
185 for (
auto j = std::next(i); j != computeDevices.end(); ++j)
196 computeDevices.end());
199 struct TensorPrinter :
public boost::static_visitor<>
201 TensorPrinter(
const std::string& binding,
203 const std::string& outputTensorFile,
204 bool dequantizeOutput)
205 : m_OutputBinding(binding)
206 , m_Scale(info.GetQuantizationScale())
207 , m_Offset(info.GetQuantizationOffset())
208 , m_OutputTensorFile(outputTensorFile)
209 , m_DequantizeOutput(dequantizeOutput)
212 void operator()(
const std::vector<float>& values)
214 ForEachValue(values, [](
float value)
216 printf(
"%f ", value);
221 void operator()(
const std::vector<uint8_t>& values)
223 if(m_DequantizeOutput)
225 auto& scale = m_Scale;
226 auto& offset = m_Offset;
227 std::vector<float> dequantizedValues;
228 ForEachValue(values, [&scale, &offset, &dequantizedValues](uint8_t value)
231 printf(
"%f ", dequantizedValue);
232 dequantizedValues.push_back(dequantizedValue);
234 WriteToFile(dequantizedValues);
238 const std::vector<int> intValues(values.begin(), values.end());
239 operator()(intValues);
243 void operator()(
const std::vector<int>& values)
245 ForEachValue(values, [](
int value)
247 printf(
"%d ", value);
253 template<
typename Container,
typename Delegate>
254 void ForEachValue(
const Container& c, Delegate delegate)
256 std::cout << m_OutputBinding <<
": ";
257 for (
const auto& value : c)
265 void WriteToFile(
const std::vector<T>& values)
267 if (!m_OutputTensorFile.empty())
269 std::ofstream outputTensorFile;
270 outputTensorFile.open(m_OutputTensorFile, std::ofstream::out | std::ofstream::trunc);
271 if (outputTensorFile.is_open())
273 outputTensorFile << m_OutputBinding <<
": ";
274 std::copy(values.begin(), values.end(), std::ostream_iterator<T>(outputTensorFile,
" "));
278 ARMNN_LOG(info) <<
"Output Tensor File: " << m_OutputTensorFile <<
" could not be opened!";
280 outputTensorFile.close();
284 std::string m_OutputBinding;
287 std::string m_OutputTensorFile;
288 bool m_DequantizeOutput =
false;
293 template<armnn::DataType ArmnnType,
typename T = armnn::ResolveType<ArmnnType>>
294 std::vector<T> GenerateDummyTensorData(
unsigned int numElements)
296 return std::vector<T>(numElements,
static_cast<T
>(0));
299 using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;
302 void PopulateTensorWithData(
TContainer& tensorData,
303 unsigned int numElements,
304 const std::string& dataTypeStr,
308 const bool readFromFile = dataFile.
has_value() && !dataFile.
value().empty();
309 const bool quantizeData = qParams.
has_value();
311 std::ifstream inputTensorFile;
314 inputTensorFile = std::ifstream(dataFile.
value());
317 if (dataTypeStr.compare(
"float") == 0)
321 const float qScale = qParams.
value().first;
322 const int qOffset = qParams.
value().second;
324 tensorData = readFromFile ?
325 ParseDataArray<armnn::DataType::QAsymmU8>(inputTensorFile, qScale, qOffset) :
326 GenerateDummyTensorData<armnn::DataType::QAsymmU8>(numElements);
330 tensorData = readFromFile ?
331 ParseDataArray<armnn::DataType::Float32>(inputTensorFile) :
332 GenerateDummyTensorData<armnn::DataType::Float32>(numElements);
335 else if (dataTypeStr.compare(
"int") == 0)
337 tensorData = readFromFile ?
338 ParseDataArray<armnn::DataType::Signed32>(inputTensorFile) :
339 GenerateDummyTensorData<armnn::DataType::Signed32>(numElements);
341 else if (dataTypeStr.compare(
"qasymm8") == 0)
343 tensorData = readFromFile ?
344 ParseDataArray<armnn::DataType::QAsymmU8>(inputTensorFile) :
345 GenerateDummyTensorData<armnn::DataType::QAsymmU8>(numElements);
349 std::string errorMessage =
"Unsupported tensor data type " + dataTypeStr;
352 inputTensorFile.close();
356 inputTensorFile.close();
385 bool m_EnableLayerDetails =
false;
387 bool m_ParseUnsupported =
false;
390 template<
typename TParser,
typename TDataType>
392 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
394 using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;
396 std::vector<TContainer> inputDataContainers;
434 for(
unsigned int i = 0; i < numInputs; ++i)
453 PopulateTensorWithData(tensorData,
459 inputDataContainers.push_back(tensorData);
463 std::vector<TContainer> outputDataContainers;
465 for (
unsigned int i = 0; i < numOutputs; ++i)
469 outputDataContainers.push_back(std::vector<float>(model.
GetOutputSize(i)));
473 outputDataContainers.push_back(std::vector<int>(model.
GetOutputSize(i)));
477 outputDataContainers.push_back(std::vector<uint8_t>(model.
GetOutputSize(i)));
487 auto inference_duration = model.
Run(inputDataContainers, outputDataContainers);
491 ARMNN_LOG(warning) <<
"The input data was generated, note that the output will not be useful";
496 for (
size_t i = 0; i < numOutputs; i++)
505 boost::apply_visitor(printer, outputDataContainers[i]);
508 ARMNN_LOG(info) <<
"\nInference time: " << std::setprecision(2)
509 << std::fixed << inference_duration.count() <<
" ms";
514 ARMNN_LOG(info) <<
"Threshold time: " << std::setprecision(2)
516 auto thresholdMinusInference = params.
m_ThresholdTime - inference_duration.count();
517 ARMNN_LOG(info) <<
"Threshold time - Inference time: " << std::setprecision(2)
518 << std::fixed << thresholdMinusInference <<
" ms" <<
"\n";
520 if (thresholdMinusInference < 0)
522 std::string errorMessage =
"Elapsed inference time is greater than provided threshold time.";
538 const std::string& inputTensorShapesStr,
539 const vector<armnn::BackendId>& computeDevices,
540 const std::string& dynamicBackendsPath,
541 const std::string& path,
542 const std::string& inputNames,
543 const std::string& inputTensorDataFilePaths,
544 const std::string& inputTypes,
546 const std::string& outputTypes,
547 const std::string& outputNames,
548 const std::string& outputTensorFiles,
549 bool dequantizeOuput,
550 bool enableProfiling,
551 bool enableFp16TurboMode,
552 const double& thresholdTime,
553 bool printIntermediate,
554 const size_t subgraphId,
555 bool enableLayerDetails =
false,
556 bool parseUnsupported =
false,
557 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
559 std::string modelFormat = boost::trim_copy(format);
560 std::string modelPath = boost::trim_copy(path);
561 std::vector<std::string> inputNamesVector = ParseStringList(inputNames,
",");
562 std::vector<std::string> inputTensorShapesVector = ParseStringList(inputTensorShapesStr,
":");
563 std::vector<std::string> inputTensorDataFilePathsVector = ParseStringList(
564 inputTensorDataFilePaths,
",");
565 std::vector<std::string> outputNamesVector = ParseStringList(outputNames,
",");
566 std::vector<std::string> inputTypesVector = ParseStringList(inputTypes,
",");
567 std::vector<std::string> outputTypesVector = ParseStringList(outputTypes,
",");
568 std::vector<std::string> outputTensorFilesVector = ParseStringList(outputTensorFiles,
",");
572 if (modelFormat.find(
"bin") != std::string::npos)
574 isModelBinary =
true;
576 else if (modelFormat.find(
"txt") != std::string::npos || modelFormat.find(
"text") != std::string::npos)
578 isModelBinary =
false;
582 ARMNN_LOG(fatal) <<
"Unknown model format: '" << modelFormat <<
"'. Please include 'binary' or 'text'";
586 if ((inputTensorShapesVector.size() != 0) && (inputTensorShapesVector.size() != inputNamesVector.size()))
588 ARMNN_LOG(fatal) <<
"input-name and input-tensor-shape must have the same amount of elements.";
592 if ((inputTensorDataFilePathsVector.size() != 0) &&
593 (inputTensorDataFilePathsVector.size() != inputNamesVector.size()))
595 ARMNN_LOG(fatal) <<
"input-name and input-tensor-data must have the same amount of elements.";
599 if ((outputTensorFilesVector.size() != 0) &&
600 (outputTensorFilesVector.size() != outputNamesVector.size()))
602 ARMNN_LOG(fatal) <<
"output-name and write-outputs-to-file must have the same amount of elements.";
606 if (inputTypesVector.size() == 0)
609 inputTypesVector.assign(inputNamesVector.size(),
"float");
611 else if ((inputTypesVector.size() != 0) && (inputTypesVector.size() != inputNamesVector.size()))
613 ARMNN_LOG(fatal) <<
"input-name and input-type must have the same amount of elements.";
617 if (outputTypesVector.size() == 0)
620 outputTypesVector.assign(outputNamesVector.size(),
"float");
622 else if ((outputTypesVector.size() != 0) && (outputTypesVector.size() != outputNamesVector.size()))
624 ARMNN_LOG(fatal) <<
"output-name and output-type must have the same amount of elements.";
629 std::vector<std::unique_ptr<armnn::TensorShape>> inputTensorShapes;
631 if (!inputTensorShapesVector.empty())
633 inputTensorShapes.reserve(inputTensorShapesVector.size());
635 for(
const std::string& shape : inputTensorShapesVector)
637 std::stringstream ss(shape);
638 std::vector<unsigned int> dims = ParseArray(ss);
643 inputTensorShapes.push_back(std::make_unique<armnn::TensorShape>(dims.size(), dims.data()));
647 ARMNN_LOG(fatal) <<
"Cannot create tensor shape: " << e.
what();
654 if (thresholdTime < 0)
656 ARMNN_LOG(fatal) <<
"Threshold time supplied as a command line argument is less than zero.";
686 ARMNN_LOG(warning) <<
"No input files provided, input tensors will be filled with 0s.";
690 if (modelFormat.find(
"armnn") != std::string::npos)
692 #if defined(ARMNN_SERIALIZER) 693 return MainImpl<armnnDeserializer::IDeserializer, float>(params, runtime);
695 ARMNN_LOG(fatal) <<
"Not built with serialization support.";
699 else if (modelFormat.find(
"caffe") != std::string::npos)
701 #if defined(ARMNN_CAFFE_PARSER) 702 return MainImpl<armnnCaffeParser::ICaffeParser, float>(params, runtime);
704 ARMNN_LOG(fatal) <<
"Not built with Caffe parser support.";
708 else if (modelFormat.find(
"onnx") != std::string::npos)
710 #if defined(ARMNN_ONNX_PARSER) 711 return MainImpl<armnnOnnxParser::IOnnxParser, float>(params, runtime);
713 ARMNN_LOG(fatal) <<
"Not built with Onnx parser support.";
717 else if (modelFormat.find(
"tensorflow") != std::string::npos)
719 #if defined(ARMNN_TF_PARSER) 720 return MainImpl<armnnTfParser::ITfParser, float>(params, runtime);
722 ARMNN_LOG(fatal) <<
"Not built with Tensorflow parser support.";
726 else if(modelFormat.find(
"tflite") != std::string::npos)
728 #if defined(ARMNN_TF_LITE_PARSER) 731 ARMNN_LOG(fatal) <<
"Unknown model format: '" << modelFormat <<
"'. Only 'binary' format supported \ 735 return MainImpl<armnnTfLiteParser::ITfLiteParser, float>(params, runtime);
737 ARMNN_LOG(fatal) <<
"Unknown model format: '" << modelFormat <<
738 "'. Please include 'caffe', 'tensorflow', 'tflite' or 'onnx'";
744 ARMNN_LOG(fatal) <<
"Unknown model format: '" << modelFormat <<
745 "'. Please include 'caffe', 'tensorflow', 'tflite' or 'onnx'";
751 const bool enableProfiling,
const bool enableFp16TurboMode,
const double& thresholdTime,
752 const bool printIntermediate,
bool enableLayerDetails =
false,
bool parseUnuspported =
false)
755 std::string modelFormat;
756 std::string modelPath;
757 std::string inputNames;
758 std::string inputTensorShapes;
759 std::string inputTensorDataFilePaths;
760 std::string outputNames;
761 std::string inputTypes;
762 std::string outputTypes;
763 std::string dynamicBackendsPath;
764 std::string outputTensorFiles;
766 size_t subgraphId = 0;
768 const std::string backendsMessage = std::string(
"The preferred order of devices to run layers on by default. ")
769 + std::string(
"Possible choices: ")
772 po::options_description desc(
"Options");
776 (
"model-format,f", po::value(&modelFormat),
777 "armnn-binary, caffe-binary, caffe-text, tflite-binary, onnx-binary, onnx-text, tensorflow-binary or " 779 (
"model-path,m", po::value(&modelPath),
"Path to model file, e.g. .armnn, .caffemodel, .prototxt, " 781 (
"compute,c", po::value<std::vector<armnn::BackendId>>()->multitoken(),
782 backendsMessage.c_str())
783 (
"dynamic-backends-path,b", po::value(&dynamicBackendsPath),
784 "Path where to load any available dynamic backend from. " 785 "If left empty (the default), dynamic backends will not be used.")
786 (
"input-name,i", po::value(&inputNames),
"Identifier of the input tensors in the network separated by comma.")
787 (
"subgraph-number,n", po::value<size_t>(&subgraphId)->default_value(0),
"Id of the subgraph to be " 788 "executed. Defaults to 0.")
789 (
"input-tensor-shape,s", po::value(&inputTensorShapes),
790 "The shape of the input tensors in the network as a flat array of integers separated by comma. " 791 "Several shapes can be passed separating them by semicolon. " 792 "This parameter is optional, depending on the network.")
793 (
"input-tensor-data,d", po::value(&inputTensorDataFilePaths)->default_value(
""),
794 "Path to files containing the input data as a flat array separated by whitespace. " 795 "Several paths can be passed separating them by comma. If not specified, the network will be run with dummy " 796 "data (useful for profiling).")
797 (
"input-type,y",po::value(&inputTypes),
"The type of the input tensors in the network separated by comma. " 798 "If unset, defaults to \"float\" for all defined inputs. " 799 "Accepted values (float, int or qasymm8).")
800 (
"quantize-input,q",po::bool_switch()->default_value(
false),
801 "If this option is enabled, all float inputs will be quantized to qasymm8. " 802 "If unset, default to not quantized. " 803 "Accepted values (true or false)")
804 (
"output-type,z",po::value(&outputTypes),
"The type of the output tensors in the network separated by comma. " 805 "If unset, defaults to \"float\" for all defined outputs. " 806 "Accepted values (float, int or qasymm8).")
807 (
"output-name,o", po::value(&outputNames),
808 "Identifier of the output tensors in the network separated by comma.")
809 (
"dequantize-output,l",po::bool_switch()->default_value(
false),
810 "If this option is enabled, all quantized outputs will be dequantized to float. " 811 "If unset, default to not get dequantized. " 812 "Accepted values (true or false)")
813 (
"write-outputs-to-file,w", po::value(&outputTensorFiles),
814 "Comma-separated list of output file paths keyed with the binding-id of the output slot. " 815 "If left empty (the default), the output tensors will not be written to a file.");
817 catch (
const std::exception& e)
822 BOOST_ASSERT_MSG(
false,
"Caught unexpected exception");
823 ARMNN_LOG(fatal) <<
"Fatal internal error: " << e.what();
827 std::vector<const char*> clOptions;
828 clOptions.reserve(csvRow.
values.size());
829 for (
const std::string& value : csvRow.
values)
831 clOptions.push_back(value.c_str());
834 po::variables_map vm;
837 po::store(po::parse_command_line(static_cast<int>(clOptions.size()), clOptions.data(), desc), vm);
841 CheckOptionDependencies(vm);
843 catch (
const po::error& e)
845 std::cerr << e.what() << std::endl << std::endl;
846 std::cerr << desc << std::endl;
851 bool quantizeInput = vm[
"quantize-input"].as<
bool>();
852 bool dequantizeOutput = vm[
"dequantize-output"].as<
bool>();
855 std::vector<armnn::BackendId> computeDevices = vm[
"compute"].as<std::vector<armnn::BackendId>>();
858 RemoveDuplicateDevices(computeDevices);
861 std::string invalidBackends;
864 ARMNN_LOG(fatal) <<
"The list of preferred devices contains invalid backend IDs: " 869 return RunTest(modelFormat, inputTensorShapes, computeDevices, dynamicBackendsPath, modelPath, inputNames,
870 inputTensorDataFilePaths, inputTypes, quantizeInput, outputTypes, outputNames, outputTensorFiles,
871 dequantizeOutput, enableProfiling, enableFp16TurboMode, thresholdTime, printIntermediate, subgraphId,
872 enableLayerDetails, parseUnuspported);
std::vector< string > m_OutputTypes
float Dequantize(QuantizedType value, float scale, int32_t offset)
Dequantize an 8-bit data type into a floating point data type.
std::vector< string > m_OutputTensorFiles
std::unique_ptr< armnn::TensorShape > TensorShapePtr
int RunTest(const std::string &format, const std::string &inputTensorShapesStr, const vector< armnn::BackendId > &computeDevices, const std::string &dynamicBackendsPath, const std::string &path, const std::string &inputNames, const std::string &inputTensorDataFilePaths, const std::string &inputTypes, bool quantizeInput, const std::string &outputTypes, const std::string &outputNames, const std::string &outputTensorFiles, bool dequantizeOuput, bool enableProfiling, bool enableFp16TurboMode, const double &thresholdTime, bool printIntermediate, const size_t subgraphId, bool enableLayerDetails=false, bool parseUnsupported=false, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
std::vector< TensorShapePtr > m_InputTensorShapes
QuantizationParams GetInputQuantizationParams(unsigned int inputIndex=0u) const
const std::vector< armnn::BindingPointInfo > & GetOutputBindingInfos() const
bool m_EnableFp16TurboMode
std::string m_DynamicBackendsPath
std::vector< std::string > values
int RunCsvTest(const armnnUtils::CsvRow &csvRow, const std::shared_ptr< armnn::IRuntime > &runtime, const bool enableProfiling, const bool enableFp16TurboMode, const double &thresholdTime, const bool printIntermediate, bool enableLayerDetails=false, bool parseUnuspported=false)
std::string m_DynamicBackendsPath
virtual const char * what() const noexcept override
#define ARMNN_LOG(severity)
BackendRegistry & BackendRegistryInstance()
bool m_EnableFp16TurboMode
void IgnoreUnused(Ts &&...)
std::string GetBackendIdsAsString() const
bool m_VisualizePostOptimizationModel
unsigned int GetOutputSize(unsigned int outputIndex=0u) const
boost::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > > TContainer
std::vector< std::string > m_InputBindings
std::vector< armnn::BackendId > m_ComputeDevices
std::vector< string > m_InputNames
std::vector< armnn::TensorShape > m_InputShapes
bool m_GenerateTensorData
std::vector< std::string > m_OutputBindings
std::vector< armnn::BackendId > m_ComputeDevices
bool has_value() const noexcept
std::vector< string > m_OutputNames
std::enable_if_t< std::is_unsigned< Source >::value &&std::is_unsigned< Dest >::value, Dest > numeric_cast(Source source)
std::pair< float, int32_t > QuantizationParams
std::vector< string > m_InputTensorDataFilePaths
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
bool m_PrintIntermediateLayers
std::chrono::duration< double, std::milli > Run(const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
bool m_EnableLayerDetails
Base class for all ArmNN exceptions so that users can filter to just those.
std::vector< string > m_InputTypes
unsigned int GetInputSize(unsigned int inputIndex=0u) const
int MainImpl(const ExecuteNetworkParams ¶ms, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)