11 #if defined(ARMNN_SERIALIZER) 14 #if defined(ARMNN_TF_LITE_PARSER) 17 #if defined(ARMNN_ONNX_PARSER) 24 #include <boost/algorithm/string/join.hpp> 25 #include <boost/exception/exception.hpp> 26 #include <boost/exception/diagnostic_information.hpp> 27 #include <boost/format.hpp> 28 #include <boost/program_options.hpp> 29 #include <boost/filesystem.hpp> 30 #include <boost/lexical_cast.hpp> 31 #include <boost/variant.hpp> 40 #include <type_traits> 45 inline bool CheckRequestedBackendsAreValid(
const std::vector<armnn::BackendId>& backendIds,
48 if (backendIds.empty())
56 for (
const auto& backendId : backendIds)
58 if (std::find(validBackendIds.begin(), validBackendIds.end(), backendId) == validBackendIds.end())
61 if (invalidBackendIds)
63 if (!invalidBackendIds.value().empty())
65 invalidBackendIds.value() +=
", ";
67 invalidBackendIds.value() += backendId;
110 template <
typename IParser>
117 std::vector<armnn::BindingPointInfo>& inputBindings,
118 std::vector<armnn::BindingPointInfo>& outputBindings)
123 auto parser(IParser::Create());
125 std::map<std::string, armnn::TensorShape> inputShapes;
130 if (numInputShapes < numInputBindings)
133 "Not every input has its tensor shape specified: expected=%1%, got=%2%")
134 % numInputBindings % numInputShapes));
137 for (
size_t i = 0; i < numInputShapes; i++)
150 parser->CreateNetworkFromBinaryFile(modelPath.c_str(), inputShapes, requestedOutputs) :
151 parser->CreateNetworkFromTextFile(modelPath.c_str(), inputShapes, requestedOutputs));
156 inputBindings.push_back(parser->GetNetworkInputBindingInfo(inputLayerName));
161 outputBindings.push_back(parser->GetNetworkOutputBindingInfo(outputLayerName));
168 #if defined(ARMNN_SERIALIZER) 177 std::vector<armnn::BindingPointInfo>& inputBindings,
178 std::vector<armnn::BindingPointInfo>& outputBindings)
180 auto parser(IParser::Create());
181 BOOST_ASSERT(parser);
188 boost::system::error_code errorCode;
189 boost::filesystem::path pathToFile(params.
m_ModelPath);
190 if (!boost::filesystem::exists(pathToFile, errorCode))
193 boost::format(
"Cannot find the file (%1%) errorCode: %2% %3%") %
198 std::ifstream file(params.
m_ModelPath, std::ios::binary);
200 network = parser->CreateNetworkFromBinary(file);
203 unsigned int subgraphId = boost::numeric_cast<
unsigned int>(params.
m_SubgraphId);
208 parser->GetNetworkInputBindingInfo(subgraphId, inputLayerName);
215 parser->GetNetworkOutputBindingInfo(subgraphId, outputLayerName);
224 #if defined(ARMNN_TF_LITE_PARSER) 233 std::vector<armnn::BindingPointInfo>& inputBindings,
234 std::vector<armnn::BindingPointInfo>& outputBindings)
239 IParser::TfLiteParserOptions
options;
241 auto parser(IParser::Create(options));
247 network = parser->CreateNetworkFromBinaryFile(modelPath.c_str());
253 parser->GetNetworkInputBindingInfo(params.
m_SubgraphId, inputLayerName);
254 inputBindings.push_back(inputBinding);
260 parser->GetNetworkOutputBindingInfo(params.
m_SubgraphId, outputLayerName);
261 outputBindings.push_back(outputBinding);
269 #if defined(ARMNN_ONNX_PARSER) 279 std::vector<BindingPointInfo>& inputBindings,
280 std::vector<BindingPointInfo>& outputBindings)
285 auto parser(IParser::Create());
292 parser->CreateNetworkFromBinaryFile(modelPath.c_str()) :
293 parser->CreateNetworkFromTextFile(modelPath.c_str()));
298 BindingPointInfo inputBinding = parser->GetNetworkInputBindingInfo(inputLayerName);
299 inputBindings.push_back(inputBinding);
304 BindingPointInfo outputBinding = parser->GetNetworkOutputBindingInfo(outputLayerName);
305 outputBindings.push_back(outputBinding);
315 template <
typename IParser,
typename TDataType>
322 using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;
335 std::vector<armnn::BackendId> backendIds;
336 std::copy(m_ComputeDevices.begin(), m_ComputeDevices.end(), std::back_inserter(backendIds));
343 namespace po = boost::program_options;
345 const std::vector<std::string> defaultComputes = {
"CpuAcc",
"CpuRef" };
347 const std::string backendsMessage =
"Which device to run layers on by default. Possible choices: " 351 (
"model-dir,m", po::value<std::string>(&options.
m_ModelDir)->required(),
352 "Path to directory containing model files (.caffemodel/.prototxt/.tflite)")
353 (
"compute,c", po::value<std::vector<std::string>>(&options.
m_ComputeDevices)->
354 default_value(defaultComputes, boost::algorithm::join(defaultComputes,
", "))->
355 multitoken(), backendsMessage.c_str())
357 "Path where to load any available dynamic backend from. " 358 "If left empty (the default), dynamic backends will not be used.")
359 (
"labels,l", po::value<std::string>(&options.
m_Labels),
360 "Text file containing one image filename - correct label pair per line, " 361 "used to test the accuracy of the network.")
362 (
"visualize-optimized-model,v",
364 "Produce a dot file useful for visualizing the graph post optimization." 365 "The file will have the same name as the model with the .dot extention.")
367 "If this option is enabled FP32 layers, weights and biases will be converted " 368 "to FP16 where the backend supports it.");
372 bool enableProfiling,
373 const std::string& dynamicBackendsPath,
374 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
390 std::string invalidBackends;
415 boost::filesystem::path filename = params.
m_ModelPath;
416 filename.replace_extension(
"dot");
417 std::fstream file(filename.c_str(), std::ios_base::out);
418 optNet->SerializeToDot(file);
424 ret = m_Runtime->LoadNetwork(m_NetworkIdentifier, std::move(optNet));
437 throw armnn::Exception(boost::str(boost::format(
"Input index out of range: %1%") % inputIndex));
445 throw armnn::Exception(boost::str(boost::format(
"Output index out of range: %1%") % outputIndex));
451 CheckInputIndexIsValid(inputIndex);
457 CheckOutputIndexIsValid(outputIndex);
461 std::chrono::duration<double, std::milli>
Run(
462 const std::vector<TContainer>& inputContainers,
463 std::vector<TContainer>& outputContainers)
465 for (
unsigned int i = 0; i < outputContainers.size(); ++i)
467 const unsigned int expectedOutputDataSize = GetOutputSize(i);
469 boost::apply_visitor([expectedOutputDataSize, i](
auto&& value)
471 const unsigned int actualOutputDataSize = boost::numeric_cast<
unsigned int>(value.size());
472 if (actualOutputDataSize < expectedOutputDataSize)
474 unsigned int outputIndex = boost::numeric_cast<
unsigned int>(i);
476 boost::str(boost::format(
"Not enough data for output #%1%: expected " 477 "%2% elements, got %3%") % outputIndex % expectedOutputDataSize % actualOutputDataSize));
480 outputContainers[i]);
483 std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);
490 const auto start_time = GetCurrentTime();
492 armnn::Status ret = m_Runtime->EnqueueWorkload(m_NetworkIdentifier,
496 const auto end_time = GetCurrentTime();
499 if (profiler && profiler->IsProfilingEnabled())
501 profiler->Print(std::cout);
510 return std::chrono::duration<double, std::milli>(end_time - start_time);
516 CheckInputIndexIsValid(inputIndex);
527 CheckOutputIndexIsValid(outputIndex);
538 CheckOutputIndexIsValid(outputIndex);
539 return std::make_pair(
m_OutputBindings[outputIndex].second.GetQuantizationScale(),
545 CheckInputIndexIsValid(inputIndex);
546 return std::make_pair(
m_InputBindings[inputIndex].second.GetQuantizationScale(),
552 std::vector<QuantizationParams> quantizationParams;
555 quantizationParams.push_back(GetQuantizationParams(i));
557 return quantizationParams;
562 std::shared_ptr<armnn::IRuntime> m_Runtime;
569 template<
typename TContainer>
575 template<
typename TContainer>
581 std::chrono::high_resolution_clock::time_point GetCurrentTime()
583 return std::chrono::high_resolution_clock::now();
586 std::chrono::duration<double, std::milli> GetTimeDuration(
587 std::chrono::high_resolution_clock::time_point& start_time,
588 std::chrono::high_resolution_clock::time_point& end_time)
590 return std::chrono::duration<double, std::milli>(end_time - start_time);
armnn::OutputTensors MakeOutputTensors(const std::vector< armnn::BindingPointInfo > &outputBindings, std::vector< TContainer > &outputDataContainers)
void CheckOutputIndexIsValid(unsigned int outputIndex) const
std::string GetBackendIdsAsString() const
bool m_VisualizePostOptimizationModel
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
bool m_EnableFp16TurboMode
IOptimizedNetworkPtr Optimize(const INetwork &network, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options=OptimizerOptions(), Optional< std::vector< std::string > &> messages=EmptyOptional())
unsigned int GetInputSize(unsigned int inputIndex=0u) const
const std::vector< armnn::BindingPointInfo > & GetInputBindingInfos() const
bool m_VisualizePostOptimizationModel
armnn::BindingPointInfo BindingPointInfo
std::pair< armnn::LayerBindingId, armnn::TensorInfo > BindingPointInfo
const armnn::BindingPointInfo & GetOutputBindingInfo(unsigned int outputIndex=0u) const
armnn::InputTensors MakeInputTensors(const std::vector< armnn::BindingPointInfo > &inputBindings, const std::vector< TContainer > &inputDataContainers)
std::vector< armnn::BackendId > GetComputeDevicesAsBackendIds()
std::pair< float, int32_t > QuantizationParams
void CheckInputIndexIsValid(unsigned int inputIndex) const
std::chrono::duration< double, std::milli > Run(const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
std::vector< std::string > m_InputBindings
std::vector< QuantizationParams > GetAllQuantizationParams() const
std::vector< std::string > m_ComputeDevices
const std::vector< armnn::BindingPointInfo > & GetOutputBindingInfos() const
BackendRegistry & BackendRegistryInstance()
BackendIdSet GetBackendIds() const
bool m_PrintIntermediateLayers
std::vector< armnn::BackendId > m_ComputeDevices
InferenceModel(const Params ¶ms, bool enableProfiling, const std::string &dynamicBackendsPath, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
QuantizationParams GetQuantizationParams(unsigned int outputIndex=0u) const
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
const armnn::BindingPointInfo & GetInputBindingInfo(unsigned int inputIndex=0u) const
#define ARMNN_SCOPED_HEAP_PROFILING(TAG)
options m_EnableProfiling
bool m_EnableGpuProfiling
bool m_EnableFp16TurboMode
QuantizationParams GetInputQuantizationParams(unsigned int inputIndex=0u) const
Base class for all ArmNN exceptions so that users can filter to just those.
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
static IRuntimePtr Create(const CreationOptions &options)
std::string m_DynamicBackendsPath
std::string m_DynamicBackendsPath
static armnn::INetworkPtr Create(const Params ¶ms, std::vector< armnn::BindingPointInfo > &inputBindings, std::vector< armnn::BindingPointInfo > &outputBindings)
static void AddCommandLineOptions(boost::program_options::options_description &desc, CommandLineOptions &options)
std::string m_DynamicBackendsPath
std::unordered_set< BackendId > BackendIdSet
armnn::TensorInfo m_TensorInfo
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
unsigned int GetOutputSize(unsigned int outputIndex=0u) const
armnn::LayerBindingId m_BindingId
std::vector< std::string > m_OutputBindings
std::vector< armnn::TensorShape > m_InputShapes
boost::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > > TContainer
armnn::Runtime::CreationOptions::ExternalProfilingOptions options
InferenceModelInternal::QuantizationParams QuantizationParams