14 #if defined(ARMNN_SERIALIZER) 17 #if defined(ARMNN_TF_LITE_PARSER) 20 #if defined(ARMNN_ONNX_PARSER) 28 #include <boost/exception/exception.hpp> 29 #include <boost/exception/diagnostic_information.hpp> 30 #include <boost/format.hpp> 31 #include <boost/program_options.hpp> 32 #include <boost/filesystem.hpp> 33 #include <boost/variant.hpp> 41 #include <type_traits> 46 inline bool CheckRequestedBackendsAreValid(
const std::vector<armnn::BackendId>& backendIds,
49 if (backendIds.empty())
57 for (
const auto& backendId : backendIds)
59 if (std::find(validBackendIds.begin(), validBackendIds.end(), backendId) == validBackendIds.end())
62 if (invalidBackendIds)
64 if (!invalidBackendIds.value().empty())
66 invalidBackendIds.value() +=
", ";
68 invalidBackendIds.value() += backendId;
113 template <
typename IParser>
120 std::vector<armnn::BindingPointInfo>& inputBindings,
121 std::vector<armnn::BindingPointInfo>& outputBindings)
126 auto parser(IParser::Create());
128 std::map<std::string, armnn::TensorShape> inputShapes;
133 if (numInputShapes < numInputBindings)
136 "Not every input has its tensor shape specified: expected=%1%, got=%2%")
137 % numInputBindings % numInputShapes));
140 for (
size_t i = 0; i < numInputShapes; i++)
153 parser->CreateNetworkFromBinaryFile(modelPath.c_str(), inputShapes, requestedOutputs) :
154 parser->CreateNetworkFromTextFile(modelPath.c_str(), inputShapes, requestedOutputs));
159 inputBindings.push_back(parser->GetNetworkInputBindingInfo(inputLayerName));
164 outputBindings.push_back(parser->GetNetworkOutputBindingInfo(outputLayerName));
171 #if defined(ARMNN_SERIALIZER) 180 std::vector<armnn::BindingPointInfo>& inputBindings,
181 std::vector<armnn::BindingPointInfo>& outputBindings)
183 auto parser(IParser::Create());
191 boost::system::error_code errorCode;
192 boost::filesystem::path pathToFile(params.
m_ModelPath);
193 if (!boost::filesystem::exists(pathToFile, errorCode))
196 boost::format(
"Cannot find the file (%1%) errorCode: %2% %3%") %
201 std::ifstream file(params.
m_ModelPath, std::ios::binary);
203 network = parser->CreateNetworkFromBinary(file);
211 parser->GetNetworkInputBindingInfo(subgraphId, inputLayerName);
218 parser->GetNetworkOutputBindingInfo(subgraphId, outputLayerName);
227 #if defined(ARMNN_TF_LITE_PARSER) 236 std::vector<armnn::BindingPointInfo>& inputBindings,
237 std::vector<armnn::BindingPointInfo>& outputBindings)
242 IParser::TfLiteParserOptions
options;
244 auto parser(IParser::Create(options));
250 network = parser->CreateNetworkFromBinaryFile(modelPath.c_str());
256 parser->GetNetworkInputBindingInfo(params.
m_SubgraphId, inputLayerName);
257 inputBindings.push_back(inputBinding);
263 parser->GetNetworkOutputBindingInfo(params.
m_SubgraphId, outputLayerName);
264 outputBindings.push_back(outputBinding);
272 #if defined(ARMNN_ONNX_PARSER) 282 std::vector<BindingPointInfo>& inputBindings,
283 std::vector<BindingPointInfo>& outputBindings)
288 auto parser(IParser::Create());
295 parser->CreateNetworkFromBinaryFile(modelPath.c_str()) :
296 parser->CreateNetworkFromTextFile(modelPath.c_str()));
301 BindingPointInfo inputBinding = parser->GetNetworkInputBindingInfo(inputLayerName);
302 inputBindings.push_back(inputBinding);
307 BindingPointInfo outputBinding = parser->GetNetworkOutputBindingInfo(outputLayerName);
308 outputBindings.push_back(outputBinding);
318 template <
typename IParser,
typename TDataType>
325 using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;
339 std::vector<armnn::BackendId> backendIds;
340 std::copy(m_ComputeDevices.begin(), m_ComputeDevices.end(), std::back_inserter(backendIds));
347 namespace po = boost::program_options;
349 const std::vector<std::string> defaultComputes = {
"CpuAcc",
"CpuRef" };
351 const std::string backendsMessage =
"Which device to run layers on by default. Possible choices: " 355 (
"model-dir,m", po::value<std::string>(&options.
m_ModelDir)->required(),
356 "Path to directory containing model files (.caffemodel/.prototxt/.tflite)")
357 (
"compute,c", po::value<std::vector<std::string>>(&options.
m_ComputeDevices)->
359 multitoken(), backendsMessage.c_str())
361 "Path where to load any available dynamic backend from. " 362 "If left empty (the default), dynamic backends will not be used.")
363 (
"labels,l", po::value<std::string>(&options.
m_Labels),
364 "Text file containing one image filename - correct label pair per line, " 365 "used to test the accuracy of the network.")
366 (
"visualize-optimized-model,v",
368 "Produce a dot file useful for visualizing the graph post optimization." 369 "The file will have the same name as the model with the .dot extention.")
371 "If this option is enabled FP32 layers, weights and biases will be converted " 372 "to FP16 where the backend supports it.")
374 "If this option is enabled FP32 layers, weights and biases will be converted " 375 "to BF16 where the backend supports it.");
379 bool enableProfiling,
380 const std::string& dynamicBackendsPath,
381 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
397 std::string invalidBackends;
406 ARMNN_LOG(info) <<
"Network parsing time: " << std::setprecision(2)
421 ARMNN_LOG(info) <<
"Optimization time: " << std::setprecision(2)
432 boost::filesystem::path filename = params.
m_ModelPath;
433 filename.replace_extension(
"dot");
434 std::fstream file(filename.c_str(), std::ios_base::out);
435 optNet->SerializeToDot(file);
441 ret = m_Runtime->LoadNetwork(m_NetworkIdentifier, std::move(optNet));
454 throw armnn::Exception(boost::str(boost::format(
"Input index out of range: %1%") % inputIndex));
462 throw armnn::Exception(boost::str(boost::format(
"Output index out of range: %1%") % outputIndex));
468 CheckInputIndexIsValid(inputIndex);
474 CheckOutputIndexIsValid(outputIndex);
478 std::chrono::duration<double, std::milli>
Run(
479 const std::vector<TContainer>& inputContainers,
480 std::vector<TContainer>& outputContainers)
482 for (
unsigned int i = 0; i < outputContainers.size(); ++i)
484 const unsigned int expectedOutputDataSize = GetOutputSize(i);
486 boost::apply_visitor([expectedOutputDataSize, i](
auto&& value)
489 if (actualOutputDataSize < expectedOutputDataSize)
493 boost::str(boost::format(
"Not enough data for output #%1%: expected " 494 "%2% elements, got %3%") % outputIndex % expectedOutputDataSize % actualOutputDataSize));
497 outputContainers[i]);
500 std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);
509 armnn::Status ret = m_Runtime->EnqueueWorkload(m_NetworkIdentifier,
516 if (profiler && profiler->IsProfilingEnabled())
518 profiler->Print(std::cout);
533 CheckInputIndexIsValid(inputIndex);
544 CheckOutputIndexIsValid(outputIndex);
555 CheckOutputIndexIsValid(outputIndex);
556 return std::make_pair(
m_OutputBindings[outputIndex].second.GetQuantizationScale(),
562 CheckInputIndexIsValid(inputIndex);
563 return std::make_pair(
m_InputBindings[inputIndex].second.GetQuantizationScale(),
569 std::vector<QuantizationParams> quantizationParams;
572 quantizationParams.push_back(GetQuantizationParams(i));
574 return quantizationParams;
579 std::shared_ptr<armnn::IRuntime> m_Runtime;
586 template<
typename TContainer>
592 template<
typename TContainer>
boost::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > > TContainer
static IRuntimePtr Create(const CreationOptions &options)
options m_EnableProfiling
BackendIdSet GetBackendIds() const
std::chrono::duration< double, std::milli > GetTimeDuration(std::chrono::high_resolution_clock::time_point start_time)
std::unordered_set< BackendId > BackendIdSet
QuantizationParams GetInputQuantizationParams(unsigned int inputIndex=0u) const
bool m_EnableFp16TurboMode
const std::vector< armnn::BindingPointInfo > & GetOutputBindingInfos() const
std::string m_DynamicBackendsPath
const armnn::BindingPointInfo & GetOutputBindingInfo(unsigned int outputIndex=0u) const
#define ARMNN_LOG(severity)
Main network class which provides the interface for building up a neural network. ...
BackendRegistry & BackendRegistryInstance()
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
const armnn::BindingPointInfo & GetInputBindingInfo(unsigned int inputIndex=0u) const
armnn::BindingPointInfo BindingPointInfo
bool m_EnableFp16TurboMode
std::chrono::high_resolution_clock::time_point GetTimeNow()
InferenceModelInternal::QuantizationParams QuantizationParams
std::string m_DynamicBackendsPath
std::string GetBackendIdsAsString() const
bool m_VisualizePostOptimizationModel
armnn::OutputTensors MakeOutputTensors(const std::vector< armnn::BindingPointInfo > &outputBindings, const std::vector< TContainer > &outputDataContainers)
void CheckInputIndexIsValid(unsigned int inputIndex) const
bool m_EnableBf16TurboMode
unsigned int GetOutputSize(unsigned int outputIndex=0u) const
std::vector< std::string > m_InputBindings
InferenceModel(const Params ¶ms, bool enableProfiling, const std::string &dynamicBackendsPath, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
std::vector< armnn::TensorShape > m_InputShapes
armnn::InputTensors MakeInputTensors(const std::vector< armnn::BindingPointInfo > &inputBindings, const std::vector< TContainer > &inputDataContainers)
IOptimizedNetworkPtr Optimize(const INetwork &network, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options=OptimizerOptions(), Optional< std::vector< std::string > &> messages=EmptyOptional())
Create an optimized version of the network.
std::vector< std::string > m_OutputBindings
std::vector< armnn::BackendId > m_ComputeDevices
#define ARMNN_SCOPED_HEAP_PROFILING(TAG)
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
QuantizationParams GetQuantizationParams(unsigned int outputIndex=0u) const
#define ARMNN_ASSERT(COND)
std::enable_if_t< std::is_unsigned< Source >::value &&std::is_unsigned< Dest >::value, Dest > numeric_cast(Source source)
std::vector< QuantizationParams > GetAllQuantizationParams() const
std::pair< float, int32_t > QuantizationParams
armnn::OutputTensors MakeOutputTensors(const std::vector< armnn::BindingPointInfo > &outputBindings, std::vector< TContainer > &outputDataContainers)
bool m_EnableBf16TurboMode
bool m_VisualizePostOptimizationModel
std::string m_DynamicBackendsPath
Setting this value will override the paths set by the DYNAMIC_BACKEND_PATHS compiler directive Only a...
bool m_EnableGpuProfiling
Setting this flag will allow the user to obtain GPU profiling information from the runtime...
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
bool m_PrintIntermediateLayers
std::pair< armnn::LayerBindingId, armnn::TensorInfo > BindingPointInfo
static armnn::INetworkPtr Create(const Params ¶ms, std::vector< armnn::BindingPointInfo > &inputBindings, std::vector< armnn::BindingPointInfo > &outputBindings)
std::chrono::duration< double, std::milli > Run(const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
std::vector< armnn::BackendId > GetComputeDevicesAsBackendIds()
armnn::TensorInfo m_TensorInfo
Base class for all ArmNN exceptions so that users can filter to just those.
std::vector< std::string > m_ComputeDevices
unsigned int GetInputSize(unsigned int inputIndex=0u) const
static void AddCommandLineOptions(boost::program_options::options_description &desc, CommandLineOptions &options)
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
armnn::LayerBindingId m_BindingId
armnn::Runtime::CreationOptions::ExternalProfilingOptions options
std::string StringConcat(const std::vector< std::string > &strings, std::string seperator="")
Takes a vector of strings and concatenates them together into one long std::string with an optional s...
armnn::InputTensors MakeInputTensors(const std::vector< armnn::BindingPointInfo > &inputBindings, const std::vector< TContainer > &inputDataContainers)
void CheckOutputIndexIsValid(unsigned int outputIndex) const
const std::vector< armnn::BindingPointInfo > & GetInputBindingInfos() const