14 #if defined(ARMNN_SERIALIZER) 17 #if defined(ARMNN_TF_LITE_PARSER) 20 #if defined(ARMNN_ONNX_PARSER) 29 #include <boost/exception/exception.hpp> 30 #include <boost/exception/diagnostic_information.hpp> 31 #include <boost/format.hpp> 32 #include <boost/program_options.hpp> 33 #include <boost/variant.hpp> 41 #include <type_traits> 46 inline bool CheckRequestedBackendsAreValid(
const std::vector<armnn::BackendId>& backendIds,
49 if (backendIds.empty())
57 for (
const auto& backendId : backendIds)
59 if (std::find(validBackendIds.begin(), validBackendIds.end(), backendId) == validBackendIds.end())
62 if (invalidBackendIds)
64 if (!invalidBackendIds.value().empty())
66 invalidBackendIds.value() +=
", ";
68 invalidBackendIds.value() += backendId;
115 template <
typename IParser>
122 std::vector<armnn::BindingPointInfo>& inputBindings,
123 std::vector<armnn::BindingPointInfo>& outputBindings)
128 auto parser(IParser::Create());
130 std::map<std::string, armnn::TensorShape> inputShapes;
135 if (numInputShapes < numInputBindings)
138 "Not every input has its tensor shape specified: expected=%1%, got=%2%")
139 % numInputBindings % numInputShapes));
142 for (
size_t i = 0; i < numInputShapes; i++)
155 parser->CreateNetworkFromBinaryFile(modelPath.c_str(), inputShapes, requestedOutputs) :
156 parser->CreateNetworkFromTextFile(modelPath.c_str(), inputShapes, requestedOutputs));
161 inputBindings.push_back(parser->GetNetworkInputBindingInfo(inputLayerName));
166 outputBindings.push_back(parser->GetNetworkOutputBindingInfo(outputLayerName));
173 #if defined(ARMNN_SERIALIZER) 182 std::vector<armnn::BindingPointInfo>& inputBindings,
183 std::vector<armnn::BindingPointInfo>& outputBindings)
185 auto parser(IParser::Create());
193 std::error_code errorCode;
195 if (!fs::exists(pathToFile, errorCode))
198 boost::format(
"Cannot find the file (%1%) errorCode: %2% %3%") %
203 std::ifstream file(params.
m_ModelPath, std::ios::binary);
205 network = parser->CreateNetworkFromBinary(file);
213 parser->GetNetworkInputBindingInfo(subgraphId, inputLayerName);
220 parser->GetNetworkOutputBindingInfo(subgraphId, outputLayerName);
229 #if defined(ARMNN_TF_LITE_PARSER) 238 std::vector<armnn::BindingPointInfo>& inputBindings,
239 std::vector<armnn::BindingPointInfo>& outputBindings)
244 IParser::TfLiteParserOptions options;
247 auto parser(IParser::Create(options));
253 network = parser->CreateNetworkFromBinaryFile(modelPath.c_str());
259 parser->GetNetworkInputBindingInfo(params.
m_SubgraphId, inputLayerName);
260 inputBindings.push_back(inputBinding);
266 parser->GetNetworkOutputBindingInfo(params.
m_SubgraphId, outputLayerName);
267 outputBindings.push_back(outputBinding);
275 #if defined(ARMNN_ONNX_PARSER) 285 std::vector<BindingPointInfo>& inputBindings,
286 std::vector<BindingPointInfo>& outputBindings)
291 auto parser(IParser::Create());
298 parser->CreateNetworkFromBinaryFile(modelPath.c_str()) :
299 parser->CreateNetworkFromTextFile(modelPath.c_str()));
304 BindingPointInfo inputBinding = parser->GetNetworkInputBindingInfo(inputLayerName);
305 inputBindings.push_back(inputBinding);
310 BindingPointInfo outputBinding = parser->GetNetworkOutputBindingInfo(outputLayerName);
311 outputBindings.push_back(outputBinding);
321 template <
typename IParser,
typename TDataType>
328 using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;
342 std::vector<armnn::BackendId> backendIds;
343 std::copy(m_ComputeDevices.begin(), m_ComputeDevices.end(), std::back_inserter(backendIds));
350 namespace po = boost::program_options;
352 const std::vector<std::string> defaultComputes = {
"CpuAcc",
"CpuRef" };
354 const std::string backendsMessage =
"Which device to run layers on by default. Possible choices: " 358 (
"model-dir,m", po::value<std::string>(&options.
m_ModelDir)->required(),
359 "Path to directory containing model files (.caffemodel/.prototxt/.tflite)")
360 (
"compute,c", po::value<std::vector<std::string>>(&options.
m_ComputeDevices)->
362 multitoken(), backendsMessage.c_str())
364 "Path where to load any available dynamic backend from. " 365 "If left empty (the default), dynamic backends will not be used.")
366 (
"labels,l", po::value<std::string>(&options.
m_Labels),
367 "Text file containing one image filename - correct label pair per line, " 368 "used to test the accuracy of the network.")
369 (
"visualize-optimized-model,v",
371 "Produce a dot file useful for visualizing the graph post optimization." 372 "The file will have the same name as the model with the .dot extention.")
374 "If this option is enabled FP32 layers, weights and biases will be converted " 375 "to FP16 where the backend supports it.")
377 "If this option is enabled FP32 layers, weights and biases will be converted " 378 "to BF16 where the backend supports it.");
382 bool enableProfiling,
383 const std::string& dynamicBackendsPath,
384 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
385 : m_EnableProfiling(enableProfiling)
400 std::string invalidBackends;
409 ARMNN_LOG(info) <<
"Network parsing time: " << std::setprecision(2)
424 ARMNN_LOG(info) <<
"Optimization time: " << std::setprecision(2)
436 filename.replace_extension(
"dot");
437 std::fstream file(filename.c_str(), std::ios_base::out);
438 optNet->SerializeToDot(file);
444 ret = m_Runtime->LoadNetwork(m_NetworkIdentifier, std::move(optNet));
457 throw armnn::Exception(boost::str(boost::format(
"Input index out of range: %1%") % inputIndex));
465 throw armnn::Exception(boost::str(boost::format(
"Output index out of range: %1%") % outputIndex));
471 CheckInputIndexIsValid(inputIndex);
477 CheckOutputIndexIsValid(outputIndex);
481 std::chrono::duration<double, std::milli>
Run(
482 const std::vector<TContainer>& inputContainers,
483 std::vector<TContainer>& outputContainers)
485 for (
unsigned int i = 0; i < outputContainers.size(); ++i)
487 const unsigned int expectedOutputDataSize = GetOutputSize(i);
489 boost::apply_visitor([expectedOutputDataSize, i](
auto&& value)
492 if (actualOutputDataSize < expectedOutputDataSize)
496 boost::str(boost::format(
"Not enough data for output #%1%: expected " 497 "%2% elements, got %3%") % outputIndex % expectedOutputDataSize % actualOutputDataSize));
500 outputContainers[i]);
503 std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);
506 profiler->EnableProfiling(m_EnableProfiling);
512 armnn::Status ret = m_Runtime->EnqueueWorkload(m_NetworkIdentifier,
519 if (profiler && profiler->IsProfilingEnabled())
521 profiler->Print(std::cout);
536 CheckInputIndexIsValid(inputIndex);
547 CheckOutputIndexIsValid(outputIndex);
558 CheckOutputIndexIsValid(outputIndex);
559 return std::make_pair(
m_OutputBindings[outputIndex].second.GetQuantizationScale(),
565 CheckInputIndexIsValid(inputIndex);
566 return std::make_pair(
m_InputBindings[inputIndex].second.GetQuantizationScale(),
572 std::vector<QuantizationParams> quantizationParams;
575 quantizationParams.push_back(GetQuantizationParams(i));
577 return quantizationParams;
582 std::shared_ptr<armnn::IRuntime> m_Runtime;
586 bool m_EnableProfiling;
589 template<
typename TContainer>
595 template<
typename TContainer>
boost::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > > TContainer
static IRuntimePtr Create(const CreationOptions &options)
BackendIdSet GetBackendIds() const
std::chrono::duration< double, std::milli > GetTimeDuration(std::chrono::high_resolution_clock::time_point start_time)
std::unordered_set< BackendId > BackendIdSet
QuantizationParams GetInputQuantizationParams(unsigned int inputIndex=0u) const
bool m_EnableFp16TurboMode
const std::vector< armnn::BindingPointInfo > & GetOutputBindingInfos() const
armnn::InputTensors MakeInputTensors(const std::vector< armnn::BindingPointInfo > &inputBindings, const std::vector< std::reference_wrapper< TContainer >> &inputDataContainers)
std::string m_DynamicBackendsPath
const armnn::BindingPointInfo & GetOutputBindingInfo(unsigned int outputIndex=0u) const
#define ARMNN_LOG(severity)
Main network class which provides the interface for building up a neural network. ...
BackendRegistry & BackendRegistryInstance()
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
const armnn::BindingPointInfo & GetInputBindingInfo(unsigned int inputIndex=0u) const
armnn::BindingPointInfo BindingPointInfo
bool m_EnableFp16TurboMode
std::chrono::high_resolution_clock::time_point GetTimeNow()
InferenceModelInternal::QuantizationParams QuantizationParams
std::string m_DynamicBackendsPath
std::string GetBackendIdsAsString() const
bool m_VisualizePostOptimizationModel
void CheckInputIndexIsValid(unsigned int inputIndex) const
bool m_EnableBf16TurboMode
unsigned int GetOutputSize(unsigned int outputIndex=0u) const
std::vector< std::string > m_InputBindings
InferenceModel(const Params ¶ms, bool enableProfiling, const std::string &dynamicBackendsPath, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
std::vector< armnn::TensorShape > m_InputShapes
armnn::InputTensors MakeInputTensors(const std::vector< armnn::BindingPointInfo > &inputBindings, const std::vector< TContainer > &inputDataContainers)
IOptimizedNetworkPtr Optimize(const INetwork &network, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options=OptimizerOptions(), Optional< std::vector< std::string > &> messages=EmptyOptional())
Create an optimized version of the network.
std::vector< std::string > m_OutputBindings
std::vector< armnn::BackendId > m_ComputeDevices
#define ARMNN_SCOPED_HEAP_PROFILING(TAG)
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
QuantizationParams GetQuantizationParams(unsigned int outputIndex=0u) const
#define ARMNN_ASSERT(COND)
std::enable_if_t< std::is_unsigned< Source >::value &&std::is_unsigned< Dest >::value, Dest > numeric_cast(Source source)
std::vector< QuantizationParams > GetAllQuantizationParams() const
std::pair< float, int32_t > QuantizationParams
armnn::OutputTensors MakeOutputTensors(const std::vector< armnn::BindingPointInfo > &outputBindings, std::vector< TContainer > &outputDataContainers)
bool m_EnableBf16TurboMode
bool m_VisualizePostOptimizationModel
std::string m_DynamicBackendsPath
Setting this value will override the paths set by the DYNAMIC_BACKEND_PATHS compiler directive Only a...
bool m_EnableGpuProfiling
Setting this flag will allow the user to obtain GPU profiling information from the runtime...
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
bool m_PrintIntermediateLayers
std::pair< armnn::LayerBindingId, armnn::TensorInfo > BindingPointInfo
static armnn::INetworkPtr Create(const Params ¶ms, std::vector< armnn::BindingPointInfo > &inputBindings, std::vector< armnn::BindingPointInfo > &outputBindings)
std::chrono::duration< double, std::milli > Run(const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
std::vector< armnn::BackendId > GetComputeDevicesAsBackendIds()
armnn::TensorInfo m_TensorInfo
Base class for all ArmNN exceptions so that users can filter to just those.
std::vector< std::string > m_ComputeDevices
unsigned int GetInputSize(unsigned int inputIndex=0u) const
armnn::OutputTensors MakeOutputTensors(const std::vector< armnn::BindingPointInfo > &outputBindings, const std::vector< std::reference_wrapper< TContainer >> &outputDataContainers)
static void AddCommandLineOptions(boost::program_options::options_description &desc, CommandLineOptions &options)
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
armnn::LayerBindingId m_BindingId
std::string StringConcat(const std::vector< std::string > &strings, std::string seperator="")
Takes a vector of strings and concatenates them together into one long std::string with an optional s...
void CheckOutputIndexIsValid(unsigned int outputIndex) const
const std::vector< armnn::BindingPointInfo > & GetInputBindingInfos() const