15 #if defined(ARMNN_SERIALIZER) 18 #if defined(ARMNN_TF_LITE_PARSER) 21 #if defined(ARMNN_ONNX_PARSER) 30 #include <cxxopts/cxxopts.hpp> 32 #include <fmt/format.h> 33 #include <mapbox/variant.hpp> 41 #include <type_traits> 46 inline bool CheckRequestedBackendsAreValid(
const std::vector<armnn::BackendId>& backendIds,
49 if (backendIds.empty())
57 for (
const auto& backendId : backendIds)
59 if (std::find(validBackendIds.begin(), validBackendIds.end(), backendId) == validBackendIds.end())
62 if (invalidBackendIds)
64 if (!invalidBackendIds.value().empty())
66 invalidBackendIds.value() +=
", ";
68 invalidBackendIds.value() += backendId;
126 template <
typename IParser>
133 std::vector<armnn::BindingPointInfo>& inputBindings,
134 std::vector<armnn::BindingPointInfo>& outputBindings)
139 auto parser(IParser::Create());
141 std::map<std::string, armnn::TensorShape> inputShapes;
146 if (numInputShapes < numInputBindings)
149 "Not every input has its tensor shape specified: expected={0}, got={1}",
150 numInputBindings, numInputShapes));
153 for (
size_t i = 0; i < numInputShapes; i++)
166 parser->CreateNetworkFromBinaryFile(modelPath.c_str(), inputShapes, requestedOutputs) :
167 parser->CreateNetworkFromTextFile(modelPath.c_str(), inputShapes, requestedOutputs));
172 inputBindings.push_back(parser->GetNetworkInputBindingInfo(inputLayerName));
177 outputBindings.push_back(parser->GetNetworkOutputBindingInfo(outputLayerName));
184 #if defined(ARMNN_SERIALIZER) 193 std::vector<armnn::BindingPointInfo>& inputBindings,
194 std::vector<armnn::BindingPointInfo>& outputBindings)
196 auto parser(IParser::Create());
204 std::error_code errorCode;
206 if (!fs::exists(pathToFile, errorCode))
213 std::ifstream file(params.
m_ModelPath, std::ios::binary);
215 network = parser->CreateNetworkFromBinary(file);
223 parser->GetNetworkInputBindingInfo(subgraphId, inputLayerName);
230 parser->GetNetworkOutputBindingInfo(subgraphId, outputLayerName);
239 #if defined(ARMNN_TF_LITE_PARSER) 248 std::vector<armnn::BindingPointInfo>& inputBindings,
249 std::vector<armnn::BindingPointInfo>& outputBindings)
254 IParser::TfLiteParserOptions options;
257 auto parser(IParser::Create(options));
263 network = parser->CreateNetworkFromBinaryFile(modelPath.c_str());
269 parser->GetNetworkInputBindingInfo(params.
m_SubgraphId, inputLayerName);
270 inputBindings.push_back(inputBinding);
276 parser->GetNetworkOutputBindingInfo(params.
m_SubgraphId, outputLayerName);
277 outputBindings.push_back(outputBinding);
285 #if defined(ARMNN_ONNX_PARSER) 295 std::vector<BindingPointInfo>& inputBindings,
296 std::vector<BindingPointInfo>& outputBindings)
301 auto parser(IParser::Create());
308 parser->CreateNetworkFromBinaryFile(modelPath.c_str()) :
309 parser->CreateNetworkFromTextFile(modelPath.c_str()));
314 BindingPointInfo inputBinding = parser->GetNetworkInputBindingInfo(inputLayerName);
315 inputBindings.push_back(inputBinding);
320 BindingPointInfo outputBinding = parser->GetNetworkOutputBindingInfo(outputLayerName);
321 outputBindings.push_back(outputBinding);
331 template <
typename IParser,
typename TDataType>
338 using TContainer = mapbox::util::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;
352 std::vector<armnn::BackendId> backendIds;
353 std::copy(m_ComputeDevices.begin(), m_ComputeDevices.end(), std::back_inserter(backendIds));
361 const std::vector<std::string> defaultComputes = {
"CpuAcc",
"CpuRef" };
363 const std::string backendsMessage =
"Which device to run layers on by default. Possible choices: " 367 .allow_unrecognised_options()
369 (
"m,model-dir",
"Path to directory containing model files (.caffemodel/.prototxt/.tflite)",
370 cxxopts::value<std::string>(cLineOptions.
m_ModelDir))
371 (
"c,compute", backendsMessage.c_str(),
372 cxxopts::value<std::vector<std::string>>(cLineOptions.
m_ComputeDevices)->default_value(
"CpuRef"))
373 (
"b,dynamic-backends-path",
374 "Path where to load any available dynamic backend from. " 375 "If left empty (the default), dynamic backends will not be used.",
378 "Text file containing one image filename - correct label pair per line, " 379 "used to test the accuracy of the network.", cxxopts::value<std::string>(cLineOptions.
m_Labels))
380 (
"v,visualize-optimized-model",
381 "Produce a dot file useful for visualizing the graph post optimization." 382 "The file will have the same name as the model with the .dot extention.",
385 "If this option is enabled FP32 layers, weights and biases will be converted " 386 "to FP16 where the backend supports it.",
389 "If this option is enabled FP32 layers, weights and biases will be converted " 390 "to BF16 where the backend supports it.",
393 required.emplace_back(
"model-dir");
397 bool enableProfiling,
398 const std::string& dynamicBackendsPath,
399 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
400 : m_EnableProfiling(enableProfiling)
415 std::string invalidBackends;
424 ARMNN_LOG(info) <<
"Network parsing time: " << std::setprecision(2)
455 ARMNN_LOG(info) <<
"Optimization time: " << std::setprecision(2)
467 filename.replace_extension(
"dot");
468 std::fstream file(filename.c_str(), std::ios_base::out);
469 optNet->SerializeToDot(file);
477 ret = m_Runtime->LoadNetwork(m_NetworkIdentifier, std::move(optNet));
479 ARMNN_LOG(info) <<
"Network loading time: " << std::setprecision(2)
493 throw armnn::Exception(fmt::format(
"Input index out of range: {}", inputIndex));
501 throw armnn::Exception(fmt::format(
"Output index out of range: {}", outputIndex));
507 CheckInputIndexIsValid(inputIndex);
513 CheckOutputIndexIsValid(outputIndex);
517 std::chrono::duration<double, std::milli>
Run(
518 const std::vector<TContainer>& inputContainers,
519 std::vector<TContainer>& outputContainers)
521 for (
unsigned int i = 0; i < outputContainers.size(); ++i)
523 const unsigned int expectedOutputDataSize = GetOutputSize(i);
525 mapbox::util::apply_visitor([expectedOutputDataSize, i](
auto&& value)
528 if (actualOutputDataSize < expectedOutputDataSize)
530 unsigned int outputIndex = i;
532 fmt::format(
"Not enough data for output #{0}: expected " 533 "{1} elements, got {2}", outputIndex, expectedOutputDataSize, actualOutputDataSize));
536 outputContainers[i]);
539 std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);
542 profiler->EnableProfiling(m_EnableProfiling);
548 armnn::Status ret = m_Runtime->EnqueueWorkload(m_NetworkIdentifier,
555 if (profiler && profiler->IsProfilingEnabled())
557 profiler->Print(std::cout);
572 CheckInputIndexIsValid(inputIndex);
583 CheckOutputIndexIsValid(outputIndex);
594 CheckOutputIndexIsValid(outputIndex);
595 return std::make_pair(
m_OutputBindings[outputIndex].second.GetQuantizationScale(),
601 CheckInputIndexIsValid(inputIndex);
602 return std::make_pair(
m_InputBindings[inputIndex].second.GetQuantizationScale(),
608 std::vector<QuantizationParams> quantizationParams;
611 quantizationParams.push_back(GetQuantizationParams(i));
613 return quantizationParams;
618 std::shared_ptr<armnn::IRuntime> m_Runtime;
622 bool m_EnableProfiling;
625 template<
typename TContainer>
631 template<
typename TContainer>
ModelOptions m_ModelOptions
static IRuntimePtr Create(const CreationOptions &options)
BackendIdSet GetBackendIds() const
std::chrono::duration< double, std::milli > GetTimeDuration(std::chrono::high_resolution_clock::time_point start_time)
std::unordered_set< BackendId > BackendIdSet
QuantizationParams GetInputQuantizationParams(unsigned int inputIndex=0u) const
bool m_EnableFp16TurboMode
const std::vector< armnn::BindingPointInfo > & GetOutputBindingInfos() const
armnn::InputTensors MakeInputTensors(const std::vector< armnn::BindingPointInfo > &inputBindings, const std::vector< std::reference_wrapper< TContainer >> &inputDataContainers)
static void AddCommandLineOptions(cxxopts::Options &options, CommandLineOptions &cLineOptions, std::vector< std::string > &required)
std::string m_DynamicBackendsPath
const armnn::BindingPointInfo & GetOutputBindingInfo(unsigned int outputIndex=0u) const
#define ARMNN_LOG(severity)
Main network class which provides the interface for building up a neural network. ...
BackendRegistry & BackendRegistryInstance()
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
const armnn::BindingPointInfo & GetInputBindingInfo(unsigned int inputIndex=0u) const
armnn::BindingPointInfo BindingPointInfo
bool m_EnableFp16TurboMode
std::chrono::high_resolution_clock::time_point GetTimeNow()
InferenceModelInternal::QuantizationParams QuantizationParams
std::string m_DynamicBackendsPath
std::string GetBackendIdsAsString() const
bool m_VisualizePostOptimizationModel
void CheckInputIndexIsValid(unsigned int inputIndex) const
bool m_EnableBf16TurboMode
unsigned int GetOutputSize(unsigned int outputIndex=0u) const
std::vector< std::string > m_InputBindings
InferenceModel(const Params ¶ms, bool enableProfiling, const std::string &dynamicBackendsPath, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
std::vector< armnn::TensorShape > m_InputShapes
armnn::InputTensors MakeInputTensors(const std::vector< armnn::BindingPointInfo > &inputBindings, const std::vector< TContainer > &inputDataContainers)
IOptimizedNetworkPtr Optimize(const INetwork &network, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options=OptimizerOptions(), Optional< std::vector< std::string > &> messages=EmptyOptional())
Create an optimized version of the network.
std::vector< std::string > m_OutputBindings
std::vector< armnn::BackendId > m_ComputeDevices
#define ARMNN_SCOPED_HEAP_PROFILING(TAG)
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
QuantizationParams GetQuantizationParams(unsigned int outputIndex=0u) const
#define ARMNN_ASSERT(COND)
std::vector< QuantizationParams > GetAllQuantizationParams() const
std::pair< float, int32_t > QuantizationParams
mapbox::util::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > > TContainer
armnn::OutputTensors MakeOutputTensors(const std::vector< armnn::BindingPointInfo > &outputBindings, std::vector< TContainer > &outputDataContainers)
bool m_EnableBf16TurboMode
bool m_VisualizePostOptimizationModel
Struct for the users to pass backend specific options.
std::string m_DynamicBackendsPath
Setting this value will override the paths set by the DYNAMIC_BACKEND_PATHS compiler directive Only a...
bool m_EnableGpuProfiling
Setting this flag will allow the user to obtain GPU profiling information from the runtime...
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
bool m_PrintIntermediateLayers
std::string m_CachedNetworkFilePath
std::pair< armnn::LayerBindingId, armnn::TensorInfo > BindingPointInfo
static armnn::INetworkPtr Create(const Params ¶ms, std::vector< armnn::BindingPointInfo > &inputBindings, std::vector< armnn::BindingPointInfo > &outputBindings)
std::chrono::duration< double, std::milli > Run(const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
std::vector< armnn::BackendId > GetComputeDevicesAsBackendIds()
armnn::TensorInfo m_TensorInfo
Base class for all ArmNN exceptions so that users can filter to just those.
std::vector< std::string > m_ComputeDevices
unsigned int GetInputSize(unsigned int inputIndex=0u) const
armnn::OutputTensors MakeOutputTensors(const std::vector< armnn::BindingPointInfo > &outputBindings, const std::vector< std::reference_wrapper< TContainer >> &outputDataContainers)
std::string m_MLGOTuningFilePath
unsigned int m_NumberOfThreads
std::enable_if_t< std::is_unsigned< Source >::value &&std::is_unsigned< Dest >::value, Dest > numeric_cast(Source source)
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
armnn::LayerBindingId m_BindingId
void CheckOutputIndexIsValid(unsigned int outputIndex) const
const std::vector< armnn::BindingPointInfo > & GetInputBindingInfos() const