15 #if defined(ARMNN_SERIALIZER) 18 #if defined(ARMNN_TF_LITE_PARSER) 21 #if defined(ARMNN_ONNX_PARSER) 30 #include <cxxopts/cxxopts.hpp> 32 #include <fmt/format.h> 33 #include <mapbox/variant.hpp> 41 #include <type_traits> 46 inline bool CheckRequestedBackendsAreValid(
const std::vector<armnn::BackendId>& backendIds,
49 if (backendIds.empty())
57 for (
const auto& backendId : backendIds)
59 if (std::find(validBackendIds.begin(), validBackendIds.end(), backendId) == validBackendIds.end())
62 if (invalidBackendIds)
64 if (!invalidBackendIds.value().empty())
66 invalidBackendIds.value() +=
", ";
68 invalidBackendIds.value() += backendId;
117 template <
typename IParser>
124 std::vector<armnn::BindingPointInfo>& inputBindings,
125 std::vector<armnn::BindingPointInfo>& outputBindings)
130 auto parser(IParser::Create());
132 std::map<std::string, armnn::TensorShape> inputShapes;
137 if (numInputShapes < numInputBindings)
140 "Not every input has its tensor shape specified: expected={0}, got={1}",
141 numInputBindings, numInputShapes));
144 for (
size_t i = 0; i < numInputShapes; i++)
157 parser->CreateNetworkFromBinaryFile(modelPath.c_str(), inputShapes, requestedOutputs) :
158 parser->CreateNetworkFromTextFile(modelPath.c_str(), inputShapes, requestedOutputs));
163 inputBindings.push_back(parser->GetNetworkInputBindingInfo(inputLayerName));
168 outputBindings.push_back(parser->GetNetworkOutputBindingInfo(outputLayerName));
175 #if defined(ARMNN_SERIALIZER) 184 std::vector<armnn::BindingPointInfo>& inputBindings,
185 std::vector<armnn::BindingPointInfo>& outputBindings)
187 auto parser(IParser::Create());
195 std::error_code errorCode;
197 if (!fs::exists(pathToFile, errorCode))
204 std::ifstream file(params.
m_ModelPath, std::ios::binary);
206 network = parser->CreateNetworkFromBinary(file);
214 parser->GetNetworkInputBindingInfo(subgraphId, inputLayerName);
221 parser->GetNetworkOutputBindingInfo(subgraphId, outputLayerName);
230 #if defined(ARMNN_TF_LITE_PARSER) 239 std::vector<armnn::BindingPointInfo>& inputBindings,
240 std::vector<armnn::BindingPointInfo>& outputBindings)
245 IParser::TfLiteParserOptions options;
248 auto parser(IParser::Create(options));
254 network = parser->CreateNetworkFromBinaryFile(modelPath.c_str());
260 parser->GetNetworkInputBindingInfo(params.
m_SubgraphId, inputLayerName);
261 inputBindings.push_back(inputBinding);
267 parser->GetNetworkOutputBindingInfo(params.
m_SubgraphId, outputLayerName);
268 outputBindings.push_back(outputBinding);
276 #if defined(ARMNN_ONNX_PARSER) 286 std::vector<BindingPointInfo>& inputBindings,
287 std::vector<BindingPointInfo>& outputBindings)
292 auto parser(IParser::Create());
299 parser->CreateNetworkFromBinaryFile(modelPath.c_str()) :
300 parser->CreateNetworkFromTextFile(modelPath.c_str()));
305 BindingPointInfo inputBinding = parser->GetNetworkInputBindingInfo(inputLayerName);
306 inputBindings.push_back(inputBinding);
311 BindingPointInfo outputBinding = parser->GetNetworkOutputBindingInfo(outputLayerName);
312 outputBindings.push_back(outputBinding);
322 template <
typename IParser,
typename TDataType>
329 using TContainer = mapbox::util::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;
343 std::vector<armnn::BackendId> backendIds;
344 std::copy(m_ComputeDevices.begin(), m_ComputeDevices.end(), std::back_inserter(backendIds));
352 const std::vector<std::string> defaultComputes = {
"CpuAcc",
"CpuRef" };
354 const std::string backendsMessage =
"Which device to run layers on by default. Possible choices: " 358 .allow_unrecognised_options()
360 (
"m,model-dir",
"Path to directory containing model files (.caffemodel/.prototxt/.tflite)",
361 cxxopts::value<std::string>(cLineOptions.
m_ModelDir))
362 (
"c,compute", backendsMessage.c_str(),
363 cxxopts::value<std::vector<std::string>>(cLineOptions.
m_ComputeDevices)->default_value(
"CpuRef"))
364 (
"b,dynamic-backends-path",
365 "Path where to load any available dynamic backend from. " 366 "If left empty (the default), dynamic backends will not be used.",
369 "Text file containing one image filename - correct label pair per line, " 370 "used to test the accuracy of the network.", cxxopts::value<std::string>(cLineOptions.
m_Labels))
371 (
"v,visualize-optimized-model",
372 "Produce a dot file useful for visualizing the graph post optimization." 373 "The file will have the same name as the model with the .dot extention.",
376 "If this option is enabled FP32 layers, weights and biases will be converted " 377 "to FP16 where the backend supports it.",
380 "If this option is enabled FP32 layers, weights and biases will be converted " 381 "to BF16 where the backend supports it.",
384 required.emplace_back(
"model-dir");
388 bool enableProfiling,
389 const std::string& dynamicBackendsPath,
390 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
391 : m_EnableProfiling(enableProfiling)
406 std::string invalidBackends;
415 ARMNN_LOG(info) <<
"Network parsing time: " << std::setprecision(2)
441 ARMNN_LOG(info) <<
"Optimization time: " << std::setprecision(2)
453 filename.replace_extension(
"dot");
454 std::fstream file(filename.c_str(), std::ios_base::out);
455 optNet->SerializeToDot(file);
461 ret = m_Runtime->LoadNetwork(m_NetworkIdentifier, std::move(optNet));
474 throw armnn::Exception(fmt::format(
"Input index out of range: {}", inputIndex));
482 throw armnn::Exception(fmt::format(
"Output index out of range: {}", outputIndex));
488 CheckInputIndexIsValid(inputIndex);
494 CheckOutputIndexIsValid(outputIndex);
498 std::chrono::duration<double, std::milli>
Run(
499 const std::vector<TContainer>& inputContainers,
500 std::vector<TContainer>& outputContainers)
502 for (
unsigned int i = 0; i < outputContainers.size(); ++i)
504 const unsigned int expectedOutputDataSize = GetOutputSize(i);
506 mapbox::util::apply_visitor([expectedOutputDataSize, i](
auto&& value)
509 if (actualOutputDataSize < expectedOutputDataSize)
511 unsigned int outputIndex = i;
513 fmt::format(
"Not enough data for output #{0}: expected " 514 "{1} elements, got {2}", outputIndex, expectedOutputDataSize, actualOutputDataSize));
517 outputContainers[i]);
520 std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);
523 profiler->EnableProfiling(m_EnableProfiling);
529 armnn::Status ret = m_Runtime->EnqueueWorkload(m_NetworkIdentifier,
536 if (profiler && profiler->IsProfilingEnabled())
538 profiler->Print(std::cout);
553 CheckInputIndexIsValid(inputIndex);
564 CheckOutputIndexIsValid(outputIndex);
575 CheckOutputIndexIsValid(outputIndex);
576 return std::make_pair(
m_OutputBindings[outputIndex].second.GetQuantizationScale(),
582 CheckInputIndexIsValid(inputIndex);
583 return std::make_pair(
m_InputBindings[inputIndex].second.GetQuantizationScale(),
589 std::vector<QuantizationParams> quantizationParams;
592 quantizationParams.push_back(GetQuantizationParams(i));
594 return quantizationParams;
599 std::shared_ptr<armnn::IRuntime> m_Runtime;
603 bool m_EnableProfiling;
606 template<
typename TContainer>
612 template<
typename TContainer>
ModelOptions m_ModelOptions
static IRuntimePtr Create(const CreationOptions &options)
BackendIdSet GetBackendIds() const
std::chrono::duration< double, std::milli > GetTimeDuration(std::chrono::high_resolution_clock::time_point start_time)
std::unordered_set< BackendId > BackendIdSet
QuantizationParams GetInputQuantizationParams(unsigned int inputIndex=0u) const
bool m_EnableFp16TurboMode
const std::vector< armnn::BindingPointInfo > & GetOutputBindingInfos() const
armnn::InputTensors MakeInputTensors(const std::vector< armnn::BindingPointInfo > &inputBindings, const std::vector< std::reference_wrapper< TContainer >> &inputDataContainers)
static void AddCommandLineOptions(cxxopts::Options &options, CommandLineOptions &cLineOptions, std::vector< std::string > &required)
std::string m_DynamicBackendsPath
const armnn::BindingPointInfo & GetOutputBindingInfo(unsigned int outputIndex=0u) const
#define ARMNN_LOG(severity)
Main network class which provides the interface for building up a neural network. ...
BackendRegistry & BackendRegistryInstance()
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
const armnn::BindingPointInfo & GetInputBindingInfo(unsigned int inputIndex=0u) const
armnn::BindingPointInfo BindingPointInfo
bool m_EnableFp16TurboMode
std::chrono::high_resolution_clock::time_point GetTimeNow()
InferenceModelInternal::QuantizationParams QuantizationParams
std::string m_DynamicBackendsPath
std::string GetBackendIdsAsString() const
bool m_VisualizePostOptimizationModel
void CheckInputIndexIsValid(unsigned int inputIndex) const
bool m_EnableBf16TurboMode
unsigned int GetOutputSize(unsigned int outputIndex=0u) const
std::vector< std::string > m_InputBindings
InferenceModel(const Params ¶ms, bool enableProfiling, const std::string &dynamicBackendsPath, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
std::vector< armnn::TensorShape > m_InputShapes
armnn::InputTensors MakeInputTensors(const std::vector< armnn::BindingPointInfo > &inputBindings, const std::vector< TContainer > &inputDataContainers)
IOptimizedNetworkPtr Optimize(const INetwork &network, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options=OptimizerOptions(), Optional< std::vector< std::string > &> messages=EmptyOptional())
Create an optimized version of the network.
std::vector< std::string > m_OutputBindings
std::vector< armnn::BackendId > m_ComputeDevices
#define ARMNN_SCOPED_HEAP_PROFILING(TAG)
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
QuantizationParams GetQuantizationParams(unsigned int outputIndex=0u) const
#define ARMNN_ASSERT(COND)
std::vector< QuantizationParams > GetAllQuantizationParams() const
std::pair< float, int32_t > QuantizationParams
mapbox::util::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > > TContainer
armnn::OutputTensors MakeOutputTensors(const std::vector< armnn::BindingPointInfo > &outputBindings, std::vector< TContainer > &outputDataContainers)
bool m_EnableBf16TurboMode
bool m_VisualizePostOptimizationModel
Struct for the users to pass backend specific options.
std::string m_DynamicBackendsPath
Setting this value will override the paths set by the DYNAMIC_BACKEND_PATHS compiler directive Only a...
bool m_EnableGpuProfiling
Setting this flag will allow the user to obtain GPU profiling information from the runtime...
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
bool m_PrintIntermediateLayers
std::pair< armnn::LayerBindingId, armnn::TensorInfo > BindingPointInfo
static armnn::INetworkPtr Create(const Params ¶ms, std::vector< armnn::BindingPointInfo > &inputBindings, std::vector< armnn::BindingPointInfo > &outputBindings)
std::chrono::duration< double, std::milli > Run(const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
std::vector< armnn::BackendId > GetComputeDevicesAsBackendIds()
armnn::TensorInfo m_TensorInfo
Base class for all ArmNN exceptions so that users can filter to just those.
std::vector< std::string > m_ComputeDevices
unsigned int GetInputSize(unsigned int inputIndex=0u) const
armnn::OutputTensors MakeOutputTensors(const std::vector< armnn::BindingPointInfo > &outputBindings, const std::vector< std::reference_wrapper< TContainer >> &outputDataContainers)
std::enable_if_t< std::is_unsigned< Source >::value &&std::is_unsigned< Dest >::value, Dest > numeric_cast(Source source)
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
armnn::LayerBindingId m_BindingId
void CheckOutputIndexIsValid(unsigned int outputIndex) const
const std::vector< armnn::BindingPointInfo > & GetInputBindingInfos() const