11 #if defined(ARMNN_SERIALIZER) 14 #if defined(ARMNN_TF_LITE_PARSER) 17 #if defined(ARMNN_ONNX_PARSER) 24 #include <boost/algorithm/string/join.hpp> 25 #include <boost/exception/exception.hpp> 26 #include <boost/exception/diagnostic_information.hpp> 27 #include <boost/format.hpp> 28 #include <boost/program_options.hpp> 29 #include <boost/filesystem.hpp> 30 #include <boost/lexical_cast.hpp> 31 #include <boost/variant.hpp> 40 #include <type_traits> 45 inline bool CheckRequestedBackendsAreValid(
const std::vector<armnn::BackendId>& backendIds,
48 if (backendIds.empty())
56 for (
const auto& backendId : backendIds)
58 if (std::find(validBackendIds.begin(), validBackendIds.end(), backendId) == validBackendIds.end())
61 if (invalidBackendIds)
63 if (!invalidBackendIds.value().empty())
65 invalidBackendIds.value() +=
", ";
67 invalidBackendIds.value() += backendId;
110 template <
typename IParser>
117 std::vector<armnn::BindingPointInfo>& inputBindings,
118 std::vector<armnn::BindingPointInfo>& outputBindings)
123 auto parser(IParser::Create());
125 std::map<std::string, armnn::TensorShape> inputShapes;
130 if (numInputShapes < numInputBindings)
133 "Not every input has its tensor shape specified: expected=%1%, got=%2%")
134 % numInputBindings % numInputShapes));
137 for (
size_t i = 0; i < numInputShapes; i++)
150 parser->CreateNetworkFromBinaryFile(modelPath.c_str(), inputShapes, requestedOutputs) :
151 parser->CreateNetworkFromTextFile(modelPath.c_str(), inputShapes, requestedOutputs));
156 inputBindings.push_back(parser->GetNetworkInputBindingInfo(inputLayerName));
161 outputBindings.push_back(parser->GetNetworkOutputBindingInfo(outputLayerName));
168 #if defined(ARMNN_SERIALIZER) 177 std::vector<armnn::BindingPointInfo>& inputBindings,
178 std::vector<armnn::BindingPointInfo>& outputBindings)
180 auto parser(IParser::Create());
181 BOOST_ASSERT(parser);
188 boost::system::error_code errorCode;
189 boost::filesystem::path pathToFile(params.
m_ModelPath);
190 if (!boost::filesystem::exists(pathToFile, errorCode))
193 boost::format(
"Cannot find the file (%1%) errorCode: %2% %3%") %
198 std::ifstream file(params.
m_ModelPath, std::ios::binary);
200 network = parser->CreateNetworkFromBinary(file);
208 parser->GetNetworkInputBindingInfo(subgraphId, inputLayerName);
215 parser->GetNetworkOutputBindingInfo(subgraphId, outputLayerName);
224 #if defined(ARMNN_TF_LITE_PARSER) 233 std::vector<armnn::BindingPointInfo>& inputBindings,
234 std::vector<armnn::BindingPointInfo>& outputBindings)
239 IParser::TfLiteParserOptions
options;
241 auto parser(IParser::Create(options));
247 network = parser->CreateNetworkFromBinaryFile(modelPath.c_str());
253 parser->GetNetworkInputBindingInfo(params.
m_SubgraphId, inputLayerName);
254 inputBindings.push_back(inputBinding);
260 parser->GetNetworkOutputBindingInfo(params.
m_SubgraphId, outputLayerName);
261 outputBindings.push_back(outputBinding);
269 #if defined(ARMNN_ONNX_PARSER) 279 std::vector<BindingPointInfo>& inputBindings,
280 std::vector<BindingPointInfo>& outputBindings)
285 auto parser(IParser::Create());
292 parser->CreateNetworkFromBinaryFile(modelPath.c_str()) :
293 parser->CreateNetworkFromTextFile(modelPath.c_str()));
298 BindingPointInfo inputBinding = parser->GetNetworkInputBindingInfo(inputLayerName);
299 inputBindings.push_back(inputBinding);
304 BindingPointInfo outputBinding = parser->GetNetworkOutputBindingInfo(outputLayerName);
305 outputBindings.push_back(outputBinding);
315 template <
typename IParser,
typename TDataType>
322 using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>>;
335 std::vector<armnn::BackendId> backendIds;
336 std::copy(m_ComputeDevices.begin(), m_ComputeDevices.end(), std::back_inserter(backendIds));
343 namespace po = boost::program_options;
345 const std::vector<std::string> defaultComputes = {
"CpuAcc",
"CpuRef" };
347 const std::string backendsMessage =
"Which device to run layers on by default. Possible choices: " 351 (
"model-dir,m", po::value<std::string>(&options.
m_ModelDir)->required(),
352 "Path to directory containing model files (.caffemodel/.prototxt/.tflite)")
353 (
"compute,c", po::value<std::vector<std::string>>(&options.
m_ComputeDevices)->
354 default_value(defaultComputes, boost::algorithm::join(defaultComputes,
", "))->
355 multitoken(), backendsMessage.c_str())
357 "Path where to load any available dynamic backend from. " 358 "If left empty (the default), dynamic backends will not be used.")
359 (
"labels,l", po::value<std::string>(&options.
m_Labels),
360 "Text file containing one image filename - correct label pair per line, " 361 "used to test the accuracy of the network.")
362 (
"visualize-optimized-model,v",
364 "Produce a dot file useful for visualizing the graph post optimization." 365 "The file will have the same name as the model with the .dot extention.")
367 "If this option is enabled FP32 layers, weights and biases will be converted " 368 "to FP16 where the backend supports it.");
372 bool enableProfiling,
373 const std::string& dynamicBackendsPath,
374 const std::shared_ptr<armnn::IRuntime>& runtime =
nullptr)
390 std::string invalidBackends;
415 boost::filesystem::path filename = params.
m_ModelPath;
416 filename.replace_extension(
"dot");
417 std::fstream file(filename.c_str(), std::ios_base::out);
418 optNet->SerializeToDot(file);
424 ret = m_Runtime->LoadNetwork(m_NetworkIdentifier, std::move(optNet));
437 throw armnn::Exception(boost::str(boost::format(
"Input index out of range: %1%") % inputIndex));
445 throw armnn::Exception(boost::str(boost::format(
"Output index out of range: %1%") % outputIndex));
451 CheckInputIndexIsValid(inputIndex);
457 CheckOutputIndexIsValid(outputIndex);
461 std::chrono::duration<double, std::milli>
Run(
462 const std::vector<TContainer>& inputContainers,
463 std::vector<TContainer>& outputContainers)
465 for (
unsigned int i = 0; i < outputContainers.size(); ++i)
467 const unsigned int expectedOutputDataSize = GetOutputSize(i);
469 boost::apply_visitor([expectedOutputDataSize, i](
auto&& value)
472 if (actualOutputDataSize < expectedOutputDataSize)
476 boost::str(boost::format(
"Not enough data for output #%1%: expected " 477 "%2% elements, got %3%") % outputIndex % expectedOutputDataSize % actualOutputDataSize));
480 outputContainers[i]);
483 std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);
490 const auto start_time = GetCurrentTime();
492 armnn::Status ret = m_Runtime->EnqueueWorkload(m_NetworkIdentifier,
496 const auto end_time = GetCurrentTime();
499 if (profiler && profiler->IsProfilingEnabled())
501 profiler->Print(std::cout);
510 return std::chrono::duration<double, std::milli>(end_time - start_time);
516 CheckInputIndexIsValid(inputIndex);
527 CheckOutputIndexIsValid(outputIndex);
538 CheckOutputIndexIsValid(outputIndex);
539 return std::make_pair(
m_OutputBindings[outputIndex].second.GetQuantizationScale(),
545 CheckInputIndexIsValid(inputIndex);
546 return std::make_pair(
m_InputBindings[inputIndex].second.GetQuantizationScale(),
552 std::vector<QuantizationParams> quantizationParams;
555 quantizationParams.push_back(GetQuantizationParams(i));
557 return quantizationParams;
562 std::shared_ptr<armnn::IRuntime> m_Runtime;
569 template<
typename TContainer>
575 template<
typename TContainer>
581 std::chrono::high_resolution_clock::time_point GetCurrentTime()
583 return std::chrono::high_resolution_clock::now();
586 std::chrono::duration<double, std::milli> GetTimeDuration(
587 std::chrono::high_resolution_clock::time_point& start_time,
588 std::chrono::high_resolution_clock::time_point& end_time)
590 return std::chrono::duration<double, std::milli>(end_time - start_time);
boost::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > > TContainer
static IRuntimePtr Create(const CreationOptions &options)
options m_EnableProfiling
BackendIdSet GetBackendIds() const
std::unordered_set< BackendId > BackendIdSet
QuantizationParams GetInputQuantizationParams(unsigned int inputIndex=0u) const
bool m_EnableFp16TurboMode
const std::vector< armnn::BindingPointInfo > & GetOutputBindingInfos() const
std::string m_DynamicBackendsPath
const armnn::BindingPointInfo & GetOutputBindingInfo(unsigned int outputIndex=0u) const
Main network class which provides the interface for building up a neural network. ...
BackendRegistry & BackendRegistryInstance()
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
const armnn::BindingPointInfo & GetInputBindingInfo(unsigned int inputIndex=0u) const
armnn::BindingPointInfo BindingPointInfo
bool m_EnableFp16TurboMode
InferenceModelInternal::QuantizationParams QuantizationParams
std::string m_DynamicBackendsPath
std::string GetBackendIdsAsString() const
bool m_VisualizePostOptimizationModel
void CheckInputIndexIsValid(unsigned int inputIndex) const
unsigned int GetOutputSize(unsigned int outputIndex=0u) const
std::vector< std::string > m_InputBindings
InferenceModel(const Params ¶ms, bool enableProfiling, const std::string &dynamicBackendsPath, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
std::vector< armnn::TensorShape > m_InputShapes
armnn::InputTensors MakeInputTensors(const std::vector< armnn::BindingPointInfo > &inputBindings, const std::vector< TContainer > &inputDataContainers)
IOptimizedNetworkPtr Optimize(const INetwork &network, const std::vector< BackendId > &backendPreferences, const IDeviceSpec &deviceSpec, const OptimizerOptions &options=OptimizerOptions(), Optional< std::vector< std::string > &> messages=EmptyOptional())
Create an optimized version of the network.
std::vector< std::string > m_OutputBindings
std::vector< armnn::BackendId > m_ComputeDevices
#define ARMNN_SCOPED_HEAP_PROFILING(TAG)
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
QuantizationParams GetQuantizationParams(unsigned int outputIndex=0u) const
std::enable_if_t< std::is_unsigned< Source >::value &&std::is_unsigned< Dest >::value, Dest > numeric_cast(Source source)
std::vector< QuantizationParams > GetAllQuantizationParams() const
std::pair< float, int32_t > QuantizationParams
armnn::OutputTensors MakeOutputTensors(const std::vector< armnn::BindingPointInfo > &outputBindings, std::vector< TContainer > &outputDataContainers)
bool m_VisualizePostOptimizationModel
std::string m_DynamicBackendsPath
Setting this value will override the paths set by the DYNAMIC_BACKEND_PATHS compiler directive Only a...
bool m_EnableGpuProfiling
Setting this flag will allow the user to obtain GPU profiling information from the runtime...
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
bool m_PrintIntermediateLayers
std::pair< armnn::LayerBindingId, armnn::TensorInfo > BindingPointInfo
static armnn::INetworkPtr Create(const Params ¶ms, std::vector< armnn::BindingPointInfo > &inputBindings, std::vector< armnn::BindingPointInfo > &outputBindings)
std::chrono::duration< double, std::milli > Run(const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
std::vector< armnn::BackendId > GetComputeDevicesAsBackendIds()
armnn::TensorInfo m_TensorInfo
Base class for all ArmNN exceptions so that users can filter to just those.
std::vector< std::string > m_ComputeDevices
unsigned int GetInputSize(unsigned int inputIndex=0u) const
static void AddCommandLineOptions(boost::program_options::options_description &desc, CommandLineOptions &options)
std::unique_ptr< INetwork, void(*)(INetwork *network)> INetworkPtr
armnn::LayerBindingId m_BindingId
armnn::Runtime::CreationOptions::ExternalProfilingOptions options
void CheckOutputIndexIsValid(unsigned int outputIndex) const
const std::vector< armnn::BindingPointInfo > & GetInputBindingInfos() const