+ +

#include <InferenceModel.hpp>

+ + + + +

+Classes
struct	CommandLineOptions

+ + + + + + + + + +

+Public Types
using	DataType = TDataType

using	Params = InferenceModelInternal::Params

using	QuantizationParams = InferenceModelInternal::QuantizationParams

using	TContainer = boost::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > >

+ + + + + + + + + + + + + + + + + + + + + + + + + + + +

+Public Member Functions
	InferenceModel (const Params &params, bool enableProfiling, const std::string &dynamicBackendsPath, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)

void	CheckInputIndexIsValid (unsigned int inputIndex) const

void	CheckOutputIndexIsValid (unsigned int outputIndex) const

unsigned int	GetInputSize (unsigned int inputIndex=0u) const

unsigned int	GetOutputSize (unsigned int outputIndex=0u) const

std::chrono::duration< double, std::milli >	Run (const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)

const armnn::BindingPointInfo &	GetInputBindingInfo (unsigned int inputIndex=0u) const

const std::vector< armnn::BindingPointInfo > &	GetInputBindingInfos () const

const armnn::BindingPointInfo &	GetOutputBindingInfo (unsigned int outputIndex=0u) const

const std::vector< armnn::BindingPointInfo > &	GetOutputBindingInfos () const

QuantizationParams	GetQuantizationParams (unsigned int outputIndex=0u) const

QuantizationParams	GetInputQuantizationParams (unsigned int inputIndex=0u) const

std::vector< QuantizationParams >	GetAllQuantizationParams () const

+ + + +

+Static Public Member Functions
static void	AddCommandLineOptions (boost::program_options::options_description &desc, CommandLineOptions &options)

+

Detailed Description

+

template<typename IParser, typename TDataType>
+class InferenceModel< IParser, TDataType >

+ + +

Definition at line 316 of file InferenceModel.hpp.

+

Member Typedef Documentation

+ +

◆ DataType

+ +

+

+ + + + +

using DataType = TDataType

+

+ +

Definition at line 319 of file InferenceModel.hpp.

+ +

+

+ +

◆ Params

+ +

+

+ + + + +

using Params = InferenceModelInternal::Params

+

+ +

Definition at line 320 of file InferenceModel.hpp.

+ +

+

+ +

◆ QuantizationParams

+ +

+

+ + + + +

using QuantizationParams = InferenceModelInternal::QuantizationParams

+

+ +

Definition at line 321 of file InferenceModel.hpp.

+ +

+

+ +

◆ TContainer

+ +

+

+ + + + +

using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char> >

+

+ +

Definition at line 322 of file InferenceModel.hpp.

+ +

+

Constructor & Destructor Documentation

+ +

◆ InferenceModel()

+ +

+

+ + + + + +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

InferenceModel	(	const Params &	params,
		bool	enableProfiling,
		const std::string &	dynamicBackendsPath,
		const std::shared_ptr< armnn::IRuntime > &	runtime = `nullptr`
	)

+

+inline

+

+ +

Definition at line 371 of file InferenceModel.hpp.

+ +

References ARMNN_SCOPED_HEAP_PROFILING, IRuntime::Create(), CreateNetworkImpl< IParser >::Create(), armnn::Failure, Params::m_ComputeDevices, OptimizerOptions::m_Debug, IRuntime::CreationOptions::m_DynamicBackendsPath, Params::m_DynamicBackendsPath, Params::m_EnableFp16TurboMode, IRuntime::CreationOptions::m_EnableGpuProfiling, m_EnableProfiling, Params::m_InputBindings, Params::m_ModelPath, Params::m_OutputBindings, Params::m_PrintIntermediateLayers, OptimizerOptions::m_ReduceFp32ToFp16, Params::m_VisualizePostOptimizationModel, armnn::Optimize(), and options.

+

         : m_EnableProfiling(enableProfiling)
         , m_DynamicBackendsPath(dynamicBackendsPath)
     {
         if (runtime)
         {
             m_Runtime = runtime;
         }
         else
         {
             armnn::IRuntime::CreationOptions options;
             options.m_EnableGpuProfiling = m_EnableProfiling;
             options.m_DynamicBackendsPath = m_DynamicBackendsPath;
             m_Runtime = std::move(armnn::IRuntime::Create(options));
         }
 
         std::string invalidBackends;
         if (!CheckRequestedBackendsAreValid(params.m_ComputeDevices, armnn::Optional<std::string&>(invalidBackends)))
         {
             throw armnn::Exception("Some backend IDs are invalid: " + invalidBackends);
         }
 
         armnn::INetworkPtr network = CreateNetworkImpl<IParser>::Create(params, m_InputBindings, m_OutputBindings);
 
         armnn::IOptimizedNetworkPtr optNet{nullptr, [](armnn::IOptimizedNetwork*){}};
         {
             ARMNN_SCOPED_HEAP_PROFILING("Optimizing");
 
             armnn::OptimizerOptions options;
             options.m_ReduceFp32ToFp16 = params.m_EnableFp16TurboMode;
             options.m_Debug = params.m_PrintIntermediateLayers;
 
             optNet = armnn::Optimize(*network, params.m_ComputeDevices, m_Runtime->GetDeviceSpec(), options);
             if (!optNet)
             {
                 throw armnn::Exception("Optimize returned nullptr");
             }
         }
 
         if (params.m_VisualizePostOptimizationModel)
         {
             boost::filesystem::path filename = params.m_ModelPath;
             filename.replace_extension("dot");
             std::fstream file(filename.c_str(), std::ios_base::out);
             optNet->SerializeToDot(file);
         }
 
         armnn::Status ret;
         {
             ARMNN_SCOPED_HEAP_PROFILING("LoadNetwork");
             ret = m_Runtime->LoadNetwork(m_NetworkIdentifier, std::move(optNet));
         }
 
         if (ret == armnn::Status::Failure)
         {
             throw armnn::Exception("IRuntime::LoadNetwork failed");
         }
     }

+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

+

Member Function Documentation

+ +

◆ AddCommandLineOptions()

+ +

+

+ + + + + +

+ + + + + + + + + + + + + + + + + + +

static void AddCommandLineOptions	(	boost::program_options::options_description &	desc,
		CommandLineOptions &	options
	)

+

+inlinestatic

+

+ +

Definition at line 341 of file InferenceModel.hpp.

+ +

References armnn::BackendRegistryInstance(), BackendRegistry::GetBackendIdsAsString(), InferenceModel< IParser, TDataType >::CommandLineOptions::m_ComputeDevices, InferenceModel< IParser, TDataType >::CommandLineOptions::m_DynamicBackendsPath, InferenceModel< IParser, TDataType >::CommandLineOptions::m_EnableFp16TurboMode, InferenceModel< IParser, TDataType >::CommandLineOptions::m_Labels, InferenceModel< IParser, TDataType >::CommandLineOptions::m_ModelDir, and InferenceModel< IParser, TDataType >::CommandLineOptions::m_VisualizePostOptimizationModel.

+ +

Referenced by ClassifierTestCaseProvider< TDatabase, InferenceModel >::AddCommandLineOptions().

+

     {
         namespace po = boost::program_options;
 
         const std::vector<std::string> defaultComputes = { "CpuAcc", "CpuRef" };
 
         const std::string backendsMessage = "Which device to run layers on by default. Possible choices: "
                                           + armnn::BackendRegistryInstance().GetBackendIdsAsString();
 
         desc.add_options()
             ("model-dir,m", po::value<std::string>(&options.m_ModelDir)->required(),
                 "Path to directory containing model files (.caffemodel/.prototxt/.tflite)")
             ("compute,c", po::value<std::vector<std::string>>(&options.m_ComputeDevices)->
                 default_value(defaultComputes, boost::algorithm::join(defaultComputes, ", "))->
                 multitoken(), backendsMessage.c_str())
             ("dynamic-backends-path,b", po::value(&options.m_DynamicBackendsPath),
                 "Path where to load any available dynamic backend from. "
                 "If left empty (the default), dynamic backends will not be used.")
             ("labels,l", po::value<std::string>(&options.m_Labels),
                 "Text file containing one image filename - correct label pair per line, "
                 "used to test the accuracy of the network.")
             ("visualize-optimized-model,v",
                 po::value<bool>(&options.m_VisualizePostOptimizationModel)->default_value(false),
              "Produce a dot file useful for visualizing the graph post optimization."
                 "The file will have the same name as the model with the .dot extention.")
             ("fp16-turbo-mode", po::value<bool>(&options.m_EnableFp16TurboMode)->default_value(false),
                 "If this option is enabled FP32 layers, weights and biases will be converted "
                 "to FP16 where the backend supports it.");
     }

+
+
+

+

+ +

◆ CheckInputIndexIsValid()

+ +

+

+ + + + + +

+ + + + + + + + +

void CheckInputIndexIsValid ( unsigned int inputIndex ) const

+

+inline

+

+ +

Definition at line 433 of file InferenceModel.hpp.

+ +

References Params::m_InputBindings.

+

     {
         if (m_InputBindings.size() < inputIndex + 1)
         {
             throw armnn::Exception(boost::str(boost::format("Input index out of range: %1%") % inputIndex));
         }
     }

+

+

+ +

◆ CheckOutputIndexIsValid()

+ +

+

+ + + + + +

+ + + + + + + + +

void CheckOutputIndexIsValid ( unsigned int outputIndex ) const

+

+inline

+

+ +

Definition at line 441 of file InferenceModel.hpp.

+ +

References Params::m_OutputBindings.

+

     {
         if (m_OutputBindings.size() < outputIndex + 1)
         {
             throw armnn::Exception(boost::str(boost::format("Output index out of range: %1%") % outputIndex));
         }
     }

+

+

+ +

◆ GetAllQuantizationParams()

+ +

+

+ + + + + +

+ + + + + + + +

std::vector<QuantizationParams> GetAllQuantizationParams ( ) const

+

+inline

+

+ +

Definition at line 550 of file InferenceModel.hpp.

+ +

References Params::m_DynamicBackendsPath, m_EnableProfiling, Params::m_InputBindings, Params::m_OutputBindings, armnnUtils::MakeInputTensors(), and armnnUtils::MakeOutputTensors().

+

     {
         std::vector<QuantizationParams> quantizationParams;
         for (unsigned int i = 0u; i < m_OutputBindings.size(); i++)
         {
             quantizationParams.push_back(GetQuantizationParams(i));
         }
         return quantizationParams;
     }

+

+

+ +

◆ GetInputBindingInfo()

+ +

+

+ + + + + +

+ + + + + + + + +

const armnn::BindingPointInfo& GetInputBindingInfo ( unsigned int inputIndex = 0u ) const

+

+inline

+

+ +

Definition at line 514 of file InferenceModel.hpp.

+ +

References Params::m_InputBindings.

+ +

Referenced by main().

+

     {
         CheckInputIndexIsValid(inputIndex);
         return m_InputBindings[inputIndex];
     }

+

+

+ +

◆ GetInputBindingInfos()

+ +

+

+ + + + + +

+ + + + + + + +

const std::vector<armnn::BindingPointInfo>& GetInputBindingInfos ( ) const

+

+inline

+

+ +

Definition at line 520 of file InferenceModel.hpp.

+ +

References Params::m_InputBindings.

+

     {
         return m_InputBindings;
     }

+

+ +

◆ GetInputQuantizationParams()

+ +

+

+ + + + + +

+ + + + + + + + +

QuantizationParams GetInputQuantizationParams ( unsigned int inputIndex = 0u ) const

+

+inline

+

+ +

Definition at line 543 of file InferenceModel.hpp.

+ +

References Params::m_InputBindings.

+ +

Referenced by MainImpl().

+

     {
         CheckInputIndexIsValid(inputIndex);
         return std::make_pair(m_InputBindings[inputIndex].second.GetQuantizationScale(),
                               m_InputBindings[inputIndex].second.GetQuantizationOffset());
     }

+

+

+ +

◆ GetInputSize()

+ +

+

+ + + + + +

+ + + + + + + + +

unsigned int GetInputSize ( unsigned int inputIndex = 0u ) const

+

+inline

+

+ +

Definition at line 449 of file InferenceModel.hpp.

+ +

References Params::m_InputBindings.

+ +

Referenced by MainImpl().

+

     {
         CheckInputIndexIsValid(inputIndex);
         return m_InputBindings[inputIndex].second.GetNumElements();
     }

+

+

+ +

◆ GetOutputBindingInfo()

+ +

+

+ + + + + +

+ + + + + + + + +

const armnn::BindingPointInfo& GetOutputBindingInfo ( unsigned int outputIndex = 0u ) const

+

+inline

+

+ +

Definition at line 525 of file InferenceModel.hpp.

+ +

References Params::m_OutputBindings.

+

     {
         CheckOutputIndexIsValid(outputIndex);
         return m_OutputBindings[outputIndex];
     }

+

+

+ +

◆ GetOutputBindingInfos()

+ +

+

+ + + + + +

+ + + + + + + +

const std::vector<armnn::BindingPointInfo>& GetOutputBindingInfos ( ) const

+

+inline

+

+ +

Definition at line 531 of file InferenceModel.hpp.

+ +

References Params::m_OutputBindings.

+ +

Referenced by MainImpl().

+

     {
         return m_OutputBindings;
     }

+

+ +

◆ GetOutputSize()

+ +

+

+ + + + + +

+ + + + + + + + +

unsigned int GetOutputSize ( unsigned int outputIndex = 0u ) const

+

+inline

+

+ +

Definition at line 455 of file InferenceModel.hpp.

+ +

References Params::m_OutputBindings.

+ +

Referenced by main(), and MainImpl().

+

     {
         CheckOutputIndexIsValid(outputIndex);
         return m_OutputBindings[outputIndex].second.GetNumElements();
     }

+

+

+ +

◆ GetQuantizationParams()

+ +

+

+ + + + + +

+ + + + + + + + +

QuantizationParams GetQuantizationParams ( unsigned int outputIndex = 0u ) const

+

+inline

+

+ +

Definition at line 536 of file InferenceModel.hpp.

+ +

References Params::m_OutputBindings.

+

     {
         CheckOutputIndexIsValid(outputIndex);
         return std::make_pair(m_OutputBindings[outputIndex].second.GetQuantizationScale(),
                               m_OutputBindings[outputIndex].second.GetQuantizationOffset());
     }

+

+

+ +

◆ Run()

+ +

+

+ + + + + +

+ + + + + + + + + + + + + + + + + + +

std::chrono::duration<double, std::milli> Run	(	const std::vector< TContainer > &	inputContainers,
		std::vector< TContainer > &	outputContainers
	)

+

+inline

+

+ +

Definition at line 461 of file InferenceModel.hpp.

+ +

References armnn::Failure, m_EnableProfiling, armnnUtils::MakeInputTensors(), armnnUtils::MakeOutputTensors(), and armnn::numeric_cast().

+ +

Referenced by MainImpl().

+

     {
         for (unsigned int i = 0; i < outputContainers.size(); ++i)
         {
             const unsigned int expectedOutputDataSize = GetOutputSize(i);
 
             boost::apply_visitor([expectedOutputDataSize, i](auto&& value)
             {
                 const unsigned int actualOutputDataSize   = boost::numeric_cast<unsigned int>(value.size());
                 if (actualOutputDataSize < expectedOutputDataSize)
                 {
                     unsigned int outputIndex = boost::numeric_cast<unsigned int>(i);
                     throw armnn::Exception(
                             boost::str(boost::format("Not enough data for output #%1%: expected "
                             "%2% elements, got %3%") % outputIndex % expectedOutputDataSize % actualOutputDataSize));
                 }
             },
             outputContainers[i]);
         }
 
         std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);
         if (profiler)
         {
             profiler->EnableProfiling(m_EnableProfiling);
         }
 
         // Start timer to record inference time in EnqueueWorkload (in milliseconds)
         const auto start_time = GetCurrentTime();
 
         armnn::Status ret = m_Runtime->EnqueueWorkload(m_NetworkIdentifier,
                                                        MakeInputTensors(inputContainers),
                                                        MakeOutputTensors(outputContainers));
 
         const auto end_time = GetCurrentTime();
 
         // if profiling is enabled print out the results
         if (profiler && profiler->IsProfilingEnabled())
         {
             profiler->Print(std::cout);
         }
 
         if (ret == armnn::Status::Failure)
         {
             throw armnn::Exception("IRuntime::EnqueueWorkload failed");
         }
         else
         {
             return std::chrono::duration<double, std::milli>(end_time - start_time);
         }
     }

+
+
+
+
+
+
+
+

+

The documentation for this class was generated from the following file:

tests/InferenceModel.hpp

+

+Classes

+Public Types

+Public Member Functions

+Static Public Member Functions

Detailed Description

template<typename IParser, typename TDataType> +class InferenceModel< IParser, TDataType >

Member Typedef Documentation

◆ DataType

◆ Params

◆ QuantizationParams

◆ TContainer

Constructor & Destructor Documentation

◆ InferenceModel()

Member Function Documentation

◆ AddCommandLineOptions()

◆ CheckInputIndexIsValid()

◆ CheckOutputIndexIsValid()

◆ GetAllQuantizationParams()

◆ GetInputBindingInfo()

◆ GetInputBindingInfos()

◆ GetInputQuantizationParams()

◆ GetInputSize()

◆ GetOutputBindingInfo()

◆ GetOutputBindingInfos()

◆ GetOutputSize()

◆ GetQuantizationParams()

◆ Run()

template<typename IParser, typename TDataType>
+class InferenceModel< IParser, TDataType >