InferenceModel< IParser, TDataType > Class Template Reference

#include <InferenceModel.hpp>

Classes
struct	CommandLineOptions

Public Types
using	DataType = TDataType
using	Params = InferenceModelInternal::Params
using	QuantizationParams = InferenceModelInternal::QuantizationParams
using	TContainer = boost::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char > >

Public Member Functions
	InferenceModel (const Params &params, bool enableProfiling, const std::string &dynamicBackendsPath, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
void	CheckInputIndexIsValid (unsigned int inputIndex) const
void	CheckOutputIndexIsValid (unsigned int outputIndex) const
unsigned int	GetInputSize (unsigned int inputIndex=0u) const
unsigned int	GetOutputSize (unsigned int outputIndex=0u) const
std::chrono::duration< double, std::milli >	Run (const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
const armnn::BindingPointInfo &	GetInputBindingInfo (unsigned int inputIndex=0u) const
const std::vector< armnn::BindingPointInfo > &	GetInputBindingInfos () const
const armnn::BindingPointInfo &	GetOutputBindingInfo (unsigned int outputIndex=0u) const
const std::vector< armnn::BindingPointInfo > &	GetOutputBindingInfos () const
QuantizationParams	GetQuantizationParams (unsigned int outputIndex=0u) const
QuantizationParams	GetInputQuantizationParams (unsigned int inputIndex=0u) const
std::vector< QuantizationParams >	GetAllQuantizationParams () const

Static Public Member Functions
static void	AddCommandLineOptions (boost::program_options::options_description &desc, CommandLineOptions &options)

Detailed Description

template<typename IParser, typename TDataType>
class InferenceModel< IParser, TDataType >

Definition at line 322 of file InferenceModel.hpp.

Member Typedef Documentation

DataType

using DataType = TDataType

Definition at line 325 of file InferenceModel.hpp.

Params

using Params = InferenceModelInternal::Params

Definition at line 326 of file InferenceModel.hpp.

QuantizationParams

using QuantizationParams = InferenceModelInternal::QuantizationParams

Definition at line 327 of file InferenceModel.hpp.

TContainer

using TContainer = boost::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char> >

Definition at line 328 of file InferenceModel.hpp.

Constructor & Destructor Documentation

InferenceModel()

InferenceModel ( const Params &  params, bool  enableProfiling, const std::string &  dynamicBackendsPath, const std::shared_ptr< armnn::IRuntime > &  runtime = nullptr  )

inline

Definition at line 381 of file InferenceModel.hpp.

References ARMNN_LOG, ARMNN_SCOPED_HEAP_PROFILING, IRuntime::Create(), CreateNetworkImpl< IParser >::Create(), armnn::Failure, armnn::GetTimeDuration(), armnn::GetTimeNow(), Params::m_ComputeDevices, OptimizerOptions::m_Debug, IRuntime::CreationOptions::m_DynamicBackendsPath, Params::m_DynamicBackendsPath, Params::m_EnableBf16TurboMode, Params::m_EnableFp16TurboMode, IRuntime::CreationOptions::m_EnableGpuProfiling, Params::m_InputBindings, Params::m_ModelPath, Params::m_OutputBindings, Params::m_PrintIntermediateLayers, OptimizerOptions::m_ReduceFp32ToBf16, OptimizerOptions::m_ReduceFp32ToFp16, Params::m_VisualizePostOptimizationModel, and armnn::Optimize().

        : m_EnableProfiling(enableProfiling)
        , m_DynamicBackendsPath(dynamicBackendsPath)
    {
        if (runtime)
        {
            m_Runtime = runtime;
        }
        else
        {
            armnn::IRuntime::CreationOptions options;
            options.m_EnableGpuProfiling = m_EnableProfiling;
            options.m_DynamicBackendsPath = m_DynamicBackendsPath;
            m_Runtime = std::move(armnn::IRuntime::Create(options));
        }

        std::string invalidBackends;
        if (!CheckRequestedBackendsAreValid(params.m_ComputeDevices, armnn::Optional<std::string&>(invalidBackends)))
        {
            throw armnn::Exception("Some backend IDs are invalid: " + invalidBackends);
        }

        const auto parsing_start_time = armnn::GetTimeNow();
        armnn::INetworkPtr network = CreateNetworkImpl<IParser>::Create(params, m_InputBindings, m_OutputBindings);

        ARMNN_LOG(info) << "Network parsing time: " << std::setprecision(2)
                        << std::fixed << armnn::GetTimeDuration(parsing_start_time).count() << " ms\n";

        armnn::IOptimizedNetworkPtr optNet{nullptr, [](armnn::IOptimizedNetwork*){}};
        {
            ARMNN_SCOPED_HEAP_PROFILING("Optimizing");

            armnn::OptimizerOptions options;
            options.m_ReduceFp32ToFp16 = params.m_EnableFp16TurboMode;
            options.m_ReduceFp32ToBf16 = params.m_EnableBf16TurboMode;
            options.m_Debug = params.m_PrintIntermediateLayers;

            const auto optimization_start_time = armnn::GetTimeNow();
            optNet = armnn::Optimize(*network, params.m_ComputeDevices, m_Runtime->GetDeviceSpec(), options);

            ARMNN_LOG(info) << "Optimization time: " << std::setprecision(2)
                            << std::fixed << armnn::GetTimeDuration(optimization_start_time).count() << " ms\n";

            if (!optNet)
            {
                throw armnn::Exception("Optimize returned nullptr");
            }
        }

        if (params.m_VisualizePostOptimizationModel)
        {
            fs::path filename = params.m_ModelPath;
            filename.replace_extension("dot");
            std::fstream file(filename.c_str(), std::ios_base::out);
            optNet->SerializeToDot(file);
        }

        armnn::Status ret;
        {
            ARMNN_SCOPED_HEAP_PROFILING("LoadNetwork");
            ret = m_Runtime->LoadNetwork(m_NetworkIdentifier, std::move(optNet));
        }

        if (ret == armnn::Status::Failure)
        {
            throw armnn::Exception("IRuntime::LoadNetwork failed");
        }
    }

Member Function Documentation

AddCommandLineOptions()

static void AddCommandLineOptions ( boost::program_options::options_description &  desc, CommandLineOptions &  options  )

inlinestatic

Definition at line 348 of file InferenceModel.hpp.

References armnn::BackendRegistryInstance(), BackendRegistry::GetBackendIdsAsString(), InferenceModel< IParser, TDataType >::CommandLineOptions::m_ComputeDevices, InferenceModel< IParser, TDataType >::CommandLineOptions::m_DynamicBackendsPath, InferenceModel< IParser, TDataType >::CommandLineOptions::m_EnableBf16TurboMode, InferenceModel< IParser, TDataType >::CommandLineOptions::m_EnableFp16TurboMode, InferenceModel< IParser, TDataType >::CommandLineOptions::m_Labels, InferenceModel< IParser, TDataType >::CommandLineOptions::m_ModelDir, InferenceModel< IParser, TDataType >::CommandLineOptions::m_VisualizePostOptimizationModel, and armnn::stringUtils::StringConcat().

Referenced by ClassifierTestCaseProvider< TDatabase, InferenceModel >::AddCommandLineOptions().

    {
        namespace po = boost::program_options;

        const std::vector<std::string> defaultComputes = { "CpuAcc", "CpuRef" };

        const std::string backendsMessage = "Which device to run layers on by default. Possible choices: "
                                          + armnn::BackendRegistryInstance().GetBackendIdsAsString();

        desc.add_options()
            ("model-dir,m", po::value<std::string>(&options.m_ModelDir)->required(),
                "Path to directory containing model files (.caffemodel/.prototxt/.tflite)")
            ("compute,c", po::value<std::vector<std::string>>(&options.m_ComputeDevices)->
                default_value(defaultComputes, armnn::stringUtils::StringConcat(defaultComputes, ", "))->
                multitoken(), backendsMessage.c_str())
            ("dynamic-backends-path,b", po::value(&options.m_DynamicBackendsPath),
                "Path where to load any available dynamic backend from. "
                "If left empty (the default), dynamic backends will not be used.")
            ("labels,l", po::value<std::string>(&options.m_Labels),
                "Text file containing one image filename - correct label pair per line, "
                "used to test the accuracy of the network.")
            ("visualize-optimized-model,v",
                po::value<bool>(&options.m_VisualizePostOptimizationModel)->default_value(false),
             "Produce a dot file useful for visualizing the graph post optimization."
                "The file will have the same name as the model with the .dot extention.")
            ("fp16-turbo-mode", po::value<bool>(&options.m_EnableFp16TurboMode)->default_value(false),
                "If this option is enabled FP32 layers, weights and biases will be converted "
                "to FP16 where the backend supports it.")
            ("bf16-turbo-mode", po::value<bool>(&options.m_EnableBf16TurboMode)->default_value(false),
                "If this option is enabled FP32 layers, weights and biases will be converted "
                "to BF16 where the backend supports it.");
    }

CheckInputIndexIsValid()

void CheckInputIndexIsValid ( unsigned int  inputIndex ) const

inline

Definition at line 453 of file InferenceModel.hpp.

References Params::m_InputBindings.

    {
        if (m_InputBindings.size() < inputIndex + 1)
        {
            throw armnn::Exception(boost::str(boost::format("Input index out of range: %1%") % inputIndex));
        }
    }

CheckOutputIndexIsValid()

void CheckOutputIndexIsValid ( unsigned int  outputIndex ) const

inline

Definition at line 461 of file InferenceModel.hpp.

References Params::m_OutputBindings.

    {
        if (m_OutputBindings.size() < outputIndex + 1)
        {
            throw armnn::Exception(boost::str(boost::format("Output index out of range: %1%") % outputIndex));
        }
    }

GetAllQuantizationParams()

std::vector<QuantizationParams> GetAllQuantizationParams ( ) const

inline

Definition at line 570 of file InferenceModel.hpp.

References Params::m_DynamicBackendsPath, Params::m_InputBindings, Params::m_OutputBindings, armnnUtils::MakeInputTensors(), MakeInputTensors(), armnnUtils::MakeOutputTensors(), and MakeOutputTensors().

    {
        std::vector<QuantizationParams> quantizationParams;
        for (unsigned int i = 0u; i < m_OutputBindings.size(); i++)
        {
            quantizationParams.push_back(GetQuantizationParams(i));
        }
        return quantizationParams;
    }

GetInputBindingInfo()

const armnn::BindingPointInfo& GetInputBindingInfo ( unsigned int  inputIndex = 0u ) const

inline

Definition at line 534 of file InferenceModel.hpp.

References Params::m_InputBindings.

Referenced by main().

    {
        CheckInputIndexIsValid(inputIndex);
        return m_InputBindings[inputIndex];
    }

GetInputBindingInfos()

const std::vector<armnn::BindingPointInfo>& GetInputBindingInfos ( ) const

inline

Definition at line 540 of file InferenceModel.hpp.

References Params::m_InputBindings.

    {
        return m_InputBindings;
    }

GetInputQuantizationParams()

QuantizationParams GetInputQuantizationParams ( unsigned int  inputIndex = 0u ) const

inline

Definition at line 563 of file InferenceModel.hpp.

References Params::m_InputBindings.

Referenced by MainImpl().

    {
        CheckInputIndexIsValid(inputIndex);
        return std::make_pair(m_InputBindings[inputIndex].second.GetQuantizationScale(),
                              m_InputBindings[inputIndex].second.GetQuantizationOffset());
    }

GetInputSize()

unsigned int GetInputSize ( unsigned int  inputIndex = 0u ) const

inline

Definition at line 469 of file InferenceModel.hpp.

References Params::m_InputBindings.

Referenced by MainImpl().

    {
        CheckInputIndexIsValid(inputIndex);
        return m_InputBindings[inputIndex].second.GetNumElements();
    }

GetOutputBindingInfo()

const armnn::BindingPointInfo& GetOutputBindingInfo ( unsigned int  outputIndex = 0u ) const

inline

Definition at line 545 of file InferenceModel.hpp.

References Params::m_OutputBindings.

    {
        CheckOutputIndexIsValid(outputIndex);
        return m_OutputBindings[outputIndex];
    }

GetOutputBindingInfos()

const std::vector<armnn::BindingPointInfo>& GetOutputBindingInfos ( ) const

inline

Definition at line 551 of file InferenceModel.hpp.

References Params::m_OutputBindings.

Referenced by MainImpl().

    {
        return m_OutputBindings;
    }

GetOutputSize()

unsigned int GetOutputSize ( unsigned int  outputIndex = 0u ) const

inline

Definition at line 475 of file InferenceModel.hpp.

References Params::m_OutputBindings.

Referenced by main(), and MainImpl().

    {
        CheckOutputIndexIsValid(outputIndex);
        return m_OutputBindings[outputIndex].second.GetNumElements();
    }

GetQuantizationParams()

QuantizationParams GetQuantizationParams ( unsigned int  outputIndex = 0u ) const

inline

Definition at line 556 of file InferenceModel.hpp.

References Params::m_OutputBindings.

    {
        CheckOutputIndexIsValid(outputIndex);
        return std::make_pair(m_OutputBindings[outputIndex].second.GetQuantizationScale(),
                              m_OutputBindings[outputIndex].second.GetQuantizationOffset());
    }

Run()

std::chrono::duration<double, std::milli> Run ( const std::vector< TContainer > &  inputContainers, std::vector< TContainer > &  outputContainers  )

inline

Definition at line 481 of file InferenceModel.hpp.

References armnn::Failure, armnn::GetTimeDuration(), armnn::GetTimeNow(), MakeInputTensors(), MakeOutputTensors(), and armnn::numeric_cast().

Referenced by MainImpl().

    {
        for (unsigned int i = 0; i < outputContainers.size(); ++i)
        {
            const unsigned int expectedOutputDataSize = GetOutputSize(i);

            boost::apply_visitor([expectedOutputDataSize, i](auto&& value)
            {
                const unsigned int actualOutputDataSize   = boost::numeric_cast<unsigned int>(value.size());
                if (actualOutputDataSize < expectedOutputDataSize)
                {
                    unsigned int outputIndex = boost::numeric_cast<unsigned int>(i);
                    throw armnn::Exception(
                            boost::str(boost::format("Not enough data for output #%1%: expected "
                            "%2% elements, got %3%") % outputIndex % expectedOutputDataSize % actualOutputDataSize));
                }
            },
            outputContainers[i]);
        }

        std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);
        if (profiler)
        {
            profiler->EnableProfiling(m_EnableProfiling);
        }

        // Start timer to record inference time in EnqueueWorkload (in milliseconds)
        const auto start_time = armnn::GetTimeNow();

        armnn::Status ret = m_Runtime->EnqueueWorkload(m_NetworkIdentifier,
                                                       MakeInputTensors(inputContainers),
                                                       MakeOutputTensors(outputContainers));

        const auto duration = armnn::GetTimeDuration(start_time);

        // if profiling is enabled print out the results
        if (profiler && profiler->IsProfilingEnabled())
        {
            profiler->Print(std::cout);
        }

        if (ret == armnn::Status::Failure)
        {
            throw armnn::Exception("IRuntime::EnqueueWorkload failed");
        }
        else
        {
            return duration;
        }
    }

---

The documentation for this class was generated from the following file:

• tests/InferenceModel.hpp