InferenceModel< IParser, TDataType > Class Template Reference

#include <InferenceModel.hpp>

Classes
struct	CommandLineOptions

Public Types
using	DataType = TDataType
using	Params = InferenceModelInternal::Params
using	QuantizationParams = InferenceModelInternal::QuantizationParams
using	TContainer = mapbox::util::variant< std::vector< float >, std::vector< int >, std::vector< unsigned char >, std::vector< int8_t > >

Public Member Functions
	InferenceModel (const Params &params, bool enableProfiling, const std::string &dynamicBackendsPath, const std::shared_ptr< armnn::IRuntime > &runtime=nullptr)
void	CheckInputIndexIsValid (unsigned int inputIndex) const
void	CheckOutputIndexIsValid (unsigned int outputIndex) const
unsigned int	GetInputSize (unsigned int inputIndex=0u) const
unsigned int	GetOutputSize (unsigned int outputIndex=0u) const
std::chrono::duration< double, std::milli >	Run (const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers)
std::tuple< unsigned int, std::chrono::duration< double, std::milli > >	RunAsync (armnn::experimental::IWorkingMemHandle &workingMemHandleRef, const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers, unsigned int inferenceID)
void	RunAsync (const std::vector< TContainer > &inputContainers, std::vector< TContainer > &outputContainers, std::shared_ptr< armnn::IAsyncExecutionCallback > cb)
const armnn::BindingPointInfo &	GetInputBindingInfo (unsigned int inputIndex=0u) const
const std::vector< armnn::BindingPointInfo > &	GetInputBindingInfos () const
const armnn::BindingPointInfo &	GetOutputBindingInfo (unsigned int outputIndex=0u) const
const std::vector< armnn::BindingPointInfo > &	GetOutputBindingInfos () const
QuantizationParams	GetQuantizationParams (unsigned int outputIndex=0u) const
QuantizationParams	GetInputQuantizationParams (unsigned int inputIndex=0u) const
std::vector< QuantizationParams >	GetAllQuantizationParams () const
std::unique_ptr< armnn::experimental::IWorkingMemHandle >	CreateWorkingMemHandle ()

Static Public Member Functions
static void	AddCommandLineOptions (cxxopts::Options &options, CommandLineOptions &cLineOptions, std::vector< std::string > &required)

Detailed Description

template<typename IParser, typename TDataType>
class InferenceModel< IParser, TDataType >

Definition at line 340 of file InferenceModel.hpp.

Member Typedef Documentation

DataType

using DataType = TDataType

Definition at line 343 of file InferenceModel.hpp.

Params

using Params = InferenceModelInternal::Params

Definition at line 344 of file InferenceModel.hpp.

QuantizationParams

using QuantizationParams = InferenceModelInternal::QuantizationParams

Definition at line 345 of file InferenceModel.hpp.

TContainer

using TContainer = mapbox::util::variant<std::vector<float>, std::vector<int>, std::vector<unsigned char>, std::vector<int8_t> >

Definition at line 347 of file InferenceModel.hpp.

Constructor & Destructor Documentation

InferenceModel()

InferenceModel ( const Params &  params, bool  enableProfiling, const std::string &  dynamicBackendsPath, const std::shared_ptr< armnn::IRuntime > &  runtime = nullptr  )

inline

Definition at line 405 of file InferenceModel.hpp.

References ARMNN_LOG, ARMNN_SCOPED_HEAP_PROFILING, CreateNetworkImpl< IParser >::Create(), IRuntime::Create(), armnn::Failure, armnn::GetTimeDuration(), armnn::GetTimeNow(), armnn::InferAndValidate, Params::m_AsyncEnabled, Params::m_CachedNetworkFilePath, Params::m_ComputeDevices, OptimizerOptions::m_Debug, Params::m_DynamicBackendsPath, IRuntime::CreationOptions::m_DynamicBackendsPath, Params::m_EnableBf16TurboMode, Params::m_EnableFastMath, Params::m_EnableFp16TurboMode, IRuntime::CreationOptions::m_EnableGpuProfiling, Params::m_InferOutputShape, Params::m_InputBindings, Params::m_MLGOTuningFilePath, OptimizerOptions::m_ModelOptions, Params::m_ModelPath, Params::m_NumberOfThreads, Params::m_OutputBindings, Params::m_OutputDetailsToStdOut, Params::m_PrintIntermediateLayers, OptimizerOptions::m_ReduceFp32ToBf16, OptimizerOptions::m_ReduceFp32ToFp16, Params::m_SaveCachedNetwork, OptimizerOptions::m_shapeInferenceMethod, Params::m_ThreadPoolSize, Params::m_VisualizePostOptimizationModel, armnn::Optimize(), armnn::Undefined, and armnn::ValidateOnly.

        : m_EnableProfiling(enableProfiling)
        , m_DynamicBackendsPath(dynamicBackendsPath)
    {
        if (runtime)
        {
            m_Runtime = runtime;
        }
        else
        {
            armnn::IRuntime::CreationOptions options;
            options.m_EnableGpuProfiling = m_EnableProfiling;
            options.m_DynamicBackendsPath = m_DynamicBackendsPath;
            m_Runtime = armnn::IRuntime::Create(options);
        }

        std::string invalidBackends;
        if (!CheckRequestedBackendsAreValid(params.m_ComputeDevices, armnn::Optional<std::string&>(invalidBackends)))
        {
            throw armnn::Exception("Some backend IDs are invalid: " + invalidBackends);
        }

        armnn::IOptimizedNetworkPtr optNet{nullptr, [](armnn::IOptimizedNetwork*){}};
        {
            const auto parsing_start_time = armnn::GetTimeNow();
            armnn::INetworkPtr network = CreateNetworkImpl<IParser>::Create(params, m_InputBindings, m_OutputBindings);

            ARMNN_LOG(info) << "Network parsing time: " << std::setprecision(2)
                            << std::fixed << armnn::GetTimeDuration(parsing_start_time).count() << " ms\n";

            ARMNN_SCOPED_HEAP_PROFILING("Optimizing");

            armnn::OptimizerOptions options;
            options.m_ReduceFp32ToFp16 = params.m_EnableFp16TurboMode;
            options.m_ReduceFp32ToBf16 = params.m_EnableBf16TurboMode;
            options.m_Debug = params.m_PrintIntermediateLayers;

            options.m_shapeInferenceMethod = params.m_InferOutputShape ?
                    armnn::ShapeInferenceMethod::InferAndValidate : armnn::ShapeInferenceMethod::ValidateOnly;

            armnn::BackendOptions gpuAcc("GpuAcc",
            {
                { "FastMathEnabled", params.m_EnableFastMath },
                { "SaveCachedNetwork", params.m_SaveCachedNetwork },
                { "CachedNetworkFilePath", params.m_CachedNetworkFilePath },
                { "MLGOTuningFilePath", params.m_MLGOTuningFilePath }
            });

            armnn::BackendOptions cpuAcc("CpuAcc",
            {
                { "FastMathEnabled", params.m_EnableFastMath },
                { "NumberOfThreads", params.m_NumberOfThreads }
            });
            options.m_ModelOptions.push_back(gpuAcc);
            options.m_ModelOptions.push_back(cpuAcc);

            const auto optimization_start_time = armnn::GetTimeNow();
            optNet = armnn::Optimize(*network, params.m_ComputeDevices, m_Runtime->GetDeviceSpec(), options);

            ARMNN_LOG(info) << "Optimization time: " << std::setprecision(2)
                            << std::fixed << armnn::GetTimeDuration(optimization_start_time).count() << " ms\n";

            if (!optNet)
            {
                throw armnn::Exception("Optimize returned nullptr");
            }


        }

        if (params.m_VisualizePostOptimizationModel)
        {
            fs::path filename = params.m_ModelPath;
            filename.replace_extension("dot");
            std::fstream file(filename.c_str(), std::ios_base::out);
            optNet->SerializeToDot(file);
        }

        armnn::Status ret;
        {
            ARMNN_SCOPED_HEAP_PROFILING("LoadNetwork");

            const auto loading_start_time = armnn::GetTimeNow();
            armnn::INetworkProperties networkProperties(params.m_AsyncEnabled,
                                                        armnn::MemorySource::Undefined,
                                                        armnn::MemorySource::Undefined,
                                                        enableProfiling,
                                                        params.m_OutputDetailsToStdOut);
            std::string errorMessage;
            ret = m_Runtime->LoadNetwork(m_NetworkIdentifier, std::move(optNet), errorMessage, networkProperties);

            ARMNN_LOG(info) << "Network loading time: " << std::setprecision(2)
                            << std::fixed << armnn::GetTimeDuration(loading_start_time).count() << " ms\n";

            if (params.m_AsyncEnabled && params.m_ThreadPoolSize > 0)
            {
                std::vector<std::shared_ptr<armnn::IWorkingMemHandle>> memHandles;
                for (size_t i = 0; i < params.m_ThreadPoolSize; ++i)
                {
                    memHandles.emplace_back(m_Runtime->CreateWorkingMemHandle(m_NetworkIdentifier));
                }

                m_Threadpool = std::make_unique<armnn::Threadpool>(params.m_ThreadPoolSize,
                                                                   m_Runtime.get(),
                                                                   memHandles);
            }
        }

        if (ret == armnn::Status::Failure)
        {
            throw armnn::Exception("IRuntime::LoadNetwork failed");
        }
    }

Member Function Documentation

AddCommandLineOptions()

static void AddCommandLineOptions ( cxxopts::Options &  options, CommandLineOptions &  cLineOptions, std::vector< std::string > &  required  )

inlinestatic

Definition at line 367 of file InferenceModel.hpp.

References armnn::BackendRegistryInstance(), BackendRegistry::GetBackendIdsAsString(), InferenceModel< IParser, TDataType >::CommandLineOptions::m_ComputeDevices, InferenceModel< IParser, TDataType >::CommandLineOptions::m_DynamicBackendsPath, InferenceModel< IParser, TDataType >::CommandLineOptions::m_EnableBf16TurboMode, InferenceModel< IParser, TDataType >::CommandLineOptions::m_EnableFp16TurboMode, InferenceModel< IParser, TDataType >::CommandLineOptions::m_Labels, InferenceModel< IParser, TDataType >::CommandLineOptions::m_ModelDir, and InferenceModel< IParser, TDataType >::CommandLineOptions::m_VisualizePostOptimizationModel.

Referenced by ClassifierTestCaseProvider< TDatabase, InferenceModel >::AddCommandLineOptions().

    {
        const std::vector<std::string> defaultComputes = { "CpuAcc", "CpuRef" };

        const std::string backendsMessage = "Which device to run layers on by default. Possible choices: "
                                          + armnn::BackendRegistryInstance().GetBackendIdsAsString();

        options
            .allow_unrecognised_options()
            .add_options()
                ("m,model-dir", "Path to directory containing model files (.prototxt/.tflite)",
                 cxxopts::value<std::string>(cLineOptions.m_ModelDir))
                ("c,compute", backendsMessage.c_str(),
                 cxxopts::value<std::vector<std::string>>(cLineOptions.m_ComputeDevices)->default_value("CpuRef"))
                ("b,dynamic-backends-path",
                 "Path where to load any available dynamic backend from. "
                 "If left empty (the default), dynamic backends will not be used.",
                 cxxopts::value(cLineOptions.m_DynamicBackendsPath))
                ("l,labels",
                 "Text file containing one image filename - correct label pair per line, "
                 "used to test the accuracy of the network.", cxxopts::value<std::string>(cLineOptions.m_Labels))
                ("v,visualize-optimized-model",
                 "Produce a dot file useful for visualizing the graph post optimization."
                 "The file will have the same name as the model with the .dot extention.",
                 cxxopts::value<bool>(cLineOptions.m_VisualizePostOptimizationModel)->default_value("false"))
                ("fp16-turbo-mode",
                 "If this option is enabled FP32 layers, weights and biases will be converted "
                 "to FP16 where the backend supports it.",
                 cxxopts::value<bool>(cLineOptions.m_EnableFp16TurboMode)->default_value("false"))
                ("bf16-turbo-mode",
                 "If this option is enabled FP32 layers, weights and biases will be converted "
                 "to BF16 where the backend supports it.",
                 cxxopts::value<bool>(cLineOptions.m_EnableBf16TurboMode)->default_value("false"));

        required.emplace_back("model-dir");
    }

CheckInputIndexIsValid()

void CheckInputIndexIsValid ( unsigned int  inputIndex ) const

inline

Definition at line 522 of file InferenceModel.hpp.

References Params::m_InputBindings.

    {
        if (m_InputBindings.size() < inputIndex + 1)
        {
            throw armnn::Exception(fmt::format("Input index out of range: {}", inputIndex));
        }
    }

CheckOutputIndexIsValid()

void CheckOutputIndexIsValid ( unsigned int  outputIndex ) const

inline

Definition at line 530 of file InferenceModel.hpp.

References Params::m_OutputBindings.

    {
        if (m_OutputBindings.size() < outputIndex + 1)
        {
            throw armnn::Exception(fmt::format("Output index out of range: {}", outputIndex));
        }
    }

CreateWorkingMemHandle()

std::unique_ptr<armnn::experimental::IWorkingMemHandle> CreateWorkingMemHandle ( )

inline

Definition at line 734 of file InferenceModel.hpp.

References Params::m_DynamicBackendsPath, Params::m_InputBindings, Params::m_OutputBindings, armnnUtils::MakeInputTensors(), MakeInputTensors(), armnnUtils::MakeOutputTensors(), and MakeOutputTensors().

Referenced by MainImpl().

    {
        return m_Runtime->CreateWorkingMemHandle(m_NetworkIdentifier);
    }

GetAllQuantizationParams()

std::vector<QuantizationParams> GetAllQuantizationParams ( ) const

inline

Definition at line 724 of file InferenceModel.hpp.

References Params::m_OutputBindings.

    {
        std::vector<QuantizationParams> quantizationParams;
        for (unsigned int i = 0u; i < m_OutputBindings.size(); i++)
        {
            quantizationParams.push_back(GetQuantizationParams(i));
        }
        return quantizationParams;
    }

GetInputBindingInfo()

const armnn::BindingPointInfo& GetInputBindingInfo ( unsigned int  inputIndex = 0u ) const

inline

Definition at line 688 of file InferenceModel.hpp.

References Params::m_InputBindings.

Referenced by main().

    {
        CheckInputIndexIsValid(inputIndex);
        return m_InputBindings[inputIndex];
    }

GetInputBindingInfos()

const std::vector<armnn::BindingPointInfo>& GetInputBindingInfos ( ) const

inline

Definition at line 694 of file InferenceModel.hpp.

References Params::m_InputBindings.

    {
        return m_InputBindings;
    }

GetInputQuantizationParams()

QuantizationParams GetInputQuantizationParams ( unsigned int  inputIndex = 0u ) const

inline

Definition at line 717 of file InferenceModel.hpp.

References Params::m_InputBindings.

Referenced by MainImpl().

    {
        CheckInputIndexIsValid(inputIndex);
        return std::make_pair(m_InputBindings[inputIndex].second.GetQuantizationScale(),
                              m_InputBindings[inputIndex].second.GetQuantizationOffset());
    }

GetInputSize()

unsigned int GetInputSize ( unsigned int  inputIndex = 0u ) const

inline

Definition at line 538 of file InferenceModel.hpp.

References Params::m_InputBindings.

Referenced by MainImpl().

    {
        CheckInputIndexIsValid(inputIndex);
        return m_InputBindings[inputIndex].second.GetNumElements();
    }

GetOutputBindingInfo()

const armnn::BindingPointInfo& GetOutputBindingInfo ( unsigned int  outputIndex = 0u ) const

inline

Definition at line 699 of file InferenceModel.hpp.

References Params::m_OutputBindings.

    {
        CheckOutputIndexIsValid(outputIndex);
        return m_OutputBindings[outputIndex];
    }

GetOutputBindingInfos()

const std::vector<armnn::BindingPointInfo>& GetOutputBindingInfos ( ) const

inline

Definition at line 705 of file InferenceModel.hpp.

References Params::m_OutputBindings.

Referenced by MainImpl().

    {
        return m_OutputBindings;
    }

GetOutputSize()

unsigned int GetOutputSize ( unsigned int  outputIndex = 0u ) const

inline

Definition at line 544 of file InferenceModel.hpp.

References Params::m_OutputBindings.

Referenced by main(), and MainImpl().

    {
        CheckOutputIndexIsValid(outputIndex);
        return m_OutputBindings[outputIndex].second.GetNumElements();
    }

GetQuantizationParams()

QuantizationParams GetQuantizationParams ( unsigned int  outputIndex = 0u ) const

inline

Definition at line 710 of file InferenceModel.hpp.

References Params::m_OutputBindings.

    {
        CheckOutputIndexIsValid(outputIndex);
        return std::make_pair(m_OutputBindings[outputIndex].second.GetQuantizationScale(),
                              m_OutputBindings[outputIndex].second.GetQuantizationOffset());
    }

Run()

std::chrono::duration<double, std::milli> Run ( const std::vector< TContainer > &  inputContainers, std::vector< TContainer > &  outputContainers  )

inline

Definition at line 550 of file InferenceModel.hpp.

References armnn::Failure, armnn::GetTimeDuration(), armnn::GetTimeNow(), MakeInputTensors(), MakeOutputTensors(), and armnn::numeric_cast().

Referenced by MainImpl().

    {
        for (unsigned int i = 0; i < outputContainers.size(); ++i)
        {
            const unsigned int expectedOutputDataSize = GetOutputSize(i);

            mapbox::util::apply_visitor([expectedOutputDataSize, i](auto&& value)
            {
                const unsigned int actualOutputDataSize   = armnn::numeric_cast<unsigned int>(value.size());
                if (actualOutputDataSize < expectedOutputDataSize)
                {
                    unsigned int outputIndex = i;
                    throw armnn::Exception(
                            fmt::format("Not enough data for output #{0}: expected "
                            "{1} elements, got {2}", outputIndex, expectedOutputDataSize, actualOutputDataSize));
                }
            },
            outputContainers[i]);
        }

        std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);

        // Start timer to record inference time in EnqueueWorkload (in milliseconds)
        const auto start_time = armnn::GetTimeNow();

        armnn::Status ret = m_Runtime->EnqueueWorkload(m_NetworkIdentifier,
                                                       MakeInputTensors(inputContainers),
                                                       MakeOutputTensors(outputContainers));
        const auto duration = armnn::GetTimeDuration(start_time);

        // if profiling is enabled print out the results
        if (profiler && profiler->IsProfilingEnabled())
        {
            profiler->Print(std::cout);
        }

        if (ret == armnn::Status::Failure)
        {
            throw armnn::Exception("IRuntime::EnqueueWorkload failed");
        }
        else
        {
            return duration;
        }
    }

RunAsync() [1/2]

std::tuple<unsigned int, std::chrono::duration<double, std::milli> > RunAsync ( armnn::experimental::IWorkingMemHandle &  workingMemHandleRef, const std::vector< TContainer > &  inputContainers, std::vector< TContainer > &  outputContainers, unsigned int  inferenceID  )

inline

Definition at line 598 of file InferenceModel.hpp.

References armnn::Failure, armnn::GetTimeDuration(), armnn::GetTimeNow(), MakeInputTensors(), MakeOutputTensors(), and armnn::numeric_cast().

Referenced by MainImpl().

    {
        for (unsigned int i = 0; i < outputContainers.size(); ++i)
        {
            const unsigned int expectedOutputDataSize = GetOutputSize(i);

            mapbox::util::apply_visitor([expectedOutputDataSize, i](auto&& value)
            {
                const unsigned int actualOutputDataSize   = armnn::numeric_cast<unsigned int>(value.size());
                if (actualOutputDataSize < expectedOutputDataSize)
                {
                    unsigned int outputIndex = i;
                    throw armnn::Exception(
                            fmt::format("Not enough data for output #{0}: expected "
                            "{1} elements, got {2}", outputIndex, expectedOutputDataSize, actualOutputDataSize));
                }
            },
            outputContainers[i]);
        }

        std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);

        // Start timer to record inference time in EnqueueWorkload (in milliseconds)
        const auto start_time = armnn::GetTimeNow();

        armnn::Status ret = m_Runtime->Execute(workingMemHandleRef,
                                               MakeInputTensors(inputContainers),
                                               MakeOutputTensors(outputContainers));

        const auto duration = armnn::GetTimeDuration(start_time);

        // if profiling is enabled print out the results
        if (profiler && profiler->IsProfilingEnabled())
        {
            profiler->Print(std::cout);
        }

        if (ret == armnn::Status::Failure)
        {
            throw armnn::Exception(
                fmt::format("IRuntime::Execute asynchronously failed for network #{0} on inference #{1}",
                            m_NetworkIdentifier, inferenceID));
        }
        else
        {
            return std::make_tuple(inferenceID, duration);
        }
    }

RunAsync() [2/2]

void RunAsync ( const std::vector< TContainer > &  inputContainers, std::vector< TContainer > &  outputContainers, std::shared_ptr< armnn::IAsyncExecutionCallback >  cb  )

inline

Definition at line 651 of file InferenceModel.hpp.

References MakeInputTensors(), MakeOutputTensors(), armnn::Medium, and armnn::numeric_cast().

    {
        for (unsigned int i = 0; i < outputContainers.size(); ++i)
        {
            const unsigned int expectedOutputDataSize = GetOutputSize(i);

            mapbox::util::apply_visitor([expectedOutputDataSize, i](auto&& value)
            {
                const unsigned int actualOutputDataSize   = armnn::numeric_cast<unsigned int>(value.size());
                if (actualOutputDataSize < expectedOutputDataSize)
                {
                    unsigned int outputIndex = i;
                    throw armnn::Exception(
                            fmt::format("Not enough data for output #{0}: expected "
                            "{1} elements, got {2}", outputIndex, expectedOutputDataSize, actualOutputDataSize));
                }
            },
            outputContainers[i]);
        }

        std::shared_ptr<armnn::IProfiler> profiler = m_Runtime->GetProfiler(m_NetworkIdentifier);

        m_Threadpool->Schedule(m_NetworkIdentifier,
                               MakeInputTensors(inputContainers),
                               MakeOutputTensors(outputContainers),
                               armnn::QosExecPriority::Medium,
                               cb);

        // if profiling is enabled print out the results
        if (profiler && profiler->IsProfilingEnabled())
        {
            profiler->Print(std::cout);
        }
    }

---

The documentation for this class was generated from the following file:

• tests/InferenceModel.hpp