ArmnnPreparedModel Class Reference

#include <ArmnnPreparedModel.hpp>

Inheritance diagram for ArmnnPreparedModel:

Public Member Functions
	ArmnnPreparedModel (armnn::NetworkId networkId, armnn::IRuntime *runtime, const Model &model, const std::string &requestInputsAndOutputsDumpDir, const bool gpuProfilingEnabled, Priority priority=Priority::MEDIUM)
	ArmnnPreparedModel (armnn::NetworkId networkId, armnn::IRuntime *runtime, const std::string &requestInputsAndOutputsDumpDir, const bool gpuProfilingEnabled, Priority priority=Priority::MEDIUM, const bool prepareModelFromCache=false)
virtual	~ArmnnPreparedModel ()
ExecutionResult< std::pair< std::vector< OutputShape >, Timing > >	execute (const Request &request, MeasureTiming measureTiming, const OptionalTimePoint &deadline, const OptionalDuration &loopTimeoutDuration, const std::vector< android::nn::TokenValuePair > &hints, const std::vector< android::nn::ExtensionNameAndPrefix > &extensionNameToPrefix) const override
GeneralResult< std::pair< SyncFence, ExecuteFencedInfoCallback > >	executeFenced (const Request &request, const std::vector< SyncFence > &waitFor, MeasureTiming measureTiming, const OptionalTimePoint &deadline, const OptionalDuration &loopTimeoutDuration, const OptionalDuration &timeoutDurationAfterFence, const std::vector< android::nn::TokenValuePair > &hints, const std::vector< android::nn::ExtensionNameAndPrefix > &extensionNameToPrefix) const override
GeneralResult< android::nn::SharedExecution >	createReusableExecution (const Request &request, MeasureTiming measureTiming, const OptionalDuration &loopTimeoutDuration, const std::vector< android::nn::TokenValuePair > &hints, const std::vector< android::nn::ExtensionNameAndPrefix > &extensionNameToPrefix) const override
GeneralResult< SharedBurst >	configureExecutionBurst () const override
std::any	getUnderlyingResource () const override
ErrorStatus	ExecuteGraph (std::shared_ptr< std::vector< android::nn::RunTimePoolInfo >> &pMemPools, armnn::InputTensors &inputTensors, armnn::OutputTensors &outputTensors, CanonicalExecutionContext callback, const bool pointerMemory=false) const
	execute the graph prepared from the request More...
Priority	GetModelPriority () const
bool	ExecuteWithDummyInputs (unsigned int numInputs, unsigned int numOutputs) const
	Executes this model with dummy inputs (e.g. More...

Detailed Description

Definition at line 38 of file ArmnnPreparedModel.hpp.

Constructor & Destructor Documentation

ArmnnPreparedModel() [1/2]

ArmnnPreparedModel	(	armnn::NetworkId	networkId,
		armnn::IRuntime *	runtime,
		const Model &	model,
		const std::string &	requestInputsAndOutputsDumpDir,
		const bool	gpuProfilingEnabled,
		Priority	priority = Priority::MEDIUM
	)

Definition at line 126 of file ArmnnPreparedModel.cpp.

    : m_NetworkId(networkId)
    , m_Runtime(runtime)
    , m_Model(model)
    , m_RequestInputsAndOutputsDumpDir(requestInputsAndOutputsDumpDir)
    , m_GpuProfilingEnabled(gpuProfilingEnabled)
    , m_ModelPriority(priority)
    , m_PrepareFromCache(false)
{
    Init();
}

ArmnnPreparedModel() [2/2]

ArmnnPreparedModel	(	armnn::NetworkId	networkId,
		armnn::IRuntime *	runtime,
		const std::string &	requestInputsAndOutputsDumpDir,
		const bool	gpuProfilingEnabled,
		Priority	priority = Priority::MEDIUM,
		const bool	prepareModelFromCache = false
	)

Definition at line 143 of file ArmnnPreparedModel.cpp.

    : m_NetworkId(networkId)
    , m_Runtime(runtime)
    , m_RequestInputsAndOutputsDumpDir(requestInputsAndOutputsDumpDir)
    , m_GpuProfilingEnabled(gpuProfilingEnabled)
    , m_ModelPriority(priority)
    , m_PrepareFromCache(prepareModelFromCache)
{
    Init();
}

~ArmnnPreparedModel()

~ArmnnPreparedModel ( )

virtual

Definition at line 628 of file ArmnnPreparedModel.cpp.

{
    VLOG(DRIVER) << "ArmnnPreparedModel::~ArmnnPreparedModel()";
    // Get a hold of the profiler used by this model.
    if (m_GpuProfilingEnabled)
    {
        auto profiler = m_Runtime->GetProfiler(m_NetworkId);
        if (profiler)
        {
            // Dump the profiling info to a file if required.
            DumpJsonProfilingIfRequired(m_GpuProfilingEnabled,
                                        m_RequestInputsAndOutputsDumpDir,
                                        m_NetworkId,
                                        profiler.get());
        }
    }
    // Unload the network associated with this model
    m_Runtime->UnloadNetwork(m_NetworkId);
}

References armnn_driver::DumpJsonProfilingIfRequired(), IRuntime::GetProfiler(), and IRuntime::UnloadNetwork().

Member Function Documentation

configureExecutionBurst()

GeneralResult< SharedBurst > configureExecutionBurst ( ) const

override

Definition at line 600 of file ArmnnPreparedModel.cpp.

{
    // TODO: Implement BURST
    return nullptr;
}

createReusableExecution()

GeneralResult< SharedExecution > createReusableExecution ( const Request &  request, MeasureTiming  measureTiming, const OptionalDuration &  loopTimeoutDuration, const std::vector< android::nn::TokenValuePair > &  hints, const std::vector< android::nn::ExtensionNameAndPrefix > &  extensionNameToPrefix  ) const

override

Definition at line 586 of file ArmnnPreparedModel.cpp.

{
    VLOG(DRIVER) << "ArmnnPreparedModel::createReusableExecution()";
    return std::make_shared<DefaultExecution>(shared_from_this(),
                                              request,
                                              measureTiming,
                                              loopTimeoutDuration);
}

execute()

ExecutionResult< std::pair< std::vector< OutputShape >, Timing > > execute ( const Request &  request, MeasureTiming  measureTiming, const OptionalTimePoint &  deadline, const OptionalDuration &  loopTimeoutDuration, const std::vector< android::nn::TokenValuePair > &  hints, const std::vector< android::nn::ExtensionNameAndPrefix > &  extensionNameToPrefix  ) const

override

Definition at line 295 of file ArmnnPreparedModel.cpp.

{
    VLOG(DRIVER) << "CanonicalDriver::PreparedModel::execute()";
 
    CanonicalExecutionContext ctx;
    if (measureTiming == MeasureTiming::YES)
    {
        ctx.measureTimings = measureTiming;
        ctx.driverStart =  Clock::now();
    }
 
    if (!m_PrepareFromCache)
    {
        const auto modelRequest = validateRequestForModel(request, m_Model);
        if (!modelRequest.ok())
        {
            return NN_ERROR(ErrorStatus::INVALID_ARGUMENT) << modelRequest.error();
        }
        VLOG(DRIVER) << "ArmnnPreparedModel::execute(): " << GetModelSummary(m_Model).c_str();
    }
    if (hasDeadlinePassed(deadline))
    {
        return NN_ERROR(ErrorStatus::MISSED_DEADLINE_PERSISTENT);
    }
 
    // map the memory pool into shared pointers
    // use a shared memory pools vector on the heap, as it is passed to the request thread
    auto memPools = std::make_shared<std::vector<android::nn::RunTimePoolInfo>>();
 
    // allocate the tensors on the heap, as they are passed to the request thread
    auto inputTensors = std::make_shared<armnn::InputTensors>();
    auto outputTensors = std::make_shared<armnn::OutputTensors>();
 
    auto isPointerTypeMemory = IsPointerTypeMemory(request);
    ErrorStatus theErrorStatus = PrepareMemoryForIO(*inputTensors,
                                                    *outputTensors,
                                                    *memPools,
                                                    request,
                                                    isPointerTypeMemory);
 
    switch(theErrorStatus)
    {
        case ErrorStatus::OUTPUT_INSUFFICIENT_SIZE:
            return NN_ERROR(ErrorStatus::OUTPUT_INSUFFICIENT_SIZE);
        case ErrorStatus::GENERAL_FAILURE:
            return NN_ERROR(ErrorStatus::GENERAL_FAILURE);
        case ErrorStatus::INVALID_ARGUMENT:
            return NN_ERROR(ErrorStatus::INVALID_ARGUMENT);
        default:
        {}
    }
 
    std::vector<OutputShape> outputShapes(outputTensors->size());
    for (unsigned int i = 0; i < outputTensors->size(); i++)
    {
        std::pair<int, armnn::Tensor> outputTensorPair = (*outputTensors)[i];
        const armnn::Tensor outputTensor = outputTensorPair.second;
        const armnn::TensorInfo outputTensorInfo = outputTensor.GetInfo();
 
        outputShapes[i] = ComputeShape(outputTensorInfo);
    }
    Timing theTiming;
 
    VLOG(DRIVER) << "ArmnnPreparedModel::execute(...) before ExecuteGraph";
    auto errorStatus = ExecuteGraph(memPools, *inputTensors, *outputTensors, ctx, isPointerTypeMemory);
    if (errorStatus != ErrorStatus::NONE)
    {
        return NN_ERROR(errorStatus) << "execute() failed";
    }
    VLOG(DRIVER) << "ArmnnPreparedModel::execute(...) after ExecuteGraph";
 
    return std::make_pair(outputShapes, theTiming);
}

References armnn_driver::ComputeShape(), CanonicalExecutionContext::driverStart, ArmnnPreparedModel::ExecuteGraph(), BaseTensor< MemoryType >::GetInfo(), armnn_driver::GetModelSummary(), and CanonicalExecutionContext::measureTimings.

executeFenced()

GeneralResult< std::pair< SyncFence, ExecuteFencedInfoCallback > > executeFenced ( const Request &  request, const std::vector< SyncFence > &  waitFor, MeasureTiming  measureTiming, const OptionalTimePoint &  deadline, const OptionalDuration &  loopTimeoutDuration, const OptionalDuration &  timeoutDurationAfterFence, const std::vector< android::nn::TokenValuePair > &  hints, const std::vector< android::nn::ExtensionNameAndPrefix > &  extensionNameToPrefix  ) const

override

Definition at line 478 of file ArmnnPreparedModel.cpp.

{
    VLOG(DRIVER) << "ArmnnPreparedModel::executeFenced()";
 
    if (!m_PrepareFromCache) {
        const auto modelRequest = validateRequestForModel(request, m_Model);
        if (!modelRequest.ok())
        {
            return NN_ERROR(ErrorStatus::INVALID_ARGUMENT) << modelRequest.error();
        }
        VLOG(DRIVER) << "ArmnnPreparedModel::executeFenced(): " << GetModelSummary(m_Model).c_str();
    }
    if (hasDeadlinePassed(deadline))
    {
        return NN_ERROR(ErrorStatus::MISSED_DEADLINE_PERSISTENT);
    }
 
    CanonicalExecutionContext ctx;
    if (measureTiming == MeasureTiming::YES)
    {
        ctx.measureTimings = measureTiming;
        ctx.driverStart =  Clock::now();
    }
 
    // Wait for the dependent events to signal
    for (const auto& syncFence : waitFor)
    {
        if (!syncFence.getSharedHandle())
        {
            return NN_ERROR(ErrorStatus::INVALID_ARGUMENT);
        }
        if (syncFence.syncWait({}) != SyncFence::FenceState::SIGNALED)
        {
            return NN_ERROR(ErrorStatus::GENERAL_FAILURE) << "syncWait failed";
        }
    }
 
    android::nn::TimePoint fenceExecutionStart;
    if (measureTiming == MeasureTiming::YES)
    {
        fenceExecutionStart = Clock::now();
    }
 
    // map the memory pool into shared pointers
    // use a shared memory pools vector on the heap, as it is passed to the request thread
    auto memPools = std::make_shared<std::vector<android::nn::RunTimePoolInfo>>();
 
    // allocate the tensors on the heap, as they are passed to the request thread
    auto inputTensors = std::make_shared<armnn::InputTensors>();
    auto outputTensors = std::make_shared<armnn::OutputTensors>();
 
    auto isPointerTypeMemory = IsPointerTypeMemory(request);
    ErrorStatus theErrorStatus = PrepareMemoryForIO(*inputTensors,
                                                    *outputTensors,
                                                    *memPools,
                                                    request,
                                                    isPointerTypeMemory);
 
    if (theErrorStatus != ErrorStatus::NONE)
    {
        return NN_ERROR(ErrorStatus::INVALID_ARGUMENT) << "executeFenced() failed";
    }
 
    Timing timingSinceLaunch = {};
    Timing timingAfterFence  = {};
    if (measureTiming == MeasureTiming::YES)
    {
        timingAfterFence.timeOnDevice = ctx.deviceEnd - ctx.deviceStart;
        timingAfterFence.timeInDriver = ctx.driverEnd - fenceExecutionStart;
        VLOG(DRIVER) << "executeFenced timingSinceLaunch = " << timingAfterFence.timeOnDevice;
        VLOG(DRIVER) << "executeFenced timingAfterFence = " << timingAfterFence.timeInDriver;
    }
 
    VLOG(DRIVER) << "ArmnnCanonicalPreparedModel::executeFenced(...) before ExecuteGraph";
    auto errorStatus = ExecuteGraph(memPools, *inputTensors, *outputTensors, ctx, isPointerTypeMemory);
    VLOG(DRIVER) << "ArmnnCanonicalPreparedModel::executeFenced(...) after ExecuteGraph";
 
    ExecuteFencedInfoCallback armnnFencedExecutionCallback =
            [timingSinceLaunch, timingAfterFence, errorStatus]() {
 
                GeneralResult<std::pair<Timing, Timing>> result;
 
                switch(errorStatus)
                {
                    case ErrorStatus::OUTPUT_INSUFFICIENT_SIZE:
                        result.error().code = (ErrorStatus::OUTPUT_INSUFFICIENT_SIZE);
                    case ErrorStatus::GENERAL_FAILURE:
                        result.error().code = (ErrorStatus::GENERAL_FAILURE);
                    case ErrorStatus::INVALID_ARGUMENT:
                        result.error().code = (ErrorStatus::INVALID_ARGUMENT);
                    default:
                    {
                        result.value() = std::make_pair(timingSinceLaunch, timingAfterFence);
                    }
                }
                return result;
            };
    return std::make_pair(SyncFence::createAsSignaled(), std::move(armnnFencedExecutionCallback ));
}

References CanonicalExecutionContext::deviceEnd, CanonicalExecutionContext::deviceStart, CanonicalExecutionContext::driverEnd, CanonicalExecutionContext::driverStart, ArmnnPreparedModel::ExecuteGraph(), armnn_driver::GetModelSummary(), and CanonicalExecutionContext::measureTimings.

ExecuteGraph()

ErrorStatus ExecuteGraph	(	std::shared_ptr< std::vector< android::nn::RunTimePoolInfo >> &	pMemPools,
		armnn::InputTensors &	inputTensors,
		armnn::OutputTensors &	outputTensors,
		CanonicalExecutionContext	callback,
		const bool	pointerMemory = false
	)		const

execute the graph prepared from the request

Definition at line 375 of file ArmnnPreparedModel.cpp.

{
    VLOG(DRIVER) << "ArmnnPreparedModel::ExecuteGraph(...)";
 
    DumpTensorsIfRequired("Input", inputTensors);
    std::vector<armnn::ImportedInputId> importedInputIds;
    std::vector<armnn::ImportedOutputId> importedOutputIds;
    try
    {
        if (ctx.measureTimings == MeasureTiming::YES)
        {
            ctx.deviceStart =  Clock::now();
        }
        armnn::Status status;
        VLOG(DRIVER) << "ArmnnPreparedModel::ExecuteGraph m_AsyncModelExecutionEnabled false";
        importedInputIds = m_Runtime->ImportInputs(m_NetworkId, inputTensors, armnn::MemorySource::Malloc);
        if (!importedInputIds.empty())
        {
            // Some or all of the input tensors been imported. We need to remove the ones that could from
            // inputTensors.
            for (armnn::ImportedInputId& importedId : importedInputIds)
            {
                inputTensors.erase(
                        std::remove_if(
                                inputTensors.begin(), inputTensors.end(),
                                [&importedId](std::pair<armnn::LayerBindingId, class armnn::ConstTensor>& element) {
                                    return (element.first == static_cast<int>(importedId));
                                }),
                        inputTensors.end());
            }
        }
        importedOutputIds = m_Runtime->ImportOutputs(m_NetworkId, outputTensors, armnn::MemorySource::Malloc);
        if (!importedOutputIds.empty())
        {
            // Some or all of the output tensors could not be imported. We need to remove the ones that could
            // from outputTensors.
            for (armnn::ImportedInputId& importedId : importedOutputIds)
            {
                outputTensors.erase(
                        std::remove_if(
                                outputTensors.begin(), outputTensors.end(),
                                [&importedId](std::pair<armnn::LayerBindingId, class armnn::Tensor>& element) {
                                    return (element.first == static_cast<int>(importedId));
                                }),
                        outputTensors.end());
            }
        }
        status = m_Runtime->EnqueueWorkload(m_NetworkId,
                                            inputTensors,
                                            outputTensors,
                                            importedInputIds,
                                            importedOutputIds);
 
        if (ctx.measureTimings == MeasureTiming::YES)
        {
            ctx.deviceEnd =  Clock::now();
        }
        if (status != armnn::Status::Success)
        {
            VLOG(DRIVER) << "ArmnnPreparedModel:ExecuteGraph EnqueueWorkload failed";
            return ErrorStatus::GENERAL_FAILURE;
        }
    }
    catch (armnn::Exception& e)
    {
        VLOG(DRIVER) << "armnn:Exception caught from EnqueueWorkload: " << e.what();
        return ErrorStatus::GENERAL_FAILURE;
    }
    catch (std::exception& e)
    {
        VLOG(DRIVER) << "std::exception caught from EnqueueWorkload: " << e.what();
        return ErrorStatus::GENERAL_FAILURE;
    }
 
    if (!pointerMemory && (!importedInputIds.empty() || !importedOutputIds.empty()))
    {
        CommitPools(*pMemPools);
    }
    DumpTensorsIfRequired("Output", outputTensors);
 
    if (ctx.measureTimings == MeasureTiming::YES)
    {
        ctx.driverEnd =  Clock::now();
        Timing timing;
        timing.timeOnDevice = ctx.deviceEnd - ctx.deviceStart;
        timing.timeInDriver = ctx.driverEnd - ctx.driverStart;
        VLOG(DRIVER) << "ArmnnPreparedModel::execute timing - Device = "
                     << timing.timeOnDevice << "Driver = " <<  timing.timeInDriver;
    }
    return ErrorStatus::NONE;
}

References armnn_driver::CommitPools(), CanonicalExecutionContext::deviceEnd, CanonicalExecutionContext::deviceStart, CanonicalExecutionContext::driverEnd, CanonicalExecutionContext::driverStart, IRuntime::EnqueueWorkload(), IRuntime::ImportInputs(), IRuntime::ImportOutputs(), armnn::Malloc, CanonicalExecutionContext::measureTimings, armnn::Success, and Exception::what().

Referenced by ArmnnPreparedModel::execute(), ArmnnPreparedModel::executeFenced(), and ArmnnPreparedModel::ExecuteWithDummyInputs().

ExecuteWithDummyInputs()

bool ExecuteWithDummyInputs	(	unsigned int	numInputs,
		unsigned int	numOutputs
	)		const

Executes this model with dummy inputs (e.g.

all zeroes).

Returns

false on failure, otherwise true

Definition at line 648 of file ArmnnPreparedModel.cpp.

{
    std::vector<std::vector<char>> storage;
    armnn::InputTensors inputTensors;
    for (unsigned int i = 0; i < numInputs; i++)
    {
        armnn::TensorInfo inputTensorInfo = m_Runtime->GetInputTensorInfo(m_NetworkId, i);
        // pInputTensors (of type InputTensors) is composed of a vector of ConstTensors.
        // Therefore, set all TensorInfo isConstant parameters of input Tensors to true.
        inputTensorInfo.SetConstant();
        storage.emplace_back(inputTensorInfo.GetNumBytes());
        const armnn::ConstTensor inputTensor(inputTensorInfo, storage.back().data());
 
        inputTensors.emplace_back(i, inputTensor);
    }
 
    armnn::OutputTensors outputTensors;
    for (unsigned int i = 0; i < numOutputs; i++)
    {
        const armnn::TensorInfo outputTensorInfo = m_Runtime->GetOutputTensorInfo(m_NetworkId, i);
        storage.emplace_back(outputTensorInfo.GetNumBytes());
        const armnn::Tensor outputTensor(outputTensorInfo, storage.back().data());
 
        outputTensors.emplace_back(i, outputTensor);
    }
    CanonicalExecutionContext ctx;
    ctx.measureTimings = MeasureTiming::NO;
    auto memPools = std::make_shared<std::vector<::android::nn::RunTimePoolInfo>>();
 
    auto errorStatus = ExecuteGraph(memPools,
                                    inputTensors,
                                    outputTensors,
                                    ctx);
 
    return errorStatus == ErrorStatus::NONE;
}

References ArmnnPreparedModel::ExecuteGraph(), IRuntime::GetInputTensorInfo(), TensorInfo::GetNumBytes(), IRuntime::GetOutputTensorInfo(), CanonicalExecutionContext::measureTimings, and TensorInfo::SetConstant().

GetModelPriority()

Priority GetModelPriority

(

)

const

Definition at line 472 of file ArmnnPreparedModel.cpp.

{
    return m_ModelPriority;
}

getUnderlyingResource()

std::any getUnderlyingResource ( ) const

override

Definition at line 606 of file ArmnnPreparedModel.cpp.

{
    return &m_Model;
}

---

The documentation for this class was generated from the following files:

• shim/sl/canonical/ArmnnPreparedModel.hpp
• shim/sl/canonical/ArmnnPreparedModel.cpp