LoadedNetwork Class Reference

#include <LoadedNetwork.hpp>

Public Types
using	WorkloadQueue = std::vector< std::unique_ptr< IWorkload > >

Public Member Functions
	~LoadedNetwork ()
std::unique_ptr< IWorkingMemHandle >	CreateWorkingMemHandle (NetworkId networkId)
	Create a new unique WorkingMemHandle object. More...
TensorInfo	GetInputTensorInfo (LayerBindingId layerId) const
TensorInfo	GetOutputTensorInfo (LayerBindingId layerId) const
std::vector< ImportedInputId >	ImportInputs (const InputTensors &inputTensors)
std::vector< ImportedOutputId >	ImportOutputs (const OutputTensors &outputTensors)
void	ClearImportedInputs (const std::vector< ImportedInputId > inputIds)
void	ClearImportedOutputs (const std::vector< ImportedOutputId > outputIds)
Status	EnqueueWorkload (const InputTensors &inputTensors, const OutputTensors &outputTensors)
	Single thread execution of the loaded network. More...
Status	Execute (const InputTensors &inputTensors, const OutputTensors &outputTensors, IWorkingMemHandle &workingMemHandle, std::vector< ImportedInputId > preImportedInputs={}, std::vector< ImportedOutputId > preImportedOutputs={})
	Thread safe execution of the loaded network. More...
const std::shared_ptr< IProfiler > &	GetProfiler () const
void	FreeWorkingMemory ()
void	RegisterDebugCallback (const DebugCallbackFunction &func)
void	SendNetworkStructure ()
bool	IsAsyncEnabled ()
profiling::ProfilingGuid	GetNetworkGuid ()

Static Public Member Functions
static std::unique_ptr< LoadedNetwork >	MakeLoadedNetwork (std::unique_ptr< IOptimizedNetwork > net, std::string &errorMessage, const INetworkProperties &networkProperties, profiling::ProfilingService &profilingService)

Detailed Description

Definition at line 41 of file LoadedNetwork.hpp.

Member Typedef Documentation

WorkloadQueue

using WorkloadQueue = std::vector<std::unique_ptr<IWorkload> >

Definition at line 44 of file LoadedNetwork.hpp.

Constructor & Destructor Documentation

~LoadedNetwork()

~LoadedNetwork ( )

inline

Definition at line 46 of file LoadedNetwork.hpp.

    {
        FreeWorkingMemory();
    }

Member Function Documentation

ClearImportedInputs()

void ClearImportedInputs

(

const std::vector< ImportedInputId >

inputIds

)

Definition at line 1241 of file LoadedNetwork.cpp.

Referenced by RuntimeImpl::ClearImportedInputs().

{
    for (auto id : inputIds)
    {
        if (id > m_PreImportedInputHandles.size())
        {
            throw InvalidArgumentException(fmt::format("ClearImportedInputs::Unknown ImportedInputId: {}", id));
        }

        auto& importedTensorHandle = m_PreImportedInputHandles[id].m_TensorHandle;
        if (!importedTensorHandle)
        {
            throw InvalidArgumentException(
                    fmt::format("ClearImportedInputs::ImportedInput with id: {} has already been deleted", id));
        }
        // Call Unimport then destroy the tensorHandle
        importedTensorHandle->Unimport();
        importedTensorHandle = {};
    }
}

ClearImportedOutputs()

void ClearImportedOutputs

(

const std::vector< ImportedOutputId >

outputIds

)

Definition at line 1262 of file LoadedNetwork.cpp.

Referenced by RuntimeImpl::ClearImportedOutputs().

{
    for (auto id : outputIds)
    {
        if (id > m_PreImportedOutputHandles.size())
        {
            throw InvalidArgumentException(fmt::format("ClearImportedOutputs::Unknown ImportedOutputId: {}", id));
        }

       auto& importedTensorHandle = m_PreImportedOutputHandles[id].m_TensorHandle;
       if (!importedTensorHandle)
       {
           throw InvalidArgumentException(
                   fmt::format("ClearImportedOutputs::ImportedOutput with id: {} has already been deleted", id));
       }
       // Call Unimport then destroy the tensorHandle
       importedTensorHandle->Unimport();
       importedTensorHandle = {};
    }
}

CreateWorkingMemHandle()

std::unique_ptr< IWorkingMemHandle > CreateWorkingMemHandle

(

NetworkId

networkId

)

Create a new unique WorkingMemHandle object.

Create multiple handles if you wish to have overlapped Execution by calling this function from different threads.

Definition at line 1505 of file LoadedNetwork.cpp.

References ARMNN_ASSERT, ARMNN_NO_DEPRECATE_WARN_BEGIN, ARMNN_NO_DEPRECATE_WARN_END, ITensorHandleFactory::CreateTensorHandle(), Layer::GetBackendId(), and ITensorHandleFactory::LegacyFactoryId.

Referenced by RuntimeImpl::CreateWorkingMemHandle().

{
    Graph& order = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();

    // Tensors that will need to be allocated internally within armnn
    std::vector<std::unique_ptr<ITensorHandle>> managedTensorHandles;
    // Tensors that will be allocated externally by the user
    std::vector<std::unique_ptr<ITensorHandle>> unmanagedTensorHandles;

    std::vector<WorkingMemDescriptor> workingMemDescriptors;
    std::unordered_map<LayerGuid, WorkingMemDescriptor> workingMemDescriptorMap;

    auto GetTensorHandle = [&](Layer* layer, const OutputSlot& outputSlot)
    {
        ITensorHandleFactory::FactoryId factoryId = outputSlot.GetTensorHandleFactoryId();
        const TensorInfo& tensorInfo = outputSlot.GetTensorInfo();

        if (factoryId == ITensorHandleFactory::LegacyFactoryId)
        {
            BackendId id = layer->GetBackendId();
            ARMNN_NO_DEPRECATE_WARN_BEGIN
            return m_WorkloadFactories.at(id)->CreateTensorHandle(tensorInfo, false);
            ARMNN_NO_DEPRECATE_WARN_END
        }
        else
        {
            ITensorHandleFactory* handleFactory = m_TensorHandleFactoryRegistry.GetFactory(factoryId);
            ARMNN_ASSERT(handleFactory);
            return handleFactory->CreateTensorHandle(tensorInfo, false);
        }
    };

    struct HandleInfo
    {
        ITensorHandle* m_TensorHandle;

        bool m_IsInputLayerHandle = false;
        bool m_IsOutputLayerHandle = false;

        WorkingMemHandle::InputMemDescriptorCoords m_InputMemDescriptorCoords;
        WorkingMemHandle::OutputMemDescriptorCoords m_OutputMemDescriptorCoords;
    };

    std::unordered_map<const OutputSlot*, HandleInfo> outputToHandleInfoMap;

    unsigned int layerIndex = 0;
    for (auto&& layer : order)
    {
        // Constant layers execution and management is handled during loaded network construction
        if (layer->GetType() == LayerType::Constant)
        {
            continue;
        }

        WorkingMemDescriptor workingMemDescriptor;

        bool isMemoryManaged = true;
        bool isInputLayer = false;
        bool isOutputLayer = false;
        bool isConnectedToOutputLayer = false;

        if (layer->GetType() == LayerType::Input || layer->GetType() == LayerType::MemImport)
        {
            // Input layers/workloads will not be executed so the descriptor is not added to workingMemDescriptors
            // However we will still need to manage the tensorHandle
            isInputLayer = true;
            isMemoryManaged = !m_NetworkProperties.m_ImportEnabled;
        }
        else if (layer->GetType() == LayerType::Output)
        {
            isOutputLayer = true;
        }

        unsigned int slotIndex = 0;
        // Create a tensor handle for each output slot of a layer
        // Once we create it, we start managing its lifetime
        for (auto& slot : layer->GetOutputSlots())
        {
            for (unsigned int i = 0; i < slot.GetNumConnections(); ++i)
            {
                if ((slot.GetConnection(i)->GetOwningLayer().GetType() == LayerType::Output))
                {
                    if (!isConnectedToOutputLayer)
                    {
                        isConnectedToOutputLayer = true;
                        // If Export is enabled disable memory management, so we can export, otherwise we do a copy
                        isMemoryManaged = !m_NetworkProperties.m_ExportEnabled;
                    }
                    else
                    {
                        // Importing in this case would likely cause unexpected behaviour, so we disallow it.
                        ARMNN_LOG(warning) <<
                           fmt::format("Layer name: '{0}' guid: '{1}' has two or more OutputLayers connected to it. "
                                       "This will prevent importing on the connected OutputLayers.",
                                        layer->GetName(), layer->GetGuid());
                        isMemoryManaged = true;
                    }
                }
            }

            ITensorHandle* tensorHandle;
            if (isMemoryManaged)
            {
                managedTensorHandles.emplace_back(GetTensorHandle(layer, slot));
                tensorHandle = managedTensorHandles.back().get();
            }
            else
            {
                unmanagedTensorHandles.emplace_back(GetTensorHandle(layer, slot));
                tensorHandle = unmanagedTensorHandles.back().get();
            }

            workingMemDescriptor.m_Outputs.push_back(tensorHandle);

            HandleInfo& handleInfo = outputToHandleInfoMap[&slot];
            handleInfo.m_TensorHandle = tensorHandle;

            // Store the coordinates of the current layer's OutputSlot that is connected to the OutputLayer
            if (isConnectedToOutputLayer)
            {
                handleInfo.m_IsOutputLayerHandle = true;
                handleInfo.m_OutputMemDescriptorCoords.m_OutputSlotCoords = {layerIndex, slotIndex};
            }
            // Store the LayerBindingId of the InputLayer
            if (isInputLayer)
            {
                handleInfo.m_IsInputLayerHandle = true;
                LayerBindingId bindingId = static_cast<BindableLayer*>(layer)->GetBindingId();
                handleInfo.m_InputMemDescriptorCoords.m_LayerBindingId = bindingId;
            }
            slotIndex++;
        }
        // Loop through the input slots in the same layer and decrement the reference counter associated
        // to each tensor handle we encounter.
        // Once it reaches zero, the lifetime of the tensor handle has ended, and we mark its memory as available
        // so that the next tensor handle with a non overlapping lifetime can share its memory.
        for (auto& slot : layer->GetInputSlots())
        {
            ARMNN_ASSERT(slot.GetConnection());
            auto outputSlot = slot.GetConnectedOutputSlot();
            auto key = outputSlot->GetOwningLayer().GetGuid();

            // Constant layers execution and management is handled during loaded network construction
            auto found = m_ConstantTensorHandles.find(key);
            if (found != m_ConstantTensorHandles.end())
            {
                ITensorHandle* tensorHandle = found->second;
                workingMemDescriptor.m_Inputs.push_back(tensorHandle);

                // Odd case where a constant layer is connected to an output layer
                // We will need to create a HandleInfo to track it
                if (isOutputLayer)
                {
                    LayerBindingId bindingId = static_cast<BindableLayer*>(layer)->GetBindingId();

                    HandleInfo& handleInfo = outputToHandleInfoMap[outputSlot];
                    handleInfo.m_TensorHandle = tensorHandle;
                    handleInfo.m_IsOutputLayerHandle = true;
                    handleInfo.m_OutputMemDescriptorCoords.m_LayerBindingIds.push_back(bindingId);
                    handleInfo.m_OutputMemDescriptorCoords.m_InputSlotCoords.push_back({layerIndex, 0});
                }
                continue;
            }

            HandleInfo& handleInfo = outputToHandleInfoMap.at(outputSlot);

            ITensorHandle* inputTensorHandle = handleInfo.m_TensorHandle;
            workingMemDescriptor.m_Inputs.push_back(inputTensorHandle);

            // Store the LayerBindingId of the OutputLayer
            if (isOutputLayer)
            {
                LayerBindingId bindingId = static_cast<BindableLayer*>(layer)->GetBindingId();
                handleInfo.m_OutputMemDescriptorCoords.m_LayerBindingIds.push_back(bindingId);
                handleInfo.m_OutputMemDescriptorCoords.m_InputSlotCoords.push_back({layerIndex, 0});
            }
            // In this case the layer is not an Output Layer but shares its input tensorhandle with an OutputLayer
            // It will need to be updated as well, if we swap out the tensorhandle
            else if (handleInfo.m_IsOutputLayerHandle)
            {
                handleInfo.m_OutputMemDescriptorCoords.m_InputSlotCoords.push_back({layerIndex, slot.GetSlotIndex()});
            }

            // Store the coordinates of the InputSlots connected to the InputLayer
            // There can be more than one InputSlot connected to an InputLayer, so we use a vector
            if (handleInfo.m_IsInputLayerHandle)
            {
                std::pair<LayerGuid, unsigned int> connectionLocation{layerIndex, slot.GetSlotIndex()};
                handleInfo.m_InputMemDescriptorCoords.m_InputSlotCoords.emplace_back(connectionLocation);
            }
        }
        workingMemDescriptorMap.insert({layer->GetGuid(), workingMemDescriptor});

        // Input/Output layers/workloads will not be executed, so the descriptor is not added to workingMemDescriptors
        // However we will still need to manage the tensorHandle
        if (!isInputLayer)
        {
            workingMemDescriptors.push_back(workingMemDescriptor);
            layerIndex++;
        }
    }

    std::vector<std::pair<std::shared_ptr<TensorMemory>, MemorySource>> tensorMemory;

    auto externalMemoryManager = CreateExternalMemoryManger(tensorMemory);

    // Sort m_TensorMemory, so it's order matches the outputSlot order
    std::sort(tensorMemory.begin(), tensorMemory.end(),
              [](const std::pair<std::shared_ptr<TensorMemory>, MemorySource>& lhs,
                 const std::pair<std::shared_ptr<TensorMemory>, MemorySource>& rhs)
              {
                  return lhs.first->m_OutputSlotId < rhs.first->m_OutputSlotId;
              });

    std::vector<WorkingMemHandle::InputMemDescriptorCoords> inputConnectionsInfo;
    std::vector<WorkingMemHandle::OutputMemDescriptorCoords> outputConnectionsInfo;

    for (const auto& handleInfo: outputToHandleInfoMap)
    {
        if (handleInfo.second.m_IsOutputLayerHandle)
        {
            outputConnectionsInfo.emplace_back(handleInfo.second.m_OutputMemDescriptorCoords);
        }

        if (handleInfo.second.m_IsInputLayerHandle)
        {
            inputConnectionsInfo.emplace_back(handleInfo.second.m_InputMemDescriptorCoords);
        }
    }

    return std::make_unique<WorkingMemHandle>(networkId,
                                              inputConnectionsInfo,
                                              outputConnectionsInfo,
                                              workingMemDescriptors,
                                              workingMemDescriptorMap,
                                              std::move(externalMemoryManager),
                                              std::move(tensorMemory),
                                              std::move(managedTensorHandles),
                                              std::move(unmanagedTensorHandles));
}

EnqueueWorkload()

Status EnqueueWorkload	(	const InputTensors &	inputTensors,
		const OutputTensors &	outputTensors
	)

Single thread execution of the loaded network.

Definition at line 636 of file LoadedNetwork.cpp.

References ARMNN_ASSERT_MSG, ARMNN_LOG, ARMNN_SCOPED_HEAP_PROFILING, ARMNN_SCOPED_PROFILING_EVENT, armnn::CheckFlag(), armnn::Failure, ITensorHandle::GetImportFlags(), Graph::GetInputLayers(), Layer::GetInputSlots(), Graph::GetNumInputs(), Layer::GetNumInputSlots(), Graph::GetNumLayers(), Graph::GetNumOutputs(), Layer::GetNumOutputSlots(), Layer::GetOutputHandler(), Graph::GetOutputLayers(), TimelineUtilityMethods::GetTimelineUtils(), Layer::GetType(), armnn::IgnoreUnused(), ITensorHandle::Import(), armnn::info, armnn::Input, QueueDescriptor::m_Inputs, WorkloadInfo::m_InputTensorInfos, QueueDescriptor::m_Outputs, WorkloadInfo::m_OutputTensorInfos, ITensorHandle::Map(), armnn::Output, armnn::Success, armnn::Undefined, ITensorHandle::Unmap(), and armnn::warning.

Referenced by RuntimeImpl::EnqueueWorkload().

{
    const Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();

    // Walk graph to determine the order of execution.
    if (graph.GetNumLayers() < 2)
    {
        ARMNN_LOG(warning) << "IRuntime::EnqueueWorkload()::Less than two nodes in graph";
        return Status::Failure;
    }

    // Data that must be kept alive for the entire execution of the workload.
    WorkloadData workloadData(inputTensors, outputTensors);

    if (graph.GetNumInputs() != inputTensors.size())
    {
        throw InvalidArgumentException("Number of inputs provided does not match network.");
    }

    // For each input to the network, call EnqueueInput with the data passed by the user.
    {
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "PrepareInputs");
        m_InputQueue.clear();
        m_InputQueue.reserve(graph.GetNumInputs());
        for (const BindableLayer* inputLayer : graph.GetInputLayers())
        {
            const TensorPin& pin = workloadData.GetInputTensorPin(inputLayer->GetBindingId());
            EnqueueInput(*inputLayer, pin.GetTensorHandle(), pin.GetTensorInfo());
        }
    }

    // For each output to the network, call EnqueueOutput with the data passed by the user.
    {
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "PrepareOutputs");
        m_OutputQueue.clear();
        m_OutputQueue.reserve(graph.GetNumOutputs());
        for (const BindableLayer* outputLayer : graph.GetOutputLayers())
        {
            const TensorPin& pin = workloadData.GetOutputTensorPin(outputLayer->GetBindingId());
            EnqueueOutput(*outputLayer, pin.GetTensorHandle(), pin.GetTensorInfo());
        }
    }

    std::unique_ptr<TimelineUtilityMethods> timelineUtils =
                        TimelineUtilityMethods::GetTimelineUtils(m_ProfilingService);
    ProfilingGuid inferenceGuid = m_ProfilingService.GetNextGuid();
    if (timelineUtils)
    {
        // Add inference timeline trace if profiling is enabled.
        ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
        timelineUtils->CreateTypedEntity(inferenceGuid, LabelsAndEventClasses::INFERENCE_GUID);
        timelineUtils->CreateRelationship(ProfilingRelationshipType::RetentionLink,
                                          networkGuid,
                                          inferenceGuid,
                                          LabelsAndEventClasses::EXECUTION_OF_GUID);
        timelineUtils->RecordEvent(inferenceGuid, LabelsAndEventClasses::ARMNN_PROFILING_SOL_EVENT_CLASS);
    }

    bool executionSucceeded = true;

    {
        if (m_ProfilingService.IsProfilingEnabled())
        {
            m_ProfilingService.IncrementCounterValue(armnn::profiling::INFERENCES_RUN);
        }
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "Execute");
        ARMNN_SCOPED_HEAP_PROFILING("Executing");
        executionSucceeded = Execute(timelineUtils, inferenceGuid);
    }

    if (timelineUtils)
    {
        // Add end of life of the inference timeline if profiling is enabled.
        timelineUtils->RecordEvent(inferenceGuid, LabelsAndEventClasses::ARMNN_PROFILING_EOL_EVENT_CLASS);
        timelineUtils->Commit();
    }
    return executionSucceeded ? Status::Success : Status::Failure;
}

Execute()

Status Execute	(	const InputTensors &	inputTensors,
		const OutputTensors &	outputTensors,
		IWorkingMemHandle &	workingMemHandle,
		std::vector< ImportedInputId >	preImportedInputs = {},
		std::vector< ImportedOutputId >	preImportedOutputs = {}
	)

Thread safe execution of the loaded network.

Definition at line 1283 of file LoadedNetwork.cpp.

References WorkingMemHandle::Allocate(), ARMNN_LOG, ARMNN_SCOPED_PROFILING_EVENT, armnn::CopyToOutputTensor(), armnn::error, armnn::Failure, WorkingMemHandle::GetBindingIdVector(), WorkingMemHandle::GetInputConnections(), WorkingMemHandle::GetInputHandle(), Graph::GetNumInputs(), Graph::GetNumOutputs(), WorkingMemHandle::GetOutputConnection(), WorkingMemHandle::GetOutputHandle(), TimelineUtilityMethods::GetTimelineUtils(), WorkingMemHandle::GetWorkingMemDescriptorAt(), WorkingMemHandle::IsAllocated(), WorkingMemHandle::MemSyncOutputs(), armnn::profiling::RetentionLink, armnn::Success, armnn::Undefined, and WorkingMemHandle::ValidateBindingIds().

Referenced by RuntimeImpl::Execute(), and LoadedNetwork::FreeWorkingMemory().

{
    const Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();

    if (inputTensors.size() + preImportedInputs.size() != graph.GetNumInputs())
    {
        if (preImportedInputs.empty())
        {
            throw InvalidArgumentException("LoadedNetwork::Execute: Number of inputs provided does not match network.");
        }
        else
        {
            throw InvalidArgumentException("LoadedNetwork::Execute: "
                                           "Number of inputs + preImportedInputs provided does not match network.");
        }
    }

    if (outputTensors.size() + preImportedOutputs.size() != graph.GetNumOutputs())
    {
        if (preImportedOutputs.empty())
        {
            throw InvalidArgumentException("LoadedNetwork::Execute: "
                                           "Number of outputs provided does not match network.");
        }
        else
        {
            throw InvalidArgumentException("LoadedNetwork::Execute: "
                                           "Number of outputs + preImportedOutputs provided does not match network.");
        }
    }

    WorkingMemHandle& workingMemHandle = dynamic_cast<WorkingMemHandle&>(iWorkingMemHandle);
    // Collect all the given LayerBindingIds and check them for duplicates and unknowns.
    std::vector<LayerBindingId>& bindingIds = workingMemHandle.GetBindingIdVector();
    unsigned int index = 0;
    for (auto pair : inputTensors)
    {
        bindingIds[index++] = pair.first;
    }
    for (ImportedInputId id : preImportedInputs)
    {
        bindingIds[index++] = ValidateImportedInputID(id);
    }
    for (auto pair : outputTensors)
    {
        bindingIds[index++] = pair.first;
    }
    for (ImportedOutputId id : preImportedOutputs)
    {
        bindingIds[index++] = ValidateImportedOutputID(id);
    }

    workingMemHandle.ValidateBindingIds();

    auto resetMemHandle = [&]()
    {
        for (ImportedInputId id: preImportedInputs)
        {
            const LayerBindingId layerBindingId = m_PreImportedInputHandles[id].m_LayerBindingId;

            auto inputHandle = workingMemHandle.GetInputHandle(layerBindingId);
            auto inputConnections = workingMemHandle.GetInputConnections(layerBindingId);
            for (auto it : inputConnections)
            {
                *it = inputHandle;
            }
        }

        for (ImportedOutputId id: preImportedOutputs)
        {
            const LayerBindingId layerBindingId = m_PreImportedOutputHandles[id].m_LayerBindingId;

            auto outputHandle = workingMemHandle.GetOutputHandle(layerBindingId);
            auto outputConnections = workingMemHandle.GetOutputConnection(layerBindingId);

            for (auto it : outputConnections)
            {
                *it = outputHandle;
            }
        }
    };

    std::unique_ptr<profiling::TimelineUtilityMethods> timelineUtils =
            profiling::TimelineUtilityMethods::GetTimelineUtils(m_ProfilingService);
    profiling::ProfilingGuid inferenceGuid = m_ProfilingService.GetNextGuid();
    if (timelineUtils)
    {
        // Add inference timeline trace if profiling is enabled.
        profiling::ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
        timelineUtils->CreateTypedEntity(inferenceGuid, profiling::LabelsAndEventClasses::INFERENCE_GUID);
        timelineUtils->CreateRelationship(profiling::ProfilingRelationshipType::RetentionLink,
                                          networkGuid,
                                          inferenceGuid,
                                          profiling::LabelsAndEventClasses::EXECUTION_OF_GUID);
        timelineUtils->RecordEvent(inferenceGuid, profiling::LabelsAndEventClasses::ARMNN_PROFILING_SOL_EVENT_CLASS);
    }

    bool executionSucceeded = true;

    if (timelineUtils)
    {
        // Add end of life of the inference timeline if profiling is enabled.
        timelineUtils->RecordEvent(inferenceGuid, profiling::LabelsAndEventClasses::ARMNN_PROFILING_EOL_EVENT_CLASS);
        timelineUtils->Commit();
    }

    if (!workingMemHandle.IsAllocated())
    {
        workingMemHandle.Allocate();
    }

    {
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "PrepareInputs");
        for (auto pair : inputTensors)
        {
            EnqueueInput(pair.second, workingMemHandle.GetInputHandle(pair.first));
        }

        // Swap in the pre-imported inputs if any
        for (ImportedInputId id : preImportedInputs)
        {
            const ImportedTensorHandlePin& importedInputPin = m_PreImportedInputHandles[id];
            const LayerBindingId layerBindingId = m_PreImportedInputHandles[id].m_LayerBindingId;
            const auto& preimportedHandle = importedInputPin.m_TensorHandle;

            auto inputConnections = workingMemHandle.GetInputConnections(layerBindingId);
            for (auto it : inputConnections)
            {
                *it = preimportedHandle.get();
            }
        }
    }
    {
        ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "PrepareOutputs");
        if (m_NetworkProperties.m_ExportEnabled)
        {
            for (auto pair: outputTensors)
            {
                ImportOutputTensor(pair.second, workingMemHandle.GetOutputHandle(pair.first));
            }
        }

        for (ImportedOutputId id : preImportedOutputs)
        {
            const ImportedTensorHandlePin& importedOutputPin = m_PreImportedOutputHandles[id];
            const LayerBindingId layerBindingId = m_PreImportedOutputHandles[id].m_LayerBindingId;
            const auto& preimportedHandle = importedOutputPin.m_TensorHandle;

            auto outputConnections = workingMemHandle.GetOutputConnection(layerBindingId);

            for (auto it : outputConnections)
            {
                *it = preimportedHandle.get();
            }
        }
    }

    auto Fail = [&](const std::exception& error)
    {
        ARMNN_LOG(error) << "An error occurred attempting to execute a workload: " << error.what();
        executionSucceeded = false;
    };
    profiling::ProfilingDynamicGuid workloadInferenceID(0);

    try
    {
        for (unsigned int i = 0; i < m_WorkloadQueue.size(); ++i)
        {
            auto& workload = m_WorkloadQueue[i];
            if (timelineUtils)
            {
                workloadInferenceID = timelineUtils->RecordWorkloadInferenceAndStartOfLifeEvent(workload->GetGuid(),
                                                                                                inferenceGuid);
            }
            workload->ExecuteAsync(workingMemHandle.GetWorkingMemDescriptorAt(i));

            if (timelineUtils)
            {
                timelineUtils->RecordEndOfLifeEvent(workloadInferenceID);
            }
        }
    }
    catch (const RuntimeException& error)
    {
        resetMemHandle();
        Fail(error);
    }
    catch (const std::runtime_error& error)
    {
        resetMemHandle();
        Fail(error);
    }
    catch (...)
    {
        resetMemHandle();
        throw;
    }

    if (!m_NetworkProperties.m_ExportEnabled)
    {
        for (auto pair: outputTensors)
        {
            CopyToOutputTensor(pair.second, workingMemHandle.GetOutputHandle(pair.first));
        }
    }
    else
    {
       ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "SyncMemGeneric_Execute");
       workingMemHandle.MemSyncOutputs();
    }

    resetMemHandle();

    return executionSucceeded ? Status::Success : Status::Failure;
}

FreeWorkingMemory()

void FreeWorkingMemory

(

)

Definition at line 902 of file LoadedNetwork.cpp.

References ARMNN_ASSERT_MSG, ARMNN_LOG, armnn::CheckFlag(), armnn::CopyTensorContentsGeneric(), armnn::error, LoadedNetwork::Execute(), ITensorHandle::GetImportFlags(), BaseTensor< MemoryType >::GetInfo(), BaseTensor< MemoryType >::GetMemoryArea(), ITensorHandle::Import(), and ITensorHandle::Map().

Referenced by RuntimeImpl::CreateWorkingMemHandle(), and RuntimeImpl::EnqueueWorkload().

{
    std::lock_guard<std::mutex> lockGuard(m_WorkingMemMutex);

    if (!m_IsWorkingMemAllocated)
    {
        return;
    }

    if (m_ExternalMemoryManager)
    {
        m_ExternalMemoryManager->Deallocate();
    }

    // Informs the memory managers to release memory in its respective memory group
    for (auto&& memoryManager : m_BackendMemoryMangers)
    {
        if (memoryManager)
        {
            memoryManager->Release();
        }
    }
    m_TensorHandleFactoryRegistry.ReleaseMemory();
    m_IsWorkingMemAllocated = false;
}

GetInputTensorInfo()

TensorInfo GetInputTensorInfo

(

LayerBindingId

layerId

)

const

Definition at line 487 of file LoadedNetwork.cpp.

References ARMNN_ASSERT_MSG.

Referenced by RuntimeImpl::GetInputTensorInfo().

{
    for (auto&& inputLayer : m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetInputLayers())
    {
        ARMNN_ASSERT_MSG(inputLayer->GetNumOutputSlots() == 1, "Input layer should have exactly 1 output slot");
        if (inputLayer->GetBindingId() == layerId)
        {
            return inputLayer->GetOutputSlot(0).GetTensorInfo();
        }
    }

    throw InvalidArgumentException(fmt::format("No input layer is associated with id {}", layerId));
}

GetNetworkGuid()

profiling::ProfilingGuid GetNetworkGuid

(

)

Definition at line 482 of file LoadedNetwork.cpp.

{
    return m_OptimizedNetwork->GetGuid();
}

GetOutputTensorInfo()

TensorInfo GetOutputTensorInfo

(

LayerBindingId

layerId

)

const

Definition at line 501 of file LoadedNetwork.cpp.

References ARMNN_ASSERT_MSG, CHECK_LOCATION, BackendId::Get(), Layer::GetBackendId(), Layer::GetNameStr(), armnn::IgnoreUnused(), armnn::info, and IWorkloadFactory::IsLayerSupported().

Referenced by RuntimeImpl::GetOutputTensorInfo().

{
    for (auto&& outputLayer : m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetOutputLayers())
    {
        ARMNN_ASSERT_MSG(outputLayer->GetNumInputSlots() == 1, "Output layer should have exactly 1 input slot");
        ARMNN_ASSERT_MSG(outputLayer->GetInputSlot(0).GetConnection(), "Input slot on Output layer must be connected");
        if (outputLayer->GetBindingId() == layerId)
        {
            return outputLayer->GetInputSlot(0).GetConnection()->GetTensorInfo();
        }
    }

    throw InvalidArgumentException(fmt::format("No output layer is associated with id {}", layerId));
}

GetProfiler()

const std::shared_ptr<IProfiler>& GetProfiler ( ) const

inline

Definition at line 82 of file LoadedNetwork.hpp.

Referenced by RuntimeImpl::CreateWorkingMemHandle(), RuntimeImpl::EnqueueWorkload(), and RuntimeImpl::Execute().

{ return m_OptimizedNetwork->GetProfiler(); }

ImportInputs()

std::vector< ImportedInputId > ImportInputs

(

const InputTensors &

inputTensors

)

Definition at line 1090 of file LoadedNetwork.cpp.

References ARMNN_ASSERT, Graph::InputLayersAccessor::begin(), armnn::CheckFlag(), ITensorHandleFactory::CreateTensorHandle(), Graph::InputLayersAccessor::end(), Layer::GetBackendId(), BindableLayer::GetBindingId(), ITensorHandle::GetImportFlags(), BaseTensor< MemoryType >::GetInfo(), Graph::GetInputLayers(), BaseTensor< MemoryType >::GetMemoryArea(), Layer::GetOutputSlots(), OutputSlot::GetTensorHandleFactoryId(), OutputSlot::GetTensorInfo(), Layer::GetType(), armnn::HasCapability(), ITensorHandle::Import(), armnn::Input, and Graph::TopologicalSort().

Referenced by RuntimeImpl::ImportInputs().

{
    if (!m_NetworkProperties.m_ImportEnabled)  // Try import the input tensor
    {
        throw MemoryImportException("ImportInputs: Memory Import failed, NetworkProperties.m_ImportEnabled");
    }

    std::vector<ImportedInputId> importedInputs;
    Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().TopologicalSort();

    for (auto inputTensor : inputTensors)
    {
        auto layerBindingId = inputTensor.first;
        auto it = std::find_if(graph.GetInputLayers().begin(), graph.GetInputLayers().end(), [=](auto* layer)
        {
            return layer->GetBindingId() == layerBindingId;
        });

        if (it == graph.GetInputLayers().end())
        {
            throw MemoryImportException(fmt::format("ImportInputs: Memory Import failed, unknown LayerBindingId: {}",
                                                    layerBindingId));
        }

        const Layer* layer = *it;
        if (layer->GetType() != LayerType::Input)
        {
            throw InvalidArgumentException("ImportInputs: given layer not an InputLayer");
        }

        auto& backend = m_Backends.at(layer->GetBackendId());
        if (!HasCapability(BackendOptions::BackendOption{"PreImportIOTensors", true}, backend->GetCapabilities()))
        {
            std::string er = backend->GetId();
            er += " does not have PreImportIOTensors capability";
            throw BackendCapabilityException(er);
        }

        const OutputSlot& outputSlot = layer->GetOutputSlots()[0];

        ITensorHandleFactory::FactoryId factoryId = outputSlot.GetTensorHandleFactoryId();
        const TensorInfo& tensorInfo = outputSlot.GetTensorInfo();

        ITensorHandleFactory* handleFactory = m_TensorHandleFactoryRegistry.GetFactory(factoryId);
        ARMNN_ASSERT(handleFactory);

        ImportedTensorHandlePin importedTensorHandlePin{layerBindingId,
                                                        handleFactory->CreateTensorHandle(tensorInfo, false)};

        ITensorHandle* tensorHandle = importedTensorHandlePin.m_TensorHandle.get();

        if (!CheckFlag(tensorHandle->GetImportFlags(), m_NetworkProperties.m_InputSource))
        {
            throw MemoryImportException(
                fmt::format("ImportInputs: Memory Import failed, backend: {} does not support importing from source {}"
                            , factoryId, m_NetworkProperties.m_InputSource));
        }

        std::unique_ptr<ITensorHandle> passThroughTensorHandle =
                std::make_unique<ConstPassthroughTensorHandle>(inputTensor.second.GetInfo(),
                                                               inputTensor.second.GetMemoryArea());

        if (tensorHandle->Import(passThroughTensorHandle->Map(), m_NetworkProperties.m_InputSource))
        {
            importedInputs.push_back(m_CurImportedInputId++);
            passThroughTensorHandle->Unmap();
        }
        else
        {
            passThroughTensorHandle->Unmap();
            throw MemoryImportException("ImportInputs: Memory Import failed");
        }

        m_PreImportedInputHandles.push_back(std::move(importedTensorHandlePin));
    }

    return importedInputs;
}

ImportOutputs()

std::vector< ImportedOutputId > ImportOutputs

(

const OutputTensors &

outputTensors

)

Definition at line 1169 of file LoadedNetwork.cpp.

References ARMNN_ASSERT, Graph::OutputLayersAccessor::begin(), armnn::CheckFlag(), ITensorHandleFactory::CreateTensorHandle(), Graph::OutputLayersAccessor::end(), Layer::GetBackendId(), BindableLayer::GetBindingId(), InputSlot::GetConnectedOutputSlot(), ITensorHandle::GetImportFlags(), Layer::GetInputSlots(), BaseTensor< MemoryType >::GetMemoryArea(), Graph::GetOutputLayers(), OutputSlot::GetTensorHandleFactoryId(), OutputSlot::GetTensorInfo(), Layer::GetType(), armnn::HasCapability(), ITensorHandle::Import(), armnn::Output, and Graph::TopologicalSort().

Referenced by RuntimeImpl::ImportOutputs().

{
    if (!m_NetworkProperties.m_ExportEnabled)  // Try import the output tensor
    {
        throw MemoryImportException("ImportOutputs: Memory Import failed, NetworkProperties.m_ImportEnabled");
    }

    std::vector<ImportedOutputId> importedOutputs;
    Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().TopologicalSort();

    for (const auto& outputTensor : outputTensors)
    {
        auto layerBindingId = outputTensor.first;
        auto it = std::find_if(graph.GetOutputLayers().begin(), graph.GetOutputLayers().end(), [=](auto* layer)
        {
            return layer->GetBindingId() == layerBindingId;
        });

        if (it == graph.GetOutputLayers().end())
        {
            throw MemoryImportException(fmt::format("ImportOutputs: Memory Import failed, unknown LayerBindingId: {}",
                                                     layerBindingId));
        }

        const Layer* layer = *it;
        if (layer->GetType() != LayerType::Output)
        {
            throw InvalidArgumentException("ImportOutputs: given layer not an OutputLayer");
        }

        auto& backend = m_Backends.at(layer->GetBackendId());
        if (!HasCapability(BackendOptions::BackendOption{"PreImportIOTensors", true}, backend->GetCapabilities()))
        {
            std::string er = backend->GetId();
            er += " does not have PreImportIOTensors capability";
            throw BackendCapabilityException(er);
        }

        const InputSlot& inputSlot = layer->GetInputSlots()[0];
        ITensorHandleFactory::FactoryId factoryId = inputSlot.GetConnectedOutputSlot()->GetTensorHandleFactoryId();
        const TensorInfo& tensorInfo = inputSlot.GetConnectedOutputSlot()->GetTensorInfo();

        ITensorHandleFactory* handleFactory = m_TensorHandleFactoryRegistry.GetFactory(factoryId);
        ARMNN_ASSERT(handleFactory);

        ImportedTensorHandlePin importedTensorHandlePin{layerBindingId,
                                                        handleFactory->CreateTensorHandle(tensorInfo, false)};

        ITensorHandle* tensorHandle = importedTensorHandlePin.m_TensorHandle.get();

        if (!CheckFlag(tensorHandle->GetImportFlags(), m_NetworkProperties.m_OutputSource))
        {
            throw MemoryImportException(fmt::format("ImportInputs: Memory Import failed, backend: "
                                                    "{} does not support importing from source {}"
                                                    , factoryId, m_NetworkProperties.m_OutputSource));
        }

        if (tensorHandle->Import(outputTensor.second.GetMemoryArea(), m_NetworkProperties.m_OutputSource))
        {
            importedOutputs.push_back(m_CurImportedOutputId++);
        }
        else
        {
            throw MemoryImportException("ImportInputs: Memory Import failed");
        }

        m_PreImportedOutputHandles.push_back(std::move(importedTensorHandlePin));
    }

    return importedOutputs;
}

IsAsyncEnabled()

bool IsAsyncEnabled ( )

inline

Definition at line 90 of file LoadedNetwork.hpp.

Referenced by RuntimeImpl::CreateWorkingMemHandle(), RuntimeImpl::EnqueueWorkload(), and RuntimeImpl::Execute().

    {
        return m_NetworkProperties.m_AsyncEnabled;
    }

MakeLoadedNetwork()

std::unique_ptr< LoadedNetwork > MakeLoadedNetwork ( std::unique_ptr< IOptimizedNetwork >  net, std::string &  errorMessage, const INetworkProperties &  networkProperties, profiling::ProfilingService &  profilingService  )

static

Definition at line 82 of file LoadedNetwork.cpp.

References ITensorHandle::Allocate(), ARMNN_ASSERT, ARMNN_LOG, ARMNN_SCOPED_PROFILING_EVENT, armnn::BackendRegistryInstance(), armnn::Constant, IBackendInternal::CreateMemoryManager(), IBackendInternal::CreateWorkloadFactory(), armnn::error, IBackendInternal::GetCapabilities(), armnnUtils::Processes::GetCurrentId(), BackendRegistry::GetFactory(), IBackend::GetId(), ProfilerManager::GetInstance(), BackendRegistry::GetMemoryOptimizerStrategies(), TimelineUtilityMethods::GetTimelineUtils(), armnn::HasCapability(), armnn::Input, ITensorHandleFactory::LegacyFactoryId, INetworkProperties::m_AsyncEnabled, INetworkProperties::m_OutputNetworkDetailsMethod, WorkingMemDescriptor::m_Outputs, INetworkProperties::m_ProfilingEnabled, armnn::MemImport, armnn::Output, ProfilerManager::RegisterProfiler(), IBackendInternal::SupportsTensorAllocatorAPI(), Graph::TopologicalSort(), and armnn::Undefined.

Referenced by RuntimeImpl::LoadNetwork().

{
    std::unique_ptr<LoadedNetwork> loadedNetwork;

    auto Fail = [&](const std::exception& error) -> std::unique_ptr<LoadedNetwork>
    {
        errorMessage = ToErrorMessage("An error occurred when preparing the network workloads: ", error);
        ARMNN_LOG(error) << errorMessage;

        return std::unique_ptr<LoadedNetwork>();
    };

    try
    {
        loadedNetwork.reset(new LoadedNetwork(std::move(net), networkProperties, profilingService));
    }
    catch (const armnn::RuntimeException& error)
    {
        return Fail(error);
    }
    catch (const armnn::Exception& error)
    {
        return Fail(error);
    }
    catch (const std::runtime_error& error)
    {
        return Fail(error);
    }

    return loadedNetwork;
}

RegisterDebugCallback()

void RegisterDebugCallback

(

const DebugCallbackFunction &

func

)

Definition at line 1746 of file LoadedNetwork.cpp.

References armnn::BackendRegistryInstance(), armnn::Constant, BackendRegistry::GetAllocators(), Layer::GetBackendId(), InputSlot::GetConnectedOutputSlot(), Layer::GetInputSlot(), Layer::GetInputSlots(), OutputSlot::GetNumConnections(), Layer::GetOutputSlots(), ITensorHandle::GetParent(), Layer::GetType(), armnn::Input, BufferStorage::m_BufferSize, TensorMemory::m_Offset, BufferStorage::m_TensorMemoryVector, armnn::Output, Graph::TopologicalSort(), and armnn::Undefined.

Referenced by RuntimeImpl::RegisterDebugCallback().

{
    for (auto&& workloadPtr: m_WorkloadQueue)
    {
        workloadPtr.get()->RegisterDebugCallback(func);
    }
}

SendNetworkStructure()

void SendNetworkStructure

(

)

Definition at line 444 of file LoadedNetwork.cpp.

References ARMNN_SCOPED_PROFILING_EVENT, TimelineUtilityMethods::GetTimelineUtils(), armnn::Input, armnn::Output, Graph::TopologicalSort(), and armnn::Undefined.

{
    ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "LoadNetwork_SendNetworkStructure");
    Graph& order = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().TopologicalSort();
    ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();

    std::unique_ptr<TimelineUtilityMethods> timelineUtils =
                        TimelineUtilityMethods::GetTimelineUtils(m_ProfilingService);

    timelineUtils->CreateTypedEntity(networkGuid, LabelsAndEventClasses::NETWORK_GUID);

    for (auto&& layer : order)
    {
        // Add layer to the post-optimisation network structure
        AddLayerStructure(timelineUtils, *layer, networkGuid);
        switch (layer->GetType())
        {
        case LayerType::Input:
        case LayerType::Output:
        {
            // Inputs and outputs are treated in a special way - see EnqueueInput() and EnqueueOutput().
            break;
        }
        default:
            {
            for (auto& workload : m_WorkloadQueue)
            {
                // Add workload to the post-optimisation network structure
                AddWorkloadStructure(timelineUtils, workload, *layer);
            }
            break;
            }
        }
    }
    // Commit to send the post-optimisation network structure
    timelineUtils->Commit();
}

---

The documentation for this class was generated from the following files:

• src/armnn/LoadedNetwork.hpp
• src/armnn/LoadedNetwork.cpp