diff options
Diffstat (limited to 'src/armnn')
-rw-r--r-- | src/armnn/ExecutionData.hpp | 21 | ||||
-rw-r--r-- | src/armnn/LoadedNetwork.cpp | 47 | ||||
-rw-r--r-- | src/armnn/WorkingMemHandle.cpp | 19 | ||||
-rw-r--r-- | src/armnn/WorkingMemHandle.hpp | 32 |
4 files changed, 85 insertions, 34 deletions
diff --git a/src/armnn/ExecutionData.hpp b/src/armnn/ExecutionData.hpp new file mode 100644 index 0000000000..69c8f3fa7a --- /dev/null +++ b/src/armnn/ExecutionData.hpp @@ -0,0 +1,21 @@ +// +// Copyright © 2022 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#pragma once + +namespace armnn +{ + +namespace experimental +{ + +struct ExecutionData +{ + void* m_Data; +}; + +} // end experimental namespace + +} // end armnn namespace diff --git a/src/armnn/LoadedNetwork.cpp b/src/armnn/LoadedNetwork.cpp index 8e664e699d..5dd7b6cd2a 100644 --- a/src/armnn/LoadedNetwork.cpp +++ b/src/armnn/LoadedNetwork.cpp @@ -9,12 +9,14 @@ #include "Profiling.hpp" #include "HeapProfiling.hpp" #include "WorkingMemHandle.hpp" +#include "ExecutionData.hpp" #include <armnn/BackendHelper.hpp> #include <armnn/BackendRegistry.hpp> #include <armnn/Logging.hpp> #include <armnn/backends/TensorHandle.hpp> +#include <armnn/backends/IBackendInternal.hpp> #include <armnn/backends/IMemoryManager.hpp> #include <armnn/backends/MemCopyWorkload.hpp> @@ -642,9 +644,13 @@ void LoadedNetwork::AllocateAndExecuteConstantWorkloadsAsync() m_ConstantTensorHandles[layer->GetGuid()] = tensorHandle; tensorHandle->Allocate(); + auto& backend = m_Backends.at(layer->GetBackendId()); + WorkingMemDescriptor memDesc; memDesc.m_Outputs.push_back(tensorHandle); - m_ConstantWorkloads[layer->GetGuid()]->ExecuteAsync(memDesc); + + ExecutionData executionData = backend->CreateExecutionData(memDesc); + m_ConstantWorkloads[layer->GetGuid()]->ExecuteAsync(executionData); } } } @@ -1717,15 +1723,15 @@ void LoadedNetwork::ClearImportedOutputs(const std::vector<ImportedOutputId> out throw InvalidArgumentException(fmt::format("ClearImportedOutputs::Unknown ImportedOutputId: {}", id)); } - auto& importedTensorHandle = m_PreImportedOutputHandles[id].m_TensorHandle; - if (!importedTensorHandle) - { - throw InvalidArgumentException( - fmt::format("ClearImportedOutputs::ImportedOutput with id: {} has already been deleted", id)); - } - // Call Unimport then destroy the tensorHandle - importedTensorHandle->Unimport(); - importedTensorHandle = {}; + auto& importedTensorHandle = m_PreImportedOutputHandles[id].m_TensorHandle; + if (!importedTensorHandle) + { + throw InvalidArgumentException( + fmt::format("ClearImportedOutputs::ImportedOutput with id: {} has already been deleted", id)); + } + // Call Unimport then destroy the tensorHandle + importedTensorHandle->Unimport(); + importedTensorHandle = {}; } } @@ -1882,7 +1888,6 @@ Status LoadedNetwork::Execute(const InputTensors& inputTensors, const auto& preimportedHandle = importedOutputPin.m_TensorHandle; auto outputConnections = workingMemHandle.GetOutputConnection(layerBindingId); - for (auto it : outputConnections) { *it = preimportedHandle.get(); @@ -1895,7 +1900,7 @@ Status LoadedNetwork::Execute(const InputTensors& inputTensors, ARMNN_LOG(error) << "An error occurred attempting to execute a workload: " << error.what(); executionSucceeded = false; }; - ProfilingDynamicGuid workloadInferenceID(0); + ProfilingDynamicGuid workloadInferenceID(0); try { @@ -1907,7 +1912,8 @@ Status LoadedNetwork::Execute(const InputTensors& inputTensors, workloadInferenceID = timelineUtils->RecordWorkloadInferenceAndStartOfLifeEvent(workload->GetGuid(), inferenceGuid); } - workload->ExecuteAsync(workingMemHandle.GetWorkingMemDescriptorAt(i)); + + workload->ExecuteAsync(workingMemHandle.GetExecutionDataAt(i).second); if (timelineUtils) { @@ -1961,7 +1967,7 @@ std::unique_ptr<IWorkingMemHandle> LoadedNetwork::CreateWorkingMemHandle(Network std::vector<std::unique_ptr<ITensorHandle>> unmanagedTensorHandles; std::vector<WorkingMemDescriptor> workingMemDescriptors; - std::unordered_map<LayerGuid, WorkingMemDescriptor> workingMemDescriptorMap; + std::vector<std::pair<BackendId, ExecutionData>> executionDataVec; auto GetTensorHandle = [&](Layer* layer, const OutputSlot& outputSlot) { @@ -2142,13 +2148,19 @@ std::unique_ptr<IWorkingMemHandle> LoadedNetwork::CreateWorkingMemHandle(Network handleInfo.m_InputMemDescriptorCoords.m_InputSlotCoords.emplace_back(connectionLocation); } } - workingMemDescriptorMap.insert({layer->GetGuid(), workingMemDescriptor}); // Input/Output layers/workloads will not be executed, so the descriptor is not added to workingMemDescriptors // However we will still need to manage the tensorHandle if (!isInputLayer) { + // Simply auto initialise ExecutionData here, so it's added only for the layer that require execution. + // The memory and data will be allocated/assigned for the void* in WorkingMemHandle::Allocate. + std::pair<BackendId, ExecutionData> dataPair; + dataPair.first = layer->GetBackendId(); + + executionDataVec.push_back(dataPair); workingMemDescriptors.push_back(workingMemDescriptor); + layerIndex++; } } @@ -2185,11 +2197,12 @@ std::unique_ptr<IWorkingMemHandle> LoadedNetwork::CreateWorkingMemHandle(Network inputConnectionsInfo, outputConnectionsInfo, workingMemDescriptors, - workingMemDescriptorMap, std::move(externalMemoryManager), std::move(tensorMemory), std::move(managedTensorHandles), - std::move(unmanagedTensorHandles)); + std::move(unmanagedTensorHandles), + executionDataVec, + &m_Backends); } void LoadedNetwork::RegisterDebugCallback(const DebugCallbackFunction& func) diff --git a/src/armnn/WorkingMemHandle.cpp b/src/armnn/WorkingMemHandle.cpp index 70bdd59373..e800c428e6 100644 --- a/src/armnn/WorkingMemHandle.cpp +++ b/src/armnn/WorkingMemHandle.cpp @@ -1,5 +1,5 @@ // -// Copyright © 2021 Arm Ltd and Contributors. All rights reserved. +// Copyright © 2022 Arm Ltd and Contributors. All rights reserved. // SPDX-License-Identifier: MIT // @@ -19,20 +19,22 @@ WorkingMemHandle::WorkingMemHandle(NetworkId networkId, std::vector<InputMemDescriptorCoords> inputLayerInfo, std::vector<OutputMemDescriptorCoords> outputLayerInfo, std::vector<WorkingMemDescriptor> workingMemDescriptors, - std::unordered_map<LayerGuid, WorkingMemDescriptor> workingMemDescriptorMap, std::unique_ptr<MemoryManager> memoryManager, std::vector<std::pair<std::shared_ptr<TensorMemory>, MemorySource>> tensorMemory, std::vector<std::unique_ptr<ITensorHandle>> managedTensorHandles, - std::vector<std::unique_ptr<ITensorHandle>> unmanagedTensorHandles) + std::vector<std::unique_ptr<ITensorHandle>> unmanagedTensorHandles, + std::vector<std::pair<BackendId, ExecutionData>> executionDataVec, + BackendPtrMap* backends) : m_NetworkId(networkId) , m_WorkingMemDescriptors(workingMemDescriptors) - , m_WorkingMemDescriptorMap(workingMemDescriptorMap) , m_MemoryManager(std::move(memoryManager)) , m_TensorMemory(std::move(tensorMemory)) , m_ManagedTensorHandles(std::move(managedTensorHandles)) , m_UnmanagedTensorHandles(std::move(unmanagedTensorHandles)) , m_InputSize(numeric_cast<DifferenceType>(inputLayerInfo.size())) , m_IsAllocated(false) + , m_ExecutionDataVec(executionDataVec) + , m_Backends(backends) { for (const auto& inputInfo : inputLayerInfo) { @@ -109,6 +111,15 @@ void WorkingMemHandle::Allocate() { m_ManagedTensorHandles[i]->Import(m_TensorMemory[i].first->m_Data, m_TensorMemory[i].second); } + + // Assign previously allocated ExecutionData. Needs to be assigned after allocation so the void* are allocated. + for (unsigned int i = 0; i < m_ExecutionDataVec.size(); ++i) + { + auto& backend = m_Backends->at(m_ExecutionDataVec[i].first); + + ExecutionData executionData = backend->CreateExecutionData(GetWorkingMemDescriptorAt(i)); + m_ExecutionDataVec[i].second = executionData; + } } void WorkingMemHandle::Free() diff --git a/src/armnn/WorkingMemHandle.hpp b/src/armnn/WorkingMemHandle.hpp index bca1d2d80c..dc62b9a2f5 100644 --- a/src/armnn/WorkingMemHandle.hpp +++ b/src/armnn/WorkingMemHandle.hpp @@ -1,16 +1,18 @@ // -// Copyright © 2021 Arm Ltd and Contributors. All rights reserved. +// Copyright © 2022 Arm Ltd and Contributors. All rights reserved. // SPDX-License-Identifier: MIT // #pragma once +#include "ExecutionData.hpp" #include "Layer.hpp" #include "Network.hpp" #include "WorkingMemDescriptor.hpp" #include <armnn/IWorkingMemHandle.hpp> #include <armnn/Tensor.hpp> +#include <armnn/backends/IBackendInternal.hpp> #include <unordered_map> #include <mutex> @@ -22,6 +24,7 @@ namespace armnn namespace experimental { +using BackendPtrMap = std::unordered_map<BackendId, IBackendInternalUniquePtr>; class WorkingMemHandle final : public IWorkingMemHandle { @@ -48,11 +51,12 @@ public: std::vector<InputMemDescriptorCoords> inputLayerInfo, std::vector<OutputMemDescriptorCoords> outputLayerInfo, std::vector<WorkingMemDescriptor> workingMemDescriptors, - std::unordered_map<LayerGuid, WorkingMemDescriptor> workingMemDescriptorMap, std::unique_ptr<MemoryManager> memoryManager, std::vector<std::pair<std::shared_ptr<TensorMemory>, MemorySource>> tensorMemory, std::vector<std::unique_ptr<ITensorHandle>> managedTensorHandles, - std::vector<std::unique_ptr<ITensorHandle>> unmanagedTensorHandles); + std::vector<std::unique_ptr<ITensorHandle>> unmanagedTensorHandles, + std::vector<std::pair<BackendId, ExecutionData>> executionDataVec, + BackendPtrMap* backends); ~WorkingMemHandle() { Free(); } @@ -75,14 +79,6 @@ public: return m_IsAllocated; } - /// Get the WorkingMemDescriptor for a Layer. - WorkingMemDescriptor& GetWorkingMemDescriptor(LayerGuid id) override - { - auto result = m_WorkingMemDescriptorMap.find(id); - ARMNN_ASSERT(result != m_WorkingMemDescriptorMap.end()); - return result->second; - } - /// Get the WorkingMemDescriptor at an index. The WorkingMemDescriptors are stored in the same order as /// the Workloads in a topologically sorted graph. WorkingMemDescriptor& GetWorkingMemDescriptorAt(unsigned int id) override @@ -90,6 +86,14 @@ public: return m_WorkingMemDescriptors[id]; } + /// Get the ExecutionData at an index. + /// The ExecutionData is paired with a BackendId to be able to call backend specific functions upon it. + /// The ExecutionData are stored in the same order as the Workloads in a topologically sorted graph. + std::pair<BackendId, ExecutionData>& GetExecutionDataAt(unsigned int id) override + { + return m_ExecutionDataVec[id]; + } + ITensorHandle* GetInputHandle(LayerBindingId layerBindingId) const { return m_InputHandleMap.at(layerBindingId); @@ -129,14 +133,12 @@ private: std::unordered_map<LayerBindingId, std::vector<std::vector<ITensorHandle*>::iterator>> m_OutputConnectionMap; std::vector<WorkingMemDescriptor> m_WorkingMemDescriptors; - std::unordered_map<LayerGuid, WorkingMemDescriptor> m_WorkingMemDescriptorMap; std::unique_ptr<MemoryManager> m_MemoryManager; // Memory to be imported into the tensorHandles after allocation std::vector<std::pair<std::shared_ptr<TensorMemory>, MemorySource>> m_TensorMemory; - // Tensors that will need to be allocated internally within armnn std::vector<std::unique_ptr<ITensorHandle>> m_ManagedTensorHandles; @@ -151,6 +153,10 @@ private: DifferenceType m_InputSize; bool m_IsAllocated; + + std::vector<std::pair<BackendId, ExecutionData>> m_ExecutionDataVec; + + BackendPtrMap* m_Backends; }; } // end experimental namespace |