23 #include <fmt/format.h> 34 template <
typename ExceptionType>
35 std::string ToErrorMessage(
const char * prefix,
const ExceptionType &
error)
38 ss << prefix <<
" " << error.what();
42 void AddLayerStructure(std::unique_ptr<TimelineUtilityMethods>& timelineUtils,
44 ProfilingGuid networkGuid)
47 std::string layerName = layer.GetNameStr().empty() ?
"<Unnamed>" : layer.GetNameStr();
48 timelineUtils->CreateNamedTypedChildEntity(layer.GetGuid(),
51 LabelsAndEventClasses::LAYER_GUID);
52 for (
auto&& input : layer.GetInputSlots())
54 const IOutputSlot* source = input.GetConnectedOutputSlot();
56 timelineUtils->CreateConnectionRelationship(ProfilingRelationshipType::RetentionLink,
57 source->GetOwningLayerGuid(),
62 void AddWorkloadStructure(std::unique_ptr<TimelineUtilityMethods>& timelineUtils,
63 std::unique_ptr<IWorkload>& workload,
67 timelineUtils->CreateTypedEntity(workload->GetGuid(), LabelsAndEventClasses::WORKLOAD_GUID);
68 timelineUtils->MarkEntityWithLabel(workload->GetGuid(),
69 layer.GetBackendId().Get(),
70 LabelsAndEventClasses::BACKENDID_GUID);
73 timelineUtils->CreateRelationship(ProfilingRelationshipType::RetentionLink,
76 LabelsAndEventClasses::CHILD_GUID);
82 std::string& errorMessage,
86 std::unique_ptr<LoadedNetwork> loadedNetwork;
88 auto Fail = [&](
const std::exception&
error) -> std::unique_ptr<LoadedNetwork>
90 errorMessage = ToErrorMessage(
"An error occurred when preparing the network workloads: ", error);
93 return std::unique_ptr<LoadedNetwork>();
98 loadedNetwork.reset(
new LoadedNetwork(std::move(net), networkProperties, profilingService));
108 catch (
const std::runtime_error& error)
113 return loadedNetwork;
116 LoadedNetwork::LoadedNetwork(std::unique_ptr<IOptimizedNetwork> net,
119 m_OptimizedNetwork(std::move(net)),
120 m_NetworkProperties(networkProperties),
121 m_TensorHandleFactoryRegistry(),
122 m_ProfilingService(profilingService)
125 m_Profiler = std::make_shared<IProfiler>();
137 for (
auto&& layer : order)
139 auto const& backendId = layer->GetBackendId();
140 if (m_Backends.count(backendId) == 0)
143 auto it = m_Backends.emplace(std::make_pair(backendId, createBackend()));
150 m_TensorHandleFactoryRegistry, m_OptimizedNetwork->pOptimizedNetworkImpl->GetModelOptions(),
152 static_cast<MemorySourceFlags>(m_NetworkProperties.m_OutputSource));
153 m_WorkloadFactories.emplace(
154 std::make_pair(backendId, std::make_pair(std::move(workloadFactory),
nullptr)));
160 memoryManager, m_OptimizedNetwork->pOptimizedNetworkImpl->GetModelOptions());
162 m_WorkloadFactories.emplace(
163 std::make_pair(backendId, std::make_pair(std::move(workloadFactory), memoryManager)));
170 for (
auto&& layer : order)
172 auto& workloadFactory = GetWorkloadFactory(*layer);
174 switch (layer->GetType())
181 layer->CreateTensorHandles(m_TensorHandleFactoryRegistry,
183 !m_NetworkProperties.m_ImportEnabled);
190 if ((layer->GetNumOutputSlots() == 1) &&
191 (layer->GetOutputSlots()[0].GetNumConnections() == 1) &&
192 (layer->GetOutputSlots()[0].GetConnection(0)->GetOwningLayer().GetType() ==
LayerType::Output))
194 layer->CreateTensorHandles(m_TensorHandleFactoryRegistry,
196 !m_NetworkProperties.m_ExportEnabled);
200 layer->CreateTensorHandles(m_TensorHandleFactoryRegistry, workloadFactory);
207 ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
208 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
212 timelineUtils->CreateTypedEntity(networkGuid, LabelsAndEventClasses::NETWORK_GUID);
214 timelineUtils->RecordEvent(networkGuid, LabelsAndEventClasses::ARMNN_PROFILING_SOL_EVENT_CLASS);
217 std::stringstream ss;
219 timelineUtils->MarkEntityWithLabel(networkGuid, ss.str(), LabelsAndEventClasses::PROCESS_ID_GUID);
223 for (
auto&& layer : order)
228 AddLayerStructure(timelineUtils, *layer, networkGuid);
233 switch (layer->GetType())
243 auto workload = layer->CreateWorkload(workloadFactory);
247 const char*
const layerName =
248 layer->GetNameStr().length() != 0 ? layer->GetName() :
"<Unnamed>";
250 fmt::format(
"No workload created for layer (name: '{0}' type: '{1}') (compute '{2}')",
251 layerName, static_cast<int>(layer->GetType()), layer->GetBackendId().Get()
258 AddWorkloadStructure(timelineUtils, workload, *layer);
265 m_ConstantWorkloads[layer->GetGuid()] = std::move(workload);
269 m_WorkloadQueue.push_back(move(workload));
273 layer->ReleaseConstantData();
279 for (
auto&& workloadFactory : m_WorkloadFactories)
281 workloadFactory.second.first->AfterWorkloadsCreated();
287 timelineUtils->Commit();
293 m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().AllocateDynamicBuffers();
297 for (
auto &workload : m_WorkloadQueue)
299 workload->PostAllocationConfigure();
304 AllocateAndExecuteConstantWorkloads();
308 void LoadedNetwork::AllocateAndExecuteConstantWorkloads()
310 Graph& order = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();
311 for (
auto&& layer : order)
315 const auto& outSlot = layer->GetOutputSlots()[0];
316 const auto factoryId = outSlot.GetTensorHandleFactoryId();
318 auto& workloadFactory = GetWorkloadFactory(*layer);
320 layer->CreateTensorHandles(m_TensorHandleFactoryRegistry, workloadFactory);
321 ITensorHandle* tensorHandle = outSlot.GetOutputHandler().GetData();
323 m_ConstantTensorHandles[layer->GetGuid()] = tensorHandle;
327 memDesc.
m_Outputs.push_back(tensorHandle);
328 m_ConstantWorkloads[layer->GetGuid()]->ExecuteAsync(memDesc);
337 ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
339 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
342 timelineUtils->CreateTypedEntity(networkGuid, LabelsAndEventClasses::NETWORK_GUID);
344 for (
auto&& layer : order)
347 AddLayerStructure(timelineUtils, *layer, networkGuid);
348 switch (layer->GetType())
358 for (
auto& workload : m_WorkloadQueue)
361 AddWorkloadStructure(timelineUtils, workload, *layer);
368 timelineUtils->Commit();
373 return m_OptimizedNetwork->GetGuid();
378 for (
auto&& inputLayer : m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetInputLayers())
380 ARMNN_ASSERT_MSG(inputLayer->GetNumOutputSlots() == 1,
"Input layer should have exactly 1 output slot");
381 if (inputLayer->GetBindingId() == layerId)
383 return inputLayer->GetOutputSlot(0).GetTensorInfo();
392 for (
auto&& outputLayer : m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetOutputLayers())
394 ARMNN_ASSERT_MSG(outputLayer->GetNumInputSlots() == 1,
"Output layer should have exactly 1 input slot");
395 ARMNN_ASSERT_MSG(outputLayer->GetInputSlot(0).GetConnection(),
"Input slot on Output layer must be connected");
396 if (outputLayer->GetBindingId() == layerId)
398 return outputLayer->GetInputSlot(0).GetConnection()->GetTensorInfo();
409 auto it = m_WorkloadFactories.find(layer.
GetBackendId());
410 if (it == m_WorkloadFactories.end())
412 throw RuntimeException(fmt::format(
"No workload factory for {0} to be used for layer: {1}",
418 workloadFactory = it->second.first.get();
422 std::string reasonIfUnsupported;
426 m_OptimizedNetwork->pOptimizedNetworkImpl->GetModelOptions()),
427 "Factory does not support layer");
429 return *workloadFactory;
439 : m_TensorHandle(std::move(handle))
445 ITensorHandle* GetTensorHandle()
const {
return m_TensorHandle.get(); }
450 std::unique_ptr<ITensorHandle> m_TensorHandle;
456 const std::vector<TensorPin>& pins,
457 char const* bindingPointDesc)
459 auto it = std::find_if(pins.begin(), pins.end(),
460 [id](
const TensorPin& pin)
462 return pin.GetBindingId() == id;
465 if (it != pins.end())
481 m_InputTensorPins.reserve(inputTensors.size());
482 m_OutputTensorPins.reserve(outputTensors.size());
484 for (
auto inputTensorPair : inputTensors)
486 auto inputTensor = inputTensorPair.second;
488 std::unique_ptr<ITensorHandle> tensorHandle =
489 std::make_unique<ConstPassthroughTensorHandle>(inputTensor.GetInfo(),inputTensor.GetMemoryArea());
492 m_InputTensorPins.emplace_back(std::move(tensorHandle), inputTensor.GetInfo(), layerId);
495 for (
auto outputTensorPair : outputTensors)
497 auto outputTensor = outputTensorPair.second;
499 std::unique_ptr<ITensorHandle> tensorHandle =
500 std::make_unique<PassthroughTensorHandle>(outputTensor.GetInfo(), outputTensor.GetMemoryArea());
503 m_OutputTensorPins.emplace_back(std::move(tensorHandle), outputTensor.GetInfo(), layerId);
509 return GetTensorPin(
id, m_InputTensorPins,
"input");
514 return GetTensorPin(
id, m_OutputTensorPins,
"output");
519 std::vector<TensorPin> m_InputTensorPins;
520 std::vector<TensorPin> m_OutputTensorPins;
528 const Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();
533 ARMNN_LOG(
warning) <<
"IRuntime::EnqueueWorkload()::Less than two nodes in graph";
538 WorkloadData workloadData(inputTensors, outputTensors);
548 m_InputQueue.clear();
552 const TensorPin& pin = workloadData.GetInputTensorPin(inputLayer->GetBindingId());
553 EnqueueInput(*inputLayer, pin.GetTensorHandle(), pin.GetTensorInfo());
560 m_OutputQueue.clear();
564 const TensorPin& pin = workloadData.GetOutputTensorPin(outputLayer->GetBindingId());
565 EnqueueOutput(*outputLayer, pin.GetTensorHandle(), pin.GetTensorInfo());
569 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
571 ProfilingGuid inferenceGuid = m_ProfilingService.GetNextGuid();
575 ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
576 timelineUtils->CreateTypedEntity(inferenceGuid, LabelsAndEventClasses::INFERENCE_GUID);
577 timelineUtils->CreateRelationship(ProfilingRelationshipType::RetentionLink,
580 LabelsAndEventClasses::EXECUTION_OF_GUID);
581 timelineUtils->RecordEvent(inferenceGuid, LabelsAndEventClasses::ARMNN_PROFILING_SOL_EVENT_CLASS);
584 bool executionSucceeded =
true;
587 if (m_ProfilingService.IsProfilingEnabled())
589 m_ProfilingService.IncrementCounterValue(armnn::profiling::INFERENCES_RUN);
593 executionSucceeded = Execute(timelineUtils, inferenceGuid);
599 timelineUtils->RecordEvent(inferenceGuid, LabelsAndEventClasses::ARMNN_PROFILING_EOL_EVENT_CLASS);
600 timelineUtils->Commit();
612 if (tensorHandle ==
nullptr)
620 inputQueueDescriptor.
m_Inputs.push_back(tensorHandle);
625 const TensorInfo& outputTensorInfo = handler.GetTensorInfo();
628 "Data should have been allocated.");
629 inputQueueDescriptor.
m_Outputs.push_back(outputTensorHandle);
633 bool needMemCopy =
true;
634 if (m_NetworkProperties.m_ImportEnabled)
636 if(
CheckFlag(importFlags, m_NetworkProperties.m_InputSource))
640 void* mem = tensorHandle->
Map(
false);
641 if (outputTensorHandle->
Import(mem, m_NetworkProperties.m_InputSource))
643 tensorHandle->
Unmap();
646 tensorHandle->
Unmap();
653 std::unique_ptr<IWorkload> inputWorkload = std::make_unique<CopyMemGenericWorkload>(inputQueueDescriptor,
info);
657 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
662 AddWorkloadStructure(timelineUtils, inputWorkload, layer);
663 timelineUtils->Commit();
666 m_InputQueue.push_back(move(inputWorkload));
677 if (tensorHandle ==
nullptr)
685 outputQueueDescriptor.
m_Outputs.push_back(tensorHandle);
693 const TensorInfo& inputTensorInfo = outputHandler.GetTensorInfo();
695 ARMNN_ASSERT_MSG(inputTensorHandle !=
nullptr,
"Data should have been allocated.");
704 bool needMemCopy =
true;
705 if (m_NetworkProperties.m_ExportEnabled &&
706 (layer.
GetInputSlots()[0].GetConnectedOutputSlot()->GetNumConnections() == 1))
711 if (
CheckFlag(importFlags, m_NetworkProperties.m_OutputSource))
714 void *mem = tensorHandle->
Map(
false);
715 bool importOk = inputTensorHandle->
Import(mem, m_NetworkProperties.m_OutputSource);
716 tensorHandle->
Unmap();
722 syncDesc.
m_Inputs.push_back(inputTensorHandle);
724 auto syncWorkload = std::make_unique<SyncMemGenericWorkload>(syncDesc,
info);
726 m_OutputQueue.push_back(move(syncWorkload));
738 outputQueueDescriptor.
m_Inputs.push_back(inputTensorHandle);
741 std::unique_ptr<IWorkload> outputWorkload =
742 std::make_unique<CopyMemGenericWorkload>(outputQueueDescriptor,
info);
745 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
750 AddWorkloadStructure(timelineUtils, outputWorkload, layer);
751 timelineUtils->Commit();
754 m_OutputQueue.push_back(move(outputWorkload));
758 void LoadedNetwork::AllocateWorkingMemory(std::lock_guard<std::mutex>& lock)
765 if (m_IsWorkingMemAllocated)
769 for (
auto&& workloadFactory : m_WorkloadFactories)
774 memoryManager->Acquire();
777 m_TensorHandleFactoryRegistry.AquireMemory();
778 m_IsWorkingMemAllocated =
true;
783 std::lock_guard<std::mutex> lockGuard(m_WorkingMemMutex);
784 if (!m_IsWorkingMemAllocated)
789 for (
auto&& workloadFactory : m_WorkloadFactories)
794 memoryManager->Release();
797 m_TensorHandleFactoryRegistry.ReleaseMemory();
798 m_IsWorkingMemAllocated =
false;
802 profiling::ProfilingGuid inferenceGuid)
806 auto Fail = [&](
const std::exception&
error)
808 ARMNN_LOG(error) <<
"An error occurred attempting to execute a workload: " << error.what();
814 std::lock_guard<std::mutex> lockGuard(m_WorkingMemMutex);
815 AllocateWorkingMemory(lockGuard);
817 ProfilingDynamicGuid workloadInferenceID(0);
818 auto ExecuteQueue = [&timelineUtils, &workloadInferenceID, &inferenceGuid](
WorkloadQueue& queue)
820 for (
auto& workload : queue)
824 workloadInferenceID = timelineUtils->RecordWorkloadInferenceAndStartOfLifeEvent(workload->GetGuid(),
830 timelineUtils->RecordEndOfLifeEvent(workloadInferenceID);
835 ExecuteQueue(m_InputQueue);
836 ExecuteQueue(m_WorkloadQueue);
837 ExecuteQueue(m_OutputQueue);
843 catch (
const std::runtime_error& error)
863 if (m_NetworkProperties.m_ImportEnabled)
865 if (
CheckFlag(importFlags, m_NetworkProperties.m_InputSource) )
868 std::unique_ptr<ITensorHandle> tensorHandle =
869 std::make_unique<ConstPassthroughTensorHandle>(inputTensor.
GetInfo(),
872 void* mem = tensorHandle->
Map(
false);
873 if (descriptor.
m_Outputs[0]->Import(mem, m_NetworkProperties.m_InputSource))
875 tensorHandle->Unmap();
878 tensorHandle->Unmap();
888 std::unique_ptr<ITensorHandle> tensorHandle =
889 std::make_unique<ConstPassthroughTensorHandle>(inputTensor.
GetInfo(), inputTensor.
GetMemoryArea());
891 auto copyFunc = [](
void* dst,
const void* src,
size_t size)
893 memcpy(dst, src, size);
896 for (
const auto& input : descriptor.
m_Outputs)
915 ARMNN_ASSERT_MSG(inputTensorHandle !=
nullptr,
"Data should have been allocated.");
924 if (m_NetworkProperties.m_ExportEnabled &&
925 (layer.
GetInputSlots()[0].GetConnectedOutputSlot()->GetNumConnections() == 1))
930 if (
CheckFlag(importFlags, m_NetworkProperties.m_OutputSource))
932 std::unique_ptr<ITensorHandle> tensorHandle =
933 std::make_unique<PassthroughTensorHandle>(outputTensor.
GetInfo(),
936 void* mem = tensorHandle->
Map(
false);
937 bool importOk = inputTensorHandle->
Import(mem, m_NetworkProperties.m_OutputSource);
938 tensorHandle->Unmap();
943 inputTensorHandle->
Map(
true);
944 inputTensorHandle->
Unmap();
958 throw MemoryExportException(
"EnqueueOutput: Memory Export failed, attempting to export Input Layer");
963 auto copyFunc = [](
void* dst,
const void* src,
size_t size)
965 memcpy(dst, src, size);
968 std::unique_ptr<ITensorHandle> tensorHandle =
969 std::make_unique<PassthroughTensorHandle>(outputTensor.
GetInfo(),
979 for (
auto inputTensorPair : inputTensors)
984 return inputTensorPair.second;
992 for (
auto outputTensorPair : outputTensors)
997 return outputTensorPair.second;
1007 const Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();
1012 ARMNN_LOG(
warning) <<
"IRuntime::EnqueueWorkload()::Less than two nodes in graph";
1021 std::unique_ptr<profiling::TimelineUtilityMethods> timelineUtils =
1023 profiling::ProfilingGuid inferenceGuid = m_ProfilingService.GetNextGuid();
1027 profiling::ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
1028 timelineUtils->CreateTypedEntity(inferenceGuid, profiling::LabelsAndEventClasses::INFERENCE_GUID);
1032 profiling::LabelsAndEventClasses::EXECUTION_OF_GUID);
1033 timelineUtils->RecordEvent(inferenceGuid, profiling::LabelsAndEventClasses::ARMNN_PROFILING_SOL_EVENT_CLASS);
1036 bool executionSucceeded =
true;
1041 timelineUtils->RecordEvent(inferenceGuid, profiling::LabelsAndEventClasses::ARMNN_PROFILING_EOL_EVENT_CLASS);
1042 timelineUtils->Commit();
1045 std::lock_guard<std::mutex> lockGuard(workingMemHandle.
GetMutex());
1056 EnqueueInput(*inputLayer,
GetInputTensor(inputLayer->GetBindingId(), inputTensors), workingMemHandle);
1060 auto Fail = [&](
const std::exception&
error)
1062 ARMNN_LOG(error) <<
"An error occurred attempting to execute a workload: " << error.what();
1063 executionSucceeded =
false;
1065 profiling::ProfilingDynamicGuid workloadInferenceID(0);
1069 for (
unsigned int i = 0; i < m_WorkloadQueue.size(); ++i)
1071 auto& workload = m_WorkloadQueue[i];
1074 workloadInferenceID = timelineUtils->RecordWorkloadInferenceAndStartOfLifeEvent(workload->GetGuid(),
1081 timelineUtils->RecordEndOfLifeEvent(workloadInferenceID);
1089 catch (
const std::runtime_error& error)
1098 EnqueueOutput(*outputLayer,
GetOutputTensor(outputLayer->GetBindingId(), outputTensors), workingMemHandle);
1109 Graph& order = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();
1110 std::unordered_map<LayerGuid, std::vector<std::unique_ptr<ITensorHandle> > > tensorHandleMap;
1111 std::vector<WorkingMemDescriptor> workingMemDescriptors;
1112 std::unordered_map<LayerGuid, WorkingMemDescriptor> workingMemDescriptorMap;
1114 WorkloadFactoryMap workloadFactoryMap;
1116 std::vector<std::shared_ptr<IMemoryManager>> memoryManagers;
1118 for (
auto const& backend : m_Backends)
1120 if (backend.second->SupportsTensorAllocatorAPI())
1122 backend.second->RegisterTensorHandleFactories(
1123 tensorHandleFactoryRegistry,
1124 static_cast<MemorySourceFlags>(m_NetworkProperties.m_InputSource),
1125 static_cast<MemorySourceFlags>(m_NetworkProperties.m_OutputSource));
1126 memoryManagers.emplace_back(tensorHandleFactoryRegistry.
GetMemoryManagers().back());
1130 std::shared_ptr<IMemoryManager> memoryManager = backend.second->CreateMemoryManager();
1131 auto workloadFactory = backend.second->CreateWorkloadFactory(
1132 memoryManager, m_OptimizedNetwork->pOptimizedNetworkImpl->GetModelOptions());
1134 workloadFactoryMap.emplace(
1135 std::make_pair(backend.first, std::make_pair(std::move(workloadFactory), memoryManager)));
1136 memoryManagers.emplace_back(memoryManager);
1140 auto GetTensorHandle = [&](
Layer* layer,
const OutputSlot& outputSlot,
bool isMemoryManaged)
1143 const TensorInfo& tensorInfo = outputSlot.GetTensorInfo();
1149 return workloadFactoryMap.at(
id).first->CreateTensorHandle(tensorInfo, isMemoryManaged);
1160 std::unordered_map<const ITensorHandle*, unsigned int> handleReferenceCounts;
1161 for (
auto&& layer : order)
1170 bool isMemoryManaged =
true;
1171 bool isInputLayer =
true;
1178 isMemoryManaged = !m_NetworkProperties.m_ExportEnabled;
1184 isInputLayer =
false;
1185 isMemoryManaged = !m_NetworkProperties.m_ExportEnabled;
1192 tensorHandleMap[layer->
GetGuid()].emplace_back(GetTensorHandle(layer, slot, isMemoryManaged));
1195 workingMemDescriptor.
m_Outputs.push_back(tensorHandle);
1197 unsigned int numConnections = slot.GetNumConnections();
1200 handleReferenceCounts[tensorHandle] = numConnections;
1209 auto outputSlot = slot.GetConnectedOutputSlot();
1210 auto key = outputSlot->GetOwningLayer().GetGuid();
1213 auto found = m_ConstantTensorHandles.find(key);
1214 if (found != m_ConstantTensorHandles.end())
1216 workingMemDescriptor.
m_Inputs.push_back(found->second);
1220 auto search = tensorHandleMap.find(key);
1221 unsigned int index = outputSlot->CalculateIndexOnOwner();
1222 ITensorHandle* inputTensorHandle = search->second[index].get();
1223 workingMemDescriptor.
m_Inputs.push_back(inputTensorHandle);
1224 --handleReferenceCounts.at(inputTensorHandle);
1225 if (handleReferenceCounts.at(inputTensorHandle) == 0u)
1229 handleReferenceCounts.erase(inputTensorHandle);
1232 workingMemDescriptorMap.insert({layer->
GetGuid(), workingMemDescriptor});
1238 workingMemDescriptors.push_back(workingMemDescriptor);
1242 return std::make_unique<WorkingMemHandle>(networkId,
1243 workingMemDescriptors,
1244 workingMemDescriptorMap,
1246 std::move(tensorHandleMap));
1251 for (
auto&& workloadPtr: m_WorkloadQueue)
1253 workloadPtr.get()->RegisterDebugCallback(func);
std::unique_ptr< IWorkingMemHandle > CreateWorkingMemHandle(NetworkId networkId)
Create a new unique WorkingMemHandle object.
virtual bool Import(void *memory, MemorySource source)
Import externally allocated memory.
FactoryFunction GetFactory(const BackendId &id) const
unsigned int GetNumInputSlots() const override
Returns the number of connectable input slots.
static std::unique_ptr< TimelineUtilityMethods > GetTimelineUtils(ProfilingService &profilingService)
static ProfilerManager & GetInstance()
#define ARMNN_NO_DEPRECATE_WARN_BEGIN
virtual unsigned int GetImportFlags() const
Get flags describing supported import sources.
const bool m_AsyncEnabled
const armnn::Tensor GetOutputTensor(const LayerBindingId layerId, const OutputTensors &outputTensors)
WorkingMemDescriptor & GetWorkingMemDescriptorAt(unsigned int id) override
Get the WorkingMemDescriptor at an index.
virtual void Allocate()=0
Indicate to the memory manager that this resource is no longer active.
TensorInfo GetInputTensorInfo(LayerBindingId layerId) const
#define ARMNN_LOG(severity)
virtual void Manage()=0
Indicate to the memory manager that this resource is active.
BackendRegistry & BackendRegistryInstance()
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
unsigned int MemorySourceFlags
MemoryType GetMemoryArea() const
size_t GetNumOutputs() const
TensorInfo GetOutputTensorInfo(LayerBindingId layerId) const
Copyright (c) 2021 ARM Limited and Contributors.
void IgnoreUnused(Ts &&...)
const std::vector< InputSlot > & GetInputSlots() const
std::function< void(LayerGuid guid, unsigned int slotIndex, ITensorHandle *tensorHandle)> DebugCallbackFunction
Define the type of callback for the Debug layer to call.
unsigned int GetNumOutputSlots() const override
Returns the number of connectable output slots.
#define ARMNN_SCOPED_PROFILING_EVENT(backendId, name)
int LayerBindingId
Type of identifiers for bindable layers (inputs, outputs).
A tensor defined by a TensorInfo (shape and data type) and a mutable backing store.
Status Execute(const InputTensors &inputTensors, const OutputTensors &outputTensors, IWorkingMemHandle &workingMemHandle)
Thread safe execution of the loaded network.
virtual IWorkloadFactoryPtr CreateWorkloadFactory(const IMemoryManagerSharedPtr &memoryManager=nullptr) const =0
std::vector< ITensorHandle * > m_Inputs
std::vector< TensorInfo > m_InputTensorInfos
WorkingMemDescriptor & GetWorkingMemDescriptor(LayerGuid id) override
Get the WorkingMemDescriptor for a Layer. The mutex must be locked.
#define ARMNN_NO_DEPRECATE_WARN_END
#define ARMNN_ASSERT_MSG(COND, MSG)
bool SupportsTensorAllocatorAPI() const
std::shared_ptr< IMemoryManager > IMemoryManagerSharedPtr
#define ARMNN_SCOPED_HEAP_PROFILING(TAG)
A tensor defined by a TensorInfo (shape and data type) and an immutable backing store.
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
const std::string & GetNameStr() const
LayerType GetType() const override
Returns the armnn::LayerType of this layer.
#define ARMNN_ASSERT(COND)
std::vector< TensorInfo > m_OutputTensorInfos
static bool IsLayerSupported(const BackendId &backendId, const IConnectableLayer &layer, Optional< DataType > dataType, std::string &outReasonIfUnsupported)
std::vector< std::unique_ptr< IWorkload > > WorkloadQueue
const TensorInfo & GetInfo() const
const BackendId & GetBackendId() const
void Allocate() override
Allocate the backing memory required for execution.
const std::vector< OutputSlot > & GetOutputSlots() const
const bool m_OutputNetworkDetails
std::mutex & GetMutex() override
Get a mutex which can be used for synchronizing access to the WorkingMemHandle object.
OutputLayersAccessor GetOutputLayers() const
Returns a wrapper object with begin(), end() methods to iterate over the output layers in a range-bas...
Status EnqueueWorkload(const InputTensors &inputTensors, const OutputTensors &outputTensors)
Single thread execution of the loaded network.
void RegisterProfiler(IProfiler *profiler)
virtual const void * Map(bool blocking=true) const =0
Map the tensor data for access.
profiling::ProfilingGuid GetNetworkGuid()
virtual void Unmap() const =0
Unmap the tensor data.
bool IsAllocated() override
IsAllocated returns true if the backing memory is currently allocated. The mutex must be locked...
std::vector< ITensorHandle * > m_Outputs
Base class for all ArmNN exceptions so that users can filter to just those.
const OutputHandler & GetOutputHandler(unsigned int i=0) const
const std::string & Get() const
void RegisterDebugCallback(const DebugCallbackFunction &func)
profiling::ProfilingGuid LayerGuid
Define LayerGuid type.
ITensorHandleFactory * GetFactory(ITensorHandleFactory::FactoryId id) const
Find a TensorHandleFactory by Id Returns nullptr if not found.
std::vector< ITensorHandle * > m_Outputs
Contains information about TensorInfos of a layer.
std::vector< std::shared_ptr< IMemoryManager > > & GetMemoryManagers()
bool CheckFlag(MemorySourceFlags flags, MemorySource source)
void CopyTensorContentsGeneric(const ITensorHandle *srcTensor, ITensorHandle *dstTensor, CopyFunc copy)
Graph & TopologicalSort()
Sorts layers in topological order and return this.
InputLayersAccessor GetInputLayers() const
Returns a wrapper object with begin(), end() methods to iterate over the input layers in a range-base...
std::vector< ITensorHandle * > m_Inputs
size_t GetNumLayers() const
virtual ARMNN_NO_DEPRECATE_WARN_END IMemoryManagerUniquePtr CreateMemoryManager() const
const armnn::ConstTensor GetInputTensor(const LayerBindingId layerId, const InputTensors &inputTensors)
const TensorInfo & GetTensorInfo(const ITensorHandle *tensorHandle)
float32 helpers
static std::unique_ptr< LoadedNetwork > MakeLoadedNetwork(std::unique_ptr< IOptimizedNetwork > net, std::string &errorMessage, const INetworkProperties &networkProperties, profiling::ProfilingService &profilingService)
size_t GetNumInputs() const
virtual std::unique_ptr< ITensorHandle > CreateTensorHandle(const TensorInfo &tensorInfo) const =0
static const FactoryId LegacyFactoryId
const bool m_ProfilingEnabled
LayerGuid GetGuid() const final
Returns the unique id of the layer.
void SendNetworkStructure()