24 #include <fmt/format.h> 35 template <
typename ExceptionType>
36 std::string ToErrorMessage(
const char * prefix,
const ExceptionType &
error)
39 ss << prefix <<
" " << error.what();
43 void AddLayerStructure(std::unique_ptr<TimelineUtilityMethods>& timelineUtils,
45 ProfilingGuid networkGuid)
48 std::string layerName = layer.GetNameStr().empty() ?
"<Unnamed>" : layer.GetNameStr();
49 timelineUtils->CreateNamedTypedChildEntity(layer.GetGuid(),
52 LabelsAndEventClasses::LAYER_GUID);
53 for (
auto&& input : layer.GetInputSlots())
55 const IOutputSlot* source = input.GetConnectedOutputSlot();
57 timelineUtils->CreateConnectionRelationship(ProfilingRelationshipType::RetentionLink,
58 source->GetOwningLayerGuid(),
63 void AddWorkloadStructure(std::unique_ptr<TimelineUtilityMethods>& timelineUtils,
64 std::unique_ptr<IWorkload>& workload,
68 timelineUtils->CreateTypedEntity(workload->GetGuid(), LabelsAndEventClasses::WORKLOAD_GUID);
69 timelineUtils->MarkEntityWithLabel(workload->GetGuid(),
70 layer.GetBackendId().Get(),
71 LabelsAndEventClasses::BACKENDID_GUID);
74 timelineUtils->CreateRelationship(ProfilingRelationshipType::RetentionLink,
77 LabelsAndEventClasses::CHILD_GUID);
83 std::string& errorMessage,
87 std::unique_ptr<LoadedNetwork> loadedNetwork;
89 auto Fail = [&](
const std::exception&
error) -> std::unique_ptr<LoadedNetwork>
91 errorMessage = ToErrorMessage(
"An error occurred when preparing the network workloads: ", error);
94 return std::unique_ptr<LoadedNetwork>();
99 loadedNetwork.reset(
new LoadedNetwork(std::move(net), networkProperties, profilingService));
109 catch (
const std::runtime_error& error)
114 return loadedNetwork;
117 LoadedNetwork::LoadedNetwork(std::unique_ptr<IOptimizedNetwork> net,
120 m_OptimizedNetwork(std::move(net)),
121 m_NetworkProperties(networkProperties),
122 m_TensorHandleFactoryRegistry(),
123 m_ProfilingService(profilingService)
127 const std::shared_ptr<IProfiler>& profiler = m_OptimizedNetwork->GetProfiler();
139 bool useExternalMemoryManager =
false;
140 bool useInternalMemoryManager =
false;
142 for (
auto&& layer : order)
144 auto const& backendId = layer->GetBackendId();
145 if (m_Backends.count(backendId) == 0)
148 auto it = m_Backends.emplace(std::make_pair(backendId, createBackend()));
155 std::string er = backend->
GetId();
156 er +=
" does not support AsyncExecution";
164 std::string er = backend->
GetId();
165 er +=
" does not support ExternallyManagedMemory\n";
166 er +=
"AsyncEnabled networks require all backends to support ExternallyManagedMemory";
171 && (m_NetworkProperties.m_ExternalMemoryManagementEnabled || m_NetworkProperties.m_AsyncEnabled))
173 m_SupportsExternallyManagedMemory[backend->
GetId()] =
true;
174 useExternalMemoryManager =
true;
178 m_SupportsExternallyManagedMemory[backend->
GetId()] =
false;
179 useInternalMemoryManager =
true;
186 m_TensorHandleFactoryRegistry,
187 m_OptimizedNetwork->pOptimizedNetworkImpl->GetModelOptions(),
189 static_cast<MemorySourceFlags>(m_NetworkProperties.m_OutputSource));
195 m_BackendMemoryMangers.back(), m_OptimizedNetwork->pOptimizedNetworkImpl->GetModelOptions());
197 m_WorkloadFactories[backendId ] = std::move(workloadFactory);
203 for (
auto&& layer : order)
205 auto& workloadFactory = GetWorkloadFactory(*layer);
206 bool supportsExternalManager = m_SupportsExternallyManagedMemory[layer->GetBackendId()];
208 switch (layer->GetType())
215 layer->CreateTensorHandles(m_TensorHandleFactoryRegistry,
217 !supportsExternalManager && !m_NetworkProperties.m_ImportEnabled);
222 layer->CreateTensorHandles(m_TensorHandleFactoryRegistry, workloadFactory,
true);
229 if ((layer->GetNumOutputSlots() == 1) &&
230 (layer->GetOutputSlots()[0].GetNumConnections() == 1) &&
231 (layer->GetOutputSlots()[0].GetConnection(0)->GetOwningLayer().GetType() ==
LayerType::Output))
233 layer->CreateTensorHandles(m_TensorHandleFactoryRegistry,
235 !supportsExternalManager && !m_NetworkProperties.m_ExportEnabled);
239 layer->CreateTensorHandles(m_TensorHandleFactoryRegistry,
241 !supportsExternalManager);
248 ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
249 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
253 timelineUtils->CreateTypedEntity(networkGuid, LabelsAndEventClasses::NETWORK_GUID);
255 timelineUtils->RecordEvent(networkGuid, LabelsAndEventClasses::ARMNN_PROFILING_SOL_EVENT_CLASS);
258 std::stringstream ss;
260 timelineUtils->MarkEntityWithLabel(networkGuid, ss.str(), LabelsAndEventClasses::PROCESS_ID_GUID);
266 for (
auto&& layer: order)
271 AddLayerStructure(timelineUtils, *layer, networkGuid);
276 switch (layer->GetType())
286 auto workload = layer->CreateWorkload(workloadFactory);
290 const char*
const layerName =
291 layer->GetNameStr().length() != 0 ? layer->GetName() :
"<Unnamed>";
293 fmt::format(
"No workload created for layer (name: '{0}' type: '{1}') (compute '{2}')",
294 layerName, static_cast<int>(layer->GetType()), layer->GetBackendId().Get()
301 AddWorkloadStructure(timelineUtils, workload, *layer);
306 if((networkProperties.
m_AsyncEnabled || useExternalMemoryManager) &&
309 m_ConstantTensorHandles[layer->GetGuid()] =
310 layer->GetOutputSlot(0).GetOutputHandler().GetData();
311 m_ConstantWorkloads[layer->GetGuid()] = std::move(workload);
315 m_WorkloadQueue.push_back(std::move(workload));
319 layer->ReleaseConstantData();
326 for (
auto&& workloadFactory : m_WorkloadFactories)
328 workloadFactory.second->AfterWorkloadsCreated();
334 timelineUtils->Commit();
337 if (useExternalMemoryManager)
341 CreateMemoryProfileAsync();
345 CreateMemoryProfile();
349 for (
auto& backendMemoryProfile : m_MemBlockMap)
351 const BackendId& backendId = backendMemoryProfile.first;
352 if (backendStrategyMap.find(backendId) != backendStrategyMap.end())
354 m_MemBinMap[backendId] = backendStrategyMap[backendId]->Optimize(backendMemoryProfile.second);
358 m_MemBinMap[backendId] = m_ConstantStrategy->Optimize(backendMemoryProfile.second);
364 m_ExternalMemoryManager = CreateExternalMemoryManger(m_TensorMemory);
367 std::sort(m_TensorMemory.begin(), m_TensorMemory.end(),
368 [](
const std::pair<std::shared_ptr<TensorMemory>,
MemorySource>& lhs,
369 const std::pair<std::shared_ptr<TensorMemory>,
MemorySource>& rhs)
371 return lhs.first->m_OutputSlotId < rhs.first->m_OutputSlotId;
380 if (useInternalMemoryManager)
383 m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().AllocateDynamicBuffers();
386 for (
auto &workload : m_WorkloadQueue)
388 workload->PostAllocationConfigure();
392 if (useExternalMemoryManager)
396 AllocateAndExecuteConstantWorkloads();
400 AllocateAndExecuteConstantWorkloadsAsync();
405 void LoadedNetwork::AllocateAndExecuteConstantWorkloads()
408 for (
auto& pair : m_ConstantWorkloads)
410 auto tensorHandle = m_ConstantTensorHandles[pair.first];
411 tensorHandle->Allocate();
412 pair.second->Execute();
418 void LoadedNetwork::AllocateAndExecuteConstantWorkloadsAsync()
421 Graph& order = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();
422 for (
auto&& layer : order)
426 const auto& outSlot = layer->GetOutputSlots()[0];
427 const auto factoryId = outSlot.GetTensorHandleFactoryId();
429 auto& workloadFactory = GetWorkloadFactory(*layer);
431 layer->CreateTensorHandles(m_TensorHandleFactoryRegistry, workloadFactory);
432 ITensorHandle* tensorHandle = outSlot.GetOutputHandler().GetData();
434 m_ConstantTensorHandles[layer->GetGuid()] = tensorHandle;
438 memDesc.
m_Outputs.push_back(tensorHandle);
439 m_ConstantWorkloads[layer->GetGuid()]->ExecuteAsync(memDesc);
448 ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
450 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
453 timelineUtils->CreateTypedEntity(networkGuid, LabelsAndEventClasses::NETWORK_GUID);
455 for (
auto&& layer : order)
458 AddLayerStructure(timelineUtils, *layer, networkGuid);
459 switch (layer->GetType())
469 for (
auto& workload : m_WorkloadQueue)
472 AddWorkloadStructure(timelineUtils, workload, *layer);
479 timelineUtils->Commit();
484 return m_OptimizedNetwork->GetGuid();
489 for (
auto&& inputLayer : m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetInputLayers())
491 ARMNN_ASSERT_MSG(inputLayer->GetNumOutputSlots() == 1,
"Input layer should have exactly 1 output slot");
492 if (inputLayer->GetBindingId() == layerId)
494 return inputLayer->GetOutputSlot(0).GetTensorInfo();
503 for (
auto&& outputLayer : m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph().GetOutputLayers())
505 ARMNN_ASSERT_MSG(outputLayer->GetNumInputSlots() == 1,
"Output layer should have exactly 1 input slot");
506 ARMNN_ASSERT_MSG(outputLayer->GetInputSlot(0).GetConnection(),
"Input slot on Output layer must be connected");
507 if (outputLayer->GetBindingId() == layerId)
509 return outputLayer->GetInputSlot(0).GetConnection()->GetTensorInfo();
520 auto it = m_WorkloadFactories.find(layer.
GetBackendId());
521 if (it == m_WorkloadFactories.end())
523 throw RuntimeException(fmt::format(
"No workload factory for {0} to be used for layer: {1}",
529 workloadFactory = it->second.get();
533 std::string reasonIfUnsupported;
537 m_OptimizedNetwork->pOptimizedNetworkImpl->GetModelOptions()),
538 "Factory does not support layer");
540 return *workloadFactory;
550 : m_TensorHandle(std::move(handle))
556 ITensorHandle* GetTensorHandle()
const {
return m_TensorHandle.get(); }
561 std::unique_ptr<ITensorHandle> m_TensorHandle;
567 const std::vector<TensorPin>& pins,
568 char const* bindingPointDesc)
570 auto it = std::find_if(pins.begin(), pins.end(),
571 [id](
const TensorPin& pin)
573 return pin.GetBindingId() == id;
576 if (it != pins.end())
592 m_InputTensorPins.reserve(inputTensors.size());
593 m_OutputTensorPins.reserve(outputTensors.size());
595 for (
auto inputTensorPair : inputTensors)
597 auto inputTensor = inputTensorPair.second;
599 std::unique_ptr<ITensorHandle> tensorHandle =
600 std::make_unique<ConstPassthroughTensorHandle>(inputTensor.GetInfo(),inputTensor.GetMemoryArea());
603 m_InputTensorPins.emplace_back(std::move(tensorHandle), inputTensor.GetInfo(), layerId);
606 for (
auto outputTensorPair : outputTensors)
608 auto outputTensor = outputTensorPair.second;
610 std::unique_ptr<ITensorHandle> tensorHandle =
611 std::make_unique<PassthroughTensorHandle>(outputTensor.GetInfo(), outputTensor.GetMemoryArea());
614 m_OutputTensorPins.emplace_back(std::move(tensorHandle), outputTensor.GetInfo(), layerId);
620 return GetTensorPin(
id, m_InputTensorPins,
"input");
625 return GetTensorPin(
id, m_OutputTensorPins,
"output");
630 std::vector<TensorPin> m_InputTensorPins;
631 std::vector<TensorPin> m_OutputTensorPins;
639 const Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();
644 ARMNN_LOG(
warning) <<
"IRuntime::EnqueueWorkload()::Less than two nodes in graph";
649 WorkloadData workloadData(inputTensors, outputTensors);
659 m_InputQueue.clear();
663 const TensorPin& pin = workloadData.GetInputTensorPin(inputLayer->GetBindingId());
664 EnqueueInput(*inputLayer, pin.GetTensorHandle(), pin.GetTensorInfo());
671 m_OutputQueue.clear();
675 const TensorPin& pin = workloadData.GetOutputTensorPin(outputLayer->GetBindingId());
676 EnqueueOutput(*outputLayer, pin.GetTensorHandle(), pin.GetTensorInfo());
680 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
682 ProfilingGuid inferenceGuid = m_ProfilingService.GetNextGuid();
686 ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
687 timelineUtils->CreateTypedEntity(inferenceGuid, LabelsAndEventClasses::INFERENCE_GUID);
688 timelineUtils->CreateRelationship(ProfilingRelationshipType::RetentionLink,
691 LabelsAndEventClasses::EXECUTION_OF_GUID);
692 timelineUtils->RecordEvent(inferenceGuid, LabelsAndEventClasses::ARMNN_PROFILING_SOL_EVENT_CLASS);
695 bool executionSucceeded =
true;
698 if (m_ProfilingService.IsProfilingEnabled())
700 m_ProfilingService.IncrementCounterValue(armnn::profiling::INFERENCES_RUN);
704 executionSucceeded = Execute(timelineUtils, inferenceGuid);
710 timelineUtils->RecordEvent(inferenceGuid, LabelsAndEventClasses::ARMNN_PROFILING_EOL_EVENT_CLASS);
711 timelineUtils->Commit();
723 if (tensorHandle ==
nullptr)
731 inputQueueDescriptor.
m_Inputs.push_back(tensorHandle);
736 const TensorInfo& outputTensorInfo = handler.GetTensorInfo();
739 "Data should have been allocated.");
740 inputQueueDescriptor.
m_Outputs.push_back(outputTensorHandle);
744 bool needMemCopy =
true;
745 if (m_NetworkProperties.m_ImportEnabled)
747 if(
CheckFlag(importFlags, m_NetworkProperties.m_InputSource))
751 void* mem = tensorHandle->
Map(
false);
752 if (outputTensorHandle->
Import(mem, m_NetworkProperties.m_InputSource))
754 tensorHandle->
Unmap();
757 tensorHandle->
Unmap();
764 std::unique_ptr<IWorkload> inputWorkload = std::make_unique<CopyMemGenericWorkload>(inputQueueDescriptor,
info);
768 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
773 AddWorkloadStructure(timelineUtils, inputWorkload, layer);
774 timelineUtils->Commit();
777 m_InputQueue.push_back(move(inputWorkload));
788 if (tensorHandle ==
nullptr)
796 outputQueueDescriptor.
m_Outputs.push_back(tensorHandle);
804 const TensorInfo& inputTensorInfo = outputHandler.GetTensorInfo();
806 ARMNN_ASSERT_MSG(inputTensorHandle !=
nullptr,
"Data should have been allocated.");
815 bool needMemCopy =
true;
816 if (m_NetworkProperties.m_ExportEnabled &&
817 (layer.
GetInputSlots()[0].GetConnectedOutputSlot()->GetNumConnections() == 1))
822 if (
CheckFlag(importFlags, m_NetworkProperties.m_OutputSource))
825 void *mem = tensorHandle->
Map(
false);
826 bool importOk = inputTensorHandle->
Import(mem, m_NetworkProperties.m_OutputSource);
827 tensorHandle->
Unmap();
833 syncDesc.
m_Inputs.push_back(inputTensorHandle);
835 auto syncWorkload = std::make_unique<SyncMemGenericWorkload>(syncDesc,
info);
837 m_OutputQueue.push_back(move(syncWorkload));
849 outputQueueDescriptor.
m_Inputs.push_back(inputTensorHandle);
852 std::unique_ptr<IWorkload> outputWorkload =
853 std::make_unique<CopyMemGenericWorkload>(outputQueueDescriptor,
info);
856 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
861 AddWorkloadStructure(timelineUtils, outputWorkload, layer);
862 timelineUtils->Commit();
865 m_OutputQueue.push_back(move(outputWorkload));
869 void LoadedNetwork::AllocateWorkingMemory(std::lock_guard<std::mutex>& lock)
876 if (m_IsWorkingMemAllocated)
881 if (m_ExternalMemoryManager)
883 m_ExternalMemoryManager->Allocate();
885 for (
unsigned int i = 0; i < m_TensorMemory.size(); ++i)
887 m_Tensorhandles[i]->Import(m_TensorMemory[i].first->m_Data, m_TensorMemory[i].second);
891 for (
auto&& memoryManager : m_BackendMemoryMangers)
895 memoryManager->Acquire();
898 m_TensorHandleFactoryRegistry.AquireMemory();
899 m_IsWorkingMemAllocated =
true;
904 std::lock_guard<std::mutex> lockGuard(m_WorkingMemMutex);
906 if (!m_IsWorkingMemAllocated)
911 if (m_ExternalMemoryManager)
913 m_ExternalMemoryManager->Deallocate();
917 for (
auto&& memoryManager : m_BackendMemoryMangers)
921 memoryManager->Release();
924 m_TensorHandleFactoryRegistry.ReleaseMemory();
925 m_IsWorkingMemAllocated =
false;
929 profiling::ProfilingGuid inferenceGuid)
933 auto Fail = [&](
const std::exception&
error)
935 ARMNN_LOG(error) <<
"An error occurred attempting to execute a workload: " << error.what();
941 std::lock_guard<std::mutex> lockGuard(m_WorkingMemMutex);
942 AllocateWorkingMemory(lockGuard);
944 ProfilingDynamicGuid workloadInferenceID(0);
945 auto ExecuteQueue = [&timelineUtils, &workloadInferenceID, &inferenceGuid](
WorkloadQueue& queue)
947 for (
auto& workload : queue)
951 workloadInferenceID = timelineUtils->RecordWorkloadInferenceAndStartOfLifeEvent(workload->GetGuid(),
957 timelineUtils->RecordEndOfLifeEvent(workloadInferenceID);
962 ExecuteQueue(m_InputQueue);
963 ExecuteQueue(m_WorkloadQueue);
964 ExecuteQueue(m_OutputQueue);
970 catch (
const std::runtime_error& error)
980 if (m_NetworkProperties.m_ImportEnabled)
983 if (
CheckFlag(importFlags, m_NetworkProperties.m_InputSource) )
985 std::unique_ptr<ITensorHandle> tensorHandle =
986 std::make_unique<ConstPassthroughTensorHandle>(inputTensor.
GetInfo(),
988 void* mem = tensorHandle->
Map(
false);
990 if (inputTensorHandle->
Import(mem, m_NetworkProperties.m_InputSource))
992 tensorHandle->Unmap();
995 tensorHandle->Unmap();
1005 std::unique_ptr<ITensorHandle> tensorHandle =
1006 std::make_unique<ConstPassthroughTensorHandle>(inputTensor.
GetInfo(), inputTensor.
GetMemoryArea());
1008 auto copyFunc = [](
void* dst,
const void* src,
size_t size)
1010 memcpy(dst, src, size);
1023 void LoadedNetwork::ImportOutputTensor(
const Tensor& outputTensor,
ITensorHandle* outputTensorHandle)
1025 ARMNN_ASSERT_MSG(outputTensorHandle !=
nullptr,
"Data should have been allocated.");
1027 if (
CheckFlag(importFlags, m_NetworkProperties.m_OutputSource))
1029 std::unique_ptr<ITensorHandle> tensorHandle =
1030 std::make_unique<PassthroughTensorHandle>(outputTensor.
GetInfo(),
1033 void* mem = tensorHandle->
Map(
false);
1034 bool importOk = outputTensorHandle->
Import(mem, m_NetworkProperties.m_OutputSource);
1035 tensorHandle->Unmap();
1044 throw MemoryExportException(
"ImportOutputTensor: Memory Export failed, attempting to export Input Layer");
1051 auto copyFunc = [](
void* dst,
const void* src,
size_t size)
1053 memcpy(dst, src, size);
1056 std::unique_ptr<ITensorHandle> tensorHandle =
1057 std::make_unique<PassthroughTensorHandle>(outputTensor.
GetInfo(),
1066 for (
auto inputTensorPair : inputTensors)
1071 return inputTensorPair.second;
1079 for (
auto outputTensorPair : outputTensors)
1084 return outputTensorPair.second;
1092 if (!m_NetworkProperties.m_ImportEnabled)
1094 throw MemoryImportException(
"ImportInputs: Memory Import failed, NetworkProperties.m_ImportEnabled");
1097 std::vector<ImportedInputId> importedInputs;
1100 for (
auto inputTensor : inputTensors)
1102 auto layerBindingId = inputTensor.first;
1110 throw MemoryImportException(fmt::format(
"ImportInputs: Memory Import failed, unknown LayerBindingId: {}",
1114 const Layer* layer = *it;
1123 std::string er = backend->GetId();
1124 er +=
" does not have PreImportIOTensors capability";
1136 ImportedTensorHandlePin importedTensorHandlePin{layerBindingId,
1139 ITensorHandle* tensorHandle = importedTensorHandlePin.m_TensorHandle.get();
1144 fmt::format(
"ImportInputs: Memory Import failed, backend: {} does not support importing from source {}" 1145 , factoryId, m_NetworkProperties.m_InputSource));
1148 std::unique_ptr<ITensorHandle> passThroughTensorHandle =
1149 std::make_unique<ConstPassthroughTensorHandle>(inputTensor.second.
GetInfo(),
1152 if (tensorHandle->
Import(passThroughTensorHandle->Map(), m_NetworkProperties.m_InputSource))
1154 importedInputs.push_back(m_CurImportedInputId++);
1155 passThroughTensorHandle->Unmap();
1159 passThroughTensorHandle->Unmap();
1163 m_PreImportedInputHandles.push_back(std::move(importedTensorHandlePin));
1166 return importedInputs;
1171 if (!m_NetworkProperties.m_ExportEnabled)
1173 throw MemoryImportException(
"ImportOutputs: Memory Import failed, NetworkProperties.m_ImportEnabled");
1176 std::vector<ImportedOutputId> importedOutputs;
1179 for (
const auto& outputTensor : outputTensors)
1181 auto layerBindingId = outputTensor.first;
1189 throw MemoryImportException(fmt::format(
"ImportOutputs: Memory Import failed, unknown LayerBindingId: {}",
1193 const Layer* layer = *it;
1202 std::string er = backend->GetId();
1203 er +=
" does not have PreImportIOTensors capability";
1214 ImportedTensorHandlePin importedTensorHandlePin{layerBindingId,
1217 ITensorHandle* tensorHandle = importedTensorHandlePin.m_TensorHandle.get();
1222 "{} does not support importing from source {}" 1223 , factoryId, m_NetworkProperties.m_OutputSource));
1226 if (tensorHandle->
Import(outputTensor.second.
GetMemoryArea(), m_NetworkProperties.m_OutputSource))
1228 importedOutputs.push_back(m_CurImportedOutputId++);
1235 m_PreImportedOutputHandles.push_back(std::move(importedTensorHandlePin));
1238 return importedOutputs;
1243 for (
auto id : inputIds)
1245 if (
id > m_PreImportedInputHandles.size())
1250 auto& importedTensorHandle = m_PreImportedInputHandles[id].m_TensorHandle;
1251 if (!importedTensorHandle)
1254 fmt::format(
"ClearImportedInputs::ImportedInput with id: {} has already been deleted",
id));
1257 importedTensorHandle->Unimport();
1258 importedTensorHandle = {};
1264 for (
auto id : outputIds)
1266 if (
id > m_PreImportedOutputHandles.size())
1271 auto& importedTensorHandle = m_PreImportedOutputHandles[id].m_TensorHandle;
1272 if (!importedTensorHandle)
1275 fmt::format(
"ClearImportedOutputs::ImportedOutput with id: {} has already been deleted",
id));
1278 importedTensorHandle->Unimport();
1279 importedTensorHandle = {};
1286 std::vector<ImportedInputId> preImportedInputs,
1287 std::vector<ImportedOutputId> preImportedOutputs)
1289 const Graph& graph = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();
1291 if (inputTensors.size() + preImportedInputs.size() != graph.
GetNumInputs())
1293 if (preImportedInputs.empty())
1300 "Number of inputs + preImportedInputs provided does not match network.");
1304 if (outputTensors.size() + preImportedOutputs.size() != graph.
GetNumOutputs())
1306 if (preImportedOutputs.empty())
1309 "Number of outputs provided does not match network.");
1314 "Number of outputs + preImportedOutputs provided does not match network.");
1321 unsigned int index = 0;
1322 for (
auto pair : inputTensors)
1324 bindingIds[index++] = pair.first;
1328 bindingIds[index++] = ValidateImportedInputID(
id);
1330 for (
auto pair : outputTensors)
1332 bindingIds[index++] = pair.first;
1336 bindingIds[index++] = ValidateImportedOutputID(
id);
1341 auto resetMemHandle = [&]()
1345 const LayerBindingId layerBindingId = m_PreImportedInputHandles[id].m_LayerBindingId;
1347 auto inputHandle = workingMemHandle.
GetInputHandle(layerBindingId);
1349 for (
auto it : inputConnections)
1357 const LayerBindingId layerBindingId = m_PreImportedOutputHandles[id].m_LayerBindingId;
1362 for (
auto it : outputConnections)
1369 std::unique_ptr<profiling::TimelineUtilityMethods> timelineUtils =
1371 profiling::ProfilingGuid inferenceGuid = m_ProfilingService.GetNextGuid();
1375 profiling::ProfilingGuid networkGuid = m_OptimizedNetwork->GetGuid();
1376 timelineUtils->CreateTypedEntity(inferenceGuid, profiling::LabelsAndEventClasses::INFERENCE_GUID);
1380 profiling::LabelsAndEventClasses::EXECUTION_OF_GUID);
1381 timelineUtils->RecordEvent(inferenceGuid, profiling::LabelsAndEventClasses::ARMNN_PROFILING_SOL_EVENT_CLASS);
1384 bool executionSucceeded =
true;
1389 timelineUtils->RecordEvent(inferenceGuid, profiling::LabelsAndEventClasses::ARMNN_PROFILING_EOL_EVENT_CLASS);
1390 timelineUtils->Commit();
1400 for (
auto pair : inputTensors)
1402 EnqueueInput(pair.second, workingMemHandle.
GetInputHandle(pair.first));
1408 const ImportedTensorHandlePin& importedInputPin = m_PreImportedInputHandles[id];
1409 const LayerBindingId layerBindingId = m_PreImportedInputHandles[id].m_LayerBindingId;
1410 const auto& preimportedHandle = importedInputPin.m_TensorHandle;
1413 for (
auto it : inputConnections)
1415 *it = preimportedHandle.get();
1421 if (m_NetworkProperties.m_ExportEnabled)
1423 for (
auto pair: outputTensors)
1425 ImportOutputTensor(pair.second, workingMemHandle.
GetOutputHandle(pair.first));
1431 const ImportedTensorHandlePin& importedOutputPin = m_PreImportedOutputHandles[id];
1432 const LayerBindingId layerBindingId = m_PreImportedOutputHandles[id].m_LayerBindingId;
1433 const auto& preimportedHandle = importedOutputPin.m_TensorHandle;
1437 for (
auto it : outputConnections)
1439 *it = preimportedHandle.get();
1444 auto Fail = [&](
const std::exception&
error)
1446 ARMNN_LOG(error) <<
"An error occurred attempting to execute a workload: " << error.what();
1447 executionSucceeded =
false;
1449 profiling::ProfilingDynamicGuid workloadInferenceID(0);
1453 for (
unsigned int i = 0; i < m_WorkloadQueue.size(); ++i)
1455 auto& workload = m_WorkloadQueue[i];
1458 workloadInferenceID = timelineUtils->RecordWorkloadInferenceAndStartOfLifeEvent(workload->GetGuid(),
1465 timelineUtils->RecordEndOfLifeEvent(workloadInferenceID);
1474 catch (
const std::runtime_error& error)
1485 if (!m_NetworkProperties.m_ExportEnabled)
1487 for (
auto pair: outputTensors)
1507 Graph& order = m_OptimizedNetwork->pOptimizedNetworkImpl->GetGraph();
1510 std::vector<std::unique_ptr<ITensorHandle>> managedTensorHandles;
1512 std::vector<std::unique_ptr<ITensorHandle>> unmanagedTensorHandles;
1514 std::vector<WorkingMemDescriptor> workingMemDescriptors;
1515 std::unordered_map<LayerGuid, WorkingMemDescriptor> workingMemDescriptorMap;
1517 auto GetTensorHandle = [&](
Layer* layer,
const OutputSlot& outputSlot)
1520 const TensorInfo& tensorInfo = outputSlot.GetTensorInfo();
1526 return m_WorkloadFactories.at(
id)->CreateTensorHandle(tensorInfo,
false);
1541 bool m_IsInputLayerHandle =
false;
1542 bool m_IsOutputLayerHandle =
false;
1548 std::unordered_map<const OutputSlot*, HandleInfo> outputToHandleInfoMap;
1550 unsigned int layerIndex = 0;
1551 for (
auto&& layer : order)
1561 bool isMemoryManaged =
true;
1562 bool isInputLayer =
false;
1563 bool isOutputLayer =
false;
1564 bool isConnectedToOutputLayer =
false;
1570 isInputLayer =
true;
1571 isMemoryManaged = !m_NetworkProperties.m_ImportEnabled;
1575 isOutputLayer =
true;
1578 unsigned int slotIndex = 0;
1583 for (
unsigned int i = 0; i < slot.GetNumConnections(); ++i)
1587 if (!isConnectedToOutputLayer)
1589 isConnectedToOutputLayer =
true;
1591 isMemoryManaged = !m_NetworkProperties.m_ExportEnabled;
1597 fmt::format(
"Layer name: '{0}' guid: '{1}' has two or more OutputLayers connected to it. " 1598 "This will prevent importing on the connected OutputLayers.",
1600 isMemoryManaged =
true;
1606 if (isMemoryManaged)
1608 managedTensorHandles.emplace_back(GetTensorHandle(layer, slot));
1609 tensorHandle = managedTensorHandles.back().get();
1613 unmanagedTensorHandles.emplace_back(GetTensorHandle(layer, slot));
1614 tensorHandle = unmanagedTensorHandles.back().get();
1617 workingMemDescriptor.
m_Outputs.push_back(tensorHandle);
1619 HandleInfo& handleInfo = outputToHandleInfoMap[&slot];
1620 handleInfo.m_TensorHandle = tensorHandle;
1623 if (isConnectedToOutputLayer)
1625 handleInfo.m_IsOutputLayerHandle =
true;
1626 handleInfo.m_OutputMemDescriptorCoords.m_OutputSlotCoords = {layerIndex, slotIndex};
1631 handleInfo.m_IsInputLayerHandle =
true;
1633 handleInfo.m_InputMemDescriptorCoords.m_LayerBindingId = bindingId;
1644 auto outputSlot = slot.GetConnectedOutputSlot();
1645 auto key = outputSlot->GetOwningLayer().GetGuid();
1648 auto found = m_ConstantTensorHandles.find(key);
1649 if (found != m_ConstantTensorHandles.end())
1652 workingMemDescriptor.
m_Inputs.push_back(tensorHandle);
1660 HandleInfo& handleInfo = outputToHandleInfoMap[outputSlot];
1661 handleInfo.m_TensorHandle = tensorHandle;
1662 handleInfo.m_IsOutputLayerHandle =
true;
1663 handleInfo.m_OutputMemDescriptorCoords.m_LayerBindingIds.push_back(bindingId);
1664 handleInfo.m_OutputMemDescriptorCoords.m_InputSlotCoords.push_back({layerIndex, 0});
1669 HandleInfo& handleInfo = outputToHandleInfoMap.at(outputSlot);
1671 ITensorHandle* inputTensorHandle = handleInfo.m_TensorHandle;
1672 workingMemDescriptor.
m_Inputs.push_back(inputTensorHandle);
1678 handleInfo.m_OutputMemDescriptorCoords.m_LayerBindingIds.push_back(bindingId);
1679 handleInfo.m_OutputMemDescriptorCoords.m_InputSlotCoords.push_back({layerIndex, 0});
1683 else if (handleInfo.m_IsOutputLayerHandle)
1685 handleInfo.m_OutputMemDescriptorCoords.m_InputSlotCoords.push_back({layerIndex, slot.GetSlotIndex()});
1690 if (handleInfo.m_IsInputLayerHandle)
1692 std::pair<LayerGuid, unsigned int> connectionLocation{layerIndex, slot.GetSlotIndex()};
1693 handleInfo.m_InputMemDescriptorCoords.m_InputSlotCoords.emplace_back(connectionLocation);
1696 workingMemDescriptorMap.insert({layer->
GetGuid(), workingMemDescriptor});
1702 workingMemDescriptors.push_back(workingMemDescriptor);
1707 std::vector<std::pair<std::shared_ptr<TensorMemory>,
MemorySource>> tensorMemory;
1709 auto externalMemoryManager = CreateExternalMemoryManger(tensorMemory);
1712 std::sort(tensorMemory.begin(), tensorMemory.end(),
1713 [](
const std::pair<std::shared_ptr<TensorMemory>,
MemorySource>& lhs,
1714 const std::pair<std::shared_ptr<TensorMemory>,
MemorySource>& rhs)
1716 return lhs.first->m_OutputSlotId < rhs.first->m_OutputSlotId;
1719 std::vector<WorkingMemHandle::InputMemDescriptorCoords> inputConnectionsInfo;
1720 std::vector<WorkingMemHandle::OutputMemDescriptorCoords> outputConnectionsInfo;
1722 for (
const auto& handleInfo: outputToHandleInfoMap)
1724 if (handleInfo.second.m_IsOutputLayerHandle)
1726 outputConnectionsInfo.emplace_back(handleInfo.second.m_OutputMemDescriptorCoords);
1729 if (handleInfo.second.m_IsInputLayerHandle)
1731 inputConnectionsInfo.emplace_back(handleInfo.second.m_InputMemDescriptorCoords);
1735 return std::make_unique<WorkingMemHandle>(networkId,
1736 inputConnectionsInfo,
1737 outputConnectionsInfo,
1738 workingMemDescriptors,
1739 workingMemDescriptorMap,
1740 std::move(externalMemoryManager),
1741 std::move(tensorMemory),
1742 std::move(managedTensorHandles),
1743 std::move(unmanagedTensorHandles));
1748 for (
auto&& workloadPtr: m_WorkloadQueue)
1750 workloadPtr.get()->RegisterDebugCallback(func);
1755 void LoadedNetwork::CreateMemoryProfileAsync()
1759 unsigned int m_StartOfLife;
1760 unsigned int m_Lifetime;
1763 unsigned int m_Index;
1768 auto align = [](
size_t numToAlign)
1770 const size_t alignment =
sizeof(float);
1771 return ((numToAlign + alignment - 1) / alignment) * alignment;
1774 std::unordered_map<const OutputSlot*, PartialBlock> memBlockTrackerMap;
1779 unsigned int timestep = 0;
1780 unsigned int outputIndex = 0;
1783 for (
auto&& layer : order)
1807 if (!m_SupportsExternallyManagedMemory[backendId])
1812 PartialBlock partialBlock;
1814 partialBlock.m_StartOfLife = timestep;
1816 size_t alignedSize = align(outputSlot.GetOutputHandler().GetTensorInfo().GetNumBytes());
1817 partialBlock.m_MemSize = alignedSize;
1818 partialBlock.m_Index = outputIndex++;
1819 partialBlock.m_Lifetime = outputSlot.GetNumConnections();
1820 partialBlock.m_BackendId = backendId;
1822 if (partialBlock.m_Lifetime == 0)
1824 m_MemBlockMap[partialBlock.m_BackendId].emplace_back(partialBlock.m_StartOfLife,
1825 partialBlock.m_StartOfLife,
1826 partialBlock.m_MemSize,
1828 partialBlock.m_Index);
1832 memBlockTrackerMap[&outputSlot] = partialBlock;
1838 const Layer& connectedInputLayer = inputSlot.GetConnectedOutputSlot()->GetOwningLayer();
1850 auto outputSlot = inputSlot.GetConnectedOutputSlot();
1852 PartialBlock& partialBlock = memBlockTrackerMap.at(outputSlot);
1854 auto& lifetime = partialBlock.m_Lifetime;
1859 m_MemBlockMap[partialBlock.m_BackendId].emplace_back(partialBlock.m_StartOfLife,
1861 partialBlock.m_MemSize,
1863 partialBlock.m_Index);
1870 void LoadedNetwork::CreateMemoryProfile()
1874 auto TraceSubTensorHandleAncestry = [](
ITensorHandle*
const subTensorHandle)
1877 while (ancestor && ancestor->
GetParent())
1886 unsigned int m_StartOfLife;
1887 unsigned int m_Lifetime;
1890 unsigned int m_Index;
1895 auto align = [](
size_t numToAlign)
1897 const size_t alignment =
sizeof(float);
1898 return ((numToAlign + alignment - 1) / alignment) * alignment;
1901 std::unordered_map<ITensorHandle*, PartialBlock> memBlockTrackerMap;
1906 unsigned int timestep = 0;
1907 unsigned int outputIndex = 0;
1910 for (
auto&& layer : order)
1934 if (!m_SupportsExternallyManagedMemory[backendId])
1939 ITensorHandle* tensorHandle = outputSlot.GetOutputHandler().GetData();
1940 tensorHandle = TraceSubTensorHandleAncestry(tensorHandle);
1942 if (memBlockTrackerMap.find(tensorHandle) == memBlockTrackerMap.end())
1944 PartialBlock partialBlock;
1946 partialBlock.m_StartOfLife = timestep;
1948 size_t alignedSize = align(outputSlot.GetOutputHandler().GetTensorInfo().GetNumBytes());
1949 partialBlock.m_MemSize = alignedSize;
1950 partialBlock.m_Index = outputIndex++;
1951 partialBlock.m_Lifetime = outputSlot.GetNumConnections();
1952 partialBlock.m_BackendId = backendId;
1954 if (partialBlock.m_Lifetime == 0)
1956 m_MemBlockMap[partialBlock.m_BackendId].emplace_back(partialBlock.m_StartOfLife,
1957 partialBlock.m_StartOfLife,
1958 partialBlock.m_MemSize,
1960 partialBlock.m_Index);
1964 memBlockTrackerMap[tensorHandle] = partialBlock;
1966 m_Tensorhandles.push_back(tensorHandle);
1971 memBlockTrackerMap.at(tensorHandle).m_Lifetime += outputSlot.GetNumConnections();
1977 const Layer& connectedInputLayer = inputSlot.GetConnectedOutputSlot()->GetOwningLayer();
1988 if (!m_SupportsExternallyManagedMemory[connectedInputLayer.
GetBackendId()])
1993 auto outputSlot = inputSlot.GetConnectedOutputSlot();
1995 ITensorHandle* tensorHandle = outputSlot->GetOutputHandler().GetData();
1996 tensorHandle = TraceSubTensorHandleAncestry(tensorHandle);
1998 PartialBlock& partialBlock = memBlockTrackerMap.at(tensorHandle);
2000 auto& lifetime = partialBlock.m_Lifetime;
2005 m_MemBlockMap[partialBlock.m_BackendId].emplace_back(partialBlock.m_StartOfLife,
2007 partialBlock.m_MemSize,
2009 partialBlock.m_Index);
2017 std::unique_ptr<MemoryManager> LoadedNetwork::CreateExternalMemoryManger(
2018 std::vector<std::pair<std::shared_ptr<TensorMemory>,
MemorySource>>& tensorMemoryVec)
2020 std::unique_ptr<MemoryManager> memoryManager = std::make_unique<MemoryManager>();
2023 for (
auto& backend : m_MemBinMap)
2025 std::vector<BufferStorage> bufferStorageVec;
2027 std::shared_ptr<ICustomAllocator> backendAllocator;
2028 if (allocatorMap.find(backend.first) != allocatorMap.end())
2030 backendAllocator = allocatorMap[backend.first];
2034 backendAllocator = m_Backends[backend.first]->GetDefaultAllocator();
2037 for (
auto& memBin : backend.second)
2043 for (
auto& memBlock : memBin.m_MemBlocks)
2045 auto tensorMemory = std::make_shared<TensorMemory>(
TensorMemory{memBlock.
m_Offset, memBlock.m_Index});
2047 tensorMemoryVec.emplace_back(tensorMemory, backendAllocator->GetMemorySourceType());
2051 bufferStorageVec.emplace_back(std::move(bufferStorage));
2054 memoryManager->StoreMemToAllocate(bufferStorageVec, backendAllocator, 4);
2057 return memoryManager;
2064 const auto& importedTensorHandlePin = m_PreImportedInputHandles.at(
id);
2065 if (!importedTensorHandlePin.m_TensorHandle)
2068 "PreImportedInput: {} has been deleted",
id));
2070 return importedTensorHandlePin.m_LayerBindingId;
2072 catch (
const std::out_of_range&)
2082 const auto& importedTensorHandlePin = m_PreImportedOutputHandles.at(
id);
2083 if (!importedTensorHandlePin.m_TensorHandle)
2086 "PreImportedOutput: {} has been deleted",
id));
2088 return importedTensorHandlePin.m_LayerBindingId;
2090 catch (
const std::out_of_range&)
Status Execute(const InputTensors &inputTensors, const OutputTensors &outputTensors, IWorkingMemHandle &workingMemHandle, std::vector< ImportedInputId > preImportedInputs={}, std::vector< ImportedOutputId > preImportedOutputs={})
Thread safe execution of the loaded network.
std::vector< std::shared_ptr< TensorMemory > > m_TensorMemoryVector
Vector of pointer to .
std::unique_ptr< IWorkingMemHandle > CreateWorkingMemHandle(NetworkId networkId)
Create a new unique WorkingMemHandle object.
bool HasCapability(const std::string &name, const BackendCapabilities &capabilities)
Convenience function to check if a capability exists in a BackendCapabilites struct.
virtual bool Import(void *memory, MemorySource source)
Import externally allocated memory.
FactoryFunction GetFactory(const BackendId &id) const
std::vector< ImportedOutputId > ImportOutputs(const OutputTensors &outputTensors)
std::unique_ptr< IWorkloadFactory > IWorkloadFactoryPtr
unsigned int GetNumInputSlots() const override
Returns the number of connectable input slots.
static std::unique_ptr< TimelineUtilityMethods > GetTimelineUtils(ProfilingService &profilingService)
static ProfilerManager & GetInstance()
#define ARMNN_NO_DEPRECATE_WARN_BEGIN
virtual IMemoryManagerUniquePtr CreateMemoryManager() const
LayerBindingId GetBindingId() const
virtual unsigned int GetImportFlags() const
Get flags describing supported import sources.
const bool m_AsyncEnabled
const armnn::Tensor GetOutputTensor(const LayerBindingId layerId, const OutputTensors &outputTensors)
MemoryOptimizerStrategiesMapRef GetMemoryOptimizerStrategies()
unsigned int ImportedOutputId
WorkingMemDescriptor & GetWorkingMemDescriptorAt(unsigned int id) override
Get the WorkingMemDescriptor at an index.
size_t m_Offset
Number of bytes the value is away from the .m_Buffer.
virtual void Allocate()=0
Indicate to the memory manager that this resource is no longer active.
TensorInfo GetInputTensorInfo(LayerBindingId layerId) const
#define ARMNN_LOG(severity)
size_t m_BufferSize
Total size of the buffer.
ITensorHandle * GetOutputHandle(LayerBindingId layerBindingId) const
BackendRegistry & BackendRegistryInstance()
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
const ProfilingDetailsMethod m_OutputNetworkDetailsMethod
unsigned int MemorySourceFlags
MemoryType GetMemoryArea() const
size_t GetNumOutputs() const
void CopyToOutputTensor(const Tensor &outputTensor, ITensorHandle *outputTensorHandle)
TensorInfo GetOutputTensorInfo(LayerBindingId layerId) const
Copyright (c) 2021 ARM Limited and Contributors.
void IgnoreUnused(Ts &&...)
const std::vector< InputSlot > & GetInputSlots() const
std::function< void(LayerGuid guid, unsigned int slotIndex, ITensorHandle *tensorHandle)> DebugCallbackFunction
Define the type of callback for the Debug layer to call.
unsigned int GetNumOutputSlots() const override
Returns the number of connectable output slots.
int LayerBindingId
Type of identifiers for bindable layers (inputs, outputs).
#define ARMNN_SCOPED_PROFILING_EVENT(backendId, name)
virtual const BackendId & GetId() const =0
ConstIteratorOutputs begin() const
A tensor defined by a TensorInfo (shape and data type) and a mutable backing store.
virtual IWorkloadFactoryPtr CreateWorkloadFactory(const IMemoryManagerSharedPtr &memoryManager=nullptr) const =0
std::vector< ITensorHandle * > m_Inputs
unsigned int GetNumConnections() const override
const InputSlot & GetInputSlot(unsigned int index) const override
Get a const input slot handle by slot index.
std::vector< TensorInfo > m_InputTensorInfos
const std::vector< std::vector< ITensorHandle * >::iterator > & GetOutputConnection(LayerBindingId layerBindingId) const
void ValidateBindingIds()
#define ARMNN_NO_DEPRECATE_WARN_END
#define ARMNN_ASSERT_MSG(COND, MSG)
bool SupportsTensorAllocatorAPI() const
#define ARMNN_SCOPED_HEAP_PROFILING(TAG)
A tensor defined by a TensorInfo (shape and data type) and an immutable backing store.
virtual ITensorHandle * GetParent() const =0
Get the parent tensor if this is a subtensor.
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
const std::string & GetNameStr() const
LayerType GetType() const override
Returns the armnn::LayerType of this layer.
const std::vector< std::vector< ITensorHandle * >::iterator > & GetInputConnections(LayerBindingId layerBindingId) const
#define ARMNN_ASSERT(COND)
void ClearImportedInputs(const std::vector< ImportedInputId > inputIds)
std::vector< TensorInfo > m_OutputTensorInfos
static bool IsLayerSupported(const BackendId &backendId, const IConnectableLayer &layer, Optional< DataType > dataType, std::string &outReasonIfUnsupported)
std::vector< std::unique_ptr< IWorkload > > WorkloadQueue
const TensorInfo & GetInfo() const
const BackendId & GetBackendId() const
void Allocate() override
Allocate the backing memory required for execution.
const std::vector< OutputSlot > & GetOutputSlots() const
OutputLayersAccessor GetOutputLayers() const
Returns a wrapper object with begin(), end() methods to iterate over the output layers in a range-bas...
unsigned int ImportedInputId
Status EnqueueWorkload(const InputTensors &inputTensors, const OutputTensors &outputTensors)
Single thread execution of the loaded network.
void RegisterProfiler(IProfiler *profiler)
virtual const void * Map(bool blocking=true) const =0
Map the tensor data for access.
std::vector< LayerBindingId > & GetBindingIdVector()
profiling::ProfilingGuid GetNetworkGuid()
std::unordered_map< BackendId, std::shared_ptr< ICustomAllocator > > GetAllocators()
virtual BackendCapabilities GetCapabilities() const
Returns a BackendCapability if the backend lists the capability The BackendCapability must then be in...
virtual void Unmap() const =0
Unmap the tensor data.
bool IsAllocated() override
IsAllocated returns true if the backing memory is currently allocated.
std::vector< ITensorHandle * > m_Outputs
Base class for all ArmNN exceptions so that users can filter to just those.
const OutputHandler & GetOutputHandler(unsigned int i=0) const
MemorySource
Define the Memory Source to reduce copies.
const std::string & Get() const
void RegisterDebugCallback(const DebugCallbackFunction &func)
ConstIteratorOutputs end() const
std::vector< ImportedInputId > ImportInputs(const InputTensors &inputTensors)
std::vector< ITensorHandle * > m_Outputs
Contains information about TensorInfos of a layer.
const char * GetName() const override
Returns the name of the layer.
ITensorHandleFactory::FactoryId GetTensorHandleFactoryId() const
bool CheckFlag(MemorySourceFlags flags, MemorySource source)
void CopyTensorContentsGeneric(const ITensorHandle *srcTensor, ITensorHandle *dstTensor, CopyFunc copy)
Graph & TopologicalSort()
Sorts layers in topological order and return this.
InputLayersAccessor GetInputLayers() const
Returns a wrapper object with begin(), end() methods to iterate over the input layers in a range-base...
std::vector< ITensorHandle * > m_Inputs
size_t GetNumLayers() const
const armnn::ConstTensor GetInputTensor(const LayerBindingId layerId, const InputTensors &inputTensors)
const TensorInfo & GetTensorInfo(const ITensorHandle *tensorHandle)
float32 helpers
static std::unique_ptr< LoadedNetwork > MakeLoadedNetwork(std::unique_ptr< IOptimizedNetwork > net, std::string &errorMessage, const INetworkProperties &networkProperties, profiling::ProfilingService &profilingService)
const TensorInfo & GetTensorInfo() const override
ITensorHandle * GetInputHandle(LayerBindingId layerBindingId) const
size_t GetNumInputs() const
virtual std::unique_ptr< ITensorHandle > CreateTensorHandle(const TensorInfo &tensorInfo) const =0
static const FactoryId LegacyFactoryId
const bool m_ProfilingEnabled
void ClearImportedOutputs(const std::vector< ImportedOutputId > outputIds)
LayerType
When adding a new layer, adapt also the LastLayer enum value in the enum class LayerType below...
LayerGuid GetGuid() const final
Returns the unique id of the layer.
void SendNetworkStructure()