26 #include <boost/format.hpp> 37 template <
typename ExceptionType>
38 std::string ToErrorMessage(
const char * prefix,
const ExceptionType &
error)
41 ss << prefix <<
" " << error.what();
45 void AddLayerStructure(std::unique_ptr<TimelineUtilityMethods>& timelineUtils,
50 std::string layerName = layer.GetNameStr().empty() ?
"<Unnamed>" : layer.GetNameStr();
51 timelineUtils->CreateNamedTypedChildEntity(layer.GetGuid(),
55 for (
auto&& input : layer.GetInputSlots())
57 const IOutputSlot* source = input.GetConnectedOutputSlot();
59 timelineUtils->CreateConnectionRelationship(ProfilingRelationshipType::RetentionLink,
60 source->GetOwningLayerGuid(),
65 void AddWorkloadStructure(std::unique_ptr<TimelineUtilityMethods>& timelineUtils,
66 std::unique_ptr<IWorkload>& workload,
71 timelineUtils->MarkEntityWithLabel(workload->GetGuid(),
72 layer.GetBackendId().Get(),
76 timelineUtils->CreateRelationship(ProfilingRelationshipType::RetentionLink,
85 std::string& errorMessage,
89 std::unique_ptr<LoadedNetwork> loadedNetwork;
91 auto Fail = [&](
const std::exception&
error) -> std::unique_ptr<LoadedNetwork>
93 errorMessage = ToErrorMessage(
"An error occurred when preparing the network workloads: ", error);
96 return std::unique_ptr<LoadedNetwork>();
101 loadedNetwork.reset(
new LoadedNetwork(std::move(net), networkProperties, profilingService));
111 catch (
const std::runtime_error& error)
116 return loadedNetwork;
119 LoadedNetwork::LoadedNetwork(std::unique_ptr<OptimizedNetwork> net,
122 m_OptimizedNetwork(std::move(net)),
125 m_TensorHandleFactoryRegistry(),
126 m_ProfilingService(profilingService)
129 m_Profiler = std::make_shared<Profiler>();
137 for (
auto&& layer : order)
139 auto const& backendId = layer->GetBackendId();
140 if (m_Backends.count(backendId) == 0)
143 auto it = m_Backends.emplace(std::make_pair(backendId, createBackend()));
150 m_WorkloadFactories.emplace(
151 std::make_pair(backendId, std::make_pair(std::move(workloadFactory),
nullptr)));
158 m_WorkloadFactories.emplace(
159 std::make_pair(backendId, std::make_pair(std::move(workloadFactory), memoryManager)));
164 for (
auto&& layer : order)
166 auto& workloadFactory = GetWorkloadFactory(*layer);
168 switch (layer->GetType())
174 layer->CreateTensorHandles(m_TensorHandleFactoryRegistry, workloadFactory, !m_IsImportEnabled);
181 if((layer->GetNumOutputSlots() == 1) &&
182 (layer->GetOutputSlots()[0].GetNumConnections() == 1) &&
183 (layer->GetOutputSlots()[0].GetConnection(0)->GetOwningLayer().GetType() ==
LayerType::Output))
185 layer->CreateTensorHandles(m_TensorHandleFactoryRegistry, workloadFactory, !m_IsExportEnabled);
189 layer->CreateTensorHandles(m_TensorHandleFactoryRegistry, workloadFactory);
196 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
205 std::stringstream ss;
211 for (
auto&& layer : order)
216 AddLayerStructure(timelineUtils, *layer, networkGuid);
221 switch (layer->GetType())
231 auto workload = layer->CreateWorkload(workloadFactory);
235 const char*
const layerName =
236 layer->GetNameStr().length() != 0 ? layer->GetName() :
"<Unnamed>";
238 boost::format(
"No workload created for layer (name: '%1%' type: '%2%') (compute '%3%')")
239 % layerName % static_cast<int>(layer->GetType()) % layer->GetBackendId().Get()
246 AddWorkloadStructure(timelineUtils, workload, *layer);
249 m_WorkloadQueue.push_back(move(workload));
251 layer->ReleaseConstantData();
260 timelineUtils->Commit();
264 m_OptimizedNetwork->GetGraph().AllocateDynamicBuffers();
267 for (
auto& workload : m_WorkloadQueue)
269 workload->PostAllocationConfigure();
278 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
283 for (
auto&& layer : order)
286 AddLayerStructure(timelineUtils, *layer, networkGuid);
287 switch (layer->GetType())
297 for (
auto& workload : m_WorkloadQueue)
300 AddWorkloadStructure(timelineUtils, workload, *layer);
307 timelineUtils->Commit();
312 return m_OptimizedNetwork->GetGuid();
317 for (
auto&& inputLayer : m_OptimizedNetwork->GetGraph().GetInputLayers())
319 ARMNN_ASSERT_MSG(inputLayer->GetNumOutputSlots() == 1,
"Input layer should have exactly 1 output slot");
320 if (inputLayer->GetBindingId() == layerId)
322 return inputLayer->GetOutputSlot(0).GetTensorInfo();
331 for (
auto&& outputLayer : m_OptimizedNetwork->GetGraph().GetOutputLayers())
333 ARMNN_ASSERT_MSG(outputLayer->GetNumInputSlots() == 1,
"Output layer should have exactly 1 input slot");
334 ARMNN_ASSERT_MSG(outputLayer->GetInputSlot(0).GetConnection(),
"Input slot on Output layer must be connected");
335 if (outputLayer->GetBindingId() == layerId)
337 return outputLayer->GetInputSlot(0).GetConnection()->GetTensorInfo();
348 auto it = m_WorkloadFactories.find(layer.
GetBackendId());
349 if (it == m_WorkloadFactories.end())
353 boost::format(
"No workload factory for %1% to be used for layer: %2%")
359 workloadFactory = it->second.first.get();
363 std::string reasonIfUnsupported;
365 "Factory does not support layer");
367 return *workloadFactory;
377 : m_TensorHandle(std::move(handle))
383 ITensorHandle* GetTensorHandle()
const {
return m_TensorHandle.get(); }
388 std::unique_ptr<ITensorHandle> m_TensorHandle;
394 const std::vector<TensorPin>& pins,
395 char const* bindingPointDesc)
397 auto it = std::find_if(pins.begin(), pins.end(),
398 [id](
const TensorPin& pin)
400 return pin.GetBindingId() == id;
403 if (it != pins.end())
410 boost::format(
"No tensor supplied for %1% %2%") % bindingPointDesc %
id));
420 m_InputTensorPins.reserve(inputTensors.size());
421 m_OutputTensorPins.reserve(outputTensors.size());
423 for (
auto inputTensorPair : inputTensors)
425 auto inputTensor = inputTensorPair.second;
427 std::unique_ptr<ITensorHandle> tensorHandle =
428 std::make_unique<ConstPassthroughCpuTensorHandle>(inputTensor.GetInfo(),inputTensor.GetMemoryArea());
431 m_InputTensorPins.emplace_back(std::move(tensorHandle), inputTensor.GetInfo(), layerId);
434 for (
auto outputTensorPair : outputTensors)
436 auto outputTensor = outputTensorPair.second;
438 std::unique_ptr<ITensorHandle> tensorHandle =
439 std::make_unique<PassthroughCpuTensorHandle>(outputTensor.GetInfo(), outputTensor.GetMemoryArea());
442 m_OutputTensorPins.emplace_back(std::move(tensorHandle), outputTensor.GetInfo(), layerId);
448 return GetTensorPin(
id, m_InputTensorPins,
"input");
453 return GetTensorPin(
id, m_OutputTensorPins,
"output");
458 std::vector<TensorPin> m_InputTensorPins;
459 std::vector<TensorPin> m_OutputTensorPins;
467 const Graph& graph = m_OptimizedNetwork->GetGraph();
472 ARMNN_LOG(
warning) <<
"IRuntime::EnqueueWorkload()::Less than two nodes in graph";
477 WorkloadData workloadData(inputTensors, outputTensors);
487 m_InputQueue.clear();
491 const TensorPin& pin = workloadData.GetInputTensorPin(inputLayer->GetBindingId());
492 EnqueueInput(*inputLayer, pin.GetTensorHandle(), pin.GetTensorInfo());
499 m_OutputQueue.clear();
503 const TensorPin& pin = workloadData.GetOutputTensorPin(outputLayer->GetBindingId());
504 EnqueueOutput(*outputLayer, pin.GetTensorHandle(), pin.GetTensorInfo());
508 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
510 ProfilingGuid inferenceGuid = m_ProfilingService.GetNextGuid();
516 timelineUtils->CreateRelationship(ProfilingRelationshipType::RetentionLink,
523 bool executionSucceeded =
true;
526 if (m_ProfilingService.IsProfilingEnabled())
528 m_ProfilingService.IncrementCounterValue(armnn::profiling::INFERENCES_RUN);
532 executionSucceeded = Execute(timelineUtils, inferenceGuid);
539 timelineUtils->Commit();
551 if (tensorHandle ==
nullptr)
559 inputQueueDescriptor.
m_Inputs.push_back(tensorHandle);
564 const TensorInfo& outputTensorInfo = handler.GetTensorInfo();
567 "Data should have been allocated.");
568 inputQueueDescriptor.
m_Outputs.push_back(outputTensorHandle);
572 if (m_IsImportEnabled)
577 void* mem = tensorHandle->
Map(
false);
580 tensorHandle->
Unmap();
583 tensorHandle->
Unmap();
594 std::unique_ptr<IWorkload> inputWorkload = std::make_unique<CopyMemGenericWorkload>(inputQueueDescriptor,
info);
598 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
603 AddWorkloadStructure(timelineUtils, inputWorkload, layer);
604 timelineUtils->Commit();
607 m_InputQueue.push_back(move(inputWorkload));
618 if (tensorHandle ==
nullptr)
626 outputQueueDescriptor.
m_Outputs.push_back(tensorHandle);
634 const TensorInfo& inputTensorInfo = outputHandler.GetTensorInfo();
636 ARMNN_ASSERT_MSG(inputTensorHandle !=
nullptr,
"Data should have been allocated.");
645 if (m_IsExportEnabled && (layer.
GetInputSlots()[0].GetConnectedOutputSlot()->GetNumConnections() == 1))
652 void *mem = tensorHandle->
Map(
false);
654 tensorHandle->
Unmap();
660 syncDesc.
m_Inputs.push_back(inputTensorHandle);
662 auto syncWorkload = std::make_unique<SyncMemGenericWorkload>(syncDesc,
info);
664 m_OutputQueue.push_back(move(syncWorkload));
678 throw MemoryExportException(
"EnqueueOutput: Memory Export failed, attempting to export Input Layer");
684 outputQueueDescriptor.
m_Inputs.push_back(inputTensorHandle);
687 std::unique_ptr<IWorkload> outputWorkload =
688 std::make_unique<CopyMemGenericWorkload>(outputQueueDescriptor,
info);
691 std::unique_ptr<TimelineUtilityMethods> timelineUtils =
696 AddWorkloadStructure(timelineUtils, outputWorkload, layer);
697 timelineUtils->Commit();
700 m_OutputQueue.push_back(move(outputWorkload));
704 void LoadedNetwork::AllocateWorkingMemory(std::lock_guard<std::mutex>& lock)
711 if (m_IsWorkingMemAllocated)
715 for (
auto&& workloadFactory : m_WorkloadFactories)
720 memoryManager->Acquire();
723 m_TensorHandleFactoryRegistry.AquireMemory();
724 m_IsWorkingMemAllocated =
true;
729 std::lock_guard<std::mutex> lockGuard(m_WorkingMemMutex);
730 if (!m_IsWorkingMemAllocated)
735 for (
auto&& workloadFactory : m_WorkloadFactories)
740 memoryManager->Release();
743 m_TensorHandleFactoryRegistry.ReleaseMemory();
744 m_IsWorkingMemAllocated =
false;
747 bool LoadedNetwork::Execute(std::unique_ptr<TimelineUtilityMethods>& timelineUtils,
752 auto Fail = [&](
const std::exception&
error)
754 ARMNN_LOG(error) <<
"An error occurred attempting to execute a workload: " << error.what();
760 std::lock_guard<std::mutex> lockGuard(m_WorkingMemMutex);
761 AllocateWorkingMemory(lockGuard);
764 auto ExecuteQueue = [&timelineUtils, &workloadInferenceID, &inferenceGuid](
WorkloadQueue& queue)
766 for (
auto& workload : queue)
770 workloadInferenceID = timelineUtils->RecordWorkloadInferenceAndStartOfLifeEvent(workload->GetGuid(),
776 timelineUtils->RecordEndOfLifeEvent(workloadInferenceID);
781 ExecuteQueue(m_InputQueue);
782 ExecuteQueue(m_WorkloadQueue);
783 ExecuteQueue(m_OutputQueue);
789 catch (
const std::runtime_error& error)
799 for (
auto&& workloadPtr: m_WorkloadQueue)
801 workloadPtr.get()->RegisterDebugCallback(func);
static ARMNN_DLLEXPORT ProfilingStaticGuid INFERENCE_GUID
virtual bool Import(void *memory, MemorySource source)
Import externally allocated memory.
FactoryFunction GetFactory(const BackendId &id) const
unsigned int GetNumInputSlots() const override
Returns the number of connectable input slots.
std::vector< std::unique_ptr< IWorkload > > WorkloadQueue
const bool m_ImportEnabled
void RegisterProfiler(Profiler *profiler)
static std::unique_ptr< TimelineUtilityMethods > GetTimelineUtils(ProfilingService &profilingService)
static ProfilerManager & GetInstance()
virtual unsigned int GetImportFlags() const
Get flags describing supported import sources.
Strongly typed guids to distinguish between those generated at runtime, and those that are statically...
TensorInfo GetInputTensorInfo(LayerBindingId layerId) const
#define ARMNN_LOG(severity)
BackendRegistry & BackendRegistryInstance()
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
unsigned int MemorySourceFlags
size_t GetNumOutputs() const
TensorInfo GetOutputTensorInfo(LayerBindingId layerId) const
Copyright (c) 2020 ARM Limited.
void IgnoreUnused(Ts &&...)
const std::vector< InputSlot > & GetInputSlots() const
std::function< void(LayerGuid guid, unsigned int slotIndex, ITensorHandle *tensorHandle)> DebugCallbackFunction
Define the type of callback for the Debug layer to call.
unsigned int GetNumOutputSlots() const override
Returns the number of connectable output slots.
static ARMNN_DLLEXPORT ProfilingStaticGuid WORKLOAD_GUID
static ARMNN_DLLEXPORT ProfilingStaticGuid ARMNN_PROFILING_EOL_EVENT_CLASS
#define ARMNN_SCOPED_PROFILING_EVENT(backendId, name)
int LayerBindingId
Type of identifiers for bindable layers (inputs, outputs).
static ARMNN_DLLEXPORT ProfilingStaticGuid ARMNN_PROFILING_SOL_EVENT_CLASS
virtual IWorkloadFactoryPtr CreateWorkloadFactory(const IMemoryManagerSharedPtr &memoryManager=nullptr) const =0
std::vector< TensorInfo > m_InputTensorInfos
static ARMNN_DLLEXPORT ProfilingStaticGuid LAYER_GUID
#define ARMNN_ASSERT_MSG(COND, MSG)
bool SupportsTensorAllocatorAPI() const
std::shared_ptr< IMemoryManager > IMemoryManagerSharedPtr
#define ARMNN_SCOPED_HEAP_PROFILING(TAG)
static ARMNN_DLLEXPORT ProfilingStaticGuid EXECUTION_OF_GUID
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
const std::string & GetNameStr() const
const bool m_ExportEnabled
#define ARMNN_ASSERT(COND)
std::vector< TensorInfo > m_OutputTensorInfos
static bool IsLayerSupported(const BackendId &backendId, const IConnectableLayer &layer, Optional< DataType > dataType, std::string &outReasonIfUnsupported)
const BackendId & GetBackendId() const
OutputLayersAccessor GetOutputLayers() const
Returns a wrapper object with begin(), end() methods to iterate over the output layers in a range-bas...
static ARMNN_DLLEXPORT ProfilingStaticGuid NETWORK_GUID
Status EnqueueWorkload(const InputTensors &inputTensors, const OutputTensors &outputTensors)
virtual const void * Map(bool blocking=true) const =0
Map the tensor data for access.
profiling::ProfilingGuid GetNetworkGuid()
virtual void Unmap() const =0
Unmap the tensor data.
std::vector< ITensorHandle * > m_Outputs
Base class for all ArmNN exceptions so that users can filter to just those.
const OutputHandler & GetOutputHandler(unsigned int i=0) const
const std::string & Get() const
void RegisterDebugCallback(const DebugCallbackFunction &func)
LayerType GetType() const
Contains information about inputs and outputs to a layer.
bool CheckFlag(MemorySourceFlags flags, MemorySource source)
Graph & TopologicalSort()
Sorts layers in topological order and return this.
InputLayersAccessor GetInputLayers() const
Returns a wrapper object with begin(), end() methods to iterate over the input layers in a range-base...
std::vector< ITensorHandle * > m_Inputs
static ARMNN_DLLEXPORT ProfilingStaticGuid PROCESS_ID_GUID
size_t GetNumLayers() const
virtual ARMNN_NO_DEPRECATE_WARN_END IMemoryManagerUniquePtr CreateMemoryManager() const
const TensorInfo & GetTensorInfo(const ITensorHandle *tensorHandle)
float32 helpers
size_t GetNumInputs() const
static ARMNN_DLLEXPORT ProfilingStaticGuid BACKENDID_GUID
static std::unique_ptr< LoadedNetwork > MakeLoadedNetwork(std::unique_ptr< OptimizedNetwork > net, std::string &errorMessage, const INetworkProperties &networkProperties, profiling::ProfilingService &profilingService)
static ARMNN_DLLEXPORT ProfilingStaticGuid CHILD_GUID
void SendNetworkStructure()