23 #include <client/include/IProfilingService.hpp> 24 #include <client/include/TimelineUtilityMethods.hpp> 26 #include <common/include/LabelsAndEventClasses.hpp> 29 #include <condition_variable> 30 #include <unordered_map> 54 std::unique_ptr<IWorkingMemHandle> CreateWorkingMemHandle(
NetworkId networkId);
59 std::vector<ImportedInputId> ImportInputs(
const InputTensors& inputTensors,
60 MemorySource forceImportMemorySource = MemorySource::Undefined);
61 std::vector<ImportedOutputId> ImportOutputs(
const OutputTensors& outputTensors,
62 MemorySource forceImportMemorySource = MemorySource::Undefined);
64 void ClearImportedInputs(
const std::vector<ImportedInputId> inputIds);
65 void ClearImportedOutputs(
const std::vector<ImportedOutputId> outputIds);
69 std::vector<ImportedInputId> preImportedInputIds = {},
70 std::vector<ImportedOutputId> preImportedOutputIds = {});
76 std::vector<ImportedInputId> preImportedInputs = {},
77 std::vector<ImportedOutputId> preImportedOutputs = {});
79 static std::unique_ptr<LoadedNetwork> MakeLoadedNetwork(std::unique_ptr<IOptimizedNetwork> net,
80 std::string& errorMessage,
82 arm::pipe::IProfilingService* profilingService);
87 const std::shared_ptr<IProfiler>&
GetProfiler()
const {
return m_OptimizedNetwork->GetProfiler(); }
89 void FreeWorkingMemory();
93 void SendNetworkStructure(arm::pipe::IProfilingService& profilingService);
97 return m_NetworkProperties.m_AsyncEnabled;
100 arm::pipe::ProfilingGuid GetNetworkGuid();
105 void AllocateWorkingMemory(
106 #
if !defined(ARMNN_DISABLE_THREADS)
107 std::lock_guard<std::mutex>& lock
110 void AllocateAndExecuteConstantWorkloads();
111 void AllocateAndExecuteConstantWorkloadsAsync();
113 std::unordered_map<LayerGuid, std::unique_ptr<IWorkload>> m_ConstantWorkloads;
114 std::unordered_map<LayerGuid, ITensorHandle*> m_ConstantTensorHandles;
116 std::unique_ptr<IMemoryOptimizerStrategy> m_ConstantStrategy = std::make_unique<SingleAxisPriorityList>();
120 arm::pipe::IProfilingService* profilingService);
130 bool Execute(std::unique_ptr<arm::pipe::TimelineUtilityMethods>& timelineUtils,
131 arm::pipe::ProfilingGuid inferenceGuid);
138 void CreateMemoryProfile();
139 void CreateMemoryProfileAsync();
141 std::unique_ptr<MemoryManager> CreateExternalMemoryManger(
142 std::vector<std::pair<std::shared_ptr<TensorMemory>,
MemorySource>>& tensorMemory);
144 using BackendPtrMap = std::unordered_map<BackendId, IBackendInternalUniquePtr>;
146 BackendPtrMap m_Backends;
147 std::vector<IBackendInternal::IMemoryManagerSharedPtr> m_BackendMemoryMangers;
149 using WorkloadFactoryMap = std::unordered_map<BackendId, IBackendInternal::IWorkloadFactoryPtr>;
150 WorkloadFactoryMap m_WorkloadFactories;
152 std::unique_ptr<IOptimizedNetwork> m_OptimizedNetwork;
158 #if !defined(ARMNN_DISABLE_THREADS) 159 mutable std::mutex m_WorkingMemMutex;
162 bool m_IsWorkingMemAllocated =
false;
169 arm::pipe::IProfilingService* m_ProfilingService;
171 struct ImportedTensorHandlePin
173 ImportedTensorHandlePin()
177 std::unique_ptr<ITensorHandle> tensorHandle)
178 : m_LayerBindingId(layerBindingId)
179 , m_TensorHandle(std::move(tensorHandle))
182 ImportedTensorHandlePin(ImportedTensorHandlePin&&) =
default;
184 ~ImportedTensorHandlePin()
188 m_TensorHandle->Unimport();
193 std::unique_ptr<ITensorHandle> m_TensorHandle;
196 std::vector<ImportedTensorHandlePin> m_PreImportedInputHandles;
197 std::vector<ImportedTensorHandlePin> m_PreImportedOutputHandles;
202 std::unordered_map<BackendId, std::vector<MemBlock>> m_MemBlockMap;
203 std::unordered_map<BackendId, std::vector<MemBin>> m_MemBinMap;
205 std::vector<ITensorHandle*> m_Tensorhandles;
207 std::vector<std::pair<std::shared_ptr<TensorMemory>,
MemorySource>> m_TensorMemory;
209 std::unique_ptr<MemoryManager> m_ExternalMemoryManager;
211 std::unordered_map<BackendId, bool> m_SupportsExternallyManagedMemory;
215 struct WorkloadIndices
217 unsigned int m_WorkloadIndex;
218 unsigned int m_SlotIndex;
221 struct OutputWorkloadIndices
223 WorkloadIndices m_OutputSlotIndices;
224 std::vector<WorkloadIndices> m_InputSlotIndices;
226 std::unordered_map<LayerBindingId, std::vector<WorkloadIndices>> m_InputWorkloadSlotPairs;
227 std::unordered_map<LayerBindingId, OutputWorkloadIndices> m_OutputWorkloadSlotPairs;
228 std::vector<bool> m_IsInputImported;
229 std::vector<bool> m_IsOutputImported;
unsigned int ImportedOutputId
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
Copyright (c) 2021 ARM Limited and Contributors.
std::function< void(LayerGuid guid, unsigned int slotIndex, ITensorHandle *tensorHandle)> DebugCallbackFunction
Define the type of callback for the Debug layer to call.
int LayerBindingId
Type of identifiers for bindable layers (inputs, outputs).
A tensor defined by a TensorInfo (shape and data type) and a mutable backing store.
A tensor defined by a TensorInfo (shape and data type) and an immutable backing store.
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
std::vector< std::unique_ptr< IWorkload > > WorkloadQueue
unsigned int ImportedInputId
const std::shared_ptr< IProfiler > & GetProfiler() const
MemorySource
Define the Memory Source to reduce copies.