1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
|
//
// Copyright © 2017 Arm Ltd and Contributors. All rights reserved.
// SPDX-License-Identifier: MIT
//
#pragma once
#include <armnn/Tensor.hpp>
#include <armnn/Types.hpp>
#include "Network.hpp"
#include "LayerFwd.hpp"
#include "Profiling.hpp"
#include <armnn/backends/IBackendInternal.hpp>
#include <backendsCommon/TensorHandleFactoryRegistry.hpp>
#include <backendsCommon/Workload.hpp>
#include <backendsCommon/WorkloadFactory.hpp>
#include <ProfilingService.hpp>
#include <TimelineUtilityMethods.hpp>
#include <mutex>
#include <unordered_map>
namespace cl
{
class Context;
class CommandQueue;
class Device;
}
namespace armnn
{
class LoadedNetwork
{
public:
using WorkloadQueue = std::vector< std::unique_ptr<IWorkload> >;
~LoadedNetwork(){ FreeWorkingMemory(); }
/// Create a new unique WorkingMemHandle object. Create multiple handles if you wish to have
/// overlapped Execution by calling this function from different threads.
std::unique_ptr<IWorkingMemHandle> CreateWorkingMemHandle(NetworkId networkId);
TensorInfo GetInputTensorInfo(LayerBindingId layerId) const;
TensorInfo GetOutputTensorInfo(LayerBindingId layerId) const;
Status EnqueueWorkload(const InputTensors& inputTensors, const OutputTensors& outputTensors);
Status Execute(const InputTensors& inputTensors,
const OutputTensors& outputTensors,
IWorkingMemHandle& workingMemHandle);
static std::unique_ptr<LoadedNetwork> MakeLoadedNetwork(std::unique_ptr<IOptimizedNetwork> net,
std::string & errorMessage,
const INetworkProperties& networkProperties,
profiling::ProfilingService& profilingService);
// NOTE we return by reference as the purpose of this method is only to provide
// access to the private m_Profiler and in theory we should not need to increment
// the shared_ptr's reference counter
const std::shared_ptr<IProfiler>& GetProfiler() const { return m_Profiler; }
void FreeWorkingMemory();
void RegisterDebugCallback(const DebugCallbackFunction& func);
void SendNetworkStructure();
bool IsAsyncEnabled()
{
return m_NetworkProperties.m_AsyncEnabled;
}
profiling::ProfilingGuid GetNetworkGuid();
private:
using WorkloadFactoryWithMemoryManager =
std::pair<IBackendInternal::IWorkloadFactoryPtr, IBackendInternal::IMemoryManagerSharedPtr>;
using WorkloadFactoryMap = std::unordered_map<BackendId, WorkloadFactoryWithMemoryManager>;
void AllocateWorkingMemory(std::lock_guard<std::mutex>& lock);
void AllocateAndExecuteConstantWorkloads();
std::unordered_map<LayerGuid, ITensorHandle* > m_ConstantTensorHandles;
std::unordered_map<LayerGuid, std::unique_ptr<IWorkload> > m_ConstantWorkloads;
LoadedNetwork(std::unique_ptr<IOptimizedNetwork> net,
const INetworkProperties& networkProperties,
profiling::ProfilingService& profilingService);
void EnqueueInput(const BindableLayer& layer, ITensorHandle* tensorHandle, const TensorInfo& tensorInfo);
void EnqueueOutput(const BindableLayer& layer, ITensorHandle* tensorHandle, const TensorInfo& tensorInfo);
void EnqueueInput(const BindableLayer& layer, const ConstTensor& inputTensor, WorkingMemHandle& handle);
void EnqueueOutput(const BindableLayer& layer, const Tensor& outputTensor, WorkingMemHandle& handle);
bool Execute(std::unique_ptr<profiling::TimelineUtilityMethods>& timelineUtils,
profiling::ProfilingGuid inferenceGuid);
const IWorkloadFactory& GetWorkloadFactory(const Layer& layer) const;
using BackendPtrMap = std::unordered_map<BackendId, IBackendInternalUniquePtr>;
BackendPtrMap m_Backends;
WorkloadFactoryMap m_WorkloadFactories;
std::unique_ptr<IOptimizedNetwork> m_OptimizedNetwork;
WorkloadQueue m_InputQueue;
WorkloadQueue m_WorkloadQueue;
WorkloadQueue m_OutputQueue;
std::shared_ptr<IProfiler> m_Profiler;
mutable std::mutex m_WorkingMemMutex;
bool m_IsWorkingMemAllocated=false;
INetworkProperties m_NetworkProperties;
TensorHandleFactoryRegistry m_TensorHandleFactoryRegistry;
profiling::ProfilingService& m_ProfilingService;
};
}
|
