17 #include <common/include/LabelsAndEventClasses.hpp> 23 using namespace armnn;
51 return pRuntimeImpl->LoadNetwork(networkIdOut, std::move(network));
56 std::string& errorMessage)
58 return pRuntimeImpl->LoadNetwork(networkIdOut, std::move(network), errorMessage);
63 std::string& errorMessage,
66 return pRuntimeImpl->LoadNetwork(networkIdOut, std::move(network), errorMessage, networkProperties);
71 return pRuntimeImpl->GetInputTensorInfo(networkId, layerId);
76 return pRuntimeImpl->GetOutputTensorInfo(networkId, layerId);
83 return pRuntimeImpl->EnqueueWorkload(networkId, inputTensors, outputTensors);
90 return pRuntimeImpl->Execute(workingMemHandle, inputTensors, outputTensors);
115 return pRuntimeImpl->RegisterDebugCallback(networkId, func);
118 int RuntimeImpl::GenerateNetworkId()
120 return m_NetworkIdCounter++;
125 std::string ignoredErrorMessage;
126 return LoadNetwork(networkIdOut, std::move(inNetwork), ignoredErrorMessage);
131 std::string& errorMessage)
135 return LoadNetwork(networkIdOut, std::move(inNetwork), errorMessage, networkProperties);
140 std::string& errorMessage,
145 networkIdOut = GenerateNetworkId();
147 for (
auto&& context : m_BackendContexts)
149 context.second->BeforeLoadNetwork(networkIdOut);
153 std::unique_ptr<IOptimizedNetwork>(rawNetwork),
164 std::lock_guard<std::mutex> lockGuard(m_Mutex);
167 m_LoadedNetworks[networkIdOut] = std::move(loadedNetwork);
170 for (
auto&& context : m_BackendContexts)
172 context.second->AfterLoadNetwork(networkIdOut);
175 if (m_ProfilingService.IsProfilingEnabled())
177 m_ProfilingService.IncrementCounterValue(armnn::profiling::NETWORK_LOADS);
185 bool unloadOk =
true;
186 for (
auto&& context : m_BackendContexts)
188 unloadOk &= context.second->BeforeUnloadNetwork(networkId);
194 "network with ID:" << networkId <<
" because BeforeUnloadNetwork failed";
198 std::unique_ptr<profiling::TimelineUtilityMethods> timelineUtils =
201 std::lock_guard<std::mutex> lockGuard(m_Mutex);
206 auto search = m_LoadedNetworks.find(networkId);
207 if (search != m_LoadedNetworks.end())
209 profiling::ProfilingGuid networkGuid = search->second->GetNetworkGuid();
210 timelineUtils->RecordEvent(networkGuid,
211 profiling::LabelsAndEventClasses::ARMNN_PROFILING_EOL_EVENT_CLASS);
214 if (m_LoadedNetworks.erase(networkId) == 0)
216 ARMNN_LOG(
warning) <<
"WARNING: RuntimeImpl::UnloadNetwork(): " << networkId <<
" not found!";
220 if (m_ProfilingService.IsProfilingEnabled())
222 m_ProfilingService.IncrementCounterValue(armnn::profiling::NETWORK_UNLOADS);
226 for (
auto&& context : m_BackendContexts)
228 context.second->AfterUnloadNetwork(networkId);
231 ARMNN_LOG(
debug) <<
"RuntimeImpl::UnloadNetwork(): Unloaded network with ID: " << networkId;
237 auto it = m_LoadedNetworks.find(networkId);
238 if (it != m_LoadedNetworks.end())
240 auto& loadedNetwork = it->second;
241 return loadedNetwork->GetProfiler();
252 LoadedNetworks::iterator it = m_LoadedNetworks.begin();
253 while (it != m_LoadedNetworks.end())
255 auto& loadedNetwork = it->second;
256 loadedNetwork->SendNetworkStructure();
263 : m_NetworkIdCounter(0),
264 m_ProfilingService(*this)
271 "It is not possible to enable timeline reporting without profiling being enabled");
285 auto backend = factoryFun();
291 customAllocatorMapIterator->second ==
nullptr)
296 throw armnn::Exception(
"Allocator associated with id " +
id.Get() +
" is null");
304 BackendCapability protectedContentCapability {
"ProtectedContentAllocation",
true};
311 <<
" is not registered as does not support protected content allocation \n";
319 if (customAllocatorMapIterator->second->GetMemorySourceType()
322 if (!backend->UseCustomMemoryAllocator(customAllocatorMapIterator->second, err))
326 <<
" reported an error when entering protected mode. Backend won't be" 327 <<
" used. ErrorMsg: " << err;
335 ARMNN_LOG(
error) <<
"The CustomAllocator provided with the runtime options doesn't support " 336 "protected memory. Protected mode can't be activated. The backend " 338 <<
" is not going to be used. MemorySource must be MemorySource::DmaBufProtected";
344 ARMNN_LOG(
error) <<
"Protected mode can't be activated for backend: " 346 <<
" no custom allocator was provided to the runtime options.";
356 if (!backend->UseCustomMemoryAllocator(customAllocatorMapIterator->second, err))
360 <<
" reported an error when trying to use the provided custom allocator." 361 " Backend won't be used." 362 <<
" ErrorMsg: " << err;
369 auto context = backend->CreateBackendContext(options);
375 m_BackendContexts.emplace(std::make_pair(
id, std::move(context)));
377 supportedBackends.emplace(
id);
379 unique_ptr<armnn::profiling::IBackendProfiling> profilingIface =
381 options, m_ProfilingService,
id));
384 auto profilingContext = backend->CreateBackendProfilingContext(options, profilingIface);
386 if (profilingContext)
409 ARMNN_LOG(
info) <<
"Initialization time: " << std::setprecision(2)
416 std::vector<int> networkIDs;
420 std::transform(m_LoadedNetworks.begin(), m_LoadedNetworks.end(),
421 std::back_inserter(networkIDs),
422 [](
const auto &pair) {
return pair.first; });
424 catch (
const std::exception& e)
429 std::cerr <<
"WARNING: An error has occurred when getting the IDs of the networks to unload: " << e.what()
430 <<
"\nSome of the loaded networks may not be unloaded" << std::endl;
435 for (
auto networkID : networkIDs)
443 catch (
const std::exception& e)
448 std::cerr <<
"WARNING: An error has occurred when unloading network " << networkID <<
": " << e.what()
456 m_BackendContexts.clear();
465 std::lock_guard<std::mutex> lockGuard(m_Mutex);
466 return m_LoadedNetworks.at(networkId).get();
484 LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
488 ARMNN_LOG(
error) <<
"A Network with an id of " << networkId <<
" does not exist.\n";
493 ARMNN_LOG(
error) <<
"Network " << networkId <<
" is async enabled.\n";
500 static thread_local
NetworkId lastId = networkId;
501 if (lastId != networkId)
518 LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
522 ARMNN_LOG(
error) <<
"A Network with an id of " << networkId <<
" does not exist.\n";
527 ARMNN_LOG(
error) <<
"Attempting execute " << networkId <<
" when it is not async enabled.\n";
534 return loadedNetwork->
Execute(inputTensors, outputTensors, iWorkingMemHandle);
541 LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
545 ARMNN_LOG(
error) <<
"A Network with an id of " << networkId <<
" does not exist.\n";
550 ARMNN_LOG(
error) <<
"Network " << networkId <<
" is not async enabled.\n";
557 static thread_local
NetworkId lastId = networkId;
558 if (lastId != networkId)
572 LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
576 void RuntimeImpl::LoadDynamicBackends(
const std::string& overrideBackendPath)
void AddSupportedBackends(const BackendIdSet &backendIds, bool isDynamic=false)
TensorInfo GetInputTensorInfo(NetworkId networkId, LayerBindingId layerId) const
std::unique_ptr< IWorkingMemHandle > CreateWorkingMemHandle(NetworkId networkId)
Create a new unique WorkingMemHandle object.
void WaitForProfilingServiceActivation(unsigned int timeout) override
bool HasCapability(const std::string &name, const BackendCapabilities &capabilities)
Convenience function to check if a capability exists in a BackendCapabilites struct.
static IRuntimePtr Create(const CreationOptions &options)
FactoryFunction GetFactory(const BackendId &id) const
const std::shared_ptr< IProfiler > GetProfiler(NetworkId networkId) const
Gets the profiler corresponding to the given network id.
std::chrono::duration< double, std::milli > GetTimeDuration(std::chrono::high_resolution_clock::time_point start_time)
static void DeregisterDynamicBackends(const BackendIdSet &dynamicBackends)
virtual NetworkId GetNetworkId()=0
Returns the NetworkId of the Network that this IWorkingMemHandle works with.
std::unordered_set< BackendId > BackendIdSet
static std::unique_ptr< TimelineUtilityMethods > GetTimelineUtils(ProfilingService &profilingService)
static ProfilerManager & GetInstance()
TensorInfo GetInputTensorInfo(NetworkId networkId, LayerBindingId layerId) const
#define ARMNN_VERSION
ARMNN_VERSION: "X.Y.Z" where: X = Major version number Y = Minor version number Z = Patch version num...
Status UnloadNetwork(NetworkId networkId)
Unloads a network from the Runtime.
std::unique_ptr< IRuntime, void(*)(IRuntime *runtime)> IRuntimePtr
void RegisterAllocator(const BackendId &id, std::shared_ptr< ICustomAllocator > alloc)
TensorInfo GetInputTensorInfo(LayerBindingId layerId) const
#define ARMNN_LOG(severity)
BackendRegistry & BackendRegistryInstance()
Status LoadNetwork(NetworkId &networkIdOut, IOptimizedNetworkPtr network)
Loads a complete network into the Runtime.
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
Status Execute(IWorkingMemHandle &workingMemHandle, const InputTensors &inputTensors, const OutputTensors &outputTensors)
This is an experimental function.
static std::vector< DynamicBackendPtr > CreateDynamicBackends(const std::vector< std::string > &sharedObjects)
TensorInfo GetOutputTensorInfo(NetworkId networkId, LayerBindingId layerId) const
std::chrono::high_resolution_clock::time_point GetTimeNow()
TensorInfo GetOutputTensorInfo(LayerBindingId layerId) const
Copyright (c) 2021 ARM Limited and Contributors.
Status EnqueueWorkload(NetworkId networkId, const InputTensors &inputTensors, const OutputTensors &outputTensors)
void RegisterDebugCallback(NetworkId networkId, const DebugCallbackFunction &func)
Registers a callback function to debug layers performing custom computations on intermediate tensors...
const std::shared_ptr< IProfiler > GetProfiler(NetworkId networkId) const
Gets the profiler corresponding to the given network id.
std::function< void(LayerGuid guid, unsigned int slotIndex, ITensorHandle *tensorHandle)> DebugCallbackFunction
Define the type of callback for the Debug layer to call.
static std::vector< std::string > GetBackendPaths(const std::string &overrideBackendPath="")
#define ARMNN_SCOPED_PROFILING_EVENT(backendId, name)
int LayerBindingId
Type of identifiers for bindable layers (inputs, outputs).
Status Execute(IWorkingMemHandle &workingMemHandle, const InputTensors &inputTensors, const OutputTensors &outputTensors)
This is an experimental function.
static void Destroy(IRuntime *runtime)
Status Execute(const InputTensors &inputTensors, const OutputTensors &outputTensors, IWorkingMemHandle &workingMemHandle)
Thread safe execution of the loaded network.
void RegisterDebugCallback(NetworkId networkId, const DebugCallbackFunction &func)
Registers a callback function to debug layers performing custom computations on intermediate tensors...
BackendCapability
BackendCapability class.
TensorInfo GetOutputTensorInfo(NetworkId networkId, LayerBindingId layerId) const
static std::vector< std::string > GetSharedObjects(const std::vector< std::string > &backendPaths)
Status UnloadNetwork(NetworkId networkId)
Unloads a network from the IRuntime.
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
void SetProfilingService(armnn::Optional< profiling::ProfilingService &> profilingService)
std::map< BackendId, std::shared_ptr< ICustomAllocator > > m_CustomAllocatorMap
A map to define a custom memory allocator for specific backend Ids.
#define ARMNN_ASSERT(COND)
std::unique_ptr< RuntimeImpl > pRuntimeImpl
Status LoadNetwork(NetworkId &networkIdOut, IOptimizedNetworkPtr network)
Loads a complete network into the IRuntime.
Device specific knowledge to be passed to the optimizer.
static IRuntime * CreateRaw(const CreationOptions &options)
const IDeviceSpec & GetDeviceSpec() const
bool m_ProtectedMode
Setting this flag will allow the user to create the Runtime in protected mode.
Status EnqueueWorkload(const InputTensors &inputTensors, const OutputTensors &outputTensors)
Single thread execution of the loaded network.
std::string m_DynamicBackendsPath
Setting this value will override the paths set by the DYNAMIC_BACKEND_PATHS compiler directive Only a...
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
void RegisterProfiler(IProfiler *profiler)
Base class for all ArmNN exceptions so that users can filter to just those.
const std::shared_ptr< IProfiler > & GetProfiler() const
void RegisterDebugCallback(const DebugCallbackFunction &func)
RuntimeImpl(const IRuntime::CreationOptions &options)
Creates a runtime for workload execution.
const BackendIdSet & GetDynamicBackends() const
std::unique_ptr< IWorkingMemHandle > CreateWorkingMemHandle(NetworkId networkId)
Create a new unique WorkingMemHandle object.
void ClearDynamicBackends()
std::unique_ptr< IWorkingMemHandle > CreateWorkingMemHandle(NetworkId networkId)
Create a new unique WorkingMemHandle object.
static BackendIdSet RegisterDynamicBackends(const std::vector< DynamicBackendPtr > &dynamicBackends)
void AddBackendProfilingContext(const BackendId backendId, std::shared_ptr< armnn::profiling::IBackendProfilingContext > profilingContext)
ExternalProfilingOptions m_ProfilingOptions
ProfilingState ConfigureProfilingService(const ExternalProfilingOptions &options, bool resetProfilingService=false)
static std::unique_ptr< LoadedNetwork > MakeLoadedNetwork(std::unique_ptr< IOptimizedNetwork > net, std::string &errorMessage, const INetworkProperties &networkProperties, profiling::ProfilingService &profilingService)
Status EnqueueWorkload(NetworkId networkId, const InputTensors &inputTensors, const OutputTensors &outputTensors)
Evaluates a network using input in inputTensors and outputs filled into outputTensors.
Class for non-fatal exceptions raised while initialising a backend.