26 #include <client/include/backends/IBackendProfiling.hpp> 28 #include <common/include/LabelsAndEventClasses.hpp> 33 using namespace armnn;
61 return pRuntimeImpl->LoadNetwork(networkIdOut, std::move(network));
66 std::string& errorMessage)
68 return pRuntimeImpl->LoadNetwork(networkIdOut, std::move(network), errorMessage);
73 std::string& errorMessage,
76 return pRuntimeImpl->LoadNetwork(networkIdOut, std::move(network), errorMessage, networkProperties);
81 return pRuntimeImpl->GetInputTensorInfo(networkId, layerId);
86 return pRuntimeImpl->GetOutputTensorInfo(networkId, layerId);
92 return pRuntimeImpl->ImportInputs(networkId, inputTensors, forceImportMemorySource);
98 return pRuntimeImpl->ImportOutputs(networkId, outputTensors, forceImportMemorySource);
103 return pRuntimeImpl->ClearImportedInputs(networkId, inputIds);
107 return pRuntimeImpl->ClearImportedOutputs(networkId, outputIds);
113 std::vector<ImportedInputId> preImportedInputIds,
114 std::vector<ImportedOutputId> preImportedOutputIds)
116 return pRuntimeImpl->EnqueueWorkload(networkId, inputTensors, outputTensors,
117 preImportedInputIds, preImportedOutputIds);
123 std::vector<ImportedInputId> preImportedInputs,
124 std::vector<ImportedOutputId> preImportedOutputs)
155 return pRuntimeImpl->RegisterDebugCallback(networkId, func);
158 int RuntimeImpl::GenerateNetworkId()
160 return m_NetworkIdCounter++;
165 std::string ignoredErrorMessage;
166 return LoadNetwork(networkIdOut, std::move(inNetwork), ignoredErrorMessage);
171 std::string& errorMessage)
175 return LoadNetwork(networkIdOut, std::move(inNetwork), errorMessage, networkProperties);
180 std::string& errorMessage,
184 auto profiler = inNetwork->GetProfiler();
189 networkIdOut = GenerateNetworkId();
191 for (
auto&& context : m_BackendContexts)
193 context.second->BeforeLoadNetwork(networkIdOut);
197 std::unique_ptr<IOptimizedNetwork>(rawNetwork),
200 m_ProfilingService.get());
208 #if !defined(ARMNN_DISABLE_THREADS) 209 std::lock_guard<std::mutex> lockGuard(m_Mutex);
213 m_LoadedNetworks[networkIdOut] = std::move(loadedNetwork);
216 for (
auto&& context : m_BackendContexts)
218 context.second->AfterLoadNetwork(networkIdOut);
221 if (m_ProfilingService->IsProfilingEnabled())
223 m_ProfilingService->IncrementCounterValue(arm::pipe::NETWORK_LOADS);
231 bool unloadOk =
true;
232 for (
auto&& context : m_BackendContexts)
234 unloadOk &= context.second->BeforeUnloadNetwork(networkId);
240 "network with ID:" << networkId <<
" because BeforeUnloadNetwork failed";
244 std::unique_ptr<arm::pipe::TimelineUtilityMethods> timelineUtils =
245 arm::pipe::TimelineUtilityMethods::GetTimelineUtils(*m_ProfilingService.get());
247 #if !defined(ARMNN_DISABLE_THREADS) 248 std::lock_guard<std::mutex> lockGuard(m_Mutex);
254 auto search = m_LoadedNetworks.find(networkId);
255 if (search != m_LoadedNetworks.end())
257 arm::pipe::ProfilingGuid networkGuid = search->second->GetNetworkGuid();
258 timelineUtils->RecordEvent(networkGuid,
259 arm::pipe::LabelsAndEventClasses::ARMNN_PROFILING_EOL_EVENT_CLASS);
263 if (m_LoadedNetworks.erase(networkId) == 0)
265 ARMNN_LOG(
warning) <<
"WARNING: RuntimeImpl::UnloadNetwork(): " << networkId <<
" not found!";
269 if (m_ProfilingService->IsProfilingEnabled())
271 m_ProfilingService->IncrementCounterValue(arm::pipe::NETWORK_UNLOADS);
275 for (
auto&& context : m_BackendContexts)
277 context.second->AfterUnloadNetwork(networkId);
283 ARMNN_LOG(
debug) <<
"RuntimeImpl::UnloadNetwork(): Unloaded network with ID: " << networkId;
289 auto it = m_LoadedNetworks.find(networkId);
290 if (it != m_LoadedNetworks.end())
292 auto& loadedNetwork = it->second;
293 return loadedNetwork->GetProfiler();
301 if (profilingService.IsProfilingEnabled())
303 LoadedNetworks::iterator it = m_LoadedNetworks.begin();
304 while (it != m_LoadedNetworks.end())
306 auto& loadedNetwork = it->second;
307 loadedNetwork->SendNetworkStructure(profilingService);
321 : m_NetworkIdCounter(0)
323 m_ProfilingService = arm::pipe::IProfilingService::CreateProfilingService(
324 arm::pipe::MAX_ARMNN_COUNTER,
326 arm::pipe::ARMNN_SOFTWARE_INFO,
327 arm::pipe::ARMNN_SOFTWARE_VERSION,
328 arm::pipe::ARMNN_HARDWARE_VERSION,
335 "It is not possible to enable timeline reporting without profiling being enabled");
349 auto backend = factoryFun();
355 customAllocatorMapIterator->second ==
nullptr)
360 throw armnn::Exception(
"Allocator associated with id " +
id.Get() +
" is null");
368 BackendCapability protectedContentCapability {
"ProtectedContentAllocation",
true};
375 <<
" is not registered as does not support protected content allocation.";
383 if (customAllocatorMapIterator->second->GetMemorySourceType()
386 if (!backend->UseCustomMemoryAllocator(customAllocatorMapIterator->second, err))
390 <<
" reported an error when entering protected mode. Backend won't be" 391 <<
" used. ErrorMsg: " << err;
399 ARMNN_LOG(
error) <<
"The CustomAllocator provided with the runtime options doesn't support " 400 "protected memory. Protected mode can't be activated. The backend " 402 <<
" is not going to be used. MemorySource must be MemorySource::DmaBufProtected";
408 ARMNN_LOG(
error) <<
"Protected mode can't be activated for backend: " 410 <<
" no custom allocator was provided to the runtime options.";
420 if (!backend->UseCustomMemoryAllocator(customAllocatorMapIterator->second, err))
424 <<
" reported an error when trying to use the provided custom allocator." 425 " Backend won't be used." 426 <<
" ErrorMsg: " << err;
443 id, customMemoryOptimizerStrategyMapIterator->second);
446 << customMemoryOptimizerStrategyMapIterator->second->GetName()
447 <<
" set for the backend " <<
id <<
".";
453 std::string memoryOptimizerStrategyName =
"";
456 if (name ==
"MemoryOptimizerStrategy")
458 memoryOptimizerStrategyName = ParseStringBackendOption(value,
"");
461 if (memoryOptimizerStrategyName !=
"")
463 std::shared_ptr<IMemoryOptimizerStrategy> strategy =
468 ARMNN_LOG(
warning) <<
"MemoryOptimizerStrategy: " << memoryOptimizerStrategyName
469 <<
" was not found.";
481 << memoryOptimizerStrategyName <<
" set for the backend " <<
id <<
".";
487 <<
" does not have multi-axis packing capability and cannot support" 488 <<
"MemoryOptimizerStrategy: " << memoryOptimizerStrategyName <<
".";
494 auto context = backend->CreateBackendContext(options);
500 m_BackendContexts.emplace(std::make_pair(
id, std::move(context)));
502 supportedBackends.emplace(
id);
504 unique_ptr<arm::pipe::IBackendProfiling> profilingIface =
505 arm::pipe::IBackendProfiling::CreateBackendProfiling(
507 *m_ProfilingService.get(),
511 auto profilingContext = backend->CreateBackendProfilingContext(options, profilingIface);
513 if (profilingContext)
516 m_ProfilingService->AddBackendProfilingContext(
id, profilingContext);
527 m_ProfilingService->ConfigureProfilingService(
529 if (options.m_ProfilingOptions.m_EnableProfiling)
532 m_ProfilingService->WaitForProfilingServiceActivation(3000);
537 ARMNN_LOG(
info) <<
"Initialization time: " << std::setprecision(2)
544 std::vector<int> networkIDs;
548 std::transform(m_LoadedNetworks.begin(), m_LoadedNetworks.end(),
549 std::back_inserter(networkIDs),
550 [](
const auto &pair) {
return pair.first; });
552 catch (
const std::exception& e)
557 std::cerr <<
"WARNING: An error has occurred when getting the IDs of the networks to unload: " << e.what()
558 <<
"\nSome of the loaded networks may not be unloaded" << std::endl;
563 for (
auto networkID : networkIDs)
571 catch (
const std::exception& e)
576 std::cerr <<
"WARNING: An error has occurred when unloading network " << networkID <<
": " << e.what()
584 m_BackendContexts.clear();
593 #if !defined(ARMNN_DISABLE_THREADS) 594 std::lock_guard<std::mutex> lockGuard(m_Mutex);
596 return m_LoadedNetworks.at(networkId).get();
612 return GetLoadedNetworkPtr(networkId)->
ImportInputs(inputTensors, forceImportMemorySource);
618 return GetLoadedNetworkPtr(networkId)->
ImportOutputs(outputTensors, forceImportMemorySource);
633 std::vector<ImportedInputId> preImportedInputIds,
634 std::vector<ImportedOutputId> preImportedOutputIds)
638 LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
642 ARMNN_LOG(
error) <<
"A Network with an id of " << networkId <<
" does not exist.";
647 ARMNN_LOG(
error) <<
"Network " << networkId <<
" is async enabled.";
654 static thread_local
NetworkId lastId = networkId;
655 if (lastId != networkId)
664 auto status = loadedNetwork->
EnqueueWorkload(inputTensors, outputTensors,
665 preImportedInputIds, preImportedOutputIds);
669 if (!preImportedInputIds.empty() || !preImportedOutputIds.empty())
672 for (
auto&& context : m_BackendContexts)
674 context.second->AfterEnqueueWorkload(networkId);
677 ARMNN_LOG(
info) <<
"Execution time: " << std::setprecision(2)
685 std::vector<ImportedInputId> preImportedInputs,
686 std::vector<ImportedOutputId> preImportedOutputs)
691 LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
695 ARMNN_LOG(
error) <<
"A Network with an id of " << networkId <<
" does not exist.";
700 ARMNN_LOG(
error) <<
"Attempting execute " << networkId <<
" when it is not async enabled.";
707 auto status = loadedNetwork->
Execute(inputTensors,
713 ARMNN_LOG(
info) <<
"Execution time: " << std::setprecision(2)
723 LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
727 ARMNN_LOG(
error) <<
"A Network with an id of " << networkId <<
" does not exist.";
732 ARMNN_LOG(
error) <<
"Network " << networkId <<
" is not async enabled.";
739 static thread_local
NetworkId lastId = networkId;
740 if (lastId != networkId)
754 LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
758 void RuntimeImpl::LoadDynamicBackends(
const std::string& overrideBackendPath)
void AddSupportedBackends(const BackendIdSet &backendIds, bool isDynamic=false)
Status Execute(const InputTensors &inputTensors, const OutputTensors &outputTensors, IWorkingMemHandle &workingMemHandle, std::vector< ImportedInputId > preImportedInputs={}, std::vector< ImportedOutputId > preImportedOutputs={})
Thread safe execution of the loaded network.
armnn::TensorInfo GetInputTensorInfo(NetworkId networkId, LayerBindingId layerId) const
std::unique_ptr< IWorkingMemHandle > CreateWorkingMemHandle(NetworkId networkId)
Create a new unique WorkingMemHandle object.
bool HasCapability(const std::string &name, const BackendCapabilities &capabilities)
Convenience function to check if a capability exists in a BackendCapabilites struct.
static IRuntimePtr Create(const CreationOptions &options)
FactoryFunction GetFactory(const BackendId &id) const
const std::shared_ptr< IProfiler > GetProfiler(NetworkId networkId) const
Gets the profiler corresponding to the given network id.
std::chrono::duration< double, std::milli > GetTimeDuration(std::chrono::high_resolution_clock::time_point start_time)
Very basic type safe variant.
static void DeregisterDynamicBackends(const BackendIdSet &dynamicBackends)
virtual NetworkId GetNetworkId()=0
Returns the NetworkId of the Network that this IWorkingMemHandle works with.
std::unordered_set< BackendId > BackendIdSet
static ProfilerManager & GetInstance()
void ParseOptions(const std::vector< BackendOptions > &options, BackendId backend, F f)
Status Execute(IWorkingMemHandle &workingMemHandle, const InputTensors &inputTensors, const OutputTensors &outputTensors, std::vector< ImportedInputId > preImportedInputs, std::vector< ImportedOutputId > preImportedOutputs)
This is an experimental function.
#define ARMNN_VERSION
ARMNN_VERSION: "X.Y.Z" where: X = Major version number Y = Minor version number Z = Patch version num...
Status UnloadNetwork(NetworkId networkId)
Unloads a network from the Runtime.
void RegisterMemoryOptimizerStrategy(const BackendId &id, std::shared_ptr< IMemoryOptimizerStrategy > strategy)
std::unique_ptr< IRuntime, void(*)(IRuntime *runtime)> IRuntimePtr
TensorInfo GetInputTensorInfo(NetworkId networkId, LayerBindingId layerId) const
void RegisterAllocator(const BackendId &id, std::shared_ptr< ICustomAllocator > alloc)
TensorInfo GetInputTensorInfo(LayerBindingId layerId) const
#define ARMNN_LOG(severity)
BackendRegistry & BackendRegistryInstance()
Status LoadNetwork(NetworkId &networkIdOut, IOptimizedNetworkPtr network)
Loads a complete network into the Runtime.
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
static std::vector< DynamicBackendPtr > CreateDynamicBackends(const std::vector< std::string > &sharedObjects)
armnn::TensorInfo GetOutputTensorInfo(NetworkId networkId, LayerBindingId layerId) const
std::chrono::high_resolution_clock::time_point GetTimeNow()
std::map< BackendId, std::shared_ptr< IMemoryOptimizerStrategy > > m_MemoryOptimizerStrategyMap
A map to define a custom memory optimizer strategy for specific backend Ids.
TensorInfo GetOutputTensorInfo(LayerBindingId layerId) const
Copyright (c) 2021 ARM Limited and Contributors.
std::vector< ImportedInputId > ImportInputs(const InputTensors &inputTensors, MemorySource forceImportMemorySource=MemorySource::Undefined)
bool m_EnableProfiling
Indicates whether external profiling is enabled or not.
void RegisterDebugCallback(NetworkId networkId, const DebugCallbackFunction &func)
Registers a callback function to debug layers performing custom computations on intermediate tensors...
const std::shared_ptr< IProfiler > GetProfiler(NetworkId networkId) const
Gets the profiler corresponding to the given network id.
std::function< void(LayerGuid guid, unsigned int slotIndex, ITensorHandle *tensorHandle)> DebugCallbackFunction
Define the type of callback for the Debug layer to call.
static std::vector< std::string > GetBackendPaths(const std::string &overrideBackendPath="")
#define ARMNN_SCOPED_PROFILING_EVENT(backendId, name)
int LayerBindingId
Type of identifiers for bindable layers (inputs, outputs).
void InitialiseProfilingService(arm::pipe::IProfilingService &profilingService) override
static void Destroy(IRuntime *runtime)
void RegisterDebugCallback(NetworkId networkId, const DebugCallbackFunction &func)
Registers a callback function to debug layers performing custom computations on intermediate tensors...
std::vector< BackendOptions > m_BackendOptions
Pass backend specific options.
BackendCapability
BackendCapability class.
static std::vector< std::string > GetSharedObjects(const std::vector< std::string > &backendPaths)
Status UnloadNetwork(NetworkId networkId)
Unloads a network from the IRuntime.
void ClearImportedInputs(NetworkId networkId, const std::vector< ImportedInputId > inputIds)
Un-import and delete the imported InputTensor/s This function is not thread safe and must not be used...
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
void ClearImportedInputs(NetworkId networkId, const std::vector< ImportedInputId > inputIds)
std::map< BackendId, std::shared_ptr< ICustomAllocator > > m_CustomAllocatorMap
A map to define a custom memory allocator for specific backend Ids.
#define ARMNN_ASSERT(COND)
Status Execute(IWorkingMemHandle &workingMemHandle, const InputTensors &inputTensors, const OutputTensors &outputTensors, std::vector< ImportedInputId > preImportedInputs={}, std::vector< ImportedOutputId > preImportedOutputs={})
This is an experimental function.
std::unique_ptr< RuntimeImpl > pRuntimeImpl
ProfilingOptions ConvertExternalProfilingOptions(const armnn::IRuntime::CreationOptions::ExternalProfilingOptions &options)
Status EnqueueWorkload(NetworkId networkId, const InputTensors &inputTensors, const OutputTensors &outputTensors, std::vector< ImportedInputId > preImportedInputIds={}, std::vector< ImportedOutputId > preImportedOutputIds={})
Evaluates a network using input in inputTensors and outputs filled into outputTensors.
void ClearImportedInputs(const std::vector< ImportedInputId > inputIds)
Status LoadNetwork(NetworkId &networkIdOut, IOptimizedNetworkPtr network)
Loads a complete network into the IRuntime.
Device specific knowledge to be passed to the optimizer.
static IRuntime * CreateRaw(const CreationOptions &options)
std::unique_ptr< IMemoryOptimizerStrategy > GetMemoryOptimizerStrategy(const std::string &strategyName)
void ReportStructure(arm::pipe::IProfilingService &profilingService) override
const IDeviceSpec & GetDeviceSpec() const
bool m_ProtectedMode
Setting this flag will allow the user to create the Runtime in protected mode.
std::vector< ImportedInputId > ImportInputs(NetworkId networkId, const InputTensors &inputTensors, MemorySource forceImportMemorySource=MemorySource::Undefined)
ImportInputs separates the importing and mapping of InputTensors from network execution.
std::string m_DynamicBackendsPath
Setting this value will override the paths set by the DYNAMIC_BACKEND_PATHS compiler directive Only a...
TensorInfo GetOutputTensorInfo(NetworkId networkId, LayerBindingId layerId) const
Status EnqueueWorkload(const InputTensors &inputTensors, const OutputTensors &outputTensors, std::vector< ImportedInputId > preImportedInputIds={}, std::vector< ImportedOutputId > preImportedOutputIds={})
Single thread execution of the loaded network.
void InitialiseProfilingService(arm::pipe::IProfilingService &profilingService) override
EmptyOptional is used to initialize the Optional class in case we want to have default value for an O...
void RegisterProfiler(IProfiler *profiler)
Status EnqueueWorkload(NetworkId networkId, const InputTensors &inputTensors, const OutputTensors &outputTensors, std::vector< ImportedInputId > preImportedInputIds={}, std::vector< ImportedOutputId > preImportedOutputIds={})
Base class for all ArmNN exceptions so that users can filter to just those.
const std::shared_ptr< IProfiler > & GetProfiler() const
std::vector< ImportedOutputId > ImportOutputs(const OutputTensors &outputTensors, MemorySource forceImportMemorySource=MemorySource::Undefined)
MemorySource
Define the Memory Source to reduce copies.
void RegisterDebugCallback(const DebugCallbackFunction &func)
constexpr const char * GetMemBlockStrategyTypeName(MemBlockStrategyType memBlockStrategyType)
std::vector< ImportedInputId > ImportInputs(NetworkId networkId, const InputTensors &inputTensors, MemorySource forceImportMemorySource=MemorySource::Undefined)
RuntimeImpl(const IRuntime::CreationOptions &options)
Creates a runtime for workload execution.
const BackendIdSet & GetDynamicBackends() const
std::unique_ptr< IWorkingMemHandle > CreateWorkingMemHandle(NetworkId networkId)
Create a new unique WorkingMemHandle object.
bool m_TimelineEnabled
Indicates whether external timeline profiling is enabled or not.
void ClearDynamicBackends()
std::vector< ImportedOutputId > ImportOutputs(NetworkId networkId, const OutputTensors &outputTensors, MemorySource forceImportMemorySource=MemorySource::Undefined)
ImportOutputs separates the importing and mapping of OutputTensors from network execution.
void ClearImportedOutputs(NetworkId networkId, const std::vector< ImportedOutputId > outputIds)
Un-import and delete the imported OutputTensor/s This function is not thread safe and must not be use...
std::unique_ptr< IWorkingMemHandle > CreateWorkingMemHandle(NetworkId networkId)
Create a new unique WorkingMemHandle object.
static BackendIdSet RegisterDynamicBackends(const std::vector< DynamicBackendPtr > &dynamicBackends)
ExternalProfilingOptions m_ProfilingOptions
std::vector< ImportedOutputId > ImportOutputs(NetworkId networkId, const OutputTensors &outputTensors, MemorySource forceImportMemorySource=MemorySource::Undefined)
void SetProfilingService(armnn::Optional< arm::pipe::IProfilingService &> profilingService)
static std::unique_ptr< LoadedNetwork > MakeLoadedNetwork(std::unique_ptr< IOptimizedNetwork > net, std::string &errorMessage, const INetworkProperties &networkProperties, arm::pipe::IProfilingService *profilingService)
void ClearImportedOutputs(NetworkId networkId, const std::vector< ImportedOutputId > outputIds)
void ClearImportedOutputs(const std::vector< ImportedOutputId > outputIds)
Class for non-fatal exceptions raised while initialising a backend.