18 #include <boost/polymorphic_cast.hpp> 21 using namespace armnn;
39 delete boost::polymorphic_downcast<Runtime*>(runtime);
42 int Runtime::GenerateNetworkId()
44 return m_NetworkIdCounter++;
49 std::string ignoredErrorMessage;
50 return LoadNetwork(networkIdOut, std::move(inNetwork), ignoredErrorMessage);
55 std::string& errorMessage)
58 return LoadNetwork(networkIdOut, std::move(inNetwork), errorMessage, networkProperties);
63 std::string& errorMessage,
68 networkIdOut = GenerateNetworkId();
70 for (
auto&& context : m_BackendContexts)
72 context.second->BeforeLoadNetwork(networkIdOut);
76 std::unique_ptr<OptimizedNetwork>(boost::polymorphic_downcast<OptimizedNetwork*>(rawNetwork)),
86 std::lock_guard<std::mutex> lockGuard(m_Mutex);
89 m_LoadedNetworks[networkIdOut] = std::move(loadedNetwork);
92 for (
auto&& context : m_BackendContexts)
94 context.second->AfterLoadNetwork(networkIdOut);
107 bool unloadOk =
true;
108 for (
auto&& context : m_BackendContexts)
110 unloadOk &= context.second->BeforeUnloadNetwork(networkId);
116 "network with ID:" << networkId <<
" because BeforeUnloadNetwork failed";
121 std::lock_guard<std::mutex> lockGuard(m_Mutex);
123 if (m_LoadedNetworks.erase(networkId) == 0)
125 ARMNN_LOG(
warning) <<
"WARNING: Runtime::UnloadNetwork(): " << networkId <<
" not found!";
134 for (
auto&& context : m_BackendContexts)
136 context.second->AfterUnloadNetwork(networkId);
139 ARMNN_LOG(
debug) <<
"Runtime::UnloadNetwork(): Unloaded network with ID: " << networkId;
145 auto it = m_LoadedNetworks.find(networkId);
146 if (it != m_LoadedNetworks.end())
148 auto& loadedNetwork = it->second;
149 return loadedNetwork->GetProfiler();
156 : m_NetworkIdCounter(0)
173 auto backend = factoryFun();
174 BOOST_ASSERT(backend.get() !=
nullptr);
176 auto context = backend->CreateBackendContext(options);
182 m_BackendContexts.emplace(std::make_pair(
id, std::move(context)));
184 supportedBackends.emplace(
id);
186 unique_ptr<armnn::profiling::IBackendProfiling> profilingIface =
191 auto profilingContext = backend->CreateBackendProfilingContext(options, profilingIface);
193 if (profilingContext)
196 if(profilingContext->EnableProfiling(
true))
218 std::vector<int> networkIDs;
222 std::transform(m_LoadedNetworks.begin(), m_LoadedNetworks.end(),
223 std::back_inserter(networkIDs),
224 [](
const auto &pair) {
return pair.first; });
226 catch (
const std::exception& e)
231 std::cerr <<
"WARNING: An error has occurred when getting the IDs of the networks to unload: " << e.what()
232 <<
"\nSome of the loaded networks may not be unloaded" << std::endl;
237 for (
auto networkID : networkIDs)
245 catch (
const std::exception& e)
250 std::cerr <<
"WARNING: An error has occurred when unloading network " << networkID <<
": " << e.what()
259 m_BackendContexts.clear();
264 std::lock_guard<std::mutex> lockGuard(m_Mutex);
265 return m_LoadedNetworks.at(networkId).get();
283 LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
285 static thread_local
NetworkId lastId = networkId;
286 if (lastId != networkId)
300 LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
304 void Runtime::LoadDynamicBackends(
const std::string& overrideBackendPath)
void AddSupportedBackends(const BackendIdSet &backendIds, bool isDynamic=false)
#define ARMNN_VERSION
ARMNN_VERSION: "YYYYMMPP" where: YYYY = 4-digit year number MM = 2-digit month number PP = 2-digit pa...
virtual TensorInfo GetOutputTensorInfo(NetworkId networkId, LayerBindingId layerId) const override
static IRuntimePtr Create(const CreationOptions &options)
static ProfilingService & Instance()
FactoryFunction GetFactory(const BackendId &id) const
static void DeregisterDynamicBackends(const BackendIdSet &dynamicBackends)
std::unordered_set< BackendId > BackendIdSet
std::unique_ptr< IRuntime, void(*)(IRuntime *runtime)> IRuntimePtr
TensorInfo GetInputTensorInfo(LayerBindingId layerId) const
#define ARMNN_LOG(severity)
BackendRegistry & BackendRegistryInstance()
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
static std::vector< DynamicBackendPtr > CreateDynamicBackends(const std::vector< std::string > &sharedObjects)
TensorInfo GetOutputTensorInfo(LayerBindingId layerId) const
Copyright (c) 2020 ARM Limited.
uint32_t IncrementCounterValue(uint16_t counterUid) override
std::function< void(LayerGuid guid, unsigned int slotIndex, ITensorHandle *tensorHandle)> DebugCallbackFunction
Define the type of callback for the Debug layer to call.
virtual const std::shared_ptr< IProfiler > GetProfiler(NetworkId networkId) const override
Gets the profiler corresponding to the given network id.
static std::vector< std::string > GetBackendPaths(const std::string &overrideBackendPath="")
int LayerBindingId
Type of identifiers for bindable layers (inputs, outputs).
static void Destroy(IRuntime *runtime)
static std::unique_ptr< LoadedNetwork > MakeLoadedNetwork(std::unique_ptr< OptimizedNetwork > net, std::string &errorMessage, const INetworkProperties &networkProperties)
virtual void RegisterDebugCallback(NetworkId networkId, const DebugCallbackFunction &func) override
Registers a callback function to debug layers performing custom computations on intermediate tensors...
static std::vector< std::string > GetSharedObjects(const std::vector< std::string > &backendPaths)
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
std::unique_ptr< IOptimizedNetwork, void(*)(IOptimizedNetwork *network)> IOptimizedNetworkPtr
static IRuntime * CreateRaw(const CreationOptions &options)
Status EnqueueWorkload(const InputTensors &inputTensors, const OutputTensors &outputTensors)
std::string m_DynamicBackendsPath
Setting this value will override the paths set by the DYNAMIC_BACKEND_PATHS compiler directive Only a...
virtual Status LoadNetwork(NetworkId &networkIdOut, IOptimizedNetworkPtr network) override
Loads a complete network into the Runtime.
void RegisterDebugCallback(const DebugCallbackFunction &func)
const BackendIdSet & GetDynamicBackends() const
void ClearDynamicBackends()
virtual Status UnloadNetwork(NetworkId networkId) override
Unloads a network from the Runtime.
static BackendIdSet RegisterDynamicBackends(const std::vector< DynamicBackendPtr > &dynamicBackends)
void AddBackendProfilingContext(const BackendId backendId, std::shared_ptr< armnn::profiling::IBackendProfilingContext > profilingContext)
armnn::Runtime::CreationOptions::ExternalProfilingOptions options
ExternalProfilingOptions m_ProfilingOptions
Runtime(const CreationOptions &options)
Creates a runtime for workload execution.
virtual Status EnqueueWorkload(NetworkId networkId, const InputTensors &inputTensors, const OutputTensors &outputTensors) override
Evaluates a network using input in inputTensors and outputs filled into outputTensors.
ProfilingState ConfigureProfilingService(const ExternalProfilingOptions &options, bool resetProfilingService=false)
virtual TensorInfo GetInputTensorInfo(NetworkId networkId, LayerBindingId layerId) const override
Class for non-fatal exceptions raised while initialising a backend.