+ +

#include <Runtime.hpp>

+

+Inheritance diagram for Runtime:

+

+ + + +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+Public Member Functions
virtual Status	LoadNetwork (NetworkId &networkIdOut, IOptimizedNetworkPtr network) override
	Loads a complete network into the Runtime. More...

virtual Status	LoadNetwork (NetworkId &networkIdOut, IOptimizedNetworkPtr network, std::string &errorMessage) override
	Load a complete network into the IRuntime. More...

virtual Status	LoadNetwork (NetworkId &networkIdOut, IOptimizedNetworkPtr network, std::string &errorMessage, const INetworkProperties &networkProperties) override

virtual TensorInfo	GetInputTensorInfo (NetworkId networkId, LayerBindingId layerId) const override

virtual TensorInfo	GetOutputTensorInfo (NetworkId networkId, LayerBindingId layerId) const override

virtual Status	EnqueueWorkload (NetworkId networkId, const InputTensors &inputTensors, const OutputTensors &outputTensors) override
	Evaluates a network using input in inputTensors and outputs filled into outputTensors. More...

virtual Status	UnloadNetwork (NetworkId networkId) override
	Unloads a network from the Runtime. More...

virtual const IDeviceSpec &	GetDeviceSpec () const override

virtual const std::shared_ptr< IProfiler >	GetProfiler (NetworkId networkId) const override
	Gets the profiler corresponding to the given network id. More...

virtual void	RegisterDebugCallback (NetworkId networkId, const DebugCallbackFunction &func) override
	Registers a callback function to debug layers performing custom computations on intermediate tensors. More...

	Runtime (const CreationOptions &options)
	Creates a runtime for workload execution. More...

	~Runtime ()

+ + + +

+Friends
void	RuntimeLoadedNetworksReserve (armnn::Runtime *runtime)

+ + + + + + + + + + + +

+Additional Inherited Members
Static Public Member Functions inherited from IRuntime
static IRuntime *	CreateRaw (const CreationOptions &options)

static IRuntimePtr	Create (const CreationOptions &options)

static void	Destroy (IRuntime *runtime)

Protected Member Functions inherited from IRuntime
	~IRuntime ()

+

Detailed Description

+

Definition at line 23 of file Runtime.hpp.

+

Constructor & Destructor Documentation

+ +

◆ Runtime()

+ +

+

+ + + + + + + + +

Runtime ( const CreationOptions & options )

+

+ +

Creates a runtime for workload execution.

+ +

Definition at line 155 of file Runtime.cpp.

+ +

References ProfilingService::AddBackendProfilingContext(), DeviceSpec::AddSupportedBackends(), ARMNN_LOG, ARMNN_VERSION, armnn::BackendRegistryInstance(), ProfilingService::ConfigureProfilingService(), BackendRegistry::GetFactory(), armnn::info, ProfilingService::Instance(), IRuntime::CreationOptions::m_DynamicBackendsPath, and IRuntime::CreationOptions::m_ProfilingOptions.

+ +

Referenced by Runtime::GetDeviceSpec().

+

     : m_NetworkIdCounter(0)
 {
     ARMNN_LOG(info) << "ArmNN v" << ARMNN_VERSION << "\n";
 
     // pass configuration info to the profiling service
     armnn::profiling::ProfilingService::Instance().ConfigureProfilingService(options.m_ProfilingOptions);
 
     // Load any available/compatible dynamic backend before the runtime
     // goes through the backend registry
     LoadDynamicBackends(options.m_DynamicBackendsPath);
 
     BackendIdSet supportedBackends;
     for (const auto& id : BackendRegistryInstance().GetBackendIds())
     {
         // Store backend contexts for the supported ones
         try {
             auto factoryFun = BackendRegistryInstance().GetFactory(id);
             auto backend = factoryFun();
             BOOST_ASSERT(backend.get() != nullptr);
 
             auto context = backend->CreateBackendContext(options);
 
             // backends are allowed to return nullptrs if they
             // don't wish to create a backend specific context
             if (context)
             {
                 m_BackendContexts.emplace(std::make_pair(id, std::move(context)));
             }
             supportedBackends.emplace(id);
 
             unique_ptr<armnn::profiling::IBackendProfiling> profilingIface =
                 std::make_unique<armnn::profiling::BackendProfiling>(armnn::profiling::BackendProfiling(
                     options, armnn::profiling::ProfilingService::Instance(), id));
 
             // Backends may also provide a profiling context. Ask for it now.
             auto profilingContext = backend->CreateBackendProfilingContext(options, profilingIface);
             // Backends that don't support profiling will return a null profiling context.
             if (profilingContext)
             {
                 // Enable profiling on the backend and assert that it returns true
                 if(profilingContext->EnableProfiling(true))
                 {
                     // Pass the context onto the profiling service.
                     armnn::profiling::ProfilingService::Instance().AddBackendProfilingContext(id, profilingContext);
                 }
                 else
                 {
                     throw BackendProfilingException("Unable to enable profiling on Backend Id: " + id.Get());
                 }
             }
         }
         catch (const BackendUnavailableException&)
         {
             // Ignore backends which are unavailable
         }
 
     }
     m_DeviceSpec.AddSupportedBackends(supportedBackends);
 }

+
+
+
+
+
+
+
+
+
+
+
+
+
+

+

+ +

◆ ~Runtime()

+ +

+

+ + + + + + + +

~Runtime ( )

+

+ +

Definition at line 216 of file Runtime.cpp.

+ +

References DeviceSpec::ClearDynamicBackends(), DynamicBackendUtils::DeregisterDynamicBackends(), DeviceSpec::GetDynamicBackends(), and Runtime::UnloadNetwork().

+ +

Referenced by Runtime::GetDeviceSpec().

+

 {
     std::vector<int> networkIDs;
     try
     {
         // Coverity fix: The following code may throw an exception of type std::length_error.
         std::transform(m_LoadedNetworks.begin(), m_LoadedNetworks.end(),
                        std::back_inserter(networkIDs),
                        [](const auto &pair) { return pair.first; });
     }
     catch (const std::exception& e)
     {
         // Coverity fix: BOOST_LOG_TRIVIAL (typically used to report errors) may throw an
         // exception of type std::length_error.
         // Using stderr instead in this context as there is no point in nesting try-catch blocks here.
         std::cerr << "WARNING: An error has occurred when getting the IDs of the networks to unload: " << e.what()
                   << "\nSome of the loaded networks may not be unloaded" << std::endl;
     }
     // We then proceed to unload all the networks which IDs have been appended to the list
     // up to the point the exception was thrown (if any).
 
     for (auto networkID : networkIDs)
     {
         try
         {
             // Coverity fix: UnloadNetwork() may throw an exception of type std::length_error,
             // boost::log::v2s_mt_posix::odr_violation or boost::log::v2s_mt_posix::system_error
             UnloadNetwork(networkID);
         }
         catch (const std::exception& e)
         {
             // Coverity fix: BOOST_LOG_TRIVIAL (typically used to report errors) may throw an
             // exception of type std::length_error.
             // Using stderr instead in this context as there is no point in nesting try-catch blocks here.
             std::cerr << "WARNING: An error has occurred when unloading network " << networkID << ": " << e.what()
                       << std::endl;
         }
     }
 
 
     // Clear all dynamic backends.
     DynamicBackendUtils::DeregisterDynamicBackends(m_DeviceSpec.GetDynamicBackends());
     m_DeviceSpec.ClearDynamicBackends();
     m_BackendContexts.clear();
 }

+
+
+
+

+

Member Function Documentation

+ +

◆ EnqueueWorkload()

+ +

+

+ + + + + +

+ + + + + + + + + + + + + + + + + + + + + + + + +

Status EnqueueWorkload	(	NetworkId	networkId,
		const InputTensors &	inputTensors,
		const OutputTensors &	outputTensors
	)

+

+overridevirtual

+

+ +

Evaluates a network using input in inputTensors and outputs filled into outputTensors.

+ +

Implements IRuntime.

+ +

Definition at line 279 of file Runtime.cpp.

+ +

References LoadedNetwork::EnqueueWorkload(), and LoadedNetwork::FreeWorkingMemory().

+

 {
     LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
 
     static thread_local NetworkId lastId = networkId;
     if (lastId != networkId)
     {
         LoadedNetworkFuncSafe(lastId, [](LoadedNetwork* network)
             {
                 network->FreeWorkingMemory();
             });
     }
     lastId=networkId;
 
     return loadedNetwork->EnqueueWorkload(inputTensors, outputTensors);
 }

+
+
+
+

+

+ +

◆ GetDeviceSpec()

+ +

+

+ + + + + +

+ + + + + + + +

virtual const IDeviceSpec& GetDeviceSpec ( ) const

+

+inlineoverridevirtual

+

+ +

Implements IRuntime.

+ +

Definition at line 63 of file Runtime.hpp.

+ +

References Runtime::GetProfiler(), options, Runtime::RegisterDebugCallback(), Runtime::Runtime(), Runtime::RuntimeLoadedNetworksReserve, and Runtime::~Runtime().

+ +

Referenced by BOOST_AUTO_TEST_CASE().

+

63 { return m_DeviceSpec; }

+

+ +

◆ GetInputTensorInfo()

+ +

+

+ + + + + +

+ + + + + + + + + + + + + + + + + + +

TensorInfo GetInputTensorInfo	(	NetworkId	networkId,
		LayerBindingId	layerId
	)		const

+

+overridevirtual

+

+ +

Implements IRuntime.

+ +

Definition at line 268 of file Runtime.cpp.

+ +

References LoadedNetwork::GetInputTensorInfo().

+

 {
     return GetLoadedNetworkPtr(networkId)->GetInputTensorInfo(layerId);
 }

+

+

+ +

◆ GetOutputTensorInfo()

+ +

+

+ + + + + +

+ + + + + + + + + + + + + + + + + + +

TensorInfo GetOutputTensorInfo	(	NetworkId	networkId,
		LayerBindingId	layerId
	)		const

+

+overridevirtual

+

+ +

Implements IRuntime.

+ +

Definition at line 273 of file Runtime.cpp.

+ +

References LoadedNetwork::GetOutputTensorInfo().

+

 {
     return GetLoadedNetworkPtr(networkId)->GetOutputTensorInfo(layerId);
 }

+

+

+ +

◆ GetProfiler()

+ +

+

+ + + + + +

+ + + + + + + + +

const std::shared_ptr< IProfiler > GetProfiler ( NetworkId networkId ) const

+

+overridevirtual

+

+ +

Gets the profiler corresponding to the given network id.

+

Parameters

+ + +

networkId The id of the network for which to get the profile.

+

Returns: A pointer to the requested profiler, or nullptr if not found.

+ +

Implements IRuntime.

+ +

Definition at line 143 of file Runtime.cpp.

+ +

Referenced by Runtime::GetDeviceSpec().

+

 {
     auto it = m_LoadedNetworks.find(networkId);
     if (it != m_LoadedNetworks.end())
     {
         auto& loadedNetwork = it->second;
         return loadedNetwork->GetProfiler();
     }
 
     return nullptr;
 }

+

+ +

◆ LoadNetwork() [1/3]

+ +

+

+ + + + + +

+ + + + + + + + + + + + + + + + + + +

Status LoadNetwork	(	NetworkId &	networkIdOut,
		IOptimizedNetworkPtr	network
	)

+

+overridevirtual

+

+ +

Loads a complete network into the Runtime.

+

Parameters

+ + + +

[out]	networkIdOut	- Unique identifier for the network is returned in this reference.
[in]	network	- Complete network to load into the Runtime. The runtime takes ownership of the network once passed in.

+

Returns: armnn::Status

+ +

Implements IRuntime.

+ +

Definition at line 47 of file Runtime.cpp.

+ +

Referenced by BOOST_AUTO_TEST_CASE().

+

 {
     std::string ignoredErrorMessage;
     return LoadNetwork(networkIdOut, std::move(inNetwork), ignoredErrorMessage);
 }

+

+

+ +

◆ LoadNetwork() [2/3]

+ +

+

+ + + + + +

+ + + + + + + + + + + + + + + + + + + + + + + + +

Status LoadNetwork	(	NetworkId &	networkIdOut,
		IOptimizedNetworkPtr	network,
		std::string &	errorMessage
	)

+

+overridevirtual

+

+ +

Load a complete network into the IRuntime.

+

Parameters

+ + + + +

[out]	networkIdOut	Unique identifier for the network is returned in this reference.
[in]	network	Complete network to load into the IRuntime.
[out]	errorMessage	Error message if there were any errors. The runtime takes ownership of the network once passed in.

+

Returns: armnn::Status

+ +

Implements IRuntime.

+ +

Definition at line 53 of file Runtime.cpp.

+

 {
     INetworkProperties networkProperties;
     return LoadNetwork(networkIdOut, std::move(inNetwork), errorMessage, networkProperties);
 }

+
+

+

+ +

◆ LoadNetwork() [3/3]

+ +

+

+ + + + + +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

Status LoadNetwork	(	NetworkId &	networkIdOut,
		IOptimizedNetworkPtr	network,
		std::string &	errorMessage,
		const INetworkProperties &	networkProperties
	)

+

+overridevirtual

+

+ +

Implements IRuntime.

+ +

Definition at line 61 of file Runtime.cpp.

+ +

References armnn::Failure, ProfilingService::IncrementCounterValue(), ProfilingService::Instance(), LoadedNetwork::MakeLoadedNetwork(), and armnn::Success.

+

 {
     IOptimizedNetwork* rawNetwork = inNetwork.release();
 
     networkIdOut = GenerateNetworkId();
 
     for (auto&& context : m_BackendContexts)
     {
         context.second->BeforeLoadNetwork(networkIdOut);
     }
 
     unique_ptr<LoadedNetwork> loadedNetwork = LoadedNetwork::MakeLoadedNetwork(
         std::unique_ptr<OptimizedNetwork>(boost::polymorphic_downcast<OptimizedNetwork*>(rawNetwork)),
         errorMessage,
         networkProperties);
 
     if (!loadedNetwork)
     {
         return Status::Failure;
     }
 
     {
         std::lock_guard<std::mutex> lockGuard(m_Mutex);
 
         // Stores the network
         m_LoadedNetworks[networkIdOut] = std::move(loadedNetwork);
     }
 
     for (auto&& context : m_BackendContexts)
     {
         context.second->AfterLoadNetwork(networkIdOut);
     }
 
     if (profiling::ProfilingService::Instance().IsProfilingEnabled())
     {
         profiling::ProfilingService::Instance().IncrementCounterValue(armnn::profiling::NETWORK_LOADS);
     }
 
     return Status::Success;
 }

+
+
+
+
+
+

+

+ +

◆ RegisterDebugCallback()

+ +

+

+ + + + + +

+ + + + + + + + + + + + + + + + + + +

void RegisterDebugCallback	(	NetworkId	networkId,
		const DebugCallbackFunction &	func
	)

+

+overridevirtual

+

+ +

Registers a callback function to debug layers performing custom computations on intermediate tensors.

+

Parameters

+ + + +

networkId	The id of the network to register the callback.
func	callback function to pass to the debug layer.

+

+ +

Implements IRuntime.

+ +

Definition at line 298 of file Runtime.cpp.

+ +

References DeviceSpec::AddSupportedBackends(), DynamicBackendUtils::CreateDynamicBackends(), DynamicBackendUtils::GetBackendPaths(), DynamicBackendUtils::GetSharedObjects(), LoadedNetwork::RegisterDebugCallback(), and DynamicBackendUtils::RegisterDynamicBackends().

+ +

Referenced by Runtime::GetDeviceSpec().

+

 {
     LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
     loadedNetwork->RegisterDebugCallback(func);
 }

+
+

+

+ +

◆ UnloadNetwork()

+ +

+

+ + + + + +

+ + + + + + + + +

Status UnloadNetwork ( NetworkId networkId )

+

+overridevirtual

+

+ +

Unloads a network from the Runtime.

+

At the moment this only removes the network from the m_Impl->m_Network. This might need more work in the future to be AndroidNN compliant.

Parameters

+ + +

[in] networkId Unique identifier for the network to be unloaded. Generated in LoadNetwork().

+

Returns: armnn::Status

+ +

Implements IRuntime.

+ +

Definition at line 105 of file Runtime.cpp.

+ +

References ARMNN_LOG, armnn::debug, armnn::Failure, ProfilingService::IncrementCounterValue(), ProfilingService::Instance(), armnn::Success, and armnn::warning.

+ +

Referenced by BOOST_AUTO_TEST_CASE(), and Runtime::~Runtime().

+

 {
     bool unloadOk = true;
     for (auto&& context : m_BackendContexts)
     {
         unloadOk &= context.second->BeforeUnloadNetwork(networkId);
     }
 
     if (!unloadOk)
     {
         ARMNN_LOG(warning) << "Runtime::UnloadNetwork(): failed to unload "
                               "network with ID:" << networkId << " because BeforeUnloadNetwork failed";
         return Status::Failure;
     }
 
     {
         std::lock_guard<std::mutex> lockGuard(m_Mutex);
 
         if (m_LoadedNetworks.erase(networkId) == 0)
         {
             ARMNN_LOG(warning) << "WARNING: Runtime::UnloadNetwork(): " << networkId << " not found!";
             return Status::Failure;
         }
         if (profiling::ProfilingService::Instance().IsProfilingEnabled())
         {
             profiling::ProfilingService::Instance().IncrementCounterValue(armnn::profiling::NETWORK_UNLOADS);
         }
     }
 
     for (auto&& context : m_BackendContexts)
     {
         context.second->AfterUnloadNetwork(networkId);
     }
 
     ARMNN_LOG(debug) << "Runtime::UnloadNetwork(): Unloaded network with ID: " << networkId;
     return Status::Success;
 }

+
+
+
+
+
+
+

+

Friends And Related Function Documentation

+ +

◆ RuntimeLoadedNetworksReserve

+ +

+

+ + + + + +

+ + + + + + + + +

void RuntimeLoadedNetworksReserve ( armnn::Runtime * runtime )

+

+friend

+

+ +

Definition at line 28 of file RuntimeTests.cpp.

+ +

Referenced by Runtime::GetDeviceSpec().

+

 {
     runtime->m_LoadedNetworks.reserve(1);
 }

+

The documentation for this class was generated from the following files:

src/armnn/Runtime.hpp
src/armnn/Runtime.cpp

+

+Public Member Functions

+Friends

+Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ Runtime()

◆ ~Runtime()

Member Function Documentation

◆ EnqueueWorkload()

◆ GetDeviceSpec()

◆ GetInputTensorInfo()

◆ GetOutputTensorInfo()

◆ GetProfiler()

◆ LoadNetwork() [1/3]

◆ LoadNetwork() [2/3]

◆ LoadNetwork() [3/3]

◆ RegisterDebugCallback()

◆ UnloadNetwork()

Friends And Related Function Documentation

◆ RuntimeLoadedNetworksReserve