Runtime Class Reference

#include <Runtime.hpp>

Inheritance diagram for Runtime:

Public Member Functions
virtual Status	LoadNetwork (NetworkId &networkIdOut, IOptimizedNetworkPtr network) override
	Loads a complete network into the Runtime. More...
virtual Status	LoadNetwork (NetworkId &networkIdOut, IOptimizedNetworkPtr network, std::string &errorMessage) override
	Load a complete network into the IRuntime. More...
virtual Status	LoadNetwork (NetworkId &networkIdOut, IOptimizedNetworkPtr network, std::string &errorMessage, const INetworkProperties &networkProperties) override
virtual TensorInfo	GetInputTensorInfo (NetworkId networkId, LayerBindingId layerId) const override
virtual TensorInfo	GetOutputTensorInfo (NetworkId networkId, LayerBindingId layerId) const override
virtual Status	EnqueueWorkload (NetworkId networkId, const InputTensors &inputTensors, const OutputTensors &outputTensors) override
	Evaluates a network using input in inputTensors and outputs filled into outputTensors. More...
virtual Status	UnloadNetwork (NetworkId networkId) override
	Unloads a network from the Runtime. More...
virtual const IDeviceSpec &	GetDeviceSpec () const override
virtual const std::shared_ptr< IProfiler >	GetProfiler (NetworkId networkId) const override
	Gets the profiler corresponding to the given network id. More...
virtual void	RegisterDebugCallback (NetworkId networkId, const DebugCallbackFunction &func) override
	Registers a callback function to debug layers performing custom computations on intermediate tensors. More...
	Runtime (const CreationOptions &options)
	Creates a runtime for workload execution. More...
	~Runtime ()

Friends
void	RuntimeLoadedNetworksReserve (armnn::Runtime *runtime)

Additional Inherited Members
Static Public Member Functions inherited from IRuntime
static IRuntime *	CreateRaw (const CreationOptions &options)
static IRuntimePtr	Create (const CreationOptions &options)
static void	Destroy (IRuntime *runtime)
Protected Member Functions inherited from IRuntime
	~IRuntime ()

Detailed Description

Definition at line 23 of file Runtime.hpp.

Constructor & Destructor Documentation

Runtime()

Runtime

(

const CreationOptions &

options

)

Creates a runtime for workload execution.

Definition at line 155 of file Runtime.cpp.

References ProfilingService::AddBackendProfilingContext(), DeviceSpec::AddSupportedBackends(), ARMNN_LOG, ARMNN_VERSION, armnn::BackendRegistryInstance(), ProfilingService::ConfigureProfilingService(), BackendRegistry::GetFactory(), armnn::info, ProfilingService::Instance(), IRuntime::CreationOptions::m_DynamicBackendsPath, and IRuntime::CreationOptions::m_ProfilingOptions.

Referenced by Runtime::GetDeviceSpec().

    : m_NetworkIdCounter(0)
{
    ARMNN_LOG(info) << "ArmNN v" << ARMNN_VERSION << "\n";

    // pass configuration info to the profiling service
    armnn::profiling::ProfilingService::Instance().ConfigureProfilingService(options.m_ProfilingOptions);

    // Load any available/compatible dynamic backend before the runtime
    // goes through the backend registry
    LoadDynamicBackends(options.m_DynamicBackendsPath);

    BackendIdSet supportedBackends;
    for (const auto& id : BackendRegistryInstance().GetBackendIds())
    {
        // Store backend contexts for the supported ones
        try {
            auto factoryFun = BackendRegistryInstance().GetFactory(id);
            auto backend = factoryFun();
            BOOST_ASSERT(backend.get() != nullptr);

            auto context = backend->CreateBackendContext(options);

            // backends are allowed to return nullptrs if they
            // don't wish to create a backend specific context
            if (context)
            {
                m_BackendContexts.emplace(std::make_pair(id, std::move(context)));
            }
            supportedBackends.emplace(id);

            unique_ptr<armnn::profiling::IBackendProfiling> profilingIface =
                std::make_unique<armnn::profiling::BackendProfiling>(armnn::profiling::BackendProfiling(
                    options, armnn::profiling::ProfilingService::Instance(), id));

            // Backends may also provide a profiling context. Ask for it now.
            auto profilingContext = backend->CreateBackendProfilingContext(options, profilingIface);
            // Backends that don't support profiling will return a null profiling context.
            if (profilingContext)
            {
                // Enable profiling on the backend and assert that it returns true
                if(profilingContext->EnableProfiling(true))
                {
                    // Pass the context onto the profiling service.
                    armnn::profiling::ProfilingService::Instance().AddBackendProfilingContext(id, profilingContext);
                }
                else
                {
                    throw BackendProfilingException("Unable to enable profiling on Backend Id: " + id.Get());
                }
            }
        }
        catch (const BackendUnavailableException&)
        {
            // Ignore backends which are unavailable
        }

    }
    m_DeviceSpec.AddSupportedBackends(supportedBackends);
}

~Runtime()

~Runtime

(

)

Definition at line 216 of file Runtime.cpp.

References DeviceSpec::ClearDynamicBackends(), DynamicBackendUtils::DeregisterDynamicBackends(), DeviceSpec::GetDynamicBackends(), and Runtime::UnloadNetwork().

Referenced by Runtime::GetDeviceSpec().

{
    std::vector<int> networkIDs;
    try
    {
        // Coverity fix: The following code may throw an exception of type std::length_error.
        std::transform(m_LoadedNetworks.begin(), m_LoadedNetworks.end(),
                       std::back_inserter(networkIDs),
                       [](const auto &pair) { return pair.first; });
    }
    catch (const std::exception& e)
    {
        // Coverity fix: BOOST_LOG_TRIVIAL (typically used to report errors) may throw an
        // exception of type std::length_error.
        // Using stderr instead in this context as there is no point in nesting try-catch blocks here.
        std::cerr << "WARNING: An error has occurred when getting the IDs of the networks to unload: " << e.what()
                  << "\nSome of the loaded networks may not be unloaded" << std::endl;
    }
    // We then proceed to unload all the networks which IDs have been appended to the list
    // up to the point the exception was thrown (if any).

    for (auto networkID : networkIDs)
    {
        try
        {
            // Coverity fix: UnloadNetwork() may throw an exception of type std::length_error,
            // boost::log::v2s_mt_posix::odr_violation or boost::log::v2s_mt_posix::system_error
            UnloadNetwork(networkID);
        }
        catch (const std::exception& e)
        {
            // Coverity fix: BOOST_LOG_TRIVIAL (typically used to report errors) may throw an
            // exception of type std::length_error.
            // Using stderr instead in this context as there is no point in nesting try-catch blocks here.
            std::cerr << "WARNING: An error has occurred when unloading network " << networkID << ": " << e.what()
                      << std::endl;
        }
    }


    // Clear all dynamic backends.
    DynamicBackendUtils::DeregisterDynamicBackends(m_DeviceSpec.GetDynamicBackends());
    m_DeviceSpec.ClearDynamicBackends();
    m_BackendContexts.clear();
}

Member Function Documentation

EnqueueWorkload()

Status EnqueueWorkload ( NetworkId  networkId, const InputTensors &  inputTensors, const OutputTensors &  outputTensors  )

overridevirtual

Evaluates a network using input in inputTensors and outputs filled into outputTensors.

Implements IRuntime.

Definition at line 279 of file Runtime.cpp.

References LoadedNetwork::EnqueueWorkload(), and LoadedNetwork::FreeWorkingMemory().

{
    LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);

    static thread_local NetworkId lastId = networkId;
    if (lastId != networkId)
    {
        LoadedNetworkFuncSafe(lastId, [](LoadedNetwork* network)
            {
                network->FreeWorkingMemory();
            });
    }
    lastId=networkId;

    return loadedNetwork->EnqueueWorkload(inputTensors, outputTensors);
}

GetDeviceSpec()

virtual const IDeviceSpec& GetDeviceSpec ( ) const

inlineoverridevirtual

Implements IRuntime.

Definition at line 63 of file Runtime.hpp.

References Runtime::GetProfiler(), options, Runtime::RegisterDebugCallback(), Runtime::Runtime(), Runtime::RuntimeLoadedNetworksReserve, and Runtime::~Runtime().

Referenced by BOOST_AUTO_TEST_CASE().

{ return m_DeviceSpec; }

GetInputTensorInfo()

TensorInfo GetInputTensorInfo ( NetworkId  networkId, LayerBindingId  layerId  ) const

overridevirtual

Implements IRuntime.

Definition at line 268 of file Runtime.cpp.

References LoadedNetwork::GetInputTensorInfo().

{
    return GetLoadedNetworkPtr(networkId)->GetInputTensorInfo(layerId);
}

GetOutputTensorInfo()

TensorInfo GetOutputTensorInfo ( NetworkId  networkId, LayerBindingId  layerId  ) const

overridevirtual

Implements IRuntime.

Definition at line 273 of file Runtime.cpp.

References LoadedNetwork::GetOutputTensorInfo().

{
    return GetLoadedNetworkPtr(networkId)->GetOutputTensorInfo(layerId);
}

GetProfiler()

const std::shared_ptr< IProfiler > GetProfiler ( NetworkId  networkId ) const

overridevirtual

Gets the profiler corresponding to the given network id.

Parameters

networkId

The id of the network for which to get the profile.

Returns

A pointer to the requested profiler, or nullptr if not found.

Implements IRuntime.

Definition at line 143 of file Runtime.cpp.

Referenced by Runtime::GetDeviceSpec().

{
    auto it = m_LoadedNetworks.find(networkId);
    if (it != m_LoadedNetworks.end())
    {
        auto& loadedNetwork = it->second;
        return loadedNetwork->GetProfiler();
    }

    return nullptr;
}

LoadNetwork() [1/3]

Status LoadNetwork ( NetworkId &  networkIdOut, IOptimizedNetworkPtr  network  )

overridevirtual

Loads a complete network into the Runtime.

Parameters

[out]	networkIdOut	- Unique identifier for the network is returned in this reference.
[in]	network	- Complete network to load into the Runtime. The runtime takes ownership of the network once passed in.

Returns

armnn::Status

Implements IRuntime.

Definition at line 47 of file Runtime.cpp.

Referenced by BOOST_AUTO_TEST_CASE().

{
    std::string ignoredErrorMessage;
    return LoadNetwork(networkIdOut, std::move(inNetwork), ignoredErrorMessage);
}

LoadNetwork() [2/3]

Status LoadNetwork ( NetworkId &  networkIdOut, IOptimizedNetworkPtr  network, std::string &  errorMessage  )

overridevirtual

Load a complete network into the IRuntime.

Parameters

[out]	networkIdOut	Unique identifier for the network is returned in this reference.
[in]	network	Complete network to load into the IRuntime.
[out]	errorMessage	Error message if there were any errors. The runtime takes ownership of the network once passed in.

Returns

armnn::Status

Implements IRuntime.

Definition at line 53 of file Runtime.cpp.

{
    INetworkProperties networkProperties;
    return LoadNetwork(networkIdOut, std::move(inNetwork), errorMessage, networkProperties);
}

LoadNetwork() [3/3]

Status LoadNetwork ( NetworkId &  networkIdOut, IOptimizedNetworkPtr  network, std::string &  errorMessage, const INetworkProperties &  networkProperties  )

overridevirtual

Implements IRuntime.

Definition at line 61 of file Runtime.cpp.

References armnn::Failure, ProfilingService::IncrementCounterValue(), ProfilingService::Instance(), LoadedNetwork::MakeLoadedNetwork(), and armnn::Success.

{
    IOptimizedNetwork* rawNetwork = inNetwork.release();

    networkIdOut = GenerateNetworkId();

    for (auto&& context : m_BackendContexts)
    {
        context.second->BeforeLoadNetwork(networkIdOut);
    }

    unique_ptr<LoadedNetwork> loadedNetwork = LoadedNetwork::MakeLoadedNetwork(
        std::unique_ptr<OptimizedNetwork>(boost::polymorphic_downcast<OptimizedNetwork*>(rawNetwork)),
        errorMessage,
        networkProperties);

    if (!loadedNetwork)
    {
        return Status::Failure;
    }

    {
        std::lock_guard<std::mutex> lockGuard(m_Mutex);

        // Stores the network
        m_LoadedNetworks[networkIdOut] = std::move(loadedNetwork);
    }

    for (auto&& context : m_BackendContexts)
    {
        context.second->AfterLoadNetwork(networkIdOut);
    }

    if (profiling::ProfilingService::Instance().IsProfilingEnabled())
    {
        profiling::ProfilingService::Instance().IncrementCounterValue(armnn::profiling::NETWORK_LOADS);
    }

    return Status::Success;
}

RegisterDebugCallback()

void RegisterDebugCallback ( NetworkId  networkId, const DebugCallbackFunction &  func  )

overridevirtual

Registers a callback function to debug layers performing custom computations on intermediate tensors.

Parameters

networkId	The id of the network to register the callback.
func	callback function to pass to the debug layer.

Implements IRuntime.

Definition at line 298 of file Runtime.cpp.

References DeviceSpec::AddSupportedBackends(), DynamicBackendUtils::CreateDynamicBackends(), DynamicBackendUtils::GetBackendPaths(), DynamicBackendUtils::GetSharedObjects(), LoadedNetwork::RegisterDebugCallback(), and DynamicBackendUtils::RegisterDynamicBackends().

Referenced by Runtime::GetDeviceSpec().

{
    LoadedNetwork* loadedNetwork = GetLoadedNetworkPtr(networkId);
    loadedNetwork->RegisterDebugCallback(func);
}

UnloadNetwork()

Status UnloadNetwork ( NetworkId  networkId )

overridevirtual

Unloads a network from the Runtime.

At the moment this only removes the network from the m_Impl->m_Network. This might need more work in the future to be AndroidNN compliant.

Parameters

[in]

networkId

Unique identifier for the network to be unloaded. Generated in LoadNetwork().

Returns

armnn::Status

Implements IRuntime.

Definition at line 105 of file Runtime.cpp.

References ARMNN_LOG, armnn::debug, armnn::Failure, ProfilingService::IncrementCounterValue(), ProfilingService::Instance(), armnn::Success, and armnn::warning.

Referenced by BOOST_AUTO_TEST_CASE(), and Runtime::~Runtime().

{
    bool unloadOk = true;
    for (auto&& context : m_BackendContexts)
    {
        unloadOk &= context.second->BeforeUnloadNetwork(networkId);
    }

    if (!unloadOk)
    {
        ARMNN_LOG(warning) << "Runtime::UnloadNetwork(): failed to unload "
                              "network with ID:" << networkId << " because BeforeUnloadNetwork failed";
        return Status::Failure;
    }

    {
        std::lock_guard<std::mutex> lockGuard(m_Mutex);

        if (m_LoadedNetworks.erase(networkId) == 0)
        {
            ARMNN_LOG(warning) << "WARNING: Runtime::UnloadNetwork(): " << networkId << " not found!";
            return Status::Failure;
        }
        if (profiling::ProfilingService::Instance().IsProfilingEnabled())
        {
            profiling::ProfilingService::Instance().IncrementCounterValue(armnn::profiling::NETWORK_UNLOADS);
        }
    }

    for (auto&& context : m_BackendContexts)
    {
        context.second->AfterUnloadNetwork(networkId);
    }

    ARMNN_LOG(debug) << "Runtime::UnloadNetwork(): Unloaded network with ID: " << networkId;
    return Status::Success;
}

Friends And Related Function Documentation

RuntimeLoadedNetworksReserve

void RuntimeLoadedNetworksReserve ( armnn::Runtime *  runtime )

friend

Definition at line 28 of file RuntimeTests.cpp.

Referenced by Runtime::GetDeviceSpec().

{
    runtime->m_LoadedNetworks.reserve(1);
}

---

The documentation for this class was generated from the following files:

• src/armnn/Runtime.hpp
• src/armnn/Runtime.cpp