From f364d5391b08e9071cd965f5765385ec9156b652 Mon Sep 17 00:00:00 2001 From: Finn Williams Date: Wed, 9 Jun 2021 17:07:33 +0100 Subject: IVGCVSW-6062 Rework the async threadpool !android-nn-driver:5802 * Extract the threadpool from LoadedNetwork/Runtime * Refactor the threadpool to be handle multiple networks * Trim IAsyncExecutionCallback and add an InferenceId to AsyncExecutionCallback * Add AsyncCallbackManager class Signed-off-by: Finn Williams Change-Id: I36aa2ad29c16bc10ee0706adfeb6b27f60012afb --- include/armnn/IAsyncExecutionCallback.hpp | 13 ------ include/armnn/IRuntime.hpp | 31 ++++++------ include/armnn/IWorkingMemHandle.hpp | 3 -- include/armnn/Threadpool.hpp | 78 +++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 32 deletions(-) create mode 100644 include/armnn/Threadpool.hpp (limited to 'include/armnn') diff --git a/include/armnn/IAsyncExecutionCallback.hpp b/include/armnn/IAsyncExecutionCallback.hpp index 045ec4581f..3e0cacccee 100644 --- a/include/armnn/IAsyncExecutionCallback.hpp +++ b/include/armnn/IAsyncExecutionCallback.hpp @@ -23,19 +23,6 @@ public: // Notify the AsyncExecutionCallback object of the armnn execution status virtual void Notify(armnn::Status status, InferenceTimingPair timeTaken) = 0; - - // Block the calling thread until the AsyncExecutionCallback object allows it to proceed - virtual void Wait() const = 0; - - // Retrieve the ArmNN Status from the AsyncExecutionCallback that has been notified - virtual armnn::Status GetStatus() const = 0; - - // Retrieve the start time before executing the inference - virtual HighResolutionClock GetStartTime() const = 0; - - // Retrieve the time after executing the inference - virtual HighResolutionClock GetEndTime() const = 0; - }; } // experimental diff --git a/include/armnn/IRuntime.hpp b/include/armnn/IRuntime.hpp index bfc13c9c01..f88b6b664f 100644 --- a/include/armnn/IRuntime.hpp +++ b/include/armnn/IRuntime.hpp @@ -32,24 +32,34 @@ struct INetworkProperties ARMNN_DEPRECATED_MSG("Please use INetworkProperties constructor with MemorySource argument") INetworkProperties(bool importEnabled = false, bool exportEnabled = false, - bool asyncEnabled = false, - size_t numThreads = 1) + bool asyncEnabled = false) : m_ImportEnabled(importEnabled) , m_ExportEnabled(exportEnabled) , m_AsyncEnabled(asyncEnabled) - , m_NumThreads(numThreads) , m_InputSource(m_ImportEnabled ? MemorySource::Malloc : MemorySource::Undefined) , m_OutputSource(m_ExportEnabled ? MemorySource::Malloc : MemorySource::Undefined) {} + ARMNN_DEPRECATED_MSG("Please use INetworkProperties constructor without numThreads argument") INetworkProperties(bool asyncEnabled, MemorySource m_InputSource, MemorySource m_OutputSource, - size_t numThreads = 1) + size_t numThreads) + : m_ImportEnabled(m_InputSource != MemorySource::Undefined) + , m_ExportEnabled(m_OutputSource != MemorySource::Undefined) + , m_AsyncEnabled(asyncEnabled) + , m_InputSource(m_InputSource) + , m_OutputSource(m_OutputSource) + { + armnn::IgnoreUnused(numThreads); + } + + INetworkProperties(bool asyncEnabled, + MemorySource m_InputSource, + MemorySource m_OutputSource) : m_ImportEnabled(m_InputSource != MemorySource::Undefined) , m_ExportEnabled(m_OutputSource != MemorySource::Undefined) , m_AsyncEnabled(asyncEnabled) - , m_NumThreads(numThreads) , m_InputSource(m_InputSource) , m_OutputSource(m_OutputSource) {} @@ -60,7 +70,6 @@ struct INetworkProperties const bool m_ExportEnabled; const bool m_AsyncEnabled; - const size_t m_NumThreads; const MemorySource m_InputSource; const MemorySource m_OutputSource; @@ -191,16 +200,6 @@ public: const InputTensors& inputTensors, const OutputTensors& outputTensors); - /// This is an experimental function - /// Schedule a thread safe execution by taking the input tensors and an execution priority for Quality of Service. - /// The output tensors will then be filled and the callback object will notify that the execution has either - /// succeeded or failed. - void Schedule(NetworkId networkId, - const InputTensors& inputTensors, - const OutputTensors& outputTensors, - const QosExecPriority priority, - std::shared_ptr callback); - /// Unloads a network from the IRuntime. /// At the moment this only removes the network from the m_Impl->m_Network. /// This might need more work in the future to be AndroidNN compliant. diff --git a/include/armnn/IWorkingMemHandle.hpp b/include/armnn/IWorkingMemHandle.hpp index 6fb2f9fe5f..171fa3d81c 100644 --- a/include/armnn/IWorkingMemHandle.hpp +++ b/include/armnn/IWorkingMemHandle.hpp @@ -25,9 +25,6 @@ public: /// Returns the NetworkId of the Network that this IWorkingMemHandle works with. virtual NetworkId GetNetworkId() = 0; - /// Returns the InferenceId of the Inference that this IWorkingMemHandle works with. - virtual profiling::ProfilingGuid GetInferenceId() = 0; - /// Allocate the backing memory required for execution. If this is not called, then allocation will be /// deferred to execution time. The mutex must be locked. virtual void Allocate() = 0; diff --git a/include/armnn/Threadpool.hpp b/include/armnn/Threadpool.hpp new file mode 100644 index 0000000000..e2458dbb65 --- /dev/null +++ b/include/armnn/Threadpool.hpp @@ -0,0 +1,78 @@ +// +// Copyright © 2021 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#pragma once + +#include +#include + +#include "INetwork.hpp" +#include "IRuntime.hpp" + +#include +#include +#include +#include +#include + +namespace armnn +{ +namespace experimental +{ +class Threadpool +{ +public: + Threadpool(std::size_t numThreads, + IRuntime* runtimePtr, + std::vector> memHandles); + + ~Threadpool() + { + TerminateThreadPool(); + } + + void LoadMemHandles(std::vector> memHandles); + void UnloadMemHandles(NetworkId networkId); + + /// Schedule an asynchronous execution on the loaded network + void Schedule(NetworkId networkId, + const InputTensors &inputTensors, + const OutputTensors &outputTensors, + const QosExecPriority priority, + std::shared_ptr cb); + + void TerminateThreadPool() noexcept; + +private: + using ExecutionTuple = std::tuple>; + + using ExecutionQueue = std::queue>; + + void ProcessExecPriorities(uint32_t index); + + IRuntime* m_RuntimePtr; + + ExecutionQueue m_HighPriorityQueue; + ExecutionQueue m_MediumPriorityQueue; + ExecutionQueue m_LowPriorityQueue; + + // Condition Variables require mutex which will guard the shared state. + // Has an event happened? Stop signal for example + std::condition_variable m_ThreadPoolEvent; + std::mutex m_ThreadPoolMutex; + + // The shared state for conditional variable + bool m_TerminatePool = false; + + std::unordered_map>> m_WorkingMemHandleMap; + std::vector> m_Threads; +}; + +} // namespace experimental + +} // namespace armnn -- cgit v1.2.1