ArmNN
 24.02
Threadpool.hpp
Go to the documentation of this file.
1 //
2 // Copyright © 2021-2022 Arm Ltd and Contributors. All rights reserved.
3 // SPDX-License-Identifier: MIT
4 //
5 #if !defined(ARMNN_DISABLE_THREADS)
6 
7 #pragma once
8 
9 #include "IRuntime.hpp"
10 #include <armnn/Tensor.hpp>
11 #include <armnn/Types.hpp>
12 #include <stdint.h>
13 #include <thread>
14 #include <mutex>
15 #include <condition_variable>
16 #include <unordered_map>
17 #include <queue>
18 #include <iosfwd>
19 #include <memory>
20 #include <tuple>
21 #include <vector>
22 
23 namespace armnn
24 {
25 namespace experimental
26 {
27 class IAsyncExecutionCallback;
28 class IWorkingMemHandle;
29 
31 {
32 public:
33  Threadpool(std::size_t numThreads,
34  IRuntime* runtimePtr,
35  std::vector<std::shared_ptr<IWorkingMemHandle>> memHandles);
36 
38  {
40  }
41 
42  void LoadMemHandles(std::vector<std::shared_ptr<IWorkingMemHandle>> memHandles);
43  void UnloadMemHandles(NetworkId networkId);
44 
45  /// Schedule an asynchronous execution on the loaded network
46  void Schedule(NetworkId networkId,
47  const InputTensors &inputTensors,
48  const OutputTensors &outputTensors,
49  const QosExecPriority priority,
50  std::shared_ptr<IAsyncExecutionCallback> cb);
51 
52  void TerminateThreadPool() noexcept;
53 
54 private:
55  using ExecutionTuple = std::tuple<NetworkId,
58  std::shared_ptr<IAsyncExecutionCallback>>;
59 
60  using ExecutionQueue = std::queue<std::shared_ptr<ExecutionTuple>>;
61 
62  void ProcessExecPriorities(uint32_t index);
63 
64  IRuntime* m_RuntimePtr;
65 
66  ExecutionQueue m_HighPriorityQueue;
67  ExecutionQueue m_MediumPriorityQueue;
68  ExecutionQueue m_LowPriorityQueue;
69 
70  // Condition Variables require mutex which will guard the shared state.
71  // Has an event happened? Stop signal for example
72  std::condition_variable m_ThreadPoolEvent;
73  std::mutex m_ThreadPoolMutex;
74 
75  // The shared state for conditional variable
76  bool m_TerminatePool = false;
77 
78  std::unordered_map<NetworkId, std::vector<std::shared_ptr<IWorkingMemHandle>>> m_WorkingMemHandleMap;
79  std::vector<std::unique_ptr<std::thread>> m_Threads;
80 };
81 
82 } // namespace experimental
83 
84 } // namespace armnn
85 
86 #endif
IRuntime.hpp
armnn::InputTensors
std::vector< std::pair< LayerBindingId, class ConstTensor > > InputTensors
Definition: Tensor.hpp:394
armnn::experimental::Threadpool::Threadpool
Threadpool(std::size_t numThreads, IRuntime *runtimePtr, std::vector< std::shared_ptr< IWorkingMemHandle >> memHandles)
Definition: Threadpool.cpp:16
armnn::experimental::Threadpool::~Threadpool
~Threadpool()
Definition: Threadpool.hpp:37
armnn::experimental::Threadpool
Definition: Threadpool.hpp:30
armnn::OutputTensors
std::vector< std::pair< LayerBindingId, class Tensor > > OutputTensors
Definition: Tensor.hpp:395
armnn::experimental::IWorkingMemHandle
Definition: IWorkingMemHandle.hpp:20
armnn::IRuntime
Definition: IRuntime.hpp:75
armnn::experimental::Threadpool::LoadMemHandles
void LoadMemHandles(std::vector< std::shared_ptr< IWorkingMemHandle >> memHandles)
Definition: Threadpool.cpp:29
armnn::NetworkId
int NetworkId
Definition: IRuntime.hpp:35
armnn::QosExecPriority
QosExecPriority
Definition: Types.hpp:79
Tensor.hpp
armnn::experimental::IAsyncExecutionCallback
Definition: IAsyncExecutionCallback.hpp:19
std
Definition: BackendId.hpp:149
armnn::experimental::Threadpool::Schedule
void Schedule(NetworkId networkId, const InputTensors &inputTensors, const OutputTensors &outputTensors, const QosExecPriority priority, std::shared_ptr< IAsyncExecutionCallback > cb)
Schedule an asynchronous execution on the loaded network.
Definition: Threadpool.cpp:69
armnn::experimental::Threadpool::UnloadMemHandles
void UnloadMemHandles(NetworkId networkId)
Definition: Threadpool.cpp:57
armnn
Copyright (c) 2021 ARM Limited and Contributors.
Definition: 01_00_quick_start.dox:6
Types.hpp
armnn::experimental::Threadpool::TerminateThreadPool
void TerminateThreadPool() noexcept
Definition: Threadpool.cpp:102