IVGCVSW-5823 Refactor Async Network API

 * Moved IAsyncNetwork into IRuntime.
 * All LoadedNetworks can be executed Asynchronously.

Signed-off-by: Mike Kelly <mike.kelly@arm.com>
Change-Id: Ibbc901ab9110dc2f881425b75489bccf9ad54169

1 files changed, 30 insertions, 3 deletions

diff --git a/src/armnn/LoadedNetwork.hpp b/src/armnn/LoadedNetwork.hpp
index c7dd37fdea..2bcf5c8c08 100644
--- a/src/armnn/LoadedNetwork.hpp
+++ b/src/armnn/LoadedNetwork.hpp
@@ -37,11 +37,19 @@ public:
     using WorkloadQueue = std::vector< std::unique_ptr<IWorkload> >;
     ~LoadedNetwork(){ FreeWorkingMemory(); }
 
+    /// Create a new unique WorkingMemHandle object. Create multiple handles if you wish to have
+    /// overlapped Execution by calling this function from different threads.
+    std::unique_ptr<IWorkingMemHandle> CreateWorkingMemHandle(NetworkId networkId);
+
     TensorInfo GetInputTensorInfo(LayerBindingId layerId) const;
     TensorInfo GetOutputTensorInfo(LayerBindingId layerId) const;
 
     Status EnqueueWorkload(const InputTensors& inputTensors, const OutputTensors& outputTensors);
 
+    Status Execute(const InputTensors& inputTensors,
+                   const OutputTensors& outputTensors,
+                   IWorkingMemHandle& workingMemHandle);
+
     static std::unique_ptr<LoadedNetwork> MakeLoadedNetwork(std::unique_ptr<IOptimizedNetwork> net,
                                                             std::string & errorMessage,
                                                             const INetworkProperties& networkProperties,
@@ -58,6 +66,11 @@ public:
 
     void SendNetworkStructure();
 
+    bool IsAsyncEnabled()
+    {
+        return m_NetworkProperties.m_AsyncEnabled;
+    }
+
     profiling::ProfilingGuid GetNetworkGuid();
 
 private:
@@ -67,14 +80,29 @@ private:
                   const INetworkProperties& networkProperties,
                   profiling::ProfilingService& profilingService);
 
+    void CollectInputTensorHandles(std::unordered_map<LayerGuid, std::vector<ITensorHandle*> >& tensorHandles,
+                                   std::vector<ITensorHandle*>& inputs,
+                                   const armnn::Layer* layer,
+                                   const TensorHandleFactoryRegistry& registry,
+                                   const bool isMemoryManaged = false);
+
+    void CreateOutputTensorHandles(std::unordered_map<LayerGuid, std::vector<ITensorHandle*> >& tensorHandles,
+                                   std::vector<ITensorHandle*>& outputs,
+                                   const armnn::Layer* layer,
+                                   const TensorHandleFactoryRegistry& registry,
+                                   const bool isMemoryManaged = false);
+
     void EnqueueInput(const BindableLayer& layer, ITensorHandle* tensorHandle, const TensorInfo& tensorInfo);
 
     void EnqueueOutput(const BindableLayer& layer, ITensorHandle* tensorHandle, const TensorInfo& tensorInfo);
 
+    void EnqueueInput(const BindableLayer& layer, const ConstTensor& inputTensor, WorkingMemHandle& handle);
+
+    void EnqueueOutput(const BindableLayer& layer, const Tensor& outputTensor, WorkingMemHandle& handle);
+
     bool Execute(std::unique_ptr<profiling::TimelineUtilityMethods>& timelineUtils,
                  profiling::ProfilingGuid inferenceGuid);
 
-
     const IWorkloadFactory& GetWorkloadFactory(const Layer& layer) const;
 
     using BackendPtrMap = std::unordered_map<BackendId, IBackendInternalUniquePtr>;
@@ -96,8 +124,7 @@ private:
     mutable std::mutex m_WorkingMemMutex;
 
     bool m_IsWorkingMemAllocated=false;
-    bool m_IsImportEnabled=false;
-    bool m_IsExportEnabled=false;
+    INetworkProperties m_NetworkProperties;
 
     TensorHandleFactoryRegistry m_TensorHandleFactoryRegistry;