IVGCVSW-3277 Refactor TensorHandle factory API

 * Added backend support for multiple types of TensorHandle factories
 * Refactored the backend API to enable new tensor strategies
 * Added mechanism to determine memory strategies during optimization
 * Perform mem-copy only when Direct access is not found
 * Explicitly deleted the copy-constructor from OutputSlot to prevent
   accidental local copies that would cause the DisconnectAll to be
   called by the destructor

Change-Id: I7e812c8e5e6c1c20db1c5932749ac70fd93db7f8
Signed-off-by: Derek Lamberti <derek.lamberti@arm.com>
Signed-off-by: Matteo Martincigh <matteo.martincigh@arm.com>

9 files changed, 232 insertions, 13 deletions

diff --git a/src/backends/backendsCommon/CMakeLists.txt b/src/backends/backendsCommon/CMakeLists.txt
index e1e387bc6f..bc1c15beef 100644
--- a/src/backends/backendsCommon/CMakeLists.txt
+++ b/src/backends/backendsCommon/CMakeLists.txt
@@ -11,6 +11,8 @@ list(APPEND armnnBackendsCommon_sources
     CpuTensorHandle.hpp
     IBackendInternal.hpp
     IBackendContext.hpp
+    ITensorHandleFactory.cpp
+    ITensorHandleFactory.hpp
     LayerSupportBase.cpp
     LayerSupportBase.hpp
     IMemoryManager.hpp
@@ -22,6 +24,8 @@ list(APPEND armnnBackendsCommon_sources
     OptimizationViews.hpp
     OutputHandler.cpp
     OutputHandler.hpp
+    TensorHandleFactoryRegistry.cpp
+    TensorHandleFactoryRegistry.hpp
     WorkloadDataCollector.hpp
     WorkloadData.cpp
     WorkloadDataFwd.hpp
diff --git a/src/backends/backendsCommon/IBackendInternal.hpp b/src/backends/backendsCommon/IBackendInternal.hpp
index fe9d620278..a0d6569949 100644
--- a/src/backends/backendsCommon/IBackendInternal.hpp
+++ b/src/backends/backendsCommon/IBackendInternal.hpp
@@ -10,7 +10,11 @@
 
 #include <ISubgraphViewConverter.hpp>
 #include <SubgraphView.hpp>
+#include <optimizations/Optimization.hpp>
 
+#include "IBackendContext.hpp"
+#include "IMemoryManager.hpp"
+#include "ITensorHandleFactory.hpp"
 #include "OptimizationViews.hpp"
 
 #include <vector>
@@ -18,9 +22,7 @@
 namespace armnn
 {
 class IWorkloadFactory;
-class IBackendContext;
 class IMemoryManager;
-class Optimization;
 class ILayerSupport;
 
 class IBackendInternal : public IBackend
@@ -60,7 +62,10 @@ public:
     }
 
     ARMNN_DEPRECATED_MSG("Use \"OptimizationViews OptimizeSubgraphView(const SubgraphView&)\" instead")
-    virtual Optimizations GetOptimizations() const = 0;
+    virtual Optimizations GetOptimizations() const
+    {
+        return Optimizations{};
+    }
 
     ARMNN_DEPRECATED_MSG("Use \"OptimizationViews OptimizeSubgraphView(const SubgraphView&)\" instead")
     virtual SubGraphUniquePtr OptimizeSubGraph(const SubGraph& subGraph, bool& optimizationAttempted) const
@@ -70,12 +75,19 @@ public:
     }
     ARMNN_NO_DEPRECATE_WARN_END
 
-    virtual IMemoryManagerUniquePtr CreateMemoryManager() const = 0;
+
+    virtual IMemoryManagerUniquePtr CreateMemoryManager() const
+    {
+        return IMemoryManagerUniquePtr();
+    };
 
     virtual IWorkloadFactoryPtr CreateWorkloadFactory(
         const IMemoryManagerSharedPtr& memoryManager = nullptr) const = 0;
 
-    virtual IBackendContextPtr CreateBackendContext(const IRuntime::CreationOptions&) const = 0;
+    virtual IBackendContextPtr CreateBackendContext(const IRuntime::CreationOptions&) const
+    {
+        return IBackendContextPtr{};
+    }
 
     virtual ILayerSupportSharedPtr GetLayerSupport() const = 0;
 
@@ -107,6 +119,29 @@ public:
         }
         return result;
     }
+
+    bool SupportsTensorAllocatorAPI() const { return GetHandleFactoryPreferences().empty() == false; }
+
+    ITensorHandleFactory::FactoryId GetBackwardCompatibleFavoriteHandleFactory()
+    {
+        auto favorites = GetHandleFactoryPreferences();
+        if (favorites.empty())
+        {
+            return ITensorHandleFactory::LegacyFactoryId;
+        }
+        return favorites[0];
+    }
+
+    /// (Optional) Returns a vector of supported TensorHandleFactory ids in preference order.
+    virtual std::vector<ITensorHandleFactory::FactoryId> GetHandleFactoryPreferences() const
+    {
+        return std::vector<ITensorHandleFactory::FactoryId>();
+    }
+
+    /// (Optional) Register TensorHandleFactories
+    /// Either this method or CreateMemoryManager() and
+    /// IWorkloadFactory::CreateTensor()/IWorkloadFactory::CreateSubtensor() methods must be implemented.
+    virtual void RegisterTensorHandleFactories(class TensorHandleFactoryRegistry& registry) {}
 };
 
 using IBackendInternalUniquePtr = std::unique_ptr<IBackendInternal>;
diff --git a/src/backends/backendsCommon/ITensorHandleFactory.cpp b/src/backends/backendsCommon/ITensorHandleFactory.cpp
new file mode 100644
index 0000000000..91f5692723
--- /dev/null
+++ b/src/backends/backendsCommon/ITensorHandleFactory.cpp
@@ -0,0 +1,14 @@
+//
+// Copyright © 2017 Arm Ltd. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#include "ITensorHandleFactory.hpp"
+
+namespace armnn
+{
+
+const ITensorHandleFactory::FactoryId ITensorHandleFactory::LegacyFactoryId = "armnn_legacy_factory";
+const ITensorHandleFactory::FactoryId ITensorHandleFactory::DeferredFactoryId = "armnn_deferred_factory";
+
+} // namespace armnn
diff --git a/src/backends/backendsCommon/ITensorHandleFactory.hpp b/src/backends/backendsCommon/ITensorHandleFactory.hpp
new file mode 100644
index 0000000000..7685061eb3
--- /dev/null
+++ b/src/backends/backendsCommon/ITensorHandleFactory.hpp
@@ -0,0 +1,49 @@
+//
+// Copyright © 2017 Arm Ltd. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#pragma once
+
+#include <armnn/Types.hpp>
+#include <armnn/IRuntime.hpp>
+
+namespace armnn
+{
+
+class ITensorHandleFactory
+{
+public:
+    using FactoryId = std::string;
+    static const FactoryId LegacyFactoryId;   // Use the workload factory to create the tensor handle
+    static const FactoryId DeferredFactoryId; // Some TensorHandleFactory decisions are deferred to run-time
+
+    virtual ~ITensorHandleFactory() {}
+
+
+    virtual std::unique_ptr<ITensorHandle> CreateSubTensorHandle(ITensorHandle& parent,
+                                                                 TensorShape const& subTensorShape,
+                                                                 unsigned int const* subTensorOrigin) const = 0;
+
+    virtual std::unique_ptr<ITensorHandle> CreateTensorHandle(const TensorInfo& tensorInfo) const = 0;
+
+    virtual const FactoryId GetId() const = 0;
+
+    virtual bool SupportsSubTensors() const = 0;
+
+    virtual bool SupportsMapUnmap() const final { return true; }
+
+    virtual bool SupportsExport() const final { return false; }
+
+    virtual bool SupportsImport() const final { return false; }
+};
+
+enum class MemoryStrategy
+{
+    Undefined,
+    DirectCompatibility,    // Only allocate the tensorhandle using the assigned factory
+    CopyToTarget,           // Default + Insert MemCopy node before target
+    ExportToTarget,         // Default + Insert Import node
+};
+
+} //namespace armnn
diff --git a/src/backends/backendsCommon/OutputHandler.cpp b/src/backends/backendsCommon/OutputHandler.cpp
index 2df2fb5181..8f4942d8ba 100644
--- a/src/backends/backendsCommon/OutputHandler.cpp
+++ b/src/backends/backendsCommon/OutputHandler.cpp
@@ -27,9 +27,9 @@ void OutputHandler::CreateTensorHandles(const IWorkloadFactory& factory)
     m_TensorHandle = factory.CreateTensorHandle(m_TensorInfo);
 }
 
-void OutputHandler::CreateTensorHandles(const IWorkloadFactory& factory, DataLayout dataLayout)
+void OutputHandler::CreateTensorHandles(const ITensorHandleFactory& factory)
 {
-    m_TensorHandle = factory.CreateTensorHandle(m_TensorInfo, dataLayout);
+    m_TensorHandle = factory.CreateTensorHandle(m_TensorInfo);
 }
 
 void OutputHandler::CollectWorkloadOutputs(WorkloadDataCollector& dataCollector) const
diff --git a/src/backends/backendsCommon/OutputHandler.hpp b/src/backends/backendsCommon/OutputHandler.hpp
index 240b369fab..01e255deaa 100644
--- a/src/backends/backendsCommon/OutputHandler.hpp
+++ b/src/backends/backendsCommon/OutputHandler.hpp
@@ -5,6 +5,7 @@
 #pragma once
 
 #include "ITensorHandle.hpp"
+#include "ITensorHandleFactory.hpp"
 
 #include <armnn/Descriptors.hpp>
 #include <armnn/INetwork.hpp>
@@ -35,14 +36,10 @@ public:
     /// @param tensorInfo - TensorInfo for the output.
     void SetTensorInfo(const TensorInfo& tensorInfo);
 
-    /// @brief - Creates tensor handlers used by the intermediate tensors. Does not allocate memory.
+    /// @brief - Creates tensor handles used by the intermediate tensors. Does not allocate memory.
     /// @param factory - Factory to be used for handler creation.
     void CreateTensorHandles(const IWorkloadFactory& factory);
-
-    /// @brief - Creates tensor handlers used by the intermediate tensors. Does not allocate memory.
-    /// @param factory - Factory to be used for handler creation.
-    /// @param dataLayout - Data Layout to be used for handler creation.
-    void CreateTensorHandles(const IWorkloadFactory& factory, DataLayout dataLayout);
+    void CreateTensorHandles(const ITensorHandleFactory& factory);
 
     /// @brief - Gets the matching TensorInfo for the output.
     /// @return - References to the output TensorInfo.
diff --git a/src/backends/backendsCommon/TensorHandleFactoryRegistry.cpp b/src/backends/backendsCommon/TensorHandleFactoryRegistry.cpp
new file mode 100644
index 0000000000..4692b9f960
--- /dev/null
+++ b/src/backends/backendsCommon/TensorHandleFactoryRegistry.cpp
@@ -0,0 +1,69 @@
+//
+// Copyright © 2017 Arm Ltd. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#include "TensorHandleFactoryRegistry.hpp"
+#include "IMemoryManager.hpp"
+
+
+namespace armnn
+{
+
+void TensorHandleFactoryRegistry::RegisterFactory(std::unique_ptr <ITensorHandleFactory> newFactory)
+{
+    if (!newFactory)
+    {
+        return;
+    }
+
+    ITensorHandleFactory::FactoryId id = newFactory->GetId();
+
+    // Don't register duplicates
+    for (auto& registeredFactory : m_Factories)
+    {
+        if (id == registeredFactory->GetId())
+        {
+            return;
+        }
+    }
+
+    // Take ownership of the new allocator
+    m_Factories.push_back(std::move(newFactory));
+}
+
+void TensorHandleFactoryRegistry::RegisterMemoryManager(std::shared_ptr<armnn::IMemoryManager> memoryManger)
+{
+    m_MemoryManagers.push_back(memoryManger);
+}
+
+ITensorHandleFactory* TensorHandleFactoryRegistry::GetFactory(ITensorHandleFactory::FactoryId id) const
+{
+    for (auto& factory : m_Factories)
+    {
+        if (factory->GetId() == id)
+        {
+            return factory.get();
+        }
+    }
+
+    return nullptr;
+}
+
+void TensorHandleFactoryRegistry::AquireMemory()
+{
+    for (auto& mgr : m_MemoryManagers)
+    {
+        mgr->Acquire();
+    }
+}
+
+void TensorHandleFactoryRegistry::ReleaseMemory()
+{
+    for (auto& mgr : m_MemoryManagers)
+    {
+        mgr->Release();
+    }
+}
+
+} // namespace armnn
diff --git a/src/backends/backendsCommon/TensorHandleFactoryRegistry.hpp b/src/backends/backendsCommon/TensorHandleFactoryRegistry.hpp
new file mode 100644
index 0000000000..9e02985301
--- /dev/null
+++ b/src/backends/backendsCommon/TensorHandleFactoryRegistry.hpp
@@ -0,0 +1,49 @@
+//
+// Copyright © 2017 Arm Ltd. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#pragma once
+
+#include "ITensorHandleFactory.hpp"
+
+#include <memory>
+#include <vector>
+
+namespace armnn
+{
+
+//Forward
+class IMemoryManager;
+
+///
+class TensorHandleFactoryRegistry
+{
+public:
+    TensorHandleFactoryRegistry() = default;
+
+    TensorHandleFactoryRegistry(const TensorHandleFactoryRegistry& other) = delete;
+    TensorHandleFactoryRegistry(TensorHandleFactoryRegistry&& other) = delete;
+
+    /// Register a TensorHandleFactory and transfer ownership
+    void RegisterFactory(std::unique_ptr<ITensorHandleFactory> allocator);
+
+    /// Register a memory manager with shared ownership
+    void RegisterMemoryManager(std::shared_ptr<IMemoryManager> memoryManger);
+
+    /// Find a TensorHandleFactory by Id
+    /// Returns nullptr if not found
+    ITensorHandleFactory* GetFactory(ITensorHandleFactory::FactoryId id) const;
+
+    /// Aquire memory required for inference
+    void AquireMemory();
+
+    /// Release memory required for inference
+    void ReleaseMemory();
+
+private:
+    std::vector<std::unique_ptr<ITensorHandleFactory>> m_Factories;
+    std::vector<std::shared_ptr<IMemoryManager>> m_MemoryManagers;
+};
+
+} // namespace armnn
diff --git a/src/backends/backendsCommon/common.mk b/src/backends/backendsCommon/common.mk
index 90d3d16ebb..8df5ab9203 100644
--- a/src/backends/backendsCommon/common.mk
+++ b/src/backends/backendsCommon/common.mk
@@ -10,10 +10,12 @@
 COMMON_SOURCES := \
     BackendRegistry.cpp \
     CpuTensorHandle.cpp \
+    ITensorHandleFactory.cpp \
     LayerSupportBase.cpp \
     MemCopyWorkload.cpp \
     OptimizationViews.cpp \
     OutputHandler.cpp \
+    TensorHandleFactoryRegistry.cpp \
     WorkloadData.cpp \
     WorkloadFactory.cpp \
     WorkloadUtils.cpp