IVGCVSW-3623 Implement NeonTensorHandle::Import

Signed-off-by: David Monahan <david.monahan@arm.com>
Change-Id: I7213788725fd4e4cf1176998604e999d0b7ed6cc

6 files changed, 241 insertions, 21 deletions

diff --git a/src/backends/neon/NeonTensorHandle.hpp b/src/backends/neon/NeonTensorHandle.hpp
index 9077f34888..c3662c1211 100644
--- a/src/backends/neon/NeonTensorHandle.hpp
+++ b/src/backends/neon/NeonTensorHandle.hpp
@@ -24,11 +24,20 @@ class NeonTensorHandle : public IAclTensorHandle
 {
 public:
     NeonTensorHandle(const TensorInfo& tensorInfo)
+                     : m_ImportFlags(static_cast<MemorySourceFlags>(MemorySource::Malloc)),
+                       m_Imported(false),
+                       m_IsImportEnabled(false)
     {
         armnn::armcomputetensorutils::BuildArmComputeTensor(m_Tensor, tensorInfo);
     }
 
-    NeonTensorHandle(const TensorInfo& tensorInfo, DataLayout dataLayout)
+    NeonTensorHandle(const TensorInfo& tensorInfo,
+                     DataLayout dataLayout,
+                     MemorySourceFlags importFlags = static_cast<MemorySourceFlags>(MemorySource::Malloc))
+                     : m_ImportFlags(importFlags),
+                       m_Imported(false),
+                       m_IsImportEnabled(false)
+
     {
         armnn::armcomputetensorutils::BuildArmComputeTensor(m_Tensor, tensorInfo, dataLayout);
     }
@@ -38,13 +47,21 @@ public:
 
     virtual void Allocate() override
     {
-        armnn::armcomputetensorutils::InitialiseArmComputeTensorEmpty(m_Tensor);
+        // If we have enabled Importing, don't Allocate the tensor
+        if (!m_IsImportEnabled)
+        {
+            armnn::armcomputetensorutils::InitialiseArmComputeTensorEmpty(m_Tensor);
+        }
     };
 
     virtual void Manage() override
     {
-        BOOST_ASSERT(m_MemoryGroup != nullptr);
-        m_MemoryGroup->manage(&m_Tensor);
+        // If we have enabled Importing, don't manage the tensor
+        if (!m_IsImportEnabled)
+        {
+            BOOST_ASSERT(m_MemoryGroup != nullptr);
+            m_MemoryGroup->manage(&m_Tensor);
+        }
     }
 
     virtual ITensorHandle* GetParent() const override { return nullptr; }
@@ -63,8 +80,8 @@ public:
     {
         return static_cast<const void*>(m_Tensor.buffer() + m_Tensor.info()->offset_first_element_in_bytes());
     }
-    virtual void Unmap() const override {}
 
+    virtual void Unmap() const override {}
 
     TensorShape GetStrides() const override
     {
@@ -76,6 +93,73 @@ public:
         return armcomputetensorutils::GetShape(m_Tensor.info()->tensor_shape());
     }
 
+    void SetImportFlags(MemorySourceFlags importFlags)
+    {
+        m_ImportFlags = importFlags;
+    }
+
+    MemorySourceFlags GetImportFlags() const override
+    {
+        return m_ImportFlags;
+    }
+
+    void SetImportEnabledFlag(bool importEnabledFlag)
+    {
+        m_IsImportEnabled = importEnabledFlag;
+    }
+
+    virtual bool Import(void* memory, MemorySource source) override
+    {
+        if (m_ImportFlags & static_cast<MemorySourceFlags>(source))
+        {
+            if (source == MemorySource::Malloc && m_IsImportEnabled)
+            {
+                // Checks the 16 byte memory alignment
+                constexpr uintptr_t alignment = sizeof(size_t);
+                if (reinterpret_cast<uintptr_t>(memory) % alignment)
+                {
+                    throw MemoryImportException("NeonTensorHandle::Import Attempting to import unaligned memory");
+                }
+
+                // m_Tensor not yet Allocated
+                if (!m_Imported && !m_Tensor.buffer())
+                {
+                    arm_compute::Status status = m_Tensor.allocator()->import_memory(memory);
+                    // Use the overloaded bool operator of Status to check if it worked, if not throw an exception
+                    // with the Status error message
+                    m_Imported = bool(status);
+                    if (!m_Imported)
+                    {
+                        throw MemoryImportException(status.error_description());
+                    }
+                    return m_Imported;
+                }
+
+                // m_Tensor.buffer() initially allocated with Allocate().
+                if (!m_Imported && m_Tensor.buffer())
+                {
+                    throw MemoryImportException(
+                        "NeonTensorHandle::Import Attempting to import on an already allocated tensor");
+                }
+
+                // m_Tensor.buffer() previously imported.
+                if (m_Imported)
+                {
+                    arm_compute::Status status = m_Tensor.allocator()->import_memory(memory);
+                    // Use the overloaded bool operator of Status to check if it worked, if not throw an exception
+                    // with the Status error message
+                    m_Imported = bool(status);
+                    if (!m_Imported)
+                    {
+                        throw MemoryImportException(status.error_description());
+                    }
+                    return m_Imported;
+                }
+            }
+        }
+        return false;
+    }
+
 private:
     // Only used for testing
     void CopyOutTo(void* memory) const override
@@ -131,6 +215,9 @@ private:
 
     arm_compute::Tensor m_Tensor;
     std::shared_ptr<arm_compute::MemoryGroup> m_MemoryGroup;
+    MemorySourceFlags m_ImportFlags;
+    bool m_Imported;
+    bool m_IsImportEnabled;
 };
 
 class NeonSubTensorHandle : public IAclTensorHandle
diff --git a/src/backends/neon/NeonTensorHandleFactory.cpp b/src/backends/neon/NeonTensorHandleFactory.cpp
index ff4e238cd8..8296b8315c 100644
--- a/src/backends/neon/NeonTensorHandleFactory.cpp
+++ b/src/backends/neon/NeonTensorHandleFactory.cpp
@@ -39,19 +39,33 @@ std::unique_ptr<ITensorHandle> NeonTensorHandleFactory::CreateSubTensorHandle(IT
             boost::polymorphic_downcast<IAclTensorHandle*>(&parent), shape, coords);
 }
 
-std::unique_ptr<ITensorHandle> NeonTensorHandleFactory::CreateTensorHandle(const TensorInfo& tensorInfo) const
+std::unique_ptr<ITensorHandle> NeonTensorHandleFactory::CreateTensorHandle(const TensorInfo& tensorInfo,
+                                                                           const bool IsMemoryManaged) const
 {
     auto tensorHandle = std::make_unique<NeonTensorHandle>(tensorInfo);
-    tensorHandle->SetMemoryGroup(m_MemoryManager->GetInterLayerMemoryGroup());
+    if (IsMemoryManaged)
+    {
+        tensorHandle->SetMemoryGroup(m_MemoryManager->GetInterLayerMemoryGroup());
+    }
+    // If we are not Managing the Memory then we must be importing
+    tensorHandle->SetImportEnabledFlag(!IsMemoryManaged);
+    tensorHandle->SetImportFlags(m_ImportFlags);
 
     return tensorHandle;
 }
 
 std::unique_ptr<ITensorHandle> NeonTensorHandleFactory::CreateTensorHandle(const TensorInfo& tensorInfo,
-                                                                           DataLayout dataLayout) const
+                                                                           DataLayout dataLayout,
+                                                                           const bool IsMemoryManaged) const
 {
     auto tensorHandle = std::make_unique<NeonTensorHandle>(tensorInfo, dataLayout);
-    tensorHandle->SetMemoryGroup(m_MemoryManager->GetInterLayerMemoryGroup());
+    if (IsMemoryManaged)
+    {
+        tensorHandle->SetMemoryGroup(m_MemoryManager->GetInterLayerMemoryGroup());
+    }
+    // If we are not Managing the Memory then we must be importing
+    tensorHandle->SetImportEnabledFlag(!IsMemoryManaged);
+    tensorHandle->SetImportFlags(m_ImportFlags);
 
     return tensorHandle;
 }
diff --git a/src/backends/neon/NeonTensorHandleFactory.hpp b/src/backends/neon/NeonTensorHandleFactory.hpp
index 82e388e629..b03433352e 100644
--- a/src/backends/neon/NeonTensorHandleFactory.hpp
+++ b/src/backends/neon/NeonTensorHandleFactory.hpp
@@ -18,18 +18,20 @@ class NeonTensorHandleFactory : public ITensorHandleFactory
 public:
     NeonTensorHandleFactory(std::weak_ptr<NeonMemoryManager> mgr)
                             : m_MemoryManager(mgr),
-                              m_ImportFlags(static_cast<MemorySourceFlags>(MemorySource::Undefined)),
-                              m_ExportFlags(static_cast<MemorySourceFlags>(MemorySource::Undefined))
+                              m_ImportFlags(static_cast<MemorySourceFlags>(MemorySource::Malloc)),
+                              m_ExportFlags(static_cast<MemorySourceFlags>(MemorySource::Malloc))
     {}
 
     std::unique_ptr<ITensorHandle> CreateSubTensorHandle(ITensorHandle& parent,
                                                          const TensorShape& subTensorShape,
                                                          const unsigned int* subTensorOrigin) const override;
 
-    std::unique_ptr<ITensorHandle> CreateTensorHandle(const TensorInfo& tensorInfo) const override;
+    std::unique_ptr<ITensorHandle> CreateTensorHandle(const TensorInfo& tensorInfo,
+                                                      const bool IsMemoryManaged = true) const override;
 
     std::unique_ptr<ITensorHandle> CreateTensorHandle(const TensorInfo& tensorInfo,
-                                                      DataLayout dataLayout) const override;
+                                                      DataLayout dataLayout,
+                                                      const bool IsMemoryManaged = true) const override;
 
     static const FactoryId& GetIdStatic();
 
diff --git a/src/backends/neon/NeonWorkloadFactory.cpp b/src/backends/neon/NeonWorkloadFactory.cpp
index 441e27f5cf..fb81008644 100644
--- a/src/backends/neon/NeonWorkloadFactory.cpp
+++ b/src/backends/neon/NeonWorkloadFactory.cpp
@@ -72,20 +72,26 @@ std::unique_ptr<ITensorHandle> NeonWorkloadFactory::CreateSubTensorHandle(ITenso
         boost::polymorphic_downcast<IAclTensorHandle*>(&parent), shape, coords);
 }
 
-std::unique_ptr<ITensorHandle> NeonWorkloadFactory::CreateTensorHandle(const TensorInfo& tensorInfo) const
+std::unique_ptr<ITensorHandle> NeonWorkloadFactory::CreateTensorHandle(const TensorInfo& tensorInfo,
+                                                                       const bool IsMemoryManaged) const
 {
     auto tensorHandle = std::make_unique<NeonTensorHandle>(tensorInfo);
-    tensorHandle->SetMemoryGroup(m_MemoryManager->GetInterLayerMemoryGroup());
-
+    if (IsMemoryManaged)
+    {
+        tensorHandle->SetMemoryGroup(m_MemoryManager->GetInterLayerMemoryGroup());
+    }
     return tensorHandle;
 }
 
 std::unique_ptr<ITensorHandle> NeonWorkloadFactory::CreateTensorHandle(const TensorInfo& tensorInfo,
-                                                                       DataLayout dataLayout) const
+                                                                       DataLayout dataLayout,
+                                                                       const bool IsMemoryManaged) const
 {
     auto tensorHandle = std::make_unique<NeonTensorHandle>(tensorInfo, dataLayout);
-    tensorHandle->SetMemoryGroup(m_MemoryManager->GetInterLayerMemoryGroup());
-
+    if (IsMemoryManaged)
+    {
+        tensorHandle->SetMemoryGroup(m_MemoryManager->GetInterLayerMemoryGroup());
+    }
     return tensorHandle;
 }
 
diff --git a/src/backends/neon/NeonWorkloadFactory.hpp b/src/backends/neon/NeonWorkloadFactory.hpp
index 935f9e4b4c..0bee365d8a 100644
--- a/src/backends/neon/NeonWorkloadFactory.hpp
+++ b/src/backends/neon/NeonWorkloadFactory.hpp
@@ -32,10 +32,12 @@ public:
                                                          TensorShape const& subTensorShape,
                                                          unsigned int const* subTensorOrigin) const override;
 
-    std::unique_ptr<ITensorHandle> CreateTensorHandle(const TensorInfo& tensorInfo) const override;
+    std::unique_ptr<ITensorHandle> CreateTensorHandle(const TensorInfo& tensorInfo,
+                                                      const bool IsMemoryManaged = true) const override;
 
     std::unique_ptr<ITensorHandle> CreateTensorHandle(const TensorInfo& tensorInfo,
-                                                      DataLayout dataLayout) const override;
+                                                      DataLayout dataLayout,
+                                                      const bool IsMemoryManaged = true) const override;
 
     std::unique_ptr<IWorkload> CreateInput(const InputQueueDescriptor& descriptor,
                                            const WorkloadInfo& info) const override;
diff --git a/src/backends/neon/test/NeonEndToEndTests.cpp b/src/backends/neon/test/NeonEndToEndTests.cpp
index eb41e949b3..a09b95ed0e 100644
--- a/src/backends/neon/test/NeonEndToEndTests.cpp
+++ b/src/backends/neon/test/NeonEndToEndTests.cpp
@@ -312,4 +312,113 @@ BOOST_AUTO_TEST_CASE(NeonTransposeConvolution2dEndToEndUint8NhwcTest)
         defaultBackends, armnn::DataLayout::NHWC);
 }
 
+BOOST_AUTO_TEST_CASE(NeonImportNonAlignedInputPointerTest)
+{
+    ImportNonAlignedInputPointerTest(defaultBackends);
+}
+
+// Utility function to find the number of instances of a substring within a string.
+int SubStringCounter(std::string& string, std::string&& substring)
+{
+    std::size_t found = 0;
+    int count = 0;
+    // Look for the substring starting from where we last found the substring
+    while((found = string.find(substring, found)) != std::string::npos)
+    {
+        count++;
+        // Offset by substring length to avoid finding the same substring twice
+        found += substring.length();
+    }
+    return count;
+}
+
+BOOST_AUTO_TEST_CASE(NeonImportOnlyWorkload)
+{
+    using namespace armnn;
+
+    IRuntime::CreationOptions options;
+    IRuntimePtr runtime(IRuntime::Create(options));
+
+    // Builds up the structure of the network.
+    INetworkPtr net(INetwork::Create());
+
+    IConnectableLayer* input = net->AddInputLayer(0);
+
+    ActivationDescriptor descriptor;
+    descriptor.m_Function = ActivationFunction::Square;
+    IConnectableLayer* pooling = net->AddActivationLayer(descriptor);
+
+    IConnectableLayer* output = net->AddOutputLayer(0);
+
+    input->GetOutputSlot(0).Connect(pooling->GetInputSlot(0));
+    pooling->GetOutputSlot(0).Connect(output->GetInputSlot(0));
+
+    input->GetOutputSlot(0).SetTensorInfo(TensorInfo({ 1, 1, 1, 4 }, DataType::Float32));
+    pooling->GetOutputSlot(0).SetTensorInfo(TensorInfo({ 1, 1, 1, 4 }, DataType::Float32));
+
+    // optimize the network
+    std::vector<BackendId> backends = {Compute::CpuAcc};
+    IOptimizedNetworkPtr optNet = Optimize(*net, backends, runtime->GetDeviceSpec());
+
+    BOOST_TEST_CHECKPOINT("Load Network");
+    // Load it into the runtime. It should pass.
+    NetworkId netId;
+    std::string ignoredErrorMessage;
+    INetworkProperties networkProperties(true, false);
+    BOOST_TEST(runtime->LoadNetwork(netId, std::move(optNet),ignoredErrorMessage, networkProperties)
+               == Status::Success);
+
+    BOOST_TEST_CHECKPOINT("Generate Data");
+    // Creates structures for input & output
+    std::vector<float> inputData
+    {
+        1.0f, 2.0f, 3.0f, 4.0f
+    };
+
+    std::vector<float> outputData(4);
+
+    std::vector<float> expectedOutput
+    {
+         1.0f, 4.0f, 9.0f, 16.0f
+    };
+
+    BOOST_TEST_CHECKPOINT("Create Network");
+    InputTensors inputTensors
+    {
+        {0,armnn::ConstTensor(runtime->GetInputTensorInfo(netId, 0), inputData.data())},
+    };
+    OutputTensors outputTensors
+    {
+        {0,armnn::Tensor(runtime->GetOutputTensorInfo(netId, 0), outputData.data())}
+    };
+
+    BOOST_TEST_CHECKPOINT("Get Profiler");
+
+    runtime->GetProfiler(netId)->EnableProfiling(true);
+
+    BOOST_TEST_CHECKPOINT("Run Inference");
+    // Do the inference
+    runtime->EnqueueWorkload(netId, inputTensors, outputTensors);
+
+    BOOST_TEST_CHECKPOINT("Print Profiler");
+    // Retrieve the Profiler.Print() output to get the workload execution
+    ProfilerManager& profilerManager = armnn::ProfilerManager::GetInstance();
+    std::stringstream ss;
+    profilerManager.GetProfiler()->Print(ss);;
+    std::string dump = ss.str();
+
+    // Check there are no SyncMemGeneric workloads as we didn't export
+    BOOST_TEST_CHECKPOINT("Find SyncMemGeneric");
+    int count = SubStringCounter(dump, "SyncMemGeneric");
+    BOOST_TEST(count == 0);
+
+    // Should only be 1 CopyMemGeneric for the output as we imported
+    BOOST_TEST_CHECKPOINT("Find CopyMemGeneric");
+    count = SubStringCounter(dump, "CopyMemGeneric");
+    BOOST_TEST(count == 1);
+
+    // Check the output is correct
+    BOOST_CHECK_EQUAL_COLLECTIONS(outputData.begin(), outputData.end(), expectedOutput.begin(), expectedOutput.end());
+}
+
 BOOST_AUTO_TEST_SUITE_END()