IVGCVSW-6114 Create multiple LoadedNetworks from one OptimizedNetwork

 * Added IOptimizedNetwork constructor that takes another
   IOptimizedNetwork and a ModelOptions.
 * Changed PreCompiledLayer to use shared_ptr rather than unique_ptr
   to store the PreCompiledObject (no interface changes).
 * Added unit tests to ensure that PreCompiledLayer::Clone() clones
   the pointer to the PreCompiledObject correctly.

Signed-off-by: Mike Kelly <mike.kelly@arm.com>
Change-Id: I3ef56055e0d189ffce9e651882d34da16c70a240

9 files changed, 272 insertions, 3 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 78c2f17e6d..9ac7ecffaa 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -561,6 +561,7 @@ set_target_properties(armnn PROPERTIES VERSION ${GENERIC_LIB_VERSION} SOVERSION
 if(BUILD_UNIT_TESTS)
     set(unittest_sources)
     list(APPEND unittest_sources
+        src/armnn/test/CloneTests.cpp
         src/armnn/test/ConstTensorLayerVisitor.hpp
         src/armnn/test/ConstTensorLayerVisitor.cpp
         src/armnn/test/EndToEndTest.cpp
diff --git a/include/armnn/INetwork.hpp b/include/armnn/INetwork.hpp
index 3c45262f1d..b40db62a59 100644
--- a/include/armnn/INetwork.hpp
+++ b/include/armnn/INetwork.hpp
@@ -731,6 +731,9 @@ public:
 
     profiling::ProfilingGuid GetGuid() const;
 
+    // Creates a copy of the IOptimizedNetwork. The IOptimizedNetwork will not be reoptimized,
+    // the provided ModelOptions will only be used when creating a LoadedNetwork. 
+    IOptimizedNetwork(const IOptimizedNetwork& other, const ModelOptions& modelOptions);
     IOptimizedNetwork(std::unique_ptr<Graph> graph);
     IOptimizedNetwork(std::unique_ptr<OptimizedNetworkImpl> impl);
     ~IOptimizedNetwork();
diff --git a/src/armnn/Network.cpp b/src/armnn/Network.cpp
index 71f19313b8..74c195f676 100644
--- a/src/armnn/Network.cpp
+++ b/src/armnn/Network.cpp
@@ -543,6 +543,8 @@ void INetwork::Destroy(INetwork* network)
     delete network;
 }
 
+IOptimizedNetwork::IOptimizedNetwork(const IOptimizedNetwork& other, const ModelOptions& modelOptions)
+    : pOptimizedNetworkImpl(new OptimizedNetworkImpl(*other.pOptimizedNetworkImpl.get(), modelOptions)) {}
 
 IOptimizedNetwork::IOptimizedNetwork(std::unique_ptr<Graph> graph)
     : pOptimizedNetworkImpl(new OptimizedNetworkImpl(std::move(graph))) {}
@@ -2621,6 +2623,13 @@ void NetworkImpl::ExecuteStrategy(IStrategy& strategy) const
     };
 }
 
+OptimizedNetworkImpl::OptimizedNetworkImpl(const OptimizedNetworkImpl& other, const ModelOptions& modelOptions)
+    : m_Graph(new Graph(*other.m_Graph.get()))
+    , m_Guid(profiling::ProfilingService::GetNextGuid())
+    , m_ModelOptions(modelOptions)
+{
+}
+
 OptimizedNetworkImpl::OptimizedNetworkImpl(std::unique_ptr<Graph> graph)
     : m_Graph(std::move(graph)), m_Guid(profiling::ProfilingService::GetNextGuid())
 {
diff --git a/src/armnn/OptimizedNetworkImpl.hpp b/src/armnn/OptimizedNetworkImpl.hpp
index fe55ca233b..d42cff7346 100644
--- a/src/armnn/OptimizedNetworkImpl.hpp
+++ b/src/armnn/OptimizedNetworkImpl.hpp
@@ -11,6 +11,7 @@ namespace armnn
 class OptimizedNetworkImpl
 {
 public:
+    OptimizedNetworkImpl(const OptimizedNetworkImpl& other, const ModelOptions& modelOptions);
     OptimizedNetworkImpl(std::unique_ptr<Graph> graph);
     OptimizedNetworkImpl(std::unique_ptr<Graph> graph, const ModelOptions& modelOptions);
     virtual ~OptimizedNetworkImpl();
diff --git a/src/armnn/layers/PreCompiledLayer.cpp b/src/armnn/layers/PreCompiledLayer.cpp
index 75c1e46a84..0cc9c5a75a 100644
--- a/src/armnn/layers/PreCompiledLayer.cpp
+++ b/src/armnn/layers/PreCompiledLayer.cpp
@@ -24,7 +24,7 @@ PreCompiledLayer::~PreCompiledLayer()
 PreCompiledLayer* PreCompiledLayer::Clone(Graph& graph) const
 {
     PreCompiledLayer* clone = CloneBase<PreCompiledLayer>(graph, m_Param, GetName());
-    clone->m_PreCompiledObject.reset(const_cast<PreCompiledLayer*>(this)->m_PreCompiledObject.release());
+    clone->m_PreCompiledObject = const_cast<PreCompiledLayer*>(this)->m_PreCompiledObject;
     return clone;
 }
 
@@ -46,7 +46,7 @@ void PreCompiledLayer::ValidateTensorShapesFromInputs()
 
 void PreCompiledLayer::SetPreCompiledObject(PreCompiledObjectPtr preCompiledObject)
 {
-    m_PreCompiledObject = std::move(preCompiledObject);
+    m_PreCompiledObject = std::make_shared<const void*>(preCompiledObject.release());
 }
 
 void PreCompiledLayer::Accept(ILayerVisitor& visitor) const
diff --git a/src/armnn/layers/PreCompiledLayer.hpp b/src/armnn/layers/PreCompiledLayer.hpp
index 2ed87578a4..6a8ac683ac 100644
--- a/src/armnn/layers/PreCompiledLayer.hpp
+++ b/src/armnn/layers/PreCompiledLayer.hpp
@@ -41,7 +41,7 @@ private:
     PreCompiledLayer(const PreCompiledLayer& other) = delete;
     PreCompiledLayer& operator=(const PreCompiledLayer& other) = delete;
 
-    PreCompiledObjectPtr m_PreCompiledObject;
+    std::shared_ptr<const void*> m_PreCompiledObject;
 };
 
 } // namespace armnn
diff --git a/src/armnn/test/CloneTests.cpp b/src/armnn/test/CloneTests.cpp
new file mode 100644
index 0000000000..2ee2cdad0e
--- /dev/null
+++ b/src/armnn/test/CloneTests.cpp
@@ -0,0 +1,103 @@
+//
+// Copyright © 2021 Arm Ltd and Contributors. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#include <Graph.hpp>
+#include <Layer.hpp>
+
+#include <armnn/TypesUtils.hpp>
+#include <armnn/Exceptions.hpp>
+#include <armnn/Optional.hpp>
+#include <armnn/backends/IBackendInternal.hpp>
+#include <armnn/utility/PolymorphicDowncast.hpp>
+
+#include <backendsCommon/TensorHandle.hpp>
+#include <backendsCommon/WorkloadFactoryBase.hpp>
+
+#include <doctest/doctest.h>
+
+namespace {
+
+const armnn::BackendId& GetCloneIdStatic()
+{
+    static const armnn::BackendId s_Id{"Tests"};
+    return s_Id;
+}
+
+class TestWorkloadFactory : public armnn::WorkloadFactoryBase
+{
+public:
+
+    TestWorkloadFactory()
+        : m_Ptr(nullptr)
+    {}
+
+    const armnn::BackendId& GetBackendId() const override
+    {
+        return GetCloneIdStatic();
+    }
+
+    std::unique_ptr<armnn::IWorkload> CreatePreCompiled(const armnn::PreCompiledQueueDescriptor& descriptor,
+                                                        const armnn::WorkloadInfo&) const override
+    {
+        if (m_Ptr)
+        {
+            CHECK(descriptor.m_PreCompiledObject == m_Ptr);
+        }
+        else
+        {
+            m_Ptr = descriptor.m_PreCompiledObject;
+        }
+        return nullptr;
+    }
+
+    mutable void* m_Ptr;
+};
+
+TEST_SUITE("CloneTests")
+{
+
+TEST_CASE ("PreCompiledLayerClonePreservesObject")
+{
+    armnn::Graph graph1;
+    armnn::Graph graph2;
+
+    armnn::PreCompiledDescriptor descriptor(0u, 0u);
+
+    armnn::Layer* const preCompiledLayer = graph1.AddLayer<armnn::PreCompiledLayer>(descriptor, "preCompiled");
+    armnn::PreCompiledLayer* layer = armnn::PolymorphicDowncast<armnn::PreCompiledLayer*>(preCompiledLayer);
+
+    armnn::PreCompiledObjectPtr payloadObject;
+    TestWorkloadFactory factory;
+
+    layer->SetPreCompiledObject(std::move(payloadObject));
+    layer->CreateWorkload(factory);
+
+    armnn::PreCompiledLayer* clone = layer->Clone(graph2);
+    CHECK(std::strcmp(clone->GetName(), "preCompiled") == 0);
+    clone->CreateWorkload(factory);
+}
+
+TEST_CASE ("PreCompiledLayerCloneNoObject")
+{
+    armnn::Graph graph1;
+
+    armnn::Graph graph2;
+
+    armnn::PreCompiledDescriptor descriptor(0u, 0u);
+
+    armnn::Layer* const preCompiledLayer = graph1.AddLayer<armnn::PreCompiledLayer>(descriptor, "preCompiled");
+    armnn::PreCompiledLayer* layer = armnn::PolymorphicDowncast<armnn::PreCompiledLayer*>(preCompiledLayer);
+
+    TestWorkloadFactory factory;
+    layer->CreateWorkload(factory);
+
+    armnn::PreCompiledLayer* clone = layer->Clone(graph2);
+    CHECK(std::strcmp(clone->GetName(), "preCompiled") == 0);
+    clone->CreateWorkload(factory);
+}
+
+}
+
+} // end anonymous namespace
diff --git a/src/backends/backendsCommon/test/CommonTestUtils.hpp b/src/backends/backendsCommon/test/CommonTestUtils.hpp
index 99412b9694..c7537f1eed 100644
--- a/src/backends/backendsCommon/test/CommonTestUtils.hpp
+++ b/src/backends/backendsCommon/test/CommonTestUtils.hpp
@@ -18,6 +18,8 @@
 #include <test/TestUtils.hpp>
 
 #include <algorithm>
+#include <random>
+#include <vector>
 
 // Checks that two collections have the exact same contents (in any order)
 // The given collections do not have to contain duplicates
@@ -94,3 +96,24 @@ armnn::TensorShape MakeTensorShape(unsigned int batches,
                                    unsigned int height,
                                    unsigned int width,
                                    armnn::DataLayout layout);
+
+template<typename DataType>
+static std::vector<DataType> GenerateRandomData(size_t size)
+{
+    constexpr bool isIntegerType = std::is_integral<DataType>::value;
+    using Distribution =
+    typename std::conditional<isIntegerType,
+            std::uniform_int_distribution<DataType>,
+            std::uniform_real_distribution<DataType>>::type;
+
+    static constexpr DataType lowerLimit = std::numeric_limits<DataType>::min();
+    static constexpr DataType upperLimit = std::numeric_limits<DataType>::max();
+
+    static Distribution distribution(lowerLimit, upperLimit);
+    static std::default_random_engine generator;
+
+    std::vector<DataType> randomData(size);
+    generate(randomData.begin(), randomData.end(), []() { return distribution(generator); });
+
+    return randomData;
+}
diff --git a/src/backends/backendsCommon/test/OptimizedNetworkTests.cpp b/src/backends/backendsCommon/test/OptimizedNetworkTests.cpp
index 2c74690e6e..dcea9ef72e 100644
--- a/src/backends/backendsCommon/test/OptimizedNetworkTests.cpp
+++ b/src/backends/backendsCommon/test/OptimizedNetworkTests.cpp
@@ -3,6 +3,8 @@
 // SPDX-License-Identifier: MIT
 //
 
+#include "CommonTestUtils.hpp"
+
 #include <Graph.hpp>
 #include <Network.hpp>
 
@@ -358,4 +360,131 @@ TEST_CASE("OptimizeValidateWorkloadsDuplicateComputeDeviceWithFallback")
     }
 }
 
+TEST_CASE("OptimizeNetworkCopy")
+{
+    armnn::IRuntime::CreationOptions options;
+    armnn::IRuntimePtr runtime = armnn::IRuntime::Create(options);
+    std::vector<armnn::NetworkId> networkIds;
+
+    const std::string layerName("convolution2d");
+    const armnn::TensorInfo inputInfo ({ 1, 5, 5, 1 }, armnn::DataType::Float32);
+    const armnn::TensorInfo outputInfo({ 1, 2, 2, 1 }, armnn::DataType::Float32);
+
+    const armnn::TensorInfo weightsInfo({ 1, 3, 3, 1 }, armnn::DataType::Float32);
+    const armnn::TensorInfo biasesInfo ({ 1 }, armnn::DataType::Float32);
+
+    std::vector<float> weightsData = GenerateRandomData<float>(weightsInfo.GetNumElements());
+    armnn::ConstTensor weights(weightsInfo, weightsData);
+
+    std::vector<float> biasesData = GenerateRandomData<float>(biasesInfo.GetNumElements());
+    armnn::ConstTensor biases(biasesInfo, biasesData);
+
+    armnn::Convolution2dDescriptor descriptor;
+    descriptor.m_PadLeft     = 1;
+    descriptor.m_PadRight    = 1;
+    descriptor.m_PadTop      = 1;
+    descriptor.m_PadBottom   = 1;
+    descriptor.m_StrideX     = 2;
+    descriptor.m_StrideY     = 2;
+    descriptor.m_DilationX   = 2;
+    descriptor.m_DilationY   = 2;
+    descriptor.m_BiasEnabled = true;
+    descriptor.m_DataLayout  = armnn::DataLayout::NHWC;
+
+    armnn::INetworkPtr network = armnn::INetwork::Create();
+    armnn::IConnectableLayer* const inputLayer  = network->AddInputLayer(0);
+    armnn::IConnectableLayer* const convLayer   =
+            network->AddConvolution2dLayer(descriptor,
+                                           weights,
+                                           armnn::Optional<armnn::ConstTensor>(biases),
+                                           layerName.c_str());
+    armnn::IConnectableLayer* const outputLayer = network->AddOutputLayer(0);
+
+    inputLayer->GetOutputSlot(0).Connect(convLayer->GetInputSlot(0));
+    convLayer->GetOutputSlot(0).Connect(outputLayer->GetInputSlot(0));
+
+    inputLayer->GetOutputSlot(0).SetTensorInfo(inputInfo);
+    convLayer->GetOutputSlot(0).SetTensorInfo(outputInfo);
+
+    std::vector<armnn::BackendId> preferredBackends { "CpuRef" };
+    armnn::ModelOptions modelOptions;
+    armnn::OptimizerOptions optimizerOptions(false, false, false, false, modelOptions);
+    std::vector<std::string> errorMessages;
+
+    // optimize the network.
+    armnn::IOptimizedNetworkPtr optNet = Optimize(*network,
+                                                  preferredBackends,
+                                                  runtime->GetDeviceSpec(),
+                                                  optimizerOptions,
+                                                  armnn::Optional<std::vector<std::string>&>(errorMessages));
+
+    for (unsigned int i = 0; i < 2; ++i)
+    {
+        armnn::ModelOptions optimizedModelOptions;
+        auto copy = armnn::IOptimizedNetworkPtr(new armnn::IOptimizedNetwork(*optNet.get(), optimizedModelOptions),
+                                               &armnn::IOptimizedNetwork::Destroy);
+
+        CHECK(copy);
+
+        armnn::NetworkId netId;
+        std::string errorMessage;
+
+        CHECK(armnn::Status::Success == runtime->LoadNetwork(netId, std::move(copy), errorMessage));
+
+        // Record the networkID for the loaded network
+        networkIds.emplace_back(netId);
+    }
+    armnn::NetworkId optNetId;
+    std::string errorMessage;
+
+    // Load the original optNet
+    CHECK(armnn::Status::Success == runtime->LoadNetwork(optNetId, std::move(optNet), errorMessage));
+
+    std::vector<float> inputData = GenerateRandomData<float>(runtime->GetInputTensorInfo(optNetId, 0).GetNumElements());
+    std::vector<float> outputData(runtime->GetOutputTensorInfo(optNetId, 0).GetNumElements());
+
+    armnn::InputTensors inputTensors
+    {
+        {
+            0 ,armnn::ConstTensor(runtime->GetInputTensorInfo(optNetId, 0), inputData.data())
+        }
+    };
+    armnn::OutputTensors outputTensors
+    {
+        {
+            0, armnn::Tensor(runtime->GetOutputTensorInfo(optNetId, 0), outputData.data())
+        }
+    };
+    runtime->EnqueueWorkload(optNetId, inputTensors, outputTensors);
+    runtime->UnloadNetwork(optNetId);
+
+    // Record the networkID for the loaded network
+    for (unsigned int i = 0; i < networkIds.size(); ++i)
+    {
+        armnn::NetworkId netId = networkIds[i];
+        std::vector<float> copyOutputData(runtime->GetOutputTensorInfo(netId, 0).GetNumElements());
+
+        armnn::InputTensors copyInputTensors
+        {
+            {
+                0, armnn::ConstTensor(runtime->GetInputTensorInfo(netId, 0), inputData.data())
+            }
+        };
+        armnn::OutputTensors copyOutputTensors
+        {
+            {
+                0, armnn::Tensor(runtime->GetOutputTensorInfo(netId, 0), copyOutputData.data())
+            }
+        };
+        runtime->EnqueueWorkload(netId, copyInputTensors, copyOutputTensors);
+        runtime->UnloadNetwork(netId);
+
+        // Check results are identical to "original" version
+        for (unsigned int j = 0; j < outputData.size(); ++j)
+        {
+            CHECK(outputData[j] == copyOutputData[j]);
+        }
+    }
+}
+
 }