IVGCVSW-6127 ConstTensorsAsInput: DepthwiseConvolution2d

!android-nn-driver:7418

 * Update Front-end and Tools.
 * Updated Serializer, Deserializer and unit tests to reflect this.
 * Updated TfLiteDelegate, TfLiteParser and OnnxParser.
 * Change NNDriver to new API.
 * Updated Ref.
 * Neon and Cl backend partially completed (Backend.cpp files).
 * Added dynamic or constant input EndToEnd tests.
 * Added ConstantTensorAsInputMemeberVariableRedirect Optimization.

Signed-off-by: Cathal Corbett <cathal.corbett@arm.com>
Change-Id: Ib18b6c10a093042e165e25237dc04a4c67ba82da

21 files changed, 463 insertions, 187 deletions

diff --git a/src/armnn/BackendHelper.cpp b/src/armnn/BackendHelper.cpp
index e2aa67275f..03f32ac191 100644
--- a/src/armnn/BackendHelper.cpp
+++ b/src/armnn/BackendHelper.cpp
@@ -439,6 +439,32 @@ bool LayerSupportHandle::IsDepthwiseConvolutionSupported(
     TensorInfo biasesVal =  biases.has_value() ? biases.value() : TensorInfo();
     TensorInfos infos{input, output, weights, biasesVal};
 
+    Optional<const BackendOptions::BackendOption> capability ;
+    if(!m_BackendId.IsUndefined())
+    {
+        capability = GetCapability("ConstantTensorsAsInputs", m_BackendId);
+        if(!capability.has_value() || capability.value().GetValue().AsBool() == false)
+        {
+            if(!weights.IsConstant())
+            {
+                return false;
+            }
+            if(descriptor.m_BiasEnabled)
+            {
+                if(!biases.value().IsConstant())
+                {
+                    return false;
+                }
+            }
+            // At the first stage we will only print a warning. this is to give
+            // backend developers a chance to adopt and read weights from input slots.
+            ARMNN_LOG(warning) << "The backend makes use of a deprecated interface to read constant tensors. "
+                                  "If you are a backend developer please find more information in our "
+                                  "doxygen documentation on github https://github.com/ARM-software/armnn "
+                                  "under the keyword 'ConstTensorsAsInputs'.";
+        }
+    }
+
     return m_LayerSupport->IsLayerSupported(LayerType::DepthwiseConvolution2d,
                                             infos,
                                             descriptor,
@@ -492,6 +518,32 @@ bool LayerSupportHandle::IsDilatedDepthwiseConvolutionSupported(
     TensorInfo biasesVal =  biases.has_value() ? biases.value() : TensorInfo();
     TensorInfos infos{input, output, weights, biasesVal};
 
+    Optional<const BackendOptions::BackendOption> capability ;
+    if(!m_BackendId.IsUndefined())
+    {
+        capability = GetCapability("ConstantTensorsAsInputs", m_BackendId);
+        if(!capability.has_value() || capability.value().GetValue().AsBool() == false)
+        {
+            if(!weights.IsConstant())
+            {
+                return false;
+            }
+            if(descriptor.m_BiasEnabled)
+            {
+                if(!biases.value().IsConstant())
+                {
+                    return false;
+                }
+            }
+            // At the first stage we will only print a warning. this is to give
+            // backend developers a chance to adopt and read weights from input slots.
+            ARMNN_LOG(warning) << "The backend makes use of a deprecated interface to read constant tensors. "
+                                  "If you are a backend developer please find more information in our "
+                                  "doxygen documentation on github https://github.com/ARM-software/armnn "
+                                  "under the keyword 'ConstTensorsAsInputs'.";
+        }
+    }
+
     return m_LayerSupport->IsLayerSupported(LayerType::DepthwiseConvolution2d,
                                             infos,
                                             descriptor,
@@ -590,8 +642,8 @@ bool LayerSupportHandle::IsFullyConnectedSupported(const TensorInfo& input,
                 if (reasonIfUnsupported.has_value())
                 {
                     reasonIfUnsupported.value() =
-                            "This backend might not support non constant weights. "
-                            "If weights are constant make sure to set IsConstant when creating TensorInfo";
+                        "This backend might not support non constant weights. "
+                        "If weights are constant make sure to set IsConstant when creating TensorInfo";
                 }
 
                 return false;
@@ -603,8 +655,8 @@ bool LayerSupportHandle::IsFullyConnectedSupported(const TensorInfo& input,
                     if (reasonIfUnsupported.has_value())
                     {
                         reasonIfUnsupported.value() =
-                                "This backend might not support non constant weights. "
-                                "If weights are constant make sure to set IsConstant when creating TensorInfo";
+                            "This backend might not support non constant bias. "
+                            "If bias are constant make sure to set IsConstant when creating TensorInfo";
                     }
                     return false;
                 }
diff --git a/src/armnn/Descriptors.cpp b/src/armnn/Descriptors.cpp
index ef55ee7bb5..d67d4404e0 100644
--- a/src/armnn/Descriptors.cpp
+++ b/src/armnn/Descriptors.cpp
@@ -452,4 +452,15 @@ uint32_t Convolution3dDescriptor::GetNumInputs() const
     return numInputs;
 }
 
+uint32_t DepthwiseConvolution2dDescriptor::GetNumInputs() const
+{
+    // Return 2 otherwise check if bias is enabled
+    unsigned int numInputs = 2;
+    if (m_BiasEnabled)
+    {
+        numInputs = 3;
+    }
+    return numInputs;
+}
+
 }
diff --git a/src/armnn/Graph.cpp b/src/armnn/Graph.cpp
index 1bea6cc2ae..c1cec482b6 100644
--- a/src/armnn/Graph.cpp
+++ b/src/armnn/Graph.cpp
@@ -603,7 +603,8 @@ void Graph::ConstructErrorMessageForUnconnectedInputs(Layer* const layer,
     bool noWeightsAndBias = false;
 
     if ((layer->GetType() == armnn::LayerType::FullyConnected ||
-         layer->GetType() == armnn::LayerType::Convolution3d) && slotIndex > 0)
+         layer->GetType() == armnn::LayerType::Convolution3d  ||
+         layer->GetType() == armnn::LayerType::DepthwiseConvolution2d) && slotIndex > 0)
     {
         // If weights are not set and is bias enabled, also check if bias is set
         if (slotIndex == 1 && layer->GetNumInputSlots() == 3)
diff --git a/src/armnn/Layer.cpp b/src/armnn/Layer.cpp
index 805612d1cb..a31119b395 100644
--- a/src/armnn/Layer.cpp
+++ b/src/armnn/Layer.cpp
@@ -23,6 +23,19 @@ namespace armnn
 // Instantiate the static member variable
 NullDescriptor Layer::m_NullDescriptor;
 
+template <typename LayerT>
+void AssertMultipleInputSlots(Layer& layer)
+{
+    if(PolymorphicDowncast<const LayerT*>(&(layer.GetParameters()))->m_BiasEnabled)
+    {
+        ARMNN_ASSERT(layer.GetNumInputSlots() == 3);
+    }
+    else
+    {
+        ARMNN_ASSERT(layer.GetNumInputSlots() == 2);
+    }
+}
+
 void InputSlot::Insert(Layer& layer)
 {
     ARMNN_ASSERT(layer.GetNumOutputSlots() == 1);
@@ -34,8 +47,21 @@ void InputSlot::Insert(Layer& layer)
         // Disconnects parent from this.
         prevSlot->Disconnect(*this);
 
+        switch (layer.GetType())
+        {
+            case LayerType::DepthwiseConvolution2d:
+            {
+                AssertMultipleInputSlots<DepthwiseConvolution2dDescriptor>(layer);
+                break;
+            }
+            default:
+            {
+                ARMNN_ASSERT(layer.GetNumInputSlots() == 1);
+                break;
+            }
+        }
+
         // Connects inserted layer to parent.
-        ARMNN_ASSERT(layer.GetNumInputSlots() == 1);
         int idx = prevSlot->Connect(layer.GetInputSlot(0));
         prevSlot->SetEdgeStrategy(armnn::numeric_cast<unsigned int>(idx), EdgeStrategy::Undefined);
 
diff --git a/src/armnn/LoadedNetwork.cpp b/src/armnn/LoadedNetwork.cpp
index f10fb89e15..a88fa5ab9c 100644
--- a/src/armnn/LoadedNetwork.cpp
+++ b/src/armnn/LoadedNetwork.cpp
@@ -272,6 +272,8 @@ LoadedNetwork::LoadedNetwork(std::unique_ptr<IOptimizedNetwork> net,
         timelineUtils->MarkEntityWithLabel(networkGuid, ss.str(), LabelsAndEventClasses::PROCESS_ID_GUID);
     }
 
+    std::vector<IWorkload*> ConstWorkloads;
+
     //Then create workloads.
     {
         ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "LoadNetwork_CreateWorkloads");
@@ -325,6 +327,11 @@ LoadedNetwork::LoadedNetwork(std::unique_ptr<IOptimizedNetwork> net,
                     else
                     {
                         m_WorkloadQueue.push_back(std::move(workload));
+
+                        if (layer->GetType() == LayerType::Constant)
+                        {
+                            ConstWorkloads.push_back(m_WorkloadQueue.back().get());
+                        }
                     }
 
                     // release the constant data in the layer..
@@ -506,6 +513,17 @@ LoadedNetwork::LoadedNetwork(std::unique_ptr<IOptimizedNetwork> net,
             AllocateAndExecuteConstantWorkloadsAsync();
         }
     }
+
+    // If synchronous, execute all constant layer workloads as the FoldPad optimization
+    // may have created a new conv2d layer prior to the input constant layers which will
+    // cause a failure if constant workloads are not executed
+    if (!networkProperties.m_AsyncEnabled)
+    {
+        for (auto workload: ConstWorkloads)
+        {
+            workload->Execute();
+        }
+    }
 }
 
 void LoadedNetwork::AllocateAndExecuteConstantWorkloads()
@@ -519,8 +537,6 @@ void LoadedNetwork::AllocateAndExecuteConstantWorkloads()
     }
 }
 
-
-
 void LoadedNetwork::AllocateAndExecuteConstantWorkloadsAsync()
 {
     ARMNN_SCOPED_PROFILING_EVENT(Compute::Undefined, "LoadNetwork_AllocateAndExecuteConstants");
diff --git a/src/armnn/Network.cpp b/src/armnn/Network.cpp
index 226d478110..1f4e72771c 100644
--- a/src/armnn/Network.cpp
+++ b/src/armnn/Network.cpp
@@ -129,12 +129,22 @@ IConnectableLayer* INetwork::AddDepthToSpaceLayer(const DepthToSpaceDescriptor&
 
 IConnectableLayer* INetwork::AddDepthwiseConvolution2dLayer(
     const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
+    const char* name)
+{
+    return pNetworkImpl->AddDepthwiseConvolution2dLayer(convolution2dDescriptor, name);
+}
+
+
+ARMNN_NO_DEPRECATE_WARN_BEGIN
+IConnectableLayer* INetwork::AddDepthwiseConvolution2dLayer(
+    const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
     const ConstTensor& weights,
     const Optional<ConstTensor>& biases,
     const char* name)
 {
     return pNetworkImpl->AddDepthwiseConvolution2dLayer(convolution2dDescriptor, weights, biases, name);
 }
+ARMNN_NO_DEPRECATE_WARN_END
 
 
 IConnectableLayer* INetwork::AddDequantizeLayer(const char* name)
@@ -1727,7 +1737,6 @@ IOptimizedNetworkPtr Optimize(const Graph& inGraph,
                                                 PermuteAsReshape(),
                                                 TransposeAsReshape(),
                                                 OptimizeConsecutiveReshapes(),
-                                                RedirectMembersToConstantInputs(),
                                                 FoldPadIntoConvolution2d(),
                                                 FoldPadIntoDepthwiseConvolution2d(),
                                                 FoldPadIntoPooling2d(),
@@ -1736,7 +1745,8 @@ IOptimizedNetworkPtr Optimize(const Graph& inGraph,
                                                 FuseBatchNormIntoConvolution2DFloat32(),
                                                 FuseBatchNormIntoConvolution2DFloat16(),
                                                 FuseBatchNormIntoDepthwiseConvolution2DFloat32(),
-                                                FuseBatchNormIntoDepthwiseConvolution2DFloat16()));
+                                                FuseBatchNormIntoDepthwiseConvolution2DFloat16(),
+                                                RedirectMembersToConstantInputs()));
 
     // If Fp32 to Fp16 optimization is set convert Fp32 network to Fp16
     if (options.m_ReduceFp32ToFp16)
@@ -2066,38 +2076,43 @@ IConnectableLayer* NetworkImpl::AddDepthToSpaceLayer(const DepthToSpaceDescripto
     return m_Graph->AddLayer<DepthToSpaceLayer>(depthToSpaceDescriptor, name);
 }
 
-IConnectableLayer* NetworkImpl::AddDepthwiseConvolution2dLayerImpl(
-        const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
-        const ConstTensor& weights,
-        const Optional<ConstTensor>& biases,
-        const char* name)
+IConnectableLayer* NetworkImpl::AddDepthwiseConvolution2dLayer(
+    const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
+    const char* name)
 {
-    if (convolution2dDescriptor.m_BiasEnabled && !biases.has_value())
-    {
-        throw InvalidArgumentException("AddDepthwiseConvolution2dLayer: biases cannot be empty");
-    }
+    return m_Graph->AddLayer<DepthwiseConvolution2dLayer>(convolution2dDescriptor, name);
+}
 
-    const auto layer = m_Graph->AddLayer<DepthwiseConvolution2dLayer>(convolution2dDescriptor, name);
+IConnectableLayer* NetworkImpl::AddDepthwiseConvolution2dLayer(
+    const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
+    const ConstTensor& weights,
+    const Optional<ConstTensor>& biases,
+    const char* name)
+{
+    auto layer = m_Graph->AddLayer<DepthwiseConvolution2dLayer>(convolution2dDescriptor, name);
 
+    // Add a constant layer for weights
+    ConstantLayer* weightsLayer = m_Graph->AddLayer<ConstantLayer>("Weights");
+    weightsLayer->m_LayerOutput = std::make_shared<ScopedTensorHandle>(weights);
     layer->m_Weight = std::make_shared<ScopedTensorHandle>(weights);
 
-    if (convolution2dDescriptor.m_BiasEnabled)
+    weightsLayer->GetOutputSlot(0).SetTensorInfo(weightsLayer->m_LayerOutput->GetTensorInfo());
+    weightsLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(1));
+
+    // Add a constant layer for biases
+    if (biases.has_value() && convolution2dDescriptor.m_BiasEnabled)
     {
+        ConstantLayer* biasLayer = m_Graph->AddLayer<ConstantLayer>("Bias");
+        biasLayer->m_LayerOutput = std::make_shared<ScopedTensorHandle>(biases.value());
         layer->m_Bias = std::make_shared<ScopedTensorHandle>(biases.value());
+
+        biasLayer->GetOutputSlot(0).SetTensorInfo(biasLayer->m_LayerOutput->GetTensorInfo());
+        biasLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(2));
     }
 
     return layer;
 }
 
-IConnectableLayer* NetworkImpl::AddDepthwiseConvolution2dLayer(
-        const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
-        const ConstTensor& weights,
-        const Optional<ConstTensor>& biases,
-        const char* name)
-{
-    return AddDepthwiseConvolution2dLayerImpl(convolution2dDescriptor, weights, biases, name);
-}
-
 IConnectableLayer* NetworkImpl::AddDetectionPostProcessLayer(const armnn::DetectionPostProcessDescriptor& descriptor,
                                                          const ConstTensor& anchors, const char* name)
 {
diff --git a/src/armnn/Network.hpp b/src/armnn/Network.hpp
index 6e4d29e490..c5ed8de50d 100644
--- a/src/armnn/Network.hpp
+++ b/src/armnn/Network.hpp
@@ -96,6 +96,11 @@ public:
 
     IConnectableLayer* AddDepthwiseConvolution2dLayer(
         const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
+        const char* name = nullptr);
+
+    ARMNN_DEPRECATED_MSG("This AddDepthwiseConvolution2dLayer overload is deprecated")
+    IConnectableLayer* AddDepthwiseConvolution2dLayer(
+        const DepthwiseConvolution2dDescriptor& convolution2dDescriptor,
         const ConstTensor& weights,
         const Optional<ConstTensor>& biases,
         const char* name = nullptr);
@@ -256,11 +261,6 @@ private:
                                                  const Optional<ConstTensor>& biases,
                                                  const char* name);
 
-    IConnectableLayer* AddDepthwiseConvolution2dLayerImpl(const DepthwiseConvolution2dDescriptor& conv2dDescriptor,
-                                                          const ConstTensor& weights,
-                                                          const Optional<ConstTensor>& biases,
-                                                          const char* name);
-
     bool GetShapeInferenceMethod();
     NetworkOptions m_NetworkOptions;
 
diff --git a/src/armnn/layers/DepthwiseConvolution2dLayer.cpp b/src/armnn/layers/DepthwiseConvolution2dLayer.cpp
index b23661b4a8..08f6fafa1b 100644
--- a/src/armnn/layers/DepthwiseConvolution2dLayer.cpp
+++ b/src/armnn/layers/DepthwiseConvolution2dLayer.cpp
@@ -22,7 +22,7 @@ namespace armnn
 
 DepthwiseConvolution2dLayer::DepthwiseConvolution2dLayer(const DepthwiseConvolution2dDescriptor& param,
                                                          const char* name)
-    : LayerWithParameters(1, 1, LayerType::DepthwiseConvolution2d, param, name)
+    : LayerWithParameters(param.GetNumInputs(), 1, LayerType::DepthwiseConvolution2d, param, name)
 {
 }
 
@@ -31,10 +31,9 @@ void DepthwiseConvolution2dLayer::SerializeLayerParameters(ParameterStringifyFun
     const std::vector<TensorShape>& inputShapes =
     {
         GetInputSlot(0).GetConnection()->GetTensorInfo().GetShape(),
-        m_Weight->GetTensorInfo().GetShape()
+        GetInputSlot(1).GetConnection()->GetTensorInfo().GetShape()
     };
     const TensorShape filterShape = inputShapes[1];
-    DataLayoutIndexed dataLayoutIndex(m_Param.m_DataLayout);
     unsigned int inputChannels = filterShape[1];
     unsigned int filterWidth = filterShape[3];
     unsigned int filterHeight = filterShape[2];
@@ -50,16 +49,14 @@ void DepthwiseConvolution2dLayer::SerializeLayerParameters(ParameterStringifyFun
 
 std::unique_ptr<IWorkload> DepthwiseConvolution2dLayer::CreateWorkload(const IWorkloadFactory& factory) const
 {
-    // on this level constant data should not be released..
-    ARMNN_ASSERT_MSG(m_Weight != nullptr, "DepthwiseConvolution2dLayer: Weights data should not be null.");
-
     DepthwiseConvolution2dQueueDescriptor descriptor;
 
-    descriptor.m_Weight = m_Weight.get();
-
-    if (m_Param.m_BiasEnabled)
+    if (m_Weight)
+    {
+        descriptor.m_Weight = m_Weight.get();
+    }
+    if (m_Param.m_BiasEnabled && m_Bias)
     {
-        ARMNN_ASSERT_MSG(m_Bias != nullptr, "DepthwiseConvolution2dLayer: Bias data should not be null.");
         descriptor.m_Bias = m_Bias.get();
     }
 
@@ -124,19 +121,19 @@ DepthwiseConvolution2dLayer::InferOutputShapes(const std::vector<TensorShape>& i
 
 void DepthwiseConvolution2dLayer::ValidateTensorShapesFromInputs()
 {
-    VerifyLayerConnections(1, CHECK_LOCATION());
+    VerifyLayerConnections(m_Param.GetNumInputs(), CHECK_LOCATION());
 
     const TensorShape& outputShape = GetOutputSlot(0).GetTensorInfo().GetShape();
 
     VerifyShapeInferenceType(outputShape, m_ShapeInferenceMethod);
 
-    // on this level constant data should not be released..
-    ARMNN_ASSERT_MSG(m_Weight != nullptr, "DepthwiseConvolution2dLayer: Weights data should not be null.");
+    ARMNN_ASSERT_MSG(GetInputSlot(1).GetConnection(),
+                     "DepthwiseConvolution2dLayer: Weights data should not be null.");
 
     auto inferredShapes = InferOutputShapes({
         GetInputSlot(0).GetConnection()->GetTensorInfo().GetShape(),
-        m_Weight->GetTensorInfo().GetShape()
-     });
+        GetInputSlot(1).GetConnection()->GetTensorInfo().GetShape()
+    });
 
     ARMNN_ASSERT(inferredShapes.size() == 1);
 
@@ -152,33 +149,13 @@ Layer::ConstantTensors DepthwiseConvolution2dLayer::GetConstantTensorsByRef()
 ARMNN_NO_DEPRECATE_WARN_BEGIN
 void DepthwiseConvolution2dLayer::Accept(ILayerVisitor& visitor) const
 {
-    ManagedConstTensorHandle managedWeight(m_Weight);
-    ConstTensor weightsTensor(managedWeight.GetTensorInfo(), managedWeight.Map());
-    Optional<ConstTensor> optionalBiasTensor = EmptyOptional();
-
-    ManagedConstTensorHandle managedBias(m_Bias);
-    if (GetParameters().m_BiasEnabled)
-    {
-        ConstTensor biasTensor(managedBias.GetTensorInfo(), managedBias.Map());
-        optionalBiasTensor = Optional<ConstTensor>(biasTensor);
-    }
-
-    visitor.VisitDepthwiseConvolution2dLayer(this, GetParameters(), weightsTensor, optionalBiasTensor, GetName());
+    visitor.VisitDepthwiseConvolution2dLayer(this, GetParameters(), GetName());
 }
 ARMNN_NO_DEPRECATE_WARN_END
 
 void DepthwiseConvolution2dLayer::ExecuteStrategy(IStrategy& strategy) const
 {
-    ManagedConstTensorHandle managedWeight(m_Weight);
-    std::vector<armnn::ConstTensor> constTensors { { managedWeight.GetTensorInfo(), managedWeight.Map() } };
-
-    ManagedConstTensorHandle managedBias(m_Bias);
-    if (GetParameters().m_BiasEnabled)
-    {
-        constTensors.emplace_back(ConstTensor(managedBias.GetTensorInfo(), managedBias.Map(true)));
-    }
-
-    strategy.ExecuteStrategy(this, GetParameters(), constTensors, GetName());
+    strategy.ExecuteStrategy(this, GetParameters(), {}, GetName());
 }
 
 } // namespace armnn
diff --git a/src/armnn/layers/DepthwiseConvolution2dLayer.hpp b/src/armnn/layers/DepthwiseConvolution2dLayer.hpp
index 8f8f020a0f..e8ae9a6e79 100644
--- a/src/armnn/layers/DepthwiseConvolution2dLayer.hpp
+++ b/src/armnn/layers/DepthwiseConvolution2dLayer.hpp
@@ -16,8 +16,10 @@ class DepthwiseConvolution2dLayer : public LayerWithParameters<DepthwiseConvolut
 {
 public:
     /// A unique pointer to store Weight values.
+    /// @Note Deprecated. Bias are stored in ConstantLayers now.
     std::shared_ptr<ConstTensorHandle> m_Weight;
     /// A unique pointer to store Bias values.
+    /// @Note Deprecated. Bias are stored in ConstantLayers now.
     std::shared_ptr<ConstTensorHandle> m_Bias;
 
     /// Makes a workload for the DepthwiseConvolution2d type.
@@ -60,6 +62,8 @@ protected:
 
     /// Retrieve the handles to the constant values stored by the layer.
     /// @return A vector of the constant tensors stored by this layer.
+    /// @Note Deprecated. GetConstantTensorsByRef is deprecated. m_Weights and m_Bias
+    ///                   should be connected to layer as Constant Layers instead."
     ConstantTensors GetConstantTensorsByRef() override;
 };
 
diff --git a/src/armnn/optimizations/FoldPadIntoLayer2d.hpp b/src/armnn/optimizations/FoldPadIntoLayer2d.hpp
index 87117debe9..bbaabb815e 100644
--- a/src/armnn/optimizations/FoldPadIntoLayer2d.hpp
+++ b/src/armnn/optimizations/FoldPadIntoLayer2d.hpp
@@ -191,21 +191,26 @@ class FoldPadIntoDepthwiseConvolution2dImpl
 public:
     void Run(Graph& graph, InputSlot& connection) const
     {
-        const auto newConv2dLayer = FoldPadIntoLayer2dImpl<DepthwiseConvolution2dLayer>(graph, connection);
+        const auto newLayer2d = FoldPadIntoLayer2dImpl<DepthwiseConvolution2dLayer>(graph, connection);
 
-        if (newConv2dLayer != nullptr)
+        if (newLayer2d != nullptr)
         {
-            const auto conv2dLayer = PolymorphicDowncast<DepthwiseConvolution2dLayer*>(&connection.GetOwningLayer());
-            // Copy weights and bias to the new convolution layer
-            ARMNN_ASSERT_MSG(conv2dLayer->m_Weight != nullptr,
+            const auto layer2d = PolymorphicDowncast<DepthwiseConvolution2dLayer*>(&connection.GetOwningLayer());
+
+            // Move weights and bias layer connections to the new convolution layer
+            ARMNN_ASSERT_MSG(layer2d->GetInputSlot(1).GetConnection() != nullptr,
                              "FoldPadIntoDepthwiseConvolution2d: Weights data should not be null.");
-            newConv2dLayer->m_Weight = std::move(conv2dLayer->m_Weight);
+            Layer& weightLayer = layer2d->GetInputSlot(1).GetConnectedOutputSlot()->GetOwningLayer();
+            weightLayer.GetOutputSlot(0).Disconnect(layer2d->GetInputSlot(1));
+            weightLayer.GetOutputSlot(0).Connect(newLayer2d->GetInputSlot(1));
 
-            if (conv2dLayer->GetParameters().m_BiasEnabled)
+            if (layer2d->GetParameters().m_BiasEnabled)
             {
-                ARMNN_ASSERT_MSG(conv2dLayer->m_Bias != nullptr,
+                ARMNN_ASSERT_MSG(layer2d->GetInputSlot(2).GetConnection() != nullptr,
                                 "FoldPadIntoDepthwiseConvolution2d: Bias data should not be null if bias is enabled.");
-                newConv2dLayer->m_Bias = std::move(conv2dLayer->m_Bias);
+                Layer& biasLayer = layer2d->GetInputSlot(2).GetConnectedOutputSlot()->GetOwningLayer();
+                biasLayer.GetOutputSlot(0).Disconnect(layer2d->GetInputSlot(2));
+                biasLayer.GetOutputSlot(0).Connect(newLayer2d->GetInputSlot(2));
             }
         }
     }
diff --git a/src/armnn/optimizations/FuseBatchNorm.hpp b/src/armnn/optimizations/FuseBatchNorm.hpp
index 66f722a8ef..6a50fc4a0c 100644
--- a/src/armnn/optimizations/FuseBatchNorm.hpp
+++ b/src/armnn/optimizations/FuseBatchNorm.hpp
@@ -50,12 +50,28 @@ public:
             ConstTensor meanTensor(batchNormLayer->m_Mean->GetTensorInfo(), batchNormLayer->m_Mean->Map(true));
             ConstTensor varTensor(batchNormLayer->m_Variance->GetTensorInfo(), batchNormLayer->m_Variance->Map(true));
 
-            auto convDescriptor = convLayer->GetParameters();
-            auto weightsInfo(convLayer->m_Weight->GetTensorInfo());
-            ConstTensor weightsTensor(weightsInfo, convLayer->m_Weight->Map(true));
+            auto        convDescriptor = convLayer->GetParameters();
+            ConstTensor weightsTensor;
+            if (convLayer->GetNumInputSlots() > 1)
+            {
+                ARMNN_ASSERT_MSG(convLayer->GetInputSlots()[1].GetConnection() != nullptr,
+                                 "FuseBatchNorm: Weight data should not be null.");
+                InputSlot    & oldSlotWeights      = const_cast<InputSlot&>(convLayer->GetInputSlots()[1]);
+                OutputSlot   & constantSlotWeights = const_cast<OutputSlot&>(*oldSlotWeights.GetConnectedOutputSlot());
+                ConstantLayer* weightLayer         = PolymorphicDowncast<ConstantLayer*>(
+                    &constantSlotWeights.GetOwningLayer());
+                weightsTensor = ConstTensor(weightLayer->m_LayerOutput->GetTensorInfo(),
+                                            weightLayer->m_LayerOutput->Map(true));
+            }
+            else
+            {
+                ARMNN_ASSERT_MSG(convLayer->m_Weight != nullptr,
+                                 "FuseBatchNorm: Bias data should not be null if bias is enabled.");
+                weightsTensor = ConstTensor(convLayer->m_Weight->GetTensorInfo(), convLayer->m_Weight->Map(true));
+            }
 
             armnnUtils::DataLayoutIndexed dataLayout(convDescriptor.m_DataLayout);
-            auto weightsShape = weightsInfo.GetShape();
+            auto weightsShape = weightsTensor.GetInfo().GetShape();
             const unsigned int inputChannels   = parentOut->GetTensorInfo().GetShape()[dataLayout.GetChannelsIndex()];
             const unsigned int depthMultiplier = depthwise ? weightsShape[3] / inputChannels : 1;
             const unsigned int outputChannels  = depthwise ? weightsShape[3] : weightsShape[0];
@@ -116,16 +132,32 @@ public:
                     }
                 }
             }
-            ConstTensor fusedWeightsTensor(weightsInfo, fusedWeightsVector);
+            ConstTensor fusedWeightsTensor(weightsTensor.GetInfo(), fusedWeightsVector);
 
             //  fusedBias = (gamma * (bias - mean)) / (variance - epsilon) + beta;
             std::vector<T> fusedBiasVector(outputChannels);
-            if (convDescriptor.m_BiasEnabled)
+            bool biasWasEnabledBeforeOpt = convDescriptor.m_BiasEnabled;
+            if (biasWasEnabledBeforeOpt)
             {
-                ARMNN_ASSERT_MSG(convLayer->m_Bias != nullptr,
-                                 "FuseBatchNorm: Bias data should not be null if bias is enabled.");
+                ConstTensor biasTensor;
+                if (convLayer->GetNumInputSlots() > 1)
+                {
+                    ARMNN_ASSERT_MSG(convLayer->GetInputSlots()[2].GetConnection() != nullptr,
+                                     "FuseBatchNorm: Bias data should not be null if bias is enabled.");
+                    InputSlot    & oldSlotBias      = const_cast<InputSlot&>(convLayer->GetInputSlots()[2]);
+                    OutputSlot   & constantSlotBias = const_cast<OutputSlot&>(*oldSlotBias.GetConnectedOutputSlot());
+                    ConstantLayer* biasLayer        = PolymorphicDowncast<ConstantLayer*>(
+                        &constantSlotBias.GetOwningLayer());
+                    biasTensor = ConstTensor(biasLayer->m_LayerOutput->GetTensorInfo(),
+                                             biasLayer->m_LayerOutput->Map(true));
+                }
+                else
+                {
+                    ARMNN_ASSERT_MSG(convLayer->m_Bias != nullptr,
+                                     "FuseBatchNorm: Bias data should not be null if bias is enabled.");
+                    biasTensor = ConstTensor(convLayer->m_Bias->GetTensorInfo(), convLayer->m_Bias->Map(true));
+                }
 
-                ConstTensor biasTensor(convLayer->m_Bias->GetTensorInfo(), convLayer->m_Bias->Map(true));
                 const auto* biasBuffer = static_cast<const T*>(biasTensor.GetMemoryArea());
                 std::vector<T> biasVector(biasBuffer, biasBuffer + biasTensor.GetNumElements());
 
@@ -156,6 +188,40 @@ public:
             newConv2dLayer.m_Weight = std::make_unique<ScopedTensorHandle>(fusedWeightsTensor);
             newConv2dLayer.m_Bias = std::make_unique<ScopedTensorHandle>(ConstTensor(fusedBiasTensor));
 
+            // Connect weights and bias from old to new Conv2d layer
+            // This optimization will always have 3 input slots on the Conv2d base layer
+            if (newConv2dLayer.GetNumInputSlots() > 1)
+            {
+                ConstantLayer* weightLayer = PolymorphicDowncast<ConstantLayer*>(
+                    &base.GetInputSlot(1).GetConnectedOutputSlot()->GetOwningLayer());
+                // Remove old connection and connect to new layer2d
+                weightLayer->GetOutputSlot(0).Disconnect(base.GetInputSlot(1));
+                weightLayer->GetOutputSlot(0).Connect(newConv2dLayer.GetInputSlot(1));
+                weightLayer->m_LayerOutput = newConv2dLayer.m_Weight;
+
+                // Move bias const layers as normal if it was enabled before the optimisation
+                ConstantLayer* biasLayer;
+                if (biasWasEnabledBeforeOpt)
+                {
+                    biasLayer = PolymorphicDowncast<ConstantLayer*>(
+                        &base.GetInputSlot(2).GetConnectedOutputSlot()->GetOwningLayer());
+                    // Remove old connection and connect to new layer2d
+                    biasLayer->GetOutputSlot(0).Disconnect(base.GetInputSlot(2));
+                    biasLayer->GetOutputSlot(0).Connect(newConv2dLayer.GetInputSlot(2));
+
+                }
+                // Otherwise create a new bias layer and add to the new convolution2d
+                else
+                {
+                    // Add in bias constant layer
+                    biasLayer = graph.AddLayer<ConstantLayer>("Bias");
+                    biasLayer->GetOutputSlot(0).SetTensorInfo(fusedBiasTensor.GetInfo());
+                    biasLayer->GetOutputSlot(0).Connect(newConv2dLayer.GetInputSlot(2));
+                }
+                biasLayer->m_LayerOutput = newConv2dLayer.m_Bias;
+            }
+
+
             // Reconnects with original parent.
             newConv2dLayer.GetOutputSlot().MoveAllConnections(*parentOut);
             // Parent is now the new convolution2d layer.
diff --git a/src/armnn/optimizations/RedirectMembersToConstantInputs.hpp b/src/armnn/optimizations/RedirectMembersToConstantInputs.hpp
index 85d715c6b1..cb97a0fe32 100644
--- a/src/armnn/optimizations/RedirectMembersToConstantInputs.hpp
+++ b/src/armnn/optimizations/RedirectMembersToConstantInputs.hpp
@@ -31,6 +31,7 @@ public:
             case LayerType::Convolution2d:
                 break;
             case LayerType::DepthwiseConvolution2d:
+                RedirectWeightsAndBiases<DepthwiseConvolution2dLayer>(&layer);
                 break;
             case LayerType::DetectionPostProcess:
                 break;
@@ -80,7 +81,7 @@ private:
     }
 };
 
-using RedirectMembersToConstantInputs = OptimizeForType<FullyConnectedLayer, RedirectMembersToConstantInputsImpl>;
+using RedirectMembersToConstantInputs = OptimizeForType<Layer, RedirectMembersToConstantInputsImpl>;
 
 } // namespace optimizations
 } // namespace armnn
diff --git a/src/armnn/test/ConstTensorLayerVisitor.cpp b/src/armnn/test/ConstTensorLayerVisitor.cpp
index cbc97b3c0e..af0581ce4c 100644
--- a/src/armnn/test/ConstTensorLayerVisitor.cpp
+++ b/src/armnn/test/ConstTensorLayerVisitor.cpp
@@ -230,11 +230,16 @@ TEST_CASE("CheckDepthwiseConvolution2dLayer")
     std::vector<unsigned int> dimensions = {1, 1, 3, 3};
     ConstTensor weights(TensorInfo(4, dimensions.data(), DataType::Float32, 0.0f, 0, true), data);
 
-    TestDepthwiseConvolution2dLayerVisitor visitor(descriptor, weights, EmptyOptional());
-
     NetworkImpl net;
 
-    IConnectableLayer* const layer = net.AddDepthwiseConvolution2dLayer(descriptor, weights, EmptyOptional());
+    TestConstantLayerVisitor weightsVisitor(weights);
+    TestDepthwiseConvolution2dLayerVisitor visitor(descriptor);
+
+    IConnectableLayer* const weightsLayer = net.AddConstantLayer(weights);
+    IConnectableLayer* const layer = net.AddDepthwiseConvolution2dLayer(descriptor);
+    weightsLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(1));
+
+    weightsLayer->ExecuteStrategy(weightsVisitor);
     layer->ExecuteStrategy(visitor);
 }
 
@@ -254,14 +259,16 @@ TEST_CASE("CheckNamedDepthwiseConvolution2dLayer")
     std::vector<unsigned int> dimensions = {1, 1, 3, 3};
     ConstTensor weights(TensorInfo(4, dimensions.data(), DataType::Float32, 0.0f, 0, true), data);
 
-    TestDepthwiseConvolution2dLayerVisitor visitor(descriptor, weights, EmptyOptional(), layerName);
-
     NetworkImpl net;
 
-    IConnectableLayer* const layer = net.AddDepthwiseConvolution2dLayer(descriptor,
-                                                                        weights,
-                                                                        EmptyOptional(),
-                                                                        layerName);
+    TestConstantLayerVisitor weightsVisitor(weights);
+    TestDepthwiseConvolution2dLayerVisitor visitor(descriptor, layerName);
+
+    IConnectableLayer* const weightsLayer = net.AddConstantLayer(weights);
+    IConnectableLayer* const layer = net.AddDepthwiseConvolution2dLayer(descriptor, layerName);
+    weightsLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(1));
+
+    weightsLayer->ExecuteStrategy(weightsVisitor);
     layer->ExecuteStrategy(visitor);
 }
 
@@ -284,13 +291,21 @@ TEST_CASE("CheckDepthwiseConvolution2dLayerWithBiases")
     std::vector<float> biasData = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0};
     std::vector<unsigned int> biasDimensions = {1, 1, 3, 3};
     ConstTensor biases(TensorInfo(4, biasDimensions.data(), DataType::Float32, 0.0f, 0, true), biasData);
-    Optional<ConstTensor> optionalBiases(biases);
 
-    TestDepthwiseConvolution2dLayerVisitor visitor(descriptor, weights, optionalBiases);
+    TestConstantLayerVisitor weightsVisitor(weights);
+    TestConstantLayerVisitor biasesVisitor(biases);
+    TestDepthwiseConvolution2dLayerVisitor visitor(descriptor);
 
     NetworkImpl net;
 
-    IConnectableLayer* const layer = net.AddDepthwiseConvolution2dLayer(descriptor, weights, optionalBiases);
+    IConnectableLayer* const weightsLayer = net.AddConstantLayer(weights);
+    IConnectableLayer* const biasesLayer = net.AddConstantLayer(biases);
+    IConnectableLayer* const layer = net.AddDepthwiseConvolution2dLayer(descriptor);
+    weightsLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(1));
+    biasesLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(2));
+
+    weightsLayer->ExecuteStrategy(weightsVisitor);
+    biasesLayer->ExecuteStrategy(biasesVisitor);
     layer->ExecuteStrategy(visitor);
 }
 
@@ -314,13 +329,21 @@ TEST_CASE("CheckNamedDepthwiseConvolution2dLayerWithBiases")
     std::vector<float> biasData = {1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0};
     std::vector<unsigned int> biasDimensions = {1, 1, 3, 3};
     ConstTensor biases(TensorInfo(4, biasDimensions.data(), DataType::Float32, 0.0f, 0, true), biasData);
-    Optional<ConstTensor> optionalBiases(biases);
 
-    TestDepthwiseConvolution2dLayerVisitor visitor(descriptor, weights, optionalBiases, layerName);
+    TestConstantLayerVisitor weightsVisitor(weights);
+    TestConstantLayerVisitor biasesVisitor(biases);
+    TestDepthwiseConvolution2dLayerVisitor visitor(descriptor, layerName);
 
     NetworkImpl net;
 
-    IConnectableLayer* const layer = net.AddDepthwiseConvolution2dLayer(descriptor, weights, optionalBiases, layerName);
+    IConnectableLayer* const weightsLayer = net.AddConstantLayer(weights);
+    IConnectableLayer* const biasesLayer = net.AddConstantLayer(biases);
+    IConnectableLayer* const layer = net.AddDepthwiseConvolution2dLayer(descriptor, layerName);
+    weightsLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(1));
+    biasesLayer->GetOutputSlot(0).Connect(layer->GetInputSlot(2));
+
+    weightsLayer->ExecuteStrategy(weightsVisitor);
+    biasesLayer->ExecuteStrategy(biasesVisitor);
     layer->ExecuteStrategy(visitor);
 }
 
diff --git a/src/armnn/test/ConstTensorLayerVisitor.hpp b/src/armnn/test/ConstTensorLayerVisitor.hpp
index 4d887c8e37..00d17b4ae8 100644
--- a/src/armnn/test/ConstTensorLayerVisitor.hpp
+++ b/src/armnn/test/ConstTensorLayerVisitor.hpp
@@ -74,13 +74,9 @@ class TestDepthwiseConvolution2dLayerVisitor : public TestLayerVisitor
 {
 public:
     explicit TestDepthwiseConvolution2dLayerVisitor(const DepthwiseConvolution2dDescriptor& descriptor,
-                                                    const ConstTensor& weights,
-                                                    const Optional<ConstTensor>& biases,
                                                     const char* name = nullptr)
         : TestLayerVisitor(name)
         , m_Descriptor(descriptor)
-        , m_Weights(weights)
-        , m_Biases(biases)
     {}
 
     virtual ~TestDepthwiseConvolution2dLayerVisitor() {}
@@ -99,12 +95,6 @@ public:
                 CheckLayerPointer(layer);
                 CheckLayerName(name);
                 CheckDescriptor(static_cast<const armnn::DepthwiseConvolution2dDescriptor&>(descriptor));
-                CheckConstTensors(m_Weights, constants[0]);
-                if (m_Biases.has_value())
-                {
-                    CHECK(constants.size() == 2);
-                    CheckConstTensors(m_Biases.value(), constants[1]);
-                }
                 break;
             }
             default:
@@ -119,8 +109,6 @@ protected:
 
 private:
     DepthwiseConvolution2dDescriptor m_Descriptor;
-    ConstTensor m_Weights;
-    Optional<ConstTensor> m_Biases;
 };
 
 class TestFullyConnectedLayerVistor : public TestLayerVisitor
diff --git a/src/armnn/test/GraphTests.cpp b/src/armnn/test/GraphTests.cpp
index d3dd499850..95421c5683 100644
--- a/src/armnn/test/GraphTests.cpp
+++ b/src/armnn/test/GraphTests.cpp
@@ -632,8 +632,10 @@ TEST_CASE("IConnectableLayerConstantTensorsByRef")
     TensorInfo weightsInfo = constInfo;
     ConstTensor weights(weightsInfo, weightData);
     DepthwiseConvolution2dDescriptor desc;
+    ARMNN_NO_DEPRECATE_WARN_BEGIN
+    // GetConstantTensorsByRef() returns {m_Weights, m_Bias} so we need to use the old AddDepthwiseConvolution2dLayer()
     const auto depthwiseLayer = net->AddDepthwiseConvolution2dLayer(desc, weights, EmptyOptional(), "Depthwise");
-
+    ARMNN_NO_DEPRECATE_WARN_END
     const void* resultData = depthwiseLayer->GetConstantTensorsByRef()[0].get()->GetConstTensor<void>();
     auto resultValue = reinterpret_cast<const uint8_t*>(resultData);
     CHECK(resultValue[0] == 3);
diff --git a/src/armnn/test/OptimizerTests.cpp b/src/armnn/test/OptimizerTests.cpp
index a7277b78b5..6a13dc6456 100644
--- a/src/armnn/test/OptimizerTests.cpp
+++ b/src/armnn/test/OptimizerTests.cpp
@@ -478,11 +478,10 @@ void CreateDepthwiseConvolution2dGraph(Graph &graph, const unsigned int* inputSh
 {
     armnn::TensorInfo inputInfo(4, inputShape, DataType::Float32);
     armnn::TensorInfo outputInfo(4, outputShape, DataType::Float32);
+    armnn::TensorInfo weightsInfo(TensorShape(4, weightsShape), armnn::DataType::Float32, 0.0f, 0, true);
 
     std::vector<float> weightsVector(18);
-    armnn::ConstTensor weights(
-            armnn::TensorInfo(4, weightsShape, armnn::DataType::Float32, 0.0f, 0, true),
-            weightsVector);
+    armnn::ConstTensor weights(weightsInfo, weightsVector);
 
     DepthwiseConvolution2dDescriptor desc;
     desc.m_BiasEnabled = false;
@@ -490,15 +489,19 @@ void CreateDepthwiseConvolution2dGraph(Graph &graph, const unsigned int* inputSh
     desc.m_StrideY     = 1;
     desc.m_DataLayout  = dataLayout;
 
-    Layer* input = graph.AddLayer<InputLayer>(0, "input");
-    input->GetOutputSlot().SetTensorInfo(inputInfo);
-
+    InputLayer* input                  = graph.AddLayer<InputLayer>(0, "input");
     DepthwiseConvolution2dLayer* layer = graph.AddLayer<DepthwiseConvolution2dLayer>(desc, "depthwiseConv2d");
-    layer->m_Weight                    = std::make_unique<armnn::ScopedTensorHandle>(weights);
+    ConstantLayer* weightsLayer        = graph.AddLayer<ConstantLayer>("weights");
+    OutputLayer* output                = graph.AddLayer<OutputLayer>(0, "output");
+
+    input->GetOutputSlot().SetTensorInfo(inputInfo);
     layer->GetOutputSlot().SetTensorInfo(outputInfo);
+    weightsLayer->GetOutputSlot().SetTensorInfo(weightsInfo);
+
+    weightsLayer->m_LayerOutput = std::make_unique<armnn::ScopedTensorHandle>(weights);
 
-    Layer* output = graph.AddLayer<OutputLayer>(0, "output");
     input->GetOutputSlot().Connect(layer->GetInputSlot(0));
+    weightsLayer->GetOutputSlot().Connect(layer->GetInputSlot(1));
     layer->GetOutputSlot().Connect(output->GetInputSlot(0));
 }
 
diff --git a/src/armnn/test/ShapeInferenceTests.cpp b/src/armnn/test/ShapeInferenceTests.cpp
index 687462dfb5..d45c9900c0 100644
--- a/src/armnn/test/ShapeInferenceTests.cpp
+++ b/src/armnn/test/ShapeInferenceTests.cpp
@@ -331,18 +331,11 @@ TEST_CASE("DepthwiseConvolutionTest")
     descriptor.m_DataLayout = DataLayout::NHWC;
     descriptor.m_BiasEnabled = false;
 
-    Graph graph;
-
-    auto layer = BuildGraph<DepthwiseConvolution2dLayer>(&graph,
-                                                        {{ 8, 16, 2, 1 }},
-                                                        descriptor,
-                                                        "depthwiseconv2d");
-
-    const float Datum = 0.0f;
-    ConstTensor weights({{ 2, 5, 3, 2 }, DataType::Float32, 0.0f, 0, true}, &Datum);
-    layer->m_Weight = std::make_unique<ScopedTensorHandle>(weights);
-
-    RunShapeInferenceTest<DepthwiseConvolution2dLayer>(layer, {{ 8, 18, 1, 2 }});
+    CreateGraphAndRunTest<DepthwiseConvolution2dLayer>({{ 8, 16, 2, 1 },   // input
+                                                        { 2, 5, 3, 2 }},   // weights
+                                                       {{ 8, 18, 1, 2 }}, // output
+                                                       descriptor,
+                                                       "conv2d");
 }
 
 TEST_CASE("DequantizeTest")
diff --git a/src/armnn/test/SubgraphViewTests.cpp b/src/armnn/test/SubgraphViewTests.cpp
index 212ae0ee01..048c4f51fd 100644
--- a/src/armnn/test/SubgraphViewTests.cpp
+++ b/src/armnn/test/SubgraphViewTests.cpp
@@ -1928,6 +1928,7 @@ bool ReplaceConstantMultiplicationWithDepthwise(SubgraphView& subgraph,
     if (layer->GetType() == LayerType::Multiplication)
     {
         IInputSlot* patternSubgraphInput = &layer->GetInputSlot(0);
+        IInputSlot* patternSubgraphConstant = &layer->GetInputSlot(1);
 
         const IConnectableLayer* inputLayer    = &patternSubgraphInput->GetConnection()->GetOwningIConnectableLayer();
         const IConnectableLayer* constantLayer = &layer->GetInputSlot(1).GetConnection()->GetOwningIConnectableLayer();
@@ -1935,7 +1936,7 @@ bool ReplaceConstantMultiplicationWithDepthwise(SubgraphView& subgraph,
         // Figure out which of the two inputs is the constant
         if (constantLayer->GetType() != LayerType::Constant)
         {
-            patternSubgraphInput = &layer->GetInputSlot(1);
+            std::swap(patternSubgraphInput, patternSubgraphConstant);
             std::swap(inputLayer, constantLayer);
         }
 
@@ -1965,7 +1966,7 @@ bool ReplaceConstantMultiplicationWithDepthwise(SubgraphView& subgraph,
                 ConstTensor           weights(weightsInfo, weightData);
 
                 const auto depthwiseLayer = replacementGraph->AddDepthwiseConvolution2dLayer(
-                        desc, weights, armnn::EmptyOptional(), "Replacement for Constant-Multiplication");
+                        desc, "Replacement for Constant-Multiplication");
 
                 auto& outslot = layer->GetOutputSlot(0);
                 SubgraphView::IOutputSlots outputs{ &outslot };
@@ -1973,7 +1974,9 @@ bool ReplaceConstantMultiplicationWithDepthwise(SubgraphView& subgraph,
                 layers.push_back(layer);
                 layers.push_back(const_cast<IConnectableLayer*>(constantLayer));
 
-                SubgraphView patternSubgraph(std::move(layers), {patternSubgraphInput}, {&layer->GetOutputSlot(0)});
+                SubgraphView patternSubgraph(std::move(layers),
+                                             {patternSubgraphInput, patternSubgraphConstant},
+                                             {&layer->GetOutputSlot(0)});
 
                 subgraph.SubstituteSubgraph(patternSubgraph, depthwiseLayer );
 
diff --git a/src/armnn/test/optimizations/FoldPadTests.cpp b/src/armnn/test/optimizations/FoldPadTests.cpp
index 2f9e1c6d31..9919c6d0e6 100644
--- a/src/armnn/test/optimizations/FoldPadTests.cpp
+++ b/src/armnn/test/optimizations/FoldPadTests.cpp
@@ -126,14 +126,18 @@ TEST_CASE("FoldPadLayerIntoDepthwiseConvolution2dLayer")
 
     auto* depthwiseConv2dLayer = graph.AddLayer<DepthwiseConvolution2dLayer>(depthwiseConvolution2dDescriptor,
                                                                              "depthwiseConv2d");
-    depthwiseConv2dLayer->m_Weight = std::make_unique<ScopedTensorHandle>(weights);
+    auto* weightsLayer = graph.AddLayer<ConstantLayer>("weights");
+
+    weightsLayer->GetOutputSlot().SetTensorInfo(weights.GetInfo());
     depthwiseConv2dLayer->GetOutputSlot().SetTensorInfo(outputInfo);
+    depthwiseConv2dLayer->m_Weight = std::make_shared<ScopedTensorHandle>(weights);
 
     Layer* output = graph.AddLayer<OutputLayer>(0, "output");
 
     // Connect up layers - input -> pad -> depthwiseConv2d -> output
     input->GetOutputSlot().Connect(padLayer->GetInputSlot(0));
     padLayer->GetOutputSlot().Connect(depthwiseConv2dLayer->GetInputSlot(0));
+    weightsLayer->GetOutputSlot().Connect(depthwiseConv2dLayer->GetInputSlot(1));
     depthwiseConv2dLayer->GetOutputSlot().Connect(output->GetInputSlot(0));
 
     auto checkSimpleDepthwiseConv2d = [](const Layer* const layer)->bool {
@@ -151,6 +155,7 @@ TEST_CASE("FoldPadLayerIntoDepthwiseConvolution2dLayer")
                              &IsLayerOfType<InputLayer>,
                              &IsLayerOfType<PadLayer>,
                              checkSimpleDepthwiseConv2d,
+                             &IsLayerOfType<ConstantLayer>,
                              &IsLayerOfType<OutputLayer>));
 
     armnn::Optimizer::Pass(graph, MakeOptimizations(FoldPadIntoDepthwiseConvolution2d()));
@@ -170,6 +175,7 @@ TEST_CASE("FoldPadLayerIntoDepthwiseConvolution2dLayer")
     CHECK(CheckSequence(graph.cbegin(), graph.cend(),
                              &IsLayerOfType<InputLayer>,
                              checkPadFoldedIntoDepthwiseConv2d,
+                             &IsLayerOfType<ConstantLayer>,
                              &IsLayerOfType<OutputLayer>));
 }
 
@@ -741,11 +747,8 @@ TEST_CASE("FoldPadLayerIntoDepthwiseConv2dLayer_ExecuteInferenceWithAndWithoutOp
         std::vector<float>    biasVector   = {5, 6, 7, 8, 9, 10, 11, 12, 5, 6, 7, 8};
         TensorInfo            biasInfo({12}, DataType::Float32, 0.0f, 0, true);
         ConstTensor           bias(biasInfo, biasVector);
-        Optional<ConstTensor> optionalBias = Optional<ConstTensor>(bias);
 
         IConnectableLayer* conv2dLayer = network->AddDepthwiseConvolution2dLayer(convDescriptor,
-                                                                                 weights,
-                                                                                 optionalBias,
                                                                                  "DepthwiseConv2D");
 
         TensorInfo outputInfo(4, outputShape, DataType::Float32);
@@ -758,6 +761,14 @@ TEST_CASE("FoldPadLayerIntoDepthwiseConv2dLayer_ExecuteInferenceWithAndWithoutOp
         padLayer->GetOutputSlot(0).Connect(conv2dLayer->GetInputSlot(0));
         conv2dLayer->GetOutputSlot(0).Connect(outputLayer->GetInputSlot(0));
 
+        auto weightsLayer = network->AddConstantLayer(weights, "Weights");
+        weightsLayer->GetOutputSlot(0).SetTensorInfo(weights.GetInfo());
+        weightsLayer->GetOutputSlot(0).Connect(conv2dLayer->GetInputSlot(1));
+
+        auto biasLayer = network->AddConstantLayer(bias, "Bias");
+        biasLayer->GetOutputSlot(0).SetTensorInfo(bias.GetInfo());
+        biasLayer->GetOutputSlot(0).Connect(conv2dLayer->GetInputSlot(2));
+
         // Create ArmNN runtime
         IRuntimePtr          run              = IRuntime::Create(IRuntime::CreationOptions());    // default options
         // Optimise the network
diff --git a/src/armnn/test/optimizations/FuseActivationTests.cpp b/src/armnn/test/optimizations/FuseActivationTests.cpp
index ac327bb609..e5f54208f0 100644
--- a/src/armnn/test/optimizations/FuseActivationTests.cpp
+++ b/src/armnn/test/optimizations/FuseActivationTests.cpp
@@ -90,7 +90,7 @@ struct DWConvolution2dTest
 public:
     using LayerType = DepthwiseConvolution2dLayer;
     static const bool isElementWise = false;
-    static const bool isConstTensorAsInputSupported = false;
+    static const bool isConstTensorAsInputSupported = true;
 
     static TensorShape GetInputShape()   { return TensorShape( {1, 4, 4, 3}); }   // [N,H,W,Cin]
     static TensorShape GetOutputShape()  { return TensorShape( {1, 3, 3, 12}); }  // [N,H,W,Cout]
@@ -104,32 +104,35 @@ public:
                                                float scale = 1.f,
                                                int32_t offset = 0)
     {
+        IgnoreUnused(scale);
+        IgnoreUnused(offset);
+
         DepthwiseConvolution2dDescriptor descriptor;
         descriptor.m_BiasEnabled = false;
         descriptor.m_DataLayout  = DataLayout::NHWC;
         descriptor.m_StrideX     = 1;
         descriptor.m_StrideY     = 1;
 
-        std::vector<float> weightsData   = { 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12,
-                                            11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22,
-                                            21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
-                                            31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42};
-        std::vector<T>     weightsVector = armnnUtils::QuantizedVector<T>(weightsData, scale, offset);
-        TensorInfo         weightsInfo(GetWeightsShape(), ArmnnType, scale, offset, true);
-        ConstTensor        weights(weightsInfo, weightsVector);
-        Optional<ConstTensor> optionalBias;
-
-        return network->AddDepthwiseConvolution2dLayer(descriptor, weights, optionalBias, name);
+        return network->AddDepthwiseConvolution2dLayer(descriptor, name);
     }
 
     static std::vector<IConnectableLayer*> AddConstantLayers(INetwork* network,
                                                              float scale = 1.f,
                                                              int32_t offset = 0)
     {
-        IgnoreUnused(network);
-        IgnoreUnused(scale);
-        IgnoreUnused(offset);
-        return {};
+        std::vector<float> weightsData   = { 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12,
+                                             11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22,
+                                             21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32,
+                                             31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42};
+        std::vector<T>     weightsVector = armnnUtils::QuantizedVector<T>(weightsData, scale, offset);
+        TensorInfo         weightsInfo(GetWeightsShape(), ArmnnType, scale, offset, true);
+        ConstTensor        weights(weightsInfo, weightsVector);
+
+        IConnectableLayer* weightsLayer = network->AddConstantLayer(weights, "Weights");
+        weightsLayer->GetOutputSlot(0).SetTensorInfo(weightsInfo);
+
+        std::vector<IConnectableLayer*> layers = { weightsLayer };
+        return layers;
     }
 };
 
@@ -390,10 +393,10 @@ INetworkPtr CreateNetwork(ActivationDescriptor activationDescriptor, bool preven
                                                                      "activation");
 
     IConnectableLayer* outputLayer  = network->AddOutputLayer(0);
-    IConnectableLayer* output2Layer = preventFusing?network->AddOutputLayer(1):nullptr;
+    IConnectableLayer* output2Layer = preventFusing ? network->AddOutputLayer(1) : nullptr;
 
     // If ConstTensorAsInputs is supported weights and bias are stored as constant layers.
-    if(LayerTest::isConstTensorAsInputSupported)
+    if (LayerTest::isConstTensorAsInputSupported)
     {
         std::vector<IConnectableLayer*> constantLayers = LayerTest::AddConstantLayers(network.get(),
                                                                                       scale,
diff --git a/src/armnn/test/optimizations/FuseBatchNormTests.cpp b/src/armnn/test/optimizations/FuseBatchNormTests.cpp
index 70cffea2b2..b28bb17773 100644
--- a/src/armnn/test/optimizations/FuseBatchNormTests.cpp
+++ b/src/armnn/test/optimizations/FuseBatchNormTests.cpp
@@ -24,6 +24,7 @@ class Conv2dTest
 public:
     using ConvDescriptorType            = armnn::Convolution2dDescriptor;
     using ConvLayerType                 = armnn::Convolution2dLayer;
+    static const bool isConstTensorAsInputSupported = false;
 
     static IConnectableLayer *AddConvolution(INetwork *network,
                                              const Convolution2dDescriptor &descriptor,
@@ -33,6 +34,19 @@ public:
     {
         return network->AddConvolution2dLayer(descriptor, weights, biases, name);
     }
+
+    static std::vector<IConnectableLayer*> AddConstantLayers(INetwork *network,
+                                                             const Convolution2dDescriptor &descriptor,
+                                                             const ConstTensor &weights,
+                                                             const Optional<ConstTensor> &biases)
+    {
+        IgnoreUnused(network);
+        IgnoreUnused(descriptor);
+        IgnoreUnused(weights);
+        IgnoreUnused(biases);
+
+        return {};
+    }
 };
 
 class DepthwiseConv2dTest
@@ -40,6 +54,7 @@ class DepthwiseConv2dTest
 public:
     using ConvDescriptorType            = armnn::DepthwiseConvolution2dDescriptor;
     using ConvLayerType                 = armnn::DepthwiseConvolution2dLayer;
+    static const bool isConstTensorAsInputSupported = true;
 
     static IConnectableLayer *AddConvolution(INetwork *network,
                                              const DepthwiseConvolution2dDescriptor &descriptor,
@@ -47,7 +62,29 @@ public:
                                              const Optional<ConstTensor> &biases,
                                              const char *name)
     {
-        return network->AddDepthwiseConvolution2dLayer(descriptor, weights, biases, name);
+        IgnoreUnused(weights);
+        IgnoreUnused(biases);
+
+        return network->AddDepthwiseConvolution2dLayer(descriptor, name);
+    }
+
+    static std::vector<IConnectableLayer*> AddConstantLayers(INetwork *network,
+                                                             const DepthwiseConvolution2dDescriptor &descriptor,
+                                                             const ConstTensor &weights,
+                                                             const Optional<ConstTensor> &biases)
+    {
+        auto weightsLayer = network->AddConstantLayer(weights, "Weights");
+        weightsLayer->GetOutputSlot(0).SetTensorInfo(weights.GetInfo());
+        std::vector<IConnectableLayer*> layers = {weightsLayer};
+
+        if (descriptor.m_BiasEnabled)
+        {
+            auto biasLayer = network->AddConstantLayer(biases.value(), "Bias");
+            biasLayer->GetOutputSlot(0).SetTensorInfo(biases.value().GetInfo());
+            layers.emplace_back(biasLayer);
+        }
+
+        return layers;
     }
 };
 
@@ -73,7 +110,7 @@ template <typename Conv2dTest,
           armnn::DataType ArmnnType,
           typename ConvDescriptorType = typename Conv2dTest::ConvDescriptorType,
           typename T = armnn::ResolveType<ArmnnType>>
-INetworkPtr CreatNetwork(bool depthwise, bool preventFusing)
+INetworkPtr CreateNetwork(bool depthwise, bool preventFusing)
 {
     // Define layers information
     ConvDescriptorType convolution2dDescriptor;
@@ -110,11 +147,6 @@ INetworkPtr CreatNetwork(bool depthwise, bool preventFusing)
     TensorInfo weightsInfo(4, weightsDimensionSizes, ArmnnType, 0.0f, 0, true);
     ConstTensor weights(weightsInfo, weightsVector);
 
-    std::vector<T> biasVector = GetVector<T>(outputDimensionSizes[3], 3.3f, 0.1f);
-    TensorInfo biasInfo(1, outputChannelSize, ArmnnType, 0.0f, 0, true);
-    ConstTensor bias(biasInfo, biasVector);
-    Optional<ConstTensor> optionalBias = Optional<ConstTensor>(bias);
-
     std::vector<T> betaVector     = GetVector<T>(outputDimensionSizes[3], 0.0f, 0.2f);
     std::vector<T> gammaVector    = GetVector<T>(outputDimensionSizes[3], 0.5f, 0.1f);
     std::vector<T> meanVector     = GetVector<T>(outputDimensionSizes[3], 0.1f, 0.1f);
@@ -133,7 +165,7 @@ INetworkPtr CreatNetwork(bool depthwise, bool preventFusing)
     IConnectableLayer* convLayer      = Conv2dTest::AddConvolution(network.get(),
                                                                    convolution2dDescriptor,
                                                                    weights,
-                                                                   optionalBias,
+                                                                   Optional<ConstTensor>(),
                                                                    "convolution");
 
     IConnectableLayer* batchNormLayer = network->AddBatchNormalizationLayer(batchNormDescriptor,
@@ -151,6 +183,21 @@ INetworkPtr CreatNetwork(bool depthwise, bool preventFusing)
         output2Layer                  = network->AddOutputLayer(1);
     }
 
+    // If ConstTensorAsInputs is supported weights and bias are stored as constant layers.
+    if (Conv2dTest::isConstTensorAsInputSupported)
+    {
+        std::vector<IConnectableLayer*> constantLayers = Conv2dTest::AddConstantLayers(network.get(),
+                                                                                       convolution2dDescriptor,
+                                                                                       weights,
+                                                                                       Optional<ConstTensor>());
+
+        // Connect constant layers to receiverLayer.
+        for (unsigned int i = 0; i < constantLayers.size(); ++i)
+        {
+            constantLayers[i]->GetOutputSlot(0).Connect(convLayer->GetInputSlot(i + 1));
+        }
+    }
+
     // Set layer information
     inputLayer    ->GetOutputSlot(0).SetTensorInfo(inputInfo);
     convLayer     ->GetOutputSlot(0).SetTensorInfo(outputInfo);
@@ -178,7 +225,7 @@ void FuseBatchNormIntoConvTest(bool depthwise, float tolerance, armnn::Compute b
 {
     // FIRST NETWORK: Fused
     // Construct ArmNN network
-    INetworkPtr networkFused = CreatNetwork<Conv2dTest, ArmnnType>(depthwise, false);
+    INetworkPtr networkFused = CreateNetwork<Conv2dTest, ArmnnType>(depthwise, false);
 
     // Create ArmNN runtime
     IRuntimePtr run = IRuntime::Create(IRuntime::CreationOptions()); // default options
@@ -194,12 +241,26 @@ void FuseBatchNormIntoConvTest(bool depthwise, float tolerance, armnn::Compute b
                (layer->GetNameStr() == "fused-batchNorm-into-convolution");
     };
 
-    CHECK(3 == graphFused.GetNumLayers());
-    CHECK(CheckSequence(graphFused.cbegin(),
-                             graphFused.cend(),
-                             &IsLayerOfType<InputLayer>,
-                             checkFusedConv2d,
-                             &IsLayerOfType<OutputLayer>));
+    if (Conv2dTest::isConstTensorAsInputSupported)
+    {
+        CHECK(5 == graphFused.GetNumLayers());
+        CHECK(CheckSequence(graphFused.cbegin(),
+                            graphFused.cend(),
+                            &IsLayerOfType<InputLayer>,
+                            &IsLayerOfType<ConstantLayer>,
+                            &IsLayerOfType<ConstantLayer>,
+                            checkFusedConv2d,
+                            &IsLayerOfType<OutputLayer>));
+    }
+    else
+    {
+        CHECK(3 == graphFused.GetNumLayers());
+        CHECK(CheckSequence(graphFused.cbegin(),
+                            graphFused.cend(),
+                            &IsLayerOfType<InputLayer>,
+                            checkFusedConv2d,
+                            &IsLayerOfType<OutputLayer>));
+    }
 
     // Load network into runtime
     NetworkId networkIdentifier;
@@ -227,7 +288,7 @@ void FuseBatchNormIntoConvTest(bool depthwise, float tolerance, armnn::Compute b
 
     // SECOND NETWORK: NotFused
     // Construct ArmNN network
-    INetworkPtr networkNotFused = CreatNetwork<Conv2dTest, ArmnnType>(depthwise, true);
+    INetworkPtr networkNotFused = CreateNetwork<Conv2dTest, ArmnnType>(depthwise, true);
 
     // Create ArmNN runtime
     IRuntimePtr runNotFused = IRuntime::Create(IRuntime::CreationOptions()); // default options
@@ -237,14 +298,29 @@ void FuseBatchNormIntoConvTest(bool depthwise, float tolerance, armnn::Compute b
 
     Graph& graphNotFused = GetGraphForTesting(optNetNotFused.get());
 
-    CHECK(5 == graphNotFused.GetNumLayers());
-    CHECK(CheckSequence(graphNotFused.cbegin(),
-                             graphNotFused.cend(),
-                             &IsLayerOfType<armnn::InputLayer>,
-                             &IsLayerOfType<ConvLayerType>,
-                             &IsLayerOfType<armnn::BatchNormalizationLayer>,
-                             &IsLayerOfType<armnn::OutputLayer>,
-                             &IsLayerOfType<armnn::OutputLayer>));
+    if (Conv2dTest::isConstTensorAsInputSupported)
+    {
+        CHECK(6 == graphNotFused.GetNumLayers());
+        CHECK(CheckSequence(graphNotFused.cbegin(),
+                            graphNotFused.cend(),
+                            &IsLayerOfType<armnn::InputLayer>,
+                            &IsLayerOfType<armnn::ConstantLayer>,
+                            &IsLayerOfType<ConvLayerType>,
+                            &IsLayerOfType<armnn::BatchNormalizationLayer>,
+                            &IsLayerOfType<armnn::OutputLayer>,
+                            &IsLayerOfType<armnn::OutputLayer>));
+    }
+    else
+    {
+        CHECK(5 == graphNotFused.GetNumLayers());
+        CHECK(CheckSequence(graphNotFused.cbegin(),
+                            graphNotFused.cend(),
+                            &IsLayerOfType<armnn::InputLayer>,
+                            &IsLayerOfType<ConvLayerType>,
+                            &IsLayerOfType<armnn::BatchNormalizationLayer>,
+                            &IsLayerOfType<armnn::OutputLayer>,
+                            &IsLayerOfType<armnn::OutputLayer>));
+    }
 
     // Load network into runtime
     NetworkId networkIdentifierNotFused;