IVGCVSW-2925: Combine Pad with Convolution2d in the Optimizer

* Added new optimization for folding pad layer into convolution2d layer following it
* Added new test in OptimizerTests.cpp
* Added new optimization into All optimizations
* Added call to new optimization in Optimize in Network.cpp
* Updated CMakeLists.txt

Signed-off-by: Nina Drozd <nina.drozd@arm.com>
Change-Id: I682e07c71bbd42c49c02dda30a848a9ab2b16e7e

5 files changed, 178 insertions, 4 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e9172f2789..fed4e9b5d6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -342,17 +342,18 @@ list(APPEND armnn_sources
     src/armnn/Utils.cpp
     src/armnn/WallClockTimer.cpp
     src/armnn/WallClockTimer.hpp
+    src/armnn/optimizations/AddDebug.hpp
     src/armnn/optimizations/All.hpp
     src/armnn/optimizations/ConvertConstants.hpp
+    src/armnn/optimizations/ConvertFp32NetworkToFp16.hpp
+    src/armnn/optimizations/FoldPadIntoConvolution2d.hpp
     src/armnn/optimizations/MovePermuteUp.hpp
     src/armnn/optimizations/Optimization.hpp
     src/armnn/optimizations/OptimizeConsecutiveReshapes.hpp
+    src/armnn/optimizations/OptimizeInverseConversions.hpp
     src/armnn/optimizations/OptimizeInversePermutes.hpp
     src/armnn/optimizations/PermuteAsReshape.hpp
     src/armnn/optimizations/SquashEqualSiblings.hpp
-    src/armnn/optimizations/OptimizeInverseConversions.hpp
-    src/armnn/optimizations/ConvertFp32NetworkToFp16.hpp
-    src/armnn/optimizations/AddDebug.hpp
     third-party/half/half.hpp
     )
 
diff --git a/src/armnn/Network.cpp b/src/armnn/Network.cpp
index c1462c090d..0bd8d4b69b 100644
--- a/src/armnn/Network.cpp
+++ b/src/armnn/Network.cpp
@@ -436,7 +436,8 @@ IOptimizedNetworkPtr Optimize(const INetwork& inNetwork,
                                                 OptimizeInversePermutes(),
                                                 MovePermuteUp(),
                                                 PermuteAsReshape(),
-                                                OptimizeConsecutiveReshapes()));
+                                                OptimizeConsecutiveReshapes(),
+                                                FoldPadIntoConvolution2d()));
 
     // Infer the tensor infos for all output slots. Throws an exception on failure
     optGraph.InferTensorInfos();
diff --git a/src/armnn/optimizations/All.hpp b/src/armnn/optimizations/All.hpp
index 0a6684ee3b..68965fd23c 100644
--- a/src/armnn/optimizations/All.hpp
+++ b/src/armnn/optimizations/All.hpp
@@ -13,3 +13,4 @@
 #include "OptimizeInverseConversions.hpp"
 #include "ConvertFp32NetworkToFp16.hpp"
 #include "AddDebug.hpp"
+#include "FoldPadIntoConvolution2d.hpp"
diff --git a/src/armnn/optimizations/FoldPadIntoConvolution2d.hpp b/src/armnn/optimizations/FoldPadIntoConvolution2d.hpp
new file mode 100644
index 0000000000..f789ffae9a
--- /dev/null
+++ b/src/armnn/optimizations/FoldPadIntoConvolution2d.hpp
@@ -0,0 +1,82 @@
+//
+// Copyright © 2017 Arm Ltd. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#pragma once
+
+#include "Optimization.hpp"
+
+namespace armnn
+{
+namespace optimizations
+{
+
+class FoldPadIntoConvolution2dImpl
+{
+public:
+
+    void Run(Graph& graph, InputSlot& connection) const
+    {
+        Layer& base = connection.GetConnectedOutputSlot()->GetOwningLayer();
+        Layer& child = connection.GetOwningLayer();
+
+        BOOST_ASSERT(base.GetType() == LayerType::Pad);
+        BOOST_ASSERT(child.GetType() == LayerType::Convolution2d);
+
+        PadLayer* padLayer = boost::polymorphic_downcast<PadLayer*>(&base);
+        Convolution2dLayer* convolution2dLayer = boost::polymorphic_downcast<Convolution2dLayer*>(&child);
+
+        OutputSlot* parentOut = base.GetInputSlot(0).GetConnectedOutputSlot();
+        const TensorInfo& outInfo = child.GetOutputHandler().GetTensorInfo();
+
+        const std::string name = std::string("folded-") + base.GetName() + std::string("-into-") + child.GetName();
+        Convolution2dDescriptor descriptor = convolution2dLayer->GetParameters();
+
+        auto padList = padLayer->GetParameters().m_PadList;
+
+        armnn::DataLayout dataLayout = descriptor.m_DataLayout;
+
+        // In Convolution2dDescriptor, padLeft and padRight are defined as paddings on width dimension
+        // whereas padTop and padBottom - paddings on height dimension, so setting these according to data layout
+        if(dataLayout == armnn::DataLayout::NHWC)
+        {
+            descriptor.m_PadLeft = padList[2].first;
+            descriptor.m_PadRight = padList[2].second;
+            descriptor.m_PadTop = padList[1].first;
+            descriptor.m_PadBottom = padList[1].second;
+        }
+        else
+        {
+            descriptor.m_PadLeft = padList[3].first;
+            descriptor.m_PadRight = padList[3].second;
+            descriptor.m_PadTop = padList[2].first;
+            descriptor.m_PadBottom = padList[2].second;
+        }
+
+        auto& newConv2dLayer = *graph.InsertNewLayer<Convolution2dLayer>(base.GetInputSlot(0),
+                                                                         descriptor,
+                                                                         name.c_str());
+        newConv2dLayer.GetOutputHandler().SetTensorInfo(outInfo);
+
+        // Reconnects with original parent.
+        newConv2dLayer.GetOutputSlot().MoveAllConnections(*parentOut);
+        // Parent is now the new convolution2d layer.
+        parentOut = &newConv2dLayer.GetOutputSlot();
+
+        // Moves connections in child output to parent layer.
+        // Child layer will be removed as it's left unconnected.
+        // Base layer will be removed if left unconnected.
+        child.GetOutputSlot().MoveAllConnections(*parentOut);
+    }
+protected:
+    FoldPadIntoConvolution2dImpl() =  default;
+    ~FoldPadIntoConvolution2dImpl() = default;
+};
+
+using FoldPadIntoConvolution2d = OptimizeForConnection<PadLayer, Convolution2dLayer, FoldPadIntoConvolution2dImpl>;
+
+} // namespace optimizations
+} // namespace armnn
+
+
diff --git a/src/armnn/test/OptimizerTests.cpp b/src/armnn/test/OptimizerTests.cpp
index f40a78a0d9..b0d8629c7f 100644
--- a/src/armnn/test/OptimizerTests.cpp
+++ b/src/armnn/test/OptimizerTests.cpp
@@ -1085,4 +1085,93 @@ BOOST_AUTO_TEST_CASE(DetectionPostProcessValidateTensorShapes)
     BOOST_CHECK_NO_THROW(graph.InferTensorInfos());
 }
 
+BOOST_AUTO_TEST_CASE(FoldPadLayerIntoConvolution2dLayer)
+{
+    Graph graph;
+    const unsigned int inputShape[] = { 1, 2, 2, 3 };
+    const unsigned int paddedShape[] = { 1, 6, 6, 3 };
+    const unsigned int weightsShape[] = { 1, 2, 3, 3 };
+    const unsigned int outputShape[] = { 1, 2, 1, 1 };
+
+
+    armnn::TensorInfo inputInfo(4, inputShape, DataType::Float32);
+    armnn::TensorInfo paddedInfo(4, paddedShape, DataType::Float32);
+    armnn::TensorInfo outputInfo(4, outputShape, DataType::Float32);
+
+    Layer* input = graph.AddLayer<InputLayer>(0, "input");
+    input->GetOutputSlot().SetTensorInfo(inputInfo);
+
+    PadDescriptor padDescriptor({{ 0, 0 }, { 2, 2 }, { 2, 2 }, { 0, 0 }});
+
+    PadLayer* padLayer = graph.AddLayer<PadLayer>(padDescriptor, "pad");
+    padLayer->GetOutputSlot().SetTensorInfo(paddedInfo);
+
+    Convolution2dDescriptor convolution2dDescriptor;
+    convolution2dDescriptor.m_BiasEnabled = false;
+    convolution2dDescriptor.m_StrideX = 1;
+    convolution2dDescriptor.m_StrideY = 1;
+    convolution2dDescriptor.m_DataLayout = DataLayout::NHWC;
+
+    std::vector<float> weightsVector(18);
+    armnn::ConstTensor weights(armnn::TensorInfo(4, weightsShape, armnn::DataType::Float32), weightsVector);
+
+    Convolution2dLayer* conv2dLayer = graph.AddLayer<Convolution2dLayer>(convolution2dDescriptor,"conv2d");
+    conv2dLayer->m_Weight = std::make_unique<armnn::ScopedCpuTensorHandle>(weights);
+    conv2dLayer->GetOutputSlot().SetTensorInfo(outputInfo);
+
+    Layer* output = graph.AddLayer<OutputLayer>(0, "output");
+
+    // Connect up layers - input -> pad -> conv2d -> output
+    input->GetOutputSlot().Connect(padLayer->GetInputSlot(0));
+    padLayer->GetOutputSlot().Connect(conv2dLayer->GetInputSlot(0));
+    conv2dLayer->GetOutputSlot().Connect(output->GetInputSlot(0));
+
+    auto checkSimpleConv2d = [ ](const armnn::Layer* const layer) -> bool
+    {
+        const auto conv2dLayer = static_cast<const armnn::Convolution2dLayer*>(layer);
+        const auto conv2dLayerParams = conv2dLayer->GetParameters();
+        return IsLayerOfType<armnn::Convolution2dLayer>(layer) &&
+            (layer->GetNameStr() == "conv2d") &&
+            (conv2dLayerParams.m_PadLeft == 0) &&
+            (conv2dLayerParams.m_PadRight == 0) &&
+            (conv2dLayerParams.m_PadTop == 0) &&
+            (conv2dLayerParams.m_PadBottom == 0) &&
+            (conv2dLayerParams.m_BiasEnabled == false) &&
+            (conv2dLayerParams.m_StrideX == 1) &&
+            (conv2dLayerParams.m_StrideY == 1) &&
+            (conv2dLayerParams.m_DataLayout == DataLayout::NHWC);
+    };
+
+    BOOST_TEST(CheckSequence(graph.cbegin(),
+        graph.cend(),
+        &IsLayerOfType<armnn::InputLayer>,
+        &IsLayerOfType<armnn::PadLayer>,
+        checkSimpleConv2d,
+        &IsLayerOfType<armnn::OutputLayer>));
+
+    armnn::Optimizer::Pass(graph, armnn::MakeOptimizations(FoldPadIntoConvolution2d()));
+
+    auto checkPadFoldedIntoConv2d = [ ](const armnn::Layer* const layer) -> bool
+    {
+        const auto conv2dLayer = static_cast<const armnn::Convolution2dLayer*>(layer);
+        const auto conv2dLayerParams = conv2dLayer->GetParameters();
+        return IsLayerOfType<armnn::Convolution2dLayer>(layer) &&
+               (layer->GetNameStr() == "folded-pad-into-conv2d") &&
+               (conv2dLayerParams.m_PadLeft == 2) &&
+               (conv2dLayerParams.m_PadRight == 2) &&
+               (conv2dLayerParams.m_PadTop == 2) &&
+               (conv2dLayerParams.m_PadBottom == 2) &&
+               (conv2dLayerParams.m_BiasEnabled == false) &&
+               (conv2dLayerParams.m_StrideX == 1) &&
+               (conv2dLayerParams.m_StrideY == 1) &&
+               (conv2dLayerParams.m_DataLayout == DataLayout::NHWC);
+    };
+
+    BOOST_TEST(CheckSequence(graph.cbegin(),
+        graph.cend(),
+        &IsLayerOfType<armnn::InputLayer>,
+        checkPadFoldedIntoConv2d,
+        &IsLayerOfType<armnn::OutputLayer>));
+}
+
 BOOST_AUTO_TEST_SUITE_END()