From 861985ff2964720a0165e109c3fc568cb245bbe9 Mon Sep 17 00:00:00 2001 From: Nina Drozd Date: Thu, 18 Apr 2019 14:48:51 +0100 Subject: IVGCVSW-2925: Combine Pad with Convolution2d in the Optimizer * Added new optimization for folding pad layer into convolution2d layer following it * Added new test in OptimizerTests.cpp * Added new optimization into All optimizations * Added call to new optimization in Optimize in Network.cpp * Updated CMakeLists.txt Signed-off-by: Nina Drozd Change-Id: I682e07c71bbd42c49c02dda30a848a9ab2b16e7e --- CMakeLists.txt | 7 +- src/armnn/Network.cpp | 3 +- src/armnn/optimizations/All.hpp | 1 + .../optimizations/FoldPadIntoConvolution2d.hpp | 82 ++++++++++++++++++++ src/armnn/test/OptimizerTests.cpp | 89 ++++++++++++++++++++++ 5 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 src/armnn/optimizations/FoldPadIntoConvolution2d.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e9172f2789..fed4e9b5d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -342,17 +342,18 @@ list(APPEND armnn_sources src/armnn/Utils.cpp src/armnn/WallClockTimer.cpp src/armnn/WallClockTimer.hpp + src/armnn/optimizations/AddDebug.hpp src/armnn/optimizations/All.hpp src/armnn/optimizations/ConvertConstants.hpp + src/armnn/optimizations/ConvertFp32NetworkToFp16.hpp + src/armnn/optimizations/FoldPadIntoConvolution2d.hpp src/armnn/optimizations/MovePermuteUp.hpp src/armnn/optimizations/Optimization.hpp src/armnn/optimizations/OptimizeConsecutiveReshapes.hpp + src/armnn/optimizations/OptimizeInverseConversions.hpp src/armnn/optimizations/OptimizeInversePermutes.hpp src/armnn/optimizations/PermuteAsReshape.hpp src/armnn/optimizations/SquashEqualSiblings.hpp - src/armnn/optimizations/OptimizeInverseConversions.hpp - src/armnn/optimizations/ConvertFp32NetworkToFp16.hpp - src/armnn/optimizations/AddDebug.hpp third-party/half/half.hpp ) diff --git a/src/armnn/Network.cpp b/src/armnn/Network.cpp index c1462c090d..0bd8d4b69b 100644 --- a/src/armnn/Network.cpp +++ b/src/armnn/Network.cpp @@ -436,7 +436,8 @@ IOptimizedNetworkPtr Optimize(const INetwork& inNetwork, OptimizeInversePermutes(), MovePermuteUp(), PermuteAsReshape(), - OptimizeConsecutiveReshapes())); + OptimizeConsecutiveReshapes(), + FoldPadIntoConvolution2d())); // Infer the tensor infos for all output slots. Throws an exception on failure optGraph.InferTensorInfos(); diff --git a/src/armnn/optimizations/All.hpp b/src/armnn/optimizations/All.hpp index 0a6684ee3b..68965fd23c 100644 --- a/src/armnn/optimizations/All.hpp +++ b/src/armnn/optimizations/All.hpp @@ -13,3 +13,4 @@ #include "OptimizeInverseConversions.hpp" #include "ConvertFp32NetworkToFp16.hpp" #include "AddDebug.hpp" +#include "FoldPadIntoConvolution2d.hpp" diff --git a/src/armnn/optimizations/FoldPadIntoConvolution2d.hpp b/src/armnn/optimizations/FoldPadIntoConvolution2d.hpp new file mode 100644 index 0000000000..f789ffae9a --- /dev/null +++ b/src/armnn/optimizations/FoldPadIntoConvolution2d.hpp @@ -0,0 +1,82 @@ +// +// Copyright © 2017 Arm Ltd. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#pragma once + +#include "Optimization.hpp" + +namespace armnn +{ +namespace optimizations +{ + +class FoldPadIntoConvolution2dImpl +{ +public: + + void Run(Graph& graph, InputSlot& connection) const + { + Layer& base = connection.GetConnectedOutputSlot()->GetOwningLayer(); + Layer& child = connection.GetOwningLayer(); + + BOOST_ASSERT(base.GetType() == LayerType::Pad); + BOOST_ASSERT(child.GetType() == LayerType::Convolution2d); + + PadLayer* padLayer = boost::polymorphic_downcast(&base); + Convolution2dLayer* convolution2dLayer = boost::polymorphic_downcast(&child); + + OutputSlot* parentOut = base.GetInputSlot(0).GetConnectedOutputSlot(); + const TensorInfo& outInfo = child.GetOutputHandler().GetTensorInfo(); + + const std::string name = std::string("folded-") + base.GetName() + std::string("-into-") + child.GetName(); + Convolution2dDescriptor descriptor = convolution2dLayer->GetParameters(); + + auto padList = padLayer->GetParameters().m_PadList; + + armnn::DataLayout dataLayout = descriptor.m_DataLayout; + + // In Convolution2dDescriptor, padLeft and padRight are defined as paddings on width dimension + // whereas padTop and padBottom - paddings on height dimension, so setting these according to data layout + if(dataLayout == armnn::DataLayout::NHWC) + { + descriptor.m_PadLeft = padList[2].first; + descriptor.m_PadRight = padList[2].second; + descriptor.m_PadTop = padList[1].first; + descriptor.m_PadBottom = padList[1].second; + } + else + { + descriptor.m_PadLeft = padList[3].first; + descriptor.m_PadRight = padList[3].second; + descriptor.m_PadTop = padList[2].first; + descriptor.m_PadBottom = padList[2].second; + } + + auto& newConv2dLayer = *graph.InsertNewLayer(base.GetInputSlot(0), + descriptor, + name.c_str()); + newConv2dLayer.GetOutputHandler().SetTensorInfo(outInfo); + + // Reconnects with original parent. + newConv2dLayer.GetOutputSlot().MoveAllConnections(*parentOut); + // Parent is now the new convolution2d layer. + parentOut = &newConv2dLayer.GetOutputSlot(); + + // Moves connections in child output to parent layer. + // Child layer will be removed as it's left unconnected. + // Base layer will be removed if left unconnected. + child.GetOutputSlot().MoveAllConnections(*parentOut); + } +protected: + FoldPadIntoConvolution2dImpl() = default; + ~FoldPadIntoConvolution2dImpl() = default; +}; + +using FoldPadIntoConvolution2d = OptimizeForConnection; + +} // namespace optimizations +} // namespace armnn + + diff --git a/src/armnn/test/OptimizerTests.cpp b/src/armnn/test/OptimizerTests.cpp index f40a78a0d9..b0d8629c7f 100644 --- a/src/armnn/test/OptimizerTests.cpp +++ b/src/armnn/test/OptimizerTests.cpp @@ -1085,4 +1085,93 @@ BOOST_AUTO_TEST_CASE(DetectionPostProcessValidateTensorShapes) BOOST_CHECK_NO_THROW(graph.InferTensorInfos()); } +BOOST_AUTO_TEST_CASE(FoldPadLayerIntoConvolution2dLayer) +{ + Graph graph; + const unsigned int inputShape[] = { 1, 2, 2, 3 }; + const unsigned int paddedShape[] = { 1, 6, 6, 3 }; + const unsigned int weightsShape[] = { 1, 2, 3, 3 }; + const unsigned int outputShape[] = { 1, 2, 1, 1 }; + + + armnn::TensorInfo inputInfo(4, inputShape, DataType::Float32); + armnn::TensorInfo paddedInfo(4, paddedShape, DataType::Float32); + armnn::TensorInfo outputInfo(4, outputShape, DataType::Float32); + + Layer* input = graph.AddLayer(0, "input"); + input->GetOutputSlot().SetTensorInfo(inputInfo); + + PadDescriptor padDescriptor({{ 0, 0 }, { 2, 2 }, { 2, 2 }, { 0, 0 }}); + + PadLayer* padLayer = graph.AddLayer(padDescriptor, "pad"); + padLayer->GetOutputSlot().SetTensorInfo(paddedInfo); + + Convolution2dDescriptor convolution2dDescriptor; + convolution2dDescriptor.m_BiasEnabled = false; + convolution2dDescriptor.m_StrideX = 1; + convolution2dDescriptor.m_StrideY = 1; + convolution2dDescriptor.m_DataLayout = DataLayout::NHWC; + + std::vector weightsVector(18); + armnn::ConstTensor weights(armnn::TensorInfo(4, weightsShape, armnn::DataType::Float32), weightsVector); + + Convolution2dLayer* conv2dLayer = graph.AddLayer(convolution2dDescriptor,"conv2d"); + conv2dLayer->m_Weight = std::make_unique(weights); + conv2dLayer->GetOutputSlot().SetTensorInfo(outputInfo); + + Layer* output = graph.AddLayer(0, "output"); + + // Connect up layers - input -> pad -> conv2d -> output + input->GetOutputSlot().Connect(padLayer->GetInputSlot(0)); + padLayer->GetOutputSlot().Connect(conv2dLayer->GetInputSlot(0)); + conv2dLayer->GetOutputSlot().Connect(output->GetInputSlot(0)); + + auto checkSimpleConv2d = [ ](const armnn::Layer* const layer) -> bool + { + const auto conv2dLayer = static_cast(layer); + const auto conv2dLayerParams = conv2dLayer->GetParameters(); + return IsLayerOfType(layer) && + (layer->GetNameStr() == "conv2d") && + (conv2dLayerParams.m_PadLeft == 0) && + (conv2dLayerParams.m_PadRight == 0) && + (conv2dLayerParams.m_PadTop == 0) && + (conv2dLayerParams.m_PadBottom == 0) && + (conv2dLayerParams.m_BiasEnabled == false) && + (conv2dLayerParams.m_StrideX == 1) && + (conv2dLayerParams.m_StrideY == 1) && + (conv2dLayerParams.m_DataLayout == DataLayout::NHWC); + }; + + BOOST_TEST(CheckSequence(graph.cbegin(), + graph.cend(), + &IsLayerOfType, + &IsLayerOfType, + checkSimpleConv2d, + &IsLayerOfType)); + + armnn::Optimizer::Pass(graph, armnn::MakeOptimizations(FoldPadIntoConvolution2d())); + + auto checkPadFoldedIntoConv2d = [ ](const armnn::Layer* const layer) -> bool + { + const auto conv2dLayer = static_cast(layer); + const auto conv2dLayerParams = conv2dLayer->GetParameters(); + return IsLayerOfType(layer) && + (layer->GetNameStr() == "folded-pad-into-conv2d") && + (conv2dLayerParams.m_PadLeft == 2) && + (conv2dLayerParams.m_PadRight == 2) && + (conv2dLayerParams.m_PadTop == 2) && + (conv2dLayerParams.m_PadBottom == 2) && + (conv2dLayerParams.m_BiasEnabled == false) && + (conv2dLayerParams.m_StrideX == 1) && + (conv2dLayerParams.m_StrideY == 1) && + (conv2dLayerParams.m_DataLayout == DataLayout::NHWC); + }; + + BOOST_TEST(CheckSequence(graph.cbegin(), + graph.cend(), + &IsLayerOfType, + checkPadFoldedIntoConv2d, + &IsLayerOfType)); +} + BOOST_AUTO_TEST_SUITE_END() -- cgit v1.2.1