diff options
author | Cathal Corbett <cathal.corbett@arm.com> | 2022-07-22 16:03:36 +0100 |
---|---|---|
committer | Nikhil Raj <nikhil.raj@arm.com> | 2022-08-05 15:50:57 +0100 |
commit | 3883b2776cec33f16f0ea9a2d795de2b7c766df7 (patch) | |
tree | 6842e15904037d73426d814d5751945b3d9c2376 /src/backends | |
parent | 9d63fee68081b65bd72de3a70da76c2696c6c6ed (diff) | |
download | armnn-3883b2776cec33f16f0ea9a2d795de2b7c766df7.tar.gz |
GitHub #667: Neon fold padding into average pool 2D quantization bug fix.
* Originated from a GitHub issue: https://github.com/ARM-software/armnn/issues/667
* Initially, Arm NN supports the pool 2D operation because there is no padding
on the pool2d. Neon failure occurs when padding is followed by average pool 2D
due to folding optimization.
* Here we prevent the folding optimization from happening for the above special case
and add it in as a backend specific optimization.
Signed-off-by: Cathal Corbett <cathal.corbett@arm.com>
Change-Id: Ia0fd90c3a6b4b9d29c81106f154617d2e893e26b
Diffstat (limited to 'src/backends')
-rw-r--r-- | src/backends/aclCommon/ArmComputeSubgraphUtils.hpp | 46 | ||||
-rw-r--r-- | src/backends/backendsCommon/SubgraphUtils.hpp | 99 | ||||
-rw-r--r-- | src/backends/cl/ClBackend.cpp | 27 | ||||
-rw-r--r-- | src/backends/reference/RefBackend.cpp | 59 | ||||
-rw-r--r-- | src/backends/reference/RefBackend.hpp | 3 |
5 files changed, 184 insertions, 50 deletions
diff --git a/src/backends/aclCommon/ArmComputeSubgraphUtils.hpp b/src/backends/aclCommon/ArmComputeSubgraphUtils.hpp index a26442cb86..766bf2d2cc 100644 --- a/src/backends/aclCommon/ArmComputeSubgraphUtils.hpp +++ b/src/backends/aclCommon/ArmComputeSubgraphUtils.hpp @@ -1,5 +1,5 @@ // -// Copyright © 2020 Arm Ltd and Contributors. All rights reserved. +// Copyright © 2022 Arm Ltd and Contributors. All rights reserved. // SPDX-License-Identifier: MIT // @@ -9,6 +9,7 @@ #include <armnn/utility/Assert.hpp> #include <aclCommon/ArmComputeUtils.hpp> +#include <backendsCommon/SubgraphUtils.hpp> namespace armnn { @@ -20,36 +21,6 @@ namespace // this helper only works if all layers where the inputs connect to are not selected // -SubgraphView::IInputSlots CreateIInputsFrom(const std::vector<armnn::IConnectableLayer*>& layers) -{ - SubgraphView::IInputSlots result; - for (auto&& layer : layers) - { - for (unsigned int i = 0 ; i < layer->GetNumInputSlots(); ++i) - { - result.push_back(&(layer->GetInputSlot(i))); - } - } - return result; -} - -// -// this helper only works if all layers where the outputs connect to are not selected -// - -SubgraphView::IOutputSlots CreateIOutputsFrom(const std::vector<armnn::IConnectableLayer*>& layers) -{ - SubgraphView::IOutputSlots result; - for (auto &&layer: layers) - { - for (unsigned int i = 0; i < layer->GetNumOutputSlots(); ++i) - { - result.push_back(&(layer->GetOutputSlot(i))); - } - } - return result; -} - bool checkDataTypeInputandOutput(const Layer& layer) { auto inputInfo = layer.GetInputSlot(0).GetConnection()->GetTensorInfo(); @@ -79,19 +50,6 @@ bool checkDataTypeInputandOutput(const Layer& layer) } // namespace -inline void ReportUntouchedLayers(OptimizationViews& optimizationViews, std::map<LayerGuid, Layer*> untouched) -{ - std::vector<Layer*> untouchedVector; - for (const auto& pair : untouched) - { - Layer* layer = pair.second; - SubgraphView subgraphView({layer}, - CreateIInputsFrom({layer}), - CreateIOutputsFrom({layer})); - optimizationViews.AddUntouchedSubgraph(std::move(subgraphView)); - } -} - template<typename LayerType> LayerType* FuseLayer(OptimizationViews& optimizationViews, LayerType* baseLayer, diff --git a/src/backends/backendsCommon/SubgraphUtils.hpp b/src/backends/backendsCommon/SubgraphUtils.hpp new file mode 100644 index 0000000000..bd3d698a98 --- /dev/null +++ b/src/backends/backendsCommon/SubgraphUtils.hpp @@ -0,0 +1,99 @@ +// +// Copyright © 2022 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#pragma once + +#include <optimizations/FoldPadIntoLayer2d.hpp> + +namespace armnn +{ + +namespace +{ + +// +// this helper only works if all layers where the inputs connect to are not selected +// + +SubgraphView::IInputSlots CreateIInputsFrom(const std::vector<armnn::IConnectableLayer*>& layers) +{ + SubgraphView::IInputSlots result; + for (auto&& layer : layers) + { + for (unsigned int i = 0 ; i < layer->GetNumInputSlots(); ++i) + { + result.push_back(&(layer->GetInputSlot(i))); + } + } + return result; +} + +// +// this helper only works if all layers where the outputs connect to are not selected +// + +SubgraphView::IOutputSlots CreateIOutputsFrom(const std::vector<armnn::IConnectableLayer*>& layers) +{ + SubgraphView::IOutputSlots result; + for (auto &&layer: layers) + { + for (unsigned int i = 0; i < layer->GetNumOutputSlots(); ++i) + { + result.push_back(&(layer->GetOutputSlot(i))); + } + } + return result; +} + +} + +inline void ReportUntouchedLayers(OptimizationViews& optimizationViews, std::map<LayerGuid, Layer*> untouched) +{ + std::vector<Layer*> untouchedVector; + for (const auto& pair : untouched) + { + Layer* layer = pair.second; + SubgraphView subgraphView({layer}, + CreateIInputsFrom({layer}), + CreateIOutputsFrom({layer})); + optimizationViews.AddUntouchedSubgraph(std::move(subgraphView)); + } +} + +template<typename LayerType> +LayerType* FoldPadLayer(OptimizationViews& optimizationViews, + LayerType* baseLayer, + LayerType* replacementLayer, + PadLayer* padLayer) +{ + SubgraphView substitutionSubgraph({padLayer, baseLayer}, + CreateIInputsFrom({padLayer}), + CreateIOutputsFrom({baseLayer})); + SubgraphView replacementSubgraph(replacementLayer); + + optimizationViews.AddSubstitution({substitutionSubgraph, replacementSubgraph}); + + return replacementLayer; +} + +template<typename LayerType> +LayerType* FoldPadIntoAveragePool2d(OptimizationViews& optimizationViews, + Pooling2dLayer* baseLayer, + Pooling2dDescriptor& poolDescriptor, + PadLayer* padLayer) +{ + IConnectableLayer* replacement = + optimizationViews.GetINetwork()->AddPooling2dLayer(poolDescriptor, "folded-pad-into-pool2d"); + LayerType* replacementLayer = PolymorphicDowncast<LayerType*>(replacement); + + FoldPadLayer(optimizationViews, + baseLayer, + replacementLayer, + padLayer); + + return replacementLayer; +} + +} // namespace armnn diff --git a/src/backends/cl/ClBackend.cpp b/src/backends/cl/ClBackend.cpp index 1fe53de62a..d2e8fbfe32 100644 --- a/src/backends/cl/ClBackend.cpp +++ b/src/backends/cl/ClBackend.cpp @@ -1,5 +1,5 @@ // -// Copyright © 2017 Arm Ltd. All rights reserved. +// Copyright © 2022 Arm Ltd. All rights reserved. // SPDX-License-Identifier: MIT // @@ -566,6 +566,31 @@ OptimizationViews ClBackend::OptimizeSubgraphView(const SubgraphView& subgraph, untouched.erase(baseLayer->GetGuid()); } } + + // Special case to fuse padding into average pooling 2d for quantized datatype. + // Required to be done as a backend specific optimization as Neon does not support this special case. + if (base.GetType() == LayerType::Pooling2d) + { + Pooling2dLayer* baseLayer = PolymorphicDowncast<Pooling2dLayer*>(&base); + Pooling2dDescriptor poolingDescriptor = baseLayer->GetParameters(); + + if (baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetOwningLayer().GetType() == LayerType::Pad) + { + PadLayer* padLayer = PolymorphicDowncast<PadLayer*>( + &baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetOwningLayer()); + if (padLayer->GetOutputSlot(0).GetNumConnections() == 1 && + optimizations::pad_fold::TryFoldPadIntoLayer2d(padLayer->GetParameters(), + poolingDescriptor, + padLayer->GetOutputSlot().GetTensorInfo(), + true)) + { + FoldPadIntoAveragePool2d<Pooling2dLayer>(optimizationViews, baseLayer, + poolingDescriptor, padLayer); + untouched.erase(baseLayer->GetGuid()); + untouched.erase(padLayer->GetGuid()); + } + } + } } if (optimizationViews.GetSubstitutions().empty()) diff --git a/src/backends/reference/RefBackend.cpp b/src/backends/reference/RefBackend.cpp index a33a7756a0..8c8879c8be 100644 --- a/src/backends/reference/RefBackend.cpp +++ b/src/backends/reference/RefBackend.cpp @@ -14,6 +14,7 @@ #include <armnn/backends/IMemoryManager.hpp> #include <armnn/utility/PolymorphicDowncast.hpp> #include <backendsCommon/DefaultAllocator.hpp> +#include <backendsCommon/SubgraphUtils.hpp> #include <Optimizer.hpp> @@ -70,11 +71,61 @@ IBackendInternal::ILayerSupportSharedPtr RefBackend::GetLayerSupport() const return layerSupport; } -OptimizationViews RefBackend::OptimizeSubgraphView(const SubgraphView& subgraph) const +OptimizationViews RefBackend::OptimizeSubgraphView(const SubgraphView& subgraph, + const ModelOptions& modelOptions) const { - OptimizationViews optimizationViews; - - optimizationViews.AddUntouchedSubgraph(SubgraphView(subgraph)); + OptimizationViews optimizationViews(modelOptions); + + auto it = subgraph.endIConnectable(); + std::map<LayerGuid, Layer*> untouched; + + while (it != subgraph.beginIConnectable()) + { + --it; + Layer& base = *(PolymorphicDowncast<Layer*>(*it)); + untouched.insert({base.GetGuid(), &base}); + } + + it = subgraph.endIConnectable(); + while (it != subgraph.beginIConnectable()) + { + --it; + Layer& base = *(PolymorphicDowncast<Layer*>(*it)); + + // Special case to fuse padding into average pooling 2d for quantized datatype. + // Required to be done as a backend specific optimization as Neon does not support this special case. + if (base.GetType() == LayerType::Pooling2d) + { + Pooling2dLayer* baseLayer = PolymorphicDowncast<Pooling2dLayer*>(&base); + Pooling2dDescriptor poolingDescriptor = baseLayer->GetParameters(); + + if (baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetOwningLayer().GetType() == LayerType::Pad) + { + PadLayer* padLayer = PolymorphicDowncast<PadLayer*>( + &baseLayer->GetInputSlot(0).GetConnectedOutputSlot()->GetOwningLayer()); + if (padLayer->GetOutputSlot(0).GetNumConnections() == 1 && + optimizations::pad_fold::TryFoldPadIntoLayer2d(padLayer->GetParameters(), + poolingDescriptor, + padLayer->GetOutputSlot().GetTensorInfo(), + true)) + { + FoldPadIntoAveragePool2d<Pooling2dLayer>(optimizationViews, baseLayer, + poolingDescriptor, padLayer); + untouched.erase(baseLayer->GetGuid()); + untouched.erase(padLayer->GetGuid()); + } + } + } + } + + if (optimizationViews.GetSubstitutions().empty()) + { + optimizationViews.AddUntouchedSubgraph(SubgraphView(subgraph)); + } + else + { + ReportUntouchedLayers(optimizationViews, untouched); + } return optimizationViews; } diff --git a/src/backends/reference/RefBackend.hpp b/src/backends/reference/RefBackend.hpp index 9828d09f51..ecbe4d5ba9 100644 --- a/src/backends/reference/RefBackend.hpp +++ b/src/backends/reference/RefBackend.hpp @@ -50,7 +50,8 @@ public: IBackendInternal::ILayerSupportSharedPtr GetLayerSupport() const override; - OptimizationViews OptimizeSubgraphView(const SubgraphView& subgraph) const override; + OptimizationViews OptimizeSubgraphView(const SubgraphView& subgraph, + const ModelOptions& modelOptions) const override; std::vector<ITensorHandleFactory::FactoryId> GetHandleFactoryPreferences() const override; |