From 4cc341cf8b5a6e6bb0543504cbbfde6fa11a2cdb Mon Sep 17 00:00:00 2001 From: Mike Kelly Date: Fri, 7 Jul 2023 15:43:06 +0100 Subject: IVGCVSW-7830 Add backend optimizations to remove Reshapes where possible * Added optimization to remove reshapes for Neon and Ref Backends by using overridden TensorInfos * Added ability to delete Subgraphs during Optimization * Fixed naming error in NeonEndToEndTests and CLEndToEndTests * Added LayerNameAndTypeCheck for testing. * Fixed error where layers were not marked as altered when removed in CLBackend Signed-off-by: Mike Kelly Change-Id: I1ac25cd4ec9821470d961831ae2c8d24882276cc --- src/backends/backendsCommon/test/CMakeLists.txt | 1 + .../backendsCommon/test/SubgraphUtilsTest.hpp | 399 +++++++++++++++++++++ 2 files changed, 400 insertions(+) create mode 100644 src/backends/backendsCommon/test/SubgraphUtilsTest.hpp (limited to 'src/backends/backendsCommon/test') diff --git a/src/backends/backendsCommon/test/CMakeLists.txt b/src/backends/backendsCommon/test/CMakeLists.txt index 0139044432..8d6891a68d 100644 --- a/src/backends/backendsCommon/test/CMakeLists.txt +++ b/src/backends/backendsCommon/test/CMakeLists.txt @@ -59,6 +59,7 @@ list(APPEND armnnBackendsCommonUnitTests_sources SpaceToDepthEndToEndTestImpl.hpp SplitterEndToEndTestImpl.hpp StridedSliceAsyncEndToEndTest.hpp + SubgraphUtilsTest.hpp SubtractionEndToEndTestImpl.hpp TransposeEndToEndTestImpl.hpp TensorCopyUtils.hpp diff --git a/src/backends/backendsCommon/test/SubgraphUtilsTest.hpp b/src/backends/backendsCommon/test/SubgraphUtilsTest.hpp new file mode 100644 index 0000000000..957726797b --- /dev/null +++ b/src/backends/backendsCommon/test/SubgraphUtilsTest.hpp @@ -0,0 +1,399 @@ +// +// Copyright © 2023 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include "backendsCommon/SubgraphUtils.hpp" + +namespace armnn +{ + +template, typename TOutput = ResolveType> +void EndToEndLayerTest(IRuntimePtr runtime, + IOptimizedNetworkPtr optNet, + const std::map>& inputTensorData, + const std::map>& expectedOutputData, + float tolerance = 0.000001f) +{ + // Loads it into the runtime. + NetworkId netId; + std::string errorMessage; + armnn::Status loadingStatus = runtime->LoadNetwork(netId, std::move(optNet), errorMessage); + CHECK_MESSAGE(loadingStatus == Status::Success, errorMessage); + + InputTensors inputTensors; + inputTensors.reserve(inputTensorData.size()); + for (auto&& it : inputTensorData) + { + inputTensors.push_back({it.first, + ConstTensor(runtime->GetInputTensorInfo(netId, it.first), it.second.data())}); + } + OutputTensors outputTensors; + outputTensors.reserve(expectedOutputData.size()); + std::map> outputStorage; + for (auto&& it : expectedOutputData) + { + std::vector out(it.second.size()); + outputStorage.emplace(it.first, out); + outputTensors.push_back({it.first, + Tensor(runtime->GetOutputTensorInfo(netId, it.first), + outputStorage.at(it.first).data())}); + } + + // Does the inference. + runtime->EnqueueWorkload(netId, inputTensors, outputTensors); + + // Checks the results. + for (auto&& it : expectedOutputData) + { + std::vector out = outputStorage.at(it.first); + for (unsigned int i = 0; i < out.size(); ++i) + { + CHECK_MESSAGE(Compare(it.second[i], out[i], tolerance) == true, + "Actual output: " << out[i] << ". Expected output:" << it.second[i]); + + } + } +} + +template> +armnn::INetworkPtr CreateReshapeInOutNetwork(const armnn::TensorShape& inputShape, + const armnn::TensorShape& outputShape, + ReshapeDescriptor& descriptor, + const float qScale = 1.0f, + const int32_t qOffset = 0) +{ + armnn::INetworkPtr network(armnn::INetwork::Create()); + + TensorInfo inputTensorInfo(inputShape, ArmnnType, qScale, qOffset, true); + TensorInfo outputTensorInfo(outputShape, ArmnnType, qScale, qOffset); + + IConnectableLayer* activation0 = network->AddActivationLayer(ActivationFunction::ReLu, "act0"); + IConnectableLayer* activation1 = network->AddActivationLayer(ActivationFunction::ReLu, "act1"); + IConnectableLayer* activation2 = network->AddActivationLayer(ActivationFunction::ReLu, "act2"); + IConnectableLayer* activation3 = network->AddActivationLayer(ActivationFunction::ReLu, "act3"); + IConnectableLayer* reshape = network->AddReshapeLayer(descriptor, "reshape"); + + IConnectableLayer* input = network->AddInputLayer(0, "input"); + IConnectableLayer* output1 = network->AddOutputLayer(0, "output1"); + IConnectableLayer* output2 = network->AddOutputLayer(1, "output2"); + IConnectableLayer* output3 = network->AddOutputLayer(2, "output3"); + + Connect(input, activation0, inputTensorInfo, 0, 0); + Connect(activation0, reshape, inputTensorInfo, 0, 0); + + Connect(reshape, activation1, outputTensorInfo, 0, 0); + Connect(reshape, activation2, outputTensorInfo, 0, 0); + Connect(reshape, activation3, outputTensorInfo, 0, 0); + Connect(activation1, output1, outputTensorInfo, 0, 0); + Connect(activation2, output2, outputTensorInfo, 0, 0); + Connect(activation3, output3, outputTensorInfo, 0, 0); + + return network; +} + +template> +armnn::INetworkPtr CreateReshapeConv2dInOutNetwork(const armnn::TensorShape& inputShape, + const armnn::TensorShape& weightsShape, + const armnn::TensorShape& convOutputShape, + const armnn::TensorShape& outputShape, + std::vector& weightsData, + ReshapeDescriptor& descriptor, + Convolution2dDescriptor& convolution2DDescriptor, + bool convFirst, + const float qScale = 1.0f, + const int32_t qOffset = 0) +{ + armnn::INetworkPtr network(armnn::INetwork::Create()); + TensorInfo weightsTensorInfo(weightsShape, ArmnnType, qScale, qOffset, true); + ConstTensor weights(weightsTensorInfo, weightsData); + + IConnectableLayer* convolution1 = network->AddConvolution2dLayer(convolution2DDescriptor, "conv2d"); + IConnectableLayer* weightsLayer = network->AddConstantLayer(weights, "weights"); + + IConnectableLayer* activation1 = network->AddActivationLayer(ActivationFunction::ReLu, "act"); + IConnectableLayer* reshape = network->AddReshapeLayer(descriptor, "reshape"); + + IConnectableLayer* input = network->AddInputLayer(0, "input"); + IConnectableLayer* output = network->AddOutputLayer(0, "output"); + + TensorInfo inputTensorInfo(inputShape, ArmnnType, qScale, qOffset, true); + TensorInfo convTensorInfo(convOutputShape, ArmnnType, qScale, qOffset, true); + TensorInfo outputTensorInfo(outputShape, ArmnnType, qScale, qOffset); + TensorInfo reshapeTensorInfo(descriptor.m_TargetShape, ArmnnType, qScale, qOffset, true); + + if (convFirst) + { + Connect(input, convolution1, inputTensorInfo, 0, 0); + Connect(weightsLayer, convolution1, weightsTensorInfo, 0, 1); + Connect(convolution1, reshape, convTensorInfo, 0, 0); + Connect(reshape, activation1, reshapeTensorInfo, 0, 0); + Connect(activation1, output, outputTensorInfo, 0, 0); + } + else + { + Connect(input, activation1, inputTensorInfo, 0, 0); + Connect(activation1, reshape, inputTensorInfo, 0, 0); + Connect(reshape, convolution1, reshapeTensorInfo, 0, 0); + Connect(weightsLayer, convolution1, weightsTensorInfo, 0, 1); + Connect(convolution1, output, outputTensorInfo, 0, 0); + } + return network; +} + +template> +void ReshapeRemovalEndToEnd(const std::vector& backends) +{ + using namespace armnn; + + const TensorShape& inputShape = { 2, 3 }; + const TensorShape& outputShape = { 6 }; + + ReshapeDescriptor descriptor; + descriptor.m_TargetShape = outputShape; + + INetworkPtr network = CreateReshapeInOutNetwork(inputShape, outputShape, descriptor); + + CHECK(network); + + std::vector data{ 1, 2, 3, + 4, 5, 6 }; + + std::map> inputTensorData = { { 0, data } }; + std::map> expectedOutputData = { { 0, data }, { 1, data }, { 2, data } }; + + // Create runtime in which test will run + IRuntime::CreationOptions options; + IRuntimePtr runtime(IRuntime::Create(options)); + + // optimize the network + IOptimizedNetworkPtr optNet = Optimize(*network, backends, runtime->GetDeviceSpec()); + + Graph& graph = GetGraphForTesting(optNet.get()); + CHECK(CheckSequence(graph.cbegin(), graph.cend(), + LayerNameAndTypeCheck(LayerType::Input, "input"), + LayerNameAndTypeCheck(LayerType::Activation, "act0"), + LayerNameAndTypeCheck(LayerType::Activation, "act1"), + LayerNameAndTypeCheck(LayerType::Activation, "act2"), + LayerNameAndTypeCheck(LayerType::Activation, "act3"), + LayerNameAndTypeCheck(LayerType::Output, "output1"), + LayerNameAndTypeCheck(LayerType::Output, "output2"), + LayerNameAndTypeCheck(LayerType::Output, "output3"))); + + EndToEndLayerTest(std::move(runtime), std::move(optNet), inputTensorData, expectedOutputData); +} + +template> +void ReshapeRemovalNCHWEndToEnd(const std::vector& backends, bool shouldBeRemoved, bool convFirst) +{ + using namespace armnn; + + // shapes are different if convFirst or not + //these are convFirst + TensorShape inputShape; + TensorShape convOutputShape; + TensorShape weightsShape; + TensorShape reshapeShape; + TensorShape outputShape; + + if (convFirst) + { + inputShape = { 1, 1, 5, 5 }; + convOutputShape = { 1, 1, 3, 3 }; + weightsShape = { 1, 1, 3, 3 }; + reshapeShape = { 9 }; + outputShape = { 9 }; + } + else + { + inputShape = { 5, 5 }; + reshapeShape = { 1, 1, 5, 5 }; + convOutputShape = { 1, 1, 3, 3 }; + weightsShape = { 1, 1, 3, 3 }; + outputShape = { 1, 1, 3, 3 }; + } + + ReshapeDescriptor descriptor; + descriptor.m_TargetShape = reshapeShape; + + Convolution2dDescriptor convolution2DDescriptor; + convolution2DDescriptor.m_PadLeft = 0; + convolution2DDescriptor.m_PadRight = 0; + convolution2DDescriptor.m_PadTop = 0; + convolution2DDescriptor.m_PadBottom = 0; + convolution2DDescriptor.m_StrideX = 1; + convolution2DDescriptor.m_StrideY = 1; + convolution2DDescriptor.m_DataLayout = DataLayout::NCHW; + convolution2DDescriptor.m_BiasEnabled = false; + + TensorInfo inputInfo(inputShape, DataType::Float32, true); + TensorInfo outputInfo(convOutputShape, DataType::Float32); + TensorInfo weightsInfo(weightsShape, DataType::Float32, true); + + std::vector inputData = + { + 1.0f, 8.0f, 3.0f, 4.0f, 6.0f, + 5.0f, 7.0f, 3.0f, 1.0f, 8.0f, + 2.0f, 3.0f, 9.0f, 8.0f, 1.0f, + 3.0f, 6.0f, 1.0f, 1.0f, 9.0f, + 5.0f, 3.0f, 9.0f, 3.0f, 2.0f + }; + + std::vector weightsData = + { + 4.0f, 0.0f, 3.0f, + 5.0f, 0.0f, 2.0f, + 6.0f, 0.0f, 1.0f + }; + + std::vector outputData = + { + 65.0f, 107.0f, 116.0f, + 76.0f, 99.0f, 98.0f, + 91.0f, 89.0f, 118.0f + }; + + INetworkPtr network = CreateReshapeConv2dInOutNetwork(inputShape, + weightsShape, + convOutputShape, + outputShape, + weightsData, + descriptor, + convolution2DDescriptor, + convFirst); + CHECK(network); + + std::map> inputTensorData = { { 0, inputData } }; + std::map> expectedOutputData = { { 0, outputData } }; + + // Create runtime in which test will run + IRuntime::CreationOptions options; + IRuntimePtr runtime(IRuntime::Create(options)); + + // optimize the network + IOptimizedNetworkPtr optNet = Optimize(*network, backends, runtime->GetDeviceSpec()); + + Graph& graph = GetGraphForTesting(optNet.get()); + + if (shouldBeRemoved) + { + if (convFirst) + { + CHECK(CheckSequence(graph.cbegin(), graph.cend(), + LayerNameAndTypeCheck(LayerType::Input, "input"), + LayerNameAndTypeCheck(LayerType::Constant, "weights"), + LayerNameAndTypeCheck(LayerType::Convolution2d, "conv2d"), + LayerNameAndTypeCheck(LayerType::Activation, "act"), + LayerNameAndTypeCheck(LayerType::Output, "output"))); + } + else + { + CHECK(CheckSequence(graph.cbegin(), graph.cend(), + LayerNameAndTypeCheck(LayerType::Input, "input"), + LayerNameAndTypeCheck(LayerType::Constant, "weights"), + LayerNameAndTypeCheck(LayerType::Activation, "act"), + LayerNameAndTypeCheck(LayerType::Convolution2d, "conv2d"), + LayerNameAndTypeCheck(LayerType::Output, "output"))); + } + } + else + { + if (convFirst) + { + CHECK(CheckSequence(graph.cbegin(), graph.cend(), + LayerNameAndTypeCheck(LayerType::Input, "input"), + LayerNameAndTypeCheck(LayerType::Constant, "weights"), + LayerNameAndTypeCheck(LayerType::Convolution2d, "conv2d"), + LayerNameAndTypeCheck(LayerType::Reshape, "reshape"), + LayerNameAndTypeCheck(LayerType::Activation, "act"), + LayerNameAndTypeCheck(LayerType::Output, "output"))); + } + else + { + CHECK(CheckSequence(graph.cbegin(), graph.cend(), + LayerNameAndTypeCheck(LayerType::Input, "input"), + LayerNameAndTypeCheck(LayerType::Constant, "weights"), + LayerNameAndTypeCheck(LayerType::Activation, "act"), + LayerNameAndTypeCheck(LayerType::Reshape, "reshape"), + LayerNameAndTypeCheck(LayerType::Convolution2d, "conv2d"), + LayerNameAndTypeCheck(LayerType::Output, "output"))); + } + } + + EndToEndLayerTest(std::move(runtime), std::move(optNet), inputTensorData, expectedOutputData); +} + +template +void CheckIsNCHW() +{ + Graph graph; + Descriptor nchwDesc; + nchwDesc.m_DataLayout = DataLayout::NCHW; + auto nchwLayer = graph.AddLayer(nchwDesc, ""); + CHECK(IsNCHW(*nchwLayer)); + + Descriptor nhwcDesc; + nhwcDesc.m_DataLayout = DataLayout::NHWC; + auto nhwcLayer = graph.AddLayer(nhwcDesc, ""); + CHECK_FALSE(IsNCHW(*nhwcLayer)); +} + +TEST_CASE("CheckIsNCHW") +{ + Graph graph; + BatchMatMulDescriptor descriptor1; + descriptor1.m_DataLayoutX = DataLayout::NHWC; + descriptor1.m_DataLayoutY = DataLayout::NHWC; + auto batchMatMulLayer1 = graph.AddLayer(descriptor1, ""); + CHECK_FALSE(IsNCHW(*batchMatMulLayer1)); + + BatchMatMulDescriptor descriptor2; + descriptor2.m_DataLayoutX = DataLayout::NCHW; + descriptor2.m_DataLayoutY = DataLayout::NHWC; + auto batchMatMulLayer2 = graph.AddLayer(descriptor2, ""); + CHECK(IsNCHW(*batchMatMulLayer2)); + + BatchMatMulDescriptor descriptor3; + descriptor3.m_DataLayoutX = DataLayout::NHWC; + descriptor3.m_DataLayoutY = DataLayout::NCHW; + auto batchMatMulLayer3 = graph.AddLayer(descriptor3, ""); + CHECK(IsNCHW(*batchMatMulLayer3)); + + BatchMatMulDescriptor descriptor4; + descriptor4.m_DataLayoutX = DataLayout::NCHW; + descriptor4.m_DataLayoutY = DataLayout::NCHW; + auto batchMatMulLayer4 = graph.AddLayer(descriptor4, ""); + CHECK(IsNCHW(*batchMatMulLayer4)); + + CheckIsNCHW(); + CheckIsNCHW(); + CheckIsNCHW(); + CheckIsNCHW(); + CheckIsNCHW(); + CheckIsNCHW(); + CheckIsNCHW(); + CheckIsNCHW(); + CheckIsNCHW(); + CheckIsNCHW(); + CheckIsNCHW(); + CheckIsNCHW(); + + // Check Default + auto elementwiseLayer = graph.AddLayer(BinaryOperation::Add, ""); + CHECK_FALSE(IsNCHW(*elementwiseLayer)); +} + + +} // Namespace -- cgit v1.2.1