From 164bf4f29f6f1b2a3e6714ef4f5a21fc0fd16c2b Mon Sep 17 00:00:00 2001 From: Matthew Sloyan Date: Fri, 28 Oct 2022 18:02:17 +0100 Subject: IVGCVSW-7164 Implement TosaRefBackend::OptimizeSubgraphView * Added TosaRefBackend::OptimizeSubgraphView implementation. * Generalised TosaRefLayerSupport::IsLayerSupported to work with any operator. * Changed TosaCommon.hpp utils to inline functions. * Added source files for TosaMappings.hpp and AdditionOperator.hpp. * Fixed multiple defines issue with HALF_ROUND_STYLE and HALF_ROUND_TIES_TO_EVEN. Signed-off-by: Matthew Sloyan Change-Id: Ib2576ec3fb97faa3a2256b2fb93ec16ac8745760 --- src/armnnUtils/Half.hpp | 8 ++- src/backends/tosaCommon/CMakeLists.txt | 1 + src/backends/tosaCommon/TosaMappings.cpp | 66 ++++++++++++++++++++++ src/backends/tosaCommon/TosaMappings.hpp | 54 +----------------- .../operatorMappings/AdditionOperator.cpp | 44 +++++++++++++++ .../operatorMappings/AdditionOperator.hpp | 36 +----------- .../tosaCommon/operatorMappings/CMakeLists.txt | 1 + .../operatorMappings/TosaOperatorUtils.hpp | 6 +- src/backends/tosaReference/TosaRefBackend.cpp | 45 ++++++++++++++- src/backends/tosaReference/TosaRefLayerSupport.cpp | 35 ++++++++---- src/backends/tosaReference/test/CMakeLists.txt | 1 + .../test/TosaRefOptimizedNetworkTests.cpp | 54 ++++++++++++++++++ 12 files changed, 246 insertions(+), 105 deletions(-) create mode 100644 src/backends/tosaCommon/TosaMappings.cpp create mode 100644 src/backends/tosaCommon/operatorMappings/AdditionOperator.cpp create mode 100644 src/backends/tosaReference/test/TosaRefOptimizedNetworkTests.cpp diff --git a/src/armnnUtils/Half.hpp b/src/armnnUtils/Half.hpp index d55c2f9ecd..e39968f54d 100644 --- a/src/armnnUtils/Half.hpp +++ b/src/armnnUtils/Half.hpp @@ -8,8 +8,12 @@ #include // Set style to round to nearest -#define HALF_ROUND_STYLE 1 -#define HALF_ROUND_TIES_TO_EVEN 1 +#ifndef HALF_ROUND_STYLE + #define HALF_ROUND_STYLE 1 +#endif +#ifndef HALF_ROUND_TIES_TO_EVEN + #define HALF_ROUND_TIES_TO_EVEN 1 +#endif #include "half/half.hpp" diff --git a/src/backends/tosaCommon/CMakeLists.txt b/src/backends/tosaCommon/CMakeLists.txt index 83737d3bd3..1b1cc55eab 100644 --- a/src/backends/tosaCommon/CMakeLists.txt +++ b/src/backends/tosaCommon/CMakeLists.txt @@ -9,6 +9,7 @@ include_directories(SYSTEM ${TOSA_SERIALIZATION_LIB_INCLUDE}) list(APPEND armnnTosaBackend_sources TosaMappings.hpp + TosaMappings.cpp TosaLayerSupportRules.hpp ) diff --git a/src/backends/tosaCommon/TosaMappings.cpp b/src/backends/tosaCommon/TosaMappings.cpp new file mode 100644 index 0000000000..3c14bfd1f9 --- /dev/null +++ b/src/backends/tosaCommon/TosaMappings.cpp @@ -0,0 +1,66 @@ +// +// Copyright © 2022 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include "TosaMappings.hpp" + +using namespace armnn; +using namespace tosa; + +void SetBasicBlockConstantTensorData(Layer* layer, TosaSerializationBasicBlock* /*basicBlock*/) +{ + switch (layer->GetType()) + { + case LayerType::Convolution2d: + { + // ToDo: using Convolution2d as an example as it has constant tensors for weights and bias. + // ToDo: manually set TosaOperator data of basicBlock where constant tensors exist. + } + default: + // If no switch statement for layer, no constant tensors exist in that layer, return + return; + } +} + +TosaSerializationBasicBlock* GetTosaMapping(const LayerType type, + const std::vector& inputs, + const std::vector& outputs, + const BaseDescriptor& /*descriptor*/) +{ + switch (type) + { + case LayerType::Addition: + { + return ConvertAdditionToTosaOperator(inputs, outputs); + } + default: + { + // empty basic block when no tosa mapping implemented/exists + TosaSerializationOperator* op = new TosaSerializationOperator(Op_UNKNOWN, Attribute_NONE, nullptr, {}, {}); + return new TosaSerializationBasicBlock("", {op}, {}, {}, {}); + } + } +} + +TosaSerializationBasicBlock* GetTosaMappingFromLayer(Layer* layer) +{ + std::vector inputs; + for (auto inputSlot : layer->GetInputSlots()) + { + inputs.push_back(&inputSlot.GetConnection()->GetTensorInfo()); + } + + std::vector outputs; + for (auto& outputSlot : layer->GetOutputSlots()) + { + outputs.push_back(&outputSlot.GetTensorInfo()); + } + + TosaSerializationBasicBlock* basicBlock = GetTosaMapping(layer->GetType(), + inputs, + outputs, + layer->GetParameters()); + SetBasicBlockConstantTensorData(layer, basicBlock); + return basicBlock; +} diff --git a/src/backends/tosaCommon/TosaMappings.hpp b/src/backends/tosaCommon/TosaMappings.hpp index 5728ff3203..c721bcaf59 100644 --- a/src/backends/tosaCommon/TosaMappings.hpp +++ b/src/backends/tosaCommon/TosaMappings.hpp @@ -15,20 +15,7 @@ using namespace tosa; // From the input armnn::Layer, set the corresponding data field in the // tosa::TosaSerializationTensor where constant tensor data exists in the armnn::Layer. -void SetBasicBlockConstantTensorData(Layer* layer, TosaSerializationBasicBlock* /*basicBlock*/) -{ - switch (layer->GetType()) - { - case LayerType::Convolution2d: - { - // ToDo: using Convolution2d as an example as it has constant tensors for weights and bias. - // ToDo: manually set TosaOperator data of basicBlock where constant tensors exist. - } - default: - // If no switch statement for layer, no constant tensors exist in that layer, return - return; - } -} +void SetBasicBlockConstantTensorData(Layer* layer, TosaSerializationBasicBlock* /*basicBlock*/); // Populates a tosa::TosaSerializationBasicBlock from constructing // tosa::TosaSerializationOperator(s) and tosa::TosaSerializationTensor(s) @@ -39,43 +26,8 @@ void SetBasicBlockConstantTensorData(Layer* layer, TosaSerializationBasicBlock* TosaSerializationBasicBlock* GetTosaMapping(const LayerType type, const std::vector& inputs, const std::vector& outputs, - const BaseDescriptor& /*descriptor*/) -{ - switch (type) - { - case LayerType::Addition: - { - return ConvertAdditionToTosaOperator(inputs, outputs); - } - default: - { - // empty basic block when no tosa mapping implemented/exists - TosaSerializationOperator* op = new TosaSerializationOperator(Op_UNKNOWN, Attribute_NONE, nullptr, {}, {}); - return new TosaSerializationBasicBlock("", {op}, {}, {}, {}); - } - } -} + const BaseDescriptor& /*descriptor*/); // Function called in armnn::OptimizeSubgraphView() when access to armnn::Layer is available // and there is an option to set tosa basic block data from constant layer tenors available from the input layer. -TosaSerializationBasicBlock* GetTosaMappingFromLayer(Layer* layer) -{ - std::vector inputs; - for (auto inputSlot : layer->GetInputSlots()) - { - inputs.push_back(&inputSlot.GetConnection()->GetTensorInfo()); - } - - std::vector outputs; - for (auto& outputSlot : layer->GetOutputSlots()) - { - outputs.push_back(&outputSlot.GetTensorInfo()); - } - - TosaSerializationBasicBlock* basicBlock = GetTosaMapping(layer->GetType(), - inputs, - outputs, - layer->GetParameters()); - SetBasicBlockConstantTensorData(layer, basicBlock); - return basicBlock; -} +TosaSerializationBasicBlock* GetTosaMappingFromLayer(Layer* layer); diff --git a/src/backends/tosaCommon/operatorMappings/AdditionOperator.cpp b/src/backends/tosaCommon/operatorMappings/AdditionOperator.cpp new file mode 100644 index 0000000000..98ea03ac3c --- /dev/null +++ b/src/backends/tosaCommon/operatorMappings/AdditionOperator.cpp @@ -0,0 +1,44 @@ +// +// Copyright © 2022 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include "AdditionOperator.hpp" + +TosaSerializationBasicBlock* ConvertAdditionToTosaOperator(const std::vector& inputs, + const std::vector& outputs) +{ + // A helper function with static global variables ensures uniqueness + // for dynamically generating input, output and block names + std::string input0Name = std::string("Op_ADD_input0_") + GetUniqueTosaMappingID(); + std::string input1Name = std::string("Op_ADD_input1_") + GetUniqueTosaMappingID(); + std::string outputName = std::string("Op_ADD_output0_") + GetUniqueTosaMappingID(); + std::string blockName = std::string("Op_ADD_block_") + GetUniqueTosaMappingID(); + + TosaSerializationOperator* op = new TosaSerializationOperator(Op_ADD, + Attribute_NONE, + nullptr, + {input0Name, input1Name}, + {outputName}); + + std::vector inputShape0 = GetTosaTensorShape(inputs[0]->GetShape()); + DType inputDType0 = ArmNNToDType(inputs[0]->GetDataType()); + + std::vector inputShape1 = GetTosaTensorShape(inputs[1]->GetShape()); + DType inputDType1 = ArmNNToDType(inputs[1]->GetDataType()); + + std::vector outputShape0 = GetTosaTensorShape(outputs[0]->GetShape()); + DType outputDType0 = ArmNNToDType(outputs[0]->GetDataType()); + + TosaSerializationTensor* inputTensor0 = new TosaSerializationTensor(input0Name, inputShape0, inputDType0, {}); + TosaSerializationTensor* inputTensor1 = new TosaSerializationTensor(input1Name, inputShape1, inputDType1, {}); + TosaSerializationTensor* outputTensor0 = new TosaSerializationTensor(outputName, outputShape0, outputDType0, {}); + + // operatorInputNames/operatorOutputNames ends up being the same as + // blockInputNames/blockOutputNames for one-to-one ArmNN to Tosa mappings + return new TosaSerializationBasicBlock(blockName, // name + {op}, // operators + {inputTensor0, inputTensor1, outputTensor0}, // tensors + {input0Name, input1Name}, // inputs + {outputName}); // outputs +} \ No newline at end of file diff --git a/src/backends/tosaCommon/operatorMappings/AdditionOperator.hpp b/src/backends/tosaCommon/operatorMappings/AdditionOperator.hpp index 98c01e2cb8..2a9c479d8e 100644 --- a/src/backends/tosaCommon/operatorMappings/AdditionOperator.hpp +++ b/src/backends/tosaCommon/operatorMappings/AdditionOperator.hpp @@ -14,39 +14,5 @@ using namespace armnn; using namespace tosa; TosaSerializationBasicBlock* ConvertAdditionToTosaOperator(const std::vector& inputs, - const std::vector& outputs) -{ - // A helper function with static global variables ensures uniqueness - // for dynamically generating input, output and block names - std::string input0Name = std::string("Op_ADD_input0_") + GetUniqueTosaMappingID(); - std::string input1Name = std::string("Op_ADD_input1_") + GetUniqueTosaMappingID(); - std::string outputName = std::string("Op_ADD_output0_") + GetUniqueTosaMappingID(); - std::string blockName = std::string("Op_ADD_block_") + GetUniqueTosaMappingID(); + const std::vector& outputs); - TosaSerializationOperator* op = new TosaSerializationOperator(Op_ADD, - Attribute_NONE, - nullptr, - {input0Name, input1Name}, - {outputName}); - - std::vector inputShape0 = GetTosaTensorShape(inputs[0]->GetShape()); - DType inputDType0 = ArmNNToDType(inputs[0]->GetDataType()); - - std::vector inputShape1 = GetTosaTensorShape(inputs[1]->GetShape()); - DType inputDType1 = ArmNNToDType(inputs[1]->GetDataType()); - - std::vector outputShape0 = GetTosaTensorShape(outputs[0]->GetShape()); - DType outputDType0 = ArmNNToDType(outputs[0]->GetDataType()); - - TosaSerializationTensor* inputTensor0 = new TosaSerializationTensor(input0Name, inputShape0, inputDType0, {}); - TosaSerializationTensor* inputTensor1 = new TosaSerializationTensor(input1Name, inputShape1, inputDType1, {}); - TosaSerializationTensor* outputTensor0 = new TosaSerializationTensor(outputName, outputShape0, outputDType0, {}); - - // operatorInputNames/operatorOutputNames ends up being the same as - // blockInputNames/blockOutputNames for one-to-one ArmNN to Tosa mappings - return new TosaSerializationBasicBlock(blockName, // name - {op}, // operators - {inputTensor0, inputTensor1, outputTensor0}, // tensors - {input0Name, input1Name}, // inputs - {outputName}); // outputs -} diff --git a/src/backends/tosaCommon/operatorMappings/CMakeLists.txt b/src/backends/tosaCommon/operatorMappings/CMakeLists.txt index 3965a6ab04..9fc33e9205 100644 --- a/src/backends/tosaCommon/operatorMappings/CMakeLists.txt +++ b/src/backends/tosaCommon/operatorMappings/CMakeLists.txt @@ -5,6 +5,7 @@ list(APPEND armnnTosaBackendOperators_sources AdditionOperator.hpp + AdditionOperator.cpp TosaOperatorUtils.hpp ) diff --git a/src/backends/tosaCommon/operatorMappings/TosaOperatorUtils.hpp b/src/backends/tosaCommon/operatorMappings/TosaOperatorUtils.hpp index e11f293b12..f580a53ebc 100644 --- a/src/backends/tosaCommon/operatorMappings/TosaOperatorUtils.hpp +++ b/src/backends/tosaCommon/operatorMappings/TosaOperatorUtils.hpp @@ -14,7 +14,7 @@ using namespace armnn; using namespace tosa; // Function to return Tosa datatype from input ArmNN datatype. -DType ArmNNToDType(const DataType& type) +inline DType ArmNNToDType(const DataType& type) { switch (type) { @@ -43,7 +43,7 @@ DType ArmNNToDType(const DataType& type) } // Function to return Tosa tensor shape from input ArmNN tensor shape. -std::vector GetTosaTensorShape(const TensorShape& shape) +inline std::vector GetTosaTensorShape(const TensorShape& shape) { std::vector returnShape; for (u_int32_t i = 0; i < shape.GetNumDimensions(); i++) @@ -55,7 +55,7 @@ std::vector GetTosaTensorShape(const TensorShape& shape) // Function to return unique int as a string to ensure uniqueness between all input, output and block names. static int uniqueTosaMappingID = 0; -std::string GetUniqueTosaMappingID() +inline std::string GetUniqueTosaMappingID() { return std::to_string(++uniqueTosaMappingID); } diff --git a/src/backends/tosaReference/TosaRefBackend.cpp b/src/backends/tosaReference/TosaRefBackend.cpp index 093802958b..688cf93b49 100644 --- a/src/backends/tosaReference/TosaRefBackend.cpp +++ b/src/backends/tosaReference/TosaRefBackend.cpp @@ -9,6 +9,7 @@ #include "TosaRefLayerSupport.hpp" #include "TosaRefTensorHandleFactory.hpp" +#include #include #include #include @@ -21,6 +22,13 @@ namespace armnn { +// Utility function to construct a valid Deleter for TosaSerializationHandler ptrs passed back to ArmNN +template +void DeleteAsType(const void* const blob) +{ + delete static_cast(blob); +} + const BackendId& TosaRefBackend::GetIdStatic() { static const BackendId s_Id{TosaRefBackendId()}; @@ -75,11 +83,44 @@ OptimizationViews TosaRefBackend::OptimizeSubgraphView(const SubgraphView& subgr const ModelOptions& modelOptions) const { OptimizationViews optimizationViews(modelOptions); - optimizationViews.AddUntouchedSubgraph(SubgraphView(subgraph)); - + auto handler = std::make_unique(); + + auto it = subgraph.endIConnectable(); + while (it != subgraph.beginIConnectable()) + { + --it; + Layer &base = *(PolymorphicDowncast(*it)); + + if(base.GetType() == armnn::LayerType::Input || + base.GetType() == armnn::LayerType::Output) + { + continue; + } + + tosa::TosaSerializationBasicBlock* mappings = GetTosaMappingFromLayer(&base); + handler.get()->GetBlocks().push_back(mappings); + } + + auto compiledBlob = + std::make_unique(handler.release(), DeleteAsType); + + IConnectableLayer* preCompiledLayer = optimizationViews.GetINetwork()->AddPrecompiledLayer( + PreCompiledDescriptor(subgraph.GetNumInputSlots(), subgraph.GetNumOutputSlots()), + std::move(*compiledBlob), + armnn::Optional(GetId()), + "TOSA_Pre_Compiled_Layer"); + + // Copy the output tensor infos from sub-graph + for (unsigned int i = 0; i < subgraph.GetNumOutputSlots(); i++) + { + preCompiledLayer->GetOutputSlot(i).SetTensorInfo(subgraph.GetIOutputSlot(i)->GetTensorInfo()); + } + + optimizationViews.AddSubstitution({ std::move(subgraph), SubgraphView(preCompiledLayer) }); return optimizationViews; } + std::vector TosaRefBackend::GetHandleFactoryPreferences() const { return std::vector { TosaRefTensorHandleFactory::GetIdStatic() }; diff --git a/src/backends/tosaReference/TosaRefLayerSupport.cpp b/src/backends/tosaReference/TosaRefLayerSupport.cpp index 18530bb535..f5f34a814b 100644 --- a/src/backends/tosaReference/TosaRefLayerSupport.cpp +++ b/src/backends/tosaReference/TosaRefLayerSupport.cpp @@ -95,25 +95,36 @@ bool TosaRefLayerSupport::IsLayerSupported(const LayerType& type, IgnoreUnused(lstmParamsInfo); IgnoreUnused(quantizedLstmInputParamsInfo); - // Setup Inputs - const auto input0 = infos[0]; - const TensorInfo* ptr0 = &input0; - const auto input1 = infos[1]; - const TensorInfo* ptr1 = &input1; - std::vector inputInfos = {ptr0, ptr1}; - - // Setup Outputs - const auto output = infos[2]; - const TensorInfo* ptr2 = &output; - std::vector outputInfos = {ptr2}; + std::vector inputInfos; + std::vector outputInfos; + + switch (type) + { + case LayerType::Addition: + // Setup inputs and outputs + inputInfos.push_back(&infos[0]); + inputInfos.push_back(&infos[1]); + outputInfos.push_back(&infos[2]); + break; + case LayerType::Input: + case LayerType::Output: + return true; + default: + break; + } auto mappings = GetTosaMapping(type, inputInfos, outputInfos, descriptor); + if (mappings->GetName() == "") + { + // There currently isn't a TOSA mapping for this layer, as the default was returned. + return false; + } // Loop through block and get each tensor and operator for (long unsigned int i = 0; i < mappings->GetOperators().size(); ++i) { // While looping over operators check for op_UNKNOWN which is unsupported - if (mappings->GetOperators()[i]->GetOp() == tosa::Op_UNKNOWN) { return false;} + if (mappings->GetOperators()[i]->GetOp() == tosa::Op_UNKNOWN) { return false; } // Loop over operators and get GetInput/OutputTensorNames, loop over resulting names and // use GetTensorByName to pass pointers to tensors on to the IsTosaLayerSupported() diff --git a/src/backends/tosaReference/test/CMakeLists.txt b/src/backends/tosaReference/test/CMakeLists.txt index 8366901605..627a1836e1 100644 --- a/src/backends/tosaReference/test/CMakeLists.txt +++ b/src/backends/tosaReference/test/CMakeLists.txt @@ -6,6 +6,7 @@ list(APPEND armnnTosaRefBackendUnitTests_sources TosaRefLayerTests.cpp TosaRefLayerSupportTests.cpp + TosaRefOptimizedNetworkTests.cpp ) add_library(armnnTosaRefBackendUnitTests OBJECT ${armnnTosaRefBackendUnitTests_sources}) diff --git a/src/backends/tosaReference/test/TosaRefOptimizedNetworkTests.cpp b/src/backends/tosaReference/test/TosaRefOptimizedNetworkTests.cpp new file mode 100644 index 0000000000..64b6805d2c --- /dev/null +++ b/src/backends/tosaReference/test/TosaRefOptimizedNetworkTests.cpp @@ -0,0 +1,54 @@ +// +// Copyright © 2022 Arm Ltd and Contributors. All rights reserved. +// SPDX-License-Identifier: MIT +// + +#include + +#include +#include + +#include + +TEST_SUITE("TosaReferenceOptimizedNetwork") +{ + +TEST_CASE("SimpleSupportedOptimizedNetwork") +{ + armnn::IRuntime::CreationOptions options; + armnn::IRuntimePtr runtime(armnn::IRuntime::Create(options)); + armnn::INetworkPtr network(armnn::INetwork::Create()); + + auto inputLayer1 = network->AddInputLayer(0, "input_1"); + auto inputLayer2 = network->AddInputLayer(1, "input_2"); + auto addLayer = network->AddAdditionLayer("add"); + auto outputLayer = network->AddOutputLayer(2, "output"); + + armnn::TensorInfo tensorInfo{{4}, armnn::DataType::Float32}; + + inputLayer1->GetOutputSlot(0).Connect(addLayer->GetInputSlot(0)); + inputLayer1->GetOutputSlot(0).SetTensorInfo(tensorInfo); + + inputLayer2->GetOutputSlot(0).Connect(addLayer->GetInputSlot(1)); + inputLayer2->GetOutputSlot(0).SetTensorInfo(tensorInfo); + + addLayer->GetOutputSlot(0).Connect(outputLayer->GetInputSlot(0)); + addLayer->GetOutputSlot(0).SetTensorInfo(tensorInfo); + + std::vector backends = { "TosaRef" }; + + armnn::OptimizerOptions optimizedOptions; + armnn::IOptimizedNetworkPtr optNet = Optimize(*network, backends, runtime->GetDeviceSpec(), optimizedOptions); + CHECK(optNet); + + armnn::Graph& graph = GetGraphForTesting(optNet.get()); + + // Check graph layer sequence to ensure that the network has been replaced with a PreCompiledLayer + CHECK(CheckSequence(graph.cbegin(), graph.cend(), + &IsLayerOfType, + &IsLayerOfType, + &IsLayerOfType, + &IsLayerOfType)); +} + +} -- cgit v1.2.1