From f4261adf78bdb9f8b2d6f2970636125096c173cb Mon Sep 17 00:00:00 2001 From: Georgios Pinitas Date: Mon, 2 Dec 2019 11:58:19 +0000 Subject: COMPMID-2779: Add support for generating synthetic int8 graphs. Adds SyntheticDataTypeMutator, which is responsible for mutating graphs to int8 and thus enable performance analysis on a wider range of models. Change-Id: I9a00f0ae59421ab11952660f5115b5dcd9314aaf Signed-off-by: Georgios Pinitas Reviewed-on: https://review.mlplatform.org/c/2418 Comments-Addressed: Arm Jenkins Tested-by: Arm Jenkins Reviewed-by: Michele Di Giorgio --- src/graph/GraphManager.cpp | 7 +- src/graph/PassManager.cpp | 18 +- src/graph/Utils.cpp | 6 +- src/graph/frontend/Stream.cpp | 4 +- src/graph/mutators/DepthConcatSubTensorMutator.cpp | 5 + src/graph/mutators/GroupedConvolutionMutator.cpp | 5 + src/graph/mutators/InPlaceOperationMutator.cpp | 5 + src/graph/mutators/NodeExecutionMethodMutator.cpp | 7 +- src/graph/mutators/NodeFusionMutator.cpp | 5 + src/graph/mutators/SplitLayerSubTensorMutator.cpp | 7 +- src/graph/mutators/SyntheticDataTypeMutator.cpp | 261 +++++++++++++++++++++ 11 files changed, 320 insertions(+), 10 deletions(-) create mode 100644 src/graph/mutators/SyntheticDataTypeMutator.cpp (limited to 'src/graph') diff --git a/src/graph/GraphManager.cpp b/src/graph/GraphManager.cpp index 4f942b99e4..996e50bbbc 100644 --- a/src/graph/GraphManager.cpp +++ b/src/graph/GraphManager.cpp @@ -51,6 +51,9 @@ void GraphManager::finalize_graph(Graph &graph, GraphContext &ctx, PassManager & ARM_COMPUTE_ERROR("Graph is already registered!"); } + // Apply IR mutating passes + pm.run_type(graph, IGraphMutator::MutationType::IR); + // Force target to all graph construct // TODO (COMPMID-2014) : Support heterogeneous execution Target forced_target = target; @@ -68,8 +71,8 @@ void GraphManager::finalize_graph(Graph &graph, GraphContext &ctx, PassManager & // Configure all tensors detail::configure_all_tensors(graph); - // Apply all mutating passes - pm.run_all(graph); + // Apply backend mutating passes + pm.run_type(graph, IGraphMutator::MutationType::Backend); // Perform topological sort std::vector topological_sorted_nodes = dfs(graph); diff --git a/src/graph/PassManager.cpp b/src/graph/PassManager.cpp index 92860e2987..99d979842a 100644 --- a/src/graph/PassManager.cpp +++ b/src/graph/PassManager.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 ARM Limited. + * Copyright (c) 2018-2019 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -70,7 +70,19 @@ void PassManager::run_all(Graph &g) } } -void PassManager::run(Graph &g, size_t index) +void PassManager::run_type(Graph &g, IGraphMutator::MutationType type) +{ + for(auto &pass : _passes) + { + if(pass && (pass->type() == type)) + { + ARM_COMPUTE_LOG_GRAPH_INFO("Running mutating pass : " << pass->name() << std::endl); + pass->mutate(g); + } + } +} + +void PassManager::run_index(Graph &g, size_t index) { if(index >= _passes.size()) { @@ -78,9 +90,9 @@ void PassManager::run(Graph &g, size_t index) } auto &pass = _passes.at(index); - if(pass != nullptr) { + ARM_COMPUTE_LOG_GRAPH_INFO("Running mutating pass : " << pass->name() << std::endl); pass->mutate(g); } } diff --git a/src/graph/Utils.cpp b/src/graph/Utils.cpp index 4c34dd85a5..3bf0caca7e 100644 --- a/src/graph/Utils.cpp +++ b/src/graph/Utils.cpp @@ -74,13 +74,17 @@ void force_target_to_graph(Graph &g, Target target) } } -PassManager create_default_pass_manager(Target target) +PassManager create_default_pass_manager(Target target, const GraphConfig &cfg) { PassManager pm; const bool is_target_gc = target == Target::GC; // Passes that mutate graph IR + if(cfg.convert_to_uint8) + { + pm.append(support::cpp14::make_unique(), !is_target_gc); + } pm.append(support::cpp14::make_unique(), !is_target_gc); pm.append(support::cpp14::make_unique()); pm.append(support::cpp14::make_unique(), !is_target_gc); diff --git a/src/graph/frontend/Stream.cpp b/src/graph/frontend/Stream.cpp index 878d688995..c04a426eef 100644 --- a/src/graph/frontend/Stream.cpp +++ b/src/graph/frontend/Stream.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 ARM Limited. + * Copyright (c) 2018-2019 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -39,7 +39,7 @@ Stream::Stream(size_t id, std::string name) void Stream::finalize(Target target, const GraphConfig &config) { - PassManager pm = create_default_pass_manager(target); + PassManager pm = create_default_pass_manager(target, config); _ctx.set_config(config); _manager.finalize_graph(_g, _ctx, pm, target); } diff --git a/src/graph/mutators/DepthConcatSubTensorMutator.cpp b/src/graph/mutators/DepthConcatSubTensorMutator.cpp index 7994541b78..30d6700446 100644 --- a/src/graph/mutators/DepthConcatSubTensorMutator.cpp +++ b/src/graph/mutators/DepthConcatSubTensorMutator.cpp @@ -42,6 +42,11 @@ const char *DepthConcatSubTensorMutator::name() return "DepthConcatSubTensorMutator"; } +IGraphMutator::MutationType DepthConcatSubTensorMutator::type() const +{ + return IGraphMutator::MutationType::Backend; +} + void DepthConcatSubTensorMutator::mutate(Graph &g) { // Early exit if no Concatenation layers exist in graph diff --git a/src/graph/mutators/GroupedConvolutionMutator.cpp b/src/graph/mutators/GroupedConvolutionMutator.cpp index 3d53f49218..bb452f9b94 100644 --- a/src/graph/mutators/GroupedConvolutionMutator.cpp +++ b/src/graph/mutators/GroupedConvolutionMutator.cpp @@ -103,6 +103,11 @@ const char *GroupedConvolutionMutator::name() return "GroupedConvolutionMutator"; } +IGraphMutator::MutationType GroupedConvolutionMutator::type() const +{ + return IGraphMutator::MutationType::Backend; +} + void GroupedConvolutionMutator::mutate(Graph &g) { // Early exit if no Convolution layers exist in graph diff --git a/src/graph/mutators/InPlaceOperationMutator.cpp b/src/graph/mutators/InPlaceOperationMutator.cpp index 07e3ecf2c7..ef4ca47e48 100644 --- a/src/graph/mutators/InPlaceOperationMutator.cpp +++ b/src/graph/mutators/InPlaceOperationMutator.cpp @@ -35,6 +35,11 @@ const char *InPlaceOperationMutator::name() return "InPlaceOperationMutator"; } +IGraphMutator::MutationType InPlaceOperationMutator::type() const +{ + return IGraphMutator::MutationType::Backend; +} + void InPlaceOperationMutator::mutate(Graph &g) { std::set in_place_nodes = { NodeType::BatchNormalizationLayer, NodeType::ActivationLayer }; diff --git a/src/graph/mutators/NodeExecutionMethodMutator.cpp b/src/graph/mutators/NodeExecutionMethodMutator.cpp index b420121c42..72e2645dd2 100644 --- a/src/graph/mutators/NodeExecutionMethodMutator.cpp +++ b/src/graph/mutators/NodeExecutionMethodMutator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 ARM Limited. + * Copyright (c) 2018-2019 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -73,6 +73,11 @@ const char *NodeExecutionMethodMutator::name() return "NodeExecutionMethodMutator"; } +IGraphMutator::MutationType NodeExecutionMethodMutator::type() const +{ + return IGraphMutator::MutationType::Backend; +} + void NodeExecutionMethodMutator::mutate(Graph &g) { // Convolution Layer diff --git a/src/graph/mutators/NodeFusionMutator.cpp b/src/graph/mutators/NodeFusionMutator.cpp index abd6436d74..b7f081dc42 100644 --- a/src/graph/mutators/NodeFusionMutator.cpp +++ b/src/graph/mutators/NodeFusionMutator.cpp @@ -286,6 +286,11 @@ const char *NodeFusionMutator::name() return "NodeFusionMutator"; } +IGraphMutator::MutationType NodeFusionMutator::type() const +{ + return IGraphMutator::MutationType::Backend; +} + void NodeFusionMutator::mutate(Graph &g) { // Supported activations when fusing diff --git a/src/graph/mutators/SplitLayerSubTensorMutator.cpp b/src/graph/mutators/SplitLayerSubTensorMutator.cpp index e21252a9ed..3ba73071ed 100644 --- a/src/graph/mutators/SplitLayerSubTensorMutator.cpp +++ b/src/graph/mutators/SplitLayerSubTensorMutator.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 ARM Limited. + * Copyright (c) 2018-2019 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -42,6 +42,11 @@ const char *SplitLayerSubTensorMutator::name() return "SplitLayerSubTensorMutator"; } +IGraphMutator::MutationType SplitLayerSubTensorMutator::type() const +{ + return IGraphMutator::MutationType::Backend; +} + void SplitLayerSubTensorMutator::mutate(Graph &g) { // Early exit if no Split layers exist in graph diff --git a/src/graph/mutators/SyntheticDataTypeMutator.cpp b/src/graph/mutators/SyntheticDataTypeMutator.cpp new file mode 100644 index 0000000000..b318df956e --- /dev/null +++ b/src/graph/mutators/SyntheticDataTypeMutator.cpp @@ -0,0 +1,261 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/graph/mutators/SyntheticDataTypeMutator.h" + +#include "arm_compute/graph/GraphBuilder.h" +#include "arm_compute/graph/ITensorAccessor.h" +#include "arm_compute/graph/Logger.h" +#include "arm_compute/graph/Utils.h" +#include "arm_compute/graph/nodes/Nodes.h" + +#include "arm_compute/core/utils/misc/Cast.h" + +#include + +namespace arm_compute +{ +namespace graph +{ +namespace +{ +/** Empty accessor class */ +class EmptyAccessor final : public graph::ITensorAccessor +{ +public: + /** Default Constructor */ + EmptyAccessor() = default; + + // Inherited methods overriden: + bool access_tensor(ITensor &tensor) override + { + ARM_COMPUTE_UNUSED(tensor); + return true; + } +}; + +/** Check if the mutation pass can be applied + * + * @param[in] g Graph the mutation pass need to be applied on + * + * @return True if the pass can be applied else false + */ +bool is_mutation_supported(Graph &g) +{ + const std::set unsupported_node_types = { NodeType::DetectionOutputLayer, + NodeType::NormalizationLayer, + NodeType::PriorBoxLayer + }; + + for(const auto &utype : unsupported_node_types) + { + if(!g.nodes(utype).empty()) + { + return false; + } + } + return true; +} + +/** Remove nodes that get optimized out during conversion + * + * @param[in, out] g Graph to remove the nodes from. + */ +void remove_optimized_nodes(Graph &g) +{ + const std::set optimized_node_types = { NodeType::BatchNormalizationLayer }; + + for(const auto &opt_type : optimized_node_types) + { + const std::vector opt_nodes_ids = g.nodes(opt_type); + for(const auto &node_id : opt_nodes_ids) + { + INode *node = g.node(node_id); + + // Get input edge + Edge *input_edge = node->input_edge(0); + ARM_COMPUTE_ERROR_ON(input_edge == nullptr); + + // Get producer node + INode *producer = input_edge->producer(); + const EdgeID producer_edge_id = input_edge->producer_idx(); + ARM_COMPUTE_ERROR_ON(producer == nullptr); + + // Get driving nodes + std::vector driving_nodes = get_driving_nodes(*node); + + // Remove node + g.remove_node(node->id()); + + // Update connections + for(auto &driving_node : driving_nodes) + { + g.add_connection(producer->id(), producer_edge_id, driving_node.node_id, driving_node.index); + } + } + } +} + +/** Convert tensor meta-data + * + * @param[in,out] g Graph to convert tensors of. + */ +void convert_tensors(Graph &g) +{ + auto &tensors = g.tensors(); + for(auto &tensor : tensors) + { + if(tensor != nullptr) + { + tensor->desc().data_type = DataType::QASYMM8; + tensor->desc().quant_info = QuantizationInfo(0.125f, -10); + } + } +} + +/** Convert special node + * + * @param[in,out] g Graph to convert tensors of. + * @param[in] fnc Conversion function. + * @param[in] optional_arguments Conversion function arguments. + */ +template +void convert_special_node(Graph &g, std::function const &f) +{ + const std::vector nodes_ids = g.nodes(NT::node_type); + for(const auto &nodes_id : nodes_ids) + { + INode *node = arm_compute::utils::cast::polymorphic_downcast(g.node(nodes_id)); + ARM_COMPUTE_ERROR_ON(node == nullptr); + + Tensor *output_tensor = node->output(0); + ARM_COMPUTE_ERROR_ON(output_tensor == nullptr); + + f(node, output_tensor); + } +} + +/** Converts special tensors + * + * @param[in,out] g Graph to convert tensors of. + */ +void convert_special_tensors(Graph &g) +{ + auto softmax_func = [](INode * node, Tensor * tensor) + { + ARM_COMPUTE_UNUSED(node); + tensor->desc().quant_info = QuantizationInfo(1.f / 256.f, 0); + return true; + }; + + auto act_func = [](INode * node, Tensor * tensor) + { + auto *act_node = arm_compute::utils::cast::polymorphic_downcast(node); + if(act_node->activation_info().activation() == ActivationLayerInfo::ActivationFunction::TANH) + { + tensor->desc().quant_info = QuantizationInfo(1.f / 128.f, 128); + } + return true; + }; + + convert_special_node(g, act_func); + convert_special_node(g, softmax_func); +} + +/** Handle nodes with bias + * + * @note Special tensors are for now biases that the data type differ + * + * @param[in,out] g Graph to convert tensors of. + */ +void handle_nodes_with_bias(Graph &g) +{ + const std::set special_node_types = { NodeType::ConvolutionLayer, + NodeType::DeconvolutionLayer, + NodeType::DepthwiseConvolutionLayer, + NodeType::FullyConnectedLayer + }; + + for(const auto &spc_type : special_node_types) + { + const std::vector scp_nodes_ids = g.nodes(spc_type); + for(const auto &node_id : scp_nodes_ids) + { + INode *node = g.node(node_id); + if(node != nullptr) + { + Tensor *tensor = node->input(2); + if(tensor != nullptr) + { + tensor->desc().data_type = DataType::S32; + } + else + { + auto params = node->common_node_params(); + params.name = params.name.empty() ? "" : params.name + "Bias"; + + TensorDescriptor b_desc = node->input(1)->desc(); + auto depth = b_desc.shape[get_dimension_idx(b_desc.layout, DataLayoutDimension::BATCHES)]; + b_desc.shape = TensorShape(depth); + + auto accessor = support::cpp14::make_unique(); + auto b_nid = GraphBuilder::add_const_node(g, params, b_desc, std::move(accessor)); + g.add_connection(b_nid, 0, node_id, 2); + } + } + } + } +} +} // namespace + +const char *SyntheticDataTypeMutator::name() +{ + return "SyntheticDataTypeMutator"; +} + +IGraphMutator::MutationType SyntheticDataTypeMutator::type() const +{ + return IGraphMutator::MutationType::IR; +} + +void SyntheticDataTypeMutator::mutate(Graph &g) +{ + if(is_mutation_supported(g)) + { + // Remove nodes that get optimized out (e.g. BatchNorm) + remove_optimized_nodes(g); + + // Convert tensor + convert_tensors(g); + convert_special_tensors(g); + + // Handle special nodes + handle_nodes_with_bias(g); + } + else + { + ARM_COMPUTE_LOG_GRAPH_VERBOSE("Synthetic data type mutator couldn't be applied" << std::endl); + } +} +} // namespace graph +} // namespace arm_compute -- cgit v1.2.1