From 15ecc9a03b1238524dcf094b48a77045c8cb2549 Mon Sep 17 00:00:00 2001 From: giuros01 Date: Thu, 6 Dec 2018 10:47:34 +0000 Subject: COMPMID-1741: Implement NEFuseBatchNormalizationKernel Change-Id: Ib3ba4b22804a94a1e8ef6d7076e28c2fc1cd2fa2 Reviewed-on: https://review.mlplatform.org/385 Tested-by: Arm Jenkins Reviewed-by: Georgios Pinitas Reviewed-by: Anthony Barbier --- arm_compute/core/NEON/NEKernels.h | 1 + .../NEON/kernels/NEFuseBatchNormalizationKernel.h | 110 ++++++++ arm_compute/runtime/NEON/NEFunctions.h | 1 + .../NEON/functions/NEFuseBatchNormalization.h | 94 +++++++ .../kernels/NEFuseBatchNormalizationKernel.cpp | 286 +++++++++++++++++++++ .../NEON/functions/NEFuseBatchNormalization.cpp | 59 +++++ tests/validation/NEON/BatchNormalizationLayer.cpp | 44 +++- 7 files changed, 587 insertions(+), 8 deletions(-) create mode 100644 arm_compute/core/NEON/kernels/NEFuseBatchNormalizationKernel.h create mode 100644 arm_compute/runtime/NEON/functions/NEFuseBatchNormalization.h create mode 100644 src/core/NEON/kernels/NEFuseBatchNormalizationKernel.cpp create mode 100644 src/runtime/NEON/functions/NEFuseBatchNormalization.cpp diff --git a/arm_compute/core/NEON/NEKernels.h b/arm_compute/core/NEON/NEKernels.h index 49bba3a750..755e68a2e1 100644 --- a/arm_compute/core/NEON/NEKernels.h +++ b/arm_compute/core/NEON/NEKernels.h @@ -64,6 +64,7 @@ #include "arm_compute/core/NEON/kernels/NEFillInnerBorderKernel.h" #include "arm_compute/core/NEON/kernels/NEFlattenLayerKernel.h" #include "arm_compute/core/NEON/kernels/NEFloorKernel.h" +#include "arm_compute/core/NEON/kernels/NEFuseBatchNormalizationKernel.h" #include "arm_compute/core/NEON/kernels/NEGEMMAssemblyBaseKernel.h" #include "arm_compute/core/NEON/kernels/NEGEMMInterleave4x4Kernel.h" #include "arm_compute/core/NEON/kernels/NEGEMMLowpMatrixMultiplyKernel.h" diff --git a/arm_compute/core/NEON/kernels/NEFuseBatchNormalizationKernel.h b/arm_compute/core/NEON/kernels/NEFuseBatchNormalizationKernel.h new file mode 100644 index 0000000000..4d5855e6f5 --- /dev/null +++ b/arm_compute/core/NEON/kernels/NEFuseBatchNormalizationKernel.h @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2018 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INNEUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY NEAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef __ARM_COMPUTE_NEFUSEBATCHNORMALIZATIONKERNEL_H__ +#define __ARM_COMPUTE_NEFUSEBATCHNORMALIZATIONKERNEL_H__ + +#include "arm_compute/core/NEON/INEKernel.h" + +namespace arm_compute +{ +// Forward declarations +class ITensor; + +/** OpenNE kernel to fuse the batch normalization node to a preceding convolution node */ +class NEFuseBatchNormalizationKernel : public INEKernel +{ +public: + const char *name() const override + { + return "NEFuseBatchNormalizationKernel"; + } + /** Default constructor */ + NEFuseBatchNormalizationKernel(); + /** Prevent instances of this class from being copied (As this class contains pointers) */ + NEFuseBatchNormalizationKernel(const NEFuseBatchNormalizationKernel &) = delete; + /** Prevent instances of this class from being copied (As this class contains pointers) */ + NEFuseBatchNormalizationKernel &operator=(const NEFuseBatchNormalizationKernel &) = delete; + /** Allow instances of this class to be moved */ + NEFuseBatchNormalizationKernel(NEFuseBatchNormalizationKernel &&) = default; + /** Allow instances of this class to be moved */ + NEFuseBatchNormalizationKernel &operator=(NEFuseBatchNormalizationKernel &&) = default; + /** Default destructor */ + ~NEFuseBatchNormalizationKernel() = default; + /** Set the source, destination of the kernel + * + * @param[in] conv_weights Convolution layer weights tensor. Data type supported: F16/F32 + * @param[in] bn_mean Batch normalization layer mean tensor. Same as @p conv_weights + * @param[in] bn_var Batch normalization layer variance tensor. Same as @p conv_weights + * @param[out] fused_weights Output fused weights tensor. Same as @p conv_weights + * @param[out] fused_bias Output fused bias tensor. Same as @p conv_weights + * @param[in] conv_bias (Optional) Convolution layer bias tensor. Same as @p conv_weights + * @param[in] bn_beta (Optional) Batch normalization layer beta tensor. Same as @p conv_weights + * @param[in] bn_gamma (Optional) Batch normalization layer gamma tensor. Same as @p conv_weights + * @param[in] epsilon (Optional) Batch normalization layer epsilon parameter. Defaults to 0.001f. + */ + void configure(const ITensor *conv_weights, const ITensor *bn_mean, const ITensor *bn_var, ITensor *fused_weights, ITensor *fused_bias, + const ITensor *conv_bias = nullptr, const ITensor *bn_beta = nullptr, const ITensor *bn_gamma = nullptr, + float epsilon = 0.001f); + /** Static function to check if given info will lead to a valid configuration of @ref NEFuseBatchNormalizationKernel + * + * @param[in] conv_weights Convolution layer weights tensor. Data type supported: F16/F32 + * @param[in] bn_mean Batch normalization layer mean tensor. Same as @p conv_weights + * @param[in] bn_var Batch normalization layer variance tensor. Same as @p conv_weights + * @param[in] fused_weights Output fused weights tensor. Same as @p conv_weights + * @param[in] fused_bias Output fused bias tensor. Same as @p conv_weights + * @param[in] conv_bias (Optional) Convolution layer bias tensor. Same as @p conv_weights + * @param[in] bn_beta (Optional) Batch normalization layer beta tensor. Same as @p conv_weights + * @param[in] bn_gamma (Optional) Batch normalization layer gamma tensor. Same as @p conv_weights + * @param[in] epsilon (Optional) Batch normalization layer epsilon parameter. Defaults to 0.001f. + * + * @return a status + */ + static Status validate(const ITensorInfo *conv_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, + const ITensorInfo *fused_weights, const ITensorInfo *fused_bias, + const ITensorInfo *conv_bias = nullptr, const ITensorInfo *bn_beta = nullptr, const ITensorInfo *bn_gamma = nullptr, + float epsilon = 0.001f); + + // Inherited methods overridden: + void run(const Window &window, const ThreadInfo &info) override; + +private: + const ITensor *_conv_weights; + const ITensor *_conv_bias; + const ITensor *_bn_mean; + const ITensor *_bn_var; + const ITensor *_bn_gamma; + const ITensor *_bn_beta; + ITensor *_fused_weights; + ITensor *_fused_bias; + float _epsilon; + bool _run_in_place_weights; + bool _run_in_place_bias; + + using FuseBatchNormFunction = void(const ITensor *conv_weights, const ITensor *conv_bias, ITensor *fused_weights, ITensor *fused_bias, + const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window); + + FuseBatchNormFunction *_func; +}; +} // namespace arm_compute +#endif /*__ARM_COMPUTE_NEFUSEBATCHNORMALIZATIONKERNEL_H__ */ diff --git a/arm_compute/runtime/NEON/NEFunctions.h b/arm_compute/runtime/NEON/NEFunctions.h index 8873110982..9a023ec51f 100644 --- a/arm_compute/runtime/NEON/NEFunctions.h +++ b/arm_compute/runtime/NEON/NEFunctions.h @@ -63,6 +63,7 @@ #include "arm_compute/runtime/NEON/functions/NEFlattenLayer.h" #include "arm_compute/runtime/NEON/functions/NEFloor.h" #include "arm_compute/runtime/NEON/functions/NEFullyConnectedLayer.h" +#include "arm_compute/runtime/NEON/functions/NEFuseBatchNormalization.h" #include "arm_compute/runtime/NEON/functions/NEGEMM.h" #include "arm_compute/runtime/NEON/functions/NEGEMMAssemblyDispatch.h" #include "arm_compute/runtime/NEON/functions/NEGEMMConvolutionLayer.h" diff --git a/arm_compute/runtime/NEON/functions/NEFuseBatchNormalization.h b/arm_compute/runtime/NEON/functions/NEFuseBatchNormalization.h new file mode 100644 index 0000000000..82ed329d86 --- /dev/null +++ b/arm_compute/runtime/NEON/functions/NEFuseBatchNormalization.h @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2018 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INNEUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY NEAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef __ARM_COMPUTE_NEFUSEBATCHNORMALIZATION_H__ +#define __ARM_COMPUTE_NEFUSEBATCHNORMALIZATION_H__ + +#include "arm_compute/core/ITensor.h" +#include "arm_compute/core/NEON/kernels/NEFuseBatchNormalizationKernel.h" +#include "arm_compute/core/Types.h" +#include "arm_compute/runtime/IFunction.h" + +namespace arm_compute +{ +// Forward declarations +class ITensor; + +/** Basic function to fuse the batch normalization node to a preceding convolution node */ +class NEFuseBatchNormalization : public IFunction +{ +public: + /** Default constructor */ + NEFuseBatchNormalization(); + /** Prevent instances of this class from being copied (As this class contains pointers) */ + NEFuseBatchNormalization(const NEFuseBatchNormalization &) = delete; + /** Prevent instances of this class from being copied (As this class contains pointers) */ + NEFuseBatchNormalization &operator=(const NEFuseBatchNormalization &) = delete; + /** Allow instances of this class to be moved */ + NEFuseBatchNormalization(NEFuseBatchNormalization &&) = default; + /** Allow instances of this class to be moved */ + NEFuseBatchNormalization &operator=(NEFuseBatchNormalization &&) = default; + /** Default destructor */ + ~NEFuseBatchNormalization() = default; + /** Set the input and output tensors. + * + * @param[in] conv_weights Convolution layer weights tensor. Data type supported: F16/F32 + * @param[in] bn_mean Batch normalization layer mean tensor. Same as @p conv_weights + * @param[in] bn_var Batch normalization layer variance tensor. Same as @p conv_weights + * @param[out] fused_weights Output fused weights tensor. Same as @p conv_weights + * @param[out] fused_bias Output fused bias tensor. Same as @p conv_weights + * @param[in] conv_bias (Optional) Convolution layer bias tensor. Same as @p conv_weights + * @param[in] bn_beta (Optional) Batch normalization layer beta tensor. Same as @p conv_weights + * @param[in] bn_gamma (Optional) Batch normalization layer gamma tensor. Same as @p conv_weights + * @param[in] epsilon (Optional) Batch normalization layer epsilon parameter. Defaults to 0.001f. + */ + void configure(const ITensor *conv_weights, const ITensor *bn_mean, const ITensor *bn_var, ITensor *fused_weights, ITensor *fused_bias, + const ITensor *conv_bias = nullptr, const ITensor *bn_beta = nullptr, const ITensor *bn_gamma = nullptr, + float epsilon = 0.001f); + /** Static function to check if given info will lead to a valid configuration of @ref NEFuseBatchNormalization + * + * @param[in] conv_weights Convolution layer weights tensor. Data type supported: F16/F32 + * @param[in] bn_mean Batch normalization layer mean tensor. Same as @p conv_weights + * @param[in] bn_var Batch normalization layer variance tensor. Same as @p conv_weights + * @param[in] fused_weights Output fused weights tensor. Same as @p conv_weights + * @param[in] fused_bias Output fused bias tensor. Same as @p conv_weights + * @param[in] conv_bias (Optional) Convolution layer bias tensor. Same as @p conv_weights + * @param[in] bn_beta (Optional) Batch normalization layer beta tensor. Same as @p conv_weights + * @param[in] bn_gamma (Optional) Batch normalization layer gamma tensor. Same as @p conv_weights + * @param[in] epsilon (Optional) Batch normalization layer epsilon parameter. Defaults to 0.001f. + * + * @return a status + */ + static Status validate(const ITensorInfo *conv_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, + const ITensorInfo *fused_weights, const ITensorInfo *fused_bias, + const ITensorInfo *conv_bias = nullptr, const ITensorInfo *bn_beta = nullptr, const ITensorInfo *bn_gamma = nullptr, + float epsilon = 0.001f); + + // Inherited methods overridden: + void run() override; + +private: + NEFuseBatchNormalizationKernel _fuse_bn_kernel; +}; +} // namespace arm_compute +#endif /*__ARM_COMPUTE_NEFUSEBATCHNORMALIZATION_H__ */ diff --git a/src/core/NEON/kernels/NEFuseBatchNormalizationKernel.cpp b/src/core/NEON/kernels/NEFuseBatchNormalizationKernel.cpp new file mode 100644 index 0000000000..25a0848977 --- /dev/null +++ b/src/core/NEON/kernels/NEFuseBatchNormalizationKernel.cpp @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2018 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INNEUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY NEAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/core/NEON/kernels/NEFuseBatchNormalizationKernel.h" + +#include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/Utils.h" +#include "arm_compute/core/Validate.h" +#include "arm_compute/core/Window.h" + +#include "arm_compute/core/Helpers.h" +#include "arm_compute/core/ITensor.h" +#include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/Utils.h" +#include "arm_compute/core/Window.h" + +#include "support/ToolchainSupport.h" + +#include "arm_compute/core/NEON/wrapper/wrapper.h" +#include "utils/TypePrinter.h" +namespace arm_compute +{ +namespace +{ +Status validate_arguments(const ITensorInfo *conv_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, + const ITensorInfo *fused_weights, const ITensorInfo *fused_bias, + const ITensorInfo *conv_bias, const ITensorInfo *bn_beta, const ITensorInfo *bn_gamma, + float epsilon) +{ + ARM_COMPUTE_UNUSED(epsilon); + //ARM_COMPUTE_RETURN_ERROR_ON_F16_UNSUPPORTED(conv_weights); + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(conv_weights, 1, DataType::F16, DataType::F32); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(bn_mean, bn_var); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(conv_weights, bn_mean, bn_var); + + unsigned int kernels_idx = get_data_layout_dimension_index(conv_weights->data_layout(), DataLayoutDimension::BATCHES); + ARM_COMPUTE_RETURN_ERROR_ON(conv_weights->dimension(kernels_idx) != bn_mean->dimension(0)); + + // Validate bias + if(conv_bias != nullptr) + { + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(bn_mean, conv_bias); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(conv_weights, conv_bias); + } + // Validate beta + if(bn_beta != nullptr) + { + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(bn_mean, bn_beta); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(conv_weights, bn_beta); + } + // Validate gamma + if(bn_gamma != nullptr) + { + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(bn_mean, bn_gamma); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(conv_weights, bn_gamma); + } + + // Validate output weights + if(fused_weights != nullptr && fused_weights->total_size() != 0) + { + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(conv_weights, fused_weights); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_LAYOUT(conv_weights, fused_weights); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(conv_weights, fused_weights); + } + // Validate output bias + if(fused_bias != nullptr && fused_bias->total_size() != 0) + { + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(bn_mean, fused_bias); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(conv_weights, fused_bias); + } + + return Status{}; +} + +template +void fused_batch_normmalization(const ITensor *conv_weights, const ITensor *conv_bias, ITensor *fused_weights, ITensor *fused_bias, + const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window) +{ + using ExactTagType = typename wrapper::traits::neon_vector::tag_type; + + const bool run_in_place_weights = (fused_weights == nullptr) || (fused_weights == conv_weights); + const bool run_in_place_bias = (fused_bias == nullptr) || (conv_bias != nullptr && fused_bias == conv_bias); + + // Set build options + Window win = window; + win.set(Window::DimX, Window::Dimension(0, 1, 1)); + + const int window_step_x = size; + const auto window_start_x = static_cast(window.x().start()); + const auto window_end_x = static_cast(window.x().end()); + + Iterator conv_w_in(conv_weights, win); + Iterator conv_w_out(run_in_place_weights ? conv_weights : fused_weights, win); + + const auto conv_bias_in = (conv_bias != nullptr ? reinterpret_cast(conv_bias->ptr_to_element(Coordinates(0, 0))) : nullptr); + auto conv_bias_out = (run_in_place_bias ? conv_bias_in : reinterpret_cast(fused_bias->ptr_to_element(Coordinates(0, 0)))); + + int slice = -1; + + const auto input_mean = reinterpret_cast(bn_mean->ptr_to_element(Coordinates(0, 0))); + const auto input_var = reinterpret_cast(bn_var->ptr_to_element(Coordinates(0, 0))); + const auto input_gamma = (bn_gamma != nullptr) ? reinterpret_cast(bn_gamma->ptr_to_element(Coordinates(0, 0))) : nullptr; + const auto input_beta = (bn_beta != nullptr) ? reinterpret_cast(bn_beta->ptr_to_element(Coordinates(0, 0))) : nullptr; + + auto mean_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{}); + auto var_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{}); + auto gamma_vec = wrapper::vdup_n(ScalarType(1), ExactTagType{}); + auto beta_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{}); + auto rvar_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{}); + const auto epsilon_vec = wrapper::vdup_n(ScalarType(epsilon), ExactTagType{}); + + auto mean = ScalarType(0.0); + auto var = ScalarType(0.0); + auto gamma = ScalarType(1.0); + auto beta = ScalarType(0.0); + auto conv_bias_in_scalar = ScalarType(0.0); + execute_window_loop(win, [&](const Coordinates & id) + { + if(slice != id[3]) + { + slice = id[3]; + mean = input_mean[slice]; + var = input_var[slice]; + gamma = ScalarType(1.0); + beta = ScalarType(0.0); + + // Construct vectors + mean_vec = wrapper::vdup_n(mean, ExactTagType{}); + var_vec = wrapper::vdup_n(var, ExactTagType{}); + if(input_gamma != nullptr) + { + gamma = input_gamma[slice]; + gamma_vec = wrapper::vdup_n(gamma, ExactTagType{}); + } + if(input_beta != nullptr) + { + beta = input_beta[slice]; + beta_vec = wrapper::vdup_n(beta, ExactTagType{}); + } + if(conv_bias_in != nullptr) + { + conv_bias_in_scalar = conv_bias_in[slice]; + } + else + { + conv_bias_in_scalar = ScalarType(0); + } + + conv_bias_in_scalar = (conv_bias_in_scalar - mean) / sqrt(var + ScalarType(epsilon)); + conv_bias_in_scalar = (conv_bias_in_scalar * gamma) + beta; + conv_bias_out[slice] = conv_bias_in_scalar; + rvar_vec = wrapper::vinvsqrt(wrapper::vadd(var_vec, epsilon_vec)); + } + + int x = window_start_x; + auto conv_w_in_ptr = reinterpret_cast(conv_w_in.ptr()); + auto conv_w_out_ptr = reinterpret_cast(conv_w_out.ptr()); + + for(; x <= (window_end_x - window_step_x); x += window_step_x) + { + auto wn = wrapper::vloadq(conv_w_in_ptr + x); + wn = wrapper::vmul(wn, rvar_vec); + wn = wrapper::vmul(wn, gamma_vec); + + // Store results + wrapper::vstore(conv_w_out_ptr + x, wn); + } + + // Compute left-over elements + for(; x < window_end_x; ++x) + { + *(conv_w_out_ptr + x) = *(conv_w_in_ptr + x) / sqrt(var + ScalarType(epsilon)) * gamma; + } + }, + conv_w_in, conv_w_out); +} +} // namespace + +NEFuseBatchNormalizationKernel::NEFuseBatchNormalizationKernel() + : _conv_weights(nullptr), _conv_bias(nullptr), _bn_mean(nullptr), _bn_var(nullptr), _bn_gamma(nullptr), _bn_beta(nullptr), _fused_weights(nullptr), _fused_bias(nullptr), _epsilon(), + _run_in_place_weights(false), _run_in_place_bias(false), _func(nullptr) +{ +} + +void NEFuseBatchNormalizationKernel::configure(const ITensor *conv_weights, const ITensor *bn_mean, const ITensor *bn_var, + ITensor *fused_weights, ITensor *fused_bias, + const ITensor *conv_bias, const ITensor *bn_beta, const ITensor *bn_gamma, + float epsilon) +{ + ARM_COMPUTE_ERROR_ON_NULLPTR(conv_weights, bn_mean, bn_var); + + _conv_weights = conv_weights; + _conv_bias = conv_bias; + _bn_mean = bn_mean; + _bn_var = bn_var; + _bn_beta = bn_beta; + _bn_gamma = bn_gamma; + _fused_weights = fused_weights; + _fused_bias = fused_bias; + _epsilon = epsilon; + + _run_in_place_weights = (fused_weights == nullptr) || (fused_weights == conv_weights); + _run_in_place_bias = (fused_bias == nullptr) || (conv_bias != nullptr && fused_bias == conv_bias); + + // Auto initialize outputs + if(_fused_weights != nullptr) + { + // Output tensor auto initialization if not yet initialized + auto_init_if_empty(*_fused_weights->info(), *_conv_weights->info()->clone()); + fused_weights->info()->set_valid_region(conv_weights->info()->valid_region()); + } + if(_fused_bias != nullptr) + { + // Output tensor auto initialization if not yet initialized + auto_init_if_empty(*_fused_bias->info(), *_bn_mean->info()->clone()); + _fused_bias->info()->set_valid_region(bn_mean->info()->valid_region()); + } + + // Validate arguments + ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(conv_weights->info(), bn_mean->info(), bn_var->info(), + (fused_weights != nullptr) ? fused_weights->info() : nullptr, + (fused_bias != nullptr) ? fused_bias->info() : nullptr, + (conv_bias != nullptr) ? conv_bias->info() : nullptr, + (bn_beta != nullptr) ? bn_beta->info() : nullptr, + (bn_gamma != nullptr) ? bn_gamma->info() : nullptr, + epsilon)); + + // Configure kernel window + Window win = calculate_max_window(*conv_weights->info()); + INEKernel::configure(win); + + // Configure function to run based on different data types + const DataType data_type = _conv_weights->info()->data_type(); + switch(data_type) + { + case DataType::F32: + _func = &fused_batch_normmalization; + break; +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + case DataType::F16: + _func = &fused_batch_normmalization; + break; +#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + default: + ARM_COMPUTE_ERROR("Not Supported"); + break; + } +} + +Status NEFuseBatchNormalizationKernel::validate(const ITensorInfo *conv_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, + const ITensorInfo *fused_weights, const ITensorInfo *fused_bias, + const ITensorInfo *conv_bias, const ITensorInfo *bn_beta, const ITensorInfo *bn_gamma, + float epsilon) +{ + ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(conv_weights, bn_mean, bn_var, fused_weights, fused_bias, conv_bias, bn_beta, bn_gamma, epsilon)); + return Status{}; +} + +void NEFuseBatchNormalizationKernel::run(const Window &window, const ThreadInfo &info) +{ + ARM_COMPUTE_UNUSED(info); + ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); + ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(IKernel::window(), window); + (*_func)(_conv_weights, _conv_bias, _fused_weights, _fused_bias, _bn_mean, _bn_var, _bn_beta, _bn_gamma, _epsilon, window); +} +} // namespace arm_compute diff --git a/src/runtime/NEON/functions/NEFuseBatchNormalization.cpp b/src/runtime/NEON/functions/NEFuseBatchNormalization.cpp new file mode 100644 index 0000000000..7f17a15cba --- /dev/null +++ b/src/runtime/NEON/functions/NEFuseBatchNormalization.cpp @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2018 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INNEUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY NEAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "arm_compute/runtime/NEON/functions/NEFuseBatchNormalization.h" + +#include "arm_compute/core/Error.h" +#include "arm_compute/core/TensorInfo.h" +#include "arm_compute/core/Types.h" +#include "arm_compute/runtime/NEON/NEScheduler.h" + +namespace arm_compute +{ +NEFuseBatchNormalization::NEFuseBatchNormalization() + : _fuse_bn_kernel() +{ +} + +void NEFuseBatchNormalization::configure(const ITensor *conv_weights, const ITensor *bn_mean, const ITensor *bn_var, + ITensor *fused_weights, ITensor *fused_bias, + const ITensor *conv_bias, const ITensor *bn_beta, const ITensor *bn_gamma, + float epsilon) +{ + _fuse_bn_kernel.configure(conv_weights, bn_mean, bn_var, fused_weights, fused_bias, conv_bias, bn_beta, bn_gamma, epsilon); +} + +Status NEFuseBatchNormalization::validate(const ITensorInfo *conv_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, + const ITensorInfo *fused_weights, const ITensorInfo *fused_bias, + const ITensorInfo *conv_bias, const ITensorInfo *bn_beta, const ITensorInfo *bn_gamma, + float epsilon) +{ + return NEFuseBatchNormalizationKernel::validate(conv_weights, bn_mean, bn_var, fused_weights, fused_bias, conv_bias, bn_beta, bn_gamma, epsilon); +} + +void NEFuseBatchNormalization::run() +{ + NEScheduler::get().schedule(&_fuse_bn_kernel, Window::DimY); +} +} // namespace arm_compute diff --git a/tests/validation/NEON/BatchNormalizationLayer.cpp b/tests/validation/NEON/BatchNormalizationLayer.cpp index ca13d26495..155f6c336e 100644 --- a/tests/validation/NEON/BatchNormalizationLayer.cpp +++ b/tests/validation/NEON/BatchNormalizationLayer.cpp @@ -23,18 +23,22 @@ */ #include "arm_compute/core/Types.h" #include "arm_compute/runtime/NEON/functions/NEBatchNormalizationLayer.h" +#include "arm_compute/runtime/NEON/functions/NEConvolutionLayer.h" +#include "arm_compute/runtime/NEON/functions/NEFuseBatchNormalization.h" #include "arm_compute/runtime/Tensor.h" #include "arm_compute/runtime/TensorAllocator.h" #include "tests/NEON/Accessor.h" #include "tests/PaddingCalculator.h" #include "tests/datasets/RandomBatchNormalizationLayerDataset.h" #include "tests/datasets/ShapeDatasets.h" +#include "tests/datasets/SmallConvolutionLayerDataset.h" #include "tests/framework/Asserts.h" #include "tests/framework/Macros.h" #include "tests/framework/datasets/Datasets.h" #include "tests/validation/Helpers.h" #include "tests/validation/Validation.h" #include "tests/validation/fixtures/BatchNormalizationLayerFixture.h" +#include "tests/validation/fixtures/BatchNormalizationLayerFusionFixture.h" namespace arm_compute { @@ -44,16 +48,21 @@ namespace validation { namespace { -constexpr AbsoluteTolerance tolerance_f32(0.00001f); /**< Tolerance value for comparing reference's output against implementation's output for DataType::F32 */ +RelativeTolerance rel_tolerance_f32(0.05f); /**< Tolerance value for comparing reference's output against implementation's output for DataType::F32 */ +constexpr AbsoluteTolerance abs_tolerance_f32(0.0001f); /**< Tolerance value for comparing reference's output against implementation's output for DataType::F32 */ #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC constexpr AbsoluteTolerance tolerance_f16(0.01f); /**< Tolerance value for comparing reference's output against implementation's output for DataType::F16 */ -#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ +#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC const auto act_infos = framework::dataset::make("ActivationInfo", { ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::RELU), ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::BOUNDED_RELU, 6.f), ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU, 8.f, 2.f), }); +const auto common_fusion_dataset = combine(combine(combine(framework::dataset::make("UseBias", { false, true }), + framework::dataset::make("UseBeta", { false, true })), + framework::dataset::make("UseGamma", { false, true })), + framework::dataset::make("Epsilon", { 0.001f })); } // namespace TEST_SUITE(NEON) @@ -150,7 +159,7 @@ FIXTURE_DATA_TEST_CASE(Random, NEBatchNormalizationLayerFixture, framewor framework::dataset::make("DataLayout", { DataLayout::NCHW, DataLayout::NHWC }))) { // Validate output - validate(Accessor(_target), _reference, tolerance_f32, 0); + validate(Accessor(_target), _reference, abs_tolerance_f32, 0); } TEST_SUITE_END() @@ -166,12 +175,31 @@ FIXTURE_DATA_TEST_CASE(Random, NEBatchNormalizationLayerFixture, framework // Validate output validate(Accessor(_target), _reference, tolerance_f16, 0); } -TEST_SUITE_END() -#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ -TEST_SUITE_END() +TEST_SUITE_END() // FP16 +#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ +TEST_SUITE_END() // Float -TEST_SUITE_END() -TEST_SUITE_END() +TEST_SUITE_END() // BatchNormalizationLayer + +TEST_SUITE(BatchNormalizationLayerFusion) +template +using NEBatchNormalizationLayerFusionFixture = BatchNormalizationLayerFusionValidationFixture; + +TEST_SUITE(Float) +TEST_SUITE(FP32) +FIXTURE_DATA_TEST_CASE(RunSmall, NEBatchNormalizationLayerFusionFixture, framework::DatasetMode::PRECOMMIT, + combine(combine(combine(datasets::SmallConvolutionLayerDataset(), common_fusion_dataset), + framework::dataset::make("DataType", DataType::F32)), + framework::dataset::make("DataLayout", { DataLayout::NCHW, DataLayout::NHWC }))) +{ + // Validate output + validate(Accessor(_target), _reference, rel_tolerance_f32, 0.f, abs_tolerance_f32); +} +TEST_SUITE_END() // FP32 +TEST_SUITE_END() // Float + +TEST_SUITE_END() // BatchNormalizationLayerFusion +TEST_SUITE_END() // NEON } // namespace validation } // namespace test } // namespace arm_compute -- cgit v1.2.1