From 11091762b6cbfa26d2135677d77b0bc7127ae980 Mon Sep 17 00:00:00 2001 From: Manuel Bottini Date: Mon, 17 Jun 2019 12:04:40 +0100 Subject: COMPMID-2245: Extend NEFuseBatchNormalization to support DepthwiseConvolution weights Change-Id: I2ee4aebfd69865290ed6c78dd17ff1299353317e Signed-off-by: Manuel Bottini Reviewed-on: https://review.mlplatform.org/c/1371 Comments-Addressed: Arm Jenkins Reviewed-by: Giuseppe Rossini Tested-by: Arm Jenkins Reviewed-by: Gian Marco Iodice --- .../NEON/kernels/NEFuseBatchNormalizationKernel.h | 56 +-- .../NEON/functions/NEFuseBatchNormalization.h | 50 +-- .../kernels/NEFuseBatchNormalizationKernel.cpp | 401 ++++++++++++++++----- .../NEON/functions/NEFuseBatchNormalization.cpp | 16 +- tests/validation/NEON/FuseBatchNormalization.cpp | 220 +++++++++++ 5 files changed, 603 insertions(+), 140 deletions(-) create mode 100644 tests/validation/NEON/FuseBatchNormalization.cpp diff --git a/arm_compute/core/NEON/kernels/NEFuseBatchNormalizationKernel.h b/arm_compute/core/NEON/kernels/NEFuseBatchNormalizationKernel.h index 97d6a9f55b..947f8e49c4 100644 --- a/arm_compute/core/NEON/kernels/NEFuseBatchNormalizationKernel.h +++ b/arm_compute/core/NEON/kernels/NEFuseBatchNormalizationKernel.h @@ -53,44 +53,50 @@ public: ~NEFuseBatchNormalizationKernel() = default; /** Set the source, destination of the kernel * - * @param[in] conv_weights Convolution layer weights tensor. Data type supported: F16/F32 - * @param[in] bn_mean Batch normalization layer mean tensor. Same as @p conv_weights - * @param[in] bn_var Batch normalization layer variance tensor. Same as @p conv_weights - * @param[out] fused_weights Output fused weights tensor. Same as @p conv_weights - * @param[out] fused_bias Output fused bias tensor. Same as @p conv_weights - * @param[in] conv_bias (Optional) Convolution layer bias tensor. Same as @p conv_weights - * @param[in] bn_beta (Optional) Batch normalization layer beta tensor. Same as @p conv_weights - * @param[in] bn_gamma (Optional) Batch normalization layer gamma tensor. Same as @p conv_weights + * @param[in] input_weights Input weights tensor for convolution or depthwise convolution layer. Data type supported: F16/F32. Data layout supported: NCHW, NHWC + * @param[in] bn_mean Batch normalization layer mean tensor. Same as @p input_weights + * @param[in] bn_var Batch normalization layer variance tensor. Same as @p input_weights + * @param[out] fused_weights (Optional) Output fused weights tensor. It can be a nullptr in case of in-place computation. Same as @p input_weights + * @param[out] fused_bias (Optional) Output fused bias tensor. It can be a nullptr in case of in-place computation and input_bias != nullptr. Same as @p input_weights + * @param[in] input_bias (Optional) Input bias tensor for convolution or depthwise convolution layer. It can be a nullptr in case the bias tensor is not required. Same as @p input_weights + * @param[in] bn_beta (Optional) Batch normalization layer beta tensor. It can be a nullptr in case the beta tensor is not required. Same as @p input_weights + * @note if nullptr, bn_beta is set to 0.0 + * @param[in] bn_gamma (Optional) Batch normalization layer gamma tensor. It can be a nullptr in case the gamma tensor is not required. Same as @p input_weights + * @note if nullptr, bn_gamma is set to 1.0 * @param[in] epsilon (Optional) Batch normalization layer epsilon parameter. Defaults to 0.001f. + * @param[in] fbn_type (Optional) Fused batch normalization type. Defaults to CONVOLUTION. */ - void configure(const ITensor *conv_weights, const ITensor *bn_mean, const ITensor *bn_var, ITensor *fused_weights, ITensor *fused_bias, - const ITensor *conv_bias = nullptr, const ITensor *bn_beta = nullptr, const ITensor *bn_gamma = nullptr, - float epsilon = 0.001f); + void configure(const ITensor *input_weights, const ITensor *bn_mean, const ITensor *bn_var, ITensor *fused_weights, ITensor *fused_bias, + const ITensor *input_bias = nullptr, const ITensor *bn_beta = nullptr, const ITensor *bn_gamma = nullptr, + float epsilon = 0.001f, FuseBatchNormalizationType fbn_type = FuseBatchNormalizationType::CONVOLUTION); /** Static function to check if given info will lead to a valid configuration of @ref NEFuseBatchNormalizationKernel * - * @param[in] conv_weights Convolution layer weights tensor. Data type supported: F16/F32 - * @param[in] bn_mean Batch normalization layer mean tensor. Same as @p conv_weights - * @param[in] bn_var Batch normalization layer variance tensor. Same as @p conv_weights - * @param[in] fused_weights Output fused weights tensor. Same as @p conv_weights - * @param[in] fused_bias Output fused bias tensor. Same as @p conv_weights - * @param[in] conv_bias (Optional) Convolution layer bias tensor. Same as @p conv_weights - * @param[in] bn_beta (Optional) Batch normalization layer beta tensor. Same as @p conv_weights - * @param[in] bn_gamma (Optional) Batch normalization layer gamma tensor. Same as @p conv_weights + * @param[in] input_weights Input weights tensor info for convolution or depthwise convolution layer. Data type supported: F16/F32. Data layout supported: NCHW, NHWC + * @param[in] bn_mean Batch normalization layer mean tensor info. Same as @p input_weights + * @param[in] bn_var Batch normalization layer variance tensor info. Same as @p input_weights + * @param[in] fused_weights (Optional) Output fused weights tensor info. It can be a nullptr in case of in-place computation. Same as @p input_weights + * @param[in] fused_bias (Optional) Output fused bias tensor info. It can be a nullptr in case of in-place computation and input_bias != nullptr. Same as @p input_weights + * @param[in] input_bias (Optional) Input bias tensor info for convolution or depthwise convolution layer. It can be a nullptr in case the bias tensor is not required. Same as @p input_weights + * @param[in] bn_beta (Optional) Batch normalization layer beta tensor info. It can be a nullptr in case the beta tensor is not required. Same as @p input_weights + * @note if nullptr, bn_beta is set to 0.0 + * @param[in] bn_gamma (Optional) Batch normalization layer gamma tensor info. It can be a nullptr in case the gamma tensor is not required. Same as @p input_weights + * @note if nullptr, bn_gamma is set to 1.0 * @param[in] epsilon (Optional) Batch normalization layer epsilon parameter. Defaults to 0.001f. + * @param[in] fbn_type (Optional) Fused batch normalization type. Defaults to CONVOLUTION. * * @return a status */ - static Status validate(const ITensorInfo *conv_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, + static Status validate(const ITensorInfo *input_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, const ITensorInfo *fused_weights, const ITensorInfo *fused_bias, - const ITensorInfo *conv_bias = nullptr, const ITensorInfo *bn_beta = nullptr, const ITensorInfo *bn_gamma = nullptr, - float epsilon = 0.001f); + const ITensorInfo *input_bias = nullptr, const ITensorInfo *bn_beta = nullptr, const ITensorInfo *bn_gamma = nullptr, + float epsilon = 0.001f, FuseBatchNormalizationType fbn_type = FuseBatchNormalizationType::CONVOLUTION); // Inherited methods overridden: void run(const Window &window, const ThreadInfo &info) override; private: - const ITensor *_conv_weights; - const ITensor *_conv_bias; + const ITensor *_input_weights; + const ITensor *_input_bias; const ITensor *_bn_mean; const ITensor *_bn_var; const ITensor *_bn_gamma; @@ -101,7 +107,7 @@ private: bool _run_in_place_weights; bool _run_in_place_bias; - using FuseBatchNormFunction = void(const ITensor *conv_weights, const ITensor *conv_bias, ITensor *fused_weights, ITensor *fused_bias, + using FuseBatchNormFunction = void(const ITensor *input_weights, const ITensor *input_bias, ITensor *fused_weights, ITensor *fused_bias, const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window); FuseBatchNormFunction *_func; diff --git a/arm_compute/runtime/NEON/functions/NEFuseBatchNormalization.h b/arm_compute/runtime/NEON/functions/NEFuseBatchNormalization.h index 5e6286966a..3a2f6ccb6d 100644 --- a/arm_compute/runtime/NEON/functions/NEFuseBatchNormalization.h +++ b/arm_compute/runtime/NEON/functions/NEFuseBatchNormalization.h @@ -52,37 +52,43 @@ public: ~NEFuseBatchNormalization() = default; /** Set the input and output tensors. * - * @param[in] conv_weights Convolution layer weights tensor. Data type supported: F16/F32 - * @param[in] bn_mean Batch normalization layer mean tensor. Same as @p conv_weights - * @param[in] bn_var Batch normalization layer variance tensor. Same as @p conv_weights - * @param[out] fused_weights Output fused weights tensor. Same as @p conv_weights - * @param[out] fused_bias Output fused bias tensor. Same as @p conv_weights - * @param[in] conv_bias (Optional) Convolution layer bias tensor. Same as @p conv_weights - * @param[in] bn_beta (Optional) Batch normalization layer beta tensor. Same as @p conv_weights - * @param[in] bn_gamma (Optional) Batch normalization layer gamma tensor. Same as @p conv_weights + * @param[in] input_weights Input weights tensor for convolution or depthwise convolution layer. Data type supported: F16/F32. Data layout supported: NCHW, NHWC + * @param[in] bn_mean Batch normalization layer mean tensor. Same as @p input_weights + * @param[in] bn_var Batch normalization layer variance tensor. Same as @p input_weights + * @param[out] fused_weights (Optional) Output fused weights tensor. It can be a nullptr in case of in-place computation. Same as @p input_weights + * @param[out] fused_bias (Optional) Output fused bias tensor. It can be a nullptr in case of in-place computation and input_bias != nullptr. Same as @p input_weights + * @param[in] input_bias (Optional) Input bias tensor for convolution or depthwise convolution layer. It can be a nullptr in case the bias tensor is not required. Same as @p input_weights + * @param[in] bn_beta (Optional) Batch normalization layer beta tensor. It can be a nullptr in case the beta tensor is not required. Same as @p input_weights + * @note if nullptr, bn_beta is set to 0.0 + * @param[in] bn_gamma (Optional) Batch normalization layer gamma tensor. It can be a nullptr in case the gamma tensor is not required. Same as @p input_weights + * @note if nullptr, bn_gamma is set to 1.0 * @param[in] epsilon (Optional) Batch normalization layer epsilon parameter. Defaults to 0.001f. + * @param[in] fbn_type (Optional) Fused batch normalization type. Defaults to Convolution. */ - void configure(const ITensor *conv_weights, const ITensor *bn_mean, const ITensor *bn_var, ITensor *fused_weights, ITensor *fused_bias, - const ITensor *conv_bias = nullptr, const ITensor *bn_beta = nullptr, const ITensor *bn_gamma = nullptr, - float epsilon = 0.001f); + void configure(const ITensor *input_weights, const ITensor *bn_mean, const ITensor *bn_var, ITensor *fused_weights, ITensor *fused_bias, + const ITensor *input_bias = nullptr, const ITensor *bn_beta = nullptr, const ITensor *bn_gamma = nullptr, + float epsilon = 0.001f, FuseBatchNormalizationType fbn_type = FuseBatchNormalizationType::CONVOLUTION); /** Static function to check if given info will lead to a valid configuration of @ref NEFuseBatchNormalization * - * @param[in] conv_weights Convolution layer weights tensor. Data type supported: F16/F32 - * @param[in] bn_mean Batch normalization layer mean tensor. Same as @p conv_weights - * @param[in] bn_var Batch normalization layer variance tensor. Same as @p conv_weights - * @param[in] fused_weights Output fused weights tensor. Same as @p conv_weights - * @param[in] fused_bias Output fused bias tensor. Same as @p conv_weights - * @param[in] conv_bias (Optional) Convolution layer bias tensor. Same as @p conv_weights - * @param[in] bn_beta (Optional) Batch normalization layer beta tensor. Same as @p conv_weights - * @param[in] bn_gamma (Optional) Batch normalization layer gamma tensor. Same as @p conv_weights + * @param[in] input_weights Input weights tensor info for convolution or depthwise convolution layer. Data type supported: F16/F32. Data layout supported: NCHW, NHWC + * @param[in] bn_mean Batch normalization layer mean tensor info. Same as @p input_weights + * @param[in] bn_var Batch normalization layer variance tensor info. Same as @p input_weights + * @param[in] fused_weights (Optional) Output fused weights tensor info. It can be a nullptr in case of in-place computation. Same as @p input_weights + * @param[in] fused_bias (Optional) Output fused bias tensor info. It can be a nullptr in case of in-place computation and input_bias != nullptr. Same as @p input_weights + * @param[in] input_bias (Optional) Input bias tensor info for convolution or depthwise convolution layer. It can be a nullptr in case the bias tensor is not required. Same as @p input_weights + * @param[in] bn_beta (Optional) Batch normalization layer beta tensor info. It can be a nullptr in case the beta tensor is not required. Same as @p input_weights + * @note if nullptr, bn_beta is set to 0.0 + * @param[in] bn_gamma (Optional) Batch normalization layer gamma tensor info. It can be a nullptr in case the gamma tensor is not required. Same as @p input_weights + * @note if nullptr, bn_gamma is set to 1.0 * @param[in] epsilon (Optional) Batch normalization layer epsilon parameter. Defaults to 0.001f. + * @param[in] fbn_type (Optional) Fused batch normalization type. Defaults to Convolution. * * @return a status */ - static Status validate(const ITensorInfo *conv_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, + static Status validate(const ITensorInfo *input_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, const ITensorInfo *fused_weights, const ITensorInfo *fused_bias, - const ITensorInfo *conv_bias = nullptr, const ITensorInfo *bn_beta = nullptr, const ITensorInfo *bn_gamma = nullptr, - float epsilon = 0.001f); + const ITensorInfo *input_bias = nullptr, const ITensorInfo *bn_beta = nullptr, const ITensorInfo *bn_gamma = nullptr, + float epsilon = 0.001f, FuseBatchNormalizationType fbn_type = FuseBatchNormalizationType::CONVOLUTION); // Inherited methods overridden: void run() override; diff --git a/src/core/NEON/kernels/NEFuseBatchNormalizationKernel.cpp b/src/core/NEON/kernels/NEFuseBatchNormalizationKernel.cpp index d45e3ce56a..836e429aba 100644 --- a/src/core/NEON/kernels/NEFuseBatchNormalizationKernel.cpp +++ b/src/core/NEON/kernels/NEFuseBatchNormalizationKernel.cpp @@ -26,74 +26,86 @@ #include "arm_compute/core/CPP/Validate.h" #include "arm_compute/core/Helpers.h" #include "arm_compute/core/ITensor.h" +#include "arm_compute/core/NEON/wrapper/wrapper.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/Window.h" - #include "support/ToolchainSupport.h" -#include "arm_compute/core/NEON/wrapper/wrapper.h" #include "utils/TypePrinter.h" +#include + namespace arm_compute { namespace { -Status validate_arguments(const ITensorInfo *conv_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, +Status validate_arguments(const ITensorInfo *input_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, const ITensorInfo *fused_weights, const ITensorInfo *fused_bias, - const ITensorInfo *conv_bias, const ITensorInfo *bn_beta, const ITensorInfo *bn_gamma, - float epsilon) + const ITensorInfo *input_bias, const ITensorInfo *bn_beta, const ITensorInfo *bn_gamma, + float epsilon, FuseBatchNormalizationType fbn_type) { ARM_COMPUTE_UNUSED(epsilon); - ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(conv_weights); - ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(conv_weights, 1, DataType::F16, DataType::F32); + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input_weights, bn_mean, bn_var); + ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(input_weights); + ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input_weights, 1, DataType::F16, DataType::F32); ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(bn_mean, bn_var); - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(conv_weights, bn_mean, bn_var); - - unsigned int kernels_idx = get_data_layout_dimension_index(conv_weights->data_layout(), DataLayoutDimension::BATCHES); - ARM_COMPUTE_RETURN_ERROR_ON(conv_weights->dimension(kernels_idx) != bn_mean->dimension(0)); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input_weights, bn_mean, bn_var); + ARM_COMPUTE_RETURN_ERROR_ON(input_bias == nullptr && fused_bias == nullptr); + ARM_COMPUTE_RETURN_ERROR_ON(bn_mean->num_dimensions() > 1); + if(fbn_type == FuseBatchNormalizationType::CONVOLUTION) + { + ARM_COMPUTE_RETURN_ERROR_ON(input_weights->dimension(3) != bn_mean->dimension(0)); + } + else + { + const size_t channel_idx = get_data_layout_dimension_index(input_weights->data_layout(), DataLayoutDimension::CHANNEL); + ARM_COMPUTE_RETURN_ERROR_ON(input_weights->dimension(channel_idx) != bn_mean->dimension(0)); + } // Validate bias - if(conv_bias != nullptr) + if(input_bias != nullptr) { - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(bn_mean, conv_bias); - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(conv_weights, conv_bias); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(bn_mean, input_bias); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input_weights, input_bias); } // Validate beta if(bn_beta != nullptr) { ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(bn_mean, bn_beta); - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(conv_weights, bn_beta); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input_weights, bn_beta); } // Validate gamma if(bn_gamma != nullptr) { ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(bn_mean, bn_gamma); - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(conv_weights, bn_gamma); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input_weights, bn_gamma); } // Validate output weights if(fused_weights != nullptr && fused_weights->total_size() != 0) { - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(conv_weights, fused_weights); - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_LAYOUT(conv_weights, fused_weights); - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(conv_weights, fused_weights); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(input_weights, fused_weights); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_LAYOUT(input_weights, fused_weights); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input_weights, fused_weights); } // Validate output bias if(fused_bias != nullptr && fused_bias->total_size() != 0) { ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(bn_mean, fused_bias); - ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(conv_weights, fused_bias); + ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input_weights, fused_bias); } return Status{}; } -template -void fused_batch_normmalization(const ITensor *conv_weights, const ITensor *conv_bias, ITensor *fused_weights, ITensor *fused_bias, - const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window) +template +void fused_batch_normalization_conv(const ITensor *conv_weights, const ITensor *conv_bias, ITensor *fused_weights, ITensor *fused_bias, + const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window) { - using ExactTagType = typename wrapper::traits::neon_vector::tag_type; + using ScalarType = typename VectorType::scalar_type; + const int size = 16 / conv_weights->info()->element_size(); + using ExactTagType = typename VectorType::tag_type; const bool run_in_place_weights = (fused_weights == nullptr) || (fused_weights == conv_weights); const bool run_in_place_bias = (fused_bias == nullptr) || (conv_bias != nullptr && fused_bias == conv_bias); @@ -112,8 +124,6 @@ void fused_batch_normmalization(const ITensor *conv_weights, const ITensor *conv const auto conv_bias_in = (conv_bias != nullptr ? reinterpret_cast(conv_bias->ptr_to_element(Coordinates(0, 0))) : nullptr); auto conv_bias_out = (run_in_place_bias ? conv_bias_in : reinterpret_cast(fused_bias->ptr_to_element(Coordinates(0, 0)))); - int slice = -1; - const auto input_mean = reinterpret_cast(bn_mean->ptr_to_element(Coordinates(0, 0))); const auto input_var = reinterpret_cast(bn_var->ptr_to_element(Coordinates(0, 0))); const auto input_gamma = (bn_gamma != nullptr) ? reinterpret_cast(bn_gamma->ptr_to_element(Coordinates(0, 0))) : nullptr; @@ -133,45 +143,38 @@ void fused_batch_normmalization(const ITensor *conv_weights, const ITensor *conv auto conv_bias_in_scalar = ScalarType(0.0); execute_window_loop(win, [&](const Coordinates & id) { - if(slice != id[3]) + var = input_var[id[3]]; + if(input_gamma != nullptr) { - slice = id[3]; - mean = input_mean[slice]; - var = input_var[slice]; - gamma = ScalarType(1.0); - beta = ScalarType(0.0); + gamma = input_gamma[id[3]]; + } - // Construct vectors - mean_vec = wrapper::vdup_n(mean, ExactTagType{}); - var_vec = wrapper::vdup_n(var, ExactTagType{}); - if(input_gamma != nullptr) - { - gamma = input_gamma[slice]; - gamma_vec = wrapper::vdup_n(gamma, ExactTagType{}); - } + if((id[0] == 0) && (id[1] == 0) && (id[2] == 0)) + { if(input_beta != nullptr) { - beta = input_beta[slice]; + beta = input_beta[id[3]]; beta_vec = wrapper::vdup_n(beta, ExactTagType{}); } + + // Construct vectors + mean = input_mean[id[3]]; + mean_vec = wrapper::vdup_n(mean, ExactTagType{}); + if(conv_bias_in != nullptr) { - conv_bias_in_scalar = conv_bias_in[slice]; - } - else - { - conv_bias_in_scalar = ScalarType(0); + conv_bias_in_scalar = conv_bias_in[id[3]]; } - - conv_bias_in_scalar = (conv_bias_in_scalar - mean) / sqrt(var + ScalarType(epsilon)); - conv_bias_in_scalar = (conv_bias_in_scalar * gamma) + beta; - conv_bias_out[slice] = conv_bias_in_scalar; - rvar_vec = wrapper::vinvsqrt(wrapper::vadd(var_vec, epsilon_vec)); + auto conv_bias_tmp_scalar = (conv_bias_in_scalar - mean) / std::sqrt(var + ScalarType(epsilon)); + conv_bias_out[id[3]] = (conv_bias_tmp_scalar * gamma) + beta; } int x = window_start_x; auto conv_w_in_ptr = reinterpret_cast(conv_w_in.ptr()); auto conv_w_out_ptr = reinterpret_cast(conv_w_out.ptr()); + var_vec = wrapper::vdup_n(var, ExactTagType{}); + gamma_vec = wrapper::vdup_n(gamma, ExactTagType{}); + rvar_vec = wrapper::vinvsqrt(wrapper::vadd(var_vec, epsilon_vec)); for(; x <= (window_end_x - window_step_x); x += window_step_x) { @@ -186,28 +189,245 @@ void fused_batch_normmalization(const ITensor *conv_weights, const ITensor *conv // Compute left-over elements for(; x < window_end_x; ++x) { - *(conv_w_out_ptr + x) = *(conv_w_in_ptr + x) / sqrt(var + ScalarType(epsilon)) * gamma; + *(conv_w_out_ptr + x) = *(conv_w_in_ptr + x) / std::sqrt(var + ScalarType(epsilon)) * gamma; } }, conv_w_in, conv_w_out); } + +template +void fused_batch_normalization_dwc_nhwc(const ITensor *dwc_weights, const ITensor *dwc_bias, ITensor *fused_weights, ITensor *fused_bias, + const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window) +{ + using ScalarType = typename VectorType::scalar_type; + const int size = 16 / dwc_weights->info()->element_size(); + using ExactTagType = typename VectorType::tag_type; + + const bool run_in_place_weights = (fused_weights == nullptr) || (fused_weights == dwc_weights); + const bool run_in_place_bias = (fused_bias == nullptr) || (dwc_bias != nullptr && fused_bias == dwc_bias); + + // Set build options + Window win = window; + win.set(Window::DimX, Window::Dimension(0, 1, 1)); + + const int window_step_x = size; + const auto window_start_x = static_cast(window.x().start()); + const auto window_end_x = static_cast(window.x().end()); + + Iterator dwc_w_in(dwc_weights, win); + Iterator dwc_w_out(run_in_place_weights ? dwc_weights : fused_weights, win); + + const auto dwc_bias_in = (dwc_bias != nullptr ? reinterpret_cast(dwc_bias->ptr_to_element(Coordinates(0, 0))) : nullptr); + auto dwc_bias_out = (run_in_place_bias ? dwc_bias_in : reinterpret_cast(fused_bias->ptr_to_element(Coordinates(0, 0)))); + + const auto input_mean = reinterpret_cast(bn_mean->ptr_to_element(Coordinates(0, 0))); + const auto input_var = reinterpret_cast(bn_var->ptr_to_element(Coordinates(0, 0))); + const auto input_gamma = (bn_gamma != nullptr) ? reinterpret_cast(bn_gamma->ptr_to_element(Coordinates(0, 0))) : nullptr; + const auto input_beta = (bn_beta != nullptr) ? reinterpret_cast(bn_beta->ptr_to_element(Coordinates(0, 0))) : nullptr; + + auto mean_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{}); + auto var_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{}); + auto gamma_vec = wrapper::vdup_n(ScalarType(1), ExactTagType{}); + auto beta_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{}); + auto rvar_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{}); + auto dwc_bias_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{}); + const auto epsilon_vec = wrapper::vdup_n(ScalarType(epsilon), ExactTagType{}); + + auto gamma = ScalarType(1.0); + auto beta = ScalarType(0.0); + auto dwc_bias_in_scalar = ScalarType(0); + + execute_window_loop(win, [&](const Coordinates & id) + { + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) + { + var_vec = wrapper::vloadq(input_var + x); + if(input_gamma != nullptr) + { + gamma_vec = wrapper::vloadq(input_gamma + x); + } + + if((id[2] == 0) && (id[1] == 0)) + { + mean_vec = wrapper::vloadq(input_mean + x); + + // Construct vectors + if(input_beta != nullptr) + { + beta_vec = wrapper::vloadq(input_beta + x); + } + + if(dwc_bias_in != nullptr) + { + dwc_bias_vec = wrapper::vloadq(dwc_bias_in + x); + } + + auto dwc_bias_tmp_vec = wrapper::vmul(wrapper::vsub(dwc_bias_vec, mean_vec), wrapper::vinvsqrt(wrapper::vadd(var_vec, epsilon_vec))); + dwc_bias_tmp_vec = wrapper::vadd(wrapper::vmul(dwc_bias_tmp_vec, gamma_vec), beta_vec); + wrapper::vstore(dwc_bias_out + x, dwc_bias_tmp_vec); + } + + auto dwc_w_in_ptr = reinterpret_cast(dwc_w_in.ptr()); + auto dwc_w_out_ptr = reinterpret_cast(dwc_w_out.ptr()); + + auto wn = wrapper::vloadq(dwc_w_in_ptr + x); + rvar_vec = wrapper::vinvsqrt(wrapper::vadd(var_vec, epsilon_vec)); + wn = wrapper::vmul(wn, rvar_vec); + wn = wrapper::vmul(wn, gamma_vec); + + // Store results + wrapper::vstore(dwc_w_out_ptr + x, wn); + } + + // Compute left-over elements + for(; x < window_end_x; ++x) + { + auto var = input_var[x]; + if(input_gamma != nullptr) + { + gamma = input_gamma[x]; + } + + if(id[2] == 0 && id[1] == 0) + { + auto mean = input_mean[x]; + if(input_beta != nullptr) + { + beta = input_beta[x]; + } + if(dwc_bias_in != nullptr) + { + dwc_bias_in_scalar = dwc_bias_in[x]; + } + + auto dwc_bias_tmp_scalar = (dwc_bias_in_scalar - mean) / std::sqrt(var + ScalarType(epsilon)); + dwc_bias_out[x] = (dwc_bias_tmp_scalar * gamma) + beta; + } + + const auto dwc_w_in_ptr = reinterpret_cast(dwc_w_in.ptr()); + auto dwc_w_out_ptr = reinterpret_cast(dwc_w_out.ptr()); + + *(dwc_w_out_ptr + x) = *(dwc_w_in_ptr + x) / std::sqrt(var + ScalarType(epsilon)) * gamma; + } + }, + dwc_w_in, dwc_w_out); +} + +template +void fused_batch_normalization_dwc_nchw(const ITensor *dwc_weights, const ITensor *dwc_bias, ITensor *fused_weights, ITensor *fused_bias, + const ITensor *bn_mean, const ITensor *bn_var, const ITensor *bn_beta, const ITensor *bn_gamma, float epsilon, const Window &window) +{ + using ScalarType = typename VectorType::scalar_type; + const int size = 16 / dwc_weights->info()->element_size(); + using ExactTagType = typename VectorType::tag_type; + + const bool run_in_place_weights = (fused_weights == nullptr) || (fused_weights == dwc_weights); + const bool run_in_place_bias = (fused_bias == nullptr) || (dwc_bias != nullptr && fused_bias == dwc_bias); + + // Set build options + Window win = window; + win.set(Window::DimX, Window::Dimension(0, 1, 1)); + + const int window_step_x = size; + const auto window_start_x = static_cast(window.x().start()); + const auto window_end_x = static_cast(window.x().end()); + + Iterator dwc_w_in(dwc_weights, win); + Iterator dwc_w_out(run_in_place_weights ? dwc_weights : fused_weights, win); + + const auto dwc_bias_in = (dwc_bias != nullptr ? reinterpret_cast(dwc_bias->ptr_to_element(Coordinates(0, 0))) : nullptr); + auto dwc_bias_out = (run_in_place_bias ? dwc_bias_in : reinterpret_cast(fused_bias->ptr_to_element(Coordinates(0, 0)))); + + const auto input_mean = reinterpret_cast(bn_mean->ptr_to_element(Coordinates(0, 0))); + const auto input_var = reinterpret_cast(bn_var->ptr_to_element(Coordinates(0, 0))); + const auto input_gamma = (bn_gamma != nullptr) ? reinterpret_cast(bn_gamma->ptr_to_element(Coordinates(0, 0))) : nullptr; + const auto input_beta = (bn_beta != nullptr) ? reinterpret_cast(bn_beta->ptr_to_element(Coordinates(0, 0))) : nullptr; + + auto mean_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{}); + auto var_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{}); + auto gamma_vec = wrapper::vdup_n(ScalarType(1), ExactTagType{}); + auto beta_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{}); + auto rvar_vec = wrapper::vdup_n(ScalarType(0), ExactTagType{}); + const auto epsilon_vec = wrapper::vdup_n(ScalarType(epsilon), ExactTagType{}); + + auto mean = ScalarType(0.0); + auto var = ScalarType(0.0); + auto gamma = ScalarType(1.0); + auto beta = ScalarType(0.0); + auto dwc_bias_in_scalar = ScalarType(0.0); + execute_window_loop(win, [&](const Coordinates & id) + { + var = input_var[id[2]]; + if(input_gamma != nullptr) + { + gamma = input_gamma[id[2]]; + } + + if(id[1] == 0) + { + mean = input_mean[id[2]]; + + // Construct vectors + mean_vec = wrapper::vdup_n(mean, ExactTagType{}); + if(input_beta != nullptr) + { + beta = input_beta[id[2]]; + beta_vec = wrapper::vdup_n(beta, ExactTagType{}); + } + + if(dwc_bias_in != nullptr) + { + dwc_bias_in_scalar = dwc_bias_in[id[2]]; + } + + auto dwc_bias_tmp_scalar = (dwc_bias_in_scalar - mean) / std::sqrt(var + ScalarType(epsilon)); + dwc_bias_out[id[2]] = (dwc_bias_tmp_scalar * gamma) + beta; + } + + int x = window_start_x; + auto dwc_w_in_ptr = reinterpret_cast(dwc_w_in.ptr()); + auto dwc_w_out_ptr = reinterpret_cast(dwc_w_out.ptr()); + var_vec = wrapper::vdup_n(var, ExactTagType{}); + gamma_vec = wrapper::vdup_n(gamma, ExactTagType{}); + rvar_vec = wrapper::vinvsqrt(wrapper::vadd(var_vec, epsilon_vec)); + + for(; x <= (window_end_x - window_step_x); x += window_step_x) + { + auto wn = wrapper::vloadq(dwc_w_in_ptr + x); + wn = wrapper::vmul(wn, rvar_vec); + wn = wrapper::vmul(wn, gamma_vec); + + // Store results + wrapper::vstore(dwc_w_out_ptr + x, wn); + } + + // Compute left-over elements + for(; x < window_end_x; ++x) + { + *(dwc_w_out_ptr + x) = *(dwc_w_in_ptr + x) / std::sqrt(var + ScalarType(epsilon)) * gamma; + } + }, + dwc_w_in, dwc_w_out); +} + } // namespace NEFuseBatchNormalizationKernel::NEFuseBatchNormalizationKernel() - : _conv_weights(nullptr), _conv_bias(nullptr), _bn_mean(nullptr), _bn_var(nullptr), _bn_gamma(nullptr), _bn_beta(nullptr), _fused_weights(nullptr), _fused_bias(nullptr), _epsilon(), + : _input_weights(nullptr), _input_bias(nullptr), _bn_mean(nullptr), _bn_var(nullptr), _bn_gamma(nullptr), _bn_beta(nullptr), _fused_weights(nullptr), _fused_bias(nullptr), _epsilon(), _run_in_place_weights(false), _run_in_place_bias(false), _func(nullptr) { } -void NEFuseBatchNormalizationKernel::configure(const ITensor *conv_weights, const ITensor *bn_mean, const ITensor *bn_var, +void NEFuseBatchNormalizationKernel::configure(const ITensor *input_weights, const ITensor *bn_mean, const ITensor *bn_var, ITensor *fused_weights, ITensor *fused_bias, - const ITensor *conv_bias, const ITensor *bn_beta, const ITensor *bn_gamma, - float epsilon) + const ITensor *input_bias, const ITensor *bn_beta, const ITensor *bn_gamma, + float epsilon, FuseBatchNormalizationType fbn_type) { - ARM_COMPUTE_ERROR_ON_NULLPTR(conv_weights, bn_mean, bn_var); + ARM_COMPUTE_ERROR_ON_NULLPTR(input_weights, bn_mean, bn_var); - _conv_weights = conv_weights; - _conv_bias = conv_bias; + _input_weights = input_weights; + _input_bias = input_bias; _bn_mean = bn_mean; _bn_var = bn_var; _bn_beta = bn_beta; @@ -216,15 +436,15 @@ void NEFuseBatchNormalizationKernel::configure(const ITensor *conv_weights, cons _fused_bias = fused_bias; _epsilon = epsilon; - _run_in_place_weights = (fused_weights == nullptr) || (fused_weights == conv_weights); - _run_in_place_bias = (fused_bias == nullptr) || (conv_bias != nullptr && fused_bias == conv_bias); + _run_in_place_weights = (fused_weights == nullptr) || (fused_weights == input_weights); + _run_in_place_bias = (fused_bias == nullptr) || (input_bias != nullptr && fused_bias == input_bias); // Auto initialize outputs if(_fused_weights != nullptr) { // Output tensor auto initialization if not yet initialized - auto_init_if_empty(*_fused_weights->info(), *_conv_weights->info()->clone()); - fused_weights->info()->set_valid_region(conv_weights->info()->valid_region()); + auto_init_if_empty(*_fused_weights->info(), *_input_weights->info()->clone()); + fused_weights->info()->set_valid_region(input_weights->info()->valid_region()); } if(_fused_bias != nullptr) { @@ -234,42 +454,53 @@ void NEFuseBatchNormalizationKernel::configure(const ITensor *conv_weights, cons } // Validate arguments - ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(conv_weights->info(), bn_mean->info(), bn_var->info(), + ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input_weights->info(), bn_mean->info(), bn_var->info(), (fused_weights != nullptr) ? fused_weights->info() : nullptr, (fused_bias != nullptr) ? fused_bias->info() : nullptr, - (conv_bias != nullptr) ? conv_bias->info() : nullptr, + (input_bias != nullptr) ? input_bias->info() : nullptr, (bn_beta != nullptr) ? bn_beta->info() : nullptr, (bn_gamma != nullptr) ? bn_gamma->info() : nullptr, - epsilon)); + epsilon, fbn_type)); // Configure kernel window - Window win = calculate_max_window(*conv_weights->info()); + Window win = calculate_max_window(*input_weights->info()); INEKernel::configure(win); - // Configure function to run based on different data types - const DataType data_type = _conv_weights->info()->data_type(); - switch(data_type) + // Configure function + static std::map map_function = { - case DataType::F32: - _func = &fused_batch_normmalization; - break; + { "fused_batch_normalization_conv_NHWC_F32", &fused_batch_normalization_conv> }, + { "fused_batch_normalization_conv_NCHW_F32", &fused_batch_normalization_conv> }, + { "fused_batch_normalization_dwc_NHWC_F32", &fused_batch_normalization_dwc_nhwc> }, + { "fused_batch_normalization_dwc_NCHW_F32", &fused_batch_normalization_dwc_nchw> }, #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - case DataType::F16: - _func = &fused_batch_normmalization; - break; -#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - default: - ARM_COMPUTE_ERROR("Not Supported"); - break; + { "fused_batch_normalization_conv_NHWC_F16", &fused_batch_normalization_conv> }, + { "fused_batch_normalization_conv_NCHW_F16", &fused_batch_normalization_conv> }, + { "fused_batch_normalization_dwc_NHWC_F16", &fused_batch_normalization_dwc_nhwc> }, + { "fused_batch_normalization_dwc_NCHW_F16", &fused_batch_normalization_dwc_nchw> }, +#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ + }; + + std::string function_to_call("fused_batch_normalization_"); + function_to_call += fbn_type == FuseBatchNormalizationType::CONVOLUTION ? "conv_" : "dwc_"; + function_to_call += string_from_data_layout(_input_weights->info()->data_layout()); + function_to_call += "_"; + function_to_call += string_from_data_type(_input_weights->info()->data_type()); + + auto it = map_function.find(function_to_call); + + if(it != map_function.end()) + { + _func = it->second; } } -Status NEFuseBatchNormalizationKernel::validate(const ITensorInfo *conv_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, +Status NEFuseBatchNormalizationKernel::validate(const ITensorInfo *input_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, const ITensorInfo *fused_weights, const ITensorInfo *fused_bias, - const ITensorInfo *conv_bias, const ITensorInfo *bn_beta, const ITensorInfo *bn_gamma, - float epsilon) + const ITensorInfo *input_bias, const ITensorInfo *bn_beta, const ITensorInfo *bn_gamma, + float epsilon, FuseBatchNormalizationType fbn_type) { - ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(conv_weights, bn_mean, bn_var, fused_weights, fused_bias, conv_bias, bn_beta, bn_gamma, epsilon)); + ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input_weights, bn_mean, bn_var, fused_weights, fused_bias, input_bias, bn_beta, bn_gamma, epsilon, fbn_type)); return Status{}; } @@ -278,6 +509,6 @@ void NEFuseBatchNormalizationKernel::run(const Window &window, const ThreadInfo ARM_COMPUTE_UNUSED(info); ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(IKernel::window(), window); - (*_func)(_conv_weights, _conv_bias, _fused_weights, _fused_bias, _bn_mean, _bn_var, _bn_beta, _bn_gamma, _epsilon, window); + (*_func)(_input_weights, _input_bias, _fused_weights, _fused_bias, _bn_mean, _bn_var, _bn_beta, _bn_gamma, _epsilon, window); } } // namespace arm_compute diff --git a/src/runtime/NEON/functions/NEFuseBatchNormalization.cpp b/src/runtime/NEON/functions/NEFuseBatchNormalization.cpp index dc48731c4d..68dc159f75 100644 --- a/src/runtime/NEON/functions/NEFuseBatchNormalization.cpp +++ b/src/runtime/NEON/functions/NEFuseBatchNormalization.cpp @@ -36,20 +36,20 @@ NEFuseBatchNormalization::NEFuseBatchNormalization() { } -void NEFuseBatchNormalization::configure(const ITensor *conv_weights, const ITensor *bn_mean, const ITensor *bn_var, +void NEFuseBatchNormalization::configure(const ITensor *input_weights, const ITensor *bn_mean, const ITensor *bn_var, ITensor *fused_weights, ITensor *fused_bias, - const ITensor *conv_bias, const ITensor *bn_beta, const ITensor *bn_gamma, - float epsilon) + const ITensor *input_bias, const ITensor *bn_beta, const ITensor *bn_gamma, + float epsilon, FuseBatchNormalizationType fbn_type) { - _fuse_bn_kernel.configure(conv_weights, bn_mean, bn_var, fused_weights, fused_bias, conv_bias, bn_beta, bn_gamma, epsilon); + _fuse_bn_kernel.configure(input_weights, bn_mean, bn_var, fused_weights, fused_bias, input_bias, bn_beta, bn_gamma, epsilon, fbn_type); } -Status NEFuseBatchNormalization::validate(const ITensorInfo *conv_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, +Status NEFuseBatchNormalization::validate(const ITensorInfo *input_weights, const ITensorInfo *bn_mean, const ITensorInfo *bn_var, const ITensorInfo *fused_weights, const ITensorInfo *fused_bias, - const ITensorInfo *conv_bias, const ITensorInfo *bn_beta, const ITensorInfo *bn_gamma, - float epsilon) + const ITensorInfo *input_bias, const ITensorInfo *bn_beta, const ITensorInfo *bn_gamma, + float epsilon, FuseBatchNormalizationType fbn_type) { - return NEFuseBatchNormalizationKernel::validate(conv_weights, bn_mean, bn_var, fused_weights, fused_bias, conv_bias, bn_beta, bn_gamma, epsilon); + return NEFuseBatchNormalizationKernel::validate(input_weights, bn_mean, bn_var, fused_weights, fused_bias, input_bias, bn_beta, bn_gamma, epsilon, fbn_type); } void NEFuseBatchNormalization::run() diff --git a/tests/validation/NEON/FuseBatchNormalization.cpp b/tests/validation/NEON/FuseBatchNormalization.cpp new file mode 100644 index 0000000000..1a8f928c6e --- /dev/null +++ b/tests/validation/NEON/FuseBatchNormalization.cpp @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/runtime/NEON/functions/NEFuseBatchNormalization.h" +#include "tests/Globals.h" +#include "tests/NEON/Accessor.h" +#include "tests/datasets/ShapeDatasets.h" +#include "tests/framework/Macros.h" +#include "tests/framework/datasets/Datasets.h" +#include "tests/validation/Validation.h" +#include "tests/validation/fixtures/FuseBatchNormalizationFixture.h" + +namespace arm_compute +{ +namespace test +{ +namespace validation +{ +namespace +{ +AbsoluteTolerance absolute_tolerance_f32(0.001f); +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +AbsoluteTolerance absolute_tolerance_f16(0.2f); +#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ +} // namespace + +template +using NEFuseBatchNormalizationConvFixture = FuseBatchNormalizationFixture; +template +using NEFuseBatchNormalizationDWCFixture = FuseBatchNormalizationFixture; + +// *INDENT-OFF* +// clang-format off + +/** Shapes to test - Precommit */ +const auto shape_conv_values_precommit = concat(datasets::Small4DShapes(), datasets::Small3DShapes()); + +/** Shapes to test - Nightly */ +const auto shape_conv_values_nightly = concat(datasets::Large4DShapes(), datasets::Large3DShapes()); + +/** Data layout to test */ +const auto data_layout_values = framework::dataset::make("DataLayout", { DataLayout::NHWC, DataLayout::NCHW }); + +/** In-place flags to test */ +const auto in_place_values = framework::dataset::make("InPlace", { true, false }); + +/** With bias flags to test */ +const auto with_bias_values = framework::dataset::make("WithBias", { true, false }); + +/** With gamma flags to test */ +const auto with_gamma_values = framework::dataset::make("WithGamma", { true, false }); + +/** With beta flags to test */ +const auto with_beta_values = framework::dataset::make("WithBeta", { true, false }); + +TEST_SUITE(NEON) +TEST_SUITE(FuseBatchNormalization) +TEST_SUITE(Convolution) +TEST_SUITE(Float) +TEST_SUITE(FP32) +FIXTURE_DATA_TEST_CASE(RunSmall, NEFuseBatchNormalizationConvFixture, framework::DatasetMode::PRECOMMIT, + combine(combine(combine(combine(combine(combine( + shape_conv_values_precommit, + framework::dataset::make("DataType", { DataType::F32 })), + data_layout_values), + in_place_values), + with_bias_values), + with_gamma_values), + with_beta_values)) +{ + // Validate outputs + validate(Accessor(_target_w), _reference_w, absolute_tolerance_f32); + validate(Accessor(_target_b), _reference_b, absolute_tolerance_f32); +} + +FIXTURE_DATA_TEST_CASE(RunLarge, NEFuseBatchNormalizationConvFixture, framework::DatasetMode::NIGHTLY, + combine(combine(combine(combine(combine(combine( + shape_conv_values_nightly, + framework::dataset::make("DataType", { DataType::F32 })), + data_layout_values), + in_place_values), + with_bias_values), + with_gamma_values), + with_beta_values)) +{ + // Validate outputs + validate(Accessor(_target_w), _reference_w, absolute_tolerance_f32); + validate(Accessor(_target_b), _reference_b, absolute_tolerance_f32); +} +TEST_SUITE_END() // FP32 +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +TEST_SUITE(FP16) +FIXTURE_DATA_TEST_CASE(RunSmall, NEFuseBatchNormalizationConvFixture, framework::DatasetMode::PRECOMMIT, + combine(combine(combine(combine(combine(combine( + shape_conv_values_precommit, + framework::dataset::make("DataType", { DataType::F16 })), + data_layout_values), + in_place_values), + with_bias_values), + with_gamma_values), + with_beta_values)) +{ + // Validate outputs + validate(Accessor(_target_w), _reference_w, absolute_tolerance_f16); + validate(Accessor(_target_b), _reference_b, absolute_tolerance_f16); +} + +FIXTURE_DATA_TEST_CASE(RunLarge, NEFuseBatchNormalizationConvFixture, framework::DatasetMode::NIGHTLY, + combine(combine(combine(combine(combine(combine( + shape_conv_values_nightly, + framework::dataset::make("DataType", { DataType::F16 })), + data_layout_values), + in_place_values), + with_bias_values), + with_gamma_values), + with_beta_values)) +{ + // Validate outputs + validate(Accessor(_target_w), _reference_w, absolute_tolerance_f16); + validate(Accessor(_target_b), _reference_b, absolute_tolerance_f16); +} +TEST_SUITE_END() // FP16 +#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ +TEST_SUITE_END() // Float +TEST_SUITE_END() // Convolution +TEST_SUITE(DepthwiseConvolution) +TEST_SUITE(Float) +TEST_SUITE(FP32) +FIXTURE_DATA_TEST_CASE(RunSmall, NEFuseBatchNormalizationDWCFixture, framework::DatasetMode::PRECOMMIT, + combine(combine(combine(combine(combine(combine( + datasets::Small3DShapes(), + framework::dataset::make("DataType", { DataType::F32 })), + data_layout_values), + in_place_values), + with_bias_values), + with_gamma_values), + with_beta_values)) +{ + // Validate outputs + validate(Accessor(_target_w), _reference_w, absolute_tolerance_f32); + validate(Accessor(_target_b), _reference_b, absolute_tolerance_f32); +} + +FIXTURE_DATA_TEST_CASE(RunLarge, NEFuseBatchNormalizationDWCFixture, framework::DatasetMode::NIGHTLY, + combine(combine(combine(combine(combine(combine( + datasets::Large3DShapes(), + framework::dataset::make("DataType", { DataType::F32 })), + data_layout_values), + in_place_values), + with_bias_values), + with_gamma_values), + with_beta_values)) +{ + // Validate outputs + validate(Accessor(_target_w), _reference_w, absolute_tolerance_f32); + validate(Accessor(_target_b), _reference_b, absolute_tolerance_f32); +} + +TEST_SUITE_END() // FP32 +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +TEST_SUITE(FP16) +FIXTURE_DATA_TEST_CASE(RunSmall, NEFuseBatchNormalizationDWCFixture, framework::DatasetMode::PRECOMMIT, + combine(combine(combine(combine(combine(combine( + datasets::Small3DShapes(), + framework::dataset::make("DataType", { DataType::F16 })), + data_layout_values), + in_place_values), + with_bias_values), + with_gamma_values), + with_beta_values)) +{ + // Validate outputs + validate(Accessor(_target_w), _reference_w, absolute_tolerance_f16); + validate(Accessor(_target_b), _reference_b, absolute_tolerance_f16); +} + +FIXTURE_DATA_TEST_CASE(RunLarge, NEFuseBatchNormalizationDWCFixture, framework::DatasetMode::NIGHTLY, + combine(combine(combine(combine(combine(combine( + datasets::Large3DShapes(), + framework::dataset::make("DataType", { DataType::F16 })), + data_layout_values), + in_place_values), + with_bias_values), + with_gamma_values), + with_beta_values)) +{ + // Validate outputs + validate(Accessor(_target_w), _reference_w, absolute_tolerance_f16); + validate(Accessor(_target_b), _reference_b, absolute_tolerance_f16); +} + +TEST_SUITE_END() // FP16 +#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ +TEST_SUITE_END() // Float +TEST_SUITE_END() // DepthwiseConvolution +TEST_SUITE_END() // FuseBatchNormalization +TEST_SUITE_END() // NEON +} // namespace validation +} // namespace test +} // namespace arm_compute \ No newline at end of file -- cgit v1.2.1