From d7e2ec51239e2075f931e0a9364e0a68534676f1 Mon Sep 17 00:00:00 2001 From: Dana Zlotnik Date: Mon, 3 Jan 2022 10:59:41 +0200 Subject: Decouple NEInstanceNormalizationLayerKernel Resolves COMPMID-4620 Signed-off-by: Dana Zlotnik Change-Id: I22c285339840493c9cfd4c1abfbc3768ad4db824 Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/6871 Tested-by: Arm Jenkins Reviewed-by: Giorgio Arena Comments-Addressed: Arm Jenkins --- .../kernels/NEInstanceNormalizationLayerKernel.cpp | 190 +++++---------------- 1 file changed, 45 insertions(+), 145 deletions(-) (limited to 'src/core/NEON/kernels/NEInstanceNormalizationLayerKernel.cpp') diff --git a/src/core/NEON/kernels/NEInstanceNormalizationLayerKernel.cpp b/src/core/NEON/kernels/NEInstanceNormalizationLayerKernel.cpp index d33431a8d2..71641404bf 100644 --- a/src/core/NEON/kernels/NEInstanceNormalizationLayerKernel.cpp +++ b/src/core/NEON/kernels/NEInstanceNormalizationLayerKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -34,8 +34,10 @@ #include "src/core/CPP/Validate.h" #include "src/core/NEON/NEMath.h" #include "src/core/NEON/wrapper/wrapper.h" +#include "src/core/common/Registrars.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" +#include "src/cpu/kernels/instancenorm/list.h" #include @@ -43,137 +45,53 @@ namespace arm_compute { namespace { -template -void vector_float_sum(AccType &result, AccType &result_square, const InputType &inputs) +struct InstanceNormSelectorData { - result = wrapper::vadd(result, inputs); - result_square = wrapper::vadd(result_square, wrapper::vmul(inputs, inputs)); -} + DataType dt; +}; -#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC -template <> -inline void vector_float_sum(float32x4_t &result, float32x4_t &result_square, const float16x8_t &inputs) -{ - vector_float_sum(result, result_square, wrapper::vcvt(wrapper::vgetlow(inputs))); - vector_float_sum(result, result_square, wrapper::vcvt(wrapper::vgethigh(inputs))); -} -#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +using InstanceNormSelctorPtr = std::add_pointer::type; +using InstanceNormUKernelPtr = std::add_pointer::type; -template -InputType vector_float_norm(const InputType &inputs, const AccType &vec_mean, const AccType &vec_multip, const AccType &vec_beta) +struct InstanceNormKernel { - return wrapper::vadd(wrapper::vmul(wrapper::vsub(inputs, vec_mean), vec_multip), vec_beta); -} + const char *name; + const InstanceNormSelctorPtr is_selected; + InstanceNormUKernelPtr ukernel; +}; -#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC -template <> -inline float16x8_t vector_float_norm(const float16x8_t &inputs, const float32x4_t &vec_mean, const float32x4_t &vec_multip, const float32x4_t &vec_beta) +static const InstanceNormKernel available_kernels[] = { - const auto input_low = wrapper::vcvt(wrapper::vgetlow(inputs)); - const auto input_high = wrapper::vcvt(wrapper::vgethigh(inputs)); - const auto result_low = wrapper::vcvt(vector_float_norm(input_low, vec_mean, vec_multip, vec_beta)); - const auto result_high = wrapper::vcvt(vector_float_norm(input_high, vec_mean, vec_multip, vec_beta)); - float16x8_t result = wrapper::vcombine(result_low, result_high); - - return result; -} + { + "fp32_neon_instancenorm", + [](const InstanceNormSelectorData & data) { return data.dt == DataType::F32; }, + REGISTER_FP32_NEON(arm_compute::cpu::neon_fp32_instancenorm) + }, +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + { + "fp16_neon_instancenorm", + [](const InstanceNormSelectorData & data) { return data.dt == DataType::F16; }, + REGISTER_FP16_NEON(arm_compute::cpu::neon_fp16_instancenorm) + }, #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +}; -template -void instance_normalization_nchw(ITensor *input, ITensor *output, float gamma, float beta, float epsilon, const Window &window) +/** Micro-kernel selector + * + * @param[in] data Selection data passed to help pick the appropriate micro-kernel + * + * @return A matching micro-kernel else nullptr + */ +const InstanceNormKernel *get_implementation(const InstanceNormSelectorData &data) { - /** SIMD vector tag type. */ - using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t; - - // Clear X/Y dimensions on execution window as we handle the planes manually - Window win = window; - win.set(Window::DimX, Window::Dimension(0, 1, 1)); - win.set(Window::DimY, Window::Dimension(0, 1, 1)); - - constexpr int window_step_x = 16 / sizeof(T); - const unsigned int elements_plane = input->info()->dimension(0) * output->info()->dimension(1); - - Iterator input_it(input, win); - execute_window_loop(win, [&](const Coordinates & id) + for(const auto &uk : available_kernels) { - Window win_plane = window; - win_plane.set(Window::DimX, Window::Dimension(0, 1, 1)); - win_plane.set(Window::DimZ, Window::Dimension(id[2], id[2] + 1, 1)); - win_plane.set(3, Window::Dimension(id[3], id[3] + 1, 1)); - - Iterator input_plane_it(input, win_plane); - Iterator output_plane_it(output, win_plane); - - auto sum_h_w = static_cast(0.f); - auto sum_squares_h_w = static_cast(0.f); - - execute_window_loop(win_plane, [&](const Coordinates &) + if(uk.is_selected(data)) { - const auto input_ptr = reinterpret_cast(input_plane_it.ptr()); - - auto vec_sum_h_w = wrapper::vdup_n(static_cast(0.f), ExactTagType{}); - auto vec_sum_squares_h_w = wrapper::vdup_n(static_cast(0.f), ExactTagType{}); - - // Compute S elements per iteration - int x = window.x().start(); - for(; x <= (window.x().end() - window_step_x); x += window_step_x) - { - auto vec_input_val = wrapper::vloadq(input_ptr + x); - vector_float_sum(vec_sum_h_w, vec_sum_squares_h_w, vec_input_val); - } - - auto vec2_sum_h_w = wrapper::vpadd(wrapper::vgethigh(vec_sum_h_w), wrapper::vgetlow(vec_sum_h_w)); - auto vec2_sum_squares_h_w = wrapper::vpadd(wrapper::vgethigh(vec_sum_squares_h_w), wrapper::vgetlow(vec_sum_squares_h_w)); - - vec2_sum_h_w = wrapper::vpadd(vec2_sum_h_w, vec2_sum_h_w); - vec2_sum_squares_h_w = wrapper::vpadd(vec2_sum_squares_h_w, vec2_sum_squares_h_w); - - sum_h_w += wrapper::vgetlane(vec2_sum_h_w, 0); - sum_squares_h_w += wrapper::vgetlane(vec2_sum_squares_h_w, 0); - - // Compute left-over elements - for(; x < window.x().end(); ++x) - { - const auto value = static_cast(*(input_ptr + x)); - sum_h_w += value; - sum_squares_h_w += value * value; - } - }, - input_plane_it, output_plane_it); - - const auto mean_h_w = sum_h_w / elements_plane; - const auto var_h_w = sum_squares_h_w / elements_plane - mean_h_w * mean_h_w; - - const auto multip_h_w = gamma / std::sqrt(var_h_w + epsilon); - const auto vec_mean_h_w = wrapper::vdup_n(static_cast(mean_h_w), ExactTagType{}); - const auto vec_multip_h_w = wrapper::vdup_n(static_cast(multip_h_w), ExactTagType{}); - const auto vec_beta = wrapper::vdup_n(static_cast(beta), ExactTagType{}); - - execute_window_loop(win_plane, [&](const Coordinates &) - { - auto input_ptr = reinterpret_cast(input_plane_it.ptr()); - auto output_ptr = reinterpret_cast(output_plane_it.ptr()); - - // Compute S elements per iteration - int x = window.x().start(); - //auto vec_val = wrapper::vdup_n(static_cast(0.0f), ExactTagType{}); - for(; x <= (window.x().end() - window_step_x); x += window_step_x) - { - const auto vec_val = wrapper::vloadq(input_ptr + x); - const auto normalized_vec = vector_float_norm(vec_val, vec_mean_h_w, vec_multip_h_w, vec_beta); - wrapper::vstore(output_ptr + x, normalized_vec); - } - - // Compute left-over elements - for(; x < window.x().end(); ++x) - { - const auto val = static_cast(*(input_ptr + x)); - *(output_ptr + x) = static_cast((val - mean_h_w) * multip_h_w + beta); - } - }, - input_plane_it, output_plane_it); - }, - input_it); + return &uk; + } + } + return nullptr; } Status validate_arguments(const ITensorInfo *input, const ITensorInfo *output, float gamma, float beta, float epsilon) @@ -210,7 +128,7 @@ std::tuple validate_and_configure_window(ITensorInfo *input, ITe } // namespace NEInstanceNormalizationLayerKernel::NEInstanceNormalizationLayerKernel() - : _func(nullptr), _input(nullptr), _output(nullptr), _gamma(1), _beta(0), _epsilon(1e-12) + : _input(nullptr), _output(nullptr), _gamma(1), _beta(0), _epsilon(1e-12) { } @@ -227,28 +145,6 @@ void NEInstanceNormalizationLayerKernel::configure(ITensor *input, ITensor *outp ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(_input->info(), _output->info(), _gamma, _beta, _epsilon)); - if(_input->info()->data_type() == DataType::F32) - { - _func = &instance_normalization_nchw; - } -#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - else if(_input->info()->data_type() == DataType::F16) - { - if(_use_mixed_precision) - { - _func = &instance_normalization_nchw; - } - else - { - _func = &instance_normalization_nchw; - } - } -#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - else - { - ARM_COMPUTE_ERROR("Unsupported data type"); - } - // Configure kernel window auto win_config = validate_and_configure_window(_input->info(), _output->info()); ARM_COMPUTE_ERROR_THROW_ON(std::get<0>(win_config)); @@ -268,6 +164,10 @@ void NEInstanceNormalizationLayerKernel::run(const Window &window, const ThreadI ARM_COMPUTE_UNUSED(info); ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INEKernel::window(), window); - (*_func)(_input, _output, _gamma, _beta, _epsilon, window); + + const auto *uk = get_implementation(InstanceNormSelectorData{ _input->info()->data_type() }); + ARM_COMPUTE_ERROR_ON(uk == nullptr || uk->ukernel == nullptr); + + uk->ukernel(_input, _output, _gamma, _beta, _epsilon, _use_mixed_precision, window); } } // namespace arm_compute -- cgit v1.2.1