diff options
Diffstat (limited to 'src/cpu/kernels/instancenorm/generic/neon')
| -rw-r--r-- | src/cpu/kernels/instancenorm/generic/neon/fp16.cpp | 201 | ||||
| -rw-r--r-- | src/cpu/kernels/instancenorm/generic/neon/fp32.cpp | 8 | ||||
| -rw-r--r-- | src/cpu/kernels/instancenorm/generic/neon/impl.cpp | 173 | ||||
| -rw-r--r-- | src/cpu/kernels/instancenorm/generic/neon/impl.h | 6 |
4 files changed, 215 insertions, 173 deletions
diff --git a/src/cpu/kernels/instancenorm/generic/neon/fp16.cpp b/src/cpu/kernels/instancenorm/generic/neon/fp16.cpp index 2b7d91b144..44418c0bb9 100644 --- a/src/cpu/kernels/instancenorm/generic/neon/fp16.cpp +++ b/src/cpu/kernels/instancenorm/generic/neon/fp16.cpp @@ -23,6 +23,7 @@ */ #if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) #include "arm_compute/core/Helpers.h" + #include "src/core/NEON/wrapper/wrapper.h" #include "src/cpu/kernels/instancenorm/generic/neon/impl.h" @@ -40,7 +41,10 @@ void vector_float_sum_fp16(AccType &result, AccType &result_square, const InputT } template <typename InputType, typename AccType> -InputType vector_float_norm_fp16(const InputType &inputs, const AccType &vec_mean, const AccType &vec_multip, const AccType &vec_beta) +InputType vector_float_norm_fp16(const InputType &inputs, + const AccType &vec_mean, + const AccType &vec_multip, + const AccType &vec_beta) { return wrapper::vadd(wrapper::vmul(wrapper::vsub(inputs, vec_mean), vec_multip), vec_beta); } @@ -52,19 +56,24 @@ inline void vector_float_sum_fp16(float32x4_t &result, float32x4_t &result_squar vector_float_sum_fp16(result, result_square, wrapper::vcvt<float>(wrapper::vgethigh(inputs))); } template <> -inline float16x8_t vector_float_norm_fp16(const float16x8_t &inputs, const float32x4_t &vec_mean, const float32x4_t &vec_multip, const float32x4_t &vec_beta) +inline float16x8_t vector_float_norm_fp16(const float16x8_t &inputs, + const float32x4_t &vec_mean, + const float32x4_t &vec_multip, + const float32x4_t &vec_beta) { - const auto input_low = wrapper::vcvt<float>(wrapper::vgetlow(inputs)); - const auto input_high = wrapper::vcvt<float>(wrapper::vgethigh(inputs)); - const auto result_low = wrapper::vcvt<float16_t>(vector_float_norm_fp16(input_low, vec_mean, vec_multip, vec_beta)); - const auto result_high = wrapper::vcvt<float16_t>(vector_float_norm_fp16(input_high, vec_mean, vec_multip, vec_beta)); - float16x8_t result = wrapper::vcombine(result_low, result_high); + const auto input_low = wrapper::vcvt<float>(wrapper::vgetlow(inputs)); + const auto input_high = wrapper::vcvt<float>(wrapper::vgethigh(inputs)); + const auto result_low = wrapper::vcvt<float16_t>(vector_float_norm_fp16(input_low, vec_mean, vec_multip, vec_beta)); + const auto result_high = + wrapper::vcvt<float16_t>(vector_float_norm_fp16(input_high, vec_mean, vec_multip, vec_beta)); + float16x8_t result = wrapper::vcombine(result_low, result_high); return result; } template <typename AccType> -void instance_normalization_nchw_fp16(const ITensor *input, ITensor *output, float gamma, float beta, float epsilon, const Window &window) +void instance_normalization_nchw_fp16( + const ITensor *input, ITensor *output, float gamma, float beta, float epsilon, const Window &window) { /** SIMD vector tag type. */ using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<float16_t, wrapper::traits::BitWidth::W128>; @@ -78,91 +87,105 @@ void instance_normalization_nchw_fp16(const ITensor *input, ITensor *output, flo const unsigned int elements_plane = input->info()->dimension(0) * output->info()->dimension(1); Iterator input_it(input, win); - execute_window_loop(win, [&](const Coordinates & id) - { - Window win_plane = window; - win_plane.set(Window::DimX, Window::Dimension(0, 1, 1)); - win_plane.set(Window::DimZ, Window::Dimension(id[2], id[2] + 1, 1)); - win_plane.set(3, Window::Dimension(id[3], id[3] + 1, 1)); - - Iterator input_plane_it(input, win_plane); - Iterator output_plane_it(output, win_plane); - - auto sum_h_w = static_cast<AccType>(0.f); - auto sum_squares_h_w = static_cast<AccType>(0.f); - - execute_window_loop(win_plane, [&](const Coordinates &) - { - const auto input_ptr = reinterpret_cast<const float16_t *>(input_plane_it.ptr()); - - auto vec_sum_h_w = wrapper::vdup_n(static_cast<AccType>(0.f), ExactTagType{}); - auto vec_sum_squares_h_w = wrapper::vdup_n(static_cast<AccType>(0.f), ExactTagType{}); - - // Compute S elements per iteration - int x = window.x().start(); - for(; x <= (window.x().end() - window_step_x); x += window_step_x) - { - auto vec_input_val = wrapper::vloadq(input_ptr + x); - vector_float_sum_fp16(vec_sum_h_w, vec_sum_squares_h_w, vec_input_val); - } - - auto vec2_sum_h_w = wrapper::vpadd(wrapper::vgethigh(vec_sum_h_w), wrapper::vgetlow(vec_sum_h_w)); - auto vec2_sum_squares_h_w = wrapper::vpadd(wrapper::vgethigh(vec_sum_squares_h_w), wrapper::vgetlow(vec_sum_squares_h_w)); - - vec2_sum_h_w = wrapper::vpadd(vec2_sum_h_w, vec2_sum_h_w); - vec2_sum_squares_h_w = wrapper::vpadd(vec2_sum_squares_h_w, vec2_sum_squares_h_w); - - sum_h_w += wrapper::vgetlane(vec2_sum_h_w, 0); - sum_squares_h_w += wrapper::vgetlane(vec2_sum_squares_h_w, 0); - - // Compute left-over elements - for(; x < window.x().end(); ++x) - { - const auto value = static_cast<AccType>(*(input_ptr + x)); - sum_h_w += value; - sum_squares_h_w += value * value; - } - }, - input_plane_it, output_plane_it); - - const auto mean_h_w = sum_h_w / elements_plane; - const auto var_h_w = sum_squares_h_w / elements_plane - mean_h_w * mean_h_w; - - const auto multip_h_w = gamma / std::sqrt(var_h_w + epsilon); - const auto vec_mean_h_w = wrapper::vdup_n(static_cast<AccType>(mean_h_w), ExactTagType{}); - const auto vec_multip_h_w = wrapper::vdup_n(static_cast<AccType>(multip_h_w), ExactTagType{}); - const auto vec_beta = wrapper::vdup_n(static_cast<AccType>(beta), ExactTagType{}); - - execute_window_loop(win_plane, [&](const Coordinates &) + execute_window_loop( + win, + [&](const Coordinates &id) { - auto input_ptr = reinterpret_cast<const float16_t *>(input_plane_it.ptr()); - auto output_ptr = reinterpret_cast<float16_t *>(output_plane_it.ptr()); - - // Compute S elements per iteration - int x = window.x().start(); - for(; x <= (window.x().end() - window_step_x); x += window_step_x) - { - const auto vec_val = wrapper::vloadq(input_ptr + x); - const auto normalized_vec = vector_float_norm_fp16(vec_val, vec_mean_h_w, vec_multip_h_w, vec_beta); - wrapper::vstore(output_ptr + x, normalized_vec); - } - - // Compute left-over elements - for(; x < window.x().end(); ++x) - { - const auto val = static_cast<AccType>(*(input_ptr + x)); - *(output_ptr + x) = static_cast<float16_t>((val - mean_h_w) * multip_h_w + beta); - } + Window win_plane = window; + win_plane.set(Window::DimX, Window::Dimension(0, 1, 1)); + win_plane.set(Window::DimZ, Window::Dimension(id[2], id[2] + 1, 1)); + win_plane.set(3, Window::Dimension(id[3], id[3] + 1, 1)); + + Iterator input_plane_it(input, win_plane); + Iterator output_plane_it(output, win_plane); + + auto sum_h_w = static_cast<AccType>(0.f); + auto sum_squares_h_w = static_cast<AccType>(0.f); + + execute_window_loop( + win_plane, + [&](const Coordinates &) + { + const auto input_ptr = reinterpret_cast<const float16_t *>(input_plane_it.ptr()); + + auto vec_sum_h_w = wrapper::vdup_n(static_cast<AccType>(0.f), ExactTagType{}); + auto vec_sum_squares_h_w = wrapper::vdup_n(static_cast<AccType>(0.f), ExactTagType{}); + + // Compute S elements per iteration + int x = window.x().start(); + for (; x <= (window.x().end() - window_step_x); x += window_step_x) + { + auto vec_input_val = wrapper::vloadq(input_ptr + x); + vector_float_sum_fp16(vec_sum_h_w, vec_sum_squares_h_w, vec_input_val); + } + + auto vec2_sum_h_w = wrapper::vpadd(wrapper::vgethigh(vec_sum_h_w), wrapper::vgetlow(vec_sum_h_w)); + auto vec2_sum_squares_h_w = + wrapper::vpadd(wrapper::vgethigh(vec_sum_squares_h_w), wrapper::vgetlow(vec_sum_squares_h_w)); + + vec2_sum_h_w = wrapper::vpadd(vec2_sum_h_w, vec2_sum_h_w); + vec2_sum_squares_h_w = wrapper::vpadd(vec2_sum_squares_h_w, vec2_sum_squares_h_w); + + sum_h_w += wrapper::vgetlane(vec2_sum_h_w, 0); + sum_squares_h_w += wrapper::vgetlane(vec2_sum_squares_h_w, 0); + + // Compute left-over elements + for (; x < window.x().end(); ++x) + { + const auto value = static_cast<AccType>(*(input_ptr + x)); + sum_h_w += value; + sum_squares_h_w += value * value; + } + }, + input_plane_it, output_plane_it); + + const auto mean_h_w = sum_h_w / elements_plane; + const auto var_h_w = sum_squares_h_w / elements_plane - mean_h_w * mean_h_w; + + const auto multip_h_w = gamma / std::sqrt(var_h_w + epsilon); + const auto vec_mean_h_w = wrapper::vdup_n(static_cast<AccType>(mean_h_w), ExactTagType{}); + const auto vec_multip_h_w = wrapper::vdup_n(static_cast<AccType>(multip_h_w), ExactTagType{}); + const auto vec_beta = wrapper::vdup_n(static_cast<AccType>(beta), ExactTagType{}); + + execute_window_loop( + win_plane, + [&](const Coordinates &) + { + auto input_ptr = reinterpret_cast<const float16_t *>(input_plane_it.ptr()); + auto output_ptr = reinterpret_cast<float16_t *>(output_plane_it.ptr()); + + // Compute S elements per iteration + int x = window.x().start(); + for (; x <= (window.x().end() - window_step_x); x += window_step_x) + { + const auto vec_val = wrapper::vloadq(input_ptr + x); + const auto normalized_vec = + vector_float_norm_fp16(vec_val, vec_mean_h_w, vec_multip_h_w, vec_beta); + wrapper::vstore(output_ptr + x, normalized_vec); + } + + // Compute left-over elements + for (; x < window.x().end(); ++x) + { + const auto val = static_cast<AccType>(*(input_ptr + x)); + *(output_ptr + x) = static_cast<float16_t>((val - mean_h_w) * multip_h_w + beta); + } + }, + input_plane_it, output_plane_it); }, - input_plane_it, output_plane_it); - }, - input_it); -} + input_it); } - -void neon_fp16_instancenorm(ITensor *input, ITensor *output, float gamma, float beta, float epsilon, bool use_mixed_precision, const Window &window) +} // namespace + +void neon_fp16_instancenorm(ITensor *input, + ITensor *output, + float gamma, + float beta, + float epsilon, + bool use_mixed_precision, + const Window &window) { - if(use_mixed_precision) + if (use_mixed_precision) { return instance_normalization_nchw_fp16<float>(input, output, gamma, beta, epsilon, window); } diff --git a/src/cpu/kernels/instancenorm/generic/neon/fp32.cpp b/src/cpu/kernels/instancenorm/generic/neon/fp32.cpp index 061dd9585c..e1ca05518d 100644 --- a/src/cpu/kernels/instancenorm/generic/neon/fp32.cpp +++ b/src/cpu/kernels/instancenorm/generic/neon/fp32.cpp @@ -26,7 +26,13 @@ namespace arm_compute { namespace cpu { -void neon_fp32_instancenorm(ITensor *input, ITensor *output, float gamma, float beta, float epsilon, bool use_mixed_precision, const Window &window) +void neon_fp32_instancenorm(ITensor *input, + ITensor *output, + float gamma, + float beta, + float epsilon, + bool use_mixed_precision, + const Window &window) { ARM_COMPUTE_UNUSED(use_mixed_precision); return instance_normalization_nchw<float>(input, output, gamma, beta, epsilon, window); diff --git a/src/cpu/kernels/instancenorm/generic/neon/impl.cpp b/src/cpu/kernels/instancenorm/generic/neon/impl.cpp index 483b6f568b..515079e1b5 100644 --- a/src/cpu/kernels/instancenorm/generic/neon/impl.cpp +++ b/src/cpu/kernels/instancenorm/generic/neon/impl.cpp @@ -22,6 +22,7 @@ * SOFTWARE. */ #include "src/cpu/kernels/instancenorm/generic/neon/impl.h" + #include "src/core/NEON/wrapper/wrapper.h" namespace arm_compute @@ -38,13 +39,15 @@ void vector_float_sum(AccType &result, AccType &result_square, const InputType & } template <typename InputType, typename AccType> -InputType vector_float_norm(const InputType &inputs, const AccType &vec_mean, const AccType &vec_multip, const AccType &vec_beta) +InputType +vector_float_norm(const InputType &inputs, const AccType &vec_mean, const AccType &vec_multip, const AccType &vec_beta) { return wrapper::vadd(wrapper::vmul(wrapper::vsub(inputs, vec_mean), vec_multip), vec_beta); } template <typename T, typename AccType> -void instance_normalization_nchw(ITensor *input, ITensor *output, float gamma, float beta, float epsilon, const Window &window) +void instance_normalization_nchw( + ITensor *input, ITensor *output, float gamma, float beta, float epsilon, const Window &window) { /** SIMD vector tag type. */ using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<T, wrapper::traits::BitWidth::W128>; @@ -58,88 +61,96 @@ void instance_normalization_nchw(ITensor *input, ITensor *output, float gamma, f const unsigned int elements_plane = input->info()->dimension(0) * output->info()->dimension(1); Iterator input_it(input, win); - execute_window_loop(win, [&](const Coordinates & id) - { - Window win_plane = window; - win_plane.set(Window::DimX, Window::Dimension(0, 1, 1)); - win_plane.set(Window::DimZ, Window::Dimension(id[2], id[2] + 1, 1)); - win_plane.set(3, Window::Dimension(id[3], id[3] + 1, 1)); - - Iterator input_plane_it(input, win_plane); - Iterator output_plane_it(output, win_plane); - - auto sum_h_w = static_cast<AccType>(0.f); - auto sum_squares_h_w = static_cast<AccType>(0.f); - - execute_window_loop(win_plane, [&](const Coordinates &) - { - const auto input_ptr = reinterpret_cast<const T *>(input_plane_it.ptr()); - - auto vec_sum_h_w = wrapper::vdup_n(static_cast<AccType>(0.f), ExactTagType{}); - auto vec_sum_squares_h_w = wrapper::vdup_n(static_cast<AccType>(0.f), ExactTagType{}); - - // Compute S elements per iteration - int x = window.x().start(); - for(; x <= (window.x().end() - window_step_x); x += window_step_x) - { - auto vec_input_val = wrapper::vloadq(input_ptr + x); - vector_float_sum(vec_sum_h_w, vec_sum_squares_h_w, vec_input_val); - } - - auto vec2_sum_h_w = wrapper::vpadd(wrapper::vgethigh(vec_sum_h_w), wrapper::vgetlow(vec_sum_h_w)); - auto vec2_sum_squares_h_w = wrapper::vpadd(wrapper::vgethigh(vec_sum_squares_h_w), wrapper::vgetlow(vec_sum_squares_h_w)); - - vec2_sum_h_w = wrapper::vpadd(vec2_sum_h_w, vec2_sum_h_w); - vec2_sum_squares_h_w = wrapper::vpadd(vec2_sum_squares_h_w, vec2_sum_squares_h_w); - - sum_h_w += wrapper::vgetlane(vec2_sum_h_w, 0); - sum_squares_h_w += wrapper::vgetlane(vec2_sum_squares_h_w, 0); - - // Compute left-over elements - for(; x < window.x().end(); ++x) - { - const auto value = static_cast<AccType>(*(input_ptr + x)); - sum_h_w += value; - sum_squares_h_w += value * value; - } - }, - input_plane_it, output_plane_it); - - const auto mean_h_w = sum_h_w / elements_plane; - const auto var_h_w = sum_squares_h_w / elements_plane - mean_h_w * mean_h_w; - - const auto multip_h_w = gamma / std::sqrt(var_h_w + epsilon); - const auto vec_mean_h_w = wrapper::vdup_n(static_cast<AccType>(mean_h_w), ExactTagType{}); - const auto vec_multip_h_w = wrapper::vdup_n(static_cast<AccType>(multip_h_w), ExactTagType{}); - const auto vec_beta = wrapper::vdup_n(static_cast<AccType>(beta), ExactTagType{}); - - execute_window_loop(win_plane, [&](const Coordinates &) + execute_window_loop( + win, + [&](const Coordinates &id) { - auto input_ptr = reinterpret_cast<T *>(input_plane_it.ptr()); - auto output_ptr = reinterpret_cast<T *>(output_plane_it.ptr()); - - // Compute S elements per iteration - int x = window.x().start(); - //auto vec_val = wrapper::vdup_n(static_cast<T>(0.0f), ExactTagType{}); - for(; x <= (window.x().end() - window_step_x); x += window_step_x) - { - const auto vec_val = wrapper::vloadq(input_ptr + x); - const auto normalized_vec = vector_float_norm(vec_val, vec_mean_h_w, vec_multip_h_w, vec_beta); - wrapper::vstore(output_ptr + x, normalized_vec); - } - - // Compute left-over elements - for(; x < window.x().end(); ++x) - { - const auto val = static_cast<AccType>(*(input_ptr + x)); - *(output_ptr + x) = static_cast<T>((val - mean_h_w) * multip_h_w + beta); - } + Window win_plane = window; + win_plane.set(Window::DimX, Window::Dimension(0, 1, 1)); + win_plane.set(Window::DimZ, Window::Dimension(id[2], id[2] + 1, 1)); + win_plane.set(3, Window::Dimension(id[3], id[3] + 1, 1)); + + Iterator input_plane_it(input, win_plane); + Iterator output_plane_it(output, win_plane); + + auto sum_h_w = static_cast<AccType>(0.f); + auto sum_squares_h_w = static_cast<AccType>(0.f); + + execute_window_loop( + win_plane, + [&](const Coordinates &) + { + const auto input_ptr = reinterpret_cast<const T *>(input_plane_it.ptr()); + + auto vec_sum_h_w = wrapper::vdup_n(static_cast<AccType>(0.f), ExactTagType{}); + auto vec_sum_squares_h_w = wrapper::vdup_n(static_cast<AccType>(0.f), ExactTagType{}); + + // Compute S elements per iteration + int x = window.x().start(); + for (; x <= (window.x().end() - window_step_x); x += window_step_x) + { + auto vec_input_val = wrapper::vloadq(input_ptr + x); + vector_float_sum(vec_sum_h_w, vec_sum_squares_h_w, vec_input_val); + } + + auto vec2_sum_h_w = wrapper::vpadd(wrapper::vgethigh(vec_sum_h_w), wrapper::vgetlow(vec_sum_h_w)); + auto vec2_sum_squares_h_w = + wrapper::vpadd(wrapper::vgethigh(vec_sum_squares_h_w), wrapper::vgetlow(vec_sum_squares_h_w)); + + vec2_sum_h_w = wrapper::vpadd(vec2_sum_h_w, vec2_sum_h_w); + vec2_sum_squares_h_w = wrapper::vpadd(vec2_sum_squares_h_w, vec2_sum_squares_h_w); + + sum_h_w += wrapper::vgetlane(vec2_sum_h_w, 0); + sum_squares_h_w += wrapper::vgetlane(vec2_sum_squares_h_w, 0); + + // Compute left-over elements + for (; x < window.x().end(); ++x) + { + const auto value = static_cast<AccType>(*(input_ptr + x)); + sum_h_w += value; + sum_squares_h_w += value * value; + } + }, + input_plane_it, output_plane_it); + + const auto mean_h_w = sum_h_w / elements_plane; + const auto var_h_w = sum_squares_h_w / elements_plane - mean_h_w * mean_h_w; + + const auto multip_h_w = gamma / std::sqrt(var_h_w + epsilon); + const auto vec_mean_h_w = wrapper::vdup_n(static_cast<AccType>(mean_h_w), ExactTagType{}); + const auto vec_multip_h_w = wrapper::vdup_n(static_cast<AccType>(multip_h_w), ExactTagType{}); + const auto vec_beta = wrapper::vdup_n(static_cast<AccType>(beta), ExactTagType{}); + + execute_window_loop( + win_plane, + [&](const Coordinates &) + { + auto input_ptr = reinterpret_cast<T *>(input_plane_it.ptr()); + auto output_ptr = reinterpret_cast<T *>(output_plane_it.ptr()); + + // Compute S elements per iteration + int x = window.x().start(); + //auto vec_val = wrapper::vdup_n(static_cast<T>(0.0f), ExactTagType{}); + for (; x <= (window.x().end() - window_step_x); x += window_step_x) + { + const auto vec_val = wrapper::vloadq(input_ptr + x); + const auto normalized_vec = vector_float_norm(vec_val, vec_mean_h_w, vec_multip_h_w, vec_beta); + wrapper::vstore(output_ptr + x, normalized_vec); + } + + // Compute left-over elements + for (; x < window.x().end(); ++x) + { + const auto val = static_cast<AccType>(*(input_ptr + x)); + *(output_ptr + x) = static_cast<T>((val - mean_h_w) * multip_h_w + beta); + } + }, + input_plane_it, output_plane_it); }, - input_plane_it, output_plane_it); - }, - input_it); + input_it); } -template void instance_normalization_nchw<float>(ITensor *input, ITensor *output, float gamma, float beta, float epsilon, const Window &window); +template void instance_normalization_nchw<float>( + ITensor *input, ITensor *output, float gamma, float beta, float epsilon, const Window &window); } // namespace cpu } // namespace arm_compute diff --git a/src/cpu/kernels/instancenorm/generic/neon/impl.h b/src/cpu/kernels/instancenorm/generic/neon/impl.h index 0ddfcdd5ba..e1cc7487f7 100644 --- a/src/cpu/kernels/instancenorm/generic/neon/impl.h +++ b/src/cpu/kernels/instancenorm/generic/neon/impl.h @@ -32,13 +32,15 @@ namespace arm_compute namespace cpu { template <typename T, typename AccType = T> -void instance_normalization_nchw(ITensor *input, ITensor *output, float gamma, float beta, float epsilon, const Window &window); +void instance_normalization_nchw( + ITensor *input, ITensor *output, float gamma, float beta, float epsilon, const Window &window); template <typename InputType, typename AccType = InputType> void vector_float_sum(AccType &result, AccType &result_square, const InputType &inputs); template <typename InputType, typename AccType = InputType> -InputType vector_float_norm(const InputType &inputs, const AccType &vec_mean, const AccType &vec_multip, const AccType &vec_beta); +InputType +vector_float_norm(const InputType &inputs, const AccType &vec_mean, const AccType &vec_multip, const AccType &vec_beta); } // namespace cpu } // namespace arm_compute #endif //define SRC_CORE_SVE_KERNELS_INSTANCENORM_IMPL_H |