From 5f39091e502b0805f292d79a2a7da66d485f70ac Mon Sep 17 00:00:00 2001 From: Michalis Spyrou Date: Wed, 13 May 2020 00:12:08 +0100 Subject: COMPMID-3176: Remove padding from NEArithmeticSubtractionKernel COMPMID-3487: Refactor NEArithmeticSubtractionKernel Refactored code in order to remove paddings. This resulted in a big increase in libary size so after some rework the total size dropped by 4Kb. Change-Id: I4e3014c2ae49c29c6090b195ea16620afcf6c09f Signed-off-by: Michalis Spyrou Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/3206 Comments-Addressed: Arm Jenkins Reviewed-by: Michele Di Giorgio Tested-by: Arm Jenkins --- .../NEON/kernels/NEArithmeticSubtractionKernel.cpp | 1018 +++++++++++--------- 1 file changed, 588 insertions(+), 430 deletions(-) (limited to 'src/core') diff --git a/src/core/NEON/kernels/NEArithmeticSubtractionKernel.cpp b/src/core/NEON/kernels/NEArithmeticSubtractionKernel.cpp index 9b7b235c9f..8bfb37ea18 100644 --- a/src/core/NEON/kernels/NEArithmeticSubtractionKernel.cpp +++ b/src/core/NEON/kernels/NEArithmeticSubtractionKernel.cpp @@ -26,6 +26,7 @@ #include "arm_compute/core/CPP/Validate.h" #include "arm_compute/core/NEON/NEAsymm.h" #include "arm_compute/core/NEON/NESymm.h" +#include "arm_compute/core/NEON/wrapper/wrapper.h" #include "arm_compute/core/TensorInfo.h" #include "arm_compute/core/Validate.h" @@ -33,437 +34,628 @@ namespace arm_compute { namespace { -constexpr unsigned int num_elems_processed_per_iteration = 16; - -void sub_wrap_U8_U8_U8(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) +template +inline typename std::enable_if::value, int8_t>::type +quantize(float val, const QuantizationInfo &info) { - Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); - Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); - Iterator output(out, window); - - execute_window_loop(window, [&](const Coordinates &) - { - const uint8x16_t ta1 = vld1q_u8(input1.ptr()); - const uint8x16_t ta2 = vld1q_u8(input2.ptr()); + return quantize_qasymm8_signed(val, info); +} - vst1q_u8(output.ptr(), vsubq_u8(ta1, ta2)); - }, - input1, input2, output); +template +inline typename std::enable_if::value, uint8_t>::type +quantize(float val, const QuantizationInfo &info) +{ + return quantize_qasymm8(val, info); } -void sub_saturate_U8_U8_U8(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) +template +void sub_same(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window, bool is_sat) { - Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); - Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); - Iterator output(out, window); + /** NEON vector tag type. */ + using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t; - execute_window_loop(window, [&](const Coordinates &) - { - const uint8x16_t ta1 = vld1q_u8(input1.ptr()); - const uint8x16_t ta2 = vld1q_u8(input2.ptr()); + // Create input windows + Window input1_win = window.broadcast_if_dimension_le_one(in1->info()->tensor_shape()); + Window input2_win = window.broadcast_if_dimension_le_one(in2->info()->tensor_shape()); - vst1q_u8(output.ptr(), vqsubq_u8(ta1, ta2)); - }, - input1, input2, output); -} + // Clear X Dimension on execution window as we handle manually + Window win = window; + win.set(Window::DimX, Window::Dimension(0, 1, 1)); + + constexpr int window_step_x = 16 / sizeof(T); + const auto window_start_x = static_cast(window.x().start()); + const auto window_end_x = static_cast(window.x().end()); + const bool is_broadcast_across_x = (input1_win.x().step() == 0) || (input2_win.x().step() == 0); -void sub_saturate_QAYSMM8_QAYSMM8_QAYSMM8(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) -{ Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); Iterator output(out, window); - const UniformQuantizationInfo iq1_info = in1->info()->quantization_info().uniform(); - const UniformQuantizationInfo iq2_info = in2->info()->quantization_info().uniform(); - const UniformQuantizationInfo oq_info = out->info()->quantization_info().uniform(); - - execute_window_loop(window, [&](const Coordinates &) + if(is_broadcast_across_x) { - const float32x4x4_t ta1 = vdequantize(vld1q_u8(reinterpret_cast(input1.ptr())), iq1_info); - const float32x4x4_t ta2 = vdequantize(vld1q_u8(reinterpret_cast(input2.ptr())), iq2_info); + const bool is_broadcast_input_2 = input2_win.x().step() == 0; + Window broadcast_win = is_broadcast_input_2 ? input2_win : input1_win; + Window non_broadcast_win = !is_broadcast_input_2 ? input2_win : input1_win; + const ITensor *broadcast_tensor = is_broadcast_input_2 ? in2 : in1; + const ITensor *non_broadcast_tensor = !is_broadcast_input_2 ? in2 : in1; - const float32x4x4_t ta3 = - { - { - vsubq_f32(ta1.val[0], ta2.val[0]), - vsubq_f32(ta1.val[1], ta2.val[1]), - vsubq_f32(ta1.val[2], ta2.val[2]), - vsubq_f32(ta1.val[3], ta2.val[3]), - } - }; + // Clear X Dimension on execution window as we handle manually + non_broadcast_win.set(Window::DimX, Window::Dimension(0, 1, 1)); - const uint8x16_t result = vquantize(ta3, oq_info); + Iterator broadcast_input(broadcast_tensor, broadcast_win); + Iterator non_broadcast_input(non_broadcast_tensor, non_broadcast_win); + Iterator output(out, win); - vst1q_u8(reinterpret_cast(output.ptr()), result); - }, - input1, input2, output); -} + execute_window_loop(win, [&](const Coordinates &) + { + const auto non_broadcast_input_ptr = reinterpret_cast(non_broadcast_input.ptr()); + const auto output_ptr = reinterpret_cast(output.ptr()); -void sub_saturate_QASYMM8_SIGNED_QASYMM8_SIGNED_QASYMM8_SIGNED(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) -{ - Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); - Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); - Iterator output(out, window); + const T broadcast_value = *reinterpret_cast(broadcast_input.ptr()); + const auto broadcast_value_vec = wrapper::vdup_n(broadcast_value, ExactTagType{}); - const UniformQuantizationInfo iq1_info = in1->info()->quantization_info().uniform(); - const UniformQuantizationInfo iq2_info = in2->info()->quantization_info().uniform(); - const UniformQuantizationInfo oq_info = out->info()->quantization_info().uniform(); + // Compute S elements per iteration + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) + { + const auto non_broadcast_v = wrapper::vloadq(non_broadcast_input_ptr + x); + auto res = is_sat ? wrapper::vqsub(broadcast_value_vec, non_broadcast_v) : wrapper::vsub(broadcast_value_vec, non_broadcast_v); + if(is_broadcast_input_2) + { + res = wrapper::vmul(res, wrapper::vdup_n(static_cast(-1), ExactTagType{})); + } + wrapper::vstore(output_ptr + x, res); + } - execute_window_loop(window, [&](const Coordinates &) + // Compute left-over elements + for(; x < window_end_x; ++x) + { + const auto non_broadcast_v = *(non_broadcast_input_ptr + x); + auto res = is_sat ? wrapper::sub_sat(broadcast_value, non_broadcast_v) : broadcast_value - non_broadcast_v; + if(is_broadcast_input_2) + { + res = static_cast(-1) * res; + } + + *(output_ptr + x) = res; + } + }, + broadcast_input, non_broadcast_input, output); + } + else { - const float32x4x4_t ta1 = vdequantize(vld1q_s8(reinterpret_cast(input1.ptr())), iq1_info); - const float32x4x4_t ta2 = vdequantize(vld1q_s8(reinterpret_cast(input2.ptr())), iq2_info); + // Clear X Dimension on execution window as we handle manually + input1_win.set(Window::DimX, Window::Dimension(0, 1, 1)); + input2_win.set(Window::DimX, Window::Dimension(0, 1, 1)); - const float32x4x4_t ta3 = + Iterator input1(in1, input1_win); + Iterator input2(in2, input2_win); + Iterator output(out, win); + + execute_window_loop(win, [&](const Coordinates &) { + const auto input1_ptr = reinterpret_cast(input1.ptr()); + const auto input2_ptr = reinterpret_cast(input2.ptr()); + const auto output_ptr = reinterpret_cast(output.ptr()); + + // Compute S elements per iteration + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) { - vsubq_f32(ta1.val[0], ta2.val[0]), - vsubq_f32(ta1.val[1], ta2.val[1]), - vsubq_f32(ta1.val[2], ta2.val[2]), - vsubq_f32(ta1.val[3], ta2.val[3]), + const auto val1 = wrapper::vloadq(input1_ptr + x); + const auto val2 = wrapper::vloadq(input2_ptr + x); + const auto res = is_sat ? wrapper::vqsub(val1, val2) : wrapper::vsub(val1, val2); + wrapper::vstore(output_ptr + x, res); } - }; - const int8x16_t result = vquantize_signed(ta3, oq_info); - - vst1q_s8(reinterpret_cast(output.ptr()), result); - }, - input1, input2, output); + // Compute left-over elements + for(; x < window_end_x; ++x) + { + const auto val1 = *(input1_ptr + x); + const auto val2 = *(input2_ptr + x); + *(output_ptr + x) = is_sat ? wrapper::sub_sat(val1, val2) : val1 - val2; + } + }, + input1, input2, output); + } } -void sub_saturate_QSYMM16_QSYMM16_QSYMM16(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) +template +void sub_quantized(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window, bool is_sat) { - Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); - Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); - Iterator output(out, window); + ARM_COMPUTE_UNUSED(is_sat); + + // Create input windows + Window input1_win = window.broadcast_if_dimension_le_one(in1->info()->tensor_shape()); + Window input2_win = window.broadcast_if_dimension_le_one(in2->info()->tensor_shape()); + + // Clear X Dimension on execution window as we handle manually + Window win = window; + win.set(Window::DimX, Window::Dimension(0, 1, 1)); + + const int window_step_x = 16; + const auto window_start_x = static_cast(window.x().start()); + const auto window_end_x = static_cast(window.x().end()); + const bool is_broadcast_across_x = (input1_win.x().step() == 0) || (input2_win.x().step() == 0); const UniformQuantizationInfo iq1_info = in1->info()->quantization_info().uniform(); const UniformQuantizationInfo iq2_info = in2->info()->quantization_info().uniform(); const UniformQuantizationInfo oq_info = out->info()->quantization_info().uniform(); - execute_window_loop(window, [&](const Coordinates &) + const float32x4_t vscale1 = vdupq_n_f32(iq1_info.scale); + const float32x4_t vscale2 = vdupq_n_f32(iq2_info.scale); + const float32x4_t invvscaleo = vdupq_n_f32(1.f / oq_info.scale); + const int32x4_t voffset1 = vdupq_n_s32(iq1_info.offset); + const int32x4_t voffset2 = vdupq_n_s32(iq2_info.offset); + const float32x4_t voffseto = vdupq_n_f32(oq_info.offset); + + if(is_broadcast_across_x) { - const int16x8x2_t in1_s16 = + const bool is_broadcast_input_2 = input2_win.x().step() == 0; + Window broadcast_win = is_broadcast_input_2 ? input2_win : input1_win; + Window non_broadcast_win = !is_broadcast_input_2 ? input2_win : input1_win; + const ITensor *broadcast_tensor = is_broadcast_input_2 ? in2 : in1; + const ITensor *non_broadcast_tensor = !is_broadcast_input_2 ? in2 : in1; + const UniformQuantizationInfo broadcast_qinfo = broadcast_tensor->info()->quantization_info().uniform(); + const UniformQuantizationInfo non_broadcast_qinfo = non_broadcast_tensor->info()->quantization_info().uniform(); + + // Clear X Dimension on execution window as we handle manually + non_broadcast_win.set(Window::DimX, Window::Dimension(0, 1, 1)); + + Iterator broadcast_input(broadcast_tensor, broadcast_win); + Iterator non_broadcast_input(non_broadcast_tensor, non_broadcast_win); + Iterator output(out, win); + + execute_window_loop(win, [&](const Coordinates &) { + const auto non_broadcast_input_ptr = reinterpret_cast(non_broadcast_input.ptr()); + const auto output_ptr = reinterpret_cast(output.ptr()); + + const auto broadcast_value = *reinterpret_cast(broadcast_input.ptr()); + const auto broadcast_value_vec = wrapper::vdup_n(static_cast(broadcast_value), wrapper::traits::vector_128_tag{}); + + const float32x4x4_t bf = { - vld1q_s16(reinterpret_cast(input1.ptr())), - vld1q_s16(reinterpret_cast(input1.ptr()) + 8), - } - }; - const int16x8x2_t in2_s16 = - { + { + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(wrapper::vmovl(wrapper::vgetlow(broadcast_value_vec))))), voffset2)), vscale2), + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(wrapper::vmovl(wrapper::vgetlow(broadcast_value_vec))))), voffset2)), vscale2), + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(wrapper::vmovl(wrapper::vgethigh(broadcast_value_vec))))), voffset2)), vscale2), + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(wrapper::vmovl(wrapper::vgethigh(broadcast_value_vec))))), voffset2)), vscale2), + } + }; + const float bfs = static_cast(broadcast_value - broadcast_qinfo.offset) * broadcast_qinfo.scale; + + // Compute S elements per iteration + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) { - vld1q_s16(reinterpret_cast(input2.ptr())), - vld1q_s16(reinterpret_cast(input2.ptr()) + 8), + const auto a = wrapper::vloadq(non_broadcast_input_ptr + x); + + const float32x4x4_t af = + { + { + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(wrapper::vmovl(wrapper::vgetlow(a))))), voffset1)), vscale1), + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(wrapper::vmovl(wrapper::vgetlow(a))))), voffset1)), vscale1), + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(wrapper::vmovl(wrapper::vgethigh(a))))), voffset1)), vscale1), + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(wrapper::vmovl(wrapper::vgethigh(a))))), voffset1)), vscale1), + } + }; + + const int32x4x4_t rf = + { + { +#ifdef __aarch64__ + vcvtnq_s32_f32(vmlaq_f32(voffseto, is_broadcast_input_2 ? vsubq_f32(bf.val[0], af.val[0]) : vsubq_f32(af.val[0], bf.val[0]), invvscaleo)), + vcvtnq_s32_f32(vmlaq_f32(voffseto, is_broadcast_input_2 ? vsubq_f32(bf.val[1], af.val[1]) : vsubq_f32(af.val[1], bf.val[1]), invvscaleo)), + vcvtnq_s32_f32(vmlaq_f32(voffseto, is_broadcast_input_2 ? vsubq_f32(bf.val[2], af.val[2]) : vsubq_f32(af.val[2], bf.val[2]), invvscaleo)), + vcvtnq_s32_f32(vmlaq_f32(voffseto, is_broadcast_input_2 ? vsubq_f32(bf.val[3], af.val[3]) : vsubq_f32(af.val[3], bf.val[3]), invvscaleo)), +#else //__aarch64__ + vcvtq_s32_f32(vmlaq_f32(voffseto, is_broadcast_input_2 ? vsubq_f32(bf.val[0], af.val[0]) : vsubq_f32(af.val[0], bf.val[0]), invvscaleo)), + vcvtq_s32_f32(vmlaq_f32(voffseto, is_broadcast_input_2 ? vsubq_f32(bf.val[1], af.val[1]) : vsubq_f32(af.val[1], bf.val[1]), invvscaleo)), + vcvtq_s32_f32(vmlaq_f32(voffseto, is_broadcast_input_2 ? vsubq_f32(bf.val[2], af.val[2]) : vsubq_f32(af.val[2], bf.val[2]), invvscaleo)), + vcvtq_s32_f32(vmlaq_f32(voffseto, is_broadcast_input_2 ? vsubq_f32(bf.val[3], af.val[3]) : vsubq_f32(af.val[3], bf.val[3]), invvscaleo)), +#endif //__aarch64__ + } + }; + + const auto pa = wrapper::vqmov(vcombine_s16(vqmovn_s32(rf.val[0]), vqmovn_s32(rf.val[1]))); + const auto pb = wrapper::vqmov(vcombine_s16(vqmovn_s32(rf.val[2]), vqmovn_s32(rf.val[3]))); + wrapper::vstore(output_ptr + x, wrapper::vcombine(pa, pb)); } - }; - const float32x4x4_t ta1 = vdequantize(in1_s16, iq1_info); - const float32x4x4_t ta2 = vdequantize(in2_s16, iq2_info); - const float32x4x4_t ta3 = - { + // Compute left-over elements + for(; x < window_end_x; ++x) { - vsubq_f32(ta1.val[0], ta2.val[0]), - vsubq_f32(ta1.val[1], ta2.val[1]), - vsubq_f32(ta1.val[2], ta2.val[2]), - vsubq_f32(ta1.val[3], ta2.val[3]), + const float afs = static_cast(*(non_broadcast_input_ptr + x) - non_broadcast_qinfo.offset) * non_broadcast_qinfo.scale; + *(output_ptr + x) = quantize((afs - bfs), out->info()->quantization_info()); } - }; - - const int16x8x2_t result = vquantize_qsymm16(ta3, oq_info); - - vst1q_s16(reinterpret_cast(output.ptr()), result.val[0]); - vst1q_s16(reinterpret_cast(output.ptr()) + 8, result.val[1]); - }, - input1, input2, output); -} + }, + broadcast_input, non_broadcast_input, output); + } + else + { + // Clear X Dimension on execution window as we handle manually + input1_win.set(Window::DimX, Window::Dimension(0, 1, 1)); + input2_win.set(Window::DimX, Window::Dimension(0, 1, 1)); -void sub_wrap_S16_S16_S16(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) -{ - Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); - Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); - Iterator output(out, window); + Iterator input1(in1, input1_win); + Iterator input2(in2, input2_win); + Iterator output(out, win); - execute_window_loop(window, [&](const Coordinates &) - { - const int16x8x2_t ta1 = - { - { - vld1q_s16(reinterpret_cast(input1.ptr())), - vld1q_s16(reinterpret_cast(input1.ptr()) + 8), - } - }; - const int16x8x2_t ta2 = + execute_window_loop(win, [&](const Coordinates &) { + const auto input1_ptr = reinterpret_cast(input1.ptr()); + const auto input2_ptr = reinterpret_cast(input2.ptr()); + const auto output_ptr = reinterpret_cast(output.ptr()); + + // Compute S elements per iteration + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) { - vld1q_s16(reinterpret_cast(input2.ptr())), - vld1q_s16(reinterpret_cast(input2.ptr()) + 8), + const auto a = wrapper::vloadq(input1_ptr + x); + const auto b = wrapper::vloadq(input2_ptr + x); + + const float32x4x4_t af = + { + { + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(wrapper::vmovl(wrapper::vgetlow(a))))), voffset1)), vscale1), + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(wrapper::vmovl(wrapper::vgetlow(a))))), voffset1)), vscale1), + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(wrapper::vmovl(wrapper::vgethigh(a))))), voffset1)), vscale1), + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(wrapper::vmovl(wrapper::vgethigh(a))))), voffset1)), vscale1), + } + }; + + const float32x4x4_t bf = + { + { + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(wrapper::vmovl(wrapper::vgetlow(b))))), voffset2)), vscale2), + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(wrapper::vmovl(wrapper::vgetlow(b))))), voffset2)), vscale2), + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(wrapper::vmovl(wrapper::vgethigh(b))))), voffset2)), vscale2), + vmulq_f32(vcvtq_f32_s32(vsubq_s32(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(wrapper::vmovl(wrapper::vgethigh(b))))), voffset2)), vscale2), + } + }; + + const int32x4x4_t rf = + { + { +#ifdef __aarch64__ + vcvtnq_s32_f32(vmlaq_f32(voffseto, vsubq_f32(af.val[0], bf.val[0]), invvscaleo)), + vcvtnq_s32_f32(vmlaq_f32(voffseto, vsubq_f32(af.val[1], bf.val[1]), invvscaleo)), + vcvtnq_s32_f32(vmlaq_f32(voffseto, vsubq_f32(af.val[2], bf.val[2]), invvscaleo)), + vcvtnq_s32_f32(vmlaq_f32(voffseto, vsubq_f32(af.val[3], bf.val[3]), invvscaleo)), +#else //__aarch64__ + vcvtq_s32_f32(vmlaq_f32(voffseto, vsubq_f32(af.val[0], bf.val[0]), invvscaleo)), + vcvtq_s32_f32(vmlaq_f32(voffseto, vsubq_f32(af.val[1], bf.val[1]), invvscaleo)), + vcvtq_s32_f32(vmlaq_f32(voffseto, vsubq_f32(af.val[2], bf.val[2]), invvscaleo)), + vcvtq_s32_f32(vmlaq_f32(voffseto, vsubq_f32(af.val[3], bf.val[3]), invvscaleo)), +#endif //__aarch64__ + } + }; + + const auto pa = wrapper::vqmov(vcombine_s16(vqmovn_s32(rf.val[0]), vqmovn_s32(rf.val[1]))); + const auto pb = wrapper::vqmov(vcombine_s16(vqmovn_s32(rf.val[2]), vqmovn_s32(rf.val[3]))); + wrapper::vstore(output_ptr + x, wrapper::vcombine(pa, pb)); } - }; - const int16x8x2_t ta3 = - { + // Compute left-over elements + for(; x < window_end_x; ++x) { - vsubq_s16(ta1.val[0], ta2.val[0]), - vsubq_s16(ta1.val[1], ta2.val[1]) - } - }; + const float afs = static_cast((*(input1_ptr + x)) - iq1_info.offset) * iq1_info.scale; + const float bfs = static_cast((*(input2_ptr + x)) - iq2_info.offset) * iq2_info.scale; - vst1q_s16(reinterpret_cast(output.ptr()), ta3.val[0]); - vst1q_s16(reinterpret_cast(output.ptr()) + 8, ta3.val[1]); - }, - input1, input2, output); + *(output_ptr + x) = quantize((afs - bfs), out->info()->quantization_info()); + } + }, + input1, input2, output); + } } -void sub_saturate_S16_S16_S16(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) +void sub_QSYMM16_QSYMM16_QSYMM16(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window, bool is_sat) { - Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); - Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); - Iterator output(out, window); + ARM_COMPUTE_UNUSED(is_sat); + + // Create input windows + Window input1_win = window.broadcast_if_dimension_le_one(in1->info()->tensor_shape()); + Window input2_win = window.broadcast_if_dimension_le_one(in2->info()->tensor_shape()); + + // Clear X Dimension on execution window as we handle manually + Window win = window; + win.set(Window::DimX, Window::Dimension(0, 1, 1)); + + const int window_step_x = 8; + const auto window_start_x = static_cast(window.x().start()); + const auto window_end_x = static_cast(window.x().end()); + const bool is_broadcast_across_x = (input1_win.x().step() == 0) || (input2_win.x().step() == 0); + + const UniformQuantizationInfo iq1_info = in1->info()->quantization_info().uniform(); + const UniformQuantizationInfo iq2_info = in2->info()->quantization_info().uniform(); + const UniformQuantizationInfo oq_info = out->info()->quantization_info().uniform(); + + const float32x4_t vscale1 = vdupq_n_f32(iq1_info.scale); + const float32x4_t vscale2 = vdupq_n_f32(iq2_info.scale); + const float32x4_t invvscaleo = vdupq_n_f32(1.f / oq_info.scale); - execute_window_loop(window, [&](const Coordinates &) + if(is_broadcast_across_x) { - const int16x8x2_t ta1 = + const bool is_broadcast_input_2 = input2_win.x().step() == 0; + Window broadcast_win = is_broadcast_input_2 ? input2_win : input1_win; + Window non_broadcast_win = !is_broadcast_input_2 ? input2_win : input1_win; + const ITensor *broadcast_tensor = is_broadcast_input_2 ? in2 : in1; + const ITensor *non_broadcast_tensor = !is_broadcast_input_2 ? in2 : in1; + const UniformQuantizationInfo broadcast_qinfo = broadcast_tensor->info()->quantization_info().uniform(); + const UniformQuantizationInfo non_broadcast_qinfo = non_broadcast_tensor->info()->quantization_info().uniform(); + + // Clear X Dimension on execution window as we handle manually + non_broadcast_win.set(Window::DimX, Window::Dimension(0, 1, 1)); + + Iterator broadcast_input(broadcast_tensor, broadcast_win); + Iterator non_broadcast_input(non_broadcast_tensor, non_broadcast_win); + Iterator output(out, win); + + execute_window_loop(win, [&](const Coordinates &) { + const auto non_broadcast_input_ptr = reinterpret_cast(non_broadcast_input.ptr()); + const auto output_ptr = reinterpret_cast(output.ptr()); + + const int16_t broadcast_value = *reinterpret_cast(broadcast_input.ptr()); + const int16x8_t broadcast_value_vec = vdupq_n_s16(broadcast_value); + + const float32x4x2_t bf = { - vld1q_s16(reinterpret_cast(input1.ptr())), - vld1q_s16(reinterpret_cast(input1.ptr()) + 8), - } - }; - const int16x8x2_t ta2 = - { + { + vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(broadcast_value_vec))), vscale2), + vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(broadcast_value_vec))), vscale2), + } + }; + const float bfs = static_cast(broadcast_value) * broadcast_qinfo.scale; + + // Compute S elements per iteration + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) { - vld1q_s16(reinterpret_cast(input2.ptr())), - vld1q_s16(reinterpret_cast(input2.ptr()) + 8), + const int16x8_t a = vld1q_s16(non_broadcast_input_ptr + x); + const float32x4x2_t af = + { + { + vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(a))), vscale1), + vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(a))), vscale1), + } + }; + + const int32x4x4_t rf = + { + { +#ifdef __aarch64__ + vcvtnq_s32_f32(vmulq_f32(is_broadcast_input_2 ? vsubq_f32(bf.val[0], af.val[0]) : vsubq_f32(af.val[0], bf.val[0]), invvscaleo)), + vcvtnq_s32_f32(vmulq_f32(is_broadcast_input_2 ? vsubq_f32(bf.val[1], af.val[1]) : vsubq_f32(af.val[1], bf.val[1]), invvscaleo)), +#else //__aarch64__ + vcvtq_s32_f32(vmulq_f32(is_broadcast_input_2 ? vsubq_f32(bf.val[0], af.val[0]) : vsubq_f32(af.val[0], bf.val[0]), invvscaleo)), + vcvtq_s32_f32(vmulq_f32(is_broadcast_input_2 ? vsubq_f32(bf.val[1], af.val[1]) : vsubq_f32(af.val[1], bf.val[1]), invvscaleo)), +#endif //__aarch64__ + } + }; + + const int16x8_t pa = vcombine_s16(vqmovn_s32(rf.val[0]), vqmovn_s32(rf.val[1])); + vst1q_s16(output_ptr + x, pa); } - }; - const int16x8x2_t ta3 = - { + // Compute left-over elements + for(; x < window_end_x; ++x) { - vqsubq_s16(ta1.val[0], ta2.val[0]), - vqsubq_s16(ta1.val[1], ta2.val[1]) + const float afs = static_cast(*(non_broadcast_input_ptr + x)) * non_broadcast_qinfo.scale; + *(output_ptr + x) = quantize_qsymm16(is_broadcast_input_2 ? (bfs - afs) : (afs - bfs), oq_info); } - }; - - vst1q_s16(reinterpret_cast(output.ptr()), ta3.val[0]); - vst1q_s16(reinterpret_cast(output.ptr()) + 8, ta3.val[1]); - }, - input1, input2, output); -} + }, + broadcast_input, non_broadcast_input, output); + } + else + { + // Clear X Dimension on execution window as we handle manually + input1_win.set(Window::DimX, Window::Dimension(0, 1, 1)); + input2_win.set(Window::DimX, Window::Dimension(0, 1, 1)); -void sub_F16_F16_F16(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) -{ -#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); - Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); - Iterator output(out, window); + Iterator input1(in1, input1_win); + Iterator input2(in2, input2_win); + Iterator output(out, win); - execute_window_loop(window, [&](const Coordinates &) - { - const float16x8x2_t a = - { - { - vld1q_f16(reinterpret_cast(input1.ptr())), - vld1q_f16(reinterpret_cast(input1.ptr()) + 8), - } - }; - const float16x8x2_t b = + execute_window_loop(win, [&](const Coordinates &) { + const auto input1_ptr = reinterpret_cast(input1.ptr()); + const auto input2_ptr = reinterpret_cast(input2.ptr()); + const auto output_ptr = reinterpret_cast(output.ptr()); + + // Compute S elements per iteration + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) { - vld1q_f16(reinterpret_cast(input2.ptr())), - vld1q_f16(reinterpret_cast(input2.ptr()) + 8), + const int16x8_t a = vld1q_s16(input1_ptr + x); + const int16x8_t b = vld1q_s16(input2_ptr + x); + + const float32x4x2_t af = + { + { + vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(a))), vscale1), + vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(a))), vscale1), + } + }; + + const float32x4x2_t bf = + { + { + vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_low_s16(b))), vscale2), + vmulq_f32(vcvtq_f32_s32(vmovl_s16(vget_high_s16(b))), vscale2), + } + }; + + const int32x4x2_t rf = + { + { +#ifdef __aarch64__ + vcvtnq_s32_f32(vmulq_f32(vsubq_f32(af.val[0], bf.val[0]), invvscaleo)), + vcvtnq_s32_f32(vmulq_f32(vsubq_f32(af.val[1], bf.val[1]), invvscaleo)), +#else //__aarch64__ + vcvtq_s32_f32(vmulq_f32(vsubq_f32(af.val[0], bf.val[0]), invvscaleo)), + vcvtq_s32_f32(vmulq_f32(vsubq_f32(af.val[1], bf.val[1]), invvscaleo)), +#endif //__aarch64__ + } + }; + + const int16x8_t pa = vcombine_s16(vqmovn_s32(rf.val[0]), vqmovn_s32(rf.val[1])); + vst1q_s16(output_ptr + x, pa); } - }; - const float16x8x2_t res = - { + + // Compute left-over elements + for(; x < window_end_x; ++x) { - vsubq_f16(a.val[0], b.val[0]), - vsubq_f16(a.val[1], b.val[1]), + const float afs = static_cast((*(input1_ptr + x))) * iq1_info.scale; + const float bfs = static_cast((*(input2_ptr + x))) * iq2_info.scale; + *(output_ptr + x) = quantize_qsymm16((afs - bfs), out->info()->quantization_info()); } - }; - - vst1q_f16(reinterpret_cast(output.ptr()), res.val[0]); - vst1q_f16(reinterpret_cast(output.ptr()) + 8, res.val[1]); - }, - input1, input2, output); -#else /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ - ARM_COMPUTE_UNUSED(in1); - ARM_COMPUTE_UNUSED(in2); - ARM_COMPUTE_UNUSED(out); - ARM_COMPUTE_UNUSED(window); - ARM_COMPUTE_ERROR("Not supported, recompile the library with arch=arm64-v8.2-a"); -#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ + }, + input1, input2, output); + } } -void sub_F32_F32_F32(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) +void sub_S16_U8_S16_impl(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window, bool is_sat, bool is_swapped) { - Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); - Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); - Iterator output(out, window); + // Create input windows + Window win = window; + Window input1_win = window.broadcast_if_dimension_le_one(in1->info()->tensor_shape()); + Window input2_win = window.broadcast_if_dimension_le_one(in2->info()->tensor_shape()); - execute_window_loop(window, [&](const Coordinates &) + // Clear X Dimension on execution window as we handle manually + win.set(Window::DimX, Window::Dimension(0, 1, 1)); + input1_win.set(Window::DimX, Window::Dimension(0, 1, 1)); + input2_win.set(Window::DimX, Window::Dimension(0, 1, 1)); + + Iterator input1(in1, input1_win); + Iterator input2(in2, input2_win); + Iterator output(out, win); + + const int window_step_x = 8; + const auto window_start_x = static_cast(window.x().start()); + const auto window_end_x = static_cast(window.x().end()); + + execute_window_loop(win, [&](const Coordinates &) { - const float32x4x4_t ta1 = + const auto input1_ptr = reinterpret_cast(input1.ptr()); + const auto input2_ptr = reinterpret_cast(input2.ptr()); + const auto output_ptr = reinterpret_cast(output.ptr()); + + if(!is_sat) { + // Compute S elements per iteration + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) { - vld1q_f32(reinterpret_cast(input1.ptr())), - vld1q_f32(reinterpret_cast(input1.ptr()) + 4), - vld1q_f32(reinterpret_cast(input1.ptr()) + 8), - vld1q_f32(reinterpret_cast(input1.ptr()) + 12), + const auto vin1 = wrapper::vloadq(input1_ptr + x); + const auto vin2 = vreinterpretq_s16_u16(wrapper::vmovl(wrapper::vload(input2_ptr + x))); + const auto res = is_swapped ? wrapper::vsub(vin2, vin1) : wrapper::vsub(vin1, vin2); + wrapper::vstore(output_ptr + x, res); } - }; - const float32x4x4_t ta2 = - { + + // Compute left-over elements + for(; x < window_end_x; ++x) { - vld1q_f32(reinterpret_cast(input2.ptr())), - vld1q_f32(reinterpret_cast(input2.ptr()) + 4), - vld1q_f32(reinterpret_cast(input2.ptr()) + 8), - vld1q_f32(reinterpret_cast(input2.ptr()) + 12), + const auto res = is_swapped ? static_cast(*(input2_ptr + x)) - *(input1_ptr + x) : *(input1_ptr + x) - static_cast(*(input2_ptr + x)); + *(output_ptr + x) = res; } - }; - - const float32x4x4_t ta3 = + } + else { + // Compute S elements per iteration + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) { - vsubq_f32(ta1.val[0], ta2.val[0]), - vsubq_f32(ta1.val[1], ta2.val[1]), - vsubq_f32(ta1.val[2], ta2.val[2]), - vsubq_f32(ta1.val[3], ta2.val[3]), + const auto vin1 = wrapper::vloadq(input1_ptr + x); + const auto vin2 = vreinterpretq_s16_u16(wrapper::vmovl(wrapper::vload(input2_ptr + x))); + const auto res = is_swapped ? wrapper::vqsub(vin2, vin1) : wrapper::vqsub(vin1, vin2); + wrapper::vstore(output_ptr + x, res); } - }; - - vst1q_f32(reinterpret_cast(output.ptr()), ta3.val[0]); - vst1q_f32(reinterpret_cast(output.ptr()) + 4, ta3.val[1]); - vst1q_f32(reinterpret_cast(output.ptr()) + 8, ta3.val[2]); - vst1q_f32(reinterpret_cast(output.ptr()) + 12, ta3.val[3]); - }, - input1, input2, output); -} -void sub_wrap_S16_U8_S16(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) -{ - Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); - Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); - Iterator output(out, window); - execute_window_loop(window, [&](const Coordinates &) - { - const uint8x16_t bv_0 = vld1q_u8(input2.ptr()); - int16x8_t a1_0 = vld1q_s16(reinterpret_cast(input1.ptr())); - int16x8_t a2_0 = vld1q_s16(reinterpret_cast(input1.ptr()) + 8); - - a1_0 = vsubq_s16(a1_0, vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(bv_0)))); - a2_0 = vsubq_s16(a2_0, vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(bv_0)))); - - vst1q_s16(reinterpret_cast(output.ptr()), a1_0); - vst1q_s16(reinterpret_cast(output.ptr()) + 8, a2_0); + // Compute left-over elements + for(; x < window_end_x; ++x) + { + const auto res = is_swapped ? wrapper::sub_sat(static_cast(*(input2_ptr + x)), *(input1_ptr + x)) : wrapper::sub_sat(*(input1_ptr + x), static_cast(*(input2_ptr + x))); + *(output_ptr + x) = res; + } + } }, input1, input2, output); } -void sub_saturate_S16_U8_S16(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) +void sub_S16_U8_S16(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window, bool is_sat) { - Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); - Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); - Iterator output(out, window); - - execute_window_loop(window, [&](const Coordinates &) - { - const uint8x16_t bv_0 = vld1q_u8(input2.ptr()); - int16x8_t a1_0 = vld1q_s16(reinterpret_cast(input1.ptr())); - int16x8_t a2_0 = vld1q_s16(reinterpret_cast(input1.ptr()) + 8); - - a1_0 = vqsubq_s16(a1_0, vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(bv_0)))); - a2_0 = vqsubq_s16(a2_0, vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(bv_0)))); - - vst1q_s16(reinterpret_cast(output.ptr()), a1_0); - vst1q_s16(reinterpret_cast(output.ptr()) + 8, a2_0); - }, - input1, input2, output); + sub_S16_U8_S16_impl(in1, in2, out, window, is_sat, false); } -void sub_wrap_U8_S16_S16(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) +void sub_U8_S16_S16(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window, bool is_sat) { - Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); - Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); - Iterator output(out, window); - - execute_window_loop(window, [&](const Coordinates &) - { - const uint8x16_t bv_0 = vld1q_u8(input1.ptr()); - int16x8_t a1_0 = vld1q_s16(reinterpret_cast(input2.ptr())); - int16x8_t a2_0 = vld1q_s16(reinterpret_cast(input2.ptr()) + 8); - - a1_0 = vsubq_s16(vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(bv_0))), a1_0); - a2_0 = vsubq_s16(vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(bv_0))), a2_0); - - vst1q_s16(reinterpret_cast(output.ptr()), a1_0); - vst1q_s16(reinterpret_cast(output.ptr()) + 8, a2_0); - }, - input1, input2, output); + // Swap arguments + sub_S16_U8_S16_impl(in2, in1, out, window, is_sat, true); } -void sub_saturate_U8_S16_S16(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) +void sub_U8_U8_S16(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window, bool is_sat) { - Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); - Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); - Iterator output(out, window); + // Create input windows + Window win = window; + Window input1_win = window.broadcast_if_dimension_le_one(in1->info()->tensor_shape()); + Window input2_win = window.broadcast_if_dimension_le_one(in2->info()->tensor_shape()); - execute_window_loop(window, [&](const Coordinates &) - { - const uint8x16_t bv_0 = vld1q_u8(input1.ptr()); - int16x8_t a1_0 = vld1q_s16(reinterpret_cast(input2.ptr())); - int16x8_t a2_0 = vld1q_s16(reinterpret_cast(input2.ptr()) + 8); + // Clear X Dimension on execution window as we handle manually + win.set(Window::DimX, Window::Dimension(0, 1, 1)); + input1_win.set(Window::DimX, Window::Dimension(0, 1, 1)); + input2_win.set(Window::DimX, Window::Dimension(0, 1, 1)); - a1_0 = vqsubq_s16(vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(bv_0))), a1_0); - a2_0 = vqsubq_s16(vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(bv_0))), a2_0); + Iterator input1(in1, input1_win); + Iterator input2(in2, input2_win); + Iterator output(out, win); - vst1q_s16(reinterpret_cast(output.ptr()), a1_0); - vst1q_s16(reinterpret_cast(output.ptr()) + 8, a2_0); - }, - input1, input2, output); -} + const int window_step_x = 8; + const auto window_start_x = static_cast(window.x().start()); + const auto window_end_x = static_cast(window.x().end()); -void sub_wrap_U8_U8_S16(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) -{ - Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); - Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); - Iterator output(out, window); - - execute_window_loop(window, [&](const Coordinates &) + execute_window_loop(win, [&](const Coordinates &) { - const uint8x16_t av_0 = vld1q_u8(input1.ptr()); - const uint8x16_t bv_0 = vld1q_u8(input2.ptr()); + const auto input1_ptr = reinterpret_cast(input1.ptr()); + const auto input2_ptr = reinterpret_cast(input2.ptr()); + const auto output_ptr = reinterpret_cast(output.ptr()); - const int16x8_t a1_0 = vsubq_s16(vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(av_0))), - vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(bv_0)))); - const int16x8_t a2_0 = vsubq_s16(vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(av_0))), - vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(bv_0)))); - - vst1q_s16(reinterpret_cast(output.ptr()), a1_0); - vst1q_s16(reinterpret_cast(output.ptr()) + 8, a2_0); - }, - input1, input2, output); -} - -void sub_saturate_U8_U8_S16(const ITensor *in1, const ITensor *in2, ITensor *out, const Window &window) -{ - Iterator input1(in1, window.broadcast_if_dimension_le_one(in1->info()->tensor_shape())); - Iterator input2(in2, window.broadcast_if_dimension_le_one(in2->info()->tensor_shape())); - Iterator output(out, window); - - execute_window_loop(window, [&](const Coordinates &) - { - const uint8x16_t av_0 = vld1q_u8(input1.ptr()); - const uint8x16_t bv_0 = vld1q_u8(input2.ptr()); + if(!is_sat) + { + // Compute S elements per iteration + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) + { + const auto vin1 = vreinterpretq_s16_u16(wrapper::vmovl(wrapper::vload(input1_ptr + x))); + const auto vin2 = vreinterpretq_s16_u16(wrapper::vmovl(wrapper::vload(input2_ptr + x))); + wrapper::vstore(output_ptr + x, wrapper::vsub(vin1, vin2)); + } - const int16x8_t a1_0 = vqsubq_s16(vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(av_0))), - vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(bv_0)))); - const int16x8_t a2_0 = vqsubq_s16(vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(av_0))), - vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(bv_0)))); + // Compute left-over elements + for(; x < window_end_x; ++x) + { + *(output_ptr + x) = static_cast(*(input1_ptr + x)) - static_cast(*(input2_ptr + x)); + } + } + else + { + // Compute S elements per iteration + int x = window_start_x; + for(; x <= (window_end_x - window_step_x); x += window_step_x) + { + const auto vin1 = vreinterpretq_s16_u16(wrapper::vmovl(wrapper::vload(input1_ptr + x))); + const auto vin2 = vreinterpretq_s16_u16(wrapper::vmovl(wrapper::vload(input2_ptr + x))); + wrapper::vstore(output_ptr + x, wrapper::vqsub(vin1, vin2)); + } - vst1q_s16(reinterpret_cast(output.ptr()), a1_0); - vst1q_s16(reinterpret_cast(output.ptr()) + 8, a2_0); + // Compute left-over elements + for(; x < window_end_x; ++x) + { + *(output_ptr + x) = wrapper::sub_sat(static_cast(*(input1_ptr + x)), + static_cast(*(input2_ptr + x))); + } + } }, input1, input2, output); } @@ -519,64 +711,10 @@ inline Status validate_arguments(const ITensorInfo &input1, const ITensorInfo &i } return Status{}; } - -inline std::pair validate_and_configure_window(ITensorInfo &input1, ITensorInfo &input2, ITensorInfo &output) -{ - const std::pair broadcast_pair = ITensorInfo::broadcast_shape_and_valid_region(input1, input2); - const TensorShape &out_shape = broadcast_pair.first; - const ValidRegion &valid_region = broadcast_pair.second; - - // Auto initialize output if not initialized - { - set_shape_if_empty(output, out_shape); - - if(input1.data_type() == DataType::S16 || input2.data_type() == DataType::S16) - { - set_format_if_unknown(output, Format::S16); - } - else if(input1.data_type() == DataType::F16 || input2.data_type() == DataType::F16) - { - set_format_if_unknown(output, Format::F16); - } - else if(input1.data_type() == DataType::F32 || input2.data_type() == DataType::F32) - { - set_format_if_unknown(output, Format::F32); - } - else if(input1.data_type() == DataType::QASYMM8 || input2.data_type() == DataType::QASYMM8) - { - set_data_type_if_unknown(output, DataType::QASYMM8); - } - else if(input1.data_type() == DataType::QASYMM8_SIGNED || input2.data_type() == DataType::QASYMM8_SIGNED) - { - set_data_type_if_unknown(output, DataType::QASYMM8_SIGNED); - } - else if(input1.data_type() == DataType::QSYMM16 || input2.data_type() == DataType::QSYMM16) - { - set_data_type_if_unknown(output, DataType::QSYMM16); - } - } - - Window win = calculate_max_window(valid_region, Steps(num_elems_processed_per_iteration)); - Window win_input1 = win.broadcast_if_dimension_le_one(input1); - Window win_input2 = win.broadcast_if_dimension_le_one(input2); - - AccessWindowHorizontal input1_access(&input1, 0, num_elems_processed_per_iteration); - AccessWindowHorizontal input2_access(&input2, 0, num_elems_processed_per_iteration); - AccessWindowHorizontal output_access(&output, 0, num_elems_processed_per_iteration); - - bool window_changed = update_window_and_padding(win_input1, input1_access) - || update_window_and_padding(win_input2, input2_access) - || update_window_and_padding(win, output_access); - - output_access.set_valid_region(win, valid_region); - - Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{}; - return std::make_pair(err, win); -} } // namespace NEArithmeticSubtractionKernel::NEArithmeticSubtractionKernel() - : _func(nullptr), _input1(nullptr), _input2(nullptr), _output(nullptr) + : _func(nullptr), _input1(nullptr), _input2(nullptr), _output(nullptr), _policy(ConvertPolicy::WRAP) { } @@ -585,57 +723,84 @@ void NEArithmeticSubtractionKernel::configure(const ITensor *input1, const ITens ARM_COMPUTE_ERROR_ON_NULLPTR(input1, input2, output); ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(*input1->info(), *input2->info(), *output->info(), policy)); - // Configure kernel window - auto win_config = validate_and_configure_window(*input1->info(), *input2->info(), *output->info()); - ARM_COMPUTE_ERROR_THROW_ON(win_config.first); - - static std::map map_function = - { - { "sub_wrap_U8_U8_U8", &sub_wrap_U8_U8_U8 }, - { "sub_wrap_U8_U8_S16", &sub_wrap_U8_U8_S16 }, - { "sub_saturate_U8_U8_U8", &sub_saturate_U8_U8_U8 }, - { "sub_saturate_U8_U8_S16", &sub_saturate_U8_U8_S16 }, - { "sub_saturate_QASYMM8_QASYMM8_QASYMM8", &sub_saturate_QAYSMM8_QAYSMM8_QAYSMM8 }, - { "sub_saturate_QASYMM8_SIGNED_QASYMM8_SIGNED_QASYMM8_SIGNED", &sub_saturate_QASYMM8_SIGNED_QASYMM8_SIGNED_QASYMM8_SIGNED }, - { "sub_saturate_QSYMM16_QSYMM16_QSYMM16", &sub_saturate_QSYMM16_QSYMM16_QSYMM16 }, - { "sub_wrap_U8_S16_S16", &sub_wrap_U8_S16_S16 }, - { "sub_wrap_S16_U8_S16", &sub_wrap_S16_U8_S16 }, - { "sub_saturate_U8_S16_S16", &sub_saturate_U8_S16_S16 }, - { "sub_saturate_S16_U8_S16", &sub_saturate_S16_U8_S16 }, - { "sub_wrap_S16_S16_S16", &sub_wrap_S16_S16_S16 }, - { "sub_saturate_S16_S16_S16", &sub_saturate_S16_S16_S16 }, - { "sub_wrap_F32_F32_F32", &sub_F32_F32_F32 }, - { "sub_saturate_F32_F32_F32", &sub_F32_F32_F32 }, - { "sub_wrap_F16_F16_F16", &sub_F16_F16_F16 }, - { "sub_saturate_F16_F16_F16", &sub_F16_F16_F16 }, - }; - _input1 = input1; _input2 = input2; _output = output; + _policy = policy; - std::string function_to_call("sub_"); - function_to_call += policy == ConvertPolicy::WRAP ? "wrap_" : "saturate_"; - function_to_call += string_from_data_type(input1->info()->data_type()) + "_"; - function_to_call += string_from_data_type(input2->info()->data_type()) + "_"; - function_to_call += string_from_data_type(output->info()->data_type()); + const std::pair broadcast_pair = ITensorInfo::broadcast_shape_and_valid_region(*input1->info(), *input2->info()); + const TensorShape &out_shape = broadcast_pair.first; + const ValidRegion &valid_region = broadcast_pair.second; - auto it = map_function.find(function_to_call); + // Auto initialize output if not initialized + set_shape_if_empty(*output->info(), out_shape); - if(it != map_function.end()) + switch(input1->info()->data_type()) { - _func = it->second; + case DataType::U8: + if(input2->info()->data_type() == DataType::U8 && output->info()->data_type() == DataType::U8) + { + _func = &sub_same; + } + else if(input2->info()->data_type() == DataType::U8 && output->info()->data_type() == DataType::S16) + { + _func = &sub_U8_U8_S16; + } + else + { + _func = &sub_U8_S16_S16; + } + break; + case DataType::QASYMM8: + _func = &sub_quantized; + set_data_type_if_unknown(*output->info(), DataType::QASYMM8); + break; + case DataType::QASYMM8_SIGNED: + _func = &sub_quantized; + set_data_type_if_unknown(*output->info(), DataType::QASYMM8_SIGNED); + break; + case DataType::S16: + if(input2->info()->data_type() == DataType::U8) + { + _func = &sub_S16_U8_S16; + } + else + { + _func = &sub_same; + } + set_format_if_unknown(*output->info(), Format::S16); + break; + case DataType::QSYMM16: + _func = &sub_QSYMM16_QSYMM16_QSYMM16; + set_data_type_if_unknown(*output->info(), DataType::QSYMM16); + break; +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + case DataType::F16: + _func = &sub_same; + set_format_if_unknown(*output->info(), Format::F16); + break; +#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ + case DataType::F32: + _func = &sub_same; + set_format_if_unknown(*output->info(), Format::F32); + break; + default: + _func = nullptr; } - INEKernel::configure(win_config.second); + // NEArithmeticSubtractionKernel doesn't need padding so update_window_and_padding() can be skipped + Coordinates coord; + coord.set_num_dimensions(output->info()->num_dimensions()); + output->info()->set_valid_region(valid_region); + Window win = calculate_max_window(valid_region, Steps()); + + INEKernel::configure(win); } Status NEArithmeticSubtractionKernel::validate(const ITensorInfo *input1, const ITensorInfo *input2, const ITensorInfo *output, ConvertPolicy policy) { ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input1, input2, output); - ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(*input1, *input2, *output, policy)); - ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window(*input1->clone(), *input2->clone(), *output->clone()).first); return Status{}; } @@ -647,13 +812,6 @@ void NEArithmeticSubtractionKernel::run(const Window &window, const ThreadInfo & ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INEKernel::window(), window); ARM_COMPUTE_ERROR_ON(_func == nullptr); - (*_func)(_input1, _input2, _output, window); -} - -BorderSize NEArithmeticSubtractionKernel::border_size() const -{ - const unsigned int replicateSize = _output->info()->dimension(0) - std::min(_input1->info()->dimension(0), _input2->info()->dimension(0)); - const unsigned int border = std::min(num_elems_processed_per_iteration - 1U, replicateSize); - return BorderSize{ 0, border, 0, 0 }; + (*_func)(_input1, _input2, _output, window, (_policy == ConvertPolicy::SATURATE)); } -} // namespace arm_compute \ No newline at end of file +} // namespace arm_compute -- cgit v1.2.1