diff options
Diffstat (limited to 'src/cpu/kernels/scale/neon')
| -rw-r--r-- | src/cpu/kernels/scale/neon/fp16.cpp | 276 | ||||
| -rw-r--r-- | src/cpu/kernels/scale/neon/integer.cpp | 783 | ||||
| -rw-r--r-- | src/cpu/kernels/scale/neon/list.h | 617 | ||||
| -rw-r--r-- | src/cpu/kernels/scale/neon/qasymm8.cpp | 406 | ||||
| -rw-r--r-- | src/cpu/kernels/scale/neon/qasymm8_signed.cpp | 394 |
5 files changed, 2476 insertions, 0 deletions
diff --git a/src/cpu/kernels/scale/neon/fp16.cpp b/src/cpu/kernels/scale/neon/fp16.cpp new file mode 100644 index 0000000000..c8a7b7038e --- /dev/null +++ b/src/cpu/kernels/scale/neon/fp16.cpp @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2022-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) + +#include "arm_compute/core/Helpers.h" +#include "arm_compute/core/ITensorPack.h" +#include "arm_compute/core/Window.h" + +#include "src/core/helpers/ScaleHelpers.h" +#include "src/core/NEON/NEMath.h" +#include "src/core/NEON/wrapper/wrapper.h" +#include "src/core/utils/ScaleUtils.h" +#include "src/cpu/kernels/scale/neon/list.h" +#include "support/Rounding.h" + +#include <arm_neon.h> +#include <cmath> +#include <cstddef> + +namespace arm_compute +{ +namespace +{ +void fp16_neon_scale_nearest(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + float sampling_offset, + bool align_corners, + const Window &window) +{ + const size_t in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; + const size_t in_stride_w = src->info()->dimension(1) + src->info()->padding().top + src->info()->padding().bottom; + const size_t in_stride_wc = in_stride_w * in_stride_c; + const size_t in_dim_h = src->info()->dimension(2); + + // Compute the ratio between source height and destination height + const auto hr = scale_utils::calculate_resize_ratio(in_dim_h, dst->info()->dimension(2), align_corners); + const auto window_start_x = static_cast<int32_t>(window.x().start()); + const auto window_end_x = static_cast<int32_t>(window.x().end()); + const int window_step_x = 8; + + Window win(window); + win.set(Window::DimX, Window::Dimension(0, 1, 1)); + Iterator out(dst, win); + + const uint8_t *in_ptr_start = src->buffer() + src->info()->offset_first_element_in_bytes(); + const unsigned int in_stride_bytes_hwc = src->info()->strides_in_bytes()[3]; + + execute_window_loop( + win, + [&](const Coordinates &id) + { + const int32_t offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))) * in_stride_c; + const auto in_hi = static_cast<int>( + align_corners ? utils::rounding::round_half_away_from_zero((id.z() + sampling_offset) * hr) + : std::floor((id.z() + sampling_offset) * hr)); + const int offset_row = in_hi * in_stride_wc; + int32_t x = window_start_x; + const float16_t *in_ptr = reinterpret_cast<const float16_t *>(in_ptr_start + in_stride_bytes_hwc * id[3]); + + for (; x <= window_end_x - window_step_x; x += window_step_x) + { + wrapper::vstore(reinterpret_cast<float16_t *>(out.ptr()) + x, + wrapper::vloadq(in_ptr + offset + offset_row + x)); + } + for (; x < window_end_x; ++x) + { + *(reinterpret_cast<float16_t *>(out.ptr()) + x) = *(in_ptr + offset + offset_row + x); + } + }, + out); +} + +void fp16_neon_scale_bilinear(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + // Compute the ratio between source height and destination height + const auto hr = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + + Iterator out(dst, window); + const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; + const int in_dim_w = src->info()->dimension(1); + const int in_dim_h = src->info()->dimension(2); + const int in_stride_wc = in_stride_c * (in_dim_w + src->info()->padding().top + src->info()->padding().bottom); + + // Don't increment in Y and Z direction for the input tensor + // A pointer to the start of this plane is needed as base for the precomputed offsets + Window win_in(window); + win_in.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in); + + if (border_mode == BorderMode::CONSTANT) + { + using ConstType = typename std::conditional<std::is_same<float16_t, float16_t>::value, half, float16_t>::type; + + const float16_t const_border_value = static_cast<float16_t>(constant_border_value.get<ConstType>()); + execute_window_loop( + window, + [&](const Coordinates &id) + { + const auto offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); + const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); + const float16_t *in_ptr = + reinterpret_cast<const float16_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; + + const auto a00 = + (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value; + const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) + ? *(in_ptr + in_stride_c) + : const_border_value; + const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) + ? *(in_ptr + in_stride_wc) + : const_border_value; + const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) + ? *(in_ptr + in_stride_c + in_stride_wc) + : const_border_value; + + *reinterpret_cast<float16_t *>(out.ptr()) = + static_cast<float16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); + }, + in, out); + } + else if (border_mode == BorderMode::REPLICATE) + { + execute_window_loop( + window, + [&](const Coordinates &id) + { + const auto offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); + const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); + + auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1); + auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1); + auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1); + auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1); + + const auto a00 = *(reinterpret_cast<const float16_t *>(in.ptr()) + clamped_w * in_stride_c + + clamped_h * in_stride_wc); + const auto a01 = *(reinterpret_cast<const float16_t *>(in.ptr()) + clamped_w1 * in_stride_c + + clamped_h * in_stride_wc); + const auto a10 = *(reinterpret_cast<const float16_t *>(in.ptr()) + clamped_w * in_stride_c + + clamped_h1 * in_stride_wc); + const auto a11 = *(reinterpret_cast<const float16_t *>(in.ptr()) + clamped_w1 * in_stride_c + + clamped_h1 * in_stride_wc); + + *reinterpret_cast<float16_t *>(out.ptr()) = + static_cast<float16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); + }, + in, out); + } + else + { + ARM_COMPUTE_ERROR("Not implemented"); + } +} +} // namespace +namespace cpu +{ +#ifdef ENABLE_NCHW_KERNELS +void fp16_bilinear_neon_scale_nchw(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + ARM_COMPUTE_UNUSED(policy); + arm_compute::cpu::scale_bilinear_nchw<float16_t>(src, dst, dx, dy, offsets, border_mode, constant_border_value, + sampling_offset, align_corners, window); +} + +void fp16_nearest_neon_scale_nchw(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + ARM_COMPUTE_UNUSED(policy); + ARM_COMPUTE_UNUSED(border_mode); + arm_compute::cpu::scale_nearest_nchw<float16_t>(src, dst, dx, dy, offsets, constant_border_value, sampling_offset, + align_corners, window); +} +#endif // ENABLE_NCHW_KERNELS +void fp16_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + if (policy == InterpolationPolicy::BILINEAR) + { + fp16_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, + align_corners, window); + } + else if (policy == InterpolationPolicy::NEAREST_NEIGHBOR) + { + fp16_neon_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); + } +} + +void fp16_common_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + arm_compute::cpu::common_neon_scale<float16_t>(src, dst, offsets, dx, dy, policy, border_mode, + constant_border_value, sampling_offset, align_corners, window); +} + +} // namespace cpu +} // namespace arm_compute + +#endif /* defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) */ diff --git a/src/cpu/kernels/scale/neon/integer.cpp b/src/cpu/kernels/scale/neon/integer.cpp new file mode 100644 index 0000000000..bbf92e0412 --- /dev/null +++ b/src/cpu/kernels/scale/neon/integer.cpp @@ -0,0 +1,783 @@ +/* + * Copyright (c) 2021-2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "arm_compute/core/Helpers.h" + +#include "src/core/helpers/ScaleHelpers.h" +#include "src/core/NEON/wrapper/wrapper.h" +#include "src/core/utils/ScaleUtils.h" +#include "support/Rounding.h" + +#include <arm_neon.h> + +namespace arm_compute +{ +namespace +{ +void u8_neon_scale_nearest(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + float sampling_offset, + bool align_corners, + const Window &window) +{ + const size_t in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; + const size_t in_stride_w = src->info()->dimension(1) + src->info()->padding().top + src->info()->padding().bottom; + const size_t in_stride_wc = in_stride_w * in_stride_c; + const size_t in_dim_h = src->info()->dimension(2); + + // Compute the ratio between source height and destination height + const auto hr = scale_utils::calculate_resize_ratio(in_dim_h, dst->info()->dimension(2), align_corners); + const auto window_start_x = static_cast<int32_t>(window.x().start()); + const auto window_end_x = static_cast<int32_t>(window.x().end()); + const int window_step_x = 16; + + Window win(window); + win.set(Window::DimX, Window::Dimension(0, 1, 1)); + Iterator out(dst, win); + + const uint8_t *in_ptr_start = src->buffer() + src->info()->offset_first_element_in_bytes(); + const unsigned int in_stride_bytes_hwc = src->info()->strides_in_bytes()[3]; + + execute_window_loop( + win, + [&](const Coordinates &id) + { + const int32_t offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))) * in_stride_c; + const auto in_hi = static_cast<int>( + align_corners ? utils::rounding::round_half_away_from_zero((id.z() + sampling_offset) * hr) + : std::floor((id.z() + sampling_offset) * hr)); + const int offset_row = in_hi * in_stride_wc; + int32_t x = window_start_x; + const uint8_t *in_ptr = reinterpret_cast<const uint8_t *>(in_ptr_start + in_stride_bytes_hwc * id[3]); + + for (; x <= window_end_x - window_step_x; x += window_step_x) + { + wrapper::vstore(reinterpret_cast<uint8_t *>(out.ptr()) + x, + wrapper::vloadq(in_ptr + offset + offset_row + x)); + } + for (; x < window_end_x; ++x) + { + *(reinterpret_cast<uint8_t *>(out.ptr()) + x) = *(in_ptr + offset + offset_row + x); + } + }, + out); +} + +void u8_neon_scale_bilinear(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + // Compute the ratio between source and destination dimensions + const float scale_x = + scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + + const int input_width = src->info()->dimension(1); + const int input_height = src->info()->dimension(2); + + if (border_mode == BorderMode::CONSTANT) + { + Iterator out(dst, window); + const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; + const int in_stride_wc = + in_stride_c * (input_width + src->info()->padding().top + src->info()->padding().bottom); + + // Don't increment in Y and Z direction for the input tensor + // A pointer to the start of this plane is needed as base for the precomputed offsets + Window win_in(window); + win_in.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in); + + const uint8_t const_border_value = static_cast<uint8_t>(constant_border_value.get<uint8_t>()); + execute_window_loop( + window, + [&](const Coordinates &id) + { + const auto offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); + const int32_t in_hi = std::floor((id.z() + sampling_offset) * scale_y - sampling_offset); + const uint8_t *in_ptr = + reinterpret_cast<const uint8_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; + + const auto a00 = (0 <= offset && offset < input_width && 0 <= in_hi && in_hi < input_height) + ? *in_ptr + : const_border_value; + const auto a01 = (-1 <= offset && offset < input_width - 1 && 0 <= in_hi && in_hi < input_height) + ? *(in_ptr + in_stride_c) + : const_border_value; + const auto a10 = (0 <= offset && offset < input_width && -1 <= in_hi && in_hi < input_height - 1) + ? *(in_ptr + in_stride_wc) + : const_border_value; + const auto a11 = (-1 <= offset && offset < input_width - 1 && -1 <= in_hi && in_hi < input_height - 1) + ? *(in_ptr + in_stride_c + in_stride_wc) + : const_border_value; + + *reinterpret_cast<uint8_t *>(out.ptr()) = + static_cast<uint8_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); + }, + in, out); + } + else if (border_mode == BorderMode::REPLICATE) + { + using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>; + + const int in_stride_x = src->info()->strides_in_bytes()[1]; + const int in_stride_y = src->info()->strides_in_bytes()[2]; + const int in_stride_b = src->info()->strides_in_bytes()[3]; + const int out_stride_x = dst->info()->strides_in_bytes()[1]; + const int out_stride_y = dst->info()->strides_in_bytes()[2]; + const int out_stride_b = dst->info()->strides_in_bytes()[3]; + + const int out_dim_ch = dst->info()->dimension(0); + constexpr int step_cout = 16; + + Window window_execution = window; + window_execution.set(Window::DimX, Window::Dimension(0, 1, 1)); + Window win_in_out(window); + win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in_out); + Iterator out(dst, win_in_out); + + const int xo_start = window_execution[1].start(); + const int xo_end = window_execution[1].end(); + const int xo_step = window_execution[1].step(); + const int yo_start = window_execution[2].start(); + const int yo_end = window_execution[2].end(); + const int yo_step = window_execution[2].step(); + const int bo_start = window_execution[3].start(); + const int bo_end = window_execution[3].end(); + const int bo_step = window_execution[3].step(); + + const float fp_coord_offset_y = sampling_offset * (scale_y - 1); + const float fp_coord_offset_x = sampling_offset * (scale_x - 1); + + for (int bo = bo_start; bo < bo_end; bo += bo_step) + { + const uint8_t *in_ptr = in.ptr() + bo * in_stride_b; + uint8_t *out_ptr = out.ptr() + bo * out_stride_b; + + for (int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + const float yi_f = yo * scale_y + fp_coord_offset_y; + // Integer coordinate + const int yi = static_cast<int>(std::floor(yi_f)); + // Weight for the y coordinate + const float a1 = (yi_f - static_cast<float>(yi)); + const float b1 = (1.f - a1); + + const int yi0 = utility::clamp<int>(yi, 0, input_height - 1); + const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1); + + const uint8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y; + const uint8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; + + uint8_t *out_ptr_yo = out_ptr + yo * out_stride_y; + for (int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + const float xi_f = xo * scale_x + fp_coord_offset_x; + // Integer coordinate + const int xi = static_cast<int>(std::floor(xi_f)); + // Weight for the x coordinate + const float a = (xi_f - static_cast<float>(xi)); + const float b = (1.f - a); + + const float s00_s = b * b1; + const float s01_s = a * b1; + const float s10_s = b * a1; + const float s11_s = a * a1; + + const auto s00 = wrapper::vdup_n(s00_s, ExactTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, ExactTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, ExactTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, ExactTagType{}); + + const int xi0 = utility::clamp<int>(xi, 0, input_width - 1); + const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1); + + const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x; + const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x; + + uint8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; + + int cout = 0; + for (; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); + const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); + const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(uint8_t)); + const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(uint8_t)); + + const uint16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00)); + const uint16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00)); + + const auto in00_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_low))); + const auto in00_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_low))); + const auto in00_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_high))); + const auto in00_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_high))); + + const uint16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01)); + const uint16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01)); + + const auto in01_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_low))); + const auto in01_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_low))); + const auto in01_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_high))); + const auto in01_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_high))); + + const uint16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10)); + const uint16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10)); + + const auto in10_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_low))); + const auto in10_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_low))); + const auto in10_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_high))); + const auto in10_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_high))); + + const uint16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11)); + const uint16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11)); + + const auto in11_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_low))); + const auto in11_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_low))); + const auto in11_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_high))); + const auto in11_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_high))); + + auto out_0 = wrapper::vmul(in00_0, s00); + out_0 = wrapper::vmla(out_0, in01_0, s01); + out_0 = wrapper::vmla(out_0, in10_0, s10); + out_0 = wrapper::vmla(out_0, in11_0, s11); + + auto out_1 = wrapper::vmul(in00_1, s00); + out_1 = wrapper::vmla(out_1, in01_1, s01); + out_1 = wrapper::vmla(out_1, in10_1, s10); + out_1 = wrapper::vmla(out_1, in11_1, s11); + + auto out_2 = wrapper::vmul(in00_2, s00); + out_2 = wrapper::vmla(out_2, in01_2, s01); + out_2 = wrapper::vmla(out_2, in10_2, s10); + out_2 = wrapper::vmla(out_2, in11_2, s11); + + auto out_3 = wrapper::vmul(in00_3, s00); + out_3 = wrapper::vmla(out_3, in01_3, s01); + out_3 = wrapper::vmla(out_3, in10_3, s10); + out_3 = wrapper::vmla(out_3, in11_3, s11); + +#if defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvta<uint32_t>(out_0); + const auto out_1_int = wrapper::vcvta<uint32_t>(out_1); + const auto out_2_int = wrapper::vcvta<uint32_t>(out_2); + const auto out_3_int = wrapper::vcvta<uint32_t>(out_3); +#else // defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvt<uint32_t>(out_0); + const auto out_1_int = wrapper::vcvt<uint32_t>(out_1); + const auto out_2_int = wrapper::vcvt<uint32_t>(out_2); + const auto out_3_int = wrapper::vcvt<uint32_t>(out_3); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + const auto low_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); + + wrapper::vstore(out_ptr_xo_yo + cout * sizeof(uint8_t), out); + } + + for (; cout < out_dim_ch; ++cout) + { + const uint8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); + const uint8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); + const uint8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(uint8_t)); + const uint8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(uint8_t)); + + float out0 = in00 * s00_s; + out0 += in01 * s01_s; + out0 += in10 * s10_s; + out0 += in11 * s11_s; + + // Rounding modes of vector and scalar loops should match +#if defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = static_cast<uint8_t>(std::round(out0)); +#else // defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = static_cast<uint8_t>(out0); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + } + } + } + } + } + else + { + ARM_COMPUTE_ERROR("Not implemented"); + } +} + +void s8_neon_scale_bilinear(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + ARM_COMPUTE_UNUSED(dx, dy, offsets, constant_border_value); + if (border_mode == BorderMode::REPLICATE) + { + using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>; + + // Compute the ratio between source and destination dimensions + const float scale_x = + scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + + const int in_stride_x = src->info()->strides_in_bytes()[1]; + const int in_stride_y = src->info()->strides_in_bytes()[2]; + const int in_stride_b = src->info()->strides_in_bytes()[3]; + const int out_stride_x = dst->info()->strides_in_bytes()[1]; + const int out_stride_y = dst->info()->strides_in_bytes()[2]; + const int out_stride_b = dst->info()->strides_in_bytes()[3]; + const int input_width = src->info()->dimension(1); + const int input_height = src->info()->dimension(2); + const int out_dim_ch = dst->info()->dimension(0); + constexpr int step_cout = 16; + + Window window_execution = window; + window_execution.set(Window::DimX, Window::Dimension(0, 1, 1)); + Window win_in_out(window); + win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in_out); + Iterator out(dst, win_in_out); + + const int xo_start = window_execution[1].start(); + const int xo_end = window_execution[1].end(); + const int xo_step = window_execution[1].step(); + const int yo_start = window_execution[2].start(); + const int yo_end = window_execution[2].end(); + const int yo_step = window_execution[2].step(); + const int bo_start = window_execution[3].start(); + const int bo_end = window_execution[3].end(); + const int bo_step = window_execution[3].step(); + + const float fp_coord_offset_y = sampling_offset * (scale_y - 1); + const float fp_coord_offset_x = sampling_offset * (scale_x - 1); + + for (int bo = bo_start; bo < bo_end; bo += bo_step) + { + const int8_t *in_ptr = reinterpret_cast<int8_t *>(in.ptr() + bo * in_stride_b); + int8_t *out_ptr = reinterpret_cast<int8_t *>(out.ptr() + bo * out_stride_b); + + for (int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + const float yi_f = yo * scale_y + fp_coord_offset_y; + // Integer coordinate + const int yi = static_cast<int>(std::floor(yi_f)); + // Weight for the y coordinate + const float a1 = (yi_f - static_cast<float>(yi)); + const float b1 = (1.f - a1); + + const int yi0 = utility::clamp<int>(yi, 0, input_height - 1); + const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1); + + const int8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y; + const int8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; + + int8_t *out_ptr_yo = out_ptr + yo * out_stride_y; + for (int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + const float xi_f = xo * scale_x + fp_coord_offset_x; + // Integer coordinate + const int xi = static_cast<int>(std::floor(xi_f)); + // Weight for the x coordinate + const float a = (xi_f - static_cast<float>(xi)); + const float b = (1.f - a); + + const float s00_s = b * b1; + const float s01_s = a * b1; + const float s10_s = b * a1; + const float s11_s = a * a1; + + const auto s00 = wrapper::vdup_n(s00_s, ExactTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, ExactTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, ExactTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, ExactTagType{}); + + const int xi0 = utility::clamp<int>(xi, 0, input_width - 1); + const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1); + + const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x; + const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x; + + int8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; + + int cout = 0; + for (; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); + const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); + const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(int8_t)); + const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(int8_t)); + + const int16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00)); + const int16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00)); + + const auto in00_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_low))); + const auto in00_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_low))); + const auto in00_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_high))); + const auto in00_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_high))); + + const int16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01)); + const int16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01)); + + const auto in01_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_low))); + const auto in01_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_low))); + const auto in01_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_high))); + const auto in01_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_high))); + + const int16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10)); + const int16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10)); + + const auto in10_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_low))); + const auto in10_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_low))); + const auto in10_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_high))); + const auto in10_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_high))); + + const int16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11)); + const int16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11)); + + const auto in11_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_low))); + const auto in11_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_low))); + const auto in11_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_high))); + const auto in11_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_high))); + + auto out_0 = wrapper::vmul(in00_0, s00); + out_0 = wrapper::vmla(out_0, in01_0, s01); + out_0 = wrapper::vmla(out_0, in10_0, s10); + out_0 = wrapper::vmla(out_0, in11_0, s11); + + auto out_1 = wrapper::vmul(in00_1, s00); + out_1 = wrapper::vmla(out_1, in01_1, s01); + out_1 = wrapper::vmla(out_1, in10_1, s10); + out_1 = wrapper::vmla(out_1, in11_1, s11); + + auto out_2 = wrapper::vmul(in00_2, s00); + out_2 = wrapper::vmla(out_2, in01_2, s01); + out_2 = wrapper::vmla(out_2, in10_2, s10); + out_2 = wrapper::vmla(out_2, in11_2, s11); + + auto out_3 = wrapper::vmul(in00_3, s00); + out_3 = wrapper::vmla(out_3, in01_3, s01); + out_3 = wrapper::vmla(out_3, in10_3, s10); + out_3 = wrapper::vmla(out_3, in11_3, s11); + +#if defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvta<int32_t>(out_0); + const auto out_1_int = wrapper::vcvta<int32_t>(out_1); + const auto out_2_int = wrapper::vcvta<int32_t>(out_2); + const auto out_3_int = wrapper::vcvta<int32_t>(out_3); +#else // defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvt<int32_t>(out_0); + const auto out_1_int = wrapper::vcvt<int32_t>(out_1); + const auto out_2_int = wrapper::vcvt<int32_t>(out_2); + const auto out_3_int = wrapper::vcvt<int32_t>(out_3); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + const auto low_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); + + wrapper::vstore(out_ptr_xo_yo + cout * sizeof(int8_t), out); + } + + for (; cout < out_dim_ch; ++cout) + { + const int8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); + const int8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); + const int8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(int8_t)); + const int8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(int8_t)); + + float out0 = in00 * s00_s; + out0 += in01 * s01_s; + out0 += in10 * s10_s; + out0 += in11 * s11_s; + + // Rounding modes of vector and scalar loops should match +#if defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(int8_t)) = static_cast<int8_t>(std::round(out0)); +#else // defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(int8_t)) = static_cast<int8_t>(out0); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + } + } + } + } + } + else + { + ARM_COMPUTE_ERROR("Not implemented"); + } +} + +void s16_neon_scale_nearest(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + float sampling_offset, + bool align_corners, + const Window &window) +{ + const size_t in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; + const size_t in_stride_w = src->info()->dimension(1) + src->info()->padding().top + src->info()->padding().bottom; + const size_t in_stride_wc = in_stride_w * in_stride_c; + const size_t in_dim_h = src->info()->dimension(2); + + // Compute the ratio between source height and destination height + const auto hr = scale_utils::calculate_resize_ratio(in_dim_h, dst->info()->dimension(2), align_corners); + const auto window_start_x = static_cast<int32_t>(window.x().start()); + const auto window_end_x = static_cast<int32_t>(window.x().end()); + const int window_step_x = 8; + + Window win(window); + win.set(Window::DimX, Window::Dimension(0, 1, 1)); + Iterator out(dst, win); + + const uint8_t *in_ptr_start = src->buffer() + src->info()->offset_first_element_in_bytes(); + const unsigned int in_stride_bytes_hwc = src->info()->strides_in_bytes()[3]; + + execute_window_loop( + win, + [&](const Coordinates &id) + { + const int32_t offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))) * in_stride_c; + const auto in_hi = static_cast<int>( + align_corners ? utils::rounding::round_half_away_from_zero((id.z() + sampling_offset) * hr) + : std::floor((id.z() + sampling_offset) * hr)); + const int offset_row = in_hi * in_stride_wc; + int32_t x = window_start_x; + const int16_t *in_ptr = reinterpret_cast<const int16_t *>(in_ptr_start + in_stride_bytes_hwc * id[3]); + + for (; x <= window_end_x - window_step_x; x += window_step_x) + { + wrapper::vstore(reinterpret_cast<int16_t *>(out.ptr()) + x, + wrapper::vloadq(in_ptr + offset + offset_row + x)); + } + for (; x < window_end_x; ++x) + { + *(reinterpret_cast<int16_t *>(out.ptr()) + x) = *(in_ptr + offset + offset_row + x); + } + }, + out); +} + +void s16_neon_scale_bilinear(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + // Compute the ratio between source height and destination height + const auto hr = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + + Iterator out(dst, window); + const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; + const int in_dim_w = src->info()->dimension(1); + const int in_dim_h = src->info()->dimension(2); + const int in_stride_wc = in_stride_c * (in_dim_w + src->info()->padding().top + src->info()->padding().bottom); + + // Don't increment in Y and Z direction for the input tensor + // A pointer to the start of this plane is needed as base for the precomputed offsets + Window win_in(window); + win_in.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in); + + if (border_mode == BorderMode::CONSTANT) + { + const int16_t const_border_value = static_cast<int16_t>(constant_border_value.get<int16_t>()); + execute_window_loop( + window, + [&](const Coordinates &id) + { + const auto offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); + const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); + const int16_t *in_ptr = + reinterpret_cast<const int16_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; + + const auto a00 = + (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value; + const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) + ? *(in_ptr + in_stride_c) + : const_border_value; + const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) + ? *(in_ptr + in_stride_wc) + : const_border_value; + const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) + ? *(in_ptr + in_stride_c + in_stride_wc) + : const_border_value; + + *reinterpret_cast<int16_t *>(out.ptr()) = + static_cast<int16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); + }, + in, out); + } + else if (border_mode == BorderMode::REPLICATE) + { + execute_window_loop( + window, + [&](const Coordinates &id) + { + const auto offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); + const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); + + const auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1); + const auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1); + const auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1); + const auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1); + + const auto a00 = + *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc); + const auto a01 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w1 * in_stride_c + + clamped_h * in_stride_wc); + const auto a10 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w * in_stride_c + + clamped_h1 * in_stride_wc); + const auto a11 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w1 * in_stride_c + + clamped_h1 * in_stride_wc); + + *reinterpret_cast<int16_t *>(out.ptr()) = + static_cast<int16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); + }, + in, out); + } + else + { + ARM_COMPUTE_ERROR("Not implemented"); + } +} +} // namespace +namespace cpu +{ +void s8_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + if (policy == InterpolationPolicy::BILINEAR) + { + s8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, + align_corners, window); + } + else + { + ARM_COMPUTE_ERROR("Not implemented"); + } +} + +void u8_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + if (policy == InterpolationPolicy::BILINEAR) + { + u8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, + align_corners, window); + } + else if (policy == InterpolationPolicy::NEAREST_NEIGHBOR) + { + u8_neon_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); + } +} + +void s16_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + if (policy == InterpolationPolicy::BILINEAR) + { + s16_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, + align_corners, window); + } + else if (policy == InterpolationPolicy::NEAREST_NEIGHBOR) + { + s16_neon_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); + } +} +} // namespace cpu +} // namespace arm_compute diff --git a/src/cpu/kernels/scale/neon/list.h b/src/cpu/kernels/scale/neon/list.h new file mode 100644 index 0000000000..153dc67c3d --- /dev/null +++ b/src/cpu/kernels/scale/neon/list.h @@ -0,0 +1,617 @@ +/* + * Copyright (c) 2021-2023 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ACL_SRC_CPU_KERNELS_SCALE_NEON_LIST_H +#define ACL_SRC_CPU_KERNELS_SCALE_NEON_LIST_H + +#include "arm_compute/core/Helpers.h" +#include "arm_compute/core/Window.h" + +#include "src/core/NEON/wrapper/wrapper.h" +#include "src/core/utils/ScaleUtils.h" +#include "support/Rounding.h" + +namespace arm_compute +{ +namespace cpu +{ +#define DECLARE_SCALE_KERNEL(func_name) \ + void func_name(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, \ + InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, \ + float sampling_offset, bool align_corners, const Window &window) + +DECLARE_SCALE_KERNEL(s16_neon_scale); +DECLARE_SCALE_KERNEL(u8_neon_scale); +DECLARE_SCALE_KERNEL(s8_neon_scale); +DECLARE_SCALE_KERNEL(qasymm8_neon_scale); +DECLARE_SCALE_KERNEL(qasymm8_signed_neon_scale); +DECLARE_SCALE_KERNEL(fp16_common_neon_scale); +DECLARE_SCALE_KERNEL(fp16_bilinear_neon_scale_nchw); +DECLARE_SCALE_KERNEL(fp16_nearest_neon_scale_nchw); + +#undef DECLARE_SCALE_KERNEL + +#ifdef ENABLE_NCHW_KERNELS +template <typename T> +void scale_nearest_nchw(const ITensor *src, + ITensor *dst, + const ITensor *dx, + const ITensor *dy, + const ITensor *offsets, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + ARM_COMPUTE_UNUSED(dx, dy); + ARM_COMPUTE_UNUSED(constant_border_value); + const size_t in_stride_x = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; + + // Compute the ratio between source height and destination height + const auto hr = + scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + + // Don't increment in X and Y direction for the input tensor + // A pointer to the start of this plane is needed as base for the precomputed offsets + Window win_in(window); + win_in.set(Window::DimX, Window::Dimension(0, 0, 0)); + win_in.set(Window::DimY, Window::Dimension(0, 0, 0)); + + // Set offsets window + Window win_off; + win_off.set(Window::DimX, window[Window::DimX]); + win_off.set(Window::DimY, window[Window::DimY]); + for (size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) + { + win_off.set(d, Window::Dimension(0, 0, 0)); + } + + // Create iterators + Iterator src_i(src, win_in); + Iterator dst_i(dst, window); + Iterator offsets_i(offsets, win_off); + execute_window_loop( + window, + [&](const Coordinates &id) + { + const auto offsets_ptr = reinterpret_cast<const int32_t *>(offsets_i.ptr()); + const auto in_yi = static_cast<int32_t>( + align_corners ? utils::rounding::round_half_away_from_zero((id.y() + sampling_offset) * hr) + : std::floor((id.y() + sampling_offset) * hr)); + const int32_t offset_row = in_yi * in_stride_x; + *reinterpret_cast<T *>(dst_i.ptr()) = + *(reinterpret_cast<const T *>(src_i.ptr()) + offsets_ptr[0] + offset_row); + }, + src_i, offsets_i, dst_i); +} + +template <typename T> +void scale_bilinear_nchw(const ITensor *src, + ITensor *dst, + const ITensor *dx, + const ITensor *dy, + const ITensor *offsets, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + // Compute the ratio between source height and destination height + const auto hr = + scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + Window win_off; + win_off.set(Window::DimX, window.x()); + win_off.set(Window::DimY, window.y()); + + // Don't increment in X and Y direction for the input tensor + // A pointer to the start of this plane is needed as base for the precomputed offsets + Window win_in(window); + win_in.set(Window::DimX, Window::Dimension(0, 0, 0)); + win_in.set(Window::DimY, Window::Dimension(0, 0, 0)); + + for (size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) + { + win_off.set(d, Window::Dimension(0, 0, 0)); + } + + Iterator src_i(src, win_in); + Iterator dst_i(dst, window); + Iterator offsets_i(offsets, win_off); + Iterator dx_i(dx, win_off); + Iterator dy_i(dy, win_off); + + const int32_t in_dim_w = src->info()->dimension(0); + const int32_t in_dim_h = src->info()->dimension(1); + const int32_t in_stride_w = in_dim_w + src->info()->padding().left + src->info()->padding().right; + + if (border_mode == BorderMode::CONSTANT) + { +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + using ConstType = typename std::conditional<std::is_same<T, float16_t>::value, half, T>::type; +#else /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ + using ConstType = T; +#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ + const T const_border_value = static_cast<T>(constant_border_value.get<ConstType>()); + execute_window_loop( + window, + [&](const Coordinates &id) + { + const int32_t index_h = std::floor((id.y() + sampling_offset) * hr - sampling_offset); + const auto index_w = *(reinterpret_cast<const int32_t *>(offsets_i.ptr())); + const auto dx_val = *(reinterpret_cast<const float *>(dx_i.ptr())); + const auto dy_val = *(reinterpret_cast<const float *>(dy_i.ptr())); + const auto pixel_row_ptr = reinterpret_cast<const T *>(src_i.ptr()); + + const auto a00 = (0 <= index_w && index_w < in_dim_w && 0 <= index_h && index_h < in_dim_h) + ? (*(pixel_row_ptr + index_w + index_h * in_stride_w)) + : const_border_value; + const auto a01 = (-1 <= index_w && index_w < in_dim_w - 1 && 0 <= index_h && index_h < in_dim_h) + ? (*(pixel_row_ptr + index_w + 1 + index_h * in_stride_w)) + : const_border_value; + const auto a10 = (0 <= index_w && index_w < in_dim_w && -1 <= index_h && index_h < in_dim_h - 1) + ? (*(pixel_row_ptr + index_w + index_h * in_stride_w + in_stride_w)) + : const_border_value; + const auto a11 = (-1 <= index_w && index_w < in_dim_w - 1 && -1 <= index_h && index_h < in_dim_h - 1) + ? (*(pixel_row_ptr + index_w + 1 + index_h * in_stride_w + in_stride_w)) + : const_border_value; + + *reinterpret_cast<T *>(dst_i.ptr()) = + static_cast<T>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); + }, + src_i, offsets_i, dx_i, dy_i, dst_i); + } + else if (border_mode == BorderMode::REPLICATE) + { + execute_window_loop( + window, + [&](const Coordinates &id) + { + const int index_h = std::floor((id.y() + sampling_offset) * hr - sampling_offset); + const auto index_w = *(reinterpret_cast<const int32_t *>(offsets_i.ptr())); + const auto dx_val = *(reinterpret_cast<const float *>(dx_i.ptr())); + const auto dy_val = *(reinterpret_cast<const float *>(dy_i.ptr())); + const auto pixel_row_ptr = reinterpret_cast<const T *>(src_i.ptr()); + + auto clamped_x = utility::clamp<int>(index_w, 0, in_dim_w - 1); + auto clamped_x1 = utility::clamp<int>(index_w + 1, 0, in_dim_w - 1); + auto clamped_y = utility::clamp<int>(index_h, 0, in_dim_h - 1); + auto clamped_y1 = utility::clamp<int>(index_h + 1, 0, in_dim_h - 1); + + const auto a00 = *(pixel_row_ptr + clamped_x + clamped_y * in_stride_w); + const auto a01 = *(pixel_row_ptr + clamped_x1 + clamped_y * in_stride_w); + const auto a10 = *(pixel_row_ptr + clamped_x + clamped_y1 * in_stride_w); + const auto a11 = *(pixel_row_ptr + clamped_x1 + clamped_y1 * in_stride_w); + + *reinterpret_cast<T *>(dst_i.ptr()) = + static_cast<T>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); + }, + src_i, offsets_i, dx_i, dy_i, dst_i); + } + else + { + ARM_COMPUTE_ERROR("Not implemented"); + } +} +#endif // ENABLE_NCHW_KERNELS + +template <typename T> +void nearest_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + float sampling_offset, + bool align_corners, + const Window &window) +{ + ARM_COMPUTE_UNUSED(offsets); + + // Compute the ratio between source and destination dimensions + const float scale_x = + scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + + const int in_stride_y = src->info()->strides_in_bytes()[1]; + const int in_stride_z = src->info()->strides_in_bytes()[2]; + const int in_stride_w = src->info()->strides_in_bytes()[3]; + const int out_stride_y = dst->info()->strides_in_bytes()[1]; + const int out_stride_z = dst->info()->strides_in_bytes()[2]; + const int out_stride_w = dst->info()->strides_in_bytes()[3]; + const int out_dim_ch = dst->info()->dimension(0); + const int step_cout = 16 / sizeof(T); + + Window window_execution = window; + window_execution.set(Window::DimX, Window::Dimension(0, 1, 1)); + Window win_in_out(window); + win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in_out); + Iterator out(dst, win_in_out); + + const int xo_start = window_execution.y().start(); + const int xo_end = window_execution.y().end(); + const int xo_step = window_execution.y().step(); + const int yo_start = window_execution.z().start(); + const int yo_end = window_execution.z().end(); + const int yo_step = window_execution.z().step(); + const int bo_start = window_execution[3].start(); + const int bo_end = window_execution[3].end(); + const int bo_step = window_execution[3].step(); + + for (int bo = bo_start; bo < bo_end; bo += bo_step) + { + const uint8_t *in_ptr_base = in.ptr() + bo * in_stride_w; + uint8_t *out_ptr_base = out.ptr() + bo * out_stride_w; + + for (int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + float yi_f = ((yo + sampling_offset) * scale_y); + int yi = 0; + if (align_corners) + { + yi = utils::rounding::round_half_away_from_zero(yi_f); + } + else + { + yi = static_cast<int>(std::floor(yi_f)); + } + + for (int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + float xi_f = ((xo + sampling_offset) * scale_x); + int xi = 0; + if (align_corners) + { + xi = utils::rounding::round_half_away_from_zero(xi_f); + } + else + { + xi = static_cast<int>(std::floor(xi_f)); + } + + const uint8_t *in_ptr = in_ptr_base + xi * in_stride_y + yi * in_stride_z; + uint8_t *out_ptr = out_ptr_base + xo * out_stride_y + yo * out_stride_z; + + int cout = 0; + for (; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + auto out0 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T))); + wrapper::vstore(reinterpret_cast<T *>(out_ptr + cout * sizeof(T)), out0); + } + + for (; cout < out_dim_ch; ++cout) + { + auto out0 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T))); + *(reinterpret_cast<T *>(out_ptr + cout * sizeof(T))) = out0; + } + } + } + } +} + +template <typename T> +void bilinear_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + ARM_COMPUTE_UNUSED(offsets); + ARM_COMPUTE_UNUSED(dx); + ARM_COMPUTE_UNUSED(dy); + using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<T, wrapper::traits::BitWidth::W128>; + + // Compute the ratio between source and destination dimensions + const float scale_x = + scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + + const int in_stride_y = src->info()->strides_in_bytes()[1]; + const int in_stride_z = src->info()->strides_in_bytes()[2]; + const int in_stride_w = src->info()->strides_in_bytes()[3]; + const int out_stride_y = dst->info()->strides_in_bytes()[1]; + const int out_stride_z = dst->info()->strides_in_bytes()[2]; + const int out_stride_w = dst->info()->strides_in_bytes()[3]; + const int in_dim_w = src->info()->dimension(1); + const int in_dim_h = src->info()->dimension(2); + const int out_dim_ch = dst->info()->dimension(0); + const int step_cout = 16 / sizeof(T); + + Window window_execution = window; + window_execution.set(Window::DimX, Window::Dimension(0, 1, 1)); + Window win_in_out(window); + win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in_out); + Iterator out(dst, win_in_out); + + const int xo_start = window_execution.y().start(); + const int xo_end = window_execution.y().end(); + const int xo_step = window_execution.y().step(); + const int yo_start = window_execution.z().start(); + const int yo_end = window_execution.z().end(); + const int yo_step = window_execution.z().step(); + const int bo_start = window_execution[3].start(); + const int bo_end = window_execution[3].end(); + const int bo_step = window_execution[3].step(); + + if (border_mode == BorderMode::CONSTANT) + { +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + using ConstType = typename std::conditional<std::is_same<T, float16_t>::value, half, T>::type; +#else /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ + using ConstType = T; +#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ + const T const_border_value = static_cast<T>(constant_border_value.get<ConstType>()); + + for (int bo = bo_start; bo < bo_end; bo += bo_step) + { + const uint8_t *in_ptr_base = in.ptr() + bo * in_stride_w; + uint8_t *out_ptr_base = out.ptr() + bo * out_stride_w; + + for (int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + const float yi_f = ((yo + sampling_offset) * scale_y - sampling_offset); + // Integer coordinate + const auto yi = static_cast<int>(std::floor(yi_f)); + // Weight for the y coordinate + const auto a1 = (yi_f - static_cast<float>(yi)); + const auto b1 = (1.f - a1); + + for (int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + const float xi_f = ((xo + sampling_offset) * scale_x - sampling_offset); + // Integer coordinate + const auto xi = static_cast<int>(std::floor(xi_f)); + // Weight for the x coordinate + const auto a = (xi_f - static_cast<float>(xi)); + const auto b = (1.f - a); + + const auto s00_s = static_cast<T>(b * b1); + const auto s01_s = static_cast<T>(a * b1); + const auto s10_s = static_cast<T>(b * a1); + const auto s11_s = static_cast<T>(a * a1); + + const uint8_t *in_ptr = in_ptr_base + xi * in_stride_y + yi * in_stride_z; + uint8_t *out_ptr = out_ptr_base + xo * out_stride_y + yo * out_stride_z; + + int cout = 0; + for (; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + auto in00 = wrapper::vdup_n(static_cast<T>(const_border_value), ExactTagType{}); + auto in01 = wrapper::vdup_n(static_cast<T>(const_border_value), ExactTagType{}); + auto in10 = wrapper::vdup_n(static_cast<T>(const_border_value), ExactTagType{}); + auto in11 = wrapper::vdup_n(static_cast<T>(const_border_value), ExactTagType{}); + if ((yi >= 0) && (yi < in_dim_h)) + { + if ((xi >= 0) && (xi < in_dim_w)) + { + in00 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T))); + } + if (((xi + 1) >= 0) && ((xi + 1) < in_dim_w)) + { + in01 = wrapper::vloadq( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y)); + } + } + if (((yi + 1) >= 0) && ((yi + 1) < in_dim_h)) + { + if ((xi >= 0) && (xi < in_dim_w)) + { + in10 = wrapper::vloadq( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_z)); + } + if (((xi + 1) >= 0) && ((xi + 1) < in_dim_w)) + { + in11 = wrapper::vloadq( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y + in_stride_z)); + } + } + + const auto s00 = wrapper::vdup_n(s00_s, ExactTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, ExactTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, ExactTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, ExactTagType{}); + auto out0 = wrapper::vdup_n(static_cast<T>(0), ExactTagType{}); + out0 = wrapper::vmla(out0, in00, s00); + out0 = wrapper::vmla(out0, in01, s01); + out0 = wrapper::vmla(out0, in10, s10); + out0 = wrapper::vmla(out0, in11, s11); + wrapper::vstore(reinterpret_cast<T *>(out_ptr + cout * sizeof(T)), out0); + } + + for (; cout < out_dim_ch; ++cout) + { + auto in00 = static_cast<T>(const_border_value); + auto in01 = static_cast<T>(const_border_value); + auto in10 = static_cast<T>(const_border_value); + auto in11 = static_cast<T>(const_border_value); + if ((yi >= 0) && (yi < in_dim_h)) + { + if ((xi >= 0) && (xi < in_dim_w)) + { + in00 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T))); + } + if (((xi + 1) >= 0) && ((xi + 1) < in_dim_w)) + { + in01 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y)); + } + } + if (((yi + 1) >= 0) && ((yi + 1) < in_dim_h)) + { + if ((xi >= 0) && (xi < in_dim_w)) + { + in10 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_z)); + } + if (((xi + 1) >= 0) && ((xi + 1) < in_dim_w)) + { + in11 = *( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y + in_stride_z)); + } + } + auto out0 = static_cast<T>(0); + out0 += in00 * s00_s; + out0 += in01 * s01_s; + out0 += in10 * s10_s; + out0 += in11 * s11_s; + *(reinterpret_cast<T *>(out_ptr + cout * sizeof(T))) = out0; + } + } + } + } + } + else if (border_mode == BorderMode::REPLICATE) + { + for (int bo = bo_start; bo < bo_end; bo += bo_step) + { + const uint8_t *in_ptr = in.ptr() + bo * in_stride_w; + uint8_t *out_ptr = out.ptr() + bo * out_stride_w; + + for (int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + const float yi_f = ((yo + sampling_offset) * scale_y - sampling_offset); + // Integer coordinate + const auto yi = static_cast<int>(std::floor(yi_f)); + // Weight for the y coordinate + const auto a1 = (yi_f - static_cast<float>(yi)); + const auto b1 = (1.f - a1); + + const int yi0 = utility::clamp<int>(yi, 0, in_dim_h - 1); + const int yi1 = utility::clamp<int>(yi + 1, 0, in_dim_h - 1); + + const int yi0_offset = yi0 * in_stride_z; + const int yi1_offset = yi1 * in_stride_z; + + const int y_offset = yo * out_stride_z; + for (int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + const float xi_f = ((xo + sampling_offset) * scale_x - sampling_offset); + // Integer coordinate + const auto xi = static_cast<int>(std::floor(xi_f)); + // Weight for the x coordinate + const auto a = (xi_f - static_cast<float>(xi)); + const auto b = (1.f - a); + + const auto s00_s = static_cast<T>(b * b1); + const auto s01_s = static_cast<T>(a * b1); + const auto s10_s = static_cast<T>(b * a1); + const auto s11_s = static_cast<T>(a * a1); + + const auto s00 = wrapper::vdup_n(s00_s, ExactTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, ExactTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, ExactTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, ExactTagType{}); + + const int xi0 = utility::clamp<int>(xi, 0, in_dim_w - 1); + const int xi1 = utility::clamp<int>(xi + 1, 0, in_dim_w - 1); + + const int xi0_offset = xi0 * in_stride_y; + const int xi1_offset = xi1 * in_stride_y; + + const int offset = xo * out_stride_y + y_offset; + + int cout = 0; + for (; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + const auto in00 = wrapper::vloadq( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); + const auto in01 = wrapper::vloadq( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); + const auto in10 = wrapper::vloadq( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); + const auto in11 = wrapper::vloadq( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); + + auto out0 = wrapper::vmul(in00, s00); + out0 = wrapper::vmla(out0, in01, s01); + out0 = wrapper::vmla(out0, in10, s10); + out0 = wrapper::vmla(out0, in11, s11); + wrapper::vstore(reinterpret_cast<T *>(out_ptr + offset + cout * sizeof(T)), out0); + } + + for (; cout < out_dim_ch; ++cout) + { + const T in00 = + *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); + const T in01 = + *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); + const T in10 = + *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); + const T in11 = + *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); + + T out0 = in00 * s00_s; + out0 += in01 * s01_s; + out0 += in10 * s10_s; + out0 += in11 * s11_s; + *(reinterpret_cast<T *>(out_ptr + offset + cout * sizeof(T))) = out0; + } + } + } + } + } + else + { + ARM_COMPUTE_ERROR("Not implemented"); + } +} + +template <typename T> +void common_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + if (policy == InterpolationPolicy::BILINEAR) + { + bilinear_neon_scale<T>(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, + align_corners, window); + } + else if (policy == InterpolationPolicy::NEAREST_NEIGHBOR) + { + nearest_neon_scale<T>(src, dst, offsets, sampling_offset, align_corners, window); + } +} +} // namespace cpu +} // namespace arm_compute + +#endif // ACL_SRC_CPU_KERNELS_SCALE_NEON_LIST_H diff --git a/src/cpu/kernels/scale/neon/qasymm8.cpp b/src/cpu/kernels/scale/neon/qasymm8.cpp new file mode 100644 index 0000000000..62a821daa5 --- /dev/null +++ b/src/cpu/kernels/scale/neon/qasymm8.cpp @@ -0,0 +1,406 @@ +/* + * Copyright (c) 2021-2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "src/core/helpers/ScaleHelpers.h" +#include "src/cpu/kernels/scale/neon/list.h" + +namespace arm_compute +{ +namespace +{ +void qasymm8_neon_scale_bilinear(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + // Data layout is NHWC + const int32_t input_width = src->info()->dimension(1); + const int32_t input_height = src->info()->dimension(2); + + const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); + const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); + + // Compute the ratio between source and destination dimensions + const float scale_x = + scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + + if (border_mode == BorderMode::CONSTANT) + { + const int32_t in_stride_y = src->info()->strides_in_bytes()[1]; + const int32_t in_stride_z = src->info()->strides_in_bytes()[2]; + + // Compute the ratio between source height and destination height + Window win_off; + win_off.set(Window::DimX, Window::Dimension(0, 0, 0)); + win_off.set(Window::DimY, Window::Dimension(0, 0, 0)); + + // Don't increment in X and Y direction for the input tensor + // A pointer to the start of this plane is needed as base for the precomputed offsets + Window win_in(window); + win_in.set(1, Window::Dimension(0, 0, 0)); + win_in.set(2, Window::Dimension(0, 0, 0)); + + for (size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) + { + win_off.set(d, Window::Dimension(0, 0, 0)); + } + + Iterator in(src, win_in); + Iterator out(dst, window); + + const uint8_t const_border_value = static_cast<uint8_t>(constant_border_value.get<uint8_t>()); + execute_window_loop( + window, + [&](const Coordinates &id) + { + const int32_t index_h = std::floor((id[2] + sampling_offset) * scale_y - sampling_offset); + const int32_t index_w = + *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2])))); + const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2])))); + const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2])))); + const auto pixel_row_ptr = reinterpret_cast<const uint8_t *>(in.ptr()); + + const auto a00 = (0 <= index_w && index_w < input_width && 0 <= index_h && index_h < input_height) + ? (*(pixel_row_ptr + index_w * in_stride_y + index_h * in_stride_z)) + : const_border_value; + const auto a01 = (-1 <= index_w && index_w + 1 < input_width && 0 <= index_h && index_h < input_height) + ? (*(pixel_row_ptr + (index_w + 1) * in_stride_y + index_h * in_stride_z)) + : const_border_value; + const auto a10 = (0 <= index_w && index_w < input_width && -1 <= index_h && index_h < input_height - 1) + ? (*(pixel_row_ptr + index_w * in_stride_y + (index_h + 1) * in_stride_z)) + : const_border_value; + const auto a11 = + (-1 <= index_w && index_w < input_width - 1 && -1 <= index_h && index_h < input_height - 1) + ? (*(pixel_row_ptr + (index_w + 1) * in_stride_y + (index_h + 1) * in_stride_z)) + : const_border_value; + + const float inp00 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a00, iq_info); + const float inp01 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a01, iq_info); + const float inp10 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a10, iq_info); + const float inp11 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a11, iq_info); + *reinterpret_cast<uint8_t *>(out.ptr()) = Qasymm8QuantizationHelper<uint8_t>::quantize( + scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); + }, + in, out); + } + else if (border_mode == BorderMode::REPLICATE) + { + using FloatTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>; + using Int32TagType = typename wrapper::traits::neon_bitvector_tag_t<int32_t, wrapper::traits::BitWidth::W128>; + + const int in_stride_x = src->info()->strides_in_bytes()[1]; + const int in_stride_y = src->info()->strides_in_bytes()[2]; + const int in_stride_b = src->info()->strides_in_bytes()[3]; + const int out_stride_x = dst->info()->strides_in_bytes()[1]; + const int out_stride_y = dst->info()->strides_in_bytes()[2]; + const int out_stride_b = dst->info()->strides_in_bytes()[3]; + const int out_dim_ch = dst->info()->dimension(0); + constexpr int step_cout = 16; + + Window window_execution = window; + window_execution.set(Window::DimX, Window::Dimension(0, 1, 1)); + Window win_in_out(window); + win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in_out); + Iterator out(dst, win_in_out); + + const int xo_start = window_execution[1].start(); + const int xo_end = window_execution[1].end(); + const int xo_step = window_execution[1].step(); + const int yo_start = window_execution[2].start(); + const int yo_end = window_execution[2].end(); + const int yo_step = window_execution[2].step(); + const int bo_start = window_execution[3].start(); + const int bo_end = window_execution[3].end(); + const int bo_step = window_execution[3].step(); + + const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); + const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); + + const float32x4_t vscale_in = wrapper::vdup_n(iq_info.scale, FloatTagType{}); + const int32x4_t voffset_in = wrapper::vdup_n(iq_info.offset, Int32TagType{}); // Offsets will be Int32 + + const float32x4_t invvscale_o = wrapper::vdup_n(1.f / oq_info.scale, FloatTagType{}); + const float32x4_t voffset_o = vdupq_n_f32(oq_info.offset); + + const float fp_coord_offset_y = sampling_offset * (scale_y - 1); + const float fp_coord_offset_x = sampling_offset * (scale_x - 1); + + for (int bo = bo_start; bo < bo_end; bo += bo_step) + { + const uint8_t *in_ptr = in.ptr() + bo * in_stride_b; + uint8_t *out_ptr = out.ptr() + bo * out_stride_b; + + for (int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + const float yi_f = yo * scale_y + fp_coord_offset_y; + // Integer coordinate + const int yi = static_cast<int>(std::floor(yi_f)); + // Weight for the y coordinate + const float a1 = (yi_f - static_cast<float>(yi)); + const float b1 = (1.f - a1); + + const int yi0 = utility::clamp<int>(yi, 0, input_height - 1); + const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1); + + const uint8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y; + const uint8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; + + uint8_t *out_ptr_yo = out_ptr + yo * out_stride_y; + for (int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + const float xi_f = xo * scale_x + fp_coord_offset_x; + // Integer coordinate + const int xi = static_cast<int>(std::floor(xi_f)); + // Weight for the x coordinate + const float a = (xi_f - static_cast<float>(xi)); + const float b = (1.f - a); + + const float s00_s = b * b1; + const float s01_s = a * b1; + const float s10_s = b * a1; + const float s11_s = a * a1; + + const auto s00 = wrapper::vdup_n(s00_s, FloatTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, FloatTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, FloatTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, FloatTagType{}); + + const int xi0 = utility::clamp<int>(xi, 0, input_width - 1); + const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1); + + const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x; + const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x; + + uint8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; + + int cout = 0; + for (; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); + const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); + const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(uint8_t)); + const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(uint8_t)); + + const uint16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00)); + const uint16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00)); + + const auto in00_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in00_low))), voffset_in)), + vscale_in); + const auto in00_1 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in00_low))), voffset_in)), + vscale_in); + const auto in00_2 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in00_high))), voffset_in)), + vscale_in); + const auto in00_3 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in00_high))), voffset_in)), + vscale_in); + + const uint16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01)); + const uint16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01)); + + const auto in01_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in01_low))), voffset_in)), + vscale_in); + const auto in01_1 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in01_low))), voffset_in)), + vscale_in); + const auto in01_2 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in01_high))), voffset_in)), + vscale_in); + const auto in01_3 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in01_high))), voffset_in)), + vscale_in); + + const uint16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10)); + const uint16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10)); + + const auto in10_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in10_low))), voffset_in)), + vscale_in); + const auto in10_1 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in10_low))), voffset_in)), + vscale_in); + const auto in10_2 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in10_high))), voffset_in)), + vscale_in); + const auto in10_3 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in10_high))), voffset_in)), + vscale_in); + + const uint16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11)); + const uint16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11)); + + const auto in11_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in11_low))), voffset_in)), + vscale_in); + const auto in11_1 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in11_low))), voffset_in)), + vscale_in); + const auto in11_2 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in11_high))), voffset_in)), + vscale_in); + const auto in11_3 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in11_high))), voffset_in)), + vscale_in); + + auto out_0 = wrapper::vmul(in00_0, s00); + out_0 = wrapper::vmla(out_0, in01_0, s01); + out_0 = wrapper::vmla(out_0, in10_0, s10); + out_0 = wrapper::vmla(out_0, in11_0, s11); + + auto out_1 = wrapper::vmul(in00_1, s00); + out_1 = wrapper::vmla(out_1, in01_1, s01); + out_1 = wrapper::vmla(out_1, in10_1, s10); + out_1 = wrapper::vmla(out_1, in11_1, s11); + + auto out_2 = wrapper::vmul(in00_2, s00); + out_2 = wrapper::vmla(out_2, in01_2, s01); + out_2 = wrapper::vmla(out_2, in10_2, s10); + out_2 = wrapper::vmla(out_2, in11_2, s11); + + auto out_3 = wrapper::vmul(in00_3, s00); + out_3 = wrapper::vmla(out_3, in01_3, s01); + out_3 = wrapper::vmla(out_3, in10_3, s10); + out_3 = wrapper::vmla(out_3, in11_3, s11); + +#if defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o)); + const auto out_1_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o)); + const auto out_2_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o)); + const auto out_3_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o)); +#else // defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o)); + const auto out_1_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o)); + const auto out_2_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o)); + const auto out_3_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o)); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + const auto low_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); + + wrapper::vstore(out_ptr_xo_yo + cout * sizeof(uint8_t), out); + } + + for (; cout < out_dim_ch; ++cout) + { + const uint8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); + const uint8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); + const uint8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(uint8_t)); + const uint8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(uint8_t)); + + const float in00_f = (static_cast<int32_t>(in00) - iq_info.offset) * iq_info.scale; + const float in01_f = (static_cast<int32_t>(in01) - iq_info.offset) * iq_info.scale; + const float in10_f = (static_cast<int32_t>(in10) - iq_info.offset) * iq_info.scale; + const float in11_f = (static_cast<int32_t>(in11) - iq_info.offset) * iq_info.scale; + + float out = in00_f * s00_s; + out += in01_f * s01_s; + out += in10_f * s10_s; + out += in11_f * s11_s; + + // Rounding modes of vector and scalar loops should match +#if defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = quantize_qasymm8(out, oq_info); +#else // defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = + quantize_qasymm8(out, oq_info, RoundingPolicy::TO_ZERO); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + } + } + } + } + } + else + { + ARM_COMPUTE_ERROR("Not implemented"); + } +} +} // namespace +namespace cpu +{ +void qasymm8_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + if (policy == InterpolationPolicy::BILINEAR) + { + if (src->info()->quantization_info() == dst->info()->quantization_info()) + { + u8_neon_scale(src, dst, offsets, dx, dy, policy, border_mode, constant_border_value, sampling_offset, + align_corners, window); + } + else + { + qasymm8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, + align_corners, window); + } + } + else if (policy == InterpolationPolicy::NEAREST_NEIGHBOR) + { + nearest_neon_scale<uint8_t>(src, dst, offsets, sampling_offset, align_corners, window); + } +} +} // namespace cpu +} // namespace arm_compute diff --git a/src/cpu/kernels/scale/neon/qasymm8_signed.cpp b/src/cpu/kernels/scale/neon/qasymm8_signed.cpp new file mode 100644 index 0000000000..5a885178a7 --- /dev/null +++ b/src/cpu/kernels/scale/neon/qasymm8_signed.cpp @@ -0,0 +1,394 @@ +/* + * Copyright (c) 2021-2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "src/core/helpers/ScaleHelpers.h" +#include "src/cpu/kernels/scale/neon/list.h" + +namespace arm_compute +{ +namespace +{ +void qasymm8_signed_neon_scale_bilinear(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + // Data layout is NHWC + const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); + const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); + + const int32_t input_width = src->info()->dimension(1); + const int32_t input_height = src->info()->dimension(2); + + // Compute the ratio between source and destination dimensions + const float scale_x = + scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + + if (border_mode == BorderMode::CONSTANT) + { + const int32_t in_stride_y = src->info()->strides_in_bytes()[1]; + const int32_t in_stride_z = src->info()->strides_in_bytes()[2]; + + Window win_off; + win_off.set(Window::DimX, Window::Dimension(0, 0, 0)); + win_off.set(Window::DimY, Window::Dimension(0, 0, 0)); + + // Don't increment in X and Y direction for the input tensor + // A pointer to the start of this plane is needed as base for the precomputed offsets + Window win_in(window); + win_in.set(1, Window::Dimension(0, 0, 0)); + win_in.set(2, Window::Dimension(0, 0, 0)); + + for (size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) + { + win_off.set(d, Window::Dimension(0, 0, 0)); + } + + Iterator in(src, win_in); + Iterator out(dst, window); + + const int8_t const_border_value = static_cast<int8_t>(constant_border_value.get<int8_t>()); + execute_window_loop( + window, + [&](const Coordinates &id) + { + const int32_t index_h = std::floor((id[2] + sampling_offset) * scale_y - sampling_offset); + const int32_t index_w = + *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2])))); + const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2])))); + const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2])))); + const auto pixel_row_ptr = reinterpret_cast<const int8_t *>(in.ptr()); + + const auto a00 = (0 <= index_w && index_w < input_width && 0 <= index_h && index_h < input_height) + ? (*(pixel_row_ptr + index_w * in_stride_y + index_h * in_stride_z)) + : const_border_value; + const auto a01 = (-1 <= index_w && index_w + 1 < input_width && 0 <= index_h && index_h < input_height) + ? (*(pixel_row_ptr + (index_w + 1) * in_stride_y + index_h * in_stride_z)) + : const_border_value; + const auto a10 = (0 <= index_w && index_w < input_width && -1 <= index_h && index_h < input_height - 1) + ? (*(pixel_row_ptr + index_w * in_stride_y + (index_h + 1) * in_stride_z)) + : const_border_value; + const auto a11 = + (-1 <= index_w && index_w < input_width - 1 && -1 <= index_h && index_h < input_height - 1) + ? (*(pixel_row_ptr + (index_w + 1) * in_stride_y + (index_h + 1) * in_stride_z)) + : const_border_value; + + const float inp00 = Qasymm8QuantizationHelper<int8_t>::dequantize(a00, iq_info); + const float inp01 = Qasymm8QuantizationHelper<int8_t>::dequantize(a01, iq_info); + const float inp10 = Qasymm8QuantizationHelper<int8_t>::dequantize(a10, iq_info); + const float inp11 = Qasymm8QuantizationHelper<int8_t>::dequantize(a11, iq_info); + *reinterpret_cast<int8_t *>(out.ptr()) = Qasymm8QuantizationHelper<int8_t>::quantize( + scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); + }, + in, out); + } + else if (border_mode == BorderMode::REPLICATE) + { + using FloatTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>; + using Int32TagType = typename wrapper::traits::neon_bitvector_tag_t<int32_t, wrapper::traits::BitWidth::W128>; + + const int in_stride_x = src->info()->strides_in_bytes()[1]; + const int in_stride_y = src->info()->strides_in_bytes()[2]; + const int in_stride_b = src->info()->strides_in_bytes()[3]; + const int out_stride_x = dst->info()->strides_in_bytes()[1]; + const int out_stride_y = dst->info()->strides_in_bytes()[2]; + const int out_stride_b = dst->info()->strides_in_bytes()[3]; + const int out_dim_ch = dst->info()->dimension(0); + constexpr int step_cout = 16; + + Window window_execution = window; + window_execution.set(Window::DimX, Window::Dimension(0, 1, 1)); + Window win_in_out(window); + win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in_out); + Iterator out(dst, win_in_out); + + const int xo_start = window_execution[1].start(); + const int xo_end = window_execution[1].end(); + const int xo_step = window_execution[1].step(); + const int yo_start = window_execution[2].start(); + const int yo_end = window_execution[2].end(); + const int yo_step = window_execution[2].step(); + const int bo_start = window_execution[3].start(); + const int bo_end = window_execution[3].end(); + const int bo_step = window_execution[3].step(); + + const float fp_coord_offset_y = sampling_offset * (scale_y - 1); + const float fp_coord_offset_x = sampling_offset * (scale_x - 1); + + const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); + const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); + + const float32x4_t vscale_in = wrapper::vdup_n(iq_info.scale, FloatTagType{}); + const int32x4_t voffset_in = wrapper::vdup_n(iq_info.offset, Int32TagType{}); // Offsets will be Int32 + + const float32x4_t invvscale_o = wrapper::vdup_n(1.f / oq_info.scale, FloatTagType{}); + const float32x4_t voffset_o = vdupq_n_f32(oq_info.offset); + + for (int bo = bo_start; bo < bo_end; bo += bo_step) + { + const int8_t *in_ptr = reinterpret_cast<int8_t *>(in.ptr() + bo * in_stride_b); + int8_t *out_ptr = reinterpret_cast<int8_t *>(out.ptr() + bo * out_stride_b); + + for (int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + const float yi_f = yo * scale_y + fp_coord_offset_y; + // Integer coordinate + const int yi = static_cast<int>(std::floor(yi_f)); + // Weight for the y coordinate + const float a1 = (yi_f - static_cast<float>(yi)); + const float b1 = (1.f - a1); + + const int yi0 = utility::clamp<int>(yi, 0, input_height - 1); + const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1); + + const int8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y; + const int8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; + + int8_t *out_ptr_yo = out_ptr + yo * out_stride_y; + for (int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + const float xi_f = xo * scale_x + fp_coord_offset_x; + // Integer coordinate + const int xi = static_cast<int>(std::floor(xi_f)); + // Weight for the x coordinate + const float a = (xi_f - static_cast<float>(xi)); + const float b = (1.f - a); + + const float s00_s = b * b1; + const float s01_s = a * b1; + const float s10_s = b * a1; + const float s11_s = a * a1; + + const auto s00 = wrapper::vdup_n(s00_s, FloatTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, FloatTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, FloatTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, FloatTagType{}); + + const int xi0 = utility::clamp<int>(xi, 0, input_width - 1); + const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1); + + const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x; + const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x; + + int8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; + + int cout = 0; + for (; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); + const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); + const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(int8_t)); + const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(int8_t)); + + const int16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00)); + const int16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00)); + + const auto in00_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in00_low)), voffset_in)), + vscale_in); + const auto in00_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgethigh(in00_low)), voffset_in)), + vscale_in); + const auto in00_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgetlow(in00_high)), voffset_in)), + vscale_in); + const auto in00_3 = + wrapper::vmul(wrapper::vcvt<float>( + wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in00_high)), voffset_in)), + vscale_in); + + const int16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01)); + const int16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01)); + + const auto in01_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in01_low)), voffset_in)), + vscale_in); + const auto in01_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgethigh(in01_low)), voffset_in)), + vscale_in); + const auto in01_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgetlow(in01_high)), voffset_in)), + vscale_in); + const auto in01_3 = + wrapper::vmul(wrapper::vcvt<float>( + wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in01_high)), voffset_in)), + vscale_in); + + const int16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10)); + const int16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10)); + + const auto in10_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in10_low)), voffset_in)), + vscale_in); + const auto in10_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgethigh(in10_low)), voffset_in)), + vscale_in); + const auto in10_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgetlow(in10_high)), voffset_in)), + vscale_in); + const auto in10_3 = + wrapper::vmul(wrapper::vcvt<float>( + wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in10_high)), voffset_in)), + vscale_in); + + const int16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11)); + const int16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11)); + + const auto in11_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in11_low)), voffset_in)), + vscale_in); + const auto in11_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgethigh(in11_low)), voffset_in)), + vscale_in); + const auto in11_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgetlow(in11_high)), voffset_in)), + vscale_in); + const auto in11_3 = + wrapper::vmul(wrapper::vcvt<float>( + wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in11_high)), voffset_in)), + vscale_in); + + auto out_0 = wrapper::vmul(in00_0, s00); + out_0 = wrapper::vmla(out_0, in01_0, s01); + out_0 = wrapper::vmla(out_0, in10_0, s10); + out_0 = wrapper::vmla(out_0, in11_0, s11); + + auto out_1 = wrapper::vmul(in00_1, s00); + out_1 = wrapper::vmla(out_1, in01_1, s01); + out_1 = wrapper::vmla(out_1, in10_1, s10); + out_1 = wrapper::vmla(out_1, in11_1, s11); + + auto out_2 = wrapper::vmul(in00_2, s00); + out_2 = wrapper::vmla(out_2, in01_2, s01); + out_2 = wrapper::vmla(out_2, in10_2, s10); + out_2 = wrapper::vmla(out_2, in11_2, s11); + + auto out_3 = wrapper::vmul(in00_3, s00); + out_3 = wrapper::vmla(out_3, in01_3, s01); + out_3 = wrapper::vmla(out_3, in10_3, s10); + out_3 = wrapper::vmla(out_3, in11_3, s11); + +#if defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o)); + const auto out_1_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o)); + const auto out_2_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o)); + const auto out_3_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o)); +#else // defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o)); + const auto out_1_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o)); + const auto out_2_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o)); + const auto out_3_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o)); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + const auto low_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); + + wrapper::vstore(out_ptr_xo_yo + cout * sizeof(int8_t), out); + } + + for (; cout < out_dim_ch; ++cout) + { + const int8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); + const int8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); + const int8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(int8_t)); + const int8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(int8_t)); + + const float in00_f = (static_cast<int32_t>(in00) - iq_info.offset) * iq_info.scale; + const float in01_f = (static_cast<int32_t>(in01) - iq_info.offset) * iq_info.scale; + const float in10_f = (static_cast<int32_t>(in10) - iq_info.offset) * iq_info.scale; + const float in11_f = (static_cast<int32_t>(in11) - iq_info.offset) * iq_info.scale; + + float out = in00_f * s00_s; + out += in01_f * s01_s; + out += in10_f * s10_s; + out += in11_f * s11_s; + + // Rounding modes of vector and scalar loops should match +#if defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(int8_t)) = quantize_qasymm8_signed(out, oq_info); +#else // defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(int8_t)) = + quantize_qasymm8_signed(out, oq_info, RoundingPolicy::TO_ZERO); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + } + } + } + } + } + else + { + ARM_COMPUTE_ERROR("Not implemented"); + } +} +} // namespace +namespace cpu +{ +void qasymm8_signed_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) +{ + if (policy == InterpolationPolicy::BILINEAR) + { + if (src->info()->quantization_info() == dst->info()->quantization_info() && + border_mode == BorderMode::REPLICATE) + { + s8_neon_scale(src, dst, offsets, dx, dy, policy, border_mode, constant_border_value, sampling_offset, + align_corners, window); + } + else + { + qasymm8_signed_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, + sampling_offset, align_corners, window); + } + } + else if (policy == InterpolationPolicy::NEAREST_NEIGHBOR) + { + nearest_neon_scale<int8_t>(src, dst, offsets, sampling_offset, align_corners, window); + } +} +} // namespace cpu +} // namespace arm_compute |