From c4f2743951473f8d97f5a43767fdbb31a4df967c Mon Sep 17 00:00:00 2001 From: Gunes Bayir Date: Sun, 11 Sep 2022 15:59:19 +0100 Subject: =?UTF-8?q?Optimize=20Quantized/Integer=20Bilinear=20Scale=20for?= =?UTF-8?q?=20Neon=E2=84=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch introduces several performance optimizations regarding the Bilinear Scale operator with REPLICATE Border mode. Changes apply only to NHWC. This patch - Reduces the memory footprint by disabling precomputation of indices and weights when they're not used - Rewrites the kernels for QASYMM8/QASYMM8_SIGNED/U8(Uint8) - Adds S8(Int8) Bilinear Scale for Border mode REPLICATE - Removes Bilinear Scale SVE kernels for Quantized and Integer types and adjust the heuristics to choose the Neon™ implementation - Adds new test cases where the input and output of the Bilinear Scale operator have different quantization scale and offset Resolves: COMPMID-5453, COMPMID-5454 Change-Id: I3d251e76e0c6978fd5a0a1795ec62ab536bec93c Signed-off-by: Gunes Bayir Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/8250 Reviewed-by: SiCong Li Comments-Addressed: Arm Jenkins Tested-by: Arm Jenkins Benchmark: Arm Jenkins --- src/cpu/kernels/CpuScaleKernel.cpp | 33 +- src/cpu/kernels/CpuScaleKernel.h | 3 +- src/cpu/kernels/scale/neon/integer.cpp | 474 +++++++++++++++++++++++--- src/cpu/kernels/scale/neon/list.h | 20 +- src/cpu/kernels/scale/neon/qasymm8.cpp | 298 +++++++++++++--- src/cpu/kernels/scale/neon/qasymm8_signed.cpp | 299 ++++++++++++---- src/cpu/kernels/scale/sve/integer.cpp | 156 +-------- src/cpu/kernels/scale/sve/qasymm8.cpp | 111 +----- src/cpu/kernels/scale/sve/qasymm8_signed.cpp | 111 +----- 9 files changed, 966 insertions(+), 539 deletions(-) (limited to 'src/cpu/kernels') diff --git a/src/cpu/kernels/CpuScaleKernel.cpp b/src/cpu/kernels/CpuScaleKernel.cpp index e7386a385a..b8bb5ad18a 100644 --- a/src/cpu/kernels/CpuScaleKernel.cpp +++ b/src/cpu/kernels/CpuScaleKernel.cpp @@ -25,14 +25,9 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Window.h" -#include "arm_compute/core/utils/misc/Utility.h" -#include "src/core/CPP/Validate.h" -#include "src/core/NEON/wrapper/wrapper.h" #include "src/core/common/Registrars.h" -#include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/ScaleHelpers.h" #include "src/core/helpers/WindowHelpers.h" -#include "src/core/utils/ScaleUtils.h" #include "src/cpu/kernels/scale/neon/list.h" #include "src/cpu/kernels/scale/sve/list.h" #include "support/Rounding.h" @@ -68,22 +63,34 @@ static const std::vector available_kernels = }, { "sve_qu8_scale", - [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::QASYMM8 && data.isa.sve; }, + [](const ScaleKernelDataTypeISASelectorData & data) + { + return data.dt == DataType::QASYMM8 && data.isa.sve && data.interpolation_policy != InterpolationPolicy::BILINEAR; + }, REGISTER_QASYMM8_SVE(arm_compute::cpu::qasymm8_sve_scale) }, { "sve_qs8_scale", - [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::QASYMM8_SIGNED && data.isa.sve; }, + [](const ScaleKernelDataTypeISASelectorData & data) + { + return data.dt == DataType::QASYMM8_SIGNED && data.isa.sve && data.interpolation_policy != InterpolationPolicy::BILINEAR; + }, REGISTER_QASYMM8_SIGNED_SVE(arm_compute::cpu::qasymm8_signed_sve_scale) }, { "sve_u8_scale", - [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::U8 && data.isa.sve; }, + [](const ScaleKernelDataTypeISASelectorData & data) + { + return data.dt == DataType::U8 && data.isa.sve && data.interpolation_policy != InterpolationPolicy::BILINEAR; + }, REGISTER_INTEGER_SVE(arm_compute::cpu::u8_sve_scale) }, { "sve_s16_scale", - [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::S16 && data.isa.sve; }, + [](const ScaleKernelDataTypeISASelectorData & data) + { + return data.dt == DataType::S16 && data.isa.sve && data.interpolation_policy != InterpolationPolicy::BILINEAR; + }, REGISTER_INTEGER_SVE(arm_compute::cpu::s16_sve_scale) }, { @@ -111,6 +118,11 @@ static const std::vector available_kernels = [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::U8; }, REGISTER_INTEGER_NEON(arm_compute::cpu::u8_neon_scale) }, + { + "neon_s8_scale", + [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::S8; }, + REGISTER_INTEGER_NEON(arm_compute::cpu::s8_neon_scale) + }, { "neon_s16_scale", [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::S16; }, @@ -140,6 +152,9 @@ Status validate_arguments(const ITensorInfo *src, const ITensorInfo *dx, const I ARM_COMPUTE_RETURN_ERROR_ON(output_width == 0); ARM_COMPUTE_RETURN_ERROR_ON(output_height == 0); + ARM_COMPUTE_RETURN_ERROR_ON((src->data_type() == DataType::S8) && (data_layout != DataLayout::NHWC || info.interpolation_policy != InterpolationPolicy::BILINEAR + || info.border_mode != BorderMode::REPLICATE)); + if(info.interpolation_policy == InterpolationPolicy::NEAREST_NEIGHBOR && offsets != nullptr) { ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(offsets, 1, DataType::S32); diff --git a/src/cpu/kernels/CpuScaleKernel.h b/src/cpu/kernels/CpuScaleKernel.h index 416e115796..8102142fc3 100644 --- a/src/cpu/kernels/CpuScaleKernel.h +++ b/src/cpu/kernels/CpuScaleKernel.h @@ -50,8 +50,9 @@ public: * * @note dx, dy and offsets have the same dimensions (width and height) of the output tensor * @note Using @p policy Area only supports data layout NCHW and input data type U8. + * @note Using S8 data type only supports NHWC, @p border_mode Replicate, and @p policy Bilinear * - * @param[in] src Source tensor info. Data types supported: QASYMM8/QASYMM8_SIGNED/U8/S16/F16/F32. + * @param[in] src Source tensor info. Data types supported: QASYMM8/QASYMM8_SIGNED/U8/S8/S16/F16/F32. * @param[in] dx Distance x tensor info. Pixel's distance between the X real coordinate and the smallest X following integer. Data type supported: F32 * @param[in] dy Distance y tensor info. Pixel's distance between the Y real coordinate and the smallest Y following integer. Data type supported: F32 * @param[in] offsets Offset tensor info. Offset to access the pixel with NEAREST interpolation or the top-left pixel with BILINEAR interpolation in the input tensor. Data type supported: S32. diff --git a/src/cpu/kernels/scale/neon/integer.cpp b/src/cpu/kernels/scale/neon/integer.cpp index a2359aac94..2ab14cf83a 100644 --- a/src/cpu/kernels/scale/neon/integer.cpp +++ b/src/cpu/kernels/scale/neon/integer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,17 +22,12 @@ * SOFTWARE. */ #include "arm_compute/core/Helpers.h" -#include "arm_compute/core/ITensorPack.h" -#include "arm_compute/core/Window.h" -#include "src/core/NEON/NEMath.h" #include "src/core/NEON/wrapper/wrapper.h" #include "src/core/helpers/ScaleHelpers.h" #include "src/core/utils/ScaleUtils.h" #include "support/Rounding.h" #include -#include -#include namespace arm_compute { @@ -84,37 +79,39 @@ void u8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window) { - // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); - - Iterator out(dst, window); - const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; - const int in_dim_w = src->info()->dimension(1); - const int in_dim_h = src->info()->dimension(2); - const int in_stride_wc = in_stride_c * (in_dim_w + src->info()->padding().top + src->info()->padding().bottom); + // Compute the ratio between source and destination dimensions + const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); - // Don't increment in Y and Z direction for the input tensor - // A pointer to the start of this plane is needed as base for the precomputed offsets - Window win_in(window); - win_in.set(Window::DimY, Window::Dimension(0, 0, 0)); - win_in.set(Window::DimZ, Window::Dimension(0, 0, 0)); - Iterator in(src, win_in); + const int input_width = src->info()->dimension(1); + const int input_height = src->info()->dimension(2); if(border_mode == BorderMode::CONSTANT) { + Iterator out(dst, window); + const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; + const int in_stride_wc = in_stride_c * (input_width + src->info()->padding().top + src->info()->padding().bottom); + + // Don't increment in Y and Z direction for the input tensor + // A pointer to the start of this plane is needed as base for the precomputed offsets + Window win_in(window); + win_in.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in); + const uint8_t const_border_value = static_cast(constant_border_value.get()); execute_window_loop(window, [&](const Coordinates & id) { const auto offset = *reinterpret_cast(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); const auto dx_val = *reinterpret_cast(dx->ptr_to_element(Coordinates(id.y(), id.z()))); const auto dy_val = *reinterpret_cast(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); + const int32_t in_hi = std::floor((id.z() + sampling_offset) * scale_y - sampling_offset); const uint8_t *in_ptr = reinterpret_cast(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; - const auto a00 = (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value; - const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) ? *(in_ptr + in_stride_c) : const_border_value; - const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_wc) : const_border_value; - const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value; + const auto a00 = (0 <= offset && offset < input_width && 0 <= in_hi && in_hi < input_height) ? *in_ptr : const_border_value; + const auto a01 = (-1 <= offset && offset < input_width - 1 && 0 <= in_hi && in_hi < input_height) ? *(in_ptr + in_stride_c) : const_border_value; + const auto a10 = (0 <= offset && offset < input_width && -1 <= in_hi && in_hi < input_height - 1) ? *(in_ptr + in_stride_wc) : const_border_value; + const auto a11 = (-1 <= offset && offset < input_width - 1 && -1 <= in_hi && in_hi < input_height - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value; *reinterpret_cast(out.ptr()) = static_cast(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); }, @@ -122,26 +119,395 @@ void u8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off } else if(border_mode == BorderMode::REPLICATE) { - execute_window_loop(window, [&](const Coordinates & id) + using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t; + + const int in_stride_x = src->info()->strides_in_bytes()[1]; + const int in_stride_y = src->info()->strides_in_bytes()[2]; + const int in_stride_b = src->info()->strides_in_bytes()[3]; + const int out_stride_x = dst->info()->strides_in_bytes()[1]; + const int out_stride_y = dst->info()->strides_in_bytes()[2]; + const int out_stride_b = dst->info()->strides_in_bytes()[3]; + + const int out_dim_ch = dst->info()->dimension(0); + constexpr int step_cout = 16; + + Window window_execution = window; + window_execution.set(Window::DimX, Window::Dimension(0, 1, 1)); + Window win_in_out(window); + win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in_out); + Iterator out(dst, win_in_out); + + const int xo_start = window_execution[1].start(); + const int xo_end = window_execution[1].end(); + const int xo_step = window_execution[1].step(); + const int yo_start = window_execution[2].start(); + const int yo_end = window_execution[2].end(); + const int yo_step = window_execution[2].step(); + const int bo_start = window_execution[3].start(); + const int bo_end = window_execution[3].end(); + const int bo_step = window_execution[3].step(); + + const float fp_coord_offset_y = sampling_offset * (scale_y - 1); + const float fp_coord_offset_x = sampling_offset * (scale_x - 1); + + for(int bo = bo_start; bo < bo_end; bo += bo_step) { - const auto offset = *reinterpret_cast(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - - auto clamped_w = utility::clamp(offset, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp(offset + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp(in_hi, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp(in_hi + 1, 0, in_dim_h - 1); - - const auto a00 = *(reinterpret_cast(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc); - const auto a01 = *(reinterpret_cast(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc); - const auto a10 = *(reinterpret_cast(in.ptr()) + clamped_w * in_stride_c + clamped_h1 * in_stride_wc); - const auto a11 = *(reinterpret_cast(in.ptr()) + clamped_w1 * in_stride_c + clamped_h1 * in_stride_wc); + const uint8_t *in_ptr = in.ptr() + bo * in_stride_b; + uint8_t *out_ptr = out.ptr() + bo * out_stride_b; + + for(int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + const float yi_f = yo * scale_y + fp_coord_offset_y; + // Integer coordinate + const int yi = static_cast(std::floor(yi_f)); + // Weight for the y coordinate + const float a1 = (yi_f - static_cast(yi)); + const float b1 = (1.f - a1); + + const int yi0 = utility::clamp(yi, 0, input_height - 1); + const int yi1 = utility::clamp(yi + 1, 0, input_height - 1); + + const uint8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y; + const uint8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; + + uint8_t *out_ptr_yo = out_ptr + yo * out_stride_y; + for(int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + const float xi_f = xo * scale_x + fp_coord_offset_x; + // Integer coordinate + const int xi = static_cast(std::floor(xi_f)); + // Weight for the x coordinate + const float a = (xi_f - static_cast(xi)); + const float b = (1.f - a); + + const float s00_s = b * b1; + const float s01_s = a * b1; + const float s10_s = b * a1; + const float s11_s = a * a1; + + const auto s00 = wrapper::vdup_n(s00_s, ExactTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, ExactTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, ExactTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, ExactTagType{}); + + const int xi0 = utility::clamp(xi, 0, input_width - 1); + const int xi1 = utility::clamp(xi + 1, 0, input_width - 1); + + const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x; + const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x; + + uint8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; + + int cout = 0; + for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); + const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); + const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(uint8_t)); + const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(uint8_t)); + + const uint16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00)); + const uint16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00)); + + const auto in00_0 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in00_low))); + const auto in00_1 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in00_low))); + const auto in00_2 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in00_high))); + const auto in00_3 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in00_high))); + + const uint16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01)); + const uint16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01)); + + const auto in01_0 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in01_low))); + const auto in01_1 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in01_low))); + const auto in01_2 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in01_high))); + const auto in01_3 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in01_high))); + + const uint16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10)); + const uint16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10)); + + const auto in10_0 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in10_low))); + const auto in10_1 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in10_low))); + const auto in10_2 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in10_high))); + const auto in10_3 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in10_high))); + + const uint16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11)); + const uint16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11)); + + const auto in11_0 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in11_low))); + const auto in11_1 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in11_low))); + const auto in11_2 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in11_high))); + const auto in11_3 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in11_high))); + + auto out_0 = wrapper::vmul(in00_0, s00); + out_0 = wrapper::vmla(out_0, in01_0, s01); + out_0 = wrapper::vmla(out_0, in10_0, s10); + out_0 = wrapper::vmla(out_0, in11_0, s11); + + auto out_1 = wrapper::vmul(in00_1, s00); + out_1 = wrapper::vmla(out_1, in01_1, s01); + out_1 = wrapper::vmla(out_1, in10_1, s10); + out_1 = wrapper::vmla(out_1, in11_1, s11); + + auto out_2 = wrapper::vmul(in00_2, s00); + out_2 = wrapper::vmla(out_2, in01_2, s01); + out_2 = wrapper::vmla(out_2, in10_2, s10); + out_2 = wrapper::vmla(out_2, in11_2, s11); + + auto out_3 = wrapper::vmul(in00_3, s00); + out_3 = wrapper::vmla(out_3, in01_3, s01); + out_3 = wrapper::vmla(out_3, in10_3, s10); + out_3 = wrapper::vmla(out_3, in11_3, s11); + +#if defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvta(out_0); + const auto out_1_int = wrapper::vcvta(out_1); + const auto out_2_int = wrapper::vcvta(out_2); + const auto out_3_int = wrapper::vcvta(out_3); +#else // defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvt(out_0); + const auto out_1_int = wrapper::vcvt(out_1); + const auto out_2_int = wrapper::vcvt(out_2); + const auto out_3_int = wrapper::vcvt(out_3); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); + + wrapper::vstore(out_ptr_xo_yo + cout * sizeof(uint8_t), out); + } + + for(; cout < out_dim_ch; ++cout) + { + const uint8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); + const uint8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); + const uint8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(uint8_t)); + const uint8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(uint8_t)); + + float out0 = in00 * s00_s; + out0 += in01 * s01_s; + out0 += in10 * s10_s; + out0 += in11 * s11_s; + + // Rounding modes of vector and scalar loops should match +#if defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = static_cast(std::round(out0)); +#else // defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = static_cast(out0); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + } + } + } + } + } + else + { + ARM_COMPUTE_ERROR("Not implemented"); + } +} - *reinterpret_cast(out.ptr()) = static_cast(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); +void s8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, + BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, + bool align_corners, const Window &window) +{ + ARM_COMPUTE_UNUSED(dx, dy, offsets, constant_border_value); + if(border_mode == BorderMode::REPLICATE) + { + using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t; + + // Compute the ratio between source and destination dimensions + const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + + const int in_stride_x = src->info()->strides_in_bytes()[1]; + const int in_stride_y = src->info()->strides_in_bytes()[2]; + const int in_stride_b = src->info()->strides_in_bytes()[3]; + const int out_stride_x = dst->info()->strides_in_bytes()[1]; + const int out_stride_y = dst->info()->strides_in_bytes()[2]; + const int out_stride_b = dst->info()->strides_in_bytes()[3]; + const int input_width = src->info()->dimension(1); + const int input_height = src->info()->dimension(2); + const int out_dim_ch = dst->info()->dimension(0); + constexpr int step_cout = 16; + + Window window_execution = window; + window_execution.set(Window::DimX, Window::Dimension(0, 1, 1)); + Window win_in_out(window); + win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in_out); + Iterator out(dst, win_in_out); + + const int xo_start = window_execution[1].start(); + const int xo_end = window_execution[1].end(); + const int xo_step = window_execution[1].step(); + const int yo_start = window_execution[2].start(); + const int yo_end = window_execution[2].end(); + const int yo_step = window_execution[2].step(); + const int bo_start = window_execution[3].start(); + const int bo_end = window_execution[3].end(); + const int bo_step = window_execution[3].step(); + + const float fp_coord_offset_y = sampling_offset * (scale_y - 1); + const float fp_coord_offset_x = sampling_offset * (scale_x - 1); + + for(int bo = bo_start; bo < bo_end; bo += bo_step) + { + const int8_t *in_ptr = reinterpret_cast(in.ptr() + bo * in_stride_b); + int8_t *out_ptr = reinterpret_cast(out.ptr() + bo * out_stride_b); + + for(int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + const float yi_f = yo * scale_y + fp_coord_offset_y; + // Integer coordinate + const int yi = static_cast(std::floor(yi_f)); + // Weight for the y coordinate + const float a1 = (yi_f - static_cast(yi)); + const float b1 = (1.f - a1); + + const int yi0 = utility::clamp(yi, 0, input_height - 1); + const int yi1 = utility::clamp(yi + 1, 0, input_height - 1); + + const int8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y; + const int8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; + + int8_t *out_ptr_yo = out_ptr + yo * out_stride_y; + for(int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + const float xi_f = xo * scale_x + fp_coord_offset_x; + // Integer coordinate + const int xi = static_cast(std::floor(xi_f)); + // Weight for the x coordinate + const float a = (xi_f - static_cast(xi)); + const float b = (1.f - a); + + const float s00_s = b * b1; + const float s01_s = a * b1; + const float s10_s = b * a1; + const float s11_s = a * a1; + + const auto s00 = wrapper::vdup_n(s00_s, ExactTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, ExactTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, ExactTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, ExactTagType{}); + + const int xi0 = utility::clamp(xi, 0, input_width - 1); + const int xi1 = utility::clamp(xi + 1, 0, input_width - 1); + + const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x; + const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x; + + int8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; + + int cout = 0; + for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); + const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); + const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(int8_t)); + const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(int8_t)); + + const int16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00)); + const int16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00)); + + const auto in00_0 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in00_low))); + const auto in00_1 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in00_low))); + const auto in00_2 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in00_high))); + const auto in00_3 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in00_high))); + + const int16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01)); + const int16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01)); + + const auto in01_0 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in01_low))); + const auto in01_1 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in01_low))); + const auto in01_2 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in01_high))); + const auto in01_3 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in01_high))); + + const int16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10)); + const int16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10)); + + const auto in10_0 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in10_low))); + const auto in10_1 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in10_low))); + const auto in10_2 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in10_high))); + const auto in10_3 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in10_high))); + + const int16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11)); + const int16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11)); + + const auto in11_0 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in11_low))); + const auto in11_1 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in11_low))); + const auto in11_2 = wrapper::vcvt(wrapper::vmovl(wrapper::vgetlow(in11_high))); + const auto in11_3 = wrapper::vcvt(wrapper::vmovl(wrapper::vgethigh(in11_high))); + + auto out_0 = wrapper::vmul(in00_0, s00); + out_0 = wrapper::vmla(out_0, in01_0, s01); + out_0 = wrapper::vmla(out_0, in10_0, s10); + out_0 = wrapper::vmla(out_0, in11_0, s11); + + auto out_1 = wrapper::vmul(in00_1, s00); + out_1 = wrapper::vmla(out_1, in01_1, s01); + out_1 = wrapper::vmla(out_1, in10_1, s10); + out_1 = wrapper::vmla(out_1, in11_1, s11); + + auto out_2 = wrapper::vmul(in00_2, s00); + out_2 = wrapper::vmla(out_2, in01_2, s01); + out_2 = wrapper::vmla(out_2, in10_2, s10); + out_2 = wrapper::vmla(out_2, in11_2, s11); + + auto out_3 = wrapper::vmul(in00_3, s00); + out_3 = wrapper::vmla(out_3, in01_3, s01); + out_3 = wrapper::vmla(out_3, in10_3, s10); + out_3 = wrapper::vmla(out_3, in11_3, s11); + +#if defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvta(out_0); + const auto out_1_int = wrapper::vcvta(out_1); + const auto out_2_int = wrapper::vcvta(out_2); + const auto out_3_int = wrapper::vcvta(out_3); +#else // defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvt(out_0); + const auto out_1_int = wrapper::vcvt(out_1); + const auto out_2_int = wrapper::vcvt(out_2); + const auto out_3_int = wrapper::vcvt(out_3); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); + + wrapper::vstore(out_ptr_xo_yo + cout * sizeof(int8_t), out); + } + + for(; cout < out_dim_ch; ++cout) + { + const int8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); + const int8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); + const int8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(int8_t)); + const int8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(int8_t)); + + float out0 = in00 * s00_s; + out0 += in01 * s01_s; + out0 += in10 * s10_s; + out0 += in11 * s11_s; + + // Rounding modes of vector and scalar loops should match +#if defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(int8_t)) = static_cast(std::round(out0)); +#else // defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(int8_t)) = static_cast(out0); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + } + } + } + } } else { @@ -240,10 +606,10 @@ void s16_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *of const auto dy_val = *reinterpret_cast(dy->ptr_to_element(Coordinates(id.y(), id.z()))); const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - auto clamped_w = utility::clamp(offset, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp(offset + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp(in_hi, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp(in_hi + 1, 0, in_dim_h - 1); + const auto clamped_w = utility::clamp(offset, 0, in_dim_w - 1); + const auto clamped_w1 = utility::clamp(offset + 1, 0, in_dim_w - 1); + const auto clamped_h = utility::clamp(in_hi, 0, in_dim_h - 1); + const auto clamped_h1 = utility::clamp(in_hi + 1, 0, in_dim_h - 1); const auto a00 = *(reinterpret_cast(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc); const auto a01 = *(reinterpret_cast(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc); @@ -262,6 +628,20 @@ void s16_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *of } namespace cpu { +void s8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, + InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, + bool align_corners, const Window &window) +{ + if(policy == InterpolationPolicy::BILINEAR) + { + s8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + } + else + { + ARM_COMPUTE_ERROR("Not implemented"); + } +} + void u8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window) diff --git a/src/cpu/kernels/scale/neon/list.h b/src/cpu/kernels/scale/neon/list.h index 17ff4bb676..28a1087224 100644 --- a/src/cpu/kernels/scale/neon/list.h +++ b/src/cpu/kernels/scale/neon/list.h @@ -25,11 +25,8 @@ #define SRC_CORE_NEON_KERNELS_SCALE_LIST_H #include "arm_compute/core/Helpers.h" -#include "arm_compute/core/ITensorPack.h" #include "arm_compute/core/Window.h" -#include "src/core/NEON/NEMath.h" #include "src/core/NEON/wrapper/wrapper.h" -#include "src/core/helpers/ScaleHelpers.h" #include "src/core/utils/ScaleUtils.h" #include "support/Rounding.h" @@ -44,6 +41,7 @@ namespace cpu DECLARE_SCALE_KERNEL(s16_neon_scale); DECLARE_SCALE_KERNEL(u8_neon_scale); +DECLARE_SCALE_KERNEL(s8_neon_scale); DECLARE_SCALE_KERNEL(qasymm8_neon_scale); DECLARE_SCALE_KERNEL(qasymm8_signed_neon_scale); @@ -360,10 +358,10 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset int cout = 0; for(; cout <= (out_dim_ch - step_cout); cout += step_cout) { - auto in00 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); - auto in01 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); - auto in10 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); - auto in11 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); + const auto in00 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); + const auto in01 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); + const auto in10 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); + const auto in11 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); auto out0 = wrapper::vmul(in00, s00); out0 = wrapper::vmla(out0, in01, s01); @@ -374,10 +372,10 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset for(; cout < out_dim_ch; ++cout) { - T in00 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); - T in01 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); - T in10 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); - T in11 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); + const T in00 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); + const T in01 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); + const T in10 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); + const T in11 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); T out0 = in00 * s00_s; out0 += in01 * s01_s; diff --git a/src/cpu/kernels/scale/neon/qasymm8.cpp b/src/cpu/kernels/scale/neon/qasymm8.cpp index daa157ec3d..778459ae39 100644 --- a/src/cpu/kernels/scale/neon/qasymm8.cpp +++ b/src/cpu/kernels/scale/neon/qasymm8.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -21,6 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include "src/core/helpers/ScaleHelpers.h" #include "src/cpu/kernels/scale/neon/list.h" namespace arm_compute @@ -32,56 +33,60 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor bool align_corners, const Window &window) { // Data layout is NHWC - // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); - Window win_off; - win_off.set(Window::DimX, Window::Dimension(0, 0, 0)); - win_off.set(Window::DimY, Window::Dimension(0, 0, 0)); - - // Don't increment in X and Y direction for the input tensor - // A pointer to the start of this plane is needed as base for the precomputed offsets - Window win_in(window); - win_in.set(1, Window::Dimension(0, 0, 0)); - win_in.set(2, Window::Dimension(0, 0, 0)); - - for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) - { - win_off.set(d, Window::Dimension(0, 0, 0)); - } - - Iterator in(src, win_in); - Iterator out(dst, window); - - const int32_t in_dim_w = src->info()->dimension(1); - const int32_t in_dim_h = src->info()->dimension(2); - const int32_t stride_w = src->info()->strides_in_bytes()[1]; - const int32_t stride_h = src->info()->strides_in_bytes()[2]; + const int32_t input_width = src->info()->dimension(1); + const int32_t input_height = src->info()->dimension(2); const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); + // Compute the ratio between source and destination dimensions + const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + if(border_mode == BorderMode::CONSTANT) { + const int32_t in_stride_y = src->info()->strides_in_bytes()[1]; + const int32_t in_stride_z = src->info()->strides_in_bytes()[2]; + + // Compute the ratio between source height and destination height + Window win_off; + win_off.set(Window::DimX, Window::Dimension(0, 0, 0)); + win_off.set(Window::DimY, Window::Dimension(0, 0, 0)); + + // Don't increment in X and Y direction for the input tensor + // A pointer to the start of this plane is needed as base for the precomputed offsets + Window win_in(window); + win_in.set(1, Window::Dimension(0, 0, 0)); + win_in.set(2, Window::Dimension(0, 0, 0)); + + for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) + { + win_off.set(d, Window::Dimension(0, 0, 0)); + } + + Iterator in(src, win_in); + Iterator out(dst, window); + const uint8_t const_border_value = static_cast(constant_border_value.get()); execute_window_loop(window, [&](const Coordinates & id) { - const int32_t index_h = std::floor((id[2] + sampling_offset) * hr - sampling_offset); + const int32_t index_h = std::floor((id[2] + sampling_offset) * scale_y - sampling_offset); const int32_t index_w = *(reinterpret_cast(offsets->ptr_to_element(Coordinates(id[1], id[2])))); const auto dx_val = *(reinterpret_cast(dx->ptr_to_element(Coordinates(id[1], id[2])))); const auto dy_val = *(reinterpret_cast(dy->ptr_to_element(Coordinates(id[1], id[2])))); const auto pixel_row_ptr = reinterpret_cast(in.ptr()); - const auto a00 = (0 <= index_w && index_w < in_dim_w && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + index_w * stride_w + index_h * stride_h)) : + const auto a00 = (0 <= index_w && index_w < input_width && 0 <= index_h && index_h < input_height) ? + (*(pixel_row_ptr + index_w * in_stride_y + index_h * in_stride_z)) : const_border_value; - const auto a01 = (-1 <= index_w && index_w + 1 < in_dim_w && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + index_h * stride_h)) : + const auto a01 = (-1 <= index_w && index_w + 1 < input_width && 0 <= index_h && index_h < input_height) ? + (*(pixel_row_ptr + (index_w + 1) * in_stride_y + index_h * in_stride_z)) : const_border_value; - const auto a10 = (0 <= index_w && index_w < in_dim_w && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + index_w * stride_w + (index_h + 1) * stride_h)) : + const auto a10 = (0 <= index_w && index_w < input_width && -1 <= index_h && index_h < input_height - 1) ? + (*(pixel_row_ptr + index_w * in_stride_y + (index_h + 1) * in_stride_z)) : const_border_value; - const auto a11 = (-1 <= index_w && index_w < in_dim_w - 1 && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + (index_h + 1) * stride_h)) : + const auto a11 = (-1 <= index_w && index_w < input_width - 1 && -1 <= index_h && index_h < input_height - 1) ? + (*(pixel_row_ptr + (index_w + 1) * in_stride_y + (index_h + 1) * in_stride_z)) : const_border_value; const float inp00 = Qasymm8QuantizationHelper::dequantize(a00, iq_info); @@ -94,31 +99,205 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor } else if(border_mode == BorderMode::REPLICATE) { - execute_window_loop(window, [&](const Coordinates & id) + using FloatTagType = typename wrapper::traits::neon_bitvector_tag_t; + using Int32TagType = typename wrapper::traits::neon_bitvector_tag_t; + + const int in_stride_x = src->info()->strides_in_bytes()[1]; + const int in_stride_y = src->info()->strides_in_bytes()[2]; + const int in_stride_b = src->info()->strides_in_bytes()[3]; + const int out_stride_x = dst->info()->strides_in_bytes()[1]; + const int out_stride_y = dst->info()->strides_in_bytes()[2]; + const int out_stride_b = dst->info()->strides_in_bytes()[3]; + const int out_dim_ch = dst->info()->dimension(0); + constexpr int step_cout = 16; + + Window window_execution = window; + window_execution.set(Window::DimX, Window::Dimension(0, 1, 1)); + Window win_in_out(window); + win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in_out); + Iterator out(dst, win_in_out); + + const int xo_start = window_execution[1].start(); + const int xo_end = window_execution[1].end(); + const int xo_step = window_execution[1].step(); + const int yo_start = window_execution[2].start(); + const int yo_end = window_execution[2].end(); + const int yo_step = window_execution[2].step(); + const int bo_start = window_execution[3].start(); + const int bo_end = window_execution[3].end(); + const int bo_step = window_execution[3].step(); + + const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); + const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); + + const float32x4_t vscale_in = wrapper::vdup_n(iq_info.scale, FloatTagType{}); + const int32x4_t voffset_in = wrapper::vdup_n(iq_info.offset, Int32TagType{}); // Offsets will be Int32 + + const float32x4_t invvscale_o = wrapper::vdup_n(1.f / oq_info.scale, FloatTagType{}); + const float32x4_t voffset_o = vdupq_n_f32(oq_info.offset); + + const float fp_coord_offset_y = sampling_offset * (scale_y - 1); + const float fp_coord_offset_x = sampling_offset * (scale_x - 1); + + for(int bo = bo_start; bo < bo_end; bo += bo_step) { - const int index_h = std::floor((id[2] + sampling_offset) * hr - sampling_offset); - const int32_t index_w = *(reinterpret_cast(offsets->ptr_to_element(Coordinates(id[1], id[2])))); - const auto dx_val = *(reinterpret_cast(dx->ptr_to_element(Coordinates(id[1], id[2])))); - const auto dy_val = *(reinterpret_cast(dy->ptr_to_element(Coordinates(id[1], id[2])))); - const auto pixel_row_ptr = reinterpret_cast(in.ptr()); + const uint8_t *in_ptr = in.ptr() + bo * in_stride_b; + uint8_t *out_ptr = out.ptr() + bo * out_stride_b; - auto clamped_w = utility::clamp(index_w, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp(index_w + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp(index_h, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp(index_h + 1, 0, in_dim_h - 1); + for(int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + const float yi_f = yo * scale_y + fp_coord_offset_y; + // Integer coordinate + const int yi = static_cast(std::floor(yi_f)); + // Weight for the y coordinate + const float a1 = (yi_f - static_cast(yi)); + const float b1 = (1.f - a1); - const auto a00 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h * stride_h); - const auto a01 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h * stride_h); - const auto a10 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h1 * stride_h); - const auto a11 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h1 * stride_h); + const int yi0 = utility::clamp(yi, 0, input_height - 1); + const int yi1 = utility::clamp(yi + 1, 0, input_height - 1); - const float inp00 = Qasymm8QuantizationHelper::dequantize(a00, iq_info); - const float inp01 = Qasymm8QuantizationHelper::dequantize(a01, iq_info); - const float inp10 = Qasymm8QuantizationHelper::dequantize(a10, iq_info); - const float inp11 = Qasymm8QuantizationHelper::dequantize(a11, iq_info); - *reinterpret_cast(out.ptr()) = Qasymm8QuantizationHelper::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); - }, - in, out); + const uint8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y; + const uint8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; + + uint8_t *out_ptr_yo = out_ptr + yo * out_stride_y; + for(int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + const float xi_f = xo * scale_x + fp_coord_offset_x; + // Integer coordinate + const int xi = static_cast(std::floor(xi_f)); + // Weight for the x coordinate + const float a = (xi_f - static_cast(xi)); + const float b = (1.f - a); + + const float s00_s = b * b1; + const float s01_s = a * b1; + const float s10_s = b * a1; + const float s11_s = a * a1; + + const auto s00 = wrapper::vdup_n(s00_s, FloatTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, FloatTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, FloatTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, FloatTagType{}); + + const int xi0 = utility::clamp(xi, 0, input_width - 1); + const int xi1 = utility::clamp(xi + 1, 0, input_width - 1); + + const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x; + const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x; + + uint8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; + + int cout = 0; + for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); + const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); + const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(uint8_t)); + const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(uint8_t)); + + const uint16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00)); + const uint16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00)); + + const auto in00_0 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in00_low))), voffset_in)), vscale_in); + const auto in00_1 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in00_low))), voffset_in)), vscale_in); + const auto in00_2 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in00_high))), voffset_in)), vscale_in); + const auto in00_3 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in00_high))), voffset_in)), vscale_in); + + const uint16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01)); + const uint16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01)); + + const auto in01_0 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in01_low))), voffset_in)), vscale_in); + const auto in01_1 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in01_low))), voffset_in)), vscale_in); + const auto in01_2 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in01_high))), voffset_in)), vscale_in); + const auto in01_3 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in01_high))), voffset_in)), vscale_in); + + const uint16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10)); + const uint16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10)); + + const auto in10_0 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in10_low))), voffset_in)), vscale_in); + const auto in10_1 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in10_low))), voffset_in)), vscale_in); + const auto in10_2 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in10_high))), voffset_in)), vscale_in); + const auto in10_3 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in10_high))), voffset_in)), vscale_in); + + const uint16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11)); + const uint16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11)); + + const auto in11_0 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in11_low))), voffset_in)), vscale_in); + const auto in11_1 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in11_low))), voffset_in)), vscale_in); + const auto in11_2 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in11_high))), voffset_in)), vscale_in); + const auto in11_3 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in11_high))), voffset_in)), vscale_in); + + auto out_0 = wrapper::vmul(in00_0, s00); + out_0 = wrapper::vmla(out_0, in01_0, s01); + out_0 = wrapper::vmla(out_0, in10_0, s10); + out_0 = wrapper::vmla(out_0, in11_0, s11); + + auto out_1 = wrapper::vmul(in00_1, s00); + out_1 = wrapper::vmla(out_1, in01_1, s01); + out_1 = wrapper::vmla(out_1, in10_1, s10); + out_1 = wrapper::vmla(out_1, in11_1, s11); + + auto out_2 = wrapper::vmul(in00_2, s00); + out_2 = wrapper::vmla(out_2, in01_2, s01); + out_2 = wrapper::vmla(out_2, in10_2, s10); + out_2 = wrapper::vmla(out_2, in11_2, s11); + + auto out_3 = wrapper::vmul(in00_3, s00); + out_3 = wrapper::vmla(out_3, in01_3, s01); + out_3 = wrapper::vmla(out_3, in10_3, s10); + out_3 = wrapper::vmla(out_3, in11_3, s11); + +#if defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvta(wrapper::vmla(voffset_o, out_0, invvscale_o)); + const auto out_1_int = wrapper::vcvta(wrapper::vmla(voffset_o, out_1, invvscale_o)); + const auto out_2_int = wrapper::vcvta(wrapper::vmla(voffset_o, out_2, invvscale_o)); + const auto out_3_int = wrapper::vcvta(wrapper::vmla(voffset_o, out_3, invvscale_o)); +#else // defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvt(wrapper::vmla(voffset_o, out_0, invvscale_o)); + const auto out_1_int = wrapper::vcvt(wrapper::vmla(voffset_o, out_1, invvscale_o)); + const auto out_2_int = wrapper::vcvt(wrapper::vmla(voffset_o, out_2, invvscale_o)); + const auto out_3_int = wrapper::vcvt(wrapper::vmla(voffset_o, out_3, invvscale_o)); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); + + wrapper::vstore(out_ptr_xo_yo + cout * sizeof(uint8_t), out); + } + + for(; cout < out_dim_ch; ++cout) + { + const uint8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); + const uint8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); + const uint8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(uint8_t)); + const uint8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(uint8_t)); + + const float in00_f = (static_cast(in00) - iq_info.offset) * iq_info.scale; + const float in01_f = (static_cast(in01) - iq_info.offset) * iq_info.scale; + const float in10_f = (static_cast(in10) - iq_info.offset) * iq_info.scale; + const float in11_f = (static_cast(in11) - iq_info.offset) * iq_info.scale; + + float out = in00_f * s00_s; + out += in01_f * s01_s; + out += in10_f * s10_s; + out += in11_f * s11_s; + + // Rounding modes of vector and scalar loops should match +#if defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = quantize_qasymm8(out, oq_info); +#else // defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = quantize_qasymm8(out, oq_info, RoundingPolicy::TO_ZERO); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + } + } + } + } } else { @@ -134,7 +313,14 @@ void qasymm8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets { if(policy == InterpolationPolicy::BILINEAR) { - qasymm8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + if(src->info()->quantization_info() == dst->info()->quantization_info()) + { + u8_neon_scale(src, dst, offsets, dx, dy, policy, border_mode, constant_border_value, sampling_offset, align_corners, window); + } + else + { + qasymm8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + } } else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) { diff --git a/src/cpu/kernels/scale/neon/qasymm8_signed.cpp b/src/cpu/kernels/scale/neon/qasymm8_signed.cpp index 83312636b4..cd63dfba63 100644 --- a/src/cpu/kernels/scale/neon/qasymm8_signed.cpp +++ b/src/cpu/kernels/scale/neon/qasymm8_signed.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -21,6 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include "src/core/helpers/ScaleHelpers.h" #include "src/cpu/kernels/scale/neon/list.h" namespace arm_compute @@ -32,56 +33,59 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const bool align_corners, const Window &window) { // Data layout is NHWC - // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); - Window win_off; - win_off.set(Window::DimX, Window::Dimension(0, 0, 0)); - win_off.set(Window::DimY, Window::Dimension(0, 0, 0)); - - // Don't increment in X and Y direction for the input tensor - // A pointer to the start of this plane is needed as base for the precomputed offsets - Window win_in(window); - win_in.set(1, Window::Dimension(0, 0, 0)); - win_in.set(2, Window::Dimension(0, 0, 0)); - - for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) - { - win_off.set(d, Window::Dimension(0, 0, 0)); - } - - Iterator in(src, win_in); - Iterator out(dst, window); - - const int32_t in_dim_w = src->info()->dimension(1); - const int32_t in_dim_h = src->info()->dimension(2); - const int32_t stride_w = src->info()->strides_in_bytes()[1]; - const int32_t stride_h = src->info()->strides_in_bytes()[2]; - const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); + const int32_t input_width = src->info()->dimension(1); + const int32_t input_height = src->info()->dimension(2); + + // Compute the ratio between source and destination dimensions + const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + if(border_mode == BorderMode::CONSTANT) { + const int32_t in_stride_y = src->info()->strides_in_bytes()[1]; + const int32_t in_stride_z = src->info()->strides_in_bytes()[2]; + + Window win_off; + win_off.set(Window::DimX, Window::Dimension(0, 0, 0)); + win_off.set(Window::DimY, Window::Dimension(0, 0, 0)); + + // Don't increment in X and Y direction for the input tensor + // A pointer to the start of this plane is needed as base for the precomputed offsets + Window win_in(window); + win_in.set(1, Window::Dimension(0, 0, 0)); + win_in.set(2, Window::Dimension(0, 0, 0)); + + for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) + { + win_off.set(d, Window::Dimension(0, 0, 0)); + } + + Iterator in(src, win_in); + Iterator out(dst, window); + const int8_t const_border_value = static_cast(constant_border_value.get()); execute_window_loop(window, [&](const Coordinates & id) { - const int32_t index_h = std::floor((id[2] + sampling_offset) * hr - sampling_offset); + const int32_t index_h = std::floor((id[2] + sampling_offset) * scale_y - sampling_offset); const int32_t index_w = *(reinterpret_cast(offsets->ptr_to_element(Coordinates(id[1], id[2])))); const auto dx_val = *(reinterpret_cast(dx->ptr_to_element(Coordinates(id[1], id[2])))); const auto dy_val = *(reinterpret_cast(dy->ptr_to_element(Coordinates(id[1], id[2])))); const auto pixel_row_ptr = reinterpret_cast(in.ptr()); - const auto a00 = (0 <= index_w && index_w < in_dim_w && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + index_w * stride_w + index_h * stride_h)) : + const auto a00 = (0 <= index_w && index_w < input_width && 0 <= index_h && index_h < input_height) ? + (*(pixel_row_ptr + index_w * in_stride_y + index_h * in_stride_z)) : const_border_value; - const auto a01 = (-1 <= index_w && index_w + 1 < in_dim_w && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + index_h * stride_h)) : + const auto a01 = (-1 <= index_w && index_w + 1 < input_width && 0 <= index_h && index_h < input_height) ? + (*(pixel_row_ptr + (index_w + 1) * in_stride_y + index_h * in_stride_z)) : const_border_value; - const auto a10 = (0 <= index_w && index_w < in_dim_w && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + index_w * stride_w + (index_h + 1) * stride_h)) : + const auto a10 = (0 <= index_w && index_w < input_width && -1 <= index_h && index_h < input_height - 1) ? + (*(pixel_row_ptr + index_w * in_stride_y + (index_h + 1) * in_stride_z)) : const_border_value; - const auto a11 = (-1 <= index_w && index_w < in_dim_w - 1 && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + (index_h + 1) * stride_h)) : + const auto a11 = (-1 <= index_w && index_w < input_width - 1 && -1 <= index_h && index_h < input_height - 1) ? + (*(pixel_row_ptr + (index_w + 1) * in_stride_y + (index_h + 1) * in_stride_z)) : const_border_value; const float inp00 = Qasymm8QuantizationHelper::dequantize(a00, iq_info); @@ -94,31 +98,205 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const } else if(border_mode == BorderMode::REPLICATE) { - execute_window_loop(window, [&](const Coordinates & id) + using FloatTagType = typename wrapper::traits::neon_bitvector_tag_t; + using Int32TagType = typename wrapper::traits::neon_bitvector_tag_t; + + const int in_stride_x = src->info()->strides_in_bytes()[1]; + const int in_stride_y = src->info()->strides_in_bytes()[2]; + const int in_stride_b = src->info()->strides_in_bytes()[3]; + const int out_stride_x = dst->info()->strides_in_bytes()[1]; + const int out_stride_y = dst->info()->strides_in_bytes()[2]; + const int out_stride_b = dst->info()->strides_in_bytes()[3]; + const int out_dim_ch = dst->info()->dimension(0); + constexpr int step_cout = 16; + + Window window_execution = window; + window_execution.set(Window::DimX, Window::Dimension(0, 1, 1)); + Window win_in_out(window); + win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in_out); + Iterator out(dst, win_in_out); + + const int xo_start = window_execution[1].start(); + const int xo_end = window_execution[1].end(); + const int xo_step = window_execution[1].step(); + const int yo_start = window_execution[2].start(); + const int yo_end = window_execution[2].end(); + const int yo_step = window_execution[2].step(); + const int bo_start = window_execution[3].start(); + const int bo_end = window_execution[3].end(); + const int bo_step = window_execution[3].step(); + + const float fp_coord_offset_y = sampling_offset * (scale_y - 1); + const float fp_coord_offset_x = sampling_offset * (scale_x - 1); + + const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); + const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); + + const float32x4_t vscale_in = wrapper::vdup_n(iq_info.scale, FloatTagType{}); + const int32x4_t voffset_in = wrapper::vdup_n(iq_info.offset, Int32TagType{}); // Offsets will be Int32 + + const float32x4_t invvscale_o = wrapper::vdup_n(1.f / oq_info.scale, FloatTagType{}); + const float32x4_t voffset_o = vdupq_n_f32(oq_info.offset); + + for(int bo = bo_start; bo < bo_end; bo += bo_step) { - const int index_h = std::floor((id[2] + sampling_offset) * hr - sampling_offset); - const int32_t index_w = *(reinterpret_cast(offsets->ptr_to_element(Coordinates(id[1], id[2])))); - const auto dx_val = *(reinterpret_cast(dx->ptr_to_element(Coordinates(id[1], id[2])))); - const auto dy_val = *(reinterpret_cast(dy->ptr_to_element(Coordinates(id[1], id[2])))); - const auto pixel_row_ptr = reinterpret_cast(in.ptr()); + const int8_t *in_ptr = reinterpret_cast(in.ptr() + bo * in_stride_b); + int8_t *out_ptr = reinterpret_cast(out.ptr() + bo * out_stride_b); - auto clamped_w = utility::clamp(index_w, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp(index_w + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp(index_h, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp(index_h + 1, 0, in_dim_h - 1); + for(int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + const float yi_f = yo * scale_y + fp_coord_offset_y; + // Integer coordinate + const int yi = static_cast(std::floor(yi_f)); + // Weight for the y coordinate + const float a1 = (yi_f - static_cast(yi)); + const float b1 = (1.f - a1); - const auto a00 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h * stride_h); - const auto a01 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h * stride_h); - const auto a10 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h1 * stride_h); - const auto a11 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h1 * stride_h); + const int yi0 = utility::clamp(yi, 0, input_height - 1); + const int yi1 = utility::clamp(yi + 1, 0, input_height - 1); - const float inp00 = Qasymm8QuantizationHelper::dequantize(a00, iq_info); - const float inp01 = Qasymm8QuantizationHelper::dequantize(a01, iq_info); - const float inp10 = Qasymm8QuantizationHelper::dequantize(a10, iq_info); - const float inp11 = Qasymm8QuantizationHelper::dequantize(a11, iq_info); - *reinterpret_cast(out.ptr()) = Qasymm8QuantizationHelper::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); - }, - in, out); + const int8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y; + const int8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; + + int8_t *out_ptr_yo = out_ptr + yo * out_stride_y; + for(int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + const float xi_f = xo * scale_x + fp_coord_offset_x; + // Integer coordinate + const int xi = static_cast(std::floor(xi_f)); + // Weight for the x coordinate + const float a = (xi_f - static_cast(xi)); + const float b = (1.f - a); + + const float s00_s = b * b1; + const float s01_s = a * b1; + const float s10_s = b * a1; + const float s11_s = a * a1; + + const auto s00 = wrapper::vdup_n(s00_s, FloatTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, FloatTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, FloatTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, FloatTagType{}); + + const int xi0 = utility::clamp(xi, 0, input_width - 1); + const int xi1 = utility::clamp(xi + 1, 0, input_width - 1); + + const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x; + const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x; + + int8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; + + int cout = 0; + for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); + const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); + const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(int8_t)); + const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(int8_t)); + + const int16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00)); + const int16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00)); + + const auto in00_0 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in00_low)), voffset_in)), vscale_in); + const auto in00_1 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in00_low)), voffset_in)), vscale_in); + const auto in00_2 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in00_high)), voffset_in)), vscale_in); + const auto in00_3 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in00_high)), voffset_in)), vscale_in); + + const int16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01)); + const int16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01)); + + const auto in01_0 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in01_low)), voffset_in)), vscale_in); + const auto in01_1 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in01_low)), voffset_in)), vscale_in); + const auto in01_2 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in01_high)), voffset_in)), vscale_in); + const auto in01_3 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in01_high)), voffset_in)), vscale_in); + + const int16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10)); + const int16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10)); + + const auto in10_0 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in10_low)), voffset_in)), vscale_in); + const auto in10_1 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in10_low)), voffset_in)), vscale_in); + const auto in10_2 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in10_high)), voffset_in)), vscale_in); + const auto in10_3 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in10_high)), voffset_in)), vscale_in); + + const int16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11)); + const int16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11)); + + const auto in11_0 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in11_low)), voffset_in)), vscale_in); + const auto in11_1 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in11_low)), voffset_in)), vscale_in); + const auto in11_2 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in11_high)), voffset_in)), vscale_in); + const auto in11_3 = wrapper::vmul(wrapper::vcvt(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in11_high)), voffset_in)), vscale_in); + + auto out_0 = wrapper::vmul(in00_0, s00); + out_0 = wrapper::vmla(out_0, in01_0, s01); + out_0 = wrapper::vmla(out_0, in10_0, s10); + out_0 = wrapper::vmla(out_0, in11_0, s11); + + auto out_1 = wrapper::vmul(in00_1, s00); + out_1 = wrapper::vmla(out_1, in01_1, s01); + out_1 = wrapper::vmla(out_1, in10_1, s10); + out_1 = wrapper::vmla(out_1, in11_1, s11); + + auto out_2 = wrapper::vmul(in00_2, s00); + out_2 = wrapper::vmla(out_2, in01_2, s01); + out_2 = wrapper::vmla(out_2, in10_2, s10); + out_2 = wrapper::vmla(out_2, in11_2, s11); + + auto out_3 = wrapper::vmul(in00_3, s00); + out_3 = wrapper::vmla(out_3, in01_3, s01); + out_3 = wrapper::vmla(out_3, in10_3, s10); + out_3 = wrapper::vmla(out_3, in11_3, s11); + +#if defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvta(wrapper::vmla(voffset_o, out_0, invvscale_o)); + const auto out_1_int = wrapper::vcvta(wrapper::vmla(voffset_o, out_1, invvscale_o)); + const auto out_2_int = wrapper::vcvta(wrapper::vmla(voffset_o, out_2, invvscale_o)); + const auto out_3_int = wrapper::vcvta(wrapper::vmla(voffset_o, out_3, invvscale_o)); +#else // defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvt(wrapper::vmla(voffset_o, out_0, invvscale_o)); + const auto out_1_int = wrapper::vcvt(wrapper::vmla(voffset_o, out_1, invvscale_o)); + const auto out_2_int = wrapper::vcvt(wrapper::vmla(voffset_o, out_2, invvscale_o)); + const auto out_3_int = wrapper::vcvt(wrapper::vmla(voffset_o, out_3, invvscale_o)); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); + + wrapper::vstore(out_ptr_xo_yo + cout * sizeof(int8_t), out); + } + + for(; cout < out_dim_ch; ++cout) + { + const int8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); + const int8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); + const int8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(int8_t)); + const int8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(int8_t)); + + const float in00_f = (static_cast(in00) - iq_info.offset) * iq_info.scale; + const float in01_f = (static_cast(in01) - iq_info.offset) * iq_info.scale; + const float in10_f = (static_cast(in10) - iq_info.offset) * iq_info.scale; + const float in11_f = (static_cast(in11) - iq_info.offset) * iq_info.scale; + + float out = in00_f * s00_s; + out += in01_f * s01_s; + out += in10_f * s10_s; + out += in11_f * s11_s; + + // Rounding modes of vector and scalar loops should match +#if defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(int8_t)) = quantize_qasymm8_signed(out, oq_info); +#else // defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(int8_t)) = quantize_qasymm8_signed(out, oq_info, RoundingPolicy::TO_ZERO); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + } + } + } + } } else { @@ -134,7 +312,14 @@ void qasymm8_signed_neon_scale(const ITensor *src, ITensor *dst, const ITensor * { if(policy == InterpolationPolicy::BILINEAR) { - qasymm8_signed_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + if(src->info()->quantization_info() == dst->info()->quantization_info() && border_mode == BorderMode::REPLICATE) + { + s8_neon_scale(src, dst, offsets, dx, dy, policy, border_mode, constant_border_value, sampling_offset, align_corners, window); + } + else + { + qasymm8_signed_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + } } else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) { diff --git a/src/cpu/kernels/scale/sve/integer.cpp b/src/cpu/kernels/scale/sve/integer.cpp index 00a43922d9..82c70ee360 100644 --- a/src/cpu/kernels/scale/sve/integer.cpp +++ b/src/cpu/kernels/scale/sve/integer.cpp @@ -83,75 +83,6 @@ void u8_sve_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offse out); } -void u8_sve_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) -{ - // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); - - Iterator out(dst, window); - const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; - const int in_dim_w = src->info()->dimension(1); - const int in_dim_h = src->info()->dimension(2); - const int in_stride_wc = in_stride_c * (in_dim_w + src->info()->padding().top + src->info()->padding().bottom); - - // Don't increment in Y and Z direction for the input tensor - // A pointer to the start of this plane is needed as base for the precomputed offsets - Window win_in(window); - win_in.set(Window::DimY, Window::Dimension(0, 0, 0)); - win_in.set(Window::DimZ, Window::Dimension(0, 0, 0)); - Iterator in(src, win_in); - - if(border_mode == BorderMode::CONSTANT) - { - const uint8_t const_border_value = static_cast(constant_border_value.get()); - execute_window_loop(window, [&](const Coordinates & id) - { - const auto offset = *reinterpret_cast(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - const uint8_t *in_ptr = reinterpret_cast(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; - - const auto a00 = (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value; - const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) ? *(in_ptr + in_stride_c) : const_border_value; - const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_wc) : const_border_value; - const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value; - - *reinterpret_cast(out.ptr()) = static_cast(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); - } - else if(border_mode == BorderMode::REPLICATE) - { - execute_window_loop(window, [&](const Coordinates & id) - { - const auto offset = *reinterpret_cast(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - - auto clamped_w = utility::clamp(offset, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp(offset + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp(in_hi, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp(in_hi + 1, 0, in_dim_h - 1); - - const auto a00 = *(reinterpret_cast(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc); - const auto a01 = *(reinterpret_cast(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc); - const auto a10 = *(reinterpret_cast(in.ptr()) + clamped_w * in_stride_c + clamped_h1 * in_stride_wc); - const auto a11 = *(reinterpret_cast(in.ptr()) + clamped_w1 * in_stride_c + clamped_h1 * in_stride_wc); - - *reinterpret_cast(out.ptr()) = static_cast(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); - } - else - { - ARM_COMPUTE_ERROR("Not implemented"); - } -} - void s16_sve_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offsets, float sampling_offset, bool align_corners, const Window &window) { @@ -195,75 +126,6 @@ void s16_sve_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offs }, out); } - -void s16_sve_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) -{ - // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); - - Iterator out(dst, window); - const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; - const int in_dim_w = src->info()->dimension(1); - const int in_dim_h = src->info()->dimension(2); - const int in_stride_wc = in_stride_c * (in_dim_w + src->info()->padding().top + src->info()->padding().bottom); - - // Don't increment in Y and Z direction for the input tensor - // A pointer to the start of this plane is needed as base for the precomputed offsets - Window win_in(window); - win_in.set(Window::DimY, Window::Dimension(0, 0, 0)); - win_in.set(Window::DimZ, Window::Dimension(0, 0, 0)); - Iterator in(src, win_in); - - if(border_mode == BorderMode::CONSTANT) - { - const int16_t const_border_value = static_cast(constant_border_value.get()); - execute_window_loop(window, [&](const Coordinates & id) - { - const auto offset = *reinterpret_cast(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - const int16_t *in_ptr = reinterpret_cast(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; - - const auto a00 = (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value; - const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) ? *(in_ptr + in_stride_c) : const_border_value; - const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_wc) : const_border_value; - const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value; - - *reinterpret_cast(out.ptr()) = static_cast(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); - } - else if(border_mode == BorderMode::REPLICATE) - { - execute_window_loop(window, [&](const Coordinates & id) - { - const auto offset = *reinterpret_cast(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - - auto clamped_w = utility::clamp(offset, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp(offset + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp(in_hi, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp(in_hi + 1, 0, in_dim_h - 1); - - const auto a00 = *(reinterpret_cast(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc); - const auto a01 = *(reinterpret_cast(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc); - const auto a10 = *(reinterpret_cast(in.ptr()) + clamped_w * in_stride_c + clamped_h1 * in_stride_wc); - const auto a11 = *(reinterpret_cast(in.ptr()) + clamped_w1 * in_stride_c + clamped_h1 * in_stride_wc); - - *reinterpret_cast(out.ptr()) = static_cast(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); - } - else - { - ARM_COMPUTE_ERROR("Not implemented"); - } -} } namespace cpu { @@ -271,13 +133,14 @@ void u8_sve_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, cons InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + ARM_COMPUTE_UNUSED(dx, dy, border_mode, constant_border_value); + if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) { - u8_sve_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + u8_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); } - else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) + else { - u8_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); + ARM_COMPUTE_ERROR("Not Implemented"); } } @@ -285,13 +148,14 @@ void s16_sve_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, con InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + ARM_COMPUTE_UNUSED(dx, dy, border_mode, constant_border_value); + if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) { - s16_sve_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + s16_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); } - else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) + else { - s16_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); + ARM_COMPUTE_ERROR("Not Implemented"); } } } // namespace cpu diff --git a/src/cpu/kernels/scale/sve/qasymm8.cpp b/src/cpu/kernels/scale/sve/qasymm8.cpp index 09ef00a783..d45a69e43b 100644 --- a/src/cpu/kernels/scale/sve/qasymm8.cpp +++ b/src/cpu/kernels/scale/sve/qasymm8.cpp @@ -83,108 +83,6 @@ void qasymm8_sve_scale_nearest(const ITensor *src, ITensor *dst, const ITensor * }, out); } - -void qasymm8_sve_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) -{ - // Data layout is NHWC - const int idx_width = 1; - const int idx_height = 2; - - // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(idx_height), dst->info()->dimension(idx_height), align_corners); - Window win_off; - win_off.set(Window::DimX, Window::Dimension(0, 0, 0)); - win_off.set(Window::DimY, Window::Dimension(0, 0, 0)); - - // Don't increment in X and Y direction for the input tensor - // A pointer to the start of this plane is needed as base for the precomputed offsets - Window win_in(window); - win_in.set(idx_width, Window::Dimension(0, 0, 0)); - win_in.set(idx_height, Window::Dimension(0, 0, 0)); - - for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) - { - win_off.set(d, Window::Dimension(0, 0, 0)); - } - - Iterator in(src, win_in); - Iterator out(dst, window); - - const int32_t in_dim_w = src->info()->dimension(idx_width); - const int32_t in_dim_h = src->info()->dimension(idx_height); - const int32_t stride_w = src->info()->strides_in_bytes()[idx_width]; - const int32_t stride_h = src->info()->strides_in_bytes()[idx_height]; - - const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); - const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); - - if(border_mode == BorderMode::CONSTANT) - { - const uint8_t const_border_value = static_cast(constant_border_value.get()); - execute_window_loop(window, [&](const Coordinates & id) - { - const int32_t index_h = std::floor((id[idx_height] + sampling_offset) * hr - sampling_offset); - const int32_t index_w = *(reinterpret_cast(offsets->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dx_val = *(reinterpret_cast(dx->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dy_val = *(reinterpret_cast(dy->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto pixel_row_ptr = reinterpret_cast(in.ptr()); - - const auto a00 = (0 <= index_w && index_w < in_dim_w && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + index_w * stride_w + index_h * stride_h)) : - const_border_value; - const auto a01 = (-1 <= index_w && index_w < in_dim_w - 1 && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + index_h * stride_h)) : - const_border_value; - const auto a10 = (0 <= index_w && index_w < in_dim_w && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + index_w * stride_w + (index_h + 1) * stride_h)) : - const_border_value; - const auto a11 = (-1 <= index_w && index_w < in_dim_w - 1 && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + (index_h + 1) * stride_h)) : - const_border_value; - - const float inp00 = Qasymm8QuantizationHelper::dequantize(a00, iq_info); - const float inp01 = Qasymm8QuantizationHelper::dequantize(a01, iq_info); - const float inp10 = Qasymm8QuantizationHelper::dequantize(a10, iq_info); - const float inp11 = Qasymm8QuantizationHelper::dequantize(a11, iq_info); - *reinterpret_cast(out.ptr()) = Qasymm8QuantizationHelper::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); - }, - in, out); - } - else if(border_mode == BorderMode::REPLICATE) - { - execute_window_loop(window, [&](const Coordinates & id) - { - const int index_h = std::floor((id[idx_height] + sampling_offset) * hr - sampling_offset); - const int32_t index_w = *(reinterpret_cast(offsets->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dx_val = *(reinterpret_cast(dx->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dy_val = *(reinterpret_cast(dy->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto pixel_row_ptr = reinterpret_cast(in.ptr()); - - auto clamped_w = utility::clamp(index_w, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp(index_w + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp(index_h, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp(index_h + 1, 0, in_dim_h - 1); - - const auto a00 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h * stride_h); - const auto a01 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h * stride_h); - const auto a10 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h1 * stride_h); - const auto a11 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h1 * stride_h); - - const float inp00 = Qasymm8QuantizationHelper::dequantize(a00, iq_info); - const float inp01 = Qasymm8QuantizationHelper::dequantize(a01, iq_info); - const float inp10 = Qasymm8QuantizationHelper::dequantize(a10, iq_info); - const float inp11 = Qasymm8QuantizationHelper::dequantize(a11, iq_info); - *reinterpret_cast(out.ptr()) = Qasymm8QuantizationHelper::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); - }, - in, out); - } - else - { - ARM_COMPUTE_ERROR("Not implemented"); - } -} } namespace cpu { @@ -192,13 +90,14 @@ void qasymm8_sve_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + ARM_COMPUTE_UNUSED(dx, dy, border_mode, constant_border_value); + if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) { - qasymm8_sve_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + qasymm8_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); } - else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) + else { - qasymm8_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); + ARM_COMPUTE_ERROR("Not Implemented"); } } } // namespace cpu diff --git a/src/cpu/kernels/scale/sve/qasymm8_signed.cpp b/src/cpu/kernels/scale/sve/qasymm8_signed.cpp index 63f515442b..67bca65f58 100644 --- a/src/cpu/kernels/scale/sve/qasymm8_signed.cpp +++ b/src/cpu/kernels/scale/sve/qasymm8_signed.cpp @@ -83,108 +83,6 @@ void qasymm8_signed_sve_scale_nearest(const ITensor *src, ITensor *dst, const IT }, out); } - -void qasymm8_signed_sve_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) -{ - // Data layout is NHWC - const int idx_width = 1; - const int idx_height = 2; - - // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(idx_height), dst->info()->dimension(idx_height), align_corners); - Window win_off; - win_off.set(Window::DimX, Window::Dimension(0, 0, 0)); - win_off.set(Window::DimY, Window::Dimension(0, 0, 0)); - - // Don't increment in X and Y direction for the input tensor - // A pointer to the start of this plane is needed as base for the precomputed offsets - Window win_in(window); - win_in.set(idx_width, Window::Dimension(0, 0, 0)); - win_in.set(idx_height, Window::Dimension(0, 0, 0)); - - for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) - { - win_off.set(d, Window::Dimension(0, 0, 0)); - } - - Iterator in(src, win_in); - Iterator out(dst, window); - - const int32_t in_dim_w = src->info()->dimension(idx_width); - const int32_t in_dim_h = src->info()->dimension(idx_height); - const int32_t stride_w = src->info()->strides_in_bytes()[idx_width]; - const int32_t stride_h = src->info()->strides_in_bytes()[idx_height]; - - const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); - const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); - - if(border_mode == BorderMode::CONSTANT) - { - const int8_t const_border_value = static_cast(constant_border_value.get()); - execute_window_loop(window, [&](const Coordinates & id) - { - const int32_t index_h = std::floor((id[idx_height] + sampling_offset) * hr - sampling_offset); - const int32_t index_w = *(reinterpret_cast(offsets->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dx_val = *(reinterpret_cast(dx->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dy_val = *(reinterpret_cast(dy->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto pixel_row_ptr = reinterpret_cast(in.ptr()); - - const auto a00 = (0 <= index_w && index_w < in_dim_w && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + index_w * stride_w + index_h * stride_h)) : - const_border_value; - const auto a01 = (-1 <= index_w && index_w < in_dim_w - 1 && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + index_h * stride_h)) : - const_border_value; - const auto a10 = (0 <= index_w && index_w < in_dim_w && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + index_w * stride_w + (index_h + 1) * stride_h)) : - const_border_value; - const auto a11 = (-1 <= index_w && index_w < in_dim_w - 1 && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + (index_h + 1) * stride_h)) : - const_border_value; - - const float inp00 = Qasymm8QuantizationHelper::dequantize(a00, iq_info); - const float inp01 = Qasymm8QuantizationHelper::dequantize(a01, iq_info); - const float inp10 = Qasymm8QuantizationHelper::dequantize(a10, iq_info); - const float inp11 = Qasymm8QuantizationHelper::dequantize(a11, iq_info); - *reinterpret_cast(out.ptr()) = Qasymm8QuantizationHelper::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); - }, - in, out); - } - else if(border_mode == BorderMode::REPLICATE) - { - execute_window_loop(window, [&](const Coordinates & id) - { - const int index_h = std::floor((id[idx_height] + sampling_offset) * hr - sampling_offset); - const int32_t index_w = *(reinterpret_cast(offsets->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dx_val = *(reinterpret_cast(dx->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dy_val = *(reinterpret_cast(dy->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto pixel_row_ptr = reinterpret_cast(in.ptr()); - - auto clamped_w = utility::clamp(index_w, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp(index_w + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp(index_h, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp(index_h + 1, 0, in_dim_h - 1); - - const auto a00 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h * stride_h); - const auto a01 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h * stride_h); - const auto a10 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h1 * stride_h); - const auto a11 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h1 * stride_h); - - const float inp00 = Qasymm8QuantizationHelper::dequantize(a00, iq_info); - const float inp01 = Qasymm8QuantizationHelper::dequantize(a01, iq_info); - const float inp10 = Qasymm8QuantizationHelper::dequantize(a10, iq_info); - const float inp11 = Qasymm8QuantizationHelper::dequantize(a11, iq_info); - *reinterpret_cast(out.ptr()) = Qasymm8QuantizationHelper::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); - }, - in, out); - } - else - { - ARM_COMPUTE_ERROR("Not implemented"); - } -} } namespace cpu { @@ -192,13 +90,14 @@ void qasymm8_signed_sve_scale(const ITensor *src, ITensor *dst, const ITensor *o InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + ARM_COMPUTE_UNUSED(dx, dy, border_mode, constant_border_value); + if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) { - qasymm8_signed_sve_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + qasymm8_signed_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); } - else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) + else { - qasymm8_signed_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); + ARM_COMPUTE_ERROR("Not Implemented"); } } } // namespace cpu -- cgit v1.2.1