diff options
| author | Gunes Bayir <gunes.bayir@arm.com> | 2022-09-11 15:59:19 +0100 |
|---|---|---|
| committer | Gunes Bayir <gunes.bayir@arm.com> | 2022-09-16 12:38:59 +0000 |
| commit | c4f2743951473f8d97f5a43767fdbb31a4df967c (patch) | |
| tree | 26c49b1af8113bb169931f3af5e502904d455a25 | |
| parent | 0d05b6690fe69c57f63ca43d59b551f074613062 (diff) | |
| download | ComputeLibrary-c4f2743951473f8d97f5a43767fdbb31a4df967c.tar.gz | |
Optimize Quantized/Integer Bilinear Scale for Neon™
This patch introduces several performance optimizations regarding the Bilinear Scale operator with REPLICATE Border mode. Changes apply only to NHWC.
This patch
- Reduces the memory footprint by disabling precomputation of indices and weights when they're not used
- Rewrites the kernels for QASYMM8/QASYMM8_SIGNED/U8(Uint8)
- Adds S8(Int8) Bilinear Scale for Border mode REPLICATE
- Removes Bilinear Scale SVE kernels for Quantized and Integer types and adjust the heuristics to choose the Neon™ implementation
- Adds new test cases where the input and output of the Bilinear Scale operator have different quantization scale and offset
Resolves: COMPMID-5453, COMPMID-5454
Change-Id: I3d251e76e0c6978fd5a0a1795ec62ab536bec93c
Signed-off-by: Gunes Bayir <gunes.bayir@arm.com>
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/8250
Reviewed-by: SiCong Li <sicong.li@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Benchmark: Arm Jenkins <bsgcomp@arm.com>
23 files changed, 1152 insertions, 621 deletions
diff --git a/arm_compute/runtime/NEON/functions/NEScale.h b/arm_compute/runtime/NEON/functions/NEScale.h index 0b7dddacb2..0920ff3802 100644 --- a/arm_compute/runtime/NEON/functions/NEScale.h +++ b/arm_compute/runtime/NEON/functions/NEScale.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -66,16 +66,19 @@ public: * |F16 |F16 | * |F32 |F32 | * |U8 |U8 | + * |S8 |S8 | * |S16 |S16 | * - * @param[in, out] input Source tensor. Data type supported: QASYMM8/QASYMM8_SIGNED/U8/S16/F16/F32. (Written to only for @p border_mode != UNDEFINED) + * @param[in, out] input Source tensor. Data type supported: QASYMM8/QASYMM8_SIGNED/U8/S8/S16/F16/F32. (Written to only for @p border_mode != UNDEFINED) * @param[out] output Destination tensor. Data type supported: Same as @p input. All but the lowest two dimensions must be the same size as in the input tensor, i.e. scaling is only performed within the XY-plane. * @param[in] info @ref ScaleKernelInfo to be used for configuration + * + * @note Using S8 data type only supports NHWC, @p border_mode Replicate, and @p policy Bilinear */ void configure(ITensor *input, ITensor *output, const ScaleKernelInfo &info); /** Static function to check if given info will lead to a valid configuration of @ref NEScale * - * @param[in] input Source tensor. Data type supported: QASYMM8/QASYMM8_SIGNED/U8/S16/F16/F32. (Written to only for @p border_mode != UNDEFINED) + * @param[in] input Source tensor. Data type supported: QASYMM8/QASYMM8_SIGNED/U8/S8/S16/F16/F32. (Written to only for @p border_mode != UNDEFINED) * @param[in] output Destination tensor. Data type supported: Same as @p input. All but the lowest two dimensions must be the same size as in the input tensor, i.e. scaling is only performed within the XY-plane. * @param[in] info @ref ScaleKernelInfo to be used for validation * diff --git a/docs/user_guide/operator_list.dox b/docs/user_guide/operator_list.dox index 51a72bf2d7..8d34a763a5 100644 --- a/docs/user_guide/operator_list.dox +++ b/docs/user_guide/operator_list.dox @@ -2784,6 +2784,7 @@ where N = batches, C = channels, H = height, W = width, D = depth <tr><td>F16<td>F16 <tr><td>F32<td>F32 <tr><td>U8<td>U8 + <tr><td>S8<td>S8 <tr><td>S16<td>S16 </table> <tr> diff --git a/filelist.json b/filelist.json index eb39915524..431979ec41 100644 --- a/filelist.json +++ b/filelist.json @@ -1980,8 +1980,8 @@ "neon": { "fp16": [ "src/cpu/kernels/scale/neon/fp16.cpp" ], "integer": [ "src/cpu/kernels/scale/neon/integer.cpp" ], - "qasymm8": [ "src/cpu/kernels/scale/neon/qasymm8.cpp" ], - "qasymm8_signed": [ "src/cpu/kernels/scale/neon/qasymm8_signed.cpp" ] + "qasymm8": [ "src/cpu/kernels/scale/neon/qasymm8.cpp", "src/cpu/kernels/scale/neon/integer.cpp" ], + "qasymm8_signed": [ "src/cpu/kernels/scale/neon/qasymm8_signed.cpp", "src/cpu/kernels/scale/neon/integer.cpp" ] } } }, diff --git a/src/core/NEON/wrapper/intrinsics/cvt.h b/src/core/NEON/wrapper/intrinsics/cvt.h index e52e3dd0c4..baad1319b2 100644 --- a/src/core/NEON/wrapper/intrinsics/cvt.h +++ b/src/core/NEON/wrapper/intrinsics/cvt.h @@ -59,19 +59,35 @@ VCVT_TO_F16_IMPL(float16x4_t, float32x4_t, vcvt, f16, f32) #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC template <typename T> -inline typename std::enable_if<std::is_same<T, uint8_t>::value, uint32x4_t>::type +inline typename std::enable_if < std::is_same<T, uint8_t>::value || std::is_same<T, uint32_t>::value, uint32x4_t >::type vcvt(const float32x4_t &a) { return vcvtq_u32_f32(a); } template <typename T> -inline typename std::enable_if<std::is_same<T, int8_t>::value, int32x4_t>::type +inline typename std::enable_if < std::is_same<T, int8_t>::value || std::is_same<T, int32_t>::value, int32x4_t >::type vcvt(const float32x4_t &a) { return vcvtq_s32_f32(a); } +#ifdef __aarch64__ +template <typename T> +inline typename std::enable_if<std::is_same<T, uint32_t>::value, uint32x4_t>::type +vcvta(const float32x4_t &a) +{ + return vcvtaq_u32_f32(a); +} + +template <typename T> +inline typename std::enable_if<std::is_same<T, int32_t>::value, int32x4_t>::type +vcvta(const float32x4_t &a) +{ + return vcvtaq_s32_f32(a); +} +#endif //__aarch64__ + #if defined(ARM_COMPUTE_ENABLE_BF16) /** Convert 2x128-bit floating point vectors into 1x128-bit bfloat16 vector * diff --git a/src/core/utils/ScaleUtils.cpp b/src/core/utils/ScaleUtils.cpp index 82c6405e89..ee57a8e7a7 100644 --- a/src/core/utils/ScaleUtils.cpp +++ b/src/core/utils/ScaleUtils.cpp @@ -40,12 +40,26 @@ float arm_compute::scale_utils::calculate_resize_ratio(size_t input_size, size_t return static_cast<float>(in) / static_cast<float>(out); } -bool arm_compute::scale_utils::is_precomputation_required(DataLayout data_layout, DataType data_type, InterpolationPolicy policy) +bool arm_compute::scale_utils::is_precomputation_required(DataLayout data_layout, DataType data_type, + InterpolationPolicy policy, BorderMode border_mode) { - // whether to precompute indices & weights - // The Neon™ kernels (which are preferred over SVE when policy is BILINEAR) do not use - // precomputed index and weights when data type is FP32/16. - // If policy is nearest_neighbor for SVE, then precompute because it's being used - // To be revised in COMPMID-5453/5454 - return data_layout != DataLayout::NHWC || (data_type != DataType::F32 && data_type != DataType::F16) || (CPUInfo::get().get_isa().sve == true && policy == InterpolationPolicy::NEAREST_NEIGHBOR); + // Do not calculate precomputed weights and indices if kernel code doesn't use them + if(data_layout == DataLayout::NHWC) + { + switch(data_type) + { + case DataType::F32: + case DataType::F16: + return (CPUInfo::get().get_isa().sve == true && policy == InterpolationPolicy::NEAREST_NEIGHBOR); + case DataType::U8: + case DataType::S8: + case DataType::QASYMM8: + case DataType::QASYMM8_SIGNED: + return (border_mode != BorderMode::REPLICATE) || (policy == InterpolationPolicy::NEAREST_NEIGHBOR); + default: + return true; + } + } + + return true; }
\ No newline at end of file diff --git a/src/core/utils/ScaleUtils.h b/src/core/utils/ScaleUtils.h index c09509253c..1484824a7f 100644 --- a/src/core/utils/ScaleUtils.h +++ b/src/core/utils/ScaleUtils.h @@ -26,9 +26,6 @@ #include "arm_compute/core/Types.h" -#include <cstdint> -#include <cstdlib> - namespace arm_compute { namespace scale_utils @@ -59,10 +56,11 @@ inline bool is_align_corners_allowed_sampling_policy(SamplingPolicy sampling_pol * @param[in] data_layout Data layout * @param[in] data_type Data type * @param[in] policy Interpolation policy + * @param[in] border_mode Border Mode * * @return True if precomputation is required */ -bool is_precomputation_required(DataLayout data_layout, DataType data_type, InterpolationPolicy policy); +bool is_precomputation_required(DataLayout data_layout, DataType data_type, InterpolationPolicy policy, BorderMode border_mode); } // namespace scale_utils } // namespace arm_compute diff --git a/src/cpu/kernels/CpuScaleKernel.cpp b/src/cpu/kernels/CpuScaleKernel.cpp index e7386a385a..b8bb5ad18a 100644 --- a/src/cpu/kernels/CpuScaleKernel.cpp +++ b/src/cpu/kernels/CpuScaleKernel.cpp @@ -25,14 +25,9 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Window.h" -#include "arm_compute/core/utils/misc/Utility.h" -#include "src/core/CPP/Validate.h" -#include "src/core/NEON/wrapper/wrapper.h" #include "src/core/common/Registrars.h" -#include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/ScaleHelpers.h" #include "src/core/helpers/WindowHelpers.h" -#include "src/core/utils/ScaleUtils.h" #include "src/cpu/kernels/scale/neon/list.h" #include "src/cpu/kernels/scale/sve/list.h" #include "support/Rounding.h" @@ -68,22 +63,34 @@ static const std::vector<CpuScaleKernel::ScaleKernel> available_kernels = }, { "sve_qu8_scale", - [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::QASYMM8 && data.isa.sve; }, + [](const ScaleKernelDataTypeISASelectorData & data) + { + return data.dt == DataType::QASYMM8 && data.isa.sve && data.interpolation_policy != InterpolationPolicy::BILINEAR; + }, REGISTER_QASYMM8_SVE(arm_compute::cpu::qasymm8_sve_scale) }, { "sve_qs8_scale", - [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::QASYMM8_SIGNED && data.isa.sve; }, + [](const ScaleKernelDataTypeISASelectorData & data) + { + return data.dt == DataType::QASYMM8_SIGNED && data.isa.sve && data.interpolation_policy != InterpolationPolicy::BILINEAR; + }, REGISTER_QASYMM8_SIGNED_SVE(arm_compute::cpu::qasymm8_signed_sve_scale) }, { "sve_u8_scale", - [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::U8 && data.isa.sve; }, + [](const ScaleKernelDataTypeISASelectorData & data) + { + return data.dt == DataType::U8 && data.isa.sve && data.interpolation_policy != InterpolationPolicy::BILINEAR; + }, REGISTER_INTEGER_SVE(arm_compute::cpu::u8_sve_scale) }, { "sve_s16_scale", - [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::S16 && data.isa.sve; }, + [](const ScaleKernelDataTypeISASelectorData & data) + { + return data.dt == DataType::S16 && data.isa.sve && data.interpolation_policy != InterpolationPolicy::BILINEAR; + }, REGISTER_INTEGER_SVE(arm_compute::cpu::s16_sve_scale) }, { @@ -112,6 +119,11 @@ static const std::vector<CpuScaleKernel::ScaleKernel> available_kernels = REGISTER_INTEGER_NEON(arm_compute::cpu::u8_neon_scale) }, { + "neon_s8_scale", + [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::S8; }, + REGISTER_INTEGER_NEON(arm_compute::cpu::s8_neon_scale) + }, + { "neon_s16_scale", [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::S16; }, REGISTER_INTEGER_NEON(arm_compute::cpu::s16_neon_scale) @@ -140,6 +152,9 @@ Status validate_arguments(const ITensorInfo *src, const ITensorInfo *dx, const I ARM_COMPUTE_RETURN_ERROR_ON(output_width == 0); ARM_COMPUTE_RETURN_ERROR_ON(output_height == 0); + ARM_COMPUTE_RETURN_ERROR_ON((src->data_type() == DataType::S8) && (data_layout != DataLayout::NHWC || info.interpolation_policy != InterpolationPolicy::BILINEAR + || info.border_mode != BorderMode::REPLICATE)); + if(info.interpolation_policy == InterpolationPolicy::NEAREST_NEIGHBOR && offsets != nullptr) { ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(offsets, 1, DataType::S32); diff --git a/src/cpu/kernels/CpuScaleKernel.h b/src/cpu/kernels/CpuScaleKernel.h index 416e115796..8102142fc3 100644 --- a/src/cpu/kernels/CpuScaleKernel.h +++ b/src/cpu/kernels/CpuScaleKernel.h @@ -50,8 +50,9 @@ public: * * @note dx, dy and offsets have the same dimensions (width and height) of the output tensor * @note Using @p policy Area only supports data layout NCHW and input data type U8. + * @note Using S8 data type only supports NHWC, @p border_mode Replicate, and @p policy Bilinear * - * @param[in] src Source tensor info. Data types supported: QASYMM8/QASYMM8_SIGNED/U8/S16/F16/F32. + * @param[in] src Source tensor info. Data types supported: QASYMM8/QASYMM8_SIGNED/U8/S8/S16/F16/F32. * @param[in] dx Distance x tensor info. Pixel's distance between the X real coordinate and the smallest X following integer. Data type supported: F32 * @param[in] dy Distance y tensor info. Pixel's distance between the Y real coordinate and the smallest Y following integer. Data type supported: F32 * @param[in] offsets Offset tensor info. Offset to access the pixel with NEAREST interpolation or the top-left pixel with BILINEAR interpolation in the input tensor. Data type supported: S32. diff --git a/src/cpu/kernels/scale/neon/integer.cpp b/src/cpu/kernels/scale/neon/integer.cpp index a2359aac94..2ab14cf83a 100644 --- a/src/cpu/kernels/scale/neon/integer.cpp +++ b/src/cpu/kernels/scale/neon/integer.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,17 +22,12 @@ * SOFTWARE. */ #include "arm_compute/core/Helpers.h" -#include "arm_compute/core/ITensorPack.h" -#include "arm_compute/core/Window.h" -#include "src/core/NEON/NEMath.h" #include "src/core/NEON/wrapper/wrapper.h" #include "src/core/helpers/ScaleHelpers.h" #include "src/core/utils/ScaleUtils.h" #include "support/Rounding.h" #include <arm_neon.h> -#include <cmath> -#include <cstddef> namespace arm_compute { @@ -84,37 +79,39 @@ void u8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window) { - // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); - - Iterator out(dst, window); - const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; - const int in_dim_w = src->info()->dimension(1); - const int in_dim_h = src->info()->dimension(2); - const int in_stride_wc = in_stride_c * (in_dim_w + src->info()->padding().top + src->info()->padding().bottom); + // Compute the ratio between source and destination dimensions + const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); - // Don't increment in Y and Z direction for the input tensor - // A pointer to the start of this plane is needed as base for the precomputed offsets - Window win_in(window); - win_in.set(Window::DimY, Window::Dimension(0, 0, 0)); - win_in.set(Window::DimZ, Window::Dimension(0, 0, 0)); - Iterator in(src, win_in); + const int input_width = src->info()->dimension(1); + const int input_height = src->info()->dimension(2); if(border_mode == BorderMode::CONSTANT) { + Iterator out(dst, window); + const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; + const int in_stride_wc = in_stride_c * (input_width + src->info()->padding().top + src->info()->padding().bottom); + + // Don't increment in Y and Z direction for the input tensor + // A pointer to the start of this plane is needed as base for the precomputed offsets + Window win_in(window); + win_in.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in); + const uint8_t const_border_value = static_cast<uint8_t>(constant_border_value.get<uint8_t>()); execute_window_loop(window, [&](const Coordinates & id) { const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); + const int32_t in_hi = std::floor((id.z() + sampling_offset) * scale_y - sampling_offset); const uint8_t *in_ptr = reinterpret_cast<const uint8_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; - const auto a00 = (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value; - const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) ? *(in_ptr + in_stride_c) : const_border_value; - const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_wc) : const_border_value; - const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value; + const auto a00 = (0 <= offset && offset < input_width && 0 <= in_hi && in_hi < input_height) ? *in_ptr : const_border_value; + const auto a01 = (-1 <= offset && offset < input_width - 1 && 0 <= in_hi && in_hi < input_height) ? *(in_ptr + in_stride_c) : const_border_value; + const auto a10 = (0 <= offset && offset < input_width && -1 <= in_hi && in_hi < input_height - 1) ? *(in_ptr + in_stride_wc) : const_border_value; + const auto a11 = (-1 <= offset && offset < input_width - 1 && -1 <= in_hi && in_hi < input_height - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value; *reinterpret_cast<uint8_t *>(out.ptr()) = static_cast<uint8_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); }, @@ -122,26 +119,395 @@ void u8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off } else if(border_mode == BorderMode::REPLICATE) { - execute_window_loop(window, [&](const Coordinates & id) + using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>; + + const int in_stride_x = src->info()->strides_in_bytes()[1]; + const int in_stride_y = src->info()->strides_in_bytes()[2]; + const int in_stride_b = src->info()->strides_in_bytes()[3]; + const int out_stride_x = dst->info()->strides_in_bytes()[1]; + const int out_stride_y = dst->info()->strides_in_bytes()[2]; + const int out_stride_b = dst->info()->strides_in_bytes()[3]; + + const int out_dim_ch = dst->info()->dimension(0); + constexpr int step_cout = 16; + + Window window_execution = window; + window_execution.set(Window::DimX, Window::Dimension(0, 1, 1)); + Window win_in_out(window); + win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in_out); + Iterator out(dst, win_in_out); + + const int xo_start = window_execution[1].start(); + const int xo_end = window_execution[1].end(); + const int xo_step = window_execution[1].step(); + const int yo_start = window_execution[2].start(); + const int yo_end = window_execution[2].end(); + const int yo_step = window_execution[2].step(); + const int bo_start = window_execution[3].start(); + const int bo_end = window_execution[3].end(); + const int bo_step = window_execution[3].step(); + + const float fp_coord_offset_y = sampling_offset * (scale_y - 1); + const float fp_coord_offset_x = sampling_offset * (scale_x - 1); + + for(int bo = bo_start; bo < bo_end; bo += bo_step) { - const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - - auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1); - - const auto a00 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc); - const auto a01 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc); - const auto a10 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h1 * in_stride_wc); - const auto a11 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h1 * in_stride_wc); + const uint8_t *in_ptr = in.ptr() + bo * in_stride_b; + uint8_t *out_ptr = out.ptr() + bo * out_stride_b; + + for(int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + const float yi_f = yo * scale_y + fp_coord_offset_y; + // Integer coordinate + const int yi = static_cast<int>(std::floor(yi_f)); + // Weight for the y coordinate + const float a1 = (yi_f - static_cast<float>(yi)); + const float b1 = (1.f - a1); + + const int yi0 = utility::clamp<int>(yi, 0, input_height - 1); + const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1); + + const uint8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y; + const uint8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; + + uint8_t *out_ptr_yo = out_ptr + yo * out_stride_y; + for(int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + const float xi_f = xo * scale_x + fp_coord_offset_x; + // Integer coordinate + const int xi = static_cast<int>(std::floor(xi_f)); + // Weight for the x coordinate + const float a = (xi_f - static_cast<float>(xi)); + const float b = (1.f - a); + + const float s00_s = b * b1; + const float s01_s = a * b1; + const float s10_s = b * a1; + const float s11_s = a * a1; + + const auto s00 = wrapper::vdup_n(s00_s, ExactTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, ExactTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, ExactTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, ExactTagType{}); + + const int xi0 = utility::clamp<int>(xi, 0, input_width - 1); + const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1); + + const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x; + const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x; + + uint8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; + + int cout = 0; + for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); + const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); + const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(uint8_t)); + const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(uint8_t)); + + const uint16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00)); + const uint16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00)); + + const auto in00_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_low))); + const auto in00_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_low))); + const auto in00_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_high))); + const auto in00_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_high))); + + const uint16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01)); + const uint16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01)); + + const auto in01_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_low))); + const auto in01_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_low))); + const auto in01_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_high))); + const auto in01_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_high))); + + const uint16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10)); + const uint16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10)); + + const auto in10_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_low))); + const auto in10_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_low))); + const auto in10_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_high))); + const auto in10_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_high))); + + const uint16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11)); + const uint16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11)); + + const auto in11_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_low))); + const auto in11_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_low))); + const auto in11_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_high))); + const auto in11_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_high))); + + auto out_0 = wrapper::vmul(in00_0, s00); + out_0 = wrapper::vmla(out_0, in01_0, s01); + out_0 = wrapper::vmla(out_0, in10_0, s10); + out_0 = wrapper::vmla(out_0, in11_0, s11); + + auto out_1 = wrapper::vmul(in00_1, s00); + out_1 = wrapper::vmla(out_1, in01_1, s01); + out_1 = wrapper::vmla(out_1, in10_1, s10); + out_1 = wrapper::vmla(out_1, in11_1, s11); + + auto out_2 = wrapper::vmul(in00_2, s00); + out_2 = wrapper::vmla(out_2, in01_2, s01); + out_2 = wrapper::vmla(out_2, in10_2, s10); + out_2 = wrapper::vmla(out_2, in11_2, s11); + + auto out_3 = wrapper::vmul(in00_3, s00); + out_3 = wrapper::vmla(out_3, in01_3, s01); + out_3 = wrapper::vmla(out_3, in10_3, s10); + out_3 = wrapper::vmla(out_3, in11_3, s11); + +#if defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvta<uint32_t>(out_0); + const auto out_1_int = wrapper::vcvta<uint32_t>(out_1); + const auto out_2_int = wrapper::vcvta<uint32_t>(out_2); + const auto out_3_int = wrapper::vcvta<uint32_t>(out_3); +#else // defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvt<uint32_t>(out_0); + const auto out_1_int = wrapper::vcvt<uint32_t>(out_1); + const auto out_2_int = wrapper::vcvt<uint32_t>(out_2); + const auto out_3_int = wrapper::vcvt<uint32_t>(out_3); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); + + wrapper::vstore(out_ptr_xo_yo + cout * sizeof(uint8_t), out); + } + + for(; cout < out_dim_ch; ++cout) + { + const uint8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); + const uint8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); + const uint8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(uint8_t)); + const uint8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(uint8_t)); + + float out0 = in00 * s00_s; + out0 += in01 * s01_s; + out0 += in10 * s10_s; + out0 += in11 * s11_s; + + // Rounding modes of vector and scalar loops should match +#if defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = static_cast<uint8_t>(std::round(out0)); +#else // defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = static_cast<uint8_t>(out0); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + } + } + } + } + } + else + { + ARM_COMPUTE_ERROR("Not implemented"); + } +} - *reinterpret_cast<uint8_t *>(out.ptr()) = static_cast<uint8_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); +void s8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, + BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, + bool align_corners, const Window &window) +{ + ARM_COMPUTE_UNUSED(dx, dy, offsets, constant_border_value); + if(border_mode == BorderMode::REPLICATE) + { + using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>; + + // Compute the ratio between source and destination dimensions + const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + + const int in_stride_x = src->info()->strides_in_bytes()[1]; + const int in_stride_y = src->info()->strides_in_bytes()[2]; + const int in_stride_b = src->info()->strides_in_bytes()[3]; + const int out_stride_x = dst->info()->strides_in_bytes()[1]; + const int out_stride_y = dst->info()->strides_in_bytes()[2]; + const int out_stride_b = dst->info()->strides_in_bytes()[3]; + const int input_width = src->info()->dimension(1); + const int input_height = src->info()->dimension(2); + const int out_dim_ch = dst->info()->dimension(0); + constexpr int step_cout = 16; + + Window window_execution = window; + window_execution.set(Window::DimX, Window::Dimension(0, 1, 1)); + Window win_in_out(window); + win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in_out); + Iterator out(dst, win_in_out); + + const int xo_start = window_execution[1].start(); + const int xo_end = window_execution[1].end(); + const int xo_step = window_execution[1].step(); + const int yo_start = window_execution[2].start(); + const int yo_end = window_execution[2].end(); + const int yo_step = window_execution[2].step(); + const int bo_start = window_execution[3].start(); + const int bo_end = window_execution[3].end(); + const int bo_step = window_execution[3].step(); + + const float fp_coord_offset_y = sampling_offset * (scale_y - 1); + const float fp_coord_offset_x = sampling_offset * (scale_x - 1); + + for(int bo = bo_start; bo < bo_end; bo += bo_step) + { + const int8_t *in_ptr = reinterpret_cast<int8_t *>(in.ptr() + bo * in_stride_b); + int8_t *out_ptr = reinterpret_cast<int8_t *>(out.ptr() + bo * out_stride_b); + + for(int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + const float yi_f = yo * scale_y + fp_coord_offset_y; + // Integer coordinate + const int yi = static_cast<int>(std::floor(yi_f)); + // Weight for the y coordinate + const float a1 = (yi_f - static_cast<float>(yi)); + const float b1 = (1.f - a1); + + const int yi0 = utility::clamp<int>(yi, 0, input_height - 1); + const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1); + + const int8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y; + const int8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; + + int8_t *out_ptr_yo = out_ptr + yo * out_stride_y; + for(int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + const float xi_f = xo * scale_x + fp_coord_offset_x; + // Integer coordinate + const int xi = static_cast<int>(std::floor(xi_f)); + // Weight for the x coordinate + const float a = (xi_f - static_cast<float>(xi)); + const float b = (1.f - a); + + const float s00_s = b * b1; + const float s01_s = a * b1; + const float s10_s = b * a1; + const float s11_s = a * a1; + + const auto s00 = wrapper::vdup_n(s00_s, ExactTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, ExactTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, ExactTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, ExactTagType{}); + + const int xi0 = utility::clamp<int>(xi, 0, input_width - 1); + const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1); + + const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x; + const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x; + + int8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; + + int cout = 0; + for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); + const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); + const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(int8_t)); + const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(int8_t)); + + const int16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00)); + const int16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00)); + + const auto in00_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_low))); + const auto in00_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_low))); + const auto in00_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_high))); + const auto in00_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_high))); + + const int16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01)); + const int16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01)); + + const auto in01_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_low))); + const auto in01_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_low))); + const auto in01_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_high))); + const auto in01_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_high))); + + const int16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10)); + const int16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10)); + + const auto in10_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_low))); + const auto in10_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_low))); + const auto in10_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_high))); + const auto in10_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_high))); + + const int16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11)); + const int16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11)); + + const auto in11_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_low))); + const auto in11_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_low))); + const auto in11_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_high))); + const auto in11_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_high))); + + auto out_0 = wrapper::vmul(in00_0, s00); + out_0 = wrapper::vmla(out_0, in01_0, s01); + out_0 = wrapper::vmla(out_0, in10_0, s10); + out_0 = wrapper::vmla(out_0, in11_0, s11); + + auto out_1 = wrapper::vmul(in00_1, s00); + out_1 = wrapper::vmla(out_1, in01_1, s01); + out_1 = wrapper::vmla(out_1, in10_1, s10); + out_1 = wrapper::vmla(out_1, in11_1, s11); + + auto out_2 = wrapper::vmul(in00_2, s00); + out_2 = wrapper::vmla(out_2, in01_2, s01); + out_2 = wrapper::vmla(out_2, in10_2, s10); + out_2 = wrapper::vmla(out_2, in11_2, s11); + + auto out_3 = wrapper::vmul(in00_3, s00); + out_3 = wrapper::vmla(out_3, in01_3, s01); + out_3 = wrapper::vmla(out_3, in10_3, s10); + out_3 = wrapper::vmla(out_3, in11_3, s11); + +#if defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvta<int32_t>(out_0); + const auto out_1_int = wrapper::vcvta<int32_t>(out_1); + const auto out_2_int = wrapper::vcvta<int32_t>(out_2); + const auto out_3_int = wrapper::vcvta<int32_t>(out_3); +#else // defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvt<int32_t>(out_0); + const auto out_1_int = wrapper::vcvt<int32_t>(out_1); + const auto out_2_int = wrapper::vcvt<int32_t>(out_2); + const auto out_3_int = wrapper::vcvt<int32_t>(out_3); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); + + wrapper::vstore(out_ptr_xo_yo + cout * sizeof(int8_t), out); + } + + for(; cout < out_dim_ch; ++cout) + { + const int8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); + const int8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); + const int8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(int8_t)); + const int8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(int8_t)); + + float out0 = in00 * s00_s; + out0 += in01 * s01_s; + out0 += in10 * s10_s; + out0 += in11 * s11_s; + + // Rounding modes of vector and scalar loops should match +#if defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(int8_t)) = static_cast<int8_t>(std::round(out0)); +#else // defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(int8_t)) = static_cast<int8_t>(out0); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + } + } + } + } } else { @@ -240,10 +606,10 @@ void s16_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *of const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1); + const auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1); + const auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1); + const auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1); + const auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1); const auto a00 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc); const auto a01 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc); @@ -262,6 +628,20 @@ void s16_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *of } namespace cpu { +void s8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, + InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, + bool align_corners, const Window &window) +{ + if(policy == InterpolationPolicy::BILINEAR) + { + s8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + } + else + { + ARM_COMPUTE_ERROR("Not implemented"); + } +} + void u8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window) diff --git a/src/cpu/kernels/scale/neon/list.h b/src/cpu/kernels/scale/neon/list.h index 17ff4bb676..28a1087224 100644 --- a/src/cpu/kernels/scale/neon/list.h +++ b/src/cpu/kernels/scale/neon/list.h @@ -25,11 +25,8 @@ #define SRC_CORE_NEON_KERNELS_SCALE_LIST_H #include "arm_compute/core/Helpers.h" -#include "arm_compute/core/ITensorPack.h" #include "arm_compute/core/Window.h" -#include "src/core/NEON/NEMath.h" #include "src/core/NEON/wrapper/wrapper.h" -#include "src/core/helpers/ScaleHelpers.h" #include "src/core/utils/ScaleUtils.h" #include "support/Rounding.h" @@ -44,6 +41,7 @@ namespace cpu DECLARE_SCALE_KERNEL(s16_neon_scale); DECLARE_SCALE_KERNEL(u8_neon_scale); +DECLARE_SCALE_KERNEL(s8_neon_scale); DECLARE_SCALE_KERNEL(qasymm8_neon_scale); DECLARE_SCALE_KERNEL(qasymm8_signed_neon_scale); @@ -360,10 +358,10 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset int cout = 0; for(; cout <= (out_dim_ch - step_cout); cout += step_cout) { - auto in00 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); - auto in01 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); - auto in10 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); - auto in11 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); + const auto in00 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); + const auto in01 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); + const auto in10 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); + const auto in11 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); auto out0 = wrapper::vmul(in00, s00); out0 = wrapper::vmla(out0, in01, s01); @@ -374,10 +372,10 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset for(; cout < out_dim_ch; ++cout) { - T in00 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); - T in01 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); - T in10 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); - T in11 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); + const T in00 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); + const T in01 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); + const T in10 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); + const T in11 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); T out0 = in00 * s00_s; out0 += in01 * s01_s; diff --git a/src/cpu/kernels/scale/neon/qasymm8.cpp b/src/cpu/kernels/scale/neon/qasymm8.cpp index daa157ec3d..778459ae39 100644 --- a/src/cpu/kernels/scale/neon/qasymm8.cpp +++ b/src/cpu/kernels/scale/neon/qasymm8.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -21,6 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include "src/core/helpers/ScaleHelpers.h" #include "src/cpu/kernels/scale/neon/list.h" namespace arm_compute @@ -32,56 +33,60 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor bool align_corners, const Window &window) { // Data layout is NHWC - // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); - Window win_off; - win_off.set(Window::DimX, Window::Dimension(0, 0, 0)); - win_off.set(Window::DimY, Window::Dimension(0, 0, 0)); - - // Don't increment in X and Y direction for the input tensor - // A pointer to the start of this plane is needed as base for the precomputed offsets - Window win_in(window); - win_in.set(1, Window::Dimension(0, 0, 0)); - win_in.set(2, Window::Dimension(0, 0, 0)); - - for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) - { - win_off.set(d, Window::Dimension(0, 0, 0)); - } - - Iterator in(src, win_in); - Iterator out(dst, window); - - const int32_t in_dim_w = src->info()->dimension(1); - const int32_t in_dim_h = src->info()->dimension(2); - const int32_t stride_w = src->info()->strides_in_bytes()[1]; - const int32_t stride_h = src->info()->strides_in_bytes()[2]; + const int32_t input_width = src->info()->dimension(1); + const int32_t input_height = src->info()->dimension(2); const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); + // Compute the ratio between source and destination dimensions + const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + if(border_mode == BorderMode::CONSTANT) { + const int32_t in_stride_y = src->info()->strides_in_bytes()[1]; + const int32_t in_stride_z = src->info()->strides_in_bytes()[2]; + + // Compute the ratio between source height and destination height + Window win_off; + win_off.set(Window::DimX, Window::Dimension(0, 0, 0)); + win_off.set(Window::DimY, Window::Dimension(0, 0, 0)); + + // Don't increment in X and Y direction for the input tensor + // A pointer to the start of this plane is needed as base for the precomputed offsets + Window win_in(window); + win_in.set(1, Window::Dimension(0, 0, 0)); + win_in.set(2, Window::Dimension(0, 0, 0)); + + for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) + { + win_off.set(d, Window::Dimension(0, 0, 0)); + } + + Iterator in(src, win_in); + Iterator out(dst, window); + const uint8_t const_border_value = static_cast<uint8_t>(constant_border_value.get<uint8_t>()); execute_window_loop(window, [&](const Coordinates & id) { - const int32_t index_h = std::floor((id[2] + sampling_offset) * hr - sampling_offset); + const int32_t index_h = std::floor((id[2] + sampling_offset) * scale_y - sampling_offset); const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2])))); const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2])))); const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2])))); const auto pixel_row_ptr = reinterpret_cast<const uint8_t *>(in.ptr()); - const auto a00 = (0 <= index_w && index_w < in_dim_w && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + index_w * stride_w + index_h * stride_h)) : + const auto a00 = (0 <= index_w && index_w < input_width && 0 <= index_h && index_h < input_height) ? + (*(pixel_row_ptr + index_w * in_stride_y + index_h * in_stride_z)) : const_border_value; - const auto a01 = (-1 <= index_w && index_w + 1 < in_dim_w && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + index_h * stride_h)) : + const auto a01 = (-1 <= index_w && index_w + 1 < input_width && 0 <= index_h && index_h < input_height) ? + (*(pixel_row_ptr + (index_w + 1) * in_stride_y + index_h * in_stride_z)) : const_border_value; - const auto a10 = (0 <= index_w && index_w < in_dim_w && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + index_w * stride_w + (index_h + 1) * stride_h)) : + const auto a10 = (0 <= index_w && index_w < input_width && -1 <= index_h && index_h < input_height - 1) ? + (*(pixel_row_ptr + index_w * in_stride_y + (index_h + 1) * in_stride_z)) : const_border_value; - const auto a11 = (-1 <= index_w && index_w < in_dim_w - 1 && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + (index_h + 1) * stride_h)) : + const auto a11 = (-1 <= index_w && index_w < input_width - 1 && -1 <= index_h && index_h < input_height - 1) ? + (*(pixel_row_ptr + (index_w + 1) * in_stride_y + (index_h + 1) * in_stride_z)) : const_border_value; const float inp00 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a00, iq_info); @@ -94,31 +99,205 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor } else if(border_mode == BorderMode::REPLICATE) { - execute_window_loop(window, [&](const Coordinates & id) + using FloatTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>; + using Int32TagType = typename wrapper::traits::neon_bitvector_tag_t<int32_t, wrapper::traits::BitWidth::W128>; + + const int in_stride_x = src->info()->strides_in_bytes()[1]; + const int in_stride_y = src->info()->strides_in_bytes()[2]; + const int in_stride_b = src->info()->strides_in_bytes()[3]; + const int out_stride_x = dst->info()->strides_in_bytes()[1]; + const int out_stride_y = dst->info()->strides_in_bytes()[2]; + const int out_stride_b = dst->info()->strides_in_bytes()[3]; + const int out_dim_ch = dst->info()->dimension(0); + constexpr int step_cout = 16; + + Window window_execution = window; + window_execution.set(Window::DimX, Window::Dimension(0, 1, 1)); + Window win_in_out(window); + win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in_out); + Iterator out(dst, win_in_out); + + const int xo_start = window_execution[1].start(); + const int xo_end = window_execution[1].end(); + const int xo_step = window_execution[1].step(); + const int yo_start = window_execution[2].start(); + const int yo_end = window_execution[2].end(); + const int yo_step = window_execution[2].step(); + const int bo_start = window_execution[3].start(); + const int bo_end = window_execution[3].end(); + const int bo_step = window_execution[3].step(); + + const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); + const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); + + const float32x4_t vscale_in = wrapper::vdup_n(iq_info.scale, FloatTagType{}); + const int32x4_t voffset_in = wrapper::vdup_n(iq_info.offset, Int32TagType{}); // Offsets will be Int32 + + const float32x4_t invvscale_o = wrapper::vdup_n(1.f / oq_info.scale, FloatTagType{}); + const float32x4_t voffset_o = vdupq_n_f32(oq_info.offset); + + const float fp_coord_offset_y = sampling_offset * (scale_y - 1); + const float fp_coord_offset_x = sampling_offset * (scale_x - 1); + + for(int bo = bo_start; bo < bo_end; bo += bo_step) { - const int index_h = std::floor((id[2] + sampling_offset) * hr - sampling_offset); - const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2])))); - const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2])))); - const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2])))); - const auto pixel_row_ptr = reinterpret_cast<const uint8_t *>(in.ptr()); + const uint8_t *in_ptr = in.ptr() + bo * in_stride_b; + uint8_t *out_ptr = out.ptr() + bo * out_stride_b; - auto clamped_w = utility::clamp<int>(index_w, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp<int>(index_w + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp<int>(index_h, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp<int>(index_h + 1, 0, in_dim_h - 1); + for(int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + const float yi_f = yo * scale_y + fp_coord_offset_y; + // Integer coordinate + const int yi = static_cast<int>(std::floor(yi_f)); + // Weight for the y coordinate + const float a1 = (yi_f - static_cast<float>(yi)); + const float b1 = (1.f - a1); - const auto a00 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h * stride_h); - const auto a01 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h * stride_h); - const auto a10 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h1 * stride_h); - const auto a11 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h1 * stride_h); + const int yi0 = utility::clamp<int>(yi, 0, input_height - 1); + const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1); - const float inp00 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a00, iq_info); - const float inp01 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a01, iq_info); - const float inp10 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a10, iq_info); - const float inp11 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a11, iq_info); - *reinterpret_cast<uint8_t *>(out.ptr()) = Qasymm8QuantizationHelper<uint8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); - }, - in, out); + const uint8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y; + const uint8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; + + uint8_t *out_ptr_yo = out_ptr + yo * out_stride_y; + for(int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + const float xi_f = xo * scale_x + fp_coord_offset_x; + // Integer coordinate + const int xi = static_cast<int>(std::floor(xi_f)); + // Weight for the x coordinate + const float a = (xi_f - static_cast<float>(xi)); + const float b = (1.f - a); + + const float s00_s = b * b1; + const float s01_s = a * b1; + const float s10_s = b * a1; + const float s11_s = a * a1; + + const auto s00 = wrapper::vdup_n(s00_s, FloatTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, FloatTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, FloatTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, FloatTagType{}); + + const int xi0 = utility::clamp<int>(xi, 0, input_width - 1); + const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1); + + const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x; + const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x; + + uint8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; + + int cout = 0; + for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); + const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); + const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(uint8_t)); + const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(uint8_t)); + + const uint16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00)); + const uint16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00)); + + const auto in00_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in00_low))), voffset_in)), vscale_in); + const auto in00_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in00_low))), voffset_in)), vscale_in); + const auto in00_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in00_high))), voffset_in)), vscale_in); + const auto in00_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in00_high))), voffset_in)), vscale_in); + + const uint16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01)); + const uint16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01)); + + const auto in01_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in01_low))), voffset_in)), vscale_in); + const auto in01_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in01_low))), voffset_in)), vscale_in); + const auto in01_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in01_high))), voffset_in)), vscale_in); + const auto in01_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in01_high))), voffset_in)), vscale_in); + + const uint16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10)); + const uint16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10)); + + const auto in10_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in10_low))), voffset_in)), vscale_in); + const auto in10_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in10_low))), voffset_in)), vscale_in); + const auto in10_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in10_high))), voffset_in)), vscale_in); + const auto in10_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in10_high))), voffset_in)), vscale_in); + + const uint16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11)); + const uint16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11)); + + const auto in11_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in11_low))), voffset_in)), vscale_in); + const auto in11_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in11_low))), voffset_in)), vscale_in); + const auto in11_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in11_high))), voffset_in)), vscale_in); + const auto in11_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in11_high))), voffset_in)), vscale_in); + + auto out_0 = wrapper::vmul(in00_0, s00); + out_0 = wrapper::vmla(out_0, in01_0, s01); + out_0 = wrapper::vmla(out_0, in10_0, s10); + out_0 = wrapper::vmla(out_0, in11_0, s11); + + auto out_1 = wrapper::vmul(in00_1, s00); + out_1 = wrapper::vmla(out_1, in01_1, s01); + out_1 = wrapper::vmla(out_1, in10_1, s10); + out_1 = wrapper::vmla(out_1, in11_1, s11); + + auto out_2 = wrapper::vmul(in00_2, s00); + out_2 = wrapper::vmla(out_2, in01_2, s01); + out_2 = wrapper::vmla(out_2, in10_2, s10); + out_2 = wrapper::vmla(out_2, in11_2, s11); + + auto out_3 = wrapper::vmul(in00_3, s00); + out_3 = wrapper::vmla(out_3, in01_3, s01); + out_3 = wrapper::vmla(out_3, in10_3, s10); + out_3 = wrapper::vmla(out_3, in11_3, s11); + +#if defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o)); + const auto out_1_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o)); + const auto out_2_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o)); + const auto out_3_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o)); +#else // defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o)); + const auto out_1_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o)); + const auto out_2_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o)); + const auto out_3_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o)); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); + + wrapper::vstore(out_ptr_xo_yo + cout * sizeof(uint8_t), out); + } + + for(; cout < out_dim_ch; ++cout) + { + const uint8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); + const uint8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); + const uint8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(uint8_t)); + const uint8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(uint8_t)); + + const float in00_f = (static_cast<int32_t>(in00) - iq_info.offset) * iq_info.scale; + const float in01_f = (static_cast<int32_t>(in01) - iq_info.offset) * iq_info.scale; + const float in10_f = (static_cast<int32_t>(in10) - iq_info.offset) * iq_info.scale; + const float in11_f = (static_cast<int32_t>(in11) - iq_info.offset) * iq_info.scale; + + float out = in00_f * s00_s; + out += in01_f * s01_s; + out += in10_f * s10_s; + out += in11_f * s11_s; + + // Rounding modes of vector and scalar loops should match +#if defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = quantize_qasymm8(out, oq_info); +#else // defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = quantize_qasymm8(out, oq_info, RoundingPolicy::TO_ZERO); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + } + } + } + } } else { @@ -134,7 +313,14 @@ void qasymm8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets { if(policy == InterpolationPolicy::BILINEAR) { - qasymm8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + if(src->info()->quantization_info() == dst->info()->quantization_info()) + { + u8_neon_scale(src, dst, offsets, dx, dy, policy, border_mode, constant_border_value, sampling_offset, align_corners, window); + } + else + { + qasymm8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + } } else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) { diff --git a/src/cpu/kernels/scale/neon/qasymm8_signed.cpp b/src/cpu/kernels/scale/neon/qasymm8_signed.cpp index 83312636b4..cd63dfba63 100644 --- a/src/cpu/kernels/scale/neon/qasymm8_signed.cpp +++ b/src/cpu/kernels/scale/neon/qasymm8_signed.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -21,6 +21,7 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ +#include "src/core/helpers/ScaleHelpers.h" #include "src/cpu/kernels/scale/neon/list.h" namespace arm_compute @@ -32,56 +33,59 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const bool align_corners, const Window &window) { // Data layout is NHWC - // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); - Window win_off; - win_off.set(Window::DimX, Window::Dimension(0, 0, 0)); - win_off.set(Window::DimY, Window::Dimension(0, 0, 0)); - - // Don't increment in X and Y direction for the input tensor - // A pointer to the start of this plane is needed as base for the precomputed offsets - Window win_in(window); - win_in.set(1, Window::Dimension(0, 0, 0)); - win_in.set(2, Window::Dimension(0, 0, 0)); - - for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) - { - win_off.set(d, Window::Dimension(0, 0, 0)); - } - - Iterator in(src, win_in); - Iterator out(dst, window); - - const int32_t in_dim_w = src->info()->dimension(1); - const int32_t in_dim_h = src->info()->dimension(2); - const int32_t stride_w = src->info()->strides_in_bytes()[1]; - const int32_t stride_h = src->info()->strides_in_bytes()[2]; - const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); + const int32_t input_width = src->info()->dimension(1); + const int32_t input_height = src->info()->dimension(2); + + // Compute the ratio between source and destination dimensions + const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + if(border_mode == BorderMode::CONSTANT) { + const int32_t in_stride_y = src->info()->strides_in_bytes()[1]; + const int32_t in_stride_z = src->info()->strides_in_bytes()[2]; + + Window win_off; + win_off.set(Window::DimX, Window::Dimension(0, 0, 0)); + win_off.set(Window::DimY, Window::Dimension(0, 0, 0)); + + // Don't increment in X and Y direction for the input tensor + // A pointer to the start of this plane is needed as base for the precomputed offsets + Window win_in(window); + win_in.set(1, Window::Dimension(0, 0, 0)); + win_in.set(2, Window::Dimension(0, 0, 0)); + + for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) + { + win_off.set(d, Window::Dimension(0, 0, 0)); + } + + Iterator in(src, win_in); + Iterator out(dst, window); + const int8_t const_border_value = static_cast<int8_t>(constant_border_value.get<int8_t>()); execute_window_loop(window, [&](const Coordinates & id) { - const int32_t index_h = std::floor((id[2] + sampling_offset) * hr - sampling_offset); + const int32_t index_h = std::floor((id[2] + sampling_offset) * scale_y - sampling_offset); const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2])))); const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2])))); const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2])))); const auto pixel_row_ptr = reinterpret_cast<const int8_t *>(in.ptr()); - const auto a00 = (0 <= index_w && index_w < in_dim_w && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + index_w * stride_w + index_h * stride_h)) : + const auto a00 = (0 <= index_w && index_w < input_width && 0 <= index_h && index_h < input_height) ? + (*(pixel_row_ptr + index_w * in_stride_y + index_h * in_stride_z)) : const_border_value; - const auto a01 = (-1 <= index_w && index_w + 1 < in_dim_w && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + index_h * stride_h)) : + const auto a01 = (-1 <= index_w && index_w + 1 < input_width && 0 <= index_h && index_h < input_height) ? + (*(pixel_row_ptr + (index_w + 1) * in_stride_y + index_h * in_stride_z)) : const_border_value; - const auto a10 = (0 <= index_w && index_w < in_dim_w && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + index_w * stride_w + (index_h + 1) * stride_h)) : + const auto a10 = (0 <= index_w && index_w < input_width && -1 <= index_h && index_h < input_height - 1) ? + (*(pixel_row_ptr + index_w * in_stride_y + (index_h + 1) * in_stride_z)) : const_border_value; - const auto a11 = (-1 <= index_w && index_w < in_dim_w - 1 && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + (index_h + 1) * stride_h)) : + const auto a11 = (-1 <= index_w && index_w < input_width - 1 && -1 <= index_h && index_h < input_height - 1) ? + (*(pixel_row_ptr + (index_w + 1) * in_stride_y + (index_h + 1) * in_stride_z)) : const_border_value; const float inp00 = Qasymm8QuantizationHelper<int8_t>::dequantize(a00, iq_info); @@ -94,31 +98,205 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const } else if(border_mode == BorderMode::REPLICATE) { - execute_window_loop(window, [&](const Coordinates & id) + using FloatTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>; + using Int32TagType = typename wrapper::traits::neon_bitvector_tag_t<int32_t, wrapper::traits::BitWidth::W128>; + + const int in_stride_x = src->info()->strides_in_bytes()[1]; + const int in_stride_y = src->info()->strides_in_bytes()[2]; + const int in_stride_b = src->info()->strides_in_bytes()[3]; + const int out_stride_x = dst->info()->strides_in_bytes()[1]; + const int out_stride_y = dst->info()->strides_in_bytes()[2]; + const int out_stride_b = dst->info()->strides_in_bytes()[3]; + const int out_dim_ch = dst->info()->dimension(0); + constexpr int step_cout = 16; + + Window window_execution = window; + window_execution.set(Window::DimX, Window::Dimension(0, 1, 1)); + Window win_in_out(window); + win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0)); + win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0)); + Iterator in(src, win_in_out); + Iterator out(dst, win_in_out); + + const int xo_start = window_execution[1].start(); + const int xo_end = window_execution[1].end(); + const int xo_step = window_execution[1].step(); + const int yo_start = window_execution[2].start(); + const int yo_end = window_execution[2].end(); + const int yo_step = window_execution[2].step(); + const int bo_start = window_execution[3].start(); + const int bo_end = window_execution[3].end(); + const int bo_step = window_execution[3].step(); + + const float fp_coord_offset_y = sampling_offset * (scale_y - 1); + const float fp_coord_offset_x = sampling_offset * (scale_x - 1); + + const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); + const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); + + const float32x4_t vscale_in = wrapper::vdup_n(iq_info.scale, FloatTagType{}); + const int32x4_t voffset_in = wrapper::vdup_n(iq_info.offset, Int32TagType{}); // Offsets will be Int32 + + const float32x4_t invvscale_o = wrapper::vdup_n(1.f / oq_info.scale, FloatTagType{}); + const float32x4_t voffset_o = vdupq_n_f32(oq_info.offset); + + for(int bo = bo_start; bo < bo_end; bo += bo_step) { - const int index_h = std::floor((id[2] + sampling_offset) * hr - sampling_offset); - const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2])))); - const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2])))); - const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2])))); - const auto pixel_row_ptr = reinterpret_cast<const int8_t *>(in.ptr()); + const int8_t *in_ptr = reinterpret_cast<int8_t *>(in.ptr() + bo * in_stride_b); + int8_t *out_ptr = reinterpret_cast<int8_t *>(out.ptr() + bo * out_stride_b); - auto clamped_w = utility::clamp<int>(index_w, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp<int>(index_w + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp<int>(index_h, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp<int>(index_h + 1, 0, in_dim_h - 1); + for(int yo = yo_start; yo < yo_end; yo += yo_step) + { + // Floating-point coordinate + const float yi_f = yo * scale_y + fp_coord_offset_y; + // Integer coordinate + const int yi = static_cast<int>(std::floor(yi_f)); + // Weight for the y coordinate + const float a1 = (yi_f - static_cast<float>(yi)); + const float b1 = (1.f - a1); - const auto a00 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h * stride_h); - const auto a01 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h * stride_h); - const auto a10 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h1 * stride_h); - const auto a11 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h1 * stride_h); + const int yi0 = utility::clamp<int>(yi, 0, input_height - 1); + const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1); - const float inp00 = Qasymm8QuantizationHelper<int8_t>::dequantize(a00, iq_info); - const float inp01 = Qasymm8QuantizationHelper<int8_t>::dequantize(a01, iq_info); - const float inp10 = Qasymm8QuantizationHelper<int8_t>::dequantize(a10, iq_info); - const float inp11 = Qasymm8QuantizationHelper<int8_t>::dequantize(a11, iq_info); - *reinterpret_cast<int8_t *>(out.ptr()) = Qasymm8QuantizationHelper<int8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); - }, - in, out); + const int8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y; + const int8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; + + int8_t *out_ptr_yo = out_ptr + yo * out_stride_y; + for(int xo = xo_start; xo < xo_end; xo += xo_step) + { + // Floating-point coordinate + const float xi_f = xo * scale_x + fp_coord_offset_x; + // Integer coordinate + const int xi = static_cast<int>(std::floor(xi_f)); + // Weight for the x coordinate + const float a = (xi_f - static_cast<float>(xi)); + const float b = (1.f - a); + + const float s00_s = b * b1; + const float s01_s = a * b1; + const float s10_s = b * a1; + const float s11_s = a * a1; + + const auto s00 = wrapper::vdup_n(s00_s, FloatTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, FloatTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, FloatTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, FloatTagType{}); + + const int xi0 = utility::clamp<int>(xi, 0, input_width - 1); + const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1); + + const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x; + const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x; + const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x; + + int8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; + + int cout = 0; + for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + { + const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); + const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); + const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(int8_t)); + const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(int8_t)); + + const int16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00)); + const int16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00)); + + const auto in00_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in00_low)), voffset_in)), vscale_in); + const auto in00_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in00_low)), voffset_in)), vscale_in); + const auto in00_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in00_high)), voffset_in)), vscale_in); + const auto in00_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in00_high)), voffset_in)), vscale_in); + + const int16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01)); + const int16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01)); + + const auto in01_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in01_low)), voffset_in)), vscale_in); + const auto in01_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in01_low)), voffset_in)), vscale_in); + const auto in01_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in01_high)), voffset_in)), vscale_in); + const auto in01_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in01_high)), voffset_in)), vscale_in); + + const int16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10)); + const int16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10)); + + const auto in10_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in10_low)), voffset_in)), vscale_in); + const auto in10_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in10_low)), voffset_in)), vscale_in); + const auto in10_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in10_high)), voffset_in)), vscale_in); + const auto in10_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in10_high)), voffset_in)), vscale_in); + + const int16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11)); + const int16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11)); + + const auto in11_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in11_low)), voffset_in)), vscale_in); + const auto in11_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in11_low)), voffset_in)), vscale_in); + const auto in11_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in11_high)), voffset_in)), vscale_in); + const auto in11_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in11_high)), voffset_in)), vscale_in); + + auto out_0 = wrapper::vmul(in00_0, s00); + out_0 = wrapper::vmla(out_0, in01_0, s01); + out_0 = wrapper::vmla(out_0, in10_0, s10); + out_0 = wrapper::vmla(out_0, in11_0, s11); + + auto out_1 = wrapper::vmul(in00_1, s00); + out_1 = wrapper::vmla(out_1, in01_1, s01); + out_1 = wrapper::vmla(out_1, in10_1, s10); + out_1 = wrapper::vmla(out_1, in11_1, s11); + + auto out_2 = wrapper::vmul(in00_2, s00); + out_2 = wrapper::vmla(out_2, in01_2, s01); + out_2 = wrapper::vmla(out_2, in10_2, s10); + out_2 = wrapper::vmla(out_2, in11_2, s11); + + auto out_3 = wrapper::vmul(in00_3, s00); + out_3 = wrapper::vmla(out_3, in01_3, s01); + out_3 = wrapper::vmla(out_3, in10_3, s10); + out_3 = wrapper::vmla(out_3, in11_3, s11); + +#if defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o)); + const auto out_1_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o)); + const auto out_2_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o)); + const auto out_3_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o)); +#else // defined(__aarch64__) && !defined(BARE_METAL) + const auto out_0_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o)); + const auto out_1_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o)); + const auto out_2_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o)); + const auto out_3_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o)); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); + + wrapper::vstore(out_ptr_xo_yo + cout * sizeof(int8_t), out); + } + + for(; cout < out_dim_ch; ++cout) + { + const int8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); + const int8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); + const int8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(int8_t)); + const int8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(int8_t)); + + const float in00_f = (static_cast<int32_t>(in00) - iq_info.offset) * iq_info.scale; + const float in01_f = (static_cast<int32_t>(in01) - iq_info.offset) * iq_info.scale; + const float in10_f = (static_cast<int32_t>(in10) - iq_info.offset) * iq_info.scale; + const float in11_f = (static_cast<int32_t>(in11) - iq_info.offset) * iq_info.scale; + + float out = in00_f * s00_s; + out += in01_f * s01_s; + out += in10_f * s10_s; + out += in11_f * s11_s; + + // Rounding modes of vector and scalar loops should match +#if defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(int8_t)) = quantize_qasymm8_signed(out, oq_info); +#else // defined(__aarch64__) && !defined(BARE_METAL) + *(out_ptr_xo_yo + cout * sizeof(int8_t)) = quantize_qasymm8_signed(out, oq_info, RoundingPolicy::TO_ZERO); +#endif // defined(__aarch64__) && !defined(BARE_METAL) + } + } + } + } } else { @@ -134,7 +312,14 @@ void qasymm8_signed_neon_scale(const ITensor *src, ITensor *dst, const ITensor * { if(policy == InterpolationPolicy::BILINEAR) { - qasymm8_signed_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + if(src->info()->quantization_info() == dst->info()->quantization_info() && border_mode == BorderMode::REPLICATE) + { + s8_neon_scale(src, dst, offsets, dx, dy, policy, border_mode, constant_border_value, sampling_offset, align_corners, window); + } + else + { + qasymm8_signed_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + } } else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) { diff --git a/src/cpu/kernels/scale/sve/integer.cpp b/src/cpu/kernels/scale/sve/integer.cpp index 00a43922d9..82c70ee360 100644 --- a/src/cpu/kernels/scale/sve/integer.cpp +++ b/src/cpu/kernels/scale/sve/integer.cpp @@ -83,75 +83,6 @@ void u8_sve_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offse out); } -void u8_sve_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) -{ - // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); - - Iterator out(dst, window); - const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; - const int in_dim_w = src->info()->dimension(1); - const int in_dim_h = src->info()->dimension(2); - const int in_stride_wc = in_stride_c * (in_dim_w + src->info()->padding().top + src->info()->padding().bottom); - - // Don't increment in Y and Z direction for the input tensor - // A pointer to the start of this plane is needed as base for the precomputed offsets - Window win_in(window); - win_in.set(Window::DimY, Window::Dimension(0, 0, 0)); - win_in.set(Window::DimZ, Window::Dimension(0, 0, 0)); - Iterator in(src, win_in); - - if(border_mode == BorderMode::CONSTANT) - { - const uint8_t const_border_value = static_cast<uint8_t>(constant_border_value.get<uint8_t>()); - execute_window_loop(window, [&](const Coordinates & id) - { - const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - const uint8_t *in_ptr = reinterpret_cast<const uint8_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; - - const auto a00 = (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value; - const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) ? *(in_ptr + in_stride_c) : const_border_value; - const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_wc) : const_border_value; - const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value; - - *reinterpret_cast<uint8_t *>(out.ptr()) = static_cast<uint8_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); - } - else if(border_mode == BorderMode::REPLICATE) - { - execute_window_loop(window, [&](const Coordinates & id) - { - const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - - auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1); - - const auto a00 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc); - const auto a01 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc); - const auto a10 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h1 * in_stride_wc); - const auto a11 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h1 * in_stride_wc); - - *reinterpret_cast<uint8_t *>(out.ptr()) = static_cast<uint8_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); - } - else - { - ARM_COMPUTE_ERROR("Not implemented"); - } -} - void s16_sve_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offsets, float sampling_offset, bool align_corners, const Window &window) { @@ -195,75 +126,6 @@ void s16_sve_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offs }, out); } - -void s16_sve_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) -{ - // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); - - Iterator out(dst, window); - const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; - const int in_dim_w = src->info()->dimension(1); - const int in_dim_h = src->info()->dimension(2); - const int in_stride_wc = in_stride_c * (in_dim_w + src->info()->padding().top + src->info()->padding().bottom); - - // Don't increment in Y and Z direction for the input tensor - // A pointer to the start of this plane is needed as base for the precomputed offsets - Window win_in(window); - win_in.set(Window::DimY, Window::Dimension(0, 0, 0)); - win_in.set(Window::DimZ, Window::Dimension(0, 0, 0)); - Iterator in(src, win_in); - - if(border_mode == BorderMode::CONSTANT) - { - const int16_t const_border_value = static_cast<int16_t>(constant_border_value.get<int16_t>()); - execute_window_loop(window, [&](const Coordinates & id) - { - const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - const int16_t *in_ptr = reinterpret_cast<const int16_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; - - const auto a00 = (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value; - const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) ? *(in_ptr + in_stride_c) : const_border_value; - const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_wc) : const_border_value; - const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value; - - *reinterpret_cast<int16_t *>(out.ptr()) = static_cast<int16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); - } - else if(border_mode == BorderMode::REPLICATE) - { - execute_window_loop(window, [&](const Coordinates & id) - { - const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - - auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1); - - const auto a00 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc); - const auto a01 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc); - const auto a10 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h1 * in_stride_wc); - const auto a11 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h1 * in_stride_wc); - - *reinterpret_cast<int16_t *>(out.ptr()) = static_cast<int16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); - } - else - { - ARM_COMPUTE_ERROR("Not implemented"); - } -} } namespace cpu { @@ -271,13 +133,14 @@ void u8_sve_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, cons InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + ARM_COMPUTE_UNUSED(dx, dy, border_mode, constant_border_value); + if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) { - u8_sve_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + u8_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); } - else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) + else { - u8_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); + ARM_COMPUTE_ERROR("Not Implemented"); } } @@ -285,13 +148,14 @@ void s16_sve_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, con InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + ARM_COMPUTE_UNUSED(dx, dy, border_mode, constant_border_value); + if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) { - s16_sve_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + s16_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); } - else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) + else { - s16_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); + ARM_COMPUTE_ERROR("Not Implemented"); } } } // namespace cpu diff --git a/src/cpu/kernels/scale/sve/qasymm8.cpp b/src/cpu/kernels/scale/sve/qasymm8.cpp index 09ef00a783..d45a69e43b 100644 --- a/src/cpu/kernels/scale/sve/qasymm8.cpp +++ b/src/cpu/kernels/scale/sve/qasymm8.cpp @@ -83,108 +83,6 @@ void qasymm8_sve_scale_nearest(const ITensor *src, ITensor *dst, const ITensor * }, out); } - -void qasymm8_sve_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) -{ - // Data layout is NHWC - const int idx_width = 1; - const int idx_height = 2; - - // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(idx_height), dst->info()->dimension(idx_height), align_corners); - Window win_off; - win_off.set(Window::DimX, Window::Dimension(0, 0, 0)); - win_off.set(Window::DimY, Window::Dimension(0, 0, 0)); - - // Don't increment in X and Y direction for the input tensor - // A pointer to the start of this plane is needed as base for the precomputed offsets - Window win_in(window); - win_in.set(idx_width, Window::Dimension(0, 0, 0)); - win_in.set(idx_height, Window::Dimension(0, 0, 0)); - - for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) - { - win_off.set(d, Window::Dimension(0, 0, 0)); - } - - Iterator in(src, win_in); - Iterator out(dst, window); - - const int32_t in_dim_w = src->info()->dimension(idx_width); - const int32_t in_dim_h = src->info()->dimension(idx_height); - const int32_t stride_w = src->info()->strides_in_bytes()[idx_width]; - const int32_t stride_h = src->info()->strides_in_bytes()[idx_height]; - - const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); - const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); - - if(border_mode == BorderMode::CONSTANT) - { - const uint8_t const_border_value = static_cast<uint8_t>(constant_border_value.get<uint8_t>()); - execute_window_loop(window, [&](const Coordinates & id) - { - const int32_t index_h = std::floor((id[idx_height] + sampling_offset) * hr - sampling_offset); - const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto pixel_row_ptr = reinterpret_cast<const uint8_t *>(in.ptr()); - - const auto a00 = (0 <= index_w && index_w < in_dim_w && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + index_w * stride_w + index_h * stride_h)) : - const_border_value; - const auto a01 = (-1 <= index_w && index_w < in_dim_w - 1 && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + index_h * stride_h)) : - const_border_value; - const auto a10 = (0 <= index_w && index_w < in_dim_w && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + index_w * stride_w + (index_h + 1) * stride_h)) : - const_border_value; - const auto a11 = (-1 <= index_w && index_w < in_dim_w - 1 && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + (index_h + 1) * stride_h)) : - const_border_value; - - const float inp00 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a00, iq_info); - const float inp01 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a01, iq_info); - const float inp10 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a10, iq_info); - const float inp11 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a11, iq_info); - *reinterpret_cast<uint8_t *>(out.ptr()) = Qasymm8QuantizationHelper<uint8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); - }, - in, out); - } - else if(border_mode == BorderMode::REPLICATE) - { - execute_window_loop(window, [&](const Coordinates & id) - { - const int index_h = std::floor((id[idx_height] + sampling_offset) * hr - sampling_offset); - const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto pixel_row_ptr = reinterpret_cast<const uint8_t *>(in.ptr()); - - auto clamped_w = utility::clamp<int>(index_w, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp<int>(index_w + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp<int>(index_h, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp<int>(index_h + 1, 0, in_dim_h - 1); - - const auto a00 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h * stride_h); - const auto a01 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h * stride_h); - const auto a10 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h1 * stride_h); - const auto a11 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h1 * stride_h); - - const float inp00 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a00, iq_info); - const float inp01 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a01, iq_info); - const float inp10 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a10, iq_info); - const float inp11 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a11, iq_info); - *reinterpret_cast<uint8_t *>(out.ptr()) = Qasymm8QuantizationHelper<uint8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); - }, - in, out); - } - else - { - ARM_COMPUTE_ERROR("Not implemented"); - } -} } namespace cpu { @@ -192,13 +90,14 @@ void qasymm8_sve_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + ARM_COMPUTE_UNUSED(dx, dy, border_mode, constant_border_value); + if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) { - qasymm8_sve_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + qasymm8_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); } - else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) + else { - qasymm8_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); + ARM_COMPUTE_ERROR("Not Implemented"); } } } // namespace cpu diff --git a/src/cpu/kernels/scale/sve/qasymm8_signed.cpp b/src/cpu/kernels/scale/sve/qasymm8_signed.cpp index 63f515442b..67bca65f58 100644 --- a/src/cpu/kernels/scale/sve/qasymm8_signed.cpp +++ b/src/cpu/kernels/scale/sve/qasymm8_signed.cpp @@ -83,108 +83,6 @@ void qasymm8_signed_sve_scale_nearest(const ITensor *src, ITensor *dst, const IT }, out); } - -void qasymm8_signed_sve_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) -{ - // Data layout is NHWC - const int idx_width = 1; - const int idx_height = 2; - - // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(idx_height), dst->info()->dimension(idx_height), align_corners); - Window win_off; - win_off.set(Window::DimX, Window::Dimension(0, 0, 0)); - win_off.set(Window::DimY, Window::Dimension(0, 0, 0)); - - // Don't increment in X and Y direction for the input tensor - // A pointer to the start of this plane is needed as base for the precomputed offsets - Window win_in(window); - win_in.set(idx_width, Window::Dimension(0, 0, 0)); - win_in.set(idx_height, Window::Dimension(0, 0, 0)); - - for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) - { - win_off.set(d, Window::Dimension(0, 0, 0)); - } - - Iterator in(src, win_in); - Iterator out(dst, window); - - const int32_t in_dim_w = src->info()->dimension(idx_width); - const int32_t in_dim_h = src->info()->dimension(idx_height); - const int32_t stride_w = src->info()->strides_in_bytes()[idx_width]; - const int32_t stride_h = src->info()->strides_in_bytes()[idx_height]; - - const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); - const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); - - if(border_mode == BorderMode::CONSTANT) - { - const int8_t const_border_value = static_cast<int8_t>(constant_border_value.get<int8_t>()); - execute_window_loop(window, [&](const Coordinates & id) - { - const int32_t index_h = std::floor((id[idx_height] + sampling_offset) * hr - sampling_offset); - const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto pixel_row_ptr = reinterpret_cast<const int8_t *>(in.ptr()); - - const auto a00 = (0 <= index_w && index_w < in_dim_w && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + index_w * stride_w + index_h * stride_h)) : - const_border_value; - const auto a01 = (-1 <= index_w && index_w < in_dim_w - 1 && 0 <= index_h && index_h < in_dim_h) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + index_h * stride_h)) : - const_border_value; - const auto a10 = (0 <= index_w && index_w < in_dim_w && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + index_w * stride_w + (index_h + 1) * stride_h)) : - const_border_value; - const auto a11 = (-1 <= index_w && index_w < in_dim_w - 1 && -1 <= index_h && index_h < in_dim_h - 1) ? - (*(pixel_row_ptr + (index_w + 1) * stride_w + (index_h + 1) * stride_h)) : - const_border_value; - - const float inp00 = Qasymm8QuantizationHelper<int8_t>::dequantize(a00, iq_info); - const float inp01 = Qasymm8QuantizationHelper<int8_t>::dequantize(a01, iq_info); - const float inp10 = Qasymm8QuantizationHelper<int8_t>::dequantize(a10, iq_info); - const float inp11 = Qasymm8QuantizationHelper<int8_t>::dequantize(a11, iq_info); - *reinterpret_cast<int8_t *>(out.ptr()) = Qasymm8QuantizationHelper<int8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); - }, - in, out); - } - else if(border_mode == BorderMode::REPLICATE) - { - execute_window_loop(window, [&](const Coordinates & id) - { - const int index_h = std::floor((id[idx_height] + sampling_offset) * hr - sampling_offset); - const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[idx_width], id[idx_height])))); - const auto pixel_row_ptr = reinterpret_cast<const int8_t *>(in.ptr()); - - auto clamped_w = utility::clamp<int>(index_w, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp<int>(index_w + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp<int>(index_h, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp<int>(index_h + 1, 0, in_dim_h - 1); - - const auto a00 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h * stride_h); - const auto a01 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h * stride_h); - const auto a10 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h1 * stride_h); - const auto a11 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h1 * stride_h); - - const float inp00 = Qasymm8QuantizationHelper<int8_t>::dequantize(a00, iq_info); - const float inp01 = Qasymm8QuantizationHelper<int8_t>::dequantize(a01, iq_info); - const float inp10 = Qasymm8QuantizationHelper<int8_t>::dequantize(a10, iq_info); - const float inp11 = Qasymm8QuantizationHelper<int8_t>::dequantize(a11, iq_info); - *reinterpret_cast<int8_t *>(out.ptr()) = Qasymm8QuantizationHelper<int8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); - }, - in, out); - } - else - { - ARM_COMPUTE_ERROR("Not implemented"); - } -} } namespace cpu { @@ -192,13 +90,14 @@ void qasymm8_signed_sve_scale(const ITensor *src, ITensor *dst, const ITensor *o InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, bool align_corners, const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + ARM_COMPUTE_UNUSED(dx, dy, border_mode, constant_border_value); + if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) { - qasymm8_signed_sve_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + qasymm8_signed_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); } - else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) + else { - qasymm8_signed_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); + ARM_COMPUTE_ERROR("Not Implemented"); } } } // namespace cpu diff --git a/src/cpu/operators/CpuScale.cpp b/src/cpu/operators/CpuScale.cpp index fdb52e5ede..a13a0f56a2 100644 --- a/src/cpu/operators/CpuScale.cpp +++ b/src/cpu/operators/CpuScale.cpp @@ -215,9 +215,9 @@ void CpuScale::prepare(ITensorPack &tensors) _scale_info.interpolation_policy; const SamplingPolicy sampling_policy = _scale_info.sampling_policy; - bool precompute_indices_weights = arm_compute::scale_utils::is_precomputation_required(_data_layout, src->info()->data_type(), policy_to_use); + bool precompute_indices_weights = arm_compute::scale_utils::is_precomputation_required(_data_layout, src->info()->data_type(), policy_to_use, _scale_info.border_mode); - if(precompute_indices_weights == true) + if(precompute_indices_weights) { switch(policy_to_use) { diff --git a/src/cpu/operators/CpuScale.h b/src/cpu/operators/CpuScale.h index f605af6712..ee7c523bad 100644 --- a/src/cpu/operators/CpuScale.h +++ b/src/cpu/operators/CpuScale.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,6 +45,8 @@ public: * @param[in, out] src Source tensor info. Data type supported: QASYMM8/QASYMM8_SIGNED/U8/S16/F16/F32. (Written to only for @p border_mode != UNDEFINED) * @param[out] dst Destination tensor info. Data type supported: Same as @p src. All but the lowest two dimensions must be the same size as in the input tensor, i.e. scaling is only performed within the XY-plane. * @param[in] info @ref ScaleKernelInfo to be used for configuration + * + * @note Using S8 data type only supports NHWC, @p border_mode Replicate, and @p policy Bilinear */ void configure(ITensorInfo *src, ITensorInfo *dst, const ScaleKernelInfo &info); /** Static function to check if given info will lead to a valid configuration diff --git a/src/runtime/NEON/functions/NEScale.cpp b/src/runtime/NEON/functions/NEScale.cpp index 74ab860d91..686017f7c1 100644 --- a/src/runtime/NEON/functions/NEScale.cpp +++ b/src/runtime/NEON/functions/NEScale.cpp @@ -23,12 +23,9 @@ */ #include "arm_compute/runtime/NEON/functions/NEScale.h" -#include "arm_compute/core/Validate.h" -#include "arm_compute/runtime/Tensor.h" #include "src/common/utils/Log.h" #include "src/core/utils/ScaleUtils.h" #include "src/cpu/operators/CpuScale.h" -#include "support/Rounding.h" namespace arm_compute { @@ -75,9 +72,9 @@ void NEScale::configure(ITensor *input, ITensor *output, const ScaleKernelInfo & TensorShape shape(output->info()->dimension(idx_width)); shape.set(1, output->info()->dimension(idx_height), false); - bool precompute_indices_weights = arm_compute::scale_utils::is_precomputation_required(data_layout, input->info()->data_type(), policy_to_use); + bool precompute_indices_weights = arm_compute::scale_utils::is_precomputation_required(data_layout, input->info()->data_type(), policy_to_use, info.border_mode); - if(precompute_indices_weights == true) + if(precompute_indices_weights) { const TensorInfo tensor_info_dxdy(shape, Format::F32); const TensorInfo tensor_info_offsets(shape, Format::S32); diff --git a/tests/datasets/ScaleValidationDataset.h b/tests/datasets/ScaleValidationDataset.h index 11e0343582..c6987c0908 100644 --- a/tests/datasets/ScaleValidationDataset.h +++ b/tests/datasets/ScaleValidationDataset.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2021 Arm Limited. + * Copyright (c) 2020-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -24,12 +24,8 @@ #ifndef ARM_COMPUTE_TEST_SCALE_VALIDATION_DATASET #define ARM_COMPUTE_TEST_SCALE_VALIDATION_DATASET -#include "utils/TypePrinter.h" - -#include "arm_compute/core/TensorShape.h" #include "arm_compute/core/Types.h" #include "tests/datasets/BorderModeDataset.h" -#include "tests/datasets/InterpolationPolicyDataset.h" #include "tests/datasets/SamplingPolicyDataset.h" #include "tests/datasets/ShapeDatasets.h" @@ -149,7 +145,7 @@ framework::dataset::make("AlignCorners", { true })); concat(concat(concat(ScaleShapesBaseDataSet<1, 1, (element_per_iteration), 0>(), \ ScaleShapesBaseDataSet<1, 1, (element_per_iteration), 2>()), \ ScaleShapesBaseDataSet<3, 1, (element_per_iteration), 1>()), \ - ScaleShapesBaseDataSet<3, 3, (element_per_iteration), 0>()) + ScaleShapesBaseDataSet<40, 3, (element_per_iteration), 0>()) // To prevent long precommit time for OpenCL, shape set for OpenCL is separated into below two parts. /** Generated shapes for precommits to achieve essential coverage. Used by CL precommit and nightly @@ -170,13 +166,19 @@ framework::dataset::make("AlignCorners", { true })); ScaleShapesBaseDataSet<3, 1, (element_per_iteration), 0>()), \ ScaleShapesBaseDataSet<3, 3, (element_per_iteration), 0>()) -/** Generating dataset for non-quantized data tyeps with the given shapes */ +/** Generating dataset for non-quantized data types with the given shapes */ #define ASSEMBLE_DATASET(shape, samping_policy_set) \ combine(combine(combine(combine((shape), ScaleDataLayouts), \ ScaleInterpolationPolicySet), \ datasets::BorderModes()), \ samping_policy_set) +#define ASSEMBLE_S8_DATASET(shape, samping_policy_set) \ + combine(combine(combine(combine((shape), framework::dataset::make("DataLayout", DataLayout::NHWC)), \ + framework::dataset::make("InterpolationPolicy", { InterpolationPolicy::BILINEAR })), \ + framework::dataset::make("BorderMode", { BorderMode::REPLICATE })), \ + samping_policy_set) + #define ASSEMBLE_NHWC_DATASET(shape, samping_policy_set) \ combine(combine(combine(combine((shape), framework::dataset::make("DataLayout", DataLayout::NHWC)), \ ScaleInterpolationPolicySet), \ @@ -192,6 +194,16 @@ framework::dataset::make("AlignCorners", { true })); datasets::BorderModes()), \ sampling_policy_set) +/** Generating dataset for quantized data tyeps with the given shapes */ +#define ASSEMBLE_DIFFERENTLY_QUANTIZED_DATASET(shape, sampling_policy_set, input_quant_info_set, output_quant_info_set) \ + combine(combine(combine(combine(combine(combine(shape, \ + input_quant_info_set), \ + output_quant_info_set), \ + framework::dataset::make("DataLayout", { DataLayout::NHWC })), \ + framework::dataset::make("InterpolationPolicy", { InterpolationPolicy::BILINEAR })), \ + framework::dataset::make("BorderMode", { BorderMode::REPLICATE })), \ + sampling_policy_set) + } // namespace datasets } // namespace test } // namespace arm_compute diff --git a/tests/validation/NEON/Scale.cpp b/tests/validation/NEON/Scale.cpp index e386d804ca..9927e21490 100644 --- a/tests/validation/NEON/Scale.cpp +++ b/tests/validation/NEON/Scale.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,16 +22,10 @@ * SOFTWARE. */ #include "arm_compute/core/Helpers.h" -#include "arm_compute/core/Types.h" #include "arm_compute/runtime/NEON/functions/NEScale.h" -#include "arm_compute/runtime/Tensor.h" -#include "arm_compute/runtime/TensorAllocator.h" #include "tests/NEON/Accessor.h" -#include "tests/PaddingCalculator.h" #include "tests/datasets/ScaleValidationDataset.h" -#include "tests/framework/Asserts.h" #include "tests/framework/Macros.h" -#include "tests/validation/Helpers.h" #include "tests/validation/Validation.h" #include "tests/validation/fixtures/ScaleFixture.h" @@ -51,7 +45,7 @@ using datasets::ScaleAlignCornersSamplingPolicySet; /** We consider vector size in byte 64 since the maximum size of * a vector used by the kernel is currently 64-byte (float32x4x4). - * There are possibility to reduce test time further by using + * There is possibility to reduce test time further by using * smaller vector sizes for different data types where applicable. */ constexpr uint32_t vector_byte = 64; @@ -62,26 +56,31 @@ constexpr uint32_t num_elements_per_vector() return vector_byte / sizeof(T); } -/** Scale data types */ -const auto ScaleDataTypes = framework::dataset::make("DataType", +/** Quantization information data set */ +const auto QuantizationInfoSet = framework::dataset::make("QuantizationInfo", { - DataType::U8, - DataType::S16, - DataType::F32, + QuantizationInfo(0.5f, -10), }); /** Quantization information data set */ -const auto QuantizationInfoSet = framework::dataset::make("QuantizationInfo", +const auto InputQuantizationInfoSet = framework::dataset::make("InputQuantizationInfo", { QuantizationInfo(0.5f, -10), }); +/** Quantization information data set */ +const auto OutputQuantizationInfoSet = framework::dataset::make("OutputQuantizationInfo", +{ + QuantizationInfo(0.2f, 20), +}); + /** Tolerance */ constexpr AbsoluteTolerance<uint8_t> tolerance_u8(1); +constexpr AbsoluteTolerance<int8_t> tolerance_s8(1); constexpr AbsoluteTolerance<int16_t> tolerance_s16(1); RelativeTolerance<float> tolerance_f32(0.05); #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC -constexpr float abs_tolerance_f16(0.01f); +constexpr float abs_tolerance_f16(0.01f); RelativeTolerance<half> tolerance_f16(half(0.1)); #endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ @@ -321,6 +320,8 @@ using NEScaleMixedDataLayoutFixture = ScaleValidationFixture<Tensor, Accessor, N template <typename T> using NEScaleQuantizedFixture = ScaleValidationQuantizedFixture<Tensor, Accessor, NEScale, T>; template <typename T> +using NEScaleDifferentOutputQuantizedFixture = ScaleValidationDifferentOutputQuantizedFixture<Tensor, Accessor, NEScale, T>; +template <typename T> using NEScaleQuantizedMixedDataLayoutFixture = ScaleValidationQuantizedFixture<Tensor, Accessor, NEScale, T, true>; TEST_SUITE(Float) @@ -457,6 +458,27 @@ FIXTURE_DATA_TEST_CASE(RunSmallAlignCorners, NEScaleFixture<uint8_t>, framework: validate(Accessor(_target), _reference, valid_region, tolerance_u8); } TEST_SUITE_END() // U8 +TEST_SUITE(S8) +const auto s8_shape = combine((SCALE_SHAPE_DATASET(num_elements_per_vector<int8_t>())), framework::dataset::make("DataType", DataType::S8)); +FIXTURE_DATA_TEST_CASE(RunSmall, NEScaleFixture<int8_t>, framework::DatasetMode::ALL, ASSEMBLE_S8_DATASET(s8_shape, ScaleSamplingPolicySet)) +{ + //Create valid region + TensorInfo src_info(_shape, 1, _data_type); + ValidRegion valid_region = calculate_valid_region_scale(src_info, _reference.shape(), _policy, _sampling_policy, (_border_mode == BorderMode::UNDEFINED)); + + // Validate output + validate(Accessor(_target), _reference, valid_region, tolerance_s8); +} +FIXTURE_DATA_TEST_CASE(RunSmallAlignCorners, NEScaleFixture<int8_t>, framework::DatasetMode::ALL, ASSEMBLE_S8_DATASET(s8_shape, ScaleAlignCornersSamplingPolicySet)) +{ + //Create valid region + TensorInfo src_info(_shape, 1, _data_type); + ValidRegion valid_region = calculate_valid_region_scale(src_info, _reference.shape(), _policy, _sampling_policy, (_border_mode == BorderMode::UNDEFINED)); + + // Validate output + validate(Accessor(_target), _reference, valid_region, tolerance_s8); +} +TEST_SUITE_END() // S8 TEST_SUITE(S16) const auto s16_shape = combine((SCALE_SHAPE_DATASET(num_elements_per_vector<int16_t>())), framework::dataset::make("DataType", DataType::S16)); FIXTURE_DATA_TEST_CASE(RunSmall, NEScaleFixture<int16_t>, framework::DatasetMode::ALL, ASSEMBLE_DATASET(s16_shape, ScaleSamplingPolicySet)) @@ -492,6 +514,16 @@ FIXTURE_DATA_TEST_CASE(RunSmall, NEScaleQuantizedFixture<uint8_t>, framework::Da // Validate output validate(Accessor(_target), _reference, valid_region, tolerance_u8); } +FIXTURE_DATA_TEST_CASE(RunSmallDifferentOutputQuantization, NEScaleDifferentOutputQuantizedFixture<uint8_t>, framework::DatasetMode::ALL, + ASSEMBLE_DIFFERENTLY_QUANTIZED_DATASET(qasymm8_shape, ScaleSamplingPolicySet, InputQuantizationInfoSet, OutputQuantizationInfoSet)) +{ + //Create valid region + TensorInfo src_info(_shape, 1, _data_type); + ValidRegion valid_region = calculate_valid_region_scale(src_info, _reference.shape(), _policy, _sampling_policy, (_border_mode == BorderMode::UNDEFINED)); + + // Validate output + validate(Accessor(_target), _reference, valid_region, tolerance_u8); +} FIXTURE_DATA_TEST_CASE(RunMixedDataLayout, NEScaleQuantizedMixedDataLayoutFixture<uint8_t>, framework::DatasetMode::ALL, ASSEMBLE_QUANTIZED_DATASET(qasymm8_shape, ScaleSamplingPolicySet, QuantizationInfoSet)) { @@ -525,6 +557,16 @@ FIXTURE_DATA_TEST_CASE(RunSmall, NEScaleQuantizedFixture<int8_t>, framework::Dat // Validate output validate(Accessor(_target), _reference, valid_region, tolerance_qasymm8_signed); } +FIXTURE_DATA_TEST_CASE(RunSmallDifferentOutputQuantization, NEScaleDifferentOutputQuantizedFixture<int8_t>, framework::DatasetMode::ALL, + ASSEMBLE_DIFFERENTLY_QUANTIZED_DATASET(qasymm8_signed_shape, ScaleSamplingPolicySet, InputQuantizationInfoSet, OutputQuantizationInfoSet)) +{ + //Create valid region + TensorInfo src_info(_shape, 1, _data_type); + ValidRegion valid_region = calculate_valid_region_scale(src_info, _reference.shape(), _policy, _sampling_policy, (_border_mode == BorderMode::UNDEFINED)); + + // Validate output + validate(Accessor(_target), _reference, valid_region, tolerance_qasymm8_signed); +} FIXTURE_DATA_TEST_CASE(RunSmallAlignCorners, NEScaleQuantizedFixture<int8_t>, framework::DatasetMode::ALL, ASSEMBLE_QUANTIZED_DATASET(qasymm8_signed_shape, ScaleAlignCornersSamplingPolicySet, QuantizationInfoSet)) { diff --git a/tests/validation/fixtures/ScaleFixture.h b/tests/validation/fixtures/ScaleFixture.h index c0b44bcb5f..72feb62016 100644 --- a/tests/validation/fixtures/ScaleFixture.h +++ b/tests/validation/fixtures/ScaleFixture.h @@ -24,12 +24,6 @@ #ifndef ARM_COMPUTE_TEST_SCALE_FIXTURE #define ARM_COMPUTE_TEST_SCALE_FIXTURE -#include "arm_compute/core/TensorShape.h" -#include "arm_compute/core/Types.h" -#include "tests/AssetsLibrary.h" -#include "tests/Globals.h" -#include "tests/IAccessor.h" -#include "tests/framework/Asserts.h" #include "tests/framework/Fixture.h" #include "tests/validation/reference/Permute.h" #include "tests/validation/reference/Scale.h" @@ -46,16 +40,17 @@ class ScaleValidationGenericFixture : public framework::Fixture public: template <typename...> void setup(TensorShape shape, DataType data_type, QuantizationInfo quantization_info, DataLayout data_layout, InterpolationPolicy policy, BorderMode border_mode, SamplingPolicy sampling_policy, - bool align_corners, bool mixed_layout) + bool align_corners, bool mixed_layout, QuantizationInfo output_quantization_info) { - _shape = shape; - _policy = policy; - _border_mode = border_mode; - _sampling_policy = sampling_policy; - _data_type = data_type; - _quantization_info = quantization_info; - _align_corners = align_corners; - _mixed_layout = mixed_layout; + _shape = shape; + _policy = policy; + _border_mode = border_mode; + _sampling_policy = sampling_policy; + _data_type = data_type; + _input_quantization_info = quantization_info; + _output_quantization_info = output_quantization_info; + _align_corners = align_corners; + _mixed_layout = mixed_layout; generate_scale(shape); @@ -144,7 +139,7 @@ protected: } // Create tensors - TensorType src = create_tensor<TensorType>(shape, _data_type, 1, _quantization_info, data_layout); + TensorType src = create_tensor<TensorType>(shape, _data_type, 1, _input_quantization_info, data_layout); const int idx_width = get_data_layout_dimension_index(data_layout, DataLayoutDimension::WIDTH); const int idx_height = get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT); @@ -152,7 +147,7 @@ protected: TensorShape shape_scaled(shape); shape_scaled.set(idx_width, shape[idx_width] * _scale_x, /* apply_dim_correction = */ false); shape_scaled.set(idx_height, shape[idx_height] * _scale_y, /* apply_dim_correction = */ false); - TensorType dst = create_tensor<TensorType>(shape_scaled, _data_type, 1, _quantization_info, data_layout); + TensorType dst = create_tensor<TensorType>(shape_scaled, _data_type, 1, _output_quantization_info, data_layout); // Create and configure function FunctionType scale; @@ -188,12 +183,12 @@ protected: SimpleTensor<T> compute_reference(const TensorShape &shape) { // Create reference - SimpleTensor<T> src{ shape, _data_type, 1, _quantization_info }; + SimpleTensor<T> src{ shape, _data_type, 1, _input_quantization_info }; // Fill reference fill(src); - return reference::scale<T>(src, _scale_x, _scale_y, _policy, _border_mode, _constant_border_value, _sampling_policy, /* ceil_policy_scale */ false, _align_corners); + return reference::scale<T>(src, _scale_x, _scale_y, _policy, _border_mode, _constant_border_value, _sampling_policy, /* ceil_policy_scale */ false, _align_corners, _output_quantization_info); } TensorType _target{}; @@ -204,7 +199,8 @@ protected: T _constant_border_value{}; SamplingPolicy _sampling_policy{}; DataType _data_type{}; - QuantizationInfo _quantization_info{}; + QuantizationInfo _input_quantization_info{}; + QuantizationInfo _output_quantization_info{}; bool _align_corners{ false }; bool _mixed_layout{ false }; float _scale_x{ 1.f }; @@ -227,7 +223,29 @@ public: border_mode, sampling_policy, align_corners, - mixed_layout); + mixed_layout, + quantization_info); + } +}; +template <typename TensorType, typename AccessorType, typename FunctionType, typename T, bool mixed_layout = false> +class ScaleValidationDifferentOutputQuantizedFixture : public ScaleValidationGenericFixture<TensorType, AccessorType, FunctionType, T> +{ +public: + template <typename...> + void setup(TensorShape shape, DataType data_type, QuantizationInfo input_quantization_info, QuantizationInfo output_quantization_info, DataLayout data_layout, InterpolationPolicy policy, + BorderMode border_mode, SamplingPolicy sampling_policy, + bool align_corners) + { + ScaleValidationGenericFixture<TensorType, AccessorType, FunctionType, T>::setup(shape, + data_type, + input_quantization_info, + data_layout, + policy, + border_mode, + sampling_policy, + align_corners, + mixed_layout, + output_quantization_info); } }; template <typename TensorType, typename AccessorType, typename FunctionType, typename T, bool mixed_layout = false> @@ -245,7 +263,8 @@ public: border_mode, sampling_policy, align_corners, - mixed_layout); + mixed_layout, + QuantizationInfo()); } }; } // namespace validation diff --git a/tests/validation/reference/Scale.cpp b/tests/validation/reference/Scale.cpp index 71e98fd776..2f429cb29b 100644 --- a/tests/validation/reference/Scale.cpp +++ b/tests/validation/reference/Scale.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020 Arm Limited. + * Copyright (c) 2017-2020, 2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -25,7 +25,6 @@ #include "Scale.h" #include "Utils.h" -#include "arm_compute/core/utils/misc/Utility.h" #include "src/core/utils/ScaleUtils.h" #include "support/Rounding.h" @@ -183,14 +182,15 @@ SimpleTensor<T> scale_core(const SimpleTensor<T> &in, float scale_x, float scale template <typename T> SimpleTensor<T> scale(const SimpleTensor<T> &src, float scale_x, float scale_y, InterpolationPolicy policy, BorderMode border_mode, T constant_border_value, - SamplingPolicy sampling_policy, bool ceil_policy_scale, bool align_corners) + SamplingPolicy sampling_policy, bool ceil_policy_scale, bool align_corners, QuantizationInfo output_quantization_info) { + ARM_COMPUTE_UNUSED(output_quantization_info); return scale_core<T>(src, scale_x, scale_y, policy, border_mode, constant_border_value, sampling_policy, ceil_policy_scale, align_corners); } template <> SimpleTensor<uint8_t> scale(const SimpleTensor<uint8_t> &src, float scale_x, float scale_y, InterpolationPolicy policy, BorderMode border_mode, uint8_t constant_border_value, - SamplingPolicy sampling_policy, bool ceil_policy_scale, bool align_corners) + SamplingPolicy sampling_policy, bool ceil_policy_scale, bool align_corners, QuantizationInfo output_quantization_info) { SimpleTensor<uint8_t> dst; if(src.quantization_info().uniform().scale != 0.f) @@ -198,7 +198,7 @@ SimpleTensor<uint8_t> scale(const SimpleTensor<uint8_t> &src, float scale_x, flo SimpleTensor<float> src_tmp = convert_from_asymmetric(src); float constant_border_value_f = dequantize_qasymm8(constant_border_value, src.quantization_info()); SimpleTensor<float> dst_tmp = scale_core<float>(src_tmp, scale_x, scale_y, policy, border_mode, constant_border_value_f, sampling_policy, ceil_policy_scale, align_corners); - dst = convert_to_asymmetric<uint8_t>(dst_tmp, src.quantization_info()); + dst = convert_to_asymmetric<uint8_t>(dst_tmp, output_quantization_info); } else { @@ -209,7 +209,7 @@ SimpleTensor<uint8_t> scale(const SimpleTensor<uint8_t> &src, float scale_x, flo template <> SimpleTensor<int8_t> scale(const SimpleTensor<int8_t> &src, float scale_x, float scale_y, InterpolationPolicy policy, BorderMode border_mode, int8_t constant_border_value, - SamplingPolicy sampling_policy, bool ceil_policy_scale, bool align_corners) + SamplingPolicy sampling_policy, bool ceil_policy_scale, bool align_corners, QuantizationInfo output_quantization_info) { SimpleTensor<int8_t> dst; if(src.quantization_info().uniform().scale != 0.f) @@ -217,7 +217,7 @@ SimpleTensor<int8_t> scale(const SimpleTensor<int8_t> &src, float scale_x, float SimpleTensor<float> src_tmp = convert_from_asymmetric(src); float constant_border_value_f = dequantize_qasymm8_signed(constant_border_value, src.quantization_info()); SimpleTensor<float> dst_tmp = scale_core<float>(src_tmp, scale_x, scale_y, policy, border_mode, constant_border_value_f, sampling_policy, ceil_policy_scale, align_corners); - dst = convert_to_asymmetric<int8_t>(dst_tmp, src.quantization_info()); + dst = convert_to_asymmetric<int8_t>(dst_tmp, output_quantization_info); } else { @@ -227,11 +227,11 @@ SimpleTensor<int8_t> scale(const SimpleTensor<int8_t> &src, float scale_x, float } template SimpleTensor<int16_t> scale(const SimpleTensor<int16_t> &src, float scale_x, float scale_y, InterpolationPolicy policy, BorderMode border_mode, int16_t constant_border_value, - SamplingPolicy sampling_policy, bool ceil_policy_scale, bool align_corners); + SamplingPolicy sampling_policy, bool ceil_policy_scale, bool align_corners, QuantizationInfo output_quantization_info); template SimpleTensor<half> scale(const SimpleTensor<half> &src, float scale_x, float scale_y, InterpolationPolicy policy, BorderMode border_mode, half constant_border_value, - SamplingPolicy sampling_policy, bool ceil_policy_scale, bool align_corners); + SamplingPolicy sampling_policy, bool ceil_policy_scale, bool align_corners, QuantizationInfo output_quantization_info); template SimpleTensor<float> scale(const SimpleTensor<float> &src, float scale_x, float scale_y, InterpolationPolicy policy, BorderMode border_mode, float constant_border_value, - SamplingPolicy sampling_policy, bool ceil_policy_scale, bool align_corners); + SamplingPolicy sampling_policy, bool ceil_policy_scale, bool align_corners, QuantizationInfo output_quantization_info); } // namespace reference } // namespace validation } // namespace test diff --git a/tests/validation/reference/Scale.h b/tests/validation/reference/Scale.h index c66af8d94e..c32c07d1c0 100644 --- a/tests/validation/reference/Scale.h +++ b/tests/validation/reference/Scale.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020 Arm Limited. + * Copyright (c) 2017-2020, 2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -37,7 +37,7 @@ namespace reference { template <typename T> SimpleTensor<T> scale(const SimpleTensor<T> &src, float scale_x, float scale_y, InterpolationPolicy policy, BorderMode border_mode, T constant_border_value = 0, - SamplingPolicy sampling_policy = SamplingPolicy::CENTER, bool ceil_policy_scale = false, bool align_corners = false); + SamplingPolicy sampling_policy = SamplingPolicy::CENTER, bool ceil_policy_scale = false, bool align_corners = false, QuantizationInfo output_quantization_info = QuantizationInfo()); } // namespace reference } // namespace validation } // namespace test |