aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorGunes Bayir <gunes.bayir@arm.com>2022-09-11 15:59:19 +0100
committerGunes Bayir <gunes.bayir@arm.com>2022-09-16 12:38:59 +0000
commitc4f2743951473f8d97f5a43767fdbb31a4df967c (patch)
tree26c49b1af8113bb169931f3af5e502904d455a25 /src
parent0d05b6690fe69c57f63ca43d59b551f074613062 (diff)
downloadComputeLibrary-c4f2743951473f8d97f5a43767fdbb31a4df967c.tar.gz
Optimize Quantized/Integer Bilinear Scale for Neon™
This patch introduces several performance optimizations regarding the Bilinear Scale operator with REPLICATE Border mode. Changes apply only to NHWC. This patch - Reduces the memory footprint by disabling precomputation of indices and weights when they're not used - Rewrites the kernels for QASYMM8/QASYMM8_SIGNED/U8(Uint8) - Adds S8(Int8) Bilinear Scale for Border mode REPLICATE - Removes Bilinear Scale SVE kernels for Quantized and Integer types and adjust the heuristics to choose the Neon™ implementation - Adds new test cases where the input and output of the Bilinear Scale operator have different quantization scale and offset Resolves: COMPMID-5453, COMPMID-5454 Change-Id: I3d251e76e0c6978fd5a0a1795ec62ab536bec93c Signed-off-by: Gunes Bayir <gunes.bayir@arm.com> Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/8250 Reviewed-by: SiCong Li <sicong.li@arm.com> Comments-Addressed: Arm Jenkins <bsgcomp@arm.com> Tested-by: Arm Jenkins <bsgcomp@arm.com> Benchmark: Arm Jenkins <bsgcomp@arm.com>
Diffstat (limited to 'src')
-rw-r--r--src/core/NEON/wrapper/intrinsics/cvt.h20
-rw-r--r--src/core/utils/ScaleUtils.cpp28
-rw-r--r--src/core/utils/ScaleUtils.h6
-rw-r--r--src/cpu/kernels/CpuScaleKernel.cpp33
-rw-r--r--src/cpu/kernels/CpuScaleKernel.h3
-rw-r--r--src/cpu/kernels/scale/neon/integer.cpp474
-rw-r--r--src/cpu/kernels/scale/neon/list.h20
-rw-r--r--src/cpu/kernels/scale/neon/qasymm8.cpp298
-rw-r--r--src/cpu/kernels/scale/neon/qasymm8_signed.cpp299
-rw-r--r--src/cpu/kernels/scale/sve/integer.cpp156
-rw-r--r--src/cpu/kernels/scale/sve/qasymm8.cpp111
-rw-r--r--src/cpu/kernels/scale/sve/qasymm8_signed.cpp111
-rw-r--r--src/cpu/operators/CpuScale.cpp4
-rw-r--r--src/cpu/operators/CpuScale.h4
-rw-r--r--src/runtime/NEON/functions/NEScale.cpp7
15 files changed, 1014 insertions, 560 deletions
diff --git a/src/core/NEON/wrapper/intrinsics/cvt.h b/src/core/NEON/wrapper/intrinsics/cvt.h
index e52e3dd0c4..baad1319b2 100644
--- a/src/core/NEON/wrapper/intrinsics/cvt.h
+++ b/src/core/NEON/wrapper/intrinsics/cvt.h
@@ -59,19 +59,35 @@ VCVT_TO_F16_IMPL(float16x4_t, float32x4_t, vcvt, f16, f32)
#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
template <typename T>
-inline typename std::enable_if<std::is_same<T, uint8_t>::value, uint32x4_t>::type
+inline typename std::enable_if < std::is_same<T, uint8_t>::value || std::is_same<T, uint32_t>::value, uint32x4_t >::type
vcvt(const float32x4_t &a)
{
return vcvtq_u32_f32(a);
}
template <typename T>
-inline typename std::enable_if<std::is_same<T, int8_t>::value, int32x4_t>::type
+inline typename std::enable_if < std::is_same<T, int8_t>::value || std::is_same<T, int32_t>::value, int32x4_t >::type
vcvt(const float32x4_t &a)
{
return vcvtq_s32_f32(a);
}
+#ifdef __aarch64__
+template <typename T>
+inline typename std::enable_if<std::is_same<T, uint32_t>::value, uint32x4_t>::type
+vcvta(const float32x4_t &a)
+{
+ return vcvtaq_u32_f32(a);
+}
+
+template <typename T>
+inline typename std::enable_if<std::is_same<T, int32_t>::value, int32x4_t>::type
+vcvta(const float32x4_t &a)
+{
+ return vcvtaq_s32_f32(a);
+}
+#endif //__aarch64__
+
#if defined(ARM_COMPUTE_ENABLE_BF16)
/** Convert 2x128-bit floating point vectors into 1x128-bit bfloat16 vector
*
diff --git a/src/core/utils/ScaleUtils.cpp b/src/core/utils/ScaleUtils.cpp
index 82c6405e89..ee57a8e7a7 100644
--- a/src/core/utils/ScaleUtils.cpp
+++ b/src/core/utils/ScaleUtils.cpp
@@ -40,12 +40,26 @@ float arm_compute::scale_utils::calculate_resize_ratio(size_t input_size, size_t
return static_cast<float>(in) / static_cast<float>(out);
}
-bool arm_compute::scale_utils::is_precomputation_required(DataLayout data_layout, DataType data_type, InterpolationPolicy policy)
+bool arm_compute::scale_utils::is_precomputation_required(DataLayout data_layout, DataType data_type,
+ InterpolationPolicy policy, BorderMode border_mode)
{
- // whether to precompute indices & weights
- // The Neon™ kernels (which are preferred over SVE when policy is BILINEAR) do not use
- // precomputed index and weights when data type is FP32/16.
- // If policy is nearest_neighbor for SVE, then precompute because it's being used
- // To be revised in COMPMID-5453/5454
- return data_layout != DataLayout::NHWC || (data_type != DataType::F32 && data_type != DataType::F16) || (CPUInfo::get().get_isa().sve == true && policy == InterpolationPolicy::NEAREST_NEIGHBOR);
+ // Do not calculate precomputed weights and indices if kernel code doesn't use them
+ if(data_layout == DataLayout::NHWC)
+ {
+ switch(data_type)
+ {
+ case DataType::F32:
+ case DataType::F16:
+ return (CPUInfo::get().get_isa().sve == true && policy == InterpolationPolicy::NEAREST_NEIGHBOR);
+ case DataType::U8:
+ case DataType::S8:
+ case DataType::QASYMM8:
+ case DataType::QASYMM8_SIGNED:
+ return (border_mode != BorderMode::REPLICATE) || (policy == InterpolationPolicy::NEAREST_NEIGHBOR);
+ default:
+ return true;
+ }
+ }
+
+ return true;
} \ No newline at end of file
diff --git a/src/core/utils/ScaleUtils.h b/src/core/utils/ScaleUtils.h
index c09509253c..1484824a7f 100644
--- a/src/core/utils/ScaleUtils.h
+++ b/src/core/utils/ScaleUtils.h
@@ -26,9 +26,6 @@
#include "arm_compute/core/Types.h"
-#include <cstdint>
-#include <cstdlib>
-
namespace arm_compute
{
namespace scale_utils
@@ -59,10 +56,11 @@ inline bool is_align_corners_allowed_sampling_policy(SamplingPolicy sampling_pol
* @param[in] data_layout Data layout
* @param[in] data_type Data type
* @param[in] policy Interpolation policy
+ * @param[in] border_mode Border Mode
*
* @return True if precomputation is required
*/
-bool is_precomputation_required(DataLayout data_layout, DataType data_type, InterpolationPolicy policy);
+bool is_precomputation_required(DataLayout data_layout, DataType data_type, InterpolationPolicy policy, BorderMode border_mode);
} // namespace scale_utils
} // namespace arm_compute
diff --git a/src/cpu/kernels/CpuScaleKernel.cpp b/src/cpu/kernels/CpuScaleKernel.cpp
index e7386a385a..b8bb5ad18a 100644
--- a/src/cpu/kernels/CpuScaleKernel.cpp
+++ b/src/cpu/kernels/CpuScaleKernel.cpp
@@ -25,14 +25,9 @@
#include "arm_compute/core/Helpers.h"
#include "arm_compute/core/Window.h"
-#include "arm_compute/core/utils/misc/Utility.h"
-#include "src/core/CPP/Validate.h"
-#include "src/core/NEON/wrapper/wrapper.h"
#include "src/core/common/Registrars.h"
-#include "src/core/helpers/AutoConfiguration.h"
#include "src/core/helpers/ScaleHelpers.h"
#include "src/core/helpers/WindowHelpers.h"
-#include "src/core/utils/ScaleUtils.h"
#include "src/cpu/kernels/scale/neon/list.h"
#include "src/cpu/kernels/scale/sve/list.h"
#include "support/Rounding.h"
@@ -68,22 +63,34 @@ static const std::vector<CpuScaleKernel::ScaleKernel> available_kernels =
},
{
"sve_qu8_scale",
- [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::QASYMM8 && data.isa.sve; },
+ [](const ScaleKernelDataTypeISASelectorData & data)
+ {
+ return data.dt == DataType::QASYMM8 && data.isa.sve && data.interpolation_policy != InterpolationPolicy::BILINEAR;
+ },
REGISTER_QASYMM8_SVE(arm_compute::cpu::qasymm8_sve_scale)
},
{
"sve_qs8_scale",
- [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::QASYMM8_SIGNED && data.isa.sve; },
+ [](const ScaleKernelDataTypeISASelectorData & data)
+ {
+ return data.dt == DataType::QASYMM8_SIGNED && data.isa.sve && data.interpolation_policy != InterpolationPolicy::BILINEAR;
+ },
REGISTER_QASYMM8_SIGNED_SVE(arm_compute::cpu::qasymm8_signed_sve_scale)
},
{
"sve_u8_scale",
- [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::U8 && data.isa.sve; },
+ [](const ScaleKernelDataTypeISASelectorData & data)
+ {
+ return data.dt == DataType::U8 && data.isa.sve && data.interpolation_policy != InterpolationPolicy::BILINEAR;
+ },
REGISTER_INTEGER_SVE(arm_compute::cpu::u8_sve_scale)
},
{
"sve_s16_scale",
- [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::S16 && data.isa.sve; },
+ [](const ScaleKernelDataTypeISASelectorData & data)
+ {
+ return data.dt == DataType::S16 && data.isa.sve && data.interpolation_policy != InterpolationPolicy::BILINEAR;
+ },
REGISTER_INTEGER_SVE(arm_compute::cpu::s16_sve_scale)
},
{
@@ -112,6 +119,11 @@ static const std::vector<CpuScaleKernel::ScaleKernel> available_kernels =
REGISTER_INTEGER_NEON(arm_compute::cpu::u8_neon_scale)
},
{
+ "neon_s8_scale",
+ [](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::S8; },
+ REGISTER_INTEGER_NEON(arm_compute::cpu::s8_neon_scale)
+ },
+ {
"neon_s16_scale",
[](const ScaleKernelDataTypeISASelectorData & data) { return data.dt == DataType::S16; },
REGISTER_INTEGER_NEON(arm_compute::cpu::s16_neon_scale)
@@ -140,6 +152,9 @@ Status validate_arguments(const ITensorInfo *src, const ITensorInfo *dx, const I
ARM_COMPUTE_RETURN_ERROR_ON(output_width == 0);
ARM_COMPUTE_RETURN_ERROR_ON(output_height == 0);
+ ARM_COMPUTE_RETURN_ERROR_ON((src->data_type() == DataType::S8) && (data_layout != DataLayout::NHWC || info.interpolation_policy != InterpolationPolicy::BILINEAR
+ || info.border_mode != BorderMode::REPLICATE));
+
if(info.interpolation_policy == InterpolationPolicy::NEAREST_NEIGHBOR && offsets != nullptr)
{
ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(offsets, 1, DataType::S32);
diff --git a/src/cpu/kernels/CpuScaleKernel.h b/src/cpu/kernels/CpuScaleKernel.h
index 416e115796..8102142fc3 100644
--- a/src/cpu/kernels/CpuScaleKernel.h
+++ b/src/cpu/kernels/CpuScaleKernel.h
@@ -50,8 +50,9 @@ public:
*
* @note dx, dy and offsets have the same dimensions (width and height) of the output tensor
* @note Using @p policy Area only supports data layout NCHW and input data type U8.
+ * @note Using S8 data type only supports NHWC, @p border_mode Replicate, and @p policy Bilinear
*
- * @param[in] src Source tensor info. Data types supported: QASYMM8/QASYMM8_SIGNED/U8/S16/F16/F32.
+ * @param[in] src Source tensor info. Data types supported: QASYMM8/QASYMM8_SIGNED/U8/S8/S16/F16/F32.
* @param[in] dx Distance x tensor info. Pixel's distance between the X real coordinate and the smallest X following integer. Data type supported: F32
* @param[in] dy Distance y tensor info. Pixel's distance between the Y real coordinate and the smallest Y following integer. Data type supported: F32
* @param[in] offsets Offset tensor info. Offset to access the pixel with NEAREST interpolation or the top-left pixel with BILINEAR interpolation in the input tensor. Data type supported: S32.
diff --git a/src/cpu/kernels/scale/neon/integer.cpp b/src/cpu/kernels/scale/neon/integer.cpp
index a2359aac94..2ab14cf83a 100644
--- a/src/cpu/kernels/scale/neon/integer.cpp
+++ b/src/cpu/kernels/scale/neon/integer.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021-2022 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -22,17 +22,12 @@
* SOFTWARE.
*/
#include "arm_compute/core/Helpers.h"
-#include "arm_compute/core/ITensorPack.h"
-#include "arm_compute/core/Window.h"
-#include "src/core/NEON/NEMath.h"
#include "src/core/NEON/wrapper/wrapper.h"
#include "src/core/helpers/ScaleHelpers.h"
#include "src/core/utils/ScaleUtils.h"
#include "support/Rounding.h"
#include <arm_neon.h>
-#include <cmath>
-#include <cstddef>
namespace arm_compute
{
@@ -84,37 +79,39 @@ void u8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off
BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
bool align_corners, const Window &window)
{
- // Compute the ratio between source height and destination height
- const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);
-
- Iterator out(dst, window);
- const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right;
- const int in_dim_w = src->info()->dimension(1);
- const int in_dim_h = src->info()->dimension(2);
- const int in_stride_wc = in_stride_c * (in_dim_w + src->info()->padding().top + src->info()->padding().bottom);
+ // Compute the ratio between source and destination dimensions
+ const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners);
+ const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);
- // Don't increment in Y and Z direction for the input tensor
- // A pointer to the start of this plane is needed as base for the precomputed offsets
- Window win_in(window);
- win_in.set(Window::DimY, Window::Dimension(0, 0, 0));
- win_in.set(Window::DimZ, Window::Dimension(0, 0, 0));
- Iterator in(src, win_in);
+ const int input_width = src->info()->dimension(1);
+ const int input_height = src->info()->dimension(2);
if(border_mode == BorderMode::CONSTANT)
{
+ Iterator out(dst, window);
+ const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right;
+ const int in_stride_wc = in_stride_c * (input_width + src->info()->padding().top + src->info()->padding().bottom);
+
+ // Don't increment in Y and Z direction for the input tensor
+ // A pointer to the start of this plane is needed as base for the precomputed offsets
+ Window win_in(window);
+ win_in.set(Window::DimY, Window::Dimension(0, 0, 0));
+ win_in.set(Window::DimZ, Window::Dimension(0, 0, 0));
+ Iterator in(src, win_in);
+
const uint8_t const_border_value = static_cast<uint8_t>(constant_border_value.get<uint8_t>());
execute_window_loop(window, [&](const Coordinates & id)
{
const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z())));
const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z())));
const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z())));
- const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset);
+ const int32_t in_hi = std::floor((id.z() + sampling_offset) * scale_y - sampling_offset);
const uint8_t *in_ptr = reinterpret_cast<const uint8_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc;
- const auto a00 = (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value;
- const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) ? *(in_ptr + in_stride_c) : const_border_value;
- const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_wc) : const_border_value;
- const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value;
+ const auto a00 = (0 <= offset && offset < input_width && 0 <= in_hi && in_hi < input_height) ? *in_ptr : const_border_value;
+ const auto a01 = (-1 <= offset && offset < input_width - 1 && 0 <= in_hi && in_hi < input_height) ? *(in_ptr + in_stride_c) : const_border_value;
+ const auto a10 = (0 <= offset && offset < input_width && -1 <= in_hi && in_hi < input_height - 1) ? *(in_ptr + in_stride_wc) : const_border_value;
+ const auto a11 = (-1 <= offset && offset < input_width - 1 && -1 <= in_hi && in_hi < input_height - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value;
*reinterpret_cast<uint8_t *>(out.ptr()) = static_cast<uint8_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val));
},
@@ -122,26 +119,395 @@ void u8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off
}
else if(border_mode == BorderMode::REPLICATE)
{
- execute_window_loop(window, [&](const Coordinates & id)
+ using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>;
+
+ const int in_stride_x = src->info()->strides_in_bytes()[1];
+ const int in_stride_y = src->info()->strides_in_bytes()[2];
+ const int in_stride_b = src->info()->strides_in_bytes()[3];
+ const int out_stride_x = dst->info()->strides_in_bytes()[1];
+ const int out_stride_y = dst->info()->strides_in_bytes()[2];
+ const int out_stride_b = dst->info()->strides_in_bytes()[3];
+
+ const int out_dim_ch = dst->info()->dimension(0);
+ constexpr int step_cout = 16;
+
+ Window window_execution = window;
+ window_execution.set(Window::DimX, Window::Dimension(0, 1, 1));
+ Window win_in_out(window);
+ win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0));
+ win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0));
+ Iterator in(src, win_in_out);
+ Iterator out(dst, win_in_out);
+
+ const int xo_start = window_execution[1].start();
+ const int xo_end = window_execution[1].end();
+ const int xo_step = window_execution[1].step();
+ const int yo_start = window_execution[2].start();
+ const int yo_end = window_execution[2].end();
+ const int yo_step = window_execution[2].step();
+ const int bo_start = window_execution[3].start();
+ const int bo_end = window_execution[3].end();
+ const int bo_step = window_execution[3].step();
+
+ const float fp_coord_offset_y = sampling_offset * (scale_y - 1);
+ const float fp_coord_offset_x = sampling_offset * (scale_x - 1);
+
+ for(int bo = bo_start; bo < bo_end; bo += bo_step)
{
- const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z())));
- const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z())));
- const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z())));
- const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset);
-
- auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1);
- auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1);
- auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1);
- auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1);
-
- const auto a00 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc);
- const auto a01 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc);
- const auto a10 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h1 * in_stride_wc);
- const auto a11 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h1 * in_stride_wc);
+ const uint8_t *in_ptr = in.ptr() + bo * in_stride_b;
+ uint8_t *out_ptr = out.ptr() + bo * out_stride_b;
+
+ for(int yo = yo_start; yo < yo_end; yo += yo_step)
+ {
+ // Floating-point coordinate
+ const float yi_f = yo * scale_y + fp_coord_offset_y;
+ // Integer coordinate
+ const int yi = static_cast<int>(std::floor(yi_f));
+ // Weight for the y coordinate
+ const float a1 = (yi_f - static_cast<float>(yi));
+ const float b1 = (1.f - a1);
+
+ const int yi0 = utility::clamp<int>(yi, 0, input_height - 1);
+ const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1);
+
+ const uint8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y;
+ const uint8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y;
+
+ uint8_t *out_ptr_yo = out_ptr + yo * out_stride_y;
+ for(int xo = xo_start; xo < xo_end; xo += xo_step)
+ {
+ // Floating-point coordinate
+ const float xi_f = xo * scale_x + fp_coord_offset_x;
+ // Integer coordinate
+ const int xi = static_cast<int>(std::floor(xi_f));
+ // Weight for the x coordinate
+ const float a = (xi_f - static_cast<float>(xi));
+ const float b = (1.f - a);
+
+ const float s00_s = b * b1;
+ const float s01_s = a * b1;
+ const float s10_s = b * a1;
+ const float s11_s = a * a1;
+
+ const auto s00 = wrapper::vdup_n(s00_s, ExactTagType{});
+ const auto s01 = wrapper::vdup_n(s01_s, ExactTagType{});
+ const auto s10 = wrapper::vdup_n(s10_s, ExactTagType{});
+ const auto s11 = wrapper::vdup_n(s11_s, ExactTagType{});
+
+ const int xi0 = utility::clamp<int>(xi, 0, input_width - 1);
+ const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1);
+
+ const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x;
+ const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x;
+ const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x;
+ const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x;
+
+ uint8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x;
+
+ int cout = 0;
+ for(; cout <= (out_dim_ch - step_cout); cout += step_cout)
+ {
+ const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(uint8_t));
+ const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(uint8_t));
+ const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(uint8_t));
+ const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(uint8_t));
+
+ const uint16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00));
+ const uint16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00));
+
+ const auto in00_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_low)));
+ const auto in00_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_low)));
+ const auto in00_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_high)));
+ const auto in00_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_high)));
+
+ const uint16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01));
+ const uint16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01));
+
+ const auto in01_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_low)));
+ const auto in01_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_low)));
+ const auto in01_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_high)));
+ const auto in01_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_high)));
+
+ const uint16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10));
+ const uint16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10));
+
+ const auto in10_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_low)));
+ const auto in10_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_low)));
+ const auto in10_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_high)));
+ const auto in10_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_high)));
+
+ const uint16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11));
+ const uint16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11));
+
+ const auto in11_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_low)));
+ const auto in11_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_low)));
+ const auto in11_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_high)));
+ const auto in11_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_high)));
+
+ auto out_0 = wrapper::vmul(in00_0, s00);
+ out_0 = wrapper::vmla(out_0, in01_0, s01);
+ out_0 = wrapper::vmla(out_0, in10_0, s10);
+ out_0 = wrapper::vmla(out_0, in11_0, s11);
+
+ auto out_1 = wrapper::vmul(in00_1, s00);
+ out_1 = wrapper::vmla(out_1, in01_1, s01);
+ out_1 = wrapper::vmla(out_1, in10_1, s10);
+ out_1 = wrapper::vmla(out_1, in11_1, s11);
+
+ auto out_2 = wrapper::vmul(in00_2, s00);
+ out_2 = wrapper::vmla(out_2, in01_2, s01);
+ out_2 = wrapper::vmla(out_2, in10_2, s10);
+ out_2 = wrapper::vmla(out_2, in11_2, s11);
+
+ auto out_3 = wrapper::vmul(in00_3, s00);
+ out_3 = wrapper::vmla(out_3, in01_3, s01);
+ out_3 = wrapper::vmla(out_3, in10_3, s10);
+ out_3 = wrapper::vmla(out_3, in11_3, s11);
+
+#if defined(__aarch64__) && !defined(BARE_METAL)
+ const auto out_0_int = wrapper::vcvta<uint32_t>(out_0);
+ const auto out_1_int = wrapper::vcvta<uint32_t>(out_1);
+ const auto out_2_int = wrapper::vcvta<uint32_t>(out_2);
+ const auto out_3_int = wrapper::vcvta<uint32_t>(out_3);
+#else // defined(__aarch64__) && !defined(BARE_METAL)
+ const auto out_0_int = wrapper::vcvt<uint32_t>(out_0);
+ const auto out_1_int = wrapper::vcvt<uint32_t>(out_1);
+ const auto out_2_int = wrapper::vcvt<uint32_t>(out_2);
+ const auto out_3_int = wrapper::vcvt<uint32_t>(out_3);
+#endif // defined(__aarch64__) && !defined(BARE_METAL)
+ const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int)));
+ const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int)));
+ const auto out = wrapper::vcombine(low_part, high_part);
+
+ wrapper::vstore(out_ptr_xo_yo + cout * sizeof(uint8_t), out);
+ }
+
+ for(; cout < out_dim_ch; ++cout)
+ {
+ const uint8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(uint8_t));
+ const uint8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(uint8_t));
+ const uint8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(uint8_t));
+ const uint8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(uint8_t));
+
+ float out0 = in00 * s00_s;
+ out0 += in01 * s01_s;
+ out0 += in10 * s10_s;
+ out0 += in11 * s11_s;
+
+ // Rounding modes of vector and scalar loops should match
+#if defined(__aarch64__) && !defined(BARE_METAL)
+ *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = static_cast<uint8_t>(std::round(out0));
+#else // defined(__aarch64__) && !defined(BARE_METAL)
+ *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = static_cast<uint8_t>(out0);
+#endif // defined(__aarch64__) && !defined(BARE_METAL)
+ }
+ }
+ }
+ }
+ }
+ else
+ {
+ ARM_COMPUTE_ERROR("Not implemented");
+ }
+}
- *reinterpret_cast<uint8_t *>(out.ptr()) = static_cast<uint8_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val));
- },
- in, out);
+void s8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
+ BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
+ bool align_corners, const Window &window)
+{
+ ARM_COMPUTE_UNUSED(dx, dy, offsets, constant_border_value);
+ if(border_mode == BorderMode::REPLICATE)
+ {
+ using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>;
+
+ // Compute the ratio between source and destination dimensions
+ const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners);
+ const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);
+
+ const int in_stride_x = src->info()->strides_in_bytes()[1];
+ const int in_stride_y = src->info()->strides_in_bytes()[2];
+ const int in_stride_b = src->info()->strides_in_bytes()[3];
+ const int out_stride_x = dst->info()->strides_in_bytes()[1];
+ const int out_stride_y = dst->info()->strides_in_bytes()[2];
+ const int out_stride_b = dst->info()->strides_in_bytes()[3];
+ const int input_width = src->info()->dimension(1);
+ const int input_height = src->info()->dimension(2);
+ const int out_dim_ch = dst->info()->dimension(0);
+ constexpr int step_cout = 16;
+
+ Window window_execution = window;
+ window_execution.set(Window::DimX, Window::Dimension(0, 1, 1));
+ Window win_in_out(window);
+ win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0));
+ win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0));
+ Iterator in(src, win_in_out);
+ Iterator out(dst, win_in_out);
+
+ const int xo_start = window_execution[1].start();
+ const int xo_end = window_execution[1].end();
+ const int xo_step = window_execution[1].step();
+ const int yo_start = window_execution[2].start();
+ const int yo_end = window_execution[2].end();
+ const int yo_step = window_execution[2].step();
+ const int bo_start = window_execution[3].start();
+ const int bo_end = window_execution[3].end();
+ const int bo_step = window_execution[3].step();
+
+ const float fp_coord_offset_y = sampling_offset * (scale_y - 1);
+ const float fp_coord_offset_x = sampling_offset * (scale_x - 1);
+
+ for(int bo = bo_start; bo < bo_end; bo += bo_step)
+ {
+ const int8_t *in_ptr = reinterpret_cast<int8_t *>(in.ptr() + bo * in_stride_b);
+ int8_t *out_ptr = reinterpret_cast<int8_t *>(out.ptr() + bo * out_stride_b);
+
+ for(int yo = yo_start; yo < yo_end; yo += yo_step)
+ {
+ // Floating-point coordinate
+ const float yi_f = yo * scale_y + fp_coord_offset_y;
+ // Integer coordinate
+ const int yi = static_cast<int>(std::floor(yi_f));
+ // Weight for the y coordinate
+ const float a1 = (yi_f - static_cast<float>(yi));
+ const float b1 = (1.f - a1);
+
+ const int yi0 = utility::clamp<int>(yi, 0, input_height - 1);
+ const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1);
+
+ const int8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y;
+ const int8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y;
+
+ int8_t *out_ptr_yo = out_ptr + yo * out_stride_y;
+ for(int xo = xo_start; xo < xo_end; xo += xo_step)
+ {
+ // Floating-point coordinate
+ const float xi_f = xo * scale_x + fp_coord_offset_x;
+ // Integer coordinate
+ const int xi = static_cast<int>(std::floor(xi_f));
+ // Weight for the x coordinate
+ const float a = (xi_f - static_cast<float>(xi));
+ const float b = (1.f - a);
+
+ const float s00_s = b * b1;
+ const float s01_s = a * b1;
+ const float s10_s = b * a1;
+ const float s11_s = a * a1;
+
+ const auto s00 = wrapper::vdup_n(s00_s, ExactTagType{});
+ const auto s01 = wrapper::vdup_n(s01_s, ExactTagType{});
+ const auto s10 = wrapper::vdup_n(s10_s, ExactTagType{});
+ const auto s11 = wrapper::vdup_n(s11_s, ExactTagType{});
+
+ const int xi0 = utility::clamp<int>(xi, 0, input_width - 1);
+ const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1);
+
+ const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x;
+ const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x;
+ const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x;
+ const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x;
+
+ int8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x;
+
+ int cout = 0;
+ for(; cout <= (out_dim_ch - step_cout); cout += step_cout)
+ {
+ const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(int8_t));
+ const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(int8_t));
+ const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(int8_t));
+ const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(int8_t));
+
+ const int16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00));
+ const int16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00));
+
+ const auto in00_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_low)));
+ const auto in00_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_low)));
+ const auto in00_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in00_high)));
+ const auto in00_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in00_high)));
+
+ const int16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01));
+ const int16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01));
+
+ const auto in01_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_low)));
+ const auto in01_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_low)));
+ const auto in01_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in01_high)));
+ const auto in01_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in01_high)));
+
+ const int16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10));
+ const int16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10));
+
+ const auto in10_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_low)));
+ const auto in10_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_low)));
+ const auto in10_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in10_high)));
+ const auto in10_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in10_high)));
+
+ const int16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11));
+ const int16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11));
+
+ const auto in11_0 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_low)));
+ const auto in11_1 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_low)));
+ const auto in11_2 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgetlow(in11_high)));
+ const auto in11_3 = wrapper::vcvt<float>(wrapper::vmovl(wrapper::vgethigh(in11_high)));
+
+ auto out_0 = wrapper::vmul(in00_0, s00);
+ out_0 = wrapper::vmla(out_0, in01_0, s01);
+ out_0 = wrapper::vmla(out_0, in10_0, s10);
+ out_0 = wrapper::vmla(out_0, in11_0, s11);
+
+ auto out_1 = wrapper::vmul(in00_1, s00);
+ out_1 = wrapper::vmla(out_1, in01_1, s01);
+ out_1 = wrapper::vmla(out_1, in10_1, s10);
+ out_1 = wrapper::vmla(out_1, in11_1, s11);
+
+ auto out_2 = wrapper::vmul(in00_2, s00);
+ out_2 = wrapper::vmla(out_2, in01_2, s01);
+ out_2 = wrapper::vmla(out_2, in10_2, s10);
+ out_2 = wrapper::vmla(out_2, in11_2, s11);
+
+ auto out_3 = wrapper::vmul(in00_3, s00);
+ out_3 = wrapper::vmla(out_3, in01_3, s01);
+ out_3 = wrapper::vmla(out_3, in10_3, s10);
+ out_3 = wrapper::vmla(out_3, in11_3, s11);
+
+#if defined(__aarch64__) && !defined(BARE_METAL)
+ const auto out_0_int = wrapper::vcvta<int32_t>(out_0);
+ const auto out_1_int = wrapper::vcvta<int32_t>(out_1);
+ const auto out_2_int = wrapper::vcvta<int32_t>(out_2);
+ const auto out_3_int = wrapper::vcvta<int32_t>(out_3);
+#else // defined(__aarch64__) && !defined(BARE_METAL)
+ const auto out_0_int = wrapper::vcvt<int32_t>(out_0);
+ const auto out_1_int = wrapper::vcvt<int32_t>(out_1);
+ const auto out_2_int = wrapper::vcvt<int32_t>(out_2);
+ const auto out_3_int = wrapper::vcvt<int32_t>(out_3);
+#endif // defined(__aarch64__) && !defined(BARE_METAL)
+ const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int)));
+ const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int)));
+ const auto out = wrapper::vcombine(low_part, high_part);
+
+ wrapper::vstore(out_ptr_xo_yo + cout * sizeof(int8_t), out);
+ }
+
+ for(; cout < out_dim_ch; ++cout)
+ {
+ const int8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(int8_t));
+ const int8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(int8_t));
+ const int8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(int8_t));
+ const int8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(int8_t));
+
+ float out0 = in00 * s00_s;
+ out0 += in01 * s01_s;
+ out0 += in10 * s10_s;
+ out0 += in11 * s11_s;
+
+ // Rounding modes of vector and scalar loops should match
+#if defined(__aarch64__) && !defined(BARE_METAL)
+ *(out_ptr_xo_yo + cout * sizeof(int8_t)) = static_cast<int8_t>(std::round(out0));
+#else // defined(__aarch64__) && !defined(BARE_METAL)
+ *(out_ptr_xo_yo + cout * sizeof(int8_t)) = static_cast<int8_t>(out0);
+#endif // defined(__aarch64__) && !defined(BARE_METAL)
+ }
+ }
+ }
+ }
}
else
{
@@ -240,10 +606,10 @@ void s16_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *of
const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z())));
const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset);
- auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1);
- auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1);
- auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1);
- auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1);
+ const auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1);
+ const auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1);
+ const auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1);
+ const auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1);
const auto a00 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc);
const auto a01 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc);
@@ -262,6 +628,20 @@ void s16_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *of
}
namespace cpu
{
+void s8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
+ InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
+ bool align_corners, const Window &window)
+{
+ if(policy == InterpolationPolicy::BILINEAR)
+ {
+ s8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
+ }
+ else
+ {
+ ARM_COMPUTE_ERROR("Not implemented");
+ }
+}
+
void u8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
bool align_corners, const Window &window)
diff --git a/src/cpu/kernels/scale/neon/list.h b/src/cpu/kernels/scale/neon/list.h
index 17ff4bb676..28a1087224 100644
--- a/src/cpu/kernels/scale/neon/list.h
+++ b/src/cpu/kernels/scale/neon/list.h
@@ -25,11 +25,8 @@
#define SRC_CORE_NEON_KERNELS_SCALE_LIST_H
#include "arm_compute/core/Helpers.h"
-#include "arm_compute/core/ITensorPack.h"
#include "arm_compute/core/Window.h"
-#include "src/core/NEON/NEMath.h"
#include "src/core/NEON/wrapper/wrapper.h"
-#include "src/core/helpers/ScaleHelpers.h"
#include "src/core/utils/ScaleUtils.h"
#include "support/Rounding.h"
@@ -44,6 +41,7 @@ namespace cpu
DECLARE_SCALE_KERNEL(s16_neon_scale);
DECLARE_SCALE_KERNEL(u8_neon_scale);
+DECLARE_SCALE_KERNEL(s8_neon_scale);
DECLARE_SCALE_KERNEL(qasymm8_neon_scale);
DECLARE_SCALE_KERNEL(qasymm8_signed_neon_scale);
@@ -360,10 +358,10 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset
int cout = 0;
for(; cout <= (out_dim_ch - step_cout); cout += step_cout)
{
- auto in00 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset));
- auto in01 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset));
- auto in10 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset));
- auto in11 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset));
+ const auto in00 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset));
+ const auto in01 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset));
+ const auto in10 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset));
+ const auto in11 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset));
auto out0 = wrapper::vmul(in00, s00);
out0 = wrapper::vmla(out0, in01, s01);
@@ -374,10 +372,10 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset
for(; cout < out_dim_ch; ++cout)
{
- T in00 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset));
- T in01 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset));
- T in10 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset));
- T in11 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset));
+ const T in00 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset));
+ const T in01 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset));
+ const T in10 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset));
+ const T in11 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset));
T out0 = in00 * s00_s;
out0 += in01 * s01_s;
diff --git a/src/cpu/kernels/scale/neon/qasymm8.cpp b/src/cpu/kernels/scale/neon/qasymm8.cpp
index daa157ec3d..778459ae39 100644
--- a/src/cpu/kernels/scale/neon/qasymm8.cpp
+++ b/src/cpu/kernels/scale/neon/qasymm8.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021-2022 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -21,6 +21,7 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
+#include "src/core/helpers/ScaleHelpers.h"
#include "src/cpu/kernels/scale/neon/list.h"
namespace arm_compute
@@ -32,56 +33,60 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor
bool align_corners, const Window &window)
{
// Data layout is NHWC
- // Compute the ratio between source height and destination height
- const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);
- Window win_off;
- win_off.set(Window::DimX, Window::Dimension(0, 0, 0));
- win_off.set(Window::DimY, Window::Dimension(0, 0, 0));
-
- // Don't increment in X and Y direction for the input tensor
- // A pointer to the start of this plane is needed as base for the precomputed offsets
- Window win_in(window);
- win_in.set(1, Window::Dimension(0, 0, 0));
- win_in.set(2, Window::Dimension(0, 0, 0));
-
- for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d)
- {
- win_off.set(d, Window::Dimension(0, 0, 0));
- }
-
- Iterator in(src, win_in);
- Iterator out(dst, window);
-
- const int32_t in_dim_w = src->info()->dimension(1);
- const int32_t in_dim_h = src->info()->dimension(2);
- const int32_t stride_w = src->info()->strides_in_bytes()[1];
- const int32_t stride_h = src->info()->strides_in_bytes()[2];
+ const int32_t input_width = src->info()->dimension(1);
+ const int32_t input_height = src->info()->dimension(2);
const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform();
const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform();
+ // Compute the ratio between source and destination dimensions
+ const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners);
+ const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);
+
if(border_mode == BorderMode::CONSTANT)
{
+ const int32_t in_stride_y = src->info()->strides_in_bytes()[1];
+ const int32_t in_stride_z = src->info()->strides_in_bytes()[2];
+
+ // Compute the ratio between source height and destination height
+ Window win_off;
+ win_off.set(Window::DimX, Window::Dimension(0, 0, 0));
+ win_off.set(Window::DimY, Window::Dimension(0, 0, 0));
+
+ // Don't increment in X and Y direction for the input tensor
+ // A pointer to the start of this plane is needed as base for the precomputed offsets
+ Window win_in(window);
+ win_in.set(1, Window::Dimension(0, 0, 0));
+ win_in.set(2, Window::Dimension(0, 0, 0));
+
+ for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d)
+ {
+ win_off.set(d, Window::Dimension(0, 0, 0));
+ }
+
+ Iterator in(src, win_in);
+ Iterator out(dst, window);
+
const uint8_t const_border_value = static_cast<uint8_t>(constant_border_value.get<uint8_t>());
execute_window_loop(window, [&](const Coordinates & id)
{
- const int32_t index_h = std::floor((id[2] + sampling_offset) * hr - sampling_offset);
+ const int32_t index_h = std::floor((id[2] + sampling_offset) * scale_y - sampling_offset);
const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2]))));
const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2]))));
const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2]))));
const auto pixel_row_ptr = reinterpret_cast<const uint8_t *>(in.ptr());
- const auto a00 = (0 <= index_w && index_w < in_dim_w && 0 <= index_h && index_h < in_dim_h) ?
- (*(pixel_row_ptr + index_w * stride_w + index_h * stride_h)) :
+ const auto a00 = (0 <= index_w && index_w < input_width && 0 <= index_h && index_h < input_height) ?
+ (*(pixel_row_ptr + index_w * in_stride_y + index_h * in_stride_z)) :
const_border_value;
- const auto a01 = (-1 <= index_w && index_w + 1 < in_dim_w && 0 <= index_h && index_h < in_dim_h) ?
- (*(pixel_row_ptr + (index_w + 1) * stride_w + index_h * stride_h)) :
+ const auto a01 = (-1 <= index_w && index_w + 1 < input_width && 0 <= index_h && index_h < input_height) ?
+ (*(pixel_row_ptr + (index_w + 1) * in_stride_y + index_h * in_stride_z)) :
const_border_value;
- const auto a10 = (0 <= index_w && index_w < in_dim_w && -1 <= index_h && index_h < in_dim_h - 1) ?
- (*(pixel_row_ptr + index_w * stride_w + (index_h + 1) * stride_h)) :
+ const auto a10 = (0 <= index_w && index_w < input_width && -1 <= index_h && index_h < input_height - 1) ?
+ (*(pixel_row_ptr + index_w * in_stride_y + (index_h + 1) * in_stride_z)) :
const_border_value;
- const auto a11 = (-1 <= index_w && index_w < in_dim_w - 1 && -1 <= index_h && index_h < in_dim_h - 1) ?
- (*(pixel_row_ptr + (index_w + 1) * stride_w + (index_h + 1) * stride_h)) :
+ const auto a11 = (-1 <= index_w && index_w < input_width - 1 && -1 <= index_h && index_h < input_height - 1) ?
+ (*(pixel_row_ptr + (index_w + 1) * in_stride_y + (index_h + 1) * in_stride_z)) :
const_border_value;
const float inp00 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a00, iq_info);
@@ -94,31 +99,205 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor
}
else if(border_mode == BorderMode::REPLICATE)
{
- execute_window_loop(window, [&](const Coordinates & id)
+ using FloatTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>;
+ using Int32TagType = typename wrapper::traits::neon_bitvector_tag_t<int32_t, wrapper::traits::BitWidth::W128>;
+
+ const int in_stride_x = src->info()->strides_in_bytes()[1];
+ const int in_stride_y = src->info()->strides_in_bytes()[2];
+ const int in_stride_b = src->info()->strides_in_bytes()[3];
+ const int out_stride_x = dst->info()->strides_in_bytes()[1];
+ const int out_stride_y = dst->info()->strides_in_bytes()[2];
+ const int out_stride_b = dst->info()->strides_in_bytes()[3];
+ const int out_dim_ch = dst->info()->dimension(0);
+ constexpr int step_cout = 16;
+
+ Window window_execution = window;
+ window_execution.set(Window::DimX, Window::Dimension(0, 1, 1));
+ Window win_in_out(window);
+ win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0));
+ win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0));
+ Iterator in(src, win_in_out);
+ Iterator out(dst, win_in_out);
+
+ const int xo_start = window_execution[1].start();
+ const int xo_end = window_execution[1].end();
+ const int xo_step = window_execution[1].step();
+ const int yo_start = window_execution[2].start();
+ const int yo_end = window_execution[2].end();
+ const int yo_step = window_execution[2].step();
+ const int bo_start = window_execution[3].start();
+ const int bo_end = window_execution[3].end();
+ const int bo_step = window_execution[3].step();
+
+ const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform();
+ const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform();
+
+ const float32x4_t vscale_in = wrapper::vdup_n(iq_info.scale, FloatTagType{});
+ const int32x4_t voffset_in = wrapper::vdup_n(iq_info.offset, Int32TagType{}); // Offsets will be Int32
+
+ const float32x4_t invvscale_o = wrapper::vdup_n(1.f / oq_info.scale, FloatTagType{});
+ const float32x4_t voffset_o = vdupq_n_f32(oq_info.offset);
+
+ const float fp_coord_offset_y = sampling_offset * (scale_y - 1);
+ const float fp_coord_offset_x = sampling_offset * (scale_x - 1);
+
+ for(int bo = bo_start; bo < bo_end; bo += bo_step)
{
- const int index_h = std::floor((id[2] + sampling_offset) * hr - sampling_offset);
- const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2]))));
- const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2]))));
- const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2]))));
- const auto pixel_row_ptr = reinterpret_cast<const uint8_t *>(in.ptr());
+ const uint8_t *in_ptr = in.ptr() + bo * in_stride_b;
+ uint8_t *out_ptr = out.ptr() + bo * out_stride_b;
- auto clamped_w = utility::clamp<int>(index_w, 0, in_dim_w - 1);
- auto clamped_w1 = utility::clamp<int>(index_w + 1, 0, in_dim_w - 1);
- auto clamped_h = utility::clamp<int>(index_h, 0, in_dim_h - 1);
- auto clamped_h1 = utility::clamp<int>(index_h + 1, 0, in_dim_h - 1);
+ for(int yo = yo_start; yo < yo_end; yo += yo_step)
+ {
+ // Floating-point coordinate
+ const float yi_f = yo * scale_y + fp_coord_offset_y;
+ // Integer coordinate
+ const int yi = static_cast<int>(std::floor(yi_f));
+ // Weight for the y coordinate
+ const float a1 = (yi_f - static_cast<float>(yi));
+ const float b1 = (1.f - a1);
- const auto a00 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h * stride_h);
- const auto a01 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h * stride_h);
- const auto a10 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h1 * stride_h);
- const auto a11 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h1 * stride_h);
+ const int yi0 = utility::clamp<int>(yi, 0, input_height - 1);
+ const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1);
- const float inp00 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a00, iq_info);
- const float inp01 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a01, iq_info);
- const float inp10 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a10, iq_info);
- const float inp11 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a11, iq_info);
- *reinterpret_cast<uint8_t *>(out.ptr()) = Qasymm8QuantizationHelper<uint8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info);
- },
- in, out);
+ const uint8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y;
+ const uint8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y;
+
+ uint8_t *out_ptr_yo = out_ptr + yo * out_stride_y;
+ for(int xo = xo_start; xo < xo_end; xo += xo_step)
+ {
+ // Floating-point coordinate
+ const float xi_f = xo * scale_x + fp_coord_offset_x;
+ // Integer coordinate
+ const int xi = static_cast<int>(std::floor(xi_f));
+ // Weight for the x coordinate
+ const float a = (xi_f - static_cast<float>(xi));
+ const float b = (1.f - a);
+
+ const float s00_s = b * b1;
+ const float s01_s = a * b1;
+ const float s10_s = b * a1;
+ const float s11_s = a * a1;
+
+ const auto s00 = wrapper::vdup_n(s00_s, FloatTagType{});
+ const auto s01 = wrapper::vdup_n(s01_s, FloatTagType{});
+ const auto s10 = wrapper::vdup_n(s10_s, FloatTagType{});
+ const auto s11 = wrapper::vdup_n(s11_s, FloatTagType{});
+
+ const int xi0 = utility::clamp<int>(xi, 0, input_width - 1);
+ const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1);
+
+ const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x;
+ const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x;
+ const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x;
+ const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x;
+
+ uint8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x;
+
+ int cout = 0;
+ for(; cout <= (out_dim_ch - step_cout); cout += step_cout)
+ {
+ const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(uint8_t));
+ const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(uint8_t));
+ const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(uint8_t));
+ const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(uint8_t));
+
+ const uint16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00));
+ const uint16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00));
+
+ const auto in00_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in00_low))), voffset_in)), vscale_in);
+ const auto in00_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in00_low))), voffset_in)), vscale_in);
+ const auto in00_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in00_high))), voffset_in)), vscale_in);
+ const auto in00_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in00_high))), voffset_in)), vscale_in);
+
+ const uint16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01));
+ const uint16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01));
+
+ const auto in01_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in01_low))), voffset_in)), vscale_in);
+ const auto in01_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in01_low))), voffset_in)), vscale_in);
+ const auto in01_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in01_high))), voffset_in)), vscale_in);
+ const auto in01_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in01_high))), voffset_in)), vscale_in);
+
+ const uint16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10));
+ const uint16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10));
+
+ const auto in10_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in10_low))), voffset_in)), vscale_in);
+ const auto in10_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in10_low))), voffset_in)), vscale_in);
+ const auto in10_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in10_high))), voffset_in)), vscale_in);
+ const auto in10_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in10_high))), voffset_in)), vscale_in);
+
+ const uint16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11));
+ const uint16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11));
+
+ const auto in11_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in11_low))), voffset_in)), vscale_in);
+ const auto in11_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in11_low))), voffset_in)), vscale_in);
+ const auto in11_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in11_high))), voffset_in)), vscale_in);
+ const auto in11_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in11_high))), voffset_in)), vscale_in);
+
+ auto out_0 = wrapper::vmul(in00_0, s00);
+ out_0 = wrapper::vmla(out_0, in01_0, s01);
+ out_0 = wrapper::vmla(out_0, in10_0, s10);
+ out_0 = wrapper::vmla(out_0, in11_0, s11);
+
+ auto out_1 = wrapper::vmul(in00_1, s00);
+ out_1 = wrapper::vmla(out_1, in01_1, s01);
+ out_1 = wrapper::vmla(out_1, in10_1, s10);
+ out_1 = wrapper::vmla(out_1, in11_1, s11);
+
+ auto out_2 = wrapper::vmul(in00_2, s00);
+ out_2 = wrapper::vmla(out_2, in01_2, s01);
+ out_2 = wrapper::vmla(out_2, in10_2, s10);
+ out_2 = wrapper::vmla(out_2, in11_2, s11);
+
+ auto out_3 = wrapper::vmul(in00_3, s00);
+ out_3 = wrapper::vmla(out_3, in01_3, s01);
+ out_3 = wrapper::vmla(out_3, in10_3, s10);
+ out_3 = wrapper::vmla(out_3, in11_3, s11);
+
+#if defined(__aarch64__) && !defined(BARE_METAL)
+ const auto out_0_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o));
+ const auto out_1_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o));
+ const auto out_2_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o));
+ const auto out_3_int = wrapper::vcvta<uint32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o));
+#else // defined(__aarch64__) && !defined(BARE_METAL)
+ const auto out_0_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o));
+ const auto out_1_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o));
+ const auto out_2_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o));
+ const auto out_3_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o));
+#endif // defined(__aarch64__) && !defined(BARE_METAL)
+ const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int)));
+ const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int)));
+ const auto out = wrapper::vcombine(low_part, high_part);
+
+ wrapper::vstore(out_ptr_xo_yo + cout * sizeof(uint8_t), out);
+ }
+
+ for(; cout < out_dim_ch; ++cout)
+ {
+ const uint8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(uint8_t));
+ const uint8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(uint8_t));
+ const uint8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(uint8_t));
+ const uint8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(uint8_t));
+
+ const float in00_f = (static_cast<int32_t>(in00) - iq_info.offset) * iq_info.scale;
+ const float in01_f = (static_cast<int32_t>(in01) - iq_info.offset) * iq_info.scale;
+ const float in10_f = (static_cast<int32_t>(in10) - iq_info.offset) * iq_info.scale;
+ const float in11_f = (static_cast<int32_t>(in11) - iq_info.offset) * iq_info.scale;
+
+ float out = in00_f * s00_s;
+ out += in01_f * s01_s;
+ out += in10_f * s10_s;
+ out += in11_f * s11_s;
+
+ // Rounding modes of vector and scalar loops should match
+#if defined(__aarch64__) && !defined(BARE_METAL)
+ *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = quantize_qasymm8(out, oq_info);
+#else // defined(__aarch64__) && !defined(BARE_METAL)
+ *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = quantize_qasymm8(out, oq_info, RoundingPolicy::TO_ZERO);
+#endif // defined(__aarch64__) && !defined(BARE_METAL)
+ }
+ }
+ }
+ }
}
else
{
@@ -134,7 +313,14 @@ void qasymm8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets
{
if(policy == InterpolationPolicy::BILINEAR)
{
- qasymm8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
+ if(src->info()->quantization_info() == dst->info()->quantization_info())
+ {
+ u8_neon_scale(src, dst, offsets, dx, dy, policy, border_mode, constant_border_value, sampling_offset, align_corners, window);
+ }
+ else
+ {
+ qasymm8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
+ }
}
else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
{
diff --git a/src/cpu/kernels/scale/neon/qasymm8_signed.cpp b/src/cpu/kernels/scale/neon/qasymm8_signed.cpp
index 83312636b4..cd63dfba63 100644
--- a/src/cpu/kernels/scale/neon/qasymm8_signed.cpp
+++ b/src/cpu/kernels/scale/neon/qasymm8_signed.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021-2022 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -21,6 +21,7 @@
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
+#include "src/core/helpers/ScaleHelpers.h"
#include "src/cpu/kernels/scale/neon/list.h"
namespace arm_compute
@@ -32,56 +33,59 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const
bool align_corners, const Window &window)
{
// Data layout is NHWC
- // Compute the ratio between source height and destination height
- const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);
- Window win_off;
- win_off.set(Window::DimX, Window::Dimension(0, 0, 0));
- win_off.set(Window::DimY, Window::Dimension(0, 0, 0));
-
- // Don't increment in X and Y direction for the input tensor
- // A pointer to the start of this plane is needed as base for the precomputed offsets
- Window win_in(window);
- win_in.set(1, Window::Dimension(0, 0, 0));
- win_in.set(2, Window::Dimension(0, 0, 0));
-
- for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d)
- {
- win_off.set(d, Window::Dimension(0, 0, 0));
- }
-
- Iterator in(src, win_in);
- Iterator out(dst, window);
-
- const int32_t in_dim_w = src->info()->dimension(1);
- const int32_t in_dim_h = src->info()->dimension(2);
- const int32_t stride_w = src->info()->strides_in_bytes()[1];
- const int32_t stride_h = src->info()->strides_in_bytes()[2];
-
const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform();
const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform();
+ const int32_t input_width = src->info()->dimension(1);
+ const int32_t input_height = src->info()->dimension(2);
+
+ // Compute the ratio between source and destination dimensions
+ const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners);
+ const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);
+
if(border_mode == BorderMode::CONSTANT)
{
+ const int32_t in_stride_y = src->info()->strides_in_bytes()[1];
+ const int32_t in_stride_z = src->info()->strides_in_bytes()[2];
+
+ Window win_off;
+ win_off.set(Window::DimX, Window::Dimension(0, 0, 0));
+ win_off.set(Window::DimY, Window::Dimension(0, 0, 0));
+
+ // Don't increment in X and Y direction for the input tensor
+ // A pointer to the start of this plane is needed as base for the precomputed offsets
+ Window win_in(window);
+ win_in.set(1, Window::Dimension(0, 0, 0));
+ win_in.set(2, Window::Dimension(0, 0, 0));
+
+ for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d)
+ {
+ win_off.set(d, Window::Dimension(0, 0, 0));
+ }
+
+ Iterator in(src, win_in);
+ Iterator out(dst, window);
+
const int8_t const_border_value = static_cast<int8_t>(constant_border_value.get<int8_t>());
execute_window_loop(window, [&](const Coordinates & id)
{
- const int32_t index_h = std::floor((id[2] + sampling_offset) * hr - sampling_offset);
+ const int32_t index_h = std::floor((id[2] + sampling_offset) * scale_y - sampling_offset);
const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2]))));
const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2]))));
const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2]))));
const auto pixel_row_ptr = reinterpret_cast<const int8_t *>(in.ptr());
- const auto a00 = (0 <= index_w && index_w < in_dim_w && 0 <= index_h && index_h < in_dim_h) ?
- (*(pixel_row_ptr + index_w * stride_w + index_h * stride_h)) :
+ const auto a00 = (0 <= index_w && index_w < input_width && 0 <= index_h && index_h < input_height) ?
+ (*(pixel_row_ptr + index_w * in_stride_y + index_h * in_stride_z)) :
const_border_value;
- const auto a01 = (-1 <= index_w && index_w + 1 < in_dim_w && 0 <= index_h && index_h < in_dim_h) ?
- (*(pixel_row_ptr + (index_w + 1) * stride_w + index_h * stride_h)) :
+ const auto a01 = (-1 <= index_w && index_w + 1 < input_width && 0 <= index_h && index_h < input_height) ?
+ (*(pixel_row_ptr + (index_w + 1) * in_stride_y + index_h * in_stride_z)) :
const_border_value;
- const auto a10 = (0 <= index_w && index_w < in_dim_w && -1 <= index_h && index_h < in_dim_h - 1) ?
- (*(pixel_row_ptr + index_w * stride_w + (index_h + 1) * stride_h)) :
+ const auto a10 = (0 <= index_w && index_w < input_width && -1 <= index_h && index_h < input_height - 1) ?
+ (*(pixel_row_ptr + index_w * in_stride_y + (index_h + 1) * in_stride_z)) :
const_border_value;
- const auto a11 = (-1 <= index_w && index_w < in_dim_w - 1 && -1 <= index_h && index_h < in_dim_h - 1) ?
- (*(pixel_row_ptr + (index_w + 1) * stride_w + (index_h + 1) * stride_h)) :
+ const auto a11 = (-1 <= index_w && index_w < input_width - 1 && -1 <= index_h && index_h < input_height - 1) ?
+ (*(pixel_row_ptr + (index_w + 1) * in_stride_y + (index_h + 1) * in_stride_z)) :
const_border_value;
const float inp00 = Qasymm8QuantizationHelper<int8_t>::dequantize(a00, iq_info);
@@ -94,31 +98,205 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const
}
else if(border_mode == BorderMode::REPLICATE)
{
- execute_window_loop(window, [&](const Coordinates & id)
+ using FloatTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>;
+ using Int32TagType = typename wrapper::traits::neon_bitvector_tag_t<int32_t, wrapper::traits::BitWidth::W128>;
+
+ const int in_stride_x = src->info()->strides_in_bytes()[1];
+ const int in_stride_y = src->info()->strides_in_bytes()[2];
+ const int in_stride_b = src->info()->strides_in_bytes()[3];
+ const int out_stride_x = dst->info()->strides_in_bytes()[1];
+ const int out_stride_y = dst->info()->strides_in_bytes()[2];
+ const int out_stride_b = dst->info()->strides_in_bytes()[3];
+ const int out_dim_ch = dst->info()->dimension(0);
+ constexpr int step_cout = 16;
+
+ Window window_execution = window;
+ window_execution.set(Window::DimX, Window::Dimension(0, 1, 1));
+ Window win_in_out(window);
+ win_in_out.set(Window::DimY, Window::Dimension(0, 0, 0));
+ win_in_out.set(Window::DimZ, Window::Dimension(0, 0, 0));
+ Iterator in(src, win_in_out);
+ Iterator out(dst, win_in_out);
+
+ const int xo_start = window_execution[1].start();
+ const int xo_end = window_execution[1].end();
+ const int xo_step = window_execution[1].step();
+ const int yo_start = window_execution[2].start();
+ const int yo_end = window_execution[2].end();
+ const int yo_step = window_execution[2].step();
+ const int bo_start = window_execution[3].start();
+ const int bo_end = window_execution[3].end();
+ const int bo_step = window_execution[3].step();
+
+ const float fp_coord_offset_y = sampling_offset * (scale_y - 1);
+ const float fp_coord_offset_x = sampling_offset * (scale_x - 1);
+
+ const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform();
+ const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform();
+
+ const float32x4_t vscale_in = wrapper::vdup_n(iq_info.scale, FloatTagType{});
+ const int32x4_t voffset_in = wrapper::vdup_n(iq_info.offset, Int32TagType{}); // Offsets will be Int32
+
+ const float32x4_t invvscale_o = wrapper::vdup_n(1.f / oq_info.scale, FloatTagType{});
+ const float32x4_t voffset_o = vdupq_n_f32(oq_info.offset);
+
+ for(int bo = bo_start; bo < bo_end; bo += bo_step)
{
- const int index_h = std::floor((id[2] + sampling_offset) * hr - sampling_offset);
- const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2]))));
- const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2]))));
- const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2]))));
- const auto pixel_row_ptr = reinterpret_cast<const int8_t *>(in.ptr());
+ const int8_t *in_ptr = reinterpret_cast<int8_t *>(in.ptr() + bo * in_stride_b);
+ int8_t *out_ptr = reinterpret_cast<int8_t *>(out.ptr() + bo * out_stride_b);
- auto clamped_w = utility::clamp<int>(index_w, 0, in_dim_w - 1);
- auto clamped_w1 = utility::clamp<int>(index_w + 1, 0, in_dim_w - 1);
- auto clamped_h = utility::clamp<int>(index_h, 0, in_dim_h - 1);
- auto clamped_h1 = utility::clamp<int>(index_h + 1, 0, in_dim_h - 1);
+ for(int yo = yo_start; yo < yo_end; yo += yo_step)
+ {
+ // Floating-point coordinate
+ const float yi_f = yo * scale_y + fp_coord_offset_y;
+ // Integer coordinate
+ const int yi = static_cast<int>(std::floor(yi_f));
+ // Weight for the y coordinate
+ const float a1 = (yi_f - static_cast<float>(yi));
+ const float b1 = (1.f - a1);
- const auto a00 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h * stride_h);
- const auto a01 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h * stride_h);
- const auto a10 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h1 * stride_h);
- const auto a11 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h1 * stride_h);
+ const int yi0 = utility::clamp<int>(yi, 0, input_height - 1);
+ const int yi1 = utility::clamp<int>(yi + 1, 0, input_height - 1);
- const float inp00 = Qasymm8QuantizationHelper<int8_t>::dequantize(a00, iq_info);
- const float inp01 = Qasymm8QuantizationHelper<int8_t>::dequantize(a01, iq_info);
- const float inp10 = Qasymm8QuantizationHelper<int8_t>::dequantize(a10, iq_info);
- const float inp11 = Qasymm8QuantizationHelper<int8_t>::dequantize(a11, iq_info);
- *reinterpret_cast<int8_t *>(out.ptr()) = Qasymm8QuantizationHelper<int8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info);
- },
- in, out);
+ const int8_t *in_ptr_yi0 = in_ptr + yi0 * in_stride_y;
+ const int8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y;
+
+ int8_t *out_ptr_yo = out_ptr + yo * out_stride_y;
+ for(int xo = xo_start; xo < xo_end; xo += xo_step)
+ {
+ // Floating-point coordinate
+ const float xi_f = xo * scale_x + fp_coord_offset_x;
+ // Integer coordinate
+ const int xi = static_cast<int>(std::floor(xi_f));
+ // Weight for the x coordinate
+ const float a = (xi_f - static_cast<float>(xi));
+ const float b = (1.f - a);
+
+ const float s00_s = b * b1;
+ const float s01_s = a * b1;
+ const float s10_s = b * a1;
+ const float s11_s = a * a1;
+
+ const auto s00 = wrapper::vdup_n(s00_s, FloatTagType{});
+ const auto s01 = wrapper::vdup_n(s01_s, FloatTagType{});
+ const auto s10 = wrapper::vdup_n(s10_s, FloatTagType{});
+ const auto s11 = wrapper::vdup_n(s11_s, FloatTagType{});
+
+ const int xi0 = utility::clamp<int>(xi, 0, input_width - 1);
+ const int xi1 = utility::clamp<int>(xi + 1, 0, input_width - 1);
+
+ const auto in_ptr_xi0_yi0 = in_ptr_yi0 + xi0 * in_stride_x;
+ const auto in_ptr_xi1_yi0 = in_ptr_yi0 + xi1 * in_stride_x;
+ const auto in_ptr_xi0_yi1 = in_ptr_yi1 + xi0 * in_stride_x;
+ const auto in_ptr_xi1_yi1 = in_ptr_yi1 + xi1 * in_stride_x;
+
+ int8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x;
+
+ int cout = 0;
+ for(; cout <= (out_dim_ch - step_cout); cout += step_cout)
+ {
+ const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(int8_t));
+ const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(int8_t));
+ const auto in10 = wrapper::vloadq(in_ptr_xi0_yi1 + cout * sizeof(int8_t));
+ const auto in11 = wrapper::vloadq(in_ptr_xi1_yi1 + cout * sizeof(int8_t));
+
+ const int16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00));
+ const int16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00));
+
+ const auto in00_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in00_low)), voffset_in)), vscale_in);
+ const auto in00_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in00_low)), voffset_in)), vscale_in);
+ const auto in00_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in00_high)), voffset_in)), vscale_in);
+ const auto in00_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in00_high)), voffset_in)), vscale_in);
+
+ const int16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01));
+ const int16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01));
+
+ const auto in01_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in01_low)), voffset_in)), vscale_in);
+ const auto in01_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in01_low)), voffset_in)), vscale_in);
+ const auto in01_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in01_high)), voffset_in)), vscale_in);
+ const auto in01_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in01_high)), voffset_in)), vscale_in);
+
+ const int16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10));
+ const int16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10));
+
+ const auto in10_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in10_low)), voffset_in)), vscale_in);
+ const auto in10_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in10_low)), voffset_in)), vscale_in);
+ const auto in10_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in10_high)), voffset_in)), vscale_in);
+ const auto in10_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in10_high)), voffset_in)), vscale_in);
+
+ const int16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11));
+ const int16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11));
+
+ const auto in11_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in11_low)), voffset_in)), vscale_in);
+ const auto in11_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in11_low)), voffset_in)), vscale_in);
+ const auto in11_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in11_high)), voffset_in)), vscale_in);
+ const auto in11_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in11_high)), voffset_in)), vscale_in);
+
+ auto out_0 = wrapper::vmul(in00_0, s00);
+ out_0 = wrapper::vmla(out_0, in01_0, s01);
+ out_0 = wrapper::vmla(out_0, in10_0, s10);
+ out_0 = wrapper::vmla(out_0, in11_0, s11);
+
+ auto out_1 = wrapper::vmul(in00_1, s00);
+ out_1 = wrapper::vmla(out_1, in01_1, s01);
+ out_1 = wrapper::vmla(out_1, in10_1, s10);
+ out_1 = wrapper::vmla(out_1, in11_1, s11);
+
+ auto out_2 = wrapper::vmul(in00_2, s00);
+ out_2 = wrapper::vmla(out_2, in01_2, s01);
+ out_2 = wrapper::vmla(out_2, in10_2, s10);
+ out_2 = wrapper::vmla(out_2, in11_2, s11);
+
+ auto out_3 = wrapper::vmul(in00_3, s00);
+ out_3 = wrapper::vmla(out_3, in01_3, s01);
+ out_3 = wrapper::vmla(out_3, in10_3, s10);
+ out_3 = wrapper::vmla(out_3, in11_3, s11);
+
+#if defined(__aarch64__) && !defined(BARE_METAL)
+ const auto out_0_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o));
+ const auto out_1_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o));
+ const auto out_2_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o));
+ const auto out_3_int = wrapper::vcvta<int32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o));
+#else // defined(__aarch64__) && !defined(BARE_METAL)
+ const auto out_0_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_0, invvscale_o));
+ const auto out_1_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_1, invvscale_o));
+ const auto out_2_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o));
+ const auto out_3_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o));
+#endif // defined(__aarch64__) && !defined(BARE_METAL)
+ const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int)));
+ const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int)));
+ const auto out = wrapper::vcombine(low_part, high_part);
+
+ wrapper::vstore(out_ptr_xo_yo + cout * sizeof(int8_t), out);
+ }
+
+ for(; cout < out_dim_ch; ++cout)
+ {
+ const int8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(int8_t));
+ const int8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(int8_t));
+ const int8_t in10 = *(in_ptr_xi0_yi1 + cout * sizeof(int8_t));
+ const int8_t in11 = *(in_ptr_xi1_yi1 + cout * sizeof(int8_t));
+
+ const float in00_f = (static_cast<int32_t>(in00) - iq_info.offset) * iq_info.scale;
+ const float in01_f = (static_cast<int32_t>(in01) - iq_info.offset) * iq_info.scale;
+ const float in10_f = (static_cast<int32_t>(in10) - iq_info.offset) * iq_info.scale;
+ const float in11_f = (static_cast<int32_t>(in11) - iq_info.offset) * iq_info.scale;
+
+ float out = in00_f * s00_s;
+ out += in01_f * s01_s;
+ out += in10_f * s10_s;
+ out += in11_f * s11_s;
+
+ // Rounding modes of vector and scalar loops should match
+#if defined(__aarch64__) && !defined(BARE_METAL)
+ *(out_ptr_xo_yo + cout * sizeof(int8_t)) = quantize_qasymm8_signed(out, oq_info);
+#else // defined(__aarch64__) && !defined(BARE_METAL)
+ *(out_ptr_xo_yo + cout * sizeof(int8_t)) = quantize_qasymm8_signed(out, oq_info, RoundingPolicy::TO_ZERO);
+#endif // defined(__aarch64__) && !defined(BARE_METAL)
+ }
+ }
+ }
+ }
}
else
{
@@ -134,7 +312,14 @@ void qasymm8_signed_neon_scale(const ITensor *src, ITensor *dst, const ITensor *
{
if(policy == InterpolationPolicy::BILINEAR)
{
- qasymm8_signed_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
+ if(src->info()->quantization_info() == dst->info()->quantization_info() && border_mode == BorderMode::REPLICATE)
+ {
+ s8_neon_scale(src, dst, offsets, dx, dy, policy, border_mode, constant_border_value, sampling_offset, align_corners, window);
+ }
+ else
+ {
+ qasymm8_signed_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
+ }
}
else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
{
diff --git a/src/cpu/kernels/scale/sve/integer.cpp b/src/cpu/kernels/scale/sve/integer.cpp
index 00a43922d9..82c70ee360 100644
--- a/src/cpu/kernels/scale/sve/integer.cpp
+++ b/src/cpu/kernels/scale/sve/integer.cpp
@@ -83,75 +83,6 @@ void u8_sve_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offse
out);
}
-void u8_sve_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
- BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
- bool align_corners, const Window &window)
-{
- // Compute the ratio between source height and destination height
- const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);
-
- Iterator out(dst, window);
- const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right;
- const int in_dim_w = src->info()->dimension(1);
- const int in_dim_h = src->info()->dimension(2);
- const int in_stride_wc = in_stride_c * (in_dim_w + src->info()->padding().top + src->info()->padding().bottom);
-
- // Don't increment in Y and Z direction for the input tensor
- // A pointer to the start of this plane is needed as base for the precomputed offsets
- Window win_in(window);
- win_in.set(Window::DimY, Window::Dimension(0, 0, 0));
- win_in.set(Window::DimZ, Window::Dimension(0, 0, 0));
- Iterator in(src, win_in);
-
- if(border_mode == BorderMode::CONSTANT)
- {
- const uint8_t const_border_value = static_cast<uint8_t>(constant_border_value.get<uint8_t>());
- execute_window_loop(window, [&](const Coordinates & id)
- {
- const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z())));
- const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z())));
- const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z())));
- const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset);
- const uint8_t *in_ptr = reinterpret_cast<const uint8_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc;
-
- const auto a00 = (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value;
- const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) ? *(in_ptr + in_stride_c) : const_border_value;
- const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_wc) : const_border_value;
- const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value;
-
- *reinterpret_cast<uint8_t *>(out.ptr()) = static_cast<uint8_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val));
- },
- in, out);
- }
- else if(border_mode == BorderMode::REPLICATE)
- {
- execute_window_loop(window, [&](const Coordinates & id)
- {
- const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z())));
- const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z())));
- const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z())));
- const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset);
-
- auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1);
- auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1);
- auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1);
- auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1);
-
- const auto a00 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc);
- const auto a01 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc);
- const auto a10 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h1 * in_stride_wc);
- const auto a11 = *(reinterpret_cast<const uint8_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h1 * in_stride_wc);
-
- *reinterpret_cast<uint8_t *>(out.ptr()) = static_cast<uint8_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val));
- },
- in, out);
- }
- else
- {
- ARM_COMPUTE_ERROR("Not implemented");
- }
-}
-
void s16_sve_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offsets,
float sampling_offset, bool align_corners, const Window &window)
{
@@ -195,75 +126,6 @@ void s16_sve_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offs
},
out);
}
-
-void s16_sve_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
- BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
- bool align_corners, const Window &window)
-{
- // Compute the ratio between source height and destination height
- const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners);
-
- Iterator out(dst, window);
- const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right;
- const int in_dim_w = src->info()->dimension(1);
- const int in_dim_h = src->info()->dimension(2);
- const int in_stride_wc = in_stride_c * (in_dim_w + src->info()->padding().top + src->info()->padding().bottom);
-
- // Don't increment in Y and Z direction for the input tensor
- // A pointer to the start of this plane is needed as base for the precomputed offsets
- Window win_in(window);
- win_in.set(Window::DimY, Window::Dimension(0, 0, 0));
- win_in.set(Window::DimZ, Window::Dimension(0, 0, 0));
- Iterator in(src, win_in);
-
- if(border_mode == BorderMode::CONSTANT)
- {
- const int16_t const_border_value = static_cast<int16_t>(constant_border_value.get<int16_t>());
- execute_window_loop(window, [&](const Coordinates & id)
- {
- const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z())));
- const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z())));
- const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z())));
- const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset);
- const int16_t *in_ptr = reinterpret_cast<const int16_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc;
-
- const auto a00 = (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value;
- const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) ? *(in_ptr + in_stride_c) : const_border_value;
- const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_wc) : const_border_value;
- const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value;
-
- *reinterpret_cast<int16_t *>(out.ptr()) = static_cast<int16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val));
- },
- in, out);
- }
- else if(border_mode == BorderMode::REPLICATE)
- {
- execute_window_loop(window, [&](const Coordinates & id)
- {
- const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z())));
- const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z())));
- const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z())));
- const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset);
-
- auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1);
- auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1);
- auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1);
- auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1);
-
- const auto a00 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc);
- const auto a01 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc);
- const auto a10 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h1 * in_stride_wc);
- const auto a11 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h1 * in_stride_wc);
-
- *reinterpret_cast<int16_t *>(out.ptr()) = static_cast<int16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val));
- },
- in, out);
- }
- else
- {
- ARM_COMPUTE_ERROR("Not implemented");
- }
-}
}
namespace cpu
{
@@ -271,13 +133,14 @@ void u8_sve_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, cons
InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
bool align_corners, const Window &window)
{
- if(policy == InterpolationPolicy::BILINEAR)
+ ARM_COMPUTE_UNUSED(dx, dy, border_mode, constant_border_value);
+ if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
{
- u8_sve_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
+ u8_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window);
}
- else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
+ else
{
- u8_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window);
+ ARM_COMPUTE_ERROR("Not Implemented");
}
}
@@ -285,13 +148,14 @@ void s16_sve_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, con
InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
bool align_corners, const Window &window)
{
- if(policy == InterpolationPolicy::BILINEAR)
+ ARM_COMPUTE_UNUSED(dx, dy, border_mode, constant_border_value);
+ if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
{
- s16_sve_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
+ s16_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window);
}
- else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
+ else
{
- s16_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window);
+ ARM_COMPUTE_ERROR("Not Implemented");
}
}
} // namespace cpu
diff --git a/src/cpu/kernels/scale/sve/qasymm8.cpp b/src/cpu/kernels/scale/sve/qasymm8.cpp
index 09ef00a783..d45a69e43b 100644
--- a/src/cpu/kernels/scale/sve/qasymm8.cpp
+++ b/src/cpu/kernels/scale/sve/qasymm8.cpp
@@ -83,108 +83,6 @@ void qasymm8_sve_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *
},
out);
}
-
-void qasymm8_sve_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
- BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
- bool align_corners, const Window &window)
-{
- // Data layout is NHWC
- const int idx_width = 1;
- const int idx_height = 2;
-
- // Compute the ratio between source height and destination height
- const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(idx_height), dst->info()->dimension(idx_height), align_corners);
- Window win_off;
- win_off.set(Window::DimX, Window::Dimension(0, 0, 0));
- win_off.set(Window::DimY, Window::Dimension(0, 0, 0));
-
- // Don't increment in X and Y direction for the input tensor
- // A pointer to the start of this plane is needed as base for the precomputed offsets
- Window win_in(window);
- win_in.set(idx_width, Window::Dimension(0, 0, 0));
- win_in.set(idx_height, Window::Dimension(0, 0, 0));
-
- for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d)
- {
- win_off.set(d, Window::Dimension(0, 0, 0));
- }
-
- Iterator in(src, win_in);
- Iterator out(dst, window);
-
- const int32_t in_dim_w = src->info()->dimension(idx_width);
- const int32_t in_dim_h = src->info()->dimension(idx_height);
- const int32_t stride_w = src->info()->strides_in_bytes()[idx_width];
- const int32_t stride_h = src->info()->strides_in_bytes()[idx_height];
-
- const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform();
- const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform();
-
- if(border_mode == BorderMode::CONSTANT)
- {
- const uint8_t const_border_value = static_cast<uint8_t>(constant_border_value.get<uint8_t>());
- execute_window_loop(window, [&](const Coordinates & id)
- {
- const int32_t index_h = std::floor((id[idx_height] + sampling_offset) * hr - sampling_offset);
- const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[idx_width], id[idx_height]))));
- const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[idx_width], id[idx_height]))));
- const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[idx_width], id[idx_height]))));
- const auto pixel_row_ptr = reinterpret_cast<const uint8_t *>(in.ptr());
-
- const auto a00 = (0 <= index_w && index_w < in_dim_w && 0 <= index_h && index_h < in_dim_h) ?
- (*(pixel_row_ptr + index_w * stride_w + index_h * stride_h)) :
- const_border_value;
- const auto a01 = (-1 <= index_w && index_w < in_dim_w - 1 && 0 <= index_h && index_h < in_dim_h) ?
- (*(pixel_row_ptr + (index_w + 1) * stride_w + index_h * stride_h)) :
- const_border_value;
- const auto a10 = (0 <= index_w && index_w < in_dim_w && -1 <= index_h && index_h < in_dim_h - 1) ?
- (*(pixel_row_ptr + index_w * stride_w + (index_h + 1) * stride_h)) :
- const_border_value;
- const auto a11 = (-1 <= index_w && index_w < in_dim_w - 1 && -1 <= index_h && index_h < in_dim_h - 1) ?
- (*(pixel_row_ptr + (index_w + 1) * stride_w + (index_h + 1) * stride_h)) :
- const_border_value;
-
- const float inp00 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a00, iq_info);
- const float inp01 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a01, iq_info);
- const float inp10 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a10, iq_info);
- const float inp11 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a11, iq_info);
- *reinterpret_cast<uint8_t *>(out.ptr()) = Qasymm8QuantizationHelper<uint8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info);
- },
- in, out);
- }
- else if(border_mode == BorderMode::REPLICATE)
- {
- execute_window_loop(window, [&](const Coordinates & id)
- {
- const int index_h = std::floor((id[idx_height] + sampling_offset) * hr - sampling_offset);
- const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[idx_width], id[idx_height]))));
- const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[idx_width], id[idx_height]))));
- const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[idx_width], id[idx_height]))));
- const auto pixel_row_ptr = reinterpret_cast<const uint8_t *>(in.ptr());
-
- auto clamped_w = utility::clamp<int>(index_w, 0, in_dim_w - 1);
- auto clamped_w1 = utility::clamp<int>(index_w + 1, 0, in_dim_w - 1);
- auto clamped_h = utility::clamp<int>(index_h, 0, in_dim_h - 1);
- auto clamped_h1 = utility::clamp<int>(index_h + 1, 0, in_dim_h - 1);
-
- const auto a00 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h * stride_h);
- const auto a01 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h * stride_h);
- const auto a10 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h1 * stride_h);
- const auto a11 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h1 * stride_h);
-
- const float inp00 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a00, iq_info);
- const float inp01 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a01, iq_info);
- const float inp10 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a10, iq_info);
- const float inp11 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a11, iq_info);
- *reinterpret_cast<uint8_t *>(out.ptr()) = Qasymm8QuantizationHelper<uint8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info);
- },
- in, out);
- }
- else
- {
- ARM_COMPUTE_ERROR("Not implemented");
- }
-}
}
namespace cpu
{
@@ -192,13 +90,14 @@ void qasymm8_sve_scale(const ITensor *src, ITensor *dst, const ITensor *offsets,
InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
bool align_corners, const Window &window)
{
- if(policy == InterpolationPolicy::BILINEAR)
+ ARM_COMPUTE_UNUSED(dx, dy, border_mode, constant_border_value);
+ if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
{
- qasymm8_sve_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
+ qasymm8_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window);
}
- else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
+ else
{
- qasymm8_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window);
+ ARM_COMPUTE_ERROR("Not Implemented");
}
}
} // namespace cpu
diff --git a/src/cpu/kernels/scale/sve/qasymm8_signed.cpp b/src/cpu/kernels/scale/sve/qasymm8_signed.cpp
index 63f515442b..67bca65f58 100644
--- a/src/cpu/kernels/scale/sve/qasymm8_signed.cpp
+++ b/src/cpu/kernels/scale/sve/qasymm8_signed.cpp
@@ -83,108 +83,6 @@ void qasymm8_signed_sve_scale_nearest(const ITensor *src, ITensor *dst, const IT
},
out);
}
-
-void qasymm8_signed_sve_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy,
- BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
- bool align_corners, const Window &window)
-{
- // Data layout is NHWC
- const int idx_width = 1;
- const int idx_height = 2;
-
- // Compute the ratio between source height and destination height
- const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(idx_height), dst->info()->dimension(idx_height), align_corners);
- Window win_off;
- win_off.set(Window::DimX, Window::Dimension(0, 0, 0));
- win_off.set(Window::DimY, Window::Dimension(0, 0, 0));
-
- // Don't increment in X and Y direction for the input tensor
- // A pointer to the start of this plane is needed as base for the precomputed offsets
- Window win_in(window);
- win_in.set(idx_width, Window::Dimension(0, 0, 0));
- win_in.set(idx_height, Window::Dimension(0, 0, 0));
-
- for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d)
- {
- win_off.set(d, Window::Dimension(0, 0, 0));
- }
-
- Iterator in(src, win_in);
- Iterator out(dst, window);
-
- const int32_t in_dim_w = src->info()->dimension(idx_width);
- const int32_t in_dim_h = src->info()->dimension(idx_height);
- const int32_t stride_w = src->info()->strides_in_bytes()[idx_width];
- const int32_t stride_h = src->info()->strides_in_bytes()[idx_height];
-
- const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform();
- const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform();
-
- if(border_mode == BorderMode::CONSTANT)
- {
- const int8_t const_border_value = static_cast<int8_t>(constant_border_value.get<int8_t>());
- execute_window_loop(window, [&](const Coordinates & id)
- {
- const int32_t index_h = std::floor((id[idx_height] + sampling_offset) * hr - sampling_offset);
- const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[idx_width], id[idx_height]))));
- const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[idx_width], id[idx_height]))));
- const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[idx_width], id[idx_height]))));
- const auto pixel_row_ptr = reinterpret_cast<const int8_t *>(in.ptr());
-
- const auto a00 = (0 <= index_w && index_w < in_dim_w && 0 <= index_h && index_h < in_dim_h) ?
- (*(pixel_row_ptr + index_w * stride_w + index_h * stride_h)) :
- const_border_value;
- const auto a01 = (-1 <= index_w && index_w < in_dim_w - 1 && 0 <= index_h && index_h < in_dim_h) ?
- (*(pixel_row_ptr + (index_w + 1) * stride_w + index_h * stride_h)) :
- const_border_value;
- const auto a10 = (0 <= index_w && index_w < in_dim_w && -1 <= index_h && index_h < in_dim_h - 1) ?
- (*(pixel_row_ptr + index_w * stride_w + (index_h + 1) * stride_h)) :
- const_border_value;
- const auto a11 = (-1 <= index_w && index_w < in_dim_w - 1 && -1 <= index_h && index_h < in_dim_h - 1) ?
- (*(pixel_row_ptr + (index_w + 1) * stride_w + (index_h + 1) * stride_h)) :
- const_border_value;
-
- const float inp00 = Qasymm8QuantizationHelper<int8_t>::dequantize(a00, iq_info);
- const float inp01 = Qasymm8QuantizationHelper<int8_t>::dequantize(a01, iq_info);
- const float inp10 = Qasymm8QuantizationHelper<int8_t>::dequantize(a10, iq_info);
- const float inp11 = Qasymm8QuantizationHelper<int8_t>::dequantize(a11, iq_info);
- *reinterpret_cast<int8_t *>(out.ptr()) = Qasymm8QuantizationHelper<int8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info);
- },
- in, out);
- }
- else if(border_mode == BorderMode::REPLICATE)
- {
- execute_window_loop(window, [&](const Coordinates & id)
- {
- const int index_h = std::floor((id[idx_height] + sampling_offset) * hr - sampling_offset);
- const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[idx_width], id[idx_height]))));
- const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[idx_width], id[idx_height]))));
- const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[idx_width], id[idx_height]))));
- const auto pixel_row_ptr = reinterpret_cast<const int8_t *>(in.ptr());
-
- auto clamped_w = utility::clamp<int>(index_w, 0, in_dim_w - 1);
- auto clamped_w1 = utility::clamp<int>(index_w + 1, 0, in_dim_w - 1);
- auto clamped_h = utility::clamp<int>(index_h, 0, in_dim_h - 1);
- auto clamped_h1 = utility::clamp<int>(index_h + 1, 0, in_dim_h - 1);
-
- const auto a00 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h * stride_h);
- const auto a01 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h * stride_h);
- const auto a10 = *(pixel_row_ptr + clamped_w * stride_w + clamped_h1 * stride_h);
- const auto a11 = *(pixel_row_ptr + clamped_w1 * stride_w + clamped_h1 * stride_h);
-
- const float inp00 = Qasymm8QuantizationHelper<int8_t>::dequantize(a00, iq_info);
- const float inp01 = Qasymm8QuantizationHelper<int8_t>::dequantize(a01, iq_info);
- const float inp10 = Qasymm8QuantizationHelper<int8_t>::dequantize(a10, iq_info);
- const float inp11 = Qasymm8QuantizationHelper<int8_t>::dequantize(a11, iq_info);
- *reinterpret_cast<int8_t *>(out.ptr()) = Qasymm8QuantizationHelper<int8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info);
- },
- in, out);
- }
- else
- {
- ARM_COMPUTE_ERROR("Not implemented");
- }
-}
}
namespace cpu
{
@@ -192,13 +90,14 @@ void qasymm8_signed_sve_scale(const ITensor *src, ITensor *dst, const ITensor *o
InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset,
bool align_corners, const Window &window)
{
- if(policy == InterpolationPolicy::BILINEAR)
+ ARM_COMPUTE_UNUSED(dx, dy, border_mode, constant_border_value);
+ if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
{
- qasymm8_signed_sve_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window);
+ qasymm8_signed_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window);
}
- else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR)
+ else
{
- qasymm8_signed_sve_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window);
+ ARM_COMPUTE_ERROR("Not Implemented");
}
}
} // namespace cpu
diff --git a/src/cpu/operators/CpuScale.cpp b/src/cpu/operators/CpuScale.cpp
index fdb52e5ede..a13a0f56a2 100644
--- a/src/cpu/operators/CpuScale.cpp
+++ b/src/cpu/operators/CpuScale.cpp
@@ -215,9 +215,9 @@ void CpuScale::prepare(ITensorPack &tensors)
_scale_info.interpolation_policy;
const SamplingPolicy sampling_policy = _scale_info.sampling_policy;
- bool precompute_indices_weights = arm_compute::scale_utils::is_precomputation_required(_data_layout, src->info()->data_type(), policy_to_use);
+ bool precompute_indices_weights = arm_compute::scale_utils::is_precomputation_required(_data_layout, src->info()->data_type(), policy_to_use, _scale_info.border_mode);
- if(precompute_indices_weights == true)
+ if(precompute_indices_weights)
{
switch(policy_to_use)
{
diff --git a/src/cpu/operators/CpuScale.h b/src/cpu/operators/CpuScale.h
index f605af6712..ee7c523bad 100644
--- a/src/cpu/operators/CpuScale.h
+++ b/src/cpu/operators/CpuScale.h
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021 Arm Limited.
+ * Copyright (c) 2021-2022 Arm Limited.
*
* SPDX-License-Identifier: MIT
*
@@ -45,6 +45,8 @@ public:
* @param[in, out] src Source tensor info. Data type supported: QASYMM8/QASYMM8_SIGNED/U8/S16/F16/F32. (Written to only for @p border_mode != UNDEFINED)
* @param[out] dst Destination tensor info. Data type supported: Same as @p src. All but the lowest two dimensions must be the same size as in the input tensor, i.e. scaling is only performed within the XY-plane.
* @param[in] info @ref ScaleKernelInfo to be used for configuration
+ *
+ * @note Using S8 data type only supports NHWC, @p border_mode Replicate, and @p policy Bilinear
*/
void configure(ITensorInfo *src, ITensorInfo *dst, const ScaleKernelInfo &info);
/** Static function to check if given info will lead to a valid configuration
diff --git a/src/runtime/NEON/functions/NEScale.cpp b/src/runtime/NEON/functions/NEScale.cpp
index 74ab860d91..686017f7c1 100644
--- a/src/runtime/NEON/functions/NEScale.cpp
+++ b/src/runtime/NEON/functions/NEScale.cpp
@@ -23,12 +23,9 @@
*/
#include "arm_compute/runtime/NEON/functions/NEScale.h"
-#include "arm_compute/core/Validate.h"
-#include "arm_compute/runtime/Tensor.h"
#include "src/common/utils/Log.h"
#include "src/core/utils/ScaleUtils.h"
#include "src/cpu/operators/CpuScale.h"
-#include "support/Rounding.h"
namespace arm_compute
{
@@ -75,9 +72,9 @@ void NEScale::configure(ITensor *input, ITensor *output, const ScaleKernelInfo &
TensorShape shape(output->info()->dimension(idx_width));
shape.set(1, output->info()->dimension(idx_height), false);
- bool precompute_indices_weights = arm_compute::scale_utils::is_precomputation_required(data_layout, input->info()->data_type(), policy_to_use);
+ bool precompute_indices_weights = arm_compute::scale_utils::is_precomputation_required(data_layout, input->info()->data_type(), policy_to_use, info.border_mode);
- if(precompute_indices_weights == true)
+ if(precompute_indices_weights)
{
const TensorInfo tensor_info_dxdy(shape, Format::F32);
const TensorInfo tensor_info_offsets(shape, Format::S32);