diff options
Diffstat (limited to 'src/cpu/kernels/scale/neon')
| -rw-r--r-- | src/cpu/kernels/scale/neon/fp16.cpp | 189 | ||||
| -rw-r--r-- | src/cpu/kernels/scale/neon/integer.cpp | 416 | ||||
| -rw-r--r-- | src/cpu/kernels/scale/neon/list.h | 163 | ||||
| -rw-r--r-- | src/cpu/kernels/scale/neon/qasymm8.cpp | 217 | ||||
| -rw-r--r-- | src/cpu/kernels/scale/neon/qasymm8_signed.cpp | 206 |
5 files changed, 762 insertions, 429 deletions
diff --git a/src/cpu/kernels/scale/neon/fp16.cpp b/src/cpu/kernels/scale/neon/fp16.cpp index 895f42215e..bd01569cc4 100644 --- a/src/cpu/kernels/scale/neon/fp16.cpp +++ b/src/cpu/kernels/scale/neon/fp16.cpp @@ -27,9 +27,10 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/ITensorPack.h" #include "arm_compute/core/Window.h" + +#include "src/core/helpers/ScaleHelpers.h" #include "src/core/NEON/NEMath.h" #include "src/core/NEON/wrapper/wrapper.h" -#include "src/core/helpers/ScaleHelpers.h" #include "src/core/utils/ScaleUtils.h" #include "support/Rounding.h" @@ -41,8 +42,12 @@ namespace arm_compute { namespace { -void fp16_neon_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offsets, - float sampling_offset, bool align_corners, const Window &window) +void fp16_neon_scale_nearest(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + float sampling_offset, + bool align_corners, + const Window &window) { const size_t in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; const size_t in_stride_w = src->info()->dimension(1) + src->info()->padding().top + src->info()->padding().bottom; @@ -62,33 +67,46 @@ void fp16_neon_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *of const uint8_t *in_ptr_start = src->buffer() + src->info()->offset_first_element_in_bytes(); const unsigned int in_stride_bytes_hwc = src->info()->strides_in_bytes()[3]; - execute_window_loop(win, [&](const Coordinates & id) - { - const int32_t offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))) * in_stride_c; - const auto in_hi = static_cast<int>(align_corners ? utils::rounding::round_half_away_from_zero((id.z() + sampling_offset) * hr) : std::floor((id.z() + sampling_offset) * hr)); - const int offset_row = in_hi * in_stride_wc; - int32_t x = window_start_x; - const float16_t *in_ptr = reinterpret_cast<const float16_t *>(in_ptr_start + in_stride_bytes_hwc * id[3]); - - for(; x <= window_end_x - window_step_x; x += window_step_x) - { - wrapper::vstore(reinterpret_cast<float16_t *>(out.ptr()) + x, - wrapper::vloadq(in_ptr + offset + offset_row + x)); - } - for(; x < window_end_x; ++x) + execute_window_loop( + win, + [&](const Coordinates &id) { - *(reinterpret_cast<float16_t *>(out.ptr()) + x) = *(in_ptr + offset + offset_row + x); - } - }, - out); + const int32_t offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))) * in_stride_c; + const auto in_hi = static_cast<int>( + align_corners ? utils::rounding::round_half_away_from_zero((id.z() + sampling_offset) * hr) + : std::floor((id.z() + sampling_offset) * hr)); + const int offset_row = in_hi * in_stride_wc; + int32_t x = window_start_x; + const float16_t *in_ptr = reinterpret_cast<const float16_t *>(in_ptr_start + in_stride_bytes_hwc * id[3]); + + for (; x <= window_end_x - window_step_x; x += window_step_x) + { + wrapper::vstore(reinterpret_cast<float16_t *>(out.ptr()) + x, + wrapper::vloadq(in_ptr + offset + offset_row + x)); + } + for (; x < window_end_x; ++x) + { + *(reinterpret_cast<float16_t *>(out.ptr()) + x) = *(in_ptr + offset + offset_row + x); + } + }, + out); } -void fp16_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) +void fp16_neon_scale_bilinear(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) { // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + const auto hr = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); Iterator out(dst, window); const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; @@ -103,68 +121,97 @@ void fp16_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *o win_in.set(Window::DimZ, Window::Dimension(0, 0, 0)); Iterator in(src, win_in); - if(border_mode == BorderMode::CONSTANT) + if (border_mode == BorderMode::CONSTANT) { using ConstType = typename std::conditional<std::is_same<float16_t, float16_t>::value, half, float16_t>::type; const float16_t const_border_value = static_cast<float16_t>(constant_border_value.get<ConstType>()); - execute_window_loop(window, [&](const Coordinates & id) - { - const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - const float16_t *in_ptr = reinterpret_cast<const float16_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; - - const auto a00 = (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value; - const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) ? *(in_ptr + in_stride_c) : const_border_value; - const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_wc) : const_border_value; - const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value; - - *reinterpret_cast<float16_t *>(out.ptr()) = static_cast<float16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); + execute_window_loop( + window, + [&](const Coordinates &id) + { + const auto offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); + const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); + const float16_t *in_ptr = + reinterpret_cast<const float16_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; + + const auto a00 = + (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value; + const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) + ? *(in_ptr + in_stride_c) + : const_border_value; + const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) + ? *(in_ptr + in_stride_wc) + : const_border_value; + const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) + ? *(in_ptr + in_stride_c + in_stride_wc) + : const_border_value; + + *reinterpret_cast<float16_t *>(out.ptr()) = + static_cast<float16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); + }, + in, out); } - else if(border_mode == BorderMode::REPLICATE) + else if (border_mode == BorderMode::REPLICATE) { - execute_window_loop(window, [&](const Coordinates & id) - { - const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - - auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1); - auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1); - auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1); - auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1); - - const auto a00 = *(reinterpret_cast<const float16_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc); - const auto a01 = *(reinterpret_cast<const float16_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc); - const auto a10 = *(reinterpret_cast<const float16_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h1 * in_stride_wc); - const auto a11 = *(reinterpret_cast<const float16_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h1 * in_stride_wc); - - *reinterpret_cast<float16_t *>(out.ptr()) = static_cast<float16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); + execute_window_loop( + window, + [&](const Coordinates &id) + { + const auto offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); + const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); + + auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1); + auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1); + auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1); + auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1); + + const auto a00 = *(reinterpret_cast<const float16_t *>(in.ptr()) + clamped_w * in_stride_c + + clamped_h * in_stride_wc); + const auto a01 = *(reinterpret_cast<const float16_t *>(in.ptr()) + clamped_w1 * in_stride_c + + clamped_h * in_stride_wc); + const auto a10 = *(reinterpret_cast<const float16_t *>(in.ptr()) + clamped_w * in_stride_c + + clamped_h1 * in_stride_wc); + const auto a11 = *(reinterpret_cast<const float16_t *>(in.ptr()) + clamped_w1 * in_stride_c + + clamped_h1 * in_stride_wc); + + *reinterpret_cast<float16_t *>(out.ptr()) = + static_cast<float16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); + }, + in, out); } else { ARM_COMPUTE_ERROR("Not implemented"); } } -} +} // namespace namespace cpu { -void fp16_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) +void fp16_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + if (policy == InterpolationPolicy::BILINEAR) { - fp16_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + fp16_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, + align_corners, window); } - else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) + else if (policy == InterpolationPolicy::NEAREST_NEIGHBOR) { fp16_neon_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); } @@ -172,4 +219,4 @@ void fp16_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, c } // namespace cpu } // namespace arm_compute -#endif /* defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) */
\ No newline at end of file +#endif /* defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) */ diff --git a/src/cpu/kernels/scale/neon/integer.cpp b/src/cpu/kernels/scale/neon/integer.cpp index 2ab14cf83a..bbf92e0412 100644 --- a/src/cpu/kernels/scale/neon/integer.cpp +++ b/src/cpu/kernels/scale/neon/integer.cpp @@ -22,8 +22,9 @@ * SOFTWARE. */ #include "arm_compute/core/Helpers.h" -#include "src/core/NEON/wrapper/wrapper.h" + #include "src/core/helpers/ScaleHelpers.h" +#include "src/core/NEON/wrapper/wrapper.h" #include "src/core/utils/ScaleUtils.h" #include "support/Rounding.h" @@ -33,8 +34,12 @@ namespace arm_compute { namespace { -void u8_neon_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offsets, - float sampling_offset, bool align_corners, const Window &window) +void u8_neon_scale_nearest(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + float sampling_offset, + bool align_corners, + const Window &window) { const size_t in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; const size_t in_stride_w = src->info()->dimension(1) + src->info()->padding().top + src->info()->padding().bottom; @@ -54,43 +59,58 @@ void u8_neon_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offs const uint8_t *in_ptr_start = src->buffer() + src->info()->offset_first_element_in_bytes(); const unsigned int in_stride_bytes_hwc = src->info()->strides_in_bytes()[3]; - execute_window_loop(win, [&](const Coordinates & id) - { - const int32_t offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))) * in_stride_c; - const auto in_hi = static_cast<int>(align_corners ? utils::rounding::round_half_away_from_zero((id.z() + sampling_offset) * hr) : std::floor((id.z() + sampling_offset) * hr)); - const int offset_row = in_hi * in_stride_wc; - int32_t x = window_start_x; - const uint8_t *in_ptr = reinterpret_cast<const uint8_t *>(in_ptr_start + in_stride_bytes_hwc * id[3]); - - for(; x <= window_end_x - window_step_x; x += window_step_x) + execute_window_loop( + win, + [&](const Coordinates &id) { - wrapper::vstore(reinterpret_cast<uint8_t *>(out.ptr()) + x, - wrapper::vloadq(in_ptr + offset + offset_row + x)); - } - for(; x < window_end_x; ++x) - { - *(reinterpret_cast<uint8_t *>(out.ptr()) + x) = *(in_ptr + offset + offset_row + x); - } - }, - out); + const int32_t offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))) * in_stride_c; + const auto in_hi = static_cast<int>( + align_corners ? utils::rounding::round_half_away_from_zero((id.z() + sampling_offset) * hr) + : std::floor((id.z() + sampling_offset) * hr)); + const int offset_row = in_hi * in_stride_wc; + int32_t x = window_start_x; + const uint8_t *in_ptr = reinterpret_cast<const uint8_t *>(in_ptr_start + in_stride_bytes_hwc * id[3]); + + for (; x <= window_end_x - window_step_x; x += window_step_x) + { + wrapper::vstore(reinterpret_cast<uint8_t *>(out.ptr()) + x, + wrapper::vloadq(in_ptr + offset + offset_row + x)); + } + for (; x < window_end_x; ++x) + { + *(reinterpret_cast<uint8_t *>(out.ptr()) + x) = *(in_ptr + offset + offset_row + x); + } + }, + out); } -void u8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) +void u8_neon_scale_bilinear(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) { // Compute the ratio between source and destination dimensions - const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); - const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + const float scale_x = + scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); const int input_width = src->info()->dimension(1); const int input_height = src->info()->dimension(2); - if(border_mode == BorderMode::CONSTANT) + if (border_mode == BorderMode::CONSTANT) { Iterator out(dst, window); - const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; - const int in_stride_wc = in_stride_c * (input_width + src->info()->padding().top + src->info()->padding().bottom); + const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; + const int in_stride_wc = + in_stride_c * (input_width + src->info()->padding().top + src->info()->padding().bottom); // Don't increment in Y and Z direction for the input tensor // A pointer to the start of this plane is needed as base for the precomputed offsets @@ -100,24 +120,37 @@ void u8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off Iterator in(src, win_in); const uint8_t const_border_value = static_cast<uint8_t>(constant_border_value.get<uint8_t>()); - execute_window_loop(window, [&](const Coordinates & id) - { - const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int32_t in_hi = std::floor((id.z() + sampling_offset) * scale_y - sampling_offset); - const uint8_t *in_ptr = reinterpret_cast<const uint8_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; - - const auto a00 = (0 <= offset && offset < input_width && 0 <= in_hi && in_hi < input_height) ? *in_ptr : const_border_value; - const auto a01 = (-1 <= offset && offset < input_width - 1 && 0 <= in_hi && in_hi < input_height) ? *(in_ptr + in_stride_c) : const_border_value; - const auto a10 = (0 <= offset && offset < input_width && -1 <= in_hi && in_hi < input_height - 1) ? *(in_ptr + in_stride_wc) : const_border_value; - const auto a11 = (-1 <= offset && offset < input_width - 1 && -1 <= in_hi && in_hi < input_height - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value; - - *reinterpret_cast<uint8_t *>(out.ptr()) = static_cast<uint8_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); + execute_window_loop( + window, + [&](const Coordinates &id) + { + const auto offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); + const int32_t in_hi = std::floor((id.z() + sampling_offset) * scale_y - sampling_offset); + const uint8_t *in_ptr = + reinterpret_cast<const uint8_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; + + const auto a00 = (0 <= offset && offset < input_width && 0 <= in_hi && in_hi < input_height) + ? *in_ptr + : const_border_value; + const auto a01 = (-1 <= offset && offset < input_width - 1 && 0 <= in_hi && in_hi < input_height) + ? *(in_ptr + in_stride_c) + : const_border_value; + const auto a10 = (0 <= offset && offset < input_width && -1 <= in_hi && in_hi < input_height - 1) + ? *(in_ptr + in_stride_wc) + : const_border_value; + const auto a11 = (-1 <= offset && offset < input_width - 1 && -1 <= in_hi && in_hi < input_height - 1) + ? *(in_ptr + in_stride_c + in_stride_wc) + : const_border_value; + + *reinterpret_cast<uint8_t *>(out.ptr()) = + static_cast<uint8_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); + }, + in, out); } - else if(border_mode == BorderMode::REPLICATE) + else if (border_mode == BorderMode::REPLICATE) { using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>; @@ -152,12 +185,12 @@ void u8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off const float fp_coord_offset_y = sampling_offset * (scale_y - 1); const float fp_coord_offset_x = sampling_offset * (scale_x - 1); - for(int bo = bo_start; bo < bo_end; bo += bo_step) + for (int bo = bo_start; bo < bo_end; bo += bo_step) { const uint8_t *in_ptr = in.ptr() + bo * in_stride_b; uint8_t *out_ptr = out.ptr() + bo * out_stride_b; - for(int yo = yo_start; yo < yo_end; yo += yo_step) + for (int yo = yo_start; yo < yo_end; yo += yo_step) { // Floating-point coordinate const float yi_f = yo * scale_y + fp_coord_offset_y; @@ -174,7 +207,7 @@ void u8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off const uint8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; uint8_t *out_ptr_yo = out_ptr + yo * out_stride_y; - for(int xo = xo_start; xo < xo_end; xo += xo_step) + for (int xo = xo_start; xo < xo_end; xo += xo_step) { // Floating-point coordinate const float xi_f = xo * scale_x + fp_coord_offset_x; @@ -205,7 +238,7 @@ void u8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off uint8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; int cout = 0; - for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + for (; cout <= (out_dim_ch - step_cout); cout += step_cout) { const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); @@ -270,19 +303,21 @@ void u8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off const auto out_2_int = wrapper::vcvta<uint32_t>(out_2); const auto out_3_int = wrapper::vcvta<uint32_t>(out_3); #else // defined(__aarch64__) && !defined(BARE_METAL) - const auto out_0_int = wrapper::vcvt<uint32_t>(out_0); - const auto out_1_int = wrapper::vcvt<uint32_t>(out_1); - const auto out_2_int = wrapper::vcvt<uint32_t>(out_2); - const auto out_3_int = wrapper::vcvt<uint32_t>(out_3); + const auto out_0_int = wrapper::vcvt<uint32_t>(out_0); + const auto out_1_int = wrapper::vcvt<uint32_t>(out_1); + const auto out_2_int = wrapper::vcvt<uint32_t>(out_2); + const auto out_3_int = wrapper::vcvt<uint32_t>(out_3); #endif // defined(__aarch64__) && !defined(BARE_METAL) - const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); - const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); - const auto out = wrapper::vcombine(low_part, high_part); + const auto low_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); wrapper::vstore(out_ptr_xo_yo + cout * sizeof(uint8_t), out); } - for(; cout < out_dim_ch; ++cout) + for (; cout < out_dim_ch; ++cout) { const uint8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); const uint8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); @@ -311,18 +346,27 @@ void u8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off } } -void s8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) +void s8_neon_scale_bilinear(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) { ARM_COMPUTE_UNUSED(dx, dy, offsets, constant_border_value); - if(border_mode == BorderMode::REPLICATE) + if (border_mode == BorderMode::REPLICATE) { using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>; // Compute the ratio between source and destination dimensions - const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); - const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + const float scale_x = + scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); const int in_stride_x = src->info()->strides_in_bytes()[1]; const int in_stride_y = src->info()->strides_in_bytes()[2]; @@ -356,12 +400,12 @@ void s8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off const float fp_coord_offset_y = sampling_offset * (scale_y - 1); const float fp_coord_offset_x = sampling_offset * (scale_x - 1); - for(int bo = bo_start; bo < bo_end; bo += bo_step) + for (int bo = bo_start; bo < bo_end; bo += bo_step) { const int8_t *in_ptr = reinterpret_cast<int8_t *>(in.ptr() + bo * in_stride_b); int8_t *out_ptr = reinterpret_cast<int8_t *>(out.ptr() + bo * out_stride_b); - for(int yo = yo_start; yo < yo_end; yo += yo_step) + for (int yo = yo_start; yo < yo_end; yo += yo_step) { // Floating-point coordinate const float yi_f = yo * scale_y + fp_coord_offset_y; @@ -378,7 +422,7 @@ void s8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off const int8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; int8_t *out_ptr_yo = out_ptr + yo * out_stride_y; - for(int xo = xo_start; xo < xo_end; xo += xo_step) + for (int xo = xo_start; xo < xo_end; xo += xo_step) { // Floating-point coordinate const float xi_f = xo * scale_x + fp_coord_offset_x; @@ -409,7 +453,7 @@ void s8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off int8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; int cout = 0; - for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + for (; cout <= (out_dim_ch - step_cout); cout += step_cout) { const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); @@ -479,14 +523,16 @@ void s8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off const auto out_2_int = wrapper::vcvt<int32_t>(out_2); const auto out_3_int = wrapper::vcvt<int32_t>(out_3); #endif // defined(__aarch64__) && !defined(BARE_METAL) - const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); - const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); - const auto out = wrapper::vcombine(low_part, high_part); + const auto low_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); wrapper::vstore(out_ptr_xo_yo + cout * sizeof(int8_t), out); } - for(; cout < out_dim_ch; ++cout) + for (; cout < out_dim_ch; ++cout) { const int8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); const int8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); @@ -515,8 +561,12 @@ void s8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *off } } -void s16_neon_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *offsets, - float sampling_offset, bool align_corners, const Window &window) +void s16_neon_scale_nearest(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + float sampling_offset, + bool align_corners, + const Window &window) { const size_t in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; const size_t in_stride_w = src->info()->dimension(1) + src->info()->padding().top + src->info()->padding().bottom; @@ -536,33 +586,46 @@ void s16_neon_scale_nearest(const ITensor *src, ITensor *dst, const ITensor *off const uint8_t *in_ptr_start = src->buffer() + src->info()->offset_first_element_in_bytes(); const unsigned int in_stride_bytes_hwc = src->info()->strides_in_bytes()[3]; - execute_window_loop(win, [&](const Coordinates & id) - { - const int32_t offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))) * in_stride_c; - const auto in_hi = static_cast<int>(align_corners ? utils::rounding::round_half_away_from_zero((id.z() + sampling_offset) * hr) : std::floor((id.z() + sampling_offset) * hr)); - const int offset_row = in_hi * in_stride_wc; - int32_t x = window_start_x; - const int16_t *in_ptr = reinterpret_cast<const int16_t *>(in_ptr_start + in_stride_bytes_hwc * id[3]); - - for(; x <= window_end_x - window_step_x; x += window_step_x) - { - wrapper::vstore(reinterpret_cast<int16_t *>(out.ptr()) + x, - wrapper::vloadq(in_ptr + offset + offset_row + x)); - } - for(; x < window_end_x; ++x) + execute_window_loop( + win, + [&](const Coordinates &id) { - *(reinterpret_cast<int16_t *>(out.ptr()) + x) = *(in_ptr + offset + offset_row + x); - } - }, - out); + const int32_t offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))) * in_stride_c; + const auto in_hi = static_cast<int>( + align_corners ? utils::rounding::round_half_away_from_zero((id.z() + sampling_offset) * hr) + : std::floor((id.z() + sampling_offset) * hr)); + const int offset_row = in_hi * in_stride_wc; + int32_t x = window_start_x; + const int16_t *in_ptr = reinterpret_cast<const int16_t *>(in_ptr_start + in_stride_bytes_hwc * id[3]); + + for (; x <= window_end_x - window_step_x; x += window_step_x) + { + wrapper::vstore(reinterpret_cast<int16_t *>(out.ptr()) + x, + wrapper::vloadq(in_ptr + offset + offset_row + x)); + } + for (; x < window_end_x; ++x) + { + *(reinterpret_cast<int16_t *>(out.ptr()) + x) = *(in_ptr + offset + offset_row + x); + } + }, + out); } -void s16_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) +void s16_neon_scale_bilinear(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) { // Compute the ratio between source height and destination height - const auto hr = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + const auto hr = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); Iterator out(dst, window); const int in_stride_c = src->info()->dimension(0) + src->info()->padding().left + src->info()->padding().right; @@ -577,64 +640,93 @@ void s16_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *of win_in.set(Window::DimZ, Window::Dimension(0, 0, 0)); Iterator in(src, win_in); - if(border_mode == BorderMode::CONSTANT) + if (border_mode == BorderMode::CONSTANT) { const int16_t const_border_value = static_cast<int16_t>(constant_border_value.get<int16_t>()); - execute_window_loop(window, [&](const Coordinates & id) - { - const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - const int16_t *in_ptr = reinterpret_cast<const int16_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; - - const auto a00 = (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value; - const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) ? *(in_ptr + in_stride_c) : const_border_value; - const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_wc) : const_border_value; - const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) ? *(in_ptr + in_stride_c + in_stride_wc) : const_border_value; - - *reinterpret_cast<int16_t *>(out.ptr()) = static_cast<int16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); + execute_window_loop( + window, + [&](const Coordinates &id) + { + const auto offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); + const int32_t in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); + const int16_t *in_ptr = + reinterpret_cast<const int16_t *>(in.ptr()) + offset * in_stride_c + in_hi * in_stride_wc; + + const auto a00 = + (0 <= offset && offset < in_dim_w && 0 <= in_hi && in_hi < in_dim_h) ? *in_ptr : const_border_value; + const auto a01 = (-1 <= offset && offset < in_dim_w - 1 && 0 <= in_hi && in_hi < in_dim_h) + ? *(in_ptr + in_stride_c) + : const_border_value; + const auto a10 = (0 <= offset && offset < in_dim_w && -1 <= in_hi && in_hi < in_dim_h - 1) + ? *(in_ptr + in_stride_wc) + : const_border_value; + const auto a11 = (-1 <= offset && offset < in_dim_w - 1 && -1 <= in_hi && in_hi < in_dim_h - 1) + ? *(in_ptr + in_stride_c + in_stride_wc) + : const_border_value; + + *reinterpret_cast<int16_t *>(out.ptr()) = + static_cast<int16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); + }, + in, out); } - else if(border_mode == BorderMode::REPLICATE) + else if (border_mode == BorderMode::REPLICATE) { - execute_window_loop(window, [&](const Coordinates & id) - { - const auto offset = *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); - const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); - const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); - - const auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1); - const auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1); - const auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1); - const auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1); - - const auto a00 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc); - const auto a01 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h * in_stride_wc); - const auto a10 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h1 * in_stride_wc); - const auto a11 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w1 * in_stride_c + clamped_h1 * in_stride_wc); - - *reinterpret_cast<int16_t *>(out.ptr()) = static_cast<int16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); - }, - in, out); + execute_window_loop( + window, + [&](const Coordinates &id) + { + const auto offset = + *reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dx_val = *reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id.y(), id.z()))); + const auto dy_val = *reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id.y(), id.z()))); + const int in_hi = std::floor((id.z() + sampling_offset) * hr - sampling_offset); + + const auto clamped_w = utility::clamp<int>(offset, 0, in_dim_w - 1); + const auto clamped_w1 = utility::clamp<int>(offset + 1, 0, in_dim_w - 1); + const auto clamped_h = utility::clamp<int>(in_hi, 0, in_dim_h - 1); + const auto clamped_h1 = utility::clamp<int>(in_hi + 1, 0, in_dim_h - 1); + + const auto a00 = + *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w * in_stride_c + clamped_h * in_stride_wc); + const auto a01 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w1 * in_stride_c + + clamped_h * in_stride_wc); + const auto a10 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w * in_stride_c + + clamped_h1 * in_stride_wc); + const auto a11 = *(reinterpret_cast<const int16_t *>(in.ptr()) + clamped_w1 * in_stride_c + + clamped_h1 * in_stride_wc); + + *reinterpret_cast<int16_t *>(out.ptr()) = + static_cast<int16_t>(scale_helpers::delta_bilinear(a00, a01, a10, a11, dx_val, dy_val)); + }, + in, out); } else { ARM_COMPUTE_ERROR("Not implemented"); } } -} +} // namespace namespace cpu { -void s8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) +void s8_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + if (policy == InterpolationPolicy::BILINEAR) { - s8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + s8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, + align_corners, window); } else { @@ -642,32 +734,50 @@ void s8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, con } } -void u8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) +void u8_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + if (policy == InterpolationPolicy::BILINEAR) { - u8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + u8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, + align_corners, window); } - else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) + else if (policy == InterpolationPolicy::NEAREST_NEIGHBOR) { u8_neon_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); } } -void s16_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) +void s16_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + if (policy == InterpolationPolicy::BILINEAR) { - s16_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + s16_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, + align_corners, window); } - else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) + else if (policy == InterpolationPolicy::NEAREST_NEIGHBOR) { s16_neon_scale_nearest(src, dst, offsets, sampling_offset, align_corners, window); } } } // namespace cpu -} // namespace arm_compute
\ No newline at end of file +} // namespace arm_compute diff --git a/src/cpu/kernels/scale/neon/list.h b/src/cpu/kernels/scale/neon/list.h index 28a1087224..0fe87d15a6 100644 --- a/src/cpu/kernels/scale/neon/list.h +++ b/src/cpu/kernels/scale/neon/list.h @@ -26,6 +26,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Window.h" + #include "src/core/NEON/wrapper/wrapper.h" #include "src/core/utils/ScaleUtils.h" #include "support/Rounding.h" @@ -34,10 +35,10 @@ namespace arm_compute { namespace cpu { -#define DECLARE_SCALE_KERNEL(func_name) \ - void func_name(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, \ - InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, \ - bool align_corners, const Window &window) +#define DECLARE_SCALE_KERNEL(func_name) \ + void func_name(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, \ + InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, \ + float sampling_offset, bool align_corners, const Window &window) DECLARE_SCALE_KERNEL(s16_neon_scale); DECLARE_SCALE_KERNEL(u8_neon_scale); @@ -48,14 +49,20 @@ DECLARE_SCALE_KERNEL(qasymm8_signed_neon_scale); #undef DECLARE_SCALE_KERNEL template <typename T> -void nearest_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, float sampling_offset, - bool align_corners, const Window &window) +void nearest_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + float sampling_offset, + bool align_corners, + const Window &window) { ARM_COMPUTE_UNUSED(offsets); // Compute the ratio between source and destination dimensions - const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); - const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + const float scale_x = + scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); const int in_stride_y = src->info()->strides_in_bytes()[1]; const int in_stride_z = src->info()->strides_in_bytes()[2]; @@ -84,17 +91,17 @@ void nearest_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets const int bo_end = window_execution[3].end(); const int bo_step = window_execution[3].step(); - for(int bo = bo_start; bo < bo_end; bo += bo_step) + for (int bo = bo_start; bo < bo_end; bo += bo_step) { const uint8_t *in_ptr_base = in.ptr() + bo * in_stride_w; uint8_t *out_ptr_base = out.ptr() + bo * out_stride_w; - for(int yo = yo_start; yo < yo_end; yo += yo_step) + for (int yo = yo_start; yo < yo_end; yo += yo_step) { // Floating-point coordinate float yi_f = ((yo + sampling_offset) * scale_y); int yi = 0; - if(align_corners) + if (align_corners) { yi = utils::rounding::round_half_away_from_zero(yi_f); } @@ -103,12 +110,12 @@ void nearest_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets yi = static_cast<int>(std::floor(yi_f)); } - for(int xo = xo_start; xo < xo_end; xo += xo_step) + for (int xo = xo_start; xo < xo_end; xo += xo_step) { // Floating-point coordinate float xi_f = ((xo + sampling_offset) * scale_x); int xi = 0; - if(align_corners) + if (align_corners) { xi = utils::rounding::round_half_away_from_zero(xi_f); } @@ -121,15 +128,15 @@ void nearest_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets uint8_t *out_ptr = out_ptr_base + xo * out_stride_y + yo * out_stride_z; int cout = 0; - for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + for (; cout <= (out_dim_ch - step_cout); cout += step_cout) { auto out0 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T))); wrapper::vstore(reinterpret_cast<T *>(out_ptr + cout * sizeof(T)), out0); } - for(; cout < out_dim_ch; ++cout) + for (; cout < out_dim_ch; ++cout) { - auto out0 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T))); + auto out0 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T))); *(reinterpret_cast<T *>(out_ptr + cout * sizeof(T))) = out0; } } @@ -138,9 +145,16 @@ void nearest_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets } template <typename T> -void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) +void bilinear_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) { ARM_COMPUTE_UNUSED(offsets); ARM_COMPUTE_UNUSED(dx); @@ -148,8 +162,10 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset using ExactTagType = typename wrapper::traits::neon_bitvector_tag_t<T, wrapper::traits::BitWidth::W128>; // Compute the ratio between source and destination dimensions - const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); - const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + const float scale_x = + scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); const int in_stride_y = src->info()->strides_in_bytes()[1]; const int in_stride_z = src->info()->strides_in_bytes()[2]; @@ -180,7 +196,7 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset const int bo_end = window_execution[3].end(); const int bo_step = window_execution[3].step(); - if(border_mode == BorderMode::CONSTANT) + if (border_mode == BorderMode::CONSTANT) { #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC using ConstType = typename std::conditional<std::is_same<T, float16_t>::value, half, T>::type; @@ -189,12 +205,12 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset #endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ const T const_border_value = static_cast<T>(constant_border_value.get<ConstType>()); - for(int bo = bo_start; bo < bo_end; bo += bo_step) + for (int bo = bo_start; bo < bo_end; bo += bo_step) { const uint8_t *in_ptr_base = in.ptr() + bo * in_stride_w; uint8_t *out_ptr_base = out.ptr() + bo * out_stride_w; - for(int yo = yo_start; yo < yo_end; yo += yo_step) + for (int yo = yo_start; yo < yo_end; yo += yo_step) { // Floating-point coordinate const float yi_f = ((yo + sampling_offset) * scale_y - sampling_offset); @@ -204,7 +220,7 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset const auto a1 = (yi_f - static_cast<float>(yi)); const auto b1 = (1.f - a1); - for(int xo = xo_start; xo < xo_end; xo += xo_step) + for (int xo = xo_start; xo < xo_end; xo += xo_step) { // Floating-point coordinate const float xi_f = ((xo + sampling_offset) * scale_x - sampling_offset); @@ -223,32 +239,35 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset uint8_t *out_ptr = out_ptr_base + xo * out_stride_y + yo * out_stride_z; int cout = 0; - for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + for (; cout <= (out_dim_ch - step_cout); cout += step_cout) { auto in00 = wrapper::vdup_n(static_cast<T>(const_border_value), ExactTagType{}); auto in01 = wrapper::vdup_n(static_cast<T>(const_border_value), ExactTagType{}); auto in10 = wrapper::vdup_n(static_cast<T>(const_border_value), ExactTagType{}); auto in11 = wrapper::vdup_n(static_cast<T>(const_border_value), ExactTagType{}); - if((yi >= 0) && (yi < in_dim_h)) + if ((yi >= 0) && (yi < in_dim_h)) { - if((xi >= 0) && (xi < in_dim_w)) + if ((xi >= 0) && (xi < in_dim_w)) { in00 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T))); } - if(((xi + 1) >= 0) && ((xi + 1) < in_dim_w)) + if (((xi + 1) >= 0) && ((xi + 1) < in_dim_w)) { - in01 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y)); + in01 = wrapper::vloadq( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y)); } } - if(((yi + 1) >= 0) && ((yi + 1) < in_dim_h)) + if (((yi + 1) >= 0) && ((yi + 1) < in_dim_h)) { - if((xi >= 0) && (xi < in_dim_w)) + if ((xi >= 0) && (xi < in_dim_w)) { - in10 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_z)); + in10 = wrapper::vloadq( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_z)); } - if(((xi + 1) >= 0) && ((xi + 1) < in_dim_w)) + if (((xi + 1) >= 0) && ((xi + 1) < in_dim_w)) { - in11 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y + in_stride_z)); + in11 = wrapper::vloadq( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y + in_stride_z)); } } @@ -264,32 +283,33 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset wrapper::vstore(reinterpret_cast<T *>(out_ptr + cout * sizeof(T)), out0); } - for(; cout < out_dim_ch; ++cout) + for (; cout < out_dim_ch; ++cout) { auto in00 = static_cast<T>(const_border_value); auto in01 = static_cast<T>(const_border_value); auto in10 = static_cast<T>(const_border_value); auto in11 = static_cast<T>(const_border_value); - if((yi >= 0) && (yi < in_dim_h)) + if ((yi >= 0) && (yi < in_dim_h)) { - if((xi >= 0) && (xi < in_dim_w)) + if ((xi >= 0) && (xi < in_dim_w)) { in00 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T))); } - if(((xi + 1) >= 0) && ((xi + 1) < in_dim_w)) + if (((xi + 1) >= 0) && ((xi + 1) < in_dim_w)) { in01 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y)); } } - if(((yi + 1) >= 0) && ((yi + 1) < in_dim_h)) + if (((yi + 1) >= 0) && ((yi + 1) < in_dim_h)) { - if((xi >= 0) && (xi < in_dim_w)) + if ((xi >= 0) && (xi < in_dim_w)) { in10 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_z)); } - if(((xi + 1) >= 0) && ((xi + 1) < in_dim_w)) + if (((xi + 1) >= 0) && ((xi + 1) < in_dim_w)) { - in11 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y + in_stride_z)); + in11 = *( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + in_stride_y + in_stride_z)); } } auto out0 = static_cast<T>(0); @@ -303,14 +323,14 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset } } } - else if(border_mode == BorderMode::REPLICATE) + else if (border_mode == BorderMode::REPLICATE) { - for(int bo = bo_start; bo < bo_end; bo += bo_step) + for (int bo = bo_start; bo < bo_end; bo += bo_step) { const uint8_t *in_ptr = in.ptr() + bo * in_stride_w; uint8_t *out_ptr = out.ptr() + bo * out_stride_w; - for(int yo = yo_start; yo < yo_end; yo += yo_step) + for (int yo = yo_start; yo < yo_end; yo += yo_step) { // Floating-point coordinate const float yi_f = ((yo + sampling_offset) * scale_y - sampling_offset); @@ -327,7 +347,7 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset const int yi1_offset = yi1 * in_stride_z; const int y_offset = yo * out_stride_z; - for(int xo = xo_start; xo < xo_end; xo += xo_step) + for (int xo = xo_start; xo < xo_end; xo += xo_step) { // Floating-point coordinate const float xi_f = ((xo + sampling_offset) * scale_x - sampling_offset); @@ -356,12 +376,16 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset const int offset = xo * out_stride_y + y_offset; int cout = 0; - for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + for (; cout <= (out_dim_ch - step_cout); cout += step_cout) { - const auto in00 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); - const auto in01 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); - const auto in10 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); - const auto in11 = wrapper::vloadq(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); + const auto in00 = wrapper::vloadq( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); + const auto in01 = wrapper::vloadq( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); + const auto in10 = wrapper::vloadq( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); + const auto in11 = wrapper::vloadq( + reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); auto out0 = wrapper::vmul(in00, s00); out0 = wrapper::vmla(out0, in01, s01); @@ -370,12 +394,16 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset wrapper::vstore(reinterpret_cast<T *>(out_ptr + offset + cout * sizeof(T)), out0); } - for(; cout < out_dim_ch; ++cout) + for (; cout < out_dim_ch; ++cout) { - const T in00 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); - const T in01 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); - const T in10 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); - const T in11 = *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); + const T in00 = + *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); + const T in01 = + *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); + const T in10 = + *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); + const T in11 = + *(reinterpret_cast<const T *>(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); T out0 = in00 * s00_s; out0 += in01 * s01_s; @@ -394,15 +422,24 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset } template <typename T> -void common_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) +void common_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + if (policy == InterpolationPolicy::BILINEAR) { - bilinear_neon_scale<T>(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + bilinear_neon_scale<T>(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, + align_corners, window); } - else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) + else if (policy == InterpolationPolicy::NEAREST_NEIGHBOR) { nearest_neon_scale<T>(src, dst, offsets, sampling_offset, align_corners, window); } diff --git a/src/cpu/kernels/scale/neon/qasymm8.cpp b/src/cpu/kernels/scale/neon/qasymm8.cpp index 778459ae39..62a821daa5 100644 --- a/src/cpu/kernels/scale/neon/qasymm8.cpp +++ b/src/cpu/kernels/scale/neon/qasymm8.cpp @@ -28,9 +28,16 @@ namespace arm_compute { namespace { -void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) +void qasymm8_neon_scale_bilinear(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) { // Data layout is NHWC const int32_t input_width = src->info()->dimension(1); @@ -40,10 +47,12 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor const UniformQuantizationInfo oq_info = dst->info()->quantization_info().uniform(); // Compute the ratio between source and destination dimensions - const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); - const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + const float scale_x = + scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); - if(border_mode == BorderMode::CONSTANT) + if (border_mode == BorderMode::CONSTANT) { const int32_t in_stride_y = src->info()->strides_in_bytes()[1]; const int32_t in_stride_z = src->info()->strides_in_bytes()[2]; @@ -59,7 +68,7 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor win_in.set(1, Window::Dimension(0, 0, 0)); win_in.set(2, Window::Dimension(0, 0, 0)); - for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) + for (size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) { win_off.set(d, Window::Dimension(0, 0, 0)); } @@ -68,36 +77,41 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor Iterator out(dst, window); const uint8_t const_border_value = static_cast<uint8_t>(constant_border_value.get<uint8_t>()); - execute_window_loop(window, [&](const Coordinates & id) - { - const int32_t index_h = std::floor((id[2] + sampling_offset) * scale_y - sampling_offset); - const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2])))); - const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2])))); - const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2])))); - const auto pixel_row_ptr = reinterpret_cast<const uint8_t *>(in.ptr()); - - const auto a00 = (0 <= index_w && index_w < input_width && 0 <= index_h && index_h < input_height) ? - (*(pixel_row_ptr + index_w * in_stride_y + index_h * in_stride_z)) : - const_border_value; - const auto a01 = (-1 <= index_w && index_w + 1 < input_width && 0 <= index_h && index_h < input_height) ? - (*(pixel_row_ptr + (index_w + 1) * in_stride_y + index_h * in_stride_z)) : - const_border_value; - const auto a10 = (0 <= index_w && index_w < input_width && -1 <= index_h && index_h < input_height - 1) ? - (*(pixel_row_ptr + index_w * in_stride_y + (index_h + 1) * in_stride_z)) : - const_border_value; - const auto a11 = (-1 <= index_w && index_w < input_width - 1 && -1 <= index_h && index_h < input_height - 1) ? - (*(pixel_row_ptr + (index_w + 1) * in_stride_y + (index_h + 1) * in_stride_z)) : - const_border_value; - - const float inp00 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a00, iq_info); - const float inp01 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a01, iq_info); - const float inp10 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a10, iq_info); - const float inp11 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a11, iq_info); - *reinterpret_cast<uint8_t *>(out.ptr()) = Qasymm8QuantizationHelper<uint8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); - }, - in, out); + execute_window_loop( + window, + [&](const Coordinates &id) + { + const int32_t index_h = std::floor((id[2] + sampling_offset) * scale_y - sampling_offset); + const int32_t index_w = + *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2])))); + const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2])))); + const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2])))); + const auto pixel_row_ptr = reinterpret_cast<const uint8_t *>(in.ptr()); + + const auto a00 = (0 <= index_w && index_w < input_width && 0 <= index_h && index_h < input_height) + ? (*(pixel_row_ptr + index_w * in_stride_y + index_h * in_stride_z)) + : const_border_value; + const auto a01 = (-1 <= index_w && index_w + 1 < input_width && 0 <= index_h && index_h < input_height) + ? (*(pixel_row_ptr + (index_w + 1) * in_stride_y + index_h * in_stride_z)) + : const_border_value; + const auto a10 = (0 <= index_w && index_w < input_width && -1 <= index_h && index_h < input_height - 1) + ? (*(pixel_row_ptr + index_w * in_stride_y + (index_h + 1) * in_stride_z)) + : const_border_value; + const auto a11 = + (-1 <= index_w && index_w < input_width - 1 && -1 <= index_h && index_h < input_height - 1) + ? (*(pixel_row_ptr + (index_w + 1) * in_stride_y + (index_h + 1) * in_stride_z)) + : const_border_value; + + const float inp00 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a00, iq_info); + const float inp01 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a01, iq_info); + const float inp10 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a10, iq_info); + const float inp11 = Qasymm8QuantizationHelper<uint8_t>::dequantize(a11, iq_info); + *reinterpret_cast<uint8_t *>(out.ptr()) = Qasymm8QuantizationHelper<uint8_t>::quantize( + scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); + }, + in, out); } - else if(border_mode == BorderMode::REPLICATE) + else if (border_mode == BorderMode::REPLICATE) { using FloatTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>; using Int32TagType = typename wrapper::traits::neon_bitvector_tag_t<int32_t, wrapper::traits::BitWidth::W128>; @@ -141,12 +155,12 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor const float fp_coord_offset_y = sampling_offset * (scale_y - 1); const float fp_coord_offset_x = sampling_offset * (scale_x - 1); - for(int bo = bo_start; bo < bo_end; bo += bo_step) + for (int bo = bo_start; bo < bo_end; bo += bo_step) { const uint8_t *in_ptr = in.ptr() + bo * in_stride_b; uint8_t *out_ptr = out.ptr() + bo * out_stride_b; - for(int yo = yo_start; yo < yo_end; yo += yo_step) + for (int yo = yo_start; yo < yo_end; yo += yo_step) { // Floating-point coordinate const float yi_f = yo * scale_y + fp_coord_offset_y; @@ -163,7 +177,7 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor const uint8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; uint8_t *out_ptr_yo = out_ptr + yo * out_stride_y; - for(int xo = xo_start; xo < xo_end; xo += xo_step) + for (int xo = xo_start; xo < xo_end; xo += xo_step) { // Floating-point coordinate const float xi_f = xo * scale_x + fp_coord_offset_x; @@ -194,7 +208,7 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor uint8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; int cout = 0; - for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + for (; cout <= (out_dim_ch - step_cout); cout += step_cout) { const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); @@ -204,34 +218,82 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor const uint16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00)); const uint16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00)); - const auto in00_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in00_low))), voffset_in)), vscale_in); - const auto in00_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in00_low))), voffset_in)), vscale_in); - const auto in00_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in00_high))), voffset_in)), vscale_in); - const auto in00_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in00_high))), voffset_in)), vscale_in); + const auto in00_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in00_low))), voffset_in)), + vscale_in); + const auto in00_1 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in00_low))), voffset_in)), + vscale_in); + const auto in00_2 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in00_high))), voffset_in)), + vscale_in); + const auto in00_3 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in00_high))), voffset_in)), + vscale_in); const uint16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01)); const uint16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01)); - const auto in01_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in01_low))), voffset_in)), vscale_in); - const auto in01_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in01_low))), voffset_in)), vscale_in); - const auto in01_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in01_high))), voffset_in)), vscale_in); - const auto in01_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in01_high))), voffset_in)), vscale_in); + const auto in01_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in01_low))), voffset_in)), + vscale_in); + const auto in01_1 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in01_low))), voffset_in)), + vscale_in); + const auto in01_2 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in01_high))), voffset_in)), + vscale_in); + const auto in01_3 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in01_high))), voffset_in)), + vscale_in); const uint16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10)); const uint16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10)); - const auto in10_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in10_low))), voffset_in)), vscale_in); - const auto in10_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in10_low))), voffset_in)), vscale_in); - const auto in10_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in10_high))), voffset_in)), vscale_in); - const auto in10_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in10_high))), voffset_in)), vscale_in); + const auto in10_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in10_low))), voffset_in)), + vscale_in); + const auto in10_1 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in10_low))), voffset_in)), + vscale_in); + const auto in10_2 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in10_high))), voffset_in)), + vscale_in); + const auto in10_3 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in10_high))), voffset_in)), + vscale_in); const uint16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11)); const uint16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11)); - const auto in11_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in11_low))), voffset_in)), vscale_in); - const auto in11_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in11_low))), voffset_in)), vscale_in); - const auto in11_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in11_high))), voffset_in)), vscale_in); - const auto in11_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in11_high))), voffset_in)), vscale_in); + const auto in11_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in11_low))), voffset_in)), + vscale_in); + const auto in11_1 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in11_low))), voffset_in)), + vscale_in); + const auto in11_2 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgetlow(in11_high))), voffset_in)), + vscale_in); + const auto in11_3 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub( + wrapper::vreinterpret(wrapper::vmovl(wrapper::vgethigh(in11_high))), voffset_in)), + vscale_in); auto out_0 = wrapper::vmul(in00_0, s00); out_0 = wrapper::vmla(out_0, in01_0, s01); @@ -264,14 +326,16 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor const auto out_2_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o)); const auto out_3_int = wrapper::vcvt<uint32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o)); #endif // defined(__aarch64__) && !defined(BARE_METAL) - const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); - const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); - const auto out = wrapper::vcombine(low_part, high_part); + const auto low_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); wrapper::vstore(out_ptr_xo_yo + cout * sizeof(uint8_t), out); } - for(; cout < out_dim_ch; ++cout) + for (; cout < out_dim_ch; ++cout) { const uint8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(uint8_t)); const uint8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(uint8_t)); @@ -292,7 +356,8 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor #if defined(__aarch64__) && !defined(BARE_METAL) *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = quantize_qasymm8(out, oq_info); #else // defined(__aarch64__) && !defined(BARE_METAL) - *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = quantize_qasymm8(out, oq_info, RoundingPolicy::TO_ZERO); + *(out_ptr_xo_yo + cout * sizeof(uint8_t)) = + quantize_qasymm8(out, oq_info, RoundingPolicy::TO_ZERO); #endif // defined(__aarch64__) && !defined(BARE_METAL) } } @@ -304,28 +369,38 @@ void qasymm8_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor ARM_COMPUTE_ERROR("Not implemented"); } } -} +} // namespace namespace cpu { -void qasymm8_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) +void qasymm8_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + if (policy == InterpolationPolicy::BILINEAR) { - if(src->info()->quantization_info() == dst->info()->quantization_info()) + if (src->info()->quantization_info() == dst->info()->quantization_info()) { - u8_neon_scale(src, dst, offsets, dx, dy, policy, border_mode, constant_border_value, sampling_offset, align_corners, window); + u8_neon_scale(src, dst, offsets, dx, dy, policy, border_mode, constant_border_value, sampling_offset, + align_corners, window); } else { - qasymm8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + qasymm8_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, + align_corners, window); } } - else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) + else if (policy == InterpolationPolicy::NEAREST_NEIGHBOR) { nearest_neon_scale<uint8_t>(src, dst, offsets, sampling_offset, align_corners, window); } } } // namespace cpu -} // namespace arm_compute
\ No newline at end of file +} // namespace arm_compute diff --git a/src/cpu/kernels/scale/neon/qasymm8_signed.cpp b/src/cpu/kernels/scale/neon/qasymm8_signed.cpp index cd63dfba63..5a885178a7 100644 --- a/src/cpu/kernels/scale/neon/qasymm8_signed.cpp +++ b/src/cpu/kernels/scale/neon/qasymm8_signed.cpp @@ -28,9 +28,16 @@ namespace arm_compute { namespace { -void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) +void qasymm8_signed_neon_scale_bilinear(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) { // Data layout is NHWC const UniformQuantizationInfo iq_info = src->info()->quantization_info().uniform(); @@ -40,10 +47,12 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const const int32_t input_height = src->info()->dimension(2); // Compute the ratio between source and destination dimensions - const float scale_x = scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); - const float scale_y = scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); + const float scale_x = + scale_utils::calculate_resize_ratio(src->info()->dimension(1), dst->info()->dimension(1), align_corners); + const float scale_y = + scale_utils::calculate_resize_ratio(src->info()->dimension(2), dst->info()->dimension(2), align_corners); - if(border_mode == BorderMode::CONSTANT) + if (border_mode == BorderMode::CONSTANT) { const int32_t in_stride_y = src->info()->strides_in_bytes()[1]; const int32_t in_stride_z = src->info()->strides_in_bytes()[2]; @@ -58,7 +67,7 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const win_in.set(1, Window::Dimension(0, 0, 0)); win_in.set(2, Window::Dimension(0, 0, 0)); - for(size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) + for (size_t d = Window::DimZ; d < offsets->info()->num_dimensions(); ++d) { win_off.set(d, Window::Dimension(0, 0, 0)); } @@ -67,36 +76,41 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const Iterator out(dst, window); const int8_t const_border_value = static_cast<int8_t>(constant_border_value.get<int8_t>()); - execute_window_loop(window, [&](const Coordinates & id) - { - const int32_t index_h = std::floor((id[2] + sampling_offset) * scale_y - sampling_offset); - const int32_t index_w = *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2])))); - const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2])))); - const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2])))); - const auto pixel_row_ptr = reinterpret_cast<const int8_t *>(in.ptr()); - - const auto a00 = (0 <= index_w && index_w < input_width && 0 <= index_h && index_h < input_height) ? - (*(pixel_row_ptr + index_w * in_stride_y + index_h * in_stride_z)) : - const_border_value; - const auto a01 = (-1 <= index_w && index_w + 1 < input_width && 0 <= index_h && index_h < input_height) ? - (*(pixel_row_ptr + (index_w + 1) * in_stride_y + index_h * in_stride_z)) : - const_border_value; - const auto a10 = (0 <= index_w && index_w < input_width && -1 <= index_h && index_h < input_height - 1) ? - (*(pixel_row_ptr + index_w * in_stride_y + (index_h + 1) * in_stride_z)) : - const_border_value; - const auto a11 = (-1 <= index_w && index_w < input_width - 1 && -1 <= index_h && index_h < input_height - 1) ? - (*(pixel_row_ptr + (index_w + 1) * in_stride_y + (index_h + 1) * in_stride_z)) : - const_border_value; - - const float inp00 = Qasymm8QuantizationHelper<int8_t>::dequantize(a00, iq_info); - const float inp01 = Qasymm8QuantizationHelper<int8_t>::dequantize(a01, iq_info); - const float inp10 = Qasymm8QuantizationHelper<int8_t>::dequantize(a10, iq_info); - const float inp11 = Qasymm8QuantizationHelper<int8_t>::dequantize(a11, iq_info); - *reinterpret_cast<int8_t *>(out.ptr()) = Qasymm8QuantizationHelper<int8_t>::quantize(scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); - }, - in, out); + execute_window_loop( + window, + [&](const Coordinates &id) + { + const int32_t index_h = std::floor((id[2] + sampling_offset) * scale_y - sampling_offset); + const int32_t index_w = + *(reinterpret_cast<const int32_t *>(offsets->ptr_to_element(Coordinates(id[1], id[2])))); + const auto dx_val = *(reinterpret_cast<const float *>(dx->ptr_to_element(Coordinates(id[1], id[2])))); + const auto dy_val = *(reinterpret_cast<const float *>(dy->ptr_to_element(Coordinates(id[1], id[2])))); + const auto pixel_row_ptr = reinterpret_cast<const int8_t *>(in.ptr()); + + const auto a00 = (0 <= index_w && index_w < input_width && 0 <= index_h && index_h < input_height) + ? (*(pixel_row_ptr + index_w * in_stride_y + index_h * in_stride_z)) + : const_border_value; + const auto a01 = (-1 <= index_w && index_w + 1 < input_width && 0 <= index_h && index_h < input_height) + ? (*(pixel_row_ptr + (index_w + 1) * in_stride_y + index_h * in_stride_z)) + : const_border_value; + const auto a10 = (0 <= index_w && index_w < input_width && -1 <= index_h && index_h < input_height - 1) + ? (*(pixel_row_ptr + index_w * in_stride_y + (index_h + 1) * in_stride_z)) + : const_border_value; + const auto a11 = + (-1 <= index_w && index_w < input_width - 1 && -1 <= index_h && index_h < input_height - 1) + ? (*(pixel_row_ptr + (index_w + 1) * in_stride_y + (index_h + 1) * in_stride_z)) + : const_border_value; + + const float inp00 = Qasymm8QuantizationHelper<int8_t>::dequantize(a00, iq_info); + const float inp01 = Qasymm8QuantizationHelper<int8_t>::dequantize(a01, iq_info); + const float inp10 = Qasymm8QuantizationHelper<int8_t>::dequantize(a10, iq_info); + const float inp11 = Qasymm8QuantizationHelper<int8_t>::dequantize(a11, iq_info); + *reinterpret_cast<int8_t *>(out.ptr()) = Qasymm8QuantizationHelper<int8_t>::quantize( + scale_helpers::delta_bilinear(inp00, inp01, inp10, inp11, dx_val, dy_val), oq_info); + }, + in, out); } - else if(border_mode == BorderMode::REPLICATE) + else if (border_mode == BorderMode::REPLICATE) { using FloatTagType = typename wrapper::traits::neon_bitvector_tag_t<float, wrapper::traits::BitWidth::W128>; using Int32TagType = typename wrapper::traits::neon_bitvector_tag_t<int32_t, wrapper::traits::BitWidth::W128>; @@ -140,12 +154,12 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const const float32x4_t invvscale_o = wrapper::vdup_n(1.f / oq_info.scale, FloatTagType{}); const float32x4_t voffset_o = vdupq_n_f32(oq_info.offset); - for(int bo = bo_start; bo < bo_end; bo += bo_step) + for (int bo = bo_start; bo < bo_end; bo += bo_step) { const int8_t *in_ptr = reinterpret_cast<int8_t *>(in.ptr() + bo * in_stride_b); int8_t *out_ptr = reinterpret_cast<int8_t *>(out.ptr() + bo * out_stride_b); - for(int yo = yo_start; yo < yo_end; yo += yo_step) + for (int yo = yo_start; yo < yo_end; yo += yo_step) { // Floating-point coordinate const float yi_f = yo * scale_y + fp_coord_offset_y; @@ -162,7 +176,7 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const const int8_t *in_ptr_yi1 = in_ptr + yi1 * in_stride_y; int8_t *out_ptr_yo = out_ptr + yo * out_stride_y; - for(int xo = xo_start; xo < xo_end; xo += xo_step) + for (int xo = xo_start; xo < xo_end; xo += xo_step) { // Floating-point coordinate const float xi_f = xo * scale_x + fp_coord_offset_x; @@ -193,7 +207,7 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const int8_t *out_ptr_xo_yo = out_ptr_yo + xo * out_stride_x; int cout = 0; - for(; cout <= (out_dim_ch - step_cout); cout += step_cout) + for (; cout <= (out_dim_ch - step_cout); cout += step_cout) { const auto in00 = wrapper::vloadq(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); const auto in01 = wrapper::vloadq(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); @@ -203,34 +217,70 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const const int16x8_t in00_low = wrapper::vmovl(wrapper::vgetlow(in00)); const int16x8_t in00_high = wrapper::vmovl(wrapper::vgethigh(in00)); - const auto in00_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in00_low)), voffset_in)), vscale_in); - const auto in00_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in00_low)), voffset_in)), vscale_in); - const auto in00_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in00_high)), voffset_in)), vscale_in); - const auto in00_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in00_high)), voffset_in)), vscale_in); + const auto in00_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in00_low)), voffset_in)), + vscale_in); + const auto in00_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgethigh(in00_low)), voffset_in)), + vscale_in); + const auto in00_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgetlow(in00_high)), voffset_in)), + vscale_in); + const auto in00_3 = + wrapper::vmul(wrapper::vcvt<float>( + wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in00_high)), voffset_in)), + vscale_in); const int16x8_t in01_low = wrapper::vmovl(wrapper::vgetlow(in01)); const int16x8_t in01_high = wrapper::vmovl(wrapper::vgethigh(in01)); - const auto in01_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in01_low)), voffset_in)), vscale_in); - const auto in01_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in01_low)), voffset_in)), vscale_in); - const auto in01_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in01_high)), voffset_in)), vscale_in); - const auto in01_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in01_high)), voffset_in)), vscale_in); + const auto in01_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in01_low)), voffset_in)), + vscale_in); + const auto in01_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgethigh(in01_low)), voffset_in)), + vscale_in); + const auto in01_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgetlow(in01_high)), voffset_in)), + vscale_in); + const auto in01_3 = + wrapper::vmul(wrapper::vcvt<float>( + wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in01_high)), voffset_in)), + vscale_in); const int16x8_t in10_low = wrapper::vmovl(wrapper::vgetlow(in10)); const int16x8_t in10_high = wrapper::vmovl(wrapper::vgethigh(in10)); - const auto in10_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in10_low)), voffset_in)), vscale_in); - const auto in10_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in10_low)), voffset_in)), vscale_in); - const auto in10_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in10_high)), voffset_in)), vscale_in); - const auto in10_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in10_high)), voffset_in)), vscale_in); + const auto in10_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in10_low)), voffset_in)), + vscale_in); + const auto in10_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgethigh(in10_low)), voffset_in)), + vscale_in); + const auto in10_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgetlow(in10_high)), voffset_in)), + vscale_in); + const auto in10_3 = + wrapper::vmul(wrapper::vcvt<float>( + wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in10_high)), voffset_in)), + vscale_in); const int16x8_t in11_low = wrapper::vmovl(wrapper::vgetlow(in11)); const int16x8_t in11_high = wrapper::vmovl(wrapper::vgethigh(in11)); - const auto in11_0 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in11_low)), voffset_in)), vscale_in); - const auto in11_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in11_low)), voffset_in)), vscale_in); - const auto in11_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in11_high)), voffset_in)), vscale_in); - const auto in11_3 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in11_high)), voffset_in)), vscale_in); + const auto in11_0 = wrapper::vmul( + wrapper::vcvt<float>(wrapper::vsub(wrapper::vmovl(wrapper::vgetlow(in11_low)), voffset_in)), + vscale_in); + const auto in11_1 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgethigh(in11_low)), voffset_in)), + vscale_in); + const auto in11_2 = wrapper::vmul(wrapper::vcvt<float>(wrapper::vsub( + wrapper::vmovl(wrapper::vgetlow(in11_high)), voffset_in)), + vscale_in); + const auto in11_3 = + wrapper::vmul(wrapper::vcvt<float>( + wrapper::vsub(wrapper::vmovl(wrapper::vgethigh(in11_high)), voffset_in)), + vscale_in); auto out_0 = wrapper::vmul(in00_0, s00); out_0 = wrapper::vmla(out_0, in01_0, s01); @@ -263,14 +313,16 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const const auto out_2_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_2, invvscale_o)); const auto out_3_int = wrapper::vcvt<int32_t>(wrapper::vmla(voffset_o, out_3, invvscale_o)); #endif // defined(__aarch64__) && !defined(BARE_METAL) - const auto low_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); - const auto high_part = wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); - const auto out = wrapper::vcombine(low_part, high_part); + const auto low_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_0_int), wrapper::vqmovn(out_1_int))); + const auto high_part = + wrapper::vqmovn(wrapper::vcombine(wrapper::vqmovn(out_2_int), wrapper::vqmovn(out_3_int))); + const auto out = wrapper::vcombine(low_part, high_part); wrapper::vstore(out_ptr_xo_yo + cout * sizeof(int8_t), out); } - for(; cout < out_dim_ch; ++cout) + for (; cout < out_dim_ch; ++cout) { const int8_t in00 = *(in_ptr_xi0_yi0 + cout * sizeof(int8_t)); const int8_t in01 = *(in_ptr_xi1_yi0 + cout * sizeof(int8_t)); @@ -291,7 +343,8 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const #if defined(__aarch64__) && !defined(BARE_METAL) *(out_ptr_xo_yo + cout * sizeof(int8_t)) = quantize_qasymm8_signed(out, oq_info); #else // defined(__aarch64__) && !defined(BARE_METAL) - *(out_ptr_xo_yo + cout * sizeof(int8_t)) = quantize_qasymm8_signed(out, oq_info, RoundingPolicy::TO_ZERO); + *(out_ptr_xo_yo + cout * sizeof(int8_t)) = + quantize_qasymm8_signed(out, oq_info, RoundingPolicy::TO_ZERO); #endif // defined(__aarch64__) && !defined(BARE_METAL) } } @@ -303,28 +356,39 @@ void qasymm8_signed_neon_scale_bilinear(const ITensor *src, ITensor *dst, const ARM_COMPUTE_ERROR("Not implemented"); } } -} +} // namespace namespace cpu { -void qasymm8_signed_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets, const ITensor *dx, const ITensor *dy, - InterpolationPolicy policy, BorderMode border_mode, PixelValue constant_border_value, float sampling_offset, - bool align_corners, const Window &window) +void qasymm8_signed_neon_scale(const ITensor *src, + ITensor *dst, + const ITensor *offsets, + const ITensor *dx, + const ITensor *dy, + InterpolationPolicy policy, + BorderMode border_mode, + PixelValue constant_border_value, + float sampling_offset, + bool align_corners, + const Window &window) { - if(policy == InterpolationPolicy::BILINEAR) + if (policy == InterpolationPolicy::BILINEAR) { - if(src->info()->quantization_info() == dst->info()->quantization_info() && border_mode == BorderMode::REPLICATE) + if (src->info()->quantization_info() == dst->info()->quantization_info() && + border_mode == BorderMode::REPLICATE) { - s8_neon_scale(src, dst, offsets, dx, dy, policy, border_mode, constant_border_value, sampling_offset, align_corners, window); + s8_neon_scale(src, dst, offsets, dx, dy, policy, border_mode, constant_border_value, sampling_offset, + align_corners, window); } else { - qasymm8_signed_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, sampling_offset, align_corners, window); + qasymm8_signed_neon_scale_bilinear(src, dst, offsets, dx, dy, border_mode, constant_border_value, + sampling_offset, align_corners, window); } } - else if(policy == InterpolationPolicy::NEAREST_NEIGHBOR) + else if (policy == InterpolationPolicy::NEAREST_NEIGHBOR) { nearest_neon_scale<int8_t>(src, dst, offsets, sampling_offset, align_corners, window); } } } // namespace cpu -} // namespace arm_compute
\ No newline at end of file +} // namespace arm_compute |