From 0eed305680ade0c48d07f592c4c4a8aaaad077b7 Mon Sep 17 00:00:00 2001 From: Gunes Bayir Date: Sun, 4 Sep 2022 21:00:10 +0100 Subject: =?UTF-8?q?Optimize=20FP32/16=20Bilinear=20Scale=20Kernel=20for=20?= =?UTF-8?q?Neon=E2=84=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch removes index and weight pre-computations where it's not used and reduces some calculations inside the inner-most loop of Scale. Resolves: COMPMID-5452 Change-Id: Ie149b1b76a90a8cb659ada0f97aef78caf69932f Signed-off-by: Gunes Bayir Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/8220 Benchmark: Arm Jenkins Tested-by: Arm Jenkins Reviewed-by: Gian Marco Iodice Comments-Addressed: Arm Jenkins --- src/core/utils/ScaleUtils.cpp | 18 ++++++++- src/core/utils/ScaleUtils.h | 13 +++++- src/cpu/kernels/CpuScaleKernel.cpp | 4 +- src/cpu/kernels/scale/neon/list.h | 73 +++++++++++++++++----------------- src/cpu/operators/CpuScale.cpp | 47 +++++++++++++--------- src/runtime/NEON/functions/NEScale.cpp | 58 ++++++++++++++++----------- 6 files changed, 130 insertions(+), 83 deletions(-) diff --git a/src/core/utils/ScaleUtils.cpp b/src/core/utils/ScaleUtils.cpp index d46ca0ea8e..82c6405e89 100644 --- a/src/core/utils/ScaleUtils.cpp +++ b/src/core/utils/ScaleUtils.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 Arm Limited. + * Copyright (c) 2020, 2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -21,8 +21,12 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ + #include "src/core/utils/ScaleUtils.h" -#include "arm_compute/core/Helpers.h" +#include "src/common/cpuinfo/CpuIsaInfo.h" + +#include "arm_compute/core/CPP/CPPTypes.h" +#include "arm_compute/core/TensorInfo.h" float arm_compute::scale_utils::calculate_resize_ratio(size_t input_size, size_t output_size, bool align_corners) { @@ -34,4 +38,14 @@ float arm_compute::scale_utils::calculate_resize_ratio(size_t input_size, size_t ARM_COMPUTE_ERROR_ON(out == 0); return static_cast(in) / static_cast(out); +} + +bool arm_compute::scale_utils::is_precomputation_required(DataLayout data_layout, DataType data_type, InterpolationPolicy policy) +{ + // whether to precompute indices & weights + // The Neon™ kernels (which are preferred over SVE when policy is BILINEAR) do not use + // precomputed index and weights when data type is FP32/16. + // If policy is nearest_neighbor for SVE, then precompute because it's being used + // To be revised in COMPMID-5453/5454 + return data_layout != DataLayout::NHWC || (data_type != DataType::F32 && data_type != DataType::F16) || (CPUInfo::get().get_isa().sve == true && policy == InterpolationPolicy::NEAREST_NEIGHBOR); } \ No newline at end of file diff --git a/src/core/utils/ScaleUtils.h b/src/core/utils/ScaleUtils.h index 3cc986b1db..c09509253c 100644 --- a/src/core/utils/ScaleUtils.h +++ b/src/core/utils/ScaleUtils.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 Arm Limited. + * Copyright (c) 2020, 2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -53,6 +53,17 @@ inline bool is_align_corners_allowed_sampling_policy(SamplingPolicy sampling_pol { return sampling_policy != SamplingPolicy::CENTER; } + +/** Returns if precomputation of indices and/or weights is required or/not + * + * @param[in] data_layout Data layout + * @param[in] data_type Data type + * @param[in] policy Interpolation policy + * + * @return True if precomputation is required + */ +bool is_precomputation_required(DataLayout data_layout, DataType data_type, InterpolationPolicy policy); + } // namespace scale_utils } // namespace arm_compute #endif /* UTILS_CORE_SCALEUTILS_H */ \ No newline at end of file diff --git a/src/cpu/kernels/CpuScaleKernel.cpp b/src/cpu/kernels/CpuScaleKernel.cpp index c9e858fc02..e7386a385a 100644 --- a/src/cpu/kernels/CpuScaleKernel.cpp +++ b/src/cpu/kernels/CpuScaleKernel.cpp @@ -140,12 +140,12 @@ Status validate_arguments(const ITensorInfo *src, const ITensorInfo *dx, const I ARM_COMPUTE_RETURN_ERROR_ON(output_width == 0); ARM_COMPUTE_RETURN_ERROR_ON(output_height == 0); - if(info.interpolation_policy == InterpolationPolicy::NEAREST_NEIGHBOR) + if(info.interpolation_policy == InterpolationPolicy::NEAREST_NEIGHBOR && offsets != nullptr) { ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(offsets, 1, DataType::S32); } - if(info.interpolation_policy == InterpolationPolicy::BILINEAR) + if(info.interpolation_policy == InterpolationPolicy::BILINEAR && offsets != nullptr) { ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(offsets, 1, DataType::S32); if(dx != nullptr && dy != nullptr) diff --git a/src/cpu/kernels/scale/neon/list.h b/src/cpu/kernels/scale/neon/list.h index 9679f161e7..17ff4bb676 100644 --- a/src/cpu/kernels/scale/neon/list.h +++ b/src/cpu/kernels/scale/neon/list.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -131,7 +131,7 @@ void nearest_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offsets for(; cout < out_dim_ch; ++cout) { - auto out0 = *(reinterpret_cast(in_ptr + cout * sizeof(T))); + auto out0 = *(reinterpret_cast(in_ptr + cout * sizeof(T))); *(reinterpret_cast(out_ptr + cout * sizeof(T))) = out0; } } @@ -322,9 +322,13 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset const auto a1 = (yi_f - static_cast(yi)); const auto b1 = (1.f - a1); - const auto yi0 = utility::clamp(yi, 0, in_dim_h - 1); - const auto yi1 = utility::clamp(yi + 1, 0, in_dim_h - 1); + const int yi0 = utility::clamp(yi, 0, in_dim_h - 1); + const int yi1 = utility::clamp(yi + 1, 0, in_dim_h - 1); + const int yi0_offset = yi0 * in_stride_z; + const int yi1_offset = yi1 * in_stride_z; + + const int y_offset = yo * out_stride_z; for(int xo = xo_start; xo < xo_end; xo += xo_step) { // Floating-point coordinate @@ -340,49 +344,46 @@ void bilinear_neon_scale(const ITensor *src, ITensor *dst, const ITensor *offset const auto s10_s = static_cast(b * a1); const auto s11_s = static_cast(a * a1); - const auto xi0 = utility::clamp(xi, 0, in_dim_w - 1); - const auto xi1 = utility::clamp(xi + 1, 0, in_dim_w - 1); + const auto s00 = wrapper::vdup_n(s00_s, ExactTagType{}); + const auto s01 = wrapper::vdup_n(s01_s, ExactTagType{}); + const auto s10 = wrapper::vdup_n(s10_s, ExactTagType{}); + const auto s11 = wrapper::vdup_n(s11_s, ExactTagType{}); + + const int xi0 = utility::clamp(xi, 0, in_dim_w - 1); + const int xi1 = utility::clamp(xi + 1, 0, in_dim_w - 1); + + const int xi0_offset = xi0 * in_stride_y; + const int xi1_offset = xi1 * in_stride_y; + + const int offset = xo * out_stride_y + y_offset; int cout = 0; for(; cout <= (out_dim_ch - step_cout); cout += step_cout) { - auto in00 = wrapper::vdup_n(static_cast(0), ExactTagType{}); - auto in01 = wrapper::vdup_n(static_cast(0), ExactTagType{}); - auto in10 = wrapper::vdup_n(static_cast(0), ExactTagType{}); - auto in11 = wrapper::vdup_n(static_cast(0), ExactTagType{}); - in00 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + (xi0) * in_stride_y + (yi0) * in_stride_z)); - in01 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + (xi1) * in_stride_y + (yi0) * in_stride_z)); - in10 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + (xi0) * in_stride_y + (yi1) * in_stride_z)); - in11 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + (xi1) * in_stride_y + (yi1) * in_stride_z)); - - const auto s00 = wrapper::vdup_n(s00_s, ExactTagType{}); - const auto s01 = wrapper::vdup_n(s01_s, ExactTagType{}); - const auto s10 = wrapper::vdup_n(s10_s, ExactTagType{}); - const auto s11 = wrapper::vdup_n(s11_s, ExactTagType{}); - auto out0 = wrapper::vdup_n(static_cast(0), ExactTagType{}); - out0 = wrapper::vmla(out0, in00, s00); - out0 = wrapper::vmla(out0, in01, s01); - out0 = wrapper::vmla(out0, in10, s10); - out0 = wrapper::vmla(out0, in11, s11); - wrapper::vstore(reinterpret_cast(out_ptr + cout * sizeof(T) + xo * out_stride_y + yo * out_stride_z), out0); + auto in00 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); + auto in01 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); + auto in10 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); + auto in11 = wrapper::vloadq(reinterpret_cast(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); + + auto out0 = wrapper::vmul(in00, s00); + out0 = wrapper::vmla(out0, in01, s01); + out0 = wrapper::vmla(out0, in10, s10); + out0 = wrapper::vmla(out0, in11, s11); + wrapper::vstore(reinterpret_cast(out_ptr + offset + cout * sizeof(T)), out0); } for(; cout < out_dim_ch; ++cout) { - auto in00 = static_cast(0); - auto in01 = static_cast(0); - auto in10 = static_cast(0); - auto in11 = static_cast(0); - in00 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + (xi0) * in_stride_y + (yi0) * in_stride_z)); - in01 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + (xi1) * in_stride_y + (yi0) * in_stride_z)); - in10 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + (xi0) * in_stride_y + (yi1) * in_stride_z)); - in11 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + (xi1) * in_stride_y + (yi1) * in_stride_z)); - auto out0 = static_cast(0); - out0 += in00 * s00_s; + T in00 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + xi0_offset + yi0_offset)); + T in01 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + xi1_offset + yi0_offset)); + T in10 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + xi0_offset + yi1_offset)); + T in11 = *(reinterpret_cast(in_ptr + cout * sizeof(T) + xi1_offset + yi1_offset)); + + T out0 = in00 * s00_s; out0 += in01 * s01_s; out0 += in10 * s10_s; out0 += in11 * s11_s; - *(reinterpret_cast(out_ptr + cout * sizeof(T) + xo * out_stride_y + yo * out_stride_z)) = out0; + *(reinterpret_cast(out_ptr + offset + cout * sizeof(T))) = out0; } } } diff --git a/src/cpu/operators/CpuScale.cpp b/src/cpu/operators/CpuScale.cpp index 27da238c16..fdb52e5ede 100644 --- a/src/cpu/operators/CpuScale.cpp +++ b/src/cpu/operators/CpuScale.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 Arm Limited. + * Copyright (c) 2021-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,9 +23,6 @@ */ #include "src/cpu/operators/CpuScale.h" -#include "arm_compute/core/Helpers.h" -#include "arm_compute/core/TensorInfo.h" -#include "arm_compute/core/Validate.h" #include "arm_compute/runtime/NEON/NEScheduler.h" #include "src/common/utils/Log.h" #include "src/core/utils/ScaleUtils.h" @@ -218,26 +215,38 @@ void CpuScale::prepare(ITensorPack &tensors) _scale_info.interpolation_policy; const SamplingPolicy sampling_policy = _scale_info.sampling_policy; - switch(policy_to_use) + bool precompute_indices_weights = arm_compute::scale_utils::is_precomputation_required(_data_layout, src->info()->data_type(), policy_to_use); + + if(precompute_indices_weights == true) { - case InterpolationPolicy::NEAREST_NEIGHBOR: - { - // Pre-compute offsets for nearest interpolation - precompute_dx_dy_offsets(nullptr, nullptr, offsets, wr, hr, sampling_policy, is_align_corners_used); - break; - } - case InterpolationPolicy::BILINEAR: + switch(policy_to_use) { - // Pre-compute dx, dy and offsets for bilinear interpolation - precompute_dx_dy_offsets(dx, dy, offsets, wr, hr, sampling_policy, is_align_corners_used); - break; + case InterpolationPolicy::NEAREST_NEIGHBOR: + { + // Pre-compute offsets for nearest interpolation + precompute_dx_dy_offsets(nullptr, nullptr, offsets, wr, hr, sampling_policy, is_align_corners_used); + break; + } + case InterpolationPolicy::BILINEAR: + { + // Pre-compute dx, dy and offsets for bilinear interpolation + precompute_dx_dy_offsets(dx, dy, offsets, wr, hr, sampling_policy, is_align_corners_used); + break; + } + case InterpolationPolicy::AREA: + { + break; + } + default: + ARM_COMPUTE_ERROR("Unsupported interpolation mode"); } - case InterpolationPolicy::AREA: + } + else + { + if(policy_to_use != InterpolationPolicy::NEAREST_NEIGHBOR && policy_to_use != InterpolationPolicy::BILINEAR && policy_to_use != InterpolationPolicy::AREA) { - break; - } - default: ARM_COMPUTE_ERROR("Unsupported interpolation mode"); + } } } } diff --git a/src/runtime/NEON/functions/NEScale.cpp b/src/runtime/NEON/functions/NEScale.cpp index 9f48e78a5a..74ab860d91 100644 --- a/src/runtime/NEON/functions/NEScale.cpp +++ b/src/runtime/NEON/functions/NEScale.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021 Arm Limited. + * Copyright (c) 2016-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -75,34 +75,46 @@ void NEScale::configure(ITensor *input, ITensor *output, const ScaleKernelInfo & TensorShape shape(output->info()->dimension(idx_width)); shape.set(1, output->info()->dimension(idx_height), false); - const TensorInfo tensor_info_dxdy(shape, Format::F32); - const TensorInfo tensor_info_offsets(shape, Format::S32); + bool precompute_indices_weights = arm_compute::scale_utils::is_precomputation_required(data_layout, input->info()->data_type(), policy_to_use); - _impl->dx.allocator()->init(tensor_info_dxdy); - _impl->dy.allocator()->init(tensor_info_dxdy); - _impl->offsets.allocator()->init(tensor_info_offsets); - switch(policy_to_use) + if(precompute_indices_weights == true) { - case InterpolationPolicy::NEAREST_NEIGHBOR: - { - // Allocate once the configure methods have been called - _impl->offsets.allocator()->allocate(); - break; - } - case InterpolationPolicy::BILINEAR: + const TensorInfo tensor_info_dxdy(shape, Format::F32); + const TensorInfo tensor_info_offsets(shape, Format::S32); + + _impl->dx.allocator()->init(tensor_info_dxdy); + _impl->dy.allocator()->init(tensor_info_dxdy); + _impl->offsets.allocator()->init(tensor_info_offsets); + switch(policy_to_use) { - // Allocate once the configure methods have been called - _impl->dx.allocator()->allocate(); - _impl->dy.allocator()->allocate(); - _impl->offsets.allocator()->allocate(); - break; + case InterpolationPolicy::NEAREST_NEIGHBOR: + { + // Allocate once the configure methods have been called + _impl->offsets.allocator()->allocate(); + break; + } + case InterpolationPolicy::BILINEAR: + { + // Allocate once the configure methods have been called + _impl->dx.allocator()->allocate(); + _impl->dy.allocator()->allocate(); + _impl->offsets.allocator()->allocate(); + break; + } + case InterpolationPolicy::AREA: + { + break; + } + default: + ARM_COMPUTE_ERROR("Unsupported interpolation mode"); } - case InterpolationPolicy::AREA: + } + else + { + if(policy_to_use != InterpolationPolicy::NEAREST_NEIGHBOR && policy_to_use != InterpolationPolicy::BILINEAR && policy_to_use != InterpolationPolicy::AREA) { - break; - } - default: ARM_COMPUTE_ERROR("Unsupported interpolation mode"); + } } } -- cgit v1.2.1