From 37c989a58a04985dfdc21089c7dacc7e1925a4d0 Mon Sep 17 00:00:00 2001 From: Viet-Hoa Do Date: Fri, 24 Feb 2023 15:52:21 +0000 Subject: Add support for arbitrary parameters for CPU Gather * The shape of input and indices tensors, and the gather axis can be any number, as long as these are valid and the output tensor doesn't have more dimensions than the library supports. * Update the reference code to be more generic and straightforward. * Add necessary test cases. Signed-off-by: Viet-Hoa Do Resolves: COMPMID-5919 Change-Id: Ic7e2032777aa97ecc147f61d5388528697508ab1 Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/9199 Tested-by: Arm Jenkins Reviewed-by: Gunes Bayir Comments-Addressed: Arm Jenkins Benchmark: Arm Jenkins --- src/core/NEON/kernels/NEGatherKernel.cpp | 204 +++++++++++++------------------ src/core/NEON/kernels/NEGatherKernel.h | 28 +---- 2 files changed, 92 insertions(+), 140 deletions(-) (limited to 'src/core/NEON') diff --git a/src/core/NEON/kernels/NEGatherKernel.cpp b/src/core/NEON/kernels/NEGatherKernel.cpp index 085ab7cb18..d361eb93fd 100644 --- a/src/core/NEON/kernels/NEGatherKernel.cpp +++ b/src/core/NEON/kernels/NEGatherKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -30,7 +30,6 @@ #include "arm_compute/core/Validate.h" #include "arm_compute/core/Window.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" -#include "src/core/CPP/Validate.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" @@ -69,7 +68,7 @@ Status validate_arguments(const ITensorInfo *input, const ITensorInfo *indices, } ARM_COMPUTE_RETURN_ERROR_ON(0 > axis || axis >= static_cast(input->num_dimensions())); - ARM_COMPUTE_RETURN_ERROR_ON(axis != 1 && indices->num_dimensions() > 1); + ARM_COMPUTE_RETURN_ERROR_ON(input->num_dimensions() + indices->num_dimensions() - 1 > Coordinates::num_max_dimensions); ARM_COMPUTE_RETURN_ERROR_ON(input->data_type() == DataType::UNKNOWN); if(output->total_size() != 0) @@ -87,84 +86,55 @@ Status validate_arguments(const ITensorInfo *input, const ITensorInfo *indices, } // namespace NEGatherKernel::NEGatherKernel() - : _input{}, _indices{}, _axis{}, _output{}, _func{} + : _input{}, _indices{}, _axis{}, _output{}, _func{}, _src_it_strides{}, _idx_it_strides{} { } -template -inline void NEGatherKernel::gather_multiindices_1_axis(const Window &window, const ThreadInfo &info) -{ - ARM_COMPUTE_UNUSED(info); - ARM_COMPUTE_ERROR_ON(_indices->info()->num_dimensions() < 2 || _indices->info()->num_dimensions() > 3); - validate_indices(_indices); - Window win = window; - win.set(Window::DimX, Window::Dimension(0, 1, 1)); - execute_window_loop(win, [&](const Coordinates & id) - { - auto *dst_ptr = _output->ptr_to_element(id); - Coordinates index_offset; - for(uint32_t k = 0; k < _indices->info()->num_dimensions(); ++k) - { - index_offset.set(k, id[k + 1]); - } - const uint32_t row = *(reinterpret_cast(_indices->ptr_to_element(index_offset))); - Coordinates src_offset; - // Set up input coords to read the row specified by the current index - src_offset.set(0, 0); - src_offset.set(1, row); - for(uint32_t j = 2; j < _input->info()->num_dimensions(); ++j) - { - src_offset.set(j, id[1 + _indices->info()->num_dimensions() + (j - 2)]); - } - const auto in_ptr_row = _input->ptr_to_element(src_offset); - // Copy a row from input to output - memcpy(dst_ptr, in_ptr_row, _input->info()->tensor_shape()[0] * _input->info()->element_size()); - }); -} - -template -inline void NEGatherKernel::gather_0_axis(const Window &window, const ThreadInfo &info) +template +void NEGatherKernel::gather_common(const Window &window, const ThreadInfo &info) { ARM_COMPUTE_UNUSED(info); - // Validate that the indices are not negative - validate_indices(_indices); - - Iterator output_it(_output, window); - execute_window_loop(window, [&](const Coordinates & id) - { - Coordinates gather_id(id); + auto dst_win = window; - auto new_index = *(reinterpret_cast(_indices->ptr_to_element(Coordinates(id[0])))); - gather_id.set(0, new_index); + const auto src_info = _input->info(); + const auto idx_info = _indices->info(); + const auto dst_info = _output->info(); - std::copy_n(_input->ptr_to_element(gather_id), _output->info()->element_size(), output_it.ptr()); - }, - output_it); -} + const auto num_dims = dst_info->num_dimensions(); + const auto chunk_stride = src_info->strides_in_bytes()[_axis]; -template -void NEGatherKernel::gather_n_axis(const Window &window, const ThreadInfo &info) -{ - ARM_COMPUTE_UNUSED(info); + const auto window_start_x = window.x().start(); + const auto window_end_x = window.x().end(); + auto window_size_x = src_info->element_size(); - // Validate that the indices are not negative - validate_indices(_indices); + if(_axis != 0) + { + dst_win.set(0, Window::Dimension(window_start_x, window_start_x + 1, 1)); + window_size_x *= window_end_x - window_start_x; + } - Window output_window{ window }; - output_window.set(Window::DimX, Window::Dimension(0, 1, 1)); + // Compute source and index tensors window based on the output window. + auto src_win = dst_win; + Window idx_win; - Iterator output_it(_output, output_window); - execute_window_loop(output_window, [&](const Coordinates & id) + for (size_t i = 0; i < idx_info->num_dimensions(); ++i) { - Coordinates gather_id(id); + src_win.set(_axis + i, Window::Dimension(0, 1, 1)); + idx_win.set(_axis + i, window[_axis + i]); + } - auto new_index = *(reinterpret_cast(_indices->ptr_to_element(Coordinates(id[_axis])))); - gather_id.set(_axis, new_index); + // Use the custom strides to access all three tensors using the same loop. + Iterator src_it(num_dims, _src_it_strides, _input->buffer(), src_info->offset_first_element_in_bytes(), src_win); + Iterator idx_it(num_dims, _idx_it_strides, _indices->buffer(), idx_info->offset_first_element_in_bytes(), idx_win); + Iterator dst_it(num_dims, dst_info->strides_in_bytes(), _output->buffer(), dst_info->offset_first_element_in_bytes(), dst_win); - std::copy_n(_input->ptr_to_element(gather_id), _input->info()->dimension(0) * _output->info()->element_size(), output_it.ptr()); - }, - output_it); + execute_window_loop(dst_win, [&](const Coordinates &) { + const auto idx = *reinterpret_cast(idx_it.ptr()); + const auto src_ptr = src_it.ptr() + idx * chunk_stride; + + std::copy_n(src_ptr, window_size_x, dst_it.ptr()); + }, src_it, idx_it, dst_it); } void NEGatherKernel::configure(const ITensor *input, const ITensor *indices, ITensor *output, int axis) @@ -183,60 +153,17 @@ void NEGatherKernel::configure(const ITensor *input, const ITensor *indices, ITe } ARM_COMPUTE_ERROR_ON(0 > _axis || _axis >= static_cast(input->info()->num_dimensions())); - if(indices->info()->num_dimensions() == 1u) + switch(_indices->info()->data_type()) { - if(_axis == 0) - { - switch(_indices->info()->data_type()) - { - case DataType::U32: - _func = &NEGatherKernel::gather_0_axis; - break; - case DataType::S32: - _func = &NEGatherKernel::gather_0_axis; - break; - default: - ARM_COMPUTE_ERROR("Not supported"); - break; - } - } - else - { - switch(_indices->info()->data_type()) - { - case DataType::U32: - _func = &NEGatherKernel::gather_n_axis; - break; - case DataType::S32: - _func = &NEGatherKernel::gather_n_axis; - break; - default: - ARM_COMPUTE_ERROR("Not supported"); - break; - } - } - } - else - { - if(_axis == 1) - { - switch(_indices->info()->data_type()) - { - case DataType::U32: - _func = &NEGatherKernel::gather_multiindices_1_axis; - break; - case DataType::S32: - _func = &NEGatherKernel::gather_multiindices_1_axis; - break; - default: - ARM_COMPUTE_ERROR("Not supported"); - break; - } - } - else - { + case DataType::U32: + _func = &NEGatherKernel::gather_common; + break; + case DataType::S32: + _func = &NEGatherKernel::gather_common; + break; + default: ARM_COMPUTE_ERROR("Not supported"); - } + break; } // Output auto initialization if not yet initialized @@ -247,6 +174,32 @@ void NEGatherKernel::configure(const ITensor *input, const ITensor *indices, ITe Window win = calculate_max_window(*output->info(), Steps()); INEKernel::configure(win); + + // Create input and indices strides that have the same number of dimensions as the output tensor. + // These will be used to iterate lock-step through all tensors (input, indices and output). + size_t dim_no = 0; + + const auto input_info = input->info(); + const auto &input_strides = input_info->strides_in_bytes(); + + const auto indices_info = indices->info(); + const auto &indices_strides = indices_info->strides_in_bytes(); + const auto indices_num_dims = indices_info->num_dimensions(); + + for(; dim_no < static_cast(_axis); ++dim_no) + { + _src_it_strides[dim_no] = input_strides[dim_no]; + } + + for(; dim_no < static_cast(_axis) + indices_num_dims; ++dim_no) + { + _idx_it_strides[dim_no] = indices_strides[dim_no - _axis]; + } + + for(; dim_no < Coordinates::num_max_dimensions; ++dim_no) + { + _src_it_strides[dim_no] = input_strides[dim_no - indices_num_dims + 1]; + } } Status NEGatherKernel::validate(const ITensorInfo *input, const ITensorInfo *indices, const ITensorInfo *output, int axis) @@ -261,6 +214,21 @@ void NEGatherKernel::run(const Window &window, const ThreadInfo &info) ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); ARM_COMPUTE_ERROR_ON(_func == nullptr); + switch(_indices->info()->data_type()) + { + case DataType::U32: + validate_indices(_indices); + break; + + case DataType::S32: + validate_indices(_indices); + break; + + default: + ARM_COMPUTE_ERROR("Not supported"); + break; + } + (this->*_func)(window, info); } diff --git a/src/core/NEON/kernels/NEGatherKernel.h b/src/core/NEON/kernels/NEGatherKernel.h index 3dc0cad7be..ce69daeda7 100644 --- a/src/core/NEON/kernels/NEGatherKernel.h +++ b/src/core/NEON/kernels/NEGatherKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2022 Arm Limited. + * Copyright (c) 2019-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -81,27 +81,8 @@ public: void run(const Window &window, const ThreadInfo &info) override; private: - /** Implementation of the gather operation for 0 axis. - * - * For gather on the 0 axis an element by element copy is performed. - * - * @param[in] window Region on which to run the kernel. (Must be a region of the window returned by window()) - * @param[in] info Info about running thread and CPU. - */ - template - void gather_0_axis(const Window &window, const ThreadInfo &info); - - template - void gather_multiindices_1_axis(const Window &window, const ThreadInfo &info); - /** Implementation of the gather operation. - * - * For 1<=axis a row-wise copy is taking place. - * - * @param[in] window Region on which to run the kernel. (Must be a region of the window returned by window()) - * @param[in] info Info about running thread and CPU. - */ - template - void gather_n_axis(const Window &window, const ThreadInfo &info); + template + void gather_common(const Window &window, const ThreadInfo &info); using kernel_ptr = void (NEGatherKernel::*)(const Window &window, const ThreadInfo &info); @@ -110,6 +91,9 @@ private: int _axis; ITensor *_output; kernel_ptr _func; + + Strides _src_it_strides; + Strides _idx_it_strides; }; } // namespace arm_compute #endif /* ARM_COMPUTE_NEGATHERKERNEL_H */ -- cgit v1.2.1