From 95d477e0662c172c264cf495d196dfce6a791248 Mon Sep 17 00:00:00 2001 From: Viet-Hoa Do Date: Tue, 10 Oct 2023 15:10:37 +0100 Subject: Remove padding from CL comparison operator * Add support for processing left-over vector to comparison kernel. * Combine native and quantized versions of CL comparison. Resolves: COMPMID-6424 Signed-off-by: Viet-Hoa Do Change-Id: I31d43bdf0eab999cee6fa8144b5d8e921a1093e8 Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/10467 Benchmark: Arm Jenkins Tested-by: Arm Jenkins Reviewed-by: Jakub Sujak Comments-Addressed: Arm Jenkins --- src/core/CL/cl_kernels/common/comparisons.cl | 117 +++++++++++---------------- src/core/CL/kernels/CLComparisonKernel.cpp | 49 +++++------ src/core/CL/kernels/CLComparisonKernel.h | 11 ++- 3 files changed, 68 insertions(+), 109 deletions(-) (limited to 'src/core') diff --git a/src/core/CL/cl_kernels/common/comparisons.cl b/src/core/CL/cl_kernels/common/comparisons.cl index f05cb87835..00bb491f85 100644 --- a/src/core/CL/cl_kernels/common/comparisons.cl +++ b/src/core/CL/cl_kernels/common/comparisons.cl @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2021 Arm Limited. + * Copyright (c) 2018-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -30,11 +30,13 @@ #define LESS(x, y) ((x) < (y)) #define LESSEQUAL(x, y) ((x) <= (y)) -#define DEFINE_KERNEL_STR(name) compare_##name -#define DEFINE_KERNEL(name) DEFINE_KERNEL_STR(name) +#ifdef IS_QUANTIZED +# define DEFINE_KERNEL_STR(name) compare_##name##_quantized +#else // IS_QUANTIZED +# define DEFINE_KERNEL_STR(name) compare_##name +#endif // IS_QUANTIZED -#define DEFINE_KERNEL_QUANTIZED_STR(name) compare_##name##_quantized -#define DEFINE_KERNEL_QUANTIZED(name) DEFINE_KERNEL_QUANTIZED_STR(name) +#define DEFINE_KERNEL(name) DEFINE_KERNEL_STR(name) #if defined(DATA_TYPE) && defined(VEC_SIZE) && defined(OP) && defined(OP_NAME) /** This function compares two tensors. @@ -73,78 +75,49 @@ __kernel void DEFINE_KERNEL(OP_NAME)( TENSOR3D_DECLARATION(in2), TENSOR3D_DECLARATION(out)) { - // Get pixels pointer - Tensor3D in1 = CONVERT_TO_TENSOR3D_STRUCT(in1); - Tensor3D in2 = CONVERT_TO_TENSOR3D_STRUCT(in2); - Tensor3D out = CONVERT_TO_TENSOR3D_STRUCT(out); + int dst_x = max((int)get_global_id(0) * VEC_SIZE - (VEC_SIZE - VEC_SIZE_LEFTOVER) % VEC_SIZE, 0); - // Load values - VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE) - in_a = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)in1.ptr); - VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE) - in_b = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)in2.ptr); +#if VEC_SIZE_IN1 == 1 + int in1_x = 0; +#else // VEC_SIZE_IN1 == 1 + int in1_x = dst_x; +#endif // VEC_SIZE_IN1 == 1 - // Calculate and store result - VSTORE(VEC_SIZE) - (CONVERT(OP(in_a, in_b), VEC_DATA_TYPE(uchar, VEC_SIZE)), 0, (__global uchar *)out.ptr); -} -#endif /* defined(DATA_TYPE) && defined(VEC_SIZE) && defined(OP) && defined(OP_NAME) */ +#if VEC_SIZE_IN2 == 1 + int in2_x = 0; +#else // VEC_SIZE_IN2 == 1 + int in2_x = dst_x; +#endif // VEC_SIZE_IN2 == 1 -#if defined(OFFSET_IN1) && defined(OFFSET_IN2) && defined(SCALE_IN1) && defined(SCALE_IN2) -/** This function compares two quantized tensors. - * - * @note The inputs' data type need to be passed at compile time using -DDATA_TYPE: e.g. -DDATA_TYPE=uchar - * @note The quantization offset of the first operand must be passed at compile time using -DOFFSET_IN1, i.e. -DOFFSET_IN1=10 - * @note The quantization offset of the second operand must be passed at compile time using -DOFFSET_IN2, i.e. -DOFFSET_IN2=10 - * @note The quantization scale of the first operand must be passed at compile time using -DSCALE_IN1, i.e. -DSCALE_IN1=10 - * @note The quantization scale of the second operand must be passed at compile time using -DSCALE_IN2, i.e. -DSCALE_IN2=10 - * - * @param[in] in1_ptr Pointer to the source tensor. Supported data types: All quantized data types. - * @param[in] in1_stride_x Stride of the source tensor in X dimension (in bytes) - * @param[in] in1_step_x in1_stride_x * number of elements along X processed per workitem(in bytes) - * @param[in] in1_stride_y Stride of the source tensor in Y dimension (in bytes) - * @param[in] in1_step_y in1_stride_y * number of elements along Y processed per workitem(in bytes) - * @param[in] in1_stride_z Stride of the source tensor in Z dimension (in bytes) - * @param[in] in1_step_z in1_stride_z * number of elements along Z processed per workitem(in bytes) - * @param[in] in1_offset_first_element_in_bytes The offset of the first element in the source tensor - * @param[in] in2_ptr Pointer to the source tensor. Supported data types: same as @p in1_ptr - * @param[in] in2_stride_x Stride of the source tensor in X dimension (in bytes) - * @param[in] in2_step_x in2_stride_x * number of elements along X processed per workitem(in bytes) - * @param[in] in2_stride_y Stride of the source tensor in Y dimension (in bytes) - * @param[in] in2_step_y in2_stride_y * number of elements along Y processed per workitem(in bytes) - * @param[in] in2_stride_z Stride of the source tensor in Z dimension (in bytes) - * @param[in] in2_step_z in2_stride_z * number of elements along Z processed per workitem(in bytes) - * @param[in] in2_offset_first_element_in_bytes The offset of the first element in the source tensor - * @param[out] out_ptr Pointer to the destination tensor. Supported data types: U8 - * @param[in] out_stride_x Stride of the destination tensor in X dimension (in bytes) - * @param[in] out_step_x out_stride_x * number of elements along X processed per workitem(in bytes) - * @param[in] out_stride_y Stride of the destination tensor in Y dimension (in bytes) - * @param[in] out_step_y out_stride_y * number of elements along Y processed per workitem(in bytes) - * @param[in] out_stride_z Stride of the source tensor in Z dimension (in bytes) - * @param[in] out_step_z out_stride_z * number of elements along Z processed per workitem(in bytes) - * @param[in] out_offset_first_element_in_bytes The offset of the first element in the destination tensor - */ -__kernel void DEFINE_KERNEL_QUANTIZED(OP_NAME)( - TENSOR3D_DECLARATION(in1), - TENSOR3D_DECLARATION(in2), - TENSOR3D_DECLARATION(out)) -{ - // Get pixels pointer - Tensor3D in1 = CONVERT_TO_TENSOR3D_STRUCT(in1); - Tensor3D in2 = CONVERT_TO_TENSOR3D_STRUCT(in2); - Tensor3D out = CONVERT_TO_TENSOR3D_STRUCT(out); + int y = get_global_id(1); + int z = get_global_id(2); - int16 in_a = CONVERT(vload16(0, (__global DATA_TYPE *)in1.ptr), int16); - int16 in_b = CONVERT(vload16(0, (__global DATA_TYPE *)in2.ptr), int16); + in1_ptr += in1_offset_first_element_in_bytes + z * in1_stride_z + y * in1_stride_y + in1_x * sizeof(DATA_TYPE); + in2_ptr += in2_offset_first_element_in_bytes + z * in2_stride_z + y * in2_stride_y + in2_x * sizeof(DATA_TYPE); + out_ptr += out_offset_first_element_in_bytes + z * out_stride_z + y * out_stride_y + dst_x * sizeof(uchar); - in_a = in_a - (int16)((int)OFFSET_IN1); - in_b = in_b - (int16)((int)OFFSET_IN2); + // Load values + VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE) in_a = (VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE))VLOAD(VEC_SIZE_IN1)(0, (__global DATA_TYPE *)in1_ptr); + VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE) in_b = (VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE))VLOAD(VEC_SIZE_IN2)(0, (__global DATA_TYPE *)in2_ptr); + + // Calculate and store result +#ifdef IS_QUANTIZED + VEC_DATA_TYPE(int, VEC_SIZE) in_a_i32 = CONVERT(in_a, VEC_DATA_TYPE(int, VEC_SIZE)); + VEC_DATA_TYPE(int, VEC_SIZE) in_b_i32 = CONVERT(in_b, VEC_DATA_TYPE(int, VEC_SIZE)); - const float16 in1f32 = convert_float16(in_a) * (float16)((float)SCALE_IN1); - const float16 in2f32 = convert_float16(in_b) * (float16)((float)SCALE_IN2); - const int16 res = OP(in1f32, in2f32); + VEC_DATA_TYPE(float, VEC_SIZE) in_a_fp = CONVERT(in_a_i32 - OFFSET_IN1, VEC_DATA_TYPE(float, VEC_SIZE)) * SCALE_IN1; + VEC_DATA_TYPE(float, VEC_SIZE) in_b_fp = CONVERT(in_b_i32 - OFFSET_IN2, VEC_DATA_TYPE(float, VEC_SIZE)) * SCALE_IN2; +#else // IS_QUANTIZED + VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE) in_a_fp = in_a; + VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE) in_b_fp = in_b; +#endif // IS_QUANTIZED - // Store result - vstore16(convert_uchar16(res), 0, (__global uchar *)out.ptr); +#if VEC_SIZE == 1 + uchar res0 = (uchar)select(0, 255, OP(in_a_fp, in_b_fp)); +#else // VEC_SIZE == 1 + VEC_DATA_TYPE(uchar, VEC_SIZE) res0 = CONVERT(OP(in_a_fp, in_b_fp), VEC_DATA_TYPE(uchar, VEC_SIZE)); +#endif // VEC_SIZE == 1 + + STORE_VECTOR_SELECT(res, uchar, out_ptr, VEC_SIZE, VEC_SIZE_LEFTOVER, VEC_SIZE_LEFTOVER != 0 && get_global_id(0) == 0) } -#endif /* defined(OFFSET_IN1) && defined(OFFSET_IN2) && defined(SCALE_IN1) && defined(SCALE_IN2) */ \ No newline at end of file +#endif /* defined(DATA_TYPE) && defined(VEC_SIZE) && defined(OP) && defined(OP_NAME) */ diff --git a/src/core/CL/kernels/CLComparisonKernel.cpp b/src/core/CL/kernels/CLComparisonKernel.cpp index f27270733e..a0f9aca54a 100644 --- a/src/core/CL/kernels/CLComparisonKernel.cpp +++ b/src/core/CL/kernels/CLComparisonKernel.cpp @@ -25,6 +25,7 @@ #include "arm_compute/core/CL/CLHelpers.h" #include "arm_compute/core/CL/ICLTensor.h" +#include "arm_compute/core/utils/helpers/AdjustVecSize.h" #include "arm_compute/core/utils/StringUtils.h" #include "src/core/CL/CLValidate.h" @@ -45,11 +46,6 @@ const std::map supported_comparison_ops = { {ComparisonOperation::Less, "LESS"}, {ComparisonOperation::LessEqual, "LESSEQUAL"}, }; -int calculate_num_elems_processed_per_iteration(const ITensorInfo &input) -{ - return 16 / input.element_size(); -} - Status validate_arguments(const ITensorInfo &input1, const ITensorInfo &input2, const ITensorInfo &output, @@ -77,26 +73,15 @@ Status validate_arguments(const ITensorInfo &input1, std::pair validate_and_configure_window(ITensorInfo &input1, ITensorInfo &input2, ITensorInfo &output) { const TensorShape &out_shape = TensorShape::broadcast_shape(input1.tensor_shape(), input2.tensor_shape()); - const unsigned int num_elems_processed_per_iteration = calculate_num_elems_processed_per_iteration(input1); + const unsigned int num_elems_processed_per_iteration = + adjust_vec_size(16 / input1.element_size(), output.dimension(0)); // Auto initialize output if not initialized auto_init_if_empty(output, out_shape, 1, DataType::U8, QuantizationInfo()); - Window win = calculate_max_window(out_shape, Steps(num_elems_processed_per_iteration)); - Window win_input1 = win.broadcast_if_dimension_le_one(input1); - Window win_input2 = win.broadcast_if_dimension_le_one(input2); - - AccessWindowHorizontal input1_access(&input1, 0, num_elems_processed_per_iteration); - AccessWindowHorizontal input2_access(&input2, 0, num_elems_processed_per_iteration); - AccessWindowHorizontal output_access(&output, 0, num_elems_processed_per_iteration); - - bool window_changed = update_window_and_padding(win_input1, input1_access) || - update_window_and_padding(win_input2, input2_access) || - update_window_and_padding(win, output_access); + Window win = calculate_max_window(out_shape, Steps(num_elems_processed_per_iteration)); - Status err = - (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{}; - return std::make_pair(err, win); + return std::make_pair(Status{}, win); } } // namespace @@ -133,11 +118,21 @@ void CLComparisonKernel::configure(const CLCompileContext &compile_context, const std::string &operation_name = supported_comparison_ops.at(operation); std::string kernel_name = "compare_" + lower_string(operation_name); + const unsigned int num_elems_processed_per_iteration = + adjust_vec_size(16 / input1->info()->element_size(), output->info()->dimension(0)); + // Set kernel build options std::set build_opts; build_opts.emplace("-DDATA_TYPE=" + get_cl_type_from_data_type(input1->info()->data_type())); - build_opts.emplace("-DVEC_SIZE=" + - support::cpp11::to_string(calculate_num_elems_processed_per_iteration(*input1->info()))); + build_opts.emplace("-DVEC_SIZE=" + support::cpp11::to_string(num_elems_processed_per_iteration)); + build_opts.emplace("-DVEC_SIZE_LEFTOVER=" + + support::cpp11::to_string(output->info()->dimension(0) % num_elems_processed_per_iteration)); + build_opts.emplace( + "-DVEC_SIZE_IN1=" + // + support::cpp11::to_string(input1->info()->dimension(0) == 1 ? 1 : num_elems_processed_per_iteration)); + build_opts.emplace( + "-DVEC_SIZE_IN2=" + // + support::cpp11::to_string(input2->info()->dimension(0) == 1 ? 1 : num_elems_processed_per_iteration)); build_opts.emplace("-DOP=" + operation_name); build_opts.emplace("-DOP_NAME=" + lower_string(operation_name)); if (is_data_type_quantized(input1->info()->data_type())) @@ -145,6 +140,7 @@ void CLComparisonKernel::configure(const CLCompileContext &compile_context, const UniformQuantizationInfo iq1_info = input1->info()->quantization_info().uniform(); const UniformQuantizationInfo iq2_info = input2->info()->quantization_info().uniform(); + build_opts.emplace("-DIS_QUANTIZED"); build_opts.emplace("-DOFFSET_IN1=" + support::cpp11::to_string(iq1_info.offset)); build_opts.emplace("-DOFFSET_IN2=" + support::cpp11::to_string(iq2_info.offset)); build_opts.emplace("-DSCALE_IN1=" + float_to_string_with_full_precision(iq1_info.scale)); @@ -228,13 +224,4 @@ void CLComparisonKernel::run(const Window &window, cl::CommandQueue &queue) } while (collapsed.slide_window_slice_3D(slice)); } -BorderSize CLComparisonKernel::border_size() const -{ - const int num_elems_processed_per_iteration = calculate_num_elems_processed_per_iteration(*_input1->info()); - - const unsigned int replicateSize = - _output->info()->dimension(0) - std::min(_input1->info()->dimension(0), _input2->info()->dimension(0)); - const unsigned int border = std::min(num_elems_processed_per_iteration - 1U, replicateSize); - return BorderSize{0, border, 0, 0}; -} } // namespace arm_compute diff --git a/src/core/CL/kernels/CLComparisonKernel.h b/src/core/CL/kernels/CLComparisonKernel.h index 174a6c9bf9..2fb4ba06b6 100644 --- a/src/core/CL/kernels/CLComparisonKernel.h +++ b/src/core/CL/kernels/CLComparisonKernel.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2020 Arm Limited. + * Copyright (c) 2018-2020, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -21,8 +21,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#ifndef ARM_COMPUTE_CLCOMPARISONKERNEL_H -#define ARM_COMPUTE_CLCOMPARISONKERNEL_H +#ifndef ACL_SRC_CORE_CL_KERNELS_CLCOMPARISONKERNEL_H +#define ACL_SRC_CORE_CL_KERNELS_CLCOMPARISONKERNEL_H #include "arm_compute/core/Types.h" @@ -85,8 +85,7 @@ public: ComparisonOperation operation); // Inherited methods overridden: - void run(const Window &window, cl::CommandQueue &queue) override; - BorderSize border_size() const override; + void run(const Window &window, cl::CommandQueue &queue) override; private: const ICLTensor *_input1; /**< Source tensor 1 */ @@ -94,4 +93,4 @@ private: ICLTensor *_output; /**< Destination tensor */ }; } // namespace arm_compute -#endif /* ARM_COMPUTE_CLCOMPARISONKERNEL_H */ +#endif // ACL_SRC_CORE_CL_KERNELS_CLCOMPARISONKERNEL_H -- cgit v1.2.1