From ada3200f5cec0b6a37f898d5d6f8e69395d7bcb1 Mon Sep 17 00:00:00 2001 From: Gunes Bayir Date: Wed, 24 Apr 2024 10:27:13 +0100 Subject: Add update/index/output (m+1)/2d/(m+n) support for CLScatter Resolves: COMPMID-6894, COMPMID-6896 Change-Id: I9d29fd3701a7e0f28d83f81a6c42a7234c2587c3 Signed-off-by: Gunes Bayir Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/11477 Tested-by: Arm Jenkins Comments-Addressed: Arm Jenkins Reviewed-by: Ramy Elgammal Dynamic-Fusion: Ramy Elgammal Benchmark: Arm Jenkins --- src/core/CL/cl_kernels/common/scatter.cl | 147 ++++++++++++++++++++----------- 1 file changed, 98 insertions(+), 49 deletions(-) (limited to 'src/core') diff --git a/src/core/CL/cl_kernels/common/scatter.cl b/src/core/CL/cl_kernels/common/scatter.cl index 73b714e042..ac9f828df2 100644 --- a/src/core/CL/cl_kernels/common/scatter.cl +++ b/src/core/CL/cl_kernels/common/scatter.cl @@ -22,8 +22,7 @@ * SOFTWARE. */ #include "helpers.h" - -#if defined(INDICES_SHAPE_Y) && defined(DATA_TYPE) && defined(OUT_SHAPE_X) && defined(SCATTER_FUNCTION) +#include "tile_helpers.h" // The below defines the various reduce operations for our purposes. // Where a corresponds to the existing value, and b the new value. @@ -33,64 +32,114 @@ #define MIN_OP(a, b) fmin(a, b) #define UPDATE_OP(a, b) (b) -/** Performs the ScatterND operation - * @note Datatype should be given as a preprocessor argument using -DDATA_TYPE=type. e.g. -DDATA_TYPE=short - * @note the size of the dst tensor in the "x" dimension should be passed using -DOUT_SHAPE_X at compile time. - * @note the number of values in the indices tensor in the y-dim should be passed with -DINDICES_SHAPE_Y at compile time. - * @note Negative indices are treated as out of bounds. +#ifdef SCATTER_MP1D_2D_MPND + +/** This kernel performs scatter operation + * + * @note Datatype should be given as a compile-time argument using -DDATA_TYPE=type. e.g. -DDATA_TYPE=short + * @note Number of indices should be given as a compile-time argument using -DNUM_INDICES, e.g. -DNUM_INDICES=3 + * @note Index length should be given as a compile-time argument using -DINDEX_LENGTH, e.g. -DINDEX_LENGTH=2 + * @note Outermost output shapes should be given as a compile-time argument using -DOUT_SHAPE_N_MINUS_X, where + * X must be 1,2,3,4,5, e.g. -DOUT_SHAPE_N_MINUS_1=3, ... + * @note Number of elements to copy in a row should be given as a compile-time argument using -DN0, e.g. -DN0=4 + * @note Number of partial elements at the edge to copy in a row should be given as a compile-time argument using + * -DPARTIAL_N0, e.g. -DPARTIAL_N0=2 + * @note Scatter function should be given as a compile-time argument using -DSCATTER_FUNCTION, e.g. -DSCATTER_FUNCTION=ADD + * @note If the kernel should skip reading the output tensor, -DSKIP_OUTPUT_READ option should be provided. + * @note Kernel name in uppercase letters should be provided as a compile-time argument, e.g. -DSCATTER_MP1D_2D_MPND * - * @param[in] updates_ptr Pointer to the source tensor. Supported data types: All - * @param[in] updates_stride_x Stride of the source tensor in X dimension (in bytes) - * @param[in] updates_step_x updates_stride_x * number of elements along X processed per work item (in bytes) - * @param[in] updates_stride_y Stride of the source tensor in Y dimension (in bytes) - * @param[in] updates_step_y updates_stride_y * number of elements along Y processed per work item (in bytes) - * @param[in] updates_stride_z Stride of the source tensor in Y dimension (in bytes) - * @param[in] updates_step_z updates_stride_z * number of elements along Z processed per work item (in bytes) - * @param[in] updates_stride_w Stride of the source tensor in Z dimension (in bytes) - * @param[in] updates_step_w updates_stride_w * number of elements along W processed per work item (in bytes) - * @param[in] updates_offset_first_element_in_bytes Offset of the first element in the source tensor - * @param[in] indices_ptr Pointer to the indices vector. Supported data types: S32. - * @param[in] indices_stride_x Stride of the indices vector in X dimension (in bytes) - * @param[in] indices_step_x updates_stride_x * number of elements along X processed per work item (in bytes) - * @param[in] indices_offset_first_element_in_bytes Offset of the first element in the indices vector - * @param[out] output_ptr Pointer to the destination tensor. Supported data types: same as @p updates_ptr + * @param[in] updates_ptr Pointer to the updates tensor. Data Types: F32 + * @param[in] updates_stride_x Stride of the updates tensor in X dimension (in bytes) + * @param[in] updates_step_x updates_stride_x * number of elements along X processed per workitem(in bytes) + * @param[in] updates_stride_y Stride of the updates tensor in Y dimension (in bytes) + * @param[in] updates_step_y updates_stride_y * number of elements along Y processed per workitem(in bytes) + * @param[in] updates_offset_first_element_in_bytes The offset of the first element in the updates tensor + * @param[in] indices_ptr Pointer to the indices tensor. Data Types: S32 + * @param[in] indices_stride_x Stride of the indices tensor in X dimension (in bytes) + * @param[in] indices_step_x indices_stride_x * number of elements along X processed per workitem(in bytes) + * @param[in] indices_stride_y Stride of the indices tensor in Y dimension (in bytes) + * @param[in] indices_step_y indices_stride_y * number of elements along Y processed per workitem(in bytes) + * @param[in] indices_offset_first_element_in_bytes The offset of the first element in the indices tensor + * @param[out] output_ptr Pointer to the destination tensor. Same as @p upt_ptr * @param[in] output_stride_x Stride of the destination tensor in X dimension (in bytes) - * @param[in] output_step_x output_stride_x * number of elements along X processed per work item (in bytes) + * @param[in] output_step_x output_stride_x * number of elements along X processed per workitem(in bytes) * @param[in] output_stride_y Stride of the destination tensor in Y dimension (in bytes) - * @param[in] output_step_y output_stride_y * number of elements along Y processed per work item (in bytes) - * @param[in] output_stride_z Stride of the destination tensor in Z dimension (in bytes) - * @param[in] output_step_z output_stride_z * number of elements along Z processed per work item (in bytes) - * @param[in] output_stride_w Stride of the destination tensor in W dimension (in bytes) - * @param[in] output_step_w output_stride_w * number of elements along W processed per work item (in bytes) - * @param[in] output_offset_first_element_in_bytes Offset of the first element in the destination tensor + * @param[in] output_step_y output_stride_y * number of elements along Y processed per workitem(in bytes) + * @param[in] output_offset_first_element_in_bytes The offset of the first element in the destination tensor + * @param[in] upt_block_stride Update tensor data block stride in bytes + * @param[in] out_block_stride Output tensor data block stride in bytes */ -// The below kernel code is expected to be excecuted sequentially with a single thread to ensure a deterministic outcome. -__kernel void scatter1D( - TENSOR4D_DECLARATION(updates), - TENSOR4D_DECLARATION(indices), - TENSOR4D_DECLARATION(output)) +__kernel void scatter_mp1d_2d_mpnd( + IMAGE_DECLARATION(updates), + IMAGE_DECLARATION(indices), + IMAGE_DECLARATION(output), + int upt_block_stride, + int out_block_stride + ) { - // Currently 1D - only iterate through y dimension of indices. - unsigned int* indices_start_offset = (unsigned int*)(indices_ptr + indices_offset_first_element_in_bytes); - DATA_TYPE* updates_start_offset = (DATA_TYPE*)(updates_ptr + updates_offset_first_element_in_bytes); - DATA_TYPE* out_start_offset = (DATA_TYPE*)(output_ptr + output_offset_first_element_in_bytes); - for (int px = 0; px < INDICES_SHAPE_Y; px++) + const int out_shape[5] = {OUT_SHAPE_N_MINUS_1, OUT_SHAPE_N_MINUS_2, OUT_SHAPE_N_MINUS_3, + OUT_SHAPE_N_MINUS_4, OUT_SHAPE_N_MINUS_5}; + + const int x = GET_SPATIAL_IDX(0, N0, PARTIAL_N0); // x-coordinate in the tensor + const int y = get_global_id(1); // collapsed y-coordinate (ignoring the outermost dimensions) + + const bool x_cond = (PARTIAL_N0 != 0 && get_global_id(0) == 0); + + uchar *ind_ptr_raw = indices_ptr + indices_offset_first_element_in_bytes; + const uchar *out_ptr_raw = output_ptr + output_offset_first_element_in_bytes + + x * sizeof(DATA_TYPE) + y * output_stride_y; + + const uchar *upt_ptr_raw = updates_ptr + updates_offset_first_element_in_bytes + + x * sizeof(DATA_TYPE) + y * updates_stride_y; + + for(int index_element = 0; index_element < NUM_INDICES; ++index_element) { - const int index_value = *(indices_start_offset); - DATA_TYPE* out_addr = out_start_offset + index_value; - if((index_value < OUT_SHAPE_X) && (index_value >= 0)) + const int *ind_ptr = (const int *) (ind_ptr_raw); + + // Out of bounds check + bool out_of_bounds = false; + LOOP_UNROLLING(int, i, 0, 1, INDEX_LENGTH, + { + if(ind_ptr[i] >= out_shape[i] || ind_ptr[i] < 0) + { + out_of_bounds = true; + } + }); + + ind_ptr_raw += indices_stride_y; + + if(out_of_bounds) { - *(__global DATA_TYPE *)(out_addr) = SCATTER_FUNCTION(*(out_addr), *updates_start_offset); + continue; } - // Increment pointers. - indices_start_offset++; - updates_start_offset++; + + // Index calculation + int index = 0; + LOOP_UNROLLING(int, i, 0, 1, INDEX_LENGTH, + { + index = index * out_shape[i] + ind_ptr[i]; + }); + + DATA_TYPE *out_ptr = (DATA_TYPE *) (out_ptr_raw + index * out_block_stride); + + const DATA_TYPE *upt_ptr = (const DATA_TYPE *) (upt_ptr_raw + index_element * upt_block_stride); + + VEC_DATA_TYPE(DATA_TYPE, N0) data_in0 = VLOAD(N0)(0, (__global DATA_TYPE *) upt_ptr); + +#ifdef SKIP_OUTPUT_READ + STORE_VECTOR_SELECT(data_in, DATA_TYPE, (__global DATA_TYPE *) out_ptr, N0, PARTIAL_N0, x_cond); +#else // ifdef SKIP_OUTPUT_READ + VEC_DATA_TYPE(DATA_TYPE, N0) data_out0 = VLOAD(N0)(0, (__global DATA_TYPE *) out_ptr); + data_out0 = SCATTER_FUNCTION(data_out0, data_in0); + + STORE_VECTOR_SELECT(data_out, DATA_TYPE, (__global DATA_TYPE *) out_ptr, N0, PARTIAL_N0, x_cond); +#endif // ifdef SKIP_OUTPUT_READ } } -#endif //defined(DATA_TYPE) && defined(SCATTER_FUNCTION) && defined(OUT_SHAPE_X) && defined(INDICES_SHAPE_Y) +#endif // SCATTER_MP1D_2D_MPND -#if defined(DATA_TYPE) && defined(SCATTER_FUNCTION) && defined(OUT_SHAPE_X) && !defined(INDICES_SHAPE_Y) +#ifdef SCATTER1D_PARALLEL // NOTE : This code is non-deterministic and can only be excecuted with the "update" ScatterFunction // This code is currently unusued as it requires changes to the existing test suite. @@ -114,4 +163,4 @@ __kernel void scatter1D_parallel( } } -#endif //defined(DATA_TYPE) && defined(SCATTER_FUNCTION) && defined(OUT_SHAPE_X) && !defined(INDICES_SHAPE_Y) +#endif // SCATTER1D_PARALLEL -- cgit v1.2.1