diff options
| -rw-r--r-- | arm_compute/core/Utils.h | 24 | ||||
| -rw-r--r-- | src/core/CL/cl_kernels/activation_layer.cl | 17 | ||||
| -rw-r--r-- | src/core/CL/cl_kernels/activation_layer_quant.cl | 36 | ||||
| -rw-r--r-- | src/core/CL/cl_kernels/gemm_helpers.h | 327 | ||||
| -rw-r--r-- | src/core/CL/cl_kernels/helpers.h | 18 | ||||
| -rw-r--r-- | src/core/CL/cl_kernels/load_store_utility.h | 366 | ||||
| -rw-r--r-- | src/core/CL/kernels/CLActivationLayerKernel.cpp | 55 | ||||
| -rw-r--r-- | tests/datasets/ShapeDatasets.h | 1 | ||||
| -rw-r--r-- | tests/validation/CL/ActivationLayer.cpp | 48 |
9 files changed, 492 insertions, 400 deletions
diff --git a/arm_compute/core/Utils.h b/arm_compute/core/Utils.h index 13c2971930..d2bc50c7c6 100644 --- a/arm_compute/core/Utils.h +++ b/arm_compute/core/Utils.h @@ -1322,6 +1322,30 @@ bool check_value_range(T val, DataType dt, QuantizationInfo qinfo = Quantization } } +/** Returns the adjusted vector size in case it is less than the input's first dimension, getting rounded down to its closest valid vector size + * + * @param[in] vec_size vector size to be adjusted + * @param[in] dim0 size of the first dimension + * + * @return the number of element processed along the X axis per thread + */ +inline unsigned int adjust_vec_size(unsigned int vec_size, size_t dim0) +{ + ARM_COMPUTE_ERROR_ON(vec_size > 16); + + if(dim0 == 3) + { + return dim0; + } + + while(vec_size > dim0) + { + vec_size >>= 1; + } + + return vec_size; +} + #ifdef ARM_COMPUTE_ASSERTS_ENABLED /** Print consecutive elements to an output stream. * diff --git a/src/core/CL/cl_kernels/activation_layer.cl b/src/core/CL/cl_kernels/activation_layer.cl index f846cb2764..499378c87f 100644 --- a/src/core/CL/cl_kernels/activation_layer.cl +++ b/src/core/CL/cl_kernels/activation_layer.cl @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2019 Arm Limited. + * Copyright (c) 2016-2020 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -61,23 +61,24 @@ __kernel void activation_layer( #endif /* not IN_PLACE */ ) { + uint x_offs = max((int)(get_global_id(0) * VEC_SIZE * sizeof(DATA_TYPE) - (VEC_SIZE - VEC_SIZE_LEFTOVER) % VEC_SIZE * sizeof(DATA_TYPE)), 0); + // Get pixels pointer - Tensor3D input = CONVERT_TO_TENSOR3D_STRUCT(input); + __global uchar *input_addr = input_ptr + input_offset_first_element_in_bytes + x_offs + get_global_id(1) * input_stride_y + get_global_id(2) * input_stride_z; #ifdef IN_PLACE - Tensor3D output = input; + __global uchar *output_addr = input_addr; #else /* IN_PLACE */ - Tensor3D output = CONVERT_TO_TENSOR3D_STRUCT(output); + __global uchar *output_addr = output_ptr + output_offset_first_element_in_bytes + x_offs + get_global_id(1) * output_stride_y + get_global_id(2) * output_stride_z; #endif /* IN_PLACE */ // Load data - TYPE data = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)input.ptr); + TYPE data0 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)input_addr); // Perform activation - data = ACTIVATION(ACT, DATA_TYPE, data, A_VAL, B_VAL); + data0 = ACTIVATION(ACT, DATA_TYPE, data0, A_VAL, B_VAL); // Store result - VSTORE(VEC_SIZE) - (data, 0, (__global DATA_TYPE *)output.ptr); + STORE_VECTOR_SELECT(data, DATA_TYPE, output_addr, VEC_SIZE, VEC_SIZE_LEFTOVER, VEC_SIZE_LEFTOVER != 0 && get_global_id(0) == 0) } #endif /* defined(ACT) */ diff --git a/src/core/CL/cl_kernels/activation_layer_quant.cl b/src/core/CL/cl_kernels/activation_layer_quant.cl index 0481319428..d8f56c093a 100644 --- a/src/core/CL/cl_kernels/activation_layer_quant.cl +++ b/src/core/CL/cl_kernels/activation_layer_quant.cl @@ -66,34 +66,35 @@ __kernel void activation_layer_quant_f32( #endif /* not IN_PLACE */ ) { + uint x_offs = max((int)(get_global_id(0) * VEC_SIZE * sizeof(DATA_TYPE) - (VEC_SIZE - VEC_SIZE_LEFTOVER) % VEC_SIZE * sizeof(DATA_TYPE)), 0); + // Get pixels pointer - Tensor3D input = CONVERT_TO_TENSOR3D_STRUCT(input); + __global uchar *input_addr = input_ptr + input_offset_first_element_in_bytes + x_offs + get_global_id(1) * input_stride_y + get_global_id(2) * input_stride_z; #ifdef IN_PLACE - Tensor3D output = input; + __global uchar *output_addr = input_addr; #else /* IN_PLACE */ - Tensor3D output = CONVERT_TO_TENSOR3D_STRUCT(output); + __global uchar *output_addr = output_ptr + output_offset_first_element_in_bytes + x_offs + get_global_id(1) * output_stride_y + get_global_id(2) * output_stride_z; #endif /* IN_PLACE */ // Load data - TYPE data = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)input.ptr); + TYPE data0 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)input_addr); - VEC_FLOAT data_flt = CONVERT(data, VEC_FLOAT); + VEC_FLOAT data_flt = CONVERT(data0, VEC_FLOAT); #if defined(O1_VAL) data_flt = round(data_flt - (float)O1_VAL) * ((float)S1_VAL); #else // defined(O1_VAL) - data_flt = round(data_flt) * ((float)S1_VAL); + data_flt = round(data_flt) * ((float)S1_VAL); #endif // defined(O1_VAL) data_flt = ACTIVATION(ACT, float, data_flt, A_VAL, B_VAL); #if defined(O2_VAL) - data = CONVERT_SAT(round(data_flt / ((float)S2_VAL)) + (float)O2_VAL, TYPE); + data0 = CONVERT_SAT(round(data_flt / ((float)S2_VAL)) + (float)O2_VAL, TYPE); #else // defined(O2_VAL) - data = CONVERT_SAT(round(data_flt / ((float)S2_VAL)), TYPE); + data0 = CONVERT_SAT(round(data_flt / ((float)S2_VAL)), TYPE); #endif // defined(O2_VAL) // Store result - VSTORE(VEC_SIZE) - (data, 0, (__global DATA_TYPE *)output.ptr); + STORE_VECTOR_SELECT(data, DATA_TYPE, output_addr, VEC_SIZE, VEC_SIZE_LEFTOVER, VEC_SIZE_LEFTOVER != 0 && get_global_id(0) == 0) } #else // defined(FLOAT_DOMAIN) @@ -137,22 +138,23 @@ __kernel void activation_layer_quant( #endif /* not IN_PLACE */ ) { + uint x_offs = max((int)(get_global_id(0) * VEC_SIZE * sizeof(DATA_TYPE) - (VEC_SIZE - VEC_SIZE_LEFTOVER) % VEC_SIZE * sizeof(DATA_TYPE)), 0); + // Get pixels pointer - Tensor3D input = CONVERT_TO_TENSOR3D_STRUCT(input); + __global uchar *input_addr = input_ptr + input_offset_first_element_in_bytes + x_offs + get_global_id(1) * input_stride_y + get_global_id(2) * input_stride_z; #ifdef IN_PLACE - Tensor3D output = input; + __global uchar *output_addr = input_addr; #else /* IN_PLACE */ - Tensor3D output = CONVERT_TO_TENSOR3D_STRUCT(output); + __global uchar *output_addr = output_ptr + output_offset_first_element_in_bytes + x_offs + get_global_id(1) * output_stride_y + get_global_id(2) * output_stride_z; #endif /* IN_PLACE */ // Load data - TYPE data = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)input.ptr); + TYPE data0 = VLOAD(VEC_SIZE)(0, (__global DATA_TYPE *)input_addr); - data = PERFORM_ACTIVATION_QUANT(ACT, data); + data0 = PERFORM_ACTIVATION_QUANT(ACT, data0); // Store result - VSTORE(VEC_SIZE) - (data, 0, (__global DATA_TYPE *)output.ptr); + STORE_VECTOR_SELECT(data, DATA_TYPE, output_addr, VEC_SIZE, VEC_SIZE_LEFTOVER, VEC_SIZE_LEFTOVER != 0 && get_global_id(0) == 0) } #endif // defined(ACT) #endif // defined(FLOAT_DOMAIN) diff --git a/src/core/CL/cl_kernels/gemm_helpers.h b/src/core/CL/cl_kernels/gemm_helpers.h index 6f6edc1bcf..3943a8de78 100644 --- a/src/core/CL/cl_kernels/gemm_helpers.h +++ b/src/core/CL/cl_kernels/gemm_helpers.h @@ -799,101 +799,6 @@ (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F)); /** @} */ // end of groupd STORE_ROW_n -/** Partially store the 0 to (n-1)th rows of the given variables - * @name STORE_ROW_PARTIAL_n - * Within each row, store the lower @p STORE_N0 elements of vectors of width @p N0 - * - * @note in case @p STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty. - * - * @param[in] N0 The width of the passed in vector. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] STORE_N0 The **lower** size of the vectors to store. Supported: [1-16 and <= @p N0 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @{ - */ -#define STORE_ROW_PARTIAL_1(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0)); - -#define STORE_ROW_PARTIAL_2(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_1(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1)); - -#define STORE_ROW_PARTIAL_3(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_2(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2)); - -#define STORE_ROW_PARTIAL_4(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_3(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3)); - -#define STORE_ROW_PARTIAL_5(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_4(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4)); - -#define STORE_ROW_PARTIAL_6(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_5(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5)); - -#define STORE_ROW_PARTIAL_7(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_6(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6)); - -#define STORE_ROW_PARTIAL_8(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_7(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7)); - -#define STORE_ROW_PARTIAL_9(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_8(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8)); - -#define STORE_ROW_PARTIAL_10(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_9(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9)); - -#define STORE_ROW_PARTIAL_11(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_10(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A)); - -#define STORE_ROW_PARTIAL_12(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_11(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B)); - -#define STORE_ROW_PARTIAL_13(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_12(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C)); - -#define STORE_ROW_PARTIAL_14(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_13(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D)); - -#define STORE_ROW_PARTIAL_15(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_14(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E)); - -#define STORE_ROW_PARTIAL_16(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - STORE_ROW_PARTIAL_15(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ - VSTORE_PARTIAL(N0, STORE_N0) \ - (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F)); -/** @} */ // end of groupd STORE_ROW_PARTIAL_n - /** Convert and store the 0th to (n-1)th rows of the given variables * @name CONVERT_STORE_ROW_n * @@ -1008,127 +913,6 @@ #define STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) /** @} */ // end of group STORE_BLOCK -/** Partially store a block of the given size STORE_M0xSTORE_N0 - * @name STORE_BLOCK_PARTIAL - * - * @note The vector width @p N0 is also required for correct partial storing behaviour. - * @note in case @p STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty. - * - * The data to store is expected to have consecutive names for each row. - * E.g., for STORE_M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for STORE_M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * @param[in] STORE_M0 The number of rows to store. Supported: 1-16 - * @param[in] STORE_N0 The lower number of elements of vectors to store. Supported: 1-16 and <= @p N0 - * @param[in] N0 The size of each vector. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @{ - */ -#define STORE_BLOCK_PARTIAL_STR(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_ROW_PARTIAL_##STORE_M0(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) -#define STORE_BLOCK_PARTIAL(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_BLOCK_PARTIAL_STR(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) -/** Store a block that can be partial in both x and y dimensions - * - * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty. - * - * The data to store is expected to have consecutive names for each row. - * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16 - * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0) - * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0) - * @param[in] N Total number of columns. Used to detect if current block is at the boundary in x. - * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0. - * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0. - */ -#define STORE_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \ - if(!(PARTIAL_COND_X) && !(PARTIAL_COND_Y)) \ - { \ - STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ - } \ - else if((PARTIAL_COND_Y) && !(PARTIAL_COND_X)) \ - { \ - STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ - } \ - else if(!(PARTIAL_COND_Y) && (PARTIAL_COND_X)) \ - { \ - STORE_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ - } \ - else \ - { \ - STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ - } -/** Store a block that can only be partial in x but not y. - * - * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty. - * - * The data to store is expected to have consecutive names for each row. - * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16 - * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0) - * @param[in] N Total number of columns. Used to detect if current block is at the boundary in x. - * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0. - */ -#define STORE_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_N0, N, PARTIAL_COND_X) \ - if(!(PARTIAL_COND_X)) \ - { \ - STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ - } \ - else \ - { \ - STORE_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ - } -/** Store a block that can only be partial in y but not x. - * - * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty. - * - * The data to store is expected to have consecutive names for each row. - * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16 - * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0) - * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0. - */ -#define STORE_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y) \ - if(!(PARTIAL_COND_Y)) \ - { \ - STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ - } \ - else \ - { \ - STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ - } -/** @} */ // end of group STORE_BLOCK_PARTIAL - /** Convert and store a block of the given size M0xN0 * @name CONVERT_STORE_BLOCK * @@ -1732,113 +1516,4 @@ */ #define CONVERT_BLOCK_STR(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) CONVERT_ROW_##M(N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) #define CONVERT_BLOCK(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) CONVERT_BLOCK_STR(M, N, DATA_TYPE, BASENAME_SRC, BASENAME_DST) -/** @} */ // end of group CONVERT_BLOCK - -#if defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0) - -/** Boundary-aware GEMM block store - * @name STORE_BLOCK_BOUNDARY_AWARE - * This macro assumes the following schemes to achieve boundary-awareness: - * - Overlapping load in Y axis from lhs tensor. This implies lhs has no padding along y dim. - * - Non-Overlapping(normal) load from rhs tensor. This imples rhs can have paddings. - * - Overlapping load in Y axis from bias tensor. This implies rhs has no padding along y dim. - * The macro then ensures that the dst tensor can be stored without any paddings in both x and y dim. - * - * In the y dimension, we place the partial blocks **at the beginning** while in the x dimension, we place the partial - * blocks **at the end**. - * Say, the dst tensor is of shape MxN and we have M0 and N0 as the block size, this is how we define "partial blocks"/ - * "boundary block" (we use the 2 terms "partial blocks" and "boundary blocks" interchangeably) and its various parameters: - * - * *--x--> x == 0 x == 1 - * | |<------------------------------N-------------------------->| - * y |<--------------N0------------->|<----PARTIAL_STORE_N0----->| - * | -------------############################################################# - * * | | |...............................|...........................| - * y == 0 | PAR_..._M0 |......Boundary block in y......|.Boundary block in x and y.| - * | | |...............................|...........................| - * M --############################################################# - * | | | |...........................| - * y == 1 | M0 | Non-boundary block |....Boundary block in x....| - * | | | |...........................| - * |------------############################################################# - * - * Then @p PARTIAL_STORE_M0 = M % M0 and @p PARTIAL_STORE_N0 = N % N0 - * - * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty. - * - * It automatically detects if a giving M,N,M0,N0 combination can yield partial blocks in either X and Y dimension, - * and select corresponding store methods such that the boundary detection logic is only added when needed. - * - * The data to store is expected to have consecutive names for each row. - * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16 - * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0) - * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported: [0, @p N0) - * @param[in] N Total number of columns. Used to detect if current block is at the boundary in x. - * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0. - * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0. - * @{ - */ -#if PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0 -// Case1: No partial blocks in either x or y -#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \ - STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) - -#elif PARTIAL_STORE_M0 > 0 && PARTIAL_STORE_N0 == 0 -// Case2: Partial blocks in y -#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \ - STORE_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y) - -#elif PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 > 0 -// Case3: Partial blocks in x -#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \ - STORE_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_N0, N, PARTIAL_COND_X) - -#else // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0 -// Case4: Partial blocks in both x and y -#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \ - STORE_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) - -#endif // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0 - -#else // defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0) - -#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \ - STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) - -#endif // defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0) -/** @} */ // end of group STORE_BLOCK_BOUNDARY_AWARE - -#if defined(PARTIAL_STORE_M0) -/** Compute the start m0 row (LHS, BIAS and DST) in a boundary-aware way so as to avoid padding - * @name COMPUTE_M0_START_ROW - * If there're any partial blocks in y dimension, they are placed at the beginning of the rows. - * This shift amount is added to all rows such that the partial block (at the beginning) overlaps with the subsequent - * blocks in the y dimension to avoid any padding. - * EG: M0=4, PARTIAL_STORE_M0=1: - * | Non-overlapping | +M0_ROW_SHIFT (Overlapping) - * block 0 (partial)| start row = 0 | start row = 0 - * block 1 (full) | start row = 4 | start row = 1 - * block 2 (full) | start row = 8 | start row = 5 - * - * @param[in] y Global id of current block in y. - * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16 - * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0) - * @{ - */ -#define COMPUTE_M0_START_ROW(y, M0, PARTIAL_STORE_M0) \ - ((uint)(max(0, (int)(y * M0) - (int)((M0 - PARTIAL_STORE_M0) % M0)))) -#else // defined(PARTIAL_STORE_M0) -#define COMPUTE_M0_START_ROW(y, M0, PARTIAL_STORE_M0) \ - ((uint)(y * M0)) -#endif // defined(PARTIAL_STORE_M0) -/** @} */ // end of group COMPUTE_M0_START_ROW +/** @} */ // end of group CONVERT_BLOCK
\ No newline at end of file diff --git a/src/core/CL/cl_kernels/helpers.h b/src/core/CL/cl_kernels/helpers.h index 7b08233029..10f04e9e31 100644 --- a/src/core/CL/cl_kernels/helpers.h +++ b/src/core/CL/cl_kernels/helpers.h @@ -24,6 +24,8 @@ #ifndef ARM_COMPUTE_HELPER_H #define ARM_COMPUTE_HELPER_H +#include "load_store_utility.h" + #if defined(ARM_COMPUTE_OPENCL_FP16_ENABLED) && defined(cl_khr_fp16) #pragma OPENCL EXTENSION cl_khr_fp16 : enable #endif // defined(ARM_COMPUTE_OPENCL_FP16_ENABLED) && defined(cl_khr_fp16) @@ -273,21 +275,30 @@ #define VSTORE_PARTIAL_STR(size, store_size) vstore_partial_##size##_##store_size #define VSTORE_PARTIAL(size, store_size) VSTORE_PARTIAL_STR(size, store_size) +#define NO_STORE(data, offs, ptr) \ + { \ + } + // Size == 1 (scalar) +#define vstore_partial_1_0 NO_STORE #define vstore_partial_1_1 vstore1 // Size == 2 +#define vstore_partial_2_0 NO_STORE #define vstore_partial_2_1 vstore_partial_1 #define vstore_partial_2_2 vstore_partial_2 // Size == 3 +#define vstore_partial_3_0 NO_STORE #define vstore_partial_3_1 vstore_partial_1 #define vstore_partial_3_2 vstore_partial_2 #define vstore_partial_3_3 vstore_partial_3 // Size == 4 +#define vstore_partial_4_0 NO_STORE #define vstore_partial_4_1 vstore_partial_1 #define vstore_partial_4_2 vstore_partial_2 #define vstore_partial_4_3 vstore_partial_3 #define vstore_partial_4_4 vstore_partial_4 // Size == 8 +#define vstore_partial_8_0 NO_STORE #define vstore_partial_8_1 vstore_partial_1 #define vstore_partial_8_2 vstore_partial_2 #define vstore_partial_8_3 vstore_partial_3 @@ -297,6 +308,7 @@ #define vstore_partial_8_7 vstore_partial_7 #define vstore_partial_8_8 vstore_partial_8 // Size == 16 +#define vstore_partial_16_0 NO_STORE #define vstore_partial_16_1 vstore_partial_1 #define vstore_partial_16_2 vstore_partial_2 #define vstore_partial_16_3 vstore_partial_3 @@ -376,15 +388,15 @@ #define vstore_partial_13(DATA, OFFSET, PTR) \ vstore_partial_8(DATA.s01234567, OFFSET, PTR); \ - vstore_partial_5(DATA.s89abc, OFFSET, PTR + 8); + vstore_partial_5(DATA.s89abcdef, OFFSET, PTR + 8); #define vstore_partial_14(DATA, OFFSET, PTR) \ vstore_partial_8(DATA.s01234567, OFFSET, PTR); \ - vstore_partial_6(DATA.s89abcd, OFFSET, PTR + 8); + vstore_partial_6(DATA.s89abcdef, OFFSET, PTR + 8); #define vstore_partial_15(DATA, OFFSET, PTR) \ vstore_partial_8(DATA.s01234567, OFFSET, PTR); \ - vstore_partial_7(DATA.s89abcde, OFFSET, PTR + 8); + vstore_partial_7(DATA.s89abcdef, OFFSET, PTR + 8); #define vstore_partial_16(DATA, OFFSET, PTR) \ vstore16(DATA, OFFSET, PTR); diff --git a/src/core/CL/cl_kernels/load_store_utility.h b/src/core/CL/cl_kernels/load_store_utility.h new file mode 100644 index 0000000000..cb833d016b --- /dev/null +++ b/src/core/CL/cl_kernels/load_store_utility.h @@ -0,0 +1,366 @@ +/* + * Copyright (c) 2020 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +/** Partially store the 0 to (n-1)th rows of the given variables + * @name STORE_ROW_PARTIAL_n + * Within each row, store the lower @p STORE_N0 elements of vectors of width @p N0 + * + * @note in case @p STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty. + * + * @param[in] N0 The width of the passed in vector. Supported: 1, 2, 3, 4, 8, 16 + * @param[in] STORE_N0 The **lower** size of the vectors to store. Supported: [1-16 and <= @p N0 + * @param[in] DATA_TYPE The data type of the vectors + * @param[in] BASENAME The basename of the variables + * @param[in] PTR The base pointer + * @param[in] STRIDE_Y The stride value in y-axis direction + * @param[in] Z The offset in z-axis direction + * @{ + */ +#define STORE_ROW_PARTIAL_1(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + 0 * STRIDE_Y + Z##0)); + +#define STORE_ROW_PARTIAL_2(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_1(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + 1 * STRIDE_Y + Z##1)); + +#define STORE_ROW_PARTIAL_3(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_2(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + 2 * STRIDE_Y + Z##2)); + +#define STORE_ROW_PARTIAL_4(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_3(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + 3 * STRIDE_Y + Z##3)); + +#define STORE_ROW_PARTIAL_5(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_4(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + 4 * STRIDE_Y + Z##4)); + +#define STORE_ROW_PARTIAL_6(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_5(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + 5 * STRIDE_Y + Z##5)); + +#define STORE_ROW_PARTIAL_7(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_6(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + 6 * STRIDE_Y + Z##6)); + +#define STORE_ROW_PARTIAL_8(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_7(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + 7 * STRIDE_Y + Z##7)); + +#define STORE_ROW_PARTIAL_9(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_8(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + 8 * STRIDE_Y + Z##8)); + +#define STORE_ROW_PARTIAL_10(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_9(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + 9 * STRIDE_Y + Z##9)); + +#define STORE_ROW_PARTIAL_11(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_10(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + 10 * STRIDE_Y + Z##A)); + +#define STORE_ROW_PARTIAL_12(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_11(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + 11 * STRIDE_Y + Z##B)); + +#define STORE_ROW_PARTIAL_13(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_12(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + 12 * STRIDE_Y + Z##C)); + +#define STORE_ROW_PARTIAL_14(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_13(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + 13 * STRIDE_Y + Z##D)); + +#define STORE_ROW_PARTIAL_15(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_14(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + 14 * STRIDE_Y + Z##E)); + +#define STORE_ROW_PARTIAL_16(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + STORE_ROW_PARTIAL_15(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) \ + VSTORE_PARTIAL(N0, STORE_N0) \ + (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + 15 * STRIDE_Y + Z##F)); +/** @} */ // end of groupd STORE_ROW_PARTIAL_n + +/** Partially store a block of the given size STORE_M0xSTORE_N0 + * @name STORE_BLOCK_PARTIAL + * + * @note The vector width @p N0 is also required for correct partial storing behaviour. + * @note in case @p STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty. + * + * The data to store is expected to have consecutive names for each row. + * E.g., for STORE_M0=3 and basename=c, the expected names are c0, c1 and c2. + * The Z offset is expected to have consecutive names. + * E.g., for STORE_M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. + * + * @param[in] STORE_M0 The number of rows to store. Supported: 1-16 + * @param[in] STORE_N0 The lower number of elements of vectors to store. Supported: 1-16 and <= @p N0 + * @param[in] N0 The size of each vector. Supported: 1, 2, 3, 4, 8, 16 + * @param[in] DATA_TYPE The data type of the vectors + * @param[in] BASENAME The basename of the variables + * @param[in] PTR The base pointer + * @param[in] STRIDE_Y The stride value in y-axis direction + * @param[in] Z The offset in z-axis direction + * @{ + */ +#define STORE_BLOCK_PARTIAL_STR(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_ROW_PARTIAL_##STORE_M0(N0, STORE_N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) +#define STORE_BLOCK_PARTIAL(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) STORE_BLOCK_PARTIAL_STR(STORE_M0, STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) +/** Store a block that can be partial in both x and y dimensions + * + * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty. + * + * The data to store is expected to have consecutive names for each row. + * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. + * The Z offset is expected to have consecutive names. + * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. + * + * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16 + * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 + * @param[in] DATA_TYPE The data type of the vectors + * @param[in] BASENAME The basename of the variables + * @param[in] PTR The base pointer + * @param[in] STRIDE_Y The stride value in y-axis direction + * @param[in] Z The offset in z-axis direction + * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0) + * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0) + * @param[in] N Total number of columns. Used to detect if current block is at the boundary in x. + * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0. + * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0. + */ +#define STORE_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \ + if(!(PARTIAL_COND_X) && !(PARTIAL_COND_Y)) \ + { \ + STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ + } \ + else if((PARTIAL_COND_Y) && !(PARTIAL_COND_X)) \ + { \ + STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ + } \ + else if(!(PARTIAL_COND_Y) && (PARTIAL_COND_X)) \ + { \ + STORE_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ + } \ + else \ + { \ + STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ + } +/** Store a block that can only be partial in x but not y. + * + * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty. + * + * The data to store is expected to have consecutive names for each row. + * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. + * The Z offset is expected to have consecutive names. + * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. + * + * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16 + * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 + * @param[in] DATA_TYPE The data type of the vectors + * @param[in] BASENAME The basename of the variables + * @param[in] PTR The base pointer + * @param[in] STRIDE_Y The stride value in y-axis direction + * @param[in] Z The offset in z-axis direction + * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0) + * @param[in] N Total number of columns. Used to detect if current block is at the boundary in x. + * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0. + */ +#define STORE_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_N0, N, PARTIAL_COND_X) \ + if(!(PARTIAL_COND_X)) \ + { \ + STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ + } \ + else \ + { \ + STORE_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ + } +/** Store a block that can only be partial in y but not x. + * + * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty. + * + * The data to store is expected to have consecutive names for each row. + * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. + * The Z offset is expected to have consecutive names. + * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. + * + * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16 + * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 + * @param[in] DATA_TYPE The data type of the vectors + * @param[in] BASENAME The basename of the variables + * @param[in] PTR The base pointer + * @param[in] STRIDE_Y The stride value in y-axis direction + * @param[in] Z The offset in z-axis direction + * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0) + * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0. + */ +#define STORE_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y) \ + if(!(PARTIAL_COND_Y)) \ + { \ + STORE_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ + } \ + else \ + { \ + STORE_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z); \ + } +/** @} */ // end of group STORE_BLOCK_PARTIAL + +#if defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0) + +/** Boundary-aware GEMM block store + * @name STORE_BLOCK_BOUNDARY_AWARE + * This macro assumes the following schemes to achieve boundary-awareness: + * - Overlapping load in Y axis from lhs tensor. This implies lhs has no padding along y dim. + * - Non-Overlapping(normal) load from rhs tensor. This imples rhs can have paddings. + * - Overlapping load in Y axis from bias tensor. This implies rhs has no padding along y dim. + * The macro then ensures that the dst tensor can be stored without any paddings in both x and y dim. + * + * In the y dimension, we place the partial blocks **at the beginning** while in the x dimension, we place the partial + * blocks **at the end**. + * Say, the dst tensor is of shape MxN and we have M0 and N0 as the block size, this is how we define "partial blocks"/ + * "boundary block" (we use the 2 terms "partial blocks" and "boundary blocks" interchangeably) and its various parameters: + * + * *--x--> x == 0 x == 1 + * | |<------------------------------N-------------------------->| + * y |<--------------N0------------->|<----PARTIAL_STORE_N0----->| + * | -------------############################################################# + * * | | |...............................|...........................| + * y == 0 | PAR_..._M0 |......Boundary block in y......|.Boundary block in x and y.| + * | | |...............................|...........................| + * M --############################################################# + * | | | |...........................| + * y == 1 | M0 | Non-boundary block |....Boundary block in x....| + * | | | |...........................| + * |------------############################################################# + * + * Then @p PARTIAL_STORE_M0 = M % M0 and @p PARTIAL_STORE_N0 = N % N0 + * + * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty. + * + * It automatically detects if a giving M,N,M0,N0 combination can yield partial blocks in either X and Y dimension, + * and select corresponding store methods such that the boundary detection logic is only added when needed. + * + * The data to store is expected to have consecutive names for each row. + * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. + * The Z offset is expected to have consecutive names. + * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. + * + * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16 + * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 + * @param[in] DATA_TYPE The data type of the vectors + * @param[in] BASENAME The basename of the variables + * @param[in] PTR The base pointer + * @param[in] STRIDE_Y The stride value in y-axis direction + * @param[in] Z The offset in z-axis direction + * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0) + * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported: [0, @p N0) + * @param[in] N Total number of columns. Used to detect if current block is at the boundary in x. + * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0. + * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial store X. True to use PARTIAL_STORE_N0 rather than N0. + * @{ + */ +#if PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0 +// Case1: No partial blocks in either x or y +#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \ + STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) + +#elif PARTIAL_STORE_M0 > 0 && PARTIAL_STORE_N0 == 0 +// Case2: Partial blocks in y +#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \ + STORE_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y) + +#elif PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 > 0 +// Case3: Partial blocks in x +#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \ + STORE_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_N0, N, PARTIAL_COND_X) + +#else // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0 +// Case4: Partial blocks in both x and y +#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \ + STORE_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) + +#endif // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0 + +#else // defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0) + +#define STORE_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, N, PARTIAL_COND_Y, PARTIAL_COND_X) \ + STORE_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, STRIDE_Y, Z) + +#endif // defined(PARTIAL_STORE_M0) && defined(PARTIAL_STORE_N0) +/** @} */ // end of group STORE_BLOCK_BOUNDARY_AWARE + +#if defined(PARTIAL_STORE_M0) +/** Compute the start m0 row (LHS, BIAS and DST) in a boundary-aware way so as to avoid padding + * @name COMPUTE_M0_START_ROW + * If there're any partial blocks in y dimension, they are placed at the beginning of the rows. + * This shift amount is added to all rows such that the partial block (at the beginning) overlaps with the subsequent + * blocks in the y dimension to avoid any padding. + * EG: M0=4, PARTIAL_STORE_M0=1: + * | Non-overlapping | +M0_ROW_SHIFT (Overlapping) + * block 0 (partial)| start row = 0 | start row = 0 + * block 1 (full) | start row = 4 | start row = 1 + * block 2 (full) | start row = 8 | start row = 5 + * + * @param[in] y Global id of current block in y. + * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16 + * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0) + * @{ + */ +#define COMPUTE_M0_START_ROW(y, M0, PARTIAL_STORE_M0) \ + ((uint)(max(0, (int)(y * M0) - (int)((M0 - PARTIAL_STORE_M0) % M0)))) +#else // defined(PARTIAL_STORE_M0) +#define COMPUTE_M0_START_ROW(y, M0, PARTIAL_STORE_M0) \ + ((uint)(y * M0)) +#endif // defined(PARTIAL_STORE_M0) +/** @} */ // end of group COMPUTE_M0_START_ROW + +/** Store a vector that can only be partial in x. + * + * @note in case @p vec_size or @p leftover != 1, 2, 3, 4, 8, 16, extra vstore(s) will be invoked, thus incurring small performance penalty. + * + * The data to store is expected to end in a 0. + * E.g., for basename=c, the expected name is c0. + * + * @param[in] basename The name of the variable without trailing 0 + * @param[in] data_type The data type of the vector + * @param[in] ptr The base pointer + * @param[in] vec_size The vector size if cond = false. Supported: 1, 2, 3, 4, 8, 16 + * @param[in] leftover The vector size if cond = true. Supported range: [1, @p vec_size0) + * @param[in] cond Condition to select either vec_size0 or vec_size1 + * @{ + */ +#define STORE_VECTOR_SELECT(basename, data_type, ptr, vec_size, leftover, cond) \ + STORE_BLOCK_PARTIAL_IN_X(1, vec_size, data_type, basename, ptr, 0, 0, leftover, 0, cond) +/** @} */ // end of group STORE_VECTOR_SELECT
\ No newline at end of file diff --git a/src/core/CL/kernels/CLActivationLayerKernel.cpp b/src/core/CL/kernels/CLActivationLayerKernel.cpp index 62cafc5ad1..5a9434ee5a 100644 --- a/src/core/CL/kernels/CLActivationLayerKernel.cpp +++ b/src/core/CL/kernels/CLActivationLayerKernel.cpp @@ -80,37 +80,6 @@ Status validate_arguments(const ITensorInfo *input, const ITensorInfo *output, c return Status{}; } - -std::pair<Status, Window> validate_and_configure_window(ITensorInfo *input, ITensorInfo *output) -{ - if(output != nullptr) - { - ARM_COMPUTE_ERROR_ON_NULLPTR(input, output); - // Output auto inizialitation if not yet initialized - auto_init_if_empty(*output, *input); - } - - const unsigned int num_elems_processed_per_iteration = 16 / input->element_size(); - - Window win = calculate_max_window(*input, Steps(num_elems_processed_per_iteration)); - bool window_changed = false; - - if(output != nullptr) - { - AccessWindowHorizontal input_access(input, 0, num_elems_processed_per_iteration); - AccessWindowHorizontal output_access(output, 0, num_elems_processed_per_iteration); - window_changed = update_window_and_padding(win, input_access, output_access); - output_access.set_valid_region(win, input->valid_region()); - } - else - { - window_changed = update_window_and_padding(win, - AccessWindowHorizontal(input, 0, num_elems_processed_per_iteration)); - } - - Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{}; - return std::make_pair(err, win); -} } // namespace CLActivationLayerKernel::CLActivationLayerKernel() @@ -132,10 +101,11 @@ void CLActivationLayerKernel::configure(const CLCompileContext &compile_context, ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input, (output != nullptr) ? output : nullptr, act_info)); - const unsigned int num_elems_processed_per_iteration = 16 / input->element_size(); - const DataType dt = input->data_type(); - float a_const = act_info.a(); - float b_const = act_info.b(); + const unsigned int num_elems_processed_per_iteration = adjust_vec_size(16 / input->element_size(), input->dimension(0)); + + const DataType dt = input->data_type(); + float a_const = act_info.a(); + float b_const = act_info.b(); const ActivationLayerInfo::ActivationFunction f_act = act_info.activation(); const bool is_quantized = is_data_type_quantized(dt); @@ -146,9 +116,10 @@ void CLActivationLayerKernel::configure(const CLCompileContext &compile_context, CLBuildOptions build_opts; build_opts.add_option_if(perform_activation_in_float, "-DFLOAT_DOMAIN"); build_opts.add_option_if(_run_in_place, "-DIN_PLACE"); - build_opts.add_option(("-DACT=" + lower_string(string_from_activation_func(f_act)))); - build_opts.add_option(("-DDATA_TYPE=" + get_cl_type_from_data_type(dt))); - build_opts.add_option(("-DVEC_SIZE=" + support::cpp11::to_string(num_elems_processed_per_iteration))); + build_opts.add_option("-DACT=" + lower_string(string_from_activation_func(f_act))); + build_opts.add_option("-DDATA_TYPE=" + get_cl_type_from_data_type(dt)); + build_opts.add_option("-DVEC_SIZE=" + support::cpp11::to_string(num_elems_processed_per_iteration)); + build_opts.add_option("-DVEC_SIZE_LEFTOVER=" + support::cpp11::to_string(input->dimension(0) % num_elems_processed_per_iteration)); std::string kernel_name = std::string("activation_layer"); @@ -226,9 +197,8 @@ void CLActivationLayerKernel::configure(const CLCompileContext &compile_context, _kernel = create_kernel(compile_context, kernel_name, build_opts.options()); // Configure kernel window - auto win_config = validate_and_configure_window(input, (_run_in_place) ? nullptr : output); - ARM_COMPUTE_ERROR_THROW_ON(win_config.first); - ICLKernel::configure_internal(win_config.second); + Window win = calculate_max_window(*input, Steps(num_elems_processed_per_iteration)); + ICLKernel::configure_internal(win); // Set config_id for enabling LWS tuning _config_id = "activation_layer_"; @@ -241,10 +211,7 @@ void CLActivationLayerKernel::configure(const CLCompileContext &compile_context, Status CLActivationLayerKernel::validate(const ITensorInfo *input, const ITensorInfo *output, const ActivationLayerInfo &act_info) { - const bool run_in_place = (output == nullptr) || (output == input); ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, output, act_info)); - ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window(input->clone().get(), (run_in_place) ? nullptr : output->clone().get()).first); - return Status{}; } diff --git a/tests/datasets/ShapeDatasets.h b/tests/datasets/ShapeDatasets.h index ccd0756c2d..565982efb6 100644 --- a/tests/datasets/ShapeDatasets.h +++ b/tests/datasets/ShapeDatasets.h @@ -179,6 +179,7 @@ public: { // Batch size 1 TensorShape{ 11U, 11U }, + TensorShape{ 16U, 16U }, TensorShape{ 27U, 13U, 7U }, TensorShape{ 31U, 27U, 17U, 2U }, // Batch size 4 diff --git a/tests/validation/CL/ActivationLayer.cpp b/tests/validation/CL/ActivationLayer.cpp index c1844674ac..f776e334a0 100644 --- a/tests/validation/CL/ActivationLayer.cpp +++ b/tests/validation/CL/ActivationLayer.cpp @@ -89,6 +89,29 @@ const auto CNNDataTypes = framework::dataset::make("DataType", /** Input data sets. */ const auto ActivationDataset = combine(combine(framework::dataset::make("InPlace", { false, true }), datasets::ActivationFunctions()), framework::dataset::make("AlphaBeta", { 0.5f, 1.f })); + +/** Zero padding test */ +bool validate_zero_padding(unsigned int width, unsigned int height, unsigned int channels, unsigned int batches, const ActivationLayerInfo &act_info, DataType data_type) +{ + TensorShape shape(width, height, channels, batches); + + // Create tensors + CLTensor src = create_tensor<CLTensor>(shape, data_type); + CLTensor dst = create_tensor<CLTensor>(shape, data_type); + + src.info()->set_quantization_info(QuantizationInfo(1.f / 256.f, 0)); + dst.info()->set_quantization_info(QuantizationInfo(1.f / 256.f, 0)); + + ARM_COMPUTE_EXPECT(src.info()->is_resizable(), framework::LogLevel::ERRORS); + ARM_COMPUTE_EXPECT(dst.info()->is_resizable(), framework::LogLevel::ERRORS); + + // Create and configure function + CLActivationLayer act; + act.configure(&src, &dst, act_info); + + // Padding can be added along rhs and bias's X dimension + return src.info()->padding().empty() && dst.info()->padding().empty(); +} } // namespace TEST_SUITE(CL) @@ -97,7 +120,7 @@ TEST_SUITE(ActivationLayer) // clang-format off DATA_TEST_CASE(Validate, framework::DatasetMode::ALL, zip(zip(zip( framework::dataset::make("InputInfo", { TensorInfo(TensorShape(27U, 13U, 2U), 1, DataType::F32), // Mismatching data types - TensorInfo(TensorShape(27U, 13U, 2U), 1, DataType::F32), // Window shrink + TensorInfo(TensorShape(27U, 13U, 2U), 1, DataType::F32), TensorInfo(TensorShape(32U, 13U, 2U), 1, DataType::F32), TensorInfo(TensorShape(32U, 13U, 2U), 1, DataType::QASYMM8), TensorInfo(TensorShape(27U, 13U, 2U), 1, DataType::QASYMM8), // Invalid quantization info @@ -126,11 +149,32 @@ DATA_TEST_CASE(Validate, framework::DatasetMode::ALL, zip(zip(zip( ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LOGISTIC), ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::SQRT), })), - framework::dataset::make("Expected", { false, false, true, true, false, false, true, true, false })), + framework::dataset::make("Expected", { false, true, true, true, false, false, true, true, false })), input_info, output_info, act_info, expected) { ARM_COMPUTE_EXPECT(bool(CLActivationLayer::validate(&input_info.clone()->set_is_resizable(false), (output_info.total_size() == 0) ? nullptr : &output_info.clone()->set_is_resizable(false), act_info)) == expected, framework::LogLevel::ERRORS); } + +/** Validate zero padding tests + * + * A series of validation tests to check that no padding is added as part of configuration for 4 different scenarios. + * + * Checks performed in order: + * - First dimension multiple of 16 + * - First dimension non-multiple of 16 + * - First dimension less than 16 (vec_size for qasymm8) but multiple + * - First dimension less than 16 (vec_size for qasymm8) non-multiple + * - Tensor with only one element + */ +DATA_TEST_CASE(ValidateZeroPadding, framework::DatasetMode::ALL, zip( +framework::dataset::make("Width", { 32U, 37U, 12U, 13U, 1U }), +framework::dataset::make("DataType", { DataType::F32, DataType::QASYMM8 })), +width, data_type) +{ + const bool one_elem = (width == 1U); + bool status = validate_zero_padding(width, one_elem ? 1U : 17U, one_elem ? 1U : 7U, one_elem ? 1U : 2U, ActivationLayerInfo(ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU, 1U, 6U), data_type); + ARM_COMPUTE_EXPECT(status, framework::LogLevel::ERRORS); +} // clang-format on // *INDENT-ON* |