diff options
Diffstat (limited to 'src/core/CL')
-rw-r--r-- | src/core/CL/cl_kernels/common/gemm.cl | 40 | ||||
-rw-r--r-- | src/core/CL/cl_kernels/gemm_helpers.h | 300 | ||||
-rw-r--r-- | src/core/CL/cl_kernels/helpers.h | 208 |
3 files changed, 20 insertions, 528 deletions
diff --git a/src/core/CL/cl_kernels/common/gemm.cl b/src/core/CL/cl_kernels/common/gemm.cl index ff153af542..10435d376f 100644 --- a/src/core/CL/cl_kernels/common/gemm.cl +++ b/src/core/CL/cl_kernels/common/gemm.cl @@ -2705,9 +2705,6 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t(IMAGE_DECLARATION(lhs), REPEAT_VAR_INIT_TO_CONST(M0, uint, zout, 0); - const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); - const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); - #if defined(REINTERPRET_OUTPUT_AS_3D) // The plane (zin) is calculated dividing M (y * M0) by HEIGHT_GEMM3D @@ -2733,7 +2730,7 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t(IMAGE_DECLARATION(lhs), #if defined(BROADCAST_BIAS) __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (get_global_id(0) * (uint)N0 * sizeof(DATA_TYPE)); - LOAD_BLOCK_BOUNDARY_AWARE(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, 1, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(1, DATA_TYPE, bias, BETA); @@ -2751,7 +2748,7 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t(IMAGE_DECLARATION(lhs), __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (get_global_id(0) * (uint)N0 * sizeof(DATA_TYPE)) + (get_global_id(1) * (uint)M0 * bias_stride_y) + get_global_id( 2) * bias_stride_z; - LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, PARTIAL_STORE_M0, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(M0, DATA_TYPE, bias, BETA); @@ -2776,6 +2773,9 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t(IMAGE_DECLARATION(lhs), #endif // defined(MIXED_PRECISION) #endif // defined(ACTIVATION_TYPE) + const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); + const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); + // Store output block #if defined(MIXED_PRECISION) CONVERT_BLOCK(M0, N0, DATA_TYPE, c, c_lp); @@ -2975,9 +2975,6 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t_texture(IMAGE_DECLARATION(lhs), REPEAT_VAR_INIT_TO_CONST(M0, uint, zout, 0); - const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); - const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); - #if defined(REINTERPRET_OUTPUT_AS_3D) // The plane (zin) is calculated dividing M (y * M0) by HEIGHT_GEMM3D @@ -3003,7 +3000,7 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t_texture(IMAGE_DECLARATION(lhs), #if defined(BROADCAST_BIAS) __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (get_global_id(0) * (uint)N0 * sizeof(DATA_TYPE)); - LOAD_BLOCK_BOUNDARY_AWARE(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, 1, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(1, DATA_TYPE, bias, BETA); @@ -3021,7 +3018,7 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t_texture(IMAGE_DECLARATION(lhs), __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (get_global_id(0) * (uint)N0 * sizeof(DATA_TYPE)) + (get_global_id(1) * (uint)M0 * bias_stride_y) + get_global_id( 2) * bias_stride_z; - LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, PARTIAL_STORE_M0, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(M0, DATA_TYPE, bias, BETA); @@ -3046,6 +3043,9 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t_texture(IMAGE_DECLARATION(lhs), #endif // defined(MIXED_PRECISION) #endif // defined(ACTIVATION_TYPE) + const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); + const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); + // Store output block #if defined(MIXED_PRECISION) CONVERT_BLOCK(M0, N0, DATA_TYPE, c, c_lp); @@ -3284,9 +3284,6 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt(IMAGE_DECLARATION(lhs), const uint y = get_global_id(1); const uint z = get_global_id(2); - const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); - const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); - #if defined(DUMMY_WORK_ITEMS) if((x * N0 >= N) || (y * M0 >= M)) { @@ -3498,7 +3495,7 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt(IMAGE_DECLARATION(lhs), #if defined(BROADCAST_BIAS) __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (x * (uint)N0 * sizeof(DATA_TYPE)); - LOAD_BLOCK_BOUNDARY_AWARE(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, 1, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(1, DATA_TYPE, bias, BETA); @@ -3516,7 +3513,7 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt(IMAGE_DECLARATION(lhs), __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (get_global_id(0) * (uint)N0 * sizeof(DATA_TYPE)) + (get_global_id(1) * (uint)M0 * bias_stride_y) + get_global_id( 2) * bias_stride_z; - LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, PARTIAL_STORE_M0, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(M0, DATA_TYPE, bias, BETA); @@ -3540,6 +3537,9 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt(IMAGE_DECLARATION(lhs), #endif // defined(MIXED_PRECISION) #endif // defined(ACTIVATION_TYPE) + const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); + const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); + // Store output block #if defined(MIXED_PRECISION) CONVERT_BLOCK(M0, N0, DATA_TYPE, c, c_lp); @@ -3838,9 +3838,6 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt_texture(IMAGE_DECLARATION(lhs), REPEAT_VAR_INIT_TO_CONST(M0, uint, zout, 0); - const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); - const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); - #if defined(REINTERPRET_OUTPUT_AS_3D) // The plane (zin) is calculated dividing M (y * M0) by HEIGHT_GEMM3D @@ -3866,7 +3863,7 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt_texture(IMAGE_DECLARATION(lhs), #if defined(BROADCAST_BIAS) __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (x * (uint)N0 * sizeof(DATA_TYPE)); - LOAD_BLOCK_BOUNDARY_AWARE(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, 1, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(1, DATA_TYPE, bias, BETA); @@ -3883,7 +3880,7 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt_texture(IMAGE_DECLARATION(lhs), #else // defined(BROADCAST_BIAS) __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (x * (uint)N0 * sizeof(DATA_TYPE)) + (y * (uint)M0 * bias_stride_y) + z * bias_stride_z; - LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, PARTIAL_STORE_M0, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(M0, DATA_TYPE, bias, BETA); @@ -3907,6 +3904,9 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt_texture(IMAGE_DECLARATION(lhs), #endif // defined(MIXED_PRECISION) #endif // defined(ACTIVATION_TYPE) + const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); + const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); + // Store output block #if defined(MIXED_PRECISION) CONVERT_BLOCK(M0, N0, DATA_TYPE, c, c_lp); diff --git a/src/core/CL/cl_kernels/gemm_helpers.h b/src/core/CL/cl_kernels/gemm_helpers.h index e4ed47e7e6..3bbd243ff5 100644 --- a/src/core/CL/cl_kernels/gemm_helpers.h +++ b/src/core/CL/cl_kernels/gemm_helpers.h @@ -398,306 +398,6 @@ #define LOAD_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) /** @} */ // end of group LOAD_BLOCK -/** Partially load the 0 to (n-1)th rows of the given variables - * @name LOAD_ROW_PARTIAL_n - * Within each row, load the lower @p LOAD_N0 elements of vectors of width @p N0 - * - * @note in case @p LOAD_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty. - * - * @param[in] N0 The width of the passed in vector. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] LOAD_N0 The **lower** size of the vectors to load. Supported: [1-16 and <= @p N0 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] OFFSET The offset within a row - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @{ - */ -#define LOAD_ROW_PARTIAL_1(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + OFFSET + 0 * STRIDE_Y + Z##0)); - -#define LOAD_ROW_PARTIAL_2(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_1(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + OFFSET + 1 * STRIDE_Y + Z##1)); - -#define LOAD_ROW_PARTIAL_3(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_2(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + OFFSET + 2 * STRIDE_Y + Z##2)); - -#define LOAD_ROW_PARTIAL_4(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_3(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + OFFSET + 3 * STRIDE_Y + Z##3)); - -#define LOAD_ROW_PARTIAL_5(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_4(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + OFFSET + 4 * STRIDE_Y + Z##4)); - -#define LOAD_ROW_PARTIAL_6(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_5(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + OFFSET + 5 * STRIDE_Y + Z##5)); - -#define LOAD_ROW_PARTIAL_7(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_6(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + OFFSET + 6 * STRIDE_Y + Z##6)); - -#define LOAD_ROW_PARTIAL_8(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_7(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + OFFSET + 7 * STRIDE_Y + Z##7)); - -#define LOAD_ROW_PARTIAL_9(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_8(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + OFFSET + 8 * STRIDE_Y + Z##8)); - -#define LOAD_ROW_PARTIAL_10(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_9(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + OFFSET + 9 * STRIDE_Y + Z##9)); - -#define LOAD_ROW_PARTIAL_11(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_10(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + OFFSET + 10 * STRIDE_Y + Z##A)); - -#define LOAD_ROW_PARTIAL_12(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_11(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + OFFSET + 11 * STRIDE_Y + Z##B)); - -#define LOAD_ROW_PARTIAL_13(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_12(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + OFFSET + 12 * STRIDE_Y + Z##C)); - -#define LOAD_ROW_PARTIAL_14(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_13(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + OFFSET + 13 * STRIDE_Y + Z##D)); - -#define LOAD_ROW_PARTIAL_15(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_14(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + OFFSET + 14 * STRIDE_Y + Z##E)); - -#define LOAD_ROW_PARTIAL_16(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_15(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + OFFSET + 15 * STRIDE_Y + Z##F)); -/** @} */ // end of groupd LOAD_ROW_PARTIAL_n - -/** Partially load a block of the given size LOAD_M0xLOAD_N0 - * @name LOAD_BLOCK_PARTIAL - * - * @note The vector width @p N0 is also required for correct partial storing behaviour. - * @note in case @p LOAD_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty. - * - * The data to load is expected to have consecutive names for each row. - * E.g., for LOAD_M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for LOAD_M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * @param[in] LOAD_M0 The number of rows to load. Supported: 1-16 - * @param[in] LOAD_N0 The lower number of elements of vectors to load. Supported: 1-16 and <= @p N0 - * @param[in] N0 The size of each vector. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] OFFSET The offset within a row - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @{ - */ -#define LOAD_BLOCK_PARTIAL_STR(LOAD_M0, LOAD_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_ROW_PARTIAL_##LOAD_M0(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) -#define LOAD_BLOCK_PARTIAL(LOAD_M0, LOAD_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_BLOCK_PARTIAL_STR(LOAD_M0, LOAD_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) -/** Load a block that can be partial in both x and y dimensions - * - * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty. - * - * The data to load is expected to have consecutive names for each row. - * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * @param[in] M0 The number of rows to load, for non-partial blocks. Supported: 1-16 - * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] OFFSET The offset within a row - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0) - * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0) - * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial load Y. True to use PARTIAL_STORE_M0 rather than M0. - * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial load X. True to use PARTIAL_STORE_N0 rather than N0. - */ -#define LOAD_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \ - if(!(PARTIAL_COND_X) && !(PARTIAL_COND_Y)) \ - { \ - LOAD_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } \ - else if((PARTIAL_COND_Y) && !(PARTIAL_COND_X)) \ - { \ - LOAD_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } \ - else if(!(PARTIAL_COND_Y) && (PARTIAL_COND_X)) \ - { \ - LOAD_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } \ - else \ - { \ - LOAD_BLOCK_PARTIAL(PARTIAL_STORE_M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } -/** Load a block that can only be partial in x but not y. - * - * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty. - * - * The data to load is expected to have consecutive names for each row. - * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * @param[in] M0 The number of rows to load, for non-partial blocks. Supported: 1-16 - * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] OFFSET The offset within a row - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0) - * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial load X. True to use PARTIAL_STORE_N0 rather than N0. - */ -#define LOAD_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_N0, PARTIAL_COND_X) \ - if(!(PARTIAL_COND_X)) \ - { \ - LOAD_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } \ - else \ - { \ - LOAD_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } -/** Load a block that can only be partial in y but not x. - * - * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty. - * - * The data to store is expected to have consecutive names for each row. - * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16 - * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] OFFSET The offset within a row - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0) - * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0. - */ -#define LOAD_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y) \ - if(!(PARTIAL_COND_Y)) \ - { \ - LOAD_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } \ - else \ - { \ - LOAD_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } -/** @} */ // end of group LOAD_BLOCK_PARTIAL -/** Boundary-aware GeMM block load - * @name LOAD_BLOCK_BOUNDARY_AWARE - * This macro assumes the following schemes to achieve boundary-awareness: - * - Overlapping load in Y axis from lhs tensor. This implies lhs has no padding along y dim. - * - Non-Overlapping(normal) load from rhs tensor. This imples rhs can have paddings. - * - Overlapping load in Y axis from bias tensor. This implies rhs has no padding along y dim. - * The macro then ensures that the src tensor can be loaded without any paddings in both x and y dim. - * - * In the y dimension, we place the partial blocks **at the beginning** while in the x dimension, we place the partial - * blocks **at the end**. - * Say, the src tensor is of shape MxN and we have M0 and N0 as the block size, this is how we define "partial blocks"/ - * "boundary block" (we use the 2 terms "partial blocks" and "boundary blocks" interchangeably) and its various parameters: - * - * *--x--> x == 0 x == 1 - * | |<------------------------------N-------------------------->| - * y |<--------------N0------------->|<----PARTIAL_STORE_N0----->| - * | -------------############################################################# - * * | | |...............................|...........................| - * y == 0 | PAR_..._M0 |......Boundary block in y......|.Boundary block in x and y.| - * | | |...............................|...........................| - * M --############################################################# - * | | | |...........................| - * y == 1 | M0 | Non-boundary block |....Boundary block in x....| - * | | | |...........................| - * |------------############################################################# - * - * Then @p PARTIAL_STORE_M0 = M % M0 and @p PARTIAL_STORE_N0 = N % N0 - * - * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty. - * - * It automatically detects if a giving M,N,M0,N0 combination can yield partial blocks in either X and Y dimension, - * and select corresponding load methods such that the boundary detection logic is only added when needed. - * - * The data to load is expected to have consecutive names for each row. - * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * The macro will result in a declaration of @p M0 vectors of size @p N0 with data - * type @p DATA_TYPE containing values partially loaded from the specified - * address in memory. The remaining (N0 - PARTIAL_STORE_N0) elements will be - * filled with zeros. - * - * @param[in] M0 The number of rows to load, for non-partial blocks. Supported: 1-16 - * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] OFFSET The offset within a row - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0) - * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported: [0, @p N0) - * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial load Y. True to use PARTIAL_STORE_M0 rather than M0. - * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial load X. True to use PARTIAL_STORE_N0 rather than N0. - * @{ - */ -#if PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0 -// Case1: No partial blocks in either x or y -#define LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \ - LOAD_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) - -#elif PARTIAL_STORE_M0 > 0 && PARTIAL_STORE_N0 == 0 -// Case2: Partial blocks in y -#define LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \ - REPEAT_VAR_INIT_TO_CONST(M0, VEC_DATA_TYPE(DATA_TYPE, N0), BASENAME, 0); \ - LOAD_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y) - -#elif PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 > 0 -// Case3: Partial blocks in x -#define LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \ - REPEAT_VAR_INIT_TO_CONST(M0, VEC_DATA_TYPE(DATA_TYPE, N0), BASENAME, 0); \ - LOAD_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_N0, PARTIAL_COND_X) - -#else // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0 -// Case4: Partial blocks in both x and y -#define LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \ - REPEAT_VAR_INIT_TO_CONST(M0, VEC_DATA_TYPE(DATA_TYPE, N0), BASENAME, 0); \ - LOAD_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) - -#endif // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0 - /** Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1). * @name LOAD_TEXTURE2D_ROW_n * diff --git a/src/core/CL/cl_kernels/helpers.h b/src/core/CL/cl_kernels/helpers.h index fae15b2347..6cd76373d2 100644 --- a/src/core/CL/cl_kernels/helpers.h +++ b/src/core/CL/cl_kernels/helpers.h @@ -202,214 +202,6 @@ #define VLOAD_STR(size) vload##size #define VLOAD(size) VLOAD_STR(size) -/** Extended partial vload that correctly handles scalar values as well. - * Load the **lower** 0 to (n-1)th elements of the given vector while minimising the amount of load ops - * @name VLOAD_PARTIAL - * - * @note With this macro, the passed data can be both a vector and a scalar - * @note @p load_size needs to be <= @p size - * eg 1: Valid - * VLOAD_PARTIAL(16, 15) ...; - * eg 2: Invalid - * VLOAD_PARTIAL(4, 7) ...; - * - * @param[in] size The width of @p DATA. Supported values: 1(scalar), 2, 3, 4, 8, 16 - * @param[in] load_size The number of lower elements to load. Supported values: 1-16, but has to be <= @p size - * @{ - */ -#define VLOAD_PARTIAL_STR(size, load_size) vload_partial_##size##_##load_size -#define VLOAD_PARTIAL(size, load_size) VLOAD_PARTIAL_STR(size, load_size) - -#define NO_LOAD(data, offs, ptr) \ - { \ - } - -// Size == 1 (scalar) -#define vload_partial_1_0 NO_LOAD -#define vload_partial_1_1 vload1 -#define vload_partial_1_2 NO_LOAD -#define vload_partial_1_3 NO_LOAD -#define vload_partial_1_4 NO_LOAD -#define vload_partial_1_5 NO_LOAD -#define vload_partial_1_6 NO_LOAD -#define vload_partial_1_7 NO_LOAD -#define vload_partial_1_8 NO_LOAD -#define vload_partial_1_9 NO_LOAD -#define vload_partial_1_10 NO_LOAD -#define vload_partial_1_11 NO_LOAD -#define vload_partial_1_12 NO_LOAD -#define vload_partial_1_13 NO_LOAD -#define vload_partial_1_14 NO_LOAD -#define vload_partial_1_15 NO_LOAD -#define vload_partial_1_16 NO_LOAD -// Size == 2 -#define vload_partial_2_0 NO_LOAD -#define vload_partial_2_1 vload_partial_1 -#define vload_partial_2_2 vload_partial_2 -#define vload_partial_2_3 NO_LOAD -#define vload_partial_2_4 NO_LOAD -#define vload_partial_2_5 NO_LOAD -#define vload_partial_2_6 NO_LOAD -#define vload_partial_2_7 NO_LOAD -#define vload_partial_2_8 NO_LOAD -#define vload_partial_2_9 NO_LOAD -#define vload_partial_2_10 NO_LOAD -#define vload_partial_2_11 NO_LOAD -#define vload_partial_2_12 NO_LOAD -#define vload_partial_2_13 NO_LOAD -#define vload_partial_2_14 NO_LOAD -#define vload_partial_2_15 NO_LOAD -#define vload_partial_2_16 NO_LOAD -// Size == 3 -#define vload_partial_3_0 NO_LOAD -#define vload_partial_3_1 vload_partial_1 -#define vload_partial_3_2 vload_partial_2 -#define vload_partial_3_3 vload_partial_3 -#define vload_partial_3_4 NO_LOAD -#define vload_partial_3_5 NO_LOAD -#define vload_partial_3_6 NO_LOAD -#define vload_partial_3_7 NO_LOAD -#define vload_partial_3_8 NO_LOAD -#define vload_partial_3_9 NO_LOAD -#define vload_partial_3_10 NO_LOAD -#define vload_partial_3_11 NO_LOAD -#define vload_partial_3_12 NO_LOAD -#define vload_partial_3_13 NO_LOAD -#define vload_partial_3_14 NO_LOAD -#define vload_partial_3_15 NO_LOAD -#define vload_partial_3_16 NO_LOAD -// Size == 4 -#define vload_partial_4_0 NO_LOAD -#define vload_partial_4_1 vload_partial_1 -#define vload_partial_4_2 vload_partial_2 -#define vload_partial_4_3 vload_partial_3 -#define vload_partial_4_4 vload_partial_4 -#define vload_partial_4_5 NO_LOAD -#define vload_partial_4_6 NO_LOAD -#define vload_partial_4_7 NO_LOAD -#define vload_partial_4_8 NO_LOAD -#define vload_partial_4_9 NO_LOAD -#define vload_partial_4_10 NO_LOAD -#define vload_partial_4_11 NO_LOAD -#define vload_partial_4_12 NO_LOAD -#define vload_partial_4_13 NO_LOAD -#define vload_partial_4_14 NO_LOAD -#define vload_partial_4_15 NO_LOAD -#define vload_partial_4_16 NO_LOAD -// Size == 8 -#define vload_partial_8_0 NO_LOAD -#define vload_partial_8_1 vload_partial_1 -#define vload_partial_8_2 vload_partial_2 -#define vload_partial_8_3 vload_partial_3 -#define vload_partial_8_4 vload_partial_4 -#define vload_partial_8_5 vload_partial_5 -#define vload_partial_8_6 vload_partial_6 -#define vload_partial_8_7 vload_partial_7 -#define vload_partial_8_8 vload_partial_8 -#define vload_partial_8_9 NO_LOAD -#define vload_partial_8_10 NO_LOAD -#define vload_partial_8_11 NO_LOAD -#define vload_partial_8_12 NO_LOAD -#define vload_partial_8_13 NO_LOAD -#define vload_partial_8_14 NO_LOAD -#define vload_partial_8_15 NO_LOAD -#define vload_partial_8_16 NO_LOAD -// Size == 16 -#define vload_partial_16_0 NO_LOAD -#define vload_partial_16_1 vload_partial_1 -#define vload_partial_16_2 vload_partial_2 -#define vload_partial_16_3 vload_partial_3 -#define vload_partial_16_4 vload_partial_4 -#define vload_partial_16_5 vload_partial_5 -#define vload_partial_16_6 vload_partial_6 -#define vload_partial_16_7 vload_partial_7 -#define vload_partial_16_8 vload_partial_8 -#define vload_partial_16_9 vload_partial_9 -#define vload_partial_16_10 vload_partial_10 -#define vload_partial_16_11 vload_partial_11 -#define vload_partial_16_12 vload_partial_12 -#define vload_partial_16_13 vload_partial_13 -#define vload_partial_16_14 vload_partial_14 -#define vload_partial_16_15 vload_partial_15 -#define vload_partial_16_16 vload_partial_16 - -/** Partial vload. Load the **lower** 0 to (n-1)th elements of the given vector while minimising the amount of vload ops - * @name vload_partial_n - * - * @note @p DATA needs to be a vector not a scalar - * @note n needs to be <= the vector width of the input variable @p DATA - * eg 1: Valid - * vload_partial_15(var:float16, 0, 0xabcd); - * eg 2: Invalid - * vload_partial_7(var:float4, 0, 0xabcd); - * - * @note in cases n == 1, 2, 3, 4, 8, 16, no extra vload is invoked, thus there's no performance penalty. - * - * @param[in] DATA The name of the variable where to load the values - * @param[in] OFFSET Offset in n - * @param[in] PTR The base pointer - * @{ - */ -#define vload_partial_1(DATA, OFFSET, PTR) \ - DATA.s0 = vload1(OFFSET, PTR); - -#define vload_partial_2(DATA, OFFSET, PTR) \ - DATA.s01 = vload2(OFFSET, PTR); - -#define vload_partial_3(DATA, OFFSET, PTR) \ - DATA.s012 = vload3(OFFSET, PTR); - -#define vload_partial_4(DATA, OFFSET, PTR) \ - DATA.s0123 = vload4(OFFSET, PTR); - -#define vload_partial_5(DATA, OFFSET, PTR) \ - DATA.s0123 = vload_partial_4(DATA, OFFSET, PTR); \ - DATA.s4 = vload1(OFFSET, PTR + 4); - -#define vload_partial_6(DATA, OFFSET, PTR) \ - DATA.s0123 = vload_partial_4(DATA, OFFSET, PTR); \ - DATA.s45 = vload_partial_2(DATA, OFFSET, PTR + 4); - -#define vload_partial_7(DATA, OFFSET, PTR) \ - DATA.s0123 = vload_partial_4(DATA, OFFSET, PTR); \ - DATA.s456 = vload_partial_3(DATA, OFFSET, PTR + 4); - -#define vload_partial_8(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload8(OFFSET, PTR); - -#define vload_partial_9(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload_partial_8(DATA, OFFSET, PTR); \ - DATA.s8 = vload1(OFFSET, PTR + 8); - -#define vload_partial_10(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload_partial_8(DATA, OFFSET, PTR); \ - DATA.s89 = vload_partial_2(DATA, OFFSET, PTR + 8); - -#define vload_partial_11(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload_partial_8(DATA, OFFSET, PTR); \ - DATA.s89A = vload_partial_3(DATA, OFFSET, PTR + 8); - -#define vload_partial_12(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload_partial_8(DATA, OFFSET, PTR); \ - DATA.s89AB = vload_partial_4(DATA, OFFSET, PTR + 8); - -#define vload_partial_13(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload_partial_8(DATA, OFFSET, PTR); \ - DATA.s89ABC = vload_partial_5(DATA, OFFSET, PTR + 8); - -#define vload_partial_14(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload_partial_8(DATA, OFFSET, PTR); \ - DATA.s89ABCD = vload_partial_6(DATA, OFFSET, PTR + 8); - -#define vload_partial_15(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload_partial_8(DATA, OFFSET, PTR); \ - DATA.s89ABCDE = vload_partial_7(DATA, OFFSET, PTR + 8); - -#define vload_partial_16(DATA, OFFSET, PTR) \ - DATA = vload16(OFFSET, PTR); -/** @} */ // end of groupd vload_partial_n -/** @} */ // end of groupd VLOAD_PARTIAL - #define PIXEL_UNIT4 1 #define PIXEL_UNIT8 2 #define PIXEL_UNIT16 4 |