From 4e53c5ab47a713ab0ce53d076e2e4cf274fec312 Mon Sep 17 00:00:00 2001 From: Pablo Marquez Tello Date: Mon, 6 Sep 2021 13:14:26 +0100 Subject: Revert "Remove padding from ClGemmMatrixMultiplyReshapedKernel" This reverts commit 50335fd3d0734157382741fcf1bfdaf630c60c4b. Resolves COMPMID-4792 Signed-off-by: Pablo Marquez Tello Change-Id: Ia6580143d9cf5a7bd5c87ca4214022f7c241ec6f Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/6214 Tested-by: Arm Jenkins Reviewed-by: Giorgio Arena Reviewed-by: Sheri Zhang Comments-Addressed: Arm Jenkins --- src/core/CL/cl_kernels/common/gemm.cl | 40 +-- src/core/CL/cl_kernels/gemm_helpers.h | 300 --------------------- src/core/CL/cl_kernels/helpers.h | 208 -------------- .../kernels/ClGemmMatrixMultiplyReshapedKernel.cpp | 59 ++-- .../kernels/ClGemmMatrixMultiplyReshapedKernel.h | 2 +- 5 files changed, 64 insertions(+), 545 deletions(-) diff --git a/src/core/CL/cl_kernels/common/gemm.cl b/src/core/CL/cl_kernels/common/gemm.cl index ff153af542..10435d376f 100644 --- a/src/core/CL/cl_kernels/common/gemm.cl +++ b/src/core/CL/cl_kernels/common/gemm.cl @@ -2705,9 +2705,6 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t(IMAGE_DECLARATION(lhs), REPEAT_VAR_INIT_TO_CONST(M0, uint, zout, 0); - const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); - const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); - #if defined(REINTERPRET_OUTPUT_AS_3D) // The plane (zin) is calculated dividing M (y * M0) by HEIGHT_GEMM3D @@ -2733,7 +2730,7 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t(IMAGE_DECLARATION(lhs), #if defined(BROADCAST_BIAS) __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (get_global_id(0) * (uint)N0 * sizeof(DATA_TYPE)); - LOAD_BLOCK_BOUNDARY_AWARE(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, 1, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(1, DATA_TYPE, bias, BETA); @@ -2751,7 +2748,7 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t(IMAGE_DECLARATION(lhs), __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (get_global_id(0) * (uint)N0 * sizeof(DATA_TYPE)) + (get_global_id(1) * (uint)M0 * bias_stride_y) + get_global_id( 2) * bias_stride_z; - LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, PARTIAL_STORE_M0, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(M0, DATA_TYPE, bias, BETA); @@ -2776,6 +2773,9 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t(IMAGE_DECLARATION(lhs), #endif // defined(MIXED_PRECISION) #endif // defined(ACTIVATION_TYPE) + const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); + const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); + // Store output block #if defined(MIXED_PRECISION) CONVERT_BLOCK(M0, N0, DATA_TYPE, c, c_lp); @@ -2975,9 +2975,6 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t_texture(IMAGE_DECLARATION(lhs), REPEAT_VAR_INIT_TO_CONST(M0, uint, zout, 0); - const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); - const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); - #if defined(REINTERPRET_OUTPUT_AS_3D) // The plane (zin) is calculated dividing M (y * M0) by HEIGHT_GEMM3D @@ -3003,7 +3000,7 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t_texture(IMAGE_DECLARATION(lhs), #if defined(BROADCAST_BIAS) __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (get_global_id(0) * (uint)N0 * sizeof(DATA_TYPE)); - LOAD_BLOCK_BOUNDARY_AWARE(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, 1, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(1, DATA_TYPE, bias, BETA); @@ -3021,7 +3018,7 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t_texture(IMAGE_DECLARATION(lhs), __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (get_global_id(0) * (uint)N0 * sizeof(DATA_TYPE)) + (get_global_id(1) * (uint)M0 * bias_stride_y) + get_global_id( 2) * bias_stride_z; - LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, PARTIAL_STORE_M0, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(M0, DATA_TYPE, bias, BETA); @@ -3046,6 +3043,9 @@ __kernel void gemm_mm_reshaped_lhs_nt_rhs_t_texture(IMAGE_DECLARATION(lhs), #endif // defined(MIXED_PRECISION) #endif // defined(ACTIVATION_TYPE) + const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); + const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); + // Store output block #if defined(MIXED_PRECISION) CONVERT_BLOCK(M0, N0, DATA_TYPE, c, c_lp); @@ -3284,9 +3284,6 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt(IMAGE_DECLARATION(lhs), const uint y = get_global_id(1); const uint z = get_global_id(2); - const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); - const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); - #if defined(DUMMY_WORK_ITEMS) if((x * N0 >= N) || (y * M0 >= M)) { @@ -3498,7 +3495,7 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt(IMAGE_DECLARATION(lhs), #if defined(BROADCAST_BIAS) __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (x * (uint)N0 * sizeof(DATA_TYPE)); - LOAD_BLOCK_BOUNDARY_AWARE(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, 1, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(1, DATA_TYPE, bias, BETA); @@ -3516,7 +3513,7 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt(IMAGE_DECLARATION(lhs), __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (get_global_id(0) * (uint)N0 * sizeof(DATA_TYPE)) + (get_global_id(1) * (uint)M0 * bias_stride_y) + get_global_id( 2) * bias_stride_z; - LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, PARTIAL_STORE_M0, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(M0, DATA_TYPE, bias, BETA); @@ -3540,6 +3537,9 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt(IMAGE_DECLARATION(lhs), #endif // defined(MIXED_PRECISION) #endif // defined(ACTIVATION_TYPE) + const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); + const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); + // Store output block #if defined(MIXED_PRECISION) CONVERT_BLOCK(M0, N0, DATA_TYPE, c, c_lp); @@ -3838,9 +3838,6 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt_texture(IMAGE_DECLARATION(lhs), REPEAT_VAR_INIT_TO_CONST(M0, uint, zout, 0); - const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); - const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); - #if defined(REINTERPRET_OUTPUT_AS_3D) // The plane (zin) is calculated dividing M (y * M0) by HEIGHT_GEMM3D @@ -3866,7 +3863,7 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt_texture(IMAGE_DECLARATION(lhs), #if defined(BROADCAST_BIAS) __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (x * (uint)N0 * sizeof(DATA_TYPE)); - LOAD_BLOCK_BOUNDARY_AWARE(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, 1, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(1, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(1, DATA_TYPE, bias, BETA); @@ -3883,7 +3880,7 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt_texture(IMAGE_DECLARATION(lhs), #else // defined(BROADCAST_BIAS) __global uchar *bias_addr = bias_ptr + bias_offset_first_element_in_bytes + (x * (uint)N0 * sizeof(DATA_TYPE)) + (y * (uint)M0 * bias_stride_y) + z * bias_stride_z; - LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero, PARTIAL_STORE_M0, PARTIAL_STORE_N0, cond_y, cond_x); + LOAD_BLOCK(M0, N0, DATA_TYPE, bias, bias_addr, 0, bias_stride_y, zero); #ifndef UNIT_BETA SCALE_BLOCK(M0, DATA_TYPE, bias, BETA); @@ -3907,6 +3904,9 @@ __kernel void gemm_mm_reshaped_lhs_t_rhs_nt_texture(IMAGE_DECLARATION(lhs), #endif // defined(MIXED_PRECISION) #endif // defined(ACTIVATION_TYPE) + const bool cond_y = ((get_global_id(1) + 1) * M0 >= M); + const bool cond_x = ((get_global_id(0) + 1) * N0 >= N); + // Store output block #if defined(MIXED_PRECISION) CONVERT_BLOCK(M0, N0, DATA_TYPE, c, c_lp); diff --git a/src/core/CL/cl_kernels/gemm_helpers.h b/src/core/CL/cl_kernels/gemm_helpers.h index e4ed47e7e6..3bbd243ff5 100644 --- a/src/core/CL/cl_kernels/gemm_helpers.h +++ b/src/core/CL/cl_kernels/gemm_helpers.h @@ -398,306 +398,6 @@ #define LOAD_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_BLOCK_STR(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) /** @} */ // end of group LOAD_BLOCK -/** Partially load the 0 to (n-1)th rows of the given variables - * @name LOAD_ROW_PARTIAL_n - * Within each row, load the lower @p LOAD_N0 elements of vectors of width @p N0 - * - * @note in case @p LOAD_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty. - * - * @param[in] N0 The width of the passed in vector. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] LOAD_N0 The **lower** size of the vectors to load. Supported: [1-16 and <= @p N0 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] OFFSET The offset within a row - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @{ - */ -#define LOAD_ROW_PARTIAL_1(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##0, 0, (__global DATA_TYPE *)(PTR + OFFSET + 0 * STRIDE_Y + Z##0)); - -#define LOAD_ROW_PARTIAL_2(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_1(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##1, 0, (__global DATA_TYPE *)(PTR + OFFSET + 1 * STRIDE_Y + Z##1)); - -#define LOAD_ROW_PARTIAL_3(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_2(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##2, 0, (__global DATA_TYPE *)(PTR + OFFSET + 2 * STRIDE_Y + Z##2)); - -#define LOAD_ROW_PARTIAL_4(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_3(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##3, 0, (__global DATA_TYPE *)(PTR + OFFSET + 3 * STRIDE_Y + Z##3)); - -#define LOAD_ROW_PARTIAL_5(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_4(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##4, 0, (__global DATA_TYPE *)(PTR + OFFSET + 4 * STRIDE_Y + Z##4)); - -#define LOAD_ROW_PARTIAL_6(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_5(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##5, 0, (__global DATA_TYPE *)(PTR + OFFSET + 5 * STRIDE_Y + Z##5)); - -#define LOAD_ROW_PARTIAL_7(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_6(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##6, 0, (__global DATA_TYPE *)(PTR + OFFSET + 6 * STRIDE_Y + Z##6)); - -#define LOAD_ROW_PARTIAL_8(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_7(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##7, 0, (__global DATA_TYPE *)(PTR + OFFSET + 7 * STRIDE_Y + Z##7)); - -#define LOAD_ROW_PARTIAL_9(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_8(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##8, 0, (__global DATA_TYPE *)(PTR + OFFSET + 8 * STRIDE_Y + Z##8)); - -#define LOAD_ROW_PARTIAL_10(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_9(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##9, 0, (__global DATA_TYPE *)(PTR + OFFSET + 9 * STRIDE_Y + Z##9)); - -#define LOAD_ROW_PARTIAL_11(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_10(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##A, 0, (__global DATA_TYPE *)(PTR + OFFSET + 10 * STRIDE_Y + Z##A)); - -#define LOAD_ROW_PARTIAL_12(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_11(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##B, 0, (__global DATA_TYPE *)(PTR + OFFSET + 11 * STRIDE_Y + Z##B)); - -#define LOAD_ROW_PARTIAL_13(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_12(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##C, 0, (__global DATA_TYPE *)(PTR + OFFSET + 12 * STRIDE_Y + Z##C)); - -#define LOAD_ROW_PARTIAL_14(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_13(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##D, 0, (__global DATA_TYPE *)(PTR + OFFSET + 13 * STRIDE_Y + Z##D)); - -#define LOAD_ROW_PARTIAL_15(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_14(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##E, 0, (__global DATA_TYPE *)(PTR + OFFSET + 14 * STRIDE_Y + Z##E)); - -#define LOAD_ROW_PARTIAL_16(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - LOAD_ROW_PARTIAL_15(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) \ - VLOAD_PARTIAL(N0, LOAD_N0) \ - (BASENAME##F, 0, (__global DATA_TYPE *)(PTR + OFFSET + 15 * STRIDE_Y + Z##F)); -/** @} */ // end of groupd LOAD_ROW_PARTIAL_n - -/** Partially load a block of the given size LOAD_M0xLOAD_N0 - * @name LOAD_BLOCK_PARTIAL - * - * @note The vector width @p N0 is also required for correct partial storing behaviour. - * @note in case @p LOAD_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty. - * - * The data to load is expected to have consecutive names for each row. - * E.g., for LOAD_M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for LOAD_M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * @param[in] LOAD_M0 The number of rows to load. Supported: 1-16 - * @param[in] LOAD_N0 The lower number of elements of vectors to load. Supported: 1-16 and <= @p N0 - * @param[in] N0 The size of each vector. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] OFFSET The offset within a row - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @{ - */ -#define LOAD_BLOCK_PARTIAL_STR(LOAD_M0, LOAD_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_ROW_PARTIAL_##LOAD_M0(N0, LOAD_N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) -#define LOAD_BLOCK_PARTIAL(LOAD_M0, LOAD_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) LOAD_BLOCK_PARTIAL_STR(LOAD_M0, LOAD_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) -/** Load a block that can be partial in both x and y dimensions - * - * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty. - * - * The data to load is expected to have consecutive names for each row. - * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * @param[in] M0 The number of rows to load, for non-partial blocks. Supported: 1-16 - * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] OFFSET The offset within a row - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0) - * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0) - * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial load Y. True to use PARTIAL_STORE_M0 rather than M0. - * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial load X. True to use PARTIAL_STORE_N0 rather than N0. - */ -#define LOAD_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \ - if(!(PARTIAL_COND_X) && !(PARTIAL_COND_Y)) \ - { \ - LOAD_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } \ - else if((PARTIAL_COND_Y) && !(PARTIAL_COND_X)) \ - { \ - LOAD_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } \ - else if(!(PARTIAL_COND_Y) && (PARTIAL_COND_X)) \ - { \ - LOAD_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } \ - else \ - { \ - LOAD_BLOCK_PARTIAL(PARTIAL_STORE_M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } -/** Load a block that can only be partial in x but not y. - * - * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty. - * - * The data to load is expected to have consecutive names for each row. - * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * @param[in] M0 The number of rows to load, for non-partial blocks. Supported: 1-16 - * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] OFFSET The offset within a row - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported range: [1, @p N0) - * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial load X. True to use PARTIAL_STORE_N0 rather than N0. - */ -#define LOAD_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_N0, PARTIAL_COND_X) \ - if(!(PARTIAL_COND_X)) \ - { \ - LOAD_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } \ - else \ - { \ - LOAD_BLOCK_PARTIAL(M0, PARTIAL_STORE_N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } -/** Load a block that can only be partial in y but not x. - * - * @note in case @p N0 or @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty. - * - * The data to store is expected to have consecutive names for each row. - * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * @param[in] M0 The number of rows to store, for non-partial blocks. Supported: 1-16 - * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] OFFSET The offset within a row - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported range: [1, @p M0) - * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial store Y. True to use PARTIAL_STORE_M0 rather than M0. - */ -#define LOAD_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y) \ - if(!(PARTIAL_COND_Y)) \ - { \ - LOAD_BLOCK_PARTIAL(M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } \ - else \ - { \ - LOAD_BLOCK_PARTIAL(PARTIAL_STORE_M0, N0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z); \ - } -/** @} */ // end of group LOAD_BLOCK_PARTIAL -/** Boundary-aware GeMM block load - * @name LOAD_BLOCK_BOUNDARY_AWARE - * This macro assumes the following schemes to achieve boundary-awareness: - * - Overlapping load in Y axis from lhs tensor. This implies lhs has no padding along y dim. - * - Non-Overlapping(normal) load from rhs tensor. This imples rhs can have paddings. - * - Overlapping load in Y axis from bias tensor. This implies rhs has no padding along y dim. - * The macro then ensures that the src tensor can be loaded without any paddings in both x and y dim. - * - * In the y dimension, we place the partial blocks **at the beginning** while in the x dimension, we place the partial - * blocks **at the end**. - * Say, the src tensor is of shape MxN and we have M0 and N0 as the block size, this is how we define "partial blocks"/ - * "boundary block" (we use the 2 terms "partial blocks" and "boundary blocks" interchangeably) and its various parameters: - * - * *--x--> x == 0 x == 1 - * | |<------------------------------N-------------------------->| - * y |<--------------N0------------->|<----PARTIAL_STORE_N0----->| - * | -------------############################################################# - * * | | |...............................|...........................| - * y == 0 | PAR_..._M0 |......Boundary block in y......|.Boundary block in x and y.| - * | | |...............................|...........................| - * M --############################################################# - * | | | |...........................| - * y == 1 | M0 | Non-boundary block |....Boundary block in x....| - * | | | |...........................| - * |------------############################################################# - * - * Then @p PARTIAL_STORE_M0 = M % M0 and @p PARTIAL_STORE_N0 = N % N0 - * - * @note in cases @p PARTIAL_STORE_N0 != 1, 2, 3, 4, 8, 16, extra vload(s) will be invoked, thus incurring small performance penalty. - * - * It automatically detects if a giving M,N,M0,N0 combination can yield partial blocks in either X and Y dimension, - * and select corresponding load methods such that the boundary detection logic is only added when needed. - * - * The data to load is expected to have consecutive names for each row. - * E.g., for M0=3 and basename=c, the expected names are c0, c1 and c2. - * The Z offset is expected to have consecutive names. - * E.g., for M0=3 and Z=zin, the expected z offset names are zin0, zin1 and zin2. - * - * The macro will result in a declaration of @p M0 vectors of size @p N0 with data - * type @p DATA_TYPE containing values partially loaded from the specified - * address in memory. The remaining (N0 - PARTIAL_STORE_N0) elements will be - * filled with zeros. - * - * @param[in] M0 The number of rows to load, for non-partial blocks. Supported: 1-16 - * @param[in] N0 The size of each vector, for non-partial blocks. Supported: 1, 2, 3, 4, 8, 16 - * @param[in] DATA_TYPE The data type of the vectors - * @param[in] BASENAME The basename of the variables - * @param[in] PTR The base pointer - * @param[in] OFFSET The offset within a row - * @param[in] STRIDE_Y The stride value in y-axis direction - * @param[in] Z The offset in z-axis direction - * @param[in] PARTIAL_STORE_M0 The partial size in y, for partial blocks. Supported: [0, @p M0) - * @param[in] PARTIAL_STORE_N0 The partial size in x, for partial blocks. Supported: [0, @p N0) - * @param[in] PARTIAL_COND_Y Condition on the y axis to perform the partial load Y. True to use PARTIAL_STORE_M0 rather than M0. - * @param[in] PARTIAL_COND_X Condition on the x axis to perform the partial load X. True to use PARTIAL_STORE_N0 rather than N0. - * @{ - */ -#if PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0 -// Case1: No partial blocks in either x or y -#define LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \ - LOAD_BLOCK(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z) - -#elif PARTIAL_STORE_M0 > 0 && PARTIAL_STORE_N0 == 0 -// Case2: Partial blocks in y -#define LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \ - REPEAT_VAR_INIT_TO_CONST(M0, VEC_DATA_TYPE(DATA_TYPE, N0), BASENAME, 0); \ - LOAD_BLOCK_PARTIAL_IN_Y(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_COND_Y) - -#elif PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 > 0 -// Case3: Partial blocks in x -#define LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \ - REPEAT_VAR_INIT_TO_CONST(M0, VEC_DATA_TYPE(DATA_TYPE, N0), BASENAME, 0); \ - LOAD_BLOCK_PARTIAL_IN_X(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_N0, PARTIAL_COND_X) - -#else // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0 -// Case4: Partial blocks in both x and y -#define LOAD_BLOCK_BOUNDARY_AWARE(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) \ - REPEAT_VAR_INIT_TO_CONST(M0, VEC_DATA_TYPE(DATA_TYPE, N0), BASENAME, 0); \ - LOAD_BLOCK_PARTIAL_IN_X_AND_Y(M0, N0, DATA_TYPE, BASENAME, PTR, OFFSET, STRIDE_Y, Z, PARTIAL_STORE_M0, PARTIAL_STORE_N0, PARTIAL_COND_Y, PARTIAL_COND_X) - -#endif // PARTIAL_STORE_M0 == 0 && PARTIAL_STORE_N0 == 0 - /** Loads the rows from 0 to n-1 in the given variables (BASENAME0 to BASENAMEn-1). * @name LOAD_TEXTURE2D_ROW_n * diff --git a/src/core/CL/cl_kernels/helpers.h b/src/core/CL/cl_kernels/helpers.h index fae15b2347..6cd76373d2 100644 --- a/src/core/CL/cl_kernels/helpers.h +++ b/src/core/CL/cl_kernels/helpers.h @@ -202,214 +202,6 @@ #define VLOAD_STR(size) vload##size #define VLOAD(size) VLOAD_STR(size) -/** Extended partial vload that correctly handles scalar values as well. - * Load the **lower** 0 to (n-1)th elements of the given vector while minimising the amount of load ops - * @name VLOAD_PARTIAL - * - * @note With this macro, the passed data can be both a vector and a scalar - * @note @p load_size needs to be <= @p size - * eg 1: Valid - * VLOAD_PARTIAL(16, 15) ...; - * eg 2: Invalid - * VLOAD_PARTIAL(4, 7) ...; - * - * @param[in] size The width of @p DATA. Supported values: 1(scalar), 2, 3, 4, 8, 16 - * @param[in] load_size The number of lower elements to load. Supported values: 1-16, but has to be <= @p size - * @{ - */ -#define VLOAD_PARTIAL_STR(size, load_size) vload_partial_##size##_##load_size -#define VLOAD_PARTIAL(size, load_size) VLOAD_PARTIAL_STR(size, load_size) - -#define NO_LOAD(data, offs, ptr) \ - { \ - } - -// Size == 1 (scalar) -#define vload_partial_1_0 NO_LOAD -#define vload_partial_1_1 vload1 -#define vload_partial_1_2 NO_LOAD -#define vload_partial_1_3 NO_LOAD -#define vload_partial_1_4 NO_LOAD -#define vload_partial_1_5 NO_LOAD -#define vload_partial_1_6 NO_LOAD -#define vload_partial_1_7 NO_LOAD -#define vload_partial_1_8 NO_LOAD -#define vload_partial_1_9 NO_LOAD -#define vload_partial_1_10 NO_LOAD -#define vload_partial_1_11 NO_LOAD -#define vload_partial_1_12 NO_LOAD -#define vload_partial_1_13 NO_LOAD -#define vload_partial_1_14 NO_LOAD -#define vload_partial_1_15 NO_LOAD -#define vload_partial_1_16 NO_LOAD -// Size == 2 -#define vload_partial_2_0 NO_LOAD -#define vload_partial_2_1 vload_partial_1 -#define vload_partial_2_2 vload_partial_2 -#define vload_partial_2_3 NO_LOAD -#define vload_partial_2_4 NO_LOAD -#define vload_partial_2_5 NO_LOAD -#define vload_partial_2_6 NO_LOAD -#define vload_partial_2_7 NO_LOAD -#define vload_partial_2_8 NO_LOAD -#define vload_partial_2_9 NO_LOAD -#define vload_partial_2_10 NO_LOAD -#define vload_partial_2_11 NO_LOAD -#define vload_partial_2_12 NO_LOAD -#define vload_partial_2_13 NO_LOAD -#define vload_partial_2_14 NO_LOAD -#define vload_partial_2_15 NO_LOAD -#define vload_partial_2_16 NO_LOAD -// Size == 3 -#define vload_partial_3_0 NO_LOAD -#define vload_partial_3_1 vload_partial_1 -#define vload_partial_3_2 vload_partial_2 -#define vload_partial_3_3 vload_partial_3 -#define vload_partial_3_4 NO_LOAD -#define vload_partial_3_5 NO_LOAD -#define vload_partial_3_6 NO_LOAD -#define vload_partial_3_7 NO_LOAD -#define vload_partial_3_8 NO_LOAD -#define vload_partial_3_9 NO_LOAD -#define vload_partial_3_10 NO_LOAD -#define vload_partial_3_11 NO_LOAD -#define vload_partial_3_12 NO_LOAD -#define vload_partial_3_13 NO_LOAD -#define vload_partial_3_14 NO_LOAD -#define vload_partial_3_15 NO_LOAD -#define vload_partial_3_16 NO_LOAD -// Size == 4 -#define vload_partial_4_0 NO_LOAD -#define vload_partial_4_1 vload_partial_1 -#define vload_partial_4_2 vload_partial_2 -#define vload_partial_4_3 vload_partial_3 -#define vload_partial_4_4 vload_partial_4 -#define vload_partial_4_5 NO_LOAD -#define vload_partial_4_6 NO_LOAD -#define vload_partial_4_7 NO_LOAD -#define vload_partial_4_8 NO_LOAD -#define vload_partial_4_9 NO_LOAD -#define vload_partial_4_10 NO_LOAD -#define vload_partial_4_11 NO_LOAD -#define vload_partial_4_12 NO_LOAD -#define vload_partial_4_13 NO_LOAD -#define vload_partial_4_14 NO_LOAD -#define vload_partial_4_15 NO_LOAD -#define vload_partial_4_16 NO_LOAD -// Size == 8 -#define vload_partial_8_0 NO_LOAD -#define vload_partial_8_1 vload_partial_1 -#define vload_partial_8_2 vload_partial_2 -#define vload_partial_8_3 vload_partial_3 -#define vload_partial_8_4 vload_partial_4 -#define vload_partial_8_5 vload_partial_5 -#define vload_partial_8_6 vload_partial_6 -#define vload_partial_8_7 vload_partial_7 -#define vload_partial_8_8 vload_partial_8 -#define vload_partial_8_9 NO_LOAD -#define vload_partial_8_10 NO_LOAD -#define vload_partial_8_11 NO_LOAD -#define vload_partial_8_12 NO_LOAD -#define vload_partial_8_13 NO_LOAD -#define vload_partial_8_14 NO_LOAD -#define vload_partial_8_15 NO_LOAD -#define vload_partial_8_16 NO_LOAD -// Size == 16 -#define vload_partial_16_0 NO_LOAD -#define vload_partial_16_1 vload_partial_1 -#define vload_partial_16_2 vload_partial_2 -#define vload_partial_16_3 vload_partial_3 -#define vload_partial_16_4 vload_partial_4 -#define vload_partial_16_5 vload_partial_5 -#define vload_partial_16_6 vload_partial_6 -#define vload_partial_16_7 vload_partial_7 -#define vload_partial_16_8 vload_partial_8 -#define vload_partial_16_9 vload_partial_9 -#define vload_partial_16_10 vload_partial_10 -#define vload_partial_16_11 vload_partial_11 -#define vload_partial_16_12 vload_partial_12 -#define vload_partial_16_13 vload_partial_13 -#define vload_partial_16_14 vload_partial_14 -#define vload_partial_16_15 vload_partial_15 -#define vload_partial_16_16 vload_partial_16 - -/** Partial vload. Load the **lower** 0 to (n-1)th elements of the given vector while minimising the amount of vload ops - * @name vload_partial_n - * - * @note @p DATA needs to be a vector not a scalar - * @note n needs to be <= the vector width of the input variable @p DATA - * eg 1: Valid - * vload_partial_15(var:float16, 0, 0xabcd); - * eg 2: Invalid - * vload_partial_7(var:float4, 0, 0xabcd); - * - * @note in cases n == 1, 2, 3, 4, 8, 16, no extra vload is invoked, thus there's no performance penalty. - * - * @param[in] DATA The name of the variable where to load the values - * @param[in] OFFSET Offset in n - * @param[in] PTR The base pointer - * @{ - */ -#define vload_partial_1(DATA, OFFSET, PTR) \ - DATA.s0 = vload1(OFFSET, PTR); - -#define vload_partial_2(DATA, OFFSET, PTR) \ - DATA.s01 = vload2(OFFSET, PTR); - -#define vload_partial_3(DATA, OFFSET, PTR) \ - DATA.s012 = vload3(OFFSET, PTR); - -#define vload_partial_4(DATA, OFFSET, PTR) \ - DATA.s0123 = vload4(OFFSET, PTR); - -#define vload_partial_5(DATA, OFFSET, PTR) \ - DATA.s0123 = vload_partial_4(DATA, OFFSET, PTR); \ - DATA.s4 = vload1(OFFSET, PTR + 4); - -#define vload_partial_6(DATA, OFFSET, PTR) \ - DATA.s0123 = vload_partial_4(DATA, OFFSET, PTR); \ - DATA.s45 = vload_partial_2(DATA, OFFSET, PTR + 4); - -#define vload_partial_7(DATA, OFFSET, PTR) \ - DATA.s0123 = vload_partial_4(DATA, OFFSET, PTR); \ - DATA.s456 = vload_partial_3(DATA, OFFSET, PTR + 4); - -#define vload_partial_8(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload8(OFFSET, PTR); - -#define vload_partial_9(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload_partial_8(DATA, OFFSET, PTR); \ - DATA.s8 = vload1(OFFSET, PTR + 8); - -#define vload_partial_10(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload_partial_8(DATA, OFFSET, PTR); \ - DATA.s89 = vload_partial_2(DATA, OFFSET, PTR + 8); - -#define vload_partial_11(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload_partial_8(DATA, OFFSET, PTR); \ - DATA.s89A = vload_partial_3(DATA, OFFSET, PTR + 8); - -#define vload_partial_12(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload_partial_8(DATA, OFFSET, PTR); \ - DATA.s89AB = vload_partial_4(DATA, OFFSET, PTR + 8); - -#define vload_partial_13(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload_partial_8(DATA, OFFSET, PTR); \ - DATA.s89ABC = vload_partial_5(DATA, OFFSET, PTR + 8); - -#define vload_partial_14(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload_partial_8(DATA, OFFSET, PTR); \ - DATA.s89ABCD = vload_partial_6(DATA, OFFSET, PTR + 8); - -#define vload_partial_15(DATA, OFFSET, PTR) \ - DATA.s01234567 = vload_partial_8(DATA, OFFSET, PTR); \ - DATA.s89ABCDE = vload_partial_7(DATA, OFFSET, PTR + 8); - -#define vload_partial_16(DATA, OFFSET, PTR) \ - DATA = vload16(OFFSET, PTR); -/** @} */ // end of groupd vload_partial_n -/** @} */ // end of groupd VLOAD_PARTIAL - #define PIXEL_UNIT4 1 #define PIXEL_UNIT8 2 #define PIXEL_UNIT16 4 diff --git a/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedKernel.cpp b/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedKernel.cpp index 3a39128c0a..959b3ab21d 100644 --- a/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedKernel.cpp +++ b/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedKernel.cpp @@ -32,6 +32,7 @@ #include "arm_compute/core/Utils.h" #include "arm_compute/core/Validate.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" +#include "src/core/AccessWindowStatic.h" #include "src/core/CL/CLUtils.h" #include "src/core/CL/CLValidate.h" #include "src/core/helpers/AutoConfiguration.h" @@ -41,6 +42,10 @@ #include "support/Cast.h" #include "support/StringSupport.h" +#include +#include +#include + namespace arm_compute { namespace opencl @@ -126,11 +131,17 @@ std::pair validate_and_configure_window(ITensorInfo *src0, ITens const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info, ElementsProcessed &num_elements_processed) { - ARM_COMPUTE_UNUSED(src0, src1, src2); unsigned int &num_elems_processed_per_iteration_x = num_elements_processed[0]; unsigned int &num_elems_processed_per_iteration_y = num_elements_processed[1]; bool reinterpret_output_as_3d = gemm_info.depth_output_gemm3d != 0; + Window win{}; + Window win_out{}; + bool window_changed = false; + + // dst tensor auto initialization if not yet initialized + auto_init_if_empty(*dst, src0->clone()->set_tensor_shape(misc::shape_calculator::compute_mm_shape(*src0, *src1, gemm_info))); + TensorInfo tmp_info(*dst); if(reinterpret_output_as_3d) @@ -146,7 +157,21 @@ std::pair validate_and_configure_window(ITensorInfo *src0, ITens num_elems_processed_per_iteration_x = rhs_info.n0; num_elems_processed_per_iteration_y = lhs_info.m0; - Window win = calculate_max_window(tmp_info, Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y)); + win = calculate_max_window(tmp_info, Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y)); + win_out = calculate_max_window(*dst, Steps(num_elems_processed_per_iteration_x, num_elems_processed_per_iteration_y)); + + if(src2 != nullptr) + { + const int bias_processed_per_iteration_x = num_elems_processed_per_iteration_x; + + const int bias_processed_per_iteration_y = gemm_info.broadcast_bias ? 1 : num_elems_processed_per_iteration_y; + + AccessWindowStatic src2_access(src2, 0, 0, + ceil_to_multiple(src2->dimension(0), bias_processed_per_iteration_x), + ceil_to_multiple(src2->dimension(1), bias_processed_per_iteration_y)); + + window_changed = update_window_and_padding(win, src2_access); // window used by the execute_window_loop + } // Collapse along the Z direction // This collapse needs to be here in order to tune the Z dimension of LWS @@ -154,7 +179,8 @@ std::pair validate_and_configure_window(ITensorInfo *src0, ITens const unsigned int dimension_to_collapse = std::min(static_cast(dst->num_dimensions()), 2u); collapsed = win.collapse(win, dimension_to_collapse); - return std::make_pair(Status{}, collapsed); + Status err = (window_changed) ? ARM_COMPUTE_CREATE_ERROR(ErrorCode::RUNTIME_ERROR, "Insufficient Padding!") : Status{}; + return std::make_pair(err, collapsed); } } // namespace @@ -164,17 +190,14 @@ ClGemmMatrixMultiplyReshapedKernel::ClGemmMatrixMultiplyReshapedKernel() } void ClGemmMatrixMultiplyReshapedKernel::configure(const CLCompileContext &compile_context, - const ITensorInfo *src0, const ITensorInfo *src1, const ITensorInfo *src2, ITensorInfo *dst, float alpha, float beta, + ITensorInfo *src0, ITensorInfo *src1, ITensorInfo *src2, ITensorInfo *dst, float alpha, float beta, const GEMMLHSMatrixInfo &lhs_info, const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info) { ARM_COMPUTE_ERROR_ON_NULLPTR(src0, src1, dst); ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(src0, src1, src2, dst, alpha, beta, lhs_info, rhs_info, gemm_info)); - // dst tensor auto initialization if not yet initialized - auto_init_if_empty(*dst, src0->clone()->set_tensor_shape(misc::shape_calculator::compute_mm_shape(*src0, *src1, gemm_info))); - - auto padding_info = get_padding_info({ src0, src1, src2, dst }); + auto padding_info = get_padding_info({ src0, dst }); _reinterpret_output_as_3d = gemm_info.depth_output_gemm3d != 0; _use_dummy_work_items = preferred_dummy_work_items_support(CLKernelLibrary::get().get_device()); _add_bias = src2 != nullptr; @@ -188,14 +211,7 @@ void ClGemmMatrixMultiplyReshapedKernel::configure(const CLCompileContext &compi ElementsProcessed num_elements_processed{}; // Configure kernel window - auto win_config = validate_and_configure_window(src0->clone().get(), - src1->clone().get(), - (src2 != nullptr) ? src2->clone().get() : nullptr, - dst->clone().get(), - lhs_info, - rhs_info, - gemm_info, - num_elements_processed); + auto win_config = validate_and_configure_window(src0, src1, src2, dst, lhs_info, rhs_info, gemm_info, num_elements_processed); ARM_COMPUTE_ERROR_THROW_ON(win_config.first); ICLKernel::configure_internal(win_config.second); @@ -288,7 +304,18 @@ Status ClGemmMatrixMultiplyReshapedKernel::validate(const ITensorInfo *src0, con const GEMMLHSMatrixInfo &lhs_info, const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info) { + ElementsProcessed num_elements_processed{}; ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(src0, src1, src2, dst, alpha, beta, lhs_info, rhs_info, gemm_info)); + ARM_COMPUTE_RETURN_ON_ERROR(validate_and_configure_window(src0->clone().get(), + src1->clone().get(), + src2 != nullptr ? src2->clone().get() : nullptr, + dst->clone().get(), + lhs_info, + rhs_info, + gemm_info, + num_elements_processed) + .first); + return Status{}; } diff --git a/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedKernel.h b/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedKernel.h index b320d318e9..435a3a67f6 100644 --- a/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedKernel.h +++ b/src/gpu/cl/kernels/ClGemmMatrixMultiplyReshapedKernel.h @@ -84,7 +84,7 @@ public: * @note lhs_info.k0 must be equal to rhs_info.k0 */ void configure(const ClCompileContext &compile_context, - const ITensorInfo *src0, const ITensorInfo *src1, const ITensorInfo *src2, ITensorInfo *dst, float alpha, float beta, + ITensorInfo *src0, ITensorInfo *src1, ITensorInfo *src2, ITensorInfo *dst, float alpha, float beta, const GEMMLHSMatrixInfo &lhs_info, const GEMMRHSMatrixInfo &rhs_info, const GEMMKernelInfo &gemm_info); /** Static function to check if given info will lead to a valid configuration * -- cgit v1.2.1