diff options
author | Gian Marco Iodice <gianmarco.iodice@arm.com> | 2021-11-29 12:49:19 +0000 |
---|---|---|
committer | Sheri Zhang <sheri.zhang@arm.com> | 2022-01-25 12:48:53 +0000 |
commit | 10e88a73518932abc427e6b12d0267d5f52ac569 (patch) | |
tree | 641a4e5e6d4b540d90923cc105d29b131c848806 /src/core/CL/cl_kernels/common/gemm_reshaped_rhs_only.cl | |
parent | 639f0f665b4346d287014adc99e14a8497aadf18 (diff) | |
download | ComputeLibrary-10e88a73518932abc427e6b12d0267d5f52ac569.tar.gz |
Rework gemm_mm_reshaped_only_rhs_ kernels with new macros
- Rework gemm_reshaped_rhs_only with new TILE macros
- Fuse post ops in gemm_reshaped_rhs_only
Resolves COMPMID-4890
Change-Id: I944948ecec6d08deaf3545b80cd3eeac26e44205
Signed-off-by: Gian Marco Iodice <gianmarco.iodice@arm.com>
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/6944
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Sheri Zhang <sheri.zhang@arm.com>
Diffstat (limited to 'src/core/CL/cl_kernels/common/gemm_reshaped_rhs_only.cl')
-rw-r--r-- | src/core/CL/cl_kernels/common/gemm_reshaped_rhs_only.cl | 953 |
1 files changed, 953 insertions, 0 deletions
diff --git a/src/core/CL/cl_kernels/common/gemm_reshaped_rhs_only.cl b/src/core/CL/cl_kernels/common/gemm_reshaped_rhs_only.cl new file mode 100644 index 0000000000..1d6560a1c2 --- /dev/null +++ b/src/core/CL/cl_kernels/common/gemm_reshaped_rhs_only.cl @@ -0,0 +1,953 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "activation_float_helpers.h" +#include "helpers.h" +#include "tile_helpers.h" + +// *INDENT-OFF* +// clang-format off +#if defined(GEMM_MM_RESHAPED_ONLY_RHS_T) +//! @cond Doxygen_Suppress +/** This OpenCL kernel computes the matrix multiplication between 2 matrices plus 3 post ops: + * Post op 1: activation (optional) + * Post op 2: elementwise op + * Post op 3: activation (optional) + * + * The LHS matrix is NOT reshaped + * The RHS is reshaped with @ref ClGemmMatrixMultiplyReshapedOnlyRhsKernel and the block K0xN0 is transposed + * + * @note If the first two dimensions of NDRange have been dispatched with "dummy_work_items" support, the option -DDUMMY_WORK_ITEMS must be passed at compile time. + * @note The block's dimensions used for reshaping the RHS matrix (N0 and K0) must be passed at compile time using -DN0 and -DK0 (e.g. -DN0=8, -DK0=4). + * @note The number of M0 rows to process must be passed at compile time using -DM0 (e.g. -DM0=2) + * @note The number of K0xN0 horizontal blocks stored on the same output row of the reshaped RHS matrix must be passed at compile time using -DH0 (e.g. -DH0=2) + * @note If the K0xN0 blocks in the reshaped RHS matrix have been interleaved, the option -DRHS_INTERLEAVE must passed at compile time. + * @note The size of the partial store block in y must be passed at compile time using -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_M0=1) + * @note The size of the partial store block in x must be passed at compile time using -DPARTIAL_STORE_N0 (e.g. -DPARTIAL_STORE_N0=1) + * @note Only the following configurations of M0, N0 and K0 are currently supported: + * - M0 = 1, 2, 3, 4, 5, 6, 7, 8 + * - N0 = 2, 3, 4, 8, 16 + * - K0 = 2, 3, 4, 8, 16 + * - H0 >= 1 + * + * @note In case of post ops, the following information must be passed at compile time: + * @note -DPOST_OP1, -DP1_ACTIVATION_TYPE, -DP1_ACTIVATION_A_VAL, -DP1_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 1 + * @note -DPOST_OP2: The arithmetic addition post op to perform at slot 2 + * @note -DPOST_OP3, -DP3_ACTIVATION_TYPE, -DP3_ACTIVATION_A_VAL, -DP3_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 3 + * + * @param[in] lhs_ptr Pointer to the LHS tensor. Supported data types: F16/F32 + * @param[in] lhs_stride_y Stride of the LHS tensor in Y dimension (in bytes) + * @param[in] lhs_stride_z Stride of the LHS tensor in Z dimension (in bytes) + * @param[in] lhs_w The size of the width dimension of the LHS tensor + * @param[in] lhs_h The size of the height dimension of the LHS tensor + * @param[in] lhs_n The size of the depth dimension of the LHS tensor + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the LHS tensor + * @param[in] rhs_ptr Pointer to the RHS reshaped tensor. Supported data type: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the RHS tensor in Y dimension (in bytes) + * @param[in] rhs_stride_z Stride of the RHS tensor in Z dimension (in bytes) + * @param[in] rhs_w The size of the width dimension of the RHS tensor + * @param[in] rhs_h The size of the height dimension of the RHS tensor + * @param[in] rhs_n The size of the depth dimension of the RHS tensor + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the RHS tensor + * @param[in] bia_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bia_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bia_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bia_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bia_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bia_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bia_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[in] ex0_ptr (Optional) Pointer to the tensor added with POST_OP2. Supported data type: same as @p lhs_ptr + * @param[in] ex0_stride_y (Optional) Stride of the tensor added with POST_OP2 in Y dimension (in bytes) + * @param[in] ex0_stride_z (Optional) Stride of the tensor added with POST_OP2 in Z dimension (in bytes) + * @param[in] ex0_w (Optional) The size of the width dimension of the tensor added with POST_OP2 + * @param[in] ex0_h (Optional) The size of the height dimension of the tensor added with POST_OP2 + * @param[in] ex0_n (Optional) The size of the depth dimension of the tensor added with POST_OP2 + * @param[in] ex0_offset_first_element_in_bytes (Optional) The offset of the first element in the tensor added with POST_OP2 + * @param[out] dst_ptr (Optional) Pointer to the destination tensor. Supported data type: same as @p lhs_ptr + * @param[in] dst_stride_y (Optional) Stride of the destination tensor in Y dimension (in bytes) + * @param[in] dst_stride_z (Optional) Stride of the destination tensor in Z dimension (in bytes) + * @param[in] dst_w (Optional) The size of the width dimension of the destination tensor + * @param[in] dst_h (Optional) The size of the height dimension of the destination tensor + * @param[in] dst_n (Optional) The size of the depth dimension of the destination tensor + * @param[in] dst_offset_first_element_in_bytes (Optional) The offset of the first element in the destination tensor + * @param[in] M Number of rows in LHS matrix not reshaped + * @param[in] N Number of columns in RHS matrix not reshaped + * @param[in] K Number of columns in LHS matrix and rows in RHS matrix not reshaped + */ +//! @endcond +__kernel void gemm_mm_reshaped_only_rhs_t( + TENSOR3D_T(lhs, BUFFER), + TENSOR3D_T(rhs, BUFFER), +#if defined(BETA) + TENSOR3D_T(bia, BUFFER), +#endif // defined(BETA) +#if defined(POST_OP2) + TENSOR3D_T(ex0, BUFFER), +#endif // defined(POST_OP_ADD) + TENSOR3D_T(dst, BUFFER), + const int M, + const int N, + const int K +) +{ + // Block size +#define RHS_BLOCK_SIZE ((K0) * (N0)) + + // RHS offset and step X +#if defined(RHS_INTERLEAVE) +#define RHS_OFFSET_X (K0) +#define RHS_STEP_X ((K0) * (H0)) +#define RHS_STEP_LOOP (1) +#else // defined(RHS_INTERLEAVE) +#define RHS_OFFSET_X (RHS_BLOCK_SIZE) +#define RHS_STEP_X (K0) +#define RHS_STEP_LOOP (H0) +#endif // defined(RHS_INTERLEAVE) + + const uint x = GET_SPATIAL_IDX(0, N0, 0); + const uint y = GET_SPATIAL_IDX(1, M0, PARTIAL_STORE_M0); + const uint z = GET_SPATIAL_IDX(2, 1, 0); + +#if defined(DUMMY_WORK_ITEMS) + if((x >= N) || (y >= M)) + { + return; + } +#endif // defined(DUMMY_WORK_ITEMS) + + bool x_cond = PARTIAL_STORE_N0 != 0 && ((x + N0) > N); + bool y_cond = PARTIAL_STORE_M0 != 0 && y == 0; + + TILE(uint, M0, 1, dst_indirect_y); + INITIALIZE_INDIRECT_Y(M0, PARTIAL_STORE_M0, y_cond, dst_indirect_y); + + const uint x_rhs = x / N0; + + lhs_offset_first_element_in_bytes += y * (uint)lhs_stride_y + z * (uint)lhs_stride_y * M; + rhs_offset_first_element_in_bytes += (x_rhs % H0) * (uint)RHS_OFFSET_X * sizeof(DATA_TYPE) + (x_rhs / (uint)H0) * rhs_stride_y; + +#if defined(MATRIX_B_DEPTH) + // Do not slide matrix B if the matrix B has 3 dimensions and matrix A more than 3 + rhs_offset_first_element_in_bytes += (z % MATRIX_B_DEPTH) * rhs_stride_z; +#else // defined(MATRIX_B_DEPTH) + rhs_offset_first_element_in_bytes += z * rhs_stride_z; +#endif // defined(MATRIX_B_DEPTH) + + // Initialize the accumulators + TILE(DATA_TYPE, M0, N0, c); + + LOOP_UNROLLING(int, i, 0, 1, M0, + { + c[i].v = 0; + }) + + int i = 0; + for(; i <= (K - K0); i+=K0) + { + TILE(DATA_TYPE, M0, K0, a); + TILE(DATA_TYPE, N0, K0, b); + + // Load tile from the lhs/rhs tensors + T_LOAD(DATA_TYPE, M0, K0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); + T_LOAD(DATA_TYPE, N0, K0, BUFFER, rhs, 0, 0, 1, RHS_STEP_X * sizeof(DATA_TYPE), b); + + // Compute the matrix multiplication between the two tiles + T_MMUL(DATA_TYPE, DATA_TYPE, DATA_TYPE, M0, N0, K0, NT, T, a, b, c); + + lhs_offset_first_element_in_bytes += K0 * sizeof(DATA_TYPE); + rhs_offset_first_element_in_bytes += (N0 * RHS_STEP_X * RHS_STEP_LOOP) * sizeof(DATA_TYPE); + } +#if defined(RUN_LEFTOVER_K0) + for(; i < K; ++i) + { + TILE(DATA_TYPE, M0, 1, a); + TILE(DATA_TYPE, N0, 1, b); + + // Load tile from the lhs/rhs tensors + T_LOAD(DATA_TYPE, M0, 1, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); + T_LOAD(DATA_TYPE, N0, 1, BUFFER, rhs, 0, 0, 1, RHS_STEP_X * sizeof(DATA_TYPE), b); + + T_MMUL(DATA_TYPE, DATA_TYPE, DATA_TYPE, M0, N0, 1, NT, T, a, b, c); + + lhs_offset_first_element_in_bytes += sizeof(DATA_TYPE); + rhs_offset_first_element_in_bytes += sizeof(DATA_TYPE); + } +#endif // defined(RUN_LEFTOVER_K0) + + // Multiply by the weight of matrix-matrix product and store the result +#if defined(ALPHA) + T_SCALE_CONSTANT(DATA_TYPE, M0, N0, c, (DATA_TYPE)ALPHA, c); +#endif // defined(ALPHA) + + // Add beta*bias +#if defined(BETA) +#if defined(BROADCAST_BIAS) + TILE(DATA_TYPE, 1, N0, bias0); + + T_LOAD_WIDTH_SELECT(DATA_TYPE, 1, N0, PARTIAL_STORE_N0, BUFFER, bia, x, 0, 0, x_cond, bias0); + +#ifndef UNIT_BETA + T_SCALE_CONSTANT(DATA_TYPE, 1, N0, bias0, (DATA_TYPE)BETA, bias0); +#endif // UNIT_BIAS + + // c = c + bias[broadcasted] + T_ADD_BROADCAST_X(DATA_TYPE, M0, N0, c, bias0, c); +#else // defined(BROADCAST_BIAS) + TILE(DATA_TYPE, M0, N0, bias0); + + bia_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * bia_stride_y) + (z * bia_stride_y * M); + + T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, bia, 0, bia_stride_y, x_cond, bias0, dst_indirect_y); + +#ifndef UNIT_BETA + T_SCALE_CONSTANT(DATA_TYPE, M0, N0, bias0, (DATA_TYPE)BETA, bias0); +#endif // UNIT_BIAS + + // c = c + bias + T_ADD(DATA_TYPE, M0, N0, c, bias0, c); + // c = c + bias +#endif // defined(BROADCAST_BIAS) +#endif // defined(BETA) + +#if defined(POST_OP1) + T_ACTIVATION(DATA_TYPE, M0, N0, P1_ACTIVATION_TYPE, P1_ACTIVATION_A_VAL, P1_ACTIVATION_B_VAL, c, c); +#endif // defined(POST_OP1) + +#if defined(POST_OP2) + TILE(DATA_TYPE, M0, N0, extra0); + + ex0_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * ex0_stride_y) + (z * ex0_stride_y * M); + + T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, ex0, 0, ex0_stride_y, x_cond, extra0, dst_indirect_y); + + T_ADD(DATA_TYPE, M0, N0, c, extra0, c); +#endif // defined(POST_OP2) + +#if defined(POST_OP3) + T_ACTIVATION(DATA_TYPE, M0, N0, P3_ACTIVATION_TYPE, P3_ACTIVATION_A_VAL, P3_ACTIVATION_B_VAL, c, c); +#endif // defined(POST_OP3) + + dst_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * dst_stride_y) + (z * dst_stride_y * M); + + // Store the tile in reverse order so that the invalid values are overwritten with the valid ones + T_STORE_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, dst, 0, dst_stride_y, x_cond, c, dst_indirect_y); + +#undef RHS_BLOCK_SIZE +#undef RHS_OFFSET_X +#undef RHS_STEP_X +} +#endif // defined(GEMM_RESHAPED_RHS_ONLY_T) + +#if defined(GEMM_MM_RESHAPED_ONLY_RHS_T_TEXTURE) +//! @cond Doxygen_Suppress +/** This OpenCL kernel computes the matrix multiplication between 2 matrices plus 3 post ops: + * Post op 1: activation (optional) + * Post op 2: elementwise op + * Post op 3: activation (optional) + * + * The LHS matrix is NOT reshaped + * The RHS is reshaped with @ref ClGemmMatrixMultiplyReshapedOnlyRhsKernel and the block K0xN0 is transposed + * + * @note If the first two dimensions of NDRange have been dispatched with "dummy_work_items" support, the option -DDUMMY_WORK_ITEMS must be passed at compile time. + * @note The block's dimensions used for reshaping the RHS matrix (N0 and K0) must be passed at compile time using -DN0 and -DK0 (e.g. -DN0=8, -DK0=4). + * @note The number of M0 rows to process must be passed at compile time using -DM0 (e.g. -DM0=2) + * @note The number of K0xN0 horizontal blocks stored on the same output row of the reshaped RHS matrix must be passed at compile time using -DH0 (e.g. -DH0=2) + * @note If the K0xN0 blocks in the reshaped RHS matrix have been interleaved, the option -DRHS_INTERLEAVE must passed at compile time. + * @note The size of the partial store block in y must be passed at compile time using -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_M0=1) + * @note The size of the partial store block in x must be passed at compile time using -DPARTIAL_STORE_N0 (e.g. -DPARTIAL_STORE_N0=1) + * @note Only the following configurations of M0, N0 and K0 are currently supported: + * - M0 = 1, 2, 3, 4, 5, 6, 7, 8 + * - N0 = 2, 3, 4, 8, 16 + * - K0 = 2, 3, 4, 8, 16 + * - H0 >= 1 + * + * @note In case of post ops, the following information must be passed at compile time: + * @note -DPOST_OP1, -DP1_ACTIVATION_TYPE, -DP1_ACTIVATION_A_VAL, -DP1_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 1 + * @note -DPOST_OP2: The arithmetic addition post op to perform at slot 2 + * @note -DPOST_OP3, -DP3_ACTIVATION_TYPE, -DP3_ACTIVATION_A_VAL, -DP3_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 3 + * + * @param[in] lhs_ptr Pointer to the LHS tensor. Supported data types: F16/F32 + * @param[in] lhs_stride_y Stride of the LHS tensor in Y dimension (in bytes) + * @param[in] lhs_stride_z Stride of the LHS tensor in Z dimension (in bytes) + * @param[in] lhs_w The size of the width dimension of the LHS tensor + * @param[in] lhs_h The size of the height dimension of the LHS tensor + * @param[in] lhs_n The size of the depth dimension of the LHS tensor + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the LHS tensor + * @param[in] rhs_ptr Pointer to the RHS reshaped tensor. Supported data type: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the RHS tensor in Y dimension (in bytes) + * @param[in] rhs_stride_z Stride of the RHS tensor in Z dimension (in bytes) + * @param[in] rhs_w The size of the width dimension of the RHS tensor + * @param[in] rhs_h The size of the height dimension of the RHS tensor + * @param[in] rhs_n The size of the depth dimension of the RHS tensor + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the RHS tensor + * @param[in] bia_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bia_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bia_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bia_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bia_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bia_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bia_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[in] ex0_ptr (Optional) Pointer to the tensor added with POST_OP2. Supported data type: same as @p lhs_ptr + * @param[in] ex0_stride_y (Optional) Stride of the tensor added with POST_OP2 in Y dimension (in bytes) + * @param[in] ex0_stride_z (Optional) Stride of the tensor added with POST_OP2 in Z dimension (in bytes) + * @param[in] ex0_w (Optional) The size of the width dimension of the tensor added with POST_OP2 + * @param[in] ex0_h (Optional) The size of the height dimension of the tensor added with POST_OP2 + * @param[in] ex0_n (Optional) The size of the depth dimension of the tensor added with POST_OP2 + * @param[in] ex0_offset_first_element_in_bytes (Optional) The offset of the first element in the tensor added with POST_OP2 + * @param[out] dst_ptr (Optional) Pointer to the destination tensor. Supported data type: same as @p lhs_ptr + * @param[in] dst_stride_y (Optional) Stride of the destination tensor in Y dimension (in bytes) + * @param[in] dst_stride_z (Optional) Stride of the destination tensor in Z dimension (in bytes) + * @param[in] dst_w (Optional) The size of the width dimension of the destination tensor + * @param[in] dst_h (Optional) The size of the height dimension of the destination tensor + * @param[in] dst_n (Optional) The size of the depth dimension of the destination tensor + * @param[in] dst_offset_first_element_in_bytes (Optional) The offset of the first element in the destination tensor + * @param[in] M Number of rows in LHS matrix not reshaped + * @param[in] N Number of columns in RHS matrix not reshaped + * @param[in] K Number of columns in LHS matrix and rows in RHS matrix not reshaped + */ +//! @endcond +__kernel void gemm_mm_reshaped_only_rhs_t_texture( + TENSOR3D_T(lhs, BUFFER), + TENSOR3D_T(rhs, IMAGE), +#if defined(BETA) + TENSOR3D_T(bia, BUFFER), +#endif // defined(BETA) +#if defined(POST_OP2) + TENSOR3D_T(ex0, BUFFER), +#endif // defined(POST_OP_ADD) + TENSOR3D_T(dst, BUFFER), + const int M, + const int N, + const int K +) +{ + // Block size +#define RHS_BLOCK_SIZE (K0 * (N0)) + + // RHS offset and step X +#if defined(RHS_INTERLEAVE) +#define RHS_OFFSET_X (K0) +#define RHS_STEP_X (K0 * (H0)) +#define RHS_STEP_LOOP (1) +#else // defined(RHS_INTERLEAVE) +#define RHS_OFFSET_X (RHS_BLOCK_SIZE) +#define RHS_STEP_X K0 +#define RHS_STEP_LOOP (H0) +#endif // defined(RHS_INTERLEAVE) + + const uint x = GET_SPATIAL_IDX(0, N0, 0); + const uint y = GET_SPATIAL_IDX(1, M0, PARTIAL_STORE_M0); + const uint z = GET_SPATIAL_IDX(2, 1, 0); + +#if defined(DUMMY_WORK_ITEMS) + if((x >= N) || (y >= M)) + { + return; + } +#endif // defined(DUMMY_WORK_ITEMS) + + bool x_cond = PARTIAL_STORE_N0 != 0 && ((x + N0) > N); + bool y_cond = PARTIAL_STORE_M0 != 0 && y == 0; + + TILE(uint, M0, 1, dst_indirect_y); + INITIALIZE_INDIRECT_Y(M0, PARTIAL_STORE_M0, y_cond, dst_indirect_y); + + lhs_offset_first_element_in_bytes += y * (uint)lhs_stride_y + z * lhs_stride_y * M; + +#if defined(MATRIX_B_DEPTH) + // Do not slide matrix B if the matrix B has 3 dimensions and matrix A more than 3 + const uint z_rhs = z % MATRIX_B_DEPTH; +#else // defined(MATRIX_B_DEPTH) + const uint z_rhs = z; +#endif // defined(MATRIX_B_DEPTH) + + uint x_rhs = ((x / N0) % H0) * (uint)RHS_OFFSET_X; + const uint y_rhs = ((x / N0) / H0) + z_rhs * rhs_h; + + // Initialize the accumulators + TILE(DATA_TYPE, M0, N0, c); + + LOOP_UNROLLING(int, i, 0, 1, M0, + { + c[i].v = 0; + }) + + TILE(DATA_TYPE, M0, K0, a); + TILE(DATA_TYPE, N0, K0, b); + + int i = 0; + for(; i <= (K - K0); i+=K0) + { + // Load tile from the lhs/rhs tensors + T_LOAD(DATA_TYPE, M0, K0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); + T_LOAD_DILATED(DATA_TYPE, N0, K0, IMAGE, rhs, x_rhs, y_rhs, RHS_STEP_X, 0, 1, b); + + // Compute the matrix multiplication between the two tiles + T_MMUL(DATA_TYPE, DATA_TYPE, DATA_TYPE, M0, N0, K0, NT, T, a, b, c); + + lhs_offset_first_element_in_bytes += K0 * sizeof(DATA_TYPE); + x_rhs += N0 * RHS_STEP_X * RHS_STEP_LOOP; + } +#if defined(RUN_LEFTOVER_K0) + T_LOAD_DILATED(DATA_TYPE, N0, K0, IMAGE, rhs, x_rhs, y_rhs, RHS_STEP_X, 0, 1, b); + + LOOP_UNROLLING(int, k0, 0, 1, PARTIAL_K, + { + LOOP_UNROLLING(int, m0, 0, 1, M0, + { + DATA_TYPE a0 = *(__global DATA_TYPE *)(lhs_ptr + lhs_offset_first_element_in_bytes + m0 * lhs_stride_y); + LOOP_UNROLLING(int, n0, 0, 1, N0, + { + c[m0].s[n0] += a0 * b[n0].s[k0]; + }) + }) + lhs_offset_first_element_in_bytes += sizeof(DATA_TYPE); + }) +#endif // defined(RUN_LEFTOVER_K0) + + // Multiply by the weight of matrix-matrix product and store the result +#if defined(ALPHA) + T_SCALE_CONSTANT(DATA_TYPE, M0, N0, c, (DATA_TYPE)ALPHA, c); +#endif // defined(ALPHA) + + // Add beta*bias +#if defined(BETA) +#if defined(BROADCAST_BIAS) + TILE(DATA_TYPE, 1, N0, bias0); + + T_LOAD_WIDTH_SELECT(DATA_TYPE, 1, N0, PARTIAL_STORE_N0, BUFFER, bia, x, 0, 0, x_cond, bias0); + +#ifndef UNIT_BETA + T_SCALE_CONSTANT(DATA_TYPE, 1, N0, bias0, (DATA_TYPE)BETA, bias0); +#endif // UNIT_BIAS + + // c = c + bias[broadcasted] + T_ADD_BROADCAST_X(DATA_TYPE, M0, N0, c, bias0, c); +#else // defined(BROADCAST_BIAS) + TILE(DATA_TYPE, M0, N0, bias0); + + bia_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * bia_stride_y) + (z * bia_stride_y * M); + + T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, bia, 0, bia_stride_y, x_cond, bias0, dst_indirect_y); + +#ifndef UNIT_BETA + T_SCALE_CONSTANT(DATA_TYPE, M0, N0, bias0, (DATA_TYPE)BETA, bias0); +#endif // UNIT_BIAS + + // c = c + bias + T_ADD(DATA_TYPE, M0, N0, c, bias0, c); + // c = c + bias +#endif // defined(BROADCAST_BIAS) +#endif // defined(BETA) + +#if defined(POST_OP1) + T_ACTIVATION(DATA_TYPE, M0, N0, P1_ACTIVATION_TYPE, P1_ACTIVATION_A_VAL, P1_ACTIVATION_B_VAL, c, c); +#endif // defined(POST_OP1) + +#if defined(POST_OP2) + TILE(DATA_TYPE, M0, N0, extra0); + + ex0_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * ex0_stride_y) + (z * ex0_stride_y * M); + + T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, ex0, 0, ex0_stride_y, x_cond, extra0, dst_indirect_y); + + T_ADD(DATA_TYPE, M0, N0, c, extra0, c); +#endif // defined(POST_OP2) + +#if defined(POST_OP3) + T_ACTIVATION(DATA_TYPE, M0, N0, P3_ACTIVATION_TYPE, P3_ACTIVATION_A_VAL, P3_ACTIVATION_B_VAL, c, c); +#endif // defined(POST_OP3) + + dst_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + y * dst_stride_y + z * dst_stride_y * M; + + // Store the tile in reverse order so that the invalid values are overwritten with the valid ones + T_STORE_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, dst, 0, dst_stride_y, x_cond, c, dst_indirect_y); + +#undef RHS_BLOCK_SIZE +#undef RHS_OFFSET_X +#undef RHS_STEP_X +} +#endif // defined(GEMM_RESHAPED_RHS_ONLY_T_TEXTURE) + +#if defined(GEMM_MM_RESHAPED_ONLY_RHS_NT) +//! @cond Doxygen_Suppress +/** This OpenCL kernel computes the matrix multiplication between 2 matrices plus 3 post ops: + * Post op 1: activation (optional) + * Post op 2: elementwise op + * Post op 3: activation (optional) + * + * The LHS matrix is NOT reshaped + * The RHS is reshaped with @ref ClGemmMatrixMultiplyReshapedOnlyRhsKernel and the block K0xN0 is not transposed + * + * @note If the first two dimensions of NDRange have been dispatched with "dummy_work_items" support, the option -DDUMMY_WORK_ITEMS must be passed at compile time. + * @note The block's dimensions used for reshaping the RHS matrix (N0 and K0) must be passed at compile time using -DN0 and -DK0 (e.g. -DN0=8, -DK0=4). + * @note The number of M0 rows to process must be passed at compile time using -DM0 (e.g. -DM0=2) + * @note The number of K0xN0 horizontal blocks stored on the same output row of the reshaped RHS matrix must be passed at compile time using -DH0 (e.g. -DH0=2) + * @note If the K0xN0 blocks in the reshaped RHS matrix have been interleaved, the option -DRHS_INTERLEAVE must passed at compile time. + * @note The size of the partial store block in y must be passed at compile time using -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_M0=1) + * @note The size of the partial store block in x must be passed at compile time using -DPARTIAL_STORE_N0 (e.g. -DPARTIAL_STORE_N0=1) + * @note Only the following configurations of M0, N0 and K0 are currently supported: + * - M0 = 1, 2, 3, 4, 5, 6, 7, 8 + * - N0 = 2, 3, 4, 8, 16 + * - K0 = 2, 3, 4, 8, 16 + * - H0 >= 1 + * + * @note In case of post ops, the following information must be passed at compile time: + * @note -DPOST_OP1, -DP1_ACTIVATION_TYPE, -DP1_ACTIVATION_A_VAL, -DP1_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 1 + * @note -DPOST_OP2: The arithmetic addition post op to perform at slot 2 + * @note -DPOST_OP3, -DP3_ACTIVATION_TYPE, -DP3_ACTIVATION_A_VAL, -DP3_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 3 + * + * @param[in] lhs_ptr Pointer to the LHS tensor. Supported data types: F16/F32 + * @param[in] lhs_stride_y Stride of the LHS tensor in Y dimension (in bytes) + * @param[in] lhs_stride_z Stride of the LHS tensor in Z dimension (in bytes) + * @param[in] lhs_w The size of the width dimension of the LHS tensor + * @param[in] lhs_h The size of the height dimension of the LHS tensor + * @param[in] lhs_n The size of the depth dimension of the LHS tensor + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the LHS tensor + * @param[in] rhs_ptr Pointer to the RHS reshaped tensor. Supported data type: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the RHS tensor in Y dimension (in bytes) + * @param[in] rhs_stride_z Stride of the RHS tensor in Z dimension (in bytes) + * @param[in] rhs_w The size of the width dimension of the RHS tensor + * @param[in] rhs_h The size of the height dimension of the RHS tensor + * @param[in] rhs_n The size of the depth dimension of the RHS tensor + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the RHS tensor + * @param[in] bia_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bia_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bia_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bia_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bia_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bia_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bia_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[in] ex0_ptr (Optional) Pointer to the tensor added with POST_OP2. Supported data type: same as @p lhs_ptr + * @param[in] ex0_stride_y (Optional) Stride of the tensor added with POST_OP2 in Y dimension (in bytes) + * @param[in] ex0_stride_z (Optional) Stride of the tensor added with POST_OP2 in Z dimension (in bytes) + * @param[in] ex0_w (Optional) The size of the width dimension of the tensor added with POST_OP2 + * @param[in] ex0_h (Optional) The size of the height dimension of the tensor added with POST_OP2 + * @param[in] ex0_n (Optional) The size of the depth dimension of the tensor added with POST_OP2 + * @param[in] ex0_offset_first_element_in_bytes (Optional) The offset of the first element in the tensor added with POST_OP2 + * @param[out] dst_ptr (Optional) Pointer to the destination tensor. Supported data type: same as @p lhs_ptr + * @param[in] dst_stride_y (Optional) Stride of the destination tensor in Y dimension (in bytes) + * @param[in] dst_stride_z (Optional) Stride of the destination tensor in Z dimension (in bytes) + * @param[in] dst_w (Optional) The size of the width dimension of the destination tensor + * @param[in] dst_h (Optional) The size of the height dimension of the destination tensor + * @param[in] dst_n (Optional) The size of the depth dimension of the destination tensor + * @param[in] dst_offset_first_element_in_bytes (Optional) The offset of the first element in the destination tensor + * @param[in] M Number of rows in LHS matrix not reshaped + * @param[in] N Number of columns in RHS matrix not reshaped + * @param[in] K Number of columns in LHS matrix and rows in RHS matrix not reshaped + */ +//! @endcond +__kernel void gemm_mm_reshaped_only_rhs_nt( + TENSOR3D_T(lhs, BUFFER), + TENSOR3D_T(rhs, BUFFER), +#if defined(BETA) + TENSOR3D_T(bia, BUFFER), +#endif // defined(BETA) +#if defined(POST_OP2) + TENSOR3D_T(ex0, BUFFER), +#endif // defined(POST_OP_ADD) + TENSOR3D_T(dst, BUFFER), + const int M, + const int N, + const int K +) +{ + // Block size +#define RHS_BLOCK_SIZE ((K0) * (N0)) + + // RHS offset and step X +#if defined(RHS_INTERLEAVE) +#define RHS_OFFSET_X (N0) +#define RHS_STEP_X ((N0) * (H0)) +#define RHS_STEP_LOOP (1) +#else // defined(RHS_INTERLEAVE) +#define RHS_OFFSET_X (RHS_BLOCK_SIZE) +#define RHS_STEP_X (N0) +#define RHS_STEP_LOOP (H0) +#endif // defined(RHS_INTERLEAVE) + + const uint x = GET_SPATIAL_IDX(0, N0, 0); + const uint y = GET_SPATIAL_IDX(1, M0, PARTIAL_STORE_M0); + const uint z = GET_SPATIAL_IDX(2, 1, 0); + +#if defined(DUMMY_WORK_ITEMS) + if((x >= N) || (y >= M)) + { + return; + } +#endif // defined(DUMMY_WORK_ITEMS) + + bool x_cond = PARTIAL_STORE_N0 != 0 && ((x + N0) > N); + bool y_cond = PARTIAL_STORE_M0 != 0 && y == 0; + + TILE(uint, M0, 1, dst_indirect_y); + INITIALIZE_INDIRECT_Y(M0, PARTIAL_STORE_M0, y_cond, dst_indirect_y); + + const uint x_rhs = x / N0; + + lhs_offset_first_element_in_bytes += y * (uint)lhs_stride_y + z * lhs_stride_y * M; + rhs_offset_first_element_in_bytes += (x_rhs % H0) * (uint)RHS_OFFSET_X * sizeof(DATA_TYPE) + (x_rhs / (uint)H0) * rhs_stride_y; + +#if defined(MATRIX_B_DEPTH) + // Do not slide matrix B if the matrix B has 3 dimensions and matrix A more than 3 + rhs_offset_first_element_in_bytes += (z % MATRIX_B_DEPTH) * rhs_stride_z; +#else // defined(MATRIX_B_DEPTH) + rhs_offset_first_element_in_bytes += z * rhs_stride_z; +#endif // defined(MATRIX_B_DEPTH) + + // Initialize the accumulators + TILE(DATA_TYPE, M0, N0, c); + + LOOP_UNROLLING(int, i, 0, 1, M0, + { + c[i].v = 0; + }) + + int i = 0; + for(; i <= (K - K0); i+=K0) + { + TILE(DATA_TYPE, M0, K0, a); + TILE(DATA_TYPE, K0, N0, b); + + // Load tile from the lhs/rhs tensors + T_LOAD(DATA_TYPE, M0, K0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); + T_LOAD(DATA_TYPE, K0, N0, BUFFER, rhs, 0, 0, 1, RHS_STEP_X * sizeof(DATA_TYPE), b); + + // Compute the matrix multiplication between the two tiles + T_MMUL(DATA_TYPE, DATA_TYPE, DATA_TYPE, M0, N0, K0, NT, NT, a, b, c); + + lhs_offset_first_element_in_bytes += K0 * sizeof(DATA_TYPE); + rhs_offset_first_element_in_bytes += K0 * RHS_STEP_X * RHS_STEP_LOOP * sizeof(DATA_TYPE); + } +#if defined(RUN_LEFTOVER_K0) + for(; i < K; ++i) + { + TILE(DATA_TYPE, M0, 1, a); + TILE(DATA_TYPE, 1, N0, b); + + // Load tile from the lhs/rhs tensors + T_LOAD(DATA_TYPE, M0, 1, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); + T_LOAD(DATA_TYPE, 1, N0, BUFFER, rhs, 0, 0, 1, RHS_STEP_X * sizeof(DATA_TYPE), b); + + T_MMUL(DATA_TYPE, DATA_TYPE, DATA_TYPE, M0, N0, 1, NT, NT, a, b, c); + + lhs_offset_first_element_in_bytes += sizeof(DATA_TYPE); + rhs_offset_first_element_in_bytes += RHS_STEP_X * sizeof(DATA_TYPE); + } +#endif // defined(RUN_LEFTOVER_K0) + + // Multiply by the weight of matrix-matrix product and store the result +#if defined(ALPHA) + T_SCALE_CONSTANT(DATA_TYPE, M0, N0, c, (DATA_TYPE)ALPHA, c); +#endif // defined(ALPHA) + + // Add beta*bias +#if defined(BETA) +#if defined(BROADCAST_BIAS) + TILE(DATA_TYPE, 1, N0, bias0); + + T_LOAD_WIDTH_SELECT(DATA_TYPE, 1, N0, PARTIAL_STORE_N0, BUFFER, bia, x, 0, 0, x_cond, bias0); + +#ifndef UNIT_BETA + T_SCALE_CONSTANT(DATA_TYPE, 1, N0, bias0, (DATA_TYPE)BETA, bias0); +#endif // UNIT_BIAS + + // c = c + bias[broadcasted] + T_ADD_BROADCAST_X(DATA_TYPE, M0, N0, c, bias0, c); +#else // defined(BROADCAST_BIAS) + TILE(DATA_TYPE, M0, N0, bias0); + + bia_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * bia_stride_y) + (z * bia_stride_y * M); + + T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, bia, 0, bia_stride_y, x_cond, bias0, dst_indirect_y); + +#ifndef UNIT_BETA + T_SCALE_CONSTANT(DATA_TYPE, M0, N0, bias0, (DATA_TYPE)BETA, bias0); +#endif // UNIT_BIAS + + // c = c + bias + T_ADD(DATA_TYPE, M0, N0, c, bias0, c); + // c = c + bias +#endif // defined(BROADCAST_BIAS) +#endif // defined(BETA) + +#if defined(POST_OP1) + T_ACTIVATION(DATA_TYPE, M0, N0, P1_ACTIVATION_TYPE, P1_ACTIVATION_A_VAL, P1_ACTIVATION_B_VAL, c, c); +#endif // defined(POST_OP1) + +#if defined(POST_OP2) + TILE(DATA_TYPE, M0, N0, extra0); + + ex0_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * ex0_stride_y) + (z * ex0_stride_y * M); + + T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, ex0, 0, ex0_stride_y, x_cond, extra0, dst_indirect_y); + + T_ADD(DATA_TYPE, M0, N0, c, extra0, c); +#endif // defined(POST_OP2) + +#if defined(POST_OP3) + T_ACTIVATION(DATA_TYPE, M0, N0, P3_ACTIVATION_TYPE, P3_ACTIVATION_A_VAL, P3_ACTIVATION_B_VAL, c, c); +#endif // defined(POST_OP3) + + dst_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + y * dst_stride_y + z * dst_stride_y * M; + + // Store the tile in reverse order so that the invalid values are overwritten with the valid ones + T_STORE_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, dst, 0, dst_stride_y, x_cond, c, dst_indirect_y); + +#undef RHS_BLOCK_SIZE +#undef RHS_OFFSET_X +#undef RHS_STEP_X +} +#endif // defined(GEMM_RESHAPED_RHS_ONLY_NT) + +#if defined(GEMM_MM_RESHAPED_ONLY_RHS_NT_TEXTURE) +//! @cond Doxygen_Suppress +/** This OpenCL kernel computes the matrix multiplication between 2 matrices plus 3 post ops: + * Post op 1: activation (optional) + * Post op 2: elementwise op + * Post op 3: activation (optional) + * + * The LHS matrix is NOT reshaped + * The RHS is reshaped with @ref ClGemmMatrixMultiplyReshapedOnlyRhsKernel and the block K0xN0 is not transposed + * + * @note If the first two dimensions of NDRange have been dispatched with "dummy_work_items" support, the option -DDUMMY_WORK_ITEMS must be passed at compile time. + * @note The block's dimensions used for reshaping the RHS matrix (N0 and K0) must be passed at compile time using -DN0 and -DK0 (e.g. -DN0=8, -DK0=4). + * @note The number of M0 rows to process must be passed at compile time using -DM0 (e.g. -DM0=2) + * @note The number of K0xN0 horizontal blocks stored on the same output row of the reshaped RHS matrix must be passed at compile time using -DH0 (e.g. -DH0=2) + * @note If the K0xN0 blocks in the reshaped RHS matrix have been interleaved, the option -DRHS_INTERLEAVE must passed at compile time. + * @note The size of the partial store block in y must be passed at compile time using -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_M0=1) + * @note The size of the partial store block in x must be passed at compile time using -DPARTIAL_STORE_N0 (e.g. -DPARTIAL_STORE_N0=1) + * @note Only the following configurations of M0, N0 and K0 are currently supported: + * - M0 = 1, 2, 3, 4, 5, 6, 7, 8 + * - N0 = 2, 3, 4, 8, 16 + * - K0 = 2, 3, 4, 8, 16 + * - H0 >= 1 + * + * @note In case of post ops, the following information must be passed at compile time: + * @note -DPOST_OP1, -DP1_ACTIVATION_TYPE, -DP1_ACTIVATION_A_VAL, -DP1_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 1 + * @note -DPOST_OP2: The arithmetic addition post op to perform at slot 2 + * @note -DPOST_OP3, -DP3_ACTIVATION_TYPE, -DP3_ACTIVATION_A_VAL, -DP3_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 3 + * + * @param[in] lhs_ptr Pointer to the LHS tensor. Supported data types: F16/F32 + * @param[in] lhs_stride_y Stride of the LHS tensor in Y dimension (in bytes) + * @param[in] lhs_stride_z Stride of the LHS tensor in Z dimension (in bytes) + * @param[in] lhs_w The size of the width dimension of the LHS tensor + * @param[in] lhs_h The size of the height dimension of the LHS tensor + * @param[in] lhs_n The size of the depth dimension of the LHS tensor + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the LHS tensor + * @param[in] rhs_ptr Pointer to the RHS reshaped tensor. Supported data type: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the RHS tensor in Y dimension (in bytes) + * @param[in] rhs_stride_z Stride of the RHS tensor in Z dimension (in bytes) + * @param[in] rhs_w The size of the width dimension of the RHS tensor + * @param[in] rhs_h The size of the height dimension of the RHS tensor + * @param[in] rhs_n The size of the depth dimension of the RHS tensor + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the RHS tensor + * @param[in] bia_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr + * @param[in] bia_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) + * @param[in] bia_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) + * @param[in] bia_w (Optional) The size of the width dimension of the bias tensor + * @param[in] bia_h (Optional) The size of the height dimension of the bias tensor + * @param[in] bia_n (Optional) The size of the depth dimension of the bias tensor + * @param[in] bia_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor + * @param[in] ex0_ptr (Optional) Pointer to the tensor added with POST_OP2. Supported data type: same as @p lhs_ptr + * @param[in] ex0_stride_y (Optional) Stride of the tensor added with POST_OP2 in Y dimension (in bytes) + * @param[in] ex0_stride_z (Optional) Stride of the tensor added with POST_OP2 in Z dimension (in bytes) + * @param[in] ex0_w (Optional) The size of the width dimension of the tensor added with POST_OP2 + * @param[in] ex0_h (Optional) The size of the height dimension of the tensor added with POST_OP2 + * @param[in] ex0_n (Optional) The size of the depth dimension of the tensor added with POST_OP2 + * @param[in] ex0_offset_first_element_in_bytes (Optional) The offset of the first element in the tensor added with POST_OP2 + * @param[out] dst_ptr (Optional) Pointer to the destination tensor. Supported data type: same as @p lhs_ptr + * @param[in] dst_stride_y (Optional) Stride of the destination tensor in Y dimension (in bytes) + * @param[in] dst_stride_z (Optional) Stride of the destination tensor in Z dimension (in bytes) + * @param[in] dst_w (Optional) The size of the width dimension of the destination tensor + * @param[in] dst_h (Optional) The size of the height dimension of the destination tensor + * @param[in] dst_n (Optional) The size of the depth dimension of the destination tensor + * @param[in] dst_offset_first_element_in_bytes (Optional) The offset of the first element in the destination tensor + * @param[in] M Number of rows in LHS matrix not reshaped + * @param[in] N Number of columns in RHS matrix not reshaped + * @param[in] K Number of columns in LHS matrix and rows in RHS matrix not reshaped + */ +//! @endcond +__kernel void gemm_mm_reshaped_only_rhs_nt_texture( + TENSOR3D_T(lhs, BUFFER), + TENSOR3D_T(rhs, IMAGE), +#if defined(BETA) + TENSOR3D_T(bia, BUFFER), +#endif // defined(BETA) +#if defined(POST_OP2) + TENSOR3D_T(ex0, BUFFER), +#endif // defined(POST_OP_ADD) + TENSOR3D_T(dst, BUFFER), + const int M, + const int N, + const int K +) +{ + // Block size +#define RHS_BLOCK_SIZE ((K0) * (N0)) + + // RHS offset and step X +#if defined(RHS_INTERLEAVE) +#define RHS_OFFSET_X (N0) +#define RHS_STEP_X ((N0) * (H0)) +#define RHS_STEP_LOOP (1) +#else // defined(RHS_INTERLEAVE) +#define RHS_OFFSET_X (RHS_BLOCK_SIZE) +#define RHS_STEP_X (N0) +#define RHS_STEP_LOOP (H0) +#endif // defined(RHS_INTERLEAVE) + + const uint x = GET_SPATIAL_IDX(0, N0, 0); + const uint y = GET_SPATIAL_IDX(1, M0, PARTIAL_STORE_M0); + const uint z = GET_SPATIAL_IDX(2, 1, 0); + +#if defined(DUMMY_WORK_ITEMS) + if((x >= N) || (y >= M)) + { + return; + } +#endif // defined(DUMMY_WORK_ITEMS) + + bool x_cond = PARTIAL_STORE_N0 != 0 && ((x + N0) > N); + bool y_cond = PARTIAL_STORE_M0 != 0 && y == 0; + + TILE(uint, M0, 1, dst_indirect_y); + INITIALIZE_INDIRECT_Y(M0, PARTIAL_STORE_M0, y_cond, dst_indirect_y); + + lhs_offset_first_element_in_bytes += y * (uint)lhs_stride_y + z * lhs_stride_y * M; + +#if defined(MATRIX_B_DEPTH) + // Do not slide matrix B if the matrix B has 3 dimensions and matrix A more than 3 + const uint z_rhs = z % MATRIX_B_DEPTH; +#else // defined(MATRIX_B_DEPTH) + const uint z_rhs = z; +#endif // defined(MATRIX_B_DEPTH) + + uint x_rhs = ((x / N0) % H0) * (uint)RHS_OFFSET_X; + const uint y_rhs = ((x / N0) / H0) + z_rhs * rhs_h; + + // Initialize the accumulators + TILE(DATA_TYPE, M0, N0, c); + + LOOP_UNROLLING(int, i, 0, 1, M0, + { + c[i].v = 0; + }) + + int i = 0; + for(; i <= (K - K0); i+=K0) + { + TILE(DATA_TYPE, M0, K0, a); + TILE(DATA_TYPE, K0, N0, b); + + // Load tile from the lhs/rhs tensors + T_LOAD(DATA_TYPE, M0, K0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); + T_LOAD_DILATED(DATA_TYPE, K0, N0, IMAGE, rhs, x_rhs, y_rhs, RHS_STEP_X, 0, 1, b); + + // Compute the matrix multiplication between the two tiles + T_MMUL(DATA_TYPE, DATA_TYPE, DATA_TYPE, M0, N0, K0, NT, NT, a, b, c); + + lhs_offset_first_element_in_bytes += K0 * sizeof(DATA_TYPE); + x_rhs += K0 * RHS_STEP_X * RHS_STEP_LOOP; + } + +#if defined(RUN_LEFTOVER_K0) + for(; i < K; ++i) + { + TILE(DATA_TYPE, M0, 1, a); + TILE(DATA_TYPE, 1, N0, b); + + // Load tile from the lhs/rhs tensors + T_LOAD(DATA_TYPE, M0, 1, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); + T_LOAD_DILATED(DATA_TYPE, 1, N0, IMAGE, rhs, x_rhs, y_rhs, RHS_STEP_X, 0, 1, b); + + T_MMUL(DATA_TYPE, DATA_TYPE, DATA_TYPE, M0, N0, 1, NT, NT, a, b, c); + + lhs_offset_first_element_in_bytes += sizeof(DATA_TYPE); + x_rhs += RHS_STEP_X; + } +#endif // defined(RUN_LEFTOVER_K0) + + // Multiply by the weight of matrix-matrix product and store the result +#if defined(ALPHA) + T_SCALE_CONSTANT(DATA_TYPE, M0, N0, c, (DATA_TYPE)ALPHA, c); +#endif // defined(ALPHA) + + // Add beta*bias +#if defined(BETA) +#if defined(BROADCAST_BIAS) + TILE(DATA_TYPE, 1, N0, bias0); + + T_LOAD_WIDTH_SELECT(DATA_TYPE, 1, N0, PARTIAL_STORE_N0, BUFFER, bia, x, 0, 0, x_cond, bias0); + +#ifndef UNIT_BETA + T_SCALE_CONSTANT(DATA_TYPE, 1, N0, bias0, (DATA_TYPE)BETA, bias0); +#endif // UNIT_BIAS + + // c = c + bias[broadcasted] + T_ADD_BROADCAST_X(DATA_TYPE, M0, N0, c, bias0, c); +#else // defined(BROADCAST_BIAS) + TILE(DATA_TYPE, M0, N0, bias0); + + bia_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * bia_stride_y) + (z * bia_stride_y * M); + + T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, bia, 0, bia_stride_y, x_cond, bias0, dst_indirect_y); + +#ifndef UNIT_BETA + T_SCALE_CONSTANT(DATA_TYPE, M0, N0, bias0, (DATA_TYPE)BETA, bias0); +#endif // UNIT_BIAS + + // c = c + bias + T_ADD(DATA_TYPE, M0, N0, c, bias0, c); + // c = c + bias +#endif // defined(BROADCAST_BIAS) +#endif // defined(BETA) + +#if defined(POST_OP1) + T_ACTIVATION(DATA_TYPE, M0, N0, P1_ACTIVATION_TYPE, P1_ACTIVATION_A_VAL, P1_ACTIVATION_B_VAL, c, c); +#endif // defined(POST_OP1) + +#if defined(POST_OP2) + TILE(DATA_TYPE, M0, N0, extra0); + + ex0_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * ex0_stride_y) + (z * ex0_stride_y * M); + + T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, ex0, 0, ex0_stride_y, x_cond, extra0, dst_indirect_y); + + T_ADD(DATA_TYPE, M0, N0, c, extra0, c); +#endif // defined(POST_OP2) + +#if defined(POST_OP3) + T_ACTIVATION(DATA_TYPE, M0, N0, P3_ACTIVATION_TYPE, P3_ACTIVATION_A_VAL, P3_ACTIVATION_B_VAL, c, c); +#endif // defined(POST_OP3) + + dst_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + y * dst_stride_y + z * dst_stride_y * M; + + // Store the tile in reverse order so that the invalid values are overwritten with the valid ones + T_STORE_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, dst, 0, dst_stride_y, x_cond, c, dst_indirect_y); + +#undef RHS_BLOCK_SIZE +#undef RHS_OFFSET_X +#undef RHS_STEP_X +} +#endif // defined(GEMM_RESHAPED_RHS_ONLY_NT_TEXTURE)
\ No newline at end of file |