diff options
Diffstat (limited to 'src/core/CL/cl_kernels/common/gemm_reshaped_rhs_only.cl')
| -rw-r--r-- | src/core/CL/cl_kernels/common/gemm_reshaped_rhs_only.cl | 953 |
1 files changed, 0 insertions, 953 deletions
diff --git a/src/core/CL/cl_kernels/common/gemm_reshaped_rhs_only.cl b/src/core/CL/cl_kernels/common/gemm_reshaped_rhs_only.cl deleted file mode 100644 index 1d6560a1c2..0000000000 --- a/src/core/CL/cl_kernels/common/gemm_reshaped_rhs_only.cl +++ /dev/null @@ -1,953 +0,0 @@ -/* - * Copyright (c) 2022 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#include "activation_float_helpers.h" -#include "helpers.h" -#include "tile_helpers.h" - -// *INDENT-OFF* -// clang-format off -#if defined(GEMM_MM_RESHAPED_ONLY_RHS_T) -//! @cond Doxygen_Suppress -/** This OpenCL kernel computes the matrix multiplication between 2 matrices plus 3 post ops: - * Post op 1: activation (optional) - * Post op 2: elementwise op - * Post op 3: activation (optional) - * - * The LHS matrix is NOT reshaped - * The RHS is reshaped with @ref ClGemmMatrixMultiplyReshapedOnlyRhsKernel and the block K0xN0 is transposed - * - * @note If the first two dimensions of NDRange have been dispatched with "dummy_work_items" support, the option -DDUMMY_WORK_ITEMS must be passed at compile time. - * @note The block's dimensions used for reshaping the RHS matrix (N0 and K0) must be passed at compile time using -DN0 and -DK0 (e.g. -DN0=8, -DK0=4). - * @note The number of M0 rows to process must be passed at compile time using -DM0 (e.g. -DM0=2) - * @note The number of K0xN0 horizontal blocks stored on the same output row of the reshaped RHS matrix must be passed at compile time using -DH0 (e.g. -DH0=2) - * @note If the K0xN0 blocks in the reshaped RHS matrix have been interleaved, the option -DRHS_INTERLEAVE must passed at compile time. - * @note The size of the partial store block in y must be passed at compile time using -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_M0=1) - * @note The size of the partial store block in x must be passed at compile time using -DPARTIAL_STORE_N0 (e.g. -DPARTIAL_STORE_N0=1) - * @note Only the following configurations of M0, N0 and K0 are currently supported: - * - M0 = 1, 2, 3, 4, 5, 6, 7, 8 - * - N0 = 2, 3, 4, 8, 16 - * - K0 = 2, 3, 4, 8, 16 - * - H0 >= 1 - * - * @note In case of post ops, the following information must be passed at compile time: - * @note -DPOST_OP1, -DP1_ACTIVATION_TYPE, -DP1_ACTIVATION_A_VAL, -DP1_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 1 - * @note -DPOST_OP2: The arithmetic addition post op to perform at slot 2 - * @note -DPOST_OP3, -DP3_ACTIVATION_TYPE, -DP3_ACTIVATION_A_VAL, -DP3_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 3 - * - * @param[in] lhs_ptr Pointer to the LHS tensor. Supported data types: F16/F32 - * @param[in] lhs_stride_y Stride of the LHS tensor in Y dimension (in bytes) - * @param[in] lhs_stride_z Stride of the LHS tensor in Z dimension (in bytes) - * @param[in] lhs_w The size of the width dimension of the LHS tensor - * @param[in] lhs_h The size of the height dimension of the LHS tensor - * @param[in] lhs_n The size of the depth dimension of the LHS tensor - * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the LHS tensor - * @param[in] rhs_ptr Pointer to the RHS reshaped tensor. Supported data type: same as @p lhs_ptr - * @param[in] rhs_stride_y Stride of the RHS tensor in Y dimension (in bytes) - * @param[in] rhs_stride_z Stride of the RHS tensor in Z dimension (in bytes) - * @param[in] rhs_w The size of the width dimension of the RHS tensor - * @param[in] rhs_h The size of the height dimension of the RHS tensor - * @param[in] rhs_n The size of the depth dimension of the RHS tensor - * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the RHS tensor - * @param[in] bia_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr - * @param[in] bia_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) - * @param[in] bia_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) - * @param[in] bia_w (Optional) The size of the width dimension of the bias tensor - * @param[in] bia_h (Optional) The size of the height dimension of the bias tensor - * @param[in] bia_n (Optional) The size of the depth dimension of the bias tensor - * @param[in] bia_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor - * @param[in] ex0_ptr (Optional) Pointer to the tensor added with POST_OP2. Supported data type: same as @p lhs_ptr - * @param[in] ex0_stride_y (Optional) Stride of the tensor added with POST_OP2 in Y dimension (in bytes) - * @param[in] ex0_stride_z (Optional) Stride of the tensor added with POST_OP2 in Z dimension (in bytes) - * @param[in] ex0_w (Optional) The size of the width dimension of the tensor added with POST_OP2 - * @param[in] ex0_h (Optional) The size of the height dimension of the tensor added with POST_OP2 - * @param[in] ex0_n (Optional) The size of the depth dimension of the tensor added with POST_OP2 - * @param[in] ex0_offset_first_element_in_bytes (Optional) The offset of the first element in the tensor added with POST_OP2 - * @param[out] dst_ptr (Optional) Pointer to the destination tensor. Supported data type: same as @p lhs_ptr - * @param[in] dst_stride_y (Optional) Stride of the destination tensor in Y dimension (in bytes) - * @param[in] dst_stride_z (Optional) Stride of the destination tensor in Z dimension (in bytes) - * @param[in] dst_w (Optional) The size of the width dimension of the destination tensor - * @param[in] dst_h (Optional) The size of the height dimension of the destination tensor - * @param[in] dst_n (Optional) The size of the depth dimension of the destination tensor - * @param[in] dst_offset_first_element_in_bytes (Optional) The offset of the first element in the destination tensor - * @param[in] M Number of rows in LHS matrix not reshaped - * @param[in] N Number of columns in RHS matrix not reshaped - * @param[in] K Number of columns in LHS matrix and rows in RHS matrix not reshaped - */ -//! @endcond -__kernel void gemm_mm_reshaped_only_rhs_t( - TENSOR3D_T(lhs, BUFFER), - TENSOR3D_T(rhs, BUFFER), -#if defined(BETA) - TENSOR3D_T(bia, BUFFER), -#endif // defined(BETA) -#if defined(POST_OP2) - TENSOR3D_T(ex0, BUFFER), -#endif // defined(POST_OP_ADD) - TENSOR3D_T(dst, BUFFER), - const int M, - const int N, - const int K -) -{ - // Block size -#define RHS_BLOCK_SIZE ((K0) * (N0)) - - // RHS offset and step X -#if defined(RHS_INTERLEAVE) -#define RHS_OFFSET_X (K0) -#define RHS_STEP_X ((K0) * (H0)) -#define RHS_STEP_LOOP (1) -#else // defined(RHS_INTERLEAVE) -#define RHS_OFFSET_X (RHS_BLOCK_SIZE) -#define RHS_STEP_X (K0) -#define RHS_STEP_LOOP (H0) -#endif // defined(RHS_INTERLEAVE) - - const uint x = GET_SPATIAL_IDX(0, N0, 0); - const uint y = GET_SPATIAL_IDX(1, M0, PARTIAL_STORE_M0); - const uint z = GET_SPATIAL_IDX(2, 1, 0); - -#if defined(DUMMY_WORK_ITEMS) - if((x >= N) || (y >= M)) - { - return; - } -#endif // defined(DUMMY_WORK_ITEMS) - - bool x_cond = PARTIAL_STORE_N0 != 0 && ((x + N0) > N); - bool y_cond = PARTIAL_STORE_M0 != 0 && y == 0; - - TILE(uint, M0, 1, dst_indirect_y); - INITIALIZE_INDIRECT_Y(M0, PARTIAL_STORE_M0, y_cond, dst_indirect_y); - - const uint x_rhs = x / N0; - - lhs_offset_first_element_in_bytes += y * (uint)lhs_stride_y + z * (uint)lhs_stride_y * M; - rhs_offset_first_element_in_bytes += (x_rhs % H0) * (uint)RHS_OFFSET_X * sizeof(DATA_TYPE) + (x_rhs / (uint)H0) * rhs_stride_y; - -#if defined(MATRIX_B_DEPTH) - // Do not slide matrix B if the matrix B has 3 dimensions and matrix A more than 3 - rhs_offset_first_element_in_bytes += (z % MATRIX_B_DEPTH) * rhs_stride_z; -#else // defined(MATRIX_B_DEPTH) - rhs_offset_first_element_in_bytes += z * rhs_stride_z; -#endif // defined(MATRIX_B_DEPTH) - - // Initialize the accumulators - TILE(DATA_TYPE, M0, N0, c); - - LOOP_UNROLLING(int, i, 0, 1, M0, - { - c[i].v = 0; - }) - - int i = 0; - for(; i <= (K - K0); i+=K0) - { - TILE(DATA_TYPE, M0, K0, a); - TILE(DATA_TYPE, N0, K0, b); - - // Load tile from the lhs/rhs tensors - T_LOAD(DATA_TYPE, M0, K0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); - T_LOAD(DATA_TYPE, N0, K0, BUFFER, rhs, 0, 0, 1, RHS_STEP_X * sizeof(DATA_TYPE), b); - - // Compute the matrix multiplication between the two tiles - T_MMUL(DATA_TYPE, DATA_TYPE, DATA_TYPE, M0, N0, K0, NT, T, a, b, c); - - lhs_offset_first_element_in_bytes += K0 * sizeof(DATA_TYPE); - rhs_offset_first_element_in_bytes += (N0 * RHS_STEP_X * RHS_STEP_LOOP) * sizeof(DATA_TYPE); - } -#if defined(RUN_LEFTOVER_K0) - for(; i < K; ++i) - { - TILE(DATA_TYPE, M0, 1, a); - TILE(DATA_TYPE, N0, 1, b); - - // Load tile from the lhs/rhs tensors - T_LOAD(DATA_TYPE, M0, 1, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); - T_LOAD(DATA_TYPE, N0, 1, BUFFER, rhs, 0, 0, 1, RHS_STEP_X * sizeof(DATA_TYPE), b); - - T_MMUL(DATA_TYPE, DATA_TYPE, DATA_TYPE, M0, N0, 1, NT, T, a, b, c); - - lhs_offset_first_element_in_bytes += sizeof(DATA_TYPE); - rhs_offset_first_element_in_bytes += sizeof(DATA_TYPE); - } -#endif // defined(RUN_LEFTOVER_K0) - - // Multiply by the weight of matrix-matrix product and store the result -#if defined(ALPHA) - T_SCALE_CONSTANT(DATA_TYPE, M0, N0, c, (DATA_TYPE)ALPHA, c); -#endif // defined(ALPHA) - - // Add beta*bias -#if defined(BETA) -#if defined(BROADCAST_BIAS) - TILE(DATA_TYPE, 1, N0, bias0); - - T_LOAD_WIDTH_SELECT(DATA_TYPE, 1, N0, PARTIAL_STORE_N0, BUFFER, bia, x, 0, 0, x_cond, bias0); - -#ifndef UNIT_BETA - T_SCALE_CONSTANT(DATA_TYPE, 1, N0, bias0, (DATA_TYPE)BETA, bias0); -#endif // UNIT_BIAS - - // c = c + bias[broadcasted] - T_ADD_BROADCAST_X(DATA_TYPE, M0, N0, c, bias0, c); -#else // defined(BROADCAST_BIAS) - TILE(DATA_TYPE, M0, N0, bias0); - - bia_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * bia_stride_y) + (z * bia_stride_y * M); - - T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, bia, 0, bia_stride_y, x_cond, bias0, dst_indirect_y); - -#ifndef UNIT_BETA - T_SCALE_CONSTANT(DATA_TYPE, M0, N0, bias0, (DATA_TYPE)BETA, bias0); -#endif // UNIT_BIAS - - // c = c + bias - T_ADD(DATA_TYPE, M0, N0, c, bias0, c); - // c = c + bias -#endif // defined(BROADCAST_BIAS) -#endif // defined(BETA) - -#if defined(POST_OP1) - T_ACTIVATION(DATA_TYPE, M0, N0, P1_ACTIVATION_TYPE, P1_ACTIVATION_A_VAL, P1_ACTIVATION_B_VAL, c, c); -#endif // defined(POST_OP1) - -#if defined(POST_OP2) - TILE(DATA_TYPE, M0, N0, extra0); - - ex0_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * ex0_stride_y) + (z * ex0_stride_y * M); - - T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, ex0, 0, ex0_stride_y, x_cond, extra0, dst_indirect_y); - - T_ADD(DATA_TYPE, M0, N0, c, extra0, c); -#endif // defined(POST_OP2) - -#if defined(POST_OP3) - T_ACTIVATION(DATA_TYPE, M0, N0, P3_ACTIVATION_TYPE, P3_ACTIVATION_A_VAL, P3_ACTIVATION_B_VAL, c, c); -#endif // defined(POST_OP3) - - dst_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * dst_stride_y) + (z * dst_stride_y * M); - - // Store the tile in reverse order so that the invalid values are overwritten with the valid ones - T_STORE_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, dst, 0, dst_stride_y, x_cond, c, dst_indirect_y); - -#undef RHS_BLOCK_SIZE -#undef RHS_OFFSET_X -#undef RHS_STEP_X -} -#endif // defined(GEMM_RESHAPED_RHS_ONLY_T) - -#if defined(GEMM_MM_RESHAPED_ONLY_RHS_T_TEXTURE) -//! @cond Doxygen_Suppress -/** This OpenCL kernel computes the matrix multiplication between 2 matrices plus 3 post ops: - * Post op 1: activation (optional) - * Post op 2: elementwise op - * Post op 3: activation (optional) - * - * The LHS matrix is NOT reshaped - * The RHS is reshaped with @ref ClGemmMatrixMultiplyReshapedOnlyRhsKernel and the block K0xN0 is transposed - * - * @note If the first two dimensions of NDRange have been dispatched with "dummy_work_items" support, the option -DDUMMY_WORK_ITEMS must be passed at compile time. - * @note The block's dimensions used for reshaping the RHS matrix (N0 and K0) must be passed at compile time using -DN0 and -DK0 (e.g. -DN0=8, -DK0=4). - * @note The number of M0 rows to process must be passed at compile time using -DM0 (e.g. -DM0=2) - * @note The number of K0xN0 horizontal blocks stored on the same output row of the reshaped RHS matrix must be passed at compile time using -DH0 (e.g. -DH0=2) - * @note If the K0xN0 blocks in the reshaped RHS matrix have been interleaved, the option -DRHS_INTERLEAVE must passed at compile time. - * @note The size of the partial store block in y must be passed at compile time using -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_M0=1) - * @note The size of the partial store block in x must be passed at compile time using -DPARTIAL_STORE_N0 (e.g. -DPARTIAL_STORE_N0=1) - * @note Only the following configurations of M0, N0 and K0 are currently supported: - * - M0 = 1, 2, 3, 4, 5, 6, 7, 8 - * - N0 = 2, 3, 4, 8, 16 - * - K0 = 2, 3, 4, 8, 16 - * - H0 >= 1 - * - * @note In case of post ops, the following information must be passed at compile time: - * @note -DPOST_OP1, -DP1_ACTIVATION_TYPE, -DP1_ACTIVATION_A_VAL, -DP1_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 1 - * @note -DPOST_OP2: The arithmetic addition post op to perform at slot 2 - * @note -DPOST_OP3, -DP3_ACTIVATION_TYPE, -DP3_ACTIVATION_A_VAL, -DP3_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 3 - * - * @param[in] lhs_ptr Pointer to the LHS tensor. Supported data types: F16/F32 - * @param[in] lhs_stride_y Stride of the LHS tensor in Y dimension (in bytes) - * @param[in] lhs_stride_z Stride of the LHS tensor in Z dimension (in bytes) - * @param[in] lhs_w The size of the width dimension of the LHS tensor - * @param[in] lhs_h The size of the height dimension of the LHS tensor - * @param[in] lhs_n The size of the depth dimension of the LHS tensor - * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the LHS tensor - * @param[in] rhs_ptr Pointer to the RHS reshaped tensor. Supported data type: same as @p lhs_ptr - * @param[in] rhs_stride_y Stride of the RHS tensor in Y dimension (in bytes) - * @param[in] rhs_stride_z Stride of the RHS tensor in Z dimension (in bytes) - * @param[in] rhs_w The size of the width dimension of the RHS tensor - * @param[in] rhs_h The size of the height dimension of the RHS tensor - * @param[in] rhs_n The size of the depth dimension of the RHS tensor - * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the RHS tensor - * @param[in] bia_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr - * @param[in] bia_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) - * @param[in] bia_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) - * @param[in] bia_w (Optional) The size of the width dimension of the bias tensor - * @param[in] bia_h (Optional) The size of the height dimension of the bias tensor - * @param[in] bia_n (Optional) The size of the depth dimension of the bias tensor - * @param[in] bia_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor - * @param[in] ex0_ptr (Optional) Pointer to the tensor added with POST_OP2. Supported data type: same as @p lhs_ptr - * @param[in] ex0_stride_y (Optional) Stride of the tensor added with POST_OP2 in Y dimension (in bytes) - * @param[in] ex0_stride_z (Optional) Stride of the tensor added with POST_OP2 in Z dimension (in bytes) - * @param[in] ex0_w (Optional) The size of the width dimension of the tensor added with POST_OP2 - * @param[in] ex0_h (Optional) The size of the height dimension of the tensor added with POST_OP2 - * @param[in] ex0_n (Optional) The size of the depth dimension of the tensor added with POST_OP2 - * @param[in] ex0_offset_first_element_in_bytes (Optional) The offset of the first element in the tensor added with POST_OP2 - * @param[out] dst_ptr (Optional) Pointer to the destination tensor. Supported data type: same as @p lhs_ptr - * @param[in] dst_stride_y (Optional) Stride of the destination tensor in Y dimension (in bytes) - * @param[in] dst_stride_z (Optional) Stride of the destination tensor in Z dimension (in bytes) - * @param[in] dst_w (Optional) The size of the width dimension of the destination tensor - * @param[in] dst_h (Optional) The size of the height dimension of the destination tensor - * @param[in] dst_n (Optional) The size of the depth dimension of the destination tensor - * @param[in] dst_offset_first_element_in_bytes (Optional) The offset of the first element in the destination tensor - * @param[in] M Number of rows in LHS matrix not reshaped - * @param[in] N Number of columns in RHS matrix not reshaped - * @param[in] K Number of columns in LHS matrix and rows in RHS matrix not reshaped - */ -//! @endcond -__kernel void gemm_mm_reshaped_only_rhs_t_texture( - TENSOR3D_T(lhs, BUFFER), - TENSOR3D_T(rhs, IMAGE), -#if defined(BETA) - TENSOR3D_T(bia, BUFFER), -#endif // defined(BETA) -#if defined(POST_OP2) - TENSOR3D_T(ex0, BUFFER), -#endif // defined(POST_OP_ADD) - TENSOR3D_T(dst, BUFFER), - const int M, - const int N, - const int K -) -{ - // Block size -#define RHS_BLOCK_SIZE (K0 * (N0)) - - // RHS offset and step X -#if defined(RHS_INTERLEAVE) -#define RHS_OFFSET_X (K0) -#define RHS_STEP_X (K0 * (H0)) -#define RHS_STEP_LOOP (1) -#else // defined(RHS_INTERLEAVE) -#define RHS_OFFSET_X (RHS_BLOCK_SIZE) -#define RHS_STEP_X K0 -#define RHS_STEP_LOOP (H0) -#endif // defined(RHS_INTERLEAVE) - - const uint x = GET_SPATIAL_IDX(0, N0, 0); - const uint y = GET_SPATIAL_IDX(1, M0, PARTIAL_STORE_M0); - const uint z = GET_SPATIAL_IDX(2, 1, 0); - -#if defined(DUMMY_WORK_ITEMS) - if((x >= N) || (y >= M)) - { - return; - } -#endif // defined(DUMMY_WORK_ITEMS) - - bool x_cond = PARTIAL_STORE_N0 != 0 && ((x + N0) > N); - bool y_cond = PARTIAL_STORE_M0 != 0 && y == 0; - - TILE(uint, M0, 1, dst_indirect_y); - INITIALIZE_INDIRECT_Y(M0, PARTIAL_STORE_M0, y_cond, dst_indirect_y); - - lhs_offset_first_element_in_bytes += y * (uint)lhs_stride_y + z * lhs_stride_y * M; - -#if defined(MATRIX_B_DEPTH) - // Do not slide matrix B if the matrix B has 3 dimensions and matrix A more than 3 - const uint z_rhs = z % MATRIX_B_DEPTH; -#else // defined(MATRIX_B_DEPTH) - const uint z_rhs = z; -#endif // defined(MATRIX_B_DEPTH) - - uint x_rhs = ((x / N0) % H0) * (uint)RHS_OFFSET_X; - const uint y_rhs = ((x / N0) / H0) + z_rhs * rhs_h; - - // Initialize the accumulators - TILE(DATA_TYPE, M0, N0, c); - - LOOP_UNROLLING(int, i, 0, 1, M0, - { - c[i].v = 0; - }) - - TILE(DATA_TYPE, M0, K0, a); - TILE(DATA_TYPE, N0, K0, b); - - int i = 0; - for(; i <= (K - K0); i+=K0) - { - // Load tile from the lhs/rhs tensors - T_LOAD(DATA_TYPE, M0, K0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); - T_LOAD_DILATED(DATA_TYPE, N0, K0, IMAGE, rhs, x_rhs, y_rhs, RHS_STEP_X, 0, 1, b); - - // Compute the matrix multiplication between the two tiles - T_MMUL(DATA_TYPE, DATA_TYPE, DATA_TYPE, M0, N0, K0, NT, T, a, b, c); - - lhs_offset_first_element_in_bytes += K0 * sizeof(DATA_TYPE); - x_rhs += N0 * RHS_STEP_X * RHS_STEP_LOOP; - } -#if defined(RUN_LEFTOVER_K0) - T_LOAD_DILATED(DATA_TYPE, N0, K0, IMAGE, rhs, x_rhs, y_rhs, RHS_STEP_X, 0, 1, b); - - LOOP_UNROLLING(int, k0, 0, 1, PARTIAL_K, - { - LOOP_UNROLLING(int, m0, 0, 1, M0, - { - DATA_TYPE a0 = *(__global DATA_TYPE *)(lhs_ptr + lhs_offset_first_element_in_bytes + m0 * lhs_stride_y); - LOOP_UNROLLING(int, n0, 0, 1, N0, - { - c[m0].s[n0] += a0 * b[n0].s[k0]; - }) - }) - lhs_offset_first_element_in_bytes += sizeof(DATA_TYPE); - }) -#endif // defined(RUN_LEFTOVER_K0) - - // Multiply by the weight of matrix-matrix product and store the result -#if defined(ALPHA) - T_SCALE_CONSTANT(DATA_TYPE, M0, N0, c, (DATA_TYPE)ALPHA, c); -#endif // defined(ALPHA) - - // Add beta*bias -#if defined(BETA) -#if defined(BROADCAST_BIAS) - TILE(DATA_TYPE, 1, N0, bias0); - - T_LOAD_WIDTH_SELECT(DATA_TYPE, 1, N0, PARTIAL_STORE_N0, BUFFER, bia, x, 0, 0, x_cond, bias0); - -#ifndef UNIT_BETA - T_SCALE_CONSTANT(DATA_TYPE, 1, N0, bias0, (DATA_TYPE)BETA, bias0); -#endif // UNIT_BIAS - - // c = c + bias[broadcasted] - T_ADD_BROADCAST_X(DATA_TYPE, M0, N0, c, bias0, c); -#else // defined(BROADCAST_BIAS) - TILE(DATA_TYPE, M0, N0, bias0); - - bia_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * bia_stride_y) + (z * bia_stride_y * M); - - T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, bia, 0, bia_stride_y, x_cond, bias0, dst_indirect_y); - -#ifndef UNIT_BETA - T_SCALE_CONSTANT(DATA_TYPE, M0, N0, bias0, (DATA_TYPE)BETA, bias0); -#endif // UNIT_BIAS - - // c = c + bias - T_ADD(DATA_TYPE, M0, N0, c, bias0, c); - // c = c + bias -#endif // defined(BROADCAST_BIAS) -#endif // defined(BETA) - -#if defined(POST_OP1) - T_ACTIVATION(DATA_TYPE, M0, N0, P1_ACTIVATION_TYPE, P1_ACTIVATION_A_VAL, P1_ACTIVATION_B_VAL, c, c); -#endif // defined(POST_OP1) - -#if defined(POST_OP2) - TILE(DATA_TYPE, M0, N0, extra0); - - ex0_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * ex0_stride_y) + (z * ex0_stride_y * M); - - T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, ex0, 0, ex0_stride_y, x_cond, extra0, dst_indirect_y); - - T_ADD(DATA_TYPE, M0, N0, c, extra0, c); -#endif // defined(POST_OP2) - -#if defined(POST_OP3) - T_ACTIVATION(DATA_TYPE, M0, N0, P3_ACTIVATION_TYPE, P3_ACTIVATION_A_VAL, P3_ACTIVATION_B_VAL, c, c); -#endif // defined(POST_OP3) - - dst_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + y * dst_stride_y + z * dst_stride_y * M; - - // Store the tile in reverse order so that the invalid values are overwritten with the valid ones - T_STORE_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, dst, 0, dst_stride_y, x_cond, c, dst_indirect_y); - -#undef RHS_BLOCK_SIZE -#undef RHS_OFFSET_X -#undef RHS_STEP_X -} -#endif // defined(GEMM_RESHAPED_RHS_ONLY_T_TEXTURE) - -#if defined(GEMM_MM_RESHAPED_ONLY_RHS_NT) -//! @cond Doxygen_Suppress -/** This OpenCL kernel computes the matrix multiplication between 2 matrices plus 3 post ops: - * Post op 1: activation (optional) - * Post op 2: elementwise op - * Post op 3: activation (optional) - * - * The LHS matrix is NOT reshaped - * The RHS is reshaped with @ref ClGemmMatrixMultiplyReshapedOnlyRhsKernel and the block K0xN0 is not transposed - * - * @note If the first two dimensions of NDRange have been dispatched with "dummy_work_items" support, the option -DDUMMY_WORK_ITEMS must be passed at compile time. - * @note The block's dimensions used for reshaping the RHS matrix (N0 and K0) must be passed at compile time using -DN0 and -DK0 (e.g. -DN0=8, -DK0=4). - * @note The number of M0 rows to process must be passed at compile time using -DM0 (e.g. -DM0=2) - * @note The number of K0xN0 horizontal blocks stored on the same output row of the reshaped RHS matrix must be passed at compile time using -DH0 (e.g. -DH0=2) - * @note If the K0xN0 blocks in the reshaped RHS matrix have been interleaved, the option -DRHS_INTERLEAVE must passed at compile time. - * @note The size of the partial store block in y must be passed at compile time using -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_M0=1) - * @note The size of the partial store block in x must be passed at compile time using -DPARTIAL_STORE_N0 (e.g. -DPARTIAL_STORE_N0=1) - * @note Only the following configurations of M0, N0 and K0 are currently supported: - * - M0 = 1, 2, 3, 4, 5, 6, 7, 8 - * - N0 = 2, 3, 4, 8, 16 - * - K0 = 2, 3, 4, 8, 16 - * - H0 >= 1 - * - * @note In case of post ops, the following information must be passed at compile time: - * @note -DPOST_OP1, -DP1_ACTIVATION_TYPE, -DP1_ACTIVATION_A_VAL, -DP1_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 1 - * @note -DPOST_OP2: The arithmetic addition post op to perform at slot 2 - * @note -DPOST_OP3, -DP3_ACTIVATION_TYPE, -DP3_ACTIVATION_A_VAL, -DP3_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 3 - * - * @param[in] lhs_ptr Pointer to the LHS tensor. Supported data types: F16/F32 - * @param[in] lhs_stride_y Stride of the LHS tensor in Y dimension (in bytes) - * @param[in] lhs_stride_z Stride of the LHS tensor in Z dimension (in bytes) - * @param[in] lhs_w The size of the width dimension of the LHS tensor - * @param[in] lhs_h The size of the height dimension of the LHS tensor - * @param[in] lhs_n The size of the depth dimension of the LHS tensor - * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the LHS tensor - * @param[in] rhs_ptr Pointer to the RHS reshaped tensor. Supported data type: same as @p lhs_ptr - * @param[in] rhs_stride_y Stride of the RHS tensor in Y dimension (in bytes) - * @param[in] rhs_stride_z Stride of the RHS tensor in Z dimension (in bytes) - * @param[in] rhs_w The size of the width dimension of the RHS tensor - * @param[in] rhs_h The size of the height dimension of the RHS tensor - * @param[in] rhs_n The size of the depth dimension of the RHS tensor - * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the RHS tensor - * @param[in] bia_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr - * @param[in] bia_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) - * @param[in] bia_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) - * @param[in] bia_w (Optional) The size of the width dimension of the bias tensor - * @param[in] bia_h (Optional) The size of the height dimension of the bias tensor - * @param[in] bia_n (Optional) The size of the depth dimension of the bias tensor - * @param[in] bia_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor - * @param[in] ex0_ptr (Optional) Pointer to the tensor added with POST_OP2. Supported data type: same as @p lhs_ptr - * @param[in] ex0_stride_y (Optional) Stride of the tensor added with POST_OP2 in Y dimension (in bytes) - * @param[in] ex0_stride_z (Optional) Stride of the tensor added with POST_OP2 in Z dimension (in bytes) - * @param[in] ex0_w (Optional) The size of the width dimension of the tensor added with POST_OP2 - * @param[in] ex0_h (Optional) The size of the height dimension of the tensor added with POST_OP2 - * @param[in] ex0_n (Optional) The size of the depth dimension of the tensor added with POST_OP2 - * @param[in] ex0_offset_first_element_in_bytes (Optional) The offset of the first element in the tensor added with POST_OP2 - * @param[out] dst_ptr (Optional) Pointer to the destination tensor. Supported data type: same as @p lhs_ptr - * @param[in] dst_stride_y (Optional) Stride of the destination tensor in Y dimension (in bytes) - * @param[in] dst_stride_z (Optional) Stride of the destination tensor in Z dimension (in bytes) - * @param[in] dst_w (Optional) The size of the width dimension of the destination tensor - * @param[in] dst_h (Optional) The size of the height dimension of the destination tensor - * @param[in] dst_n (Optional) The size of the depth dimension of the destination tensor - * @param[in] dst_offset_first_element_in_bytes (Optional) The offset of the first element in the destination tensor - * @param[in] M Number of rows in LHS matrix not reshaped - * @param[in] N Number of columns in RHS matrix not reshaped - * @param[in] K Number of columns in LHS matrix and rows in RHS matrix not reshaped - */ -//! @endcond -__kernel void gemm_mm_reshaped_only_rhs_nt( - TENSOR3D_T(lhs, BUFFER), - TENSOR3D_T(rhs, BUFFER), -#if defined(BETA) - TENSOR3D_T(bia, BUFFER), -#endif // defined(BETA) -#if defined(POST_OP2) - TENSOR3D_T(ex0, BUFFER), -#endif // defined(POST_OP_ADD) - TENSOR3D_T(dst, BUFFER), - const int M, - const int N, - const int K -) -{ - // Block size -#define RHS_BLOCK_SIZE ((K0) * (N0)) - - // RHS offset and step X -#if defined(RHS_INTERLEAVE) -#define RHS_OFFSET_X (N0) -#define RHS_STEP_X ((N0) * (H0)) -#define RHS_STEP_LOOP (1) -#else // defined(RHS_INTERLEAVE) -#define RHS_OFFSET_X (RHS_BLOCK_SIZE) -#define RHS_STEP_X (N0) -#define RHS_STEP_LOOP (H0) -#endif // defined(RHS_INTERLEAVE) - - const uint x = GET_SPATIAL_IDX(0, N0, 0); - const uint y = GET_SPATIAL_IDX(1, M0, PARTIAL_STORE_M0); - const uint z = GET_SPATIAL_IDX(2, 1, 0); - -#if defined(DUMMY_WORK_ITEMS) - if((x >= N) || (y >= M)) - { - return; - } -#endif // defined(DUMMY_WORK_ITEMS) - - bool x_cond = PARTIAL_STORE_N0 != 0 && ((x + N0) > N); - bool y_cond = PARTIAL_STORE_M0 != 0 && y == 0; - - TILE(uint, M0, 1, dst_indirect_y); - INITIALIZE_INDIRECT_Y(M0, PARTIAL_STORE_M0, y_cond, dst_indirect_y); - - const uint x_rhs = x / N0; - - lhs_offset_first_element_in_bytes += y * (uint)lhs_stride_y + z * lhs_stride_y * M; - rhs_offset_first_element_in_bytes += (x_rhs % H0) * (uint)RHS_OFFSET_X * sizeof(DATA_TYPE) + (x_rhs / (uint)H0) * rhs_stride_y; - -#if defined(MATRIX_B_DEPTH) - // Do not slide matrix B if the matrix B has 3 dimensions and matrix A more than 3 - rhs_offset_first_element_in_bytes += (z % MATRIX_B_DEPTH) * rhs_stride_z; -#else // defined(MATRIX_B_DEPTH) - rhs_offset_first_element_in_bytes += z * rhs_stride_z; -#endif // defined(MATRIX_B_DEPTH) - - // Initialize the accumulators - TILE(DATA_TYPE, M0, N0, c); - - LOOP_UNROLLING(int, i, 0, 1, M0, - { - c[i].v = 0; - }) - - int i = 0; - for(; i <= (K - K0); i+=K0) - { - TILE(DATA_TYPE, M0, K0, a); - TILE(DATA_TYPE, K0, N0, b); - - // Load tile from the lhs/rhs tensors - T_LOAD(DATA_TYPE, M0, K0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); - T_LOAD(DATA_TYPE, K0, N0, BUFFER, rhs, 0, 0, 1, RHS_STEP_X * sizeof(DATA_TYPE), b); - - // Compute the matrix multiplication between the two tiles - T_MMUL(DATA_TYPE, DATA_TYPE, DATA_TYPE, M0, N0, K0, NT, NT, a, b, c); - - lhs_offset_first_element_in_bytes += K0 * sizeof(DATA_TYPE); - rhs_offset_first_element_in_bytes += K0 * RHS_STEP_X * RHS_STEP_LOOP * sizeof(DATA_TYPE); - } -#if defined(RUN_LEFTOVER_K0) - for(; i < K; ++i) - { - TILE(DATA_TYPE, M0, 1, a); - TILE(DATA_TYPE, 1, N0, b); - - // Load tile from the lhs/rhs tensors - T_LOAD(DATA_TYPE, M0, 1, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); - T_LOAD(DATA_TYPE, 1, N0, BUFFER, rhs, 0, 0, 1, RHS_STEP_X * sizeof(DATA_TYPE), b); - - T_MMUL(DATA_TYPE, DATA_TYPE, DATA_TYPE, M0, N0, 1, NT, NT, a, b, c); - - lhs_offset_first_element_in_bytes += sizeof(DATA_TYPE); - rhs_offset_first_element_in_bytes += RHS_STEP_X * sizeof(DATA_TYPE); - } -#endif // defined(RUN_LEFTOVER_K0) - - // Multiply by the weight of matrix-matrix product and store the result -#if defined(ALPHA) - T_SCALE_CONSTANT(DATA_TYPE, M0, N0, c, (DATA_TYPE)ALPHA, c); -#endif // defined(ALPHA) - - // Add beta*bias -#if defined(BETA) -#if defined(BROADCAST_BIAS) - TILE(DATA_TYPE, 1, N0, bias0); - - T_LOAD_WIDTH_SELECT(DATA_TYPE, 1, N0, PARTIAL_STORE_N0, BUFFER, bia, x, 0, 0, x_cond, bias0); - -#ifndef UNIT_BETA - T_SCALE_CONSTANT(DATA_TYPE, 1, N0, bias0, (DATA_TYPE)BETA, bias0); -#endif // UNIT_BIAS - - // c = c + bias[broadcasted] - T_ADD_BROADCAST_X(DATA_TYPE, M0, N0, c, bias0, c); -#else // defined(BROADCAST_BIAS) - TILE(DATA_TYPE, M0, N0, bias0); - - bia_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * bia_stride_y) + (z * bia_stride_y * M); - - T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, bia, 0, bia_stride_y, x_cond, bias0, dst_indirect_y); - -#ifndef UNIT_BETA - T_SCALE_CONSTANT(DATA_TYPE, M0, N0, bias0, (DATA_TYPE)BETA, bias0); -#endif // UNIT_BIAS - - // c = c + bias - T_ADD(DATA_TYPE, M0, N0, c, bias0, c); - // c = c + bias -#endif // defined(BROADCAST_BIAS) -#endif // defined(BETA) - -#if defined(POST_OP1) - T_ACTIVATION(DATA_TYPE, M0, N0, P1_ACTIVATION_TYPE, P1_ACTIVATION_A_VAL, P1_ACTIVATION_B_VAL, c, c); -#endif // defined(POST_OP1) - -#if defined(POST_OP2) - TILE(DATA_TYPE, M0, N0, extra0); - - ex0_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * ex0_stride_y) + (z * ex0_stride_y * M); - - T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, ex0, 0, ex0_stride_y, x_cond, extra0, dst_indirect_y); - - T_ADD(DATA_TYPE, M0, N0, c, extra0, c); -#endif // defined(POST_OP2) - -#if defined(POST_OP3) - T_ACTIVATION(DATA_TYPE, M0, N0, P3_ACTIVATION_TYPE, P3_ACTIVATION_A_VAL, P3_ACTIVATION_B_VAL, c, c); -#endif // defined(POST_OP3) - - dst_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + y * dst_stride_y + z * dst_stride_y * M; - - // Store the tile in reverse order so that the invalid values are overwritten with the valid ones - T_STORE_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, dst, 0, dst_stride_y, x_cond, c, dst_indirect_y); - -#undef RHS_BLOCK_SIZE -#undef RHS_OFFSET_X -#undef RHS_STEP_X -} -#endif // defined(GEMM_RESHAPED_RHS_ONLY_NT) - -#if defined(GEMM_MM_RESHAPED_ONLY_RHS_NT_TEXTURE) -//! @cond Doxygen_Suppress -/** This OpenCL kernel computes the matrix multiplication between 2 matrices plus 3 post ops: - * Post op 1: activation (optional) - * Post op 2: elementwise op - * Post op 3: activation (optional) - * - * The LHS matrix is NOT reshaped - * The RHS is reshaped with @ref ClGemmMatrixMultiplyReshapedOnlyRhsKernel and the block K0xN0 is not transposed - * - * @note If the first two dimensions of NDRange have been dispatched with "dummy_work_items" support, the option -DDUMMY_WORK_ITEMS must be passed at compile time. - * @note The block's dimensions used for reshaping the RHS matrix (N0 and K0) must be passed at compile time using -DN0 and -DK0 (e.g. -DN0=8, -DK0=4). - * @note The number of M0 rows to process must be passed at compile time using -DM0 (e.g. -DM0=2) - * @note The number of K0xN0 horizontal blocks stored on the same output row of the reshaped RHS matrix must be passed at compile time using -DH0 (e.g. -DH0=2) - * @note If the K0xN0 blocks in the reshaped RHS matrix have been interleaved, the option -DRHS_INTERLEAVE must passed at compile time. - * @note The size of the partial store block in y must be passed at compile time using -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_M0=1) - * @note The size of the partial store block in x must be passed at compile time using -DPARTIAL_STORE_N0 (e.g. -DPARTIAL_STORE_N0=1) - * @note Only the following configurations of M0, N0 and K0 are currently supported: - * - M0 = 1, 2, 3, 4, 5, 6, 7, 8 - * - N0 = 2, 3, 4, 8, 16 - * - K0 = 2, 3, 4, 8, 16 - * - H0 >= 1 - * - * @note In case of post ops, the following information must be passed at compile time: - * @note -DPOST_OP1, -DP1_ACTIVATION_TYPE, -DP1_ACTIVATION_A_VAL, -DP1_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 1 - * @note -DPOST_OP2: The arithmetic addition post op to perform at slot 2 - * @note -DPOST_OP3, -DP3_ACTIVATION_TYPE, -DP3_ACTIVATION_A_VAL, -DP3_ACTIVATION_B_VAL: The activation type, alpha and beta values of the activation post op at slot 3 - * - * @param[in] lhs_ptr Pointer to the LHS tensor. Supported data types: F16/F32 - * @param[in] lhs_stride_y Stride of the LHS tensor in Y dimension (in bytes) - * @param[in] lhs_stride_z Stride of the LHS tensor in Z dimension (in bytes) - * @param[in] lhs_w The size of the width dimension of the LHS tensor - * @param[in] lhs_h The size of the height dimension of the LHS tensor - * @param[in] lhs_n The size of the depth dimension of the LHS tensor - * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the LHS tensor - * @param[in] rhs_ptr Pointer to the RHS reshaped tensor. Supported data type: same as @p lhs_ptr - * @param[in] rhs_stride_y Stride of the RHS tensor in Y dimension (in bytes) - * @param[in] rhs_stride_z Stride of the RHS tensor in Z dimension (in bytes) - * @param[in] rhs_w The size of the width dimension of the RHS tensor - * @param[in] rhs_h The size of the height dimension of the RHS tensor - * @param[in] rhs_n The size of the depth dimension of the RHS tensor - * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the RHS tensor - * @param[in] bia_ptr (Optional) Pointer to the bias tensor. Supported data type: same as @p lhs_ptr - * @param[in] bia_stride_y (Optional) Stride of the bias tensor in Y dimension (in bytes) - * @param[in] bia_stride_z (Optional) Stride of the bias tensor in Z dimension (in bytes) - * @param[in] bia_w (Optional) The size of the width dimension of the bias tensor - * @param[in] bia_h (Optional) The size of the height dimension of the bias tensor - * @param[in] bia_n (Optional) The size of the depth dimension of the bias tensor - * @param[in] bia_offset_first_element_in_bytes (Optional) The offset of the first element in the bias tensor - * @param[in] ex0_ptr (Optional) Pointer to the tensor added with POST_OP2. Supported data type: same as @p lhs_ptr - * @param[in] ex0_stride_y (Optional) Stride of the tensor added with POST_OP2 in Y dimension (in bytes) - * @param[in] ex0_stride_z (Optional) Stride of the tensor added with POST_OP2 in Z dimension (in bytes) - * @param[in] ex0_w (Optional) The size of the width dimension of the tensor added with POST_OP2 - * @param[in] ex0_h (Optional) The size of the height dimension of the tensor added with POST_OP2 - * @param[in] ex0_n (Optional) The size of the depth dimension of the tensor added with POST_OP2 - * @param[in] ex0_offset_first_element_in_bytes (Optional) The offset of the first element in the tensor added with POST_OP2 - * @param[out] dst_ptr (Optional) Pointer to the destination tensor. Supported data type: same as @p lhs_ptr - * @param[in] dst_stride_y (Optional) Stride of the destination tensor in Y dimension (in bytes) - * @param[in] dst_stride_z (Optional) Stride of the destination tensor in Z dimension (in bytes) - * @param[in] dst_w (Optional) The size of the width dimension of the destination tensor - * @param[in] dst_h (Optional) The size of the height dimension of the destination tensor - * @param[in] dst_n (Optional) The size of the depth dimension of the destination tensor - * @param[in] dst_offset_first_element_in_bytes (Optional) The offset of the first element in the destination tensor - * @param[in] M Number of rows in LHS matrix not reshaped - * @param[in] N Number of columns in RHS matrix not reshaped - * @param[in] K Number of columns in LHS matrix and rows in RHS matrix not reshaped - */ -//! @endcond -__kernel void gemm_mm_reshaped_only_rhs_nt_texture( - TENSOR3D_T(lhs, BUFFER), - TENSOR3D_T(rhs, IMAGE), -#if defined(BETA) - TENSOR3D_T(bia, BUFFER), -#endif // defined(BETA) -#if defined(POST_OP2) - TENSOR3D_T(ex0, BUFFER), -#endif // defined(POST_OP_ADD) - TENSOR3D_T(dst, BUFFER), - const int M, - const int N, - const int K -) -{ - // Block size -#define RHS_BLOCK_SIZE ((K0) * (N0)) - - // RHS offset and step X -#if defined(RHS_INTERLEAVE) -#define RHS_OFFSET_X (N0) -#define RHS_STEP_X ((N0) * (H0)) -#define RHS_STEP_LOOP (1) -#else // defined(RHS_INTERLEAVE) -#define RHS_OFFSET_X (RHS_BLOCK_SIZE) -#define RHS_STEP_X (N0) -#define RHS_STEP_LOOP (H0) -#endif // defined(RHS_INTERLEAVE) - - const uint x = GET_SPATIAL_IDX(0, N0, 0); - const uint y = GET_SPATIAL_IDX(1, M0, PARTIAL_STORE_M0); - const uint z = GET_SPATIAL_IDX(2, 1, 0); - -#if defined(DUMMY_WORK_ITEMS) - if((x >= N) || (y >= M)) - { - return; - } -#endif // defined(DUMMY_WORK_ITEMS) - - bool x_cond = PARTIAL_STORE_N0 != 0 && ((x + N0) > N); - bool y_cond = PARTIAL_STORE_M0 != 0 && y == 0; - - TILE(uint, M0, 1, dst_indirect_y); - INITIALIZE_INDIRECT_Y(M0, PARTIAL_STORE_M0, y_cond, dst_indirect_y); - - lhs_offset_first_element_in_bytes += y * (uint)lhs_stride_y + z * lhs_stride_y * M; - -#if defined(MATRIX_B_DEPTH) - // Do not slide matrix B if the matrix B has 3 dimensions and matrix A more than 3 - const uint z_rhs = z % MATRIX_B_DEPTH; -#else // defined(MATRIX_B_DEPTH) - const uint z_rhs = z; -#endif // defined(MATRIX_B_DEPTH) - - uint x_rhs = ((x / N0) % H0) * (uint)RHS_OFFSET_X; - const uint y_rhs = ((x / N0) / H0) + z_rhs * rhs_h; - - // Initialize the accumulators - TILE(DATA_TYPE, M0, N0, c); - - LOOP_UNROLLING(int, i, 0, 1, M0, - { - c[i].v = 0; - }) - - int i = 0; - for(; i <= (K - K0); i+=K0) - { - TILE(DATA_TYPE, M0, K0, a); - TILE(DATA_TYPE, K0, N0, b); - - // Load tile from the lhs/rhs tensors - T_LOAD(DATA_TYPE, M0, K0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); - T_LOAD_DILATED(DATA_TYPE, K0, N0, IMAGE, rhs, x_rhs, y_rhs, RHS_STEP_X, 0, 1, b); - - // Compute the matrix multiplication between the two tiles - T_MMUL(DATA_TYPE, DATA_TYPE, DATA_TYPE, M0, N0, K0, NT, NT, a, b, c); - - lhs_offset_first_element_in_bytes += K0 * sizeof(DATA_TYPE); - x_rhs += K0 * RHS_STEP_X * RHS_STEP_LOOP; - } - -#if defined(RUN_LEFTOVER_K0) - for(; i < K; ++i) - { - TILE(DATA_TYPE, M0, 1, a); - TILE(DATA_TYPE, 1, N0, b); - - // Load tile from the lhs/rhs tensors - T_LOAD(DATA_TYPE, M0, 1, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); - T_LOAD_DILATED(DATA_TYPE, 1, N0, IMAGE, rhs, x_rhs, y_rhs, RHS_STEP_X, 0, 1, b); - - T_MMUL(DATA_TYPE, DATA_TYPE, DATA_TYPE, M0, N0, 1, NT, NT, a, b, c); - - lhs_offset_first_element_in_bytes += sizeof(DATA_TYPE); - x_rhs += RHS_STEP_X; - } -#endif // defined(RUN_LEFTOVER_K0) - - // Multiply by the weight of matrix-matrix product and store the result -#if defined(ALPHA) - T_SCALE_CONSTANT(DATA_TYPE, M0, N0, c, (DATA_TYPE)ALPHA, c); -#endif // defined(ALPHA) - - // Add beta*bias -#if defined(BETA) -#if defined(BROADCAST_BIAS) - TILE(DATA_TYPE, 1, N0, bias0); - - T_LOAD_WIDTH_SELECT(DATA_TYPE, 1, N0, PARTIAL_STORE_N0, BUFFER, bia, x, 0, 0, x_cond, bias0); - -#ifndef UNIT_BETA - T_SCALE_CONSTANT(DATA_TYPE, 1, N0, bias0, (DATA_TYPE)BETA, bias0); -#endif // UNIT_BIAS - - // c = c + bias[broadcasted] - T_ADD_BROADCAST_X(DATA_TYPE, M0, N0, c, bias0, c); -#else // defined(BROADCAST_BIAS) - TILE(DATA_TYPE, M0, N0, bias0); - - bia_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * bia_stride_y) + (z * bia_stride_y * M); - - T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, bia, 0, bia_stride_y, x_cond, bias0, dst_indirect_y); - -#ifndef UNIT_BETA - T_SCALE_CONSTANT(DATA_TYPE, M0, N0, bias0, (DATA_TYPE)BETA, bias0); -#endif // UNIT_BIAS - - // c = c + bias - T_ADD(DATA_TYPE, M0, N0, c, bias0, c); - // c = c + bias -#endif // defined(BROADCAST_BIAS) -#endif // defined(BETA) - -#if defined(POST_OP1) - T_ACTIVATION(DATA_TYPE, M0, N0, P1_ACTIVATION_TYPE, P1_ACTIVATION_A_VAL, P1_ACTIVATION_B_VAL, c, c); -#endif // defined(POST_OP1) - -#if defined(POST_OP2) - TILE(DATA_TYPE, M0, N0, extra0); - - ex0_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + (y * ex0_stride_y) + (z * ex0_stride_y * M); - - T_LOAD_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, ex0, 0, ex0_stride_y, x_cond, extra0, dst_indirect_y); - - T_ADD(DATA_TYPE, M0, N0, c, extra0, c); -#endif // defined(POST_OP2) - -#if defined(POST_OP3) - T_ACTIVATION(DATA_TYPE, M0, N0, P3_ACTIVATION_TYPE, P3_ACTIVATION_A_VAL, P3_ACTIVATION_B_VAL, c, c); -#endif // defined(POST_OP3) - - dst_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + y * dst_stride_y + z * dst_stride_y * M; - - // Store the tile in reverse order so that the invalid values are overwritten with the valid ones - T_STORE_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, dst, 0, dst_stride_y, x_cond, c, dst_indirect_y); - -#undef RHS_BLOCK_SIZE -#undef RHS_OFFSET_X -#undef RHS_STEP_X -} -#endif // defined(GEMM_RESHAPED_RHS_ONLY_NT_TEXTURE)
\ No newline at end of file |