From 5e99a3e4d65f814c5e6938c31a0ef505d0fb8f17 Mon Sep 17 00:00:00 2001 From: Jakub Sujak Date: Tue, 18 Apr 2023 08:33:56 +0100 Subject: Add quantized CL MatMul kernel for LHS NT, RHS T Implement a native kernel for batched Matrix Multiplication for the quantized data types QASYMM8 and QASYMM8_SIGNED and with the MatMul attributes `adj_x = false, adj_y = true`. Resolves: COMPMID-5923 Change-Id: I477b2dd886edfe83beaba9efc7d6b05ed19f5da4 Signed-off-by: Jakub Sujak Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/9467 Tested-by: Arm Jenkins Reviewed-by: SiCong Li Comments-Addressed: Arm Jenkins Benchmark: Arm Jenkins --- src/core/CL/cl_kernels/common/mat_mul_quantized.cl | 176 +++++++++++++++++++++ src/gpu/cl/ClKernelLibrary.cpp | 1 + tests/validation/CL/MatMulLowpNativeKernel.cpp | 97 ++++-------- 3 files changed, 208 insertions(+), 66 deletions(-) diff --git a/src/core/CL/cl_kernels/common/mat_mul_quantized.cl b/src/core/CL/cl_kernels/common/mat_mul_quantized.cl index 5c931d2fc1..0c3cbca9a6 100644 --- a/src/core/CL/cl_kernels/common/mat_mul_quantized.cl +++ b/src/core/CL/cl_kernels/common/mat_mul_quantized.cl @@ -208,6 +208,182 @@ __kernel void mat_mul_native_quantized_nt_nt( } #endif // defined(MAT_MUL_NATIVE_QUANTIZED_NT_NT) +#if defined(MAT_MUL_NATIVE_QUANTIZED_NT_T) +/** This OpenCL kernel performs the batch matrix multiplication (BatchMatMul): LHS non-transposed, RHS transposed - buffer only + * + * @note the "batch" here expresses the number of matrix multiplications to run in parallel. However, it + * should NOT be confused with the batch size of the model. For NHWC the "batch" is the "H" dimension + * @note The data type must be passed at compile time using -DDATA_TYPE (e.g. -DDATA_TYPE=uchar) + * @note The block's dimensions used for the LHS and RHS matrices (M0, N0 and K0) must be passed at compile time using -DN0, -DM0 and -DK0 (e.g. -DN0=8, -DM0=4, -DK0=4). + * @note The number of leftover outputs rows/columns must be passed using -DPARTIAL_STORE_N0 and -DPARTIAL_STORE_M0 (e.g. -DPARTIAL_STORE_N0=2, -DPARTIAL_STORE_M0=3) + * @note The dimension K must be passed at compile time using -DK (e.g. -DK=6) + * @note The kernel name in uppercase must be passed at compile time (e.g. -DMAT_MUL_NATIVE_QUANTIZED_NT_T) + * @note Only the following configurations of M0, N0 and K0 are currently supported: + * - M0 > 0 + * - N0 = 1, 2, 3, 4, 8, 16 + * - K0 = 1, 2, 3, 4, 8, 16 + * @note Values > 8 for M0, N0, K0 are not expected to be efficient + * + * @param[in] lhs_ptr Pointer to the lhs matrix. Supported data types: QASYMM8/QASYMM8_SIGNED + * @param[in] lhs_stride_y Stride of the lhs matrix in Y (2nd) dimension (in bytes) + * @param[in] lhs_stride_z Stride of the lhs tensor in Z (3rd) dimension (in bytes) + * @param[in] lhs_w The width of the lhs tensor + * @param[in] lhs_h The height of the lhs tensor + * @param[in] lhs_n Number of the matrices (buffers) in the batch + * @param[in] lhs_offset_first_element_in_bytes The offset of the first element in the lhs matrix + * @param[in] rhs_ptr Pointer to the rhs matrix. Supported data types: same as @p lhs_ptr + * @param[in] rhs_stride_y Stride of the rhs matrix in Y (2nd) dimension (in bytes) + * @param[in] rhs_stride_z Stride of the rhs tensor in Z (3rd) dimension (in bytes) + * @param[in] rhs_w The width of the rhs tensor + * @param[in] rhs_h The height of the rhs tensor + * @param[in] rhs_n Number of the matrices (buffers) in the batch + * @param[in] rhs_offset_first_element_in_bytes The offset of the first element in the rhs matrix + * @param[out] dst_ptr Pointer to the dst matrix. Supported data types: same as @p lhs_ptr + * @param[in] dst_stride_y Stride of the dst matrix in Y (2nd) dimension (in bytes) + * @param[in] dst_stride_z Stride of the dst tensor in Z (3rd) dimension (in bytes) + * @param[in] dst_w The width of the dst tensor + * @param[in] dst_h The height of the dst tensor + * @param[in] dst_n Number of the matrices (buffers) in the batch + * @param[in] dst_offset_first_element_in_bytes The offset of the first element in the dst matrix + */ +__kernel void mat_mul_native_quantized_nt_t( + TENSOR3D_T(lhs, BUFFER), + TENSOR3D_T(rhs, BUFFER), + TENSOR3D_T(dst, BUFFER)) +{ + const uint x = GET_SPATIAL_IDX(0, N0, PARTIAL_STORE_N0); + const uint y = GET_SPATIAL_IDX(1, M0, PARTIAL_STORE_M0); + const uint z = GET_SPATIAL_IDX(2, 1, 0); + + // Compute LHS/RHS/DST matrix address + lhs_offset_first_element_in_bytes += y * lhs_stride_y + z * lhs_stride_z; + rhs_offset_first_element_in_bytes += x * rhs_stride_y + z * rhs_stride_z; + dst_offset_first_element_in_bytes += x * sizeof(DATA_TYPE) + y * dst_stride_y + z * dst_stride_z; + + // Initialize the accumulators + TILE(int, M0, N0, acc); + LOOP_UNROLLING(int, i, 0, 1, M0, + { + acc[i].v = K * ((int)LHS_OFFSET) * ((int)RHS_OFFSET); + }) + + TILE(int, 1, M0, a_sum); + a_sum[0].v = 0; + + TILE(int, 1, N0, b_sum); + b_sum[0].v = 0; + + int k; + for(k = 0; k <= K - K0; k += K0) + { + TILE(DATA_TYPE, M0, K0, a); + TILE(DATA_TYPE, N0, K0, b); + + LOOP_UNROLLING(int, i, 0, 1, M0, + { + a[i].v = 0; + }) + + LOOP_UNROLLING(int, i, 0, 1, N0, + { + b[i].v = 0; + }) + + // Load tile from lhs/rhs tensors + T_LOAD(DATA_TYPE, M0, K0, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); + T_LOAD(DATA_TYPE, N0, K0, BUFFER, rhs, 0, 0, 1, rhs_stride_y, b); + + T_MMUL(DATA_TYPE, DATA_TYPE, int, M0, N0, K0, NT, T, a, b, acc); + + LOOP_UNROLLING(int, i, 0, 1, M0, + { + LOOP_UNROLLING(int, j, 0, 1, K0, + { + a_sum[0].s[i] += (int)a[i].s[j]; + }) + }) + + LOOP_UNROLLING(int, i, 0, 1, N0, + { + LOOP_UNROLLING(int, j, 0, 1, K0, + { + b_sum[0].s[i] += (int)b[i].s[j]; + }) + }) + + lhs_offset_first_element_in_bytes += K0 * sizeof(DATA_TYPE); + rhs_offset_first_element_in_bytes += K0 * sizeof(DATA_TYPE); + } + +#if ((K % K0) != 0) + // Leftover loop + for(; k < K; ++k) + { + TILE(DATA_TYPE, M0, 1, a); + TILE(DATA_TYPE, N0, 1, b); + + LOOP_UNROLLING(int, i, 0, 1, M0, + { + a[i].v = 0; + }) + + LOOP_UNROLLING(int, i, 0, 1, N0, + { + b[i].v = 0; + }) + + // Load tile from lhs/rhs tensors + T_LOAD(DATA_TYPE, M0, 1, BUFFER, lhs, 0, 0, 1, lhs_stride_y, a); + T_LOAD(DATA_TYPE, N0, 1, BUFFER, rhs, 0, 0, 1, rhs_stride_y, b); + + T_MMUL(DATA_TYPE, DATA_TYPE, int, M0, N0, 1, NT, T, a, b, acc); + + LOOP_UNROLLING(int, i, 0, 1, M0, + { + LOOP_UNROLLING(int, j, 0, 1, 1, + { + a_sum[0].s[i] += (int)a[i].s[j]; + }) + }) + + LOOP_UNROLLING(int, i, 0, 1, N0, + { + LOOP_UNROLLING(int, j, 0, 1, 1, + { + b_sum[0].s[i] += (int)b[i].s[j]; + }) + }) + + lhs_offset_first_element_in_bytes += 1 * sizeof(DATA_TYPE); + rhs_offset_first_element_in_bytes += 1 * sizeof(DATA_TYPE); + } +#endif // ((K % K0) != 0) + + LOOP_UNROLLING(int, i, 0, 1, M0, + { + LOOP_UNROLLING(int, j, 0, 1, N0, + { + acc[i].s[j] += ((int)(RHS_OFFSET)) * a_sum[0].s[i] + ((int)(LHS_OFFSET)) * b_sum[0].s[j]; + }) + }) + + const bool x_cond = PARTIAL_STORE_N0 != 0 && get_global_id(0) == 0; + const bool y_cond = PARTIAL_STORE_M0 != 0 && get_global_id(1) == 0; + + // Quantize the tile + TILE(DATA_TYPE, M0, N0, accq); + T_QUANTIZE8_ASYMMETRIC(int, DATA_TYPE, M0, N0, DST_OFFSET, DST_SHIFT, DST_MULTIPLIER, acc, accq); + + TILE(int, M0, 1, indirect_buffer); + LOOP_UNROLLING(int, _i, 0, 1, M0, + { + indirect_buffer[_i].v = min(_i, select(M0 - 1, PARTIAL_STORE_M0 - 1, y_cond)); + }); + + T_STORE_INDIRECT_WIDTH_SELECT(DATA_TYPE, M0, N0, PARTIAL_STORE_N0, BUFFER, dst, 0, dst_stride_y, x_cond, accq, indirect_buffer); +} +#endif // defined(MAT_MUL_NATIVE_QUANTIZED_NT_T) + #if defined(MAT_MUL_NATIVE_QUANTIZED_T_NT) /** This OpenCL kernel performs the batch matrix multiplication (BatchMatMul): LHS transposed, RHS non-transposed * diff --git a/src/gpu/cl/ClKernelLibrary.cpp b/src/gpu/cl/ClKernelLibrary.cpp index 4612ca35b8..a9080049b5 100644 --- a/src/gpu/cl/ClKernelLibrary.cpp +++ b/src/gpu/cl/ClKernelLibrary.cpp @@ -324,6 +324,7 @@ const std::map ClKernelLibrary::_kernel_program_map = { "mat_mul_native_t_nt", "common/mat_mul.cl" }, { "mat_mul_native_t_t", "common/mat_mul.cl" }, { "mat_mul_native_quantized_nt_nt", "common/mat_mul_quantized.cl" }, + { "mat_mul_native_quantized_nt_t", "common/mat_mul_quantized.cl" }, { "mat_mul_native_quantized_t_nt", "common/mat_mul_quantized.cl" }, { "mat_mul_native_quantized_t_t", "common/mat_mul_quantized.cl" }, { "max_unpooling_layer_2", "common/unpooling_layer.cl" }, diff --git a/tests/validation/CL/MatMulLowpNativeKernel.cpp b/tests/validation/CL/MatMulLowpNativeKernel.cpp index a0b2a37b4b..fd7a4cb156 100644 --- a/tests/validation/CL/MatMulLowpNativeKernel.cpp +++ b/tests/validation/CL/MatMulLowpNativeKernel.cpp @@ -212,7 +212,7 @@ TEST_SUITE(Quantized) TEST_SUITE(QASYMM8_SIGNED) FIXTURE_DATA_TEST_CASE(RunTiny, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::TinyMatMulDataset(), framework::dataset::make("TransposeA", { true, false })), - framework::dataset::make("TransposeB", { false })), + framework::dataset::make("TransposeB", { true, false })), m0_values_precommit), n0_values_precommit), k0_values_precommit), @@ -224,7 +224,7 @@ FIXTURE_DATA_TEST_CASE(RunTiny, CLMatMulLowpNativeKernelFixture, framewo } FIXTURE_DATA_TEST_CASE(RunSmall, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::SmallMatMulDataset(), framework::dataset::make("TransposeA", { true, false })), - framework::dataset::make("TransposeB", { false })), + framework::dataset::make("TransposeB", { true, false })), m0_values_precommit), n0_values_precommit), k0_values_precommit), @@ -234,30 +234,6 @@ FIXTURE_DATA_TEST_CASE(RunSmall, CLMatMulLowpNativeKernelFixture, framew // Validate output validate(CLAccessor(_target), _reference, tolerance_quant); } -FIXTURE_DATA_TEST_CASE(RunTiny_T_T, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::TinyMatMulDataset(), - framework::dataset::make("TransposeA", { true })), - framework::dataset::make("TransposeB", { true })), - m0_values_precommit), - n0_values_precommit), - k0_values_precommit), - framework::dataset::make("ExportRhsToCLImage", { false })), - framework::dataset::make("DataType", DataType::QASYMM8_SIGNED))) -{ - // Validate output - validate(CLAccessor(_target), _reference, tolerance_quant); -} -FIXTURE_DATA_TEST_CASE(RunSmall_T_T, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::SmallMatMulDataset(), - framework::dataset::make("TransposeA", { true })), - framework::dataset::make("TransposeB", { true })), - m0_values_precommit), - n0_values_precommit), - k0_values_precommit), - framework::dataset::make("ExportRhsToCLImage", { false })), - framework::dataset::make("DataType", DataType::QASYMM8_SIGNED))) -{ - // Validate output - validate(CLAccessor(_target), _reference, tolerance_quant); -} FIXTURE_DATA_TEST_CASE(RunLargeNoTranspose, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulDataset(), framework::dataset::make("TransposeA", { false })), @@ -271,6 +247,19 @@ FIXTURE_DATA_TEST_CASE(RunLargeNoTranspose, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::NIGHTLY, + combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulDataset(), + framework::dataset::make("TransposeA", { false })), + framework::dataset::make("TransposeB", { true })), + m0_values_nightly_lhs_nt), + n0_values_nightly_rhs_t), + k0_values_nightly_rhs_t), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::QASYMM8_SIGNED))) +{ + // Validate output + validate(CLAccessor(_target), _reference, tolerance_quant); +} FIXTURE_DATA_TEST_CASE(RunLargeLhsTransposed, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulDataset(), framework::dataset::make("TransposeA", { true })), @@ -302,20 +291,7 @@ FIXTURE_DATA_TEST_CASE(RunLargeLhsTransposedRhsTransposed, CLMatMulLowpNativeKer FIXTURE_DATA_TEST_CASE(RunHighDimensional, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::HighDimensionalMatMulDataset(), framework::dataset::make("TransposeA", { true, false })), - framework::dataset::make("TransposeB", { false })), - framework::dataset::make("M0", { 2 })), - framework::dataset::make("N0", { 2 })), - framework::dataset::make("K0", { 2 })), - framework::dataset::make("ExportRhsToCLImage", { false })), - framework::dataset::make("DataType", DataType::QASYMM8_SIGNED))) -{ - // Validate output - validate(CLAccessor(_target), _reference, tolerance_quant); -} -FIXTURE_DATA_TEST_CASE(RunHighDimensional_T_T, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::ALL, - combine(combine(combine(combine(combine(combine(combine(datasets::HighDimensionalMatMulDataset(), - framework::dataset::make("TransposeA", { true })), - framework::dataset::make("TransposeB", { true })), + framework::dataset::make("TransposeB", { true, false })), framework::dataset::make("M0", { 2 })), framework::dataset::make("N0", { 2 })), framework::dataset::make("K0", { 2 })), @@ -330,7 +306,7 @@ TEST_SUITE_END() // QASYMM8_SIGNED TEST_SUITE(QASYMM8) FIXTURE_DATA_TEST_CASE(RunTiny, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::TinyMatMulDataset(), framework::dataset::make("TransposeA", { true, false })), - framework::dataset::make("TransposeB", { false })), + framework::dataset::make("TransposeB", { true, false })), m0_values_precommit), n0_values_precommit), k0_values_precommit), @@ -342,7 +318,7 @@ FIXTURE_DATA_TEST_CASE(RunTiny, CLMatMulLowpNativeKernelFixture, framew } FIXTURE_DATA_TEST_CASE(RunSmall, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::SmallMatMulDataset(), framework::dataset::make("TransposeA", { true, false })), - framework::dataset::make("TransposeB", { false })), + framework::dataset::make("TransposeB", { true, false })), m0_values_precommit), n0_values_precommit), k0_values_precommit), @@ -352,30 +328,6 @@ FIXTURE_DATA_TEST_CASE(RunSmall, CLMatMulLowpNativeKernelFixture, frame // Validate output validate(CLAccessor(_target), _reference, tolerance_quant); } -FIXTURE_DATA_TEST_CASE(RunTiny_T_T, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::TinyMatMulDataset(), - framework::dataset::make("TransposeA", { true })), - framework::dataset::make("TransposeB", { true })), - m0_values_precommit), - n0_values_precommit), - k0_values_precommit), - framework::dataset::make("ExportRhsToCLImage", { false })), - framework::dataset::make("DataType", DataType::QASYMM8))) -{ - // Validate output - validate(CLAccessor(_target), _reference, tolerance_quant); -} -FIXTURE_DATA_TEST_CASE(RunSmall_T_T, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::ALL, combine(combine(combine(combine(combine(combine(combine(datasets::SmallMatMulDataset(), - framework::dataset::make("TransposeA", { true })), - framework::dataset::make("TransposeB", { true })), - m0_values_precommit), - n0_values_precommit), - k0_values_precommit), - framework::dataset::make("ExportRhsToCLImage", { false })), - framework::dataset::make("DataType", DataType::QASYMM8))) -{ - // Validate output - validate(CLAccessor(_target), _reference, tolerance_quant); -} FIXTURE_DATA_TEST_CASE(RunLargeNoTranspose, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulDataset(), framework::dataset::make("TransposeA", { false })), @@ -389,6 +341,19 @@ FIXTURE_DATA_TEST_CASE(RunLargeNoTranspose, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::NIGHTLY, + combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulDataset(), + framework::dataset::make("TransposeA", { false })), + framework::dataset::make("TransposeB", { true })), + m0_values_nightly_lhs_nt), + n0_values_nightly_rhs_t), + k0_values_nightly_rhs_t), + framework::dataset::make("ExportRhsToCLImage", { false })), + framework::dataset::make("DataType", DataType::QASYMM8))) +{ + // Validate output + validate(CLAccessor(_target), _reference, tolerance_quant); +} FIXTURE_DATA_TEST_CASE(RunLargeLhsTransposed, CLMatMulLowpNativeKernelFixture, framework::DatasetMode::NIGHTLY, combine(combine(combine(combine(combine(combine(combine(datasets::LargeMatMulDataset(), framework::dataset::make("TransposeA", { true })), -- cgit v1.2.1