From 229757be70e7884b1a64972c2bdf9919075401a2 Mon Sep 17 00:00:00 2001 From: Gian Marco Iodice Date: Sun, 15 Nov 2020 16:06:10 +0000 Subject: COMPMID-3975: Update CLGEMM heuristic for Mali-G52 F16 - Update the GEMM heuristic for Mali-G52 - Fix the GEMV heuristic on Mali-G76 Change-Id: I895153d02d09e73c51ecf158e2ae43f81f6d7b05 Signed-off-by: Gian Marco Iodice Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/4425 Tested-by: Arm Jenkins Reviewed-by: Michele Di Giorgio Comments-Addressed: Arm Jenkins --- .../CLGEMMReshapedKernelConfigurationBifrost.cpp | 18 ++- .../CLGEMMReshapedKernelConfigurationBifrost.h | 1 + ...MMReshapedOnlyRHSKernelConfigurationBifrost.cpp | 85 +++++++++++++- ...GEMMReshapedOnlyRHSKernelConfigurationBifrost.h | 1 + .../CL/gemm/CLGEMMKernelSelectionBifrost.cpp | 122 ++++++++++++++++++++- src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.h | 1 + 6 files changed, 221 insertions(+), 7 deletions(-) diff --git a/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.cpp b/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.cpp index 70992974a3..e5c89f89af 100644 --- a/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.cpp +++ b/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.cpp @@ -64,7 +64,7 @@ std::pair CLGEMMReshapedKernelConfiguratio static std::map gemm_configs_G52 = { { DataType::F32, &CLGEMMReshapedKernelConfigurationBifrost::configure_G52_f32 }, - { DataType::F16, &CLGEMMReshapedKernelConfigurationBifrost::configure_G7x_f16 }, + { DataType::F16, &CLGEMMReshapedKernelConfigurationBifrost::configure_G52_f16 }, { DataType::QASYMM8, &CLGEMMReshapedKernelConfigurationBifrost::configure_G7x_u8 }, { DataType::QSYMM8, &CLGEMMReshapedKernelConfigurationBifrost::configure_G7x_u8 }, { DataType::QASYMM8_SIGNED, &CLGEMMReshapedKernelConfigurationBifrost::configure_G7x_u8 }, @@ -263,6 +263,22 @@ std::pair CLGEMMReshapedKernelConfiguratio } } +std::pair CLGEMMReshapedKernelConfigurationBifrost::configure_G52_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b) +{ + ARM_COMPUTE_UNUSED(k); + + const float workload = (static_cast(m) * static_cast(n) * static_cast(b)) / 20.0f; + + if(workload <= 232.8000f) + { + return configure_lhs_rhs_info(m, n, 2, 4, 4, 4, 4, true, true, true, false, false); + } + else + { + return configure_lhs_rhs_info(m, n, 4, 4, 4, 4, 4, true, true, true, false, false); + } +} + std::pair CLGEMMReshapedKernelConfigurationBifrost::configure_G76_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b) { ARM_COMPUTE_UNUSED(k); diff --git a/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.h b/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.h index 6c67b70962..715e7abfa3 100644 --- a/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.h +++ b/src/core/CL/gemm/reshaped/CLGEMMReshapedKernelConfigurationBifrost.h @@ -48,6 +48,7 @@ private: std::pair configure_G52_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b); std::pair configure_G76_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b); std::pair configure_G7x_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b); + std::pair configure_G52_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b); std::pair configure_G76_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b); std::pair configure_G7x_u8(unsigned int m, unsigned int n, unsigned int k, unsigned int b); std::pair configure_G76_u8(unsigned int m, unsigned int n, unsigned int k, unsigned int b); diff --git a/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.cpp b/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.cpp index 188ba4d9c5..a2c1ed2c8e 100644 --- a/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.cpp +++ b/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.cpp @@ -65,7 +65,7 @@ std::pair CLGEMMReshapedOnlyRHSKernelConfi static std::map gemm_configs_G52 = { { DataType::F32, &CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G52_f32 }, - { DataType::F16, &CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G7x_f16 }, + { DataType::F16, &CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G52_f16 }, { DataType::QASYMM8, &CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G7x_u8 }, { DataType::QSYMM8, &CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G7x_u8 }, { DataType::QASYMM8_SIGNED, &CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G7x_u8 }, @@ -168,16 +168,25 @@ std::pair CLGEMMReshapedOnlyRHSKernelConfi GEMMRHSMatrixInfo rhs_info_img; const bool is_workload_big = ((m * n * b) / 16) >= 2048; - // Get lhs_info/rhs_info in case of OpenCL buffer + if(m == 1) { - if(n <= 204.0) + if(n >= 8192) { - return configure_lhs_rhs_info(m, n, 1, 2, 16, 1, 16, false, true, false, true, false); + const unsigned int h0 = std::max(n / 4, 1U); + return configure_lhs_rhs_info(m, n, 1, 4, 8, 1, h0, false, true, false, true, false); } else { - return configure_lhs_rhs_info(m, n, 1, 2, 8, 1, 32, false, true, false, true, false); + const unsigned int h0 = std::max(n / 2, 1U); + if(n <= 204) + { + return configure_lhs_rhs_info(m, n, 1, 2, 16, 1, h0, false, true, false, true, false); + } + else + { + return configure_lhs_rhs_info(m, n, 1, 2, 8, 1, h0, false, true, false, true, false); + } } } else @@ -306,6 +315,72 @@ std::pair CLGEMMReshapedOnlyRHSKernelConfi } } +std::pair CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G52_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b) +{ + const float r_mn = static_cast(m) / static_cast(n); + const float workload = (static_cast(m) * static_cast(n) * static_cast(b)) / 20.0f; + const float r_mk = static_cast(m) / static_cast(k); + const float r_nk = static_cast(n) / static_cast(k); + + if(m == 1) + { + if(r_mk <= 0.0026f) + { + if(r_nk <= 0.4664f) + { + return configure_lhs_rhs_info(m, n, 1, 2, 16, 1, 32, false, true, false, true, false); + } + else + { + return configure_lhs_rhs_info(m, n, 1, 4, 16, 1, 16, false, true, false, false, true); + } + } + else + { + if(r_mk <= 0.0148f) + { + return configure_lhs_rhs_info(m, n, 1, 2, 16, 1, 32, false, true, false, true, false); + } + else + { + return configure_lhs_rhs_info(m, n, 1, 4, 16, 1, 16, false, true, false, false, true); + } + } + } + else + { + if(workload <= 362.6000f) + { + return configure_lhs_rhs_info(m, n, 2, 2, 8, 1, 16, false, false, false, true, false); + } + else + { + if(r_mn <= 22.6067f) + { + if(workload <= 708.8000f) + { + return configure_lhs_rhs_info(m, n, 5, 4, 4, 1, 2, false, false, false, false, true); + } + else + { + return configure_lhs_rhs_info(m, n, 5, 8, 2, 1, 16, false, false, false, false, false); + } + } + else + { + if(r_nk <= 0.0917f) + { + return configure_lhs_rhs_info(m, n, 2, 2, 8, 1, 16, false, false, false, true, false); + } + else + { + return configure_lhs_rhs_info(m, n, 5, 4, 4, 1, 2, false, false, false, false, true); + } + } + } + } +} + std::pair CLGEMMReshapedOnlyRHSKernelConfigurationBifrost::configure_G76_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b) { ARM_COMPUTE_UNUSED(k); diff --git a/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.h b/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.h index 3dfd96a822..4d284ed3e8 100644 --- a/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.h +++ b/src/core/CL/gemm/reshaped_only_rhs/CLGEMMReshapedOnlyRHSKernelConfigurationBifrost.h @@ -49,6 +49,7 @@ private: std::pair configure_G52_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b); std::pair configure_G51_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b); std::pair configure_G7x_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b); + std::pair configure_G52_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b); std::pair configure_G76_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b); std::pair configure_G51_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b); std::pair configure_G7x_u8(unsigned int m, unsigned int n, unsigned int k, unsigned int b); diff --git a/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.cpp b/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.cpp index c77746a044..46d07fffba 100644 --- a/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.cpp +++ b/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.cpp @@ -72,7 +72,7 @@ CLGEMMKernelType CLGEMMKernelSelectionBifrost::select_kernel(const CLGEMMKernelS static std::map gemm_g52_configs = { { DataType::F32, &CLGEMMKernelSelectionBifrost::g52_f32 }, - { DataType::F16, &CLGEMMKernelSelectionBifrost::default_f16 }, + { DataType::F16, &CLGEMMKernelSelectionBifrost::g52_f16 }, { DataType::QASYMM8, &CLGEMMKernelSelectionBifrost::default_q8 }, { DataType::QASYMM8_SIGNED, &CLGEMMKernelSelectionBifrost::default_q8 }, { DataType::QSYMM8, &CLGEMMKernelSelectionBifrost::default_q8 }, @@ -443,6 +443,126 @@ CLGEMMKernelType CLGEMMKernelSelectionBifrost::g76_f16(unsigned int m, unsigned } } +CLGEMMKernelType CLGEMMKernelSelectionBifrost::g52_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant) +{ + ARM_COMPUTE_UNUSED(b); + + if (!is_rhs_constant) + { + return CLGEMMKernelType::NATIVE_V1; + } + + if (m == 1) + { + return CLGEMMKernelType::RESHAPED_ONLY_RHS; + } + + const float r_mn = static_cast(m) / static_cast(n); + const float r_mk = static_cast(m) / static_cast(k); + const float r_nk = static_cast(n) / static_cast(k); + const float r_mnk = static_cast(m) / (static_cast(n) * static_cast(k)); + + if(r_mn <= 22.9200f) + { + if(r_mk <= 0.0157f) + { + return CLGEMMKernelType::RESHAPED_ONLY_RHS; + } + else + { + if(r_mnk <= 7809.3750f) + { + if(r_mnk <= 101.7937f) + { + if(r_mn <= 0.4594f) + { + if(r_mk <= 0.0557f) + { + return CLGEMMKernelType::RESHAPED_ONLY_RHS; + } + else + { + return CLGEMMKernelType::RESHAPED; + } + } + else + { + return CLGEMMKernelType::RESHAPED_ONLY_RHS; + } + } + else + { + if(r_nk <= 0.4396f) + { + if(r_mn <= 1.5182f) + { + if(r_mnk <= 1709.9167f) + { + return CLGEMMKernelType::RESHAPED; + } + else + { + return CLGEMMKernelType::RESHAPED_ONLY_RHS; + } + } + else + { + if(r_mnk <= 1330.6000f) + { + return CLGEMMKernelType::RESHAPED_ONLY_RHS; + } + else + { + return CLGEMMKernelType::RESHAPED; + } + } + } + else + { + if(r_mn <= 2.5896f) + { + return CLGEMMKernelType::RESHAPED; + } + else + { + if(r_mnk <= 326.6667f) + { + return CLGEMMKernelType::RESHAPED_ONLY_RHS; + } + else + { + return CLGEMMKernelType::RESHAPED; + } + } + } + } + } + else + { + return CLGEMMKernelType::RESHAPED_ONLY_RHS; + } + } + } + else + { + if(r_mn <= 86.7578f) + { + if(r_mnk <= 11231.6406f) + { + return CLGEMMKernelType::RESHAPED_ONLY_RHS; + } + else + { + return CLGEMMKernelType::RESHAPED; + } + } + else + { + return CLGEMMKernelType::RESHAPED_ONLY_RHS; + } + } +} + CLGEMMKernelType CLGEMMKernelSelectionBifrost::g71_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant) { ARM_COMPUTE_UNUSED(b); diff --git a/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.h b/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.h index fbafc531f5..6831a12aec 100644 --- a/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.h +++ b/src/runtime/CL/gemm/CLGEMMKernelSelectionBifrost.h @@ -47,6 +47,7 @@ private: CLGEMMKernelType g52_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant); CLGEMMKernelType g76_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant); CLGEMMKernelType g76_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant); + CLGEMMKernelType g52_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant); CLGEMMKernelType g71_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant); CLGEMMKernelType default_f32(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant); CLGEMMKernelType default_f16(unsigned int m, unsigned int n, unsigned int k, unsigned int b, bool is_rhs_constant); -- cgit v1.2.1