From 4ee8b1599dbaf7634d25607fa5ac96ba3dc6b0f2 Mon Sep 17 00:00:00 2001
From: Georgios Pinitas <georgios.pinitas@arm.com>
Date: Fri, 16 Jul 2021 16:16:43 +0100
Subject: Update GEMM assembly kernels

- Introduce Fp32 kernels with internal calculations in Bfloat16 when
fast_mode is enabled
- Improve kernel selection heuristics

Signed-off-by: Georgios Pinitas <georgios.pinitas@arm.com>
Change-Id: I68a9e7e862b6fd2721b46e0d7cc791091c4ab279
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/5965
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
---
 .../kernels/sve_interleaved_u8u32_dot_8x3VL.hpp    | 75 ++++++++++++++++------
 1 file changed, 57 insertions(+), 18 deletions(-)

(limited to 'src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL.hpp')

diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL.hpp
index 7d39485164..f66a9bf51e 100644
--- a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL.hpp
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_8x3VL.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019-2020 Arm Limited.
+ * Copyright (c) 2019-2021 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -10,64 +10,103 @@
  * sell copies of the Software, and to permit persons to whom the Software is
  * furnished to do so, subject to the following conditions:
  *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
  */
 #pragma once
 
 #ifdef ARM_COMPUTE_ENABLE_SVE
-
-#include <cstdint>
 #include "../std_transforms_sve.hpp"
+#include "../performance_parameters.hpp"
 
-namespace arm_gemm {
+#define ARGLIST  \
+    const uint8_t *, const uint8_t *, \
+    uint32_t *, int, int, int
 
+namespace arm_gemm
+{
 // Actual kernel implementations
-void sve_interleaved_u8u32_dot_8x3VL(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
+void sve_interleaved_u8u32_dot_8x3VL( ARGLIST );
 
-class cls_sve_interleaved_u8u32_dot_8x3VL {
+class cls_sve_interleaved_u8u32_dot_8x3VL
+{
 public:
     typedef uint8_t operand_type;
     typedef uint32_t result_type;
 
-    typedef void (*kern_type)(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
+    typedef void (*kern_type)( ARGLIST );
 
     /* Kernel blocking parameters */
+    static constexpr unsigned int out_height()
+    {
+        return 8;
+    }
+
     static unsigned int out_width()
     {
         return get_vector_length<uint32_t>() * 3;
     }
 
-    static unsigned int out_height()
+    static unsigned int stripe_width()
     {
-        return 8;
+        return get_vector_length<uint32_t>();
     }
 
-    static unsigned int k_unroll()
+    static constexpr unsigned int k_unroll()
     {
         return 4;
     }
 
-    // Use the standard fixed size transforms.
+
     StdTransformsSVE<operand_type, result_type, 8, 3, 4, 1> transforms = {};
     StdTransformsSVE<operand_type, result_type, 8, 3, 4, 1, true> transforms_quantized = {};
+    template<typename T>
+    static inline PerformanceParameters get_performance_parameters(const CPUInfo *ci)
+    {
 
-    kern_type kernel=sve_interleaved_u8u32_dot_8x3VL;
+        if (std::is_same<T, uint32_t>::value) {
+            switch (ci->get_cpu_model()) {
+                default:
+                    return { 31.66, 4.11, 7.94 };
+                case CPUModel::A510:
+                    return { 27.44, 3.41, 2.90 };
+                case CPUModel::V1:
+                    return { 63.30, 4.97, 11.52 };
+            }
+        }
+
+
+        if (std::is_same<T, uint8_t>::value) {
+            switch (ci->get_cpu_model()) {
+                default:
+                    return { 31.67, 4.04, 0.50 };
+                case CPUModel::A510:
+                    return { 27.45, 1.65, 0.28 };
+                case CPUModel::V1:
+                    return { 63.35, 4.96, 0.77 };
+            }
+        }
+
+        return { 1.0 };
+    }
 
+    // Default to the generic kernel
+    kern_type kernel=sve_interleaved_u8u32_dot_8x3VL;
     cls_sve_interleaved_u8u32_dot_8x3VL(const CPUInfo *)
     {
-
     }
 };
 
 } // namespace arm_gemm
 
+#undef ARGLIST
+
 #endif // ARM_COMPUTE_ENABLE_SVE
-- 
cgit v1.2.1