From 4ee8b1599dbaf7634d25607fa5ac96ba3dc6b0f2 Mon Sep 17 00:00:00 2001 From: Georgios Pinitas Date: Fri, 16 Jul 2021 16:16:43 +0100 Subject: Update GEMM assembly kernels - Introduce Fp32 kernels with internal calculations in Bfloat16 when fast_mode is enabled - Improve kernel selection heuristics Signed-off-by: Georgios Pinitas Change-Id: I68a9e7e862b6fd2721b46e0d7cc791091c4ab279 Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/5965 Tested-by: Arm Jenkins Comments-Addressed: Arm Jenkins --- .../kernels/a64_hybrid_s8qa_dot_4x16/a55.cpp | 18 +-- .../kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp | 174 ++++++++++----------- 2 files changed, 96 insertions(+), 96 deletions(-) (limited to 'src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16') diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/a55.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/a55.cpp index 11aa05a9b7..ee7e55f179 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/a55.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/a55.cpp @@ -406,10 +406,10 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( "b 122f\n" "31:" // Height 2 "movi v11.4s, #0x0\n" - "movi v12.4s, #0x0\n" - "movi v15.16b, #0x1\n" "ldr x11, [%x[args_ptr], %[offsetof_N]]\n" + "movi v12.4s, #0x0\n" "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "movi v15.16b, #0x1\n" "mov x9, %x[col_bias]\n" "bic %x[flags], %x[flags], #0x80000000\n" "mov x28, %x[output_ptr]\n" @@ -853,12 +853,12 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( "b 122f\n" "61:" // Height 3 "movi v11.4s, #0x0\n" + "ldr x11, [%x[args_ptr], %[offsetof_N]]\n" "movi v12.4s, #0x0\n" + "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n" "movi v13.4s, #0x0\n" - "movi v15.16b, #0x1\n" - "ldr x11, [%x[args_ptr], %[offsetof_N]]\n" "mov x9, %x[col_bias]\n" - "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n" + "movi v15.16b, #0x1\n" "bic %x[flags], %x[flags], #0x80000000\n" "mov x28, %x[output_ptr]\n" "62:" // Height 3: Column loop @@ -1426,14 +1426,14 @@ void a64_hybrid_s8qa_dot_4x16_a55 ( "b 122f\n" "91:" // Height 4 "movi v11.4s, #0x0\n" + "ldr x11, [%x[args_ptr], %[offsetof_N]]\n" "movi v12.4s, #0x0\n" + "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n" "movi v13.4s, #0x0\n" + "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" "movi v14.4s, #0x0\n" - "movi v15.16b, #0x1\n" - "ldr x11, [%x[args_ptr], %[offsetof_N]]\n" - "ldr x10, [%x[args_ptr], %[offsetof_B_ptr]]\n" "mov x9, %x[col_bias]\n" - "ldr x20, [%x[args_ptr], %[offsetof_output_offset]]\n" + "movi v15.16b, #0x1\n" "bic %x[flags], %x[flags], #0x80000000\n" "mov x28, %x[output_ptr]\n" "mov x19, #0x4\n" diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp index 0adfb99f23..a1c4b34d38 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp @@ -283,16 +283,16 @@ void a64_hybrid_s8qa_dot_4x16 ( "sqrdmulh v19.4s, v19.4s, v4.4s\n" "tbz %x[flags], #5, 20f\n" "and v4.16b, v16.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" "and v5.16b, v17.16b, v0.16b\n" "and v6.16b, v18.16b, v0.16b\n" - "sshr v5.4s, v5.4s, #0x1f\n" "and v7.16b, v19.16b, v0.16b\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sshr v6.4s, v6.4s, #0x1f\n" "sqadd v16.4s, v16.4s, v4.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" "sqadd v17.4s, v17.4s, v5.4s\n" "sqadd v18.4s, v18.4s, v6.4s\n" + "sshr v7.4s, v7.4s, #0x1f\n" "sqadd v19.4s, v19.4s, v7.4s\n" "20:" // Height 1: no shift correction "srshl v16.4s, v16.4s, v0.4s\n" @@ -612,8 +612,8 @@ void a64_hybrid_s8qa_dot_4x16 ( "ld1r { v2.4s }, [x22]\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v11.4s, v11.4s, v11.4s\n" - "addp v12.4s, v12.4s, v12.4s\n" "neg v2.4s, v2.4s\n" + "addp v12.4s, v12.4s, v12.4s\n" "mul v11.4s, v11.4s, v2.4s\n" "mul v12.4s, v12.4s, v2.4s\n" "49:" // Height 2: skip row sum fixup @@ -653,27 +653,27 @@ void a64_hybrid_s8qa_dot_4x16 ( "sqrdmulh v23.4s, v23.4s, v4.4s\n" "tbz %x[flags], #5, 50f\n" "and v4.16b, v16.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" "and v5.16b, v17.16b, v0.16b\n" "and v6.16b, v18.16b, v0.16b\n" + "sshr v4.4s, v4.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v4.4s\n" + "sqadd v17.4s, v17.4s, v5.4s\n" + "sqadd v18.4s, v18.4s, v6.4s\n" "and v7.16b, v19.16b, v0.16b\n" "and v8.16b, v20.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" "and v9.16b, v21.16b, v0.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "and v10.16b, v22.16b, v0.16b\n" "sshr v8.4s, v8.4s, #0x1f\n" - "and v4.16b, v23.16b, v0.16b\n" "sshr v9.4s, v9.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" "sqadd v19.4s, v19.4s, v7.4s\n" "sqadd v20.4s, v20.4s, v8.4s\n" "sqadd v21.4s, v21.4s, v9.4s\n" + "and v10.16b, v22.16b, v0.16b\n" + "and v4.16b, v23.16b, v0.16b\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" "sqadd v22.4s, v22.4s, v10.4s\n" "sqadd v23.4s, v23.4s, v4.4s\n" "50:" // Height 2: no shift correction @@ -690,8 +690,6 @@ void a64_hybrid_s8qa_dot_4x16 ( "cmp x9, #0x10\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" - "srshl v22.4s, v22.4s, v0.4s\n" - "srshl v23.4s, v23.4s, v0.4s\n" "add v16.4s, v16.4s, v4.4s\n" "add v17.4s, v17.4s, v4.4s\n" "add v18.4s, v18.4s, v4.4s\n" @@ -710,16 +708,18 @@ void a64_hybrid_s8qa_dot_4x16 ( "smax v19.4s, v19.4s, v5.4s\n" "smax v20.4s, v20.4s, v5.4s\n" "smax v21.4s, v21.4s, v5.4s\n" + "srshl v22.4s, v22.4s, v0.4s\n" + "srshl v23.4s, v23.4s, v0.4s\n" + "uzp1 v16.8h, v16.8h, v17.8h\n" + "uzp1 v17.8h, v18.8h, v19.8h\n" "add v22.4s, v22.4s, v4.4s\n" "add v23.4s, v23.4s, v4.4s\n" - "uzp1 v16.8h, v16.8h, v17.8h\n" + "uzp1 v20.8h, v20.8h, v21.8h\n" "smin v22.4s, v22.4s, v6.4s\n" "smin v23.4s, v23.4s, v6.4s\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v16.16b, v16.16b, v17.16b\n" "smax v22.4s, v22.4s, v5.4s\n" "smax v23.4s, v23.4s, v5.4s\n" - "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" "uzp1 v21.8h, v22.8h, v23.8h\n" "uzp1 v20.16b, v20.16b, v21.16b\n" "bge 59f\n" @@ -1094,9 +1094,9 @@ void a64_hybrid_s8qa_dot_4x16 ( "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" "addp v11.4s, v11.4s, v11.4s\n" + "neg v3.4s, v3.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" - "neg v3.4s, v3.4s\n" "mul v11.4s, v11.4s, v3.4s\n" "mul v12.4s, v12.4s, v3.4s\n" "mul v13.4s, v13.4s, v3.4s\n" @@ -1149,39 +1149,39 @@ void a64_hybrid_s8qa_dot_4x16 ( "sqrdmulh v27.4s, v27.4s, v4.4s\n" "tbz %x[flags], #5, 80f\n" "and v4.16b, v16.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" "and v5.16b, v17.16b, v0.16b\n" "and v6.16b, v18.16b, v0.16b\n" + "sshr v4.4s, v4.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v4.4s\n" + "sqadd v17.4s, v17.4s, v5.4s\n" + "sqadd v18.4s, v18.4s, v6.4s\n" "and v7.16b, v19.16b, v0.16b\n" "and v8.16b, v20.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" "and v9.16b, v21.16b, v0.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "and v10.16b, v22.16b, v0.16b\n" "sshr v8.4s, v8.4s, #0x1f\n" - "and v4.16b, v23.16b, v0.16b\n" "sshr v9.4s, v9.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "and v5.16b, v24.16b, v0.16b\n" - "sshr v5.4s, v5.4s, #0x1f\n" "sqadd v19.4s, v19.4s, v7.4s\n" "sqadd v20.4s, v20.4s, v8.4s\n" "sqadd v21.4s, v21.4s, v9.4s\n" + "and v10.16b, v22.16b, v0.16b\n" + "and v4.16b, v23.16b, v0.16b\n" + "and v5.16b, v24.16b, v0.16b\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sqadd v22.4s, v22.4s, v10.4s\n" "sqadd v23.4s, v23.4s, v4.4s\n" - "and v6.16b, v25.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" "sqadd v24.4s, v24.4s, v5.4s\n" + "and v6.16b, v25.16b, v0.16b\n" "and v7.16b, v26.16b, v0.16b\n" - "sshr v7.4s, v7.4s, #0x1f\n" "and v8.16b, v27.16b, v0.16b\n" - "sqadd v25.4s, v25.4s, v6.4s\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v7.4s, v7.4s, #0x1f\n" "sshr v8.4s, v8.4s, #0x1f\n" + "sqadd v25.4s, v25.4s, v6.4s\n" "sqadd v26.4s, v26.4s, v7.4s\n" "sqadd v27.4s, v27.4s, v8.4s\n" "80:" // Height 3: no shift correction @@ -1198,8 +1198,6 @@ void a64_hybrid_s8qa_dot_4x16 ( "cmp x9, #0x10\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" - "srshl v22.4s, v22.4s, v0.4s\n" - "srshl v23.4s, v23.4s, v0.4s\n" "add v16.4s, v16.4s, v4.4s\n" "add v17.4s, v17.4s, v4.4s\n" "add v18.4s, v18.4s, v4.4s\n" @@ -1218,31 +1216,33 @@ void a64_hybrid_s8qa_dot_4x16 ( "smax v19.4s, v19.4s, v5.4s\n" "smax v20.4s, v20.4s, v5.4s\n" "smax v21.4s, v21.4s, v5.4s\n" + "srshl v22.4s, v22.4s, v0.4s\n" + "srshl v23.4s, v23.4s, v0.4s\n" + "srshl v24.4s, v24.4s, v0.4s\n" + "srshl v25.4s, v25.4s, v0.4s\n" "add v22.4s, v22.4s, v4.4s\n" "add v23.4s, v23.4s, v4.4s\n" - "srshl v24.4s, v24.4s, v0.4s\n" + "add v24.4s, v24.4s, v4.4s\n" "smin v22.4s, v22.4s, v6.4s\n" "smin v23.4s, v23.4s, v6.4s\n" - "srshl v25.4s, v25.4s, v0.4s\n" + "smin v24.4s, v24.4s, v6.4s\n" "smax v22.4s, v22.4s, v5.4s\n" "smax v23.4s, v23.4s, v5.4s\n" - "add v24.4s, v24.4s, v4.4s\n" + "smax v24.4s, v24.4s, v5.4s\n" "add v25.4s, v25.4s, v4.4s\n" "srshl v26.4s, v26.4s, v0.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" "srshl v27.4s, v27.4s, v0.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" + "smin v25.4s, v25.4s, v6.4s\n" + "uzp1 v16.8h, v16.8h, v17.8h\n" "add v26.4s, v26.4s, v4.4s\n" + "smax v25.4s, v25.4s, v5.4s\n" "add v27.4s, v27.4s, v4.4s\n" - "uzp1 v16.8h, v16.8h, v17.8h\n" "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" "uzp1 v17.8h, v18.8h, v19.8h\n" + "smin v27.4s, v27.4s, v6.4s\n" "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" "uzp1 v20.8h, v20.8h, v21.8h\n" + "smax v27.4s, v27.4s, v5.4s\n" "uzp1 v21.8h, v22.8h, v23.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" "uzp1 v25.8h, v26.8h, v27.8h\n" @@ -1705,10 +1705,10 @@ void a64_hybrid_s8qa_dot_4x16 ( "addp v13.4s, v13.4s, v13.4s\n" "addp v14.4s, v14.4s, v14.4s\n" "addp v11.4s, v11.4s, v11.4s\n" + "neg v4.4s, v4.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" "addp v14.4s, v14.4s, v14.4s\n" - "neg v4.4s, v4.4s\n" "mul v11.4s, v11.4s, v4.4s\n" "mul v12.4s, v12.4s, v4.4s\n" "mul v13.4s, v13.4s, v4.4s\n" @@ -1774,52 +1774,52 @@ void a64_hybrid_s8qa_dot_4x16 ( "sqrdmulh v31.4s, v31.4s, v4.4s\n" "tbz %x[flags], #5, 110f\n" "and v4.16b, v16.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" "and v5.16b, v17.16b, v0.16b\n" "and v6.16b, v18.16b, v0.16b\n" + "sshr v4.4s, v4.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v4.4s\n" + "sqadd v17.4s, v17.4s, v5.4s\n" + "sqadd v18.4s, v18.4s, v6.4s\n" "and v7.16b, v19.16b, v0.16b\n" "and v8.16b, v20.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" "and v9.16b, v21.16b, v0.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "and v10.16b, v22.16b, v0.16b\n" "sshr v8.4s, v8.4s, #0x1f\n" - "and v4.16b, v23.16b, v0.16b\n" "sshr v9.4s, v9.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "and v5.16b, v24.16b, v0.16b\n" - "sshr v5.4s, v5.4s, #0x1f\n" "sqadd v19.4s, v19.4s, v7.4s\n" "sqadd v20.4s, v20.4s, v8.4s\n" "sqadd v21.4s, v21.4s, v9.4s\n" + "and v10.16b, v22.16b, v0.16b\n" + "and v4.16b, v23.16b, v0.16b\n" + "and v5.16b, v24.16b, v0.16b\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sqadd v22.4s, v22.4s, v10.4s\n" "sqadd v23.4s, v23.4s, v4.4s\n" - "and v6.16b, v25.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" "sqadd v24.4s, v24.4s, v5.4s\n" + "and v6.16b, v25.16b, v0.16b\n" "and v7.16b, v26.16b, v0.16b\n" - "sshr v7.4s, v7.4s, #0x1f\n" "and v8.16b, v27.16b, v0.16b\n" - "and v9.16b, v28.16b, v0.16b\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v7.4s, v7.4s, #0x1f\n" "sshr v8.4s, v8.4s, #0x1f\n" "sqadd v25.4s, v25.4s, v6.4s\n" + "sqadd v26.4s, v26.4s, v7.4s\n" + "sqadd v27.4s, v27.4s, v8.4s\n" + "and v9.16b, v28.16b, v0.16b\n" "and v10.16b, v29.16b, v0.16b\n" - "sshr v9.4s, v9.4s, #0x1f\n" "and v4.16b, v30.16b, v0.16b\n" + "sshr v9.4s, v9.4s, #0x1f\n" "sshr v10.4s, v10.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "and v5.16b, v31.16b, v0.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v27.4s, v27.4s, v8.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" "sqadd v28.4s, v28.4s, v9.4s\n" "sqadd v29.4s, v29.4s, v10.4s\n" "sqadd v30.4s, v30.4s, v4.4s\n" + "and v5.16b, v31.16b, v0.16b\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sqadd v31.4s, v31.4s, v5.4s\n" "110:" // Height 4: no shift correction "srshl v16.4s, v16.4s, v0.4s\n" @@ -1835,8 +1835,6 @@ void a64_hybrid_s8qa_dot_4x16 ( "cmp x9, #0x10\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" - "srshl v22.4s, v22.4s, v0.4s\n" - "srshl v23.4s, v23.4s, v0.4s\n" "add v16.4s, v16.4s, v4.4s\n" "add v17.4s, v17.4s, v4.4s\n" "add v18.4s, v18.4s, v4.4s\n" @@ -1855,45 +1853,47 @@ void a64_hybrid_s8qa_dot_4x16 ( "smax v19.4s, v19.4s, v5.4s\n" "smax v20.4s, v20.4s, v5.4s\n" "smax v21.4s, v21.4s, v5.4s\n" + "srshl v22.4s, v22.4s, v0.4s\n" + "srshl v23.4s, v23.4s, v0.4s\n" + "srshl v24.4s, v24.4s, v0.4s\n" + "srshl v25.4s, v25.4s, v0.4s\n" "add v22.4s, v22.4s, v4.4s\n" "add v23.4s, v23.4s, v4.4s\n" - "srshl v24.4s, v24.4s, v0.4s\n" + "add v24.4s, v24.4s, v4.4s\n" "smin v22.4s, v22.4s, v6.4s\n" "smin v23.4s, v23.4s, v6.4s\n" - "srshl v25.4s, v25.4s, v0.4s\n" + "smin v24.4s, v24.4s, v6.4s\n" "smax v22.4s, v22.4s, v5.4s\n" "smax v23.4s, v23.4s, v5.4s\n" - "add v24.4s, v24.4s, v4.4s\n" + "smax v24.4s, v24.4s, v5.4s\n" "add v25.4s, v25.4s, v4.4s\n" "srshl v26.4s, v26.4s, v0.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" "srshl v27.4s, v27.4s, v0.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" + "smin v25.4s, v25.4s, v6.4s\n" + "srshl v28.4s, v28.4s, v0.4s\n" "add v26.4s, v26.4s, v4.4s\n" + "smax v25.4s, v25.4s, v5.4s\n" "add v27.4s, v27.4s, v4.4s\n" - "srshl v28.4s, v28.4s, v0.4s\n" "smin v26.4s, v26.4s, v6.4s\n" + "add v28.4s, v28.4s, v4.4s\n" "smin v27.4s, v27.4s, v6.4s\n" - "srshl v29.4s, v29.4s, v0.4s\n" "smax v26.4s, v26.4s, v5.4s\n" + "smin v28.4s, v28.4s, v6.4s\n" "smax v27.4s, v27.4s, v5.4s\n" - "add v28.4s, v28.4s, v4.4s\n" - "add v29.4s, v29.4s, v4.4s\n" + "srshl v29.4s, v29.4s, v0.4s\n" + "smax v28.4s, v28.4s, v5.4s\n" "srshl v30.4s, v30.4s, v0.4s\n" - "smin v28.4s, v28.4s, v6.4s\n" - "smin v29.4s, v29.4s, v6.4s\n" "srshl v31.4s, v31.4s, v0.4s\n" - "smax v28.4s, v28.4s, v5.4s\n" - "smax v29.4s, v29.4s, v5.4s\n" + "add v29.4s, v29.4s, v4.4s\n" + "uzp1 v16.8h, v16.8h, v17.8h\n" "add v30.4s, v30.4s, v4.4s\n" + "smin v29.4s, v29.4s, v6.4s\n" "add v31.4s, v31.4s, v4.4s\n" - "uzp1 v16.8h, v16.8h, v17.8h\n" "smin v30.4s, v30.4s, v6.4s\n" + "smax v29.4s, v29.4s, v5.4s\n" "smin v31.4s, v31.4s, v6.4s\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" "smax v30.4s, v30.4s, v5.4s\n" + "uzp1 v17.8h, v18.8h, v19.8h\n" "smax v31.4s, v31.4s, v5.4s\n" "uzp1 v20.8h, v20.8h, v21.8h\n" "uzp1 v21.8h, v22.8h, v23.8h\n" -- cgit v1.2.1