diff options
Diffstat (limited to 'src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp')
| -rw-r--r-- | src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp | 174 |
1 files changed, 87 insertions, 87 deletions
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp index 0adfb99f23..a1c4b34d38 100644 --- a/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp +++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8qa_dot_4x16/generic.cpp @@ -283,16 +283,16 @@ void a64_hybrid_s8qa_dot_4x16 ( "sqrdmulh v19.4s, v19.4s, v4.4s\n" "tbz %x[flags], #5, 20f\n" "and v4.16b, v16.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" "and v5.16b, v17.16b, v0.16b\n" "and v6.16b, v18.16b, v0.16b\n" - "sshr v5.4s, v5.4s, #0x1f\n" "and v7.16b, v19.16b, v0.16b\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sshr v6.4s, v6.4s, #0x1f\n" "sqadd v16.4s, v16.4s, v4.4s\n" - "sshr v7.4s, v7.4s, #0x1f\n" "sqadd v17.4s, v17.4s, v5.4s\n" "sqadd v18.4s, v18.4s, v6.4s\n" + "sshr v7.4s, v7.4s, #0x1f\n" "sqadd v19.4s, v19.4s, v7.4s\n" "20:" // Height 1: no shift correction "srshl v16.4s, v16.4s, v0.4s\n" @@ -612,8 +612,8 @@ void a64_hybrid_s8qa_dot_4x16 ( "ld1r { v2.4s }, [x22]\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v11.4s, v11.4s, v11.4s\n" - "addp v12.4s, v12.4s, v12.4s\n" "neg v2.4s, v2.4s\n" + "addp v12.4s, v12.4s, v12.4s\n" "mul v11.4s, v11.4s, v2.4s\n" "mul v12.4s, v12.4s, v2.4s\n" "49:" // Height 2: skip row sum fixup @@ -653,27 +653,27 @@ void a64_hybrid_s8qa_dot_4x16 ( "sqrdmulh v23.4s, v23.4s, v4.4s\n" "tbz %x[flags], #5, 50f\n" "and v4.16b, v16.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" "and v5.16b, v17.16b, v0.16b\n" "and v6.16b, v18.16b, v0.16b\n" + "sshr v4.4s, v4.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v4.4s\n" + "sqadd v17.4s, v17.4s, v5.4s\n" + "sqadd v18.4s, v18.4s, v6.4s\n" "and v7.16b, v19.16b, v0.16b\n" "and v8.16b, v20.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" "and v9.16b, v21.16b, v0.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "and v10.16b, v22.16b, v0.16b\n" "sshr v8.4s, v8.4s, #0x1f\n" - "and v4.16b, v23.16b, v0.16b\n" "sshr v9.4s, v9.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" "sqadd v19.4s, v19.4s, v7.4s\n" "sqadd v20.4s, v20.4s, v8.4s\n" "sqadd v21.4s, v21.4s, v9.4s\n" + "and v10.16b, v22.16b, v0.16b\n" + "and v4.16b, v23.16b, v0.16b\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" "sqadd v22.4s, v22.4s, v10.4s\n" "sqadd v23.4s, v23.4s, v4.4s\n" "50:" // Height 2: no shift correction @@ -690,8 +690,6 @@ void a64_hybrid_s8qa_dot_4x16 ( "cmp x9, #0x10\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" - "srshl v22.4s, v22.4s, v0.4s\n" - "srshl v23.4s, v23.4s, v0.4s\n" "add v16.4s, v16.4s, v4.4s\n" "add v17.4s, v17.4s, v4.4s\n" "add v18.4s, v18.4s, v4.4s\n" @@ -710,16 +708,18 @@ void a64_hybrid_s8qa_dot_4x16 ( "smax v19.4s, v19.4s, v5.4s\n" "smax v20.4s, v20.4s, v5.4s\n" "smax v21.4s, v21.4s, v5.4s\n" + "srshl v22.4s, v22.4s, v0.4s\n" + "srshl v23.4s, v23.4s, v0.4s\n" + "uzp1 v16.8h, v16.8h, v17.8h\n" + "uzp1 v17.8h, v18.8h, v19.8h\n" "add v22.4s, v22.4s, v4.4s\n" "add v23.4s, v23.4s, v4.4s\n" - "uzp1 v16.8h, v16.8h, v17.8h\n" + "uzp1 v20.8h, v20.8h, v21.8h\n" "smin v22.4s, v22.4s, v6.4s\n" "smin v23.4s, v23.4s, v6.4s\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" + "uzp1 v16.16b, v16.16b, v17.16b\n" "smax v22.4s, v22.4s, v5.4s\n" "smax v23.4s, v23.4s, v5.4s\n" - "uzp1 v20.8h, v20.8h, v21.8h\n" - "uzp1 v16.16b, v16.16b, v17.16b\n" "uzp1 v21.8h, v22.8h, v23.8h\n" "uzp1 v20.16b, v20.16b, v21.16b\n" "bge 59f\n" @@ -1094,9 +1094,9 @@ void a64_hybrid_s8qa_dot_4x16 ( "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" "addp v11.4s, v11.4s, v11.4s\n" + "neg v3.4s, v3.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" - "neg v3.4s, v3.4s\n" "mul v11.4s, v11.4s, v3.4s\n" "mul v12.4s, v12.4s, v3.4s\n" "mul v13.4s, v13.4s, v3.4s\n" @@ -1149,39 +1149,39 @@ void a64_hybrid_s8qa_dot_4x16 ( "sqrdmulh v27.4s, v27.4s, v4.4s\n" "tbz %x[flags], #5, 80f\n" "and v4.16b, v16.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" "and v5.16b, v17.16b, v0.16b\n" "and v6.16b, v18.16b, v0.16b\n" + "sshr v4.4s, v4.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v4.4s\n" + "sqadd v17.4s, v17.4s, v5.4s\n" + "sqadd v18.4s, v18.4s, v6.4s\n" "and v7.16b, v19.16b, v0.16b\n" "and v8.16b, v20.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" "and v9.16b, v21.16b, v0.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "and v10.16b, v22.16b, v0.16b\n" "sshr v8.4s, v8.4s, #0x1f\n" - "and v4.16b, v23.16b, v0.16b\n" "sshr v9.4s, v9.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "and v5.16b, v24.16b, v0.16b\n" - "sshr v5.4s, v5.4s, #0x1f\n" "sqadd v19.4s, v19.4s, v7.4s\n" "sqadd v20.4s, v20.4s, v8.4s\n" "sqadd v21.4s, v21.4s, v9.4s\n" + "and v10.16b, v22.16b, v0.16b\n" + "and v4.16b, v23.16b, v0.16b\n" + "and v5.16b, v24.16b, v0.16b\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sqadd v22.4s, v22.4s, v10.4s\n" "sqadd v23.4s, v23.4s, v4.4s\n" - "and v6.16b, v25.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" "sqadd v24.4s, v24.4s, v5.4s\n" + "and v6.16b, v25.16b, v0.16b\n" "and v7.16b, v26.16b, v0.16b\n" - "sshr v7.4s, v7.4s, #0x1f\n" "and v8.16b, v27.16b, v0.16b\n" - "sqadd v25.4s, v25.4s, v6.4s\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v7.4s, v7.4s, #0x1f\n" "sshr v8.4s, v8.4s, #0x1f\n" + "sqadd v25.4s, v25.4s, v6.4s\n" "sqadd v26.4s, v26.4s, v7.4s\n" "sqadd v27.4s, v27.4s, v8.4s\n" "80:" // Height 3: no shift correction @@ -1198,8 +1198,6 @@ void a64_hybrid_s8qa_dot_4x16 ( "cmp x9, #0x10\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" - "srshl v22.4s, v22.4s, v0.4s\n" - "srshl v23.4s, v23.4s, v0.4s\n" "add v16.4s, v16.4s, v4.4s\n" "add v17.4s, v17.4s, v4.4s\n" "add v18.4s, v18.4s, v4.4s\n" @@ -1218,31 +1216,33 @@ void a64_hybrid_s8qa_dot_4x16 ( "smax v19.4s, v19.4s, v5.4s\n" "smax v20.4s, v20.4s, v5.4s\n" "smax v21.4s, v21.4s, v5.4s\n" + "srshl v22.4s, v22.4s, v0.4s\n" + "srshl v23.4s, v23.4s, v0.4s\n" + "srshl v24.4s, v24.4s, v0.4s\n" + "srshl v25.4s, v25.4s, v0.4s\n" "add v22.4s, v22.4s, v4.4s\n" "add v23.4s, v23.4s, v4.4s\n" - "srshl v24.4s, v24.4s, v0.4s\n" + "add v24.4s, v24.4s, v4.4s\n" "smin v22.4s, v22.4s, v6.4s\n" "smin v23.4s, v23.4s, v6.4s\n" - "srshl v25.4s, v25.4s, v0.4s\n" + "smin v24.4s, v24.4s, v6.4s\n" "smax v22.4s, v22.4s, v5.4s\n" "smax v23.4s, v23.4s, v5.4s\n" - "add v24.4s, v24.4s, v4.4s\n" + "smax v24.4s, v24.4s, v5.4s\n" "add v25.4s, v25.4s, v4.4s\n" "srshl v26.4s, v26.4s, v0.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" "srshl v27.4s, v27.4s, v0.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" + "smin v25.4s, v25.4s, v6.4s\n" + "uzp1 v16.8h, v16.8h, v17.8h\n" "add v26.4s, v26.4s, v4.4s\n" + "smax v25.4s, v25.4s, v5.4s\n" "add v27.4s, v27.4s, v4.4s\n" - "uzp1 v16.8h, v16.8h, v17.8h\n" "smin v26.4s, v26.4s, v6.4s\n" - "smin v27.4s, v27.4s, v6.4s\n" "uzp1 v17.8h, v18.8h, v19.8h\n" + "smin v27.4s, v27.4s, v6.4s\n" "smax v26.4s, v26.4s, v5.4s\n" - "smax v27.4s, v27.4s, v5.4s\n" "uzp1 v20.8h, v20.8h, v21.8h\n" + "smax v27.4s, v27.4s, v5.4s\n" "uzp1 v21.8h, v22.8h, v23.8h\n" "uzp1 v24.8h, v24.8h, v25.8h\n" "uzp1 v25.8h, v26.8h, v27.8h\n" @@ -1705,10 +1705,10 @@ void a64_hybrid_s8qa_dot_4x16 ( "addp v13.4s, v13.4s, v13.4s\n" "addp v14.4s, v14.4s, v14.4s\n" "addp v11.4s, v11.4s, v11.4s\n" + "neg v4.4s, v4.4s\n" "addp v12.4s, v12.4s, v12.4s\n" "addp v13.4s, v13.4s, v13.4s\n" "addp v14.4s, v14.4s, v14.4s\n" - "neg v4.4s, v4.4s\n" "mul v11.4s, v11.4s, v4.4s\n" "mul v12.4s, v12.4s, v4.4s\n" "mul v13.4s, v13.4s, v4.4s\n" @@ -1774,52 +1774,52 @@ void a64_hybrid_s8qa_dot_4x16 ( "sqrdmulh v31.4s, v31.4s, v4.4s\n" "tbz %x[flags], #5, 110f\n" "and v4.16b, v16.16b, v0.16b\n" - "sshr v4.4s, v4.4s, #0x1f\n" "and v5.16b, v17.16b, v0.16b\n" "and v6.16b, v18.16b, v0.16b\n" + "sshr v4.4s, v4.4s, #0x1f\n" "sshr v5.4s, v5.4s, #0x1f\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sqadd v16.4s, v16.4s, v4.4s\n" + "sqadd v17.4s, v17.4s, v5.4s\n" + "sqadd v18.4s, v18.4s, v6.4s\n" "and v7.16b, v19.16b, v0.16b\n" "and v8.16b, v20.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" "and v9.16b, v21.16b, v0.16b\n" "sshr v7.4s, v7.4s, #0x1f\n" - "sqadd v16.4s, v16.4s, v4.4s\n" - "and v10.16b, v22.16b, v0.16b\n" "sshr v8.4s, v8.4s, #0x1f\n" - "and v4.16b, v23.16b, v0.16b\n" "sshr v9.4s, v9.4s, #0x1f\n" - "sqadd v17.4s, v17.4s, v5.4s\n" - "sshr v10.4s, v10.4s, #0x1f\n" - "sqadd v18.4s, v18.4s, v6.4s\n" - "sshr v4.4s, v4.4s, #0x1f\n" - "and v5.16b, v24.16b, v0.16b\n" - "sshr v5.4s, v5.4s, #0x1f\n" "sqadd v19.4s, v19.4s, v7.4s\n" "sqadd v20.4s, v20.4s, v8.4s\n" "sqadd v21.4s, v21.4s, v9.4s\n" + "and v10.16b, v22.16b, v0.16b\n" + "and v4.16b, v23.16b, v0.16b\n" + "and v5.16b, v24.16b, v0.16b\n" + "sshr v10.4s, v10.4s, #0x1f\n" + "sshr v4.4s, v4.4s, #0x1f\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sqadd v22.4s, v22.4s, v10.4s\n" "sqadd v23.4s, v23.4s, v4.4s\n" - "and v6.16b, v25.16b, v0.16b\n" - "sshr v6.4s, v6.4s, #0x1f\n" "sqadd v24.4s, v24.4s, v5.4s\n" + "and v6.16b, v25.16b, v0.16b\n" "and v7.16b, v26.16b, v0.16b\n" - "sshr v7.4s, v7.4s, #0x1f\n" "and v8.16b, v27.16b, v0.16b\n" - "and v9.16b, v28.16b, v0.16b\n" + "sshr v6.4s, v6.4s, #0x1f\n" + "sshr v7.4s, v7.4s, #0x1f\n" "sshr v8.4s, v8.4s, #0x1f\n" "sqadd v25.4s, v25.4s, v6.4s\n" + "sqadd v26.4s, v26.4s, v7.4s\n" + "sqadd v27.4s, v27.4s, v8.4s\n" + "and v9.16b, v28.16b, v0.16b\n" "and v10.16b, v29.16b, v0.16b\n" - "sshr v9.4s, v9.4s, #0x1f\n" "and v4.16b, v30.16b, v0.16b\n" + "sshr v9.4s, v9.4s, #0x1f\n" "sshr v10.4s, v10.4s, #0x1f\n" - "sqadd v26.4s, v26.4s, v7.4s\n" - "and v5.16b, v31.16b, v0.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" - "sqadd v27.4s, v27.4s, v8.4s\n" - "sshr v5.4s, v5.4s, #0x1f\n" "sqadd v28.4s, v28.4s, v9.4s\n" "sqadd v29.4s, v29.4s, v10.4s\n" "sqadd v30.4s, v30.4s, v4.4s\n" + "and v5.16b, v31.16b, v0.16b\n" + "sshr v5.4s, v5.4s, #0x1f\n" "sqadd v31.4s, v31.4s, v5.4s\n" "110:" // Height 4: no shift correction "srshl v16.4s, v16.4s, v0.4s\n" @@ -1835,8 +1835,6 @@ void a64_hybrid_s8qa_dot_4x16 ( "cmp x9, #0x10\n" "srshl v20.4s, v20.4s, v0.4s\n" "srshl v21.4s, v21.4s, v0.4s\n" - "srshl v22.4s, v22.4s, v0.4s\n" - "srshl v23.4s, v23.4s, v0.4s\n" "add v16.4s, v16.4s, v4.4s\n" "add v17.4s, v17.4s, v4.4s\n" "add v18.4s, v18.4s, v4.4s\n" @@ -1855,45 +1853,47 @@ void a64_hybrid_s8qa_dot_4x16 ( "smax v19.4s, v19.4s, v5.4s\n" "smax v20.4s, v20.4s, v5.4s\n" "smax v21.4s, v21.4s, v5.4s\n" + "srshl v22.4s, v22.4s, v0.4s\n" + "srshl v23.4s, v23.4s, v0.4s\n" + "srshl v24.4s, v24.4s, v0.4s\n" + "srshl v25.4s, v25.4s, v0.4s\n" "add v22.4s, v22.4s, v4.4s\n" "add v23.4s, v23.4s, v4.4s\n" - "srshl v24.4s, v24.4s, v0.4s\n" + "add v24.4s, v24.4s, v4.4s\n" "smin v22.4s, v22.4s, v6.4s\n" "smin v23.4s, v23.4s, v6.4s\n" - "srshl v25.4s, v25.4s, v0.4s\n" + "smin v24.4s, v24.4s, v6.4s\n" "smax v22.4s, v22.4s, v5.4s\n" "smax v23.4s, v23.4s, v5.4s\n" - "add v24.4s, v24.4s, v4.4s\n" + "smax v24.4s, v24.4s, v5.4s\n" "add v25.4s, v25.4s, v4.4s\n" "srshl v26.4s, v26.4s, v0.4s\n" - "smin v24.4s, v24.4s, v6.4s\n" - "smin v25.4s, v25.4s, v6.4s\n" "srshl v27.4s, v27.4s, v0.4s\n" - "smax v24.4s, v24.4s, v5.4s\n" - "smax v25.4s, v25.4s, v5.4s\n" + "smin v25.4s, v25.4s, v6.4s\n" + "srshl v28.4s, v28.4s, v0.4s\n" "add v26.4s, v26.4s, v4.4s\n" + "smax v25.4s, v25.4s, v5.4s\n" "add v27.4s, v27.4s, v4.4s\n" - "srshl v28.4s, v28.4s, v0.4s\n" "smin v26.4s, v26.4s, v6.4s\n" + "add v28.4s, v28.4s, v4.4s\n" "smin v27.4s, v27.4s, v6.4s\n" - "srshl v29.4s, v29.4s, v0.4s\n" "smax v26.4s, v26.4s, v5.4s\n" + "smin v28.4s, v28.4s, v6.4s\n" "smax v27.4s, v27.4s, v5.4s\n" - "add v28.4s, v28.4s, v4.4s\n" - "add v29.4s, v29.4s, v4.4s\n" + "srshl v29.4s, v29.4s, v0.4s\n" + "smax v28.4s, v28.4s, v5.4s\n" "srshl v30.4s, v30.4s, v0.4s\n" - "smin v28.4s, v28.4s, v6.4s\n" - "smin v29.4s, v29.4s, v6.4s\n" "srshl v31.4s, v31.4s, v0.4s\n" - "smax v28.4s, v28.4s, v5.4s\n" - "smax v29.4s, v29.4s, v5.4s\n" + "add v29.4s, v29.4s, v4.4s\n" + "uzp1 v16.8h, v16.8h, v17.8h\n" "add v30.4s, v30.4s, v4.4s\n" + "smin v29.4s, v29.4s, v6.4s\n" "add v31.4s, v31.4s, v4.4s\n" - "uzp1 v16.8h, v16.8h, v17.8h\n" "smin v30.4s, v30.4s, v6.4s\n" + "smax v29.4s, v29.4s, v5.4s\n" "smin v31.4s, v31.4s, v6.4s\n" - "uzp1 v17.8h, v18.8h, v19.8h\n" "smax v30.4s, v30.4s, v5.4s\n" + "uzp1 v17.8h, v18.8h, v19.8h\n" "smax v31.4s, v31.4s, v5.4s\n" "uzp1 v20.8h, v20.8h, v21.8h\n" "uzp1 v21.8h, v22.8h, v23.8h\n" |