COMPMID-881: RSH new arm_gemm interface.

Change-Id: I1e2a1a77097d8017c274af3f97eba6964f80f5fa
Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/122592
Tested-by: Jenkins <bsgcomp@arm.com>
Reviewed-by: Anthony Barbier <anthony.barbier@arm.com>

1 files changed, 273 insertions, 0 deletions

diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_4x4/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_4x4/generic.cpp
new file mode 100644
index 0000000000..0a881ffde3
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_gemm_u8_4x4/generic.cpp
@@ -0,0 +1,273 @@
+/*
+ * Copyright (c) 2017-2018 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include <arm_neon.h>
+
+#include "../../asmlib.hpp"
+
+namespace arm_gemm
+{
+void a64_gemm_u8_4x4(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K)
+{
+    const uint8_t *a_ptr = Apanel;
+    uint32_t      *c_ptr = Cpanel;
+    K /= 16;
+
+    for(int yb = 0; yb < ablocks; yb++)
+    {
+        const uint8_t *a_ptr0 = a_ptr;
+        const uint8_t *b_ptr  = Bpanel;
+
+        for(int xb = 0; xb < bblocks; xb++)
+        {
+            a_ptr = a_ptr0;
+
+            int k = K - 1;
+
+            register uint8x16_t b0 asm("v4");
+            register uint8x16_t b1 asm("v5");
+            register uint8x16_t b2 asm("v6");
+            register uint8x16_t b3 asm("v7");
+
+            __asm __volatile(
+                "movi    v16.4s, #0x0\n"
+                "ldr    q0, [%[a_ptr]]\n"
+                "movi    v17.4s, #0x0\n"
+                "ldr    %q[b0], [%[b_ptr]]\n"
+                "movi    v18.4s, #0x0\n"
+                "ldr    %q[b1], [%[b_ptr], #16]\n"
+                "movi    v19.4s, #0x0\n"
+                "ldr    %q[b2], [%[b_ptr], #32]\n"
+                "movi    v20.4s, #0x0\n"
+                "ldr    %q[b3], [%[b_ptr], #48]\n"
+                "movi    v21.4s, #0x0\n"
+                "ldr    q1, [%[a_ptr], #16]\n"
+                "movi    v22.4s, #0x0\n"
+                "ldr    q2, [%[a_ptr], #32]\n"
+                "movi    v23.4s, #0x0\n"
+                "ldr    q3, [%[a_ptr], #48]\n"
+                "movi    v24.4s, #0x0\n" ASM_PREFETCH("[%[b_ptr], #64]") "movi    v25.4s, #0x0\n" ASM_PREFETCH("[%[a_ptr], #64]") "movi    v26.4s, #0x0\n" ASM_PREFETCH("[%[b_ptr], #128]") "movi    v27.4s, #0x0\n"
+                ASM_PREFETCH("[%[a_ptr], #128]") "movi    v28.4s, #0x0\n" ASM_PREFETCH("[%[b_ptr], #192]") "movi    v29.4s, #0x0\n" ASM_PREFETCH("[%[a_ptr], #192]") "movi    v30.4s, #0x0\n"
+                ASM_PREFETCH("[%[b_ptr], #256]") "movi    v31.4s, #0x0\n" ASM_PREFETCH("[%[a_ptr], #256]")
+
+                "umull    v12.8h, v0.8b, %[b0].8b\n"
+                "add    %[a_ptr], %[a_ptr], #64\n"
+                "umull    v13.8h, v0.8b, %[b1].8b\n"
+                "umull    v14.8h, v0.8b, %[b2].8b\n"
+                "add    %[b_ptr], %[b_ptr], #64\n"
+                "umull    v15.8h, v0.8b, %[b3].8b\n"
+
+                // Skip loop if we are doing zero iterations of it.
+                "cbz    %w[k], 2f\n"
+
+                "1:\n"
+                "uadalp    v16.4s, v12.8h\n"
+                "umull2    v12.8h, v0.16b, %[b0].16b\n"
+                "uadalp    v17.4s, v13.8h\n"
+                "umull2    v13.8h, v0.16b, %[b1].16b\n"
+                "uadalp    v18.4s, v14.8h\n"
+                "umull2    v14.8h, v0.16b, %[b2].16b\n"
+                "uadalp    v19.4s, v15.8h\n"
+                "umull2    v15.8h, v0.16b, %[b3].16b\n"
+                "ldr     q0, [%[a_ptr]]\n"
+
+                "uadalp    v16.4s, v12.8h\n"
+                "umull    v12.8h, v1.8b, %[b0].8b\n"
+                "uadalp    v17.4s, v13.8h\n"
+                "umull    v13.8h, v1.8b, %[b1].8b\n"
+                "subs    %w[k], %w[k], #1\n"
+                "uadalp    v18.4s, v14.8h\n"
+                "umull    v14.8h, v1.8b, %[b2].8b\n"
+                "uadalp    v19.4s, v15.8h\n"
+                "umull    v15.8h, v1.8b, %[b3].8b\n"
+
+                "uadalp    v20.4s, v12.8h\n"
+                "umull2    v12.8h, v1.16b, %[b0].16b\n"
+                "uadalp    v21.4s, v13.8h\n"
+                "umull2    v13.8h, v1.16b, %[b1].16b\n" ASM_PREFETCH("[%[a_ptr], #256]")
+                "uadalp    v22.4s, v14.8h\n"
+                "umull2    v14.8h, v1.16b, %[b2].16b\n"
+                "uadalp    v23.4s, v15.8h\n"
+                "umull2    v15.8h, v1.16b, %[b3].16b\n"
+                "ldr     q1, [%[a_ptr], #16]\n"
+
+                "uadalp    v20.4s, v12.8h\n"
+                "umull    v12.8h, v2.8b, %[b0].8b\n"
+                "uadalp    v21.4s, v13.8h\n"
+                "umull    v13.8h, v2.8b, %[b1].8b\n" ASM_PREFETCH("[%[b_ptr], #256]")
+                "uadalp    v22.4s, v14.8h\n"
+                "umull    v14.8h, v2.8b, %[b2].8b\n"
+                "uadalp    v23.4s, v15.8h\n"
+                "umull    v15.8h, v2.8b, %[b3].8b\n"
+
+                "uadalp    v24.4s, v12.8h\n"
+                "umull2    v12.8h, v2.16b, %[b0].16b\n"
+                "uadalp    v25.4s, v13.8h\n"
+                "umull2    v13.8h, v2.16b, %[b1].16b\n"
+                "uadalp    v26.4s, v14.8h\n"
+                "umull2    v14.8h, v2.16b, %[b2].16b\n"
+                "uadalp    v27.4s, v15.8h\n"
+                "umull2    v15.8h, v2.16b, %[b3].16b\n"
+                "ldr    q2, [%[a_ptr], #32]\n"
+
+                "uadalp    v24.4s, v12.8h\n"
+                "umull    v12.8h, v3.8b, %[b0].8b\n"
+                "uadalp    v25.4s, v13.8h\n"
+                "umull    v13.8h, v3.8b, %[b1].8b\n"
+                "uadalp    v26.4s, v14.8h\n"
+                "umull    v14.8h, v3.8b, %[b2].8b\n"
+                "uadalp    v27.4s, v15.8h\n"
+                "umull    v15.8h, v3.8b, %[b3].8b\n"
+
+                "uadalp    v28.4s, v12.8h\n"
+                "umull2    v12.8h, v3.16b, %[b0].16b\n"
+                "ldr    %q[b0], [%[b_ptr]]\n"
+                "uadalp    v29.4s, v13.8h\n"
+                "umull2    v13.8h, v3.16b, %[b1].16b\n"
+                "ldr    %q[b1], [%[b_ptr], #16]\n"
+                "uadalp    v30.4s, v14.8h\n"
+                "umull2    v14.8h, v3.16b, %[b2].16b\n"
+                "ldr    %q[b2], [%[b_ptr], #32]\n"
+                "uadalp    v31.4s, v15.8h\n"
+                "umull2    v15.8h, v3.16b, %[b3].16b\n"
+                "ldr    %q[b3], [%[b_ptr], #48]\n"
+
+                "uadalp    v28.4s, v12.8h\n"
+                "umull    v12.8h, v0.8b, %[b0].8b\n"
+                "add    %[b_ptr], %[b_ptr], #64\n"
+                "uadalp    v29.4s, v13.8h\n"
+                "umull    v13.8h, v0.8b, %[b1].8b\n"
+                "ldr    q3, [%[a_ptr], #48]\n"
+                "uadalp    v30.4s, v14.8h\n"
+                "umull    v14.8h, v0.8b, %[b2].8b\n"
+                "add    %[a_ptr], %[a_ptr], #64\n"
+                "uadalp    v31.4s, v15.8h\n"
+                "umull    v15.8h, v0.8b, %[b3].8b\n"
+                "bne    1b\n"
+
+                // Branch target
+                "2:\n"
+                "uadalp    v16.4s, v12.8h\n"
+                "umull2    v12.8h, v0.16b, %[b0].16b\n"
+                "uadalp    v17.4s, v13.8h\n"
+                "umull2    v13.8h, v0.16b, %[b1].16b\n"
+                "uadalp    v18.4s, v14.8h\n"
+                "umull2    v14.8h, v0.16b, %[b2].16b\n"
+                "uadalp    v19.4s, v15.8h\n"
+                "umull2    v15.8h, v0.16b, %[b3].16b\n"
+
+                "uadalp    v16.4s, v12.8h\n"
+                "umull    v12.8h, v1.8b, %[b0].8b\n"
+                "uadalp    v17.4s, v13.8h\n"
+                "umull    v13.8h, v1.8b, %[b1].8b\n"
+                "uadalp    v18.4s, v14.8h\n"
+                "umull    v14.8h, v1.8b, %[b2].8b\n"
+                "uadalp    v19.4s, v15.8h\n"
+                "umull    v15.8h, v1.8b, %[b3].8b\n"
+
+                "uadalp    v20.4s, v12.8h\n"
+                "umull2    v12.8h, v1.16b, %[b0].16b\n"
+                "uadalp    v21.4s, v13.8h\n"
+                "umull2    v13.8h, v1.16b, %[b1].16b\n"
+                "uadalp    v22.4s, v14.8h\n"
+                "umull2    v14.8h, v1.16b, %[b2].16b\n"
+                "uadalp    v23.4s, v15.8h\n"
+                "umull2    v15.8h, v1.16b, %[b3].16b\n"
+
+                "uadalp    v20.4s, v12.8h\n"
+                "umull    v12.8h, v2.8b, %[b0].8b\n"
+                "uadalp    v21.4s, v13.8h\n"
+                "umull    v13.8h, v2.8b, %[b1].8b\n"
+                "uadalp    v22.4s, v14.8h\n"
+                "umull    v14.8h, v2.8b, %[b2].8b\n"
+                "uadalp    v23.4s, v15.8h\n"
+                "umull    v15.8h, v2.8b, %[b3].8b\n"
+
+                "uadalp    v24.4s, v12.8h\n"
+                "umull2    v12.8h, v2.16b, %[b0].16b\n"
+                "uadalp    v25.4s, v13.8h\n"
+                "umull2    v13.8h, v2.16b, %[b1].16b\n"
+                "uadalp    v26.4s, v14.8h\n"
+                "umull2    v14.8h, v2.16b, %[b2].16b\n"
+                "uadalp    v27.4s, v15.8h\n"
+                "umull2    v15.8h, v2.16b, %[b3].16b\n"
+
+                "uadalp    v24.4s, v12.8h\n"
+                "umull    v12.8h, v3.8b, %[b0].8b\n"
+                "uadalp    v25.4s, v13.8h\n"
+                "umull    v13.8h, v3.8b, %[b1].8b\n"
+                "uadalp    v26.4s, v14.8h\n"
+                "umull    v14.8h, v3.8b, %[b2].8b\n"
+                "uadalp    v27.4s, v15.8h\n"
+                "umull    v15.8h, v3.8b, %[b3].8b\n"
+
+                "uadalp    v28.4s, v12.8h\n"
+                "umull2    v12.8h, v3.16b, %[b0].16b\n"
+                "uadalp    v29.4s, v13.8h\n"
+                "umull2    v13.8h, v3.16b, %[b1].16b\n"
+                "uadalp    v30.4s, v14.8h\n"
+                "umull2    v14.8h, v3.16b, %[b2].16b\n"
+                "uadalp    v31.4s, v15.8h\n"
+                "umull2    v15.8h, v3.16b, %[b3].16b\n"
+
+                "uadalp    v28.4s, v12.8h\n"
+                "uadalp    v29.4s, v13.8h\n"
+                "uadalp    v30.4s, v14.8h\n"
+                "uadalp    v31.4s, v15.8h\n"
+
+                "addp    v16.4s, v16.4s, v17.4s\n"
+                "addp    v17.4s, v18.4s, v19.4s\n"
+                "addp    v18.4s, v20.4s, v21.4s\n"
+                "addp    v19.4s, v22.4s, v23.4s\n"
+                "addp    v20.4s, v24.4s, v25.4s\n"
+                "addp    v21.4s, v26.4s, v27.4s\n"
+                "addp    v22.4s, v28.4s, v29.4s\n"
+                "addp    v23.4s, v30.4s, v31.4s\n"
+
+                "addp    v16.4s, v16.4s, v17.4s\n"
+                "addp    v17.4s, v18.4s, v19.4s\n"
+                "addp    v18.4s, v20.4s, v21.4s\n"
+                "addp    v19.4s, v22.4s, v23.4s\n"
+
+                "str    q16, [%[c_ptr]]\n"
+                "str    q17, [%[c_ptr], #16]\n"
+                "str    q18, [%[c_ptr], #32]\n"
+                "str    q19, [%[c_ptr], #48]\n"
+                "add    %[c_ptr], %[c_ptr], #64\n"
+
+                :
+                [a_ptr] "+r"(a_ptr), [b_ptr] "+r"(b_ptr), [c_ptr] "+r"(c_ptr),
+                [b0] "+w"(b0), [b1] "+w"(b1), [b2] "+w"(b2), [b3] "+w"(b3),
+                [k] "+r"(k)
+                :
+                : "x20", "x21", "v0", "v1", "v2", "v3", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
+                "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc");
+        }
+    }
+}
+
+} // namespace arm_gemm
+
+#endif // __aarch64__