aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGeorgios Pinitas <georgios.pinitas@arm.com>2020-01-22 18:36:27 +0000
committerGeorgios Pinitas <georgios.pinitas@arm.com>2020-01-31 11:36:14 +0000
commit94672fb2af6535adc6ea7fe8b8498580ad8cf3f4 (patch)
tree189c68e7ff2c29d7800e09239da6d9a6364d5b05
parent6a342648ae50beb8457871862f14fc9baef6b74f (diff)
downloadComputeLibrary-94672fb2af6535adc6ea7fe8b8498580ad8cf3f4.tar.gz
COMPMID-3003: Integrate assembly kernels utilizing MMLA instruction.
MMLA is a matrix-multiply instruction introduced on armv8.6-A Signed-off-by: Georgios Pinitas <georgios.pinitas@arm.com> Change-Id: I572a54981d48f5a1e0e9e51102cb7ae28ad87806 Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/2663 Tested-by: Arm Jenkins <bsgcomp@arm.com> Reviewed-by: Michalis Spyrou <michalis.spyrou@arm.com> Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
-rw-r--r--Android.bp4
-rw-r--r--SConstruct19
-rw-r--r--src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp2
-rw-r--r--src/core/NEON/kernels/arm_gemm/gemm_int8.cpp20
-rw-r--r--src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp20
-rw-r--r--src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8.hpp72
-rw-r--r--src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8/generic.cpp393
-rw-r--r--src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8.hpp72
-rw-r--r--src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8/generic.cpp393
-rw-r--r--src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8.hpp72
-rw-r--r--src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8/generic.cpp395
-rw-r--r--src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8.hpp72
-rw-r--r--src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8/generic.cpp395
-rw-r--r--src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp45
-rw-r--r--src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_8way_32bit.hpp147
-rw-r--r--src/core/NEON/kernels/arm_gemm/transforms/list.hpp4
-rw-r--r--src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_16bit.hpp596
-rw-r--r--src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_16bit.hpp596
-rw-r--r--src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block8_8bit.hpp596
19 files changed, 3903 insertions, 10 deletions
diff --git a/Android.bp b/Android.bp
index 8ebe4c52d1..e7ad65187f 100644
--- a/Android.bp
+++ b/Android.bp
@@ -764,6 +764,8 @@ cc_library_static {
"src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_s8s32_dot_16x4/generic.cpp",
"src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/a55.cpp",
"src/core/NEON/kernels/arm_gemm/kernels/a64_hybrid_u8u32_dot_16x4/generic.cpp",
+ "src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8/generic.cpp",
+ "src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8/generic.cpp",
"src/core/NEON/kernels/arm_gemm/kernels/a64_native_fp32_mla_16x4/generic.cpp",
"src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/a53.cpp",
"src/core/NEON/kernels/arm_gemm/kernels/a64_sgemm_12x8/a55.cpp",
@@ -788,7 +790,9 @@ cc_library_static {
"src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp16_mla_3VLx8/generic.cpp",
"src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_fp32_mla_3VLx8/generic.cpp",
"src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_dot_3VLx8/generic.cpp",
+ "src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8/generic.cpp",
"src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_dot_3VLx8/generic.cpp",
+ "src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8/generic.cpp",
"src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp16_mla_4VLx4/generic.cpp",
"src/core/NEON/kernels/arm_gemm/kernels/sve_native_fp32_mla_4VLx4/generic.cpp",
"src/core/NEON/kernels/arm_gemm/kernels/sve_native_s8s32_dot_4VLx4/generic.cpp",
diff --git a/SConstruct b/SConstruct
index 3236e7ca30..0b491847df 100644
--- a/SConstruct
+++ b/SConstruct
@@ -41,7 +41,8 @@ vars.AddVariables(
BoolVariable("asserts", "Enable asserts (this flag is forced to 1 for debug=1)", False),
BoolVariable("logging", "Logging (this flag is forced to 1 for debug=1)", False),
EnumVariable("arch", "Target Architecture", "armv7a",
- allowed_values=("armv7a", "arm64-v8a", "arm64-v8.2-a", "arm64-v8.2-a-sve", "x86_32", "x86_64", "armv8a", "armv8.2-a", "armv8.2-a-sve", "x86")),
+ allowed_values=("armv7a", "arm64-v8a", "arm64-v8.2-a", "arm64-v8.2-a-sve", "x86_32", "x86_64",
+ "armv8a", "armv8.2-a", "armv8.2-a-sve", "armv8.6-a", "x86")),
EnumVariable("estate", "Execution State", "auto", allowed_values=("auto", "32", "64")),
EnumVariable("os", "Target OS", "linux", allowed_values=("linux", "android", "bare_metal")),
EnumVariable("build", "Build type", "cross_compile", allowed_values=("native", "cross_compile", "embed_only")),
@@ -194,17 +195,17 @@ if 'v7a' in env['arch']:
env.Append(CXXFLAGS = ['-mfloat-abi=softfp'])
else:
env.Append(CXXFLAGS = ['-mfloat-abi=hard'])
-elif 'v8a' in env['arch']:
- env.Append(CXXFLAGS = ['-march=armv8-a'])
- if env['estate'] == '32':
- env.Append(CXXFLAGS = ['-mfpu=neon-fp-armv8'])
-elif 'v8.2-a' in env['arch']:
- if env['estate'] == '32':
- env.Append(CXXFLAGS = ['-mfpu=neon-fp-armv8'])
+elif 'v8' in env['arch']:
if 'sve' in env['arch']:
env.Append(CXXFLAGS = ['-march=armv8.2-a+sve+fp16+dotprod'])
- else:
+ elif 'v8.2-a' in env['arch']:
env.Append(CXXFLAGS = ['-march=armv8.2-a+fp16']) # explicitly enable fp16 extension otherwise __ARM_FEATURE_FP16_VECTOR_ARITHMETIC is undefined
+ else:
+ env.Append(CXXFLAGS = ['-march=armv8-a'])
+
+ if 'v8.6-a' in env['arch']:
+ env.Append(CXXFLAGS = ['-DV8P6'])
+
elif 'x86' in env['arch']:
if env['estate'] == '32':
env.Append(CCFLAGS = ['-m32'])
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
index 7f171ec15a..35493a609c 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_fp32.cpp
@@ -114,7 +114,7 @@ static const GemmImplementation<float, float> gemm_fp32_methods[] =
},
{
GemmMethod::GEMM_HYBRID,
- "hybrid_fp32_mla_16x4",
+ "hybrid_fp32_mla_16x4_normal",
[](const GemmArgs &args) { return (args._Ksize >= 4) && !args._trA && args._pretransposed_hint; },
[](const GemmArgs &args) { return ((args._Ksize <= 256) && (args._Nsize <= 256)) || (args._Msize < 16) || (args._nmulti > 1); },
[](const GemmArgs &args) { return new GemmHybrid<hybrid_fp32_mla_16x4, float, float>(args); }
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
index a3446b9ddc..f7d8f65aea 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_int8.cpp
@@ -34,10 +34,12 @@
#include "kernels/a64_gemm_s8_12x8.hpp"
#include "kernels/a64_gemm_s8_4x4.hpp"
#include "kernels/a64_hybrid_s8s32_dot_16x4.hpp"
+#include "kernels/a64_interleaved_s8s32_mmla_12x8.hpp"
#include "kernels/a64_smallK_hybrid_s8s32_dot_4x6.hpp"
#include "kernels/a64_smallK_hybrid_s8s32_dot_4x8.hpp"
#include "kernels/sve_hybrid_s8s32_dot_4VLx4.hpp"
#include "kernels/sve_interleaved_s8s32_dot_3VLx8.hpp"
+#include "kernels/sve_interleaved_s8s32_mmla_3VLx8.hpp"
#include "kernels/sve_native_s8s32_dot_4VLx4.hpp"
#include "kernels/sve_smallK_hybrid_s8s32_dot_1VLx8.hpp"
@@ -45,6 +47,15 @@ namespace arm_gemm {
static const GemmImplementation<int8_t, int32_t> gemm_s8_methods[] = {
#ifdef __ARM_FEATURE_SVE
+#ifdef V8P6
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "interleaved_s8s32_mmla_3VLx8",
+ [](const GemmArgs &args) { return (args._Ksize>8); },
+ nullptr,
+ [](const GemmArgs &args) { return new GemmInterleaved<interleaved_s8s32_mmla_3VLx8, int8_t, int32_t>(args); }
+},
+#endif
{
GemmMethod::GEMM_HYBRID,
"smallK_hybrid_s8s32_dot_1VLx8",
@@ -74,6 +85,15 @@ static const GemmImplementation<int8_t, int32_t> gemm_s8_methods[] = {
[](const GemmArgs &args) { return new GemmInterleaved<interleaved_s8s32_dot_3VLx8, int8_t, int32_t>(args); }
},
#endif
+#ifdef V8P6
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "interleaved_s8s32_mmla_12x8",
+ [](const GemmArgs &args) { return (args._Ksize>8); },
+ nullptr,
+ [](const GemmArgs &args) { return new GemmInterleaved<interleaved_s8s32_mmla_12x8, int8_t, int32_t>(args); }
+},
+#endif
{
GemmMethod::GEMM_HYBRID,
"smallK_hybrid_s8s32_dot_4x8",
diff --git a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
index aead814d7e..430d35e06d 100644
--- a/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
+++ b/src/core/NEON/kernels/arm_gemm/gemm_uint8.cpp
@@ -34,10 +34,12 @@
#include "kernels/a64_gemm_u8_12x8.hpp"
#include "kernels/a64_gemm_u8_4x4.hpp"
#include "kernels/a64_hybrid_u8u32_dot_16x4.hpp"
+#include "kernels/a64_interleaved_u8u32_mmla_12x8.hpp"
#include "kernels/a64_smallK_hybrid_u8u32_dot_4x6.hpp"
#include "kernels/a64_smallK_hybrid_u8u32_dot_4x8.hpp"
#include "kernels/sve_hybrid_u8u32_dot_4VLx4.hpp"
#include "kernels/sve_interleaved_u8u32_dot_3VLx8.hpp"
+#include "kernels/sve_interleaved_u8u32_mmla_3VLx8.hpp"
#include "kernels/sve_native_u8u32_dot_4VLx4.hpp"
#include "kernels/sve_smallK_hybrid_u8u32_dot_1VLx8.hpp"
@@ -45,6 +47,15 @@ namespace arm_gemm {
static const GemmImplementation<uint8_t, uint32_t> gemm_u8_methods[] = {
#ifdef __ARM_FEATURE_SVE
+#ifdef V8P6
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "interleaved_u8u32_mmla_3VLx8",
+ [](const GemmArgs &args) { return (args._Ksize>8); },
+ nullptr,
+ [](const GemmArgs &args) { return new GemmInterleaved<interleaved_u8u32_mmla_3VLx8, uint8_t, uint32_t>(args); }
+},
+#endif
{
GemmMethod::GEMM_HYBRID,
"smallK_hybrid_u8u32_dot_1VLx8",
@@ -74,6 +85,15 @@ static const GemmImplementation<uint8_t, uint32_t> gemm_u8_methods[] = {
[](const GemmArgs &args) { return new GemmInterleaved<interleaved_u8u32_dot_3VLx8, uint8_t, uint32_t>(args); }
},
#endif
+#ifdef V8P6
+{
+ GemmMethod::GEMM_INTERLEAVED,
+ "interleaved_u8u32_mmla_12x8",
+ [](const GemmArgs &args) { return (args._Ksize>8); },
+ nullptr,
+ [](const GemmArgs &args) { return new GemmInterleaved<interleaved_u8u32_mmla_12x8, uint8_t, uint32_t>(args); }
+},
+#endif
{
GemmMethod::GEMM_HYBRID,
"smallK_hybrid_u8u32_dot_4x8",
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8.hpp
new file mode 100644
index 0000000000..f669b870c6
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8.hpp
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __aarch64__
+
+#include <cstdint>
+#include "../std_transforms_fixed.hpp"
+
+namespace arm_gemm {
+
+// Actual kernel implementations
+void a64_interleaved_s8s32_mmla_12x8(const int8_t *, const int8_t *, int32_t *, int, int, int);
+
+class interleaved_s8s32_mmla_12x8 {
+public:
+ typedef int8_t operand_type;
+ typedef int32_t result_type;
+
+ typedef void (*kern_type)(const int8_t *, const int8_t *, int32_t *, int, int, int);
+
+ /* Kernel blocking parameters */
+ static unsigned int out_width()
+ {
+ return 12;
+ }
+
+ static unsigned int out_height()
+ {
+ return 8;
+ }
+
+ static unsigned int k_unroll()
+ {
+ return 8;
+ }
+
+ // Use the standard fixed size transforms.
+ StdTransformsFixed<operand_type, result_type, 8, 12, 8> transforms = {};
+
+ kern_type kernel=a64_interleaved_s8s32_mmla_12x8;
+
+ interleaved_s8s32_mmla_12x8(const CPUInfo *ci)
+ {
+ UNUSED(ci);
+ }
+};
+
+} // namespace arm_gemm
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8/generic.cpp
new file mode 100644
index 0000000000..49dbdb866e
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_s8s32_mmla_12x8/generic.cpp
@@ -0,0 +1,393 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include <cstdint>
+#include "../../asmlib.hpp"
+
+namespace arm_gemm {
+
+void a64_interleaved_s8s32_mmla_12x8(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K) {
+ const int8_t *a_ptr = Apanel;
+ int32_t *c_ptr = Cpanel;
+
+ K /= 8;
+ const long loops_count = (K / 2) - 1;
+ const long tails_count = K % 2;
+
+ for (int yb=0; yb<ablocks; yb++) {
+ const int8_t *a_ptr0 = a_ptr;
+ const int8_t *b_ptr = Bpanel;
+
+ for (int xb=0; xb<bblocks; xb++) {
+ a_ptr = a_ptr0;
+ long loops = loops_count;
+ long tails = tails_count;
+
+ __asm __volatile (
+ "movi v8.4s, #0\n"
+ "ldr q0, [%[a_ptr]]\n"
+ "movi v9.4s, #0\n"
+ "ldr q4, [%[b_ptr]]\n"
+ "movi v10.4s, #0\n"
+ "ldr q1, [%[a_ptr], #0x10]\n"
+ "movi v11.4s, #0\n"
+ "ldr q5, [%[b_ptr], #0x10]\n"
+ "movi v12.4s, #0\n"
+ "ldr q2, [%[a_ptr], #0x20]\n"
+ "movi v13.4s, #0\n"
+ "ldr q6, [%[b_ptr], #0x20]\n"
+ "movi v14.4s, #0\n"
+ "ldr q3, [%[a_ptr], #0x30]\n"
+ "movi v15.4s, #0\n"
+ "ldr q7, [%[b_ptr], #0x30]\n"
+ "movi v16.4s, #0\n"
+ "add %[a_ptr], %[a_ptr], #0x40\n"
+ "movi v17.4s, #0\n"
+ "add %[b_ptr], %[b_ptr], #0x40\n"
+ "movi v18.4s, #0\n"
+ "movi v19.4s, #0\n"
+ "movi v20.4s, #0\n"
+ "movi v21.4s, #0\n"
+ "movi v22.4s, #0\n"
+ "movi v23.4s, #0\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ ".inst 0x4e84a408 // smmla v8.4s, v0.16b, v4.16b\n"
+ "subs %[loops], %[loops], #0x1\n"
+ ".inst 0x4e84a42e // smmla v14.4s, v1.16b, v4.16b\n"
+ ".inst 0x4e84a454 // smmla v20.4s, v2.16b, v4.16b\n"
+ ".inst 0x4e84a47a // smmla v26.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr]]\n"
+ ".inst 0x4e85a409 // smmla v9.4s, v0.16b, v5.16b\n"
+ ".inst 0x4e85a42f // smmla v15.4s, v1.16b, v5.16b\n"
+ ".inst 0x4e85a455 // smmla v21.4s, v2.16b, v5.16b\n"
+ ".inst 0x4e85a47b // smmla v27.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #0x10]\n"
+ ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n"
+ ".inst 0x4e86a430 // smmla v16.4s, v1.16b, v6.16b\n"
+ ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n"
+ ".inst 0x4e86a47c // smmla v28.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #0x20]\n"
+ ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n"
+ ".inst 0x4e87a431 // smmla v17.4s, v1.16b, v7.16b\n"
+ ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n"
+ ".inst 0x4e87a47d // smmla v29.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #0x30]\n"
+ ".inst 0x4e84a40c // smmla v12.4s, v0.16b, v4.16b\n"
+ ".inst 0x4e84a432 // smmla v18.4s, v1.16b, v4.16b\n"
+ ".inst 0x4e84a458 // smmla v24.4s, v2.16b, v4.16b\n"
+ ".inst 0x4e84a47e // smmla v30.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr], #0x40]\n"
+ ".inst 0x4e85a40d // smmla v13.4s, v0.16b, v5.16b\n"
+ "ldr q0, [%[a_ptr]]\n"
+ ".inst 0x4e85a433 // smmla v19.4s, v1.16b, v5.16b\n"
+ "ldr q1, [%[a_ptr], #0x10]\n"
+ ".inst 0x4e85a459 // smmla v25.4s, v2.16b, v5.16b\n"
+ "ldr q2, [%[a_ptr], #0x20]\n"
+ ".inst 0x4e85a47f // smmla v31.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #0x50]\n"
+ ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n"
+ "ldr q3, [%[a_ptr], #0x30]\n"
+ ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n"
+ "add %[a_ptr], %[a_ptr], #0x80\n"
+ ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n"
+ "add %[b_ptr], %[b_ptr], #0xc0\n"
+ ".inst 0x4e86a47a // smmla v26.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #-0x60]\n"
+ ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n"
+ ".inst 0x4e87a42f // smmla v15.4s, v1.16b, v7.16b\n"
+ ".inst 0x4e87a455 // smmla v21.4s, v2.16b, v7.16b\n"
+ ".inst 0x4e87a47b // smmla v27.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #-0x50]\n"
+ ".inst 0x4e84a40a // smmla v10.4s, v0.16b, v4.16b\n"
+ ".inst 0x4e84a430 // smmla v16.4s, v1.16b, v4.16b\n"
+ ".inst 0x4e84a456 // smmla v22.4s, v2.16b, v4.16b\n"
+ ".inst 0x4e84a47c // smmla v28.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr], #-0x40]\n"
+ ".inst 0x4e85a40b // smmla v11.4s, v0.16b, v5.16b\n"
+ ".inst 0x4e85a431 // smmla v17.4s, v1.16b, v5.16b\n"
+ ".inst 0x4e85a457 // smmla v23.4s, v2.16b, v5.16b\n"
+ ".inst 0x4e85a47d // smmla v29.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #-0x30]\n"
+ ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n"
+ ".inst 0x4e86a432 // smmla v18.4s, v1.16b, v6.16b\n"
+ ".inst 0x4e86a458 // smmla v24.4s, v2.16b, v6.16b\n"
+ ".inst 0x4e86a47e // smmla v30.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #-0x20]\n"
+ ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n"
+ "ldr q0, [%[a_ptr], #-0x40]\n"
+ ".inst 0x4e87a433 // smmla v19.4s, v1.16b, v7.16b\n"
+ "ldr q1, [%[a_ptr], #-0x30]\n"
+ ".inst 0x4e87a459 // smmla v25.4s, v2.16b, v7.16b\n"
+ "ldr q2, [%[a_ptr], #-0x20]\n"
+ ".inst 0x4e87a47f // smmla v31.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #-0x10]\n"
+ "ldr q3, [%[a_ptr], #-0x10]\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[tails], 3f\n"
+ ".inst 0x4e84a408 // smmla v8.4s, v0.16b, v4.16b\n"
+ ".inst 0x4e84a42e // smmla v14.4s, v1.16b, v4.16b\n"
+ ".inst 0x4e84a454 // smmla v20.4s, v2.16b, v4.16b\n"
+ ".inst 0x4e84a47a // smmla v26.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr]]\n"
+ ".inst 0x4e85a409 // smmla v9.4s, v0.16b, v5.16b\n"
+ ".inst 0x4e85a42f // smmla v15.4s, v1.16b, v5.16b\n"
+ ".inst 0x4e85a455 // smmla v21.4s, v2.16b, v5.16b\n"
+ ".inst 0x4e85a47b // smmla v27.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #0x10]\n"
+ ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n"
+ ".inst 0x4e86a430 // smmla v16.4s, v1.16b, v6.16b\n"
+ ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n"
+ ".inst 0x4e86a47c // smmla v28.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #0x20]\n"
+ ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n"
+ ".inst 0x4e87a431 // smmla v17.4s, v1.16b, v7.16b\n"
+ ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n"
+ ".inst 0x4e87a47d // smmla v29.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #0x30]\n"
+ ".inst 0x4e84a40c // smmla v12.4s, v0.16b, v4.16b\n"
+ ".inst 0x4e84a432 // smmla v18.4s, v1.16b, v4.16b\n"
+ ".inst 0x4e84a458 // smmla v24.4s, v2.16b, v4.16b\n"
+ ".inst 0x4e84a47e // smmla v30.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr], #0x40]\n"
+ ".inst 0x4e85a40d // smmla v13.4s, v0.16b, v5.16b\n"
+ "ldr q0, [%[a_ptr]]\n"
+ ".inst 0x4e85a433 // smmla v19.4s, v1.16b, v5.16b\n"
+ "ldr q1, [%[a_ptr], #0x10]\n"
+ ".inst 0x4e85a459 // smmla v25.4s, v2.16b, v5.16b\n"
+ "ldr q2, [%[a_ptr], #0x20]\n"
+ ".inst 0x4e85a47f // smmla v31.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #0x50]\n"
+ ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n"
+ "ldr q3, [%[a_ptr], #0x30]\n"
+ ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n"
+ "add %[a_ptr], %[a_ptr], #0x80\n"
+ ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n"
+ "add %[b_ptr], %[b_ptr], #0xe0\n"
+ ".inst 0x4e86a47a // smmla v26.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #-0x80]\n"
+ ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n"
+ ".inst 0x4e87a42f // smmla v15.4s, v1.16b, v7.16b\n"
+ ".inst 0x4e87a455 // smmla v21.4s, v2.16b, v7.16b\n"
+ ".inst 0x4e87a47b // smmla v27.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #-0x70]\n"
+ ".inst 0x4e84a40a // smmla v10.4s, v0.16b, v4.16b\n"
+ ".inst 0x4e84a430 // smmla v16.4s, v1.16b, v4.16b\n"
+ ".inst 0x4e84a456 // smmla v22.4s, v2.16b, v4.16b\n"
+ ".inst 0x4e84a47c // smmla v28.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr], #-0x60]\n"
+ ".inst 0x4e85a40b // smmla v11.4s, v0.16b, v5.16b\n"
+ ".inst 0x4e85a431 // smmla v17.4s, v1.16b, v5.16b\n"
+ ".inst 0x4e85a457 // smmla v23.4s, v2.16b, v5.16b\n"
+ ".inst 0x4e85a47d // smmla v29.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #-0x50]\n"
+ ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n"
+ ".inst 0x4e86a432 // smmla v18.4s, v1.16b, v6.16b\n"
+ ".inst 0x4e86a458 // smmla v24.4s, v2.16b, v6.16b\n"
+ ".inst 0x4e86a47e // smmla v30.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #-0x40]\n"
+ ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n"
+ "ldr q0, [%[a_ptr], #-0x40]\n"
+ ".inst 0x4e87a433 // smmla v19.4s, v1.16b, v7.16b\n"
+ "ldr q1, [%[a_ptr], #-0x30]\n"
+ ".inst 0x4e87a459 // smmla v25.4s, v2.16b, v7.16b\n"
+ "ldr q2, [%[a_ptr], #-0x20]\n"
+ ".inst 0x4e87a47f // smmla v31.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #-0x30]\n"
+ ".inst 0x4e84a408 // smmla v8.4s, v0.16b, v4.16b\n"
+ "ldr q3, [%[a_ptr], #-0x10]\n"
+ ".inst 0x4e84a42e // smmla v14.4s, v1.16b, v4.16b\n"
+ ".inst 0x4e84a454 // smmla v20.4s, v2.16b, v4.16b\n"
+ ".inst 0x4e85a409 // smmla v9.4s, v0.16b, v5.16b\n"
+ ".inst 0x4e84a47a // smmla v26.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr], #-0x20]\n"
+ ".inst 0x4e85a42f // smmla v15.4s, v1.16b, v5.16b\n"
+ ".inst 0x4e85a455 // smmla v21.4s, v2.16b, v5.16b\n"
+ ".inst 0x4e85a47b // smmla v27.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #-0x10]\n"
+ ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n"
+ ".inst 0x4e86a430 // smmla v16.4s, v1.16b, v6.16b\n"
+ ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n"
+ ".inst 0x4e86a47c // smmla v28.4s, v3.16b, v6.16b\n"
+ "uzp1 v6.2d, v14.2d, v15.2d\n"
+ ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n"
+ ".inst 0x4e87a431 // smmla v17.4s, v1.16b, v7.16b\n"
+ ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n"
+ ".inst 0x4e87a47d // smmla v29.4s, v3.16b, v7.16b\n"
+ ".inst 0x4e84a40c // smmla v12.4s, v0.16b, v4.16b\n"
+ "uzp1 v7.2d, v16.2d, v17.2d\n"
+ ".inst 0x4e84a432 // smmla v18.4s, v1.16b, v4.16b\n"
+ ".inst 0x4e84a458 // smmla v24.4s, v2.16b, v4.16b\n"
+ ".inst 0x4e84a47e // smmla v30.4s, v3.16b, v4.16b\n"
+ "uzp2 v4.2d, v10.2d, v11.2d\n"
+ ".inst 0x4e85a40d // smmla v13.4s, v0.16b, v5.16b\n"
+ "uzp1 v0.2d, v8.2d, v9.2d\n"
+ ".inst 0x4e85a433 // smmla v19.4s, v1.16b, v5.16b\n"
+ "uzp1 v1.2d, v10.2d, v11.2d\n"
+ ".inst 0x4e85a459 // smmla v25.4s, v2.16b, v5.16b\n"
+ "str q0, [%[c_ptr]]\n"
+ "uzp1 v2.2d, v12.2d, v13.2d\n"
+ "uzp1 v0.2d, v18.2d, v19.2d\n"
+ ".inst 0x4e85a47f // smmla v31.4s, v3.16b, v5.16b\n"
+ "str q1, [%[c_ptr], #0x10]\n"
+ "uzp2 v3.2d, v8.2d, v9.2d\n"
+ "uzp2 v5.2d, v12.2d, v13.2d\n"
+ "uzp2 v1.2d, v14.2d, v15.2d\n"
+ "str q2, [%[c_ptr], #0x20]\n"
+ "b 4f\n"
+ "3:\n"
+ ".inst 0x4e84a408 // smmla v8.4s, v0.16b, v4.16b\n"
+ "add %[a_ptr], %[a_ptr], #0x40\n"
+ ".inst 0x4e84a42e // smmla v14.4s, v1.16b, v4.16b\n"
+ "add %[b_ptr], %[b_ptr], #0x80\n"
+ ".inst 0x4e84a454 // smmla v20.4s, v2.16b, v4.16b\n"
+ ".inst 0x4e84a47a // smmla v26.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr], #-0x80]\n"
+ ".inst 0x4e85a409 // smmla v9.4s, v0.16b, v5.16b\n"
+ ".inst 0x4e85a42f // smmla v15.4s, v1.16b, v5.16b\n"
+ ".inst 0x4e85a455 // smmla v21.4s, v2.16b, v5.16b\n"
+ ".inst 0x4e85a47b // smmla v27.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #-0x70]\n"
+ ".inst 0x4e86a40a // smmla v10.4s, v0.16b, v6.16b\n"
+ ".inst 0x4e86a430 // smmla v16.4s, v1.16b, v6.16b\n"
+ ".inst 0x4e86a456 // smmla v22.4s, v2.16b, v6.16b\n"
+ ".inst 0x4e86a47c // smmla v28.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #-0x60]\n"
+ ".inst 0x4e87a40b // smmla v11.4s, v0.16b, v7.16b\n"
+ ".inst 0x4e87a431 // smmla v17.4s, v1.16b, v7.16b\n"
+ ".inst 0x4e87a457 // smmla v23.4s, v2.16b, v7.16b\n"
+ ".inst 0x4e87a47d // smmla v29.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #-0x50]\n"
+ ".inst 0x4e84a40c // smmla v12.4s, v0.16b, v4.16b\n"
+ ".inst 0x4e84a432 // smmla v18.4s, v1.16b, v4.16b\n"
+ ".inst 0x4e84a458 // smmla v24.4s, v2.16b, v4.16b\n"
+ ".inst 0x4e84a47e // smmla v30.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr], #-0x40]\n"
+ ".inst 0x4e85a40d // smmla v13.4s, v0.16b, v5.16b\n"
+ "ldr q0, [%[a_ptr], #-0x40]\n"
+ ".inst 0x4e85a433 // smmla v19.4s, v1.16b, v5.16b\n"
+ "ldr q1, [%[a_ptr], #-0x30]\n"
+ ".inst 0x4e85a459 // smmla v25.4s, v2.16b, v5.16b\n"
+ "ldr q2, [%[a_ptr], #-0x20]\n"
+ ".inst 0x4e85a47f // smmla v31.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #-0x30]\n"
+ ".inst 0x4e86a408 // smmla v8.4s, v0.16b, v6.16b\n"
+ "ldr q3, [%[a_ptr], #-0x10]\n"
+ ".inst 0x4e86a42e // smmla v14.4s, v1.16b, v6.16b\n"
+ ".inst 0x4e86a454 // smmla v20.4s, v2.16b, v6.16b\n"
+ ".inst 0x4e87a409 // smmla v9.4s, v0.16b, v7.16b\n"
+ ".inst 0x4e86a47a // smmla v26.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #-0x20]\n"
+ ".inst 0x4e87a42f // smmla v15.4s, v1.16b, v7.16b\n"
+ ".inst 0x4e87a455 // smmla v21.4s, v2.16b, v7.16b\n"
+ ".inst 0x4e87a47b // smmla v27.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #-0x10]\n"
+ ".inst 0x4e84a40a // smmla v10.4s, v0.16b, v4.16b\n"
+ ".inst 0x4e84a430 // smmla v16.4s, v1.16b, v4.16b\n"
+ ".inst 0x4e84a456 // smmla v22.4s, v2.16b, v4.16b\n"
+ ".inst 0x4e84a47c // smmla v28.4s, v3.16b, v4.16b\n"
+ ".inst 0x4e85a40b // smmla v11.4s, v0.16b, v5.16b\n"
+ ".inst 0x4e85a431 // smmla v17.4s, v1.16b, v5.16b\n"
+ ".inst 0x4e85a457 // smmla v23.4s, v2.16b, v5.16b\n"
+ ".inst 0x4e85a47d // smmla v29.4s, v3.16b, v5.16b\n"
+ "uzp2 v4.2d, v10.2d, v11.2d\n"
+ ".inst 0x4e86a40c // smmla v12.4s, v0.16b, v6.16b\n"
+ ".inst 0x4e86a432 // smmla v18.4s, v1.16b, v6.16b\n"
+ ".inst 0x4e86a458 // smmla v24.4s, v2.16b, v6.16b\n"
+ ".inst 0x4e86a47e // smmla v30.4s, v3.16b, v6.16b\n"
+ "uzp1 v6.2d, v14.2d, v15.2d\n"
+ ".inst 0x4e87a40d // smmla v13.4s, v0.16b, v7.16b\n"
+ "uzp1 v0.2d, v8.2d, v9.2d\n"
+ ".inst 0x4e87a433 // smmla v19.4s, v1.16b, v7.16b\n"
+ "uzp1 v1.2d, v10.2d, v11.2d\n"
+ "uzp2 v5.2d, v12.2d, v13.2d\n"
+ "str q0, [%[c_ptr]]\n"
+ ".inst 0x4e87a459 // smmla v25.4s, v2.16b, v7.16b\n"
+ "uzp1 v2.2d, v12.2d, v13.2d\n"
+ "uzp1 v0.2d, v18.2d, v19.2d\n"
+ "str q1, [%[c_ptr], #0x10]\n"
+ "uzp2 v1.2d, v14.2d, v15.2d\n"
+ ".inst 0x4e87a47f // smmla v31.4s, v3.16b, v7.16b\n"
+ "uzp2 v3.2d, v8.2d, v9.2d\n"
+ "str q2, [%[c_ptr], #0x20]\n"
+ "uzp1 v7.2d, v16.2d, v17.2d\n"
+ "4:\n"
+ "uzp2 v2.2d, v16.2d, v17.2d\n"
+ "str q3, [%[c_ptr], #0x30]\n"
+ "uzp2 v3.2d, v18.2d, v19.2d\n"
+ "str q4, [%[c_ptr], #0x40]\n"
+ "uzp1 v4.2d, v20.2d, v21.2d\n"
+ "str q5, [%[c_ptr], #0x50]\n"
+ "uzp1 v5.2d, v22.2d, v23.2d\n"
+ "str q6, [%[c_ptr], #0x60]\n"
+ "uzp1 v6.2d, v24.2d, v25.2d\n"
+ "str q7, [%[c_ptr], #0x70]\n"
+ "uzp2 v7.2d, v20.2d, v21.2d\n"
+ "str q0, [%[c_ptr], #0x80]\n"
+ "uzp2 v0.2d, v22.2d, v23.2d\n"
+ "str q1, [%[c_ptr], #0x90]\n"
+ "uzp2 v1.2d, v24.2d, v25.2d\n"
+ "str q2, [%[c_ptr], #0xa0]\n"
+ "uzp1 v2.2d, v26.2d, v27.2d\n"
+ "str q3, [%[c_ptr], #0xb0]\n"
+ "uzp1 v3.2d, v28.2d, v29.2d\n"
+ "str q4, [%[c_ptr], #0xc0]\n"
+ "uzp1 v4.2d, v30.2d, v31.2d\n"
+ "str q5, [%[c_ptr], #0xd0]\n"
+ "uzp2 v5.2d, v26.2d, v27.2d\n"
+ "str q6, [%[c_ptr], #0xe0]\n"
+ "uzp2 v6.2d, v28.2d, v29.2d\n"
+ "str q7, [%[c_ptr], #0xf0]\n"
+ "uzp2 v7.2d, v30.2d, v31.2d\n"
+ "str q0, [%[c_ptr], #0x100]\n"
+ "str q1, [%[c_ptr], #0x110]\n"
+ "str q2, [%[c_ptr], #0x120]\n"
+ "str q3, [%[c_ptr], #0x130]\n"
+ "str q4, [%[c_ptr], #0x140]\n"
+ "str q5, [%[c_ptr], #0x150]\n"
+ "str q6, [%[c_ptr], #0x160]\n"
+ "str q7, [%[c_ptr], #0x170]\n"
+ "add %[c_ptr], %[c_ptr], #0x180\n"
+ : [a_ptr] "+r" (a_ptr), [b_ptr] "+r" (b_ptr), [c_ptr] "+r" (c_ptr),
+ [loops] "+r" (loops), [tails] "+r" (tails)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ }
+ }
+}
+
+} // namespace arm_gemm
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8.hpp
new file mode 100644
index 0000000000..d66edd832a
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8.hpp
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __aarch64__
+
+#include <cstdint>
+#include "../std_transforms_fixed.hpp"
+
+namespace arm_gemm {
+
+// Actual kernel implementations
+void a64_interleaved_u8u32_mmla_12x8(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
+
+class interleaved_u8u32_mmla_12x8 {
+public:
+ typedef uint8_t operand_type;
+ typedef uint32_t result_type;
+
+ typedef void (*kern_type)(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
+
+ /* Kernel blocking parameters */
+ static unsigned int out_width()
+ {
+ return 12;
+ }
+
+ static unsigned int out_height()
+ {
+ return 8;
+ }
+
+ static unsigned int k_unroll()
+ {
+ return 8;
+ }
+
+ // Use the standard fixed size transforms.
+ StdTransformsFixed<operand_type, result_type, 8, 12, 8> transforms = {};
+
+ kern_type kernel=a64_interleaved_u8u32_mmla_12x8;
+
+ interleaved_u8u32_mmla_12x8(const CPUInfo *ci)
+ {
+ UNUSED(ci);
+ }
+};
+
+} // namespace arm_gemm
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8/generic.cpp
new file mode 100644
index 0000000000..e182a425f4
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/a64_interleaved_u8u32_mmla_12x8/generic.cpp
@@ -0,0 +1,393 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __aarch64__
+
+#include <cstdint>
+#include "../../asmlib.hpp"
+
+namespace arm_gemm {
+
+void a64_interleaved_u8u32_mmla_12x8(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K) {
+ const uint8_t *a_ptr = Apanel;
+ uint32_t *c_ptr = Cpanel;
+
+ K /= 8;
+ const long loops_count = (K / 2) - 1;
+ const long tails_count = K % 2;
+
+ for (int yb=0; yb<ablocks; yb++) {
+ const uint8_t *a_ptr0 = a_ptr;
+ const uint8_t *b_ptr = Bpanel;
+
+ for (int xb=0; xb<bblocks; xb++) {
+ a_ptr = a_ptr0;
+ long loops = loops_count;
+ long tails = tails_count;
+
+ __asm __volatile (
+ "movi v8.4s, #0\n"
+ "ldr q0, [%[a_ptr]]\n"
+ "movi v9.4s, #0\n"
+ "ldr q4, [%[b_ptr]]\n"
+ "movi v10.4s, #0\n"
+ "ldr q1, [%[a_ptr], #0x10]\n"
+ "movi v11.4s, #0\n"
+ "ldr q5, [%[b_ptr], #0x10]\n"
+ "movi v12.4s, #0\n"
+ "ldr q2, [%[a_ptr], #0x20]\n"
+ "movi v13.4s, #0\n"
+ "ldr q6, [%[b_ptr], #0x20]\n"
+ "movi v14.4s, #0\n"
+ "ldr q3, [%[a_ptr], #0x30]\n"
+ "movi v15.4s, #0\n"
+ "ldr q7, [%[b_ptr], #0x30]\n"
+ "movi v16.4s, #0\n"
+ "add %[a_ptr], %[a_ptr], #0x40\n"
+ "movi v17.4s, #0\n"
+ "add %[b_ptr], %[b_ptr], #0x40\n"
+ "movi v18.4s, #0\n"
+ "movi v19.4s, #0\n"
+ "movi v20.4s, #0\n"
+ "movi v21.4s, #0\n"
+ "movi v22.4s, #0\n"
+ "movi v23.4s, #0\n"
+ "movi v24.4s, #0\n"
+ "movi v25.4s, #0\n"
+ "movi v26.4s, #0\n"
+ "movi v27.4s, #0\n"
+ "movi v28.4s, #0\n"
+ "movi v29.4s, #0\n"
+ "movi v30.4s, #0\n"
+ "movi v31.4s, #0\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ ".inst 0x6e84a408 // ummla v8.4s, v0.16b, v4.16b\n"
+ "subs %[loops], %[loops], #0x1\n"
+ ".inst 0x6e84a42e // ummla v14.4s, v1.16b, v4.16b\n"
+ ".inst 0x6e84a454 // ummla v20.4s, v2.16b, v4.16b\n"
+ ".inst 0x6e84a47a // ummla v26.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr]]\n"
+ ".inst 0x6e85a409 // ummla v9.4s, v0.16b, v5.16b\n"
+ ".inst 0x6e85a42f // ummla v15.4s, v1.16b, v5.16b\n"
+ ".inst 0x6e85a455 // ummla v21.4s, v2.16b, v5.16b\n"
+ ".inst 0x6e85a47b // ummla v27.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #0x10]\n"
+ ".inst 0x6e86a40a // ummla v10.4s, v0.16b, v6.16b\n"
+ ".inst 0x6e86a430 // ummla v16.4s, v1.16b, v6.16b\n"
+ ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n"
+ ".inst 0x6e86a47c // ummla v28.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #0x20]\n"
+ ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n"
+ ".inst 0x6e87a431 // ummla v17.4s, v1.16b, v7.16b\n"
+ ".inst 0x6e87a457 // ummla v23.4s, v2.16b, v7.16b\n"
+ ".inst 0x6e87a47d // ummla v29.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #0x30]\n"
+ ".inst 0x6e84a40c // ummla v12.4s, v0.16b, v4.16b\n"
+ ".inst 0x6e84a432 // ummla v18.4s, v1.16b, v4.16b\n"
+ ".inst 0x6e84a458 // ummla v24.4s, v2.16b, v4.16b\n"
+ ".inst 0x6e84a47e // ummla v30.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr], #0x40]\n"
+ ".inst 0x6e85a40d // ummla v13.4s, v0.16b, v5.16b\n"
+ "ldr q0, [%[a_ptr]]\n"
+ ".inst 0x6e85a433 // ummla v19.4s, v1.16b, v5.16b\n"
+ "ldr q1, [%[a_ptr], #0x10]\n"
+ ".inst 0x6e85a459 // ummla v25.4s, v2.16b, v5.16b\n"
+ "ldr q2, [%[a_ptr], #0x20]\n"
+ ".inst 0x6e85a47f // ummla v31.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #0x50]\n"
+ ".inst 0x6e86a408 // ummla v8.4s, v0.16b, v6.16b\n"
+ "ldr q3, [%[a_ptr], #0x30]\n"
+ ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n"
+ "add %[a_ptr], %[a_ptr], #0x80\n"
+ ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n"
+ "add %[b_ptr], %[b_ptr], #0xc0\n"
+ ".inst 0x6e86a47a // ummla v26.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #-0x60]\n"
+ ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n"
+ ".inst 0x6e87a42f // ummla v15.4s, v1.16b, v7.16b\n"
+ ".inst 0x6e87a455 // ummla v21.4s, v2.16b, v7.16b\n"
+ ".inst 0x6e87a47b // ummla v27.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #-0x50]\n"
+ ".inst 0x6e84a40a // ummla v10.4s, v0.16b, v4.16b\n"
+ ".inst 0x6e84a430 // ummla v16.4s, v1.16b, v4.16b\n"
+ ".inst 0x6e84a456 // ummla v22.4s, v2.16b, v4.16b\n"
+ ".inst 0x6e84a47c // ummla v28.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr], #-0x40]\n"
+ ".inst 0x6e85a40b // ummla v11.4s, v0.16b, v5.16b\n"
+ ".inst 0x6e85a431 // ummla v17.4s, v1.16b, v5.16b\n"
+ ".inst 0x6e85a457 // ummla v23.4s, v2.16b, v5.16b\n"
+ ".inst 0x6e85a47d // ummla v29.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #-0x30]\n"
+ ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n"
+ ".inst 0x6e86a432 // ummla v18.4s, v1.16b, v6.16b\n"
+ ".inst 0x6e86a458 // ummla v24.4s, v2.16b, v6.16b\n"
+ ".inst 0x6e86a47e // ummla v30.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #-0x20]\n"
+ ".inst 0x6e87a40d // ummla v13.4s, v0.16b, v7.16b\n"
+ "ldr q0, [%[a_ptr], #-0x40]\n"
+ ".inst 0x6e87a433 // ummla v19.4s, v1.16b, v7.16b\n"
+ "ldr q1, [%[a_ptr], #-0x30]\n"
+ ".inst 0x6e87a459 // ummla v25.4s, v2.16b, v7.16b\n"
+ "ldr q2, [%[a_ptr], #-0x20]\n"
+ ".inst 0x6e87a47f // ummla v31.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #-0x10]\n"
+ "ldr q3, [%[a_ptr], #-0x10]\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[tails], 3f\n"
+ ".inst 0x6e84a408 // ummla v8.4s, v0.16b, v4.16b\n"
+ ".inst 0x6e84a42e // ummla v14.4s, v1.16b, v4.16b\n"
+ ".inst 0x6e84a454 // ummla v20.4s, v2.16b, v4.16b\n"
+ ".inst 0x6e84a47a // ummla v26.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr]]\n"
+ ".inst 0x6e85a409 // ummla v9.4s, v0.16b, v5.16b\n"
+ ".inst 0x6e85a42f // ummla v15.4s, v1.16b, v5.16b\n"
+ ".inst 0x6e85a455 // ummla v21.4s, v2.16b, v5.16b\n"
+ ".inst 0x6e85a47b // ummla v27.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #0x10]\n"
+ ".inst 0x6e86a40a // ummla v10.4s, v0.16b, v6.16b\n"
+ ".inst 0x6e86a430 // ummla v16.4s, v1.16b, v6.16b\n"
+ ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n"
+ ".inst 0x6e86a47c // ummla v28.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #0x20]\n"
+ ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n"
+ ".inst 0x6e87a431 // ummla v17.4s, v1.16b, v7.16b\n"
+ ".inst 0x6e87a457 // ummla v23.4s, v2.16b, v7.16b\n"
+ ".inst 0x6e87a47d // ummla v29.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #0x30]\n"
+ ".inst 0x6e84a40c // ummla v12.4s, v0.16b, v4.16b\n"
+ ".inst 0x6e84a432 // ummla v18.4s, v1.16b, v4.16b\n"
+ ".inst 0x6e84a458 // ummla v24.4s, v2.16b, v4.16b\n"
+ ".inst 0x6e84a47e // ummla v30.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr], #0x40]\n"
+ ".inst 0x6e85a40d // ummla v13.4s, v0.16b, v5.16b\n"
+ "ldr q0, [%[a_ptr]]\n"
+ ".inst 0x6e85a433 // ummla v19.4s, v1.16b, v5.16b\n"
+ "ldr q1, [%[a_ptr], #0x10]\n"
+ ".inst 0x6e85a459 // ummla v25.4s, v2.16b, v5.16b\n"
+ "ldr q2, [%[a_ptr], #0x20]\n"
+ ".inst 0x6e85a47f // ummla v31.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #0x50]\n"
+ ".inst 0x6e86a408 // ummla v8.4s, v0.16b, v6.16b\n"
+ "ldr q3, [%[a_ptr], #0x30]\n"
+ ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n"
+ "add %[a_ptr], %[a_ptr], #0x80\n"
+ ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n"
+ "add %[b_ptr], %[b_ptr], #0xe0\n"
+ ".inst 0x6e86a47a // ummla v26.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #-0x80]\n"
+ ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n"
+ ".inst 0x6e87a42f // ummla v15.4s, v1.16b, v7.16b\n"
+ ".inst 0x6e87a455 // ummla v21.4s, v2.16b, v7.16b\n"
+ ".inst 0x6e87a47b // ummla v27.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #-0x70]\n"
+ ".inst 0x6e84a40a // ummla v10.4s, v0.16b, v4.16b\n"
+ ".inst 0x6e84a430 // ummla v16.4s, v1.16b, v4.16b\n"
+ ".inst 0x6e84a456 // ummla v22.4s, v2.16b, v4.16b\n"
+ ".inst 0x6e84a47c // ummla v28.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr], #-0x60]\n"
+ ".inst 0x6e85a40b // ummla v11.4s, v0.16b, v5.16b\n"
+ ".inst 0x6e85a431 // ummla v17.4s, v1.16b, v5.16b\n"
+ ".inst 0x6e85a457 // ummla v23.4s, v2.16b, v5.16b\n"
+ ".inst 0x6e85a47d // ummla v29.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #-0x50]\n"
+ ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n"
+ ".inst 0x6e86a432 // ummla v18.4s, v1.16b, v6.16b\n"
+ ".inst 0x6e86a458 // ummla v24.4s, v2.16b, v6.16b\n"
+ ".inst 0x6e86a47e // ummla v30.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #-0x40]\n"
+ ".inst 0x6e87a40d // ummla v13.4s, v0.16b, v7.16b\n"
+ "ldr q0, [%[a_ptr], #-0x40]\n"
+ ".inst 0x6e87a433 // ummla v19.4s, v1.16b, v7.16b\n"
+ "ldr q1, [%[a_ptr], #-0x30]\n"
+ ".inst 0x6e87a459 // ummla v25.4s, v2.16b, v7.16b\n"
+ "ldr q2, [%[a_ptr], #-0x20]\n"
+ ".inst 0x6e87a47f // ummla v31.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #-0x30]\n"
+ ".inst 0x6e84a408 // ummla v8.4s, v0.16b, v4.16b\n"
+ "ldr q3, [%[a_ptr], #-0x10]\n"
+ ".inst 0x6e84a42e // ummla v14.4s, v1.16b, v4.16b\n"
+ ".inst 0x6e84a454 // ummla v20.4s, v2.16b, v4.16b\n"
+ ".inst 0x6e85a409 // ummla v9.4s, v0.16b, v5.16b\n"
+ ".inst 0x6e84a47a // ummla v26.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr], #-0x20]\n"
+ ".inst 0x6e85a42f // ummla v15.4s, v1.16b, v5.16b\n"
+ ".inst 0x6e85a455 // ummla v21.4s, v2.16b, v5.16b\n"
+ ".inst 0x6e85a47b // ummla v27.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #-0x10]\n"
+ ".inst 0x6e86a40a // ummla v10.4s, v0.16b, v6.16b\n"
+ ".inst 0x6e86a430 // ummla v16.4s, v1.16b, v6.16b\n"
+ ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n"
+ ".inst 0x6e86a47c // ummla v28.4s, v3.16b, v6.16b\n"
+ "uzp1 v6.2d, v14.2d, v15.2d\n"
+ ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n"
+ ".inst 0x6e87a431 // ummla v17.4s, v1.16b, v7.16b\n"
+ ".inst 0x6e87a457 // ummla v23.4s, v2.16b, v7.16b\n"
+ ".inst 0x6e87a47d // ummla v29.4s, v3.16b, v7.16b\n"
+ ".inst 0x6e84a40c // ummla v12.4s, v0.16b, v4.16b\n"
+ "uzp1 v7.2d, v16.2d, v17.2d\n"
+ ".inst 0x6e84a432 // ummla v18.4s, v1.16b, v4.16b\n"
+ ".inst 0x6e84a458 // ummla v24.4s, v2.16b, v4.16b\n"
+ ".inst 0x6e84a47e // ummla v30.4s, v3.16b, v4.16b\n"
+ "uzp2 v4.2d, v10.2d, v11.2d\n"
+ ".inst 0x6e85a40d // ummla v13.4s, v0.16b, v5.16b\n"
+ "uzp1 v0.2d, v8.2d, v9.2d\n"
+ ".inst 0x6e85a433 // ummla v19.4s, v1.16b, v5.16b\n"
+ "uzp1 v1.2d, v10.2d, v11.2d\n"
+ ".inst 0x6e85a459 // ummla v25.4s, v2.16b, v5.16b\n"
+ "str q0, [%[c_ptr]]\n"
+ "uzp1 v2.2d, v12.2d, v13.2d\n"
+ "uzp1 v0.2d, v18.2d, v19.2d\n"
+ ".inst 0x6e85a47f // ummla v31.4s, v3.16b, v5.16b\n"
+ "str q1, [%[c_ptr], #0x10]\n"
+ "uzp2 v3.2d, v8.2d, v9.2d\n"
+ "uzp2 v5.2d, v12.2d, v13.2d\n"
+ "uzp2 v1.2d, v14.2d, v15.2d\n"
+ "str q2, [%[c_ptr], #0x20]\n"
+ "b 4f\n"
+ "3:\n"
+ ".inst 0x6e84a408 // ummla v8.4s, v0.16b, v4.16b\n"
+ "add %[a_ptr], %[a_ptr], #0x40\n"
+ ".inst 0x6e84a42e // ummla v14.4s, v1.16b, v4.16b\n"
+ "add %[b_ptr], %[b_ptr], #0x80\n"
+ ".inst 0x6e84a454 // ummla v20.4s, v2.16b, v4.16b\n"
+ ".inst 0x6e84a47a // ummla v26.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr], #-0x80]\n"
+ ".inst 0x6e85a409 // ummla v9.4s, v0.16b, v5.16b\n"
+ ".inst 0x6e85a42f // ummla v15.4s, v1.16b, v5.16b\n"
+ ".inst 0x6e85a455 // ummla v21.4s, v2.16b, v5.16b\n"
+ ".inst 0x6e85a47b // ummla v27.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #-0x70]\n"
+ ".inst 0x6e86a40a // ummla v10.4s, v0.16b, v6.16b\n"
+ ".inst 0x6e86a430 // ummla v16.4s, v1.16b, v6.16b\n"
+ ".inst 0x6e86a456 // ummla v22.4s, v2.16b, v6.16b\n"
+ ".inst 0x6e86a47c // ummla v28.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #-0x60]\n"
+ ".inst 0x6e87a40b // ummla v11.4s, v0.16b, v7.16b\n"
+ ".inst 0x6e87a431 // ummla v17.4s, v1.16b, v7.16b\n"
+ ".inst 0x6e87a457 // ummla v23.4s, v2.16b, v7.16b\n"
+ ".inst 0x6e87a47d // ummla v29.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #-0x50]\n"
+ ".inst 0x6e84a40c // ummla v12.4s, v0.16b, v4.16b\n"
+ ".inst 0x6e84a432 // ummla v18.4s, v1.16b, v4.16b\n"
+ ".inst 0x6e84a458 // ummla v24.4s, v2.16b, v4.16b\n"
+ ".inst 0x6e84a47e // ummla v30.4s, v3.16b, v4.16b\n"
+ "ldr q4, [%[b_ptr], #-0x40]\n"
+ ".inst 0x6e85a40d // ummla v13.4s, v0.16b, v5.16b\n"
+ "ldr q0, [%[a_ptr], #-0x40]\n"
+ ".inst 0x6e85a433 // ummla v19.4s, v1.16b, v5.16b\n"
+ "ldr q1, [%[a_ptr], #-0x30]\n"
+ ".inst 0x6e85a459 // ummla v25.4s, v2.16b, v5.16b\n"
+ "ldr q2, [%[a_ptr], #-0x20]\n"
+ ".inst 0x6e85a47f // ummla v31.4s, v3.16b, v5.16b\n"
+ "ldr q5, [%[b_ptr], #-0x30]\n"
+ ".inst 0x6e86a408 // ummla v8.4s, v0.16b, v6.16b\n"
+ "ldr q3, [%[a_ptr], #-0x10]\n"
+ ".inst 0x6e86a42e // ummla v14.4s, v1.16b, v6.16b\n"
+ ".inst 0x6e86a454 // ummla v20.4s, v2.16b, v6.16b\n"
+ ".inst 0x6e87a409 // ummla v9.4s, v0.16b, v7.16b\n"
+ ".inst 0x6e86a47a // ummla v26.4s, v3.16b, v6.16b\n"
+ "ldr q6, [%[b_ptr], #-0x20]\n"
+ ".inst 0x6e87a42f // ummla v15.4s, v1.16b, v7.16b\n"
+ ".inst 0x6e87a455 // ummla v21.4s, v2.16b, v7.16b\n"
+ ".inst 0x6e87a47b // ummla v27.4s, v3.16b, v7.16b\n"
+ "ldr q7, [%[b_ptr], #-0x10]\n"
+ ".inst 0x6e84a40a // ummla v10.4s, v0.16b, v4.16b\n"
+ ".inst 0x6e84a430 // ummla v16.4s, v1.16b, v4.16b\n"
+ ".inst 0x6e84a456 // ummla v22.4s, v2.16b, v4.16b\n"
+ ".inst 0x6e84a47c // ummla v28.4s, v3.16b, v4.16b\n"
+ ".inst 0x6e85a40b // ummla v11.4s, v0.16b, v5.16b\n"
+ ".inst 0x6e85a431 // ummla v17.4s, v1.16b, v5.16b\n"
+ ".inst 0x6e85a457 // ummla v23.4s, v2.16b, v5.16b\n"
+ ".inst 0x6e85a47d // ummla v29.4s, v3.16b, v5.16b\n"
+ "uzp2 v4.2d, v10.2d, v11.2d\n"
+ ".inst 0x6e86a40c // ummla v12.4s, v0.16b, v6.16b\n"
+ ".inst 0x6e86a432 // ummla v18.4s, v1.16b, v6.16b\n"
+ ".inst 0x6e86a458 // ummla v24.4s, v2.16b, v6.16b\n"
+ ".inst 0x6e86a47e // ummla v30.4s, v3.16b, v6.16b\n"
+ "uzp1 v6.2d, v14.2d, v15.2d\n"
+ ".inst 0x6e87a40d // ummla v13.4s, v0.16b, v7.16b\n"
+ "uzp1 v0.2d, v8.2d, v9.2d\n"
+ ".inst 0x6e87a433 // ummla v19.4s, v1.16b, v7.16b\n"
+ "uzp1 v1.2d, v10.2d, v11.2d\n"
+ "uzp2 v5.2d, v12.2d, v13.2d\n"
+ "str q0, [%[c_ptr]]\n"
+ ".inst 0x6e87a459 // ummla v25.4s, v2.16b, v7.16b\n"
+ "uzp1 v2.2d, v12.2d, v13.2d\n"
+ "uzp1 v0.2d, v18.2d, v19.2d\n"
+ "str q1, [%[c_ptr], #0x10]\n"
+ "uzp2 v1.2d, v14.2d, v15.2d\n"
+ ".inst 0x6e87a47f // ummla v31.4s, v3.16b, v7.16b\n"
+ "uzp2 v3.2d, v8.2d, v9.2d\n"
+ "str q2, [%[c_ptr], #0x20]\n"
+ "uzp1 v7.2d, v16.2d, v17.2d\n"
+ "4:\n"
+ "uzp2 v2.2d, v16.2d, v17.2d\n"
+ "str q3, [%[c_ptr], #0x30]\n"
+ "uzp2 v3.2d, v18.2d, v19.2d\n"
+ "str q4, [%[c_ptr], #0x40]\n"
+ "uzp1 v4.2d, v20.2d, v21.2d\n"
+ "str q5, [%[c_ptr], #0x50]\n"
+ "uzp1 v5.2d, v22.2d, v23.2d\n"
+ "str q6, [%[c_ptr], #0x60]\n"
+ "uzp1 v6.2d, v24.2d, v25.2d\n"
+ "str q7, [%[c_ptr], #0x70]\n"
+ "uzp2 v7.2d, v20.2d, v21.2d\n"
+ "str q0, [%[c_ptr], #0x80]\n"
+ "uzp2 v0.2d, v22.2d, v23.2d\n"
+ "str q1, [%[c_ptr], #0x90]\n"
+ "uzp2 v1.2d, v24.2d, v25.2d\n"
+ "str q2, [%[c_ptr], #0xa0]\n"
+ "uzp1 v2.2d, v26.2d, v27.2d\n"
+ "str q3, [%[c_ptr], #0xb0]\n"
+ "uzp1 v3.2d, v28.2d, v29.2d\n"
+ "str q4, [%[c_ptr], #0xc0]\n"
+ "uzp1 v4.2d, v30.2d, v31.2d\n"
+ "str q5, [%[c_ptr], #0xd0]\n"
+ "uzp2 v5.2d, v26.2d, v27.2d\n"
+ "str q6, [%[c_ptr], #0xe0]\n"
+ "uzp2 v6.2d, v28.2d, v29.2d\n"
+ "str q7, [%[c_ptr], #0xf0]\n"
+ "uzp2 v7.2d, v30.2d, v31.2d\n"
+ "str q0, [%[c_ptr], #0x100]\n"
+ "str q1, [%[c_ptr], #0x110]\n"
+ "str q2, [%[c_ptr], #0x120]\n"
+ "str q3, [%[c_ptr], #0x130]\n"
+ "str q4, [%[c_ptr], #0x140]\n"
+ "str q5, [%[c_ptr], #0x150]\n"
+ "str q6, [%[c_ptr], #0x160]\n"
+ "str q7, [%[c_ptr], #0x170]\n"
+ "add %[c_ptr], %[c_ptr], #0x180\n"
+ : [a_ptr] "+r" (a_ptr), [b_ptr] "+r" (b_ptr), [c_ptr] "+r" (c_ptr),
+ [loops] "+r" (loops), [tails] "+r" (tails)
+ :
+ : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "cc", "memory"
+ );
+ }
+ }
+}
+
+} // namespace arm_gemm
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8.hpp
new file mode 100644
index 0000000000..cd50d0ded3
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8.hpp
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __ARM_FEATURE_SVE
+
+#include <cstdint>
+#include "../std_transforms_sve.hpp"
+
+namespace arm_gemm {
+
+// Actual kernel implementations
+void sve_interleaved_s8s32_mmla_3VLx8(const int8_t *, const int8_t *, int32_t *, int, int, int);
+
+class interleaved_s8s32_mmla_3VLx8 {
+public:
+ typedef int8_t operand_type;
+ typedef int32_t result_type;
+
+ typedef void (*kern_type)(const int8_t *, const int8_t *, int32_t *, int, int, int);
+
+ /* Kernel blocking parameters */
+ static unsigned int out_width()
+ {
+ return get_vector_length<int32_t>() * 3;
+ }
+
+ static unsigned int out_height()
+ {
+ return 8;
+ }
+
+ static unsigned int k_unroll()
+ {
+ return 8;
+ }
+
+ // Use the standard fixed size transforms.
+ StdTransformsSVE<operand_type, result_type, 8, 6, 8, 2> transforms = {};
+
+ kern_type kernel=sve_interleaved_s8s32_mmla_3VLx8;
+
+ interleaved_s8s32_mmla_3VLx8(const CPUInfo *ci)
+ {
+ UNUSED(ci);
+ }
+};
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8/generic.cpp
new file mode 100644
index 0000000000..d636c9d2a4
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_s8s32_mmla_3VLx8/generic.cpp
@@ -0,0 +1,395 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include <cstdint>
+#include "../../asmlib.hpp"
+
+namespace arm_gemm {
+
+void sve_interleaved_s8s32_mmla_3VLx8(const int8_t *Apanel, const int8_t *Bpanel, int32_t *Cpanel, int ablocks, int bblocks, int K) {
+ const int8_t *a_ptr = Apanel;
+ int32_t *c_ptr = Cpanel;
+
+ K /= 8;
+ const long loops_count = (K / 2) - 1;
+ const long tails_count = K % 2;
+
+ for (int yb=0; yb<ablocks; yb++) {
+ const int8_t *a_ptr0 = a_ptr;
+ const int8_t *b_ptr = Bpanel;
+
+ for (int xb=0; xb<bblocks; xb++) {
+ a_ptr = a_ptr0;
+ long loops = loops_count;
+ long tails = tails_count;
+
+ __asm __volatile (
+ "mov z8.s, #0\n"
+ "ptrue p0.b\n"
+ "mov z9.s, #0\n"
+ "mov z10.s, #0\n"
+ "mov z11.s, #0\n"
+ "ld1rqb z0.b, p0/z, [%[a_ptr]]\n"
+ "mov z12.s, #0\n"
+ "ld1b z4.b, p0/z, [%[b_ptr]]\n"
+ "mov z13.s, #0\n"
+ "ld1rqb z1.b, p0/z, [%[a_ptr], #0x10]\n"
+ "mov z14.s, #0\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #1, MUL VL]\n"
+ "mov z15.s, #0\n"
+ "ld1rqb z2.b, p0/z, [%[a_ptr], #0x20]\n"
+ "mov z16.s, #0\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #2, MUL VL]\n"
+ "mov z17.s, #0\n"
+ "ld1rqb z3.b, p0/z, [%[a_ptr], #0x30]\n"
+ "mov z18.s, #0\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #3, MUL VL]\n"
+ "mov z19.s, #0\n"
+ "add %[a_ptr], %[a_ptr], #0x40\n"
+ "mov z20.s, #0\n"
+ "addvl %[b_ptr], %[b_ptr], #4\n"
+ "mov z21.s, #0\n"
+ "mov z22.s, #0\n"
+ "mov z23.s, #0\n"
+ "mov z24.s, #0\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ ".inst 0x45049808 // smmla z8.s, z0.b, z4.b\n"
+ "subs %[loops], %[loops], #0x1\n"
+ ".inst 0x4504982e // smmla z14.s, z1.b, z4.b\n"
+ ".inst 0x45049854 // smmla z20.s, z2.b, z4.b\n"
+ ".inst 0x4504987a // smmla z26.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr]]\n"
+ ".inst 0x45059809 // smmla z9.s, z0.b, z5.b\n"
+ ".inst 0x4505982f // smmla z15.s, z1.b, z5.b\n"
+ ".inst 0x45059855 // smmla z21.s, z2.b, z5.b\n"
+ ".inst 0x4505987b // smmla z27.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #1, MUL VL]\n"
+ ".inst 0x4506980a // smmla z10.s, z0.b, z6.b\n"
+ ".inst 0x45069830 // smmla z16.s, z1.b, z6.b\n"
+ ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n"
+ ".inst 0x4506987c // smmla z28.s, z3.b, z6.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #2, MUL VL]\n"
+ ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n"
+ ".inst 0x45079831 // smmla z17.s, z1.b, z7.b\n"
+ ".inst 0x45079857 // smmla z23.s, z2.b, z7.b\n"
+ ".inst 0x4507987d // smmla z29.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #3, MUL VL]\n"
+ ".inst 0x4504980c // smmla z12.s, z0.b, z4.b\n"
+ ".inst 0x45049832 // smmla z18.s, z1.b, z4.b\n"
+ ".inst 0x45049858 // smmla z24.s, z2.b, z4.b\n"
+ ".inst 0x4504987e // smmla z30.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr], #4, MUL VL]\n"
+ ".inst 0x4505980d // smmla z13.s, z0.b, z5.b\n"
+ "ld1rqb z0.b, p0/z, [%[a_ptr]]\n"
+ ".inst 0x45059833 // smmla z19.s, z1.b, z5.b\n"
+ "ld1rqb z1.b, p0/z, [%[a_ptr], #0x10]\n"
+ ".inst 0x45059859 // smmla z25.s, z2.b, z5.b\n"
+ "ld1rqb z2.b, p0/z, [%[a_ptr], #0x20]\n"
+ ".inst 0x4505987f // smmla z31.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #5, MUL VL]\n"
+ ".inst 0x45069808 // smmla z8.s, z0.b, z6.b\n"
+ "ld1rqb z3.b, p0/z, [%[a_ptr], #0x30]\n"
+ ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n"
+ "add %[a_ptr], %[a_ptr], #0x80\n"
+ ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n"
+ "addvl %[b_ptr], %[b_ptr], #12\n"
+ ".inst 0x4506987a // smmla z26.s, z3.b, z6.b\n"
+ ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n"
+ ".inst 0x4507982f // smmla z15.s, z1.b, z7.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #-6, MUL VL]\n"
+ ".inst 0x45079855 // smmla z21.s, z2.b, z7.b\n"
+ ".inst 0x4507987b // smmla z27.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #-5, MUL VL]\n"
+ ".inst 0x4504980a // smmla z10.s, z0.b, z4.b\n"
+ ".inst 0x45049830 // smmla z16.s, z1.b, z4.b\n"
+ ".inst 0x45049856 // smmla z22.s, z2.b, z4.b\n"
+ ".inst 0x4504987c // smmla z28.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr], #-4, MUL VL]\n"
+ ".inst 0x4505980b // smmla z11.s, z0.b, z5.b\n"
+ ".inst 0x45059831 // smmla z17.s, z1.b, z5.b\n"
+ ".inst 0x45059857 // smmla z23.s, z2.b, z5.b\n"
+ ".inst 0x4505987d // smmla z29.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #-3, MUL VL]\n"
+ ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n"
+ ".inst 0x45069832 // smmla z18.s, z1.b, z6.b\n"
+ ".inst 0x45069858 // smmla z24.s, z2.b, z6.b\n"
+ ".inst 0x4506987e // smmla z30.s, z3.b, z6.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #-2, MUL VL]\n"
+ ".inst 0x4507980d // smmla z13.s, z0.b, z7.b\n"
+ "ld1rqb z0.b, p0/z, [%[a_ptr], #-0x40]\n"
+ ".inst 0x45079833 // smmla z19.s, z1.b, z7.b\n"
+ "ld1rqb z1.b, p0/z, [%[a_ptr], #-0x30]\n"
+ ".inst 0x45079859 // smmla z25.s, z2.b, z7.b\n"
+ "ld1rqb z2.b, p0/z, [%[a_ptr], #-0x20]\n"
+ ".inst 0x4507987f // smmla z31.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #-1, MUL VL]\n"
+ "ld1rqb z3.b, p0/z, [%[a_ptr], #-0x10]\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[tails], 3f\n"
+ ".inst 0x45049808 // smmla z8.s, z0.b, z4.b\n"
+ ".inst 0x4504982e // smmla z14.s, z1.b, z4.b\n"
+ ".inst 0x45049854 // smmla z20.s, z2.b, z4.b\n"
+ ".inst 0x4504987a // smmla z26.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr]]\n"
+ ".inst 0x45059809 // smmla z9.s, z0.b, z5.b\n"
+ ".inst 0x4505982f // smmla z15.s, z1.b, z5.b\n"
+ ".inst 0x45059855 // smmla z21.s, z2.b, z5.b\n"
+ ".inst 0x4505987b // smmla z27.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #1, MUL VL]\n"
+ ".inst 0x4506980a // smmla z10.s, z0.b, z6.b\n"
+ ".inst 0x45069830 // smmla z16.s, z1.b, z6.b\n"
+ ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n"
+ ".inst 0x4506987c // smmla z28.s, z3.b, z6.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #2, MUL VL]\n"
+ ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n"
+ ".inst 0x45079831 // smmla z17.s, z1.b, z7.b\n"
+ ".inst 0x45079857 // smmla z23.s, z2.b, z7.b\n"
+ ".inst 0x4507987d // smmla z29.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #3, MUL VL]\n"
+ ".inst 0x4504980c // smmla z12.s, z0.b, z4.b\n"
+ ".inst 0x45049832 // smmla z18.s, z1.b, z4.b\n"
+ ".inst 0x45049858 // smmla z24.s, z2.b, z4.b\n"
+ ".inst 0x4504987e // smmla z30.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr], #4, MUL VL]\n"
+ ".inst 0x4505980d // smmla z13.s, z0.b, z5.b\n"
+ "ld1rqb z0.b, p0/z, [%[a_ptr]]\n"
+ ".inst 0x45059833 // smmla z19.s, z1.b, z5.b\n"
+ "ld1rqb z1.b, p0/z, [%[a_ptr], #0x10]\n"
+ ".inst 0x45059859 // smmla z25.s, z2.b, z5.b\n"
+ "ld1rqb z2.b, p0/z, [%[a_ptr], #0x20]\n"
+ ".inst 0x4505987f // smmla z31.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #5, MUL VL]\n"
+ ".inst 0x45069808 // smmla z8.s, z0.b, z6.b\n"
+ "ld1rqb z3.b, p0/z, [%[a_ptr], #0x30]\n"
+ ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n"
+ "add %[a_ptr], %[a_ptr], #0x80\n"
+ ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n"
+ "addvl %[b_ptr], %[b_ptr], #14\n"
+ ".inst 0x4506987a // smmla z26.s, z3.b, z6.b\n"
+ ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n"
+ ".inst 0x4507982f // smmla z15.s, z1.b, z7.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #-8, MUL VL]\n"
+ ".inst 0x45079855 // smmla z21.s, z2.b, z7.b\n"
+ ".inst 0x4507987b // smmla z27.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #-7, MUL VL]\n"
+ ".inst 0x4504980a // smmla z10.s, z0.b, z4.b\n"
+ ".inst 0x45049830 // smmla z16.s, z1.b, z4.b\n"
+ ".inst 0x45049856 // smmla z22.s, z2.b, z4.b\n"
+ ".inst 0x4504987c // smmla z28.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr], #-6, MUL VL]\n"
+ ".inst 0x4505980b // smmla z11.s, z0.b, z5.b\n"
+ ".inst 0x45059831 // smmla z17.s, z1.b, z5.b\n"
+ ".inst 0x45059857 // smmla z23.s, z2.b, z5.b\n"
+ ".inst 0x4505987d // smmla z29.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #-5, MUL VL]\n"
+ ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n"
+ ".inst 0x45069832 // smmla z18.s, z1.b, z6.b\n"
+ ".inst 0x45069858 // smmla z24.s, z2.b, z6.b\n"
+ ".inst 0x4506987e // smmla z30.s, z3.b, z6.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #-4, MUL VL]\n"
+ ".inst 0x4507980d // smmla z13.s, z0.b, z7.b\n"
+ "ld1rqb z0.b, p0/z, [%[a_ptr], #-0x40]\n"
+ ".inst 0x45079833 // smmla z19.s, z1.b, z7.b\n"
+ "ld1rqb z1.b, p0/z, [%[a_ptr], #-0x30]\n"
+ ".inst 0x45079859 // smmla z25.s, z2.b, z7.b\n"
+ "ld1rqb z2.b, p0/z, [%[a_ptr], #-0x20]\n"
+ ".inst 0x4507987f // smmla z31.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #-3, MUL VL]\n"
+ ".inst 0x45049808 // smmla z8.s, z0.b, z4.b\n"
+ "ld1rqb z3.b, p0/z, [%[a_ptr], #-0x10]\n"
+ ".inst 0x4504982e // smmla z14.s, z1.b, z4.b\n"
+ ".inst 0x45049854 // smmla z20.s, z2.b, z4.b\n"
+ ".inst 0x45059809 // smmla z9.s, z0.b, z5.b\n"
+ ".inst 0x4504987a // smmla z26.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr], #-2, MUL VL]\n"
+ ".inst 0x4505982f // smmla z15.s, z1.b, z5.b\n"
+ ".inst 0x45059855 // smmla z21.s, z2.b, z5.b\n"
+ ".inst 0x4505987b // smmla z27.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #-1, MUL VL]\n"
+ ".inst 0x4506980a // smmla z10.s, z0.b, z6.b\n"
+ ".inst 0x45069830 // smmla z16.s, z1.b, z6.b\n"
+ ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n"
+ ".inst 0x4506987c // smmla z28.s, z3.b, z6.b\n"
+ "uzp1 z6.d, z14.d, z15.d\n"
+ ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n"
+ ".inst 0x45079831 // smmla z17.s, z1.b, z7.b\n"
+ ".inst 0x45079857 // smmla z23.s, z2.b, z7.b\n"
+ ".inst 0x4507987d // smmla z29.s, z3.b, z7.b\n"
+ ".inst 0x4504980c // smmla z12.s, z0.b, z4.b\n"
+ "uzp1 z7.d, z16.d, z17.d\n"
+ ".inst 0x45049832 // smmla z18.s, z1.b, z4.b\n"
+ ".inst 0x45049858 // smmla z24.s, z2.b, z4.b\n"
+ ".inst 0x4504987e // smmla z30.s, z3.b, z4.b\n"
+ "uzp2 z4.d, z10.d, z11.d\n"
+ ".inst 0x4505980d // smmla z13.s, z0.b, z5.b\n"
+ "uzp1 z0.d, z8.d, z9.d\n"
+ ".inst 0x45059833 // smmla z19.s, z1.b, z5.b\n"
+ "uzp1 z1.d, z10.d, z11.d\n"
+ ".inst 0x45059859 // smmla z25.s, z2.b, z5.b\n"
+ "st1w z0.s, p0, [%[c_ptr]]\n"
+ "uzp1 z2.d, z12.d, z13.d\n"
+ "uzp1 z0.d, z18.d, z19.d\n"
+ ".inst 0x4505987f // smmla z31.s, z3.b, z5.b\n"
+ "st1w z1.s, p0, [%[c_ptr], #1, MUL VL]\n"
+ "uzp2 z3.d, z8.d, z9.d\n"
+ "uzp2 z5.d, z12.d, z13.d\n"
+ "uzp2 z1.d, z14.d, z15.d\n"
+ "st1w z2.s, p0, [%[c_ptr], #2, MUL VL]\n"
+ "b 4f\n"
+ "3:\n"
+ ".inst 0x45049808 // smmla z8.s, z0.b, z4.b\n"
+ "add %[a_ptr], %[a_ptr], #0x40\n"
+ ".inst 0x4504982e // smmla z14.s, z1.b, z4.b\n"
+ "addvl %[b_ptr], %[b_ptr], #8\n"
+ ".inst 0x45049854 // smmla z20.s, z2.b, z4.b\n"
+ ".inst 0x4504987a // smmla z26.s, z3.b, z4.b\n"
+ ".inst 0x45059809 // smmla z9.s, z0.b, z5.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr], #-8, MUL VL]\n"
+ ".inst 0x4505982f // smmla z15.s, z1.b, z5.b\n"
+ ".inst 0x45059855 // smmla z21.s, z2.b, z5.b\n"
+ ".inst 0x4505987b // smmla z27.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #-7, MUL VL]\n"
+ ".inst 0x4506980a // smmla z10.s, z0.b, z6.b\n"
+ ".inst 0x45069830 // smmla z16.s, z1.b, z6.b\n"
+ ".inst 0x45069856 // smmla z22.s, z2.b, z6.b\n"
+ ".inst 0x4506987c // smmla z28.s, z3.b, z6.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #-6, MUL VL]\n"
+ ".inst 0x4507980b // smmla z11.s, z0.b, z7.b\n"
+ ".inst 0x45079831 // smmla z17.s, z1.b, z7.b\n"
+ ".inst 0x45079857 // smmla z23.s, z2.b, z7.b\n"
+ ".inst 0x4507987d // smmla z29.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #-5, MUL VL]\n"
+ ".inst 0x4504980c // smmla z12.s, z0.b, z4.b\n"
+ ".inst 0x45049832 // smmla z18.s, z1.b, z4.b\n"
+ ".inst 0x45049858 // smmla z24.s, z2.b, z4.b\n"
+ ".inst 0x4504987e // smmla z30.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr], #-4, MUL VL]\n"
+ ".inst 0x4505980d // smmla z13.s, z0.b, z5.b\n"
+ "ld1rqb z0.b, p0/z, [%[a_ptr], #-0x40]\n"
+ ".inst 0x45059833 // smmla z19.s, z1.b, z5.b\n"
+ "ld1rqb z1.b, p0/z, [%[a_ptr], #-0x30]\n"
+ ".inst 0x45059859 // smmla z25.s, z2.b, z5.b\n"
+ "ld1rqb z2.b, p0/z, [%[a_ptr], #-0x20]\n"
+ ".inst 0x4505987f // smmla z31.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #-3, MUL VL]\n"
+ ".inst 0x45069808 // smmla z8.s, z0.b, z6.b\n"
+ "ld1rqb z3.b, p0/z, [%[a_ptr], #-0x10]\n"
+ ".inst 0x4506982e // smmla z14.s, z1.b, z6.b\n"
+ ".inst 0x45069854 // smmla z20.s, z2.b, z6.b\n"
+ ".inst 0x45079809 // smmla z9.s, z0.b, z7.b\n"
+ ".inst 0x4506987a // smmla z26.s, z3.b, z6.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #-2, MUL VL]\n"
+ ".inst 0x4507982f // smmla z15.s, z1.b, z7.b\n"
+ ".inst 0x45079855 // smmla z21.s, z2.b, z7.b\n"
+ ".inst 0x4507987b // smmla z27.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #-1, MUL VL]\n"
+ ".inst 0x4504980a // smmla z10.s, z0.b, z4.b\n"
+ ".inst 0x45049830 // smmla z16.s, z1.b, z4.b\n"
+ ".inst 0x45049856 // smmla z22.s, z2.b, z4.b\n"
+ ".inst 0x4504987c // smmla z28.s, z3.b, z4.b\n"
+ ".inst 0x4505980b // smmla z11.s, z0.b, z5.b\n"
+ ".inst 0x45059831 // smmla z17.s, z1.b, z5.b\n"
+ ".inst 0x45059857 // smmla z23.s, z2.b, z5.b\n"
+ ".inst 0x4505987d // smmla z29.s, z3.b, z5.b\n"
+ "uzp2 z4.d, z10.d, z11.d\n"
+ ".inst 0x4506980c // smmla z12.s, z0.b, z6.b\n"
+ ".inst 0x45069832 // smmla z18.s, z1.b, z6.b\n"
+ ".inst 0x45069858 // smmla z24.s, z2.b, z6.b\n"
+ ".inst 0x4506987e // smmla z30.s, z3.b, z6.b\n"
+ "uzp1 z6.d, z14.d, z15.d\n"
+ ".inst 0x4507980d // smmla z13.s, z0.b, z7.b\n"
+ "uzp1 z0.d, z8.d, z9.d\n"
+ ".inst 0x45079833 // smmla z19.s, z1.b, z7.b\n"
+ "uzp1 z1.d, z10.d, z11.d\n"
+ "uzp2 z5.d, z12.d, z13.d\n"
+ "st1w z0.s, p0, [%[c_ptr]]\n"
+ ".inst 0x45079859 // smmla z25.s, z2.b, z7.b\n"
+ "uzp1 z2.d, z12.d, z13.d\n"
+ "uzp1 z0.d, z18.d, z19.d\n"
+ "st1w z1.s, p0, [%[c_ptr], #1, MUL VL]\n"
+ "uzp2 z1.d, z14.d, z15.d\n"
+ ".inst 0x4507987f // smmla z31.s, z3.b, z7.b\n"
+ "uzp2 z3.d, z8.d, z9.d\n"
+ "st1w z2.s, p0, [%[c_ptr], #2, MUL VL]\n"
+ "uzp1 z7.d, z16.d, z17.d\n"
+ "4:\n"
+ "uzp2 z2.d, z16.d, z17.d\n"
+ "st1w z3.s, p0, [%[c_ptr], #3, MUL VL]\n"
+ "uzp2 z3.d, z18.d, z19.d\n"
+ "st1w z4.s, p0, [%[c_ptr], #4, MUL VL]\n"
+ "uzp1 z4.d, z20.d, z21.d\n"
+ "st1w z5.s, p0, [%[c_ptr], #5, MUL VL]\n"
+ "uzp1 z5.d, z22.d, z23.d\n"
+ "st1w z6.s, p0, [%[c_ptr], #6, MUL VL]\n"
+ "uzp1 z6.d, z24.d, z25.d\n"
+ "st1w z7.s, p0, [%[c_ptr], #7, MUL VL]\n"
+ "addvl %[c_ptr], %[c_ptr], #16\n"
+ "uzp2 z7.d, z20.d, z21.d\n"
+ "st1w z0.s, p0, [%[c_ptr], #-8, MUL VL]\n"
+ "uzp2 z0.d, z22.d, z23.d\n"
+ "st1w z1.s, p0, [%[c_ptr], #-7, MUL VL]\n"
+ "uzp2 z1.d, z24.d, z25.d\n"
+ "st1w z2.s, p0, [%[c_ptr], #-6, MUL VL]\n"
+ "uzp1 z2.d, z26.d, z27.d\n"
+ "st1w z3.s, p0, [%[c_ptr], #-5, MUL VL]\n"
+ "uzp1 z3.d, z28.d, z29.d\n"
+ "st1w z4.s, p0, [%[c_ptr], #-4, MUL VL]\n"
+ "uzp1 z4.d, z30.d, z31.d\n"
+ "st1w z5.s, p0, [%[c_ptr], #-3, MUL VL]\n"
+ "uzp2 z5.d, z26.d, z27.d\n"
+ "st1w z6.s, p0, [%[c_ptr], #-2, MUL VL]\n"
+ "uzp2 z6.d, z28.d, z29.d\n"
+ "st1w z7.s, p0, [%[c_ptr], #-1, MUL VL]\n"
+ "uzp2 z7.d, z30.d, z31.d\n"
+ "st1w z0.s, p0, [%[c_ptr]]\n"
+ "st1w z1.s, p0, [%[c_ptr], #1, MUL VL]\n"
+ "st1w z2.s, p0, [%[c_ptr], #2, MUL VL]\n"
+ "st1w z3.s, p0, [%[c_ptr], #3, MUL VL]\n"
+ "st1w z4.s, p0, [%[c_ptr], #4, MUL VL]\n"
+ "st1w z5.s, p0, [%[c_ptr], #5, MUL VL]\n"
+ "st1w z6.s, p0, [%[c_ptr], #6, MUL VL]\n"
+ "st1w z7.s, p0, [%[c_ptr], #7, MUL VL]\n"
+ "addvl %[c_ptr], %[c_ptr], #8\n"
+ : [a_ptr] "+r" (a_ptr), [b_ptr] "+r" (b_ptr), [c_ptr] "+r" (c_ptr),
+ [loops] "+r" (loops), [tails] "+r" (tails)
+ :
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ }
+ }
+}
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8.hpp
new file mode 100644
index 0000000000..9b5ca1049e
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8.hpp
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __ARM_FEATURE_SVE
+
+#include <cstdint>
+#include "../std_transforms_sve.hpp"
+
+namespace arm_gemm {
+
+// Actual kernel implementations
+void sve_interleaved_u8u32_mmla_3VLx8(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
+
+class interleaved_u8u32_mmla_3VLx8 {
+public:
+ typedef uint8_t operand_type;
+ typedef uint32_t result_type;
+
+ typedef void (*kern_type)(const uint8_t *, const uint8_t *, uint32_t *, int, int, int);
+
+ /* Kernel blocking parameters */
+ static unsigned int out_width()
+ {
+ return get_vector_length<uint32_t>() * 3;
+ }
+
+ static unsigned int out_height()
+ {
+ return 8;
+ }
+
+ static unsigned int k_unroll()
+ {
+ return 8;
+ }
+
+ // Use the standard fixed size transforms.
+ StdTransformsSVE<operand_type, result_type, 8, 6, 8, 2> transforms = {};
+
+ kern_type kernel=sve_interleaved_u8u32_mmla_3VLx8;
+
+ interleaved_u8u32_mmla_3VLx8(const CPUInfo *ci)
+ {
+ UNUSED(ci);
+ }
+};
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8/generic.cpp b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8/generic.cpp
new file mode 100644
index 0000000000..15cc8fb897
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/kernels/sve_interleaved_u8u32_mmla_3VLx8/generic.cpp
@@ -0,0 +1,395 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifdef __ARM_FEATURE_SVE
+
+#include <cstdint>
+#include "../../asmlib.hpp"
+
+namespace arm_gemm {
+
+void sve_interleaved_u8u32_mmla_3VLx8(const uint8_t *Apanel, const uint8_t *Bpanel, uint32_t *Cpanel, int ablocks, int bblocks, int K) {
+ const uint8_t *a_ptr = Apanel;
+ uint32_t *c_ptr = Cpanel;
+
+ K /= 8;
+ const long loops_count = (K / 2) - 1;
+ const long tails_count = K % 2;
+
+ for (int yb=0; yb<ablocks; yb++) {
+ const uint8_t *a_ptr0 = a_ptr;
+ const uint8_t *b_ptr = Bpanel;
+
+ for (int xb=0; xb<bblocks; xb++) {
+ a_ptr = a_ptr0;
+ long loops = loops_count;
+ long tails = tails_count;
+
+ __asm __volatile (
+ "mov z8.s, #0\n"
+ "ptrue p0.b\n"
+ "mov z9.s, #0\n"
+ "mov z10.s, #0\n"
+ "mov z11.s, #0\n"
+ "ld1rqb z0.b, p0/z, [%[a_ptr]]\n"
+ "mov z12.s, #0\n"
+ "ld1b z4.b, p0/z, [%[b_ptr]]\n"
+ "mov z13.s, #0\n"
+ "ld1rqb z1.b, p0/z, [%[a_ptr], #0x10]\n"
+ "mov z14.s, #0\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #1, MUL VL]\n"
+ "mov z15.s, #0\n"
+ "ld1rqb z2.b, p0/z, [%[a_ptr], #0x20]\n"
+ "mov z16.s, #0\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #2, MUL VL]\n"
+ "mov z17.s, #0\n"
+ "ld1rqb z3.b, p0/z, [%[a_ptr], #0x30]\n"
+ "mov z18.s, #0\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #3, MUL VL]\n"
+ "mov z19.s, #0\n"
+ "add %[a_ptr], %[a_ptr], #0x40\n"
+ "mov z20.s, #0\n"
+ "addvl %[b_ptr], %[b_ptr], #4\n"
+ "mov z21.s, #0\n"
+ "mov z22.s, #0\n"
+ "mov z23.s, #0\n"
+ "mov z24.s, #0\n"
+ "mov z25.s, #0\n"
+ "mov z26.s, #0\n"
+ "mov z27.s, #0\n"
+ "mov z28.s, #0\n"
+ "mov z29.s, #0\n"
+ "mov z30.s, #0\n"
+ "mov z31.s, #0\n"
+ "cbz %[loops], 1f\n"
+ "2:\n"
+ ".inst 0x45c49808 // ummla z8.s, z0.b, z4.b\n"
+ "subs %[loops], %[loops], #0x1\n"
+ ".inst 0x45c4982e // ummla z14.s, z1.b, z4.b\n"
+ ".inst 0x45c49854 // ummla z20.s, z2.b, z4.b\n"
+ ".inst 0x45c4987a // ummla z26.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr]]\n"
+ ".inst 0x45c59809 // ummla z9.s, z0.b, z5.b\n"
+ ".inst 0x45c5982f // ummla z15.s, z1.b, z5.b\n"
+ ".inst 0x45c59855 // ummla z21.s, z2.b, z5.b\n"
+ ".inst 0x45c5987b // ummla z27.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #1, MUL VL]\n"
+ ".inst 0x45c6980a // ummla z10.s, z0.b, z6.b\n"
+ ".inst 0x45c69830 // ummla z16.s, z1.b, z6.b\n"
+ ".inst 0x45c69856 // ummla z22.s, z2.b, z6.b\n"
+ ".inst 0x45c6987c // ummla z28.s, z3.b, z6.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #2, MUL VL]\n"
+ ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n"
+ ".inst 0x45c79831 // ummla z17.s, z1.b, z7.b\n"
+ ".inst 0x45c79857 // ummla z23.s, z2.b, z7.b\n"
+ ".inst 0x45c7987d // ummla z29.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #3, MUL VL]\n"
+ ".inst 0x45c4980c // ummla z12.s, z0.b, z4.b\n"
+ ".inst 0x45c49832 // ummla z18.s, z1.b, z4.b\n"
+ ".inst 0x45c49858 // ummla z24.s, z2.b, z4.b\n"
+ ".inst 0x45c4987e // ummla z30.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr], #4, MUL VL]\n"
+ ".inst 0x45c5980d // ummla z13.s, z0.b, z5.b\n"
+ "ld1rqb z0.b, p0/z, [%[a_ptr]]\n"
+ ".inst 0x45c59833 // ummla z19.s, z1.b, z5.b\n"
+ "ld1rqb z1.b, p0/z, [%[a_ptr], #0x10]\n"
+ ".inst 0x45c59859 // ummla z25.s, z2.b, z5.b\n"
+ "ld1rqb z2.b, p0/z, [%[a_ptr], #0x20]\n"
+ ".inst 0x45c5987f // ummla z31.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #5, MUL VL]\n"
+ ".inst 0x45c69808 // ummla z8.s, z0.b, z6.b\n"
+ "ld1rqb z3.b, p0/z, [%[a_ptr], #0x30]\n"
+ ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n"
+ "add %[a_ptr], %[a_ptr], #0x80\n"
+ ".inst 0x45c69854 // ummla z20.s, z2.b, z6.b\n"
+ "addvl %[b_ptr], %[b_ptr], #12\n"
+ ".inst 0x45c6987a // ummla z26.s, z3.b, z6.b\n"
+ ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n"
+ ".inst 0x45c7982f // ummla z15.s, z1.b, z7.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #-6, MUL VL]\n"
+ ".inst 0x45c79855 // ummla z21.s, z2.b, z7.b\n"
+ ".inst 0x45c7987b // ummla z27.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #-5, MUL VL]\n"
+ ".inst 0x45c4980a // ummla z10.s, z0.b, z4.b\n"
+ ".inst 0x45c49830 // ummla z16.s, z1.b, z4.b\n"
+ ".inst 0x45c49856 // ummla z22.s, z2.b, z4.b\n"
+ ".inst 0x45c4987c // ummla z28.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr], #-4, MUL VL]\n"
+ ".inst 0x45c5980b // ummla z11.s, z0.b, z5.b\n"
+ ".inst 0x45c59831 // ummla z17.s, z1.b, z5.b\n"
+ ".inst 0x45c59857 // ummla z23.s, z2.b, z5.b\n"
+ ".inst 0x45c5987d // ummla z29.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #-3, MUL VL]\n"
+ ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n"
+ ".inst 0x45c69832 // ummla z18.s, z1.b, z6.b\n"
+ ".inst 0x45c69858 // ummla z24.s, z2.b, z6.b\n"
+ ".inst 0x45c6987e // ummla z30.s, z3.b, z6.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #-2, MUL VL]\n"
+ ".inst 0x45c7980d // ummla z13.s, z0.b, z7.b\n"
+ "ld1rqb z0.b, p0/z, [%[a_ptr], #-0x40]\n"
+ ".inst 0x45c79833 // ummla z19.s, z1.b, z7.b\n"
+ "ld1rqb z1.b, p0/z, [%[a_ptr], #-0x30]\n"
+ ".inst 0x45c79859 // ummla z25.s, z2.b, z7.b\n"
+ "ld1rqb z2.b, p0/z, [%[a_ptr], #-0x20]\n"
+ ".inst 0x45c7987f // ummla z31.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #-1, MUL VL]\n"
+ "ld1rqb z3.b, p0/z, [%[a_ptr], #-0x10]\n"
+ "b.ne 2b\n"
+ "1:\n"
+ "cbz %[tails], 3f\n"
+ ".inst 0x45c49808 // ummla z8.s, z0.b, z4.b\n"
+ ".inst 0x45c4982e // ummla z14.s, z1.b, z4.b\n"
+ ".inst 0x45c49854 // ummla z20.s, z2.b, z4.b\n"
+ ".inst 0x45c4987a // ummla z26.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr]]\n"
+ ".inst 0x45c59809 // ummla z9.s, z0.b, z5.b\n"
+ ".inst 0x45c5982f // ummla z15.s, z1.b, z5.b\n"
+ ".inst 0x45c59855 // ummla z21.s, z2.b, z5.b\n"
+ ".inst 0x45c5987b // ummla z27.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #1, MUL VL]\n"
+ ".inst 0x45c6980a // ummla z10.s, z0.b, z6.b\n"
+ ".inst 0x45c69830 // ummla z16.s, z1.b, z6.b\n"
+ ".inst 0x45c69856 // ummla z22.s, z2.b, z6.b\n"
+ ".inst 0x45c6987c // ummla z28.s, z3.b, z6.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #2, MUL VL]\n"
+ ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n"
+ ".inst 0x45c79831 // ummla z17.s, z1.b, z7.b\n"
+ ".inst 0x45c79857 // ummla z23.s, z2.b, z7.b\n"
+ ".inst 0x45c7987d // ummla z29.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #3, MUL VL]\n"
+ ".inst 0x45c4980c // ummla z12.s, z0.b, z4.b\n"
+ ".inst 0x45c49832 // ummla z18.s, z1.b, z4.b\n"
+ ".inst 0x45c49858 // ummla z24.s, z2.b, z4.b\n"
+ ".inst 0x45c4987e // ummla z30.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr], #4, MUL VL]\n"
+ ".inst 0x45c5980d // ummla z13.s, z0.b, z5.b\n"
+ "ld1rqb z0.b, p0/z, [%[a_ptr]]\n"
+ ".inst 0x45c59833 // ummla z19.s, z1.b, z5.b\n"
+ "ld1rqb z1.b, p0/z, [%[a_ptr], #0x10]\n"
+ ".inst 0x45c59859 // ummla z25.s, z2.b, z5.b\n"
+ "ld1rqb z2.b, p0/z, [%[a_ptr], #0x20]\n"
+ ".inst 0x45c5987f // ummla z31.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #5, MUL VL]\n"
+ ".inst 0x45c69808 // ummla z8.s, z0.b, z6.b\n"
+ "ld1rqb z3.b, p0/z, [%[a_ptr], #0x30]\n"
+ ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n"
+ "add %[a_ptr], %[a_ptr], #0x80\n"
+ ".inst 0x45c69854 // ummla z20.s, z2.b, z6.b\n"
+ "addvl %[b_ptr], %[b_ptr], #14\n"
+ ".inst 0x45c6987a // ummla z26.s, z3.b, z6.b\n"
+ ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n"
+ ".inst 0x45c7982f // ummla z15.s, z1.b, z7.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #-8, MUL VL]\n"
+ ".inst 0x45c79855 // ummla z21.s, z2.b, z7.b\n"
+ ".inst 0x45c7987b // ummla z27.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #-7, MUL VL]\n"
+ ".inst 0x45c4980a // ummla z10.s, z0.b, z4.b\n"
+ ".inst 0x45c49830 // ummla z16.s, z1.b, z4.b\n"
+ ".inst 0x45c49856 // ummla z22.s, z2.b, z4.b\n"
+ ".inst 0x45c4987c // ummla z28.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr], #-6, MUL VL]\n"
+ ".inst 0x45c5980b // ummla z11.s, z0.b, z5.b\n"
+ ".inst 0x45c59831 // ummla z17.s, z1.b, z5.b\n"
+ ".inst 0x45c59857 // ummla z23.s, z2.b, z5.b\n"
+ ".inst 0x45c5987d // ummla z29.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #-5, MUL VL]\n"
+ ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n"
+ ".inst 0x45c69832 // ummla z18.s, z1.b, z6.b\n"
+ ".inst 0x45c69858 // ummla z24.s, z2.b, z6.b\n"
+ ".inst 0x45c6987e // ummla z30.s, z3.b, z6.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #-4, MUL VL]\n"
+ ".inst 0x45c7980d // ummla z13.s, z0.b, z7.b\n"
+ "ld1rqb z0.b, p0/z, [%[a_ptr], #-0x40]\n"
+ ".inst 0x45c79833 // ummla z19.s, z1.b, z7.b\n"
+ "ld1rqb z1.b, p0/z, [%[a_ptr], #-0x30]\n"
+ ".inst 0x45c79859 // ummla z25.s, z2.b, z7.b\n"
+ "ld1rqb z2.b, p0/z, [%[a_ptr], #-0x20]\n"
+ ".inst 0x45c7987f // ummla z31.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #-3, MUL VL]\n"
+ ".inst 0x45c49808 // ummla z8.s, z0.b, z4.b\n"
+ "ld1rqb z3.b, p0/z, [%[a_ptr], #-0x10]\n"
+ ".inst 0x45c4982e // ummla z14.s, z1.b, z4.b\n"
+ ".inst 0x45c49854 // ummla z20.s, z2.b, z4.b\n"
+ ".inst 0x45c59809 // ummla z9.s, z0.b, z5.b\n"
+ ".inst 0x45c4987a // ummla z26.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr], #-2, MUL VL]\n"
+ ".inst 0x45c5982f // ummla z15.s, z1.b, z5.b\n"
+ ".inst 0x45c59855 // ummla z21.s, z2.b, z5.b\n"
+ ".inst 0x45c5987b // ummla z27.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #-1, MUL VL]\n"
+ ".inst 0x45c6980a // ummla z10.s, z0.b, z6.b\n"
+ ".inst 0x45c69830 // ummla z16.s, z1.b, z6.b\n"
+ ".inst 0x45c69856 // ummla z22.s, z2.b, z6.b\n"
+ ".inst 0x45c6987c // ummla z28.s, z3.b, z6.b\n"
+ "uzp1 z6.d, z14.d, z15.d\n"
+ ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n"
+ ".inst 0x45c79831 // ummla z17.s, z1.b, z7.b\n"
+ ".inst 0x45c79857 // ummla z23.s, z2.b, z7.b\n"
+ ".inst 0x45c7987d // ummla z29.s, z3.b, z7.b\n"
+ ".inst 0x45c4980c // ummla z12.s, z0.b, z4.b\n"
+ "uzp1 z7.d, z16.d, z17.d\n"
+ ".inst 0x45c49832 // ummla z18.s, z1.b, z4.b\n"
+ ".inst 0x45c49858 // ummla z24.s, z2.b, z4.b\n"
+ ".inst 0x45c4987e // ummla z30.s, z3.b, z4.b\n"
+ "uzp2 z4.d, z10.d, z11.d\n"
+ ".inst 0x45c5980d // ummla z13.s, z0.b, z5.b\n"
+ "uzp1 z0.d, z8.d, z9.d\n"
+ ".inst 0x45c59833 // ummla z19.s, z1.b, z5.b\n"
+ "uzp1 z1.d, z10.d, z11.d\n"
+ ".inst 0x45c59859 // ummla z25.s, z2.b, z5.b\n"
+ "st1w z0.s, p0, [%[c_ptr]]\n"
+ "uzp1 z2.d, z12.d, z13.d\n"
+ "uzp1 z0.d, z18.d, z19.d\n"
+ ".inst 0x45c5987f // ummla z31.s, z3.b, z5.b\n"
+ "st1w z1.s, p0, [%[c_ptr], #1, MUL VL]\n"
+ "uzp2 z3.d, z8.d, z9.d\n"
+ "uzp2 z5.d, z12.d, z13.d\n"
+ "uzp2 z1.d, z14.d, z15.d\n"
+ "st1w z2.s, p0, [%[c_ptr], #2, MUL VL]\n"
+ "b 4f\n"
+ "3:\n"
+ ".inst 0x45c49808 // ummla z8.s, z0.b, z4.b\n"
+ "add %[a_ptr], %[a_ptr], #0x40\n"
+ ".inst 0x45c4982e // ummla z14.s, z1.b, z4.b\n"
+ "addvl %[b_ptr], %[b_ptr], #8\n"
+ ".inst 0x45c49854 // ummla z20.s, z2.b, z4.b\n"
+ ".inst 0x45c4987a // ummla z26.s, z3.b, z4.b\n"
+ ".inst 0x45c59809 // ummla z9.s, z0.b, z5.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr], #-8, MUL VL]\n"
+ ".inst 0x45c5982f // ummla z15.s, z1.b, z5.b\n"
+ ".inst 0x45c59855 // ummla z21.s, z2.b, z5.b\n"
+ ".inst 0x45c5987b // ummla z27.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #-7, MUL VL]\n"
+ ".inst 0x45c6980a // ummla z10.s, z0.b, z6.b\n"
+ ".inst 0x45c69830 // ummla z16.s, z1.b, z6.b\n"
+ ".inst 0x45c69856 // ummla z22.s, z2.b, z6.b\n"
+ ".inst 0x45c6987c // ummla z28.s, z3.b, z6.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #-6, MUL VL]\n"
+ ".inst 0x45c7980b // ummla z11.s, z0.b, z7.b\n"
+ ".inst 0x45c79831 // ummla z17.s, z1.b, z7.b\n"
+ ".inst 0x45c79857 // ummla z23.s, z2.b, z7.b\n"
+ ".inst 0x45c7987d // ummla z29.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #-5, MUL VL]\n"
+ ".inst 0x45c4980c // ummla z12.s, z0.b, z4.b\n"
+ ".inst 0x45c49832 // ummla z18.s, z1.b, z4.b\n"
+ ".inst 0x45c49858 // ummla z24.s, z2.b, z4.b\n"
+ ".inst 0x45c4987e // ummla z30.s, z3.b, z4.b\n"
+ "ld1b z4.b, p0/z, [%[b_ptr], #-4, MUL VL]\n"
+ ".inst 0x45c5980d // ummla z13.s, z0.b, z5.b\n"
+ "ld1rqb z0.b, p0/z, [%[a_ptr], #-0x40]\n"
+ ".inst 0x45c59833 // ummla z19.s, z1.b, z5.b\n"
+ "ld1rqb z1.b, p0/z, [%[a_ptr], #-0x30]\n"
+ ".inst 0x45c59859 // ummla z25.s, z2.b, z5.b\n"
+ "ld1rqb z2.b, p0/z, [%[a_ptr], #-0x20]\n"
+ ".inst 0x45c5987f // ummla z31.s, z3.b, z5.b\n"
+ "ld1b z5.b, p0/z, [%[b_ptr], #-3, MUL VL]\n"
+ ".inst 0x45c69808 // ummla z8.s, z0.b, z6.b\n"
+ "ld1rqb z3.b, p0/z, [%[a_ptr], #-0x10]\n"
+ ".inst 0x45c6982e // ummla z14.s, z1.b, z6.b\n"
+ ".inst 0x45c69854 // ummla z20.s, z2.b, z6.b\n"
+ ".inst 0x45c79809 // ummla z9.s, z0.b, z7.b\n"
+ ".inst 0x45c6987a // ummla z26.s, z3.b, z6.b\n"
+ "ld1b z6.b, p0/z, [%[b_ptr], #-2, MUL VL]\n"
+ ".inst 0x45c7982f // ummla z15.s, z1.b, z7.b\n"
+ ".inst 0x45c79855 // ummla z21.s, z2.b, z7.b\n"
+ ".inst 0x45c7987b // ummla z27.s, z3.b, z7.b\n"
+ "ld1b z7.b, p0/z, [%[b_ptr], #-1, MUL VL]\n"
+ ".inst 0x45c4980a // ummla z10.s, z0.b, z4.b\n"
+ ".inst 0x45c49830 // ummla z16.s, z1.b, z4.b\n"
+ ".inst 0x45c49856 // ummla z22.s, z2.b, z4.b\n"
+ ".inst 0x45c4987c // ummla z28.s, z3.b, z4.b\n"
+ ".inst 0x45c5980b // ummla z11.s, z0.b, z5.b\n"
+ ".inst 0x45c59831 // ummla z17.s, z1.b, z5.b\n"
+ ".inst 0x45c59857 // ummla z23.s, z2.b, z5.b\n"
+ ".inst 0x45c5987d // ummla z29.s, z3.b, z5.b\n"
+ "uzp2 z4.d, z10.d, z11.d\n"
+ ".inst 0x45c6980c // ummla z12.s, z0.b, z6.b\n"
+ ".inst 0x45c69832 // ummla z18.s, z1.b, z6.b\n"
+ ".inst 0x45c69858 // ummla z24.s, z2.b, z6.b\n"
+ ".inst 0x45c6987e // ummla z30.s, z3.b, z6.b\n"
+ "uzp1 z6.d, z14.d, z15.d\n"
+ ".inst 0x45c7980d // ummla z13.s, z0.b, z7.b\n"
+ "uzp1 z0.d, z8.d, z9.d\n"
+ ".inst 0x45c79833 // ummla z19.s, z1.b, z7.b\n"
+ "uzp1 z1.d, z10.d, z11.d\n"
+ "uzp2 z5.d, z12.d, z13.d\n"
+ "st1w z0.s, p0, [%[c_ptr]]\n"
+ ".inst 0x45c79859 // ummla z25.s, z2.b, z7.b\n"
+ "uzp1 z2.d, z12.d, z13.d\n"
+ "uzp1 z0.d, z18.d, z19.d\n"
+ "st1w z1.s, p0, [%[c_ptr], #1, MUL VL]\n"
+ "uzp2 z1.d, z14.d, z15.d\n"
+ ".inst 0x45c7987f // ummla z31.s, z3.b, z7.b\n"
+ "uzp2 z3.d, z8.d, z9.d\n"
+ "st1w z2.s, p0, [%[c_ptr], #2, MUL VL]\n"
+ "uzp1 z7.d, z16.d, z17.d\n"
+ "4:\n"
+ "uzp2 z2.d, z16.d, z17.d\n"
+ "st1w z3.s, p0, [%[c_ptr], #3, MUL VL]\n"
+ "uzp2 z3.d, z18.d, z19.d\n"
+ "st1w z4.s, p0, [%[c_ptr], #4, MUL VL]\n"
+ "uzp1 z4.d, z20.d, z21.d\n"
+ "st1w z5.s, p0, [%[c_ptr], #5, MUL VL]\n"
+ "uzp1 z5.d, z22.d, z23.d\n"
+ "st1w z6.s, p0, [%[c_ptr], #6, MUL VL]\n"
+ "uzp1 z6.d, z24.d, z25.d\n"
+ "st1w z7.s, p0, [%[c_ptr], #7, MUL VL]\n"
+ "addvl %[c_ptr], %[c_ptr], #16\n"
+ "uzp2 z7.d, z20.d, z21.d\n"
+ "st1w z0.s, p0, [%[c_ptr], #-8, MUL VL]\n"
+ "uzp2 z0.d, z22.d, z23.d\n"
+ "st1w z1.s, p0, [%[c_ptr], #-7, MUL VL]\n"
+ "uzp2 z1.d, z24.d, z25.d\n"
+ "st1w z2.s, p0, [%[c_ptr], #-6, MUL VL]\n"
+ "uzp1 z2.d, z26.d, z27.d\n"
+ "st1w z3.s, p0, [%[c_ptr], #-5, MUL VL]\n"
+ "uzp1 z3.d, z28.d, z29.d\n"
+ "st1w z4.s, p0, [%[c_ptr], #-4, MUL VL]\n"
+ "uzp1 z4.d, z30.d, z31.d\n"
+ "st1w z5.s, p0, [%[c_ptr], #-3, MUL VL]\n"
+ "uzp2 z5.d, z26.d, z27.d\n"
+ "st1w z6.s, p0, [%[c_ptr], #-2, MUL VL]\n"
+ "uzp2 z6.d, z28.d, z29.d\n"
+ "st1w z7.s, p0, [%[c_ptr], #-1, MUL VL]\n"
+ "uzp2 z7.d, z30.d, z31.d\n"
+ "st1w z0.s, p0, [%[c_ptr]]\n"
+ "st1w z1.s, p0, [%[c_ptr], #1, MUL VL]\n"
+ "st1w z2.s, p0, [%[c_ptr], #2, MUL VL]\n"
+ "st1w z3.s, p0, [%[c_ptr], #3, MUL VL]\n"
+ "st1w z4.s, p0, [%[c_ptr], #4, MUL VL]\n"
+ "st1w z5.s, p0, [%[c_ptr], #5, MUL VL]\n"
+ "st1w z6.s, p0, [%[c_ptr], #6, MUL VL]\n"
+ "st1w z7.s, p0, [%[c_ptr], #7, MUL VL]\n"
+ "addvl %[c_ptr], %[c_ptr], #8\n"
+ : [a_ptr] "+r" (a_ptr), [b_ptr] "+r" (b_ptr), [c_ptr] "+r" (c_ptr),
+ [loops] "+r" (loops), [tails] "+r" (tails)
+ :
+ : "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31", "cc", "memory"
+ );
+ }
+ }
+}
+
+} // namespace arm_gemm
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp
index 7bfab412ca..f82e7b4e47 100644
--- a/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp
+++ b/src/core/NEON/kernels/arm_gemm/merges/a64_merge_fp16_24x8.hpp
@@ -140,6 +140,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[outptr0]]\n"
"dup v1.8h, %[minval].h[0]\n"
@@ -214,6 +217,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[outptr0]]\n"
"dup v1.8h, %[minval].h[0]\n"
@@ -311,6 +317,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[outptr0]]\n"
"dup v1.8h, %[minval].h[0]\n"
@@ -430,6 +439,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[outptr0]]\n"
"dup v1.8h, %[minval].h[0]\n"
@@ -572,6 +584,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[outptr0]]\n"
"dup v1.8h, %[minval].h[0]\n"
@@ -737,6 +752,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[outptr0]]\n"
"dup v1.8h, %[minval].h[0]\n"
@@ -926,6 +944,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[outptr0]]\n"
"dup v1.8h, %[minval].h[0]\n"
@@ -1133,6 +1154,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[biasptr]]\n"
"dup v1.8h, %[minval].h[0]\n"
@@ -1184,6 +1208,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[biasptr]]\n"
"dup v1.8h, %[minval].h[0]\n"
@@ -1255,6 +1282,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[biasptr]]\n"
"dup v1.8h, %[minval].h[0]\n"
@@ -1346,6 +1376,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[biasptr]]\n"
"dup v1.8h, %[minval].h[0]\n"
@@ -1456,6 +1489,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[biasptr]]\n"
"dup v1.8h, %[minval].h[0]\n"
@@ -1586,6 +1622,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[biasptr]]\n"
"dup v1.8h, %[minval].h[0]\n"
@@ -1736,6 +1775,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[biasptr]]\n"
"dup v1.8h, %[minval].h[0]\n"
@@ -1907,6 +1949,9 @@ void MergeResults<24, 8, false>(__fp16 *out, const __fp16 *in, const int ldout,
} else {
/* Optimized routine to copy an entire block */
__asm __volatile (
+#ifndef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+ ".arch armv8.2-a+fp16\n"
+#endif
"dup v0.8h, %[maxval].h[0]\n"
"ldr q2, [%[biasptr]]\n"
"dup v1.8h, %[minval].h[0]\n"
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_8way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_8way_32bit.hpp
new file mode 100644
index 0000000000..0080c91b18
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_8way_32bit.hpp
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) 2017-2019 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __aarch64__
+
+#include "transpose_interleave_common.hpp"
+
+// Generic unblocked transposed 8x32-bit sized specialisation
+template <>
+template <typename T>
+inline void TransformImpl<8, 1, true, 4, 4, false>::Transform(
+ T* out, const T* const in, const int stride,
+ const int x0, const int xmax, const int k0, const int kmax
+) {
+ // Redirect to a 16 x uint16_t specialisation
+ TransformImpl<16, 1, true, 2, 2, false>::Transform(
+ reinterpret_cast<uint16_t *>(out),
+ reinterpret_cast<const uint16_t *>(in),
+ stride*2, x0*2, xmax*2, k0, kmax
+ );
+}
+
+// Generic 16x16-bit sized specialisation
+template <>
+template <typename T>
+inline void TransformImpl<16, 1, true, 2, 2, false>::Transform(
+ T* out, const T* const in, const int stride,
+ const int x0, const int xmax, const int k0, const int kmax
+) {
+ // Redirect to a uint16_t specialisation
+ Transform(
+ reinterpret_cast<uint16_t *>(out),
+ reinterpret_cast<const uint16_t *>(in),
+ stride, x0, xmax, k0, kmax
+ );
+}
+
+// Specialised 16 x uint16_t version
+template <>
+inline void TransposeInterleaveCommon<16, uint16_t, uint16_t>::moveblock_1x1(const uint16_t *&in0, uint16_t *const out) {
+ __asm volatile (
+ "LDR q0, [%[in0]]\n"
+ "STR q0, [%[out]]\n"
+ "LDR q1, [%[in0], #0x10]\n"
+ "STR q1, [%[out], #0x10]\n"
+ "ADD %x[in0], %x[in0], #0x20\n"
+ ASM_PREFETCH("[%[in0], #192]")
+ : [in0] "+r" (in0)
+ : [out] "r" (out)
+ : "v0", "v1", "memory"
+ );
+}
+
+template <>
+inline void TransposeInterleaveCommon<16, uint16_t, uint16_t>::moveblock_1x2(const uint16_t *&in0, const uint16_t *&in1, uint16_t *const out) {
+ __asm volatile (
+ "LDR q0, [%[in0]]\n"
+ "STR q0, [%[out]]\n"
+ "LDR q1, [%[in0], #0x10]\n"
+ "STR q1, [%[out], #0x10]\n"
+ "ADD %x[in0], %x[in0], #0x20\n"
+ ASM_PREFETCH("[%[in0], #192]")
+
+ "LDR q2, [%[in1]]\n"
+ "STR q2, [%[out], #0x20]\n"
+ "LDR q3, [%[in1], #0x10]\n"
+ "STR q3, [%[out], #0x30]\n"
+ "ADD %x[in1], %x[in1], #0x20\n"
+ ASM_PREFETCH("[%[in1], #192]")
+ : [in0] "+r" (in0),
+ [in1] "+r" (in1)
+ : [out] "r" (out)
+ : "v0", "v1", "v2", "v3", "memory"
+ );
+}
+
+template <>
+inline void TransposeInterleaveCommon<16, uint16_t, uint16_t>::moveblock_1x4(const uint16_t *&in0, const uint16_t *&in1, const uint16_t *&in2, const uint16_t *&in3, uint16_t *const out) {
+ __asm __volatile (
+ "LDR q0, [%[in0]]\n"
+ "STR q0, [%[out]]\n"
+ "LDR q1, [%[in0], #0x10]\n"
+ "STR q1, [%[out], #0x10]\n"
+ "ADD %x[in0], %x[in0], #0x20\n"
+ ASM_PREFETCH("[%[in0], #192]")
+
+ "LDR q2, [%[in1]]\n"
+ "STR q2, [%[out], #0x20]\n"
+ "LDR q3, [%[in1], #0x10]\n"
+ "STR q3, [%[out], #0x30]\n"
+ "ADD %x[in1], %x[in1], #0x20\n"
+ ASM_PREFETCH("[%[in1], #192]")
+
+ "LDR q0, [%[in2]]\n"
+ "STR q0, [%[out], #0x40]\n"
+ "LDR q1, [%[in2], #0x10]\n"
+ "STR q1, [%[out], #0x50]\n"
+ "ADD %x[in2], %x[in2], #0x20\n"
+ ASM_PREFETCH("[%[in2], #192]")
+
+ "LDR q2, [%[in3]]\n"
+ "STR q2, [%[out], #0x60]\n"
+ "LDR q3, [%[in3], #0x10]\n"
+ "STR q3, [%[out], #0x70]\n"
+ "ADD %x[in3], %x[in3], #0x20\n"
+ ASM_PREFETCH("[%[in3], #192]")
+ : [in0] "+r" (in0),
+ [in1] "+r" (in1),
+ [in2] "+r" (in2),
+ [in3] "+r" (in3)
+ : [out] "r" (out)
+ : "v0", "v1", "v2", "v3", "memory"
+ );
+}
+
+template <>
+template <>
+inline void TransformImpl<16, 1, true, 2, 2, false>::Transform(
+ uint16_t* out, const uint16_t* const in, const int stride,
+ const int x0, const int xmax, const int k0, const int kmax
+) {
+ TransposeInterleaveCommon<16, uint16_t, uint16_t>::Transform(out, in, stride, x0, xmax, k0, kmax);
+}
+
+#endif // __aarch64__
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/list.hpp b/src/core/NEON/kernels/arm_gemm/transforms/list.hpp
index c0c2ca19d7..be66cd42ff 100644
--- a/src/core/NEON/kernels/arm_gemm/transforms/list.hpp
+++ b/src/core/NEON/kernels/arm_gemm/transforms/list.hpp
@@ -31,5 +31,9 @@
#include "a64_transpose_interleave_12way_16bit.hpp"
#include "a64_transpose_interleave_12way_half_to_float.hpp"
#include "a64_transpose_interleave_24way_16bit.hpp"
+#include "a64_transpose_interleave_8way_32bit.hpp"
#include "sve_interleave_8way_32bit.hpp"
+#include "sve_interleave_8way_block2_16bit.hpp"
+#include "sve_interleave_8way_block4_16bit.hpp"
#include "sve_interleave_8way_block4_8bit.hpp"
+#include "sve_interleave_8way_block8_8bit.hpp"
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_16bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_16bit.hpp
new file mode 100644
index 0000000000..234433a0f1
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_16bit.hpp
@@ -0,0 +1,596 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __ARM_FEATURE_SVE
+
+template<>
+template<typename T>
+inline void TransformImpl<8, 2, false, 2, 2, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax)
+{
+ uint16_t *master_outptr = reinterpret_cast<uint16_t *>(out);
+ const uint16_t *inptr = reinterpret_cast<const uint16_t *>(in);
+
+ for (int y=y0; y<ymax; y+=8)
+ {
+ const int height = ymax-y;
+ const long inwidth = (kmax - k0);
+ const long outwidth = ((inwidth + 1) / 2) * 16;
+ long inpos = 0;
+ long outpos = 0;
+
+ uint16_t *outptr = master_outptr;
+ master_outptr += outwidth;
+
+ const uint16_t *inptr0 = inptr + y * ldin + k0;
+ const uint16_t *inptr1 = inptr0 + ldin;
+ const uint16_t *inptr2 = inptr1 + ldin;
+ const uint16_t *inptr3 = inptr2 + ldin;
+ const uint16_t *inptr4 = inptr3 + ldin;
+ const uint16_t *inptr5 = inptr4 + ldin;
+ const uint16_t *inptr6 = inptr5 + ldin;
+ const uint16_t *inptr7 = inptr6 + ldin;
+
+ switch(height)
+ {
+ case 1:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z4.h, #0\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip1 z0.s, z8.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z1.s, z8.s, z4.s\n"
+ "zip1 z2.s, z9.s, z4.s\n"
+ "zip2 z3.s, z9.s, z4.s\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "zip1 z10.s, z1.s, z4.s\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip2 z11.s, z1.s, z4.s\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z4.s\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip2 z13.s, z2.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z14.s, z3.s, z4.s\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "zip2 z15.s, z3.s, z4.s\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 2:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z4.h, #0\n"
+ "mov z14.h, #0\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "zip1 z10.s, z1.s, z4.s\n"
+ "zip2 z11.s, z1.s, z4.s\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip1 z0.s, z8.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z1.s, z8.s, z4.s\n"
+ "zip1 z2.s, z9.s, z4.s\n"
+ "zip2 z3.s, z9.s, z4.s\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip1 z4.s, z10.s, z14.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z5.s, z10.s, z14.s\n"
+ "zip1 z6.s, z11.s, z14.s\n"
+ "zip2 z7.s, z11.s, z14.s\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 3:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z4.h, #0\n"
+ "mov z14.h, #0\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
+ "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z4.s\n"
+ "zip2 z11.s, z1.s, z4.s\n"
+ "zip1 z12.s, z2.s, z4.s\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z0.s, z8.s, z12.s\n"
+ "zip2 z1.s, z8.s, z12.s\n"
+ "zip1 z2.s, z9.s, z13.s\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip2 z3.s, z9.s, z13.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z4.s, z10.s, z14.s\n"
+ "zip2 z5.s, z10.s, z14.s\n"
+ "zip1 z6.s, z11.s, z14.s\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip2 z7.s, z11.s, z14.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 4:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z4.h, #0\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
+ "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
+ "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z4.s\n"
+ "zip2 z11.s, z1.s, z4.s\n"
+ "zip1 z12.s, z2.s, z4.s\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z14.s, z3.s, z4.s\n"
+ "zip2 z15.s, z3.s, z4.s\n"
+ "zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip2 z1.s, z8.s, z12.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z2.s, z9.s, z13.s\n"
+ "zip2 z3.s, z9.s, z13.s\n"
+ "zip1 z4.s, z10.s, z14.s\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip2 z5.s, z10.s, z14.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z6.s, z11.s, z15.s\n"
+ "zip2 z7.s, z11.s, z15.s\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 5:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z5.h, #0\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
+ "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
+ "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
+ "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "zip1 z12.s, z2.s, z5.s\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z5.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z14.s, z3.s, z5.s\n"
+ "zip2 z15.s, z3.s, z5.s\n"
+ "zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip2 z1.s, z8.s, z12.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z2.s, z9.s, z13.s\n"
+ "zip2 z3.s, z9.s, z13.s\n"
+ "zip1 z4.s, z10.s, z14.s\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip2 z5.s, z10.s, z14.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z6.s, z11.s, z15.s\n"
+ "zip2 z7.s, z11.s, z15.s\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 6:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z6.h, #0\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
+ "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
+ "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
+ "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
+ "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z14.s, z3.s, z6.s\n"
+ "zip2 z15.s, z3.s, z6.s\n"
+ "zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip2 z1.s, z8.s, z12.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z2.s, z9.s, z13.s\n"
+ "zip2 z3.s, z9.s, z13.s\n"
+ "zip1 z4.s, z10.s, z14.s\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip2 z5.s, z10.s, z14.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z6.s, z11.s, z15.s\n"
+ "zip2 z7.s, z11.s, z15.s\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 7:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z7.h, #0\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
+ "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
+ "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
+ "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
+ "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n"
+ "ld1h z6.h, p0/z, [%[inptr6], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip2 z1.s, z8.s, z12.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z2.s, z9.s, z13.s\n"
+ "zip2 z3.s, z9.s, z13.s\n"
+ "zip1 z4.s, z10.s, z14.s\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip2 z5.s, z10.s, z14.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z6.s, z11.s, z15.s\n"
+ "zip2 z7.s, z11.s, z15.s\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ default:
+ case 8:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
+ "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
+ "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
+ "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
+ "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n"
+ "ld1h z6.h, p0/z, [%[inptr6], %[inpos], LSL #1]\n"
+ "ld1h z7.h, p0/z, [%[inptr7], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "zip1 z0.s, z8.s, z12.s\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip2 z1.s, z8.s, z12.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z2.s, z9.s, z13.s\n"
+ "zip2 z3.s, z9.s, z13.s\n"
+ "zip1 z4.s, z10.s, z14.s\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip2 z5.s, z10.s, z14.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z6.s, z11.s, z15.s\n"
+ "zip2 z7.s, z11.s, z15.s\n"
+ "zip1 z8.s, z0.s, z4.s\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.s, z0.s, z4.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.s, z1.s, z5.s\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip2 z11.s, z1.s, z5.s\n"
+ "zip1 z12.s, z2.s, z6.s\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.s, z2.s, z6.s\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.s, z3.s, z7.s\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z15.s, z3.s, z7.s\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+
+ }
+ }
+}
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_16bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_16bit.hpp
new file mode 100644
index 0000000000..26e10511a6
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_16bit.hpp
@@ -0,0 +1,596 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __ARM_FEATURE_SVE
+
+template<>
+template<typename T>
+inline void TransformImpl<8, 4, false, 2, 2, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax)
+{
+ uint16_t *master_outptr = reinterpret_cast<uint16_t *>(out);
+ const uint16_t *inptr = reinterpret_cast<const uint16_t *>(in);
+
+ for (int y=y0; y<ymax; y+=8)
+ {
+ const int height = ymax-y;
+ const long inwidth = (kmax - k0);
+ const long outwidth = ((inwidth + 3) / 4) * 32;
+ long inpos = 0;
+ long outpos = 0;
+
+ uint16_t *outptr = master_outptr;
+ master_outptr += outwidth;
+
+ const uint16_t *inptr0 = inptr + y * ldin + k0;
+ const uint16_t *inptr1 = inptr0 + ldin;
+ const uint16_t *inptr2 = inptr1 + ldin;
+ const uint16_t *inptr3 = inptr2 + ldin;
+ const uint16_t *inptr4 = inptr3 + ldin;
+ const uint16_t *inptr5 = inptr4 + ldin;
+ const uint16_t *inptr6 = inptr5 + ldin;
+ const uint16_t *inptr7 = inptr6 + ldin;
+
+ switch(height)
+ {
+ case 1:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z4.h, #0\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip1 z0.d, z8.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z1.d, z8.d, z4.d\n"
+ "zip1 z2.d, z9.d, z4.d\n"
+ "zip2 z3.d, z9.d, z4.d\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "zip1 z10.d, z1.d, z4.d\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip2 z11.d, z1.d, z4.d\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip1 z12.d, z2.d, z4.d\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip2 z13.d, z2.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z14.d, z3.d, z4.d\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "zip2 z15.d, z3.d, z4.d\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 2:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z4.h, #0\n"
+ "mov z14.h, #0\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "zip1 z10.d, z1.d, z4.d\n"
+ "zip2 z11.d, z1.d, z4.d\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip1 z0.d, z8.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z1.d, z8.d, z4.d\n"
+ "zip1 z2.d, z9.d, z4.d\n"
+ "zip2 z3.d, z9.d, z4.d\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip1 z4.d, z10.d, z14.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z5.d, z10.d, z14.d\n"
+ "zip1 z6.d, z11.d, z14.d\n"
+ "zip2 z7.d, z11.d, z14.d\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 3:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z4.h, #0\n"
+ "mov z14.h, #0\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
+ "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z4.d\n"
+ "zip2 z11.d, z1.d, z4.d\n"
+ "zip1 z12.d, z2.d, z4.d\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z0.d, z8.d, z12.d\n"
+ "zip2 z1.d, z8.d, z12.d\n"
+ "zip1 z2.d, z9.d, z13.d\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip2 z3.d, z9.d, z13.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z4.d, z10.d, z14.d\n"
+ "zip2 z5.d, z10.d, z14.d\n"
+ "zip1 z6.d, z11.d, z14.d\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip2 z7.d, z11.d, z14.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 4:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z4.h, #0\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
+ "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
+ "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z4.d\n"
+ "zip2 z11.d, z1.d, z4.d\n"
+ "zip1 z12.d, z2.d, z4.d\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z14.d, z3.d, z4.d\n"
+ "zip2 z15.d, z3.d, z4.d\n"
+ "zip1 z0.d, z8.d, z12.d\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip2 z1.d, z8.d, z12.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z2.d, z9.d, z13.d\n"
+ "zip2 z3.d, z9.d, z13.d\n"
+ "zip1 z4.d, z10.d, z14.d\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip2 z5.d, z10.d, z14.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z6.d, z11.d, z15.d\n"
+ "zip2 z7.d, z11.d, z15.d\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 5:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z5.h, #0\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
+ "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
+ "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
+ "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z5.d\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z5.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z14.d, z3.d, z5.d\n"
+ "zip2 z15.d, z3.d, z5.d\n"
+ "zip1 z0.d, z8.d, z12.d\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip2 z1.d, z8.d, z12.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z2.d, z9.d, z13.d\n"
+ "zip2 z3.d, z9.d, z13.d\n"
+ "zip1 z4.d, z10.d, z14.d\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip2 z5.d, z10.d, z14.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z6.d, z11.d, z15.d\n"
+ "zip2 z7.d, z11.d, z15.d\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 6:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z6.h, #0\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
+ "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
+ "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
+ "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
+ "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z14.d, z3.d, z6.d\n"
+ "zip2 z15.d, z3.d, z6.d\n"
+ "zip1 z0.d, z8.d, z12.d\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip2 z1.d, z8.d, z12.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z2.d, z9.d, z13.d\n"
+ "zip2 z3.d, z9.d, z13.d\n"
+ "zip1 z4.d, z10.d, z14.d\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip2 z5.d, z10.d, z14.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z6.d, z11.d, z15.d\n"
+ "zip2 z7.d, z11.d, z15.d\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 7:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z7.h, #0\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
+ "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
+ "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
+ "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
+ "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n"
+ "ld1h z6.h, p0/z, [%[inptr6], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "zip1 z0.d, z8.d, z12.d\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip2 z1.d, z8.d, z12.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z2.d, z9.d, z13.d\n"
+ "zip2 z3.d, z9.d, z13.d\n"
+ "zip1 z4.d, z10.d, z14.d\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip2 z5.d, z10.d, z14.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z6.d, z11.d, z15.d\n"
+ "zip2 z7.d, z11.d, z15.d\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ default:
+ case 8:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.h, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n"
+ "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n"
+ "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n"
+ "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n"
+ "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n"
+ "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n"
+ "ld1h z6.h, p0/z, [%[inptr6], %[inpos], LSL #1]\n"
+ "ld1h z7.h, p0/z, [%[inptr7], %[inpos], LSL #1]\n"
+ "inch %[inpos], all, mul #1\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p0.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p1.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "zip1 z0.d, z8.d, z12.d\n"
+ "whilelt p2.h, %[outpos], %[outwidth]\n"
+ "zip2 z1.d, z8.d, z12.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z2.d, z9.d, z13.d\n"
+ "zip2 z3.d, z9.d, z13.d\n"
+ "zip1 z4.d, z10.d, z14.d\n"
+ "whilelt p3.h, %[outpos], %[outwidth]\n"
+ "zip2 z5.d, z10.d, z14.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z6.d, z11.d, z15.d\n"
+ "zip2 z7.d, z11.d, z15.d\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p4.h, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "st1h z8.h, p0, [%[outptr]]\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p5.h, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "inch %[outpos], all, mul #1\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.h, %[outpos], %[outwidth]\n"
+ "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.h, %[outpos], %[outwidth]\n"
+ "inch %[outpos], all, mul #1\n"
+ "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n"
+ "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n"
+ "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+
+ }
+ }
+}
+
+#endif // __ARM_FEATURE_SVE
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block8_8bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block8_8bit.hpp
new file mode 100644
index 0000000000..b4935e6417
--- /dev/null
+++ b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block8_8bit.hpp
@@ -0,0 +1,596 @@
+/*
+ * Copyright (c) 2019 Arm Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#pragma once
+
+#ifdef __ARM_FEATURE_SVE
+
+template<>
+template<typename T>
+inline void TransformImpl<8, 8, false, 1, 1, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax)
+{
+ uint8_t *master_outptr = reinterpret_cast<uint8_t *>(out);
+ const uint8_t *inptr = reinterpret_cast<const uint8_t *>(in);
+
+ for (int y=y0; y<ymax; y+=8)
+ {
+ const int height = ymax-y;
+ const long inwidth = (kmax - k0);
+ const long outwidth = ((inwidth + 7) / 8) * 64;
+ long inpos = 0;
+ long outpos = 0;
+
+ uint8_t *outptr = master_outptr;
+ master_outptr += outwidth;
+
+ const uint8_t *inptr0 = inptr + y * ldin + k0;
+ const uint8_t *inptr1 = inptr0 + ldin;
+ const uint8_t *inptr2 = inptr1 + ldin;
+ const uint8_t *inptr3 = inptr2 + ldin;
+ const uint8_t *inptr4 = inptr3 + ldin;
+ const uint8_t *inptr5 = inptr4 + ldin;
+ const uint8_t *inptr6 = inptr5 + ldin;
+ const uint8_t *inptr7 = inptr6 + ldin;
+
+ switch(height)
+ {
+ case 1:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.b, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z4.b, #0\n"
+ "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
+ "incb %[inpos], all, mul #1\n"
+ "whilelt p0.b, %[outpos], %[outwidth]\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "whilelt p1.b, %[outpos], %[outwidth]\n"
+ "zip1 z0.d, z8.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip2 z1.d, z8.d, z4.d\n"
+ "zip1 z2.d, z9.d, z4.d\n"
+ "zip2 z3.d, z9.d, z4.d\n"
+ "whilelt p2.b, %[outpos], %[outwidth]\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "zip1 z10.d, z1.d, z4.d\n"
+ "st1b z8.b, p0, [%[outptr]]\n"
+ "zip2 z11.d, z1.d, z4.d\n"
+ "whilelt p3.b, %[outpos], %[outwidth]\n"
+ "zip1 z12.d, z2.d, z4.d\n"
+ "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
+ "zip2 z13.d, z2.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z14.d, z3.d, z4.d\n"
+ "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
+ "zip2 z15.d, z3.d, z4.d\n"
+ "whilelt p4.b, %[outpos], %[outwidth]\n"
+ "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p5.b, %[outpos], %[outwidth]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
+ "whilelt p6.b, %[outpos], %[outwidth]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
+ "whilelt p7.b, %[outpos], %[outwidth]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 2:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.b, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z4.b, #0\n"
+ "mov z14.b, #0\n"
+ "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
+ "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
+ "incb %[inpos], all, mul #1\n"
+ "whilelt p0.b, %[outpos], %[outwidth]\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "zip1 z10.d, z1.d, z4.d\n"
+ "zip2 z11.d, z1.d, z4.d\n"
+ "whilelt p1.b, %[outpos], %[outwidth]\n"
+ "zip1 z0.d, z8.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip2 z1.d, z8.d, z4.d\n"
+ "zip1 z2.d, z9.d, z4.d\n"
+ "zip2 z3.d, z9.d, z4.d\n"
+ "whilelt p2.b, %[outpos], %[outwidth]\n"
+ "zip1 z4.d, z10.d, z14.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip2 z5.d, z10.d, z14.d\n"
+ "zip1 z6.d, z11.d, z14.d\n"
+ "zip2 z7.d, z11.d, z14.d\n"
+ "whilelt p3.b, %[outpos], %[outwidth]\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "st1b z8.b, p0, [%[outptr]]\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "whilelt p4.b, %[outpos], %[outwidth]\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "whilelt p5.b, %[outpos], %[outwidth]\n"
+ "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p6.b, %[outpos], %[outwidth]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
+ "whilelt p7.b, %[outpos], %[outwidth]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
+ "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 3:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.b, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z4.b, #0\n"
+ "mov z14.b, #0\n"
+ "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
+ "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
+ "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
+ "incb %[inpos], all, mul #1\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p0.b, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z4.d\n"
+ "zip2 z11.d, z1.d, z4.d\n"
+ "zip1 z12.d, z2.d, z4.d\n"
+ "whilelt p1.b, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z0.d, z8.d, z12.d\n"
+ "zip2 z1.d, z8.d, z12.d\n"
+ "zip1 z2.d, z9.d, z13.d\n"
+ "whilelt p2.b, %[outpos], %[outwidth]\n"
+ "zip2 z3.d, z9.d, z13.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z4.d, z10.d, z14.d\n"
+ "zip2 z5.d, z10.d, z14.d\n"
+ "zip1 z6.d, z11.d, z14.d\n"
+ "whilelt p3.b, %[outpos], %[outwidth]\n"
+ "zip2 z7.d, z11.d, z14.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "whilelt p4.b, %[outpos], %[outwidth]\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "st1b z8.b, p0, [%[outptr]]\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "whilelt p5.b, %[outpos], %[outwidth]\n"
+ "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
+ "whilelt p6.b, %[outpos], %[outwidth]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.b, %[outpos], %[outwidth]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
+ "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
+ "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 4:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.b, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z4.b, #0\n"
+ "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
+ "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
+ "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
+ "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
+ "incb %[inpos], all, mul #1\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p0.b, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z4.d\n"
+ "zip2 z11.d, z1.d, z4.d\n"
+ "zip1 z12.d, z2.d, z4.d\n"
+ "whilelt p1.b, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z14.d, z3.d, z4.d\n"
+ "zip2 z15.d, z3.d, z4.d\n"
+ "zip1 z0.d, z8.d, z12.d\n"
+ "whilelt p2.b, %[outpos], %[outwidth]\n"
+ "zip2 z1.d, z8.d, z12.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z2.d, z9.d, z13.d\n"
+ "zip2 z3.d, z9.d, z13.d\n"
+ "zip1 z4.d, z10.d, z14.d\n"
+ "whilelt p3.b, %[outpos], %[outwidth]\n"
+ "zip2 z5.d, z10.d, z14.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z6.d, z11.d, z15.d\n"
+ "zip2 z7.d, z11.d, z15.d\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p4.b, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "st1b z8.b, p0, [%[outptr]]\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p5.b, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.b, %[outpos], %[outwidth]\n"
+ "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.b, %[outpos], %[outwidth]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
+ "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
+ "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 5:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.b, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z5.b, #0\n"
+ "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
+ "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
+ "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
+ "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
+ "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n"
+ "incb %[inpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "whilelt p0.b, %[outpos], %[outwidth]\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z5.d\n"
+ "whilelt p1.b, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z5.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z14.d, z3.d, z5.d\n"
+ "zip2 z15.d, z3.d, z5.d\n"
+ "zip1 z0.d, z8.d, z12.d\n"
+ "whilelt p2.b, %[outpos], %[outwidth]\n"
+ "zip2 z1.d, z8.d, z12.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z2.d, z9.d, z13.d\n"
+ "zip2 z3.d, z9.d, z13.d\n"
+ "zip1 z4.d, z10.d, z14.d\n"
+ "whilelt p3.b, %[outpos], %[outwidth]\n"
+ "zip2 z5.d, z10.d, z14.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z6.d, z11.d, z15.d\n"
+ "zip2 z7.d, z11.d, z15.d\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p4.b, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "st1b z8.b, p0, [%[outptr]]\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p5.b, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.b, %[outpos], %[outwidth]\n"
+ "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.b, %[outpos], %[outwidth]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
+ "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
+ "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 6:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.b, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z6.b, #0\n"
+ "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
+ "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
+ "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
+ "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
+ "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n"
+ "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n"
+ "incb %[inpos], all, mul #1\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p0.b, %[outpos], %[outwidth]\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "whilelt p1.b, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z14.d, z3.d, z6.d\n"
+ "zip2 z15.d, z3.d, z6.d\n"
+ "zip1 z0.d, z8.d, z12.d\n"
+ "whilelt p2.b, %[outpos], %[outwidth]\n"
+ "zip2 z1.d, z8.d, z12.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z2.d, z9.d, z13.d\n"
+ "zip2 z3.d, z9.d, z13.d\n"
+ "zip1 z4.d, z10.d, z14.d\n"
+ "whilelt p3.b, %[outpos], %[outwidth]\n"
+ "zip2 z5.d, z10.d, z14.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z6.d, z11.d, z15.d\n"
+ "zip2 z7.d, z11.d, z15.d\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p4.b, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "st1b z8.b, p0, [%[outptr]]\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p5.b, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.b, %[outpos], %[outwidth]\n"
+ "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.b, %[outpos], %[outwidth]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
+ "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
+ "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ case 7:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.b, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "mov z7.b, #0\n"
+ "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
+ "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
+ "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
+ "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
+ "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n"
+ "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n"
+ "ld1b z6.b, p0/z, [%[inptr6], %[inpos]]\n"
+ "incb %[inpos], all, mul #1\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p0.b, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p1.b, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "zip1 z0.d, z8.d, z12.d\n"
+ "whilelt p2.b, %[outpos], %[outwidth]\n"
+ "zip2 z1.d, z8.d, z12.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z2.d, z9.d, z13.d\n"
+ "zip2 z3.d, z9.d, z13.d\n"
+ "zip1 z4.d, z10.d, z14.d\n"
+ "whilelt p3.b, %[outpos], %[outwidth]\n"
+ "zip2 z5.d, z10.d, z14.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z6.d, z11.d, z15.d\n"
+ "zip2 z7.d, z11.d, z15.d\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p4.b, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "st1b z8.b, p0, [%[outptr]]\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p5.b, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.b, %[outpos], %[outwidth]\n"
+ "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.b, %[outpos], %[outwidth]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
+ "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
+ "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+ default:
+ case 8:
+ __asm __volatile(
+ "1:\n"
+ "whilelt p0.b, %[inpos], %[inwidth]\n"
+ "b.none 2f\n"
+ "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n"
+ "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n"
+ "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n"
+ "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n"
+ "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n"
+ "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n"
+ "ld1b z6.b, p0/z, [%[inptr6], %[inpos]]\n"
+ "ld1b z7.b, p0/z, [%[inptr7], %[inpos]]\n"
+ "incb %[inpos], all, mul #1\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p0.b, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p1.b, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "zip1 z0.d, z8.d, z12.d\n"
+ "whilelt p2.b, %[outpos], %[outwidth]\n"
+ "zip2 z1.d, z8.d, z12.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z2.d, z9.d, z13.d\n"
+ "zip2 z3.d, z9.d, z13.d\n"
+ "zip1 z4.d, z10.d, z14.d\n"
+ "whilelt p3.b, %[outpos], %[outwidth]\n"
+ "zip2 z5.d, z10.d, z14.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z6.d, z11.d, z15.d\n"
+ "zip2 z7.d, z11.d, z15.d\n"
+ "zip1 z8.d, z0.d, z4.d\n"
+ "whilelt p4.b, %[outpos], %[outwidth]\n"
+ "zip2 z9.d, z0.d, z4.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip1 z10.d, z1.d, z5.d\n"
+ "st1b z8.b, p0, [%[outptr]]\n"
+ "zip2 z11.d, z1.d, z5.d\n"
+ "zip1 z12.d, z2.d, z6.d\n"
+ "whilelt p5.b, %[outpos], %[outwidth]\n"
+ "zip2 z13.d, z2.d, z6.d\n"
+ "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n"
+ "zip1 z14.d, z3.d, z7.d\n"
+ "incb %[outpos], all, mul #1\n"
+ "zip2 z15.d, z3.d, z7.d\n"
+ "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n"
+ "whilelt p6.b, %[outpos], %[outwidth]\n"
+ "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n"
+ "whilelt p7.b, %[outpos], %[outwidth]\n"
+ "incb %[outpos], all, mul #1\n"
+ "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n"
+ "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n"
+ "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n"
+ "addvl %[outptr], %[outptr], #8\n"
+ "b 1b\n"
+ "2:\n"
+ : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7)
+ : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth)
+ : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory"
+ );
+ break;
+
+
+ }
+ }
+}
+
+#endif // __ARM_FEATURE_SVE