diff options
Diffstat (limited to 'src/core/NEON/kernels/arm_gemm/transforms')
21 files changed, 20 insertions, 5159 deletions
diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a32_interleave_6way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a32_interleave_6way_32bit.hpp deleted file mode 100644 index 2df5d1bd28..0000000000 --- a/src/core/NEON/kernels/arm_gemm/transforms/a32_interleave_6way_32bit.hpp +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (c) 2017-2018 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __arm__ - -#include <arm_neon.h> - -#include "../asmlib.hpp" - -template<> -template<typename T> -inline void TransformImpl<6, 1, false, 4, 4, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) { - uint32_t *outptr = reinterpret_cast<uint32_t *>(out); - const uint32_t *inptr = reinterpret_cast<const uint32_t *>(in); - bool first = true; - - uint32_t zerobuff[16] = { 0 }; // 8 for asm loop plus up to 7 for overflow loop - - for (int y=y0; y<ymax; y+=6) { - const uint32_t *inptr0 = inptr + y * ldin + k0; - const uint32_t *inptr1 = inptr0 + ldin; - const uint32_t *inptr2 = inptr1 + ldin; - const uint32_t *inptr3 = inptr2 + ldin; - const uint32_t *inptr4 = inptr3 + ldin; - const uint32_t *inptr5 = inptr4 + ldin; - - //prefetch_2x(inptr0); - //prefetch_2x(inptr1); - //prefetch_2x(inptr2); - //prefetch_2x(inptr3); - //prefetch_2x(inptr4); - //prefetch_2x(inptr5); - - int x=(kmax-k0); - for (;(x>7) || first;x-=8) { - /* Cope with ragged cases by copying from a buffer of zeroes instead */ - /* 'first' forces this to always run at least once, needed if the total size is <=7. */ - if ((y + 5) >= ymax) { - switch ((y + 5) - ymax) { - case 4: - inptr1 = zerobuff; - // fall through - case 3: - inptr2 = zerobuff; - // fall through - case 2: - inptr3 = zerobuff; - // fall through - case 1: - inptr4 = zerobuff; - // fall through - case 0: - inptr5 = zerobuff; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - if (first) { - if (x<=7) { - break; - } - - first = false; - } - - __asm __volatile ( - // Load up 8 elements (2 vectors) from each of 8 sources. - "VLD1.32 {d0-d3}, [%[inptr0]]!\n" // q0=A0A1A2A3 - "VLD1.32 {d4-d7}, [%[inptr1]]!\n" // q2=B0B1B2B3 - "VLD1.32 {d8-d11}, [%[inptr2]]!\n" // q4=C0C1C2C3 - "VZIP.32 q0, q4\n" // q0=A0C0A1C1, q4 = A2C2A3C3 - "VLD1.32 {d12-d15}, [%[inptr3]]!\n" // q6=D0D1D2D3 - "VZIP.32 q2, q6\n" // q2=B0D0B1D1, q6 = B2D2B3D3 - "VLD1.32 {d16-d19}, [%[inptr4]]!\n" - "VLD1.32 {d20-d23}, [%[inptr5]]!\n" - "VZIP.32 q8, q10\n" // q8=E0F0E1F1, q10 = E2F2E3F3 - ASM_PREFETCH("[%[inptr0], #128]") - "VZIP.32 q0, q2\n" // q0 = A0B0C0D0, q2 = A1B1C1D1 - - // Store first elements - "VST1.32 {d0-d1}, [%[outptr]]!\n" - "VST1.32 {d16}, [%[outptr]]!\n" - - "VZIP.32 q4, q6\n" // q4 = A2B2C2D2, q6 = A3B3C3D3 - - // Store second elements - "VST1.32 {d4-d5}, [%[outptr]]!\n" - "VZIP.32 q1, q5\n" - ASM_PREFETCH("[%[inptr1], #128]") - "VST1.32 {d17}, [%[outptr]]!\n" - "VZIP.32 q3, q7\n" - - // Store third elements - "VZIP.32 q9, q11\n" - "VST1.32 {d8-d9}, [%[outptr]]!\n" - "VZIP.32 q1, q3\n" - ASM_PREFETCH("[%[inptr2], #128]") - "VST1.32 {d20}, [%[outptr]]!\n" - - // Store fourth elements - "VZIP.32 q5, q7\n" - "VST1.32 {d12-d13}, [%[outptr]]!\n" - ASM_PREFETCH("[%[inptr3], #128]") - "VST1.32 {d21}, [%[outptr]]!\n" - - // Fifth - "VST1.32 {d2-d3}, [%[outptr]]!\n" - ASM_PREFETCH("[%[inptr4], #128]") - "VST1.32 {d18}, [%[outptr]]!\n" - - // Sixth - "VST1.32 {d6-d7}, [%[outptr]]!\n" - ASM_PREFETCH("[%[inptr5], #128]") - "VST1.32 {d19}, [%[outptr]]!\n" - - // Seventh - "VST1.32 {d10-d11}, [%[outptr]]!\n" - "VST1.32 {d22}, [%[outptr]]!\n" - - // Eighth - "VST1.32 {d14-d15}, [%[outptr]]!\n" - "VST1.32 {d23}, [%[outptr]]!\n" - - : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), - [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [outptr] "+r" (outptr) - : - : "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", "q9", "q10", "q11", "q12", "memory" - ); - } - - for (;x>0;x--) { - *outptr++ = *inptr0++; - *outptr++ = *inptr1++; - *outptr++ = *inptr2++; - *outptr++ = *inptr3++; - *outptr++ = *inptr4++; - *outptr++ = *inptr5++; - } - } -} - -#endif // __arm__ diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a32_transpose_interleave_8way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a32_transpose_interleave_8way_32bit.hpp index 8f0b8ae63f..3ce1d328a7 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a32_transpose_interleave_8way_32bit.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a32_transpose_interleave_8way_32bit.hpp @@ -30,22 +30,22 @@ // Generic unblocked transposed 8x32-bit sized specialisation template <> template <typename T> -inline void TransformImpl<8, 1, true, 4, 4, false>::Transform( +inline void TransformImpl<8, 1, true, 4, 4, VLType::None>::Transform( T* out, const T* const in, const int stride, const int x0, const int xmax, const int k0, const int kmax ) { // Redirect to a 16x uint16_t specialisation - TransformImpl<16, 1, true, 2, 2, false>::Transform( + TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform( reinterpret_cast<uint16_t *>(out), reinterpret_cast<const uint16_t *>(in), stride*2, x0*2, xmax*2, k0, kmax ); } -// Generic 12x16-bit sized specialisation +// Generic 16x16-bit sized specialisation template <> template <typename T> -inline void TransformImpl<16, 1, true, 2, 2, false>::Transform( +inline void TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform( T* out, const T* const in, const int stride, const int x0, const int xmax, const int k0, const int kmax ) { @@ -117,7 +117,7 @@ inline void TransposeInterleaveCommon<16, uint16_t, uint16_t>::moveblock_1x4(con template <> template <> -inline void TransformImpl<16, 1, true, 2, 2, false>::Transform( +inline void TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform( uint16_t* out, const uint16_t* const in, const int stride, const int x0, const int xmax, const int k0, const int kmax ) { diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_block16_interleave4_8bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_block16_interleave4_8bit.hpp deleted file mode 100644 index 9b6f4de543..0000000000 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_block16_interleave4_8bit.hpp +++ /dev/null @@ -1,128 +0,0 @@ -/* - * Copyright (c) 2017-2020 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __aarch64__ - -#include <arm_neon.h> - -#include "../asmlib.hpp" -#include "../utils.hpp" - -template<> -template<typename T> -void TransformImpl<4, 16, false, 1, 1, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) { - uint8_t *outptr = (uint8_t *)out; - const uint8_t *inptr = (uint8_t *)in; - - uint8_t zerobuff[16] = { 0 }; - - for (int y=y0; y<ymax; y+=4) { - const uint8_t *inptr0 = inptr + static_cast<intptr_t>(y) * ldin + k0; - const uint8_t *inptr1 = inptr0 + ldin; - const uint8_t *inptr2 = inptr1 + ldin; - const uint8_t *inptr3 = inptr2 + ldin; - - prefetch_2x(inptr0); - prefetch_2x(inptr1); - prefetch_2x(inptr2); - prefetch_2x(inptr3); - - int x=(kmax-k0); - for (;x>15;x-=16) { - /* Cope with ragged cases by copying from a buffer of zeroes instead */ - if ((y + 3) >= ymax) { - switch ((y + 3) - ymax) { - case 2: - inptr1 = zerobuff; - // fall through - case 1: - inptr2 = zerobuff; - // fall through - case 0: - inptr3 = zerobuff; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - __asm __volatile ( - "LDR q0, [%[inptr0]], #16\n" - ASM_PREFETCH("[%[inptr0], #176]") - "LDR q1, [%[inptr1]], #16\n" - ASM_PREFETCH("[%[inptr1], #176]") - "STP q0, q1, [%[outptr]], #32\n" - "LDR q0, [%[inptr2]], #16\n" - ASM_PREFETCH("[%[inptr2], #176]") - "LDR q1, [%[inptr3]], #16\n" - ASM_PREFETCH("[%[inptr3], #176]") - "STP q0, q1, [%[outptr]], #32\n" - : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), - [outptr] "+r" (outptr) - : - : "v0", "v1" - ); - } - - if (x>0) { - /* Need to duplicate this here, in case we didn't run the main loop. */ - if ((y + 3) >= ymax) { - switch ((y + 3) - ymax) { - case 2: - inptr1 = zerobuff; - // fall through - case 1: - inptr2 = zerobuff; - // fall through - case 0: - inptr3 = zerobuff; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - /* We have to write out 16 values, copy as many legal values as there are and pad with 0 */ - auto f = [&outptr, x](const uint8_t *&p) { - for (int i=0; i<16; i++) { - if (i < x) { - *outptr++ = *p++; - } else { - *outptr++ = 0; - } - } - }; - - f(inptr0); - f(inptr1); - f(inptr2); - f(inptr3); - } - } -} - -#endif // __aarch64__
\ No newline at end of file diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_16bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_16bit.hpp deleted file mode 100644 index 3d912c4675..0000000000 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_16bit.hpp +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright (c) 2017-2020 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __aarch64__ - -#include <arm_neon.h> - -#include "../asmlib.hpp" - -template<> -template<typename T> -void TransformImpl<8, 1, false, 2, 2, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) { - uint16_t *outptr = (uint16_t *)out; - const uint16_t *inptr = (const uint16_t *)in; - bool first=true; - - uint16_t zerobuff[16] = { 0 }; // 8 for asm loop plus up to 7 for overflow loop - - for (int y=y0; y<ymax; y+=8) { - const uint16_t *inptr0 = inptr + y * ldin + k0; - const uint16_t *inptr1 = inptr0 + ldin; - const uint16_t *inptr2 = inptr1 + ldin; - const uint16_t *inptr3 = inptr2 + ldin; - const uint16_t *inptr4 = inptr3 + ldin; - const uint16_t *inptr5 = inptr4 + ldin; - const uint16_t *inptr6 = inptr5 + ldin; - const uint16_t *inptr7 = inptr6 + ldin; - - prefetch_2x(inptr0); - prefetch_2x(inptr1); - prefetch_2x(inptr2); - prefetch_2x(inptr3); - prefetch_2x(inptr4); - prefetch_2x(inptr5); - prefetch_2x(inptr6); - prefetch_2x(inptr7); - - int x=(kmax-k0); - for (;(x>7) || first;x-=8) { - /* Cope with ragged cases by copying from a buffer of zeroes instead */ - /* 'first' forces this to always run at least once, needed if the total size is <=7. */ - if ((y + 7) >= ymax) { - switch ((y + 7) - ymax) { - case 6: - inptr1 = zerobuff; - // fall through - case 5: - inptr2 = zerobuff; - // fall through - case 4: - inptr3 = zerobuff; - // fall through - case 3: - inptr4 = zerobuff; - // fall through - case 2: - inptr5 = zerobuff; - // fall through - case 1: - inptr6 = zerobuff; - // fall through - case 0: - inptr7 = zerobuff; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - if (first) { - if (x <= 7) { - break; - } - - first = false; - } - - int skippf = (x & 31); - __asm __volatile ( - // Load up 8 elements (1 vector) from each of 8 sources. - "CBNZ %w[skippf], 1f\n" - ASM_PREFETCH("[%[inptr0], #128]") - ASM_PREFETCH("[%[inptr1], #128]") - ASM_PREFETCH("[%[inptr2], #128]") - ASM_PREFETCH("[%[inptr3], #128]") - "1:\n" - - "LDR q0, [%[inptr0]], #16\n" // q0=A0A1A2A3A4A5A6A7 - "LDR q4, [%[inptr4]], #16\n" // q8=E0E1E2E3E4E5E6E7 - "LDR q2, [%[inptr2]], #16\n" // q4=C0C1C2C3... - "LDR q6, [%[inptr6]], #16\n" - "ZIP1 v8.8h, v0.8h, v4.8h\n" // q8=A0E0A1E1A2E2A3E3 - "ZIP2 v16.8h, v0.8h, v4.8h\n" // q16=A4E4A5E5A6E6A7E7 - "ZIP1 v9.8h, v2.8h, v6.8h\n" // q9=C0G0C1G1C2G2C3G3 - "ZIP2 v17.8h, v2.8h, v6.8h\n" // q17=C4G4C5G5C6G6C7G7 - "LDR q1, [%[inptr1]], #16\n" // q1=B0B1B2B3B4B5B6B7 - "LDR q5, [%[inptr5]], #16\n" - "LDR q3, [%[inptr3]], #16\n" // q3=D0D1D2D3.... - "LDR q7, [%[inptr7]], #16\n" - "ZIP1 v10.8h, v1.8h, v5.8h\n" // q18=B0F0B1F1B2F2B3F3 - "ZIP2 v18.8h, v1.8h, v5.8h\n" // q18=B4F4B5F5B6F6B7F7 - "ZIP1 v11.8h, v3.8h, v7.8h\n" // q19=D0H0D1H1D2H2D3H3 - "ZIP2 v19.8h, v3.8h, v7.8h\n" // q19=D4H4D5H5D6H6D7H7 - - "ZIP1 v12.8h, v8.8h, v9.8h\n" // q20=A0C0E0G0A1C1E1G1 - "ZIP2 v20.8h, v8.8h, v9.8h\n" - "ZIP1 v13.8h, v10.8h, v11.8h\n" // q21=B0D0F0H0B1I1F1H1 - "ZIP2 v21.8h, v10.8h, v11.8h\n" - - "CBNZ %w[skippf], 2f\n" - ASM_PREFETCH("[%[inptr4], #112]") - ASM_PREFETCH("[%[inptr5], #112]") - ASM_PREFETCH("[%[inptr6], #112]") - ASM_PREFETCH("[%[inptr7], #112]") - "2:\n" - - "ZIP1 v22.8h, v16.8h, v17.8h\n" - "ZIP2 v30.8h, v16.8h, v17.8h\n" - "ZIP1 v23.8h, v18.8h, v19.8h\n" - "ZIP2 v31.8h, v18.8h, v19.8h\n" - - "ZIP1 v14.8h, v12.8h, v13.8h\n" // q22=A0B0C0D0E0F0G0H0 - "ZIP2 v15.8h, v12.8h, v13.8h\n" // q23=A1B1C1D1E1F1G1H1 - "STP q14, q15, [%[outptr]], #32\n" // Write back first two elements - - "ZIP1 v0.8h, v20.8h, v21.8h\n" - "ZIP2 v1.8h, v20.8h, v21.8h\n" - "STP q0, q1, [%[outptr]], #32\n" // Write back next two elements - - "ZIP1 v2.8h, v22.8h, v23.8h\n" - "ZIP2 v3.8h, v22.8h, v23.8h\n" - "STP q2, q3, [%[outptr]], #32\n" // Write back next two elements - - "ZIP1 v4.8h, v30.8h, v31.8h\n" - "ZIP2 v5.8h, v30.8h, v31.8h\n" - "STP q4, q5, [%[outptr]], #32\n" // Write back last two elements - : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), - [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7), [outptr] "+r" (outptr) - : [skippf] "r" (skippf) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", - "v25", "v26", "v27", "v28", "v29", "v30", "v31", "memory" - ); - } - - for (;x>0;x--) { - *outptr++ = *inptr0++; - *outptr++ = *inptr1++; - *outptr++ = *inptr2++; - *outptr++ = *inptr3++; - *outptr++ = *inptr4++; - *outptr++ = *inptr5++; - *outptr++ = *inptr6++; - *outptr++ = *inptr7++; - } - } -} - -#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_32bit.hpp deleted file mode 100644 index 701d688af2..0000000000 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_32bit.hpp +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Copyright (c) 2017-2020 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#if defined(__aarch64__) && !defined(__ARM_FEATURE_SVE) - -#include <arm_neon.h> - -#include "../asmlib.hpp" - -template<> -template<typename T> -inline void TransformImpl<8, 1, false, 4, 4, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) { - uint32_t *outptr = (uint32_t *)out; - const uint32_t *inptr = (uint32_t *)in; - bool first = true; - - uint32_t zerobuff[16] = { 0 }; // 8 for asm loop plus up to 7 for overflow loop - - for (int y=y0; y<ymax; y+=8) { - const uint32_t *inptr0 = inptr + y * ldin + k0; - const uint32_t *inptr1 = inptr0 + ldin; - const uint32_t *inptr2 = inptr1 + ldin; - const uint32_t *inptr3 = inptr2 + ldin; - const uint32_t *inptr4 = inptr3 + ldin; - const uint32_t *inptr5 = inptr4 + ldin; - const uint32_t *inptr6 = inptr5 + ldin; - const uint32_t *inptr7 = inptr6 + ldin; - - prefetch_2x(inptr0); - prefetch_2x(inptr1); - prefetch_2x(inptr2); - prefetch_2x(inptr3); - prefetch_2x(inptr4); - prefetch_2x(inptr5); - prefetch_2x(inptr6); - prefetch_2x(inptr7); - - int x=(kmax-k0); - for (;(x>7) || first;x-=8) { - /* Cope with ragged cases by copying from a buffer of zeroes instead */ - /* 'first' forces this to always run at least once, needed if the total size is <=7. */ - if ((y + 7) >= ymax) { - switch ((y + 7) - ymax) { - case 6: - inptr1 = zerobuff; - // fall through - case 5: - inptr2 = zerobuff; - // fall through - case 4: - inptr3 = zerobuff; - // fall through - case 3: - inptr4 = zerobuff; - // fall through - case 2: - inptr5 = zerobuff; - // fall through - case 1: - inptr6 = zerobuff; - // fall through - case 0: - inptr7 = zerobuff; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - if (first) { - if (x<=7) { - break; - } - - first = false; - } - - __asm __volatile ( - // Load up 8 elements (2 vectors) from each of 8 sources. - "LDP q0, q1, [%[inptr0]], #32\n" // q0=A0A1A2A3 - "LDP q2, q3, [%[inptr1]], #32\n" // q2=B0B1B2B3 - "LDP q4, q5, [%[inptr2]], #32\n" // q4=C0C1C2C3 - "ZIP1 v16.4s, v0.4s, v4.4s\n" // q16=A0C0A1C1 - ASM_PREFETCH("[%[inptr0], #128]") - "LDP q6, q7, [%[inptr3]], #32\n" // q6=D0D1D2D3 - "ZIP1 v17.4s, v2.4s, v6.4s\n" // q17=B0D0B1D1 - "LDP q8, q9, [%[inptr4]], #32\n" - "LDP q10, q11, [%[inptr5]], #32\n" - "LDP q12, q13, [%[inptr6]], #32\n" - "ZIP1 v18.4s, v8.4s, v12.4s\n" - ASM_PREFETCH("[%[inptr1], #128]") - "LDP q14, q15, [%[inptr7]], #32\n" - "ZIP1 v19.4s, v10.4s, v14.4s\n" - - "ZIP1 v20.4s, v16.4s, v17.4s\n" // q20=A0B0C0D0 - ASM_PREFETCH("[%[inptr2], #128]") - "ZIP1 v21.4s, v18.4s, v19.4s\n" - "ZIP2 v22.4s, v16.4s, v17.4s\n" - "ZIP2 v23.4s, v18.4s, v19.4s\n" - - "ZIP2 v16.4s, v0.4s, v4.4s\n" - ASM_PREFETCH("[%[inptr3], #128]") - "ZIP2 v17.4s, v2.4s, v6.4s\n" - "STP q20, q21, [%[outptr]], #32\n" // Write back the first element of each source - - "ZIP2 v18.4s, v8.4s, v12.4s\n" - "ZIP2 v19.4s, v10.4s, v14.4s\n" - "STP q22, q23, [%[outptr]], #32\n" // Write back the second element of each source - - "ZIP1 v20.4s, v16.4s, v17.4s\n" - ASM_PREFETCH("[%[inptr4], #128]") - "ZIP1 v21.4s, v18.4s, v19.4s\n" - "ZIP2 v22.4s, v16.4s, v17.4s\n" - "ZIP2 v23.4s, v18.4s, v19.4s\n" - - "ZIP1 v16.4s, v1.4s, v5.4s\n" - ASM_PREFETCH("[%[inptr5], #128]") - "ZIP1 v17.4s, v3.4s, v7.4s\n" - "STP q20, q21, [%[outptr]], #32\n" // Third element - - "ZIP1 v18.4s, v9.4s, v13.4s\n" - "ZIP1 v19.4s, v11.4s, v15.4s\n" - "STP q22, q23, [%[outptr]], #32\n" // Fourth element - - "ZIP1 v20.4s, v16.4s, v17.4s\n" - "ZIP1 v21.4s, v18.4s, v19.4s\n" - "ZIP2 v22.4s, v16.4s, v17.4s\n" - ASM_PREFETCH("[%[inptr6], #128]") - "ZIP2 v23.4s, v18.4s, v19.4s\n" - - "ZIP2 v16.4s, v1.4s, v5.4s\n" - "ZIP2 v17.4s, v3.4s, v7.4s\n" - "STP q20, q21, [%[outptr]], #32\n" // Fifth element - - "ZIP2 v18.4s, v9.4s, v13.4s\n" - ASM_PREFETCH("[%[inptr7], #128]") - "ZIP2 v19.4s, v11.4s, v15.4s\n" - "STP q22, q23, [%[outptr]], #32\n" // Sixth element - - "ZIP1 v20.4s, v16.4s, v17.4s\n" - "ZIP1 v21.4s, v18.4s, v19.4s\n" - "STP q20, q21, [%[outptr]], #32\n" // Seventh element - - "ZIP2 v22.4s, v16.4s, v17.4s\n" - "ZIP2 v23.4s, v18.4s, v19.4s\n" - "STP q22, q23, [%[outptr]], #32\n" // Eighth element - : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), - [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7), [outptr] "+r" (outptr) - : - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "memory" - ); - } - - for (;x>0;x--) { - *outptr++ = *inptr0++; - *outptr++ = *inptr1++; - *outptr++ = *inptr2++; - *outptr++ = *inptr3++; - *outptr++ = *inptr4++; - *outptr++ = *inptr5++; - *outptr++ = *inptr6++; - *outptr++ = *inptr7++; - } - } -} - -#endif // __aarch64__ && !__ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_block4_8bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_block4_8bit.hpp deleted file mode 100644 index 2546cc571a..0000000000 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_block4_8bit.hpp +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright (c) 2017-2020 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#if defined(__aarch64__) && !defined(__ARM_FEATURE_SVE) - -#include <arm_neon.h> - -#include "../asmlib.hpp" - -template<> -template<typename T> -inline void TransformImpl<8, 4, false, 1, 1, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) { - uint8_t *outptr = reinterpret_cast<uint8_t *>(out); - const uint8_t *inptr = reinterpret_cast<const uint8_t *>(in); - bool first = true; - - /* Helper functions to copy blocks about used for odd case. */ - class t { - public: - static inline void copy_4_inc(uint8_t *&out, const uint8_t *&in) { - uint32_t *out_word = reinterpret_cast<uint32_t *>(out); - const uint32_t *in_word = reinterpret_cast<const uint32_t *>(in); - - *out_word++ = *in_word++; - - out = reinterpret_cast<uint8_t *>(out_word); - in = reinterpret_cast<const uint8_t *>(in_word); - } - - static inline void copy_pad(uint8_t *&out, const uint8_t *&in, size_t count) { - for (unsigned int i=0; i<4; i++) { - if (i < count) { - *out++ = *in++; - } else { - *out++ = 0; - } - } - } - }; - - uint8_t zerobuff[64] = { 0 }; // 32 for asm loop plus up to 31 for overflow loop - - for (int y=y0; y<ymax; y+=8) { - const uint8_t *inptr0 = inptr + y * ldin + k0; - const uint8_t *inptr1 = inptr0 + ldin; - const uint8_t *inptr2 = inptr1 + ldin; - const uint8_t *inptr3 = inptr2 + ldin; - const uint8_t *inptr4 = inptr3 + ldin; - const uint8_t *inptr5 = inptr4 + ldin; - const uint8_t *inptr6 = inptr5 + ldin; - const uint8_t *inptr7 = inptr6 + ldin; - - prefetch_2x(inptr0); - prefetch_2x(inptr1); - prefetch_2x(inptr2); - prefetch_2x(inptr3); - prefetch_2x(inptr4); - prefetch_2x(inptr5); - prefetch_2x(inptr6); - prefetch_2x(inptr7); - - int x=(kmax-k0); - for (;(x>31) || first;x-=32) { - /* Cope with ragged cases by copying from a buffer of zeroes instead */ - /* 'first' forces this to always run at least once, needed if the total size is <=32. */ - if ((y + 7) >= ymax) { - switch ((y + 7) - ymax) { - case 6: - inptr1 = zerobuff; - // fall through - case 5: - inptr2 = zerobuff; - // fall through - case 4: - inptr3 = zerobuff; - // fall through - case 3: - inptr4 = zerobuff; - // fall through - case 2: - inptr5 = zerobuff; - // fall through - case 1: - inptr6 = zerobuff; - // fall through - case 0: - inptr7 = zerobuff; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - if (first) { - if (x<=31) { - break; - } - - first = false; - } - - __asm __volatile ( - // Load up 8 elements (2 vectors) from each of 8 sources. - "LDP q0, q1, [%[inptr0]], #32\n" // q0=A0A1A2A3 - "LDP q2, q3, [%[inptr1]], #32\n" // q2=B0B1B2B3 - "LDP q4, q5, [%[inptr2]], #32\n" // q4=C0C1C2C3 - "ZIP1 v16.4s, v0.4s, v4.4s\n" // q16=A0C0A1C1 - ASM_PREFETCH("[%[inptr0], #128]") - "LDP q6, q7, [%[inptr3]], #32\n" // q6=D0D1D2D3 - "ZIP1 v17.4s, v2.4s, v6.4s\n" // q17=B0D0B1D1 - "LDP q8, q9, [%[inptr4]], #32\n" - "LDP q10, q11, [%[inptr5]], #32\n" - "LDP q12, q13, [%[inptr6]], #32\n" - "ZIP1 v18.4s, v8.4s, v12.4s\n" - ASM_PREFETCH("[%[inptr1], #128]") - "LDP q14, q15, [%[inptr7]], #32\n" - "ZIP1 v19.4s, v10.4s, v14.4s\n" - - "ZIP1 v20.4s, v16.4s, v17.4s\n" // q20=A0B0C0D0 - ASM_PREFETCH("[%[inptr2], #128]") - "ZIP1 v21.4s, v18.4s, v19.4s\n" - "ZIP2 v22.4s, v16.4s, v17.4s\n" - "ZIP2 v23.4s, v18.4s, v19.4s\n" - - "ZIP2 v16.4s, v0.4s, v4.4s\n" - ASM_PREFETCH("[%[inptr3], #128]") - "ZIP2 v17.4s, v2.4s, v6.4s\n" - "STP q20, q21, [%[outptr]], #32\n" // Write back the first element of each source - - "ZIP2 v18.4s, v8.4s, v12.4s\n" - "ZIP2 v19.4s, v10.4s, v14.4s\n" - "STP q22, q23, [%[outptr]], #32\n" // Write back the second element of each source - - "ZIP1 v20.4s, v16.4s, v17.4s\n" - ASM_PREFETCH("[%[inptr4], #128]") - "ZIP1 v21.4s, v18.4s, v19.4s\n" - "ZIP2 v22.4s, v16.4s, v17.4s\n" - "ZIP2 v23.4s, v18.4s, v19.4s\n" - - "ZIP1 v16.4s, v1.4s, v5.4s\n" - ASM_PREFETCH("[%[inptr5], #128]") - "ZIP1 v17.4s, v3.4s, v7.4s\n" - "STP q20, q21, [%[outptr]], #32\n" // Third element - - "ZIP1 v18.4s, v9.4s, v13.4s\n" - "ZIP1 v19.4s, v11.4s, v15.4s\n" - "STP q22, q23, [%[outptr]], #32\n" // Fourth element - - "ZIP1 v20.4s, v16.4s, v17.4s\n" - "ZIP1 v21.4s, v18.4s, v19.4s\n" - "ZIP2 v22.4s, v16.4s, v17.4s\n" - ASM_PREFETCH("[%[inptr6], #128]") - "ZIP2 v23.4s, v18.4s, v19.4s\n" - - "ZIP2 v16.4s, v1.4s, v5.4s\n" - "ZIP2 v17.4s, v3.4s, v7.4s\n" - "STP q20, q21, [%[outptr]], #32\n" // Fifth element - - "ZIP2 v18.4s, v9.4s, v13.4s\n" - ASM_PREFETCH("[%[inptr7], #128]") - "ZIP2 v19.4s, v11.4s, v15.4s\n" - "STP q22, q23, [%[outptr]], #32\n" // Sixth element - - "ZIP1 v20.4s, v16.4s, v17.4s\n" - "ZIP1 v21.4s, v18.4s, v19.4s\n" - "STP q20, q21, [%[outptr]], #32\n" // Seventh element - - "ZIP2 v22.4s, v16.4s, v17.4s\n" - "ZIP2 v23.4s, v18.4s, v19.4s\n" - "STP q22, q23, [%[outptr]], #32\n" // Eighth element - : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), - [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7), [outptr] "+r" (outptr) - : - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "memory" - ); - } - - // Copy any leftover blocks of 4 a complete block at a time. - for (;x>4;x-=4) { - t::copy_4_inc(outptr, inptr0); - t::copy_4_inc(outptr, inptr1); - t::copy_4_inc(outptr, inptr2); - t::copy_4_inc(outptr, inptr3); - t::copy_4_inc(outptr, inptr4); - t::copy_4_inc(outptr, inptr5); - t::copy_4_inc(outptr, inptr6); - t::copy_4_inc(outptr, inptr7); - } - - // Final block with padding, if any. - if (x > 0) { - t::copy_pad(outptr, inptr0, x); - t::copy_pad(outptr, inptr1, x); - t::copy_pad(outptr, inptr2, x); - t::copy_pad(outptr, inptr3, x); - t::copy_pad(outptr, inptr4, x); - t::copy_pad(outptr, inptr5, x); - t::copy_pad(outptr, inptr6, x); - t::copy_pad(outptr, inptr7, x); - } - } -} - -#endif // __aarch64__ && !__ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_half_to_float.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_half_to_float.hpp deleted file mode 100644 index a342d6c3d1..0000000000 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_half_to_float.hpp +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright (c) 2017-2020 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) - -#include <arm_neon.h> - -#include "../asmlib.hpp" - -template<> -template<> -inline void TransformImpl<8, 1, false, 4, 2, false>::Transform(float *out, const __fp16 *in, int ldin, int y0, int ymax, int k0, int kmax) { - float *outptr = out; - const __fp16 *inptr = in; - bool first = true; - - __fp16 zerobuff[16] = { 0 }; // 8 for asm loop plus up to 7 for overflow loop - - for (int y=y0; y<ymax; y+=8) { - const __fp16 *inptr0 = inptr + y * ldin + k0; - const __fp16 *inptr1 = inptr0 + ldin; - const __fp16 *inptr2 = inptr1 + ldin; - const __fp16 *inptr3 = inptr2 + ldin; - const __fp16 *inptr4 = inptr3 + ldin; - const __fp16 *inptr5 = inptr4 + ldin; - const __fp16 *inptr6 = inptr5 + ldin; - const __fp16 *inptr7 = inptr6 + ldin; - - prefetch_2x(inptr0); - prefetch_2x(inptr1); - prefetch_2x(inptr2); - prefetch_2x(inptr3); - prefetch_2x(inptr4); - prefetch_2x(inptr5); - prefetch_2x(inptr6); - prefetch_2x(inptr7); - - int x=(kmax-k0); - for (;(x>7) || first;x-=8) { - /* Cope with ragged cases by copying from a buffer of zeroes instead */ - /* 'first' forces this to always run at least once, needed if the total size is <=7. */ - if ((y + 7) >= ymax) { - switch ((y + 7) - ymax) { - case 6: - inptr1 = zerobuff; - // fall through - case 5: - inptr2 = zerobuff; - // fall through - case 4: - inptr3 = zerobuff; - // fall through - case 3: - inptr4 = zerobuff; - // fall through - case 2: - inptr5 = zerobuff; - // fall through - case 1: - inptr6 = zerobuff; - // fall through - case 0: - inptr7 = zerobuff; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - if (first) { - if (x<=7) { - break; - } - - first = false; - } - - __asm __volatile ( - // Load up 8 elements (2 vectors) from each of 8 sources. - "LDR q0, [%[inptr0]], #16\n" - "LDR q2, [%[inptr1]], #16\n" - "FCVTL2 v1.4s, v0.8h\n" - "FCVTL v0.4s, v0.4h\n" - "LDR q4, [%[inptr2]], #16\n" // q4=C0C1C2C3 - "FCVTL2 v3.4s, v2.8h\n" - "FCVTL v2.4s, v2.4h\n" - "FCVTL2 v5.4s, v4.8h\n" - "FCVTL v4.4s, v4.4h\n" - "ZIP1 v16.4s, v0.4s, v4.4s\n" // q16=A0C0A1C1 - ASM_PREFETCH("[%[inptr0], #128]") - "LDR q6, [%[inptr3]], #16\n" // q6=D0D1D2D3 - "FCVTL2 v7.4s, v6.8h\n" - "FCVTL v6.4s, v6.4h\n" - "ZIP1 v17.4s, v2.4s, v6.4s\n" // q17=B0D0B1D1 - "LDR q8, [%[inptr4]], #16\n" - "LDR q10, [%[inptr5]], #16\n" - "FCVTL2 v9.4s, v8.8h\n" - "FCVTL v8.4s, v8.4h\n" - ASM_PREFETCH("[%[inptr1], #128]") - "LDR q12, [%[inptr6]], #16\n" - "FCVTL2 v11.4s, v10.8h\n" - "FCVTL v10.4s, v10.4h\n" - "FCVTL2 v13.4s, v12.8h\n" - "FCVTL v12.4s, v12.4h\n" - "ZIP1 v18.4s, v8.4s, v12.4s\n" - "LDR q14, [%[inptr7]], #16\n" - "FCVTL2 v15.4s, v14.8h\n" - "FCVTL v14.4s, v14.4h\n" - "ZIP1 v19.4s, v10.4s, v14.4s\n" - - ASM_PREFETCH("[%[inptr2], #128]") - "ZIP1 v20.4s, v16.4s, v17.4s\n" // q20=A0B0C0D0 - "ZIP1 v21.4s, v18.4s, v19.4s\n" - "ZIP2 v22.4s, v16.4s, v17.4s\n" - "ZIP2 v23.4s, v18.4s, v19.4s\n" - ASM_PREFETCH("[%[inptr3], #128]") - - "ZIP2 v16.4s, v0.4s, v4.4s\n" - "ZIP2 v17.4s, v2.4s, v6.4s\n" - "STP q20, q21, [%[outptr]], #32\n" // Write back the first element of each source - - "ZIP2 v18.4s, v8.4s, v12.4s\n" - ASM_PREFETCH("[%[inptr4], #128]") - "ZIP2 v19.4s, v10.4s, v14.4s\n" - "STP q22, q23, [%[outptr]], #32\n" // Write back the second element of each source - - "ZIP1 v20.4s, v16.4s, v17.4s\n" - "ZIP1 v21.4s, v18.4s, v19.4s\n" - ASM_PREFETCH("[%[inptr5], #128]") - "ZIP2 v22.4s, v16.4s, v17.4s\n" - "ZIP2 v23.4s, v18.4s, v19.4s\n" - - "ZIP1 v16.4s, v1.4s, v5.4s\n" - "ZIP1 v17.4s, v3.4s, v7.4s\n" - ASM_PREFETCH("[%[inptr6], #128]") - "STP q20, q21, [%[outptr]], #32\n" // Third element - - "ZIP1 v18.4s, v9.4s, v13.4s\n" - "ZIP1 v19.4s, v11.4s, v15.4s\n" - "STP q22, q23, [%[outptr]], #32\n" // Fourth element - ASM_PREFETCH("[%[inptr7], #128]") - - "ZIP1 v20.4s, v16.4s, v17.4s\n" - "ZIP1 v21.4s, v18.4s, v19.4s\n" - "ZIP2 v22.4s, v16.4s, v17.4s\n" - "ZIP2 v23.4s, v18.4s, v19.4s\n" - - "ZIP2 v16.4s, v1.4s, v5.4s\n" - "ZIP2 v17.4s, v3.4s, v7.4s\n" - "STP q20, q21, [%[outptr]], #32\n" // Fifth element - - "ZIP2 v18.4s, v9.4s, v13.4s\n" - "ZIP2 v19.4s, v11.4s, v15.4s\n" - "STP q22, q23, [%[outptr]], #32\n" // Sixth element - - "ZIP1 v20.4s, v16.4s, v17.4s\n" - "ZIP1 v21.4s, v18.4s, v19.4s\n" - "STP q20, q21, [%[outptr]], #32\n" // Seventh element - - "ZIP2 v22.4s, v16.4s, v17.4s\n" - "ZIP2 v23.4s, v18.4s, v19.4s\n" - "STP q22, q23, [%[outptr]], #32\n" // Eighth element - : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), - [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7), [outptr] "+r" (outptr) - : - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "memory" - ); - } - - for (;x>0;x--) { - *outptr++ = *inptr0++; - *outptr++ = *inptr1++; - *outptr++ = *inptr2++; - *outptr++ = *inptr3++; - *outptr++ = *inptr4++; - *outptr++ = *inptr5++; - *outptr++ = *inptr6++; - *outptr++ = *inptr7++; - } - } -} - -#endif // __aarch64__ && __ARM_FP16_ARGS diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_s8_to_s16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_s8_to_s16.hpp deleted file mode 100644 index 37344a82a9..0000000000 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_s8_to_s16.hpp +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright (c) 2017-2020 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) - -#include <arm_neon.h> -#include <cstdint> - -#include "../asmlib.hpp" - -template<> -template<> -inline void TransformImpl<8, 1, false, 2, 1, false>::Transform(int16_t *out, const int8_t *in, int ldin, int y0, int ymax, int k0, int kmax) { - int16_t *outptr = out; - const int8_t *inptr = in; - bool first = true; - - int8_t zerobuff[32] = { 0 }; // 16 for asm loop plus up to 15 for overflow loop - - for (int y=y0; y<ymax; y+=8) { - const int8_t *inptr0 = inptr + y * ldin + k0; - const int8_t *inptr1 = inptr0 + ldin; - const int8_t *inptr2 = inptr1 + ldin; - const int8_t *inptr3 = inptr2 + ldin; - const int8_t *inptr4 = inptr3 + ldin; - const int8_t *inptr5 = inptr4 + ldin; - const int8_t *inptr6 = inptr5 + ldin; - const int8_t *inptr7 = inptr6 + ldin; - - prefetch_2x(inptr0); - prefetch_2x(inptr1); - prefetch_2x(inptr2); - prefetch_2x(inptr3); - prefetch_2x(inptr4); - prefetch_2x(inptr5); - prefetch_2x(inptr6); - prefetch_2x(inptr7); - - int x=(kmax-k0); - for (;(x>15) || first;x-=16) { - /* Cope with ragged cases by copying from a buffer of zeroes instead */ - /* 'first' forces this to always run at least once, needed if the total size is <=7. */ - if ((y + 7) >= ymax) { - switch ((y + 7) - ymax) { - case 6: - inptr1 = zerobuff; - // fall through - case 5: - inptr2 = zerobuff; - // fall through - case 4: - inptr3 = zerobuff; - // fall through - case 3: - inptr4 = zerobuff; - // fall through - case 2: - inptr5 = zerobuff; - // fall through - case 1: - inptr6 = zerobuff; - // fall through - case 0: - inptr7 = zerobuff; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - if (first) { - if (x<=15) { - break; - } - - first = false; - } - - __asm __volatile ( - // Load up 16 elements (1 source vector, 2 destination vectors) from each of 8 sources. - "LDR q0, [%[inptr0]], #16\n" - "LDR q2, [%[inptr1]], #16\n" - "SSHLL2 v1.8h, v0.16b, #0\n" - "SSHLL v0.8h, v0.8b, #0\n" - "LDR q4, [%[inptr2]], #16\n" // q4=C0C1C2C3 - "SSHLL2 v3.8h, v2.16b, #0\n" - "SSHLL v2.8h, v2.8b, #0\n" - "SSHLL2 v5.8h, v4.16b, #0\n" - "SSHLL v4.8h, v4.8b, #0\n" - "ZIP1 v16.8h, v0.8h, v4.8h\n" // q16=A0C0A1C1 - ASM_PREFETCH("[%[inptr0], #128]") - "LDR q6, [%[inptr3]], #16\n" // q6=D0D1D2D3 - "SSHLL2 v7.8h, v6.16b, #0\n" - "SSHLL v6.8h, v6.8b, #0\n" - "ZIP1 v17.8h, v2.8h, v6.8h\n" // q17=B0D0B1D1 - "LDR q8, [%[inptr4]], #16\n" - "LDR q10, [%[inptr5]], #16\n" - "SSHLL2 v9.8h, v8.16b, #0\n" - "SSHLL v8.8h, v8.8b, #0\n" - ASM_PREFETCH("[%[inptr1], #128]") - "LDR q12, [%[inptr6]], #16\n" - "SSHLL2 v11.8h, v10.16b, #0\n" - "SSHLL v10.8h, v10.8b, #0\n" - "SSHLL2 v13.8h, v12.16b, #0\n" - "SSHLL v12.8h, v12.8b, #0\n" - "ZIP1 v18.8h, v8.8h, v12.8h\n" - "LDR q14, [%[inptr7]], #16\n" - "SSHLL2 v15.8h, v14.16b, #0\n" - "SSHLL v14.8h, v14.8b, #0\n" - "ZIP1 v19.8h, v10.8h, v14.8h\n" - - ASM_PREFETCH("[%[inptr2], #128]") - "ZIP1 v20.8h, v16.8h, v17.8h\n" // q20=A0B0C0D0A1B1C1D1 - "ZIP1 v21.8h, v18.8h, v19.8h\n" // q21=E0F0G0H0E1F1G1H1 - "ZIP2 v22.8h, v16.8h, v17.8h\n" // q22=A2B2C2D2A3B3C3D3 - "ZIP2 v23.8h, v18.8h, v19.8h\n" // q23=E2F2G2H1E3F3G3H3 - ASM_PREFETCH("[%[inptr3], #128]") - - "ZIP2 v16.8h, v0.8h, v4.8h\n" - "ZIP2 v17.8h, v2.8h, v6.8h\n" - "TRN1 v24.2d, v20.2d, v21.2d\n" - "TRN2 v25.2d, v20.2d, v21.2d\n" - - "ZIP2 v18.8h, v8.8h, v12.8h\n" - ASM_PREFETCH("[%[inptr4], #128]") - "ZIP2 v19.8h, v10.8h, v14.8h\n" - "STP q24, q25, [%[outptr]], #32\n" // Write back the first element of each source - "TRN1 v24.2d, v22.2d, v23.2d\n" - "TRN2 v25.2d, v22.2d, v23.2d\n" - - "ZIP1 v20.8h, v16.8h, v17.8h\n" - "ZIP1 v21.8h, v18.8h, v19.8h\n" - ASM_PREFETCH("[%[inptr5], #128]") - "ZIP2 v22.8h, v16.8h, v17.8h\n" - "ZIP2 v23.8h, v18.8h, v19.8h\n" - "STP q24, q25, [%[outptr]], #32\n" // Write back the second element of each source - - "ZIP1 v16.8h, v1.8h, v5.8h\n" - "ZIP1 v17.8h, v3.8h, v7.8h\n" - ASM_PREFETCH("[%[inptr6], #128]") - "TRN1 v24.2d, v20.2d, v21.2d\n" - "TRN2 v25.2d, v20.2d, v21.2d\n" - - "ZIP1 v18.8h, v9.8h, v13.8h\n" - "ZIP1 v19.8h, v11.8h, v15.8h\n" - "STP q24, q25, [%[outptr]], #32\n" // Third element - "TRN1 v24.2d, v22.2d, v23.2d\n" - "TRN2 v25.2d, v22.2d, v23.2d\n" - ASM_PREFETCH("[%[inptr7], #128]") - - "ZIP1 v20.8h, v16.8h, v17.8h\n" - "ZIP1 v21.8h, v18.8h, v19.8h\n" - "STP q24, q25, [%[outptr]], #32\n" // Fourth element - "ZIP2 v22.8h, v16.8h, v17.8h\n" - "ZIP2 v23.8h, v18.8h, v19.8h\n" - - "ZIP2 v16.8h, v1.8h, v5.8h\n" - "ZIP2 v17.8h, v3.8h, v7.8h\n" - "TRN1 v24.2d, v20.2d, v21.2d\n" - "TRN2 v25.2d, v20.2d, v21.2d\n" - - "ZIP2 v18.8h, v9.8h, v13.8h\n" - "ZIP2 v19.8h, v11.8h, v15.8h\n" - "STP q24, q25, [%[outptr]], #32\n" // Fifth element - "TRN1 v24.2d, v22.2d, v23.2d\n" - "TRN2 v25.2d, v22.2d, v23.2d\n" - - "ZIP1 v20.8h, v16.8h, v17.8h\n" - "ZIP1 v21.8h, v18.8h, v19.8h\n" - "STP q24, q25, [%[outptr]], #32\n" // Sixth element - "TRN1 v24.2d, v20.2d, v21.2d\n" - "TRN2 v25.2d, v20.2d, v21.2d\n" - - "ZIP2 v22.8h, v16.8h, v17.8h\n" - "ZIP2 v23.8h, v18.8h, v19.8h\n" - "STP q24, q25, [%[outptr]], #32\n" // Seventh element - "TRN1 v24.2d, v22.2d, v23.2d\n" - "TRN2 v25.2d, v22.2d, v23.2d\n" - "STP q24, q25, [%[outptr]], #32\n" // Eighth element - : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), - [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7), [outptr] "+r" (outptr) - : - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "memory" - ); - } - - for (;x>0;x--) { - *outptr++ = *inptr0++; - *outptr++ = *inptr1++; - *outptr++ = *inptr2++; - *outptr++ = *inptr3++; - *outptr++ = *inptr4++; - *outptr++ = *inptr5++; - *outptr++ = *inptr6++; - *outptr++ = *inptr7++; - } - } -} - -#endif // __aarch64__ && __ARM_FP16_ARGS diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_u8_to_u16.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_u8_to_u16.hpp deleted file mode 100644 index a3a269c9cd..0000000000 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_interleave_8way_u8_to_u16.hpp +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright (c) 2017-2020 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) - -#include <arm_neon.h> -#include <cstdint> - -#include "../asmlib.hpp" - -template<> -template<> -inline void TransformImpl<8, 1, false, 2, 1, false>::Transform(uint16_t *out, const uint8_t *in, int ldin, int y0, int ymax, int k0, int kmax) { - uint16_t *outptr = out; - const uint8_t *inptr = in; - bool first = true; - - uint8_t zerobuff[32] = { 0 }; // 16 for asm loop plus up to 15 for overflow loop - - for (int y=y0; y<ymax; y+=8) { - const uint8_t *inptr0 = inptr + y * ldin + k0; - const uint8_t *inptr1 = inptr0 + ldin; - const uint8_t *inptr2 = inptr1 + ldin; - const uint8_t *inptr3 = inptr2 + ldin; - const uint8_t *inptr4 = inptr3 + ldin; - const uint8_t *inptr5 = inptr4 + ldin; - const uint8_t *inptr6 = inptr5 + ldin; - const uint8_t *inptr7 = inptr6 + ldin; - - prefetch_2x(inptr0); - prefetch_2x(inptr1); - prefetch_2x(inptr2); - prefetch_2x(inptr3); - prefetch_2x(inptr4); - prefetch_2x(inptr5); - prefetch_2x(inptr6); - prefetch_2x(inptr7); - - int x=(kmax-k0); - for (;(x>15) || first;x-=16) { - /* Cope with ragged cases by copying from a buffer of zeroes instead */ - /* 'first' forces this to always run at least once, needed if the total size is <=7. */ - if ((y + 7) >= ymax) { - switch ((y + 7) - ymax) { - case 6: - inptr1 = zerobuff; - // fall through - case 5: - inptr2 = zerobuff; - // fall through - case 4: - inptr3 = zerobuff; - // fall through - case 3: - inptr4 = zerobuff; - // fall through - case 2: - inptr5 = zerobuff; - // fall through - case 1: - inptr6 = zerobuff; - // fall through - case 0: - inptr7 = zerobuff; - break; - - default: - UNREACHABLE("Impossible."); - } - } - - if (first) { - if (x<=15) { - break; - } - - first = false; - } - - __asm __volatile ( - // Load up 16 elements (1 source vector, 2 destination vectors) from each of 8 sources. - "LDR q0, [%[inptr0]], #16\n" - "LDR q2, [%[inptr1]], #16\n" - "USHLL2 v1.8h, v0.16b, #0\n" - "USHLL v0.8h, v0.8b, #0\n" - "LDR q4, [%[inptr2]], #16\n" // q4=C0C1C2C3 - "USHLL2 v3.8h, v2.16b, #0\n" - "USHLL v2.8h, v2.8b, #0\n" - "USHLL2 v5.8h, v4.16b, #0\n" - "USHLL v4.8h, v4.8b, #0\n" - "ZIP1 v16.8h, v0.8h, v4.8h\n" // q16=A0C0A1C1 - ASM_PREFETCH("[%[inptr0], #128]") - "LDR q6, [%[inptr3]], #16\n" // q6=D0D1D2D3 - "USHLL2 v7.8h, v6.16b, #0\n" - "USHLL v6.8h, v6.8b, #0\n" - "ZIP1 v17.8h, v2.8h, v6.8h\n" // q17=B0D0B1D1 - "LDR q8, [%[inptr4]], #16\n" - "LDR q10, [%[inptr5]], #16\n" - "USHLL2 v9.8h, v8.16b, #0\n" - "USHLL v8.8h, v8.8b, #0\n" - ASM_PREFETCH("[%[inptr1], #128]") - "LDR q12, [%[inptr6]], #16\n" - "USHLL2 v11.8h, v10.16b, #0\n" - "USHLL v10.8h, v10.8b, #0\n" - "USHLL2 v13.8h, v12.16b, #0\n" - "USHLL v12.8h, v12.8b, #0\n" - "ZIP1 v18.8h, v8.8h, v12.8h\n" - "LDR q14, [%[inptr7]], #16\n" - "USHLL2 v15.8h, v14.16b, #0\n" - "USHLL v14.8h, v14.8b, #0\n" - "ZIP1 v19.8h, v10.8h, v14.8h\n" - - ASM_PREFETCH("[%[inptr2], #128]") - "ZIP1 v20.8h, v16.8h, v17.8h\n" // q20=A0B0C0D0A1B1C1D1 - "ZIP1 v21.8h, v18.8h, v19.8h\n" // q21=E0F0G0H0E1F1G1H1 - "ZIP2 v22.8h, v16.8h, v17.8h\n" // q22=A2B2C2D2A3B3C3D3 - "ZIP2 v23.8h, v18.8h, v19.8h\n" // q23=E2F2G2H1E3F3G3H3 - ASM_PREFETCH("[%[inptr3], #128]") - - "ZIP2 v16.8h, v0.8h, v4.8h\n" - "ZIP2 v17.8h, v2.8h, v6.8h\n" - "TRN1 v24.2d, v20.2d, v21.2d\n" - "TRN2 v25.2d, v20.2d, v21.2d\n" - - "ZIP2 v18.8h, v8.8h, v12.8h\n" - ASM_PREFETCH("[%[inptr4], #128]") - "ZIP2 v19.8h, v10.8h, v14.8h\n" - "STP q24, q25, [%[outptr]], #32\n" // Write back the first element of each source - "TRN1 v24.2d, v22.2d, v23.2d\n" - "TRN2 v25.2d, v22.2d, v23.2d\n" - - "ZIP1 v20.8h, v16.8h, v17.8h\n" - "ZIP1 v21.8h, v18.8h, v19.8h\n" - ASM_PREFETCH("[%[inptr5], #128]") - "ZIP2 v22.8h, v16.8h, v17.8h\n" - "ZIP2 v23.8h, v18.8h, v19.8h\n" - "STP q24, q25, [%[outptr]], #32\n" // Write back the second element of each source - - "ZIP1 v16.8h, v1.8h, v5.8h\n" - "ZIP1 v17.8h, v3.8h, v7.8h\n" - ASM_PREFETCH("[%[inptr6], #128]") - "TRN1 v24.2d, v20.2d, v21.2d\n" - "TRN2 v25.2d, v20.2d, v21.2d\n" - - "ZIP1 v18.8h, v9.8h, v13.8h\n" - "ZIP1 v19.8h, v11.8h, v15.8h\n" - "STP q24, q25, [%[outptr]], #32\n" // Third element - "TRN1 v24.2d, v22.2d, v23.2d\n" - "TRN2 v25.2d, v22.2d, v23.2d\n" - ASM_PREFETCH("[%[inptr7], #128]") - - "ZIP1 v20.8h, v16.8h, v17.8h\n" - "ZIP1 v21.8h, v18.8h, v19.8h\n" - "STP q24, q25, [%[outptr]], #32\n" // Fourth element - "ZIP2 v22.8h, v16.8h, v17.8h\n" - "ZIP2 v23.8h, v18.8h, v19.8h\n" - - "ZIP2 v16.8h, v1.8h, v5.8h\n" - "ZIP2 v17.8h, v3.8h, v7.8h\n" - "TRN1 v24.2d, v20.2d, v21.2d\n" - "TRN2 v25.2d, v20.2d, v21.2d\n" - - "ZIP2 v18.8h, v9.8h, v13.8h\n" - "ZIP2 v19.8h, v11.8h, v15.8h\n" - "STP q24, q25, [%[outptr]], #32\n" // Fifth element - "TRN1 v24.2d, v22.2d, v23.2d\n" - "TRN2 v25.2d, v22.2d, v23.2d\n" - - "ZIP1 v20.8h, v16.8h, v17.8h\n" - "ZIP1 v21.8h, v18.8h, v19.8h\n" - "STP q24, q25, [%[outptr]], #32\n" // Sixth element - "TRN1 v24.2d, v20.2d, v21.2d\n" - "TRN2 v25.2d, v20.2d, v21.2d\n" - - "ZIP2 v22.8h, v16.8h, v17.8h\n" - "ZIP2 v23.8h, v18.8h, v19.8h\n" - "STP q24, q25, [%[outptr]], #32\n" // Seventh element - "TRN1 v24.2d, v22.2d, v23.2d\n" - "TRN2 v25.2d, v22.2d, v23.2d\n" - "STP q24, q25, [%[outptr]], #32\n" // Eighth element - : [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), - [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7), [outptr] "+r" (outptr) - : - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "memory" - ); - } - - for (;x>0;x--) { - *outptr++ = *inptr0++; - *outptr++ = *inptr1++; - *outptr++ = *inptr2++; - *outptr++ = *inptr3++; - *outptr++ = *inptr4++; - *outptr++ = *inptr5++; - *outptr++ = *inptr6++; - *outptr++ = *inptr7++; - } - } -} - -#endif // __aarch64__ && __ARM_FP16_ARGS diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_16bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_16bit.hpp index 5ab5774751..f6233ef503 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_16bit.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_16bit.hpp @@ -30,12 +30,12 @@ // Generic unblocked transposed 6x32-bit sized specialisation template <> template <typename T> -inline void TransformImpl<6, 1, true, 4, 4, false>::Transform( +inline void TransformImpl<6, 1, true, 4, 4, VLType::None>::Transform( T* out, const T* const in, const int stride, const int x0, const int xmax, const int k0, const int kmax ) { // Redirect to a 12 x uint16_t specialisation - TransformImpl<12, 1, true, 2, 2, false>::Transform( + TransformImpl<12, 1, true, 2, 2, VLType::None>::Transform( reinterpret_cast<uint16_t *>(out), reinterpret_cast<const uint16_t *>(in), stride*2, x0*2, xmax*2, k0, kmax @@ -45,7 +45,7 @@ inline void TransformImpl<6, 1, true, 4, 4, false>::Transform( // Generic 12x16-bit sized specialisation template <> template <typename T> -inline void TransformImpl<12, 1, true, 2, 2, false>::Transform( +inline void TransformImpl<12, 1, true, 2, 2, VLType::None>::Transform( T* out, const T* const in, const int stride, const int x0, const int xmax, const int k0, const int kmax ) { @@ -135,7 +135,7 @@ inline void TransposeInterleaveCommon<12, uint16_t, uint16_t>::moveblock_1x4(con template <> template <> -inline void TransformImpl<12, 1, true, 2, 2, false>::Transform( +inline void TransformImpl<12, 1, true, 2, 2, VLType::None>::Transform( uint16_t* out, const uint16_t* const in, const int stride, const int x0, const int xmax, const int k0, const int kmax ) { diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_half_to_float.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_half_to_float.hpp index d7de9ff934..c0f3e17d31 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_half_to_float.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_12way_half_to_float.hpp @@ -110,7 +110,7 @@ inline void TransposeInterleaveCommon<12, __fp16, float>::moveblock_1x4(const __ template <> template <> -inline void TransformImpl<12, 1, true, 4, 2, false>::Transform( +inline void TransformImpl<12, 1, true, 4, 2, VLType::None>::Transform( float* out, const __fp16* const in, const int stride, const int x0, const int xmax, const int k0, const int kmax ) { diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24way_16bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24way_16bit.hpp index a137f9360a..bcbe2b84d8 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24way_16bit.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_24way_16bit.hpp @@ -30,12 +30,12 @@ // Generic unblocked transposed 12x32-bit sized specialisation template <> template <typename T> -inline void TransformImpl<12, 1, true, 4, 4, false>::Transform( +inline void TransformImpl<12, 1, true, 4, 4, VLType::None>::Transform( T* out, const T* const in, const int stride, const int x0, const int xmax, const int k0, const int kmax ) { // Redirect to a 24 x uint16_t specialisation - TransformImpl<24, 1, true, 2, 2, false>::Transform( + TransformImpl<24, 1, true, 2, 2, VLType::None>::Transform( reinterpret_cast<uint16_t *>(out), reinterpret_cast<const uint16_t * const>(in), stride*2, x0*2, xmax*2, k0, kmax @@ -45,7 +45,7 @@ inline void TransformImpl<12, 1, true, 4, 4, false>::Transform( // Generic 24x16-bit sized specialisation template <> template <typename T> -inline void TransformImpl<24, 1, true, 2, 2, false>::Transform( +inline void TransformImpl<24, 1, true, 2, 2, VLType::None>::Transform( T* out, const T* const in, const int stride, const int x0, const int xmax, const int k0, const int kmax ) { @@ -120,7 +120,7 @@ inline void TransposeInterleaveCommon<24, uint16_t, uint16_t>::moveblock_1x4(con template <> template <> -inline void TransformImpl<24, 1, true, 2, 2, false>::Transform( +inline void TransformImpl<24, 1, true, 2, 2, VLType::None>::Transform( uint16_t* out, const uint16_t* const in, const int stride, const int x0, const int xmax, const int k0, const int kmax ) { diff --git a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_8way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_8way_32bit.hpp index 974be481e7..df68740bb4 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_8way_32bit.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/a64_transpose_interleave_8way_32bit.hpp @@ -30,12 +30,12 @@ // Generic unblocked transposed 8x32-bit sized specialisation template <> template <typename T> -inline void TransformImpl<8, 1, true, 4, 4, false>::Transform( +inline void TransformImpl<8, 1, true, 4, 4, VLType::None>::Transform( T* out, const T* const in, const int stride, const int x0, const int xmax, const int k0, const int kmax ) { // Redirect to a 16 x uint16_t specialisation - TransformImpl<16, 1, true, 2, 2, false>::Transform( + TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform( reinterpret_cast<uint16_t *>(out), reinterpret_cast<const uint16_t *>(in), stride*2, x0*2, xmax*2, k0, kmax @@ -45,7 +45,7 @@ inline void TransformImpl<8, 1, true, 4, 4, false>::Transform( // Generic 16x16-bit sized specialisation template <> template <typename T> -inline void TransformImpl<16, 1, true, 2, 2, false>::Transform( +inline void TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform( T* out, const T* const in, const int stride, const int x0, const int xmax, const int k0, const int kmax ) { @@ -137,7 +137,7 @@ inline void TransposeInterleaveCommon<16, uint16_t, uint16_t>::moveblock_1x4(con template <> template <> -inline void TransformImpl<16, 1, true, 2, 2, false>::Transform( +inline void TransformImpl<16, 1, true, 2, 2, VLType::None>::Transform( uint16_t* out, const uint16_t* const in, const int stride, const int x0, const int xmax, const int k0, const int kmax ) { diff --git a/src/core/NEON/kernels/arm_gemm/transforms/list.hpp b/src/core/NEON/kernels/arm_gemm/transforms/list.hpp index b825e1c358..e092c729ba 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/list.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/list.hpp @@ -21,22 +21,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -#include "a32_interleave_6way_32bit.hpp" #include "a32_transpose_interleave_8way_32bit.hpp" -#include "a64_block16_interleave4_8bit.hpp" -#include "a64_interleave_8way_16bit.hpp" -#include "a64_interleave_8way_32bit.hpp" -#include "a64_interleave_8way_block4_8bit.hpp" -#include "a64_interleave_8way_half_to_float.hpp" -#include "a64_interleave_8way_s8_to_s16.hpp" -#include "a64_interleave_8way_u8_to_u16.hpp" #include "a64_transpose_interleave_12way_16bit.hpp" #include "a64_transpose_interleave_12way_half_to_float.hpp" #include "a64_transpose_interleave_24way_16bit.hpp" #include "a64_transpose_interleave_8way_32bit.hpp" -#include "sve_interleave_8way_32bit.hpp" -#include "sve_interleave_8way_block2_16bit.hpp" -#include "sve_interleave_8way_block2_32bit.hpp" -#include "sve_interleave_8way_block4_16bit.hpp" -#include "sve_interleave_8way_block4_8bit.hpp" -#include "sve_interleave_8way_block8_8bit.hpp" diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp deleted file mode 100644 index 348d78e3f5..0000000000 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_32bit.hpp +++ /dev/null @@ -1,596 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __ARM_FEATURE_SVE - -template<> -template<typename T> -inline void TransformImpl<8, 1, false, 4, 4, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) -{ - uint32_t *master_outptr = reinterpret_cast<uint32_t *>(out); - const uint32_t *inptr = reinterpret_cast<const uint32_t *>(in); - - for (int y=y0; y<ymax; y+=8) - { - const int height = ymax-y; - const long inwidth = (kmax - k0); - const long outwidth = inwidth * 8; - long inpos = 0; - long outpos = 0; - - uint32_t *outptr = master_outptr; - master_outptr += outwidth; - - const uint32_t *inptr0 = inptr + y * ldin + k0; - const uint32_t *inptr1 = inptr0 + ldin; - const uint32_t *inptr2 = inptr1 + ldin; - const uint32_t *inptr3 = inptr2 + ldin; - const uint32_t *inptr4 = inptr3 + ldin; - const uint32_t *inptr5 = inptr4 + ldin; - const uint32_t *inptr6 = inptr5 + ldin; - const uint32_t *inptr7 = inptr6 + ldin; - - switch(height) - { - case 1: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.s, #0\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "zip2 z9.s, z0.s, z4.s\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip1 z0.s, z8.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip2 z1.s, z8.s, z4.s\n" - "zip1 z2.s, z9.s, z4.s\n" - "zip2 z3.s, z9.s, z4.s\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z4.s\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z4.s\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip1 z12.s, z2.s, z4.s\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip2 z13.s, z2.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z4.s\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "zip2 z15.s, z3.s, z4.s\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 2: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.s, #0\n" - "mov z14.s, #0\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z4.s\n" - "zip2 z11.s, z1.s, z4.s\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip1 z0.s, z8.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip2 z1.s, z8.s, z4.s\n" - "zip1 z2.s, z9.s, z4.s\n" - "zip2 z3.s, z9.s, z4.s\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip1 z4.s, z10.s, z14.s\n" - "incw %[outpos], all, mul #1\n" - "zip2 z5.s, z10.s, z14.s\n" - "zip1 z6.s, z11.s, z14.s\n" - "zip2 z7.s, z11.s, z14.s\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "zip1 z12.s, z2.s, z6.s\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip2 z13.s, z2.s, z6.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z7.s\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "zip2 z15.s, z3.s, z7.s\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 3: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.s, #0\n" - "mov z14.s, #0\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n" - "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z4.s\n" - "zip2 z11.s, z1.s, z4.s\n" - "zip1 z12.s, z2.s, z4.s\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z0.s, z8.s, z12.s\n" - "zip2 z1.s, z8.s, z12.s\n" - "zip1 z2.s, z9.s, z13.s\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip2 z3.s, z9.s, z13.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z4.s, z10.s, z14.s\n" - "zip2 z5.s, z10.s, z14.s\n" - "zip1 z6.s, z11.s, z14.s\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip2 z7.s, z11.s, z14.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z5.s\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "zip2 z11.s, z1.s, z5.s\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip1 z12.s, z2.s, z6.s\n" - "incw %[outpos], all, mul #1\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "zip2 z15.s, z3.s, z7.s\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 4: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.s, #0\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n" - "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n" - "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z4.s\n" - "zip2 z11.s, z1.s, z4.s\n" - "zip1 z12.s, z2.s, z4.s\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z4.s\n" - "zip2 z15.s, z3.s, z4.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "incw %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 5: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z5.s, #0\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n" - "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n" - "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n" - "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z5.s\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z5.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z5.s\n" - "zip2 z15.s, z3.s, z5.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "incw %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 6: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z6.s, #0\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n" - "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n" - "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n" - "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n" - "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z5.s\n" - "zip2 z11.s, z1.s, z5.s\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z6.s\n" - "zip2 z15.s, z3.s, z6.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "incw %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 7: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z7.s, #0\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n" - "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n" - "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n" - "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n" - "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n" - "ld1w z6.s, p0/z, [%[inptr6], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z7.s\n" - "zip2 z15.s, z3.s, z7.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "incw %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - default: - case 8: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n" - "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n" - "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n" - "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n" - "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n" - "ld1w z6.s, p0/z, [%[inptr6], %[inpos], LSL #2]\n" - "ld1w z7.s, p0/z, [%[inptr7], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z7.s\n" - "zip2 z15.s, z3.s, z7.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "incw %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - - } - } -} - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_16bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_16bit.hpp deleted file mode 100644 index 234433a0f1..0000000000 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_16bit.hpp +++ /dev/null @@ -1,596 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __ARM_FEATURE_SVE - -template<> -template<typename T> -inline void TransformImpl<8, 2, false, 2, 2, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) -{ - uint16_t *master_outptr = reinterpret_cast<uint16_t *>(out); - const uint16_t *inptr = reinterpret_cast<const uint16_t *>(in); - - for (int y=y0; y<ymax; y+=8) - { - const int height = ymax-y; - const long inwidth = (kmax - k0); - const long outwidth = ((inwidth + 1) / 2) * 16; - long inpos = 0; - long outpos = 0; - - uint16_t *outptr = master_outptr; - master_outptr += outwidth; - - const uint16_t *inptr0 = inptr + y * ldin + k0; - const uint16_t *inptr1 = inptr0 + ldin; - const uint16_t *inptr2 = inptr1 + ldin; - const uint16_t *inptr3 = inptr2 + ldin; - const uint16_t *inptr4 = inptr3 + ldin; - const uint16_t *inptr5 = inptr4 + ldin; - const uint16_t *inptr6 = inptr5 + ldin; - const uint16_t *inptr7 = inptr6 + ldin; - - switch(height) - { - case 1: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.h, #0\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "zip2 z9.s, z0.s, z4.s\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip1 z0.s, z8.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip2 z1.s, z8.s, z4.s\n" - "zip1 z2.s, z9.s, z4.s\n" - "zip2 z3.s, z9.s, z4.s\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z4.s\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z4.s\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip1 z12.s, z2.s, z4.s\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip2 z13.s, z2.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z4.s\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "zip2 z15.s, z3.s, z4.s\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 2: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.h, #0\n" - "mov z14.h, #0\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z4.s\n" - "zip2 z11.s, z1.s, z4.s\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip1 z0.s, z8.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip2 z1.s, z8.s, z4.s\n" - "zip1 z2.s, z9.s, z4.s\n" - "zip2 z3.s, z9.s, z4.s\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip1 z4.s, z10.s, z14.s\n" - "inch %[outpos], all, mul #1\n" - "zip2 z5.s, z10.s, z14.s\n" - "zip1 z6.s, z11.s, z14.s\n" - "zip2 z7.s, z11.s, z14.s\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "zip1 z12.s, z2.s, z6.s\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip2 z13.s, z2.s, z6.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z7.s\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "zip2 z15.s, z3.s, z7.s\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 3: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.h, #0\n" - "mov z14.h, #0\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n" - "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z4.s\n" - "zip2 z11.s, z1.s, z4.s\n" - "zip1 z12.s, z2.s, z4.s\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z0.s, z8.s, z12.s\n" - "zip2 z1.s, z8.s, z12.s\n" - "zip1 z2.s, z9.s, z13.s\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip2 z3.s, z9.s, z13.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z4.s, z10.s, z14.s\n" - "zip2 z5.s, z10.s, z14.s\n" - "zip1 z6.s, z11.s, z14.s\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip2 z7.s, z11.s, z14.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z5.s\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "zip2 z11.s, z1.s, z5.s\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip1 z12.s, z2.s, z6.s\n" - "inch %[outpos], all, mul #1\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "zip2 z15.s, z3.s, z7.s\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 4: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.h, #0\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n" - "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n" - "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z4.s\n" - "zip2 z11.s, z1.s, z4.s\n" - "zip1 z12.s, z2.s, z4.s\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z4.s\n" - "zip2 z15.s, z3.s, z4.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "inch %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 5: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z5.h, #0\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n" - "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n" - "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n" - "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z5.s\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z5.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z5.s\n" - "zip2 z15.s, z3.s, z5.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "inch %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 6: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z6.h, #0\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n" - "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n" - "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n" - "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n" - "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z5.s\n" - "zip2 z11.s, z1.s, z5.s\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z6.s\n" - "zip2 z15.s, z3.s, z6.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "inch %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 7: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z7.h, #0\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n" - "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n" - "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n" - "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n" - "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n" - "ld1h z6.h, p0/z, [%[inptr6], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z7.s\n" - "zip2 z15.s, z3.s, z7.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "inch %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - default: - case 8: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n" - "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n" - "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n" - "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n" - "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n" - "ld1h z6.h, p0/z, [%[inptr6], %[inpos], LSL #1]\n" - "ld1h z7.h, p0/z, [%[inptr7], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z7.s\n" - "zip2 z15.s, z3.s, z7.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "inch %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - - } - } -} - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_32bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_32bit.hpp deleted file mode 100644 index f21933b8de..0000000000 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block2_32bit.hpp +++ /dev/null @@ -1,596 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __ARM_FEATURE_SVE - -template<> -template<typename T> -inline void TransformImpl<8, 2, false, 4, 4, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) -{ - uint32_t *master_outptr = reinterpret_cast<uint32_t *>(out); - const uint32_t *inptr = reinterpret_cast<const uint32_t *>(in); - - for (int y=y0; y<ymax; y+=8) - { - const int height = ymax-y; - const long inwidth = (kmax - k0); - const long outwidth = ((inwidth + 1) / 2) * 16; - long inpos = 0; - long outpos = 0; - - uint32_t *outptr = master_outptr; - master_outptr += outwidth; - - const uint32_t *inptr0 = inptr + y * ldin + k0; - const uint32_t *inptr1 = inptr0 + ldin; - const uint32_t *inptr2 = inptr1 + ldin; - const uint32_t *inptr3 = inptr2 + ldin; - const uint32_t *inptr4 = inptr3 + ldin; - const uint32_t *inptr5 = inptr4 + ldin; - const uint32_t *inptr6 = inptr5 + ldin; - const uint32_t *inptr7 = inptr6 + ldin; - - switch(height) - { - case 1: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.s, #0\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "zip2 z9.d, z0.d, z4.d\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip1 z0.d, z8.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip2 z1.d, z8.d, z4.d\n" - "zip1 z2.d, z9.d, z4.d\n" - "zip2 z3.d, z9.d, z4.d\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z4.d\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z4.d\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip1 z12.d, z2.d, z4.d\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip2 z13.d, z2.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z4.d\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "zip2 z15.d, z3.d, z4.d\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 2: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.s, #0\n" - "mov z14.s, #0\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z4.d\n" - "zip2 z11.d, z1.d, z4.d\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip1 z0.d, z8.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip2 z1.d, z8.d, z4.d\n" - "zip1 z2.d, z9.d, z4.d\n" - "zip2 z3.d, z9.d, z4.d\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip1 z4.d, z10.d, z14.d\n" - "incw %[outpos], all, mul #1\n" - "zip2 z5.d, z10.d, z14.d\n" - "zip1 z6.d, z11.d, z14.d\n" - "zip2 z7.d, z11.d, z14.d\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "zip1 z12.d, z2.d, z6.d\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip2 z13.d, z2.d, z6.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z7.d\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "zip2 z15.d, z3.d, z7.d\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 3: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.s, #0\n" - "mov z14.s, #0\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n" - "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z4.d\n" - "zip2 z11.d, z1.d, z4.d\n" - "zip1 z12.d, z2.d, z4.d\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z0.d, z8.d, z12.d\n" - "zip2 z1.d, z8.d, z12.d\n" - "zip1 z2.d, z9.d, z13.d\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip2 z3.d, z9.d, z13.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z4.d, z10.d, z14.d\n" - "zip2 z5.d, z10.d, z14.d\n" - "zip1 z6.d, z11.d, z14.d\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip2 z7.d, z11.d, z14.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z5.d\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "zip2 z11.d, z1.d, z5.d\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip1 z12.d, z2.d, z6.d\n" - "incw %[outpos], all, mul #1\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "zip2 z15.d, z3.d, z7.d\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 4: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.s, #0\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n" - "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n" - "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z4.d\n" - "zip2 z11.d, z1.d, z4.d\n" - "zip1 z12.d, z2.d, z4.d\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z4.d\n" - "zip2 z15.d, z3.d, z4.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "incw %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 5: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z5.s, #0\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n" - "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n" - "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n" - "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z5.d\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z5.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z5.d\n" - "zip2 z15.d, z3.d, z5.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "incw %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 6: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z6.s, #0\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n" - "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n" - "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n" - "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n" - "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z5.d\n" - "zip2 z11.d, z1.d, z5.d\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z6.d\n" - "zip2 z15.d, z3.d, z6.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "incw %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 7: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z7.s, #0\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n" - "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n" - "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n" - "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n" - "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n" - "ld1w z6.s, p0/z, [%[inptr6], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z7.d\n" - "zip2 z15.d, z3.d, z7.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "incw %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - default: - case 8: - __asm __volatile( - "1:\n" - "whilelt p0.s, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "ld1w z0.s, p0/z, [%[inptr0], %[inpos], LSL #2]\n" - "ld1w z1.s, p0/z, [%[inptr1], %[inpos], LSL #2]\n" - "ld1w z2.s, p0/z, [%[inptr2], %[inpos], LSL #2]\n" - "ld1w z3.s, p0/z, [%[inptr3], %[inpos], LSL #2]\n" - "ld1w z4.s, p0/z, [%[inptr4], %[inpos], LSL #2]\n" - "ld1w z5.s, p0/z, [%[inptr5], %[inpos], LSL #2]\n" - "ld1w z6.s, p0/z, [%[inptr6], %[inpos], LSL #2]\n" - "ld1w z7.s, p0/z, [%[inptr7], %[inpos], LSL #2]\n" - "incw %[inpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p0.s, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p1.s, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z7.d\n" - "zip2 z15.d, z3.d, z7.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.s, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.s, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.s, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incw %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1w z8.s, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.s, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1w z9.s, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "incw %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1w z10.s, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.s, %[outpos], %[outwidth]\n" - "st1w z11.s, p3, [%[outptr], #3, MUL VL]\n" - "incw %[outpos], all, mul #1\n" - "st1w z12.s, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.s, %[outpos], %[outwidth]\n" - "incw %[outpos], all, mul #1\n" - "st1w z13.s, p5, [%[outptr], #5, MUL VL]\n" - "st1w z14.s, p6, [%[outptr], #6, MUL VL]\n" - "st1w z15.s, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - - } - } -} - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_16bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_16bit.hpp deleted file mode 100644 index 26e10511a6..0000000000 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_16bit.hpp +++ /dev/null @@ -1,596 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __ARM_FEATURE_SVE - -template<> -template<typename T> -inline void TransformImpl<8, 4, false, 2, 2, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) -{ - uint16_t *master_outptr = reinterpret_cast<uint16_t *>(out); - const uint16_t *inptr = reinterpret_cast<const uint16_t *>(in); - - for (int y=y0; y<ymax; y+=8) - { - const int height = ymax-y; - const long inwidth = (kmax - k0); - const long outwidth = ((inwidth + 3) / 4) * 32; - long inpos = 0; - long outpos = 0; - - uint16_t *outptr = master_outptr; - master_outptr += outwidth; - - const uint16_t *inptr0 = inptr + y * ldin + k0; - const uint16_t *inptr1 = inptr0 + ldin; - const uint16_t *inptr2 = inptr1 + ldin; - const uint16_t *inptr3 = inptr2 + ldin; - const uint16_t *inptr4 = inptr3 + ldin; - const uint16_t *inptr5 = inptr4 + ldin; - const uint16_t *inptr6 = inptr5 + ldin; - const uint16_t *inptr7 = inptr6 + ldin; - - switch(height) - { - case 1: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.h, #0\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "zip2 z9.d, z0.d, z4.d\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip1 z0.d, z8.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip2 z1.d, z8.d, z4.d\n" - "zip1 z2.d, z9.d, z4.d\n" - "zip2 z3.d, z9.d, z4.d\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z4.d\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z4.d\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip1 z12.d, z2.d, z4.d\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip2 z13.d, z2.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z4.d\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "zip2 z15.d, z3.d, z4.d\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 2: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.h, #0\n" - "mov z14.h, #0\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z4.d\n" - "zip2 z11.d, z1.d, z4.d\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip1 z0.d, z8.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip2 z1.d, z8.d, z4.d\n" - "zip1 z2.d, z9.d, z4.d\n" - "zip2 z3.d, z9.d, z4.d\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip1 z4.d, z10.d, z14.d\n" - "inch %[outpos], all, mul #1\n" - "zip2 z5.d, z10.d, z14.d\n" - "zip1 z6.d, z11.d, z14.d\n" - "zip2 z7.d, z11.d, z14.d\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "zip1 z12.d, z2.d, z6.d\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip2 z13.d, z2.d, z6.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z7.d\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "zip2 z15.d, z3.d, z7.d\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 3: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.h, #0\n" - "mov z14.h, #0\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n" - "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z4.d\n" - "zip2 z11.d, z1.d, z4.d\n" - "zip1 z12.d, z2.d, z4.d\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z0.d, z8.d, z12.d\n" - "zip2 z1.d, z8.d, z12.d\n" - "zip1 z2.d, z9.d, z13.d\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip2 z3.d, z9.d, z13.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z4.d, z10.d, z14.d\n" - "zip2 z5.d, z10.d, z14.d\n" - "zip1 z6.d, z11.d, z14.d\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip2 z7.d, z11.d, z14.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z5.d\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "zip2 z11.d, z1.d, z5.d\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip1 z12.d, z2.d, z6.d\n" - "inch %[outpos], all, mul #1\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "zip2 z15.d, z3.d, z7.d\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 4: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.h, #0\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n" - "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n" - "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z4.d\n" - "zip2 z11.d, z1.d, z4.d\n" - "zip1 z12.d, z2.d, z4.d\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z4.d\n" - "zip2 z15.d, z3.d, z4.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "inch %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 5: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z5.h, #0\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n" - "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n" - "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n" - "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z5.d\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z5.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z5.d\n" - "zip2 z15.d, z3.d, z5.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "inch %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 6: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z6.h, #0\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n" - "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n" - "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n" - "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n" - "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z5.d\n" - "zip2 z11.d, z1.d, z5.d\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z6.d\n" - "zip2 z15.d, z3.d, z6.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "inch %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 7: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z7.h, #0\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n" - "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n" - "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n" - "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n" - "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n" - "ld1h z6.h, p0/z, [%[inptr6], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z7.d\n" - "zip2 z15.d, z3.d, z7.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "inch %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - default: - case 8: - __asm __volatile( - "1:\n" - "whilelt p0.h, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "ld1h z0.h, p0/z, [%[inptr0], %[inpos], LSL #1]\n" - "ld1h z1.h, p0/z, [%[inptr1], %[inpos], LSL #1]\n" - "ld1h z2.h, p0/z, [%[inptr2], %[inpos], LSL #1]\n" - "ld1h z3.h, p0/z, [%[inptr3], %[inpos], LSL #1]\n" - "ld1h z4.h, p0/z, [%[inptr4], %[inpos], LSL #1]\n" - "ld1h z5.h, p0/z, [%[inptr5], %[inpos], LSL #1]\n" - "ld1h z6.h, p0/z, [%[inptr6], %[inpos], LSL #1]\n" - "ld1h z7.h, p0/z, [%[inptr7], %[inpos], LSL #1]\n" - "inch %[inpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p0.h, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p1.h, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z7.d\n" - "zip2 z15.d, z3.d, z7.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.h, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.h, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.h, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "inch %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1h z8.h, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.h, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1h z9.h, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "inch %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1h z10.h, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.h, %[outpos], %[outwidth]\n" - "st1h z11.h, p3, [%[outptr], #3, MUL VL]\n" - "inch %[outpos], all, mul #1\n" - "st1h z12.h, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.h, %[outpos], %[outwidth]\n" - "inch %[outpos], all, mul #1\n" - "st1h z13.h, p5, [%[outptr], #5, MUL VL]\n" - "st1h z14.h, p6, [%[outptr], #6, MUL VL]\n" - "st1h z15.h, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - - } - } -} - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_8bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_8bit.hpp deleted file mode 100644 index ed0d58aa91..0000000000 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block4_8bit.hpp +++ /dev/null @@ -1,596 +0,0 @@ -/* - * Copyright (c) 2018-2020 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __ARM_FEATURE_SVE - -template<> -template<typename T> -inline void TransformImpl<8, 4, false, 1, 1, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) -{ - uint8_t *master_outptr = reinterpret_cast<uint8_t *>(out); - const uint8_t *inptr = reinterpret_cast<const uint8_t *>(in); - - for (int y=y0; y<ymax; y+=8) - { - const int height = ymax-y; - const long inwidth = (kmax - k0); - const long outwidth = ((inwidth + 3) / 4) * 32; - long inpos = 0; - long outpos = 0; - - uint8_t *outptr = master_outptr; - master_outptr += outwidth; - - const uint8_t *inptr0 = inptr + y * ldin + k0; - const uint8_t *inptr1 = inptr0 + ldin; - const uint8_t *inptr2 = inptr1 + ldin; - const uint8_t *inptr3 = inptr2 + ldin; - const uint8_t *inptr4 = inptr3 + ldin; - const uint8_t *inptr5 = inptr4 + ldin; - const uint8_t *inptr6 = inptr5 + ldin; - const uint8_t *inptr7 = inptr6 + ldin; - - switch(height) - { - case 1: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.b, #0\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "zip2 z9.s, z0.s, z4.s\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip1 z0.s, z8.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip2 z1.s, z8.s, z4.s\n" - "zip1 z2.s, z9.s, z4.s\n" - "zip2 z3.s, z9.s, z4.s\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z4.s\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z4.s\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip1 z12.s, z2.s, z4.s\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip2 z13.s, z2.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z4.s\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "zip2 z15.s, z3.s, z4.s\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 2: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.b, #0\n" - "mov z14.b, #0\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z4.s\n" - "zip2 z11.s, z1.s, z4.s\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip1 z0.s, z8.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip2 z1.s, z8.s, z4.s\n" - "zip1 z2.s, z9.s, z4.s\n" - "zip2 z3.s, z9.s, z4.s\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip1 z4.s, z10.s, z14.s\n" - "incb %[outpos], all, mul #1\n" - "zip2 z5.s, z10.s, z14.s\n" - "zip1 z6.s, z11.s, z14.s\n" - "zip2 z7.s, z11.s, z14.s\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "zip1 z12.s, z2.s, z6.s\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip2 z13.s, z2.s, z6.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z7.s\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "zip2 z15.s, z3.s, z7.s\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 3: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.b, #0\n" - "mov z14.b, #0\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n" - "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z4.s\n" - "zip2 z11.s, z1.s, z4.s\n" - "zip1 z12.s, z2.s, z4.s\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z0.s, z8.s, z12.s\n" - "zip2 z1.s, z8.s, z12.s\n" - "zip1 z2.s, z9.s, z13.s\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip2 z3.s, z9.s, z13.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z4.s, z10.s, z14.s\n" - "zip2 z5.s, z10.s, z14.s\n" - "zip1 z6.s, z11.s, z14.s\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip2 z7.s, z11.s, z14.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z5.s\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "zip2 z11.s, z1.s, z5.s\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip1 z12.s, z2.s, z6.s\n" - "incb %[outpos], all, mul #1\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "zip2 z15.s, z3.s, z7.s\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 4: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.b, #0\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n" - "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n" - "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z4.s\n" - "zip2 z11.s, z1.s, z4.s\n" - "zip1 z12.s, z2.s, z4.s\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z4.s\n" - "zip2 z15.s, z3.s, z4.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "incb %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 5: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z5.b, #0\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n" - "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n" - "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n" - "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z5.s\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z5.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z5.s\n" - "zip2 z15.s, z3.s, z5.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "incb %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 6: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z6.b, #0\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n" - "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n" - "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n" - "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n" - "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "zip1 z8.s, z0.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip2 z9.s, z0.s, z4.s\n" - "zip1 z10.s, z1.s, z5.s\n" - "zip2 z11.s, z1.s, z5.s\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z6.s\n" - "zip2 z15.s, z3.s, z6.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "incb %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 7: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z7.b, #0\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n" - "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n" - "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n" - "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n" - "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n" - "ld1b z6.b, p0/z, [%[inptr6], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z7.s\n" - "zip2 z15.s, z3.s, z7.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "incb %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - default: - case 8: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n" - "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n" - "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n" - "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n" - "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n" - "ld1b z6.b, p0/z, [%[inptr6], %[inpos]]\n" - "ld1b z7.b, p0/z, [%[inptr7], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z14.s, z3.s, z7.s\n" - "zip2 z15.s, z3.s, z7.s\n" - "zip1 z0.s, z8.s, z12.s\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip2 z1.s, z8.s, z12.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z2.s, z9.s, z13.s\n" - "zip2 z3.s, z9.s, z13.s\n" - "zip1 z4.s, z10.s, z14.s\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip2 z5.s, z10.s, z14.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z6.s, z11.s, z15.s\n" - "zip2 z7.s, z11.s, z15.s\n" - "zip1 z8.s, z0.s, z4.s\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "zip2 z9.s, z0.s, z4.s\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.s, z1.s, z5.s\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip2 z11.s, z1.s, z5.s\n" - "zip1 z12.s, z2.s, z6.s\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "zip2 z13.s, z2.s, z6.s\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.s, z3.s, z7.s\n" - "incb %[outpos], all, mul #1\n" - "zip2 z15.s, z3.s, z7.s\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - - } - } -} - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block8_8bit.hpp b/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block8_8bit.hpp deleted file mode 100644 index b4935e6417..0000000000 --- a/src/core/NEON/kernels/arm_gemm/transforms/sve_interleave_8way_block8_8bit.hpp +++ /dev/null @@ -1,596 +0,0 @@ -/* - * Copyright (c) 2019 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once - -#ifdef __ARM_FEATURE_SVE - -template<> -template<typename T> -inline void TransformImpl<8, 8, false, 1, 1, false>::Transform(T *out, const T *in, int ldin, int y0, int ymax, int k0, int kmax) -{ - uint8_t *master_outptr = reinterpret_cast<uint8_t *>(out); - const uint8_t *inptr = reinterpret_cast<const uint8_t *>(in); - - for (int y=y0; y<ymax; y+=8) - { - const int height = ymax-y; - const long inwidth = (kmax - k0); - const long outwidth = ((inwidth + 7) / 8) * 64; - long inpos = 0; - long outpos = 0; - - uint8_t *outptr = master_outptr; - master_outptr += outwidth; - - const uint8_t *inptr0 = inptr + y * ldin + k0; - const uint8_t *inptr1 = inptr0 + ldin; - const uint8_t *inptr2 = inptr1 + ldin; - const uint8_t *inptr3 = inptr2 + ldin; - const uint8_t *inptr4 = inptr3 + ldin; - const uint8_t *inptr5 = inptr4 + ldin; - const uint8_t *inptr6 = inptr5 + ldin; - const uint8_t *inptr7 = inptr6 + ldin; - - switch(height) - { - case 1: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.b, #0\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "zip2 z9.d, z0.d, z4.d\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip1 z0.d, z8.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip2 z1.d, z8.d, z4.d\n" - "zip1 z2.d, z9.d, z4.d\n" - "zip2 z3.d, z9.d, z4.d\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z4.d\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z4.d\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip1 z12.d, z2.d, z4.d\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip2 z13.d, z2.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z4.d\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "zip2 z15.d, z3.d, z4.d\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 2: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.b, #0\n" - "mov z14.b, #0\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z4.d\n" - "zip2 z11.d, z1.d, z4.d\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip1 z0.d, z8.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip2 z1.d, z8.d, z4.d\n" - "zip1 z2.d, z9.d, z4.d\n" - "zip2 z3.d, z9.d, z4.d\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip1 z4.d, z10.d, z14.d\n" - "incb %[outpos], all, mul #1\n" - "zip2 z5.d, z10.d, z14.d\n" - "zip1 z6.d, z11.d, z14.d\n" - "zip2 z7.d, z11.d, z14.d\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "zip1 z12.d, z2.d, z6.d\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip2 z13.d, z2.d, z6.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z7.d\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "zip2 z15.d, z3.d, z7.d\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 3: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.b, #0\n" - "mov z14.b, #0\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n" - "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z4.d\n" - "zip2 z11.d, z1.d, z4.d\n" - "zip1 z12.d, z2.d, z4.d\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z0.d, z8.d, z12.d\n" - "zip2 z1.d, z8.d, z12.d\n" - "zip1 z2.d, z9.d, z13.d\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip2 z3.d, z9.d, z13.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z4.d, z10.d, z14.d\n" - "zip2 z5.d, z10.d, z14.d\n" - "zip1 z6.d, z11.d, z14.d\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip2 z7.d, z11.d, z14.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z5.d\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "zip2 z11.d, z1.d, z5.d\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip1 z12.d, z2.d, z6.d\n" - "incb %[outpos], all, mul #1\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "zip2 z15.d, z3.d, z7.d\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 4: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z4.b, #0\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n" - "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n" - "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z4.d\n" - "zip2 z11.d, z1.d, z4.d\n" - "zip1 z12.d, z2.d, z4.d\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z4.d\n" - "zip2 z15.d, z3.d, z4.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "incb %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 5: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z5.b, #0\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n" - "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n" - "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n" - "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z5.d\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z5.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z5.d\n" - "zip2 z15.d, z3.d, z5.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "incb %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 6: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z6.b, #0\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n" - "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n" - "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n" - "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n" - "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "zip1 z8.d, z0.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip2 z9.d, z0.d, z4.d\n" - "zip1 z10.d, z1.d, z5.d\n" - "zip2 z11.d, z1.d, z5.d\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z6.d\n" - "zip2 z15.d, z3.d, z6.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "incb %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - case 7: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "mov z7.b, #0\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n" - "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n" - "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n" - "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n" - "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n" - "ld1b z6.b, p0/z, [%[inptr6], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z7.d\n" - "zip2 z15.d, z3.d, z7.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "incb %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - default: - case 8: - __asm __volatile( - "1:\n" - "whilelt p0.b, %[inpos], %[inwidth]\n" - "b.none 2f\n" - "ld1b z0.b, p0/z, [%[inptr0], %[inpos]]\n" - "ld1b z1.b, p0/z, [%[inptr1], %[inpos]]\n" - "ld1b z2.b, p0/z, [%[inptr2], %[inpos]]\n" - "ld1b z3.b, p0/z, [%[inptr3], %[inpos]]\n" - "ld1b z4.b, p0/z, [%[inptr4], %[inpos]]\n" - "ld1b z5.b, p0/z, [%[inptr5], %[inpos]]\n" - "ld1b z6.b, p0/z, [%[inptr6], %[inpos]]\n" - "ld1b z7.b, p0/z, [%[inptr7], %[inpos]]\n" - "incb %[inpos], all, mul #1\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p0.b, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p1.b, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z14.d, z3.d, z7.d\n" - "zip2 z15.d, z3.d, z7.d\n" - "zip1 z0.d, z8.d, z12.d\n" - "whilelt p2.b, %[outpos], %[outwidth]\n" - "zip2 z1.d, z8.d, z12.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z2.d, z9.d, z13.d\n" - "zip2 z3.d, z9.d, z13.d\n" - "zip1 z4.d, z10.d, z14.d\n" - "whilelt p3.b, %[outpos], %[outwidth]\n" - "zip2 z5.d, z10.d, z14.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z6.d, z11.d, z15.d\n" - "zip2 z7.d, z11.d, z15.d\n" - "zip1 z8.d, z0.d, z4.d\n" - "whilelt p4.b, %[outpos], %[outwidth]\n" - "zip2 z9.d, z0.d, z4.d\n" - "incb %[outpos], all, mul #1\n" - "zip1 z10.d, z1.d, z5.d\n" - "st1b z8.b, p0, [%[outptr]]\n" - "zip2 z11.d, z1.d, z5.d\n" - "zip1 z12.d, z2.d, z6.d\n" - "whilelt p5.b, %[outpos], %[outwidth]\n" - "zip2 z13.d, z2.d, z6.d\n" - "st1b z9.b, p1, [%[outptr], #1, MUL VL]\n" - "zip1 z14.d, z3.d, z7.d\n" - "incb %[outpos], all, mul #1\n" - "zip2 z15.d, z3.d, z7.d\n" - "st1b z10.b, p2, [%[outptr], #2, MUL VL]\n" - "whilelt p6.b, %[outpos], %[outwidth]\n" - "st1b z11.b, p3, [%[outptr], #3, MUL VL]\n" - "incb %[outpos], all, mul #1\n" - "st1b z12.b, p4, [%[outptr], #4, MUL VL]\n" - "whilelt p7.b, %[outpos], %[outwidth]\n" - "incb %[outpos], all, mul #1\n" - "st1b z13.b, p5, [%[outptr], #5, MUL VL]\n" - "st1b z14.b, p6, [%[outptr], #6, MUL VL]\n" - "st1b z15.b, p7, [%[outptr], #7, MUL VL]\n" - "addvl %[outptr], %[outptr], #8\n" - "b 1b\n" - "2:\n" - : [inpos] "+r" (inpos), [outpos] "+r" (outpos), [outptr] "+r" (outptr), [inptr0] "+r" (inptr0), [inptr1] "+r" (inptr1), [inptr2] "+r" (inptr2), [inptr3] "+r" (inptr3), [inptr4] "+r" (inptr4), [inptr5] "+r" (inptr5), [inptr6] "+r" (inptr6), [inptr7] "+r" (inptr7) - : [outwidth] "r" (outwidth), [inwidth] "r" (inwidth) - : "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "cc", "memory" - ); - break; - - - } - } -} - -#endif // __ARM_FEATURE_SVE diff --git a/src/core/NEON/kernels/arm_gemm/transforms/transpose_interleave_common.hpp b/src/core/NEON/kernels/arm_gemm/transforms/transpose_interleave_common.hpp index aac5e19ebe..a3216c494f 100644 --- a/src/core/NEON/kernels/arm_gemm/transforms/transpose_interleave_common.hpp +++ b/src/core/NEON/kernels/arm_gemm/transforms/transpose_interleave_common.hpp @@ -23,6 +23,8 @@ */ #pragma once +#include "../asmlib.hpp" + template <unsigned int IntBy, typename TIn, typename TOut> struct TransposeInterleaveCommon { // Override the moveblock_1xY methods to improve performance |