diff options
Diffstat (limited to 'src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp')
| -rw-r--r-- | src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp | 604 |
1 files changed, 302 insertions, 302 deletions
diff --git a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp index b0d10c085d..2da48922e3 100644 --- a/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp +++ b/src/core/NEON/kernels/arm_gemm/merges/sve_merge_fp32_3VLx8.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2020 Arm Limited. + * Copyright (c) 2019-2020,2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -23,7 +23,7 @@ */ #pragma once -#ifdef __ARM_FEATURE_SVE +#ifdef ARM_COMPUTE_ENABLE_SVE template<> void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, const int y0, const int ymax, const int x0, const int xmax, const float *bias, Activation act, bool append) @@ -1029,25 +1029,25 @@ void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, cons "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" "ld1w z2.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[p], %[w]\n" - "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" - "incw %[p], all, mul #1\n" - "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" "ld1w z13.s, p0/z, [%[inptr]]\n" + "incw %[p], all, mul #1\n" + "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "fadd z13.s, z13.s, z2.s\n" "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" "whilelt p2.s, %[p], %[w]\n" - "fadd z13.s, z13.s, z2.s\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" + "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n" "fadd z14.s, z14.s, z3.s\n" "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" "addvl %[inptr], %[inptr], #24\n" - "fmin z13.s, p0/m, z13.s, z0.s\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" "fmin z14.s, p1/m, z14.s, z0.s\n" "fadd z15.s, z15.s, z4.s\n" - "fmax z13.s, p0/m, z13.s, z1.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" "fmax z14.s, p1/m, z14.s, z1.s\n" "fmin z15.s, p2/m, z15.s, z0.s\n" - "st1w z13.s, p0, [%[outptr0]]\n" - "fmax z15.s, p2/m, z15.s, z1.s\n" "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" : [outptr0] "+r" (outptr0), [outptr1] "+r" (outptr1), [outptr2] "+r" (outptr2), [outptr3] "+r" (outptr3), [outptr4] "+r" (outptr4), [outptr5] "+r" (outptr5), [outptr6] "+r" (outptr6), [outptr7] "+r" (outptr7), @@ -1073,42 +1073,42 @@ void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, cons "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" "ld1w z2.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[p], %[w]\n" - "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z13.s, p0/z, [%[inptr]]\n" - "whilelt p2.s, %[p], %[w]\n" + "fadd z13.s, z13.s, z2.s\n" + "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n" "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "fadd z16.s, z16.s, z2.s\n" + "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "fmin z13.s, p0/m, z13.s, z0.s\n" "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" - "fadd z13.s, z13.s, z2.s\n" - "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" - "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "fadd z14.s, z14.s, z3.s\n" - "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" + "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "fmin z16.s, p0/m, z16.s, z0.s\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "fmax z13.s, p0/m, z13.s, z1.s\n" "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" "addvl %[inptr], %[inptr], #24\n" - "fmin z13.s, p0/m, z13.s, z0.s\n" "fmin z14.s, p1/m, z14.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" "fadd z15.s, z15.s, z4.s\n" - "fadd z16.s, z16.s, z2.s\n" - "fmax z13.s, p0/m, z13.s, z1.s\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" "fmax z14.s, p1/m, z14.s, z1.s\n" "fmin z15.s, p2/m, z15.s, z0.s\n" - "fmin z16.s, p0/m, z16.s, z0.s\n" - "st1w z13.s, p0, [%[outptr0]]\n" - "fadd z17.s, z17.s, z3.s\n" "fadd z18.s, z18.s, z4.s\n" - "fmax z15.s, p2/m, z15.s, z1.s\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" - "fmax z16.s, p0/m, z16.s, z1.s\n" - "fmin z17.s, p1/m, z17.s, z0.s\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" "fmin z18.s, p2/m, z18.s, z0.s\n" "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "addvl %[outptr0], %[outptr0], #3\n" - "fmax z17.s, p1/m, z17.s, z1.s\n" - "st1w z16.s, p0, [%[outptr1]]\n" "fmax z18.s, p2/m, z18.s, z1.s\n" + "st1w z16.s, p0, [%[outptr1]]\n" "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "addvl %[outptr1], %[outptr1], #3\n" @@ -1135,60 +1135,60 @@ void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, cons "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" "ld1w z2.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[p], %[w]\n" - "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z13.s, p0/z, [%[inptr]]\n" - "whilelt p2.s, %[p], %[w]\n" - "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" "fadd z13.s, z13.s, z2.s\n" - "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" - "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fadd z14.s, z14.s, z3.s\n" + "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "fadd z16.s, z16.s, z2.s\n" "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" "fmin z13.s, p0/m, z13.s, z0.s\n" - "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" - "fadd z15.s, z15.s, z4.s\n" "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fadd z16.s, z16.s, z2.s\n" - "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" - "fmin z14.s, p1/m, z14.s, z0.s\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" "fmax z13.s, p0/m, z13.s, z1.s\n" - "addvl %[inptr], %[inptr], #24\n" - "fmax z14.s, p1/m, z14.s, z1.s\n" - "fmin z15.s, p2/m, z15.s, z0.s\n" - "st1w z13.s, p0, [%[outptr0]]\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" "fmin z16.s, p0/m, z16.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" - "fadd z17.s, z17.s, z3.s\n" - "fadd z18.s, z18.s, z4.s\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" + "addvl %[inptr], %[inptr], #24\n" "fmax z15.s, p2/m, z15.s, z1.s\n" - "fmax z16.s, p0/m, z16.s, z1.s\n" + "fadd z18.s, z18.s, z4.s\n" "fmin z17.s, p1/m, z17.s, z0.s\n" - "fmin z18.s, p2/m, z18.s, z0.s\n" - "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "fadd z19.s, z19.s, z2.s\n" + "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" + "fadd z20.s, z20.s, z3.s\n" "addvl %[outptr0], %[outptr0], #3\n" "fmax z17.s, p1/m, z17.s, z1.s\n" "st1w z16.s, p0, [%[outptr1]]\n" - "fmax z18.s, p2/m, z18.s, z1.s\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" "fmin z19.s, p0/m, z19.s, z0.s\n" - "fadd z20.s, z20.s, z3.s\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" "fadd z13.s, z13.s, z4.s\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" "fmax z19.s, p0/m, z19.s, z1.s\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" - "fmin z20.s, p1/m, z20.s, z0.s\n" "addvl %[outptr1], %[outptr1], #3\n" - "fmin z13.s, p2/m, z13.s, z0.s\n" - "st1w z19.s, p0, [%[outptr2]]\n" - "fmax z20.s, p1/m, z20.s, z1.s\n" "fmax z13.s, p2/m, z13.s, z1.s\n" + "st1w z19.s, p0, [%[outptr2]]\n" "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" @@ -1215,75 +1215,75 @@ void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, cons "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" "ld1w z2.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[p], %[w]\n" - "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z13.s, p0/z, [%[inptr]]\n" - "whilelt p2.s, %[p], %[w]\n" - "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" "fadd z13.s, z13.s, z2.s\n" - "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" - "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fadd z14.s, z14.s, z3.s\n" + "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "fadd z16.s, z16.s, z2.s\n" "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" "fmin z13.s, p0/m, z13.s, z0.s\n" - "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" - "fadd z15.s, z15.s, z4.s\n" "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fadd z16.s, z16.s, z2.s\n" - "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" - "fmin z14.s, p1/m, z14.s, z0.s\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" "fmax z13.s, p0/m, z13.s, z1.s\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmax z14.s, p1/m, z14.s, z1.s\n" - "addvl %[inptr], %[inptr], #24\n" - "fmin z15.s, p2/m, z15.s, z0.s\n" - "st1w z13.s, p0, [%[outptr0]]\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" "fmin z16.s, p0/m, z16.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" - "fadd z17.s, z17.s, z3.s\n" - "fadd z18.s, z18.s, z4.s\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fmax z16.s, p0/m, z16.s, z1.s\n" "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" - "fmax z15.s, p2/m, z15.s, z1.s\n" + "fadd z17.s, z17.s, z3.s\n" "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" - "fmax z16.s, p0/m, z16.s, z1.s\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" + "fadd z18.s, z18.s, z4.s\n" + "addvl %[inptr], %[inptr], #24\n" "fmin z17.s, p1/m, z17.s, z0.s\n" - "fmin z18.s, p2/m, z18.s, z0.s\n" "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "fadd z19.s, z19.s, z2.s\n" "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" - "fadd z20.s, z20.s, z3.s\n" + "fmin z18.s, p2/m, z18.s, z0.s\n" "addvl %[outptr0], %[outptr0], #3\n" "fmax z17.s, p1/m, z17.s, z1.s\n" "st1w z16.s, p0, [%[outptr1]]\n" - "fmax z18.s, p2/m, z18.s, z1.s\n" - "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" "fmin z19.s, p0/m, z19.s, z0.s\n" - "fmin z20.s, p1/m, z20.s, z0.s\n" + "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "fadd z20.s, z20.s, z3.s\n" "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" "fadd z13.s, z13.s, z4.s\n" - "fadd z14.s, z14.s, z2.s\n" "fmax z19.s, p0/m, z19.s, z1.s\n" + "fadd z14.s, z14.s, z2.s\n" "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" - "fmax z20.s, p1/m, z20.s, z1.s\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" "addvl %[outptr1], %[outptr1], #3\n" "fmin z13.s, p2/m, z13.s, z0.s\n" "st1w z19.s, p0, [%[outptr2]]\n" "fmin z14.s, p0/m, z14.s, z0.s\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" "fadd z15.s, z15.s, z3.s\n" - "fadd z16.s, z16.s, z4.s\n" - "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" "fmax z13.s, p2/m, z13.s, z1.s\n" "fmax z14.s, p0/m, z14.s, z1.s\n" + "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" + "fadd z16.s, z16.s, z4.s\n" "fmin z15.s, p1/m, z15.s, z0.s\n" - "fmin z16.s, p2/m, z16.s, z0.s\n" "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "addvl %[outptr2], %[outptr2], #3\n" "fmax z15.s, p1/m, z15.s, z1.s\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" "st1w z14.s, p0, [%[outptr3]]\n" "fmax z16.s, p2/m, z16.s, z1.s\n" "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" @@ -1312,93 +1312,93 @@ void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, cons "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" "ld1w z2.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[p], %[w]\n" - "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z13.s, p0/z, [%[inptr]]\n" - "whilelt p2.s, %[p], %[w]\n" - "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" "fadd z13.s, z13.s, z2.s\n" - "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" - "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fadd z14.s, z14.s, z3.s\n" + "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "fadd z16.s, z16.s, z2.s\n" "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" "fmin z13.s, p0/m, z13.s, z0.s\n" - "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" - "fadd z15.s, z15.s, z4.s\n" "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fadd z16.s, z16.s, z2.s\n" - "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" - "fmin z14.s, p1/m, z14.s, z0.s\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" "fmax z13.s, p0/m, z13.s, z1.s\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmax z14.s, p1/m, z14.s, z1.s\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmin z15.s, p2/m, z15.s, z0.s\n" - "st1w z13.s, p0, [%[outptr0]]\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" "fmin z16.s, p0/m, z16.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" - "fadd z17.s, z17.s, z3.s\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "fmax z15.s, p2/m, z15.s, z1.s\n" - "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" - "fmin z17.s, p1/m, z17.s, z0.s\n" - "addvl %[inptr], %[inptr], #24\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" "fadd z18.s, z18.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "fadd z19.s, z19.s, z2.s\n" "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" - "fmax z17.s, p1/m, z17.s, z1.s\n" - "addvl %[outptr0], %[outptr0], #3\n" "fmin z18.s, p2/m, z18.s, z0.s\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" "st1w z16.s, p0, [%[outptr1]]\n" "fmin z19.s, p0/m, z19.s, z0.s\n" "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" + "fmax z18.s, p2/m, z18.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" "fadd z20.s, z20.s, z3.s\n" - "fadd z13.s, z13.s, z4.s\n" "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" - "fmax z18.s, p2/m, z18.s, z1.s\n" - "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" "fmax z19.s, p0/m, z19.s, z1.s\n" + "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" + "fadd z13.s, z13.s, z4.s\n" + "addvl %[inptr], %[inptr], #24\n" "fmin z20.s, p1/m, z20.s, z0.s\n" - "fmin z13.s, p2/m, z13.s, z0.s\n" "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "fadd z14.s, z14.s, z2.s\n" "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" - "fadd z15.s, z15.s, z3.s\n" + "fmin z13.s, p2/m, z13.s, z0.s\n" "addvl %[outptr1], %[outptr1], #3\n" "fmax z20.s, p1/m, z20.s, z1.s\n" "st1w z19.s, p0, [%[outptr2]]\n" - "fmax z13.s, p2/m, z13.s, z1.s\n" - "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" "fmin z14.s, p0/m, z14.s, z0.s\n" - "fmin z15.s, p1/m, z15.s, z0.s\n" + "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "fadd z15.s, z15.s, z3.s\n" "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" "fadd z16.s, z16.s, z4.s\n" - "fadd z17.s, z17.s, z2.s\n" "fmax z14.s, p0/m, z14.s, z1.s\n" + "fadd z17.s, z17.s, z2.s\n" "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" - "fmax z15.s, p1/m, z15.s, z1.s\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" "addvl %[outptr2], %[outptr2], #3\n" "fmin z16.s, p2/m, z16.s, z0.s\n" "st1w z14.s, p0, [%[outptr3]]\n" "fmin z17.s, p0/m, z17.s, z0.s\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" "fadd z18.s, z18.s, z3.s\n" - "fadd z19.s, z19.s, z4.s\n" - "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" "fmax z16.s, p2/m, z16.s, z1.s\n" "fmax z17.s, p0/m, z17.s, z1.s\n" + "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" + "fadd z19.s, z19.s, z4.s\n" "fmin z18.s, p1/m, z18.s, z0.s\n" - "fmin z19.s, p2/m, z19.s, z0.s\n" "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "addvl %[outptr3], %[outptr3], #3\n" "fmax z18.s, p1/m, z18.s, z1.s\n" + "fmin z19.s, p2/m, z19.s, z0.s\n" "st1w z17.s, p0, [%[outptr4]]\n" "fmax z19.s, p2/m, z19.s, z1.s\n" "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" @@ -1427,111 +1427,111 @@ void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, cons "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" "ld1w z2.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[p], %[w]\n" - "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z13.s, p0/z, [%[inptr]]\n" - "whilelt p2.s, %[p], %[w]\n" - "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" "fadd z13.s, z13.s, z2.s\n" - "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" - "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fadd z14.s, z14.s, z3.s\n" + "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "fadd z16.s, z16.s, z2.s\n" "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" "fmin z13.s, p0/m, z13.s, z0.s\n" - "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" - "fadd z15.s, z15.s, z4.s\n" "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fadd z16.s, z16.s, z2.s\n" - "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" - "fmin z14.s, p1/m, z14.s, z0.s\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" "fmax z13.s, p0/m, z13.s, z1.s\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmax z14.s, p1/m, z14.s, z1.s\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmin z15.s, p2/m, z15.s, z0.s\n" - "st1w z13.s, p0, [%[outptr0]]\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" "fmin z16.s, p0/m, z16.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" - "fadd z17.s, z17.s, z3.s\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "fmax z15.s, p2/m, z15.s, z1.s\n" - "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" - "fmin z17.s, p1/m, z17.s, z0.s\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" "fadd z18.s, z18.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "fadd z19.s, z19.s, z2.s\n" "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" - "fmax z17.s, p1/m, z17.s, z1.s\n" - "addvl %[outptr0], %[outptr0], #3\n" "fmin z18.s, p2/m, z18.s, z0.s\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" "st1w z16.s, p0, [%[outptr1]]\n" "fmin z19.s, p0/m, z19.s, z0.s\n" "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" - "fadd z20.s, z20.s, z3.s\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" "fmax z18.s, p2/m, z18.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd z20.s, z20.s, z3.s\n" "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" "fmax z19.s, p0/m, z19.s, z1.s\n" "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" - "fmin z20.s, p1/m, z20.s, z0.s\n" - "addvl %[inptr], %[inptr], #24\n" "fadd z13.s, z13.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "fadd z14.s, z14.s, z2.s\n" "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" - "fmax z20.s, p1/m, z20.s, z1.s\n" - "addvl %[outptr1], %[outptr1], #3\n" "fmin z13.s, p2/m, z13.s, z0.s\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" "st1w z19.s, p0, [%[outptr2]]\n" "fmin z14.s, p0/m, z14.s, z0.s\n" "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" + "fmax z13.s, p2/m, z13.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" "fadd z15.s, z15.s, z3.s\n" - "fadd z16.s, z16.s, z4.s\n" "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" - "fmax z13.s, p2/m, z13.s, z1.s\n" - "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" "fmax z14.s, p0/m, z14.s, z1.s\n" + "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" + "fadd z16.s, z16.s, z4.s\n" + "addvl %[inptr], %[inptr], #24\n" "fmin z15.s, p1/m, z15.s, z0.s\n" - "fmin z16.s, p2/m, z16.s, z0.s\n" "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "fadd z17.s, z17.s, z2.s\n" "ld1w z13.s, p1/z, [x8]\n" - "fadd z18.s, z18.s, z3.s\n" + "fmin z16.s, p2/m, z16.s, z0.s\n" "addvl %[outptr2], %[outptr2], #3\n" "fmax z15.s, p1/m, z15.s, z1.s\n" "st1w z14.s, p0, [%[outptr3]]\n" - "fmax z16.s, p2/m, z16.s, z1.s\n" - "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" "fmin z17.s, p0/m, z17.s, z0.s\n" - "fmin z18.s, p1/m, z18.s, z0.s\n" + "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "fadd z18.s, z18.s, z3.s\n" "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" "fadd z19.s, z19.s, z4.s\n" - "fadd z20.s, z20.s, z2.s\n" "fmax z17.s, p0/m, z17.s, z1.s\n" + "fadd z20.s, z20.s, z2.s\n" "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" - "fmax z18.s, p1/m, z18.s, z1.s\n" + "fmin z18.s, p1/m, z18.s, z0.s\n" "addvl %[outptr3], %[outptr3], #3\n" "fmin z19.s, p2/m, z19.s, z0.s\n" "st1w z17.s, p0, [%[outptr4]]\n" "fmin z20.s, p0/m, z20.s, z0.s\n" + "fmax z18.s, p1/m, z18.s, z1.s\n" "fadd z13.s, z13.s, z3.s\n" - "fadd z14.s, z14.s, z4.s\n" - "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" "fmax z19.s, p2/m, z19.s, z1.s\n" "fmax z20.s, p0/m, z20.s, z1.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" + "fadd z14.s, z14.s, z4.s\n" "fmin z13.s, p1/m, z13.s, z0.s\n" - "fmin z14.s, p2/m, z14.s, z0.s\n" "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" "addvl %[outptr4], %[outptr4], #3\n" "fmax z13.s, p1/m, z13.s, z1.s\n" + "fmin z14.s, p2/m, z14.s, z0.s\n" "st1w z20.s, p0, [%[outptr5]]\n" "fmax z14.s, p2/m, z14.s, z1.s\n" "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" @@ -1560,129 +1560,129 @@ void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, cons "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" "ld1w z2.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[p], %[w]\n" - "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z13.s, p0/z, [%[inptr]]\n" - "whilelt p2.s, %[p], %[w]\n" - "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" "fadd z13.s, z13.s, z2.s\n" - "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" - "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fadd z14.s, z14.s, z3.s\n" + "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "fadd z16.s, z16.s, z2.s\n" "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" "fmin z13.s, p0/m, z13.s, z0.s\n" - "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" - "fadd z15.s, z15.s, z4.s\n" "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fadd z16.s, z16.s, z2.s\n" - "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" - "fmin z14.s, p1/m, z14.s, z0.s\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" "fmax z13.s, p0/m, z13.s, z1.s\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmax z14.s, p1/m, z14.s, z1.s\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmin z15.s, p2/m, z15.s, z0.s\n" - "st1w z13.s, p0, [%[outptr0]]\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" "fmin z16.s, p0/m, z16.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" - "fadd z17.s, z17.s, z3.s\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "fmax z15.s, p2/m, z15.s, z1.s\n" - "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" - "fmin z17.s, p1/m, z17.s, z0.s\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" "fadd z18.s, z18.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "fadd z19.s, z19.s, z2.s\n" "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" - "fmax z17.s, p1/m, z17.s, z1.s\n" - "addvl %[outptr0], %[outptr0], #3\n" "fmin z18.s, p2/m, z18.s, z0.s\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" "st1w z16.s, p0, [%[outptr1]]\n" "fmin z19.s, p0/m, z19.s, z0.s\n" "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" - "fadd z20.s, z20.s, z3.s\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" "fmax z18.s, p2/m, z18.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd z20.s, z20.s, z3.s\n" "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" "fmax z19.s, p0/m, z19.s, z1.s\n" "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" - "fmin z20.s, p1/m, z20.s, z0.s\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" "fadd z13.s, z13.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "fadd z14.s, z14.s, z2.s\n" "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" - "fmax z20.s, p1/m, z20.s, z1.s\n" - "addvl %[outptr1], %[outptr1], #3\n" "fmin z13.s, p2/m, z13.s, z0.s\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" "st1w z19.s, p0, [%[outptr2]]\n" "fmin z14.s, p0/m, z14.s, z0.s\n" "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" - "fadd z15.s, z15.s, z3.s\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" "fmax z13.s, p2/m, z13.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fadd z15.s, z15.s, z3.s\n" "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" "fmax z14.s, p0/m, z14.s, z1.s\n" "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" - "fmin z15.s, p1/m, z15.s, z0.s\n" - "addvl %[inptr], %[inptr], #24\n" "fadd z16.s, z16.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "fadd z17.s, z17.s, z2.s\n" "ld1w z13.s, p1/z, [x8]\n" - "fmax z15.s, p1/m, z15.s, z1.s\n" - "addvl %[outptr2], %[outptr2], #3\n" "fmin z16.s, p2/m, z16.s, z0.s\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" "st1w z14.s, p0, [%[outptr3]]\n" "fmin z17.s, p0/m, z17.s, z0.s\n" "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" + "fmax z16.s, p2/m, z16.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" "fadd z18.s, z18.s, z3.s\n" - "fadd z19.s, z19.s, z4.s\n" "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" - "fmax z16.s, p2/m, z16.s, z1.s\n" - "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" "fmax z17.s, p0/m, z17.s, z1.s\n" + "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" + "fadd z19.s, z19.s, z4.s\n" + "addvl %[inptr], %[inptr], #24\n" "fmin z18.s, p1/m, z18.s, z0.s\n" - "fmin z19.s, p2/m, z19.s, z0.s\n" "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "fadd z20.s, z20.s, z2.s\n" "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n" - "fadd z13.s, z13.s, z3.s\n" + "fmin z19.s, p2/m, z19.s, z0.s\n" "addvl %[outptr3], %[outptr3], #3\n" "fmax z18.s, p1/m, z18.s, z1.s\n" "st1w z17.s, p0, [%[outptr4]]\n" - "fmax z19.s, p2/m, z19.s, z1.s\n" - "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" "fmin z20.s, p0/m, z20.s, z0.s\n" - "fmin z13.s, p1/m, z13.s, z0.s\n" + "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" + "fmax z19.s, p2/m, z19.s, z1.s\n" + "fadd z13.s, z13.s, z3.s\n" "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" "fadd z14.s, z14.s, z4.s\n" - "fadd z15.s, z15.s, z2.s\n" "fmax z20.s, p0/m, z20.s, z1.s\n" + "fadd z15.s, z15.s, z2.s\n" "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" - "fmax z13.s, p1/m, z13.s, z1.s\n" + "fmin z13.s, p1/m, z13.s, z0.s\n" "addvl %[outptr4], %[outptr4], #3\n" "fmin z14.s, p2/m, z14.s, z0.s\n" "st1w z20.s, p0, [%[outptr5]]\n" "fmin z15.s, p0/m, z15.s, z0.s\n" + "fmax z13.s, p1/m, z13.s, z1.s\n" "fadd z16.s, z16.s, z3.s\n" - "fadd z17.s, z17.s, z4.s\n" - "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" "fmax z14.s, p2/m, z14.s, z1.s\n" "fmax z15.s, p0/m, z15.s, z1.s\n" + "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" + "fadd z17.s, z17.s, z4.s\n" "fmin z16.s, p1/m, z16.s, z0.s\n" - "fmin z17.s, p2/m, z17.s, z0.s\n" "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" "addvl %[outptr5], %[outptr5], #3\n" "fmax z16.s, p1/m, z16.s, z1.s\n" + "fmin z17.s, p2/m, z17.s, z0.s\n" "st1w z15.s, p0, [%[outptr6]]\n" "fmax z17.s, p2/m, z17.s, z1.s\n" "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n" @@ -1712,142 +1712,142 @@ void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, cons "prfm PSTL1KEEP, [%[outptr0], #0x60]\n" "ld1w z2.s, p0/z, [%[biasptr]]\n" "whilelt p1.s, %[p], %[w]\n" - "ld1w z3.s, p0/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z13.s, p0/z, [%[inptr]]\n" "incw %[p], all, mul #1\n" - "ld1w z4.s, p0/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" "prfm PLDL1KEEP, [%[inptr], #0x1c0]\n" - "ld1w z13.s, p0/z, [%[inptr]]\n" - "whilelt p2.s, %[p], %[w]\n" - "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" - "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" "fadd z13.s, z13.s, z2.s\n" - "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" - "ld1w z16.s, p0/z, [%[inptr], #3, MUL VL]\n" - "prfm PLDL1KEEP, [%[inptr], #0x200]\n" - "fadd z14.s, z14.s, z3.s\n" + "ld1w z3.s, p1/z, [%[biasptr], #1, MUL VL]\n" + "ld1w z14.s, p1/z, [%[inptr], #1, MUL VL]\n" + "whilelt p2.s, %[p], %[w]\n" + "fadd z16.s, z16.s, z2.s\n" "ld1w z17.s, p1/z, [%[inptr], #4, MUL VL]\n" "fmin z13.s, p0/m, z13.s, z0.s\n" - "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" - "fadd z15.s, z15.s, z4.s\n" "ld1w z19.s, p0/z, [%[inptr], #6, MUL VL]\n" - "fadd z16.s, z16.s, z2.s\n" - "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" - "fmin z14.s, p1/m, z14.s, z0.s\n" - "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" + "fadd z14.s, z14.s, z3.s\n" + "ld1w z4.s, p2/z, [%[biasptr], #2, MUL VL]\n" + "ld1w z15.s, p2/z, [%[inptr], #2, MUL VL]\n" + "prfm PSTL1KEEP, [%[outptr1], #0x60]\n" "fmax z13.s, p0/m, z13.s, z1.s\n" - "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" - "fmax z14.s, p1/m, z14.s, z1.s\n" - "prfm PLDL1KEEP, [%[inptr], #0x240]\n" - "fmin z15.s, p2/m, z15.s, z0.s\n" - "st1w z13.s, p0, [%[outptr0]]\n" + "ld1w z18.s, p2/z, [%[inptr], #5, MUL VL]\n" + "fmin z14.s, p1/m, z14.s, z0.s\n" + "ld1w z20.s, p1/z, [%[inptr], #7, MUL VL]\n" + "fadd z15.s, z15.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x200]\n" "fmin z16.s, p0/m, z16.s, z0.s\n" + "st1w z13.s, p0, [%[outptr0]]\n" + "fmax z14.s, p1/m, z14.s, z1.s\n" "ld1w z13.s, p2/z, [x8, #-8, MUL VL]\n" - "fadd z17.s, z17.s, z3.s\n" - "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" - "fmax z15.s, p2/m, z15.s, z1.s\n" - "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fmin z15.s, p2/m, z15.s, z0.s\n" + "prfm PSTL1KEEP, [%[outptr2], #0x60]\n" "fmax z16.s, p0/m, z16.s, z1.s\n" + "st1w z14.s, p1, [%[outptr0], #1, MUL VL]\n" + "fadd z17.s, z17.s, z3.s\n" "ld1w z14.s, p0/z, [x8, #-7, MUL VL]\n" - "fmin z17.s, p1/m, z17.s, z0.s\n" - "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmax z15.s, p2/m, z15.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr3], #0x60]\n" "fadd z18.s, z18.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x240]\n" + "fmin z17.s, p1/m, z17.s, z0.s\n" "st1w z15.s, p2, [%[outptr0], #2, MUL VL]\n" "fadd z19.s, z19.s, z2.s\n" "ld1w z15.s, p1/z, [x8, #-6, MUL VL]\n" - "fmax z17.s, p1/m, z17.s, z1.s\n" - "addvl %[outptr0], %[outptr0], #3\n" "fmin z18.s, p2/m, z18.s, z0.s\n" + "addvl %[outptr0], %[outptr0], #3\n" + "fmax z17.s, p1/m, z17.s, z1.s\n" "st1w z16.s, p0, [%[outptr1]]\n" "fmin z19.s, p0/m, z19.s, z0.s\n" "ld1w z16.s, p2/z, [x8, #-5, MUL VL]\n" - "fadd z20.s, z20.s, z3.s\n" - "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" "fmax z18.s, p2/m, z18.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr4], #0x60]\n" + "fadd z20.s, z20.s, z3.s\n" "st1w z17.s, p1, [%[outptr1], #1, MUL VL]\n" "fmax z19.s, p0/m, z19.s, z1.s\n" "ld1w z17.s, p0/z, [x8, #-4, MUL VL]\n" - "fmin z20.s, p1/m, z20.s, z0.s\n" - "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" "fadd z13.s, z13.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x280]\n" + "fmin z20.s, p1/m, z20.s, z0.s\n" "st1w z18.s, p2, [%[outptr1], #2, MUL VL]\n" "fadd z14.s, z14.s, z2.s\n" "ld1w z18.s, p1/z, [x8, #-3, MUL VL]\n" - "fmax z20.s, p1/m, z20.s, z1.s\n" - "addvl %[outptr1], %[outptr1], #3\n" "fmin z13.s, p2/m, z13.s, z0.s\n" + "addvl %[outptr1], %[outptr1], #3\n" + "fmax z20.s, p1/m, z20.s, z1.s\n" "st1w z19.s, p0, [%[outptr2]]\n" "fmin z14.s, p0/m, z14.s, z0.s\n" "ld1w z19.s, p2/z, [x8, #-2, MUL VL]\n" - "fadd z15.s, z15.s, z3.s\n" - "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" "fmax z13.s, p2/m, z13.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr5], #0x60]\n" + "fadd z15.s, z15.s, z3.s\n" "st1w z20.s, p1, [%[outptr2], #1, MUL VL]\n" "fmax z14.s, p0/m, z14.s, z1.s\n" "ld1w z20.s, p0/z, [x8, #-1, MUL VL]\n" - "fmin z15.s, p1/m, z15.s, z0.s\n" - "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" "fadd z16.s, z16.s, z4.s\n" + "prfm PLDL1KEEP, [%[inptr], #0x2c0]\n" + "fmin z15.s, p1/m, z15.s, z0.s\n" "st1w z13.s, p2, [%[outptr2], #2, MUL VL]\n" "fadd z17.s, z17.s, z2.s\n" "ld1w z13.s, p1/z, [x8]\n" - "fmax z15.s, p1/m, z15.s, z1.s\n" - "addvl %[outptr2], %[outptr2], #3\n" "fmin z16.s, p2/m, z16.s, z0.s\n" + "addvl %[outptr2], %[outptr2], #3\n" + "fmax z15.s, p1/m, z15.s, z1.s\n" "st1w z14.s, p0, [%[outptr3]]\n" "fmin z17.s, p0/m, z17.s, z0.s\n" "ld1w z14.s, p2/z, [x8, #1, MUL VL]\n" - "fadd z18.s, z18.s, z3.s\n" - "addvl %[inptr], %[inptr], #24\n" "fmax z16.s, p2/m, z16.s, z1.s\n" + "prfm PSTL1KEEP, [%[outptr6], #0x60]\n" + "fadd z18.s, z18.s, z3.s\n" "st1w z15.s, p1, [%[outptr3], #1, MUL VL]\n" "fmax z17.s, p0/m, z17.s, z1.s\n" "ld1w z15.s, p0/z, [x8, #2, MUL VL]\n" - "fmin z18.s, p1/m, z18.s, z0.s\n" "fadd z19.s, z19.s, z4.s\n" + "prfm PSTL1KEEP, [%[outptr7], #0x60]\n" + "fmin z18.s, p1/m, z18.s, z0.s\n" "st1w z16.s, p2, [%[outptr3], #2, MUL VL]\n" "fadd z20.s, z20.s, z2.s\n" "ld1w z16.s, p1/z, [x8, #3, MUL VL]\n" - "fadd z13.s, z13.s, z3.s\n" + "fmin z19.s, p2/m, z19.s, z0.s\n" "addvl %[outptr3], %[outptr3], #3\n" "fmax z18.s, p1/m, z18.s, z1.s\n" "st1w z17.s, p0, [%[outptr4]]\n" - "fmin z19.s, p2/m, z19.s, z0.s\n" - "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" "fmin z20.s, p0/m, z20.s, z0.s\n" - "fmin z13.s, p1/m, z13.s, z0.s\n" - "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" - "fadd z14.s, z14.s, z4.s\n" - "ld1w z18.s, p0/z, [x8, #5, MUL VL]\n" + "ld1w z17.s, p2/z, [x8, #4, MUL VL]\n" "fmax z19.s, p2/m, z19.s, z1.s\n" + "addvl %[inptr], %[inptr], #24\n" + "fadd z13.s, z13.s, z3.s\n" + "st1w z18.s, p1, [%[outptr4], #1, MUL VL]\n" "fmax z20.s, p0/m, z20.s, z1.s\n" - "fmax z13.s, p1/m, z13.s, z1.s\n" - "fmin z14.s, p2/m, z14.s, z0.s\n" - "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" + "ld1w z18.s, p0/z, [x8, #5, MUL VL]\n" + "fadd z14.s, z14.s, z4.s\n" "fadd z15.s, z15.s, z2.s\n" + "st1w z19.s, p2, [%[outptr4], #2, MUL VL]\n" + "fmin z13.s, p1/m, z13.s, z0.s\n" "ld1w z19.s, p1/z, [x8, #6, MUL VL]\n" "fadd z16.s, z16.s, z3.s\n" "addvl %[outptr4], %[outptr4], #3\n" - "fmax z14.s, p2/m, z14.s, z1.s\n" + "fmin z14.s, p2/m, z14.s, z0.s\n" "st1w z20.s, p0, [%[outptr5]]\n" - "fmin z15.s, p0/m, z15.s, z0.s\n" + "fmax z13.s, p1/m, z13.s, z1.s\n" "ld1w z20.s, p2/z, [x8, #7, MUL VL]\n" + "fmin z15.s, p0/m, z15.s, z0.s\n" "fmin z16.s, p1/m, z16.s, z0.s\n" - "fadd z17.s, z17.s, z4.s\n" + "fmax z14.s, p2/m, z14.s, z1.s\n" "st1w z13.s, p1, [%[outptr5], #1, MUL VL]\n" - "fadd z18.s, z18.s, z2.s\n" + "fadd z17.s, z17.s, z4.s\n" "fmax z15.s, p0/m, z15.s, z1.s\n" "fmax z16.s, p1/m, z16.s, z1.s\n" "st1w z14.s, p2, [%[outptr5], #2, MUL VL]\n" - "fmin z17.s, p2/m, z17.s, z0.s\n" + "fadd z18.s, z18.s, z2.s\n" "addvl %[outptr5], %[outptr5], #3\n" - "fmin z18.s, p0/m, z18.s, z0.s\n" + "fmin z17.s, p2/m, z17.s, z0.s\n" "st1w z15.s, p0, [%[outptr6]]\n" "fadd z19.s, z19.s, z3.s\n" - "fmax z17.s, p2/m, z17.s, z1.s\n" + "fmin z18.s, p0/m, z18.s, z0.s\n" "fadd z20.s, z20.s, z4.s\n" "st1w z16.s, p1, [%[outptr6], #1, MUL VL]\n" - "fmax z18.s, p0/m, z18.s, z1.s\n" + "fmax z17.s, p2/m, z17.s, z1.s\n" "fmin z19.s, p1/m, z19.s, z0.s\n" + "fmax z18.s, p0/m, z18.s, z1.s\n" "fmin z20.s, p2/m, z20.s, z0.s\n" "st1w z17.s, p2, [%[outptr6], #2, MUL VL]\n" "addvl %[outptr6], %[outptr6], #3\n" @@ -1872,4 +1872,4 @@ void MergeResults<3, 8, true>(float *out, const float *in, const int ldout, cons } } -#endif // __ARM_FEATURE_SVE +#endif // ARM_COMPUTE_ENABLE_SVE |