diff options
Diffstat (limited to 'src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8q_nhwc_avg_generic_depthfirst/generic.cpp')
| -rw-r--r-- | src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8q_nhwc_avg_generic_depthfirst/generic.cpp | 202 |
1 files changed, 101 insertions, 101 deletions
diff --git a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8q_nhwc_avg_generic_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8q_nhwc_avg_generic_depthfirst/generic.cpp index d08863105b..6ad693547b 100644 --- a/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8q_nhwc_avg_generic_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/pooling/kernels/sme_u8q_nhwc_avg_generic_depthfirst/generic.cpp @@ -133,11 +133,11 @@ void sme_u8q_nhwc_avg_generic_depthfirst_impl( "1:" // 4-vectors of channels "ld1rw { z15.s }, p0/Z, [%x[accumulator_init]]\n" "lsr x23, %x[n_valid_cells], #0x1\n" - "mov x22, %x[inptrs]\n" "mov z14.d, z15.d\n" "mov z13.d, z15.d\n" "mov z12.d, z15.d\n" "mov z11.d, z15.d\n" + "mov x22, %x[inptrs]\n" "mov z10.d, z15.d\n" "mov z9.d, z15.d\n" "mov z8.d, z15.d\n" @@ -170,32 +170,32 @@ void sme_u8q_nhwc_avg_generic_depthfirst_impl( ".inst 0x455c0bb5 // uaddlb z21.h, z29.b, z28.b\n" ".inst 0x455c0fb4 // uaddlt z20.h, z29.b, z28.b\n" "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" ".inst 0x455a0b73 // uaddlb z19.h, z27.b, z26.b\n" ".inst 0x455a0f72 // uaddlt z18.h, z27.b, z26.b\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" ".inst 0x45580b31 // uaddlb z17.h, z25.b, z24.b\n" ".inst 0x45580f30 // uaddlt z16.h, z25.b, z24.b\n" - "ld1b { z31.b }, p4/Z, [x21, x27]\n" + "ld1b { z29.b }, p3/Z, [x21, x26]\n" ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" ".inst 0x45974dce // uaddwt z14.s, z14.s, z23.h\n" - "ld1b { z30.b }, p4/Z, [x20, x27]\n" + "ld1b { z28.b }, p3/Z, [x20, x26]\n" ".inst 0x459649ad // uaddwb z13.s, z13.s, z22.h\n" ".inst 0x45964d8c // uaddwt z12.s, z12.s, z22.h\n" - "ld1b { z29.b }, p3/Z, [x21, x26]\n" + "ld1b { z27.b }, p2/Z, [x21, x25]\n" ".inst 0x4595496b // uaddwb z11.s, z11.s, z21.h\n" ".inst 0x45954d4a // uaddwt z10.s, z10.s, z21.h\n" - "ld1b { z28.b }, p3/Z, [x20, x26]\n" + "ld1b { z26.b }, p2/Z, [x20, x25]\n" ".inst 0x45944929 // uaddwb z9.s, z9.s, z20.h\n" ".inst 0x45944d08 // uaddwt z8.s, z8.s, z20.h\n" - "ld1b { z27.b }, p2/Z, [x21, x25]\n" + "ld1b { z25.b }, p1/Z, [x21, x24]\n" ".inst 0x459348e7 // uaddwb z7.s, z7.s, z19.h\n" ".inst 0x45934cc6 // uaddwt z6.s, z6.s, z19.h\n" - "ld1b { z26.b }, p2/Z, [x20, x25]\n" + "ld1b { z24.b }, p1/Z, [x20, x24]\n" ".inst 0x459248a5 // uaddwb z5.s, z5.s, z18.h\n" ".inst 0x45924c84 // uaddwt z4.s, z4.s, z18.h\n" - "ld1b { z25.b }, p1/Z, [x21, x24]\n" ".inst 0x45914863 // uaddwb z3.s, z3.s, z17.h\n" ".inst 0x45914c42 // uaddwt z2.s, z2.s, z17.h\n" - "ld1b { z24.b }, p1/Z, [x20, x24]\n" ".inst 0x45904821 // uaddwb z1.s, z1.s, z16.h\n" ".inst 0x45904c00 // uaddwt z0.s, z0.s, z16.h\n" "bgt 2b\n" @@ -229,17 +229,17 @@ void sme_u8q_nhwc_avg_generic_depthfirst_impl( "beq 6f\n" "5:" // 4-vectors of channels: Single input loop "ldr x20, [x22], #0x8\n" + "ld1b { z16.b }, p4/Z, [x20, x27]\n" + ".inst 0x4508aa17 // ushllb z23.h, z16.b, #0x0\n" + ".inst 0x4508ae16 // ushllt z22.h, z16.b, #0x0\n" + "ld1b { z16.b }, p3/Z, [x20, x26]\n" + ".inst 0x4508aa15 // ushllb z21.h, z16.b, #0x0\n" + ".inst 0x4508ae14 // ushllt z20.h, z16.b, #0x0\n" "subs x21, x21, #0x1\n" - "ld1b { z19.b }, p4/Z, [x20, x27]\n" - "ld1b { z18.b }, p3/Z, [x20, x26]\n" - "ld1b { z17.b }, p2/Z, [x20, x25]\n" + "ld1b { z16.b }, p2/Z, [x20, x25]\n" + ".inst 0x4508aa13 // ushllb z19.h, z16.b, #0x0\n" + ".inst 0x4508ae12 // ushllt z18.h, z16.b, #0x0\n" "ld1b { z16.b }, p1/Z, [x20, x24]\n" - ".inst 0x4508aa77 // ushllb z23.h, z19.b, #0x0\n" - ".inst 0x4508ae76 // ushllt z22.h, z19.b, #0x0\n" - ".inst 0x4508aa55 // ushllb z21.h, z18.b, #0x0\n" - ".inst 0x4508ae54 // ushllt z20.h, z18.b, #0x0\n" - ".inst 0x4508aa33 // ushllb z19.h, z17.b, #0x0\n" - ".inst 0x4508ae32 // ushllt z18.h, z17.b, #0x0\n" ".inst 0x4508aa11 // ushllb z17.h, z16.b, #0x0\n" ".inst 0x4508ae10 // ushllt z16.h, z16.b, #0x0\n" ".inst 0x459749ef // uaddwb z15.s, z15.s, z23.h\n" @@ -260,29 +260,27 @@ void sme_u8q_nhwc_avg_generic_depthfirst_impl( ".inst 0x45904c00 // uaddwt z0.s, z0.s, z16.h\n" "bgt 5b\n" "6:" // 4-vectors of channels: Single input loop: End - "ld1rw { z21.s }, p0/Z, [%x[left_shift]]\n" + "ld1rw { z19.s }, p0/Z, [%x[left_shift]]\n" + ".inst 0x4482826f // srshl z15.s, p0/M, z15.s, z19.s\n" + ".inst 0x4482826e // srshl z14.s, p0/M, z14.s, z19.s\n" "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" - "mov z19.s, #0x0\n" - "mov z20.s, #0xff\n" + ".inst 0x4482826d // srshl z13.s, p0/M, z13.s, z19.s\n" + ".inst 0x4482826c // srshl z12.s, p0/M, z12.s, z19.s\n" "ld1rw { z18.s }, p0/Z, [%x[combined_rescale_value]]\n" + ".inst 0x4482826b // srshl z11.s, p0/M, z11.s, z19.s\n" + ".inst 0x4482826a // srshl z10.s, p0/M, z10.s, z19.s\n" "ld1rw { z17.s }, p0/Z, [%x[right_shift]]\n" - ".inst 0x448282af // srshl z15.s, p0/M, z15.s, z21.s\n" - ".inst 0x448282ae // srshl z14.s, p0/M, z14.s, z21.s\n" + ".inst 0x44828269 // srshl z9.s, p0/M, z9.s, z19.s\n" + ".inst 0x44828268 // srshl z8.s, p0/M, z8.s, z19.s\n" "ld1rw { z16.s }, p0/Z, [x20]\n" - ".inst 0x448282ad // srshl z13.s, p0/M, z13.s, z21.s\n" - ".inst 0x448282ac // srshl z12.s, p0/M, z12.s, z21.s\n" - ".inst 0x448282ab // srshl z11.s, p0/M, z11.s, z21.s\n" - ".inst 0x448282aa // srshl z10.s, p0/M, z10.s, z21.s\n" - ".inst 0x448282a9 // srshl z9.s, p0/M, z9.s, z21.s\n" - ".inst 0x448282a8 // srshl z8.s, p0/M, z8.s, z21.s\n" - ".inst 0x448282a7 // srshl z7.s, p0/M, z7.s, z21.s\n" - ".inst 0x448282a6 // srshl z6.s, p0/M, z6.s, z21.s\n" - ".inst 0x448282a5 // srshl z5.s, p0/M, z5.s, z21.s\n" - ".inst 0x448282a4 // srshl z4.s, p0/M, z4.s, z21.s\n" - ".inst 0x448282a3 // srshl z3.s, p0/M, z3.s, z21.s\n" - ".inst 0x448282a2 // srshl z2.s, p0/M, z2.s, z21.s\n" - ".inst 0x448282a1 // srshl z1.s, p0/M, z1.s, z21.s\n" - ".inst 0x448282a0 // srshl z0.s, p0/M, z0.s, z21.s\n" + ".inst 0x44828267 // srshl z7.s, p0/M, z7.s, z19.s\n" + ".inst 0x44828266 // srshl z6.s, p0/M, z6.s, z19.s\n" + ".inst 0x44828265 // srshl z5.s, p0/M, z5.s, z19.s\n" + ".inst 0x44828264 // srshl z4.s, p0/M, z4.s, z19.s\n" + ".inst 0x44828263 // srshl z3.s, p0/M, z3.s, z19.s\n" + ".inst 0x44828262 // srshl z2.s, p0/M, z2.s, z19.s\n" + ".inst 0x44828261 // srshl z1.s, p0/M, z1.s, z19.s\n" + ".inst 0x44828260 // srshl z0.s, p0/M, z0.s, z19.s\n" ".inst 0x04b275ef // sqrdmulh z15.s, z15.s, z18.s\n" ".inst 0x04b275ce // sqrdmulh z14.s, z14.s, z18.s\n" ".inst 0x04b275ad // sqrdmulh z13.s, z13.s, z18.s\n" @@ -331,52 +329,54 @@ void sme_u8q_nhwc_avg_generic_depthfirst_impl( "add z2.s, z2.s, z16.s\n" "add z1.s, z1.s, z16.s\n" "add z0.s, z0.s, z16.s\n" - "smax z15.s, p0/M, z15.s, z19.s\n" - "smax z14.s, p0/M, z14.s, z19.s\n" - "smax z13.s, p0/M, z13.s, z19.s\n" - "smax z12.s, p0/M, z12.s, z19.s\n" - "smax z11.s, p0/M, z11.s, z19.s\n" - "smax z10.s, p0/M, z10.s, z19.s\n" - "smax z9.s, p0/M, z9.s, z19.s\n" - "smax z8.s, p0/M, z8.s, z19.s\n" - "smax z7.s, p0/M, z7.s, z19.s\n" - "smax z6.s, p0/M, z6.s, z19.s\n" - "smax z5.s, p0/M, z5.s, z19.s\n" - "smax z4.s, p0/M, z4.s, z19.s\n" - "smax z3.s, p0/M, z3.s, z19.s\n" - "smax z2.s, p0/M, z2.s, z19.s\n" - "smax z1.s, p0/M, z1.s, z19.s\n" - "smax z0.s, p0/M, z0.s, z19.s\n" - "smin z15.s, p0/M, z15.s, z20.s\n" - "smin z14.s, p0/M, z14.s, z20.s\n" - "smin z13.s, p0/M, z13.s, z20.s\n" - "smin z12.s, p0/M, z12.s, z20.s\n" - "smin z11.s, p0/M, z11.s, z20.s\n" - "smin z10.s, p0/M, z10.s, z20.s\n" - "smin z9.s, p0/M, z9.s, z20.s\n" - "smin z8.s, p0/M, z8.s, z20.s\n" - "smin z7.s, p0/M, z7.s, z20.s\n" - "smin z6.s, p0/M, z6.s, z20.s\n" - "trn1 z19.h, z15.h, z14.h\n" - "smin z5.s, p0/M, z5.s, z20.s\n" - "smin z4.s, p0/M, z4.s, z20.s\n" + "mov z16.s, #0x0\n" + "mov z19.s, #0xff\n" + "smax z15.s, p0/M, z15.s, z16.s\n" + "smax z14.s, p0/M, z14.s, z16.s\n" + "smax z13.s, p0/M, z13.s, z16.s\n" + "smax z12.s, p0/M, z12.s, z16.s\n" + "smax z11.s, p0/M, z11.s, z16.s\n" + "smax z10.s, p0/M, z10.s, z16.s\n" + "smax z9.s, p0/M, z9.s, z16.s\n" + "smax z8.s, p0/M, z8.s, z16.s\n" + "smax z7.s, p0/M, z7.s, z16.s\n" + "smax z6.s, p0/M, z6.s, z16.s\n" + "smax z5.s, p0/M, z5.s, z16.s\n" + "smax z4.s, p0/M, z4.s, z16.s\n" + "smax z3.s, p0/M, z3.s, z16.s\n" + "smax z2.s, p0/M, z2.s, z16.s\n" + "smax z1.s, p0/M, z1.s, z16.s\n" + "smax z0.s, p0/M, z0.s, z16.s\n" + "smin z15.s, p0/M, z15.s, z19.s\n" + "smin z14.s, p0/M, z14.s, z19.s\n" + "trn1 z23.h, z15.h, z14.h\n" + "smin z13.s, p0/M, z13.s, z19.s\n" + "smin z12.s, p0/M, z12.s, z19.s\n" "trn1 z16.h, z13.h, z12.h\n" - "smin z3.s, p0/M, z3.s, z20.s\n" - "smin z2.s, p0/M, z2.s, z20.s\n" + "smin z11.s, p0/M, z11.s, z19.s\n" + "smin z10.s, p0/M, z10.s, z19.s\n" "trn1 z22.h, z11.h, z10.h\n" - "smin z1.s, p0/M, z1.s, z20.s\n" - "smin z0.s, p0/M, z0.s, z20.s\n" + "smin z9.s, p0/M, z9.s, z19.s\n" + "smin z8.s, p0/M, z8.s, z19.s\n" "trn1 z18.h, z9.h, z8.h\n" + "smin z7.s, p0/M, z7.s, z19.s\n" + "smin z6.s, p0/M, z6.s, z19.s\n" "trn1 z21.h, z7.h, z6.h\n" + "smin z5.s, p0/M, z5.s, z19.s\n" + "smin z4.s, p0/M, z4.s, z19.s\n" "trn1 z17.h, z5.h, z4.h\n" - "trn1 z20.b, z19.b, z16.b\n" - "trn1 z19.h, z3.h, z2.h\n" - "trn1 z16.h, z1.h, z0.h\n" + "smin z3.s, p0/M, z3.s, z19.s\n" + "smin z2.s, p0/M, z2.s, z19.s\n" + "trn1 z20.h, z3.h, z2.h\n" + "smin z1.s, p0/M, z1.s, z19.s\n" + "smin z0.s, p0/M, z0.s, z19.s\n" + "trn1 z19.h, z1.h, z0.h\n" + "trn1 z16.b, z23.b, z16.b\n" "trn1 z18.b, z22.b, z18.b\n" - "trn1 z17.b, z21.b, z17.b\n" - "st1b { z20.b }, p4, [%x[outptr], x27]\n" + "st1b { z16.b }, p4, [%x[outptr], x27]\n" "incb x27, ALL, MUL #4\n" - "trn1 z16.b, z19.b, z16.b\n" + "trn1 z17.b, z21.b, z17.b\n" + "trn1 z16.b, z20.b, z19.b\n" "st1b { z18.b }, p3, [%x[outptr], x26]\n" "incb x26, ALL, MUL #4\n" "st1b { z17.b }, p2, [%x[outptr], x25]\n" @@ -391,10 +391,10 @@ void sme_u8q_nhwc_avg_generic_depthfirst_impl( "8:" // Single vector of channels: Loop "ld1rw { z15.s }, p0/Z, [%x[accumulator_init]]\n" "lsr x23, %x[n_valid_cells], #0x1\n" - "mov x22, %x[inptrs]\n" "mov z14.d, z15.d\n" "mov z13.d, z15.d\n" "mov z12.d, z15.d\n" + "mov x22, %x[inptrs]\n" "cbz x23, 11f\n" "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" @@ -407,13 +407,13 @@ void sme_u8q_nhwc_avg_generic_depthfirst_impl( ".inst 0x455e0ff0 // uaddlt z16.h, z31.b, z30.b\n" "ldp x21, x20, [x22, #0x0]\n" "subs x23, x23, #0x1\n" - "add x22, x22, #0x10\n" - "ld1b { z31.b }, p4/Z, [x21, x27]\n" ".inst 0x459149ef // uaddwb z15.s, z15.s, z17.h\n" ".inst 0x45914dce // uaddwt z14.s, z14.s, z17.h\n" - "ld1b { z30.b }, p4/Z, [x20, x27]\n" + "add x22, x22, #0x10\n" + "ld1b { z31.b }, p4/Z, [x21, x27]\n" ".inst 0x459049ad // uaddwb z13.s, z13.s, z16.h\n" ".inst 0x45904d8c // uaddwt z12.s, z12.s, z16.h\n" + "ld1b { z30.b }, p4/Z, [x20, x27]\n" "bgt 9b\n" "10:" // Single vector of channels: Loop: 2 inputs tail ".inst 0x455e0bf1 // uaddlb z17.h, z31.b, z30.b\n" @@ -427,31 +427,29 @@ void sme_u8q_nhwc_avg_generic_depthfirst_impl( "beq 13f\n" "12:" // Single vector of channels: Loop: Single input loop "ldr x20, [x22], #0x8\n" - "subs x21, x21, #0x1\n" "ld1b { z16.b }, p4/Z, [x20, x27]\n" ".inst 0x4508aa11 // ushllb z17.h, z16.b, #0x0\n" ".inst 0x4508ae10 // ushllt z16.h, z16.b, #0x0\n" + "subs x21, x21, #0x1\n" ".inst 0x459149ef // uaddwb z15.s, z15.s, z17.h\n" ".inst 0x45914dce // uaddwt z14.s, z14.s, z17.h\n" ".inst 0x459049ad // uaddwb z13.s, z13.s, z16.h\n" ".inst 0x45904d8c // uaddwt z12.s, z12.s, z16.h\n" "bgt 12b\n" "13:" // Single vector of channels: Loop: Single input loop: End - "ld1rw { z21.s }, p0/Z, [%x[left_shift]]\n" + "ld1rw { z16.s }, p0/Z, [%x[left_shift]]\n" + ".inst 0x4482820f // srshl z15.s, p0/M, z15.s, z16.s\n" + ".inst 0x4482820e // srshl z14.s, p0/M, z14.s, z16.s\n" "add x20, %x[quant_params], %[offsetof_qp_output_offset]\n" - "mov z20.s, #0x0\n" - "mov z19.s, #0xff\n" - "ld1rw { z18.s }, p0/Z, [%x[combined_rescale_value]]\n" + ".inst 0x4482820d // srshl z13.s, p0/M, z13.s, z16.s\n" + ".inst 0x4482820c // srshl z12.s, p0/M, z12.s, z16.s\n" + "ld1rw { z16.s }, p0/Z, [%x[combined_rescale_value]]\n" + ".inst 0x04b075ef // sqrdmulh z15.s, z15.s, z16.s\n" + ".inst 0x04b075ce // sqrdmulh z14.s, z14.s, z16.s\n" "ld1rw { z17.s }, p0/Z, [%x[right_shift]]\n" - ".inst 0x448282af // srshl z15.s, p0/M, z15.s, z21.s\n" - ".inst 0x448282ae // srshl z14.s, p0/M, z14.s, z21.s\n" + ".inst 0x04b075ad // sqrdmulh z13.s, z13.s, z16.s\n" + ".inst 0x04b0758c // sqrdmulh z12.s, z12.s, z16.s\n" "ld1rw { z16.s }, p0/Z, [x20]\n" - ".inst 0x448282ad // srshl z13.s, p0/M, z13.s, z21.s\n" - ".inst 0x448282ac // srshl z12.s, p0/M, z12.s, z21.s\n" - ".inst 0x04b275ef // sqrdmulh z15.s, z15.s, z18.s\n" - ".inst 0x04b275ce // sqrdmulh z14.s, z14.s, z18.s\n" - ".inst 0x04b275ad // sqrdmulh z13.s, z13.s, z18.s\n" - ".inst 0x04b2758c // sqrdmulh z12.s, z12.s, z18.s\n" ".inst 0x4482822f // srshl z15.s, p0/M, z15.s, z17.s\n" ".inst 0x4482822e // srshl z14.s, p0/M, z14.s, z17.s\n" ".inst 0x4482822d // srshl z13.s, p0/M, z13.s, z17.s\n" @@ -460,15 +458,17 @@ void sme_u8q_nhwc_avg_generic_depthfirst_impl( "add z14.s, z14.s, z16.s\n" "add z13.s, z13.s, z16.s\n" "add z12.s, z12.s, z16.s\n" - "smax z15.s, p0/M, z15.s, z20.s\n" - "smax z14.s, p0/M, z14.s, z20.s\n" - "smax z13.s, p0/M, z13.s, z20.s\n" - "smax z12.s, p0/M, z12.s, z20.s\n" - "smin z15.s, p0/M, z15.s, z19.s\n" - "smin z14.s, p0/M, z14.s, z19.s\n" - "smin z13.s, p0/M, z13.s, z19.s\n" - "smin z12.s, p0/M, z12.s, z19.s\n" + "mov z17.s, #0x0\n" + "mov z16.s, #0xff\n" + "smax z15.s, p0/M, z15.s, z17.s\n" + "smax z14.s, p0/M, z14.s, z17.s\n" + "smax z13.s, p0/M, z13.s, z17.s\n" + "smax z12.s, p0/M, z12.s, z17.s\n" + "smin z15.s, p0/M, z15.s, z16.s\n" + "smin z14.s, p0/M, z14.s, z16.s\n" "trn1 z17.h, z15.h, z14.h\n" + "smin z13.s, p0/M, z13.s, z16.s\n" + "smin z12.s, p0/M, z12.s, z16.s\n" "trn1 z16.h, z13.h, z12.h\n" "trn1 z16.b, z17.b, z16.b\n" "st1b { z16.b }, p4, [%x[outptr], x27]\n" |