diff options
Diffstat (limited to 'src/core/NEON/kernels/convolution/winograd/output_transforms')
8 files changed, 2253 insertions, 0 deletions
diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/a64_fp16_4x4_3x3.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/a64_fp16_4x4_3x3.cpp new file mode 100644 index 0000000000..295005a2ee --- /dev/null +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/a64_fp16_4x4_3x3.cpp @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2022, 2024 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + +#include <algorithm> +#include <arm_neon.h> +#include <cstddef> + +namespace arm_conv { +namespace winograd { +namespace output_transform { + +void a64_fp16_4x4_3x3( + unsigned int n_channels, + const __fp16* inptr, + size_t matrix_stride, + const __fp16* bptr, + __fp16* const output, + size_t output_row_stride, + size_t output_col_stride, + __fp16 output_min, + __fp16 output_max +) +{ + constexpr int output_tile_rows = 4, output_tile_cols = 4; + + // Construct a map to the output cells + __fp16 *outptrs[output_tile_rows][output_tile_cols]; + for (int i = 0; i < output_tile_rows; i++) + { + for (int j = 0; j < output_tile_cols; j++) + { + outptrs[i][j] = output + i*output_row_stride + j*output_col_stride; + } + } + + // For each channel of the output + int channels_remaining = n_channels; + +#ifdef __aarch64__ + for (; channels_remaining >= 8; channels_remaining -= 8) + { + // Matrices used and computed during this transform + float16x8_t F[6][6], FZ[6][4], f[4][4], b; + + // Read a 6x6 tile in the Winograd domain + for (int i = 0, m = 0; i < 6; i++) + { + for (int j = 0; j < 6; j++, m++) + { + F[i][j] = vld1q_f16(inptr + m*matrix_stride); + } + } + inptr += 8; + + // Compute the matrix F Z + for (int i = 0; i < 6; i++) + { + // FZ[i][0] = 1*F[i][0] + 1*F[i][1] + 1*F[i][2] + 1*F[i][3] + 1*F[i][4]; + FZ[i][0] = vaddq_f16(vaddq_f16(vaddq_f16(F[i][0], F[i][1]), vaddq_f16(F[i][2], F[i][3])), F[i][4]); + + // FZ[i][1] = 1*F[i][1] + -1*F[i][2] + 2*F[i][3] + -2*F[i][4]; + FZ[i][1] = vaddq_f16(vsubq_f16(F[i][1], F[i][2]), vmulq_f16(vsubq_f16(F[i][3], F[i][4]), vdupq_n_f16(2.0f))); + + // FZ[i][2] = 1*F[i][1] + 1*F[i][2] + 4*F[i][3] + 4*F[i][4]; + FZ[i][2] = vaddq_f16(vaddq_f16(F[i][1], F[i][2]), vmulq_f16(vaddq_f16(F[i][3], F[i][4]), vdupq_n_f16(4.0f))); + + // FZ[i][3] = 1*F[i][1] + -1*F[i][2] + 8*F[i][3] + -8*F[i][4] + 1*F[i][5]; + FZ[i][3] = vaddq_f16(vaddq_f16(vsubq_f16(F[i][1], F[i][2]), vmulq_f16(vsubq_f16(F[i][3], F[i][4]), vdupq_n_f16(8.0f))), F[i][5]); + } + + // Compute the output tile f = ZT F Z + for (int j = 0; j < 4; j++) + { + // f[0][j] = 1*FZ[0][j] + 1*FZ[1][j] + 1*FZ[2][j] + 1*FZ[3][j] + 1*FZ[4][j]; + f[0][j] = vaddq_f16(vaddq_f16(vaddq_f16(FZ[0][j], FZ[1][j]), vaddq_f16(FZ[2][j], FZ[3][j])), FZ[4][j]); + + // f[1][j] = 1*FZ[1][j] + -1*FZ[2][j] + 2*FZ[3][j] + -2*FZ[4][j]; + f[1][j] = vaddq_f16(vsubq_f16(FZ[1][j], FZ[2][j]), vmulq_f16(vsubq_f16(FZ[3][j], FZ[4][j]), vdupq_n_f16(2.0f))); + + // f[2][j] = 1*FZ[1][j] + 1*FZ[2][j] + 4*FZ[3][j] + 4*FZ[4][j]; + f[2][j] = vaddq_f16(vaddq_f16(FZ[1][j], FZ[2][j]), vmulq_f16(vaddq_f16(FZ[3][j], FZ[4][j]), vdupq_n_f16(4.0f))); + + // f[3][j] = 1*FZ[1][j] + -1*FZ[2][j] + 8*FZ[3][j] + -8*FZ[4][j] + 1*FZ[5][j]; + f[3][j] = vaddq_f16(vaddq_f16(vsubq_f16(FZ[1][j], FZ[2][j]), vmulq_f16(vsubq_f16(FZ[3][j], FZ[4][j]), vdupq_n_f16(8.0f))), FZ[5][j]); + } + + // Write out the output tile + if (bptr != nullptr) + { + b = vld1q_f16(bptr); + bptr += 8; + } + else + { + b = vdupq_n_f16(0.0f); + } + for (int i = 0; i < output_tile_rows; i++) + { + for (int j = 0; j < output_tile_cols; j++) + { + const auto y = + vmaxq_f16(vminq_f16(vaddq_f16(f[i][j], b), vdupq_n_f16(output_max)), + vdupq_n_f16(output_min)); + vst1q_f16(outptrs[i][j], y); + outptrs[i][j] += 8; + } + } + } +#endif // __aarch64__ +#ifdef __arm_any__ + for (; channels_remaining >= 4; channels_remaining -= 4) + { + // Matrices used and computed during this transform + float16x4_t F[6][6], FZ[6][4], f[4][4], b; + + // Read a 6x6 tile in the Winograd domain + for (int i = 0, m = 0; i < 6; i++) + { + for (int j = 0; j < 6; j++, m++) + { + F[i][j] = vld1_f16(inptr + m*matrix_stride); + } + } + inptr += 4; + + // Compute the matrix F Z + for (int i = 0; i < 6; i++) + { + // FZ[i][0] = 1*F[i][0] + 1*F[i][1] + 1*F[i][2] + 1*F[i][3] + 1*F[i][4]; + FZ[i][0] = vadd_f16(vadd_f16(vadd_f16(F[i][0], F[i][1]), vadd_f16(F[i][2], F[i][3])), F[i][4]); + + // FZ[i][1] = 1*F[i][1] + -1*F[i][2] + 2*F[i][3] + -2*F[i][4]; + FZ[i][1] = vadd_f16(vsub_f16(F[i][1], F[i][2]), vmul_f16(vsub_f16(F[i][3], F[i][4]), vdup_n_f16(2.0f))); + + // FZ[i][2] = 1*F[i][1] + 1*F[i][2] + 4*F[i][3] + 4*F[i][4]; + FZ[i][2] = vadd_f16(vadd_f16(F[i][1], F[i][2]), vmul_f16(vadd_f16(F[i][3], F[i][4]), vdup_n_f16(4.0f))); + + // FZ[i][3] = 1*F[i][1] + -1*F[i][2] + 8*F[i][3] + -8*F[i][4] + 1*F[i][5]; + FZ[i][3] = vadd_f16(vadd_f16(vsub_f16(F[i][1], F[i][2]), vmul_f16(vsub_f16(F[i][3], F[i][4]), vdup_n_f16(8.0f))), F[i][5]); + } + + // Compute the output tile f = ZT F Z + for (int j = 0; j < 4; j++) + { + // f[0][j] = 1*FZ[0][j] + 1*FZ[1][j] + 1*FZ[2][j] + 1*FZ[3][j] + 1*FZ[4][j]; + f[0][j] = vadd_f16(vadd_f16(vadd_f16(FZ[0][j], FZ[1][j]), vadd_f16(FZ[2][j], FZ[3][j])), FZ[4][j]); + + // f[1][j] = 1*FZ[1][j] + -1*FZ[2][j] + 2*FZ[3][j] + -2*FZ[4][j]; + f[1][j] = vadd_f16(vsub_f16(FZ[1][j], FZ[2][j]), vmul_f16(vsub_f16(FZ[3][j], FZ[4][j]), vdup_n_f16(2.0f))); + + // f[2][j] = 1*FZ[1][j] + 1*FZ[2][j] + 4*FZ[3][j] + 4*FZ[4][j]; + f[2][j] = vadd_f16(vadd_f16(FZ[1][j], FZ[2][j]), vmul_f16(vadd_f16(FZ[3][j], FZ[4][j]), vdup_n_f16(4.0f))); + + // f[3][j] = 1*FZ[1][j] + -1*FZ[2][j] + 8*FZ[3][j] + -8*FZ[4][j] + 1*FZ[5][j]; + f[3][j] = vadd_f16(vadd_f16(vsub_f16(FZ[1][j], FZ[2][j]), vmul_f16(vsub_f16(FZ[3][j], FZ[4][j]), vdup_n_f16(8.0f))), FZ[5][j]); + } + + // Write out the output tile + if (bptr != nullptr) + { + b = vld1_f16(bptr); + bptr += 4; + } + else + { + b = vdup_n_f16(0.0f); + } + for (int i = 0; i < output_tile_rows; i++) + { + for (int j = 0; j < output_tile_cols; j++) + { + const auto y = + vmax_f16(vmin_f16(vadd_f16(f[i][j], b), vdup_n_f16(output_max)), + vdup_n_f16(output_min)); + vst1_f16(outptrs[i][j], y); + outptrs[i][j] += 4; + } + } + } +#endif // __arm_any__ + for (; channels_remaining; channels_remaining--) + { + // Matrices used and computed during this transform + __fp16 F[6][6], FZ[6][4], f[4][4], b; + + // Read a 6x6 tile in the Winograd domain + for (int i = 0, m = 0; i < 6; i++) + { + for (int j = 0; j < 6; j++, m++) + { + F[i][j] = *(inptr + m*matrix_stride); + } + } + inptr++; + + // Compute the matrix F Z + for (int i = 0; i < 6; i++) + { + FZ[i][0] = 1*F[i][0] + 1*F[i][1] + 1*F[i][2] + 1*F[i][3] + 1*F[i][4]; + FZ[i][1] = 1*F[i][1] + -1*F[i][2] + 2*F[i][3] + -2*F[i][4]; + FZ[i][2] = 1*F[i][1] + 1*F[i][2] + 4*F[i][3] + 4*F[i][4]; + FZ[i][3] = 1*F[i][1] + -1*F[i][2] + 8*F[i][3] + -8*F[i][4] + 1*F[i][5]; + } + + // Compute the output tile f = ZT F Z + for (int j = 0; j < 4; j++) + { + f[0][j] = 1*FZ[0][j] + 1*FZ[1][j] + 1*FZ[2][j] + 1*FZ[3][j] + 1*FZ[4][j]; + f[1][j] = 1*FZ[1][j] + -1*FZ[2][j] + 2*FZ[3][j] + -2*FZ[4][j]; + f[2][j] = 1*FZ[1][j] + 1*FZ[2][j] + 4*FZ[3][j] + 4*FZ[4][j]; + f[3][j] = 1*FZ[1][j] + -1*FZ[2][j] + 8*FZ[3][j] + -8*FZ[4][j] + 1*FZ[5][j]; + } + + // Write out the output tile + if (bptr != nullptr) + { + b = *(bptr++); + } + else + { + b = 0.0f; + } + for (int i = 0; i < output_tile_rows; i++) + { + for (int j = 0; j < output_tile_cols; j++) + { + const auto y = std::max(std::min<__fp16>(f[i][j] + b, output_max), output_min); + *(outptrs[i][j]++) = y; + } + } + } +} + +} // namespace output_transform +} // namespace winograd +} // namespace arm_conv + +#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x2_1x7.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x2_1x7.cpp new file mode 100644 index 0000000000..8c6cf9725e --- /dev/null +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x2_1x7.cpp @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2022-2024 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <algorithm> +#include <cstddef> +#include <arm_neon.h> + +namespace arm_conv { +namespace winograd { +namespace output_transform { + +void arm_fp32_1x2_1x7( + unsigned int n_channels, + const float* inptr, + size_t matrix_stride, + const float* bptr, + float *outptr, + size_t, // No need to stride across rows + size_t output_col_stride, + float output_min, + float output_max +) +{ + constexpr auto inner_tile_cols = 8u, output_tile_cols = 2u; + + // For each channel of the output + for (; n_channels >= 4; n_channels -= 4) + { + // Matrices used and computed during this transform + float32x4_t F[inner_tile_cols], f[output_tile_cols], b = vdupq_n_f32(0.0f); + + // Read a 1x8 tile in the Winograd domain + for (auto j = 0u; j < inner_tile_cols; j++) + { + F[j] = vld1q_f32(inptr + j*matrix_stride); + } + inptr += 4; + + f[0] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[6], 1), F[5], 1), F[4], 1), F[3], 1), F[2], 1), F[1], 1), F[0], 1); + f[1] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[7], 1), F[2], 1), F[6], 3), F[4], 2), F[3], -2), F[5], -3), F[1], -1); + + // Write out the output tile + if (bptr != 0) + { + b = vld1q_f32(bptr); + bptr += 4; + } + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = vminq_f32(vmaxq_f32(f[j] + b, vdupq_n_f32(output_min)), + vdupq_n_f32(output_max)); + vst1q_f32(outptr + j*output_col_stride, y); + } + outptr += 4; + } + for (; n_channels >= 2; n_channels -= 2) + { + // Matrices used and computed during this transform + float32x2_t F[inner_tile_cols], f[output_tile_cols], b = vdup_n_f32(0.0f); + + // Read a 1x8 tile in the Winograd domain + for (auto j = 0u; j < inner_tile_cols; j++) + { + F[j] = vld1_f32(inptr + j*matrix_stride); + } + inptr += 2; + + f[0] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[6], 1), F[5], 1), F[4], 1), F[3], 1), F[2], 1), F[1], 1), F[0], 1); + f[1] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[7], 1), F[2], 1), F[6], 3), F[4], 2), F[3], -2), F[5], -3), F[1], -1); + + // Write out the output tile + if (bptr != 0) + { + b = vld1_f32(bptr); + bptr += 2; + } + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = vmin_f32(vmax_f32(f[j] + b, vdup_n_f32(output_min)), + vdup_n_f32(output_max)); + vst1_f32(outptr + j*output_col_stride, y); + } + outptr += 2; + } + if (n_channels) + { + // Matrices used and computed during this transform + float F[inner_tile_cols], f[output_tile_cols], b = 0.0f; + + // Read a 1x8 tile in the Winograd domain + for (auto j = 0u; j < inner_tile_cols; j++) + { + F[j] = *(inptr + j*matrix_stride); + } + + f[0] = F[0]*1 + F[1]*1 + F[2]*1 + F[3]*1 + F[4]*1 + F[5]*1 + F[6]*1; + f[1] = F[1]*-1 + F[5]*-3 + F[3]*-2 + F[4]*2 + F[6]*3 + F[2]*1 + F[7]*1; + + // Write out the output tile + if (bptr != 0) + { + b = *(bptr++); + } + for (auto j = 0u; j < output_tile_cols; j++) + { + *(outptr + j*output_col_stride) = std::max(std::min(f[j] + b, output_max), output_min); + } + } +} + +} // namespace output_transform +} // namespace winograd +} // namespace arm_conv diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x4_1x5.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x4_1x5.cpp new file mode 100644 index 0000000000..ac05f23221 --- /dev/null +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x4_1x5.cpp @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2022-2024 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <algorithm> +#include <cstddef> +#include <arm_neon.h> + +namespace arm_conv { +namespace winograd { +namespace output_transform { + +void arm_fp32_1x4_1x5( + unsigned int n_channels, + const float* inptr, + size_t matrix_stride, + const float* bptr, + float *outptr, + size_t, // No need to stride across rows + size_t output_col_stride, + float output_min, + float output_max +) +{ + constexpr auto inner_tile_cols = 8u, output_tile_cols = 4u; + + // For each channel of the output + for (; n_channels >= 4; n_channels -= 4) + { + // Matrices used and computed during this transform + float32x4_t F[inner_tile_cols], f[output_tile_cols], b = vdupq_n_f32(0.0f); + + // Read a 1x8 tile in the Winograd domain + for (auto j = 0u; j < inner_tile_cols; j++) + { + F[j] = vld1q_f32(inptr + j*matrix_stride); + } + inptr += 4; + + f[0] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[6], 1), F[5], 1), F[4], 1), F[3], 1), F[2], 1), F[1], 1), F[0], 1); + f[1] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[2], 1), F[6], 3), F[4], 2), F[3], -2), F[5], -3), F[1], -1); + f[2] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[2], 1), F[1], 1), F[6], 9), F[5], 9), F[4], 4), F[3], 4); + f[3] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[7], 1), F[2], 1), F[6], 27), F[4], 8), F[3], -8), F[5], -27), F[1], -1); + + // Write out the output tile + if (bptr != 0) + { + b = vld1q_f32(bptr); + bptr += 4; + } + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = + vmaxq_f32(vminq_f32(vaddq_f32(f[j], b), vdupq_n_f32(output_max)), + vdupq_n_f32(output_min)); + vst1q_f32(outptr + j*output_col_stride, y); + } + outptr += 4; + } + for (; n_channels >= 2; n_channels -= 2) + { + // Matrices used and computed during this transform + float32x2_t F[inner_tile_cols], f[output_tile_cols], b = vdup_n_f32(0.0f); + + // Read a 1x8 tile in the Winograd domain + for (auto j = 0u; j < inner_tile_cols; j++) + { + F[j] = vld1_f32(inptr + j*matrix_stride); + } + inptr += 2; + + f[0] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[6], 1), F[5], 1), F[4], 1), F[3], 1), F[2], 1), F[1], 1), F[0], 1); + f[1] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[2], 1), F[6], 3), F[4], 2), F[3], -2), F[5], -3), F[1], -1); + f[2] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[2], 1), F[1], 1), F[6], 9), F[5], 9), F[4], 4), F[3], 4); + f[3] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[7], 1), F[2], 1), F[6], 27), F[4], 8), F[3], -8), F[5], -27), F[1], -1); + + // Write out the output tile + if (bptr != 0) + { + b = vld1_f32(bptr); + bptr += 2; + } + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = + vmax_f32(vmin_f32(vadd_f32(f[j], b), vdup_n_f32(output_max)), + vdup_n_f32(output_min)); + vst1_f32(outptr + j*output_col_stride, y); + } + outptr += 2; + } + for (; n_channels; n_channels--) + { + // Matrices used and computed during this transform + float F[inner_tile_cols], f[output_tile_cols], b = 0.0f; + + // Read a 1x8 tile in the Winograd domain + for (auto j = 0u; j < inner_tile_cols; j++) + { + F[j] = *(inptr + j*matrix_stride); + } + inptr++; + + f[0] = F[0]*1 + F[1]*1 + F[2]*1 + F[3]*1 + F[4]*1 + F[5]*1 + F[6]*1; + f[1] = F[1]*-1 + F[5]*-3 + F[3]*-2 + F[4]*2 + F[6]*3 + F[2]*1; + f[2] = F[3]*4 + F[4]*4 + F[5]*9 + F[6]*9 + F[1]*1 + F[2]*1; + f[3] = F[1]*-1 + F[5]*-27 + F[3]*-8 + F[4]*8 + F[6]*27 + F[2]*1 + F[7]*1; + + // Write out the output tile + if (bptr != 0) + { + b = *(bptr++); + } + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = std::max(std::min(f[j] + b, output_max), output_min); + *(outptr + j*output_col_stride) = y; + } + outptr++; + } +} + +} // namespace output_transform +} // namespace winograd +} // namespace arm_conv diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x6_1x3.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x6_1x3.cpp new file mode 100644 index 0000000000..154dc6fe1a --- /dev/null +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_1x6_1x3.cpp @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2022-2024 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <algorithm> +#include <cstddef> + +#include <arm_neon.h> + +namespace arm_conv { +namespace winograd { +namespace output_transform { + +void arm_fp32_1x6_1x3( + unsigned int n_channels, + const float* inptr, + size_t matrix_stride, + const float* bptr, + float *outptr, + size_t, // No need to stride across rows + size_t output_col_stride, + float output_min, + float output_max +) +{ + constexpr unsigned int inner_tile_cols = 8, output_tile_cols = 6; + + // For each channel of the output + for (; n_channels >= 4; n_channels -= 4) + { + // Matrices used and computed during this transform + float32x4_t F[inner_tile_cols], f[output_tile_cols], b = vdupq_n_f32(0.0f); + + // Read a 1x8 tile in the Winograd domain + for (auto j = 0u; j < inner_tile_cols; j++) + { + F[j] = vld1q_f32(inptr + j*matrix_stride); + } + inptr += 4; + + f[0] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[6], 1), F[5], 1), F[4], 1), F[3], 1), F[2], 1), F[1], 1), F[0], 1); + f[1] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[2], 1), F[6], 3), F[4], 2), F[3], -2), F[5], -3), F[1], -1); + f[2] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[2], 1), F[1], 1), F[6], 9), F[5], 9), F[4], 4), F[3], 4); + f[3] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[2], 1), F[6], 27), F[4], 8), F[3], -8), F[5], -27), F[1], -1); + f[4] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[2], 1), F[1], 1), F[6], 81), F[5], 81), F[4], 16), F[3], 16); + f[5] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[7], 1), F[2], 1), F[6], 243), F[4], 32), F[3], -32), F[5], -243), F[1], -1); + + // Write out the output tile + if (bptr != 0) + { + b = vld1q_f32(bptr); + bptr += 4; + } + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = vminq_f32(vmaxq_f32(f[j] + b, vdupq_n_f32(output_min)), + vdupq_n_f32(output_max)); + vst1q_f32(outptr + j*output_col_stride, y); + } + outptr += 4; + } + for (; n_channels >= 2; n_channels -= 2) + { + // Matrices used and computed during this transform + float32x2_t F[inner_tile_cols], f[output_tile_cols], b = vdup_n_f32(0.0f); + + // Read a 1x8 tile in the Winograd domain + for (auto j = 0u; j < inner_tile_cols; j++) + { + F[j] = vld1_f32(inptr + j*matrix_stride); + } + inptr += 2; + + f[0] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[6], 1), F[5], 1), F[4], 1), F[3], 1), F[2], 1), F[1], 1), F[0], 1); + f[1] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[2], 1), F[6], 3), F[4], 2), F[3], -2), F[5], -3), F[1], -1); + f[2] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[2], 1), F[1], 1), F[6], 9), F[5], 9), F[4], 4), F[3], 4); + f[3] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[2], 1), F[6], 27), F[4], 8), F[3], -8), F[5], -27), F[1], -1); + f[4] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[2], 1), F[1], 1), F[6], 81), F[5], 81), F[4], 16), F[3], 16); + f[5] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[7], 1), F[2], 1), F[6], 243), F[4], 32), F[3], -32), F[5], -243), F[1], -1); + + // Write out the output tile + if (bptr != 0) + { + b = vld1_f32(bptr); + bptr += 2; + } + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = vmin_f32(vmax_f32(f[j] + b, vdup_n_f32(output_min)), + vdup_n_f32(output_max)); + vst1_f32(outptr + j*output_col_stride, y); + } + outptr += 2; + } + for (; n_channels; n_channels--) + { + // Matrices used and computed during this transform + float F[inner_tile_cols], f[output_tile_cols], b = 0.0f; + + // Read a 1x8 tile in the Winograd domain + for (auto j = 0u; j < inner_tile_cols; j++) + { + F[j] = *(inptr + j*matrix_stride); + } + inptr++; + + f[0] = F[0]*1 + F[1]*1 + F[2]*1 + F[3]*1 + F[4]*1 + F[5]*1 + F[6]*1; + f[1] = F[1]*-1 + F[5]*-3 + F[3]*-2 + F[4]*2 + F[6]*3 + F[2]*1; + f[2] = F[3]*4 + F[4]*4 + F[5]*9 + F[6]*9 + F[1]*1 + F[2]*1; + f[3] = F[1]*-1 + F[5]*-27 + F[3]*-8 + F[4]*8 + F[6]*27 + F[2]*1; + f[4] = F[3]*16 + F[4]*16 + F[5]*81 + F[6]*81 + F[1]*1 + F[2]*1; + f[5] = F[1]*-1 + F[5]*-243 + F[3]*-32 + F[4]*32 + F[6]*243 + F[2]*1 + F[7]*1; + + // Write out the output tile + if (bptr != 0) + { + b = *(bptr++); + } + for (auto j = 0u; j < output_tile_cols; j++) + { + *(outptr + j*output_col_stride) = std::max(std::min(f[j] + b, output_max), output_min); + } + outptr++; + } +} + +} // namespace output_transform +} // namespace winograd +} // namespace arm_conv diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_2x2_3x3.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_2x2_3x3.cpp new file mode 100644 index 0000000000..28f042bcbf --- /dev/null +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_2x2_3x3.cpp @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2022, 2024 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <algorithm> +#include <cstddef> +#include <arm_neon.h> + +namespace arm_conv { +namespace winograd { +namespace output_transform { + +void arm_fp32_2x2_3x3( + unsigned int n_channels, + const float* inptr, + size_t matrix_stride, + const float* bptr, + float *outptr, + size_t output_row_stride, + size_t output_col_stride, + float output_min, + float output_max +) +{ + constexpr auto output_tile_rows = 2u, output_tile_cols = 2u; + + // For each channel of the output + for (; n_channels >= 4; n_channels -= 4) + { + // Matrices used and computed during this transform + float32x4_t F[4][4], FZ[4][2], f[2][2], b; + + // Read a 4x4 tile in the Winograd domain + for (auto i = 0u, m = 0u; i < 4; i++) + { + for (auto j = 0u; j < 4; j++, m++) + { + F[i][j] = vld1q_f32(inptr + m*matrix_stride); + } + } + inptr += 4; + + // Compute the matrix F Z + for (auto i = 0u; i < 4; i++) + { + // FZ[i][0] = F[i][0] + F[i][1] + F[i][2]; + FZ[i][0] = vaddq_f32(vaddq_f32(F[i][0], F[i][1]), F[i][2]); + + // FZ[i][1] = F[i][1] - F[i][2] - F[i][3]; + FZ[i][1] = vsubq_f32(vsubq_f32(F[i][1], F[i][2]), F[i][3]); + } + + // Compute the output tile f = ZT F Z + for (auto j = 0u; j < 2; j++) + { + // f[0][j] = FZ[0][j] + FZ[1][j] + FZ[2][j]; + f[0][j] = vaddq_f32(vaddq_f32(FZ[0][j], FZ[1][j]), FZ[2][j]); + + // f[1][j] = FZ[1][j] - FZ[2][j] - FZ[3][j]; + f[1][j] = vsubq_f32(vsubq_f32(FZ[1][j], FZ[2][j]), FZ[3][j]); + } + + // Load the bias vector + if (bptr != nullptr) + { + b = vld1q_f32(bptr); + bptr += 4; + } + else + { + b = vdupq_n_f32(0.0f); + } + + // Write out the output tile + for (auto i = 0u; i < output_tile_rows; i++) + { + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = + vmaxq_f32(vminq_f32(vaddq_f32(f[i][j], b), vdupq_n_f32(output_max)), + vdupq_n_f32(output_min)); + vst1q_f32(outptr + i*output_row_stride + j*output_col_stride, y); + } + } + outptr += 4; + } + for (; n_channels >= 2; n_channels -= 2) + { + // Matrices used and computed during this transform + float32x2_t F[4][4], FZ[4][2], f[2][2], b; + + // Read a 4x4 tile in the Winograd domain + for (auto i = 0u, m = 0u; i < 4; i++) + { + for (auto j = 0u; j < 4; j++, m++) + { + F[i][j] = vld1_f32(inptr + m*matrix_stride); + } + } + inptr += 2; + + // Compute the matrix F Z + for (auto i = 0u; i < 4; i++) + { + // FZ[i][0] = F[i][0] + F[i][1] + F[i][2]; + FZ[i][0] = vadd_f32(vadd_f32(F[i][0], F[i][1]), F[i][2]); + + // FZ[i][1] = F[i][1] - F[i][2] - F[i][3]; + FZ[i][1] = vsub_f32(vsub_f32(F[i][1], F[i][2]), F[i][3]); + } + + // Compute the output tile f = ZT F Z + for (auto j = 0u; j < 2; j++) + { + // f[0][j] = FZ[0][j] + FZ[1][j] + FZ[2][j]; + f[0][j] = vadd_f32(vadd_f32(FZ[0][j], FZ[1][j]), FZ[2][j]); + + // f[1][j] = FZ[1][j] - FZ[2][j] - FZ[3][j]; + f[1][j] = vsub_f32(vsub_f32(FZ[1][j], FZ[2][j]), FZ[3][j]); + } + + // Load the bias vector + if (bptr != nullptr) + { + b = vld1_f32(bptr); + bptr += 2; + } + else + { + b = vdup_n_f32(0.0f); + } + + // Write out the output tile + for (auto i = 0u; i < output_tile_rows; i++) + { + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = + vmax_f32(vmin_f32(vadd_f32(f[i][j], b), vdup_n_f32(output_max)), + vdup_n_f32(output_min)); + vst1_f32(outptr + i*output_row_stride + j*output_col_stride, y); + } + } + outptr += 2; + } + for (; n_channels; n_channels--) + { + // Matrices used and computed during this transform + float F[4][4], FZ[4][2], f[2][2], b; + + // Read a 4x4 tile in the Winograd domain + for (auto i = 0u, m = 0u; i < 4; i++) + { + for (auto j = 0u; j < 4; j++, m++) + { + F[i][j] = *(inptr + m*matrix_stride); + } + } + inptr++; + + // Compute the matrix F Z + for (auto i = 0u; i < 4; i++) + { + FZ[i][0] = F[i][0] + F[i][1] + F[i][2]; + FZ[i][1] = F[i][1] - F[i][2] - F[i][3]; + } + + // Compute the output tile f = ZT F Z + for (auto j = 0u; j < 2; j++) + { + f[0][j] = FZ[0][j] + FZ[1][j] + FZ[2][j]; + f[1][j] = FZ[1][j] - FZ[2][j] - FZ[3][j]; + } + + // Load the bias + if (bptr != nullptr) + { + b = *(bptr++); + } + else + { + b = 0.0f; + } + + // Write out the output tile + for (auto i = 0u; i < output_tile_rows; i++) + { + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = std::max(std::min(f[i][j] + b, output_max), output_min); + *(outptr + i*output_row_stride + j*output_col_stride) = y; + } + } + outptr++; + } +} + +} // namespace output_transform +} // namespace winograd +} // namespace arm_conv diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_2x2_5x5.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_2x2_5x5.cpp new file mode 100644 index 0000000000..8e5ba74ac3 --- /dev/null +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_2x2_5x5.cpp @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2022, 2024 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <algorithm> +#include <cstddef> +#include <arm_neon.h> + +namespace arm_conv { +namespace winograd { +namespace output_transform { + +void arm_fp32_2x2_5x5( + unsigned int n_channels, + const float* inptr, + size_t matrix_stride, + const float* bptr, + float *outptr, + size_t output_row_stride, + size_t output_col_stride, + float output_min, + float output_max +) +{ + constexpr auto output_tile_rows = 2u, output_tile_cols = 2u; + + // For each channel of the output + for (; n_channels >= 4; n_channels -= 4) + { + // Matrices used and computed during this transform + float32x4_t F[6][6], FZ[6][2], f[2][2], b; + + // Read a 6x6 tile in the Winograd domain + for (auto i = 0u, m = 0u; i < 6; i++) + { + for (auto j = 0u; j < 6; j++, m++) + { + F[i][j] = vld1q_f32(inptr + m*matrix_stride); + } + } + inptr += 4; + + // Compute the matrix F Z + for (auto i = 0u; i < 6; i++) + { + // FZ[i][0] = 1*F[i][0] + 1*F[i][1] + 1*F[i][2] + 1*F[i][3] + 1*F[i][4]; + FZ[i][0] = vaddq_f32(vaddq_f32(vaddq_f32(F[i][0], F[i][1]), vaddq_f32(F[i][2], F[i][3])), F[i][4]); + + // FZ[i][1] = 1*F[i][1] + -1*F[i][2] + 2*F[i][3] + -2*F[i][4] + 1*F[i][5]; + FZ[i][1] = vaddq_f32(vmlaq_n_f32(vsubq_f32(F[i][1], F[i][2]), vsubq_f32(F[i][3], F[i][4]), 2.0f), F[i][5]); + } + + // Compute the output tile f = ZT F Z + for (auto j = 0u; j < 2; j++) + { + // f[0][j] = 1*FZ[0][j] + 1*FZ[1][j] + 1*FZ[2][j] + 1*FZ[3][j] + 1*FZ[4][j]; + f[0][j] = vaddq_f32(vaddq_f32(vaddq_f32(FZ[0][j], FZ[1][j]), vaddq_f32(FZ[2][j], FZ[3][j])), FZ[4][j]); + + // f[1][j] = 1*FZ[1][j] + -1*FZ[2][j] + 2*FZ[3][j] + -2*FZ[4][j] + 1*FZ[5][j]; + f[1][j] = vaddq_f32(vmlaq_n_f32(vsubq_f32(FZ[1][j], FZ[2][j]), vsubq_f32(FZ[3][j], FZ[4][j]), 2.0f), FZ[5][j]); + } + + // Write out the output tile + if (bptr != nullptr) + { + b = vld1q_f32(bptr); + bptr += 4; + } + else + { + b = vdupq_n_f32(0.0f); + } + for (auto i = 0u; i < output_tile_rows; i++) + { + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = + vmaxq_f32(vminq_f32(vaddq_f32(f[i][j], b), vdupq_n_f32(output_max)), + vdupq_n_f32(output_min)); + vst1q_f32(outptr + i*output_row_stride + j*output_col_stride, y); + } + } + outptr += 4; + } + for (; n_channels >= 2; n_channels -= 2) + { + // Matrices used and computed during this transform + float32x2_t F[6][6], FZ[6][2], f[2][2], b; + + // Read a 6x6 tile in the Winograd domain + for (auto i = 0u, m = 0u; i < 6; i++) + { + for (auto j = 0u; j < 6; j++, m++) + { + F[i][j] = vld1_f32(inptr + m*matrix_stride); + } + } + inptr += 2; + + // Compute the matrix F Z + for (auto i = 0u; i < 6; i++) + { + // FZ[i][0] = 1*F[i][0] + 1*F[i][1] + 1*F[i][2] + 1*F[i][3] + 1*F[i][4]; + FZ[i][0] = vadd_f32(vadd_f32(vadd_f32(F[i][0], F[i][1]), vadd_f32(F[i][2], F[i][3])), F[i][4]); + + // FZ[i][1] = 1*F[i][1] + -1*F[i][2] + 2*F[i][3] + -2*F[i][4] + 1*F[i][5]; + FZ[i][1] = vadd_f32(vmla_n_f32(vsub_f32(F[i][1], F[i][2]), vsub_f32(F[i][3], F[i][4]), 2.0f), F[i][5]); + } + + // Compute the output tile f = ZT F Z + for (auto j = 0u; j < 2; j++) + { + // f[0][j] = 1*FZ[0][j] + 1*FZ[1][j] + 1*FZ[2][j] + 1*FZ[3][j] + 1*FZ[4][j]; + f[0][j] = vadd_f32(vadd_f32(vadd_f32(FZ[0][j], FZ[1][j]), vadd_f32(FZ[2][j], FZ[3][j])), FZ[4][j]); + + // f[1][j] = 1*FZ[1][j] + -1*FZ[2][j] + 2*FZ[3][j] + -2*FZ[4][j] + 1*FZ[5][j]; + f[1][j] = vadd_f32(vmla_n_f32(vsub_f32(FZ[1][j], FZ[2][j]), vsub_f32(FZ[3][j], FZ[4][j]), 2.0f), FZ[5][j]); + } + + // Write out the output tile + if (bptr != nullptr) + { + b = vld1_f32(bptr); + bptr += 2; + } + else + { + b = vdup_n_f32(0.0f); + } + for (auto i = 0u; i < output_tile_rows; i++) + { + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = + vmax_f32(vmin_f32(vadd_f32(f[i][j], b), vdup_n_f32(output_max)), + vdup_n_f32(output_min)); + vst1_f32(outptr + i*output_row_stride + j*output_col_stride, y); + } + } + outptr += 2; + } + if (n_channels) + { + // Matrices used and computed during this transform + float F[6][6], FZ[6][2], f[2][2], b; + + // Read a 6x6 tile in the Winograd domain + for (auto i = 0u, m = 0u; i < 6; i++) + { + for (auto j = 0u; j < 6; j++, m++) + { + F[i][j] = *(inptr + m*matrix_stride); + } + } + + // Compute the matrix F Z + for (auto i = 0u; i < 6; i++) + { + FZ[i][0] = 1*F[i][0] + 1*F[i][1] + 1*F[i][2] + 1*F[i][3] + 1*F[i][4]; + FZ[i][1] = 1*F[i][1] + -1*F[i][2] + 2*F[i][3] + -2*F[i][4] + 1*F[i][5]; + } + + // Compute the output tile f = ZT F Z + for (auto j = 0u; j < 2; j++) + { + f[0][j] = 1*FZ[0][j] + 1*FZ[1][j] + 1*FZ[2][j] + 1*FZ[3][j] + 1*FZ[4][j]; + f[1][j] = 1*FZ[1][j] + -1*FZ[2][j] + 2*FZ[3][j] + -2*FZ[4][j] + 1*FZ[5][j]; + } + + // Write out the output tile + if (bptr != nullptr) + { + b = *(bptr++); + } + else + { + b = 0.0f; + } + for (auto i = 0u; i < output_tile_rows; i++) + { + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = std::max(std::min(f[i][j] + b, output_max), output_min); + *(outptr + i*output_row_stride + j*output_col_stride) = y; + } + } + } +} + +} // namespace output_transform +} // namespace winograd +} // namespace arm_conv diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_4x4_3x3.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_4x4_3x3.cpp new file mode 100644 index 0000000000..72c43019fa --- /dev/null +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/arm_fp32_4x4_3x3.cpp @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2022, 2024 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include <algorithm> +#include <cstddef> +#include <arm_neon.h> + +namespace arm_conv { +namespace winograd { +namespace output_transform { + +void arm_fp32_4x4_3x3( + unsigned int n_channels, + const float* inptr, + size_t matrix_stride, + const float* bptr, + float *outptr, + size_t output_row_stride, + size_t output_col_stride, + float output_min, + float output_max +) +{ + constexpr auto output_tile_rows = 4u, output_tile_cols = 4u; + + // For each channel of the output + for (; n_channels >= 4; n_channels -= 4) + { + // Matrices used and computed during this transform + float32x4_t F[6][6], FZ[6][4], f[4][4], b; + + // Read a 6x6 tile in the Winograd domain + for (auto i = 0u, m = 0u; i < 6; i++) + { + for (auto j = 0u; j < 6; j++, m++) + { + F[i][j] = vld1q_f32(inptr + m*matrix_stride); + } + } + inptr += 4; + + // Compute the matrix F Z + for (auto i = 0u; i < 6; i++) + { + // FZ[i][0] = 1*F[i][0] + 1*F[i][1] + 1*F[i][2] + 1*F[i][3] + 1*F[i][4]; + FZ[i][0] = vaddq_f32(vaddq_f32(vaddq_f32(F[i][0], F[i][1]), vaddq_f32(F[i][2], F[i][3])), F[i][4]); + + // FZ[i][1] = 1*F[i][1] + -1*F[i][2] + 2*F[i][3] + -2*F[i][4]; + FZ[i][1] = vmlaq_n_f32(vsubq_f32(F[i][1], F[i][2]), vsubq_f32(F[i][3], F[i][4]), 2.0f); + + // FZ[i][2] = 1*F[i][1] + 1*F[i][2] + 4*F[i][3] + 4*F[i][4]; + FZ[i][2] = vmlaq_n_f32(vaddq_f32(F[i][1], F[i][2]), vaddq_f32(F[i][3], F[i][4]), 4.0f); + + // FZ[i][3] = 1*F[i][1] + -1*F[i][2] + 8*F[i][3] + -8*F[i][4] + 1*F[i][5]; + FZ[i][3] = vaddq_f32(vmlaq_n_f32(vsubq_f32(F[i][1], F[i][2]), vsubq_f32(F[i][3], F[i][4]), 8.0f), F[i][5]); + } + + // Compute the output tile f = ZT F Z + for (auto j = 0u; j < 4; j++) + { + // f[0][j] = 1*FZ[0][j] + 1*FZ[1][j] + 1*FZ[2][j] + 1*FZ[3][j] + 1*FZ[4][j]; + f[0][j] = vaddq_f32(vaddq_f32(vaddq_f32(FZ[0][j], FZ[1][j]), vaddq_f32(FZ[2][j], FZ[3][j])), FZ[4][j]); + + // f[1][j] = 1*FZ[1][j] + -1*FZ[2][j] + 2*FZ[3][j] + -2*FZ[4][j]; + f[1][j] = vmlaq_n_f32(vsubq_f32(FZ[1][j], FZ[2][j]), vsubq_f32(FZ[3][j], FZ[4][j]), 2.0f); + + // f[2][j] = 1*FZ[1][j] + 1*FZ[2][j] + 4*FZ[3][j] + 4*FZ[4][j]; + f[2][j] = vmlaq_n_f32(vaddq_f32(FZ[1][j], FZ[2][j]), vaddq_f32(FZ[3][j], FZ[4][j]), 4.0f); + + // f[3][j] = 1*FZ[1][j] + -1*FZ[2][j] + 8*FZ[3][j] + -8*FZ[4][j] + 1*FZ[5][j]; + f[3][j] = vaddq_f32(vmlaq_n_f32(vsubq_f32(FZ[1][j], FZ[2][j]), vsubq_f32(FZ[3][j], FZ[4][j]), 8.0f), FZ[5][j]); + } + + // Write out the output tile + if (bptr != nullptr) + { + b = vld1q_f32(bptr); + bptr += 4; + } + else + { + b = vdupq_n_f32(0.0f); + } + for (auto i = 0u; i < output_tile_rows; i++) + { + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = + vmaxq_f32(vminq_f32(vaddq_f32(f[i][j], b), vdupq_n_f32(output_max)), + vdupq_n_f32(output_min)); + vst1q_f32(outptr + i*output_row_stride + j*output_col_stride, y); + } + } + outptr += 4; + } + for (; n_channels >= 2; n_channels -= 2) + { + // Matrices used and computed during this transform + float32x2_t F[6][6], FZ[6][4], f[4][4], b; + + // Read a 6x6 tile in the Winograd domain + for (auto i = 0u, m = 0u; i < 6; i++) + { + for (auto j = 0u; j < 6; j++, m++) + { + F[i][j] = vld1_f32(inptr + m*matrix_stride); + } + } + inptr += 2; + + // Compute the matrix F Z + for (auto i = 0u; i < 6; i++) + { + // FZ[i][0] = 1*F[i][0] + 1*F[i][1] + 1*F[i][2] + 1*F[i][3] + 1*F[i][4]; + FZ[i][0] = vadd_f32(vadd_f32(vadd_f32(F[i][0], F[i][1]), vadd_f32(F[i][2], F[i][3])), F[i][4]); + + // FZ[i][1] = 1*F[i][1] + -1*F[i][2] + 2*F[i][3] + -2*F[i][4]; + FZ[i][1] = vmla_n_f32(vsub_f32(F[i][1], F[i][2]), vsub_f32(F[i][3], F[i][4]), 2.0f); + + // FZ[i][2] = 1*F[i][1] + 1*F[i][2] + 4*F[i][3] + 4*F[i][4]; + FZ[i][2] = vmla_n_f32(vadd_f32(F[i][1], F[i][2]), vadd_f32(F[i][3], F[i][4]), 4.0f); + + // FZ[i][3] = 1*F[i][1] + -1*F[i][2] + 8*F[i][3] + -8*F[i][4] + 1*F[i][5]; + FZ[i][3] = vadd_f32(vmla_n_f32(vsub_f32(F[i][1], F[i][2]), vsub_f32(F[i][3], F[i][4]), 8.0f), F[i][5]); + } + + // Compute the output tile f = ZT F Z + for (auto j = 0u; j < 4; j++) + { + // f[0][j] = 1*FZ[0][j] + 1*FZ[1][j] + 1*FZ[2][j] + 1*FZ[3][j] + 1*FZ[4][j]; + f[0][j] = vadd_f32(vadd_f32(vadd_f32(FZ[0][j], FZ[1][j]), vadd_f32(FZ[2][j], FZ[3][j])), FZ[4][j]); + + // f[1][j] = 1*FZ[1][j] + -1*FZ[2][j] + 2*FZ[3][j] + -2*FZ[4][j]; + f[1][j] = vmla_n_f32(vsub_f32(FZ[1][j], FZ[2][j]), vsub_f32(FZ[3][j], FZ[4][j]), 2.0f); + + // f[2][j] = 1*FZ[1][j] + 1*FZ[2][j] + 4*FZ[3][j] + 4*FZ[4][j]; + f[2][j] = vmla_n_f32(vadd_f32(FZ[1][j], FZ[2][j]), vadd_f32(FZ[3][j], FZ[4][j]), 4.0f); + + // f[3][j] = 1*FZ[1][j] + -1*FZ[2][j] + 8*FZ[3][j] + -8*FZ[4][j] + 1*FZ[5][j]; + f[3][j] = vadd_f32(vmla_n_f32(vsub_f32(FZ[1][j], FZ[2][j]), vsub_f32(FZ[3][j], FZ[4][j]), 8.0f), FZ[5][j]); + } + + // Write out the output tile + if (bptr != nullptr) + { + b = vld1_f32(bptr); + bptr += 2; + } + else + { + b = vdup_n_f32(0.0f); + } + for (auto i = 0u; i < output_tile_rows; i++) + { + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = + vmax_f32(vmin_f32(vadd_f32(f[i][j], b), vdup_n_f32(output_max)), + vdup_n_f32(output_min)); + vst1_f32(outptr + i*output_row_stride + j*output_col_stride, y); + } + } + outptr += 2; + } + for (; n_channels; n_channels--) + { + // Matrices used and computed during this transform + float F[6][6], FZ[6][4], f[4][4], b; + + // Read a 6x6 tile in the Winograd domain + for (auto i = 0u, m = 0u; i < 6; i++) + { + for (auto j = 0u; j < 6; j++, m++) + { + F[i][j] = *(inptr + m*matrix_stride); + } + } + inptr++; + + // Compute the matrix F Z + for (auto i = 0u; i < 6; i++) + { + FZ[i][0] = 1*F[i][0] + 1*F[i][1] + 1*F[i][2] + 1*F[i][3] + 1*F[i][4]; + FZ[i][1] = 1*F[i][1] + -1*F[i][2] + 2*F[i][3] + -2*F[i][4]; + FZ[i][2] = 1*F[i][1] + 1*F[i][2] + 4*F[i][3] + 4*F[i][4]; + FZ[i][3] = 1*F[i][1] + -1*F[i][2] + 8*F[i][3] + -8*F[i][4] + 1*F[i][5]; + } + + // Compute the output tile f = ZT F Z + for (auto j = 0u; j < 4; j++) + { + f[0][j] = 1*FZ[0][j] + 1*FZ[1][j] + 1*FZ[2][j] + 1*FZ[3][j] + 1*FZ[4][j]; + f[1][j] = 1*FZ[1][j] + -1*FZ[2][j] + 2*FZ[3][j] + -2*FZ[4][j]; + f[2][j] = 1*FZ[1][j] + 1*FZ[2][j] + 4*FZ[3][j] + 4*FZ[4][j]; + f[3][j] = 1*FZ[1][j] + -1*FZ[2][j] + 8*FZ[3][j] + -8*FZ[4][j] + 1*FZ[5][j]; + } + + // Write out the output tile + if (bptr != nullptr) + { + b = *(bptr++); + } + else + { + b = 0.0f; + } + for (auto i = 0u; i < output_tile_rows; i++) + { + for (auto j = 0u; j < output_tile_cols; j++) + { + const auto y = std::max(std::min(f[i][j] + b, output_max), output_min); + *(outptr + i*output_row_stride + j*output_col_stride) = y; + } + } + outptr++; + } +} + +} // namespace output_transform +} // namespace winograd +} // namespace arm_conv diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp new file mode 100644 index 0000000000..043914d590 --- /dev/null +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp @@ -0,0 +1,891 @@ +/* + * Copyright (c) 2022-2024 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +#include <cstddef> + +namespace arm_conv { +namespace winograd { +namespace output_transform { + +void sme_fp32_mopa_4x4_3x3( + unsigned int n_channels, + const float* inptr, + size_t matrix_stride, + const float* bptr, + float* const output, + size_t output_row_stride, + size_t output_col_stride, + float output_min, + float output_max +) +{ + // The below assembler uses the Kronecker product and the "vec trick" to + // implement the Winograd output transform (y = AT Y A) using the SME + // array. This code REQUIRES that the vectors are 512b long (or longer, if + // we add some predication). + // + // The "vec trick" uses the identity $vec(AT Y A) = (AT (x) AT) vec(Y)$ to + // convert the chain of matrix multiplications into a matrix-vector + // product. We then stack multiple channels of vec(Y) together to allow us + // to perform multiple channels of the transformation simultaneously. + // + // Since the complete matrix (AT (x) AT) is quite big [16 x 36], we compute + // it on the fly. To do so, we store two representations of the matrix AT. + // The first representation (the outer terms) contains, within each quad, + // four coefficients of the matrix AT. + const float outer_terms[32] = { + 1, 1, 1, 1, + 0, 1, -1, 2, + 0, 1, 1, 4, + 0, 1, -1, 8, + // The following rows are continuations of the first four rows, and each + // contains two columns of padding values which aren't used in the + // computation but are there to ensure that the coefficients end up in + // the right quads of the vector into which they're read. + 1, 0, 0, 0, + -2, 0, 0, 0, + 4, 0, 0, 0, + -8, 1, 0, 0 + }; + // This should be loaded completely into two Z registers. + // + // We can then use by-element FMLA to construct columns of (AT (x) AT) by + // multiplying elements of the outer terms against the following inner + // terms (again split into quads, but expected to be loaded replicated such + // that each of the six required Z registers contains a repeated quad of + // the values). + const float inner_terms[24] = { + 1, 0, 0, 0, + 1, 1, 1, 1, + 1, -1, 1, -1, + 1, 2, 4, 8, + 1, -2, 4, -8, + 0, 0, 0, 1 + }; + + struct Params + { + const float *outer_terms; + const float *inner_terms; + float act_min; + float act_max; + + Params(const float *outer_terms, + const float *inner_terms, + float act_min, + float act_max) + : outer_terms(outer_terms), inner_terms(inner_terms), + act_min(act_min), act_max(act_max) + { + } + }; + + Params params(outer_terms, inner_terms, output_min, output_max); + + __asm__ __volatile__( + "ldr x20, [%x[params], %[offsetof_Params_outer_terms]]\n" + ".inst 0xd503477f // SMSTART ZA\n" + "ptrue p5.b\n" + "ld1rw { z12.s }, p5/Z, [%x[params], %[offsetof_Params_act_min]]\n" + "ld1rw { z10.s }, p5/Z, [%x[params], %[offsetof_Params_act_max]]\n" + "pfalse p8.b\n" + "ldr x8, [%x[params], %[offsetof_Params_inner_terms]]\n" + "ld1w { z6.s }, p5/Z, [x20]\n" + "ld1w { z7.s }, p5/Z, [x20, #1, MUL VL]\n" + "ld1rqw { z9.s }, p5/Z, [x8]\n" + "ld1rqw { z8.s }, p5/Z, [x8, #16]\n" + "ld1rqw { z15.s }, p5/Z, [x8, #32]\n" + "fmul z11.s, z9.s, z6.s[0]\n" + "fmul z5.s, z9.s, z6.s[1]\n" + "ld1rqw { z4.s }, p5/Z, [x8, #48]\n" + "ld1rqw { z3.s }, p5/Z, [x8, #64]\n" + "ld1rqw { z2.s }, p5/Z, [x8, #80]\n" + "cbz %x[bptr], 1f\n" + "ptrue p8.s\n" + "1:" // Set bias predicate: Done + ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" + "fmov z1.s, #1.0\n" + "mov x25, #0x0\n" + "cntw x24\n" + "cntw x23, ALL, MUL #2\n" + "cntw x22, ALL, MUL #3\n" + "whilelt p4.s, x25, %x[n_channels]\n" + "whilelt p3.s, x24, %x[n_channels]\n" + "ld1w { z31.s }, p4/Z, [%x[inptr], x25, LSL #2]\n" + "ld1w { z30.s }, p3/Z, [%x[inptr], x24, LSL #2]\n" + "whilelt p2.s, x23, %x[n_channels]\n" + "whilelt p1.s, x22, %x[n_channels]\n" + "ld1w { z29.s }, p2/Z, [%x[inptr], x23, LSL #2]\n" + "add x21, %x[inptr], %x[matrix_stride], LSL #2\n" + "and p0.b, p5/Z, p8.b, p4.b\n" + "ld1w { z28.s }, p1/Z, [%x[inptr], x22, LSL #2]\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x25, LSL #2]\n" + "and p0.b, p5/Z, p8.b, p3.b\n" + ".inst 0x8080b420 // fmopa za0.s, p5/M, p5/M, z1.s, z0.s\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x24, LSL #2]\n" + "and p0.b, p5/Z, p8.b, p2.b\n" + ".inst 0x8080b421 // fmopa za1.s, p5/M, p5/M, z1.s, z0.s\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x23, LSL #2]\n" + "and p0.b, p5/Z, p8.b, p1.b\n" + ".inst 0x8080b422 // fmopa za2.s, p5/M, p5/M, z1.s, z0.s\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x22, LSL #2]\n" + ".inst 0x8080b423 // fmopa za3.s, p5/M, p5/M, z1.s, z0.s\n" + "2:" // Loop + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + "mov x14, #0xc\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + "whilelt p0.s, x25, %x[n_channels]\n" + "add x20, %x[output], %x[output_col_stride], LSL #2\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + "add x8, %x[output], %x[output_row_stride], LSL #2\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z9.s, z6.s[2]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z9.s, z6.s[3]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z9.s, z7.s[0]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z9.s, z7.s[1]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z8.s, z6.s[0]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z8.s, z6.s[1]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z8.s, z6.s[2]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z8.s, z6.s[3]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z8.s, z7.s[0]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z8.s, z7.s[1]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z15.s, z6.s[0]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z15.s, z6.s[1]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z15.s, z6.s[2]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z15.s, z6.s[3]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z15.s, z7.s[0]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z15.s, z7.s[1]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z4.s, z6.s[0]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z4.s, z6.s[1]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z4.s, z6.s[2]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z4.s, z6.s[3]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z4.s, z7.s[0]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z4.s, z7.s[1]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z3.s, z6.s[0]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z3.s, z6.s[1]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z3.s, z6.s[2]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z3.s, z6.s[3]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z3.s, z7.s[0]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z3.s, z7.s[1]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z2.s, z6.s[0]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z2.s, z6.s[1]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z2.s, z6.s[2]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z2.s, z6.s[3]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z2.s, z7.s[0]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z2.s, z7.s[1]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z9.s, z6.s[0]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z9.s, z6.s[1]\n" + ".inst 0xc082741f // mova z31.s, p5/M, za0h.s[XZR]\n" + ".inst 0xc082541c // mova z28.s, p5/M, za0h.s[x14]\n" + "fmin z31.s, p5/M, z31.s, z10.s\n" + ".inst 0xc082743b // mova z27.s, p5/M, za0h.s[XZR, #1]\n" + "fmin z28.s, p5/M, z28.s, z10.s\n" + ".inst 0xc0825438 // mova z24.s, p5/M, za0h.s[x14, #1]\n" + "fmin z27.s, p5/M, z27.s, z10.s\n" + "mov x13, #0x4\n" + "mov x12, #0x8\n" + ".inst 0xc082341e // mova z30.s, p5/M, za0h.s[x13]\n" + "fmin z24.s, p5/M, z24.s, z10.s\n" + ".inst 0xc082141d // mova z29.s, p5/M, za0h.s[x12]\n" + "fmax z31.s, p5/M, z31.s, z12.s\n" + "fmin z30.s, p5/M, z30.s, z10.s\n" + ".inst 0xc082343a // mova z26.s, p5/M, za0h.s[x13, #1]\n" + "fmin z29.s, p5/M, z29.s, z10.s\n" + "fmax z28.s, p5/M, z28.s, z12.s\n" + ".inst 0xc0821439 // mova z25.s, p5/M, za0h.s[x12, #1]\n" + "fmax z27.s, p5/M, z27.s, z12.s\n" + "fmin z26.s, p5/M, z26.s, z10.s\n" + ".inst 0xc0827457 // mova z23.s, p5/M, za0h.s[XZR, #2]\n" + "fmin z25.s, p5/M, z25.s, z10.s\n" + "fmax z24.s, p5/M, z24.s, z12.s\n" + ".inst 0xc0823456 // mova z22.s, p5/M, za0h.s[x13, #2]\n" + "fmax z30.s, p5/M, z30.s, z12.s\n" + "fmin z23.s, p5/M, z23.s, z10.s\n" + ".inst 0xc0821455 // mova z21.s, p5/M, za0h.s[x12, #2]\n" + "fmax z29.s, p5/M, z29.s, z12.s\n" + "fmin z22.s, p5/M, z22.s, z10.s\n" + ".inst 0xc0825454 // mova z20.s, p5/M, za0h.s[x14, #2]\n" + "fmax z26.s, p5/M, z26.s, z12.s\n" + "fmin z21.s, p5/M, z21.s, z10.s\n" + ".inst 0xc0827473 // mova z19.s, p5/M, za0h.s[XZR, #3]\n" + "fmax z25.s, p5/M, z25.s, z12.s\n" + "fmin z20.s, p5/M, z20.s, z10.s\n" + ".inst 0xc0823472 // mova z18.s, p5/M, za0h.s[x13, #3]\n" + "fmax z23.s, p5/M, z23.s, z12.s\n" + "fmin z19.s, p5/M, z19.s, z10.s\n" + ".inst 0xc0821471 // mova z17.s, p5/M, za0h.s[x12, #3]\n" + "fmax z22.s, p5/M, z22.s, z12.s\n" + "fmin z18.s, p5/M, z18.s, z10.s\n" + ".inst 0xc0825470 // mova z16.s, p5/M, za0h.s[x14, #3]\n" + "fmax z21.s, p5/M, z21.s, z12.s\n" + "fmin z17.s, p5/M, z17.s, z10.s\n" + "fmax z20.s, p5/M, z20.s, z12.s\n" + "fmin z16.s, p5/M, z16.s, z10.s\n" + "st1w { z31.s }, p0, [%x[output], x25, LSL #2]\n" + "fmax z19.s, p5/M, z19.s, z12.s\n" + "st1w { z30.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z18.s, p5/M, z18.s, z12.s\n" + "st1w { z29.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z17.s, p5/M, z17.s, z12.s\n" + "st1w { z28.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "fmax z16.s, p5/M, z16.s, z12.s\n" + "st1w { z27.s }, p0, [x8, x25, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z26.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z25.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z24.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z23.s }, p0, [x8, x25, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z22.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z21.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z20.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z19.s }, p0, [x8, x25, LSL #2]\n" + "st1w { z18.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z17.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z16.s }, p0, [x20, x25, LSL #2]\n" + "whilelt p0.s, x24, %x[n_channels]\n" + "b.none 3f\n" + ".inst 0xc082749f // mova z31.s, p5/M, za1h.s[XZR]\n" + ".inst 0xc082349e // mova z30.s, p5/M, za1h.s[x13]\n" + "fmin z31.s, p5/M, z31.s, z10.s\n" + ".inst 0xc082149d // mova z29.s, p5/M, za1h.s[x12]\n" + "fmin z30.s, p5/M, z30.s, z10.s\n" + ".inst 0xc082549c // mova z28.s, p5/M, za1h.s[x14]\n" + "fmin z29.s, p5/M, z29.s, z10.s\n" + ".inst 0xc08274bb // mova z27.s, p5/M, za1h.s[XZR, #1]\n" + "fmin z28.s, p5/M, z28.s, z10.s\n" + ".inst 0xc08234ba // mova z26.s, p5/M, za1h.s[x13, #1]\n" + "fmax z31.s, p5/M, z31.s, z12.s\n" + "fmin z27.s, p5/M, z27.s, z10.s\n" + ".inst 0xc08214b9 // mova z25.s, p5/M, za1h.s[x12, #1]\n" + "fmax z30.s, p5/M, z30.s, z12.s\n" + "fmin z26.s, p5/M, z26.s, z10.s\n" + ".inst 0xc08254b8 // mova z24.s, p5/M, za1h.s[x14, #1]\n" + "fmax z29.s, p5/M, z29.s, z12.s\n" + "fmin z25.s, p5/M, z25.s, z10.s\n" + ".inst 0xc08274d7 // mova z23.s, p5/M, za1h.s[XZR, #2]\n" + "fmax z28.s, p5/M, z28.s, z12.s\n" + "fmin z24.s, p5/M, z24.s, z10.s\n" + ".inst 0xc08234d6 // mova z22.s, p5/M, za1h.s[x13, #2]\n" + "fmax z27.s, p5/M, z27.s, z12.s\n" + "fmin z23.s, p5/M, z23.s, z10.s\n" + ".inst 0xc08214d5 // mova z21.s, p5/M, za1h.s[x12, #2]\n" + "fmax z26.s, p5/M, z26.s, z12.s\n" + "fmin z22.s, p5/M, z22.s, z10.s\n" + "add x20, %x[output], %x[output_col_stride], LSL #2\n" + ".inst 0xc08254d4 // mova z20.s, p5/M, za1h.s[x14, #2]\n" + "fmax z25.s, p5/M, z25.s, z12.s\n" + "fmin z21.s, p5/M, z21.s, z10.s\n" + "add x8, %x[output], %x[output_row_stride], LSL #2\n" + ".inst 0xc08274f3 // mova z19.s, p5/M, za1h.s[XZR, #3]\n" + "fmax z24.s, p5/M, z24.s, z12.s\n" + "fmin z20.s, p5/M, z20.s, z10.s\n" + ".inst 0xc08234f2 // mova z18.s, p5/M, za1h.s[x13, #3]\n" + "fmax z23.s, p5/M, z23.s, z12.s\n" + "fmin z19.s, p5/M, z19.s, z10.s\n" + ".inst 0xc08214f1 // mova z17.s, p5/M, za1h.s[x12, #3]\n" + "fmax z22.s, p5/M, z22.s, z12.s\n" + "fmin z18.s, p5/M, z18.s, z10.s\n" + ".inst 0xc08254f0 // mova z16.s, p5/M, za1h.s[x14, #3]\n" + "fmax z21.s, p5/M, z21.s, z12.s\n" + "fmin z17.s, p5/M, z17.s, z10.s\n" + "fmax z20.s, p5/M, z20.s, z12.s\n" + "fmin z16.s, p5/M, z16.s, z10.s\n" + "st1w { z31.s }, p0, [%x[output], x24, LSL #2]\n" + "fmax z19.s, p5/M, z19.s, z12.s\n" + "st1w { z30.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z18.s, p5/M, z18.s, z12.s\n" + "st1w { z29.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z17.s, p5/M, z17.s, z12.s\n" + "st1w { z28.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "fmax z16.s, p5/M, z16.s, z12.s\n" + "st1w { z27.s }, p0, [x8, x24, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z26.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z25.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z24.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z23.s }, p0, [x8, x24, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z22.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z21.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z20.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z19.s }, p0, [x8, x24, LSL #2]\n" + "st1w { z18.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z17.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z16.s }, p0, [x20, x24, LSL #2]\n" + "whilelt p0.s, x23, %x[n_channels]\n" + "b.none 3f\n" + ".inst 0xc082751f // mova z31.s, p5/M, za2h.s[XZR]\n" + ".inst 0xc082351e // mova z30.s, p5/M, za2h.s[x13]\n" + "fmin z31.s, p5/M, z31.s, z10.s\n" + ".inst 0xc082151d // mova z29.s, p5/M, za2h.s[x12]\n" + "fmin z30.s, p5/M, z30.s, z10.s\n" + ".inst 0xc082551c // mova z28.s, p5/M, za2h.s[x14]\n" + "fmin z29.s, p5/M, z29.s, z10.s\n" + ".inst 0xc082753b // mova z27.s, p5/M, za2h.s[XZR, #1]\n" + "fmin z28.s, p5/M, z28.s, z10.s\n" + ".inst 0xc082353a // mova z26.s, p5/M, za2h.s[x13, #1]\n" + "fmax z31.s, p5/M, z31.s, z12.s\n" + "fmin z27.s, p5/M, z27.s, z10.s\n" + ".inst 0xc0821539 // mova z25.s, p5/M, za2h.s[x12, #1]\n" + "fmax z30.s, p5/M, z30.s, z12.s\n" + "fmin z26.s, p5/M, z26.s, z10.s\n" + ".inst 0xc0825538 // mova z24.s, p5/M, za2h.s[x14, #1]\n" + "fmax z29.s, p5/M, z29.s, z12.s\n" + "fmin z25.s, p5/M, z25.s, z10.s\n" + ".inst 0xc0827557 // mova z23.s, p5/M, za2h.s[XZR, #2]\n" + "fmax z28.s, p5/M, z28.s, z12.s\n" + "fmin z24.s, p5/M, z24.s, z10.s\n" + ".inst 0xc0823556 // mova z22.s, p5/M, za2h.s[x13, #2]\n" + "fmax z27.s, p5/M, z27.s, z12.s\n" + "fmin z23.s, p5/M, z23.s, z10.s\n" + ".inst 0xc0821555 // mova z21.s, p5/M, za2h.s[x12, #2]\n" + "fmax z26.s, p5/M, z26.s, z12.s\n" + "fmin z22.s, p5/M, z22.s, z10.s\n" + "add x20, %x[output], %x[output_col_stride], LSL #2\n" + ".inst 0xc0825554 // mova z20.s, p5/M, za2h.s[x14, #2]\n" + "fmax z25.s, p5/M, z25.s, z12.s\n" + "fmin z21.s, p5/M, z21.s, z10.s\n" + "add x8, %x[output], %x[output_row_stride], LSL #2\n" + ".inst 0xc0827573 // mova z19.s, p5/M, za2h.s[XZR, #3]\n" + "fmax z24.s, p5/M, z24.s, z12.s\n" + "fmin z20.s, p5/M, z20.s, z10.s\n" + ".inst 0xc0823572 // mova z18.s, p5/M, za2h.s[x13, #3]\n" + "fmax z23.s, p5/M, z23.s, z12.s\n" + "fmin z19.s, p5/M, z19.s, z10.s\n" + ".inst 0xc0821571 // mova z17.s, p5/M, za2h.s[x12, #3]\n" + "fmax z22.s, p5/M, z22.s, z12.s\n" + "fmin z18.s, p5/M, z18.s, z10.s\n" + ".inst 0xc0825570 // mova z16.s, p5/M, za2h.s[x14, #3]\n" + "fmax z21.s, p5/M, z21.s, z12.s\n" + "fmin z17.s, p5/M, z17.s, z10.s\n" + "fmax z20.s, p5/M, z20.s, z12.s\n" + "fmin z16.s, p5/M, z16.s, z10.s\n" + "st1w { z31.s }, p0, [%x[output], x23, LSL #2]\n" + "fmax z19.s, p5/M, z19.s, z12.s\n" + "st1w { z30.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z18.s, p5/M, z18.s, z12.s\n" + "st1w { z29.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z17.s, p5/M, z17.s, z12.s\n" + "st1w { z28.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "fmax z16.s, p5/M, z16.s, z12.s\n" + "st1w { z27.s }, p0, [x8, x23, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z26.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z25.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z24.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z23.s }, p0, [x8, x23, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z22.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z21.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z20.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z19.s }, p0, [x8, x23, LSL #2]\n" + "st1w { z18.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z17.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z16.s }, p0, [x20, x23, LSL #2]\n" + "whilelt p0.s, x22, %x[n_channels]\n" + "b.none 3f\n" + "fmov z1.s, #1.0\n" + ".inst 0xc082759f // mova z31.s, p5/M, za3h.s[XZR]\n" + ".inst 0xc082359e // mova z30.s, p5/M, za3h.s[x13]\n" + "fmin z31.s, p5/M, z31.s, z10.s\n" + ".inst 0xc082159d // mova z29.s, p5/M, za3h.s[x12]\n" + "fmin z30.s, p5/M, z30.s, z10.s\n" + ".inst 0xc082559c // mova z28.s, p5/M, za3h.s[x14]\n" + "fmin z29.s, p5/M, z29.s, z10.s\n" + ".inst 0xc08275bb // mova z27.s, p5/M, za3h.s[XZR, #1]\n" + "fmin z28.s, p5/M, z28.s, z10.s\n" + ".inst 0xc08235ba // mova z26.s, p5/M, za3h.s[x13, #1]\n" + "fmax z31.s, p5/M, z31.s, z12.s\n" + "fmin z27.s, p5/M, z27.s, z10.s\n" + ".inst 0xc08215b9 // mova z25.s, p5/M, za3h.s[x12, #1]\n" + "fmax z30.s, p5/M, z30.s, z12.s\n" + "fmin z26.s, p5/M, z26.s, z10.s\n" + ".inst 0xc08255b8 // mova z24.s, p5/M, za3h.s[x14, #1]\n" + "fmax z29.s, p5/M, z29.s, z12.s\n" + "fmin z25.s, p5/M, z25.s, z10.s\n" + ".inst 0xc08275d7 // mova z23.s, p5/M, za3h.s[XZR, #2]\n" + "fmax z28.s, p5/M, z28.s, z12.s\n" + "fmin z24.s, p5/M, z24.s, z10.s\n" + ".inst 0xc08235d6 // mova z22.s, p5/M, za3h.s[x13, #2]\n" + "fmax z27.s, p5/M, z27.s, z12.s\n" + "fmin z23.s, p5/M, z23.s, z10.s\n" + ".inst 0xc08215d5 // mova z21.s, p5/M, za3h.s[x12, #2]\n" + "fmax z26.s, p5/M, z26.s, z12.s\n" + "fmin z22.s, p5/M, z22.s, z10.s\n" + ".inst 0xc08255d4 // mova z20.s, p5/M, za3h.s[x14, #2]\n" + "fmax z25.s, p5/M, z25.s, z12.s\n" + "fmin z21.s, p5/M, z21.s, z10.s\n" + "add x20, %x[output], %x[output_col_stride], LSL #2\n" + ".inst 0xc08275f3 // mova z19.s, p5/M, za3h.s[XZR, #3]\n" + "fmax z24.s, p5/M, z24.s, z12.s\n" + "fmin z20.s, p5/M, z20.s, z10.s\n" + "add x8, %x[output], %x[output_row_stride], LSL #2\n" + ".inst 0xc08235f2 // mova z18.s, p5/M, za3h.s[x13, #3]\n" + "fmax z23.s, p5/M, z23.s, z12.s\n" + "fmin z19.s, p5/M, z19.s, z10.s\n" + "incw x25, ALL, MUL #4\n" + ".inst 0xc08215f1 // mova z17.s, p5/M, za3h.s[x12, #3]\n" + "fmax z22.s, p5/M, z22.s, z12.s\n" + "fmin z18.s, p5/M, z18.s, z10.s\n" + "incw x24, ALL, MUL #4\n" + ".inst 0xc08255f0 // mova z16.s, p5/M, za3h.s[x14, #3]\n" + "fmax z21.s, p5/M, z21.s, z12.s\n" + "fmin z17.s, p5/M, z17.s, z10.s\n" + "incw x23, ALL, MUL #4\n" + ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" + "fmax z20.s, p5/M, z20.s, z12.s\n" + "fmin z16.s, p5/M, z16.s, z10.s\n" + "add x21, %x[inptr], %x[matrix_stride], LSL #2\n" + "fmax z19.s, p5/M, z19.s, z12.s\n" + "st1w { z31.s }, p0, [%x[output], x22, LSL #2]\n" + "fmax z18.s, p5/M, z18.s, z12.s\n" + "st1w { z30.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z17.s, p5/M, z17.s, z12.s\n" + "st1w { z29.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z16.s, p5/M, z16.s, z12.s\n" + "st1w { z28.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z27.s }, p0, [x8, x22, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z26.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z25.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z24.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z23.s }, p0, [x8, x22, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z22.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z21.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z20.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z19.s }, p0, [x8, x22, LSL #2]\n" + "st1w { z18.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z17.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z16.s }, p0, [x20, x22, LSL #2]\n" + "incw x22, ALL, MUL #4\n" + "whilelt p1.s, x22, %x[n_channels]\n" + "ld1w { z28.s }, p1/Z, [%x[inptr], x22, LSL #2]\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "whilelt p2.s, x23, %x[n_channels]\n" + "whilelt p3.s, x24, %x[n_channels]\n" + "ld1w { z30.s }, p3/Z, [%x[inptr], x24, LSL #2]\n" + "whilelt p4.s, x25, %x[n_channels]\n" + "ld1w { z31.s }, p4/Z, [%x[inptr], x25, LSL #2]\n" + "and p0.b, p5/Z, p8.b, p4.b\n" + "ld1w { z29.s }, p2/Z, [%x[inptr], x23, LSL #2]\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x25, LSL #2]\n" + "and p0.b, p5/Z, p8.b, p3.b\n" + ".inst 0x8080b420 // fmopa za0.s, p5/M, p5/M, z1.s, z0.s\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x24, LSL #2]\n" + "and p0.b, p5/Z, p8.b, p2.b\n" + ".inst 0x8080b421 // fmopa za1.s, p5/M, p5/M, z1.s, z0.s\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x23, LSL #2]\n" + "and p0.b, p5/Z, p8.b, p1.b\n" + ".inst 0x8080b422 // fmopa za2.s, p5/M, p5/M, z1.s, z0.s\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x22, LSL #2]\n" + ".inst 0x8080b423 // fmopa za3.s, p5/M, p5/M, z1.s, z0.s\n" + "b.any 2b\n" + "3:" // End + ".inst 0xd503467f // SMSTOP\n" + : + : [bptr] "r" (bptr), [inptr] "r" (inptr), [matrix_stride] "r" (matrix_stride), [n_channels] "r" (n_channels), [offsetof_Params_act_max] "I" (offsetof(Params, act_max)), [offsetof_Params_act_min] "I" (offsetof(Params, act_min)), [offsetof_Params_inner_terms] "I" (offsetof(Params, inner_terms)), [offsetof_Params_outer_terms] "I" (offsetof(Params, outer_terms)), [output] "r" (output), [output_col_stride] "r" (output_col_stride), [output_row_stride] "r" (output_row_stride), [params] "r" (¶ms) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p8", "x12", "x13", "x14", "x8", "x20", "x21", "x22", "x23", "x24", "x25", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +} // namespace output_transform +} // namespace winograd +} // namespace arm_conv + +#endif // defined(ARM_COMPUTE_ENABLE_SME) |