From 638b7e4f6b1125b74f27f90dea2cd23eca52bfe8 Mon Sep 17 00:00:00 2001 From: ramelg01 Date: Wed, 4 May 2022 15:12:21 +0100 Subject: =?UTF-8?q?Fix=20for=20Neon=E2=84=A2=20Depthwise=20Android=20P=20V?= =?UTF-8?q?TS=20test=20failure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves: COMPMID-5237 Signed-off-by: ramy.elgammal@arm.com Change-Id: Ib1f5e262030e915a038cef587001708bbaf14c56 Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/7508 Reviewed-by: David Mansell Reviewed-by: Pablo Marquez Tello Comments-Addressed: Arm Jenkins Tested-by: Arm Jenkins --- .../arm_conv/depthwise/depthwise_depthfirst.hpp | 6 ++- .../generic.cpp | 48 +++++++++++----------- .../generic.cpp | 48 +++++++++++----------- .../generic.cpp | 48 +++++++++++----------- .../generic.cpp | 48 +++++++++++----------- .../generic.cpp | 48 +++++++++++----------- .../generic.cpp | 48 +++++++++++----------- .../generic.cpp | 48 +++++++++++----------- .../generic.cpp | 48 +++++++++++----------- .../generic.cpp | 48 +++++++++++----------- .../generic.cpp | 48 +++++++++++----------- .../generic.cpp | 48 +++++++++++----------- .../generic.cpp | 48 +++++++++++----------- 13 files changed, 292 insertions(+), 290 deletions(-) (limited to 'src/core/NEON/kernels') diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp index 6905076357..70b12919b0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp @@ -437,7 +437,9 @@ class DepthwiseDepthfirst const auto input_i = static_cast(ii < 0 ? 0 : ii); const auto input_j = output_j * this->m_args.stride_cols - this->m_args.padding.left; - const auto valid_input_rows = std::min(strat->get_input_rows(), this->m_args.input_rows - input_i); + // Valid input rows is the smallest of the input rows that aren't padding for this tile, and the number of rows + // available. + const auto valid_input_rows = std::min(strat->get_input_rows() - input_pad_top, this->m_args.input_rows - input_i); const auto valid_output_rows = std::min(strat->get_output_rows(), this->m_args.output_rows - output_i); const auto input_point_stride = input.ld_col * this->m_strat->get_output_cols() * this->m_args.stride_cols; @@ -471,7 +473,7 @@ class DepthwiseDepthfirst // Update all unpadded pointers { auto ptr = ws->inptr_array + strat->get_input_cols() * input_pad_top; - for (auto n = input_pad_top; n < valid_input_rows; n++) + for (auto n = input_pad_top; n < (valid_input_rows + input_pad_top); n++) { for (auto m = 0u; m < strat->get_input_cols(); m++) { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index 0e8d16fc40..71729e0d1e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -267,24 +267,24 @@ void a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( "smlal2 v23.4s, v29.8h, v6.8h\n" "smlal v13.4s, v31.4h, v6.4h\n" "smlal v19.4s, v30.4h, v8.4h\n" - "sqdmulh v13.4s, v13.4s, v21.4s\n" + "sqrdmulh v13.4s, v13.4s, v21.4s\n" "smlal v18.4s, v28.4h, v8.4h\n" "smlal v9.4s, v28.4h, v7.4h\n" - "sqdmulh v19.4s, v19.4s, v21.4s\n" + "sqrdmulh v19.4s, v19.4s, v21.4s\n" "smlal2 v26.4s, v31.8h, v6.8h\n" "smlal2 v11.4s, v30.8h, v8.8h\n" - "sqdmulh v18.4s, v18.4s, v21.4s\n" + "sqrdmulh v18.4s, v18.4s, v21.4s\n" "smlal2 v24.4s, v28.8h, v8.8h\n" "smlal2 v23.4s, v28.8h, v7.8h\n" - "sqdmulh v9.4s, v9.4s, v21.4s\n" + "sqrdmulh v9.4s, v9.4s, v21.4s\n" "and v7.16b, v13.16b, v25.16b\n" - "sqdmulh v26.4s, v26.4s, v10.4s\n" + "sqrdmulh v26.4s, v26.4s, v10.4s\n" "and v4.16b, v19.16b, v25.16b\n" - "sqdmulh v11.4s, v11.4s, v10.4s\n" + "sqrdmulh v11.4s, v11.4s, v10.4s\n" "and v21.16b, v18.16b, v25.16b\n" - "sqdmulh v24.4s, v24.4s, v10.4s\n" + "sqrdmulh v24.4s, v24.4s, v10.4s\n" "and v20.16b, v9.16b, v25.16b\n" - "sqdmulh v23.4s, v23.4s, v10.4s\n" + "sqrdmulh v23.4s, v23.4s, v10.4s\n" "sshr v7.4s, v7.4s, #0x1f\n" "and v29.16b, v26.16b, v16.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" @@ -493,24 +493,24 @@ void a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( "smlal2 v23.4s, v29.8h, v6.8h\n" "smlal v13.4s, v31.4h, v6.4h\n" "smlal v19.4s, v30.4h, v8.4h\n" - "sqdmulh v13.4s, v13.4s, v21.4s\n" + "sqrdmulh v13.4s, v13.4s, v21.4s\n" "smlal v18.4s, v28.4h, v8.4h\n" "smlal v9.4s, v28.4h, v7.4h\n" - "sqdmulh v19.4s, v19.4s, v21.4s\n" + "sqrdmulh v19.4s, v19.4s, v21.4s\n" "smlal2 v26.4s, v31.8h, v6.8h\n" "smlal2 v11.4s, v30.8h, v8.8h\n" - "sqdmulh v18.4s, v18.4s, v21.4s\n" + "sqrdmulh v18.4s, v18.4s, v21.4s\n" "smlal2 v24.4s, v28.8h, v8.8h\n" "smlal2 v23.4s, v28.8h, v7.8h\n" - "sqdmulh v9.4s, v9.4s, v21.4s\n" + "sqrdmulh v9.4s, v9.4s, v21.4s\n" "and v7.16b, v13.16b, v25.16b\n" - "sqdmulh v26.4s, v26.4s, v10.4s\n" + "sqrdmulh v26.4s, v26.4s, v10.4s\n" "and v4.16b, v19.16b, v25.16b\n" - "sqdmulh v11.4s, v11.4s, v10.4s\n" + "sqrdmulh v11.4s, v11.4s, v10.4s\n" "and v21.16b, v18.16b, v25.16b\n" - "sqdmulh v24.4s, v24.4s, v10.4s\n" + "sqrdmulh v24.4s, v24.4s, v10.4s\n" "and v20.16b, v9.16b, v25.16b\n" - "sqdmulh v23.4s, v23.4s, v10.4s\n" + "sqrdmulh v23.4s, v23.4s, v10.4s\n" "sshr v7.4s, v7.4s, #0x1f\n" "and v29.16b, v26.16b, v16.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" @@ -1043,22 +1043,22 @@ void a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( "ld1 { v21.s }[0], [x13]\n" "ld1 { v25.s }[0], [x11]\n" "59:" // Oddments: Load requant params: Bit 2: End - "sqdmulh v13.4s, v13.4s, v21.4s\n" - "sqdmulh v19.4s, v19.4s, v21.4s\n" + "sqrdmulh v13.4s, v13.4s, v21.4s\n" + "sqrdmulh v19.4s, v19.4s, v21.4s\n" "add x10, x10, x14\n" "add x9, x9, x14\n" - "sqdmulh v18.4s, v18.4s, v21.4s\n" - "sqdmulh v9.4s, v9.4s, v21.4s\n" + "sqrdmulh v18.4s, v18.4s, v21.4s\n" + "sqrdmulh v9.4s, v9.4s, v21.4s\n" "add x28, x28, x14\n" "add x27, x27, x14\n" "and v7.16b, v13.16b, v25.16b\n" - "sqdmulh v26.4s, v26.4s, v10.4s\n" + "sqrdmulh v26.4s, v26.4s, v10.4s\n" "and v4.16b, v19.16b, v25.16b\n" - "sqdmulh v11.4s, v11.4s, v10.4s\n" + "sqrdmulh v11.4s, v11.4s, v10.4s\n" "and v21.16b, v18.16b, v25.16b\n" - "sqdmulh v24.4s, v24.4s, v10.4s\n" + "sqrdmulh v24.4s, v24.4s, v10.4s\n" "and v20.16b, v9.16b, v25.16b\n" - "sqdmulh v23.4s, v23.4s, v10.4s\n" + "sqrdmulh v23.4s, v23.4s, v10.4s\n" "sshr v7.4s, v7.4s, #0x1f\n" "and v29.16b, v26.16b, v16.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index 5499392d6d..0dc377c5c1 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -269,7 +269,7 @@ void a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr q4, [x13, #0x10]\n" "smlal2 v10.4s, v24.8h, v7.8h\n" "smlal v22.4s, v24.4h, v1.4h\n" - "sqdmulh v15.4s, v15.4s, v19.4s\n" + "sqrdmulh v15.4s, v15.4s, v19.4s\n" "ldr q31, [x11, #0x10]\n" "smlal2 v21.4s, v24.8h, v1.8h\n" "ldr d24, [x22, x15]\n" @@ -285,7 +285,7 @@ void a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "add x17, x17, #0x48\n" "smlal v9.4s, v28.4h, v7.4h\n" "smlal2 v16.4s, v28.8h, v7.8h\n" - "sqdmulh v10.4s, v10.4s, v4.4s\n" + "sqrdmulh v10.4s, v10.4s, v4.4s\n" "subs x16, x16, #0x1\n" "smlal2 v21.4s, v25.8h, v6.8h\n" "ldr d25, [x20, x15]\n" @@ -301,7 +301,7 @@ void a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ssubl v29.8h, v29.8b, v12.8b\n" "smlal2 v21.4s, v27.8h, v7.8h\n" "smlal2 v18.4s, v26.8h, v7.8h\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "add x15, x15, #0x8\n" "smlal v22.4s, v24.4h, v5.4h\n" "smlal v23.4s, v25.4h, v6.4h\n" @@ -309,17 +309,17 @@ void a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "add x11, x11, #0x20\n" "smlal2 v21.4s, v24.8h, v5.8h\n" "smlal2 v18.4s, v25.8h, v6.8h\n" - "sqdmulh v16.4s, v16.4s, v4.4s\n" + "sqrdmulh v16.4s, v16.4s, v4.4s\n" "smlal v22.4s, v25.4h, v8.4h\n" "smlal v23.4s, v29.4h, v8.4h\n" - "sqdmulh v22.4s, v22.4s, v19.4s\n" + "sqrdmulh v22.4s, v22.4s, v19.4s\n" "smlal2 v21.4s, v25.8h, v8.8h\n" "smlal2 v18.4s, v29.8h, v8.8h\n" - "sqdmulh v23.4s, v23.4s, v19.4s\n" + "sqrdmulh v23.4s, v23.4s, v19.4s\n" "and v29.16b, v22.16b, v0.16b\n" - "sqdmulh v21.4s, v21.4s, v4.4s\n" + "sqrdmulh v21.4s, v21.4s, v4.4s\n" "and v20.16b, v23.16b, v0.16b\n" - "sqdmulh v18.4s, v18.4s, v4.4s\n" + "sqrdmulh v18.4s, v18.4s, v4.4s\n" "and v19.16b, v10.16b, v31.16b\n" "sshr v28.4s, v28.4s, #0x1f\n" "and v4.16b, v16.16b, v31.16b\n" @@ -521,7 +521,7 @@ void a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr q4, [x13, #0x10]\n" "smlal2 v10.4s, v24.8h, v7.8h\n" "smlal v22.4s, v24.4h, v1.4h\n" - "sqdmulh v15.4s, v15.4s, v19.4s\n" + "sqrdmulh v15.4s, v15.4s, v19.4s\n" "ldr q31, [x11, #0x10]\n" "smlal2 v21.4s, v24.8h, v1.8h\n" "ldr d24, [x22, x15]\n" @@ -537,7 +537,7 @@ void a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "tst x8, #0x7\n" "smlal v9.4s, v28.4h, v7.4h\n" "smlal2 v16.4s, v28.8h, v7.8h\n" - "sqdmulh v10.4s, v10.4s, v4.4s\n" + "sqrdmulh v10.4s, v10.4s, v4.4s\n" "add x13, x13, #0x20\n" "smlal2 v21.4s, v25.8h, v6.8h\n" "ldr d25, [x20, x15]\n" @@ -553,24 +553,24 @@ void a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ssubl v29.8h, v29.8b, v12.8b\n" "smlal2 v21.4s, v27.8h, v7.8h\n" "smlal2 v18.4s, v26.8h, v7.8h\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "add x15, x15, #0x8\n" "smlal v22.4s, v24.4h, v5.4h\n" "smlal v23.4s, v25.4h, v6.4h\n" "and v28.16b, v9.16b, v0.16b\n" "smlal2 v21.4s, v24.8h, v5.8h\n" "smlal2 v18.4s, v25.8h, v6.8h\n" - "sqdmulh v16.4s, v16.4s, v4.4s\n" + "sqrdmulh v16.4s, v16.4s, v4.4s\n" "smlal v22.4s, v25.4h, v8.4h\n" "smlal v23.4s, v29.4h, v8.4h\n" - "sqdmulh v22.4s, v22.4s, v19.4s\n" + "sqrdmulh v22.4s, v22.4s, v19.4s\n" "smlal2 v21.4s, v25.8h, v8.8h\n" "smlal2 v18.4s, v29.8h, v8.8h\n" - "sqdmulh v23.4s, v23.4s, v19.4s\n" + "sqrdmulh v23.4s, v23.4s, v19.4s\n" "and v29.16b, v22.16b, v0.16b\n" - "sqdmulh v21.4s, v21.4s, v4.4s\n" + "sqrdmulh v21.4s, v21.4s, v4.4s\n" "and v20.16b, v23.16b, v0.16b\n" - "sqdmulh v18.4s, v18.4s, v4.4s\n" + "sqrdmulh v18.4s, v18.4s, v4.4s\n" "and v19.16b, v10.16b, v31.16b\n" "sshr v28.4s, v28.4s, #0x1f\n" "and v4.16b, v16.16b, v31.16b\n" @@ -1274,22 +1274,22 @@ void a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ld1 { v19.s }[0], [x13]\n" "ld1 { v0.s }[0], [x11]\n" "83:" // Oddments: Load requant params: Bit 2: End - "sqdmulh v15.4s, v15.4s, v19.4s\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v15.4s, v15.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "add x10, x10, x14\n" "add x9, x9, x14\n" - "sqdmulh v22.4s, v22.4s, v19.4s\n" - "sqdmulh v23.4s, v23.4s, v19.4s\n" + "sqrdmulh v22.4s, v22.4s, v19.4s\n" + "sqrdmulh v23.4s, v23.4s, v19.4s\n" "add x28, x28, x14\n" "add x27, x27, x14\n" "and v30.16b, v15.16b, v0.16b\n" - "sqdmulh v10.4s, v10.4s, v4.4s\n" + "sqrdmulh v10.4s, v10.4s, v4.4s\n" "and v28.16b, v9.16b, v0.16b\n" - "sqdmulh v16.4s, v16.4s, v4.4s\n" + "sqrdmulh v16.4s, v16.4s, v4.4s\n" "and v29.16b, v22.16b, v0.16b\n" - "sqdmulh v21.4s, v21.4s, v4.4s\n" + "sqrdmulh v21.4s, v21.4s, v4.4s\n" "and v20.16b, v23.16b, v0.16b\n" - "sqdmulh v18.4s, v18.4s, v4.4s\n" + "sqrdmulh v18.4s, v18.4s, v4.4s\n" "sshr v30.4s, v30.4s, #0x1f\n" "and v19.16b, v10.16b, v31.16b\n" "sshr v28.4s, v28.4s, #0x1f\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index bd71b65908..663ea59a98 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -504,24 +504,24 @@ void a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "smlal2 v5.4s, v24.8h, v3.8h\n" "smlal v15.4s, v28.4h, v4.4h\n" "smlal v17.4s, v26.4h, v4.4h\n" - "sqdmulh v15.4s, v15.4s, v12.4s\n" + "sqrdmulh v15.4s, v15.4s, v12.4s\n" "smlal v10.4s, v24.4h, v4.4h\n" "smlal v6.4s, v27.4h, v4.4h\n" - "sqdmulh v17.4s, v17.4s, v12.4s\n" + "sqrdmulh v17.4s, v17.4s, v12.4s\n" "smlal2 v16.4s, v28.8h, v4.8h\n" "smlal2 v8.4s, v26.8h, v4.8h\n" - "sqdmulh v10.4s, v10.4s, v12.4s\n" + "sqrdmulh v10.4s, v10.4s, v12.4s\n" "smlal2 v7.4s, v24.8h, v4.8h\n" "smlal2 v5.4s, v27.8h, v4.8h\n" - "sqdmulh v6.4s, v6.4s, v12.4s\n" + "sqrdmulh v6.4s, v6.4s, v12.4s\n" "and v23.16b, v15.16b, v19.16b\n" - "sqdmulh v16.4s, v16.4s, v20.4s\n" + "sqrdmulh v16.4s, v16.4s, v20.4s\n" "and v22.16b, v17.16b, v19.16b\n" - "sqdmulh v8.4s, v8.4s, v20.4s\n" + "sqrdmulh v8.4s, v8.4s, v20.4s\n" "and v21.16b, v10.16b, v19.16b\n" - "sqdmulh v7.4s, v7.4s, v20.4s\n" + "sqrdmulh v7.4s, v7.4s, v20.4s\n" "and v26.16b, v6.16b, v19.16b\n" - "sqdmulh v5.4s, v5.4s, v20.4s\n" + "sqrdmulh v5.4s, v5.4s, v20.4s\n" "sshr v23.4s, v23.4s, #0x1f\n" "and v4.16b, v16.16b, v29.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" @@ -947,24 +947,24 @@ void a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "smlal2 v5.4s, v24.8h, v3.8h\n" "smlal v15.4s, v28.4h, v4.4h\n" "smlal v17.4s, v26.4h, v4.4h\n" - "sqdmulh v15.4s, v15.4s, v12.4s\n" + "sqrdmulh v15.4s, v15.4s, v12.4s\n" "smlal v10.4s, v24.4h, v4.4h\n" "smlal v6.4s, v27.4h, v4.4h\n" - "sqdmulh v17.4s, v17.4s, v12.4s\n" + "sqrdmulh v17.4s, v17.4s, v12.4s\n" "smlal2 v16.4s, v28.8h, v4.8h\n" "smlal2 v8.4s, v26.8h, v4.8h\n" - "sqdmulh v10.4s, v10.4s, v12.4s\n" + "sqrdmulh v10.4s, v10.4s, v12.4s\n" "smlal2 v7.4s, v24.8h, v4.8h\n" "smlal2 v5.4s, v27.8h, v4.8h\n" - "sqdmulh v6.4s, v6.4s, v12.4s\n" + "sqrdmulh v6.4s, v6.4s, v12.4s\n" "and v23.16b, v15.16b, v19.16b\n" - "sqdmulh v16.4s, v16.4s, v20.4s\n" + "sqrdmulh v16.4s, v16.4s, v20.4s\n" "and v22.16b, v17.16b, v19.16b\n" - "sqdmulh v8.4s, v8.4s, v20.4s\n" + "sqrdmulh v8.4s, v8.4s, v20.4s\n" "and v21.16b, v10.16b, v19.16b\n" - "sqdmulh v7.4s, v7.4s, v20.4s\n" + "sqrdmulh v7.4s, v7.4s, v20.4s\n" "and v26.16b, v6.16b, v19.16b\n" - "sqdmulh v5.4s, v5.4s, v20.4s\n" + "sqrdmulh v5.4s, v5.4s, v20.4s\n" "sshr v23.4s, v23.4s, #0x1f\n" "and v4.16b, v16.16b, v29.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" @@ -2064,22 +2064,22 @@ void a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "ld1 { v12.s }[0], [x10]\n" "ld1 { v19.s }[0], [x1]\n" "119:" // Oddments: Load requant params: Bit 2: End - "sqdmulh v15.4s, v15.4s, v12.4s\n" - "sqdmulh v17.4s, v17.4s, v12.4s\n" + "sqrdmulh v15.4s, v15.4s, v12.4s\n" + "sqrdmulh v17.4s, v17.4s, v12.4s\n" "add x16, x16, x22\n" "add x8, x8, x22\n" - "sqdmulh v10.4s, v10.4s, v12.4s\n" - "sqdmulh v6.4s, v6.4s, v12.4s\n" + "sqrdmulh v10.4s, v10.4s, v12.4s\n" + "sqrdmulh v6.4s, v6.4s, v12.4s\n" "add x4, x4, x22\n" "add x7, x7, x22\n" "and v23.16b, v15.16b, v19.16b\n" - "sqdmulh v16.4s, v16.4s, v20.4s\n" + "sqrdmulh v16.4s, v16.4s, v20.4s\n" "and v22.16b, v17.16b, v19.16b\n" - "sqdmulh v8.4s, v8.4s, v20.4s\n" + "sqrdmulh v8.4s, v8.4s, v20.4s\n" "and v21.16b, v10.16b, v19.16b\n" - "sqdmulh v7.4s, v7.4s, v20.4s\n" + "sqrdmulh v7.4s, v7.4s, v20.4s\n" "and v26.16b, v6.16b, v19.16b\n" - "sqdmulh v5.4s, v5.4s, v20.4s\n" + "sqrdmulh v5.4s, v5.4s, v20.4s\n" "sshr v23.4s, v23.4s, #0x1f\n" "and v4.16b, v16.16b, v29.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index 453f9cfa92..4b0fca77f1 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -267,24 +267,24 @@ void a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( "smlal2 v23.4s, v29.8h, v6.8h\n" "smlal v13.4s, v31.4h, v6.4h\n" "smlal v19.4s, v30.4h, v8.4h\n" - "sqdmulh v13.4s, v13.4s, v21.4s\n" + "sqrdmulh v13.4s, v13.4s, v21.4s\n" "smlal v18.4s, v28.4h, v8.4h\n" "smlal v9.4s, v28.4h, v7.4h\n" - "sqdmulh v19.4s, v19.4s, v21.4s\n" + "sqrdmulh v19.4s, v19.4s, v21.4s\n" "smlal2 v26.4s, v31.8h, v6.8h\n" "smlal2 v11.4s, v30.8h, v8.8h\n" - "sqdmulh v18.4s, v18.4s, v21.4s\n" + "sqrdmulh v18.4s, v18.4s, v21.4s\n" "smlal2 v24.4s, v28.8h, v8.8h\n" "smlal2 v23.4s, v28.8h, v7.8h\n" - "sqdmulh v9.4s, v9.4s, v21.4s\n" + "sqrdmulh v9.4s, v9.4s, v21.4s\n" "and v7.16b, v13.16b, v25.16b\n" - "sqdmulh v26.4s, v26.4s, v10.4s\n" + "sqrdmulh v26.4s, v26.4s, v10.4s\n" "and v4.16b, v19.16b, v25.16b\n" - "sqdmulh v11.4s, v11.4s, v10.4s\n" + "sqrdmulh v11.4s, v11.4s, v10.4s\n" "and v21.16b, v18.16b, v25.16b\n" - "sqdmulh v24.4s, v24.4s, v10.4s\n" + "sqrdmulh v24.4s, v24.4s, v10.4s\n" "and v20.16b, v9.16b, v25.16b\n" - "sqdmulh v23.4s, v23.4s, v10.4s\n" + "sqrdmulh v23.4s, v23.4s, v10.4s\n" "sshr v7.4s, v7.4s, #0x1f\n" "and v29.16b, v26.16b, v16.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" @@ -493,24 +493,24 @@ void a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( "smlal2 v23.4s, v29.8h, v6.8h\n" "smlal v13.4s, v31.4h, v6.4h\n" "smlal v19.4s, v30.4h, v8.4h\n" - "sqdmulh v13.4s, v13.4s, v21.4s\n" + "sqrdmulh v13.4s, v13.4s, v21.4s\n" "smlal v18.4s, v28.4h, v8.4h\n" "smlal v9.4s, v28.4h, v7.4h\n" - "sqdmulh v19.4s, v19.4s, v21.4s\n" + "sqrdmulh v19.4s, v19.4s, v21.4s\n" "smlal2 v26.4s, v31.8h, v6.8h\n" "smlal2 v11.4s, v30.8h, v8.8h\n" - "sqdmulh v18.4s, v18.4s, v21.4s\n" + "sqrdmulh v18.4s, v18.4s, v21.4s\n" "smlal2 v24.4s, v28.8h, v8.8h\n" "smlal2 v23.4s, v28.8h, v7.8h\n" - "sqdmulh v9.4s, v9.4s, v21.4s\n" + "sqrdmulh v9.4s, v9.4s, v21.4s\n" "and v7.16b, v13.16b, v25.16b\n" - "sqdmulh v26.4s, v26.4s, v10.4s\n" + "sqrdmulh v26.4s, v26.4s, v10.4s\n" "and v4.16b, v19.16b, v25.16b\n" - "sqdmulh v11.4s, v11.4s, v10.4s\n" + "sqrdmulh v11.4s, v11.4s, v10.4s\n" "and v21.16b, v18.16b, v25.16b\n" - "sqdmulh v24.4s, v24.4s, v10.4s\n" + "sqrdmulh v24.4s, v24.4s, v10.4s\n" "and v20.16b, v9.16b, v25.16b\n" - "sqdmulh v23.4s, v23.4s, v10.4s\n" + "sqrdmulh v23.4s, v23.4s, v10.4s\n" "sshr v7.4s, v7.4s, #0x1f\n" "and v29.16b, v26.16b, v16.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" @@ -1043,22 +1043,22 @@ void a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( "ld1 { v21.s }[0], [x13]\n" "ld1 { v25.s }[0], [x11]\n" "59:" // Oddments: Load requant params: Bit 2: End - "sqdmulh v13.4s, v13.4s, v21.4s\n" - "sqdmulh v19.4s, v19.4s, v21.4s\n" + "sqrdmulh v13.4s, v13.4s, v21.4s\n" + "sqrdmulh v19.4s, v19.4s, v21.4s\n" "add x10, x10, x14\n" "add x9, x9, x14\n" - "sqdmulh v18.4s, v18.4s, v21.4s\n" - "sqdmulh v9.4s, v9.4s, v21.4s\n" + "sqrdmulh v18.4s, v18.4s, v21.4s\n" + "sqrdmulh v9.4s, v9.4s, v21.4s\n" "add x28, x28, x14\n" "add x27, x27, x14\n" "and v7.16b, v13.16b, v25.16b\n" - "sqdmulh v26.4s, v26.4s, v10.4s\n" + "sqrdmulh v26.4s, v26.4s, v10.4s\n" "and v4.16b, v19.16b, v25.16b\n" - "sqdmulh v11.4s, v11.4s, v10.4s\n" + "sqrdmulh v11.4s, v11.4s, v10.4s\n" "and v21.16b, v18.16b, v25.16b\n" - "sqdmulh v24.4s, v24.4s, v10.4s\n" + "sqrdmulh v24.4s, v24.4s, v10.4s\n" "and v20.16b, v9.16b, v25.16b\n" - "sqdmulh v23.4s, v23.4s, v10.4s\n" + "sqrdmulh v23.4s, v23.4s, v10.4s\n" "sshr v7.4s, v7.4s, #0x1f\n" "and v29.16b, v26.16b, v16.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index 872f665b40..0216786c6f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -269,7 +269,7 @@ void a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr q4, [x13, #0x10]\n" "smlal2 v10.4s, v24.8h, v7.8h\n" "smlal v22.4s, v24.4h, v1.4h\n" - "sqdmulh v15.4s, v15.4s, v19.4s\n" + "sqrdmulh v15.4s, v15.4s, v19.4s\n" "ldr q31, [x11, #0x10]\n" "smlal2 v21.4s, v24.8h, v1.8h\n" "ldr d24, [x22, x15]\n" @@ -285,7 +285,7 @@ void a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "add x17, x17, #0x48\n" "smlal v9.4s, v28.4h, v7.4h\n" "smlal2 v16.4s, v28.8h, v7.8h\n" - "sqdmulh v10.4s, v10.4s, v4.4s\n" + "sqrdmulh v10.4s, v10.4s, v4.4s\n" "subs x16, x16, #0x1\n" "smlal2 v21.4s, v25.8h, v6.8h\n" "ldr d25, [x20, x15]\n" @@ -301,7 +301,7 @@ void a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "usubl v29.8h, v29.8b, v12.8b\n" "smlal2 v21.4s, v27.8h, v7.8h\n" "smlal2 v18.4s, v26.8h, v7.8h\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "add x15, x15, #0x8\n" "smlal v22.4s, v24.4h, v5.4h\n" "smlal v23.4s, v25.4h, v6.4h\n" @@ -309,17 +309,17 @@ void a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "add x11, x11, #0x20\n" "smlal2 v21.4s, v24.8h, v5.8h\n" "smlal2 v18.4s, v25.8h, v6.8h\n" - "sqdmulh v16.4s, v16.4s, v4.4s\n" + "sqrdmulh v16.4s, v16.4s, v4.4s\n" "smlal v22.4s, v25.4h, v8.4h\n" "smlal v23.4s, v29.4h, v8.4h\n" - "sqdmulh v22.4s, v22.4s, v19.4s\n" + "sqrdmulh v22.4s, v22.4s, v19.4s\n" "smlal2 v21.4s, v25.8h, v8.8h\n" "smlal2 v18.4s, v29.8h, v8.8h\n" - "sqdmulh v23.4s, v23.4s, v19.4s\n" + "sqrdmulh v23.4s, v23.4s, v19.4s\n" "and v29.16b, v22.16b, v0.16b\n" - "sqdmulh v21.4s, v21.4s, v4.4s\n" + "sqrdmulh v21.4s, v21.4s, v4.4s\n" "and v20.16b, v23.16b, v0.16b\n" - "sqdmulh v18.4s, v18.4s, v4.4s\n" + "sqrdmulh v18.4s, v18.4s, v4.4s\n" "and v19.16b, v10.16b, v31.16b\n" "sshr v28.4s, v28.4s, #0x1f\n" "and v4.16b, v16.16b, v31.16b\n" @@ -521,7 +521,7 @@ void a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr q4, [x13, #0x10]\n" "smlal2 v10.4s, v24.8h, v7.8h\n" "smlal v22.4s, v24.4h, v1.4h\n" - "sqdmulh v15.4s, v15.4s, v19.4s\n" + "sqrdmulh v15.4s, v15.4s, v19.4s\n" "ldr q31, [x11, #0x10]\n" "smlal2 v21.4s, v24.8h, v1.8h\n" "ldr d24, [x22, x15]\n" @@ -537,7 +537,7 @@ void a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "tst x8, #0x7\n" "smlal v9.4s, v28.4h, v7.4h\n" "smlal2 v16.4s, v28.8h, v7.8h\n" - "sqdmulh v10.4s, v10.4s, v4.4s\n" + "sqrdmulh v10.4s, v10.4s, v4.4s\n" "add x13, x13, #0x20\n" "smlal2 v21.4s, v25.8h, v6.8h\n" "ldr d25, [x20, x15]\n" @@ -553,24 +553,24 @@ void a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "usubl v29.8h, v29.8b, v12.8b\n" "smlal2 v21.4s, v27.8h, v7.8h\n" "smlal2 v18.4s, v26.8h, v7.8h\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "add x15, x15, #0x8\n" "smlal v22.4s, v24.4h, v5.4h\n" "smlal v23.4s, v25.4h, v6.4h\n" "and v28.16b, v9.16b, v0.16b\n" "smlal2 v21.4s, v24.8h, v5.8h\n" "smlal2 v18.4s, v25.8h, v6.8h\n" - "sqdmulh v16.4s, v16.4s, v4.4s\n" + "sqrdmulh v16.4s, v16.4s, v4.4s\n" "smlal v22.4s, v25.4h, v8.4h\n" "smlal v23.4s, v29.4h, v8.4h\n" - "sqdmulh v22.4s, v22.4s, v19.4s\n" + "sqrdmulh v22.4s, v22.4s, v19.4s\n" "smlal2 v21.4s, v25.8h, v8.8h\n" "smlal2 v18.4s, v29.8h, v8.8h\n" - "sqdmulh v23.4s, v23.4s, v19.4s\n" + "sqrdmulh v23.4s, v23.4s, v19.4s\n" "and v29.16b, v22.16b, v0.16b\n" - "sqdmulh v21.4s, v21.4s, v4.4s\n" + "sqrdmulh v21.4s, v21.4s, v4.4s\n" "and v20.16b, v23.16b, v0.16b\n" - "sqdmulh v18.4s, v18.4s, v4.4s\n" + "sqrdmulh v18.4s, v18.4s, v4.4s\n" "and v19.16b, v10.16b, v31.16b\n" "sshr v28.4s, v28.4s, #0x1f\n" "and v4.16b, v16.16b, v31.16b\n" @@ -1274,22 +1274,22 @@ void a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ld1 { v19.s }[0], [x13]\n" "ld1 { v0.s }[0], [x11]\n" "83:" // Oddments: Load requant params: Bit 2: End - "sqdmulh v15.4s, v15.4s, v19.4s\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v15.4s, v15.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "add x10, x10, x14\n" "add x9, x9, x14\n" - "sqdmulh v22.4s, v22.4s, v19.4s\n" - "sqdmulh v23.4s, v23.4s, v19.4s\n" + "sqrdmulh v22.4s, v22.4s, v19.4s\n" + "sqrdmulh v23.4s, v23.4s, v19.4s\n" "add x28, x28, x14\n" "add x27, x27, x14\n" "and v30.16b, v15.16b, v0.16b\n" - "sqdmulh v10.4s, v10.4s, v4.4s\n" + "sqrdmulh v10.4s, v10.4s, v4.4s\n" "and v28.16b, v9.16b, v0.16b\n" - "sqdmulh v16.4s, v16.4s, v4.4s\n" + "sqrdmulh v16.4s, v16.4s, v4.4s\n" "and v29.16b, v22.16b, v0.16b\n" - "sqdmulh v21.4s, v21.4s, v4.4s\n" + "sqrdmulh v21.4s, v21.4s, v4.4s\n" "and v20.16b, v23.16b, v0.16b\n" - "sqdmulh v18.4s, v18.4s, v4.4s\n" + "sqrdmulh v18.4s, v18.4s, v4.4s\n" "sshr v30.4s, v30.4s, #0x1f\n" "and v19.16b, v10.16b, v31.16b\n" "sshr v28.4s, v28.4s, #0x1f\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index 6934dffc98..bd6fa1d443 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -504,24 +504,24 @@ void a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "smlal2 v5.4s, v24.8h, v3.8h\n" "smlal v15.4s, v28.4h, v4.4h\n" "smlal v17.4s, v26.4h, v4.4h\n" - "sqdmulh v15.4s, v15.4s, v12.4s\n" + "sqrdmulh v15.4s, v15.4s, v12.4s\n" "smlal v10.4s, v24.4h, v4.4h\n" "smlal v6.4s, v27.4h, v4.4h\n" - "sqdmulh v17.4s, v17.4s, v12.4s\n" + "sqrdmulh v17.4s, v17.4s, v12.4s\n" "smlal2 v16.4s, v28.8h, v4.8h\n" "smlal2 v8.4s, v26.8h, v4.8h\n" - "sqdmulh v10.4s, v10.4s, v12.4s\n" + "sqrdmulh v10.4s, v10.4s, v12.4s\n" "smlal2 v7.4s, v24.8h, v4.8h\n" "smlal2 v5.4s, v27.8h, v4.8h\n" - "sqdmulh v6.4s, v6.4s, v12.4s\n" + "sqrdmulh v6.4s, v6.4s, v12.4s\n" "and v23.16b, v15.16b, v19.16b\n" - "sqdmulh v16.4s, v16.4s, v20.4s\n" + "sqrdmulh v16.4s, v16.4s, v20.4s\n" "and v22.16b, v17.16b, v19.16b\n" - "sqdmulh v8.4s, v8.4s, v20.4s\n" + "sqrdmulh v8.4s, v8.4s, v20.4s\n" "and v21.16b, v10.16b, v19.16b\n" - "sqdmulh v7.4s, v7.4s, v20.4s\n" + "sqrdmulh v7.4s, v7.4s, v20.4s\n" "and v26.16b, v6.16b, v19.16b\n" - "sqdmulh v5.4s, v5.4s, v20.4s\n" + "sqrdmulh v5.4s, v5.4s, v20.4s\n" "sshr v23.4s, v23.4s, #0x1f\n" "and v4.16b, v16.16b, v29.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" @@ -947,24 +947,24 @@ void a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "smlal2 v5.4s, v24.8h, v3.8h\n" "smlal v15.4s, v28.4h, v4.4h\n" "smlal v17.4s, v26.4h, v4.4h\n" - "sqdmulh v15.4s, v15.4s, v12.4s\n" + "sqrdmulh v15.4s, v15.4s, v12.4s\n" "smlal v10.4s, v24.4h, v4.4h\n" "smlal v6.4s, v27.4h, v4.4h\n" - "sqdmulh v17.4s, v17.4s, v12.4s\n" + "sqrdmulh v17.4s, v17.4s, v12.4s\n" "smlal2 v16.4s, v28.8h, v4.8h\n" "smlal2 v8.4s, v26.8h, v4.8h\n" - "sqdmulh v10.4s, v10.4s, v12.4s\n" + "sqrdmulh v10.4s, v10.4s, v12.4s\n" "smlal2 v7.4s, v24.8h, v4.8h\n" "smlal2 v5.4s, v27.8h, v4.8h\n" - "sqdmulh v6.4s, v6.4s, v12.4s\n" + "sqrdmulh v6.4s, v6.4s, v12.4s\n" "and v23.16b, v15.16b, v19.16b\n" - "sqdmulh v16.4s, v16.4s, v20.4s\n" + "sqrdmulh v16.4s, v16.4s, v20.4s\n" "and v22.16b, v17.16b, v19.16b\n" - "sqdmulh v8.4s, v8.4s, v20.4s\n" + "sqrdmulh v8.4s, v8.4s, v20.4s\n" "and v21.16b, v10.16b, v19.16b\n" - "sqdmulh v7.4s, v7.4s, v20.4s\n" + "sqrdmulh v7.4s, v7.4s, v20.4s\n" "and v26.16b, v6.16b, v19.16b\n" - "sqdmulh v5.4s, v5.4s, v20.4s\n" + "sqrdmulh v5.4s, v5.4s, v20.4s\n" "sshr v23.4s, v23.4s, #0x1f\n" "and v4.16b, v16.16b, v29.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" @@ -2064,22 +2064,22 @@ void a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "ld1 { v12.s }[0], [x10]\n" "ld1 { v19.s }[0], [x1]\n" "119:" // Oddments: Load requant params: Bit 2: End - "sqdmulh v15.4s, v15.4s, v12.4s\n" - "sqdmulh v17.4s, v17.4s, v12.4s\n" + "sqrdmulh v15.4s, v15.4s, v12.4s\n" + "sqrdmulh v17.4s, v17.4s, v12.4s\n" "add x16, x16, x22\n" "add x8, x8, x22\n" - "sqdmulh v10.4s, v10.4s, v12.4s\n" - "sqdmulh v6.4s, v6.4s, v12.4s\n" + "sqrdmulh v10.4s, v10.4s, v12.4s\n" + "sqrdmulh v6.4s, v6.4s, v12.4s\n" "add x4, x4, x22\n" "add x7, x7, x22\n" "and v23.16b, v15.16b, v19.16b\n" - "sqdmulh v16.4s, v16.4s, v20.4s\n" + "sqrdmulh v16.4s, v16.4s, v20.4s\n" "and v22.16b, v17.16b, v19.16b\n" - "sqdmulh v8.4s, v8.4s, v20.4s\n" + "sqrdmulh v8.4s, v8.4s, v20.4s\n" "and v21.16b, v10.16b, v19.16b\n" - "sqdmulh v7.4s, v7.4s, v20.4s\n" + "sqrdmulh v7.4s, v7.4s, v20.4s\n" "and v26.16b, v6.16b, v19.16b\n" - "sqdmulh v5.4s, v5.4s, v20.4s\n" + "sqrdmulh v5.4s, v5.4s, v20.4s\n" "sshr v23.4s, v23.4s, #0x1f\n" "and v4.16b, v16.16b, v29.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index 2410d38512..89cb2ec380 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -265,24 +265,24 @@ void a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( "smlal2 v26.4s, v29.8h, v6.8h\n" "smlal v12.4s, v31.4h, v6.4h\n" "smlal v14.4s, v30.4h, v8.4h\n" - "sqdmulh v12.4s, v12.4s, v21.4s\n" + "sqrdmulh v12.4s, v12.4s, v21.4s\n" "smlal v16.4s, v28.4h, v8.4h\n" "smlal v18.4s, v28.4h, v7.4h\n" - "sqdmulh v14.4s, v14.4s, v21.4s\n" + "sqrdmulh v14.4s, v14.4s, v21.4s\n" "smlal2 v17.4s, v31.8h, v6.8h\n" "smlal2 v9.4s, v30.8h, v8.8h\n" - "sqdmulh v16.4s, v16.4s, v21.4s\n" + "sqrdmulh v16.4s, v16.4s, v21.4s\n" "smlal2 v10.4s, v28.8h, v8.8h\n" "smlal2 v26.4s, v28.8h, v7.8h\n" - "sqdmulh v18.4s, v18.4s, v21.4s\n" + "sqrdmulh v18.4s, v18.4s, v21.4s\n" "and v29.16b, v12.16b, v24.16b\n" - "sqdmulh v17.4s, v17.4s, v19.4s\n" + "sqrdmulh v17.4s, v17.4s, v19.4s\n" "and v22.16b, v14.16b, v24.16b\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "and v21.16b, v16.16b, v24.16b\n" - "sqdmulh v10.4s, v10.4s, v19.4s\n" + "sqrdmulh v10.4s, v10.4s, v19.4s\n" "and v20.16b, v18.16b, v24.16b\n" - "sqdmulh v26.4s, v26.4s, v19.4s\n" + "sqrdmulh v26.4s, v26.4s, v19.4s\n" "sshr v29.4s, v29.4s, #0x1f\n" "and v19.16b, v17.16b, v23.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" @@ -491,24 +491,24 @@ void a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( "smlal2 v26.4s, v29.8h, v6.8h\n" "smlal v12.4s, v31.4h, v6.4h\n" "smlal v14.4s, v30.4h, v8.4h\n" - "sqdmulh v12.4s, v12.4s, v21.4s\n" + "sqrdmulh v12.4s, v12.4s, v21.4s\n" "smlal v16.4s, v28.4h, v8.4h\n" "smlal v18.4s, v28.4h, v7.4h\n" - "sqdmulh v14.4s, v14.4s, v21.4s\n" + "sqrdmulh v14.4s, v14.4s, v21.4s\n" "smlal2 v17.4s, v31.8h, v6.8h\n" "smlal2 v9.4s, v30.8h, v8.8h\n" - "sqdmulh v16.4s, v16.4s, v21.4s\n" + "sqrdmulh v16.4s, v16.4s, v21.4s\n" "smlal2 v10.4s, v28.8h, v8.8h\n" "smlal2 v26.4s, v28.8h, v7.8h\n" - "sqdmulh v18.4s, v18.4s, v21.4s\n" + "sqrdmulh v18.4s, v18.4s, v21.4s\n" "and v29.16b, v12.16b, v24.16b\n" - "sqdmulh v17.4s, v17.4s, v19.4s\n" + "sqrdmulh v17.4s, v17.4s, v19.4s\n" "and v22.16b, v14.16b, v24.16b\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "and v21.16b, v16.16b, v24.16b\n" - "sqdmulh v10.4s, v10.4s, v19.4s\n" + "sqrdmulh v10.4s, v10.4s, v19.4s\n" "and v20.16b, v18.16b, v24.16b\n" - "sqdmulh v26.4s, v26.4s, v19.4s\n" + "sqrdmulh v26.4s, v26.4s, v19.4s\n" "sshr v29.4s, v29.4s, #0x1f\n" "and v19.16b, v17.16b, v23.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" @@ -1041,22 +1041,22 @@ void a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( "ld1 { v21.s }[0], [x12]\n" "ld1 { v24.s }[0], [x11]\n" "59:" // Oddments: Load requant params: Bit 2: End - "sqdmulh v12.4s, v12.4s, v21.4s\n" - "sqdmulh v14.4s, v14.4s, v21.4s\n" + "sqrdmulh v12.4s, v12.4s, v21.4s\n" + "sqrdmulh v14.4s, v14.4s, v21.4s\n" "add x10, x10, x14\n" "add x9, x9, x14\n" - "sqdmulh v16.4s, v16.4s, v21.4s\n" - "sqdmulh v18.4s, v18.4s, v21.4s\n" + "sqrdmulh v16.4s, v16.4s, v21.4s\n" + "sqrdmulh v18.4s, v18.4s, v21.4s\n" "add x28, x28, x14\n" "add x27, x27, x14\n" "and v29.16b, v12.16b, v24.16b\n" - "sqdmulh v17.4s, v17.4s, v19.4s\n" + "sqrdmulh v17.4s, v17.4s, v19.4s\n" "and v22.16b, v14.16b, v24.16b\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "and v21.16b, v16.16b, v24.16b\n" - "sqdmulh v10.4s, v10.4s, v19.4s\n" + "sqrdmulh v10.4s, v10.4s, v19.4s\n" "and v20.16b, v18.16b, v24.16b\n" - "sqdmulh v26.4s, v26.4s, v19.4s\n" + "sqrdmulh v26.4s, v26.4s, v19.4s\n" "sshr v29.4s, v29.4s, #0x1f\n" "and v19.16b, v17.16b, v23.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index 49f69c4204..42ff502b0f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -267,7 +267,7 @@ void a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr q19, [x12, #0x10]\n" "smlal2 v18.4s, v24.8h, v7.8h\n" "smlal v11.4s, v24.4h, v1.4h\n" - "sqdmulh v15.4s, v15.4s, v22.4s\n" + "sqrdmulh v15.4s, v15.4s, v22.4s\n" "ldr q30, [x11, #0x10]\n" "smlal2 v10.4s, v24.8h, v1.8h\n" "ldr d24, [x22, x15]\n" @@ -283,7 +283,7 @@ void a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "add x17, x17, #0x48\n" "smlal v13.4s, v28.4h, v7.4h\n" "smlal2 v17.4s, v28.8h, v7.8h\n" - "sqdmulh v18.4s, v18.4s, v19.4s\n" + "sqrdmulh v18.4s, v18.4s, v19.4s\n" "subs x16, x16, #0x1\n" "smlal2 v10.4s, v25.8h, v6.8h\n" "ldr d25, [x20, x15]\n" @@ -299,7 +299,7 @@ void a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ushll v29.8h, v29.8b, #0x0\n" "smlal2 v10.4s, v27.8h, v7.8h\n" "smlal2 v9.4s, v26.8h, v7.8h\n" - "sqdmulh v13.4s, v13.4s, v22.4s\n" + "sqrdmulh v13.4s, v13.4s, v22.4s\n" "add x15, x15, #0x8\n" "smlal v11.4s, v24.4h, v5.4h\n" "smlal v23.4s, v25.4h, v6.4h\n" @@ -307,17 +307,17 @@ void a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "add x11, x11, #0x20\n" "smlal2 v10.4s, v24.8h, v5.8h\n" "smlal2 v9.4s, v25.8h, v6.8h\n" - "sqdmulh v17.4s, v17.4s, v19.4s\n" + "sqrdmulh v17.4s, v17.4s, v19.4s\n" "smlal v11.4s, v25.4h, v8.4h\n" "smlal v23.4s, v29.4h, v8.4h\n" - "sqdmulh v11.4s, v11.4s, v22.4s\n" + "sqrdmulh v11.4s, v11.4s, v22.4s\n" "smlal2 v10.4s, v25.8h, v8.8h\n" "smlal2 v9.4s, v29.8h, v8.8h\n" - "sqdmulh v23.4s, v23.4s, v22.4s\n" + "sqrdmulh v23.4s, v23.4s, v22.4s\n" "and v22.16b, v11.16b, v31.16b\n" - "sqdmulh v10.4s, v10.4s, v19.4s\n" + "sqrdmulh v10.4s, v10.4s, v19.4s\n" "and v20.16b, v23.16b, v31.16b\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "and v19.16b, v18.16b, v30.16b\n" "sshr v1.4s, v1.4s, #0x1f\n" "and v27.16b, v17.16b, v30.16b\n" @@ -519,7 +519,7 @@ void a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr q19, [x12, #0x10]\n" "smlal2 v18.4s, v24.8h, v7.8h\n" "smlal v11.4s, v24.4h, v1.4h\n" - "sqdmulh v15.4s, v15.4s, v22.4s\n" + "sqrdmulh v15.4s, v15.4s, v22.4s\n" "ldr q30, [x11, #0x10]\n" "smlal2 v10.4s, v24.8h, v1.8h\n" "ldr d24, [x22, x15]\n" @@ -535,7 +535,7 @@ void a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "tst x8, #0x7\n" "smlal v13.4s, v28.4h, v7.4h\n" "smlal2 v17.4s, v28.8h, v7.8h\n" - "sqdmulh v18.4s, v18.4s, v19.4s\n" + "sqrdmulh v18.4s, v18.4s, v19.4s\n" "add x12, x12, #0x20\n" "smlal2 v10.4s, v25.8h, v6.8h\n" "ldr d25, [x20, x15]\n" @@ -551,24 +551,24 @@ void a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ushll v29.8h, v29.8b, #0x0\n" "smlal2 v10.4s, v27.8h, v7.8h\n" "smlal2 v9.4s, v26.8h, v7.8h\n" - "sqdmulh v13.4s, v13.4s, v22.4s\n" + "sqrdmulh v13.4s, v13.4s, v22.4s\n" "add x15, x15, #0x8\n" "smlal v11.4s, v24.4h, v5.4h\n" "smlal v23.4s, v25.4h, v6.4h\n" "and v1.16b, v13.16b, v31.16b\n" "smlal2 v10.4s, v24.8h, v5.8h\n" "smlal2 v9.4s, v25.8h, v6.8h\n" - "sqdmulh v17.4s, v17.4s, v19.4s\n" + "sqrdmulh v17.4s, v17.4s, v19.4s\n" "smlal v11.4s, v25.4h, v8.4h\n" "smlal v23.4s, v29.4h, v8.4h\n" - "sqdmulh v11.4s, v11.4s, v22.4s\n" + "sqrdmulh v11.4s, v11.4s, v22.4s\n" "smlal2 v10.4s, v25.8h, v8.8h\n" "smlal2 v9.4s, v29.8h, v8.8h\n" - "sqdmulh v23.4s, v23.4s, v22.4s\n" + "sqrdmulh v23.4s, v23.4s, v22.4s\n" "and v22.16b, v11.16b, v31.16b\n" - "sqdmulh v10.4s, v10.4s, v19.4s\n" + "sqrdmulh v10.4s, v10.4s, v19.4s\n" "and v20.16b, v23.16b, v31.16b\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "and v19.16b, v18.16b, v30.16b\n" "sshr v1.4s, v1.4s, #0x1f\n" "and v27.16b, v17.16b, v30.16b\n" @@ -1272,22 +1272,22 @@ void a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ld1 { v22.s }[0], [x12]\n" "ld1 { v31.s }[0], [x11]\n" "83:" // Oddments: Load requant params: Bit 2: End - "sqdmulh v15.4s, v15.4s, v22.4s\n" - "sqdmulh v13.4s, v13.4s, v22.4s\n" + "sqrdmulh v15.4s, v15.4s, v22.4s\n" + "sqrdmulh v13.4s, v13.4s, v22.4s\n" "add x10, x10, x14\n" "add x9, x9, x14\n" - "sqdmulh v11.4s, v11.4s, v22.4s\n" - "sqdmulh v23.4s, v23.4s, v22.4s\n" + "sqrdmulh v11.4s, v11.4s, v22.4s\n" + "sqrdmulh v23.4s, v23.4s, v22.4s\n" "add x28, x28, x14\n" "add x27, x27, x14\n" "and v4.16b, v15.16b, v31.16b\n" - "sqdmulh v18.4s, v18.4s, v19.4s\n" + "sqrdmulh v18.4s, v18.4s, v19.4s\n" "and v1.16b, v13.16b, v31.16b\n" - "sqdmulh v17.4s, v17.4s, v19.4s\n" + "sqrdmulh v17.4s, v17.4s, v19.4s\n" "and v22.16b, v11.16b, v31.16b\n" - "sqdmulh v10.4s, v10.4s, v19.4s\n" + "sqrdmulh v10.4s, v10.4s, v19.4s\n" "and v20.16b, v23.16b, v31.16b\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "sshr v4.4s, v4.4s, #0x1f\n" "and v19.16b, v18.16b, v30.16b\n" "sshr v1.4s, v1.4s, #0x1f\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index 7f1fd1d568..9ac7173b4c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -502,24 +502,24 @@ void a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "smlal2 v6.4s, v24.8h, v3.8h\n" "smlal v11.4s, v28.4h, v4.4h\n" "smlal v14.4s, v26.4h, v4.4h\n" - "sqdmulh v11.4s, v11.4s, v17.4s\n" + "sqrdmulh v11.4s, v11.4s, v17.4s\n" "smlal v9.4s, v24.4h, v4.4h\n" "smlal v7.4s, v27.4h, v4.4h\n" - "sqdmulh v14.4s, v14.4s, v17.4s\n" + "sqrdmulh v14.4s, v14.4s, v17.4s\n" "smlal2 v21.4s, v28.8h, v4.8h\n" "smlal2 v10.4s, v26.8h, v4.8h\n" - "sqdmulh v9.4s, v9.4s, v17.4s\n" + "sqrdmulh v9.4s, v9.4s, v17.4s\n" "smlal2 v8.4s, v24.8h, v4.8h\n" "smlal2 v6.4s, v27.8h, v4.8h\n" - "sqdmulh v7.4s, v7.4s, v17.4s\n" + "sqrdmulh v7.4s, v7.4s, v17.4s\n" "and v23.16b, v11.16b, v5.16b\n" - "sqdmulh v21.4s, v21.4s, v18.4s\n" + "sqrdmulh v21.4s, v21.4s, v18.4s\n" "and v22.16b, v14.16b, v5.16b\n" - "sqdmulh v10.4s, v10.4s, v18.4s\n" + "sqrdmulh v10.4s, v10.4s, v18.4s\n" "and v17.16b, v9.16b, v5.16b\n" - "sqdmulh v8.4s, v8.4s, v18.4s\n" + "sqrdmulh v8.4s, v8.4s, v18.4s\n" "and v20.16b, v7.16b, v5.16b\n" - "sqdmulh v6.4s, v6.4s, v18.4s\n" + "sqrdmulh v6.4s, v6.4s, v18.4s\n" "sshr v23.4s, v23.4s, #0x1f\n" "and v19.16b, v21.16b, v29.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" @@ -945,24 +945,24 @@ void a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "smlal2 v6.4s, v24.8h, v3.8h\n" "smlal v11.4s, v28.4h, v4.4h\n" "smlal v14.4s, v26.4h, v4.4h\n" - "sqdmulh v11.4s, v11.4s, v17.4s\n" + "sqrdmulh v11.4s, v11.4s, v17.4s\n" "smlal v9.4s, v24.4h, v4.4h\n" "smlal v7.4s, v27.4h, v4.4h\n" - "sqdmulh v14.4s, v14.4s, v17.4s\n" + "sqrdmulh v14.4s, v14.4s, v17.4s\n" "smlal2 v21.4s, v28.8h, v4.8h\n" "smlal2 v10.4s, v26.8h, v4.8h\n" - "sqdmulh v9.4s, v9.4s, v17.4s\n" + "sqrdmulh v9.4s, v9.4s, v17.4s\n" "smlal2 v8.4s, v24.8h, v4.8h\n" "smlal2 v6.4s, v27.8h, v4.8h\n" - "sqdmulh v7.4s, v7.4s, v17.4s\n" + "sqrdmulh v7.4s, v7.4s, v17.4s\n" "and v23.16b, v11.16b, v5.16b\n" - "sqdmulh v21.4s, v21.4s, v18.4s\n" + "sqrdmulh v21.4s, v21.4s, v18.4s\n" "and v22.16b, v14.16b, v5.16b\n" - "sqdmulh v10.4s, v10.4s, v18.4s\n" + "sqrdmulh v10.4s, v10.4s, v18.4s\n" "and v17.16b, v9.16b, v5.16b\n" - "sqdmulh v8.4s, v8.4s, v18.4s\n" + "sqrdmulh v8.4s, v8.4s, v18.4s\n" "and v20.16b, v7.16b, v5.16b\n" - "sqdmulh v6.4s, v6.4s, v18.4s\n" + "sqrdmulh v6.4s, v6.4s, v18.4s\n" "sshr v23.4s, v23.4s, #0x1f\n" "and v19.16b, v21.16b, v29.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" @@ -2062,22 +2062,22 @@ void a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "ld1 { v17.s }[0], [x5]\n" "ld1 { v5.s }[0], [x6]\n" "119:" // Oddments: Load requant params: Bit 2: End - "sqdmulh v11.4s, v11.4s, v17.4s\n" - "sqdmulh v14.4s, v14.4s, v17.4s\n" + "sqrdmulh v11.4s, v11.4s, v17.4s\n" + "sqrdmulh v14.4s, v14.4s, v17.4s\n" "add x21, x21, x2\n" "add x15, x15, x2\n" - "sqdmulh v9.4s, v9.4s, v17.4s\n" - "sqdmulh v7.4s, v7.4s, v17.4s\n" + "sqrdmulh v9.4s, v9.4s, v17.4s\n" + "sqrdmulh v7.4s, v7.4s, v17.4s\n" "add x17, x17, x2\n" "add x16, x16, x2\n" "and v23.16b, v11.16b, v5.16b\n" - "sqdmulh v21.4s, v21.4s, v18.4s\n" + "sqrdmulh v21.4s, v21.4s, v18.4s\n" "and v22.16b, v14.16b, v5.16b\n" - "sqdmulh v10.4s, v10.4s, v18.4s\n" + "sqrdmulh v10.4s, v10.4s, v18.4s\n" "and v17.16b, v9.16b, v5.16b\n" - "sqdmulh v8.4s, v8.4s, v18.4s\n" + "sqrdmulh v8.4s, v8.4s, v18.4s\n" "and v20.16b, v7.16b, v5.16b\n" - "sqdmulh v6.4s, v6.4s, v18.4s\n" + "sqrdmulh v6.4s, v6.4s, v18.4s\n" "sshr v23.4s, v23.4s, #0x1f\n" "and v19.16b, v21.16b, v29.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index 22f95746fc..96cde40e04 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -267,24 +267,24 @@ void a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( "smlal2 v23.4s, v29.8h, v6.8h\n" "smlal v13.4s, v31.4h, v6.4h\n" "smlal v19.4s, v30.4h, v8.4h\n" - "sqdmulh v13.4s, v13.4s, v21.4s\n" + "sqrdmulh v13.4s, v13.4s, v21.4s\n" "smlal v18.4s, v28.4h, v8.4h\n" "smlal v9.4s, v28.4h, v7.4h\n" - "sqdmulh v19.4s, v19.4s, v21.4s\n" + "sqrdmulh v19.4s, v19.4s, v21.4s\n" "smlal2 v26.4s, v31.8h, v6.8h\n" "smlal2 v11.4s, v30.8h, v8.8h\n" - "sqdmulh v18.4s, v18.4s, v21.4s\n" + "sqrdmulh v18.4s, v18.4s, v21.4s\n" "smlal2 v24.4s, v28.8h, v8.8h\n" "smlal2 v23.4s, v28.8h, v7.8h\n" - "sqdmulh v9.4s, v9.4s, v21.4s\n" + "sqrdmulh v9.4s, v9.4s, v21.4s\n" "and v7.16b, v13.16b, v25.16b\n" - "sqdmulh v26.4s, v26.4s, v10.4s\n" + "sqrdmulh v26.4s, v26.4s, v10.4s\n" "and v4.16b, v19.16b, v25.16b\n" - "sqdmulh v11.4s, v11.4s, v10.4s\n" + "sqrdmulh v11.4s, v11.4s, v10.4s\n" "and v21.16b, v18.16b, v25.16b\n" - "sqdmulh v24.4s, v24.4s, v10.4s\n" + "sqrdmulh v24.4s, v24.4s, v10.4s\n" "and v20.16b, v9.16b, v25.16b\n" - "sqdmulh v23.4s, v23.4s, v10.4s\n" + "sqrdmulh v23.4s, v23.4s, v10.4s\n" "sshr v7.4s, v7.4s, #0x1f\n" "and v29.16b, v26.16b, v16.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" @@ -493,24 +493,24 @@ void a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( "smlal2 v23.4s, v29.8h, v6.8h\n" "smlal v13.4s, v31.4h, v6.4h\n" "smlal v19.4s, v30.4h, v8.4h\n" - "sqdmulh v13.4s, v13.4s, v21.4s\n" + "sqrdmulh v13.4s, v13.4s, v21.4s\n" "smlal v18.4s, v28.4h, v8.4h\n" "smlal v9.4s, v28.4h, v7.4h\n" - "sqdmulh v19.4s, v19.4s, v21.4s\n" + "sqrdmulh v19.4s, v19.4s, v21.4s\n" "smlal2 v26.4s, v31.8h, v6.8h\n" "smlal2 v11.4s, v30.8h, v8.8h\n" - "sqdmulh v18.4s, v18.4s, v21.4s\n" + "sqrdmulh v18.4s, v18.4s, v21.4s\n" "smlal2 v24.4s, v28.8h, v8.8h\n" "smlal2 v23.4s, v28.8h, v7.8h\n" - "sqdmulh v9.4s, v9.4s, v21.4s\n" + "sqrdmulh v9.4s, v9.4s, v21.4s\n" "and v7.16b, v13.16b, v25.16b\n" - "sqdmulh v26.4s, v26.4s, v10.4s\n" + "sqrdmulh v26.4s, v26.4s, v10.4s\n" "and v4.16b, v19.16b, v25.16b\n" - "sqdmulh v11.4s, v11.4s, v10.4s\n" + "sqrdmulh v11.4s, v11.4s, v10.4s\n" "and v21.16b, v18.16b, v25.16b\n" - "sqdmulh v24.4s, v24.4s, v10.4s\n" + "sqrdmulh v24.4s, v24.4s, v10.4s\n" "and v20.16b, v9.16b, v25.16b\n" - "sqdmulh v23.4s, v23.4s, v10.4s\n" + "sqrdmulh v23.4s, v23.4s, v10.4s\n" "sshr v7.4s, v7.4s, #0x1f\n" "and v29.16b, v26.16b, v16.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" @@ -1043,22 +1043,22 @@ void a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst_impl( "ld1 { v21.s }[0], [x13]\n" "ld1 { v25.s }[0], [x11]\n" "59:" // Oddments: Load requant params: Bit 2: End - "sqdmulh v13.4s, v13.4s, v21.4s\n" - "sqdmulh v19.4s, v19.4s, v21.4s\n" + "sqrdmulh v13.4s, v13.4s, v21.4s\n" + "sqrdmulh v19.4s, v19.4s, v21.4s\n" "add x10, x10, x14\n" "add x9, x9, x14\n" - "sqdmulh v18.4s, v18.4s, v21.4s\n" - "sqdmulh v9.4s, v9.4s, v21.4s\n" + "sqrdmulh v18.4s, v18.4s, v21.4s\n" + "sqrdmulh v9.4s, v9.4s, v21.4s\n" "add x28, x28, x14\n" "add x27, x27, x14\n" "and v7.16b, v13.16b, v25.16b\n" - "sqdmulh v26.4s, v26.4s, v10.4s\n" + "sqrdmulh v26.4s, v26.4s, v10.4s\n" "and v4.16b, v19.16b, v25.16b\n" - "sqdmulh v11.4s, v11.4s, v10.4s\n" + "sqrdmulh v11.4s, v11.4s, v10.4s\n" "and v21.16b, v18.16b, v25.16b\n" - "sqdmulh v24.4s, v24.4s, v10.4s\n" + "sqrdmulh v24.4s, v24.4s, v10.4s\n" "and v20.16b, v9.16b, v25.16b\n" - "sqdmulh v23.4s, v23.4s, v10.4s\n" + "sqrdmulh v23.4s, v23.4s, v10.4s\n" "sshr v7.4s, v7.4s, #0x1f\n" "and v29.16b, v26.16b, v16.16b\n" "sshr v4.4s, v4.4s, #0x1f\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index 790d26bc6d..079b212e6c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -269,7 +269,7 @@ void a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr q4, [x13, #0x10]\n" "smlal2 v10.4s, v24.8h, v7.8h\n" "smlal v22.4s, v24.4h, v1.4h\n" - "sqdmulh v15.4s, v15.4s, v19.4s\n" + "sqrdmulh v15.4s, v15.4s, v19.4s\n" "ldr q31, [x11, #0x10]\n" "smlal2 v21.4s, v24.8h, v1.8h\n" "ldr d24, [x22, x15]\n" @@ -285,7 +285,7 @@ void a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "add x17, x17, #0x48\n" "smlal v9.4s, v28.4h, v7.4h\n" "smlal2 v16.4s, v28.8h, v7.8h\n" - "sqdmulh v10.4s, v10.4s, v4.4s\n" + "sqrdmulh v10.4s, v10.4s, v4.4s\n" "subs x16, x16, #0x1\n" "smlal2 v21.4s, v25.8h, v6.8h\n" "ldr d25, [x20, x15]\n" @@ -301,7 +301,7 @@ void a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "usubl v29.8h, v29.8b, v12.8b\n" "smlal2 v21.4s, v27.8h, v7.8h\n" "smlal2 v18.4s, v26.8h, v7.8h\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "add x15, x15, #0x8\n" "smlal v22.4s, v24.4h, v5.4h\n" "smlal v23.4s, v25.4h, v6.4h\n" @@ -309,17 +309,17 @@ void a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "add x11, x11, #0x20\n" "smlal2 v21.4s, v24.8h, v5.8h\n" "smlal2 v18.4s, v25.8h, v6.8h\n" - "sqdmulh v16.4s, v16.4s, v4.4s\n" + "sqrdmulh v16.4s, v16.4s, v4.4s\n" "smlal v22.4s, v25.4h, v8.4h\n" "smlal v23.4s, v29.4h, v8.4h\n" - "sqdmulh v22.4s, v22.4s, v19.4s\n" + "sqrdmulh v22.4s, v22.4s, v19.4s\n" "smlal2 v21.4s, v25.8h, v8.8h\n" "smlal2 v18.4s, v29.8h, v8.8h\n" - "sqdmulh v23.4s, v23.4s, v19.4s\n" + "sqrdmulh v23.4s, v23.4s, v19.4s\n" "and v29.16b, v22.16b, v0.16b\n" - "sqdmulh v21.4s, v21.4s, v4.4s\n" + "sqrdmulh v21.4s, v21.4s, v4.4s\n" "and v20.16b, v23.16b, v0.16b\n" - "sqdmulh v18.4s, v18.4s, v4.4s\n" + "sqrdmulh v18.4s, v18.4s, v4.4s\n" "and v19.16b, v10.16b, v31.16b\n" "sshr v28.4s, v28.4s, #0x1f\n" "and v4.16b, v16.16b, v31.16b\n" @@ -521,7 +521,7 @@ void a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ldr q4, [x13, #0x10]\n" "smlal2 v10.4s, v24.8h, v7.8h\n" "smlal v22.4s, v24.4h, v1.4h\n" - "sqdmulh v15.4s, v15.4s, v19.4s\n" + "sqrdmulh v15.4s, v15.4s, v19.4s\n" "ldr q31, [x11, #0x10]\n" "smlal2 v21.4s, v24.8h, v1.8h\n" "ldr d24, [x22, x15]\n" @@ -537,7 +537,7 @@ void a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "tst x8, #0x7\n" "smlal v9.4s, v28.4h, v7.4h\n" "smlal2 v16.4s, v28.8h, v7.8h\n" - "sqdmulh v10.4s, v10.4s, v4.4s\n" + "sqrdmulh v10.4s, v10.4s, v4.4s\n" "add x13, x13, #0x20\n" "smlal2 v21.4s, v25.8h, v6.8h\n" "ldr d25, [x20, x15]\n" @@ -553,24 +553,24 @@ void a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "usubl v29.8h, v29.8b, v12.8b\n" "smlal2 v21.4s, v27.8h, v7.8h\n" "smlal2 v18.4s, v26.8h, v7.8h\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "add x15, x15, #0x8\n" "smlal v22.4s, v24.4h, v5.4h\n" "smlal v23.4s, v25.4h, v6.4h\n" "and v28.16b, v9.16b, v0.16b\n" "smlal2 v21.4s, v24.8h, v5.8h\n" "smlal2 v18.4s, v25.8h, v6.8h\n" - "sqdmulh v16.4s, v16.4s, v4.4s\n" + "sqrdmulh v16.4s, v16.4s, v4.4s\n" "smlal v22.4s, v25.4h, v8.4h\n" "smlal v23.4s, v29.4h, v8.4h\n" - "sqdmulh v22.4s, v22.4s, v19.4s\n" + "sqrdmulh v22.4s, v22.4s, v19.4s\n" "smlal2 v21.4s, v25.8h, v8.8h\n" "smlal2 v18.4s, v29.8h, v8.8h\n" - "sqdmulh v23.4s, v23.4s, v19.4s\n" + "sqrdmulh v23.4s, v23.4s, v19.4s\n" "and v29.16b, v22.16b, v0.16b\n" - "sqdmulh v21.4s, v21.4s, v4.4s\n" + "sqrdmulh v21.4s, v21.4s, v4.4s\n" "and v20.16b, v23.16b, v0.16b\n" - "sqdmulh v18.4s, v18.4s, v4.4s\n" + "sqrdmulh v18.4s, v18.4s, v4.4s\n" "and v19.16b, v10.16b, v31.16b\n" "sshr v28.4s, v28.4s, #0x1f\n" "and v4.16b, v16.16b, v31.16b\n" @@ -1274,22 +1274,22 @@ void a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst_impl( "ld1 { v19.s }[0], [x13]\n" "ld1 { v0.s }[0], [x11]\n" "83:" // Oddments: Load requant params: Bit 2: End - "sqdmulh v15.4s, v15.4s, v19.4s\n" - "sqdmulh v9.4s, v9.4s, v19.4s\n" + "sqrdmulh v15.4s, v15.4s, v19.4s\n" + "sqrdmulh v9.4s, v9.4s, v19.4s\n" "add x10, x10, x14\n" "add x9, x9, x14\n" - "sqdmulh v22.4s, v22.4s, v19.4s\n" - "sqdmulh v23.4s, v23.4s, v19.4s\n" + "sqrdmulh v22.4s, v22.4s, v19.4s\n" + "sqrdmulh v23.4s, v23.4s, v19.4s\n" "add x28, x28, x14\n" "add x27, x27, x14\n" "and v30.16b, v15.16b, v0.16b\n" - "sqdmulh v10.4s, v10.4s, v4.4s\n" + "sqrdmulh v10.4s, v10.4s, v4.4s\n" "and v28.16b, v9.16b, v0.16b\n" - "sqdmulh v16.4s, v16.4s, v4.4s\n" + "sqrdmulh v16.4s, v16.4s, v4.4s\n" "and v29.16b, v22.16b, v0.16b\n" - "sqdmulh v21.4s, v21.4s, v4.4s\n" + "sqrdmulh v21.4s, v21.4s, v4.4s\n" "and v20.16b, v23.16b, v0.16b\n" - "sqdmulh v18.4s, v18.4s, v4.4s\n" + "sqrdmulh v18.4s, v18.4s, v4.4s\n" "sshr v30.4s, v30.4s, #0x1f\n" "and v19.16b, v10.16b, v31.16b\n" "sshr v28.4s, v28.4s, #0x1f\n" diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index 291ffec0bb..082d8dd3e1 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -504,24 +504,24 @@ void a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "smlal2 v5.4s, v24.8h, v3.8h\n" "smlal v15.4s, v28.4h, v4.4h\n" "smlal v17.4s, v26.4h, v4.4h\n" - "sqdmulh v15.4s, v15.4s, v12.4s\n" + "sqrdmulh v15.4s, v15.4s, v12.4s\n" "smlal v10.4s, v24.4h, v4.4h\n" "smlal v6.4s, v27.4h, v4.4h\n" - "sqdmulh v17.4s, v17.4s, v12.4s\n" + "sqrdmulh v17.4s, v17.4s, v12.4s\n" "smlal2 v16.4s, v28.8h, v4.8h\n" "smlal2 v8.4s, v26.8h, v4.8h\n" - "sqdmulh v10.4s, v10.4s, v12.4s\n" + "sqrdmulh v10.4s, v10.4s, v12.4s\n" "smlal2 v7.4s, v24.8h, v4.8h\n" "smlal2 v5.4s, v27.8h, v4.8h\n" - "sqdmulh v6.4s, v6.4s, v12.4s\n" + "sqrdmulh v6.4s, v6.4s, v12.4s\n" "and v23.16b, v15.16b, v19.16b\n" - "sqdmulh v16.4s, v16.4s, v20.4s\n" + "sqrdmulh v16.4s, v16.4s, v20.4s\n" "and v22.16b, v17.16b, v19.16b\n" - "sqdmulh v8.4s, v8.4s, v20.4s\n" + "sqrdmulh v8.4s, v8.4s, v20.4s\n" "and v21.16b, v10.16b, v19.16b\n" - "sqdmulh v7.4s, v7.4s, v20.4s\n" + "sqrdmulh v7.4s, v7.4s, v20.4s\n" "and v26.16b, v6.16b, v19.16b\n" - "sqdmulh v5.4s, v5.4s, v20.4s\n" + "sqrdmulh v5.4s, v5.4s, v20.4s\n" "sshr v23.4s, v23.4s, #0x1f\n" "and v4.16b, v16.16b, v29.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" @@ -947,24 +947,24 @@ void a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "smlal2 v5.4s, v24.8h, v3.8h\n" "smlal v15.4s, v28.4h, v4.4h\n" "smlal v17.4s, v26.4h, v4.4h\n" - "sqdmulh v15.4s, v15.4s, v12.4s\n" + "sqrdmulh v15.4s, v15.4s, v12.4s\n" "smlal v10.4s, v24.4h, v4.4h\n" "smlal v6.4s, v27.4h, v4.4h\n" - "sqdmulh v17.4s, v17.4s, v12.4s\n" + "sqrdmulh v17.4s, v17.4s, v12.4s\n" "smlal2 v16.4s, v28.8h, v4.8h\n" "smlal2 v8.4s, v26.8h, v4.8h\n" - "sqdmulh v10.4s, v10.4s, v12.4s\n" + "sqrdmulh v10.4s, v10.4s, v12.4s\n" "smlal2 v7.4s, v24.8h, v4.8h\n" "smlal2 v5.4s, v27.8h, v4.8h\n" - "sqdmulh v6.4s, v6.4s, v12.4s\n" + "sqrdmulh v6.4s, v6.4s, v12.4s\n" "and v23.16b, v15.16b, v19.16b\n" - "sqdmulh v16.4s, v16.4s, v20.4s\n" + "sqrdmulh v16.4s, v16.4s, v20.4s\n" "and v22.16b, v17.16b, v19.16b\n" - "sqdmulh v8.4s, v8.4s, v20.4s\n" + "sqrdmulh v8.4s, v8.4s, v20.4s\n" "and v21.16b, v10.16b, v19.16b\n" - "sqdmulh v7.4s, v7.4s, v20.4s\n" + "sqrdmulh v7.4s, v7.4s, v20.4s\n" "and v26.16b, v6.16b, v19.16b\n" - "sqdmulh v5.4s, v5.4s, v20.4s\n" + "sqrdmulh v5.4s, v5.4s, v20.4s\n" "sshr v23.4s, v23.4s, #0x1f\n" "and v4.16b, v16.16b, v29.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" @@ -2064,22 +2064,22 @@ void a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst_impl( "ld1 { v12.s }[0], [x10]\n" "ld1 { v19.s }[0], [x1]\n" "119:" // Oddments: Load requant params: Bit 2: End - "sqdmulh v15.4s, v15.4s, v12.4s\n" - "sqdmulh v17.4s, v17.4s, v12.4s\n" + "sqrdmulh v15.4s, v15.4s, v12.4s\n" + "sqrdmulh v17.4s, v17.4s, v12.4s\n" "add x16, x16, x22\n" "add x8, x8, x22\n" - "sqdmulh v10.4s, v10.4s, v12.4s\n" - "sqdmulh v6.4s, v6.4s, v12.4s\n" + "sqrdmulh v10.4s, v10.4s, v12.4s\n" + "sqrdmulh v6.4s, v6.4s, v12.4s\n" "add x4, x4, x22\n" "add x7, x7, x22\n" "and v23.16b, v15.16b, v19.16b\n" - "sqdmulh v16.4s, v16.4s, v20.4s\n" + "sqrdmulh v16.4s, v16.4s, v20.4s\n" "and v22.16b, v17.16b, v19.16b\n" - "sqdmulh v8.4s, v8.4s, v20.4s\n" + "sqrdmulh v8.4s, v8.4s, v20.4s\n" "and v21.16b, v10.16b, v19.16b\n" - "sqdmulh v7.4s, v7.4s, v20.4s\n" + "sqrdmulh v7.4s, v7.4s, v20.4s\n" "and v26.16b, v6.16b, v19.16b\n" - "sqdmulh v5.4s, v5.4s, v20.4s\n" + "sqrdmulh v5.4s, v5.4s, v20.4s\n" "sshr v23.4s, v23.4s, #0x1f\n" "and v4.16b, v16.16b, v29.16b\n" "sshr v22.4s, v22.4s, #0x1f\n" -- cgit v1.2.1