From fc94f4d23abd4bc427b701f54ad85282e9ec7872 Mon Sep 17 00:00:00 2001 From: Michael Tyler Date: Tue, 4 Jun 2024 15:47:37 +0100 Subject: Update CPU kernels and add mixed sign GEMM support - Add support for mixed sign quantized convolution. - Add support for mixed sign dequantized GEMM. - Add SME FP16 GEMV kernel. - Change SME vector length function to use RDSVL instead of static variable. - Add GEMM dilation support internally (not exposed yet). - Remove unused "get_default_activation_values" functions. - Add SVE fixed format interleaved BF16 DOT kernel. - Updates and optimizations to assembly kernels. Resolves COMPMID-6926 Change-Id: I227f502502611d4cc4111c89e30c53ce94079544 Signed-off-by: Michael Tyler Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/11570 Tested-by: Arm Jenkins Reviewed-by: Gunes Bayir Comments-Addressed: Arm Jenkins Benchmark: Arm Jenkins --- .../sme_interleave4VL_block2_fp16_fp16.hpp | 48 +++++++++++----------- 1 file changed, 24 insertions(+), 24 deletions(-) (limited to 'src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_fp16_fp16.hpp') diff --git a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_fp16_fp16.hpp b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_fp16_fp16.hpp index 268bdbb924..9e0ab463be 100644 --- a/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_fp16_fp16.hpp +++ b/src/core/NEON/kernels/arm_gemm/indirect-interleaves/sme_interleave4VL_block2_fp16_fp16.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Arm Limited. + * Copyright (c) 2023-2024 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -10,19 +10,19 @@ * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. */ -#if defined(__ARM_FEATURE_SVE) +#if defined(ARM_COMPUTE_ENABLE_SME) template <> void interleave_block<4, 2, VLType::SME, false>( @@ -65,36 +65,36 @@ void interleave_block<4, 2, VLType::SME, false>( "ldr x21, [x20], #0x8\n" "cbz x15, 3f\n" "2:" // Loads: Loop - ".inst 0x25296582 // psel p2.h, p9.h/Z, p12.h[w13]\n" - ".inst 0x25296161 // psel p1.h, p8.h/Z, p11.h[w13]\n" - ".inst 0x25396580 // psel p0.h, p9.h/Z, p12.h[w13, #1]\n" - ".inst 0xe0502b00 // ld1h { za0h.h[x13] }, p2/Z, [x24, x16, LSL #1]\n" - ".inst 0x25396162 // psel p2.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0x25296580 // psel p0.h, p9.h/Z, p12.h[w13]\n" + ".inst 0x25296162 // psel p2.h, p8.h/Z, p11.h[w13]\n" + ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0xe0502300 // ld1h { za0h.h[x13] }, p0/Z, [x24, x16, LSL #1]\n" + ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" "ldr x24, [x27], #0x8\n" - ".inst 0xe05026e8 // ld1h { za1h.h[x13] }, p1/Z, [x23, x16, LSL #1]\n" + ".inst 0xe0502ae8 // ld1h { za1h.h[x13] }, p2/Z, [x23, x16, LSL #1]\n" "ldr x23, [x26], #0x8\n" - ".inst 0xe05022c1 // ld1h { za0h.h[x13, #1] }, p0/Z, [x22, x16, LSL #1]\n" + ".inst 0xe05026c1 // ld1h { za0h.h[x13, #1] }, p1/Z, [x22, x16, LSL #1]\n" "ldr x22, [x25], #0x8\n" - ".inst 0xe0502aa9 // ld1h { za1h.h[x13, #1] }, p2/Z, [x21, x16, LSL #1]\n" + ".inst 0xe05022a9 // ld1h { za1h.h[x13, #1] }, p0/Z, [x21, x16, LSL #1]\n" "add x13, x13, #0x2\n" "ldr x21, [x20], #0x8\n" "cmp x13, x15, LSL #1\n" "blt 2b\n" "3:" // Loads: Tail - ".inst 0x25296581 // psel p1.h, p9.h/Z, p12.h[w13]\n" - ".inst 0x25296160 // psel p0.h, p8.h/Z, p11.h[w13]\n" + ".inst 0x25296580 // psel p0.h, p9.h/Z, p12.h[w13]\n" + ".inst 0x25296162 // psel p2.h, p8.h/Z, p11.h[w13]\n" "sub x20, %x[width], x17\n" - ".inst 0x25396582 // psel p2.h, p9.h/Z, p12.h[w13, #1]\n" + ".inst 0x25396581 // psel p1.h, p9.h/Z, p12.h[w13, #1]\n" "cmp x20, x9\n" "mov x12, #0x0\n" - ".inst 0xe0502700 // ld1h { za0h.h[x13] }, p1/Z, [x24, x16, LSL #1]\n" - ".inst 0xe05022e8 // ld1h { za1h.h[x13] }, p0/Z, [x23, x16, LSL #1]\n" - ".inst 0x25396161 // psel p1.h, p8.h/Z, p11.h[w13, #1]\n" + ".inst 0xe0502300 // ld1h { za0h.h[x13] }, p0/Z, [x24, x16, LSL #1]\n" + ".inst 0x25396160 // psel p0.h, p8.h/Z, p11.h[w13, #1]\n" "csel x20, x20, x9, LT\n" + ".inst 0xe0502ae8 // ld1h { za1h.h[x13] }, p2/Z, [x23, x16, LSL #1]\n" "add x20, x20, #0x1\n" - ".inst 0xe0502ac1 // ld1h { za0h.h[x13, #1] }, p2/Z, [x22, x16, LSL #1]\n" + ".inst 0xe05026c1 // ld1h { za0h.h[x13, #1] }, p1/Z, [x22, x16, LSL #1]\n" "lsr x20, x20, #0x1\n" - ".inst 0xe05026a9 // ld1h { za1h.h[x13, #1] }, p1/Z, [x21, x16, LSL #1]\n" + ".inst 0xe05022a9 // ld1h { za1h.h[x13, #1] }, p0/Z, [x21, x16, LSL #1]\n" "4:" // Stores: Loop ".inst 0x25307540 // psel p0.s, p13.s/Z, p10.s[w12]\n" ".inst 0x25307542 // psel p2.s, p13.s/Z, p10.s[w12]\n" @@ -122,4 +122,4 @@ void interleave_block<4, 2, VLType::SME, false>( ); } -#endif // defined(__ARM_FEATURE_SVE) +#endif // defined(ARM_COMPUTE_ENABLE_SME) -- cgit v1.2.1