/*
 * Copyright (c) 2017-2018 ARM Limited.
 *
 * SPDX-License-Identifier: MIT
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
#ifdef __aarch64__

#include <algorithm>

#include <arm_neon.h>

#include "../../asmlib.hpp"
#include "../../utils.hpp"

namespace arm_gemm {

void a64_sgemv_pretransposed(const float *A, int lda, const float *X, float *Y, float beta, int M, int N) {
    const bool beta0 = (beta==0.0f);
    const bool beta1 = (beta==1.0f);

    for (int x=0; x<N; x+=32) {
        float *y_ptr = Y + x;

        // How many elements are we processing in this loop?
        int l = std::min(N - x, 32);

        register float32x4_t r0 asm("v24");
        register float32x4_t r1 asm("v25");
        register float32x4_t r2 asm("v26");
        register float32x4_t r3 asm("v27");
        register float32x4_t r4 asm("v28");
        register float32x4_t r5 asm("v29");
        register float32x4_t r6 asm("v30");
        register float32x4_t r7 asm("v31");

        register float32x4_t x0  asm("v0");
        register float32x4_t x0a asm("v1");

        const float *x_ptr = X;
        const float *a_ptr = A + ((x/32) * lda);

        if (beta0) {
            r0=r1=r2=r3=r4=r5=r6=r7=vdupq_n_f32(0.0f);
        } else {
            if (l==32) {
                // Fastest path - load all 8 vectors
                r0 = vld1q_f32(y_ptr);
                r1 = vld1q_f32(y_ptr + 4);
                r2 = vld1q_f32(y_ptr + 8);
                r3 = vld1q_f32(y_ptr + 12);
                r4 = vld1q_f32(y_ptr + 16);
                r5 = vld1q_f32(y_ptr + 20);
                r6 = vld1q_f32(y_ptr + 24);
                r7 = vld1q_f32(y_ptr + 28);
            } else {
                // Slow case - leftovers.  Note that we don't care about
                // out-of-range vectors and lanes as we will throw them away at
                // the end.
                int vecs=l/4; // How many leftover vectors?
                int oddbits=l%4; // And how many odd single values?

                if (oddbits) {
                    // Load the outstanding odd values into a vector first
                    float32x4_t oddvec = vdupq_n_f32(0.0f); // This does not really need to be initialized, but the compiler has a hard time with that.
                    float *oddbase = y_ptr + l - oddbits;

                    switch (oddbits) {
                        case 3:
                            oddvec = vld1q_lane_f32(oddbase + 2, oddvec, 2);
                            // fall through
                        case 2:
                            oddvec = vld1q_lane_f32(oddbase + 1, oddvec, 1);
                            // fall through
                        case 1:
                            oddvec = vld1q_lane_f32(oddbase, oddvec, 0);
                            break;

                        default:
                            UNREACHABLE("Impossible case in switch.");
                    }

                    // Now load the whole vectors, putting the oddments in when we run out.
                    do {
                        if (vecs==0) { r0 = oddvec; break; }

                        r0 = vld1q_f32(y_ptr);
                        if (--vecs==0) { r1 = oddvec; break; }

                        r1 = vld1q_f32(y_ptr + 4);
                        if (--vecs==0) { r2 = oddvec; break; }

                        r2 = vld1q_f32(y_ptr + 8);
                        if (--vecs==0) { r3 = oddvec; break; }

                        r3 = vld1q_f32(y_ptr + 12);
                        if (--vecs==0) { r4 = oddvec; break; }

                        r4 = vld1q_f32(y_ptr + 16);
                        if (--vecs==0) { r5 = oddvec; break; }

                        r5 = vld1q_f32(y_ptr + 20);
                        if (--vecs==0) { r6 = oddvec; break; }

                        r6 = vld1q_f32(y_ptr + 24);
                        r7 = oddvec;
                    } while (0);
                } else {
                    // Slightly less slow path - just load the whole vectors
                    do {
                        // It can't be the case that oddbits==0 AND vecs==0 or we wouldn't be here.
                        if (vecs==0) { UNREACHABLE("Impossible lack of work to do"); }

                        r0 = vld1q_f32(y_ptr);
                        if (--vecs==0) { break; }

                        r1 = vld1q_f32(y_ptr + 4);
                        if (--vecs==0) { break; }

                        r2 = vld1q_f32(y_ptr + 8);
                        if (--vecs==0) { break; }

                        r3 = vld1q_f32(y_ptr + 12);
                        if (--vecs==0) { break; }

                        r4 = vld1q_f32(y_ptr + 16);
                        if (--vecs==0) { break; }

                        r5 = vld1q_f32(y_ptr + 20);
                        if (--vecs==0) { break; }

                        r6 = vld1q_f32(y_ptr + 24);
                    } while (0);
                }
            }

            if (!beta1) {
                const float32x4_t vb = vdupq_n_f32(beta);

                r0 = vmulq_f32(r0, vb);
                r1 = vmulq_f32(r1, vb);
                r2 = vmulq_f32(r2, vb);
                r3 = vmulq_f32(r3, vb);
                r4 = vmulq_f32(r4, vb);
                r5 = vmulq_f32(r5, vb);
                r6 = vmulq_f32(r6, vb);
                r7 = vmulq_f32(r7, vb);
            }
        }

        if (M>=8) {
            int k = (M/8)-1;
            x0 = vld1q_f32(x_ptr);

            __asm __volatile (
                "ldr	q2, [%[a_ptr], #0]\n"
                "ldr	q3, [%[a_ptr], #16]\n"
                "ldr	q4, [%[a_ptr], #32]\n"
                "ldr	q5, [%[a_ptr], #48]\n"
                "ldr	q6, [%[a_ptr], #64]\n"
                "ldr	q7, [%[a_ptr], #80]\n"
                "ldr	q8, [%[a_ptr], #96]\n"
                "ldr	q9, [%[a_ptr], #112]\n"
                "ldr	q10, [%[a_ptr], #128]\n"
                "ldr	q11, [%[a_ptr], #144]\n"
                "ldr	q12, [%[a_ptr], #160]\n"
                "ldr	q13, [%[a_ptr], #176]\n"
                "ldr	q14, [%[a_ptr], #192]\n"
                "ldr	q15, [%[a_ptr], #208]\n"
                "ldr	q16, [%[a_ptr], #224]\n"
                "ldr	q17, [%[a_ptr], #240]\n"
                "ldr	q18, [%[a_ptr], #256]\n"
                "ldr	q19, [%[a_ptr], #272]\n"
                "ldr	q20, [%[a_ptr], #288]\n"
                "ldr	q21, [%[a_ptr], #304]\n"
                "ldr	q22, [%[a_ptr], #320]\n"
                "ldr	q23, [%[a_ptr], #336]\n"
                ASM_PREFETCH("[%[a_ptr], #384]")
                ASM_PREFETCH("[%[a_ptr], #448]")
                ASM_PREFETCH("[%[a_ptr], #512]")
                ASM_PREFETCH("[%[a_ptr], #576]")
                ASM_PREFETCH("[%[a_ptr], #640]")
                ASM_PREFETCH("[%[a_ptr], #704]")
                ASM_PREFETCH("[%[a_ptr], #768]")
                ASM_PREFETCH("[%[a_ptr], #832]")
                ASM_PREFETCH("[%[a_ptr], #896]")
                ASM_PREFETCH("[%[a_ptr], #960]")
                ASM_PREFETCH("[%[a_ptr], #1024]")
                ASM_PREFETCH("[%[a_ptr], #1088]")
                ASM_PREFETCH("[%[a_ptr], #1152]")
                ASM_PREFETCH("[%[a_ptr], #1216]")
                ASM_PREFETCH("[%[a_ptr], #1280]")
                ASM_PREFETCH("[%[a_ptr], #1344]")
                ASM_PREFETCH("[%[a_ptr], #1408]")
                ASM_PREFETCH("[%[a_ptr], #1472]")
                ASM_PREFETCH("[%[a_ptr], #1536]")
                ASM_PREFETCH("[%[a_ptr], #1600]")
                ASM_PREFETCH("[%[a_ptr], #1664]")
                ASM_PREFETCH("[%[a_ptr], #1728]")
                ASM_PREFETCH("[%[a_ptr], #1792]")
                ASM_PREFETCH("[%[a_ptr], #1856]")
                ASM_PREFETCH("[%[a_ptr], #1920]")
                ASM_PREFETCH("[%[a_ptr], #1984]")
                "add	%[a_ptr], %[a_ptr], #352\n"

                "cbz	%w[k], 2f\n"

                "1:\n"
                // Unroll 0
                "fmla	%[r0].4s, v2.4s, %[x0].s[0]\n"
                "ldr	%q[x0a], [%[x_ptr], #16]\n"
                "fmla	%[r1].4s, v3.4s, %[x0].s[0]\n"
                "ldr	q3, [%[a_ptr], #0]\n"
                "subs	%w[k], %w[k], #1\n"
                "fmla	%[r2].4s, v4.4s, %[x0].s[0]\n"
                "ldr	q4, [%[a_ptr], #16]\n"
                "fmla	%[r3].4s, v5.4s, %[x0].s[0]\n"
                "ldr	q5, [%[a_ptr], #32]\n"
                "add	%[x_ptr], %[x_ptr], #32\n"
                ASM_PREFETCH("[%[a_ptr], #1664]")
                "fmla	%[r4].4s, v6.4s, %[x0].s[0]\n"
                "ldr	q6, [%[a_ptr], #48]\n"
                "fmla	%[r5].4s, v7.4s, %[x0].s[0]\n"
                "ldr	q7, [%[a_ptr], #64]\n"
                "fmla	%[r6].4s, v8.4s, %[x0].s[0]\n"
                "ldr	q8, [%[a_ptr], #80]\n"
                "fmla	%[r7].4s, v9.4s, %[x0].s[0]\n"
                "ldr	q9, [%[a_ptr], #96]\n"
                ASM_PREFETCH("[%[a_ptr], #1728]")

                // Unroll 1
                "fmla	%[r0].4s, v10.4s, %[x0].s[1]\n"
                "ldr	q10, [%[a_ptr], #112]\n"
                "fmla	%[r1].4s, v11.4s, %[x0].s[1]\n"
                "ldr	q11, [%[a_ptr], #128]\n"
                "fmla	%[r2].4s, v12.4s, %[x0].s[1]\n"
                "ldr	q12, [%[a_ptr], #144]\n"
                "fmla	%[r3].4s, v13.4s, %[x0].s[1]\n"
                "ldr	q13, [%[a_ptr], #160]\n"
                ASM_PREFETCH("[%[a_ptr], #1792]")
                "fmla	%[r4].4s, v14.4s, %[x0].s[1]\n"
                "ldr	q14, [%[a_ptr], #176]\n"
                "fmla	%[r5].4s, v15.4s, %[x0].s[1]\n"
                "ldr	q15, [%[a_ptr], #192]\n"
                "fmla	%[r6].4s, v16.4s, %[x0].s[1]\n"
                "ldr	q16, [%[a_ptr], #208]\n"
                "fmla	%[r7].4s, v17.4s, %[x0].s[1]\n"
                "ldr	q17, [%[a_ptr], #224]\n"
                ASM_PREFETCH("[%[a_ptr], #1856]")

                // Unroll 2
                "fmla	%[r0].4s, v18.4s, %[x0].s[2]\n"
                "ldr	q18, [%[a_ptr], #240]\n"
                "fmla	%[r1].4s, v19.4s, %[x0].s[2]\n"
                "ldr	q19, [%[a_ptr], #256]\n"
                "fmla	%[r2].4s, v20.4s, %[x0].s[2]\n"
                "ldr	q20, [%[a_ptr], #272]\n"
                "fmla	%[r3].4s, v21.4s, %[x0].s[2]\n"
                "ldr	q21, [%[a_ptr], #288]\n"
                ASM_PREFETCH("[%[a_ptr], #1920]")
                "fmla	%[r4].4s, v22.4s, %[x0].s[2]\n"
                "ldr	q22, [%[a_ptr], #304]\n"
                "fmla	%[r5].4s, v23.4s, %[x0].s[2]\n"
                "ldr	q23, [%[a_ptr], #320]\n"
                "fmla	%[r6].4s, v3.4s, %[x0].s[2]\n"
                "ldr	q2, [%[a_ptr], #336]\n"
                "ldr	q3, [%[a_ptr], #352]\n"
                "fmla	%[r7].4s, v4.4s, %[x0].s[2]\n"
                "ldr	q4, [%[a_ptr], #368]\n"
                ASM_PREFETCH("[%[a_ptr], #1984]")

                // Unroll 3
                "fmla	%[r0].4s, v5.4s, %[x0].s[3]\n"
                "ldr	q5, [%[a_ptr], #384]\n"
                "fmla	%[r1].4s, v6.4s, %[x0].s[3]\n"
                "ldr	q6, [%[a_ptr], #400]\n"
                "fmla	%[r2].4s, v7.4s, %[x0].s[3]\n"
                "ldr	q7, [%[a_ptr], #416]\n"
                "fmla	%[r3].4s, v8.4s, %[x0].s[3]\n"
                ASM_PREFETCH("[%[a_ptr], #2048]")
                "ldr	q8, [%[a_ptr], #432]\n"
                "fmla	%[r4].4s, v9.4s, %[x0].s[3]\n"
                "ldr	q9, [%[a_ptr], #448]\n"
                "fmla	%[r5].4s, v10.4s, %[x0].s[3]\n"
                "ldr	q10, [%[a_ptr], #464]\n"
                "fmla	%[r6].4s, v11.4s, %[x0].s[3]\n"
                "ldr	q11, [%[a_ptr], #480]\n"
                "fmla	%[r7].4s, v12.4s, %[x0].s[3]\n"
                "ldr	q12, [%[a_ptr], #496]\n"
                ASM_PREFETCH("[%[a_ptr], #2112]")

                // Unroll 4
                "fmla	%[r0].4s, v13.4s, %[x0a].s[0]\n"
                "ldr	%q[x0], [%[x_ptr]]\n"
                "fmla	%[r1].4s, v14.4s, %[x0a].s[0]\n"
                "ldr	q14, [%[a_ptr], #512]\n"
                "fmla	%[r2].4s, v15.4s, %[x0a].s[0]\n"
                "ldr	q15, [%[a_ptr], #528]\n"
                "fmla	%[r3].4s, v16.4s, %[x0a].s[0]\n"
                ASM_PREFETCH("[%[a_ptr], #2176]")
                "ldr	q16, [%[a_ptr], #544]\n"
                "fmla	%[r4].4s, v17.4s, %[x0a].s[0]\n"
                "ldr	q17, [%[a_ptr], #560]\n"
                "fmla	%[r5].4s, v18.4s, %[x0a].s[0]\n"
                "ldr	q18, [%[a_ptr], #576]\n"
                "fmla	%[r6].4s, v19.4s, %[x0a].s[0]\n"
                "ldr	q19, [%[a_ptr], #592]\n"
                "fmla	%[r7].4s, v20.4s, %[x0a].s[0]\n"
                "ldr	q20, [%[a_ptr], #608]\n"
                ASM_PREFETCH("[%[a_ptr], #2240]")

                // Unroll 5
                "fmla	%[r0].4s, v21.4s, %[x0a].s[1]\n"
                "ldr	q21, [%[a_ptr], #624]\n"
                "fmla	%[r1].4s, v22.4s, %[x0a].s[1]\n"
                "ldr	q22, [%[a_ptr], #640]\n"
                "fmla	%[r2].4s, v23.4s, %[x0a].s[1]\n"
                "ldr	q23, [%[a_ptr], #656]\n"
                "fmla	%[r3].4s, v2.4s, %[x0a].s[1]\n"
                "ldr	q2, [%[a_ptr], #672]\n"
                ASM_PREFETCH("[%[a_ptr], #2304]")
                "fmla	%[r4].4s, v3.4s, %[x0a].s[1]\n"
                "ldr	q3, [%[a_ptr], #688]\n"
                "fmla	%[r5].4s, v4.4s, %[x0a].s[1]\n"
                "ldr	q4, [%[a_ptr], #704]\n"
                "fmla	%[r6].4s, v5.4s, %[x0a].s[1]\n"
                "ldr	q5, [%[a_ptr], #720]\n"
                "fmla	%[r7].4s, v6.4s, %[x0a].s[1]\n"
                "ldr	q6, [%[a_ptr], #736]\n"
                ASM_PREFETCH("[%[a_ptr], #2368]")

                // Unroll 6
                "fmla	%[r0].4s, v7.4s, %[x0a].s[2]\n"
                "ldr	q7, [%[a_ptr], #752]\n"
                "fmla	%[r1].4s, v8.4s, %[x0a].s[2]\n"
                "ldr	q8, [%[a_ptr], #768]\n"
                "fmla	%[r2].4s, v9.4s, %[x0a].s[2]\n"
                "ldr	q9, [%[a_ptr], #784]\n"
                "fmla	%[r3].4s, v10.4s, %[x0a].s[2]\n"
                "ldr	q10, [%[a_ptr], #800]\n"
                ASM_PREFETCH("[%[a_ptr], #2432]")
                "fmla	%[r4].4s, v11.4s, %[x0a].s[2]\n"
                "ldr	q11, [%[a_ptr], #816]\n"
                "fmla	%[r5].4s, v12.4s, %[x0a].s[2]\n"
                "ldr	q12, [%[a_ptr], #832]\n"
                "fmla	%[r6].4s, v14.4s, %[x0a].s[2]\n"
                "ldr	q13, [%[a_ptr], #848]\n"
                "ldr	q14, [%[a_ptr], #864]\n"
                "fmla	%[r7].4s, v15.4s, %[x0a].s[2]\n"
                "ldr	q15, [%[a_ptr], #880]\n"
                ASM_PREFETCH("[%[a_ptr], #2496]")

                // Unroll 7
                "fmla	%[r0].4s, v16.4s, %[x0a].s[3]\n"
                "ldr	q16, [%[a_ptr], #896]\n"
                "fmla	%[r1].4s, v17.4s, %[x0a].s[3]\n"
                "ldr	q17, [%[a_ptr], #912]\n"
                "fmla	%[r2].4s, v18.4s, %[x0a].s[3]\n"
                "ldr	q18, [%[a_ptr], #928]\n"
                "fmla	%[r3].4s, v19.4s, %[x0a].s[3]\n"
                ASM_PREFETCH("[%[a_ptr], #2560]")
                "ldr	q19, [%[a_ptr], #944]\n"
                "fmla	%[r4].4s, v20.4s, %[x0a].s[3]\n"
                "ldr	q20, [%[a_ptr], #960]\n"
                "fmla	%[r5].4s, v21.4s, %[x0a].s[3]\n"
                "ldr	q21, [%[a_ptr], #976]\n"
                "add	%[a_ptr], %[a_ptr], #1024\n"
                "fmla	%[r6].4s, v22.4s, %[x0a].s[3]\n"
                "ldr	q22, [%[a_ptr], #-32]\n"
                "fmla	%[r7].4s, v23.4s, %[x0a].s[3]\n"
                "ldr	q23, [%[a_ptr], #-16]\n"
                ASM_PREFETCH("[%[a_ptr], #1600]")
                "bne	1b\n"

                // Detached final iteration
                "2:\n"

                // Unroll 0
                "fmla	%[r0].4s, v2.4s, %[x0].s[0]\n"
                "ldr	%q[x0a], [%[x_ptr], #16]\n"
                "fmla	%[r1].4s, v3.4s, %[x0].s[0]\n"
                "ldr	q3, [%[a_ptr], #0]\n"
                "subs	%w[k], %w[k], #1\n"
                "fmla	%[r2].4s, v4.4s, %[x0].s[0]\n"
                "ldr	q4, [%[a_ptr], #16]\n"
                "fmla	%[r3].4s, v5.4s, %[x0].s[0]\n"
                "ldr	q5, [%[a_ptr], #32]\n"
                "add	%[x_ptr], %[x_ptr], #32\n"
                "fmla	%[r4].4s, v6.4s, %[x0].s[0]\n"
                "ldr	q6, [%[a_ptr], #48]\n"
                "fmla	%[r5].4s, v7.4s, %[x0].s[0]\n"
                "ldr	q7, [%[a_ptr], #64]\n"
                "fmla	%[r6].4s, v8.4s, %[x0].s[0]\n"
                "ldr	q8, [%[a_ptr], #80]\n"
                "fmla	%[r7].4s, v9.4s, %[x0].s[0]\n"
                "ldr	q9, [%[a_ptr], #96]\n"

                // Unroll 1
                "fmla	%[r0].4s, v10.4s, %[x0].s[1]\n"
                "ldr	q10, [%[a_ptr], #112]\n"
                "fmla	%[r1].4s, v11.4s, %[x0].s[1]\n"
                "ldr	q11, [%[a_ptr], #128]\n"
                "fmla	%[r2].4s, v12.4s, %[x0].s[1]\n"
                "ldr	q12, [%[a_ptr], #144]\n"
                "fmla	%[r3].4s, v13.4s, %[x0].s[1]\n"
                "ldr	q13, [%[a_ptr], #160]\n"
                "fmla	%[r4].4s, v14.4s, %[x0].s[1]\n"
                "ldr	q14, [%[a_ptr], #176]\n"
                "fmla	%[r5].4s, v15.4s, %[x0].s[1]\n"
                "ldr	q15, [%[a_ptr], #192]\n"
                "fmla	%[r6].4s, v16.4s, %[x0].s[1]\n"
                "ldr	q16, [%[a_ptr], #208]\n"
                "fmla	%[r7].4s, v17.4s, %[x0].s[1]\n"
                "ldr	q17, [%[a_ptr], #224]\n"

                // Unroll 2
                "fmla	%[r0].4s, v18.4s, %[x0].s[2]\n"
                "ldr	q18, [%[a_ptr], #240]\n"
                "fmla	%[r1].4s, v19.4s, %[x0].s[2]\n"
                "ldr	q19, [%[a_ptr], #256]\n"
                "fmla	%[r2].4s, v20.4s, %[x0].s[2]\n"
                "ldr	q20, [%[a_ptr], #272]\n"
                "fmla	%[r3].4s, v21.4s, %[x0].s[2]\n"
                "ldr	q21, [%[a_ptr], #288]\n"
                "fmla	%[r4].4s, v22.4s, %[x0].s[2]\n"
                "ldr	q22, [%[a_ptr], #304]\n"
                "fmla	%[r5].4s, v23.4s, %[x0].s[2]\n"
                "ldr	q23, [%[a_ptr], #320]\n"
                "fmla	%[r6].4s, v3.4s, %[x0].s[2]\n"
                "ldr	q2, [%[a_ptr], #336]\n"
                "ldr	q3, [%[a_ptr], #352]\n"
                "fmla	%[r7].4s, v4.4s, %[x0].s[2]\n"
                "ldr	q4, [%[a_ptr], #368]\n"

                // Unroll 3
                "fmla	%[r0].4s, v5.4s, %[x0].s[3]\n"
                "ldr	q5, [%[a_ptr], #384]\n"
                "fmla	%[r1].4s, v6.4s, %[x0].s[3]\n"
                "ldr	q6, [%[a_ptr], #400]\n"
                "fmla	%[r2].4s, v7.4s, %[x0].s[3]\n"
                "ldr	q7, [%[a_ptr], #416]\n"
                "fmla	%[r3].4s, v8.4s, %[x0].s[3]\n"
                "ldr	q8, [%[a_ptr], #432]\n"
                "fmla	%[r4].4s, v9.4s, %[x0].s[3]\n"
                "ldr	q9, [%[a_ptr], #448]\n"
                "fmla	%[r5].4s, v10.4s, %[x0].s[3]\n"
                "ldr	q10, [%[a_ptr], #464]\n"
                "fmla	%[r6].4s, v11.4s, %[x0].s[3]\n"
                "ldr	q11, [%[a_ptr], #480]\n"
                "fmla	%[r7].4s, v12.4s, %[x0].s[3]\n"
                "ldr	q12, [%[a_ptr], #496]\n"

                // Unroll 4
                "fmla	%[r0].4s, v13.4s, %[x0a].s[0]\n"
                "fmla	%[r1].4s, v14.4s, %[x0a].s[0]\n"
                "ldr	q14, [%[a_ptr], #512]\n"
                "fmla	%[r2].4s, v15.4s, %[x0a].s[0]\n"
                "ldr	q15, [%[a_ptr], #528]\n"
                "fmla	%[r3].4s, v16.4s, %[x0a].s[0]\n"
                "ldr	q16, [%[a_ptr], #544]\n"
                "fmla	%[r4].4s, v17.4s, %[x0a].s[0]\n"
                "ldr	q17, [%[a_ptr], #560]\n"
                "fmla	%[r5].4s, v18.4s, %[x0a].s[0]\n"
                "ldr	q18, [%[a_ptr], #576]\n"
                "fmla	%[r6].4s, v19.4s, %[x0a].s[0]\n"
                "ldr	q19, [%[a_ptr], #592]\n"
                "fmla	%[r7].4s, v20.4s, %[x0a].s[0]\n"
                "ldr	q20, [%[a_ptr], #608]\n"

                // Unroll 5
                "fmla	%[r0].4s, v21.4s, %[x0a].s[1]\n"
                "ldr	q21, [%[a_ptr], #624]\n"
                "fmla	%[r1].4s, v22.4s, %[x0a].s[1]\n"
                "ldr	q22, [%[a_ptr], #640]\n"
                "fmla	%[r2].4s, v23.4s, %[x0a].s[1]\n"
                "ldr	q23, [%[a_ptr], #656]\n"
                "fmla	%[r3].4s, v2.4s, %[x0a].s[1]\n"
                "add	%[a_ptr], %[a_ptr], #672\n"
                "fmla	%[r4].4s, v3.4s, %[x0a].s[1]\n"
                "fmla	%[r5].4s, v4.4s, %[x0a].s[1]\n"
                "fmla	%[r6].4s, v5.4s, %[x0a].s[1]\n"
                "fmla	%[r7].4s, v6.4s, %[x0a].s[1]\n"

                // Unroll 6
                "fmla	%[r0].4s, v7.4s, %[x0a].s[2]\n"
                "fmla	%[r1].4s, v8.4s, %[x0a].s[2]\n"
                "fmla	%[r2].4s, v9.4s, %[x0a].s[2]\n"
                "fmla	%[r3].4s, v10.4s, %[x0a].s[2]\n"
                "fmla	%[r4].4s, v11.4s, %[x0a].s[2]\n"
                "fmla	%[r5].4s, v12.4s, %[x0a].s[2]\n"
                "fmla	%[r6].4s, v14.4s, %[x0a].s[2]\n"
                "fmla	%[r7].4s, v15.4s, %[x0a].s[2]\n"

                // Unroll 7
                "fmla	%[r0].4s, v16.4s, %[x0a].s[3]\n"
                "fmla	%[r1].4s, v17.4s, %[x0a].s[3]\n"
                "fmla	%[r2].4s, v18.4s, %[x0a].s[3]\n"
                "fmla	%[r3].4s, v19.4s, %[x0a].s[3]\n"
                "fmla	%[r4].4s, v20.4s, %[x0a].s[3]\n"
                "fmla	%[r5].4s, v21.4s, %[x0a].s[3]\n"
                "fmla	%[r6].4s, v22.4s, %[x0a].s[3]\n"
                "fmla	%[r7].4s, v23.4s, %[x0a].s[3]\n"
            :
              [a_ptr] "+r" (a_ptr), [x_ptr] "+r" (x_ptr),
              [x0] "+w" (x0), [x0a] "+w" (x0a), [k] "+r" (k),
              [r0] "+w" (r0), [r1] "+w" (r1), [r2] "+w" (r2), [r3] "+w" (r3),
              [r4] "+w" (r4), [r5] "+w" (r5), [r6] "+w" (r6), [r7] "+w" (r7)
            :
            : "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14",
              "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "x20", "x21", "cc", "memory");
        }

        // Deal with ragged M
        if (M % 8) {
            int l=(M%8)-1;

            __asm __volatile (
                "ldr	q2, [%[a_ptr], #0]\n"
                "ldr	q3, [%[a_ptr], #16]\n"
                "ldr	q4, [%[a_ptr], #32]\n"
                "ldr	q5, [%[a_ptr], #48]\n"
                "ldr	q6, [%[a_ptr], #64]\n"
                "ldr	q7, [%[a_ptr], #80]\n"
                "ldr	q8, [%[a_ptr], #96]\n"
                "ldr	q9, [%[a_ptr], #112]\n"
                "ldr	%s[x0], [%[x_ptr]]\n"
                "add	%[a_ptr], %[a_ptr], #128\n"
                "add	%[x_ptr], %[x_ptr], #4\n"

                "cbz	%w[l], 2f\n"

                "1:\n"
                "fmla	%[r0].4s, v2.4s, %[x0].s[0]\n"
                "ldr	q2, [%[a_ptr], #0]\n"
                "subs	%w[l], %w[l], #1\n"
                "fmla	%[r1].4s, v3.4s, %[x0].s[0]\n"
                "ldr	q3, [%[a_ptr], #16]\n"
                "fmla	%[r2].4s, v4.4s, %[x0].s[0]\n"
                "ldr	q4, [%[a_ptr], #32]\n"
                "fmla	%[r3].4s, v5.4s, %[x0].s[0]\n"
                "ldr	q5, [%[a_ptr], #48]\n"
                "fmla	%[r4].4s, v6.4s, %[x0].s[0]\n"
                "ldr	q6, [%[a_ptr], #64]\n"
                "fmla	%[r5].4s, v7.4s, %[x0].s[0]\n"
                "ldr	q7, [%[a_ptr], #80]\n"
                "fmla	%[r6].4s, v8.4s, %[x0].s[0]\n"
                "ldr	q8, [%[a_ptr], #96]\n"
                "fmla	%[r7].4s, v9.4s, %[x0].s[0]\n"
                "ldr	q9, [%[a_ptr], #112]\n"
                "ldr	%s[x0], [%[x_ptr]]\n"
                "add	%[a_ptr], %[a_ptr], #128\n"
                "add	%[x_ptr], %[x_ptr], #4\n"
                "bne	1b\n"

                "2:\n"

                "fmla	%[r0].4s, v2.4s, %[x0].s[0]\n"
                "fmla	%[r1].4s, v3.4s, %[x0].s[0]\n"
                "fmla	%[r2].4s, v4.4s, %[x0].s[0]\n"
                "fmla	%[r3].4s, v5.4s, %[x0].s[0]\n"
                "fmla	%[r4].4s, v6.4s, %[x0].s[0]\n"
                "fmla	%[r5].4s, v7.4s, %[x0].s[0]\n"
                "fmla	%[r6].4s, v8.4s, %[x0].s[0]\n"
                "fmla	%[r7].4s, v9.4s, %[x0].s[0]\n"
            :
              [a_ptr] "+r" (a_ptr), [x_ptr] "+r" (x_ptr),
              [x0] "+w" (x0), [l] "+r" (l),
              [r0] "+w" (r0), [r1] "+w" (r1), [r2] "+w" (r2), [r3] "+w" (r3),
              [r4] "+w" (r4), [r5] "+w" (r5), [r6] "+w" (r6), [r7] "+w" (r7)
            :
            : "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "cc", "memory");
        }

        if (l==32) {
            // Fast path
            vst1q_f32(y_ptr, r0);
            vst1q_f32(y_ptr + 4, r1);
            vst1q_f32(y_ptr + 8, r2);
            vst1q_f32(y_ptr + 12, r3);
            vst1q_f32(y_ptr + 16, r4);
            vst1q_f32(y_ptr + 20, r5);
            vst1q_f32(y_ptr + 24, r6);
            vst1q_f32(y_ptr + 28, r7);
        } else {
            int vecs=l/4;
            int oddbits=l%4;

            if (oddbits) {
                // As above - slowest path deals with vectors plus odd bits
                float32x4_t oddvec;

                do {
                    if (vecs==0) { oddvec=r0; break; }

                    vst1q_f32(y_ptr, r0);
                    if (--vecs==0) { oddvec=r1; break; }

                    vst1q_f32(y_ptr + 4, r1);
                    if (--vecs==0) { oddvec=r2; break; }

                    vst1q_f32(y_ptr + 8, r2);
                    if (--vecs==0) { oddvec=r3; break; }

                    vst1q_f32(y_ptr + 12, r3);
                    if (--vecs==0) { oddvec=r4; break; }

                    vst1q_f32(y_ptr + 16, r4);
                    if (--vecs==0) { oddvec=r5; break; }

                    vst1q_f32(y_ptr + 20, r5);
                    if (--vecs==0) { oddvec=r6; break; }

                    vst1q_f32(y_ptr + 24, r6);
                    oddvec=r7;
                } while (0);

                float *oddbase = y_ptr + l - oddbits;

                switch(oddbits) {
                    case 3:
                        vst1q_lane_f32(oddbase + 2, oddvec, 2);
                        // fall through
                    case 2:
                        vst1q_lane_f32(oddbase + 1, oddvec, 1);
                        // fall through
                    case 1:
                        vst1q_lane_f32(oddbase, oddvec, 0);
                        break;

                    default:
                        // oddbits must be 1, 2 or 3.
                        UNREACHABLE("Impossible case in switch.");
                }
            } else {
                // As above - medium path deals with vectors only
                do {
                    if (vecs==0) { UNREACHABLE("vecs and oddbits can't both be 0"); }

                    vst1q_f32(y_ptr, r0);
                    if (--vecs==0) { break; }

                    vst1q_f32(y_ptr + 4, r1);
                    if (--vecs==0) { break; }

                    vst1q_f32(y_ptr + 8, r2);
                    if (--vecs==0) { break; }

                    vst1q_f32(y_ptr + 12, r3);
                    if (--vecs==0) { break; }

                    vst1q_f32(y_ptr + 16, r4);
                    if (--vecs==0) { break; }

                    vst1q_f32(y_ptr + 20, r5);
                    if (--vecs==0) { break; }

                    vst1q_f32(y_ptr + 24, r6);
                } while (0);
            }
        }
    }
}

} // namespace arm_gemm

#endif // aarch64