COMPMID-1005: Update Depthwise Convolution form RSH

Change-Id: I3033ddb8de183661010d6c71a83f71132037b139
Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/124338
Tested-by: Jenkins <bsgcomp@arm.com>
Reviewed-by: Pablo Tello <pablo.tello@arm.com>

1 files changed, 534 insertions, 403 deletions

diff --git a/src/core/NEON/kernels/convolution/depthwise/depthwise_2x2_3x3_1x1_fp32_fp32.cpp b/src/core/NEON/kernels/convolution/depthwise/depthwise_2x2_3x3_1x1_fp32_fp32.cpp
index fa50f79bc5..9b3a60db59 100644
--- a/src/core/NEON/kernels/convolution/depthwise/depthwise_2x2_3x3_1x1_fp32_fp32.cpp
+++ b/src/core/NEON/kernels/convolution/depthwise/depthwise_2x2_3x3_1x1_fp32_fp32.cpp
@@ -28,412 +28,543 @@ namespace depthwise
 using Conv = DepthwiseConvolution<2, 2, 3, 3, 1, 1, float, float>;
 using ConvImpl = DepthwiseConvolutionImpl<2, 2, 3, 3, 1, 1, float, float>;
 
+#ifdef __aarch64__
+
+template <>
 template <>
-const Conv::TileFn Conv::tile_fns
-  [max_in_pad_top]
-  [max_in_pad_left]
-  [max_in_pad_bottom]
-  [max_in_pad_right]
-  [max_out_pad_bottom]
-  [max_out_pad_right] = {
-  {  // Input pad top = 0
-    {  // Input pad left = 0
-      {  // Input pad bottom = 0
-        {  // Input pad right = 0
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 0, 0, 0, 0, 0>,
-            ConvImpl::template process_tile<0, 0, 0, 0, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 0, 0, 0, 1, 0>,
-            ConvImpl::template process_tile<0, 0, 0, 0, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 0
-        {  // Input pad right = 1
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 0, 0, 1, 0, 0>,
-            ConvImpl::template process_tile<0, 0, 0, 1, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 0, 0, 1, 1, 0>,
-            ConvImpl::template process_tile<0, 0, 0, 1, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 1
-        {  // Input pad right = 2
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 0, 0, 2, 0, 0>,
-            ConvImpl::template process_tile<0, 0, 0, 2, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 0, 0, 2, 1, 0>,
-            ConvImpl::template process_tile<0, 0, 0, 2, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 2
-      },  // Input pad bottom = 0
-      {  // Input pad bottom = 1
-        {  // Input pad right = 0
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 0, 1, 0, 0, 0>,
-            ConvImpl::template process_tile<0, 0, 1, 0, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 0, 1, 0, 1, 0>,
-            ConvImpl::template process_tile<0, 0, 1, 0, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 0
-        {  // Input pad right = 1
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 0, 1, 1, 0, 0>,
-            ConvImpl::template process_tile<0, 0, 1, 1, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 0, 1, 1, 1, 0>,
-            ConvImpl::template process_tile<0, 0, 1, 1, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 1
-        {  // Input pad right = 2
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 0, 1, 2, 0, 0>,
-            ConvImpl::template process_tile<0, 0, 1, 2, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 0, 1, 2, 1, 0>,
-            ConvImpl::template process_tile<0, 0, 1, 2, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 2
-      },  // Input pad bottom = 1
-      {  // Input pad bottom = 2
-        {  // Input pad right = 0
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 0, 2, 0, 0, 0>,
-            ConvImpl::template process_tile<0, 0, 2, 0, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 0, 2, 0, 1, 0>,
-            ConvImpl::template process_tile<0, 0, 2, 0, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 0
-        {  // Input pad right = 1
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 0, 2, 1, 0, 0>,
-            ConvImpl::template process_tile<0, 0, 2, 1, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 0, 2, 1, 1, 0>,
-            ConvImpl::template process_tile<0, 0, 2, 1, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 1
-        {  // Input pad right = 2
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 0, 2, 2, 0, 0>,
-            ConvImpl::template process_tile<0, 0, 2, 2, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 0, 2, 2, 1, 0>,
-            ConvImpl::template process_tile<0, 0, 2, 2, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 2
-      },  // Input pad bottom = 2
-    },  // Input pad left = 0
-    {  // Input pad left = 1
-      {  // Input pad bottom = 0
-        {  // Input pad right = 0
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 1, 0, 0, 0, 0>,
-            ConvImpl::template process_tile<0, 1, 0, 0, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 1, 0, 0, 1, 0>,
-            ConvImpl::template process_tile<0, 1, 0, 0, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 0
-        {  // Input pad right = 1
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 1, 0, 1, 0, 0>,
-            ConvImpl::template process_tile<0, 1, 0, 1, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 1, 0, 1, 1, 0>,
-            ConvImpl::template process_tile<0, 1, 0, 1, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 1
-        {  // Input pad right = 2
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 1, 0, 2, 0, 0>,
-            ConvImpl::template process_tile<0, 1, 0, 2, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 1, 0, 2, 1, 0>,
-            ConvImpl::template process_tile<0, 1, 0, 2, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 2
-      },  // Input pad bottom = 0
-      {  // Input pad bottom = 1
-        {  // Input pad right = 0
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 1, 1, 0, 0, 0>,
-            ConvImpl::template process_tile<0, 1, 1, 0, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 1, 1, 0, 1, 0>,
-            ConvImpl::template process_tile<0, 1, 1, 0, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 0
-        {  // Input pad right = 1
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 1, 1, 1, 0, 0>,
-            ConvImpl::template process_tile<0, 1, 1, 1, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 1, 1, 1, 1, 0>,
-            ConvImpl::template process_tile<0, 1, 1, 1, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 1
-        {  // Input pad right = 2
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 1, 1, 2, 0, 0>,
-            ConvImpl::template process_tile<0, 1, 1, 2, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 1, 1, 2, 1, 0>,
-            ConvImpl::template process_tile<0, 1, 1, 2, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 2
-      },  // Input pad bottom = 1
-      {  // Input pad bottom = 2
-        {  // Input pad right = 0
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 1, 2, 0, 0, 0>,
-            ConvImpl::template process_tile<0, 1, 2, 0, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 1, 2, 0, 1, 0>,
-            ConvImpl::template process_tile<0, 1, 2, 0, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 0
-        {  // Input pad right = 1
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 1, 2, 1, 0, 0>,
-            ConvImpl::template process_tile<0, 1, 2, 1, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 1, 2, 1, 1, 0>,
-            ConvImpl::template process_tile<0, 1, 2, 1, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 1
-        {  // Input pad right = 2
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<0, 1, 2, 2, 0, 0>,
-            ConvImpl::template process_tile<0, 1, 2, 2, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<0, 1, 2, 2, 1, 0>,
-            ConvImpl::template process_tile<0, 1, 2, 2, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 2
-      },  // Input pad bottom = 2
-    },  // Input pad left = 1
-  },  // Input pad top = 0
-  {  // Input pad top = 1
-    {  // Input pad left = 0
-      {  // Input pad bottom = 0
-        {  // Input pad right = 0
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 0, 0, 0, 0, 0>,
-            ConvImpl::template process_tile<1, 0, 0, 0, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 0, 0, 0, 1, 0>,
-            ConvImpl::template process_tile<1, 0, 0, 0, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 0
-        {  // Input pad right = 1
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 0, 0, 1, 0, 0>,
-            ConvImpl::template process_tile<1, 0, 0, 1, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 0, 0, 1, 1, 0>,
-            ConvImpl::template process_tile<1, 0, 0, 1, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 1
-        {  // Input pad right = 2
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 0, 0, 2, 0, 0>,
-            ConvImpl::template process_tile<1, 0, 0, 2, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 0, 0, 2, 1, 0>,
-            ConvImpl::template process_tile<1, 0, 0, 2, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 2
-      },  // Input pad bottom = 0
-      {  // Input pad bottom = 1
-        {  // Input pad right = 0
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 0, 1, 0, 0, 0>,
-            ConvImpl::template process_tile<1, 0, 1, 0, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 0, 1, 0, 1, 0>,
-            ConvImpl::template process_tile<1, 0, 1, 0, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 0
-        {  // Input pad right = 1
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 0, 1, 1, 0, 0>,
-            ConvImpl::template process_tile<1, 0, 1, 1, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 0, 1, 1, 1, 0>,
-            ConvImpl::template process_tile<1, 0, 1, 1, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 1
-        {  // Input pad right = 2
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 0, 1, 2, 0, 0>,
-            ConvImpl::template process_tile<1, 0, 1, 2, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 0, 1, 2, 1, 0>,
-            ConvImpl::template process_tile<1, 0, 1, 2, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 2
-      },  // Input pad bottom = 1
-      {  // Input pad bottom = 2
-        {  // Input pad right = 0
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 0, 2, 0, 0, 0>,
-            ConvImpl::template process_tile<1, 0, 2, 0, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 0, 2, 0, 1, 0>,
-            ConvImpl::template process_tile<1, 0, 2, 0, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 0
-        {  // Input pad right = 1
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 0, 2, 1, 0, 0>,
-            ConvImpl::template process_tile<1, 0, 2, 1, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 0, 2, 1, 1, 0>,
-            ConvImpl::template process_tile<1, 0, 2, 1, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 1
-        {  // Input pad right = 2
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 0, 2, 2, 0, 0>,
-            ConvImpl::template process_tile<1, 0, 2, 2, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 0, 2, 2, 1, 0>,
-            ConvImpl::template process_tile<1, 0, 2, 2, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 2
-      },  // Input pad bottom = 2
-    },  // Input pad left = 0
-    {  // Input pad left = 1
-      {  // Input pad bottom = 0
-        {  // Input pad right = 0
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 1, 0, 0, 0, 0>,
-            ConvImpl::template process_tile<1, 1, 0, 0, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 1, 0, 0, 1, 0>,
-            ConvImpl::template process_tile<1, 1, 0, 0, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 0
-        {  // Input pad right = 1
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 1, 0, 1, 0, 0>,
-            ConvImpl::template process_tile<1, 1, 0, 1, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 1, 0, 1, 1, 0>,
-            ConvImpl::template process_tile<1, 1, 0, 1, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 1
-        {  // Input pad right = 2
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 1, 0, 2, 0, 0>,
-            ConvImpl::template process_tile<1, 1, 0, 2, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 1, 0, 2, 1, 0>,
-            ConvImpl::template process_tile<1, 1, 0, 2, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 2
-      },  // Input pad bottom = 0
-      {  // Input pad bottom = 1
-        {  // Input pad right = 0
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 1, 1, 0, 0, 0>,
-            ConvImpl::template process_tile<1, 1, 1, 0, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 1, 1, 0, 1, 0>,
-            ConvImpl::template process_tile<1, 1, 1, 0, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 0
-        {  // Input pad right = 1
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 1, 1, 1, 0, 0>,
-            ConvImpl::template process_tile<1, 1, 1, 1, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 1, 1, 1, 1, 0>,
-            ConvImpl::template process_tile<1, 1, 1, 1, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 1
-        {  // Input pad right = 2
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 1, 1, 2, 0, 0>,
-            ConvImpl::template process_tile<1, 1, 1, 2, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 1, 1, 2, 1, 0>,
-            ConvImpl::template process_tile<1, 1, 1, 2, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 2
-      },  // Input pad bottom = 1
-      {  // Input pad bottom = 2
-        {  // Input pad right = 0
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 1, 2, 0, 0, 0>,
-            ConvImpl::template process_tile<1, 1, 2, 0, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 1, 2, 0, 1, 0>,
-            ConvImpl::template process_tile<1, 1, 2, 0, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 0
-        {  // Input pad right = 1
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 1, 2, 1, 0, 0>,
-            ConvImpl::template process_tile<1, 1, 2, 1, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 1, 2, 1, 1, 0>,
-            ConvImpl::template process_tile<1, 1, 2, 1, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 1
-        {  // Input pad right = 2
-          {  // Output pad bottom = 0
-            ConvImpl::template process_tile<1, 1, 2, 2, 0, 0>,
-            ConvImpl::template process_tile<1, 1, 2, 2, 0, 1>,
-          },  // Output pad bottom = 0
-          {  // Output pad bottom = 1
-            ConvImpl::template process_tile<1, 1, 2, 2, 1, 0>,
-            ConvImpl::template process_tile<1, 1, 2, 2, 1, 1>,
-          },  // Output pad bottom = 1
-        },  // Input pad right = 2
-      },  // Input pad bottom = 2
-    },  // Input pad left = 1
-  },  // Input pad top = 1
+void ConvImpl::process_tile<true, 0, 0, 0, 0, 0, 0>(
+  const int n_channels,
+  const float* const weights,
+  const int weight_row_stride,
+  const int weight_col_stride,
+  const float* const inptr,
+  const int in_row_stride,
+  const int in_col_stride,
+  float* const outptr,
+  const int out_row_stride,
+  const int out_col_stride,
+  const int, const int, const int, const int, const int, const int
+)
+{
+  // Copy pointers
+  const float *uptr0 = inptr;
+  const float *wptr0 = weights;
+  float *vptr0 = outptr;
+
+  int channels_remaining = n_channels;
+  if (channels_remaining >= 4)
+  {
+    // Process blocks of 4 channels at a time
+    int n_iters = ((channels_remaining / 4) + 1)/2 - 1;
+    const bool odd_tail = (channels_remaining / 4) & 1;
+    channels_remaining %= 4;
+
+    asm volatile (
+      "qW11B .req q0\n" "vW11B .req v0\n" "qW33A .req q1\n" "qU32B .req q1\n"
+      "vW33A .req v1\n" "vU32B .req v1\n" "qU44B .req q2\n" "qW21A .req q2\n"
+      "vU44B .req v2\n" "vW21A .req v2\n" "qU21B .req q3\n" "qU32A .req q3\n"
+      "vU21B .req v3\n" "vU32A .req v3\n" "qU43A .req q4\n" "qV21B .req q4\n"
+      "vU43A .req v4\n" "vV21B .req v4\n" "qU24A .req q5\n" "qU44A .req q5\n"
+      "qU33B .req q5\n" "vU24A .req v5\n" "vU44A .req v5\n" "vU33B .req v5\n"
+      "qU31A .req q6\n" "qV12B .req q6\n" "qU23A .req q6\n" "vU31A .req v6\n"
+      "vV12B .req v6\n" "vU23A .req v6\n" "qW31B .req q7\n" "qV22A .req q7\n"
+      "vW31B .req v7\n" "vV22A .req v7\n" "qV12A .req q8\n" "qW21B .req q8\n"
+      "vV12A .req v8\n" "vW21B .req v8\n" "qU22B .req q9\n" "qU34A .req q9\n"
+      "vU22B .req v9\n" "vU34A .req v9\n" "qU13B .req q10\n" "qU13A .req q10\n"
+      "vU13B .req v10\n" "vU13A .req v10\n" "qU34B .req q11\n" "qU22A .req q11\n"
+      "vU34B .req v11\n" "vU22A .req v11\n" "qU24B .req q12\n" "qU31B .req q12\n"
+      "vU24B .req v12\n" "vU31B .req v12\n" "qW12B .req q13\n" "qW13A .req q13\n"
+      "vW12B .req v13\n" "vW13A .req v13\n" "qV21A .req q14\n" "qV11B .req q14\n"
+      "vV21A .req v14\n" "vV11B .req v14\n" "qW32A .req q15\n" "qW32B .req q15\n"
+      "vW32A .req v15\n" "vW32B .req v15\n" "qW31A .req q16\n" "qV22B .req q16\n"
+      "vW31A .req v16\n" "vV22B .req v16\n"
+      "qW11A .req q17\n" "vW11A .req v17\n" "qW13B .req q18\n" "qU14A .req q18\n"
+      "vW13B .req v18\n" "vU14A .req v18\n" "qU33A .req q19\n" "qW33B .req q19\n"
+      "vU33A .req v19\n" "vW33B .req v19\n" "qW22A .req q20\n" "qU23B .req q20\n"
+      "vW22A .req v20\n" "vU23B .req v20\n" "qU12A .req q21\n" "qU42A .req q21\n"
+      "vU12A .req v21\n" "vU42A .req v21\n" "qU41A .req q22\n" "qU42B .req q22\n"
+      "vU41A .req v22\n" "vU42B .req v22\n" "qW23A .req q23\n" "qW23B .req q23\n"
+      "vW23A .req v23\n" "vW23B .req v23\n" "qU43B .req q24\n" "qU11A .req q24\n"
+      "vU43B .req v24\n" "vU11A .req v24\n" "qU12B .req q25\n" "qW12A .req q25\n"
+      "vU12B .req v25\n" "vW12A .req v25\n" "qU41B .req q26\n" "qV11A .req q26\n"
+      "vU41B .req v26\n" "vV11A .req v26\n" "qW22B .req q27\n" "vW22B .req v27\n"
+      "qU11B .req q28\n" "qU14B .req q28\n" "vU11B .req v28\n" "vU14B .req v28\n"
+      "qU21A .req q29\n" "vU21A .req v29\n"
+
+      "u_col_stride1 .req %x[u_col_stride]\n"
+      "u_col_stride2 .req x0\n"
+      "u_col_stride3 .req x1\n"
+      "uptr1 .req x2\n"
+      "uptr2 .req x3\n"
+      "uptr3 .req x4\n"
+      "wptr1 .req x5\n"
+      "wptr2 .req x6\n"
+      "vptr1 .req x7\n"
+      "w_col_stride1 .req %x[w_col_stride]\n"
+      "w_col_stride2 .req x8\n"
+
+      // Prepare strides and pointers
+      "add uptr1, %x[uptr0], %x[u_row_stride]\n"
+      "add uptr2,    uptr1 , %x[u_row_stride]\n"
+      "add uptr3,    uptr2 , %x[u_row_stride]\n"
+      "add wptr1, %x[wptr0], %x[w_row_stride]\n"
+      "add wptr2,    wptr1 , %x[w_row_stride]\n"
+      "add vptr1, %x[vptr0], %x[v_row_stride]\n"
+      "add u_col_stride2, %x[u_col_stride], %x[u_col_stride]\n"
+      "add u_col_stride3,    u_col_stride2 , %x[u_col_stride]\n"
+      "add w_col_stride2, %x[w_col_stride], %x[w_col_stride]\n"
+
+      // Load in preparation for execution
+      "ldr qU14A, [%x[uptr0], u_col_stride3]\n"
+      "ldr qW13A, [%x[wptr0], w_col_stride2]\n"
+      "ldr qU13A, [%x[uptr0], u_col_stride2]\n"
+      "ldr qW12A, [%x[wptr0], w_col_stride1]\n"
+      "ldr qU12A, [%x[uptr0], u_col_stride1]\n"
+      "ldr qW11A, [%x[wptr0]], #0x10\n"
+      "ldr qU24A, [uptr1, u_col_stride3]\n"
+      "ldr qW23A, [wptr1, w_col_stride2]\n"
+      "ldr qU23A, [uptr1, u_col_stride2]\n"
+      "ldr qW22A, [wptr1, w_col_stride1]\n"
+      "ldr qU22A, [uptr1, u_col_stride1]\n"
+      "ldr qW21A, [wptr1], #0x10\n"
+      "ldr qU34A, [uptr2, u_col_stride3]\n"
+      "ldr qW33A, [wptr2, w_col_stride2]\n"
+      "ldr qU33A, [uptr2, u_col_stride2]\n"
+      "ldr qW32A, [wptr2, w_col_stride1]\n"
+      "ldr qU32A, [uptr2, u_col_stride1]\n"
+      "ldr qW31A, [wptr2], #0x10\n"
+      "fmul vV12A.4s, vU14A.4s, vW13A.4s\n"
+      "cbz %x[iters], 2f\n"  // Jump to tail if doing zero iterations of loop
+
+      "1:"  // Main loop body
+        // A part
+        "fmul vV11A.4s, vU13A.4s, vW13A.4s\n"
+        "fmla vV12A.4s, vU13A.4s, vW12A.4s\n"
+        "fmla vV11A.4s, vU12A.4s, vW12A.4s\n"
+        "fmla vV12A.4s, vU12A.4s, vW11A.4s\n"
+        "fmla vV12A.4s, vU24A.4s, vW23A.4s\n"
+        "fmul vV22A.4s, vU24A.4s, vW13A.4s\n"
+        "fmla vV11A.4s, vU23A.4s, vW23A.4s\n"
+        "ldr qU44A, [uptr3, u_col_stride3]\n"
+        "fmla vV12A.4s, vU23A.4s, vW22A.4s\n"
+        "ldr qU43A, [uptr3, u_col_stride2]\n"
+        "fmul vV21A.4s, vU23A.4s, vW13A.4s\n"
+        "ldr qU42A, [uptr3, u_col_stride1]\n"
+        "fmla vV22A.4s, vU23A.4s, vW12A.4s\n"
+        "ldr qU11A, [%x[uptr0]], #0x10\n"
+        "fmla vV11A.4s, vU22A.4s, vW22A.4s\n"
+        "ldr qU21A, [uptr1], #0x10\n"
+        "fmla vV12A.4s, vU22A.4s, vW21A.4s\n"
+        "ldr qU31A, [uptr2], #0x10\n"
+        "fmla vV21A.4s, vU22A.4s, vW12A.4s\n"
+        "ldr qU41A, [uptr3], #0x10\n"
+        "fmla vV22A.4s, vU22A.4s, vW11A.4s\n"
+        "ldr qU14B, [%x[uptr0], u_col_stride3]\n"
+        "fmla vV12A.4s, vU34A.4s, vW33A.4s\n"
+        "ldr qW13B, [%x[wptr0], w_col_stride2]\n"
+        "fmla vV22A.4s, vU34A.4s, vW23A.4s\n"
+        "ldr qU13B, [%x[uptr0], u_col_stride2]\n"
+        "fmla vV11A.4s, vU33A.4s, vW33A.4s\n"
+        "ldr qW12B, [%x[wptr0], w_col_stride1]\n"
+        "fmla vV12A.4s, vU33A.4s, vW32A.4s\n"
+        "ldr qU12B, [%x[uptr0], u_col_stride1]\n"
+        "fmla vV21A.4s, vU33A.4s, vW23A.4s\n"
+        "ldr qW11B, [%x[wptr0]], #0x10\n"
+        "fmla vV22A.4s, vU33A.4s, vW22A.4s\n"
+        "ldr qU24B, [uptr1, u_col_stride3]\n"
+        "fmla vV11A.4s, vU32A.4s, vW32A.4s\n"
+        "ldr qW23B, [wptr1, w_col_stride2]\n"
+        "fmla vV12A.4s, vU32A.4s, vW31A.4s\n"
+        "str qV12A, [%x[vptr0], %x[v_col_stride]]\n"
+        "fmla vV21A.4s, vU32A.4s, vW22A.4s\n"
+        "ldr qU23B, [uptr1, u_col_stride2]\n"
+        "fmla vV22A.4s, vU32A.4s, vW21A.4s\n"
+        "ldr qW22B, [wptr1, w_col_stride1]\n"
+        "fmla vV22A.4s, vU44A.4s, vW33A.4s\n"
+        "ldr qU22B, [uptr1, u_col_stride1]\n"
+        "fmla vV21A.4s, vU43A.4s, vW33A.4s\n"
+        "ldr qW21B, [wptr1], #0x10\n"
+        "fmla vV22A.4s, vU43A.4s, vW32A.4s\n"
+        "ldr qU34B, [uptr2, u_col_stride3]\n"
+        "fmla vV21A.4s, vU42A.4s, vW32A.4s\n"
+        "ldr qW33B, [wptr2, w_col_stride2]\n"
+        "fmla vV22A.4s, vU42A.4s, vW31A.4s\n"
+        "str qV22A, [vptr1, %x[v_col_stride]]\n"
+        "fmla vV11A.4s, vU11A.4s, vW11A.4s\n"
+        "ldr qU33B, [uptr2, u_col_stride2]\n"
+        "fmla vV11A.4s, vU21A.4s, vW21A.4s\n"
+        "ldr qW32B, [wptr2, w_col_stride1]\n"
+        "fmla vV21A.4s, vU21A.4s, vW11A.4s\n"
+        "ldr qU32B, [uptr2, u_col_stride1]\n"
+        "fmla vV11A.4s, vU31A.4s, vW31A.4s\n"
+        "str qV11A, [%x[vptr0]], #0x10\n"
+        "fmla vV21A.4s, vU31A.4s, vW21A.4s\n"
+        "ldr qW31B, [wptr2], #0x10\n"
+        "fmla vV21A.4s, vU41A.4s, vW31A.4s\n"
+        "str qV21A, [vptr1], #0x10\n"
+
+        // B part
+        "fmul vV12B.4s, vU14B.4s, vW13B.4s\n"
+        "fmul vV11B.4s, vU13B.4s, vW13B.4s\n"
+        "fmla vV12B.4s, vU13B.4s, vW12B.4s\n"
+        "fmla vV11B.4s, vU12B.4s, vW12B.4s\n"
+        "fmla vV12B.4s, vU12B.4s, vW11B.4s\n"
+        "fmla vV12B.4s, vU24B.4s, vW23B.4s\n"
+        "fmul vV22B.4s, vU24B.4s, vW13B.4s\n"
+        "subs %x[iters], %x[iters], #1\n"
+        "fmla vV11B.4s, vU23B.4s, vW23B.4s\n"
+        "ldr qU44B, [uptr3, u_col_stride3]\n"
+        "fmla vV12B.4s, vU23B.4s, vW22B.4s\n"
+        "ldr qU43B, [uptr3, u_col_stride2]\n"
+        "fmul vV21B.4s, vU23B.4s, vW13B.4s\n"
+        "ldr qU42B, [uptr3, u_col_stride1]\n"
+        "fmla vV22B.4s, vU23B.4s, vW12B.4s\n"
+        "ldr qU11B, [%x[uptr0]], #0x10\n"
+        "fmla vV11B.4s, vU22B.4s, vW22B.4s\n"
+        "ldr qU21B, [uptr1], #0x10\n"
+        "fmla vV12B.4s, vU22B.4s, vW21B.4s\n"
+        "ldr qU31B, [uptr2], #0x10\n"
+        "fmla vV21B.4s, vU22B.4s, vW12B.4s\n"
+        "ldr qU41B, [uptr3], #0x10\n"
+        "fmla vV22B.4s, vU22B.4s, vW11B.4s\n"
+        "ldr qU14A, [%x[uptr0], u_col_stride3]\n"
+        "fmla vV12B.4s, vU34B.4s, vW33B.4s\n"
+        "ldr qW13A, [%x[wptr0], w_col_stride2]\n"
+        "fmla vV22B.4s, vU34B.4s, vW23B.4s\n"
+        "ldr qU13A, [%x[uptr0], u_col_stride2]\n"
+        "fmla vV11B.4s, vU33B.4s, vW33B.4s\n"
+        "ldr qW12A, [%x[wptr0], w_col_stride1]\n"
+        "fmla vV12B.4s, vU33B.4s, vW32B.4s\n"
+        "ldr qU12A, [%x[uptr0], u_col_stride1]\n"
+        "fmla vV21B.4s, vU33B.4s, vW23B.4s\n"
+        "ldr qW11A, [%x[wptr0]], #0x10\n"
+        "fmla vV22B.4s, vU33B.4s, vW22B.4s\n"
+        "ldr qU24A, [uptr1, u_col_stride3]\n"
+        "fmla vV11B.4s, vU32B.4s, vW32B.4s\n"
+        "ldr qW23A, [wptr1, w_col_stride2]\n"
+        "fmla vV12B.4s, vU32B.4s, vW31B.4s\n"
+        "str qV12B, [%x[vptr0], %x[v_col_stride]]\n"
+        "fmla vV21B.4s, vU32B.4s, vW22B.4s\n"
+        "ldr qU23A, [uptr1, u_col_stride2]\n"
+        "fmla vV22B.4s, vU32B.4s, vW21B.4s\n"
+        "ldr qW22A, [wptr1, w_col_stride1]\n"
+        "fmla vV22B.4s, vU44B.4s, vW33B.4s\n"
+        "ldr qU22A, [uptr1, u_col_stride1]\n"
+        "fmla vV21B.4s, vU43B.4s, vW33B.4s\n"
+        "ldr qW21A, [wptr1], #0x10\n"
+        "fmla vV22B.4s, vU43B.4s, vW32B.4s\n"
+        "ldr qU34A, [uptr2, u_col_stride3]\n"
+        "fmla vV21B.4s, vU42B.4s, vW32B.4s\n"
+        "ldr qW33A, [wptr2, w_col_stride2]\n"
+        "fmla vV22B.4s, vU42B.4s, vW31B.4s\n"
+        "str qV22B, [vptr1, %x[v_col_stride]]\n"
+        "fmla vV11B.4s, vU11B.4s, vW11B.4s\n"
+        "ldr qU33A, [uptr2, u_col_stride2]\n"
+        "fmla vV11B.4s, vU21B.4s, vW21B.4s\n"
+        "ldr qW32A, [wptr2, w_col_stride1]\n"
+        "fmla vV21B.4s, vU21B.4s, vW11B.4s\n"
+        "ldr qU32A, [uptr2, u_col_stride1]\n"
+        "fmla vV11B.4s, vU31B.4s, vW31B.4s\n"
+        "str qV11B, [%x[vptr0]], #0x10\n"
+        "fmla vV21B.4s, vU31B.4s, vW21B.4s\n"
+        "ldr qW31A, [wptr2], #0x10\n"
+        "fmla vV21B.4s, vU41B.4s, vW31B.4s\n"
+        "str qV21B, [vptr1], #0x10\n"
+        "fmul vV12A.4s, vU14A.4s, vW13A.4s\n"
+        "bne 1b\n"  // Loop
+
+      "2:"  // Branch destination for zero loops
+        "cbnz %w[odd_tail], 4f\n"
+
+      "3:"  // Even number of iterations
+        "fmul vV11A.4s, vU13A.4s, vW13A.4s\n"
+        "fmla vV12A.4s, vU13A.4s, vW12A.4s\n"
+        "fmla vV11A.4s, vU12A.4s, vW12A.4s\n"
+        "fmla vV12A.4s, vU12A.4s, vW11A.4s\n"
+        "fmla vV12A.4s, vU24A.4s, vW23A.4s\n"
+        "fmul vV22A.4s, vU24A.4s, vW13A.4s\n"
+        "fmla vV11A.4s, vU23A.4s, vW23A.4s\n"
+        "ldr qU44A, [uptr3, u_col_stride3]\n"
+        "fmla vV12A.4s, vU23A.4s, vW22A.4s\n"
+        "ldr qU43A, [uptr3, u_col_stride2]\n"
+        "fmul vV21A.4s, vU23A.4s, vW13A.4s\n"
+        "ldr qU42A, [uptr3, u_col_stride1]\n"
+        "fmla vV22A.4s, vU23A.4s, vW12A.4s\n"
+        "ldr qU11A, [%x[uptr0]], #0x10\n"
+        "fmla vV11A.4s, vU22A.4s, vW22A.4s\n"
+        "ldr qU21A, [uptr1], #0x10\n"
+        "fmla vV12A.4s, vU22A.4s, vW21A.4s\n"
+        "ldr qU31A, [uptr2], #0x10\n"
+        "fmla vV21A.4s, vU22A.4s, vW12A.4s\n"
+        "ldr qU41A, [uptr3], #0x10\n"
+        "fmla vV22A.4s, vU22A.4s, vW11A.4s\n"
+        "ldr qU14B, [%x[uptr0], u_col_stride3]\n"
+        "fmla vV12A.4s, vU34A.4s, vW33A.4s\n"
+        "ldr qW13B, [%x[wptr0], w_col_stride2]\n"
+        "fmla vV22A.4s, vU34A.4s, vW23A.4s\n"
+        "ldr qU13B, [%x[uptr0], u_col_stride2]\n"
+        "fmla vV11A.4s, vU33A.4s, vW33A.4s\n"
+        "ldr qW12B, [%x[wptr0], w_col_stride1]\n"
+        "fmla vV12A.4s, vU33A.4s, vW32A.4s\n"
+        "ldr qU12B, [%x[uptr0], u_col_stride1]\n"
+        "fmla vV21A.4s, vU33A.4s, vW23A.4s\n"
+        "ldr qW11B, [%x[wptr0]], #0x10\n"
+        "fmla vV22A.4s, vU33A.4s, vW22A.4s\n"
+        "ldr qU24B, [uptr1, u_col_stride3]\n"
+        "fmla vV11A.4s, vU32A.4s, vW32A.4s\n"
+        "ldr qW23B, [wptr1, w_col_stride2]\n"
+        "fmla vV12A.4s, vU32A.4s, vW31A.4s\n"
+        "str qV12A, [%x[vptr0], %x[v_col_stride]]\n"
+        "fmla vV21A.4s, vU32A.4s, vW22A.4s\n"
+        "ldr qU23B, [uptr1, u_col_stride2]\n"
+        "fmla vV22A.4s, vU32A.4s, vW21A.4s\n"
+        "ldr qW22B, [wptr1, w_col_stride1]\n"
+        "fmla vV22A.4s, vU44A.4s, vW33A.4s\n"
+        "ldr qU22B, [uptr1, u_col_stride1]\n"
+        "fmla vV21A.4s, vU43A.4s, vW33A.4s\n"
+        "ldr qW21B, [wptr1], #0x10\n"
+        "fmla vV22A.4s, vU43A.4s, vW32A.4s\n"
+        "ldr qU34B, [uptr2, u_col_stride3]\n"
+        "fmla vV21A.4s, vU42A.4s, vW32A.4s\n"
+        "ldr qW33B, [wptr2, w_col_stride2]\n"
+        "fmla vV22A.4s, vU42A.4s, vW31A.4s\n"
+        "str qV22A, [vptr1, %x[v_col_stride]]\n"
+        "fmla vV11A.4s, vU11A.4s, vW11A.4s\n"
+        "ldr qU33B, [uptr2, u_col_stride2]\n"
+        "fmla vV11A.4s, vU21A.4s, vW21A.4s\n"
+        "ldr qW32B, [wptr2, w_col_stride1]\n"
+        "fmla vV21A.4s, vU21A.4s, vW11A.4s\n"
+        "ldr qU32B, [uptr2, u_col_stride1]\n"
+        "fmla vV11A.4s, vU31A.4s, vW31A.4s\n"
+        "str qV11A, [%x[vptr0]], #0x10\n"
+        "fmla vV21A.4s, vU31A.4s, vW21A.4s\n"
+        "ldr qW31B, [wptr2], #0x10\n"
+        "fmla vV21A.4s, vU41A.4s, vW31A.4s\n"
+        "str qV21A, [vptr1], #0x10\n"
+
+        "fmul vV12B.4s, vU14B.4s, vW13B.4s\n"
+        "fmul vV11B.4s, vU13B.4s, vW13B.4s\n"
+        "fmla vV12B.4s, vU13B.4s, vW12B.4s\n"
+        "fmla vV11B.4s, vU12B.4s, vW12B.4s\n"
+        "fmla vV12B.4s, vU12B.4s, vW11B.4s\n"
+        "fmla vV12B.4s, vU24B.4s, vW23B.4s\n"
+        "fmul vV22B.4s, vU24B.4s, vW13B.4s\n"
+        "fmla vV11B.4s, vU23B.4s, vW23B.4s\n"
+        "ldr qU44B, [uptr3, u_col_stride3]\n"
+        "fmla vV12B.4s, vU23B.4s, vW22B.4s\n"
+        "ldr qU43B, [uptr3, u_col_stride2]\n"
+        "fmul vV21B.4s, vU23B.4s, vW13B.4s\n"
+        "ldr qU42B, [uptr3, u_col_stride1]\n"
+        "fmla vV22B.4s, vU23B.4s, vW12B.4s\n"
+        "ldr qU11B, [%x[uptr0]], #0x10\n"
+        "fmla vV11B.4s, vU22B.4s, vW22B.4s\n"
+        "ldr qU21B, [uptr1], #0x10\n"
+        "fmla vV12B.4s, vU22B.4s, vW21B.4s\n"
+        "ldr qU31B, [uptr2], #0x10\n"
+        "fmla vV21B.4s, vU22B.4s, vW12B.4s\n"
+        "ldr qU41B, [uptr3], #0x10\n"
+        "fmla vV22B.4s, vU22B.4s, vW11B.4s\n"
+        "fmla vV12B.4s, vU34B.4s, vW33B.4s\n"
+        "fmla vV22B.4s, vU34B.4s, vW23B.4s\n"
+        "fmla vV11B.4s, vU33B.4s, vW33B.4s\n"
+        "fmla vV12B.4s, vU33B.4s, vW32B.4s\n"
+        "fmla vV21B.4s, vU33B.4s, vW23B.4s\n"
+        "fmla vV22B.4s, vU33B.4s, vW22B.4s\n"
+        "fmla vV11B.4s, vU32B.4s, vW32B.4s\n"
+        "fmla vV12B.4s, vU32B.4s, vW31B.4s\n"
+        "str qV12B, [%x[vptr0], %x[v_col_stride]]\n"
+        "fmla vV21B.4s, vU32B.4s, vW22B.4s\n"
+        "fmla vV22B.4s, vU32B.4s, vW21B.4s\n"
+        "fmla vV22B.4s, vU44B.4s, vW33B.4s\n"
+        "fmla vV21B.4s, vU43B.4s, vW33B.4s\n"
+        "fmla vV22B.4s, vU43B.4s, vW32B.4s\n"
+        "fmla vV21B.4s, vU42B.4s, vW32B.4s\n"
+        "fmla vV22B.4s, vU42B.4s, vW31B.4s\n"
+        "str qV22B, [vptr1, %x[v_col_stride]]\n"
+        "fmla vV11B.4s, vU11B.4s, vW11B.4s\n"
+        "fmla vV11B.4s, vU21B.4s, vW21B.4s\n"
+        "fmla vV21B.4s, vU21B.4s, vW11B.4s\n"
+        "fmla vV11B.4s, vU31B.4s, vW31B.4s\n"
+        "str qV11B, [%x[vptr0]], #0x10\n"
+        "fmla vV21B.4s, vU31B.4s, vW21B.4s\n"
+        "fmla vV21B.4s, vU41B.4s, vW31B.4s\n"
+        "str qV21B, [vptr1], #0x10\n"
+        "b 5f\n"
+
+      "4:"  // Odd number of iterations
+        "fmul vV11A.4s, vU13A.4s, vW13A.4s\n"
+        "fmla vV12A.4s, vU13A.4s, vW12A.4s\n"
+        "fmla vV11A.4s, vU12A.4s, vW12A.4s\n"
+        "fmla vV12A.4s, vU12A.4s, vW11A.4s\n"
+        "fmla vV12A.4s, vU24A.4s, vW23A.4s\n"
+        "fmul vV22A.4s, vU24A.4s, vW13A.4s\n"
+        "fmla vV11A.4s, vU23A.4s, vW23A.4s\n"
+        "ldr qU44A, [uptr3, u_col_stride3]\n"
+        "fmla vV12A.4s, vU23A.4s, vW22A.4s\n"
+        "ldr qU43A, [uptr3, u_col_stride2]\n"
+        "fmul vV21A.4s, vU23A.4s, vW13A.4s\n"
+        "ldr qU42A, [uptr3, u_col_stride1]\n"
+        "fmla vV22A.4s, vU23A.4s, vW12A.4s\n"
+        "ldr qU11A, [%x[uptr0]], #0x10\n"
+        "fmla vV11A.4s, vU22A.4s, vW22A.4s\n"
+        "ldr qU21A, [uptr1], #0x10\n"
+        "fmla vV12A.4s, vU22A.4s, vW21A.4s\n"
+        "ldr qU31A, [uptr2], #0x10\n"
+        "fmla vV21A.4s, vU22A.4s, vW12A.4s\n"
+        "ldr qU41A, [uptr3], #0x10\n"
+        "fmla vV22A.4s, vU22A.4s, vW11A.4s\n"
+        "fmla vV12A.4s, vU34A.4s, vW33A.4s\n"
+        "fmla vV22A.4s, vU34A.4s, vW23A.4s\n"
+        "fmla vV11A.4s, vU33A.4s, vW33A.4s\n"
+        "fmla vV12A.4s, vU33A.4s, vW32A.4s\n"
+        "fmla vV21A.4s, vU33A.4s, vW23A.4s\n"
+        "fmla vV22A.4s, vU33A.4s, vW22A.4s\n"
+        "fmla vV11A.4s, vU32A.4s, vW32A.4s\n"
+        "fmla vV12A.4s, vU32A.4s, vW31A.4s\n"
+        "str qV12A, [%x[vptr0], %x[v_col_stride]]\n"
+        "fmla vV21A.4s, vU32A.4s, vW22A.4s\n"
+        "fmla vV22A.4s, vU32A.4s, vW21A.4s\n"
+        "fmla vV22A.4s, vU44A.4s, vW33A.4s\n"
+        "fmla vV21A.4s, vU43A.4s, vW33A.4s\n"
+        "fmla vV22A.4s, vU43A.4s, vW32A.4s\n"
+        "fmla vV21A.4s, vU42A.4s, vW32A.4s\n"
+        "fmla vV22A.4s, vU42A.4s, vW31A.4s\n"
+        "str qV22A, [vptr1, %x[v_col_stride]]\n"
+        "fmla vV11A.4s, vU11A.4s, vW11A.4s\n"
+        "fmla vV11A.4s, vU21A.4s, vW21A.4s\n"
+        "fmla vV21A.4s, vU21A.4s, vW11A.4s\n"
+        "fmla vV11A.4s, vU31A.4s, vW31A.4s\n"
+        "str qV11A, [%x[vptr0]], #0x10\n"
+        "fmla vV21A.4s, vU31A.4s, vW21A.4s\n"
+        "fmla vV21A.4s, vU41A.4s, vW31A.4s\n"
+        "str qV21A, [vptr1], #0x10\n"
+
+      "5:"  // End of method
+
+      ".unreq qW11B\n" ".unreq qW33A\n" ".unreq qU32B\n"
+      ".unreq qU44B\n" ".unreq qW21A\n" ".unreq qU21B\n" ".unreq qU32A\n"
+      ".unreq qU43A\n" ".unreq qV21B\n"
+      ".unreq qU24A\n" ".unreq qU44A\n" ".unreq qU33B\n"
+      ".unreq qU31A\n" ".unreq qV12B\n" ".unreq qU23A\n"
+      ".unreq qW31B\n" ".unreq qV22A\n" ".unreq qV12A\n" ".unreq qW21B\n"
+      ".unreq qU22B\n" ".unreq qU34A\n" ".unreq qU13B\n" ".unreq qU13A\n"
+      ".unreq qU34B\n" ".unreq qU22A\n" ".unreq qU24B\n" ".unreq qU31B\n"
+      ".unreq qW12B\n" ".unreq qW13A\n" ".unreq qV21A\n" ".unreq qV11B\n"
+      ".unreq qW32A\n" ".unreq qW32B\n" ".unreq qW31A\n" ".unreq qV22B\n"
+      ".unreq qW11A\n" ".unreq qW13B\n" ".unreq qU14A\n"
+      ".unreq qU33A\n" ".unreq qW33B\n" ".unreq qW22A\n" ".unreq qU23B\n"
+      ".unreq qU12A\n" ".unreq qU42A\n" ".unreq qU41A\n" ".unreq qU42B\n"
+      ".unreq qW23A\n" ".unreq qW23B\n" ".unreq qU43B\n" ".unreq qU11A\n"
+      ".unreq qU12B\n" ".unreq qW12A\n" ".unreq qU41B\n" ".unreq qV11A\n"
+      ".unreq qW22B\n" ".unreq qU11B\n" ".unreq qU14B\n" ".unreq qU21A\n"
+      ".unreq vW11B\n" ".unreq vW33A\n" ".unreq vU32B\n"
+      ".unreq vU44B\n" ".unreq vW21A\n" ".unreq vU21B\n" ".unreq vU32A\n"
+      ".unreq vU43A\n" ".unreq vV21B\n"
+      ".unreq vU24A\n" ".unreq vU44A\n" ".unreq vU33B\n"
+      ".unreq vU31A\n" ".unreq vV12B\n" ".unreq vU23A\n"
+      ".unreq vW31B\n" ".unreq vV22A\n" ".unreq vV12A\n" ".unreq vW21B\n"
+      ".unreq vU22B\n" ".unreq vU34A\n" ".unreq vU13B\n" ".unreq vU13A\n"
+      ".unreq vU34B\n" ".unreq vU22A\n" ".unreq vU24B\n" ".unreq vU31B\n"
+      ".unreq vW12B\n" ".unreq vW13A\n" ".unreq vV21A\n" ".unreq vV11B\n"
+      ".unreq vW32A\n" ".unreq vW32B\n" ".unreq vW31A\n" ".unreq vV22B\n"
+      ".unreq vW11A\n" ".unreq vW13B\n" ".unreq vU14A\n"
+      ".unreq vU33A\n" ".unreq vW33B\n" ".unreq vW22A\n" ".unreq vU23B\n"
+      ".unreq vU12A\n" ".unreq vU42A\n" ".unreq vU41A\n" ".unreq vU42B\n"
+      ".unreq vW23A\n" ".unreq vW23B\n" ".unreq vU43B\n" ".unreq vU11A\n"
+      ".unreq vU12B\n" ".unreq vW12A\n" ".unreq vU41B\n" ".unreq vV11A\n"
+      ".unreq vW22B\n" ".unreq vU11B\n" ".unreq vU14B\n" ".unreq vU21A\n"
+      ".unreq u_col_stride1\n" ".unreq u_col_stride2\n"
+      ".unreq u_col_stride3\n"
+      ".unreq uptr1\n" ".unreq uptr2\n" ".unreq uptr3\n"
+      ".unreq wptr1\n" ".unreq wptr2\n" ".unreq vptr1\n"
+      ".unreq w_col_stride1\n" ".unreq w_col_stride2\n"
+
+      : [uptr0] "+r" (uptr0), [vptr0] "+r" (vptr0), [wptr0] "+r" (wptr0),
+        [iters] "+r" (n_iters)
+      : [u_row_stride] "r" (in_row_stride * sizeof(float)),
+        [u_col_stride] "r" (in_col_stride * sizeof(float)),
+        [v_row_stride] "r" (out_row_stride * sizeof(float)),
+        [v_col_stride] "r" (out_col_stride * sizeof(float)),
+        [w_row_stride] "r" (weight_row_stride * sizeof(float)),
+        [w_col_stride] "r" (weight_col_stride * sizeof(float)),
+        [odd_tail] "r" (odd_tail)
+      : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "cc",
+        "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10",
+        "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20",
+        "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "memory"
+    );
+  }
+
+  if (channels_remaining)
+  {
+    // Fall back on the unoptimised version to clean up the tail
+    ConvImpl::process_tile<false>(
+        channels_remaining,
+        wptr0, weight_row_stride, weight_col_stride,
+        uptr0, in_row_stride, in_col_stride,
+        vptr0, out_row_stride, out_col_stride,
+        0, 0, 0, 0, 0, 0
+    );
+  }
+}
+
+#endif  // __aarch64__
+
+template <>
+const Conv::TileFn Conv::tilefn_unpadded = ConvImpl::template process_tile<true, 0, 0, 0, 0, 0, 0>;
+
+template <>
+const Conv::TileFn Conv::tilefn_top[n_in_pad_top_fns] = {
+  ConvImpl::template process_tile<true, 1, 0, 0, 0, 0, 0>,
+};
+
+template <>
+const Conv::TileFn Conv::tilefn_left[n_in_pad_left_fns] = {
+  ConvImpl::template process_tile<true, 0, 1, 0, 0, 0, 0>,
+};
+
+template <>
+const Conv::TileFn Conv::tilefn_bottom[n_in_pad_bottom_fns][n_out_pad_bottom_fns] = {
+  {
+    ConvImpl::template process_tile<true, 0, 0, 0, 0, 0, 0>,
+    ConvImpl::template process_tile<true, 0, 0, 0, 0, 1, 0>,
+  },
+  {
+    ConvImpl::template process_tile<true, 0, 0, 1, 0, 0, 0>,
+    ConvImpl::template process_tile<true, 0, 0, 1, 0, 1, 0>,
+  },
+  {
+    ConvImpl::template process_tile<true, 0, 0, 2, 0, 0, 0>,
+    ConvImpl::template process_tile<true, 0, 0, 2, 0, 1, 0>,
+  },
+  {
+    ConvImpl::template process_tile<true, 0, 0, 3, 0, 0, 0>,
+    ConvImpl::template process_tile<true, 0, 0, 3, 0, 1, 0>,
+  },
 };
 
+template <>
+const Conv::TileFn Conv::tilefn_right[n_in_pad_right_fns][n_out_pad_right_fns] = {
+  {
+    ConvImpl::template process_tile<true, 0, 0, 0, 0, 0, 0>,
+    ConvImpl::template process_tile<true, 0, 0, 0, 0, 0, 1>,
+  },
+  {
+    ConvImpl::template process_tile<true, 0, 0, 0, 1, 0, 0>,
+    ConvImpl::template process_tile<true, 0, 0, 0, 1, 0, 1>,
+  },
+  {
+    ConvImpl::template process_tile<true, 0, 0, 0, 2, 0, 0>,
+    ConvImpl::template process_tile<true, 0, 0, 0, 2, 0, 1>,
+  },
+  {
+    ConvImpl::template process_tile<true, 0, 0, 0, 3, 0, 0>,
+    ConvImpl::template process_tile<true, 0, 0, 0, 3, 0, 1>,
+  },
+};
+
+template <>
+const Conv::TileFn Conv::tilefn_generic = ConvImpl::template process_tile<false>;
 
 template class DepthwiseConvolution<2, 2, 3, 3, 1, 1, float, float>;
 }  // namespace depthwise