diff options
Diffstat (limited to 'src/core/NEON/kernels/winograd/transforms/input_2x2_3x3/a64_float.hpp')
| -rw-r--r-- | src/core/NEON/kernels/winograd/transforms/input_2x2_3x3/a64_float.hpp | 1498 |
1 files changed, 0 insertions, 1498 deletions
diff --git a/src/core/NEON/kernels/winograd/transforms/input_2x2_3x3/a64_float.hpp b/src/core/NEON/kernels/winograd/transforms/input_2x2_3x3/a64_float.hpp deleted file mode 100644 index a99cbe325b..0000000000 --- a/src/core/NEON/kernels/winograd/transforms/input_2x2_3x3/a64_float.hpp +++ /dev/null @@ -1,1498 +0,0 @@ -/* - * Copyright (c) 2017 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once -#include "../input_2x2_3x3.hpp" - -#ifdef __aarch64__ -namespace winograd { - -// Pad left by one column, pad right by one column, no upper or lower padding, 4 channels -template <> -template <> -inline void Winograd2x2_3x3GemmInput<float>::process_tile_row<0, 1, 0, 1, 4>( - const int tile_N, // Number of tiles in the row - const float* const input, // Base input pointer (appropriate to batch, channel and row) - const int input_row_stride, // Stride between rows of the input - const int input_col_stride, // Stride between columns of the input - float* const matrix, // 1st output matrix (appropriate to batch, channel and row) - const int matrix_stride, // Stride between matrices - const int matrix_row_stride // Stride between rows of the output matrix -) { - /* SIMD register allocation - * ======================== - * - * In the following code we read 4x4 tiles of a matrix `x`, with which we - * compute another matrix `X.T x` where: - * - * / 1 0 0 0 \ - * X = | 0 1 -1 1 | - * | -1 1 1 0 | - * \ 0 0 0 -1 / - * - * Hence, `X.T` is a program which operates upon rows of the matrix `X`. - * We subsequently compute and store the matrix `U = (X.T x) X`. - * - * Importantly, each iteration of the loop below loads a new matrix `x'` - * where the final two columns of `x'` are the first two columns of the - * previous `x`. That is: - * - * x11 x12 x13 x14 - * x21 x22 x23 x24 - * x31 x32 x33 x34 - * x41 x42 x43 x44 - * - * x'11 x'12 x'13 x'14 - * x'21 x'22 x'23 x'24 - * x'31 x'32 x'33 x'34 - * x'41 x'42 x'43 x'44 - * - * Consequently, while the first iteration of the below loop must load 16 - * values for `x`, the second need load only 8. *Furthermore*, since we noted - * above that the operation `X.T x` was a program which operated upon *rows* - * of the matrix `x` it follows that that the relation that `x'[i][1] = - * x[i][3]` and `x'[i][2] = x[i][4]` applies also the matrices `X.T x'` and - * `X.T x`. That is: - * - * (X.T x)11 (X.T x)12 (X.T x)13 (X.T x)14 - * (X.T x)21 (X.T x)22 (X.T x)23 (X.T x)24 - * (X.T x)31 (X.T x)32 (X.T x)33 (X.T x)34 - * (X.T x)41 (X.T x)42 (X.T x)43 (X.T x)44 - * - * (X.T x')11 (X.T x')12 (X.T x')13 (X.T x')14 - * (X.T x')12 (X.T x')12 (X.T x')12 (X.T x')12 - * (X.T x')13 (X.T x')13 (X.T x')13 (X.T x')13 - * (X.T x')14 (X.T x')14 (X.T x')14 (X.T x')14 - * - * Hence, as well as not needing to load new values for x'[i][1..2] it is - * also unnecessary to recompute values for (X.T x')[i][1..2]. - * - * Following this we break the registers into blocks `A` and `B` used by the - * two stages of the unrolled loop. These registers named such that the - * latter columns of `A` become the earlier columns of `B` and vice-versa: - * - * AXTx11 AXTx12 > AXTx13 AXTx14 | - * AXTx21 AXTx22 > AXTx23 AXTx24 | - * AXTx31 AXTx32 > AXTx33 AXTx34 | - * AXTx41 AXTx42 > AXTx43 AXTx44 | - * - * BXTx13 BXTx14 | BXTx11 BXTx12 > - * BXTx23 BXTx24 | BXTx21 BXTx22 > - * BXTx33 BXTx34 | BXTx31 BXTx32 > - * BXTx43 BXTx44 | BXTx41 BXTx42 > - * - * These 32 named registers require only 16 architectural registers. 1 - * additional architectural register is used as scratch space and 8 - * architectural registers are used to load in the values x[1..4][3,4]. - * - * Input and output addressing - * =========================== - * TODO Description - */ - const float *inptr0 = input; - const float *inptr1 = input + input_row_stride; - const float *inptr2 = input + input_row_stride * 2; - const float *inptr3 = input + input_row_stride * 3; - - float *outptr0 = matrix; - float *outptr4 = matrix + matrix_stride * 4; - float *outptr8 = matrix + matrix_stride * 8; - float *outptr12 = matrix + matrix_stride * 12; - - int tile_j = tile_N; // Tiles to process - - asm volatile ( - // Named SIMD registers according to the policy given above - // Registers into which to load the latter two columns of `x` - "x_13 .req v0\n qx_13 .req q0\n" "x_14 .req v4\n qx_14 .req q4\n" - "x_23 .req v1\n qx_23 .req q1\n" "x_24 .req v5\n qx_24 .req q5\n" - "x_33 .req v2\n qx_33 .req q2\n" "x_34 .req v6\n qx_34 .req q6\n" - "x_43 .req v3\n qx_43 .req q3\n" "x_44 .req v7\n qx_44 .req q7\n" - - // Registers for storing X.T x (both A and B halves) - "AXTx11 .req v8\n" "BXTx13 .req v8\n" - "AXTx12 .req v9\n" "BXTx14 .req v9\n" "qAXTx12 .req q9\n" - "AXTx21 .req v10\n" "BXTx23 .req v10\n" - "AXTx22 .req v11\n" "BXTx24 .req v11\n" "qAXTx22 .req q11\n" - "AXTx31 .req v12\n" "BXTx33 .req v12\n" - "AXTx32 .req v13\n" "BXTx34 .req v13\n" "qAXTx32 .req q13\n" - "AXTx41 .req v14\n" "BXTx43 .req v14\n" - "AXTx42 .req v15\n" "BXTx44 .req v15\n" "qAXTx42 .req q15\n" - "AXTx13 .req v16\n" "BXTx11 .req v16\n" - "AXTx14 .req v17\n" "BXTx12 .req v17\n" "qBXTx12 .req q17\n" - "AXTx23 .req v18\n" "BXTx21 .req v18\n" - "AXTx24 .req v19\n" "BXTx22 .req v19\n" "qBXTx22 .req q19\n" - "AXTx33 .req v20\n" "BXTx31 .req v20\n" - "AXTx34 .req v21\n" "BXTx32 .req v21\n" "qBXTx32 .req q21\n" - "AXTx43 .req v22\n" "BXTx41 .req v22\n" - "AXTx44 .req v23\n" "BXTx42 .req v23\n" "qBXTx42 .req q23\n" - - // Result register (TODO Does using more registers yield better - // performance) - "U .req v24\n qU .req q24\n" - - // ---------------------------------------------------------------------- - // Head of loop - // Loads a complete 4x4 tile of x, computes X.T x, computes and stores - // `U = X.T x X`. Prepares for the 'A' half of the loop. - // NOTE: Since the first tile has the leftmost column padded we can - // skip 4 loads and 4 calculations for the matrix X.T x X. - - // Temporarily alias registers for computing the first (non-padded) - // column of x. - "x_12 .req v0\n qx_12 .req q0\n" - "x_22 .req v1\n qx_22 .req q1\n" - "x_32 .req v2\n qx_32 .req q2\n" - "x_42 .req v3\n qx_42 .req q3\n" - - "ldr qx_12, [%x[inptr0]]\n" - "ldr qx_22, [%x[inptr1]]\n" - "ldr qx_32, [%x[inptr2]]\n" - "ldr qx_42, [%x[inptr3]]\n" - - "fsub BXTx12.4s, x_12.4s, x_32.4s\n" - "fadd BXTx22.4s, x_22.4s, x_32.4s\n" - "fsub BXTx32.4s, x_32.4s, x_22.4s\n" - "fsub BXTx42.4s, x_22.4s, x_42.4s\n" - - ".unreq x_12\n .unreq qx_12\n" - ".unreq x_22\n .unreq qx_22\n" - ".unreq x_32\n .unreq qx_32\n" - ".unreq x_42\n .unreq qx_42\n" - - // Load and compute latter two columns of the first tile. Progress the - // input pointers (by three columns so that the each points are the - // second column of the next tile, that is, each points at the first - // column which must be read for the next tile. - "ldr qx_13, [%x[inptr0], %x[colstride1]]\n" - "ldr qx_23, [%x[inptr1], %x[colstride1]]\n" - "ldr qx_33, [%x[inptr2], %x[colstride1]]\n" - "ldr qx_43, [%x[inptr3], %x[colstride1]]\n" - - "fsub BXTx13.4s, x_13.4s, x_33.4s\n" - "ldr qx_14, [%x[inptr0], %x[colstride2]]\n" - - "fadd BXTx23.4s, x_23.4s, x_33.4s\n" - "ldr qx_24, [%x[inptr1], %x[colstride2]]\n" - - "fsub BXTx33.4s, x_33.4s, x_23.4s\n" - "ldr qx_34, [%x[inptr2], %x[colstride2]]\n" - - "fsub BXTx43.4s, x_23.4s, x_43.4s\n" - "ldr qx_44, [%x[inptr3], %x[colstride2]]\n" - - "fsub BXTx14.4s, x_14.4s, x_34.4s\n" - "add %x[inptr0], %x[inptr0], %x[colstride3]\n" - - "fadd BXTx24.4s, x_24.4s, x_34.4s\n" - "add %x[inptr1], %x[inptr1], %x[colstride3]\n" - - "fsub BXTx34.4s, x_34.4s, x_24.4s\n" - "add %x[inptr2], %x[inptr2], %x[colstride3]\n" - - "fsub BXTx44.4s, x_24.4s, x_44.4s\n" - "add %x[inptr3], %x[inptr3], %x[colstride3]\n" - - // Compute and store U for the first tile - // First row - "fneg U.4s, BXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, BXTx12.4s, BXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, BXTx13.4s, BXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "fsub U.4s, BXTx12.4s, BXTx14.4s\n" - "str qU, [%x[outptr0], %x[mstride3]]\n" - "add %x[outptr0], %x[outptr0], %x[matrix_row_stride]\n" - - // Second row - "fneg U.4s, BXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, BXTx22.4s, BXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, BXTx23.4s, BXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "fsub U.4s, BXTx22.4s, BXTx24.4s\n" - "str qU, [%x[outptr4], %x[mstride3]]\n" - "add %x[outptr4], %x[outptr4], %x[matrix_row_stride]\n" - - // Third row - "fneg U.4s, BXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, BXTx32.4s, BXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, BXTx33.4s, BXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "fsub U.4s, BXTx32.4s, BXTx34.4s\n" - "str qU, [%x[outptr8], %x[mstride3]]\n" - "add %x[outptr8], %x[outptr8], %x[matrix_row_stride]\n" - - // Fourth row, simultaneously load the first column of inputs for the - // next tile. - "fneg U.4s, BXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - "ldr qx_13, [%x[inptr0]]\n" - - "fadd U.4s, BXTx42.4s, BXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "ldr qx_23, [%x[inptr1]]\n" - - "fsub U.4s, BXTx43.4s, BXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "ldr qx_33, [%x[inptr2]]\n" - - "fsub U.4s, BXTx42.4s, BXTx44.4s\n" - "str qU, [%x[outptr12], %x[mstride3]]\n" - "ldr qx_43, [%x[inptr3]]\n" - - "add %x[outptr12], %x[outptr12], %x[matrix_row_stride]\n" - - // Update the loop counter, subtract two to account for both the head and - // the tail. - "subs %x[tile_j], %x[tile_j], #2\n" - "beq 2f\n" // Jump to "A" tail if out of tiles - - // ---------------------------------------------------------------------- - "1:" - // Start part A - // Load last column of this tile (the first column has already been - // loaded) and compute latter two columns of X.T x. - "fsub AXTx13.4s, x_13.4s, x_33.4s\n" - "ldr qx_14, [%x[inptr0], %x[colstride1]]\n" - "fadd AXTx23.4s, x_23.4s, x_33.4s\n" - "ldr qx_24, [%x[inptr1], %x[colstride1]]\n" - "fsub AXTx33.4s, x_33.4s, x_23.4s\n" - "ldr qx_34, [%x[inptr2], %x[colstride1]]\n" - "fsub AXTx43.4s, x_23.4s, x_43.4s\n" - "ldr qx_44, [%x[inptr3], %x[colstride1]]\n" - "fsub AXTx14.4s, x_14.4s, x_34.4s\n" - "add %x[inptr0], %x[inptr0], %x[colstride2]\n" - "fadd AXTx24.4s, x_24.4s, x_34.4s\n" - "add %x[inptr1], %x[inptr1], %x[colstride2]\n" - "fsub AXTx34.4s, x_34.4s, x_24.4s\n" - "add %x[inptr2], %x[inptr2], %x[colstride2]\n" - "fsub AXTx44.4s, x_24.4s, x_44.4s\n" - "add %x[inptr3], %x[inptr3], %x[colstride2]\n" - - // Compute and store U. - // First row - "fsub U.4s, AXTx11.4s, AXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, AXTx12.4s, AXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, AXTx13.4s, AXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "fsub U.4s, AXTx12.4s, AXTx14.4s\n" - "str qU, [%x[outptr0], %x[mstride3]]\n" - "add %x[outptr0], %x[outptr0], %x[matrix_row_stride]\n" - - // Second row - "fsub U.4s, AXTx21.4s, AXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, AXTx22.4s, AXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, AXTx23.4s, AXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "fsub U.4s, AXTx22.4s, AXTx24.4s\n" - "str qU, [%x[outptr4], %x[mstride3]]\n" - "add %x[outptr4], %x[outptr4], %x[matrix_row_stride]\n" - - // Third row - "fsub U.4s, AXTx31.4s, AXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, AXTx32.4s, AXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, AXTx33.4s, AXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "fsub U.4s, AXTx32.4s, AXTx34.4s\n" - "str qU, [%x[outptr8], %x[mstride3]]\n" - "add %x[outptr8], %x[outptr8], %x[matrix_row_stride]\n" - - // Fourth row - "fsub U.4s, AXTx41.4s, AXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - "ldr qx_13, [%x[inptr0]]\n" - - "fadd U.4s, AXTx42.4s, AXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "ldr qx_23, [%x[inptr1]]\n" - - "fsub U.4s, AXTx43.4s, AXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "ldr qx_33, [%x[inptr2]]\n" - - "fsub U.4s, AXTx42.4s, AXTx44.4s\n" - "str qU, [%x[outptr12], %x[mstride3]]\n" - "ldr qx_43, [%x[inptr3]]\n" - - "add %x[outptr12], %x[outptr12], %x[matrix_row_stride]\n" - - "subs %x[tile_j], %x[tile_j], #1\n" - "beq 3f\n" // Jump to 'B' tail - - // Start part B - // Load last column of this tile (the first column has already been - // loaded) and compute latter two columns of X.T x. - "fsub BXTx13.4s, x_13.4s, x_33.4s\n" - "ldr qx_14, [%x[inptr0], %x[colstride1]]\n" - "fadd BXTx23.4s, x_23.4s, x_33.4s\n" - "ldr qx_24, [%x[inptr1], %x[colstride1]]\n" - "fsub BXTx33.4s, x_33.4s, x_23.4s\n" - "ldr qx_34, [%x[inptr2], %x[colstride1]]\n" - "fsub BXTx43.4s, x_23.4s, x_43.4s\n" - "ldr qx_44, [%x[inptr3], %x[colstride1]]\n" - "fsub BXTx14.4s, x_14.4s, x_34.4s\n" - "add %x[inptr0], %x[inptr0], %x[colstride2]\n" - "fadd BXTx24.4s, x_24.4s, x_34.4s\n" - "add %x[inptr1], %x[inptr1], %x[colstride2]\n" - "fsub BXTx34.4s, x_34.4s, x_24.4s\n" - "add %x[inptr2], %x[inptr2], %x[colstride2]\n" - "fsub BXTx44.4s, x_24.4s, x_44.4s\n" - "add %x[inptr3], %x[inptr3], %x[colstride2]\n" - - // Compute and store U. - // First row - "fsub U.4s, BXTx11.4s, BXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, BXTx12.4s, BXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, BXTx13.4s, BXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "fsub U.4s, BXTx12.4s, BXTx14.4s\n" - "str qU, [%x[outptr0], %x[mstride3]]\n" - "add %x[outptr0], %x[outptr0], %x[matrix_row_stride]\n" - - // Second row - "fsub U.4s, BXTx21.4s, BXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, BXTx22.4s, BXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, BXTx23.4s, BXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "fsub U.4s, BXTx22.4s, BXTx24.4s\n" - "str qU, [%x[outptr4], %x[mstride3]]\n" - "add %x[outptr4], %x[outptr4], %x[matrix_row_stride]\n" - - // Third row - "fsub U.4s, BXTx31.4s, BXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, BXTx32.4s, BXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, BXTx33.4s, BXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "fsub U.4s, BXTx32.4s, BXTx34.4s\n" - "str qU, [%x[outptr8], %x[mstride3]]\n" - "add %x[outptr8], %x[outptr8], %x[matrix_row_stride]\n" - - // Fourth row - "fsub U.4s, BXTx41.4s, BXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - "ldr qx_13, [%x[inptr0]]\n" - - "fadd U.4s, BXTx42.4s, BXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "ldr qx_23, [%x[inptr1]]\n" - - "fsub U.4s, BXTx43.4s, BXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "ldr qx_33, [%x[inptr2]]\n" - - "fsub U.4s, BXTx42.4s, BXTx44.4s\n" - "str qU, [%x[outptr12], %x[mstride3]]\n" - "ldr qx_43, [%x[inptr3]]\n" - - "add %x[outptr12], %x[outptr12], %x[matrix_row_stride]\n" - "subs %x[tile_j], %x[tile_j], #1\n" - "bne 1b\n" // Continue loop, otherwise flow into 'A' tail - - // ---------------------------------------------------------------------- - "2:" - // 'A' tail - // Since the final column is padding and the last-but-one column has - // already been loaded just compute the 3rd column of `X.T x'. - "fsub AXTx13.4s, x_13.4s, x_33.4s\n" - "fadd AXTx23.4s, x_23.4s, x_33.4s\n" - "fsub AXTx33.4s, x_33.4s, x_23.4s\n" - "fsub AXTx43.4s, x_23.4s, x_43.4s\n" - - // Compute and store U. Modified to account for the final column of X.T - // x containing padding. Note, it is also unnecessary to update the - // output pointers. - // First row - "fsub U.4s, AXTx11.4s, AXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, AXTx12.4s, AXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, AXTx13.4s, AXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "str qAXTx12, [%x[outptr0], %x[mstride3]]\n" - - // Second row - "fsub U.4s, AXTx21.4s, AXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, AXTx22.4s, AXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, AXTx23.4s, AXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "str qAXTx22, [%x[outptr4], %x[mstride3]]\n" - - // Third row - "fsub U.4s, AXTx31.4s, AXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, AXTx32.4s, AXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, AXTx33.4s, AXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "str qAXTx32, [%x[outptr8], %x[mstride3]]\n" - - // Fourth row - "fsub U.4s, AXTx41.4s, AXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - "fadd U.4s, AXTx42.4s, AXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "fsub U.4s, AXTx43.4s, AXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "str qAXTx42, [%x[outptr12], %x[mstride3]]\n" - - "b 4f\n" // Jump to end of function - - // ---------------------------------------------------------------------- - "3:" - // 'B' tail - // Since the final column is padding and the last-but-one column has - // already been loaded just compute the 3rd column of `X.T x'. - "fsub BXTx13.4s, x_13.4s, x_33.4s\n" - "fadd BXTx23.4s, x_23.4s, x_33.4s\n" - "fsub BXTx33.4s, x_33.4s, x_23.4s\n" - "fsub BXTx43.4s, x_23.4s, x_43.4s\n" - - // Compute and store U. Modified to account for the final column of X.T - // x containing padding. Note, it is also unnecessary to update the - // output pointers. - // First row - "fsub U.4s, BXTx11.4s, BXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, BXTx12.4s, BXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, BXTx13.4s, BXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "str qBXTx12, [%x[outptr0], %x[mstride3]]\n" - - // Second row - "fsub U.4s, BXTx21.4s, BXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, BXTx22.4s, BXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, BXTx23.4s, BXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "str qBXTx22, [%x[outptr4], %x[mstride3]]\n" - - // Third row - "fsub U.4s, BXTx31.4s, BXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, BXTx32.4s, BXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, BXTx33.4s, BXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "str qBXTx32, [%x[outptr8], %x[mstride3]]\n" - - // Fourth row - "fsub U.4s, BXTx41.4s, BXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - "fadd U.4s, BXTx42.4s, BXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "fsub U.4s, BXTx43.4s, BXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "str qBXTx42, [%x[outptr12], %x[mstride3]]\n" - - // ---------------------------------------------------------------------- - "4:" - // End of function - - // Clear names - ".unreq x_13\n" ".unreq qx_13\n" ".unreq x_14\n" ".unreq qx_14\n" - ".unreq x_23\n" ".unreq qx_23\n" ".unreq x_24\n" ".unreq qx_24\n" - ".unreq x_33\n" ".unreq qx_33\n" ".unreq x_34\n" ".unreq qx_34\n" - ".unreq x_43\n" ".unreq qx_43\n" ".unreq x_44\n" ".unreq qx_44\n" - ".unreq AXTx11\n" ".unreq BXTx13\n" - ".unreq AXTx12\n" ".unreq BXTx14\n" ".unreq qAXTx12\n" - ".unreq AXTx21\n" ".unreq BXTx23\n" - ".unreq AXTx22\n" ".unreq BXTx24\n" ".unreq qAXTx22\n" - ".unreq AXTx31\n" ".unreq BXTx33\n" - ".unreq AXTx32\n" ".unreq BXTx34\n" ".unreq qAXTx32\n" - ".unreq AXTx41\n" ".unreq BXTx43\n" - ".unreq AXTx42\n" ".unreq BXTx44\n" ".unreq qAXTx42\n" - ".unreq AXTx13\n" ".unreq BXTx11\n" - ".unreq AXTx14\n" ".unreq BXTx12\n" ".unreq qBXTx12\n" - ".unreq AXTx23\n" ".unreq BXTx21\n" - ".unreq AXTx24\n" ".unreq BXTx22\n" ".unreq qBXTx22\n" - ".unreq AXTx33\n" ".unreq BXTx31\n" - ".unreq AXTx34\n" ".unreq BXTx32\n" ".unreq qBXTx32\n" - ".unreq AXTx43\n" ".unreq BXTx41\n" - ".unreq AXTx44\n" ".unreq BXTx42\n" ".unreq qBXTx42\n" - ".unreq U\n" ".unreq qU\n" - : [inptr0] "+r" (inptr0), - [inptr1] "+r" (inptr1), - [inptr2] "+r" (inptr2), - [inptr3] "+r" (inptr3), - [outptr0] "+r" (outptr0), - [outptr4] "+r" (outptr4), - [outptr8] "+r" (outptr8), - [outptr12] "+r" (outptr12), - [tile_j] "+r" (tile_j) // Tile counter - : [colstride1] "r" (1 * input_col_stride * sizeof(float)), - [colstride2] "r" (2 * input_col_stride * sizeof(float)), - [colstride3] "r" (3 * input_col_stride * sizeof(float)), - [mstride1] "r" (1 * matrix_stride * sizeof(float)), - [mstride2] "r" (2 * matrix_stride * sizeof(float)), - [mstride3] "r" (3 * matrix_stride * sizeof(float)), - [matrix_row_stride] "r" (matrix_row_stride * sizeof(float)) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", - "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", - "v22", "v23", "v24" - ); -} - -// Pad top, left and right by 1. -template <> -template <> -inline void Winograd2x2_3x3GemmInput<float>::process_tile_row<1, 1, 0, 1, 4>( - const int tile_N, - const float* const input, - const int input_row_stride, - const int input_col_stride, - float* const matrix, - const int matrix_stride, - const int matrix_row_stride -) { - const float *inptr0 = input; - const float *inptr1 = input + input_row_stride; - const float *inptr2 = input + input_row_stride * 2; - - float *outptr0 = matrix; - float *outptr4 = matrix + matrix_stride * 4; - float *outptr8 = matrix + matrix_stride * 8; - float *outptr12 = matrix + matrix_stride * 12; - - int tile_j = tile_N; // Tiles to process - - asm volatile ( - // Named SIMD registers according to the policy given above - // Registers into which to load the latter two columns of `x` - // NOTE: We need only load the latter three rows since we know that the - // first row is padded. - "x_23 .req v1\n qx_23 .req q1\n" "x_24 .req v5\n qx_24 .req q5\n" - "x_33 .req v2\n qx_33 .req q2\n" "x_34 .req v6\n qx_34 .req q6\n" - "x_43 .req v3\n qx_43 .req q3\n" "x_44 .req v7\n qx_44 .req q7\n" - - // Registers for storing X.T x (both A and B halves) - "AXTx11 .req v8\n" "BXTx13 .req v8\n" - "AXTx12 .req v9\n" "BXTx14 .req v9\n" "qAXTx12 .req q9\n" - "AXTx21 .req v10\n" "BXTx23 .req v10\n" - "AXTx22 .req v11\n" "BXTx24 .req v11\n" "qAXTx22 .req q11\n" - "AXTx31 .req v12\n" "BXTx33 .req v12\n" - "AXTx32 .req v13\n" "BXTx34 .req v13\n" "qAXTx32 .req q13\n" - "AXTx41 .req v14\n" "BXTx43 .req v14\n" - "AXTx42 .req v15\n" "BXTx44 .req v15\n" "qAXTx42 .req q15\n" - "AXTx13 .req v16\n" "BXTx11 .req v16\n" - "AXTx14 .req v17\n" "BXTx12 .req v17\n" "qBXTx12 .req q17\n" - "AXTx23 .req v18\n" "BXTx21 .req v18\n" - "AXTx24 .req v19\n" "BXTx22 .req v19\n" "qBXTx22 .req q19\n" - "AXTx33 .req v20\n" "BXTx31 .req v20\n" - "AXTx34 .req v21\n" "BXTx32 .req v21\n" "qBXTx32 .req q21\n" - "AXTx43 .req v22\n" "BXTx41 .req v22\n" - "AXTx44 .req v23\n" "BXTx42 .req v23\n" "qBXTx42 .req q23\n" - - // Result register (TODO Does using more registers yield better - // performance) - "U .req v24\n qU .req q24\n" - - // ---------------------------------------------------------------------- - // Head of loop - // Loads a complete 4x4 tile of x, computes X.T x, computes and stores - // `U = X.T x X`. Prepares for the 'A' half of the loop. - // NOTE: Since the first tile has the leftmost column padded we can - // skip 4 loads and 4 calculations for the matrix X.T x X. - - // Temporarily alias registers for computing the first (non-padded) - // column of x. - "x_22 .req v1\n qx_22 .req q1\n" - "x_32 .req v2\n qx_32 .req q2\n" - "x_42 .req v3\n qx_42 .req q3\n" - - "ldr qx_22, [%x[inptr1]]\n" - "ldr qx_32, [%x[inptr2]]\n" - "ldr qx_42, [%x[inptr3]]\n" - - "fneg BXTx12.4s, x_32.4s\n" - "fadd BXTx22.4s, x_22.4s, x_32.4s\n" - "fsub BXTx32.4s, x_32.4s, x_22.4s\n" - "fsub BXTx42.4s, x_22.4s, x_42.4s\n" - - ".unreq x_22\n .unreq qx_22\n" - ".unreq x_32\n .unreq qx_32\n" - ".unreq x_42\n .unreq qx_42\n" - - // Load and compute latter two columns of the first tile. Progress the - // input pointers (by three columns so that the each points are the - // second column of the next tile, that is, each points at the first - // column which must be read for the next tile. - "ldr qx_23, [%x[inptr1], %x[colstride1]]\n" - "ldr qx_33, [%x[inptr2], %x[colstride1]]\n" - "ldr qx_43, [%x[inptr3], %x[colstride1]]\n" - - "fneg BXTx13.4s, x_33.4s\n" - - "fadd BXTx23.4s, x_23.4s, x_33.4s\n" - "ldr qx_24, [%x[inptr1], %x[colstride2]]\n" - - "fsub BXTx33.4s, x_33.4s, x_23.4s\n" - "ldr qx_34, [%x[inptr2], %x[colstride2]]\n" - - "fsub BXTx43.4s, x_23.4s, x_43.4s\n" - "ldr qx_44, [%x[inptr3], %x[colstride2]]\n" - - "fneg BXTx14.4s, x_34.4s\n" - - "fadd BXTx24.4s, x_24.4s, x_34.4s\n" - "add %x[inptr1], %x[inptr1], %x[colstride3]\n" - - "fsub BXTx34.4s, x_34.4s, x_24.4s\n" - "add %x[inptr2], %x[inptr2], %x[colstride3]\n" - - "fsub BXTx44.4s, x_24.4s, x_44.4s\n" - "add %x[inptr3], %x[inptr3], %x[colstride3]\n" - - // Compute and store U for the first tile - // First row - "fneg U.4s, BXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, BXTx12.4s, BXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, BXTx13.4s, BXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "fsub U.4s, BXTx12.4s, BXTx14.4s\n" - "str qU, [%x[outptr0], %x[mstride3]]\n" - "add %x[outptr0], %x[outptr0], %x[matrix_row_stride]\n" - - // Second row - "fneg U.4s, BXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, BXTx22.4s, BXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, BXTx23.4s, BXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "fsub U.4s, BXTx22.4s, BXTx24.4s\n" - "str qU, [%x[outptr4], %x[mstride3]]\n" - "add %x[outptr4], %x[outptr4], %x[matrix_row_stride]\n" - - // Third row - "fneg U.4s, BXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, BXTx32.4s, BXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, BXTx33.4s, BXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "fsub U.4s, BXTx32.4s, BXTx34.4s\n" - "str qU, [%x[outptr8], %x[mstride3]]\n" - "add %x[outptr8], %x[outptr8], %x[matrix_row_stride]\n" - - // Fourth row, simultaneously load the first column of inputs for the - // next tile. - "fneg U.4s, BXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - - "fadd U.4s, BXTx42.4s, BXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "ldr qx_23, [%x[inptr1]]\n" - - "fsub U.4s, BXTx43.4s, BXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "ldr qx_33, [%x[inptr2]]\n" - - "fsub U.4s, BXTx42.4s, BXTx44.4s\n" - "str qU, [%x[outptr12], %x[mstride3]]\n" - "ldr qx_43, [%x[inptr3]]\n" - - "add %x[outptr12], %x[outptr12], %x[matrix_row_stride]\n" - - // Update the loop counter, subtract two to account for both the head and - // the tail. - "subs %x[tile_j], %x[tile_j], #2\n" - "beq 2f\n" // Jump to "A" tail if out of tiles - - // ---------------------------------------------------------------------- - "1:" - // Start part A - // Load last column of this tile (the first column has already been - // loaded) and compute latter two columns of X.T x. - "fneg AXTx13.4s, x_33.4s\n" - "fadd AXTx23.4s, x_23.4s, x_33.4s\n" - "ldr qx_24, [%x[inptr1], %x[colstride1]]\n" - "fsub AXTx33.4s, x_33.4s, x_23.4s\n" - "ldr qx_34, [%x[inptr2], %x[colstride1]]\n" - "fsub AXTx43.4s, x_23.4s, x_43.4s\n" - "ldr qx_44, [%x[inptr3], %x[colstride1]]\n" - "fneg AXTx14.4s, x_34.4s\n" - "fadd AXTx24.4s, x_24.4s, x_34.4s\n" - "add %x[inptr1], %x[inptr1], %x[colstride2]\n" - "fsub AXTx34.4s, x_34.4s, x_24.4s\n" - "add %x[inptr2], %x[inptr2], %x[colstride2]\n" - "fsub AXTx44.4s, x_24.4s, x_44.4s\n" - "add %x[inptr3], %x[inptr3], %x[colstride2]\n" - - // Compute and store U. - // First row - "fsub U.4s, AXTx11.4s, AXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, AXTx12.4s, AXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, AXTx13.4s, AXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "fsub U.4s, AXTx12.4s, AXTx14.4s\n" - "str qU, [%x[outptr0], %x[mstride3]]\n" - "add %x[outptr0], %x[outptr0], %x[matrix_row_stride]\n" - - // Second row - "fsub U.4s, AXTx21.4s, AXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, AXTx22.4s, AXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, AXTx23.4s, AXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "fsub U.4s, AXTx22.4s, AXTx24.4s\n" - "str qU, [%x[outptr4], %x[mstride3]]\n" - "add %x[outptr4], %x[outptr4], %x[matrix_row_stride]\n" - - // Third row - "fsub U.4s, AXTx31.4s, AXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, AXTx32.4s, AXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, AXTx33.4s, AXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "fsub U.4s, AXTx32.4s, AXTx34.4s\n" - "str qU, [%x[outptr8], %x[mstride3]]\n" - "add %x[outptr8], %x[outptr8], %x[matrix_row_stride]\n" - - // Fourth row - "fsub U.4s, AXTx41.4s, AXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - - "fadd U.4s, AXTx42.4s, AXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "ldr qx_23, [%x[inptr1]]\n" - - "fsub U.4s, AXTx43.4s, AXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "ldr qx_33, [%x[inptr2]]\n" - - "fsub U.4s, AXTx42.4s, AXTx44.4s\n" - "str qU, [%x[outptr12], %x[mstride3]]\n" - "ldr qx_43, [%x[inptr3]]\n" - - "add %x[outptr12], %x[outptr12], %x[matrix_row_stride]\n" - - "subs %x[tile_j], %x[tile_j], #1\n" - "beq 3f\n" // Jump to 'B' tail - - // Start part B - // Load last column of this tile (the first column has already been - // loaded) and compute latter two columns of X.T x. - "fneg BXTx13.4s, x_33.4s\n" - "fadd BXTx23.4s, x_23.4s, x_33.4s\n" - "ldr qx_24, [%x[inptr1], %x[colstride1]]\n" - "fsub BXTx33.4s, x_33.4s, x_23.4s\n" - "ldr qx_34, [%x[inptr2], %x[colstride1]]\n" - "fsub BXTx43.4s, x_23.4s, x_43.4s\n" - "ldr qx_44, [%x[inptr3], %x[colstride1]]\n" - "fneg BXTx14.4s, x_34.4s\n" - "fadd BXTx24.4s, x_24.4s, x_34.4s\n" - "add %x[inptr1], %x[inptr1], %x[colstride2]\n" - "fsub BXTx34.4s, x_34.4s, x_24.4s\n" - "add %x[inptr2], %x[inptr2], %x[colstride2]\n" - "fsub BXTx44.4s, x_24.4s, x_44.4s\n" - "add %x[inptr3], %x[inptr3], %x[colstride2]\n" - - // Compute and store U. - // First row - "fsub U.4s, BXTx11.4s, BXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, BXTx12.4s, BXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, BXTx13.4s, BXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "fsub U.4s, BXTx12.4s, BXTx14.4s\n" - "str qU, [%x[outptr0], %x[mstride3]]\n" - "add %x[outptr0], %x[outptr0], %x[matrix_row_stride]\n" - - // Second row - "fsub U.4s, BXTx21.4s, BXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, BXTx22.4s, BXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, BXTx23.4s, BXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "fsub U.4s, BXTx22.4s, BXTx24.4s\n" - "str qU, [%x[outptr4], %x[mstride3]]\n" - "add %x[outptr4], %x[outptr4], %x[matrix_row_stride]\n" - - // Third row - "fsub U.4s, BXTx31.4s, BXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, BXTx32.4s, BXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, BXTx33.4s, BXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "fsub U.4s, BXTx32.4s, BXTx34.4s\n" - "str qU, [%x[outptr8], %x[mstride3]]\n" - "add %x[outptr8], %x[outptr8], %x[matrix_row_stride]\n" - - // Fourth row - "fsub U.4s, BXTx41.4s, BXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - - "fadd U.4s, BXTx42.4s, BXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "ldr qx_23, [%x[inptr1]]\n" - - "fsub U.4s, BXTx43.4s, BXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "ldr qx_33, [%x[inptr2]]\n" - - "fsub U.4s, BXTx42.4s, BXTx44.4s\n" - "str qU, [%x[outptr12], %x[mstride3]]\n" - "ldr qx_43, [%x[inptr3]]\n" - - "add %x[outptr12], %x[outptr12], %x[matrix_row_stride]\n" - "subs %x[tile_j], %x[tile_j], #1\n" - "bne 1b\n" // Continue loop, otherwise flow into 'A' tail - - // ---------------------------------------------------------------------- - "2:" - // 'A' tail - // Since the final column is padding and the last-but-one column has - // already been loaded just compute the 3rd column of `X.T x'. - "fneg AXTx13.4s, x_33.4s\n" - "fadd AXTx23.4s, x_23.4s, x_33.4s\n" - "fsub AXTx33.4s, x_33.4s, x_23.4s\n" - "fsub AXTx43.4s, x_23.4s, x_43.4s\n" - - // Compute and store U. Modified to account for the final column of X.T - // x containing padding. Note, it is also unnecessary to update the - // output pointers. - // First row - "fsub U.4s, AXTx11.4s, AXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, AXTx12.4s, AXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, AXTx13.4s, AXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "str qAXTx12, [%x[outptr0], %x[mstride3]]\n" - - // Second row - "fsub U.4s, AXTx21.4s, AXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, AXTx22.4s, AXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, AXTx23.4s, AXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "str qAXTx22, [%x[outptr4], %x[mstride3]]\n" - - // Third row - "fsub U.4s, AXTx31.4s, AXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, AXTx32.4s, AXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, AXTx33.4s, AXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "str qAXTx32, [%x[outptr8], %x[mstride3]]\n" - - // Fourth row - "fsub U.4s, AXTx41.4s, AXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - "fadd U.4s, AXTx42.4s, AXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "fsub U.4s, AXTx43.4s, AXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "str qAXTx42, [%x[outptr12], %x[mstride3]]\n" - - "b 4f\n" // Jump to end of function - - // ---------------------------------------------------------------------- - "3:" - // 'B' tail - // Since the final column is padding and the last-but-one column has - // already been loaded just compute the 3rd column of `X.T x'. - "fneg BXTx13.4s, x_33.4s\n" - "fadd BXTx23.4s, x_23.4s, x_33.4s\n" - "fsub BXTx33.4s, x_33.4s, x_23.4s\n" - "fsub BXTx43.4s, x_23.4s, x_43.4s\n" - - // Compute and store U. Modified to account for the final column of X.T - // x containing padding. Note, it is also unnecessary to update the - // output pointers. - // First row - "fsub U.4s, BXTx11.4s, BXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, BXTx12.4s, BXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, BXTx13.4s, BXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "str qBXTx12, [%x[outptr0], %x[mstride3]]\n" - - // Second row - "fsub U.4s, BXTx21.4s, BXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, BXTx22.4s, BXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, BXTx23.4s, BXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "str qBXTx22, [%x[outptr4], %x[mstride3]]\n" - - // Third row - "fsub U.4s, BXTx31.4s, BXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, BXTx32.4s, BXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, BXTx33.4s, BXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "str qBXTx32, [%x[outptr8], %x[mstride3]]\n" - - // Fourth row - "fsub U.4s, BXTx41.4s, BXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - "fadd U.4s, BXTx42.4s, BXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "fsub U.4s, BXTx43.4s, BXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "str qBXTx42, [%x[outptr12], %x[mstride3]]\n" - - // ---------------------------------------------------------------------- - "4:" - // End of function - - // Clear names - ".unreq x_23\n" ".unreq qx_23\n" ".unreq x_24\n" ".unreq qx_24\n" - ".unreq x_33\n" ".unreq qx_33\n" ".unreq x_34\n" ".unreq qx_34\n" - ".unreq x_43\n" ".unreq qx_43\n" ".unreq x_44\n" ".unreq qx_44\n" - ".unreq AXTx11\n" ".unreq BXTx13\n" - ".unreq AXTx12\n" ".unreq BXTx14\n" ".unreq qAXTx12\n" - ".unreq AXTx21\n" ".unreq BXTx23\n" - ".unreq AXTx22\n" ".unreq BXTx24\n" ".unreq qAXTx22\n" - ".unreq AXTx31\n" ".unreq BXTx33\n" - ".unreq AXTx32\n" ".unreq BXTx34\n" ".unreq qAXTx32\n" - ".unreq AXTx41\n" ".unreq BXTx43\n" - ".unreq AXTx42\n" ".unreq BXTx44\n" ".unreq qAXTx42\n" - ".unreq AXTx13\n" ".unreq BXTx11\n" - ".unreq AXTx14\n" ".unreq BXTx12\n" ".unreq qBXTx12\n" - ".unreq AXTx23\n" ".unreq BXTx21\n" - ".unreq AXTx24\n" ".unreq BXTx22\n" ".unreq qBXTx22\n" - ".unreq AXTx33\n" ".unreq BXTx31\n" - ".unreq AXTx34\n" ".unreq BXTx32\n" ".unreq qBXTx32\n" - ".unreq AXTx43\n" ".unreq BXTx41\n" - ".unreq AXTx44\n" ".unreq BXTx42\n" ".unreq qBXTx42\n" - ".unreq U\n" ".unreq qU\n" - : [inptr1] "+r" (inptr0), // Offset to account for padded row - [inptr2] "+r" (inptr1), // Offset to account for padded row - [inptr3] "+r" (inptr2), // Offset to account for padded row - [outptr0] "+r" (outptr0), - [outptr4] "+r" (outptr4), - [outptr8] "+r" (outptr8), - [outptr12] "+r" (outptr12), - [tile_j] "+r" (tile_j) // Tile counter - : [colstride1] "r" (1 * input_col_stride * sizeof(float)), - [colstride2] "r" (2 * input_col_stride * sizeof(float)), - [colstride3] "r" (3 * input_col_stride * sizeof(float)), - [mstride1] "r" (1 * matrix_stride * sizeof(float)), - [mstride2] "r" (2 * matrix_stride * sizeof(float)), - [mstride3] "r" (3 * matrix_stride * sizeof(float)), - [matrix_row_stride] "r" (matrix_row_stride * sizeof(float)) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", - "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", - "v22", "v23", "v24" - ); -} - -// Pad left, right and bottom by 1. -template <> -template <> -inline void Winograd2x2_3x3GemmInput<float>::process_tile_row<0, 1, 1, 1, 4>( - const int tile_N, - const float* const input, - const int input_row_stride, - const int input_col_stride, - float* const matrix, - const int matrix_stride, - const int matrix_row_stride -) { - const float *inptr0 = input; - const float *inptr1 = input + input_row_stride; - const float *inptr2 = input + input_row_stride * 2; - - float *outptr0 = matrix; - float *outptr4 = matrix + matrix_stride * 4; - float *outptr8 = matrix + matrix_stride * 8; - float *outptr12 = matrix + matrix_stride * 12; - - int tile_j = tile_N; // Tiles to process - - asm volatile ( - // Named SIMD registers according to the policy given above - // Registers into which to load the latter two columns of `x` - // NOTE: Bottom row is not required since since it is padded. - "x_13 .req v0\n qx_13 .req q0\n" "x_14 .req v4\n qx_14 .req q4\n" - "x_23 .req v1\n qx_23 .req q1\n" "x_24 .req v5\n qx_24 .req q5\n" - "x_33 .req v2\n qx_33 .req q2\n" "x_34 .req v6\n qx_34 .req q6\n" - - // Registers for storing X.T x (both A and B halves) - "AXTx11 .req v8\n" "BXTx13 .req v8\n" - "AXTx12 .req v9\n" "BXTx14 .req v9\n" "qAXTx12 .req q9\n" - "AXTx21 .req v10\n" "BXTx23 .req v10\n" - "AXTx22 .req v11\n" "BXTx24 .req v11\n" "qAXTx22 .req q11\n" - "AXTx31 .req v12\n" "BXTx33 .req v12\n" - "AXTx32 .req v13\n" "BXTx34 .req v13\n" "qAXTx32 .req q13\n" - "AXTx41 .req v14\n" "BXTx43 .req v14\n" - "AXTx42 .req v15\n" "BXTx44 .req v15\n" "qAXTx42 .req q15\n" - "AXTx13 .req v16\n" "BXTx11 .req v16\n" - "AXTx14 .req v17\n" "BXTx12 .req v17\n" "qBXTx12 .req q17\n" - "AXTx23 .req v18\n" "BXTx21 .req v18\n" - "AXTx24 .req v19\n" "BXTx22 .req v19\n" "qBXTx22 .req q19\n" - "AXTx33 .req v20\n" "BXTx31 .req v20\n" - "AXTx34 .req v21\n" "BXTx32 .req v21\n" "qBXTx32 .req q21\n" - "AXTx43 .req v22\n" "BXTx41 .req v22\n" - "AXTx44 .req v23\n" "BXTx42 .req v23\n" "qBXTx42 .req q23\n" - - // Result register (TODO Does using more registers yield better - // performance) - "U .req v24\n qU .req q24\n" - - // ---------------------------------------------------------------------- - // Head of loop - // Loads a complete 4x4 tile of x, computes X.T x, computes and stores - // `U = X.T x X`. Prepares for the 'A' half of the loop. - // NOTE: Since the first tile has the leftmost column padded we can - // skip 4 loads and 4 calculations for the matrix X.T x X. - - // Temporarily alias registers for computing the first (non-padded) - // column of x. - "x_12 .req v0\n qx_12 .req q0\n" - "x_22 .req v1\n qx_22 .req q1\n" - "x_32 .req v2\n qx_32 .req q2\n" - - "ldr qx_12, [%x[inptr0]]\n" - "ldr qx_22, [%x[inptr1]]\n" - "ldr qx_32, [%x[inptr2]]\n" - - "fsub BXTx12.4s, x_12.4s, x_32.4s\n" - "fadd BXTx22.4s, x_22.4s, x_32.4s\n" - "fsub BXTx32.4s, x_32.4s, x_22.4s\n" - "mov BXTx42.16b, x_22.16b\n" // Probably should do better - - ".unreq x_12\n .unreq qx_12\n" - ".unreq x_22\n .unreq qx_22\n" - ".unreq x_32\n .unreq qx_32\n" - - // Load and compute latter two columns of the first tile. Progress the - // input pointers (by three columns so that the each points are the - // second column of the next tile, that is, each points at the first - // column which must be read for the next tile. - "ldr qx_13, [%x[inptr0], %x[colstride1]]\n" - "ldr qx_23, [%x[inptr1], %x[colstride1]]\n" - "ldr qx_33, [%x[inptr2], %x[colstride1]]\n" - - "fsub BXTx13.4s, x_13.4s, x_33.4s\n" - "ldr qx_14, [%x[inptr0], %x[colstride2]]\n" - - "fadd BXTx23.4s, x_23.4s, x_33.4s\n" - "ldr qx_24, [%x[inptr1], %x[colstride2]]\n" - - "fsub BXTx33.4s, x_33.4s, x_23.4s\n" - "ldr qx_34, [%x[inptr2], %x[colstride2]]\n" - - "mov BXTx43.16b, x_23.16b\n" - "fsub BXTx14.4s, x_14.4s, x_34.4s\n" - "add %x[inptr0], %x[inptr0], %x[colstride3]\n" - - "fadd BXTx24.4s, x_24.4s, x_34.4s\n" - "add %x[inptr1], %x[inptr1], %x[colstride3]\n" - - "fsub BXTx34.4s, x_34.4s, x_24.4s\n" - "add %x[inptr2], %x[inptr2], %x[colstride3]\n" - - "mov BXTx44.16b, x_24.16b\n" - - // Compute and store U for the first tile - // First row - "fneg U.4s, BXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, BXTx12.4s, BXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, BXTx13.4s, BXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "fsub U.4s, BXTx12.4s, BXTx14.4s\n" - "str qU, [%x[outptr0], %x[mstride3]]\n" - "add %x[outptr0], %x[outptr0], %x[matrix_row_stride]\n" - - // Second row - "fneg U.4s, BXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, BXTx22.4s, BXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, BXTx23.4s, BXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "fsub U.4s, BXTx22.4s, BXTx24.4s\n" - "str qU, [%x[outptr4], %x[mstride3]]\n" - "add %x[outptr4], %x[outptr4], %x[matrix_row_stride]\n" - - // Third row - "fneg U.4s, BXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, BXTx32.4s, BXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, BXTx33.4s, BXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "fsub U.4s, BXTx32.4s, BXTx34.4s\n" - "str qU, [%x[outptr8], %x[mstride3]]\n" - "add %x[outptr8], %x[outptr8], %x[matrix_row_stride]\n" - - // Fourth row, simultaneously load the first column of inputs for the - // next tile. - "fneg U.4s, BXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - "ldr qx_13, [%x[inptr0]]\n" - - "fadd U.4s, BXTx42.4s, BXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "ldr qx_23, [%x[inptr1]]\n" - - "fsub U.4s, BXTx43.4s, BXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "ldr qx_33, [%x[inptr2]]\n" - - "fsub U.4s, BXTx42.4s, BXTx44.4s\n" - "str qU, [%x[outptr12], %x[mstride3]]\n" - - "add %x[outptr12], %x[outptr12], %x[matrix_row_stride]\n" - - // Update the loop counter, subtract two to account for both the head and - // the tail. - "subs %x[tile_j], %x[tile_j], #2\n" - "beq 2f\n" // Jump to "A" tail if out of tiles - - // ---------------------------------------------------------------------- - "1:" - // Start part A - // Load last column of this tile (the first column has already been - // loaded) and compute latter two columns of X.T x. - "fsub AXTx13.4s, x_13.4s, x_33.4s\n" - "ldr qx_14, [%x[inptr0], %x[colstride1]]\n" - "fadd AXTx23.4s, x_23.4s, x_33.4s\n" - "ldr qx_24, [%x[inptr1], %x[colstride1]]\n" - "fsub AXTx33.4s, x_33.4s, x_23.4s\n" - "ldr qx_34, [%x[inptr2], %x[colstride1]]\n" - "mov AXTx43.16b, x_23.16b\n" - - "fsub AXTx14.4s, x_14.4s, x_34.4s\n" - "add %x[inptr0], %x[inptr0], %x[colstride2]\n" - "fadd AXTx24.4s, x_24.4s, x_34.4s\n" - "add %x[inptr1], %x[inptr1], %x[colstride2]\n" - "fsub AXTx34.4s, x_34.4s, x_24.4s\n" - "add %x[inptr2], %x[inptr2], %x[colstride2]\n" - "mov AXTx44.16b, x_24.16b\n" - - // Compute and store U. - // First row - "fsub U.4s, AXTx11.4s, AXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, AXTx12.4s, AXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, AXTx13.4s, AXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "fsub U.4s, AXTx12.4s, AXTx14.4s\n" - "str qU, [%x[outptr0], %x[mstride3]]\n" - "add %x[outptr0], %x[outptr0], %x[matrix_row_stride]\n" - - // Second row - "fsub U.4s, AXTx21.4s, AXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, AXTx22.4s, AXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, AXTx23.4s, AXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "fsub U.4s, AXTx22.4s, AXTx24.4s\n" - "str qU, [%x[outptr4], %x[mstride3]]\n" - "add %x[outptr4], %x[outptr4], %x[matrix_row_stride]\n" - - // Third row - "fsub U.4s, AXTx31.4s, AXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, AXTx32.4s, AXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, AXTx33.4s, AXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "fsub U.4s, AXTx32.4s, AXTx34.4s\n" - "str qU, [%x[outptr8], %x[mstride3]]\n" - "add %x[outptr8], %x[outptr8], %x[matrix_row_stride]\n" - - // Fourth row - "fsub U.4s, AXTx41.4s, AXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - "ldr qx_13, [%x[inptr0]]\n" - - "fadd U.4s, AXTx42.4s, AXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "ldr qx_23, [%x[inptr1]]\n" - - "fsub U.4s, AXTx43.4s, AXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "ldr qx_33, [%x[inptr2]]\n" - - "fsub U.4s, AXTx42.4s, AXTx44.4s\n" - "str qU, [%x[outptr12], %x[mstride3]]\n" - - "add %x[outptr12], %x[outptr12], %x[matrix_row_stride]\n" - - "subs %x[tile_j], %x[tile_j], #1\n" - "beq 3f\n" // Jump to 'B' tail - - // Start part B - // Load last column of this tile (the first column has already been - // loaded) and compute latter two columns of X.T x. - "fsub BXTx13.4s, x_13.4s, x_33.4s\n" - "ldr qx_14, [%x[inptr0], %x[colstride1]]\n" - "fadd BXTx23.4s, x_23.4s, x_33.4s\n" - "ldr qx_24, [%x[inptr1], %x[colstride1]]\n" - "fsub BXTx33.4s, x_33.4s, x_23.4s\n" - "ldr qx_34, [%x[inptr2], %x[colstride1]]\n" - "mov BXTx43.16b, x_23.16b\n" - - "fsub BXTx14.4s, x_14.4s, x_34.4s\n" - "add %x[inptr0], %x[inptr0], %x[colstride2]\n" - "fadd BXTx24.4s, x_24.4s, x_34.4s\n" - "add %x[inptr1], %x[inptr1], %x[colstride2]\n" - "fsub BXTx34.4s, x_34.4s, x_24.4s\n" - "add %x[inptr2], %x[inptr2], %x[colstride2]\n" - "mov BXTx44.16b, x_24.16b\n" - - // Compute and store U. - // First row - "fsub U.4s, BXTx11.4s, BXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, BXTx12.4s, BXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, BXTx13.4s, BXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "fsub U.4s, BXTx12.4s, BXTx14.4s\n" - "str qU, [%x[outptr0], %x[mstride3]]\n" - "add %x[outptr0], %x[outptr0], %x[matrix_row_stride]\n" - - // Second row - "fsub U.4s, BXTx21.4s, BXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, BXTx22.4s, BXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, BXTx23.4s, BXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "fsub U.4s, BXTx22.4s, BXTx24.4s\n" - "str qU, [%x[outptr4], %x[mstride3]]\n" - "add %x[outptr4], %x[outptr4], %x[matrix_row_stride]\n" - - // Third row - "fsub U.4s, BXTx31.4s, BXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, BXTx32.4s, BXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, BXTx33.4s, BXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "fsub U.4s, BXTx32.4s, BXTx34.4s\n" - "str qU, [%x[outptr8], %x[mstride3]]\n" - "add %x[outptr8], %x[outptr8], %x[matrix_row_stride]\n" - - // Fourth row - "fsub U.4s, BXTx41.4s, BXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - "ldr qx_13, [%x[inptr0]]\n" - - "fadd U.4s, BXTx42.4s, BXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "ldr qx_23, [%x[inptr1]]\n" - - "fsub U.4s, BXTx43.4s, BXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "ldr qx_33, [%x[inptr2]]\n" - - "fsub U.4s, BXTx42.4s, BXTx44.4s\n" - "str qU, [%x[outptr12], %x[mstride3]]\n" - - "add %x[outptr12], %x[outptr12], %x[matrix_row_stride]\n" - "subs %x[tile_j], %x[tile_j], #1\n" - "bne 1b\n" // Continue loop, otherwise flow into 'A' tail - - // ---------------------------------------------------------------------- - "2:" - // 'A' tail - // Since the final column is padding and the last-but-one column has - // already been loaded just compute the 3rd column of `X.T x'. - "fsub AXTx13.4s, x_13.4s, x_33.4s\n" - "fadd AXTx23.4s, x_23.4s, x_33.4s\n" - "fsub AXTx33.4s, x_33.4s, x_23.4s\n" - "mov AXTx43.16b, x_23.16b\n" - - // Compute and store U. Modified to account for the final column of X.T - // x containing padding. Note, it is also unnecessary to update the - // output pointers. - // First row - "fsub U.4s, AXTx11.4s, AXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, AXTx12.4s, AXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, AXTx13.4s, AXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "str qAXTx12, [%x[outptr0], %x[mstride3]]\n" - - // Second row - "fsub U.4s, AXTx21.4s, AXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, AXTx22.4s, AXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, AXTx23.4s, AXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "str qAXTx22, [%x[outptr4], %x[mstride3]]\n" - - // Third row - "fsub U.4s, AXTx31.4s, AXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, AXTx32.4s, AXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, AXTx33.4s, AXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "str qAXTx32, [%x[outptr8], %x[mstride3]]\n" - - // Fourth row - "fsub U.4s, AXTx41.4s, AXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - "fadd U.4s, AXTx42.4s, AXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "fsub U.4s, AXTx43.4s, AXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "str qAXTx42, [%x[outptr12], %x[mstride3]]\n" - - "b 4f\n" // Jump to end of function - - // ---------------------------------------------------------------------- - "3:" - // 'B' tail - // Since the final column is padding and the last-but-one column has - // already been loaded just compute the 3rd column of `X.T x'. - "fsub BXTx13.4s, x_13.4s, x_33.4s\n" - "fadd BXTx23.4s, x_23.4s, x_33.4s\n" - "fsub BXTx33.4s, x_33.4s, x_23.4s\n" - "mov BXTx43.16b, x_23.16b\n" - - // Compute and store U. Modified to account for the final column of X.T - // x containing padding. Note, it is also unnecessary to update the - // output pointers. - // First row - "fsub U.4s, BXTx11.4s, BXTx13.4s\n" - "str qU, [%x[outptr0]]\n" - "fadd U.4s, BXTx12.4s, BXTx13.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, BXTx13.4s, BXTx12.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "str qBXTx12, [%x[outptr0], %x[mstride3]]\n" - - // Second row - "fsub U.4s, BXTx21.4s, BXTx23.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, BXTx22.4s, BXTx23.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fsub U.4s, BXTx23.4s, BXTx22.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "str qBXTx22, [%x[outptr4], %x[mstride3]]\n" - - // Third row - "fsub U.4s, BXTx31.4s, BXTx33.4s\n" - "str qU, [%x[outptr8]]\n" - "fadd U.4s, BXTx32.4s, BXTx33.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, BXTx33.4s, BXTx32.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "str qBXTx32, [%x[outptr8], %x[mstride3]]\n" - - // Fourth row - "fsub U.4s, BXTx41.4s, BXTx43.4s\n" - "str qU, [%x[outptr12]]\n" - "fadd U.4s, BXTx42.4s, BXTx43.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "fsub U.4s, BXTx43.4s, BXTx42.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "str qBXTx42, [%x[outptr12], %x[mstride3]]\n" - - // ---------------------------------------------------------------------- - "4:" - // End of function - - // Clear names - ".unreq x_13\n" ".unreq qx_13\n" ".unreq x_14\n" ".unreq qx_14\n" - ".unreq x_23\n" ".unreq qx_23\n" ".unreq x_24\n" ".unreq qx_24\n" - ".unreq x_33\n" ".unreq qx_33\n" ".unreq x_34\n" ".unreq qx_34\n" - ".unreq AXTx11\n" ".unreq BXTx13\n" - ".unreq AXTx12\n" ".unreq BXTx14\n" ".unreq qAXTx12\n" - ".unreq AXTx21\n" ".unreq BXTx23\n" - ".unreq AXTx22\n" ".unreq BXTx24\n" ".unreq qAXTx22\n" - ".unreq AXTx31\n" ".unreq BXTx33\n" - ".unreq AXTx32\n" ".unreq BXTx34\n" ".unreq qAXTx32\n" - ".unreq AXTx41\n" ".unreq BXTx43\n" - ".unreq AXTx42\n" ".unreq BXTx44\n" ".unreq qAXTx42\n" - ".unreq AXTx13\n" ".unreq BXTx11\n" - ".unreq AXTx14\n" ".unreq BXTx12\n" ".unreq qBXTx12\n" - ".unreq AXTx23\n" ".unreq BXTx21\n" - ".unreq AXTx24\n" ".unreq BXTx22\n" ".unreq qBXTx22\n" - ".unreq AXTx33\n" ".unreq BXTx31\n" - ".unreq AXTx34\n" ".unreq BXTx32\n" ".unreq qBXTx32\n" - ".unreq AXTx43\n" ".unreq BXTx41\n" - ".unreq AXTx44\n" ".unreq BXTx42\n" ".unreq qBXTx42\n" - ".unreq U\n" ".unreq qU\n" - : [inptr0] "+r" (inptr0), - [inptr1] "+r" (inptr1), - [inptr2] "+r" (inptr2), - [outptr0] "+r" (outptr0), - [outptr4] "+r" (outptr4), - [outptr8] "+r" (outptr8), - [outptr12] "+r" (outptr12), - [tile_j] "+r" (tile_j) // Tile counter - : [colstride1] "r" (1 * input_col_stride * sizeof(float)), - [colstride2] "r" (2 * input_col_stride * sizeof(float)), - [colstride3] "r" (3 * input_col_stride * sizeof(float)), - [mstride1] "r" (1 * matrix_stride * sizeof(float)), - [mstride2] "r" (2 * matrix_stride * sizeof(float)), - [mstride3] "r" (3 * matrix_stride * sizeof(float)), - [matrix_row_stride] "r" (matrix_row_stride * sizeof(float)) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", - "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", - "v22", "v23", "v24" - ); -} -} -#endif // __aarch64__ |