diff options
Diffstat (limited to 'src/core/NEON/kernels/winograd/transforms/input_2x2_3x3/a64_float_channelwise.hpp')
| -rw-r--r-- | src/core/NEON/kernels/winograd/transforms/input_2x2_3x3/a64_float_channelwise.hpp | 961 |
1 files changed, 0 insertions, 961 deletions
diff --git a/src/core/NEON/kernels/winograd/transforms/input_2x2_3x3/a64_float_channelwise.hpp b/src/core/NEON/kernels/winograd/transforms/input_2x2_3x3/a64_float_channelwise.hpp deleted file mode 100644 index ad1ad55291..0000000000 --- a/src/core/NEON/kernels/winograd/transforms/input_2x2_3x3/a64_float_channelwise.hpp +++ /dev/null @@ -1,961 +0,0 @@ -/* - * Copyright (c) 2017 ARM Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#pragma once -#include "../input_2x2_3x3.hpp" - -#ifdef __aarch64__ - -namespace winograd { - -template <> -template <> -inline void Winograd2x2_3x3GemmInputChannelwise<float>::_process_tile<0, 0, 0, 0, 4>( - int &n_channels, // Number of channels in the tile - const float* &inptr0, - const int input_row_stride, - const int input_col_stride, - float* &outptr0, - const int matrix_stride -) { - // We use 4 pointers to point to the starting position on each row and use - // three offsets to extract elements from each of the other 3 columns. - auto inptr1 = inptr0 + 1*input_row_stride; - auto inptr2 = inptr0 + 2*input_row_stride; - auto inptr3 = inptr0 + 3*input_row_stride; - - // We use 4 pointers to point at matrices 0, 4, 8 and 12 and use three - // offsets to access the intermediate matrices. - auto outptr1 = outptr0 + matrix_stride * 4; - auto outptr2 = outptr0 + matrix_stride * 8; - auto outptr3 = outptr0 + matrix_stride * 12; - - for (; n_channels > 3; n_channels -= 4) { - asm volatile ( - "X_11 .req v0\n" "qX_11 .req q0\n" - "X_12 .req v1\n" "qX_12 .req q1\n" - "X_13 .req v2\n" "qX_13 .req q2\n" - "X_14 .req v3\n" "qX_14 .req q3\n" - "X_21 .req v4\n" "qX_21 .req q4\n" - "X_22 .req v5\n" "qX_22 .req q5\n" - "X_23 .req v6\n" "qX_23 .req q6\n" - "X_24 .req v7\n" "qX_24 .req q7\n" - "X_31 .req v8\n" "qX_31 .req q8\n" - "X_32 .req v9\n" "qX_32 .req q9\n" - "X_33 .req v10\n" "qX_33 .req q10\n" - "X_34 .req v11\n" "qX_34 .req q11\n" - "X_41 .req v12\n" "qX_41 .req q12\n" - "X_42 .req v13\n" "qX_42 .req q13\n" - "X_43 .req v14\n" "qX_43 .req q14\n" - "X_44 .req v15\n" "qX_44 .req q15\n" - "xX_11 .req v16\n" - "xX_12 .req v17\n" - "xX_13 .req v18\n" - "xX_14 .req v19\n" - "xX_21 .req v20\n" - "xX_22 .req v21\n" - "xX_23 .req v22\n" - "xX_24 .req v23\n" - "xX_31 .req v24\n" - "xX_32 .req v25\n" - "xX_33 .req v26\n" - "xX_34 .req v27\n" - "xX_41 .req v28\n" - "xX_42 .req v29\n" - "xX_43 .req v30\n" - "xX_44 .req v31\n" - " U .req v0\n" - "qU .req q0\n" - - // Load the tile, and compute compute the matrix xX - "ldr qX_11, [%x[inptr0]]\n" - "ldr qX_12, [%x[inptr0], %x[colstride1]]\n" - "ldr qX_13, [%x[inptr0], %x[colstride2]]\n" - "ldr qX_14, [%x[inptr0], %x[colstride3]]\n" - "add %x[inptr0], %x[inptr0], #0x10\n" - - "ldr qX_21, [%x[inptr1]]\n" - "fsub xX_11.4s, x_11.4s, x_13.4s\n" - "ldr qX_22, [%x[inptr1], %x[colstride1]]\n" - "fadd xX_12.4s, x_12.4s, x_13.4s\n" - "ldr qX_23, [%x[inptr1], %x[colstride2]]\n" - "fsub xX_13.4s, x_13.4s, x_12.4s\n" - "ldr qX_24, [%x[inptr1], %x[colstride3]]\n" - "fsub xX_14.4s, x_12.4s, x_14.4s\n" - "add %x[inptr1], %x[inptr1], #0x10\n" - - "ldr qX_31, [%x[inptr2]]\n" - "fsub xX_21.4s, x_21.4s, x_23.4s\n" - "ldr qX_32, [%x[inptr2], %x[colstride1]]\n" - "fadd xX_22.4s, x_22.4s, x_23.4s\n" - "ldr qX_33, [%x[inptr2], %x[colstride2]]\n" - "fsub xX_23.4s, x_23.4s, x_22.4s\n" - "ldr qX_34, [%x[inptr2], %x[colstride3]]\n" - "fsub xX_24.4s, x_22.4s, x_24.4s\n" - "add %x[inptr2], %x[inptr2], #0x10\n" - - "ldr qX_41, [%x[inptr3]]\n" - "fsub xX_31.4s, x_31.4s, x_33.4s\n" - "ldr qX_42, [%x[inptr3], %x[colstride1]]\n" - "fadd xX_32.4s, x_32.4s, x_33.4s\n" - "ldr qX_43, [%x[inptr3], %x[colstride2]]\n" - "fsub xX_33.4s, x_33.4s, x_32.4s\n" - "ldr qX_44, [%x[inptr3], %x[colstride3]]\n" - "fsub xX_34.4s, x_32.4s, x_34.4s\n" - "add %x[inptr3], %x[inptr3], #0x10\n" - - // Complete computing xX while beginning to compute and store - // $U = X.T x X$ - - "fsub xX_41.4s, x_41.4s, x_43.4s\n" - - "fsub U.4s, xX_11.4s, xX_31.4s\n" - "str qU, [%x[outptr0]]\n" - "fsub U.4s, xX_12.4s, xX_32.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, xX_13.4s, xX_33.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "fsub U.4s, xX_14.4s, xX_34.4s\n" - "str qU, [%x[outptr0], %x[mstride3]]\n" - "add %x[outptr0], %x[outptr0], #0x10\n" - - "fadd xX_42.4s, x_42.4s, x_43.4s\n" - - "fadd U.4s, xX_21.4s, xX_31.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, xX_22.4s, xX_32.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fadd U.4s, xX_23.4s, xX_33.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "fadd U.4s, xX_24.4s, xX_34.4s\n" - "str qU, [%x[outptr4], %x[mstride3]]\n" - "add %x[outptr4], %x[outptr4], #0x10\n" - - "fsub xX_43.4s, x_43.4s, x_42.4s\n" - - "fsub U.4s, xX_31.4s, xX_21.4s\n" - "str qU, [%x[outptr8]]\n" - "fsub U.4s, xX_32.4s, xX_22.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, xX_33.4s, xX_23.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "fsub U.4s, xX_34.4s, xX_24.4s\n" - "str qU, [%x[outptr8], %x[mstride3]]\n" - "add %x[outptr8], %x[outptr8], #0x10\n" - - "fsub xX_44.4s, x_42.4s, x_44.4s\n" - - "fsub U.4s, xX_21.4s, xX_41.4s\n" - "str qU, [%x[outptr12]]\n" - "fsub U.4s, xX_22.4s, xX_42.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "fsub U.4s, xX_23.4s, xX_43.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "fsub U.4s, xX_24.4s, xX_44.4s\n" - "str qU, [%x[outptr12], %x[mstride3]]\n" - "add %x[outptr12], %x[outptr12], #0x10\n" - - ".unreq qU\n" - ".unreq U\n" - ".unreq X_11\n" ".unreq qX_11\n" - ".unreq X_12\n" ".unreq qX_12\n" - ".unreq X_13\n" ".unreq qX_13\n" - ".unreq X_14\n" ".unreq qX_14\n" - ".unreq X_21\n" ".unreq qX_21\n" - ".unreq X_22\n" ".unreq qX_22\n" - ".unreq X_23\n" ".unreq qX_23\n" - ".unreq X_24\n" ".unreq qX_24\n" - ".unreq X_31\n" ".unreq qX_31\n" - ".unreq X_32\n" ".unreq qX_32\n" - ".unreq X_33\n" ".unreq qX_33\n" - ".unreq X_34\n" ".unreq qX_34\n" - ".unreq X_41\n" ".unreq qX_41\n" - ".unreq X_42\n" ".unreq qX_42\n" - ".unreq X_43\n" ".unreq qX_43\n" - ".unreq X_44\n" ".unreq qX_44\n" - ".unreq xX_11\n" - ".unreq xX_12\n" - ".unreq xX_13\n" - ".unreq xX_14\n" - ".unreq xX_21\n" - ".unreq xX_22\n" - ".unreq xX_23\n" - ".unreq xX_24\n" - ".unreq xX_31\n" - ".unreq xX_32\n" - ".unreq xX_33\n" - ".unreq xX_34\n" - ".unreq xX_41\n" - ".unreq xX_42\n" - ".unreq xX_43\n" - ".unreq xX_44\n" - - : [inptr0] "+r" (inptr0), - [inptr1] "+r" (inptr1), - [inptr2] "+r" (inptr2), - [inptr3] "+r" (inptr3), - [outptr0] "+r" (outptr0), - [outptr4] "+r" (outptr1), - [outptr8] "+r" (outptr2), - [outptr12] "+r" (outptr3) - : [colstride1] "r" (input_col_stride * sizeof(float)), - [colstride2] "r" (input_col_stride * sizeof(float) * 2), - [colstride3] "r" (input_col_stride * sizeof(float) * 3), - [mstride1] "r" (matrix_stride * sizeof(float)), - [mstride2] "r" (matrix_stride * sizeof(float) * 2), - [mstride3] "r" (matrix_stride * sizeof(float) * 3) - : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", - "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", - "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", - "v30", "v31" - ); - } -} - -// Pad top by 1 -template <> -template <> -inline void Winograd2x2_3x3GemmInputChannelwise<float>::_process_tile<1, 0, 0, 0, 4>( - int &n_channels, // Number of channels in the tile - const float* &inptr0, - const int input_row_stride, - const int input_col_stride, - float* &outptr0, - const int matrix_stride -) { - // We use 4 pointers to point to the starting position on each row and use - // three offsets to extract elements from each of the other 3 columns. - auto inptr1 = inptr0 + 0*input_row_stride; - auto inptr2 = inptr0 + 1*input_row_stride; - - // We use 4 pointers to point at matrices 0, 4, 8 and 12 and use three - // offsets to access the intermediate matrices. - auto outptr1 = outptr0 + matrix_stride * 4; - auto outptr2 = outptr0 + matrix_stride * 8; - auto outptr3 = outptr0 + matrix_stride * 12; - - for (; n_channels > 3; n_channels -= 4) { - asm volatile ( - "X_21 .req v4\n" "qX_21 .req q4\n" - "X_22 .req v5\n" "qX_22 .req q5\n" - "X_23 .req v6\n" "qX_23 .req q6\n" - "X_24 .req v7\n" "qX_24 .req q7\n" - "X_31 .req v8\n" "qX_31 .req q8\n" - "X_32 .req v9\n" "qX_32 .req q9\n" - "X_33 .req v10\n" "qX_33 .req q10\n" - "X_34 .req v11\n" "qX_34 .req q11\n" - "X_41 .req v12\n" "qX_41 .req q12\n" - "X_42 .req v13\n" "qX_42 .req q13\n" - "X_43 .req v14\n" "qX_43 .req q14\n" - "X_44 .req v15\n" "qX_44 .req q15\n" - "xX_21 .req v20\n" - "xX_22 .req v21\n" - "xX_23 .req v22\n" - "xX_24 .req v23\n" - "xX_31 .req v24\n" - "xX_32 .req v25\n" - "xX_33 .req v26\n" - "xX_34 .req v27\n" - "xX_41 .req v28\n" - "xX_42 .req v29\n" - "xX_43 .req v30\n" - "xX_44 .req v31\n" - " U .req v0\n" - "qU .req q0\n" - - // Load the tile, and compute compute the matrix xX - "ldr qX_21, [%x[inptr1]]\n" - "ldr qX_22, [%x[inptr1], %x[colstride1]]\n" - "ldr qX_23, [%x[inptr1], %x[colstride2]]\n" - "ldr qX_24, [%x[inptr1], %x[colstride3]]\n" - "add %x[inptr1], %x[inptr1], #0x10\n" - - "ldr qX_31, [%x[inptr2]]\n" - "fsub xX_21.4s, x_21.4s, x_23.4s\n" - "ldr qX_32, [%x[inptr2], %x[colstride1]]\n" - "fadd xX_22.4s, x_22.4s, x_23.4s\n" - "ldr qX_33, [%x[inptr2], %x[colstride2]]\n" - "fsub xX_23.4s, x_23.4s, x_22.4s\n" - "ldr qX_34, [%x[inptr2], %x[colstride3]]\n" - "fsub xX_24.4s, x_22.4s, x_24.4s\n" - "add %x[inptr2], %x[inptr2], #0x10\n" - - "ldr qX_41, [%x[inptr3]]\n" - "fsub xX_31.4s, x_31.4s, x_33.4s\n" - "ldr qX_42, [%x[inptr3], %x[colstride1]]\n" - "fadd xX_32.4s, x_32.4s, x_33.4s\n" - "ldr qX_43, [%x[inptr3], %x[colstride2]]\n" - "fsub xX_33.4s, x_33.4s, x_32.4s\n" - "ldr qX_44, [%x[inptr3], %x[colstride3]]\n" - "fsub xX_34.4s, x_32.4s, x_34.4s\n" - "add %x[inptr3], %x[inptr3], #0x10\n" - - // Complete computing xX while beginning to compute and store - // $U = X.T x X$ - - "fsub xX_41.4s, x_41.4s, x_43.4s\n" - - "fneg U.4s, xX_31.4s\n" - "str qU, [%x[outptr0]]\n" - "fneg U.4s, xX_32.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fneg U.4s, xX_33.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "fneg U.4s, xX_34.4s\n" - "str qU, [%x[outptr0], %x[mstride3]]\n" - "add %x[outptr0], %x[outptr0], #0x10\n" - - "fadd xX_42.4s, x_42.4s, x_43.4s\n" - - "fadd U.4s, xX_21.4s, xX_31.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, xX_22.4s, xX_32.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fadd U.4s, xX_23.4s, xX_33.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "fadd U.4s, xX_24.4s, xX_34.4s\n" - "str qU, [%x[outptr4], %x[mstride3]]\n" - "add %x[outptr4], %x[outptr4], #0x10\n" - - "fsub xX_43.4s, x_43.4s, x_42.4s\n" - - "fsub U.4s, xX_31.4s, xX_21.4s\n" - "str qU, [%x[outptr8]]\n" - "fsub U.4s, xX_32.4s, xX_22.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, xX_33.4s, xX_23.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "fsub U.4s, xX_34.4s, xX_24.4s\n" - "str qU, [%x[outptr8], %x[mstride3]]\n" - "add %x[outptr8], %x[outptr8], #0x10\n" - - "fsub xX_44.4s, x_42.4s, x_44.4s\n" - - "fsub U.4s, xX_21.4s, xX_41.4s\n" - "str qU, [%x[outptr12]]\n" - "fsub U.4s, xX_22.4s, xX_42.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "fsub U.4s, xX_23.4s, xX_43.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "fsub U.4s, xX_24.4s, xX_44.4s\n" - "str qU, [%x[outptr12], %x[mstride3]]\n" - "add %x[outptr12], %x[outptr12], #0x10\n" - - ".unreq qU\n" - ".unreq U\n" - ".unreq X_21\n" ".unreq qX_21\n" - ".unreq X_22\n" ".unreq qX_22\n" - ".unreq X_23\n" ".unreq qX_23\n" - ".unreq X_24\n" ".unreq qX_24\n" - ".unreq X_31\n" ".unreq qX_31\n" - ".unreq X_32\n" ".unreq qX_32\n" - ".unreq X_33\n" ".unreq qX_33\n" - ".unreq X_34\n" ".unreq qX_34\n" - ".unreq X_41\n" ".unreq qX_41\n" - ".unreq X_42\n" ".unreq qX_42\n" - ".unreq X_43\n" ".unreq qX_43\n" - ".unreq X_44\n" ".unreq qX_44\n" - ".unreq xX_21\n" - ".unreq xX_22\n" - ".unreq xX_23\n" - ".unreq xX_24\n" - ".unreq xX_31\n" - ".unreq xX_32\n" - ".unreq xX_33\n" - ".unreq xX_34\n" - ".unreq xX_41\n" - ".unreq xX_42\n" - ".unreq xX_43\n" - ".unreq xX_44\n" - - : [inptr1] "+r" (inptr0), // Offset for missing row - [inptr2] "+r" (inptr1), // Offset for missing row - [inptr3] "+r" (inptr2), // Offset for missing row - [outptr0] "+r" (outptr0), - [outptr4] "+r" (outptr1), - [outptr8] "+r" (outptr2), - [outptr12] "+r" (outptr3) - : [colstride1] "r" (input_col_stride * sizeof(float)), - [colstride2] "r" (input_col_stride * sizeof(float) * 2), - [colstride3] "r" (input_col_stride * sizeof(float) * 3), - [mstride1] "r" (matrix_stride * sizeof(float)), - [mstride2] "r" (matrix_stride * sizeof(float) * 2), - [mstride3] "r" (matrix_stride * sizeof(float) * 3) - : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", - "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", - "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", - "v30", "v31" - ); - } -} - -// Pad left by 1 -template <> -template <> -inline void Winograd2x2_3x3GemmInputChannelwise<float>::_process_tile<0, 1, 0, 0, 4>( - int &n_channels, // Number of channels in the tile - const float* &inptr0, - const int input_row_stride, - const int input_col_stride, - float* &outptr0, - const int matrix_stride -) { - // We use 4 pointers to point to the starting position on each row and use - // three offsets to extract elements from each of the other 3 columns. - auto inptr1 = inptr0 + 1*input_row_stride; - auto inptr2 = inptr0 + 2*input_row_stride; - auto inptr3 = inptr0 + 3*input_row_stride; - - // We use 4 pointers to point at matrices 0, 4, 8 and 12 and use three - // offsets to access the intermediate matrices. - auto outptr1 = outptr0 + matrix_stride * 4; - auto outptr2 = outptr0 + matrix_stride * 8; - auto outptr3 = outptr0 + matrix_stride * 12; - - for (; n_channels > 3; n_channels -= 4) { - asm volatile ( - "X_12 .req v1\n" "qX_12 .req q1\n" - "X_13 .req v2\n" "qX_13 .req q2\n" - "X_14 .req v3\n" "qX_14 .req q3\n" - "X_22 .req v5\n" "qX_22 .req q5\n" - "X_23 .req v6\n" "qX_23 .req q6\n" - "X_24 .req v7\n" "qX_24 .req q7\n" - "X_32 .req v9\n" "qX_32 .req q9\n" - "X_33 .req v10\n" "qX_33 .req q10\n" - "X_34 .req v11\n" "qX_34 .req q11\n" - "X_42 .req v13\n" "qX_42 .req q13\n" - "X_43 .req v14\n" "qX_43 .req q14\n" - "X_44 .req v15\n" "qX_44 .req q15\n" - "xX_11 .req v16\n" - "xX_12 .req v17\n" - "xX_13 .req v18\n" - "xX_14 .req v19\n" - "xX_21 .req v20\n" - "xX_22 .req v21\n" - "xX_23 .req v22\n" - "xX_24 .req v23\n" - "xX_31 .req v24\n" - "xX_32 .req v25\n" - "xX_33 .req v26\n" - "xX_34 .req v27\n" - "xX_41 .req v28\n" - "xX_42 .req v29\n" - "xX_43 .req v30\n" - "xX_44 .req v31\n" - " U .req v0\n" - "qU .req q0\n" - - // Load the tile, and compute compute the matrix xX - "ldr qX_12, [%x[inptr0]]\n" - "ldr qX_13, [%x[inptr0], %x[colstride1]]\n" - "ldr qX_14, [%x[inptr0], %x[colstride2]]\n" - "add %x[inptr0], %x[inptr0], #0x10\n" - - "fneg xX_11.4s, x_13.4s\n" - "ldr qX_22, [%x[inptr1]]\n" - "fadd xX_12.4s, x_12.4s, x_13.4s\n" - "ldr qX_23, [%x[inptr1], %x[colstride1]]\n" - "fsub xX_13.4s, x_13.4s, x_12.4s\n" - "ldr qX_24, [%x[inptr1], %x[colstride2]]\n" - "fsub xX_14.4s, x_12.4s, x_14.4s\n" - "add %x[inptr1], %x[inptr1], #0x10\n" - - "fneg xX_21.4s, x_23.4s\n" - "ldr qX_32, [%x[inptr2]]\n" - "fadd xX_22.4s, x_22.4s, x_23.4s\n" - "ldr qX_33, [%x[inptr2], %x[colstride1]]\n" - "fsub xX_23.4s, x_23.4s, x_22.4s\n" - "ldr qX_34, [%x[inptr2], %x[colstride2]]\n" - "fsub xX_24.4s, x_22.4s, x_24.4s\n" - "add %x[inptr2], %x[inptr2], #0x10\n" - - "fneg xX_31.4s, x_33.4s\n" - "ldr qX_42, [%x[inptr3]]\n" - "fadd xX_32.4s, x_32.4s, x_33.4s\n" - "ldr qX_43, [%x[inptr3], %x[colstride1]]\n" - "fsub xX_33.4s, x_33.4s, x_32.4s\n" - "ldr qX_44, [%x[inptr3], %x[colstride2]]\n" - "fsub xX_34.4s, x_32.4s, x_34.4s\n" - "add %x[inptr3], %x[inptr3], #0x10\n" - - // Complete computing xX while beginning to compute and store - // $U = X.T x X$ - - "fneg xX_41.4s, x_43.4s\n" - - "fsub U.4s, xX_11.4s, xX_31.4s\n" - "str qU, [%x[outptr0]]\n" - "fsub U.4s, xX_12.4s, xX_32.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, xX_13.4s, xX_33.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "fsub U.4s, xX_14.4s, xX_34.4s\n" - "str qU, [%x[outptr0], %x[mstride3]]\n" - "add %x[outptr0], %x[outptr0], #0x10\n" - - "fadd xX_42.4s, x_42.4s, x_43.4s\n" - - "fadd U.4s, xX_21.4s, xX_31.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, xX_22.4s, xX_32.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fadd U.4s, xX_23.4s, xX_33.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "fadd U.4s, xX_24.4s, xX_34.4s\n" - "str qU, [%x[outptr4], %x[mstride3]]\n" - "add %x[outptr4], %x[outptr4], #0x10\n" - - "fsub xX_43.4s, x_43.4s, x_42.4s\n" - - "fsub U.4s, xX_31.4s, xX_21.4s\n" - "str qU, [%x[outptr8]]\n" - "fsub U.4s, xX_32.4s, xX_22.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, xX_33.4s, xX_23.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "fsub U.4s, xX_34.4s, xX_24.4s\n" - "str qU, [%x[outptr8], %x[mstride3]]\n" - "add %x[outptr8], %x[outptr8], #0x10\n" - - "fsub xX_44.4s, x_42.4s, x_44.4s\n" - - "fsub U.4s, xX_21.4s, xX_41.4s\n" - "str qU, [%x[outptr12]]\n" - "fsub U.4s, xX_22.4s, xX_42.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "fsub U.4s, xX_23.4s, xX_43.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "fsub U.4s, xX_24.4s, xX_44.4s\n" - "str qU, [%x[outptr12], %x[mstride3]]\n" - "add %x[outptr12], %x[outptr12], #0x10\n" - - ".unreq X_12\n" ".unreq qX_12\n" - ".unreq X_13\n" ".unreq qX_13\n" - ".unreq X_14\n" ".unreq qX_14\n" - ".unreq X_22\n" ".unreq qX_22\n" - ".unreq X_23\n" ".unreq qX_23\n" - ".unreq X_24\n" ".unreq qX_24\n" - ".unreq X_32\n" ".unreq qX_32\n" - ".unreq X_33\n" ".unreq qX_33\n" - ".unreq X_34\n" ".unreq qX_34\n" - ".unreq X_42\n" ".unreq qX_42\n" - ".unreq X_43\n" ".unreq qX_43\n" - ".unreq X_44\n" ".unreq qX_44\n" - ".unreq xX_11\n" - ".unreq xX_12\n" - ".unreq xX_13\n" - ".unreq xX_14\n" - ".unreq xX_21\n" - ".unreq xX_22\n" - ".unreq xX_23\n" - ".unreq xX_24\n" - ".unreq xX_31\n" - ".unreq xX_32\n" - ".unreq xX_33\n" - ".unreq xX_34\n" - ".unreq xX_41\n" - ".unreq xX_42\n" - ".unreq xX_43\n" - ".unreq xX_44\n" - ".unreq U\n" - ".unreq qU\n" - - : [inptr0] "+r" (inptr0), - [inptr1] "+r" (inptr1), - [inptr2] "+r" (inptr2), - [inptr3] "+r" (inptr3), - [outptr0] "+r" (outptr0), - [outptr4] "+r" (outptr1), - [outptr8] "+r" (outptr2), - [outptr12] "+r" (outptr3) - : [colstride1] "r" (input_col_stride * sizeof(float)), - [colstride2] "r" (input_col_stride * sizeof(float) * 2), - [mstride1] "r" (matrix_stride * sizeof(float)), - [mstride2] "r" (matrix_stride * sizeof(float) * 2), - [mstride3] "r" (matrix_stride * sizeof(float) * 3) - : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", - "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", - "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", - "v30", "v31" - ); - } -} - -// Pad bottom by 1 -template <> -template <> -inline void Winograd2x2_3x3GemmInputChannelwise<float>::_process_tile<0, 0, 1, 0, 4>( - int &n_channels, // Number of channels in the tile - const float* &inptr0, - const int input_row_stride, - const int input_col_stride, - float* &outptr0, - const int matrix_stride -) { - // We use 4 pointers to point to the starting position on each row and use - // three offsets to extract elements from each of the other 3 columns. - auto inptr1 = inptr0 + 1*input_row_stride; - auto inptr2 = inptr0 + 2*input_row_stride; - - // We use 4 pointers to point at matrices 0, 4, 8 and 12 and use three - // offsets to access the intermediate matrices. - auto outptr1 = outptr0 + matrix_stride * 4; - auto outptr2 = outptr0 + matrix_stride * 8; - auto outptr3 = outptr0 + matrix_stride * 12; - - for (; n_channels > 3; n_channels -= 4) { - asm volatile ( - "X_11 .req v0\n" "qX_11 .req q0\n" - "X_12 .req v1\n" "qX_12 .req q1\n" - "X_13 .req v2\n" "qX_13 .req q2\n" - "X_14 .req v3\n" "qX_14 .req q3\n" - "X_21 .req v4\n" "qX_21 .req q4\n" - "X_22 .req v5\n" "qX_22 .req q5\n" - "X_23 .req v6\n" "qX_23 .req q6\n" - "X_24 .req v7\n" "qX_24 .req q7\n" - "X_31 .req v8\n" "qX_31 .req q8\n" - "X_32 .req v9\n" "qX_32 .req q9\n" - "X_33 .req v10\n" "qX_33 .req q10\n" - "X_34 .req v11\n" "qX_34 .req q11\n" - "xX_11 .req v16\n" - "xX_12 .req v17\n" - "xX_13 .req v18\n" - "xX_14 .req v19\n" - "xX_21 .req v20\n" "qxX_21 .req q20\n" - "xX_22 .req v21\n" "qxX_22 .req q21\n" - "xX_23 .req v22\n" "qxX_23 .req q22\n" - "xX_24 .req v23\n" "qxX_24 .req q23\n" - "xX_31 .req v24\n" - "xX_32 .req v25\n" - "xX_33 .req v26\n" - "xX_34 .req v27\n" - " U .req v0\n" - "qU .req q0\n" - - // Load the tile, and compute compute the matrix xX - "ldr qX_11, [%x[inptr0]]\n" - "ldr qX_12, [%x[inptr0], %x[colstride1]]\n" - "ldr qX_13, [%x[inptr0], %x[colstride2]]\n" - "ldr qX_14, [%x[inptr0], %x[colstride3]]\n" - "add %x[inptr0], %x[inptr0], #0x10\n" - - "ldr qX_21, [%x[inptr1]]\n" - "fsub xX_11.4s, x_11.4s, x_13.4s\n" - "ldr qX_22, [%x[inptr1], %x[colstride1]]\n" - "fadd xX_12.4s, x_12.4s, x_13.4s\n" - "ldr qX_23, [%x[inptr1], %x[colstride2]]\n" - "fsub xX_13.4s, x_13.4s, x_12.4s\n" - "ldr qX_24, [%x[inptr1], %x[colstride3]]\n" - "fsub xX_14.4s, x_12.4s, x_14.4s\n" - "add %x[inptr1], %x[inptr1], #0x10\n" - - "ldr qX_31, [%x[inptr2]]\n" - "fsub xX_21.4s, x_21.4s, x_23.4s\n" - "ldr qX_32, [%x[inptr2], %x[colstride1]]\n" - "fadd xX_22.4s, x_22.4s, x_23.4s\n" - "ldr qX_33, [%x[inptr2], %x[colstride2]]\n" - "fsub xX_23.4s, x_23.4s, x_22.4s\n" - "ldr qX_34, [%x[inptr2], %x[colstride3]]\n" - "fsub xX_24.4s, x_22.4s, x_24.4s\n" - "add %x[inptr2], %x[inptr2], #0x10\n" - - "fsub xX_31.4s, x_31.4s, x_33.4s\n" - "fadd xX_32.4s, x_32.4s, x_33.4s\n" - "fsub xX_33.4s, x_33.4s, x_32.4s\n" - "fsub xX_34.4s, x_32.4s, x_34.4s\n" - - // Complete computing xX while beginning to compute and store - // $U = X.T x X$ - - "fsub U.4s, xX_11.4s, xX_31.4s\n" - "str qU, [%x[outptr0]]\n" - "fsub U.4s, xX_12.4s, xX_32.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, xX_13.4s, xX_33.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "fsub U.4s, xX_14.4s, xX_34.4s\n" - "str qU, [%x[outptr0], %x[mstride3]]\n" - "add %x[outptr0], %x[outptr0], #0x10\n" - - "fadd U.4s, xX_21.4s, xX_31.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, xX_22.4s, xX_32.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fadd U.4s, xX_23.4s, xX_33.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "fadd U.4s, xX_24.4s, xX_34.4s\n" - "str qU, [%x[outptr4], %x[mstride3]]\n" - "add %x[outptr4], %x[outptr4], #0x10\n" - - "fsub U.4s, xX_31.4s, xX_21.4s\n" - "str qU, [%x[outptr8]]\n" - "fsub U.4s, xX_32.4s, xX_22.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, xX_33.4s, xX_23.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "fsub U.4s, xX_34.4s, xX_24.4s\n" - "str qU, [%x[outptr8], %x[mstride3]]\n" - "add %x[outptr8], %x[outptr8], #0x10\n" - - "str qxX_21, [%x[outptr12]]\n" - "str qxX_22, [%x[outptr12], %x[mstride1]]\n" - "str qxX_23, [%x[outptr12], %x[mstride2]]\n" - "str qxX_24, [%x[outptr12], %x[mstride3]]\n" - "add %x[outptr12], %x[outptr12], #0x10\n" - - ".unreq qU\n" - ".unreq U\n" - ".unreq X_11\n" ".unreq qX_11\n" - ".unreq X_12\n" ".unreq qX_12\n" - ".unreq X_13\n" ".unreq qX_13\n" - ".unreq X_14\n" ".unreq qX_14\n" - ".unreq X_21\n" ".unreq qX_21\n" - ".unreq X_22\n" ".unreq qX_22\n" - ".unreq X_23\n" ".unreq qX_23\n" - ".unreq X_24\n" ".unreq qX_24\n" - ".unreq X_31\n" ".unreq qX_31\n" - ".unreq X_32\n" ".unreq qX_32\n" - ".unreq X_33\n" ".unreq qX_33\n" - ".unreq X_34\n" ".unreq qX_34\n" - ".unreq xX_11\n" - ".unreq xX_12\n" - ".unreq xX_13\n" - ".unreq xX_14\n" - ".unreq xX_21\n" ".unreq qxX_21\n" - ".unreq xX_22\n" ".unreq qxX_22\n" - ".unreq xX_23\n" ".unreq qxX_23\n" - ".unreq xX_24\n" ".unreq qxX_24\n" - ".unreq xX_31\n" - ".unreq xX_32\n" - ".unreq xX_33\n" - ".unreq xX_34\n" - - : [inptr0] "+r" (inptr0), - [inptr1] "+r" (inptr1), - [inptr2] "+r" (inptr2), - [outptr0] "+r" (outptr0), - [outptr4] "+r" (outptr1), - [outptr8] "+r" (outptr2), - [outptr12] "+r" (outptr3) - : [colstride1] "r" (input_col_stride * sizeof(float)), - [colstride2] "r" (input_col_stride * sizeof(float) * 2), - [colstride3] "r" (input_col_stride * sizeof(float) * 3), - [mstride1] "r" (matrix_stride * sizeof(float)), - [mstride2] "r" (matrix_stride * sizeof(float) * 2), - [mstride3] "r" (matrix_stride * sizeof(float) * 3) - : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", - "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", - "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", - "v30", "v31" - ); - } -} - -// Pad right by 1 -template <> -template <> -inline void Winograd2x2_3x3GemmInputChannelwise<float>::_process_tile<0, 0, 0, 1, 4>( - int &n_channels, // Number of channels in the tile - const float* &inptr0, - const int input_row_stride, - const int input_col_stride, - float* &outptr0, - const int matrix_stride -) { - // We use 4 pointers to point to the starting position on each row and use - // three offsets to extract elements from each of the other 3 columns. - auto inptr1 = inptr0 + 1*input_row_stride; - auto inptr2 = inptr0 + 2*input_row_stride; - auto inptr3 = inptr0 + 3*input_row_stride; - - // We use 4 pointers to point at matrices 0, 4, 8 and 12 and use three - // offsets to access the intermediate matrices. - auto outptr1 = outptr0 + matrix_stride * 4; - auto outptr2 = outptr0 + matrix_stride * 8; - auto outptr3 = outptr0 + matrix_stride * 12; - - for (; n_channels > 3; n_channels -= 4) { - asm volatile ( - "X_11 .req v0\n" "qX_11 .req q0\n" - "X_12 .req v1\n" "qX_12 .req q1\n" - "X_13 .req v2\n" "qX_13 .req q2\n" - "X_21 .req v4\n" "qX_21 .req q4\n" - "X_22 .req v5\n" "qX_22 .req q5\n" - "X_23 .req v6\n" "qX_23 .req q6\n" - "X_31 .req v8\n" "qX_31 .req q8\n" - "X_32 .req v9\n" "qX_32 .req q9\n" - "X_33 .req v10\n" "qX_33 .req q10\n" - "X_41 .req v12\n" "qX_41 .req q12\n" - "X_42 .req v13\n" "qX_42 .req q13\n" - "X_43 .req v14\n" "qX_43 .req q14\n" - "xX_11 .req v16\n" - "xX_12 .req v17\n" - "xX_13 .req v18\n" - "xX_14 .req x_12\n" - "xX_21 .req v20\n" - "xX_22 .req v21\n" - "xX_23 .req v22\n" - "xX_24 .req x_22\n" - "xX_31 .req v24\n" - "xX_32 .req v25\n" - "xX_33 .req v26\n" - "xX_34 .req x_32\n" - "xX_41 .req v28\n" - "xX_42 .req v29\n" - "xX_43 .req v30\n" - "xX_44 .req x_42\n" - " U .req v0\n" - "qU .req q0\n" - - // Load the tile, and compute compute the matrix xX - "ldr qX_11, [%x[inptr0]]\n" - "ldr qX_12, [%x[inptr0], %x[colstride1]]\n" - "ldr qX_13, [%x[inptr0], %x[colstride2]]\n" - "add %x[inptr0], %x[inptr0], #0x10\n" - - "ldr qX_21, [%x[inptr1]]\n" - "fsub xX_11.4s, x_11.4s, x_13.4s\n" - "ldr qX_22, [%x[inptr1], %x[colstride1]]\n" - "fadd xX_12.4s, x_12.4s, x_13.4s\n" - "ldr qX_23, [%x[inptr1], %x[colstride2]]\n" - "fsub xX_13.4s, x_13.4s, x_12.4s\n" - "add %x[inptr1], %x[inptr1], #0x10\n" - - "ldr qX_31, [%x[inptr2]]\n" - "fsub xX_21.4s, x_21.4s, x_23.4s\n" - "ldr qX_32, [%x[inptr2], %x[colstride1]]\n" - "fadd xX_22.4s, x_22.4s, x_23.4s\n" - "ldr qX_33, [%x[inptr2], %x[colstride2]]\n" - "fsub xX_23.4s, x_23.4s, x_22.4s\n" - "add %x[inptr2], %x[inptr2], #0x10\n" - - "ldr qX_41, [%x[inptr3]]\n" - "fsub xX_31.4s, x_31.4s, x_33.4s\n" - "ldr qX_42, [%x[inptr3], %x[colstride1]]\n" - "fadd xX_32.4s, x_32.4s, x_33.4s\n" - "ldr qX_43, [%x[inptr3], %x[colstride2]]\n" - "fsub xX_33.4s, x_33.4s, x_32.4s\n" - "add %x[inptr3], %x[inptr3], #0x10\n" - - // Complete computing xX while beginning to compute and store - // $U = X.T x X$ - - "fsub xX_41.4s, x_41.4s, x_43.4s\n" - - "fsub U.4s, xX_11.4s, xX_31.4s\n" - "str qU, [%x[outptr0]]\n" - "fsub U.4s, xX_12.4s, xX_32.4s\n" - "str qU, [%x[outptr0], %x[mstride1]]\n" - "fsub U.4s, xX_13.4s, xX_33.4s\n" - "str qU, [%x[outptr0], %x[mstride2]]\n" - "fsub U.4s, xX_14.4s, xX_34.4s\n" - "str qU, [%x[outptr0], %x[mstride3]]\n" - "add %x[outptr0], %x[outptr0], #0x10\n" - - "fadd xX_42.4s, x_42.4s, x_43.4s\n" - - "fadd U.4s, xX_21.4s, xX_31.4s\n" - "str qU, [%x[outptr4]]\n" - "fadd U.4s, xX_22.4s, xX_32.4s\n" - "str qU, [%x[outptr4], %x[mstride1]]\n" - "fadd U.4s, xX_23.4s, xX_33.4s\n" - "str qU, [%x[outptr4], %x[mstride2]]\n" - "fadd U.4s, xX_24.4s, xX_34.4s\n" - "str qU, [%x[outptr4], %x[mstride3]]\n" - "add %x[outptr4], %x[outptr4], #0x10\n" - - "fsub xX_43.4s, x_43.4s, x_42.4s\n" - - "fsub U.4s, xX_31.4s, xX_21.4s\n" - "str qU, [%x[outptr8]]\n" - "fsub U.4s, xX_32.4s, xX_22.4s\n" - "str qU, [%x[outptr8], %x[mstride1]]\n" - "fsub U.4s, xX_33.4s, xX_23.4s\n" - "str qU, [%x[outptr8], %x[mstride2]]\n" - "fsub U.4s, xX_34.4s, xX_24.4s\n" - "str qU, [%x[outptr8], %x[mstride3]]\n" - "add %x[outptr8], %x[outptr8], #0x10\n" - - "fsub U.4s, xX_21.4s, xX_41.4s\n" - "str qU, [%x[outptr12]]\n" - "fsub U.4s, xX_22.4s, xX_42.4s\n" - "str qU, [%x[outptr12], %x[mstride1]]\n" - "fsub U.4s, xX_23.4s, xX_43.4s\n" - "str qU, [%x[outptr12], %x[mstride2]]\n" - "fsub U.4s, xX_24.4s, xX_44.4s\n" - "str qU, [%x[outptr12], %x[mstride3]]\n" - "add %x[outptr12], %x[outptr12], #0x10\n" - - ".unreq qU\n" - ".unreq U\n" - ".unreq X_11\n" ".unreq qX_11\n" - ".unreq X_12\n" ".unreq qX_12\n" - ".unreq X_13\n" ".unreq qX_13\n" - ".unreq X_21\n" ".unreq qX_21\n" - ".unreq X_22\n" ".unreq qX_22\n" - ".unreq X_23\n" ".unreq qX_23\n" - ".unreq X_31\n" ".unreq qX_31\n" - ".unreq X_32\n" ".unreq qX_32\n" - ".unreq X_33\n" ".unreq qX_33\n" - ".unreq X_41\n" ".unreq qX_41\n" - ".unreq X_42\n" ".unreq qX_42\n" - ".unreq X_43\n" ".unreq qX_43\n" - ".unreq xX_11\n" - ".unreq xX_12\n" - ".unreq xX_13\n" - ".unreq xX_14\n" - ".unreq xX_21\n" - ".unreq xX_22\n" - ".unreq xX_23\n" - ".unreq xX_24\n" - ".unreq xX_31\n" - ".unreq xX_32\n" - ".unreq xX_33\n" - ".unreq xX_34\n" - ".unreq xX_41\n" - ".unreq xX_42\n" - ".unreq xX_43\n" - ".unreq xX_44\n" - - : [inptr0] "+r" (inptr0), - [inptr1] "+r" (inptr1), - [inptr2] "+r" (inptr2), - [inptr3] "+r" (inptr3), - [outptr0] "+r" (outptr0), - [outptr4] "+r" (outptr1), - [outptr8] "+r" (outptr2), - [outptr12] "+r" (outptr3) - : [colstride1] "r" (input_col_stride * sizeof(float)), - [colstride2] "r" (input_col_stride * sizeof(float) * 2), - [mstride1] "r" (matrix_stride * sizeof(float)), - [mstride2] "r" (matrix_stride * sizeof(float) * 2), - [mstride3] "r" (matrix_stride * sizeof(float) * 3) - : "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", - "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", - "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", - "v30", "v31" - ); - } -} -} -#endif |