From bda6e4b51bc4045c97100bb9d562164ba7c6c28f Mon Sep 17 00:00:00 2001 From: Pablo Tello Date: Wed, 22 Aug 2018 11:40:33 +0100 Subject: COMPMID-1247:Integrate kernel size 1x3 & 3x1 support in NEWinogradLayer. Change-Id: I6fe198881230e49864c841a3b2366ccf2a9247f9 Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/145210 Tested-by: Jenkins Reviewed-by: Georgios Pinitas --- .../winograd/transforms/input_2x2_3x3_fp32.cpp | 2 +- .../winograd/transforms/input_2x2_5x5_fp32.cpp | 2 +- .../winograd/transforms/input_4x4_3x3_fp32.cpp | 2 +- .../winograd/transforms/input_6_3_fp32.cpp | 226 +++++++++++++++++++++ .../winograd/transforms/output_6_3_fp32.cpp | 186 +++++++++++++++++ .../winograd/transforms/weights_6_3_fp32.cpp | 125 ++++++++++++ 6 files changed, 540 insertions(+), 3 deletions(-) create mode 100644 src/core/NEON/kernels/convolution/winograd/transforms/input_6_3_fp32.cpp create mode 100644 src/core/NEON/kernels/convolution/winograd/transforms/output_6_3_fp32.cpp create mode 100644 src/core/NEON/kernels/convolution/winograd/transforms/weights_6_3_fp32.cpp (limited to 'src/core/NEON/kernels/convolution/winograd/transforms') diff --git a/src/core/NEON/kernels/convolution/winograd/transforms/input_2x2_3x3_fp32.cpp b/src/core/NEON/kernels/convolution/winograd/transforms/input_2x2_3x3_fp32.cpp index 6d8afc0def..97b2695d69 100644 --- a/src/core/NEON/kernels/convolution/winograd/transforms/input_2x2_3x3_fp32.cpp +++ b/src/core/NEON/kernels/convolution/winograd/transforms/input_2x2_3x3_fp32.cpp @@ -329,7 +329,7 @@ void Transform::process_tile( template <> template <> -const Transform::TileFn Transform::tile_fns[2][2][max_pad_bottom][max_pad_right] = +const Transform::TileFn Transform::tile_fns[n_pad_top][n_pad_left][n_pad_bottom][n_pad_right] = { { { diff --git a/src/core/NEON/kernels/convolution/winograd/transforms/input_2x2_5x5_fp32.cpp b/src/core/NEON/kernels/convolution/winograd/transforms/input_2x2_5x5_fp32.cpp index fd30b6118e..30c9463bb8 100644 --- a/src/core/NEON/kernels/convolution/winograd/transforms/input_2x2_5x5_fp32.cpp +++ b/src/core/NEON/kernels/convolution/winograd/transforms/input_2x2_5x5_fp32.cpp @@ -298,7 +298,7 @@ void Transform::process_tile( template <> template <> -const Transform::TileFn Transform::tile_fns[2][2][max_pad_bottom][max_pad_right] = +const Transform::TileFn Transform::tile_fns[n_pad_top][n_pad_left][n_pad_bottom][n_pad_right] = { { { diff --git a/src/core/NEON/kernels/convolution/winograd/transforms/input_4x4_3x3_fp32.cpp b/src/core/NEON/kernels/convolution/winograd/transforms/input_4x4_3x3_fp32.cpp index 04d1573e4c..7f93187132 100644 --- a/src/core/NEON/kernels/convolution/winograd/transforms/input_4x4_3x3_fp32.cpp +++ b/src/core/NEON/kernels/convolution/winograd/transforms/input_4x4_3x3_fp32.cpp @@ -326,7 +326,7 @@ void Transform::process_tile( */ template <> template <> -const Transform::TileFn Transform::tile_fns[2][2][max_pad_bottom][max_pad_right] = +const Transform::TileFn Transform::tile_fns[n_pad_top][n_pad_left][n_pad_bottom][n_pad_right] = { { { diff --git a/src/core/NEON/kernels/convolution/winograd/transforms/input_6_3_fp32.cpp b/src/core/NEON/kernels/convolution/winograd/transforms/input_6_3_fp32.cpp new file mode 100644 index 0000000000..67e46499cd --- /dev/null +++ b/src/core/NEON/kernels/convolution/winograd/transforms/input_6_3_fp32.cpp @@ -0,0 +1,226 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "arm_compute/core/NEON/kernels/convolution/winograd/transforms/input.hpp" +#include "arm_compute/core/NEON/kernels/convolution/winograd/winograd_gemm.hpp" +#include "arm_compute/core/NEON/kernels/convolution/common/arm.hpp" + + +namespace winograd +{ + +using Transform = WinogradGEMM<1, 6, 1, 3>::InputTransform; + +template <> +template <> +int Transform::ops_performed(const Tensor4DShape &input_shape) +{ + (void) input_shape; + return 0; // TODO +} + +template <> +template <> +template +void Transform::process_tile( + int n_channels, + const float* const input_base, + const int input_row_stride, + const int input_col_stride, + float* const matrix_base, + const int matrix_stride +) +{ + (void) input_row_stride; // No rows over which to stride + constexpr int inner_tile_j = 8; + constexpr int cells_j = inner_tile_j - pad_right; + + float *outptr = matrix_base; + + // Get pointers into the input tile + const float *x_ptrs[inner_tile_j]; + for (int j = pad_left, xj = 0; j < cells_j; j++, xj++) + { + x_ptrs[j] = input_base + xj*input_col_stride; + } + + // Vectors used/computed in this kernel. + float x[inner_tile_j]; + float U[inner_tile_j]; + + for (int j = 0; j < inner_tile_j; j++) + { + x[j] = 0.0f; + } + + // Perform the Winograd input transformation for each channel in the input + // tensor. + int channels_remaining = n_channels; +#ifdef __arm_any__ + for (; channels_remaining >= 4; channels_remaining -= 4) + { + float32x4_t x[inner_tile_j], U[inner_tile_j]; + for (int j = 0; j < inner_tile_cols; j++) + { + x[j] = vdupq_n_f32(0.0f); + } + + // Load x + for (int j = pad_left; j < cells_j; j++) + { + x[j] = vld1q_f32(x_ptrs[j]); + x_ptrs[j] += 4; + } + + // Compute U = x . X + U[0] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(x[6], 1), x[2], 49), x[4], -14), x[0], -36); + U[1] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(x[6], 1), x[2], 36), x[3], 13), x[4], -13), x[1], -36), x[5], -1); + U[2] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(x[6], 1), x[5], 1), x[2], 36), x[1], 36), x[4], -13), x[3], -13); + U[3] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(x[6], 1), x[3], 20), x[2], 9), x[5], -2), x[4], -10), x[1], -18); + U[4] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(x[6], 1), x[1], 18), x[2], 9), x[5], 2), x[4], -10), x[3], -20); + U[5] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(x[6], 1), x[3], 15), x[2], 4), x[5], -3), x[4], -5), x[1], -12); + U[6] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(x[6], 1), x[1], 12), x[2], 4), x[5], 3), x[4], -5), x[3], -15); + U[7] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(x[7], 1), x[3], 49), x[5], -14), x[1], -36); + + // Store the transformed vector + for (int j = 0; j < inner_tile_j; j++) + { + vst1q_f32(outptr + j*matrix_stride, U[j]); + } + outptr += 4; + } + + for (; channels_remaining >= 2; channels_remaining -= 2) + { + float32x2_t x[inner_tile_j], U[inner_tile_j]; + for (int j = 0; j < inner_tile_cols; j++) + { + x[j] = vdup_n_f32(0.0f); + } + + // Load x + for (int j = pad_left; j < cells_j; j++) + { + x[j] = vld1_f32(x_ptrs[j]); + x_ptrs[j] += 2; + } + + // Compute U = x . X + U[0] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(x[6], 1), x[2], 49), x[4], -14), x[0], -36); + U[1] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(x[6], 1), x[2], 36), x[3], 13), x[4], -13), x[1], -36), x[5], -1); + U[2] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(x[6], 1), x[5], 1), x[2], 36), x[1], 36), x[4], -13), x[3], -13); + U[3] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(x[6], 1), x[3], 20), x[2], 9), x[5], -2), x[4], -10), x[1], -18); + U[4] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(x[6], 1), x[1], 18), x[2], 9), x[5], 2), x[4], -10), x[3], -20); + U[5] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(x[6], 1), x[3], 15), x[2], 4), x[5], -3), x[4], -5), x[1], -12); + U[6] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(x[6], 1), x[1], 12), x[2], 4), x[5], 3), x[4], -5), x[3], -15); + U[7] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(x[7], 1), x[3], 49), x[5], -14), x[1], -36); + + // Store the transformed vector + for (int j = 0; j < inner_tile_j; j++) + { + vst1_f32(outptr + j*matrix_stride, U[j]); + } + outptr += 2; + } +#endif // __arm_any__ + for (; channels_remaining; channels_remaining--) + { + // Load x + for (int j = pad_left; j < cells_j; j++) + { + x[j] = *(x_ptrs[j]++); + } + + // Compute U = x . X + U[0] = x[0]*-36 + x[4]*-14 + x[2]*49 + x[6]*1; + U[1] = x[5]*-1 + x[1]*-36 + x[4]*-13 + x[3]*13 + x[2]*36 + x[6]*1; + U[2] = x[3]*-13 + x[4]*-13 + x[1]*36 + x[2]*36 + x[5]*1 + x[6]*1; + U[3] = x[1]*-18 + x[4]*-10 + x[5]*-2 + x[2]*9 + x[3]*20 + x[6]*1; + U[4] = x[3]*-20 + x[4]*-10 + x[5]*2 + x[2]*9 + x[1]*18 + x[6]*1; + U[5] = x[1]*-12 + x[4]*-5 + x[5]*-3 + x[2]*4 + x[3]*15 + x[6]*1; + U[6] = x[3]*-15 + x[4]*-5 + x[5]*3 + x[2]*4 + x[1]*12 + x[6]*1; + U[7] = x[1]*-36 + x[5]*-14 + x[3]*49 + x[7]*1; + + // Store the transformed vector + for (int j = 0; j < inner_tile_j; j++) + { + *(outptr + j*matrix_stride) = U[j]; + } + outptr++; + } +} + +template <> +template <> +const Transform::TileFn Transform::tile_fns[n_pad_top][n_pad_left][n_pad_bottom][n_pad_right] = +{ + { + { + { + Transform::template process_tile<0, 0, 0, 0>, + Transform::template process_tile<0, 0, 0, 1>, + Transform::template process_tile<0, 0, 0, 2>, + Transform::template process_tile<0, 0, 0, 3>, + Transform::template process_tile<0, 0, 0, 4>, + Transform::template process_tile<0, 0, 0, 5>, + Transform::template process_tile<0, 0, 0, 6>, + } + }, + { + { + Transform::template process_tile<0, 1, 0, 0>, + Transform::template process_tile<0, 1, 0, 1>, + Transform::template process_tile<0, 1, 0, 2>, + Transform::template process_tile<0, 1, 0, 3>, + Transform::template process_tile<0, 1, 0, 4>, + Transform::template process_tile<0, 1, 0, 5>, + Transform::template process_tile<0, 1, 0, 6>, + } + } + } +}; + +template +using TransformTransposed = typename WinogradGEMM::template InputTransform; + +template <> +template <> +const TransformTransposed<6, 3>::TileFn + TransformTransposed<6, 3>::tile_fns[n_pad_top][n_pad_left][n_pad_bottom][n_pad_right] = {}; + +template <> +template <> +const TransformTransposed<4, 5>::TileFn + TransformTransposed<4, 5>::tile_fns[n_pad_top][n_pad_left][n_pad_bottom][n_pad_right] = {}; + +template <> +template <> +const TransformTransposed<2, 7>::TileFn + TransformTransposed<2, 7>::tile_fns[n_pad_top][n_pad_left][n_pad_bottom][n_pad_right] = {}; + + + +template struct WinogradGEMM<1, 6, 1, 3>::InputTransform; +template struct WinogradGEMM<6, 1, 3, 1>::InputTransform; +} // namespace winograd diff --git a/src/core/NEON/kernels/convolution/winograd/transforms/output_6_3_fp32.cpp b/src/core/NEON/kernels/convolution/winograd/transforms/output_6_3_fp32.cpp new file mode 100644 index 0000000000..16667ccdb6 --- /dev/null +++ b/src/core/NEON/kernels/convolution/winograd/transforms/output_6_3_fp32.cpp @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "arm_compute/core/NEON/kernels/convolution/winograd/transforms/output.hpp" +#include "arm_compute/core/NEON/kernels/convolution/winograd/winograd_gemm.hpp" +#include "arm_compute/core/NEON/kernels/convolution/common/arm.hpp" + +namespace winograd +{ + +using Transform = WinogradGEMM<1, 6, 1, 3>::OutputTransform; +using TransformTransposed = WinogradGEMM<6, 1, 3, 1>::OutputTransform; + +template <> +template <> +int Transform::ops_performed(const Tensor4DShape &shape) +{ + (void) shape; + return 0; // TODO +} + +template <> +template <> +template +void Transform::process_tile( + const int n_channels, + const float* const matrix_base, + const int matrix_stride, + const float* const biases, + float* const output, + const int output_row_stride, + const int output_col_stride +) +{ + (void) output_row_stride; + constexpr int cells_j = output_tile_cols - pad_right; + + // Construct a map to the output cells + float *outptrs[cells_j]; + for (int j = 0; j < cells_j; j++) + { + outptrs[j] = output + j*output_col_stride; + } + const float *inptr = matrix_base; + const float *bptr = biases; + + // For each channel of the output + int channels_remaining = n_channels; +#ifdef __arm_any__ + for (; channels_remaining >= 4; channels_remaining -= 4) + { + // Matrices used and computed during this transform + float32x4_t F[inner_tile_cols], f[output_tile_cols], b = vdupq_n_f32(0.0f); + + // Read a 1x8 tile in the Winograd domain + for (int j = 0; j < inner_tile_cols; j++) + { + F[j] = vld1q_f32(inptr + j*matrix_stride); + } + inptr += 4; + + f[0] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[6], 1), F[5], 1), F[4], 1), F[3], 1), F[2], 1), F[1], 1), F[0], 1); + f[1] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[2], 1), F[6], 3), F[4], 2), F[3], -2), F[5], -3), F[1], -1); + f[2] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[2], 1), F[1], 1), F[6], 9), F[5], 9), F[4], 4), F[3], 4); + f[3] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[2], 1), F[6], 27), F[4], 8), F[3], -8), F[5], -27), F[1], -1); + f[4] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[2], 1), F[1], 1), F[6], 81), F[5], 81), F[4], 16), F[3], 16); + f[5] = vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmlaq_n_f32(vmulq_n_f32(F[7], 1), F[2], 1), F[6], 243), F[4], 32), F[3], -32), F[5], -243), F[1], -1); + + // Write out the output tile + if (bptr != 0) + { + b = vld1q_f32(bptr); + bptr += 4; + } + for (int j = 0; j < cells_j; j++) + { + vst1q_f32(outptrs[j], f[j] + b); + outptrs[j] += 4; + } + } + for (; channels_remaining >= 2; channels_remaining -= 2) + { + // Matrices used and computed during this transform + float32x2_t F[inner_tile_cols], f[output_tile_cols], b = vdup_n_f32(0.0f); + + // Read a 1x8 tile in the Winograd domain + for (int j = 0; j < inner_tile_cols; j++) + { + F[j] = vld1_f32(inptr + j*matrix_stride); + } + inptr += 2; + + f[0] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[6], 1), F[5], 1), F[4], 1), F[3], 1), F[2], 1), F[1], 1), F[0], 1); + f[1] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[2], 1), F[6], 3), F[4], 2), F[3], -2), F[5], -3), F[1], -1); + f[2] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[2], 1), F[1], 1), F[6], 9), F[5], 9), F[4], 4), F[3], 4); + f[3] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[2], 1), F[6], 27), F[4], 8), F[3], -8), F[5], -27), F[1], -1); + f[4] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[2], 1), F[1], 1), F[6], 81), F[5], 81), F[4], 16), F[3], 16); + f[5] = vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmla_n_f32(vmul_n_f32(F[7], 1), F[2], 1), F[6], 243), F[4], 32), F[3], -32), F[5], -243), F[1], -1); + + // Write out the output tile + if (bptr != 0) + { + b = vld1_f32(bptr); + bptr += 2; + } + for (int j = 0; j < cells_j; j++) + { + vst1_f32(outptrs[j], f[j] + b); + outptrs[j] += 2; + } + } +#endif // __arm_any__ + for (; channels_remaining; channels_remaining--) + { + // Matrices used and computed during this transform + float F[inner_tile_cols], f[output_tile_cols], b = 0.0f; + + // Read a 1x8 tile in the Winograd domain + for (int j = 0; j < inner_tile_cols; j++) + { + F[j] = *(inptr + j*matrix_stride); + } + inptr++; + + f[0] = F[0]*1 + F[1]*1 + F[2]*1 + F[3]*1 + F[4]*1 + F[5]*1 + F[6]*1; + f[1] = F[1]*-1 + F[5]*-3 + F[3]*-2 + F[4]*2 + F[6]*3 + F[2]*1; + f[2] = F[3]*4 + F[4]*4 + F[5]*9 + F[6]*9 + F[1]*1 + F[2]*1; + f[3] = F[1]*-1 + F[5]*-27 + F[3]*-8 + F[4]*8 + F[6]*27 + F[2]*1; + f[4] = F[3]*16 + F[4]*16 + F[5]*81 + F[6]*81 + F[1]*1 + F[2]*1; + f[5] = F[1]*-1 + F[5]*-243 + F[3]*-32 + F[4]*32 + F[6]*243 + F[2]*1 + F[7]*1; + + // Write out the output tile + if (bptr != 0) + { + b = *(bptr++); + } + for (int j = 0; j < cells_j; j++) + { + *(outptrs[j]++) = f[j] + b; + } + } +} + +template <> +template <> +const Transform::TileFn Transform::tile_fns[max_pad_bottom][max_pad_right] = +{ + { + Transform::template process_tile<0, 0>, + Transform::template process_tile<0, 1>, + Transform::template process_tile<0, 2>, + Transform::template process_tile<0, 3>, + Transform::template process_tile<0, 4>, + Transform::template process_tile<0, 5>, + }, +}; + +template <> +template <> +const TransformTransposed::TileFn TransformTransposed::tile_fns[max_pad_bottom][max_pad_right] = {}; + + +template struct WinogradGEMM<1, 6, 1, 3>::OutputTransform; +template struct WinogradGEMM<6, 1, 3, 1>::OutputTransform; +} // namespace winograd diff --git a/src/core/NEON/kernels/convolution/winograd/transforms/weights_6_3_fp32.cpp b/src/core/NEON/kernels/convolution/winograd/transforms/weights_6_3_fp32.cpp new file mode 100644 index 0000000000..c560aa8c8f --- /dev/null +++ b/src/core/NEON/kernels/convolution/winograd/transforms/weights_6_3_fp32.cpp @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "arm_compute/core/NEON/kernels/convolution/common/arm.hpp" +#include "arm_compute/core/NEON/kernels/convolution/winograd/winograd_gemm.hpp" +#include "arm_compute/core/NEON/kernels/convolution/winograd/transforms/kernel.hpp" + + +namespace winograd +{ + template <> + template <> + void WinogradGEMM<1, 6, 1, 3>::WeightsTransform::execute( + const int n_output_channels, + const int n_input_channels, + const float* const input, // NOTE: Data in HWIO order + float* const output, + const int matrix_stride, + const int matrix_row_stride + ) + { + // Get pointers to each cell of the weight tensor + const auto weight_col_stride = n_input_channels * n_output_channels; + const float *inptrs[3]; + for (int j = 0; j < 3; j++) + { + inptrs[j] = input + j*weight_col_stride; + } + + // For each input channel + for (int ic = 0; ic < n_input_channels; ic++) + { + float *outptr = output + ic * matrix_row_stride; + + // For each output channel + int channels_remaining = n_output_channels; + for (; channels_remaining; channels_remaining--) + { + // Matrices used and computed in this kernel + float w[3], V[inner_tile_cols]; + + // Read weights + for (int j = 0; j < 3; j++) + { + w[j] = *(inptrs[j]++); + } + + // Compute V = w WT + V[0] = (w[0]*-1) / 36.0f; + V[1] = (w[1]*-1 + w[0]*1 + w[2]*1) / 48.0f; + V[2] = (w[0]*1 + w[1]*1 + w[2]*1) / 48.0f; + V[3] = (w[0]*-1 + w[2]*-4 + w[1]*2) / 120.0f; + V[4] = (w[0]*-1 + w[2]*-4 + w[1]*-2) / 120.0f; + V[5] = (w[1]*-3 + w[2]*9 + w[0]*1) / 720.0f; + V[6] = (w[1]*3 + w[2]*9 + w[0]*1) / 720.0f; + V[7] = (w[2]*1) / 1; + + // Store the transformed weights + for (int j = 0; j < inner_tile_cols; j++) + { + *(outptr + j*matrix_stride) = V[j]; + } + outptr++; + } + } + } + + template <> + template <> + int WinogradGEMM<1, 6, 1, 3>::WeightsTransform::ops_performed(const KernelShape &shape) + { + (void) shape; + return 0; // TODO + } + + template <> + template <> + void WinogradGEMM<6, 1, 3, 1>::WeightsTransform::execute( + const int n_output_channels, + const int n_input_channels, + const float* const input, // NOTE: Data in HWIO order + float* const output, + const int matrix_stride, + const int matrix_row_stride + ) + { + // Redirect to the 1xN implementation + WinogradGEMM<1, 6, 1, 3>::template WeightsTransform::execute( + n_output_channels, n_input_channels, input, output, matrix_stride, + matrix_row_stride + ); + } + + template <> + template <> + int WinogradGEMM<6, 1, 3, 1>::WeightsTransform::ops_performed(const KernelShape &shape) + { + (void) shape; + return 0; // TODO + } + + template struct WinogradGEMM<1, 6, 1, 3>::WeightsTransform; + template struct WinogradGEMM<6, 1, 3, 1>::WeightsTransform; +} -- cgit v1.2.1