From 72686fa6ee0f04d458ed2274b4d34917628ef14d Mon Sep 17 00:00:00 2001 From: Pablo Tello Date: Mon, 3 Sep 2018 11:40:33 +0100 Subject: COMPMID-1550: Winograd integrate RSH changes. Refactors the transforms to make use of partial specialization. Change-Id: Idff68d22817a00a7ee9eef5351a5a9fd33147540 Reviewed-on: https://eu-gerrit-1.euhpc.arm.com/146635 Tested-by: Jenkins Reviewed-by: Georgios Pinitas --- .../convolution/winograd/transforms/input.hpp | 152 ++++++++++------- .../kernels/convolution/winograd/winograd_gemm.hpp | 123 +------------- .../winograd/winograd_input_transform.hpp | 184 +++++++++++++++++++++ 3 files changed, 280 insertions(+), 179 deletions(-) create mode 100644 arm_compute/core/NEON/kernels/convolution/winograd/winograd_input_transform.hpp (limited to 'arm_compute/core/NEON/kernels') diff --git a/arm_compute/core/NEON/kernels/convolution/winograd/transforms/input.hpp b/arm_compute/core/NEON/kernels/convolution/winograd/transforms/input.hpp index 369c2ff48f..473a13c3b0 100644 --- a/arm_compute/core/NEON/kernels/convolution/winograd/transforms/input.hpp +++ b/arm_compute/core/NEON/kernels/convolution/winograd/transforms/input.hpp @@ -29,10 +29,8 @@ namespace winograd { /***************************************************************************/ /* Instance-less API */ - template - template - void WinogradGEMM::InputTransform::execute( + template + void InputTransformImpl::execute( const T* const input, /** Input tensor data */ const int n_batches, /** Number of batches in input tensor. */ const int in_batch_stride, /** Stride between batches of the input. */ @@ -50,26 +48,9 @@ namespace winograd const int matrix_row_stride /** Stride within matrices. */ ) { - // If an Nx1 kernel then transpose and redirect to the 1xN implementation - if (kernel_cols == 1) - { - WinogradGEMM:: - template InputTransform::execute( - input, - n_batches, in_batch_stride, - n_cols, in_col_stride, - n_rows, in_row_stride, - n_channels, padding, - tile_N, tile_M, - output, matrix_stride, matrix_batch_stride, matrix_row_stride - ); - return; - } - // Compute the padding required on each edge of the image - const int pad_top = (padding == PADDING_SAME) ? (kernel_rows - 1) / 2 : 0; - const int pad_left = (padding == PADDING_SAME) ? (kernel_cols - 1) / 2 : 0; - const int tile_overlap = kernel_rows - 1; + const int pad_top = (padding == PADDING_SAME) ? (KernelRows - 1) / 2 : 0; + const int pad_left = (padding == PADDING_SAME) ? (KernelCols - 1) / 2 : 0; // Compute striding values (assuming NHWC ordered data) const int output_col_stride = matrix_row_stride; @@ -85,19 +66,19 @@ namespace winograd // Loop over rows of tiles for (int tile_i = 0; tile_i < tile_M; tile_i++) { + // Padding (top + bottom) for the row + const int row_top = tile_i*(InnerTileRows - overlap_rows) - pad_top; + const int row_bottom = row_top + InnerTileRows; + const int row_pad_top = std::max(0, pad_top - tile_i*(InnerTileRows - overlap_rows)); + const int row_pad_bottom = (row_bottom <= n_rows) ? 0 : row_bottom - n_rows; + // Pointer to the row - const int row_offset = (tile_i == 0) ? 0 : pad_top; + const int row_offset = std::min(0, row_pad_top - pad_top); const T* const input_base_row = ( - input_base_batch + ((inner_tile_rows - (kernel_rows - 1))*tile_i - row_offset)*in_row_stride + input_base_batch + ((InnerTileRows - overlap_rows)*tile_i + row_offset)*in_row_stride ); T* const outptr_base_row = outptr_base_batch + tile_i*output_row_stride; - // Padding (top + bottom) for the row - const int row_top = tile_i*(inner_tile_rows - tile_overlap) - pad_top; - const int row_bottom = row_top + inner_tile_rows; - const int row_pad_top = (tile_i == 0) ? pad_top : 0; - const int row_pad_bottom = (row_bottom <= n_rows) ? 0 : row_bottom - n_rows; - // Process the row process_tile_row( tile_N, n_channels, @@ -109,10 +90,40 @@ namespace winograd } } - template - template - void WinogradGEMM::InputTransform::process_tile_row( + + template + void InputTransformImpl::execute( + const T* const input, /** Input tensor data */ + const int n_batches, /** Number of batches in input tensor. */ + const int in_batch_stride, /** Stride between batches of the input. */ + const int n_rows, /** Number of rows in input tensor. */ + const int in_row_stride, /** Stride between rows of the input. */ + const int n_cols, /** Number of columns in input tensor. */ + const int in_col_stride, /** Stride between columns of the input. */ + const int n_channels, /** Number of channels in input tensor. */ + const PaddingType padding, /** Padding type. */ + const int tile_M, + const int tile_N, + T* const output, /** Base of output matrices. */ + const int matrix_stride, /** Stride between output matrices. */ + const int matrix_batch_stride, /** Stride between batches within the matrix. */ + const int matrix_row_stride /** Stride within matrices. */ + ) + { + // If an Nx1 kernel then transpose and redirect to the 1xN implementation + InputTransformImpl<1, KernelRows, 1, InnerTileRows, T>::execute( + input, + n_batches, in_batch_stride, + n_cols, in_col_stride, + n_rows, in_row_stride, + n_channels, padding, + tile_N, tile_M, + output, matrix_stride, matrix_batch_stride, matrix_row_stride + ); + } + + template + void InputTransformImpl::process_tile_row( const int tile_N, int n_channels, const T* const input_base, @@ -127,33 +138,25 @@ namespace winograd const int n_cols ) { - if (kernel_cols == 1) - { - // If an Nx1 implementation then this should never be reached. - return; - } - - constexpr int tile_overlap = kernel_cols - 1; - // Loop over columns of tiles for (int tile_j = 0; tile_j < tile_N; tile_j++) { // Padding (left + right) for the tile - const int t_pad_left = (tile_j == 0) ? row_pad_left : 0; - const int t_start = tile_j*(inner_tile_cols - tile_overlap) - row_pad_left; - const int t_end = t_start + inner_tile_cols; + const int t_start = tile_j*(InnerTileCols - overlap_cols) - row_pad_left; + const int t_end = t_start + InnerTileCols; + const int t_pad_left = std::max(0, row_pad_left - tile_j*(InnerTileCols - overlap_cols)); const int t_pad_right = (t_end <= n_cols) ? 0 : t_end - n_cols; // Get pointers into the inputs and outputs - const int col_offset = (tile_j == 0) ? 0 : row_pad_left; + const int col_offset = std::min(0, t_pad_left - row_pad_left); const T* const input_base_col = ( - input_base + ((inner_tile_cols - tile_overlap)*tile_j - col_offset)*input_col_stride + input_base + ((InnerTileCols - overlap_cols)*tile_j + col_offset)*input_col_stride ); T* const outptr = matrix_base + tile_j*matrix_row_stride; // Apply the specific tile processing function - const int f_pad_top = pad_top ? 1 : 0; - const int f_pad_left = t_pad_left ? 1 : 0; + const int f_pad_top = iceildiv(pad_top, 2); + const int f_pad_left = iceildiv(t_pad_left, 2); tile_fns[f_pad_top][f_pad_left][pad_bottom][t_pad_right]( n_channels, input_base_col, @@ -166,9 +169,8 @@ namespace winograd } /***************************************************************************/ - template - template - WinogradGEMM::InputTransform::InputTransform( + template + InputTransform::InputTransform( const T* const input, /** Input tensor data */ const int n_batches, /** Number of batches in input tensor. */ const int n_rows, /** Number of rows in input tensor. */ @@ -184,10 +186,10 @@ namespace winograd ) : _inptr(input), _outptr(output), _n_batches(n_batches), _n_rows(n_rows), _n_cols(n_cols), _n_channels(n_channels), _matrix_stride(matrix_stride), _matrix_row_stride(matrix_row_stride), - _tiles_M(iceildiv((padding == PADDING_SAME) ? n_rows : n_rows - kr + 1, - output_tile_rows)), - _tiles_N(iceildiv((padding == PADDING_SAME) ? n_cols : n_cols - kc + 1, - output_tile_cols)), + _tiles_M(iceildiv((padding == PADDING_SAME) ? n_rows : n_rows - KernelRows + 1, + InnerTileRows - KernelRows + 1)), + _tiles_N(iceildiv((padding == PADDING_SAME) ? n_cols : n_cols - KernelCols + 1, + InnerTileCols - KernelCols + 1)), _in_col_stride(in_col_stride ? in_col_stride : n_channels), _in_row_stride(in_row_stride ? in_row_stride : n_cols * _in_col_stride), _in_batch_stride(in_batch_stride ? in_batch_stride : n_rows * _in_row_stride), @@ -195,18 +197,16 @@ namespace winograd { } - template - template - unsigned int WinogradGEMM::InputTransform::get_window() const + template + unsigned int InputTransform::get_window() const { // The final window includes the tail, all other windows will be a multiple // of the window block in size. return iceildiv(_n_channels, WINDOW_BLOCK); } - template - template - void WinogradGEMM::InputTransform::run( + template + void InputTransform::run( const unsigned int start, const unsigned int stop ) { @@ -238,4 +238,30 @@ namespace winograd _matrix_row_stride ); } + + template + void InputTransform::execute( + const T* const input, /** Input tensor data */ + const int n_batches, /** Number of batches in input tensor. */ + const int in_batch_stride, /** Stride between batches of the input. */ + const int n_rows, /** Number of rows in input tensor. */ + const int in_row_stride, /** Stride between rows of the input. */ + const int n_cols, /** Number of columns in input tensor. */ + const int in_col_stride, /** Stride between columns of the input. */ + const int n_channels, /** Number of channels in input tensor. */ + const PaddingType padding, /** Padding type. */ + const int tile_M, + const int tile_N, + T* const output, /** Base of output matrices. */ + const int matrix_stride, /** Stride between output matrices. */ + const int matrix_batch_stride, /** Stride between batches within the matrix. */ + const int matrix_row_stride /** Stride within matrices. */ + ) + { + Transform::execute( + input, n_batches, in_batch_stride, n_rows, in_row_stride, n_cols, + in_col_stride, n_channels, padding, tile_M, tile_N, output, + matrix_stride, matrix_batch_stride, matrix_row_stride + ); + } } diff --git a/arm_compute/core/NEON/kernels/convolution/winograd/winograd_gemm.hpp b/arm_compute/core/NEON/kernels/convolution/winograd/winograd_gemm.hpp index 7098fc48a1..31aee35fab 100644 --- a/arm_compute/core/NEON/kernels/convolution/winograd/winograd_gemm.hpp +++ b/arm_compute/core/NEON/kernels/convolution/winograd/winograd_gemm.hpp @@ -30,7 +30,7 @@ #include "arm_compute/core/NEON/kernels/convolution/common/shims.hpp" #include "arm_compute/core/NEON/kernels/convolution/common/tensor.hpp" #include "arm_compute/core/NEON/kernels/convolution/common/utils.hpp" - +#include "winograd_input_transform.hpp" #include #include @@ -114,121 +114,12 @@ class WinogradGEMM /** Transform input feature maps from the spatial to the Winograd domain. */ template - struct InputTransform - { - /** Get the bytes read during the transform. */ - static size_t bytes_read(const Tensor4DShape &shape) - { - return shape.size() * sizeof(T); - } - - /** Get the bytes written during the transform. */ - static size_t bytes_written(const Tensor4DShape &shape) - { - const int M = iceildiv(shape.n_rows, inner_tile_rows) * - iceildiv(shape.n_cols, inner_tile_cols); - const int K = shape.n_channels; - return inner_tile_rows * inner_tile_cols * M * K * sizeof(T); - } - - /** Get the count of operations performed by the transform. */ - static int ops_performed(const Tensor4DShape &shape); - - /** Apply the transform to a tensor. */ - static void execute( - const T* const input, /** Input tensor data */ - const int n_batches, /** Number of batches in input tensor. */ - const int in_batch_stride, /** Stride between batches of the input. */ - const int n_rows, /** Number of rows in input tensor. */ - const int in_row_stride, /** Stride between rows of the input. */ - const int n_cols, /** Number of columns in input tensor. */ - const int in_col_stride, /** Stride between columns of the input. */ - const int n_channels, /** Number of channels in input tensor. */ - const PaddingType padding, /** Padding type. */ - const int tile_M, - const int tile_N, - T* const output, /** Base of output matrices. */ - const int matrix_stride, /** Stride between output matrices. */ - const int matrix_batch_stride, /** Stride between batches within the matrix. */ - const int matrix_row_stride /** Stride within matrices. */ - ); - - /***********************************************************************/ - /** Create an InputTransform operator fixed on a given problem and set of - * pointers. - */ - InputTransform( - const T* const input, /** Input tensor data */ - const int n_batches, /** Number of batches in input tensor. */ - const int n_rows, /** Number of rows in input tensor. */ - const int n_cols, /** Number of columns in input tensor. */ - const int n_channels, /** Number of channels in input tensor. */ - const PaddingType padding, /** Padding type. */ - T* const output, /** Base of output matrices. */ - const int matrix_stride, /** Stride between output matrices. */ - const int matrix_row_stride, /** Stride within matrices. */ - const int in_batch_stride=0, /** Stride between input batches. */ - const int in_row_stride=0, /** Stride between input rows. */ - const int in_col_stride=0 /** Stride between input columns. */ - ); - - /** Get the window of work a given operator can perform. */ - unsigned int get_window() const; - static constexpr unsigned int WINDOW_BLOCK = 16; // Base size of window - - /** Perform work upon a window of the input. */ - void run(const unsigned int start, const unsigned int stop); - /***********************************************************************/ - - private: - static void process_tile_row( - const int tile_N, - int n_channels, - const T* const input_base, - const int input_row_stride, - const int input_col_stride, - T* const matrix_base, - const int matrix_stride, - const int matrix_row_stride, - const int row_pad_top, - const int row_pad_left, - const int row_pad_bottom, - const int n_cols - ); - - // Tile overlaps - static constexpr int overlap_rows = kernel_rows - 1; - static constexpr int overlap_cols = kernel_cols - 1; - - // Maximum padding and number of distinct paddings - static constexpr int max_pad_top = kernel_rows / 2; - static constexpr int n_pad_top = 1 + iceildiv(max_pad_top, inner_tile_rows - overlap_rows); - - static constexpr int max_pad_left = kernel_cols / 2; - static constexpr int n_pad_left = 1 + iceildiv(max_pad_left, inner_tile_cols - overlap_cols); - - static constexpr int n_pad_bottom = inner_tile_rows; - static constexpr int n_pad_right = inner_tile_cols; - - - - /** Process a single tile of the input tensor. */ - template - static void process_tile(int, const T*, int, int, T*, int); - - // Array of methods to transform tiles of the input tensor. - typedef void (*TileFn)(int, const T*, int, int, T*, int); - static const TileFn - tile_fns[n_pad_top][n_pad_left][n_pad_bottom][n_pad_right]; - - /* Member values for instance-based API. */ - const T* const _inptr; - T* const _outptr; - const int _n_batches, _n_rows, _n_cols, _n_channels, _matrix_stride, - _matrix_row_stride, _tiles_M, _tiles_N; - const int _in_col_stride, _in_row_stride, _in_batch_stride; - const PaddingType _padding_type; - }; + using InputTransform = InputTransform< + KernelRows, KernelCols, + (OutputTileRows + KernelRows - 1), + (OutputTileCols + KernelCols - 1), + T + >; /** Transform output feature maps from the Winograd to the spatial domain. */ diff --git a/arm_compute/core/NEON/kernels/convolution/winograd/winograd_input_transform.hpp b/arm_compute/core/NEON/kernels/convolution/winograd/winograd_input_transform.hpp new file mode 100644 index 0000000000..abcda53534 --- /dev/null +++ b/arm_compute/core/NEON/kernels/convolution/winograd/winograd_input_transform.hpp @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2018 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#pragma once + +namespace winograd +{ + +namespace +{ + +template +class InputTransformImpl +{ + public: + /** Apply the transform to a tensor. */ + static void execute( + const T* const input, /** Input tensor data */ + const int n_batches, /** Number of batches in input tensor. */ + const int in_batch_stride, /** Stride between batches of the input. */ + const int n_rows, /** Number of rows in input tensor. */ + const int in_row_stride, /** Stride between rows of the input. */ + const int n_cols, /** Number of columns in input tensor. */ + const int in_col_stride, /** Stride between columns of the input. */ + const int n_channels, /** Number of channels in input tensor. */ + const PaddingType padding, /** Padding type. */ + const int tile_M, + const int tile_N, + T* const output, /** Base of output matrices. */ + const int matrix_stride, /** Stride between output matrices. */ + const int matrix_batch_stride, /** Stride between batches within the matrix. */ + const int matrix_row_stride /** Stride within matrices. */ + ); + + private: + static void process_tile_row( + const int tile_N, + int n_channels, + const T* const input_base, + const int input_row_stride, + const int input_col_stride, + T* const matrix_base, + const int matrix_stride, + const int matrix_row_stride, + const int row_pad_top, + const int row_pad_left, + const int row_pad_bottom, + const int n_cols + ); + + // Tile overlaps + static constexpr int overlap_rows = KernelRows - 1; + static constexpr int overlap_cols = KernelCols - 1; + + // Maximum padding and number of distinct paddings + static constexpr int max_pad_top = KernelRows / 2; + static constexpr int n_pad_top = 1 + iceildiv(max_pad_top, InnerTileRows - overlap_rows); + + static constexpr int max_pad_left = KernelCols / 2; + static constexpr int n_pad_left = 1 + iceildiv(max_pad_left, InnerTileCols - overlap_cols); + + static constexpr int n_pad_bottom = InnerTileRows; + static constexpr int n_pad_right = InnerTileCols; + + /** Process a single tile of the input tensor. */ + template + static void process_tile(int, const T*, int, int, T*, int); + + // Array of methods to transform tiles of the input tensor. + typedef void (*TileFn)(int, const T*, int, int, T*, int); + static const TileFn + tile_fns[n_pad_top][n_pad_left][n_pad_bottom][n_pad_right]; +}; + + +template +class InputTransformImpl +{ + public: + /** Apply the transform to a tensor. */ + static void execute( + const T* const input, /** Input tensor data */ + const int n_batches, /** Number of batches in input tensor. */ + const int in_batch_stride, /** Stride between batches of the input. */ + const int n_rows, /** Number of rows in input tensor. */ + const int in_row_stride, /** Stride between rows of the input. */ + const int n_cols, /** Number of columns in input tensor. */ + const int in_col_stride, /** Stride between columns of the input. */ + const int n_channels, /** Number of channels in input tensor. */ + const PaddingType padding, /** Padding type. */ + const int tile_M, + const int tile_N, + T* const output, /** Base of output matrices. */ + const int matrix_stride, /** Stride between output matrices. */ + const int matrix_batch_stride, /** Stride between batches within the matrix. */ + const int matrix_row_stride /** Stride within matrices. */ + ); +}; + +} // namespace (anonymous) + +template +class InputTransform +{ + public: + /***********************************************************************/ + /** Create an InputTransform operator fixed on a given problem and set of + * pointers. + */ + InputTransform( + const T* const input, /** Input tensor data */ + const int n_batches, /** Number of batches in input tensor. */ + const int n_rows, /** Number of rows in input tensor. */ + const int n_cols, /** Number of columns in input tensor. */ + const int n_channels, /** Number of channels in input tensor. */ + const PaddingType padding, /** Padding type. */ + T* const output, /** Base of output matrices. */ + const int matrix_stride, /** Stride between output matrices. */ + const int matrix_row_stride, /** Stride within matrices. */ + const int in_batch_stride=0, /** Stride between input batches. */ + const int in_row_stride=0, /** Stride between input rows. */ + const int in_col_stride=0 /** Stride between input columns. */ + ); + + /** Get the window of work a given operator can perform. */ + unsigned int get_window() const; + static constexpr unsigned int WINDOW_BLOCK = 16; // Base size of window + + /** Perform work upon a window of the input. */ + void run(const unsigned int start, const unsigned int stop); + + /** Apply the transform to a tensor. */ + static void execute( + const T* const input, /** Input tensor data */ + const int n_batches, /** Number of batches in input tensor. */ + const int in_batch_stride, /** Stride between batches of the input. */ + const int n_rows, /** Number of rows in input tensor. */ + const int in_row_stride, /** Stride between rows of the input. */ + const int n_cols, /** Number of columns in input tensor. */ + const int in_col_stride, /** Stride between columns of the input. */ + const int n_channels, /** Number of channels in input tensor. */ + const PaddingType padding, /** Padding type. */ + const int tile_M, + const int tile_N, + T* const output, /** Base of output matrices. */ + const int matrix_stride, /** Stride between output matrices. */ + const int matrix_batch_stride, /** Stride between batches within the matrix. */ + const int matrix_row_stride /** Stride within matrices. */ + ); + + protected: + using Transform = InputTransformImpl; + + /* Member values for instance-based API. */ + const T* const _inptr; + T* const _outptr; + const int _n_batches, _n_rows, _n_cols, _n_channels, _matrix_stride, + _matrix_row_stride, _tiles_M, _tiles_N; + const int _in_col_stride, _in_row_stride, _in_batch_stride; + const PaddingType _padding_type; +}; + +} // namespace winograd -- cgit v1.2.1