diff options
Diffstat (limited to 'arm_compute/core/NEON/kernels/convolution/depthwise/impl_base.hpp')
-rw-r--r-- | arm_compute/core/NEON/kernels/convolution/depthwise/impl_base.hpp | 348 |
1 files changed, 348 insertions, 0 deletions
diff --git a/arm_compute/core/NEON/kernels/convolution/depthwise/impl_base.hpp b/arm_compute/core/NEON/kernels/convolution/depthwise/impl_base.hpp new file mode 100644 index 0000000000..f9671fc426 --- /dev/null +++ b/arm_compute/core/NEON/kernels/convolution/depthwise/impl_base.hpp @@ -0,0 +1,348 @@ +/* + * Copyright (c) 2018 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* + * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + * + * NOTE: Header to be included by implementation files only. + * + * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + */ + +#include <algorithm> +#include "arm_compute/core/NEON/kernels/convolution/depthwise/depthwise.hpp" +#include "arm_compute/core/NEON/kernels/convolution/common/utils.hpp" + +#pragma once + +namespace depthwise +{ + +template <int OTR, int OTC, int KR, int KC, int SR, int SC, typename TIn, typename TOut> +int DepthwiseConvolution<OTR, OTC, KR, KC, SR, SC, TIn, TOut>::get_output_size( + const int dim_size, const bool same_padding +) +{ + return iceildiv(dim_size - (same_padding ? 0 : (KC - 1)), SR); +} + + +template <int OTR, int OTC, int KR, int KC, int SR, int SC, typename TIn, typename TOut> +DepthwiseConvolution<OTR, OTC, KR, KC, SR, SC, TIn, TOut>::DepthwiseConvolution( + const int n_batches, const int n_input_rows, const int n_input_cols, + const int n_channels, const bool padding_same, + const TIn* const weights, + const TIn* const input, + TOut* const output +) : _weights(weights), _input(input), _output(output), + _n_batches(n_batches), + _n_input_rows(n_input_rows), + _n_input_cols(n_input_cols), + _n_channels(n_channels), + _n_output_rows(get_output_size(n_input_rows, padding_same)), + _n_output_cols(get_output_size(n_input_cols, padding_same)), + _n_tile_rows(iceildiv(_n_output_rows, output_tile_rows)), + _n_tile_cols(iceildiv(_n_output_cols, output_tile_cols)), + _padding_same(padding_same) +{ +} + + +template <int OTR, int OTC, int KR, int KC, int SR, int SC, typename TIn, typename TOut> +unsigned int DepthwiseConvolution<OTR, OTC, KR, KC, SR, SC, TIn, TOut>::get_window() const +{ + // TODO Later support parallelisation over tile rows. + return 1; // _n_tile_rows; +} + + +template <int OTR, int OTC, int KR, int KC, int SR, int SC, typename TIn, typename TOut> +void DepthwiseConvolution<OTR, OTC, KR, KC, SR, SC, TIn, TOut>::run( + const unsigned int start, + const unsigned int stop +) +{ + // TODO Later support parallelisation over tile rows. + (void) start; + (void) stop; + + // Compute input striding + const int input_col_stride = _n_channels; + const int input_row_stride = _n_input_cols * input_col_stride; + const int input_batch_stride = _n_input_rows * input_row_stride; + + // Compute output striding + const int output_col_stride = _n_channels; + const int output_row_stride = _n_output_cols * output_col_stride; + const int output_batch_stride = _n_output_rows * output_row_stride; + + // Compute top and bottom padding for input and output + const int input_pad_top = _padding_same ? + ((_n_output_rows - 1)*stride_rows + kernel_rows - _n_input_rows) / 2 : 0; + const int input_pad_left = _padding_same ? + ((_n_output_cols - 1)*stride_cols + kernel_cols - _n_input_cols) / 2 : 0; + constexpr int tile_overlap = kernel_rows - 1; + + // Perform the convolution by calling `process_tile_row` for each tile row in + // each batch. + for (int batch = 0; batch < _n_batches; batch++) + { + const TIn* const inptr_batch = _input + batch*input_batch_stride; + TOut* const outptr_batch = _output + batch*output_batch_stride; + + // Loop over rows of tiles + for (int tile_i = 0; tile_i < _n_tile_rows; tile_i++) + { + // Pointer to the row + const int input_row_offset = (tile_i == 0) ? 0 : input_pad_top; + const TIn* const inptr_row = (inptr_batch + ((inner_tile_rows - tile_overlap)*tile_i - input_row_offset)*input_row_stride); + TOut* const outptr_row = outptr_batch + output_tile_rows * tile_i * output_row_stride; + + // Input padding (top + bottom) for the row + const int input_row_top = tile_i*(inner_tile_rows - tile_overlap) - input_pad_top; + const int input_row_bottom = input_row_top + inner_tile_rows; + const int input_row_pad_top = (tile_i == 0) ? input_pad_top : 0; + const int input_row_pad_bottom = std::max(0, input_row_bottom - _n_input_rows); + + // Output padding (bottom) for the row + const int output_row_bottom = (tile_i + 1)*output_tile_rows; + const int output_row_pad_bottom = std::max(0, output_row_bottom - _n_output_rows); + + // Process the row + process_tile_row( + _n_channels, _weights, + inptr_row, input_row_stride, input_col_stride, + outptr_row, output_row_stride, output_col_stride, + input_row_pad_top, input_pad_left, input_row_pad_bottom, + output_row_pad_bottom, + _n_tile_cols, _n_input_cols, _n_output_cols + ); + } + } +} + + +template <int OTR, int OTC, int KR, int KC, int SR, int SC, typename TIn, typename TOut> +void DepthwiseConvolution<OTR, OTC, KR, KC, SR, SC, TIn, TOut>::process_tile_row( + const int n_channels, + const TIn* const weights, + const TIn* const inptr, + const int in_row_stride, + const int in_col_stride, + TOut* const outptr, + const int out_row_stride, + const int out_col_stride, + const int row_pad_in_top, + const int row_pad_in_left, + const int row_pad_in_bottom, + const int row_pad_out_bottom, + const int n_tiles, + const int n_input_cols, + const int n_output_cols +) +{ + constexpr int tile_overlap = kernel_cols - 1; + + // Loop over columns of tiles + for (int tile_j = 0; tile_j < n_tiles; tile_j++) + { + // Input padding (left + right) for the tile + const int t_pad_in_left = (tile_j == 0) ? row_pad_in_left : 0; + const int t_in_start = tile_j*(inner_tile_cols - tile_overlap) - row_pad_in_left; + const int t_in_end = t_in_start + inner_tile_cols; + const int t_pad_in_right = std::max(0, t_in_end - n_input_cols); + + // Output padding (right) for the tile + const int t_out_end = (tile_j + 1) * output_tile_cols; + const int t_pad_out_right = std::max(0, t_out_end - n_output_cols); + + // Get pointers into the inputs and outputs + const int col_offset = (tile_j == 0) ? 0 : row_pad_in_left; + const TIn* const inptr_col = (inptr + ((inner_tile_cols - tile_overlap)*tile_j - col_offset)*in_col_stride); + TOut* const outptr_col = outptr + tile_j * output_tile_cols * out_col_stride; + + // Apply the specific tile processing function + tile_fns[row_pad_in_top][t_pad_in_left][row_pad_in_bottom][t_pad_in_right][row_pad_out_bottom][t_pad_out_right]( + n_channels, weights, + inptr_col, in_row_stride, in_col_stride, + outptr_col, out_row_stride, out_col_stride + ); + } +} + + +template <int OTR, int OTC, int KR, int KC, int SR, int SC, typename TIn, typename TOut> +template < + int in_pad_top, int in_pad_left, int in_pad_bottom, int in_pad_right, + int out_pad_bottom, int out_pad_right +> +void DepthwiseConvolution<OTR, OTC, KR, KC, SR, SC, TIn, TOut>::process_tile( + const int n_channels, + const TIn* const weights, + const TIn* const inptr, + const int in_row_stride, + const int in_col_stride, + TOut* const outptr, + const int out_row_stride, + const int out_col_stride +) +{ + // Compute valid ranges of the tile + constexpr int in_cells_i = inner_tile_rows - in_pad_bottom; + constexpr int in_cells_j = inner_tile_cols - in_pad_right; + constexpr int out_cells_i = output_tile_rows - out_pad_bottom; + constexpr int out_cells_j = output_tile_cols - out_pad_right; + + // Instantiate pointers + const TIn* inptr_base = inptr; + const TIn* wptr_base = weights; + TOut* outptr_base = outptr; + + const int weight_col_stride = n_channels; + const int weight_row_stride = kernel_cols * n_channels; + + // Perform the depthwise convolution + int channels_remaining = n_channels; + for (; channels_remaining; channels_remaining--) + { + // Load input tile + TIn u[inner_tile_rows][inner_tile_cols]; + for (int i = 0; i < inner_tile_rows; i++) + { + const TIn* const inptr_row = inptr_base + (i - in_pad_top)*in_row_stride; + for (int j = 0; j < inner_tile_cols; j++) + { + if (i < in_pad_top || in_cells_i <= i || + j < in_pad_left || in_cells_j <= j) + { + u[i][j] = static_cast<TIn>(0); + } + else + { + u[i][j] = *(inptr_row + (j - in_pad_left)*in_col_stride); + } + } + } + inptr_base++; + + // Load weights tile + TIn w[kernel_rows][kernel_cols]; + for (int i = 0; i < kernel_rows; i++) + { + const TIn* const wptr_row = wptr_base + i*weight_row_stride; + for (int j = 0; j < kernel_cols; j++) + { + w[i][j] = *(wptr_row + j*weight_col_stride); + } + } + wptr_base++; + + // Perform the convolution + TOut v[out_cells_i][out_cells_j]; + for (int out_i = 0; out_i < out_cells_i; out_i++) + { + for (int out_j = 0; out_j < out_cells_j; out_j++) + { + // Clear the accumulator + v[out_i][out_j] = static_cast<TOut>(0); + + // Base co-ordinate + const int base_i = out_i * stride_rows; + const int base_j = out_j * stride_cols; + + // Fill the accumulator + for (int in_i = 0; in_i < kernel_rows; in_i++) + { + const int i = base_i + in_i; + for (int in_j = 0; in_j < kernel_cols; in_j++) + { + const int j = base_j + in_j; + v[out_i][out_j] += w[in_i][in_j] * u[i][j]; + } + } + } + } + + // Store the output tile + for (int i = 0; i < out_cells_i; i++) + { + TOut* const outptr_row = outptr_base + i*out_row_stride; + for (int j = 0; j < out_cells_j; j++) + { + *(outptr_row + j*out_col_stride) = v[i][j]; + } + } + outptr_base++; + } +} + + +// New templated struct used solely as a way to provide tile processing +// specialisations. +template <int OutputTileRows, int OutputTileCols, + int KernelRows, int KernelCols, + int StrideRows, int StrideCols, + typename TIn, typename TOut> +struct DepthwiseConvolutionImpl : public DepthwiseConvolution< + OutputTileRows, OutputTileCols, + KernelRows, KernelCols, + StrideRows, StrideCols, TIn, TOut +> +{ + template < + int in_pad_top, int in_pad_left, int in_pad_bottom, int in_pad_right, + int out_pad_bottom, int out_pad_right + > + static void process_tile( + const int n_channels, + const TIn* const weights, + const TIn* const inptr, + const int in_row_stride, + const int in_col_stride, + TOut* const outptr, + const int out_row_stride, + const int out_col_stride + ) + { + // By default, redirect to parent. Specialised implementations can be added + // by overriding this method. + DepthwiseConvolution<OutputTileRows, OutputTileCols, + KernelRows, KernelCols, + StrideRows, StrideCols, + TIn, TOut>:: + template process_tile<in_pad_top, in_pad_left, in_pad_bottom, in_pad_right, + out_pad_bottom, out_pad_right>( + n_channels, + weights, + inptr, + in_row_stride, + in_col_stride, + outptr, + out_row_stride, + out_col_stride + ); + } +}; + +} // namespace depthwise |