From 30271c779c36a2abe6995c4454674d92bbc1f91f Mon Sep 17 00:00:00 2001 From: Georgios Pinitas Date: Mon, 24 Jun 2019 14:56:34 +0100 Subject: COMPMID-2156: Optimized dilated convolution for NEON. Change-Id: I3a8abe8cc9637c8983d9bd69dcbaee1a15eac8d0 Signed-off-by: Georgios Pinitas Reviewed-on: https://review.mlplatform.org/c/1492 Comments-Addressed: Arm Jenkins Tested-by: Arm Jenkins Reviewed-by: Pablo Marquez --- .../kernels/convolution/depthwise/depthwise.hpp | 41 ++- .../convolution/depthwise/depthwise_dilated.hpp | 156 +++++++++++ .../convolution/depthwise/depthwise_quantized.hpp | 38 ++- .../depthwise/depthwise_quantized_dilated.hpp | 88 ++++++ .../kernels/convolution/depthwise/impl_base.hpp | 32 ++- .../kernels/convolution/depthwise/impl_dilated.hpp | 295 +++++++++++++++++++++ 6 files changed, 640 insertions(+), 10 deletions(-) create mode 100644 arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_dilated.hpp create mode 100644 arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_quantized_dilated.hpp create mode 100644 arm_compute/core/NEON/kernels/convolution/depthwise/impl_dilated.hpp (limited to 'arm_compute/core') diff --git a/arm_compute/core/NEON/kernels/convolution/depthwise/depthwise.hpp b/arm_compute/core/NEON/kernels/convolution/depthwise/depthwise.hpp index e0cb616a3d..a4a833d90a 100644 --- a/arm_compute/core/NEON/kernels/convolution/depthwise/depthwise.hpp +++ b/arm_compute/core/NEON/kernels/convolution/depthwise/depthwise.hpp @@ -25,8 +25,8 @@ #pragma once #include -#include "arm_compute/core/NEON/kernels/convolution/common/activation.hpp" -#include "arm_compute/core/NEON/kernels/convolution/common/padding.hpp" +#include "activation.hpp" +#include "padding.hpp" namespace depthwise { @@ -127,6 +127,23 @@ class DepthwiseConvolutionBase : public IDepthwiseConvolution unsigned int padding_right ); + /** Create a new depthwise convolution engine. + * + * @param[in] n_batches Number of batches tensors. + * @param[in] n_input_rows Number of rows in input tensor. + * @param[in] n_input_cols Number of columns in input tensor. + * @param[in] n_channels Number of channels in input and output tensors. + */ + DepthwiseConvolutionBase( + int n_batches, int n_input_rows, int n_input_cols, int n_channels, + int n_output_rows, int n_output_cols, + nck::ActivationFunction activation, + unsigned int padding_top, + unsigned int padding_left, + unsigned int padding_bottom, + unsigned int padding_right + ); + // Cannot copy or move a DepthwiseConvolution. DepthwiseConvolutionBase(DepthwiseConvolutionBase&) = delete; DepthwiseConvolutionBase operator=(DepthwiseConvolutionBase&) = delete; @@ -417,6 +434,16 @@ class DepthwiseConvolution< unsigned int padding_right ); + DepthwiseConvolution( + int n_batches, int n_input_rows, int n_input_cols, int n_channels, + int n_output_rows, int n_output_cols, + nck::ActivationFunction activation, + unsigned int padding_top, + unsigned int padding_left, + unsigned int padding_bottom, + unsigned int padding_right + ); + protected: template void execute_tile( @@ -488,6 +515,16 @@ class DepthwiseConvolution< unsigned int padding_right ); + DepthwiseConvolution( + int n_batches, int n_input_rows, int n_input_cols, int n_channels, + int n_output_rows, int n_output_cols, + nck::ActivationFunction activation, + unsigned int padding_top, + unsigned int padding_left, + unsigned int padding_bottom, + unsigned int padding_right + ); + protected: template void execute_tile( diff --git a/arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_dilated.hpp b/arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_dilated.hpp new file mode 100644 index 0000000000..e0d7f0c7f1 --- /dev/null +++ b/arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_dilated.hpp @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#pragma once + +#include +#include +#include + +#include "depthwise.hpp" + +namespace depthwise +{ + +template < + unsigned int OutputTileRows, unsigned int OutputTileCols, + unsigned int KernelRows, unsigned int KernelCols, + unsigned int StrideRows, unsigned int StrideCols, + typename TIn, typename TBias, typename TOut +> +class DilatedDepthwiseConvolution : public IDepthwiseConvolution +{ + public: + /** Create a new dilated depthwise convolution engine. + */ + DilatedDepthwiseConvolution( + int n_batches, int n_input_rows, int n_input_cols, int n_channels, + int dilation_factor, + nck::ActivationFunction activation, + unsigned int padding_top, + unsigned int padding_left, + unsigned int padding_bottom, + unsigned int padding_right + ); + + /** Create a new dilated depthwise convolution engine. + */ + DilatedDepthwiseConvolution( + int n_batches, int n_input_rows, int n_input_cols, int n_channels, + int dilation_factor, int n_output_rows, int n_output_cols, + nck::ActivationFunction activation, + unsigned int padding_top, + unsigned int padding_left, + unsigned int padding_bottom, + unsigned int padding_right + ); + + // Cannot copy or move a DilatedDepthwiseConvolution. + DilatedDepthwiseConvolution(DilatedDepthwiseConvolution&) = delete; + DilatedDepthwiseConvolution operator=(DilatedDepthwiseConvolution&) = delete; + + /* Set input tensor and stride. */ + void set_input(const void *inptr) override; + void set_input(const void *inptr, int column_stride) override; + void set_input(const void *inptr, int row_stride, int column_stride) override; + void set_input(const void *inptr, int batch_stride, int row_stride, int column_stride) override; + + /* Set output tensor and stride. */ + void set_output(void *outptr) override; + void set_output(void *outptr, int column_stride) override; + void set_output(void *outptr, int row_stride, int column_stride) override; + void set_output(void *outptr, int batch_stride, int row_stride, int column_stride) override; + + static int get_output_size( + int dim_size, + unsigned int padding_before, + unsigned int padding_after, + int dilation_factor + ); + + int output_size( + int dim_size, unsigned int padding_before, unsigned int padding_after + ) const override; + + /* Weights and biases are re-ordered to improve memory access patterns. Use + * these methods to determine the size of the re-pack buffer and to set the + * address (and implicitly reorder the weights and biases into) the buffer. + */ + size_t get_packed_params_size(void) const override; + void set_packed_params_buffer(void *) override; + + void pack_params(const void *weights, const void *biases=nullptr) const override; + void pack_params(void *buffer, const void *weights, const void *biases=nullptr) const override; + void pack_params( + void *buffer, + const void* weights, + unsigned int weight_row_stride, + unsigned int weight_col_stride, + const void *biases=nullptr + ) const override; + + /* Working space is used to pad tensors on the fly. Before running any + * inference check the amount of space required, allocate and provide a + * pointer to the convolution engine. + */ + size_t get_working_space_size(unsigned int nthreads=1) const override; + void set_working_space(void *) override; + + unsigned int get_window(void) const override; + void run(unsigned int start, unsigned int stop, unsigned int threadid=0) override; + + protected: + /** Protected constructor which also accepts a function to construct a new + * subconvolution + */ + DilatedDepthwiseConvolution( + int n_batches, int n_input_rows, int n_input_cols, int n_channels, + int dilation_factor, int n_output_rows, int n_output_cols, + nck::ActivationFunction activation, + unsigned int padding_top, + unsigned int padding_left, + unsigned int padding_bottom, + unsigned int padding_right, + std::function subconvfn + ); + + const int _dilation_factor; + const int _n_input_rows, _n_input_cols, _n_channels; + const int _padding_top, _padding_left; + const int _n_output_rows, _n_output_cols; + + /* Dilated depthwise convolution is performed through repeated calls to + * non-dilated convolutions. If the dilation factor is $n$, then we perform + * $(n + 1)^2$ depthwise convolutions. + */ + using BaseDepthwise = DepthwiseConvolution< + OutputTileRows, OutputTileCols, + KernelRows, KernelCols, + StrideRows, StrideCols, + TIn, TBias, TOut + >; + std::deque>> _convs; +}; + +} // namespace depthwise diff --git a/arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_quantized.hpp b/arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_quantized.hpp index e34023faf1..b65ced6f35 100644 --- a/arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_quantized.hpp +++ b/arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_quantized.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 ARM Limited. + * Copyright (c) 2018-2019 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -23,8 +23,8 @@ */ #pragma once -#include "arm_compute/core/NEON/kernels/convolution/depthwise/depthwise.hpp" -#include "arm_compute/core/NEON/kernels/convolution/common/qasymm8.hpp" +#include "depthwise.hpp" +#include "qasymm8.hpp" namespace depthwise { @@ -70,6 +70,33 @@ class QAsymm8DepthwiseConvolution : public DepthwiseConvolutionBase< QAsymm8DepthwiseConvolution( int n_batches, int n_input_rows, int n_input_cols, int n_channels, + int n_output_rows, int n_output_cols, + nck::ActivationFunction activation, + const qasymm8::QAsymm8Params& weight_quantisation, + const qasymm8::QAsymm8Params& input_quantisation, + const qasymm8::QAsymm8Params& output_quantisation, + unsigned int padding_top, + unsigned int padding_left, + unsigned int padding_bottom, + unsigned int padding_right + ); + + QAsymm8DepthwiseConvolution( + int n_batches, int n_input_rows, int n_input_cols, int n_channels, + nck::ActivationFunction activation, + const qasymm8::QAsymm8Params& weight_quantisation, + const qasymm8::QAsymm8Params& input_quantisation, + const qasymm8::QAsymm8Params& output_quantisation, + const qasymm8::QAsymm8RescaleParams& rescale_parameters, + unsigned int padding_top, + unsigned int padding_left, + unsigned int padding_bottom, + unsigned int padding_right + ); + + QAsymm8DepthwiseConvolution( + int n_batches, int n_input_rows, int n_input_cols, int n_channels, + int n_output_rows, int n_output_cols, nck::ActivationFunction activation, const qasymm8::QAsymm8Params& weight_quantisation, const qasymm8::QAsymm8Params& input_quantisation, @@ -82,6 +109,11 @@ class QAsymm8DepthwiseConvolution : public DepthwiseConvolutionBase< ); protected: + static nck::ActivationFunction get_activation_fn( + nck::ActivationFunction activation, + const qasymm8::QAsymm8Params& output_quantisation + ); + uint8_t _input_padding_value(void) const; void _pack_params( diff --git a/arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_quantized_dilated.hpp b/arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_quantized_dilated.hpp new file mode 100644 index 0000000000..cf1c6f581f --- /dev/null +++ b/arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_quantized_dilated.hpp @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#pragma once +#include "depthwise_dilated.hpp" +#include "depthwise_quantized.hpp" + +namespace depthwise { + +template +class QAsymm8DilatedDepthwiseConvolution + : public DilatedDepthwiseConvolution< + OutputTileRows, OutputTileCols, KernelRows, KernelCols, StrideRows, + StrideCols, uint8_t, int32_t, uint8_t> { +public: + /** Create a new dilated depthwise convolution engine. + */ + QAsymm8DilatedDepthwiseConvolution( + int n_batches, int n_input_rows, int n_input_cols, int n_channels, + int dilation_factor, nck::ActivationFunction activation, + const qasymm8::QAsymm8Params &weight_quantisation, + const qasymm8::QAsymm8Params &input_quantisation, + const qasymm8::QAsymm8Params &output_quantisation, + unsigned int padding_top, unsigned int padding_left, + unsigned int padding_bottom, unsigned int padding_right); + + /** Create a new dilated depthwise convolution engine. + */ + QAsymm8DilatedDepthwiseConvolution( + int n_batches, int n_input_rows, int n_input_cols, int n_channels, + int dilation_factor, int n_output_rows, int n_output_cols, + nck::ActivationFunction activation, + const qasymm8::QAsymm8Params &weight_quantisation, + const qasymm8::QAsymm8Params &input_quantisation, + const qasymm8::QAsymm8Params &output_quantisation, + unsigned int padding_top, unsigned int padding_left, + unsigned int padding_bottom, unsigned int padding_right); + + /** Create a new dilated depthwise convolution engine. + */ + QAsymm8DilatedDepthwiseConvolution( + int n_batches, int n_input_rows, int n_input_cols, int n_channels, + int dilation_factor, nck::ActivationFunction activation, + const qasymm8::QAsymm8Params &weight_quantisation, + const qasymm8::QAsymm8Params &input_quantisation, + const qasymm8::QAsymm8Params &output_quantisation, + const qasymm8::QAsymm8RescaleParams &rescale_parameters, + unsigned int padding_top, unsigned int padding_left, + unsigned int padding_bottom, unsigned int padding_right); + + /** Create a new dilated depthwise convolution engine. + */ + QAsymm8DilatedDepthwiseConvolution( + int n_batches, int n_input_rows, int n_input_cols, int n_channels, + int dilation_factor, int n_output_rows, int n_output_cols, + nck::ActivationFunction activation, + const qasymm8::QAsymm8Params &weight_quantisation, + const qasymm8::QAsymm8Params &input_quantisation, + const qasymm8::QAsymm8Params &output_quantisation, + const qasymm8::QAsymm8RescaleParams& rescale_parameters, + unsigned int padding_top, unsigned int padding_left, + unsigned int padding_bottom, unsigned int padding_right); +}; + +} // namespace depthwise diff --git a/arm_compute/core/NEON/kernels/convolution/depthwise/impl_base.hpp b/arm_compute/core/NEON/kernels/convolution/depthwise/impl_base.hpp index 493b2991dc..b102a24250 100644 --- a/arm_compute/core/NEON/kernels/convolution/depthwise/impl_base.hpp +++ b/arm_compute/core/NEON/kernels/convolution/depthwise/impl_base.hpp @@ -32,9 +32,9 @@ #include #include -#include "arm_compute/core/NEON/kernels/convolution/depthwise/depthwise.hpp" -#include "arm_compute/core/NEON/kernels/convolution/common/padding.hpp" -#include "arm_compute/core/NEON/kernels/convolution/common/utils.hpp" +#include "depthwise.hpp" +#include "padding.hpp" +#include "utils.hpp" #pragma once @@ -95,6 +95,28 @@ MEMBERFN()::DepthwiseConvolutionBase( const unsigned int padding_left, const unsigned int padding_bottom, const unsigned int padding_right +) : DepthwiseConvolutionBase( + n_batches, n_input_rows, n_input_cols, n_channels, + get_output_size(n_input_rows, padding_top, padding_bottom), + get_output_size(n_input_cols, padding_left, padding_right), + activation, + padding_top, padding_left, padding_bottom, padding_right + ) +{ +} + +MEMBERFN()::DepthwiseConvolutionBase( + const int n_batches, + const int n_input_rows, + const int n_input_cols, + const int n_channels, + const int n_output_rows, + const int n_output_cols, + ActivationFunction activation, + const unsigned int padding_top, + const unsigned int padding_left, + const unsigned int padding_bottom, + const unsigned int padding_right ) : _input(nullptr), _output(nullptr), _packed_parameters(nullptr), _working_space(nullptr), @@ -102,8 +124,8 @@ MEMBERFN()::DepthwiseConvolutionBase( _n_input_rows(n_input_rows), _n_input_cols(n_input_cols), _n_channels(n_channels), - _n_output_rows(get_output_size(n_input_rows, padding_top, padding_bottom)), - _n_output_cols(get_output_size(n_input_cols, padding_left, padding_right)), + _n_output_rows(n_output_rows), + _n_output_cols(n_output_cols), _n_tile_rows(iceildiv(_n_output_rows, output_tile_rows)), _n_tile_cols(iceildiv(_n_output_cols, output_tile_cols)), _padding_top(padding_top), diff --git a/arm_compute/core/NEON/kernels/convolution/depthwise/impl_dilated.hpp b/arm_compute/core/NEON/kernels/convolution/depthwise/impl_dilated.hpp new file mode 100644 index 0000000000..2ef965ba4b --- /dev/null +++ b/arm_compute/core/NEON/kernels/convolution/depthwise/impl_dilated.hpp @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2019 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "depthwise_dilated.hpp" +#include "utils.hpp" + +#define MEMBERFN(TOUT) \ + template \ + TOUT DilatedDepthwiseConvolution + +namespace depthwise { + +MEMBERFN() +::DilatedDepthwiseConvolution(const int n_batches, const int n_input_rows, + const int n_input_cols, const int n_channels, + const int dilation_factor, + nck::ActivationFunction activation, + const unsigned int padding_top, + const unsigned int padding_left, + const unsigned int padding_bottom, + const unsigned int padding_right) + : DilatedDepthwiseConvolution( + n_batches, n_input_rows, n_input_cols, n_channels, dilation_factor, + DilatedDepthwiseConvolution::get_output_size( + n_input_rows, padding_top, padding_bottom, dilation_factor), + DilatedDepthwiseConvolution::get_output_size( + n_input_cols, padding_left, padding_right, dilation_factor), + activation, padding_top, padding_left, padding_bottom, + padding_right) {} + +MEMBERFN() +::DilatedDepthwiseConvolution(const int n_batches, const int n_input_rows, + const int n_input_cols, const int n_channels, + const int dilation_factor, + const int n_output_rows, const int n_output_cols, + nck::ActivationFunction activation, + const unsigned int padding_top, + const unsigned int padding_left, + const unsigned int, // padding_bottom + const unsigned int // padding_right + ) + : DilatedDepthwiseConvolution( + n_batches, n_input_rows, n_input_cols, n_channels, dilation_factor, + n_output_rows, n_output_cols, activation, padding_top, padding_left, + 0, 0, + // Function which creates a new (standard) depthwise convolution + [](const int n_batches, const int n_input_rows, + const int n_input_cols, const int n_channels, + const int n_output_rows, const int n_output_cols, + const nck::ActivationFunction activation, + const unsigned int padding_top, const unsigned int padding_left, + const unsigned int padding_bottom, + const unsigned int padding_right) -> IDepthwiseConvolution * { + return new DepthwiseConvolution< + OutputTileRows, OutputTileColumns, KernelRows, KernelColumns, + StrideRows, StrideColumns, TIn, TBias, TOut>( + n_batches, n_input_rows, n_input_cols, n_channels, + n_output_rows, n_output_cols, activation, padding_top, + padding_left, padding_bottom, padding_right); + }) {} + +MEMBERFN() +::DilatedDepthwiseConvolution( + const int n_batches, const int n_input_rows, const int n_input_cols, + const int n_channels, const int dilation_factor, const int n_output_rows, + const int n_output_cols, nck::ActivationFunction activation, + const unsigned int padding_top, const unsigned int padding_left, + const unsigned int, // padding_bottom + const unsigned int, // padding_right + std::function + subconvfn // Function to create a new convolution + ) + : _dilation_factor(dilation_factor), _n_input_rows(n_input_rows), + _n_input_cols(n_input_cols), _n_channels(n_channels), + _padding_top(static_cast(padding_top)), + _padding_left(static_cast(padding_left)), + _n_output_rows(n_output_rows), _n_output_cols(n_output_cols), + _convs(_dilation_factor) { + // Instantiate the base convolutions + for (int i = 0; i < _dilation_factor; i++) { + // Compute properties of this row of base convolutions + const int row_top = + i * StrideRows - _padding_top; // -ve values are in the padding + const int row_pad_top = + row_top < 0 ? iceildiv(-row_top, dilation_factor) : 0; + + const int _n_input_rows = iceildiv(n_input_rows - i, dilation_factor); + const int _n_output_rows = iceildiv(n_output_rows - i, dilation_factor); + + for (int j = 0; j < _dilation_factor; j++) { + // Compute properties of the base convolution + const int col_left = + j * StrideColumns - padding_left; // -ve values are in the padding + const int col_pad_left = + col_left < 0 ? iceildiv(-col_left, dilation_factor) : 0; + + const int _n_input_cols = iceildiv(n_input_cols - j, dilation_factor); + const int _n_output_cols = iceildiv(n_output_cols - j, dilation_factor); + + // Create new depthwise convolution engine and include it in the vector + // of engines. The new depthwise convolution engine is created by calling + // the delegate function we received as an argument. + _convs[i].emplace_back(subconvfn( + n_batches, _n_input_rows, _n_input_cols, n_channels, _n_output_rows, + _n_output_cols, activation, + // Note: since we have computed the output tensor size we don't need + // to explicitly provide bottom and right padding values to the + // depthwise convolution. + row_pad_top, col_pad_left, 0, 0)); + } + } +} + +MEMBERFN(void)::set_input(const void *const inptr) { + set_input(inptr, _n_channels); +} + +MEMBERFN(void)::set_input(const void *const inptr, const int ldcol) { + set_input(inptr, _n_input_cols * ldcol, ldcol); +} + +MEMBERFN(void) +::set_input(const void *const inptr, const int ldrow, const int ldcol) { + set_input(inptr, _n_input_rows * ldrow, ldrow, ldcol); +} + +MEMBERFN(void) +::set_input(const void *const inptr, const int ldbatch, const int ldrow, + const int ldcol) { + // Compute dilated strides + const int ldrow_dilated = ldrow * _dilation_factor; + const int ldcol_dilated = ldcol * _dilation_factor; + + // Pass input parameters on to base convolutions + for (int i = 0; i < _dilation_factor; i++) { + const int top_pos = + i * StrideRows - _padding_top + + ((static_cast(i * StrideRows) < _padding_top) + ? iceildiv(_padding_top - i * StrideRows, _dilation_factor) * + _dilation_factor + : 0); + const TIn *const inptr_i = + static_cast(inptr) + top_pos * ldrow; + + for (int j = 0; j < _dilation_factor; j++) { + int left_pos = j * StrideColumns - _padding_left; + while (left_pos < 0) + left_pos += _dilation_factor; + + // Modify the pointer to point to the first element of the dilated input + // tensor, then set the input for this convolution engine. + const void *const inptr_ij = inptr_i + left_pos * ldcol; + _convs[i][j]->set_input(inptr_ij, ldbatch, ldrow_dilated, ldcol_dilated); + } + } +} + +MEMBERFN(void)::set_output(void *const outptr) { + set_output(outptr, _n_channels); +} + +MEMBERFN(void)::set_output(void *const outptr, const int ldcol) { + set_output(outptr, _n_output_cols * ldcol, ldcol); +} + +MEMBERFN(void) +::set_output(void *const outptr, const int ldrow, const int ldcol) { + set_output(outptr, _n_output_rows * ldrow, ldrow, ldcol); +} + +MEMBERFN(void) +::set_output(void *const outptr, const int ldbatch, const int ldrow, + const int ldcol) { + // Compute dilated strides + const int ldrow_dilated = ldrow * _dilation_factor; + const int ldcol_dilated = ldcol * _dilation_factor; + + // Pass input parameters on to base convolutions + for (int i = 0; i < _dilation_factor; i++) { + for (int j = 0; j < _dilation_factor; j++) { + // Modify the pointer to point to the first element of the dilated input + // tensor, then set the input for this convolution engine. + void *const outptr_ij = + static_cast(outptr) + i * ldrow + j * ldcol; + _convs[i][j]->set_output(outptr_ij, ldbatch, ldrow_dilated, + ldcol_dilated); + } + } +} + +MEMBERFN(int) +::get_output_size(const int dim_size, const unsigned int padding_before, + const unsigned int padding_after, const int dilation_factor) { + const int input_size = + dim_size + static_cast(padding_before + padding_after); + const int window_size = (KernelRows - 1) * dilation_factor + 1; + return iceildiv(input_size - window_size + 1, StrideRows); +} + +MEMBERFN(int) +::output_size(const int dim_size, const unsigned int padding_before, + const unsigned int padding_after) const { + return get_output_size(dim_size, padding_before, padding_after, + _dilation_factor); +} + +MEMBERFN(size_t)::get_packed_params_size(void) const { + return _convs[0][0]->get_packed_params_size(); +} + +MEMBERFN(void)::set_packed_params_buffer(void *buffer) { + // Set the buffer for all convolution engines + for (auto &&row : _convs) { + for (auto &&conv : row) { + conv->set_packed_params_buffer(buffer); + } + } +} + +MEMBERFN(void) +::pack_params(const void *const weights, const void *const biases) const { + _convs[0][0]->pack_params(weights, biases); +} + +MEMBERFN(void) +::pack_params(void *const buffer, const void *const weights, + const void *const biases) const { + _convs[0][0]->pack_params(buffer, weights, biases); +} + +MEMBERFN(void) +::pack_params(void *const buffer, const void *const weights, + const unsigned int ldrow, const unsigned int ldcol, + const void *const biases) const { + _convs[0][0]->pack_params(buffer, weights, ldrow, ldcol, biases); +} + +MEMBERFN(size_t)::get_working_space_size(unsigned int nthreads) const { + return _convs[0][0]->get_working_space_size(nthreads); +} + +MEMBERFN(void)::set_working_space(void *const ws) { + // Use the same working space set for all contained depthwise engines. + for (auto &&row : _convs) { + for (auto &&conv : row) { + conv->set_working_space(ws); + } + } +} + +MEMBERFN(unsigned int)::get_window(void) const { + return _convs[0][0]->get_window(); +} + +MEMBERFN(void) +::run(const unsigned int start, const unsigned int stop, + const unsigned int threadid) { + // Run each contained convolution in turn + for (auto &&row : _convs) { + for (auto &&conv : row) { + conv->run(start, stop, threadid); + } + } +} + +} // namespace depthwise -- cgit v1.2.1