COMPMID-2156: Optimized dilated convolution for NEON.

Change-Id: I3a8abe8cc9637c8983d9bd69dcbaee1a15eac8d0
Signed-off-by: Georgios Pinitas <georgios.pinitas@arm.com>
Reviewed-on: https://review.mlplatform.org/c/1492
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Pablo Marquez <pablo.tello@arm.com>

1 files changed, 72 insertions, 55 deletions

diff --git a/src/runtime/NEON/functions/assembly/NEDepthwiseConvolutionAssemblyDispatch.cpp b/src/runtime/NEON/functions/assembly/NEDepthwiseConvolutionAssemblyDispatch.cpp
index 5f57bbfe23..b28aaa715f 100644
--- a/src/runtime/NEON/functions/assembly/NEDepthwiseConvolutionAssemblyDispatch.cpp
+++ b/src/runtime/NEON/functions/assembly/NEDepthwiseConvolutionAssemblyDispatch.cpp
@@ -26,7 +26,9 @@
 
 #include "arm_compute/core/CPP/Validate.h"
 #include "arm_compute/core/ITensor.h"
-#include "arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_quantized.hpp"
+#include "arm_compute/core/NEON/kernels/assembly/NEDepthwiseConvolutionAssemblyKernelWrapper.h"
+#include "arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_dilated.hpp"
+#include "arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_quantized_dilated.hpp"
 #include "arm_compute/core/Utils.h"
 #include "arm_compute/core/utils/misc/InfoHelpers.h"
 #include "arm_compute/core/utils/misc/ShapeCalculator.h"
@@ -42,19 +44,22 @@ std::unique_ptr<depthwise::IDepthwiseConvolution> create_convolver(const ITensor
                                                                    const ITensor      *weights,
                                                                    ITensor            *output,
                                                                    PadStrideInfo       conv_info,
-                                                                   ActivationLayerInfo act_info)
+                                                                   ActivationLayerInfo act_info,
+                                                                   const Size2D       &dilation)
 {
+    ARM_COMPUTE_UNUSED(dilation);
     const DataType    data_type = input->info()->data_type();
     const TensorShape shape     = input->info()->tensor_shape();
 
-    const int n_batches      = shape[3];
-    const int in_rows        = shape.z();
-    const int in_cols        = shape.y();
-    const int n_channels     = shape.x();
-    const int padding_top    = conv_info.pad_top();
-    const int padding_left   = conv_info.pad_left();
-    const int padding_bottom = conv_info.pad_bottom();
-    const int padding_right  = conv_info.pad_right();
+    const int n_batches       = shape[3];
+    const int in_rows         = shape.z();
+    const int in_cols         = shape.y();
+    const int n_channels      = shape.x();
+    const int dilation_factor = dilation.x();
+    const int padding_top     = conv_info.pad_top();
+    const int padding_left    = conv_info.pad_left();
+    const int padding_bottom  = conv_info.pad_bottom();
+    const int padding_right   = conv_info.pad_right();
 
     const unsigned int stride_x = conv_info.stride().first;
 
@@ -95,11 +100,11 @@ std::unique_ptr<depthwise::IDepthwiseConvolution> create_convolver(const ITensor
         switch(stride_x)
         {
             case 1:
-                return arm_compute::support::cpp14::make_unique<depthwise::QAsymm8DepthwiseConvolution<2, 2, 3, 3, 1, 1>>(
-                           n_batches, in_rows, in_cols, n_channels, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
+                return arm_compute::support::cpp14::make_unique<depthwise::QAsymm8DilatedDepthwiseConvolution<2, 2, 3, 3, 1, 1>>(
+                           n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
             case 2:
-                return arm_compute::support::cpp14::make_unique<depthwise::QAsymm8DepthwiseConvolution<2, 2, 3, 3, 2, 2>>(
-                           n_batches, in_rows, in_cols, n_channels, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
+                return arm_compute::support::cpp14::make_unique<depthwise::QAsymm8DilatedDepthwiseConvolution<2, 2, 3, 3, 2, 2>>(
+                           n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right);
             default:
                 return nullptr;
         }
@@ -115,11 +120,11 @@ std::unique_ptr<depthwise::IDepthwiseConvolution> create_convolver(const ITensor
                 switch(stride_x)
                 {
                     case 1:
-                        return arm_compute::support::cpp14::make_unique<depthwise::DepthwiseConvolution<3, 3, 3, 3, 1, 1, float16_t, float16_t, float16_t>>(
-                                   n_batches, in_rows, in_cols, n_channels, activation, padding_top, padding_left, padding_bottom, padding_right);
+                        return arm_compute::support::cpp14::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 3, 3, 1, 1, float16_t, float16_t, float16_t>>(
+                                   n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
                     case 2:
-                        return arm_compute::support::cpp14::make_unique<depthwise::DepthwiseConvolution<3, 3, 3, 3, 2, 2, float16_t, float16_t, float16_t>>(
-                                   n_batches, in_rows, in_cols, n_channels, activation, padding_top, padding_left, padding_bottom, padding_right);
+                        return arm_compute::support::cpp14::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 3, 3, 2, 2, float16_t, float16_t, float16_t>>(
+                                   n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
                     default:
                         return nullptr;
                 }
@@ -131,11 +136,11 @@ std::unique_ptr<depthwise::IDepthwiseConvolution> create_convolver(const ITensor
                 switch(stride_x)
                 {
                     case 1:
-                        return arm_compute::support::cpp14::make_unique<depthwise::DepthwiseConvolution<4, 4, 3, 3, 1, 1, float, float, float>>(
-                                   n_batches, in_rows, in_cols, n_channels, activation, padding_top, padding_left, padding_bottom, padding_right);
+                        return arm_compute::support::cpp14::make_unique<depthwise::DilatedDepthwiseConvolution<4, 4, 3, 3, 1, 1, float, float, float>>(
+                                   n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
                     case 2:
-                        return arm_compute::support::cpp14::make_unique<depthwise::DepthwiseConvolution<3, 3, 3, 3, 2, 2, float, float, float>>(
-                                   n_batches, in_rows, in_cols, n_channels, activation, padding_top, padding_left, padding_bottom, padding_right);
+                        return arm_compute::support::cpp14::make_unique<depthwise::DilatedDepthwiseConvolution<3, 3, 3, 3, 2, 2, float, float, float>>(
+                                   n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right);
                     default:
                         return nullptr;
                 }
@@ -148,21 +153,30 @@ std::unique_ptr<depthwise::IDepthwiseConvolution> create_convolver(const ITensor
 }
 } // namespace
 
+struct NEDepthwiseConvolutionAssemblyDispatch::LocalImpl
+{
+    std::unique_ptr<depthwise::IDepthwiseConvolution> _dwc_assembly_kernel{ nullptr };
+    NEDepthwiseConvolutionAssemblyKernelWrapper       _dwc_acl_kernel{};
+};
+
 #ifndef DOXYGEN_SKIP_THIS
 NEDepthwiseConvolutionAssemblyDispatch::NEDepthwiseConvolutionAssemblyDispatch(std::shared_ptr<arm_compute::IMemoryManager> memory_manager)
-    : _memory_group(std::move(memory_manager)), _input(nullptr), _weights(nullptr), _bias(nullptr), _output(nullptr), _packed_weights(), _workspace(), _is_prepared(false), _dwc_assembly_kernel(nullptr),
-      _dwc_acl_kernel()
+    : _memory_group(std::move(memory_manager)), _input(nullptr), _weights(nullptr), _bias(nullptr), _output(nullptr), _packed_weights(), _workspace(), _is_prepared(false),
+      _pImpl(support::cpp14::make_unique<LocalImpl>())
 {
 }
 #endif /* DOXYGEN_SKIP_THIS */
 
+NEDepthwiseConvolutionAssemblyDispatch::~NEDepthwiseConvolutionAssemblyDispatch() = default;
+
 void NEDepthwiseConvolutionAssemblyDispatch::configure(const ITensor             *input,
                                                        const ITensor             *weights,
                                                        const ITensor             *bias,
                                                        ITensor                   *output,
                                                        const PadStrideInfo       &conv_info,
                                                        unsigned int               depth_multiplier,
-                                                       const ActivationLayerInfo &act_info)
+                                                       const ActivationLayerInfo &act_info,
+                                                       const Size2D              &dilation)
 {
     ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output);
     ARM_COMPUTE_UNUSED(depth_multiplier);
@@ -172,10 +186,11 @@ void NEDepthwiseConvolutionAssemblyDispatch::configure(const ITensor
                                                                                 output->info(),
                                                                                 conv_info,
                                                                                 depth_multiplier,
-                                                                                act_info));
+                                                                                act_info,
+                                                                                dilation));
 
     // Output auto inizialitation if not yet initialized
-    const TensorShape output_shape = misc::shape_calculator::compute_depthwise_convolution_shape(*input->info(), *weights->info(), conv_info, depth_multiplier);
+    const TensorShape output_shape = misc::shape_calculator::compute_depthwise_convolution_shape(*input->info(), *weights->info(), conv_info, depth_multiplier, dilation);
     auto_init_if_empty(*output->info(), input->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(output_shape).set_quantization_info(output->info()->quantization_info()));
 
     _input       = input;
@@ -185,24 +200,24 @@ void NEDepthwiseConvolutionAssemblyDispatch::configure(const ITensor
     _is_prepared = false;
 
     // Create convolver
-    _dwc_assembly_kernel = create_convolver(input, weights, output, conv_info, act_info);
-    ARM_COMPUTE_ERROR_ON(_dwc_assembly_kernel == nullptr);
+    _pImpl->_dwc_assembly_kernel = create_convolver(input, weights, output, conv_info, act_info, dilation);
+    ARM_COMPUTE_ERROR_ON(_pImpl->_dwc_assembly_kernel == nullptr);
 
     // Create assembly kernel wrapper
-    _dwc_acl_kernel.configure(_dwc_assembly_kernel.get());
+    _pImpl->_dwc_acl_kernel.configure(_pImpl->_dwc_assembly_kernel.get());
 
     constexpr size_t alignment = 128;
 
     // Create workspace
     const unsigned int num_threads    = NEScheduler::get().num_threads();
-    const size_t       workspace_size = _dwc_assembly_kernel->get_working_space_size(num_threads);
+    const size_t       workspace_size = _pImpl->_dwc_assembly_kernel->get_working_space_size(num_threads);
     ARM_COMPUTE_ERROR_ON_MSG(workspace_size == 0, "Workspace size cannot be 0 !");
     _workspace.allocator()->init(TensorInfo(TensorShape{ workspace_size }, 1, DataType::S8), alignment);
     _memory_group.manage(&_workspace);
     _workspace.allocator()->allocate();
 
     // Create packing tensor
-    const size_t pack_tensor_size = _dwc_assembly_kernel->get_packed_params_size();
+    const size_t pack_tensor_size = _pImpl->_dwc_assembly_kernel->get_packed_params_size();
     ARM_COMPUTE_ERROR_ON_MSG(pack_tensor_size == 0, "Pack tensor size cannot be 0 !");
     _packed_weights.allocator()->init(TensorInfo(TensorShape{ pack_tensor_size }, 1, DataType::S8), alignment);
 }
@@ -213,7 +228,8 @@ Status NEDepthwiseConvolutionAssemblyDispatch::validate(const ITensorInfo
                                                         const ITensorInfo         *output,
                                                         const PadStrideInfo       &conv_info,
                                                         unsigned int               depth_multiplier,
-                                                        const ActivationLayerInfo &act_info)
+                                                        const ActivationLayerInfo &act_info,
+                                                        const Size2D              &dilation)
 {
     ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(input);
     ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8, DataType::F16, DataType::F32);
@@ -227,6 +243,7 @@ Status NEDepthwiseConvolutionAssemblyDispatch::validate(const ITensorInfo
     ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(width_idx) != 3 || weights->dimension(height_idx) != 3);
     ARM_COMPUTE_RETURN_ERROR_ON(!((strides.first == strides.second) && ((strides.first == 1) || (strides.first == 2))));
     ARM_COMPUTE_RETURN_ERROR_ON(depth_multiplier != 1);
+    ARM_COMPUTE_RETURN_ERROR_ON(dilation.x() != dilation.y());
 
     const bool is_relu  = arm_compute::utils::info_helpers::is_relu(act_info);
     const bool is_relu6 = arm_compute::utils::info_helpers::is_relu6(act_info);
@@ -243,7 +260,7 @@ Status NEDepthwiseConvolutionAssemblyDispatch::validate(const ITensorInfo
     // Check output
     if(output->total_size() != 0)
     {
-        const TensorShape output_shape = misc::shape_calculator::compute_depthwise_convolution_shape(*input, *weights, conv_info, depth_multiplier);
+        const TensorShape output_shape = misc::shape_calculator::compute_depthwise_convolution_shape(*input, *weights, conv_info, depth_multiplier, dilation);
         ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(output->tensor_shape(), output_shape);
         ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output);
     }
@@ -283,17 +300,17 @@ bool NEDepthwiseConvolutionAssemblyDispatch::is_optimized_supported(const ITenso
     bool        supported_strides = (strides.first == strides.second) && ((strides.first == 1) || (strides.first == 2));
 
     // Check for supported padding
-    const auto    pad_top           = conv_info.pad_top();
-    const auto    pad_right         = conv_info.pad_right();
-    const auto    pad_bottom        = conv_info.pad_bottom();
-    const auto    pad_left          = conv_info.pad_left();
-    PadStrideInfo same_pad          = calculate_same_pad(in_shape, TensorShape(3U, 3U), conv_info);
-    bool          is_same_padding   = (pad_top == same_pad.pad_top()) && (pad_right == same_pad.pad_right()) && (pad_bottom == same_pad.pad_bottom()) && (pad_left == same_pad.pad_left());
-    bool          is_valid_padding  = (pad_top == 0) && (pad_right == 0) && (pad_bottom == 0) && (pad_left == 0);
-    bool          supported_padding = is_same_padding || is_valid_padding;
-    bool          is_dilation_1     = dilation.x() == 1 && dilation.y() == 1;
-
-    return is_data_type_valid && weights_supported && supported_strides && supported_padding && (depth_multiplier == 1) && is_dilation_1;
+    const auto    pad_top               = conv_info.pad_top();
+    const auto    pad_right             = conv_info.pad_right();
+    const auto    pad_bottom            = conv_info.pad_bottom();
+    const auto    pad_left              = conv_info.pad_left();
+    PadStrideInfo same_pad              = calculate_same_pad(in_shape, TensorShape(3U, 3U), conv_info);
+    bool          is_same_padding       = (pad_top == same_pad.pad_top()) && (pad_right == same_pad.pad_right()) && (pad_bottom == same_pad.pad_bottom()) && (pad_left == same_pad.pad_left());
+    bool          is_valid_padding      = (pad_top == 0) && (pad_right == 0) && (pad_bottom == 0) && (pad_left == 0);
+    bool          supported_padding     = is_same_padding || is_valid_padding;
+    bool          is_dilation_supported = (dilation.x() == dilation.y()) || (dilation == Size2D(1U, 1U));
+
+    return is_data_type_valid && weights_supported && supported_strides && supported_padding && (depth_multiplier == 1) && is_dilation_supported;
 }
 
 void NEDepthwiseConvolutionAssemblyDispatch::run()
@@ -305,7 +322,7 @@ void NEDepthwiseConvolutionAssemblyDispatch::run()
 
     // Setup inputs/outputs
     ARM_COMPUTE_ERROR_ON(_workspace.buffer() == nullptr);
-    _dwc_assembly_kernel->set_working_space(static_cast<void *>(_workspace.buffer()));
+    _pImpl->_dwc_assembly_kernel->set_working_space(static_cast<void *>(_workspace.buffer()));
 
     ARM_COMPUTE_ERROR_ON(_input->buffer() == nullptr);
     const int   input_element_size = _input->info()->element_size();
@@ -313,7 +330,7 @@ void NEDepthwiseConvolutionAssemblyDispatch::run()
     const int   input_row_stride   = _input->info()->strides_in_bytes().z() / input_element_size;
     const int   input_col_stride   = _input->info()->strides_in_bytes().y() / input_element_size;
     const void *input_ptr          = _input->buffer() + _input->info()->offset_first_element_in_bytes();
-    _dwc_assembly_kernel->set_input(input_ptr, input_batch_stride, input_row_stride, input_col_stride);
+    _pImpl->_dwc_assembly_kernel->set_input(input_ptr, input_batch_stride, input_row_stride, input_col_stride);
 
     ARM_COMPUTE_ERROR_ON(_output->buffer() == nullptr);
     const int output_element_size = _output->info()->element_size();
@@ -321,10 +338,10 @@ void NEDepthwiseConvolutionAssemblyDispatch::run()
     const int output_row_stride   = _output->info()->strides_in_bytes().z() / output_element_size;
     const int output_col_stride   = _output->info()->strides_in_bytes().y() / output_element_size;
     void     *output_ptr          = _output->buffer() + _output->info()->offset_first_element_in_bytes();
-    _dwc_assembly_kernel->set_output(output_ptr, output_batch_stride, output_row_stride, output_col_stride);
+    _pImpl->_dwc_assembly_kernel->set_output(output_ptr, output_batch_stride, output_row_stride, output_col_stride);
 
     // Schedule assembly kernel
-    NEScheduler::get().schedule(&_dwc_acl_kernel, Window::DimX);
+    NEScheduler::get().schedule(&_pImpl->_dwc_acl_kernel, Window::DimX);
 }
 
 void NEDepthwiseConvolutionAssemblyDispatch::prepare()
@@ -338,12 +355,12 @@ void NEDepthwiseConvolutionAssemblyDispatch::prepare()
         const int weights_element_size = _weights->info()->element_size();
         const int weights_row_stride   = _weights->info()->strides_in_bytes().z() / weights_element_size;
         const int weights_col_stride   = _weights->info()->strides_in_bytes().y() / weights_element_size;
-        _dwc_assembly_kernel->pack_params(_packed_weights.buffer(),
-                                          _weights->buffer() + _weights->info()->offset_first_element_in_bytes(),
-                                          weights_row_stride,
-                                          weights_col_stride,
-                                          (_bias != nullptr) ? _bias->buffer() : nullptr);
-        _dwc_assembly_kernel->set_packed_params_buffer(_packed_weights.buffer());
+        _pImpl->_dwc_assembly_kernel->pack_params(_packed_weights.buffer(),
+                                                  _weights->buffer() + _weights->info()->offset_first_element_in_bytes(),
+                                                  weights_row_stride,
+                                                  weights_col_stride,
+                                                  (_bias != nullptr) ? _bias->buffer() : nullptr);
+        _pImpl->_dwc_assembly_kernel->set_packed_params_buffer(_packed_weights.buffer());
 
         _weights->mark_as_unused();
         if(_bias != nullptr)