From 30271c779c36a2abe6995c4454674d92bbc1f91f Mon Sep 17 00:00:00 2001 From: Georgios Pinitas Date: Mon, 24 Jun 2019 14:56:34 +0100 Subject: COMPMID-2156: Optimized dilated convolution for NEON. Change-Id: I3a8abe8cc9637c8983d9bd69dcbaee1a15eac8d0 Signed-off-by: Georgios Pinitas Reviewed-on: https://review.mlplatform.org/c/1492 Comments-Addressed: Arm Jenkins Tested-by: Arm Jenkins Reviewed-by: Pablo Marquez --- .../NEON/functions/NEDepthwiseConvolutionLayer.cpp | 327 +++++++++++++++++++++ .../NEDepthwiseConvolutionAssemblyDispatch.cpp | 127 ++++---- 2 files changed, 399 insertions(+), 55 deletions(-) (limited to 'src/runtime/NEON/functions') diff --git a/src/runtime/NEON/functions/NEDepthwiseConvolutionLayer.cpp b/src/runtime/NEON/functions/NEDepthwiseConvolutionLayer.cpp index 43288ec4c6..45cc2d2762 100644 --- a/src/runtime/NEON/functions/NEDepthwiseConvolutionLayer.cpp +++ b/src/runtime/NEON/functions/NEDepthwiseConvolutionLayer.cpp @@ -363,6 +363,333 @@ void NEDepthwiseConvolutionLayer3x3::prepare() } } +NEDepthwiseConvolutionLayerOptimized::NEDepthwiseConvolutionLayerOptimized(std::shared_ptr memory_manager) + : _memory_group(memory_manager), _dwc_kernel(), _dwc_optimized_func(memory_manager), _output_stage_kernel(), _border_handler(), _permute_input(), _permute_weights(), _permute_output(), + _activationlayer_function(), _accumulator(), _permuted_input(), _permuted_weights(), _permuted_output(), _original_weights(nullptr), _has_bias(false), _is_quantized(false), _is_optimized(false), + _is_nchw(true), _permute(false), _is_activationlayer_enabled(false), _is_prepared(false) +{ +} + +void NEDepthwiseConvolutionLayerOptimized::configure_generic(ITensor *input, + const ITensor *weights, + const ITensor *biases, + ITensor *output, + const PadStrideInfo &conv_info, + unsigned int depth_multiplier, + const ActivationLayerInfo &act_info, + const Size2D &dilation) +{ + ARM_COMPUTE_UNUSED(act_info); + + PixelValue zero_value(0.f); + + // Initialize the intermediate accumulator tensor in case of quantized input + if(_is_quantized) + { + TensorShape accum_shape = output->info()->tensor_shape(); + DataLayout accum_layout = output->info()->data_layout(); + if(!_is_nchw) + { + permute(accum_shape, PermutationVector(1U, 2U, 0U)); + accum_layout = DataLayout::NCHW; + } + + _memory_group.manage(&_accumulator); + _accumulator.allocator()->init(TensorInfo(accum_shape, 1, DataType::S32, output->info()->quantization_info())); + _accumulator.info()->set_data_layout(accum_layout); + zero_value = PixelValue(static_cast(input->info()->quantization_info().uniform().offset)); + } + + if(!_is_nchw) + { + _memory_group.manage(&_permuted_input); + _memory_group.manage(&_permuted_output); + + // Configure the function to transform the input tensor from NHWC -> NCHW + _permute_input.configure(input, &_permuted_input, PermutationVector(1U, 2U, 0U)); + _permuted_input.info()->set_data_layout(DataLayout::NCHW); + + // Configure the function to transform the weights tensor from HWI -> IHW + _permute_weights.configure(weights, &_permuted_weights, PermutationVector(1U, 2U, 0U)); + _permuted_weights.info()->set_data_layout(DataLayout::NCHW); + _permuted_output.info()->set_quantization_info(output->info()->quantization_info()); + + // Configure depthwise + _dwc_kernel.configure(&_permuted_input, &_permuted_weights, (_is_quantized) ? &_accumulator : &_permuted_output, conv_info, depth_multiplier, dilation); + + // Configure border handler + _border_handler.configure(&_permuted_input, _dwc_kernel.border_size(), BorderMode::CONSTANT, zero_value); + + // Allocate tensors + _permuted_input.allocator()->allocate(); + } + else + { + // Configure depthwise convolution kernel + _dwc_kernel.configure(input, weights, (_is_quantized) ? &_accumulator : output, conv_info, depth_multiplier, dilation); + + // Configure border handler + _border_handler.configure(input, _dwc_kernel.border_size(), BorderMode::CONSTANT, zero_value); + } + + // Configure biases accumulation + if(_is_quantized) + { + const UniformQuantizationInfo iq_info = input->info()->quantization_info().uniform(); + const UniformQuantizationInfo wq_info = weights->info()->quantization_info().uniform(); + const UniformQuantizationInfo oq_info = (output->info()->total_size() == 0) ? iq_info : output->info()->quantization_info().uniform(); + + float multiplier = (iq_info.scale * wq_info.scale) / oq_info.scale; + int output_multiplier; + int output_shift; + quantization::calculate_quantized_multiplier_less_than_one(multiplier, &output_multiplier, &output_shift); + _output_stage_kernel.configure(&_accumulator, biases, _is_nchw ? output : &_permuted_output, output_multiplier, output_shift, oq_info.offset); + _accumulator.allocator()->allocate(); + } + else if(_has_bias) + { + _output_stage_kernel.configure(_is_nchw ? output : &_permuted_output, biases); + } + + // Permute output + if(!_is_nchw) + { + // Configure the function to transform the convoluted output to NHWC + _permute_output.configure(&_permuted_output, output, PermutationVector(2U, 0U, 1U)); + _permuted_output.allocator()->allocate(); + } +} + +void NEDepthwiseConvolutionLayerOptimized::configure_optimized(const ITensor *input, + const ITensor *weights, + const ITensor *biases, + ITensor *output, + const PadStrideInfo &conv_info, + unsigned int depth_multiplier, + const ActivationLayerInfo &act_info, + const Size2D &dilation) +{ + ActivationLayerInfo act_info_to_use = ActivationLayerInfo(); + const bool is_relu = arm_compute::utils::info_helpers::is_relu(act_info); + const bool is_relu6 = arm_compute::utils::info_helpers::is_relu6(act_info); + _is_activationlayer_enabled = act_info.enabled() && !(is_relu || is_relu6); + if(!_is_activationlayer_enabled) + { + act_info_to_use = act_info; + } + + if(_is_nchw) + { + _memory_group.manage(&_permuted_input); + _memory_group.manage(&_permuted_output); + + // Configure the function to transform the input tensor from NCHW -> NHWC + _permute_input.configure(input, &_permuted_input, PermutationVector(2U, 0U, 1U)); + _permuted_input.info()->set_data_layout(DataLayout::NHWC); + + // Configure the function to transform the weights tensor from IHW -> HWI + _permute_weights.configure(weights, &_permuted_weights, PermutationVector(2U, 0U, 1U)); + _permuted_weights.info()->set_data_layout(DataLayout::NHWC); + + _permuted_output.info()->set_data_layout(DataLayout::NHWC); + _permuted_output.info()->set_quantization_info(output->info()->quantization_info()); + + // Configure optimized depthwise + _dwc_optimized_func.configure(&_permuted_input, &_permuted_weights, biases, &_permuted_output, conv_info, depth_multiplier, act_info_to_use, dilation); + + // Configure the function to transform the convoluted output to ACL's native ordering format NCHW + _permuted_output.info()->set_data_layout(DataLayout::NHWC); + _permute_output.configure(&_permuted_output, output, PermutationVector(1U, 2U, 0U)); + + // Allocate tensors + _permuted_input.allocator()->allocate(); + _permuted_output.allocator()->allocate(); + } + else + { + _dwc_optimized_func.configure(input, weights, biases, output, conv_info, depth_multiplier, act_info_to_use, dilation); + } +} + +void NEDepthwiseConvolutionLayerOptimized::configure(ITensor *input, + const ITensor *weights, + const ITensor *biases, + ITensor *output, const PadStrideInfo &conv_info, + unsigned int depth_multiplier, + const ActivationLayerInfo &act_info, + const Size2D &dilation) +{ + ARM_COMPUTE_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8, DataType::F16, DataType::F32); + ARM_COMPUTE_ERROR_ON_MISMATCHING_DATA_TYPES(input, weights); + + // idx_w and idx_h only used for validation + const size_t idx_w = get_data_layout_dimension_index(input->info()->data_layout(), DataLayoutDimension::WIDTH); + const size_t idx_h = get_data_layout_dimension_index(input->info()->data_layout(), DataLayoutDimension::HEIGHT); + ARM_COMPUTE_UNUSED(idx_w); + ARM_COMPUTE_UNUSED(idx_h); + + ARM_COMPUTE_ERROR_ON(weights->info()->dimension(idx_w) + (weights->info()->dimension(idx_w) - 1) * (dilation.x() - 1) > input->info()->dimension(idx_w) + conv_info.pad_left() + conv_info.pad_right()); + ARM_COMPUTE_ERROR_ON(weights->info()->dimension(idx_h) + (weights->info()->dimension(idx_h) - 1) * (dilation.y() - 1) > input->info()->dimension(idx_h) + conv_info.pad_top() + conv_info.pad_bottom()); + + _original_weights = weights; + _is_quantized = is_data_type_quantized_asymmetric(input->info()->data_type()); + _has_bias = biases != nullptr; + _is_optimized = NEDepthwiseConvolutionAssemblyDispatch::is_optimized_supported(input->info(), + weights->info(), + conv_info, + depth_multiplier, + dilation); + _is_nchw = input->info()->data_layout() == DataLayout::NCHW; + _permute = _is_optimized == _is_nchw; + _is_prepared = false; + _is_activationlayer_enabled = act_info.enabled(); + + // Configure appropriate pipeline + if(_is_optimized) + { + configure_optimized(input, weights, biases, output, conv_info, depth_multiplier, act_info, dilation); + } + else + { + configure_generic(input, weights, biases, output, conv_info, depth_multiplier, act_info, dilation); + } + + // Configure activation + if(_is_activationlayer_enabled) + { + _activationlayer_function.configure(output, nullptr, act_info); + } +} + +Status NEDepthwiseConvolutionLayerOptimized::validate(const ITensorInfo *input, + const ITensorInfo *weights, + const ITensorInfo *biases, + const ITensorInfo *output, + const PadStrideInfo &conv_info, + unsigned int depth_multiplier, + const ActivationLayerInfo &act_info, + const Size2D &dilation) +{ + ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, weights, output); + ARM_COMPUTE_RETURN_ERROR_ON(input->data_layout() == DataLayout::UNKNOWN); + ARM_COMPUTE_RETURN_ERROR_ON(dilation.x() < 1 || dilation.y() < 1); + const size_t idx_w = get_data_layout_dimension_index(input->data_layout(), DataLayoutDimension::WIDTH); + const size_t idx_h = get_data_layout_dimension_index(input->data_layout(), DataLayoutDimension::HEIGHT); + ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(idx_w) + (weights->dimension(idx_w) - 1) * (dilation.x() - 1) > input->dimension(idx_w) + conv_info.pad_left() + conv_info.pad_right()); + ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(idx_h) + (weights->dimension(idx_h) - 1) * (dilation.y() - 1) > input->dimension(idx_h) + conv_info.pad_top() + conv_info.pad_bottom()); + + if(biases != nullptr) + { + const unsigned int channel_idx = get_data_layout_dimension_index(input->data_layout(), DataLayoutDimension::CHANNEL); + ARM_COMPUTE_RETURN_ERROR_ON(biases->num_dimensions() > 1); + ARM_COMPUTE_RETURN_ERROR_ON(biases->dimension(0) != weights->dimension(channel_idx)); + } + + if(!NEDepthwiseConvolutionAssemblyDispatch::is_optimized_supported(input, weights, conv_info, depth_multiplier, dilation)) + { + const bool is_quantized = is_data_type_quantized_asymmetric(input->data_type()); + TensorInfo accumulator = TensorInfo(output->clone()->set_is_resizable(true).reset_padding().set_data_type(DataType::S32)); + ARM_COMPUTE_RETURN_ON_ERROR(NEDepthwiseConvolutionLayer3x3Kernel::validate(input, weights, is_quantized ? &accumulator : output, conv_info, depth_multiplier, dilation)); + + if(is_quantized) + { + ARM_COMPUTE_RETURN_ON_ERROR(NEDirectConvolutionLayerOutputStageKernel::validate(&accumulator, biases, output)); + } + } + else + { + ARM_COMPUTE_RETURN_ON_ERROR(NEDepthwiseConvolutionAssemblyDispatch::validate(input, weights, biases, output, conv_info, depth_multiplier, act_info, dilation)); + } + + //Validate Activation Layer + if(act_info.enabled()) + { + ARM_COMPUTE_RETURN_ON_ERROR(NEActivationLayer::validate(output, nullptr, act_info)); + } + + return Status{}; +} + +void NEDepthwiseConvolutionLayerOptimized::run_generic() +{ + // Fill border + NEScheduler::get().schedule(&_border_handler, Window::DimX); + + // Execute depthwise convolution + NEScheduler::get().schedule(&_dwc_kernel, Window::DimX); + + // Add biases + if(_has_bias || _is_quantized) + { + NEScheduler::get().schedule(&_output_stage_kernel, Window::DimX); + } + + // Permute output + if(!_is_nchw) + { + _permute_output.run(); + } +} + +void NEDepthwiseConvolutionLayerOptimized::run_optimized() +{ + // Run assembly function + _dwc_optimized_func.run(); + + // Permute output + if(_is_nchw) + { + _permute_output.run(); + } +} + +void NEDepthwiseConvolutionLayerOptimized::run() +{ + prepare(); + + MemoryGroupResourceScope scope_mg(_memory_group); + + // Permute input + if(_permute) + { + _permute_input.run(); + } + + _is_optimized ? run_optimized() : run_generic(); + + // Run activation + if(_is_activationlayer_enabled) + { + _activationlayer_function.run(); + } +} + +void NEDepthwiseConvolutionLayerOptimized::prepare() +{ + if(!_is_prepared) + { + // Permute weights + if(_permute) + { + _permuted_weights.allocator()->allocate(); + _permute_weights.run(); + _original_weights->mark_as_unused(); + } + + // Prepare optimized function + if(_is_optimized) + { + _dwc_optimized_func.prepare(); + if(!_permuted_weights.is_used()) + { + _permuted_weights.allocator()->free(); + } + } + + _is_prepared = true; + } +} + NEDepthwiseConvolutionLayer::NEDepthwiseConvolutionLayer() : _im2col_kernel(), _weights_reshape_kernel(), _v2mm_kernel(), _vector_to_tensor_kernel(), _output_stage_kernel(), _v2mm_input_fill_border(), _v2mm_weights_fill_border(), _permute_input(), _permute_weights(), _permute_output(), _activationlayer_function(), _input_reshaped(), _weights_reshaped(), _v2mm_output(), _output_reshaped(), _permuted_input(), _permuted_weights(), diff --git a/src/runtime/NEON/functions/assembly/NEDepthwiseConvolutionAssemblyDispatch.cpp b/src/runtime/NEON/functions/assembly/NEDepthwiseConvolutionAssemblyDispatch.cpp index 5f57bbfe23..b28aaa715f 100644 --- a/src/runtime/NEON/functions/assembly/NEDepthwiseConvolutionAssemblyDispatch.cpp +++ b/src/runtime/NEON/functions/assembly/NEDepthwiseConvolutionAssemblyDispatch.cpp @@ -26,7 +26,9 @@ #include "arm_compute/core/CPP/Validate.h" #include "arm_compute/core/ITensor.h" -#include "arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_quantized.hpp" +#include "arm_compute/core/NEON/kernels/assembly/NEDepthwiseConvolutionAssemblyKernelWrapper.h" +#include "arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_dilated.hpp" +#include "arm_compute/core/NEON/kernels/convolution/depthwise/depthwise_quantized_dilated.hpp" #include "arm_compute/core/Utils.h" #include "arm_compute/core/utils/misc/InfoHelpers.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" @@ -42,19 +44,22 @@ std::unique_ptr create_convolver(const ITensor const ITensor *weights, ITensor *output, PadStrideInfo conv_info, - ActivationLayerInfo act_info) + ActivationLayerInfo act_info, + const Size2D &dilation) { + ARM_COMPUTE_UNUSED(dilation); const DataType data_type = input->info()->data_type(); const TensorShape shape = input->info()->tensor_shape(); - const int n_batches = shape[3]; - const int in_rows = shape.z(); - const int in_cols = shape.y(); - const int n_channels = shape.x(); - const int padding_top = conv_info.pad_top(); - const int padding_left = conv_info.pad_left(); - const int padding_bottom = conv_info.pad_bottom(); - const int padding_right = conv_info.pad_right(); + const int n_batches = shape[3]; + const int in_rows = shape.z(); + const int in_cols = shape.y(); + const int n_channels = shape.x(); + const int dilation_factor = dilation.x(); + const int padding_top = conv_info.pad_top(); + const int padding_left = conv_info.pad_left(); + const int padding_bottom = conv_info.pad_bottom(); + const int padding_right = conv_info.pad_right(); const unsigned int stride_x = conv_info.stride().first; @@ -95,11 +100,11 @@ std::unique_ptr create_convolver(const ITensor switch(stride_x) { case 1: - return arm_compute::support::cpp14::make_unique>( - n_batches, in_rows, in_cols, n_channels, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right); + return arm_compute::support::cpp14::make_unique>( + n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right); case 2: - return arm_compute::support::cpp14::make_unique>( - n_batches, in_rows, in_cols, n_channels, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right); + return arm_compute::support::cpp14::make_unique>( + n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, wqinfo, iqinfo, oqinfo, rescale_params, padding_top, padding_left, padding_bottom, padding_right); default: return nullptr; } @@ -115,11 +120,11 @@ std::unique_ptr create_convolver(const ITensor switch(stride_x) { case 1: - return arm_compute::support::cpp14::make_unique>( - n_batches, in_rows, in_cols, n_channels, activation, padding_top, padding_left, padding_bottom, padding_right); + return arm_compute::support::cpp14::make_unique>( + n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right); case 2: - return arm_compute::support::cpp14::make_unique>( - n_batches, in_rows, in_cols, n_channels, activation, padding_top, padding_left, padding_bottom, padding_right); + return arm_compute::support::cpp14::make_unique>( + n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right); default: return nullptr; } @@ -131,11 +136,11 @@ std::unique_ptr create_convolver(const ITensor switch(stride_x) { case 1: - return arm_compute::support::cpp14::make_unique>( - n_batches, in_rows, in_cols, n_channels, activation, padding_top, padding_left, padding_bottom, padding_right); + return arm_compute::support::cpp14::make_unique>( + n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right); case 2: - return arm_compute::support::cpp14::make_unique>( - n_batches, in_rows, in_cols, n_channels, activation, padding_top, padding_left, padding_bottom, padding_right); + return arm_compute::support::cpp14::make_unique>( + n_batches, in_rows, in_cols, n_channels, dilation_factor, activation, padding_top, padding_left, padding_bottom, padding_right); default: return nullptr; } @@ -148,21 +153,30 @@ std::unique_ptr create_convolver(const ITensor } } // namespace +struct NEDepthwiseConvolutionAssemblyDispatch::LocalImpl +{ + std::unique_ptr _dwc_assembly_kernel{ nullptr }; + NEDepthwiseConvolutionAssemblyKernelWrapper _dwc_acl_kernel{}; +}; + #ifndef DOXYGEN_SKIP_THIS NEDepthwiseConvolutionAssemblyDispatch::NEDepthwiseConvolutionAssemblyDispatch(std::shared_ptr memory_manager) - : _memory_group(std::move(memory_manager)), _input(nullptr), _weights(nullptr), _bias(nullptr), _output(nullptr), _packed_weights(), _workspace(), _is_prepared(false), _dwc_assembly_kernel(nullptr), - _dwc_acl_kernel() + : _memory_group(std::move(memory_manager)), _input(nullptr), _weights(nullptr), _bias(nullptr), _output(nullptr), _packed_weights(), _workspace(), _is_prepared(false), + _pImpl(support::cpp14::make_unique()) { } #endif /* DOXYGEN_SKIP_THIS */ +NEDepthwiseConvolutionAssemblyDispatch::~NEDepthwiseConvolutionAssemblyDispatch() = default; + void NEDepthwiseConvolutionAssemblyDispatch::configure(const ITensor *input, const ITensor *weights, const ITensor *bias, ITensor *output, const PadStrideInfo &conv_info, unsigned int depth_multiplier, - const ActivationLayerInfo &act_info) + const ActivationLayerInfo &act_info, + const Size2D &dilation) { ARM_COMPUTE_ERROR_ON_NULLPTR(input, weights, output); ARM_COMPUTE_UNUSED(depth_multiplier); @@ -172,10 +186,11 @@ void NEDepthwiseConvolutionAssemblyDispatch::configure(const ITensor output->info(), conv_info, depth_multiplier, - act_info)); + act_info, + dilation)); // Output auto inizialitation if not yet initialized - const TensorShape output_shape = misc::shape_calculator::compute_depthwise_convolution_shape(*input->info(), *weights->info(), conv_info, depth_multiplier); + const TensorShape output_shape = misc::shape_calculator::compute_depthwise_convolution_shape(*input->info(), *weights->info(), conv_info, depth_multiplier, dilation); auto_init_if_empty(*output->info(), input->info()->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(output_shape).set_quantization_info(output->info()->quantization_info())); _input = input; @@ -185,24 +200,24 @@ void NEDepthwiseConvolutionAssemblyDispatch::configure(const ITensor _is_prepared = false; // Create convolver - _dwc_assembly_kernel = create_convolver(input, weights, output, conv_info, act_info); - ARM_COMPUTE_ERROR_ON(_dwc_assembly_kernel == nullptr); + _pImpl->_dwc_assembly_kernel = create_convolver(input, weights, output, conv_info, act_info, dilation); + ARM_COMPUTE_ERROR_ON(_pImpl->_dwc_assembly_kernel == nullptr); // Create assembly kernel wrapper - _dwc_acl_kernel.configure(_dwc_assembly_kernel.get()); + _pImpl->_dwc_acl_kernel.configure(_pImpl->_dwc_assembly_kernel.get()); constexpr size_t alignment = 128; // Create workspace const unsigned int num_threads = NEScheduler::get().num_threads(); - const size_t workspace_size = _dwc_assembly_kernel->get_working_space_size(num_threads); + const size_t workspace_size = _pImpl->_dwc_assembly_kernel->get_working_space_size(num_threads); ARM_COMPUTE_ERROR_ON_MSG(workspace_size == 0, "Workspace size cannot be 0 !"); _workspace.allocator()->init(TensorInfo(TensorShape{ workspace_size }, 1, DataType::S8), alignment); _memory_group.manage(&_workspace); _workspace.allocator()->allocate(); // Create packing tensor - const size_t pack_tensor_size = _dwc_assembly_kernel->get_packed_params_size(); + const size_t pack_tensor_size = _pImpl->_dwc_assembly_kernel->get_packed_params_size(); ARM_COMPUTE_ERROR_ON_MSG(pack_tensor_size == 0, "Pack tensor size cannot be 0 !"); _packed_weights.allocator()->init(TensorInfo(TensorShape{ pack_tensor_size }, 1, DataType::S8), alignment); } @@ -213,7 +228,8 @@ Status NEDepthwiseConvolutionAssemblyDispatch::validate(const ITensorInfo const ITensorInfo *output, const PadStrideInfo &conv_info, unsigned int depth_multiplier, - const ActivationLayerInfo &act_info) + const ActivationLayerInfo &act_info, + const Size2D &dilation) { ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(input); ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8, DataType::F16, DataType::F32); @@ -227,6 +243,7 @@ Status NEDepthwiseConvolutionAssemblyDispatch::validate(const ITensorInfo ARM_COMPUTE_RETURN_ERROR_ON(weights->dimension(width_idx) != 3 || weights->dimension(height_idx) != 3); ARM_COMPUTE_RETURN_ERROR_ON(!((strides.first == strides.second) && ((strides.first == 1) || (strides.first == 2)))); ARM_COMPUTE_RETURN_ERROR_ON(depth_multiplier != 1); + ARM_COMPUTE_RETURN_ERROR_ON(dilation.x() != dilation.y()); const bool is_relu = arm_compute::utils::info_helpers::is_relu(act_info); const bool is_relu6 = arm_compute::utils::info_helpers::is_relu6(act_info); @@ -243,7 +260,7 @@ Status NEDepthwiseConvolutionAssemblyDispatch::validate(const ITensorInfo // Check output if(output->total_size() != 0) { - const TensorShape output_shape = misc::shape_calculator::compute_depthwise_convolution_shape(*input, *weights, conv_info, depth_multiplier); + const TensorShape output_shape = misc::shape_calculator::compute_depthwise_convolution_shape(*input, *weights, conv_info, depth_multiplier, dilation); ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DIMENSIONS(output->tensor_shape(), output_shape); ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_DATA_TYPES(input, output); } @@ -283,17 +300,17 @@ bool NEDepthwiseConvolutionAssemblyDispatch::is_optimized_supported(const ITenso bool supported_strides = (strides.first == strides.second) && ((strides.first == 1) || (strides.first == 2)); // Check for supported padding - const auto pad_top = conv_info.pad_top(); - const auto pad_right = conv_info.pad_right(); - const auto pad_bottom = conv_info.pad_bottom(); - const auto pad_left = conv_info.pad_left(); - PadStrideInfo same_pad = calculate_same_pad(in_shape, TensorShape(3U, 3U), conv_info); - bool is_same_padding = (pad_top == same_pad.pad_top()) && (pad_right == same_pad.pad_right()) && (pad_bottom == same_pad.pad_bottom()) && (pad_left == same_pad.pad_left()); - bool is_valid_padding = (pad_top == 0) && (pad_right == 0) && (pad_bottom == 0) && (pad_left == 0); - bool supported_padding = is_same_padding || is_valid_padding; - bool is_dilation_1 = dilation.x() == 1 && dilation.y() == 1; - - return is_data_type_valid && weights_supported && supported_strides && supported_padding && (depth_multiplier == 1) && is_dilation_1; + const auto pad_top = conv_info.pad_top(); + const auto pad_right = conv_info.pad_right(); + const auto pad_bottom = conv_info.pad_bottom(); + const auto pad_left = conv_info.pad_left(); + PadStrideInfo same_pad = calculate_same_pad(in_shape, TensorShape(3U, 3U), conv_info); + bool is_same_padding = (pad_top == same_pad.pad_top()) && (pad_right == same_pad.pad_right()) && (pad_bottom == same_pad.pad_bottom()) && (pad_left == same_pad.pad_left()); + bool is_valid_padding = (pad_top == 0) && (pad_right == 0) && (pad_bottom == 0) && (pad_left == 0); + bool supported_padding = is_same_padding || is_valid_padding; + bool is_dilation_supported = (dilation.x() == dilation.y()) || (dilation == Size2D(1U, 1U)); + + return is_data_type_valid && weights_supported && supported_strides && supported_padding && (depth_multiplier == 1) && is_dilation_supported; } void NEDepthwiseConvolutionAssemblyDispatch::run() @@ -305,7 +322,7 @@ void NEDepthwiseConvolutionAssemblyDispatch::run() // Setup inputs/outputs ARM_COMPUTE_ERROR_ON(_workspace.buffer() == nullptr); - _dwc_assembly_kernel->set_working_space(static_cast(_workspace.buffer())); + _pImpl->_dwc_assembly_kernel->set_working_space(static_cast(_workspace.buffer())); ARM_COMPUTE_ERROR_ON(_input->buffer() == nullptr); const int input_element_size = _input->info()->element_size(); @@ -313,7 +330,7 @@ void NEDepthwiseConvolutionAssemblyDispatch::run() const int input_row_stride = _input->info()->strides_in_bytes().z() / input_element_size; const int input_col_stride = _input->info()->strides_in_bytes().y() / input_element_size; const void *input_ptr = _input->buffer() + _input->info()->offset_first_element_in_bytes(); - _dwc_assembly_kernel->set_input(input_ptr, input_batch_stride, input_row_stride, input_col_stride); + _pImpl->_dwc_assembly_kernel->set_input(input_ptr, input_batch_stride, input_row_stride, input_col_stride); ARM_COMPUTE_ERROR_ON(_output->buffer() == nullptr); const int output_element_size = _output->info()->element_size(); @@ -321,10 +338,10 @@ void NEDepthwiseConvolutionAssemblyDispatch::run() const int output_row_stride = _output->info()->strides_in_bytes().z() / output_element_size; const int output_col_stride = _output->info()->strides_in_bytes().y() / output_element_size; void *output_ptr = _output->buffer() + _output->info()->offset_first_element_in_bytes(); - _dwc_assembly_kernel->set_output(output_ptr, output_batch_stride, output_row_stride, output_col_stride); + _pImpl->_dwc_assembly_kernel->set_output(output_ptr, output_batch_stride, output_row_stride, output_col_stride); // Schedule assembly kernel - NEScheduler::get().schedule(&_dwc_acl_kernel, Window::DimX); + NEScheduler::get().schedule(&_pImpl->_dwc_acl_kernel, Window::DimX); } void NEDepthwiseConvolutionAssemblyDispatch::prepare() @@ -338,12 +355,12 @@ void NEDepthwiseConvolutionAssemblyDispatch::prepare() const int weights_element_size = _weights->info()->element_size(); const int weights_row_stride = _weights->info()->strides_in_bytes().z() / weights_element_size; const int weights_col_stride = _weights->info()->strides_in_bytes().y() / weights_element_size; - _dwc_assembly_kernel->pack_params(_packed_weights.buffer(), - _weights->buffer() + _weights->info()->offset_first_element_in_bytes(), - weights_row_stride, - weights_col_stride, - (_bias != nullptr) ? _bias->buffer() : nullptr); - _dwc_assembly_kernel->set_packed_params_buffer(_packed_weights.buffer()); + _pImpl->_dwc_assembly_kernel->pack_params(_packed_weights.buffer(), + _weights->buffer() + _weights->info()->offset_first_element_in_bytes(), + weights_row_stride, + weights_col_stride, + (_bias != nullptr) ? _bias->buffer() : nullptr); + _pImpl->_dwc_assembly_kernel->set_packed_params_buffer(_packed_weights.buffer()); _weights->mark_as_unused(); if(_bias != nullptr) -- cgit v1.2.1