From ebbae9411f1babe6bba9966ba83c540d11e77410 Mon Sep 17 00:00:00 2001 From: Dana Zlotnik Date: Thu, 3 Feb 2022 12:52:15 +0200 Subject: Decouple CpuDepthwiseConv2dNativeKernel Resolves COMPMID-4632 Change-Id: I5e2a9f0f7801a2afaa35de871ab29cd7238923fd Signed-off-by: Dana Zlotnik Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/7115 Tested-by: Arm Jenkins Reviewed-by: Giorgio Arena Comments-Addressed: Arm Jenkins --- src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.cpp | 851 ++------------------- 1 file changed, 49 insertions(+), 802 deletions(-) (limited to 'src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.cpp') diff --git a/src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.cpp b/src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.cpp index d09cc1debf..f47df1e405 100644 --- a/src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.cpp +++ b/src/cpu/kernels/CpuDepthwiseConv2dNativeKernel.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019-2021 Arm Limited. + * Copyright (c) 2019-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,12 +26,12 @@ #include "arm_compute/core/ITensor.h" #include "arm_compute/core/ITensorInfo.h" #include "arm_compute/core/utils/misc/ShapeCalculator.h" -#include "arm_compute/core/utils/quantization/AsymmHelpers.h" #include "src/core/CPP/Validate.h" #include "src/core/NEON/wrapper/traits.h" -#include "src/core/NEON/wrapper/wrapper.h" +#include "src/core/common/Registrars.h" #include "src/core/helpers/AutoConfiguration.h" #include "src/core/helpers/WindowHelpers.h" +#include "src/cpu/kernels/depthwiseconv2d/list.h" #include "support/ToolchainSupport.h" namespace arm_compute @@ -42,717 +42,57 @@ namespace kernels { namespace { -constexpr auto data_layout = DataLayout::NHWC; -const size_t width_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::WIDTH); -const size_t height_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::HEIGHT); -const size_t channel_idx = get_data_layout_dimension_index(data_layout, DataLayoutDimension::CHANNEL); - -constexpr auto dim_manual_loop = Window::Dimension(0, 0, 0); -constexpr auto dim_single_unit_step = Window::Dimension(0, 1, 1); -constexpr size_t vector_size = 8; - -struct DepthwiseConvolutionRunInfo +static const std::vector available_kernels = { - const size_t num_read_elements_per_iteration; - const uint32_t x_start; - const uint32_t x_end; - const uint32_t x_step; - const uint32_t x_leftover_start; - const size_t input_stride_y; - const size_t input_stride_z; - const size_t input_max_offset; - const size_t weights_width; - const size_t weights_height; - const size_t weights_stride_y; - const size_t weights_stride_z; - const size_t conv_stride_x; - const size_t conv_stride_y; - const size_t conv_pad_left; - const size_t conv_pad_top; - const size_t input_height; - const size_t input_width; - const size_t input_depth; - - DepthwiseConvolutionRunInfo(const ITensorInfo &input, const ITensorInfo &weights, const PadStrideInfo &conv_info, const Window &w, uint32_t depth_multiplier = 1) // NOLINT - : num_read_elements_per_iteration((depth_multiplier == 1 ? (vector_size / element_size_from_data_type(input.data_type())) : 1)), - x_start(w.x().start()), - x_end(w.x().end()), - x_step(static_cast(num_read_elements_per_iteration * depth_multiplier)), - x_leftover_start(std::max(static_cast(w.x().end() + 1) - static_cast(x_step), int32_t(0))), - input_stride_y(input.strides_in_bytes().y()), - input_stride_z(input.strides_in_bytes().z()), - input_max_offset(input.strides_in_bytes().z() * input.dimension(height_idx) - (input.padding().bottom + input.padding().top) * input.strides_in_bytes().y()), - weights_width(weights.dimension(width_idx)), - weights_height(weights.dimension(height_idx)), - weights_stride_y(weights.strides_in_bytes().y()), - weights_stride_z(weights.strides_in_bytes().z()), - conv_stride_x(conv_info.stride().first), - conv_stride_y(conv_info.stride().second), - conv_pad_left(conv_info.pad_left()), - conv_pad_top(conv_info.pad_top()), - input_height(input.dimension(height_idx)), - input_width(input.dimension(width_idx)), - input_depth(input.dimension(channel_idx)) { - } -}; - -inline int32x4_t saturating_doubling_high_mul(const int32x4_t &a, const int32_t &b) -{ - return vqrdmulhq_n_s32(a, b); -} - -inline int32_t saturating_doubling_high_mul(const int32_t &a, const int32_t &b) -{ - return vget_lane_s32(vqrdmulh_n_s32(vdup_n_s32(a), b), 0); -} - -inline int32x4_t rounding_divide_by_exp2(const int32x4_t &x, const int exponent) -{ - const int32x4_t shift = vdupq_n_s32(-exponent); - const int32x4_t fixup = vshrq_n_s32(vandq_s32(x, shift), 31); - const int32x4_t fixed = vqaddq_s32(x, fixup); - return vrshlq_s32(fixed, shift); -} - -inline int32x2_t rounding_divide_by_exp2(const int32x2_t &x, const int exponent) -{ - const int32x2_t shift = vdup_n_s32(-exponent); - const int32x2_t fixup = vshr_n_s32(vand_s32(x, shift), 31); - const int32x2_t fixed = vqadd_s32(x, fixup); - return vrshl_s32(fixed, shift); -} - -inline int32_t rounding_divide_by_exp2(const int32_t &x, const int exponent) -{ - const int32x2_t xs = vdup_n_s32(x); - return vget_lane_s32(rounding_divide_by_exp2(xs, exponent), 0); -} - -inline bool is_valid_input_region(int32_t base_w, uint32_t base_h, uint32_t w, uint32_t h, const DepthwiseConvolutionRunInfo &run_info, const Size2D &dilation) -{ - const int32_t current_h = base_h + h * dilation.y(); - const bool is_valid_h = current_h >= 0 && current_h < static_cast(run_info.input_height); - - const int32_t current_w = base_w + w * dilation.x(); - const bool is_valid_w = current_w >= 0 && current_w < static_cast(run_info.input_width); - - return is_valid_h && is_valid_w; -} - -template -void depthwise_loop_multiplier1_fp(const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const PadStrideInfo &conv_info, - const Size2D &dilation, const Window &window, bool has_biases) -{ - constexpr auto element_per_vector = vector_size / sizeof(T); - using VectorType = typename wrapper::traits::neon_vector::type; - using TagType = typename wrapper::traits::neon_vector::tag_type; - - const auto run_info = DepthwiseConvolutionRunInfo(*src->info(), *weights->info(), conv_info, window); - - const VectorType zero_vector = wrapper::vdup_n(static_cast(0), TagType{}); - - Window execution_window = window; - execution_window.set(Window::DimX, dim_single_unit_step); - - Window win_input = window; - win_input.set(Window::DimX, dim_manual_loop); - win_input.set(Window::DimY, dim_manual_loop); - win_input.set(Window::DimZ, dim_manual_loop); - - Window win_weights = win_input; - win_weights.set(Window::DimW, dim_manual_loop); - - Window win_output = window; - win_output.set(Window::DimX, dim_manual_loop); - - Iterator input_it(src, win_input); - Iterator weights_it(weights, win_weights); - Iterator output_it(dst, win_output); - Iterator biases_it{}; - - if(has_biases) - { - biases_it = Iterator(biases, win_weights); - } - - execute_window_loop(execution_window, [&](const Coordinates & id) - { - const int32_t input_y = id.y() * run_info.conv_stride_x - run_info.conv_pad_left; - const int32_t input_z = id.z() * run_info.conv_stride_y - run_info.conv_pad_top; - const int64_t base_input_offset = input_y * run_info.input_stride_y + input_z * run_info.input_stride_z; - - auto const base_weights_ptr = weights_it.ptr(); - uint32_t x = run_info.x_start; - - for(; x < run_info.x_leftover_start; x += run_info.x_step) - { - VectorType acc = zero_vector; - auto weights_ptr = base_weights_ptr; - int64_t input_offset = base_input_offset; - - for(uint32_t h = 0; h < run_info.weights_height; ++h) - { - int64_t offs = input_offset + x * sizeof(T); - for(uint32_t w = 0; w < run_info.weights_width; ++w) - { - const bool is_valid_region = is_valid_input_region(input_y, input_z, w, h, run_info, dilation); - const auto input_vals = is_valid_region ? - wrapper::vload(reinterpret_cast(input_it.ptr() + std::min(static_cast(offs), run_info.input_max_offset))) : - zero_vector; - const auto weights_vals = wrapper::vload(reinterpret_cast(weights_ptr + w * run_info.weights_stride_y) + x); - acc = wrapper::vmla(acc, weights_vals, input_vals); - - offs += dilation.x() * run_info.input_stride_y; - } - - weights_ptr += run_info.weights_stride_z; - input_offset += dilation.y() * run_info.input_stride_z; - } - - if(has_biases) - { - const auto biases_vals = wrapper::vload(reinterpret_cast(biases_it.ptr()) + x); - acc = wrapper::vadd(acc, biases_vals); - } - - wrapper::vstore(reinterpret_cast(output_it.ptr()) + x, acc); - } - - for(; x < run_info.x_end; ++x) + "neon_qu8_deptwiseconv2dnative", + [](const DepthwiseConv2dNativeDataTypeISASelectorData & data) { - auto acc_scalar = T{ 0 }; - auto weights_ptr = base_weights_ptr; - int64_t input_offset = base_input_offset; - - for(size_t h = 0; h < run_info.weights_height; ++h) - { - int64_t offs = input_offset + x * sizeof(T); - for(size_t w = 0; w < run_info.weights_width; ++w) - { - const bool is_valid_region = is_valid_input_region(input_y, input_z, w, h, run_info, dilation); - const auto input_vals = is_valid_region ? *reinterpret_cast(input_it.ptr() + std::min(static_cast(offs), run_info.input_max_offset)) : 0; - const auto weights_vals = *(reinterpret_cast(weights_ptr + w * run_info.weights_stride_y) + x); - - acc_scalar += (input_vals * weights_vals); - - offs += dilation.x() * run_info.input_stride_y; - } - - weights_ptr += run_info.weights_stride_z; - input_offset += dilation.y() * run_info.input_stride_z; - } - - if(has_biases) - { - const auto biases_vals = *(reinterpret_cast(biases_it.ptr()) + x); - acc_scalar += biases_vals; - } - *(reinterpret_cast(output_it.ptr()) + x) = acc_scalar; - } + return (data.weights_dt == DataType::QASYMM8); + }, + REGISTER_QASYMM8_NEON(neon_qu8_deptwiseconv2dnative) }, - input_it, weights_it, biases_it, output_it); -} - -template -void depthwise_loop_generic_fp(const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const PadStrideInfo &conv_info, - const Size2D &dilation, unsigned int depth_multiplier, const Window &window, bool has_biases) -{ - const auto run_info = DepthwiseConvolutionRunInfo(*src->info(), *weights->info(), conv_info, window, depth_multiplier); - - Window execution_window = window; - execution_window.set(Window::DimX, Window::Dimension(0, run_info.input_depth, 1)); - - Window win_input = execution_window; - win_input.set(Window::DimX, Window::Dimension(0, run_info.input_depth, 1)); - win_input.set(Window::DimY, dim_manual_loop); - win_input.set(Window::DimZ, dim_manual_loop); - - Window win_weights = window; - win_weights.set_dimension_step(Window::DimX, run_info.x_step); - win_weights.set(Window::DimY, dim_manual_loop); - win_weights.set(Window::DimZ, dim_manual_loop); - win_weights.set(Window::DimW, dim_manual_loop); - - Window win_output = window; - win_output.set_dimension_step(Window::DimX, run_info.x_step); - - Iterator input_it(src, win_input); - Iterator weights_it(weights, win_weights); - Iterator output_it(dst, win_output); - Iterator biases_it{}; - - if(has_biases) { - biases_it = Iterator(biases, win_weights); - } - - execute_window_loop(execution_window, [&](const Coordinates & id) - { - std::vector acc(depth_multiplier, static_cast(0)); - - const int input_y = id.y() * run_info.conv_stride_x - run_info.conv_pad_left; - const int input_z = id.z() * run_info.conv_stride_y - run_info.conv_pad_top; - int input_offset = input_y * run_info.input_stride_y + input_z * run_info.input_stride_z; - - auto weights_ptr = weights_it.ptr(); - for(size_t h = 0; h < run_info.weights_height; ++h) - { - int offs = input_offset; - for(size_t w = 0; w < run_info.weights_width; ++w) - { - const bool is_valid_region = is_valid_input_region(input_y, input_z, w, h, run_info, dilation); - const auto input_val = is_valid_region ? *(reinterpret_cast(input_it.ptr() + std::min(static_cast(offs), run_info.input_max_offset))) : T(0); - - for(size_t m = 0; m < depth_multiplier; ++m) - { - const auto weights_val = *(reinterpret_cast(weights_ptr + m * sizeof(T) + w * run_info.weights_stride_y)); - acc.at(m) = support::cpp11::fma(weights_val, input_val, acc.at(m)); - } - - offs += dilation.x() * run_info.input_stride_y; - } - - weights_ptr += run_info.weights_stride_z; - input_offset += dilation.y() * run_info.input_stride_z; - } - - if(has_biases) - { - for(size_t m = 0; m < depth_multiplier; ++m) - { - const auto biases_val = *(reinterpret_cast(biases_it.ptr() + m * sizeof(T))); - *(reinterpret_cast(output_it.ptr() + m * sizeof(T))) = acc.at(m) + biases_val; - } - } - else + "neon_qs8_deptwiseconv2dnative", + [](const DepthwiseConv2dNativeDataTypeISASelectorData & data) { - for(size_t m = 0; m < depth_multiplier; ++m) - { - *(reinterpret_cast(output_it.ptr() + m * sizeof(T))) = acc.at(m); - } - } + return (data.weights_dt == DataType::QASYMM8_SIGNED); + }, + REGISTER_QASYMM8_SIGNED_NEON(neon_qs8_deptwiseconv2dnative) }, - input_it, weights_it, biases_it, output_it); -} - -template -void depthwise_loop_multiplier1_quantized(const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const PadStrideInfo &conv_info, - const Size2D &dilation, std::vector output_multiplier, std::vector output_shift, const Window &window, bool has_biases) // NOLINT -{ - ARM_COMPUTE_UNUSED(output_multiplier, output_shift); - constexpr auto element_per_vector = vector_size / sizeof(T); - using VectorType = typename wrapper::traits::neon_vector::type; - using TagType = typename wrapper::traits::neon_vector::tag_type; - using AccType = int32_t; - using AccArrayType = std::array; - - const auto out_of_bound_value = PixelValue(static_cast(0), src->info()->data_type(), src->info()->quantization_info()).get(); - const auto out_of_bound_vector = wrapper::vdup_n(static_cast(out_of_bound_value), TagType{}); - - const auto run_info = DepthwiseConvolutionRunInfo(*src->info(), *weights->info(), conv_info, window); - - const int32_t input_qoffset = src->info()->quantization_info().uniform().offset; - const int32_t weights_qoffset = weights->info()->quantization_info().uniform().offset; - const int32_t output_qoffset = dst->info()->quantization_info().uniform().offset; - const int32_t k_offset = run_info.weights_width * run_info.weights_height * input_qoffset * weights_qoffset; - - Window execution_window = window; - execution_window.set(Window::DimX, dim_single_unit_step); - - Window win_input = window; - win_input.set(Window::DimX, dim_manual_loop); - win_input.set(Window::DimY, dim_manual_loop); - win_input.set(Window::DimZ, dim_manual_loop); - - Window win_weights = win_input; - win_weights.set(Window::DimW, dim_manual_loop); - - Window win_output = window; - win_output.set(Window::DimX, dim_manual_loop); - - Iterator input_it(src, win_input); - Iterator weights_it(weights, win_weights); - Iterator output_it(dst, win_output); - Iterator biases_it{}; - - if(has_biases) - { - biases_it = Iterator(biases, win_weights); - } - - execute_window_loop(execution_window, [&](const Coordinates & id) { - const int32_t input_y = id.y() * run_info.conv_stride_x - run_info.conv_pad_left; - const int32_t input_z = id.z() * run_info.conv_stride_y - run_info.conv_pad_top; - const int64_t base_input_offset = input_y * run_info.input_stride_y + input_z * run_info.input_stride_z; - auto const base_weights_ptr = weights_it.ptr(); - size_t x = run_info.x_start; - - for(; x < run_info.x_leftover_start; x += run_info.x_step) - { - AccArrayType acc{}; - AccArrayType in_sum{}; - AccArrayType we_sum{}; - - auto weights_ptr = base_weights_ptr; - auto input_offset = base_input_offset; - - for(size_t h = 0; h < run_info.weights_height; ++h) - { - int64_t offs = input_offset + x * sizeof(T); - for(size_t w = 0; w < run_info.weights_width; ++w) - { - const bool is_valid_region = is_valid_input_region(input_y, input_z, w, h, run_info, dilation); - const auto input_vals = is_valid_region ? - wrapper::vload(reinterpret_cast(input_it.ptr() + std::min(static_cast(offs), run_info.input_max_offset))) : - out_of_bound_vector; - const auto weights_vals = wrapper::vload(reinterpret_cast(weights_ptr + w * run_info.weights_stride_y) + x); - - for(size_t i = 0; i < element_per_vector; ++i) - { - acc.at(i) += input_vals[i] * weights_vals[i]; - in_sum.at(i) += input_vals[i]; - we_sum.at(i) += weights_vals[i]; - } - - offs += dilation.x() * run_info.input_stride_y; - } - - weights_ptr += run_info.weights_stride_z; - input_offset += dilation.y() * run_info.input_stride_z; - } - - VectorType out_vals = wrapper::vdup_n(static_cast(0), TagType{}); - for(size_t i = 0; i < element_per_vector; ++i) - { - acc.at(i) -= in_sum.at(i) * weights_qoffset; - acc.at(i) -= we_sum.at(i) * input_qoffset; - acc.at(i) += k_offset; - - if(has_biases) - { - acc.at(i) += *(reinterpret_cast(biases_it.ptr() + i * sizeof(int32_t)) + x); - } - - const int32_t out_mul = output_multiplier.at(x + i); - const int32_t out_shift = output_shift.at(x + i); - if(out_shift < 0) - { - acc.at(i) = saturating_doubling_high_mul(acc.at(i) * (1 << (-out_shift)), out_mul) + output_qoffset; - } - else - { - acc.at(i) = rounding_divide_by_exp2(saturating_doubling_high_mul(acc.at(i), out_mul), out_shift) + output_qoffset; - } - out_vals[i] = static_cast(utility::clamp(acc.at(i))); - } - - wrapper::vstore(reinterpret_cast(output_it.ptr()) + x, out_vals); - } - - // left-over - for(; x < run_info.x_end; ++x) + "neon_fp16_deptwiseconv2dnative", + [](const DepthwiseConv2dNativeDataTypeISASelectorData & data) { - AccType acc = 0; - AccType in_sum = 0; - AccType we_sum = 0; - - auto weights_ptr = base_weights_ptr; - auto input_offset = base_input_offset; - - for(size_t h = 0; h < run_info.weights_height; ++h) - { - int64_t offs = input_offset + x * sizeof(T); - for(size_t w = 0; w < run_info.weights_width; ++w) - { - const bool is_valid_region = is_valid_input_region(input_y, input_z, w, h, run_info, dilation); - const auto input_val = is_valid_region ? - *reinterpret_cast(input_it.ptr() + std::min(static_cast(offs), run_info.input_max_offset)) : - out_of_bound_value; - const auto weights_val = *(reinterpret_cast(weights_ptr + w * run_info.weights_stride_y) + x); - - acc += input_val * weights_val; - in_sum += input_val; - we_sum += weights_val; - - offs += dilation.x() * run_info.input_stride_y; - } - - weights_ptr += run_info.weights_stride_z; - input_offset += dilation.y() * run_info.input_stride_z; - } - - T out_vals{ 0 }; - - acc -= in_sum * weights_qoffset; - acc -= we_sum * input_qoffset; - acc += k_offset; - - if(has_biases) - { - acc += *(reinterpret_cast(biases_it.ptr()) + x); - } - - const int32_t out_mul = output_multiplier.at(x); - const int32_t out_shift = output_shift.at(x); - - if(out_shift < 0) - { - acc = saturating_doubling_high_mul(acc * (1 << (-out_shift)), out_mul) + output_qoffset; - } - else - { - acc = rounding_divide_by_exp2(saturating_doubling_high_mul(acc, out_mul), out_shift) + output_qoffset; - } - - out_vals = static_cast(utility::clamp(acc)); - *(reinterpret_cast(output_it.ptr()) + x) = out_vals; - } + return (data.weights_dt == DataType::F16 && data.isa.fp16); + }, + REGISTER_FP16_NEON(neon_fp16_deptwiseconv2dnative) }, - input_it, weights_it, biases_it, output_it); -} - -template -void depthwise_loop_generic_quantized(const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const PadStrideInfo &conv_info, - const Size2D &dilation, unsigned int depth_multiplier, std::vector output_multiplier, std::vector output_shift, const Window &window, bool has_biases) // NOLINT -{ - using AccType = int32_t; - - const auto run_info = DepthwiseConvolutionRunInfo(*src->info(), *weights->info(), conv_info, window, depth_multiplier); - - const auto out_of_bound_value = PixelValue(static_cast(0), src->info()->data_type(), src->info()->quantization_info()).get(); - - const int32_t input_qoffset = src->info()->quantization_info().uniform().offset; - const int32_t weights_qoffset = weights->info()->quantization_info().uniform().offset; - const int32_t output_qoffset = dst->info()->quantization_info().uniform().offset; - const int32_t k_offset = run_info.weights_width * run_info.weights_height * input_qoffset * weights_qoffset; - - Window execution_window = window; - execution_window.set(Window::DimX, Window::Dimension(0, run_info.input_depth, 1)); - - Window win_input = execution_window; - win_input.set(Window::DimY, dim_manual_loop); - win_input.set(Window::DimZ, dim_manual_loop); - - Window win_weights = window; - win_weights.set_dimension_step(Window::DimX, run_info.x_step); - win_weights.set(Window::DimY, dim_manual_loop); - win_weights.set(Window::DimZ, dim_manual_loop); - win_weights.set(Window::DimW, dim_manual_loop); - - Window win_output = window; - win_output.set_dimension_step(Window::DimX, run_info.x_step); - - Iterator input_it(src, win_input); - Iterator weights_it(weights, win_weights); - Iterator output_it(dst, win_output); - Iterator biases_it{}; - - if(has_biases) - { - biases_it = Iterator(biases, win_weights); - } - - execute_window_loop(execution_window, [&](const Coordinates & id) { - std::vector acc(depth_multiplier, 0); - std::vector we_sum(depth_multiplier, 0); - AccType in_sum = 0; - - const int32_t input_y = id.y() * run_info.conv_stride_x - run_info.conv_pad_left; - const int32_t input_z = id.z() * run_info.conv_stride_y - run_info.conv_pad_top; - int64_t input_offset = input_y * run_info.input_stride_y + input_z * run_info.input_stride_z; - - auto weights_ptr = weights_it.ptr(); - for(size_t h = 0; h < run_info.weights_height; ++h) - { - int offs = input_offset; - for(size_t w = 0; w < run_info.weights_width; ++w) - { - const bool is_valid_region = is_valid_input_region(input_y, input_z, w, h, run_info, dilation); - const auto input_val = is_valid_region ? *(reinterpret_cast(input_it.ptr() + std::min(static_cast(offs), run_info.input_max_offset))) : out_of_bound_value; - - for(size_t m = 0; m < depth_multiplier; ++m) - { - const auto weights_val = *(reinterpret_cast(weights_ptr + m * sizeof(T) + w * run_info.weights_stride_y)); - acc.at(m) += input_val * weights_val; - - we_sum.at(m) += weights_val; - } - - offs += dilation.x() * run_info.input_stride_y; - in_sum += input_val; - } - - weights_ptr += run_info.weights_stride_z; - input_offset += dilation.y() * run_info.input_stride_z; - } - - for(size_t m = 0; m < depth_multiplier; ++m) + "neon_fp32_deptwiseconv2dnative", + [](const DepthwiseConv2dNativeDataTypeISASelectorData & data) { - acc.at(m) -= in_sum * weights_qoffset; - acc.at(m) -= we_sum.at(m) * input_qoffset; - acc.at(m) += k_offset; - - if(has_biases) - { - acc.at(m) += *(reinterpret_cast(biases_it.ptr() + m * sizeof(int32_t))); - } - - const int32_t out_mul = output_multiplier.at(id.x() * depth_multiplier + m); - const int32_t out_shift = output_shift.at(id.x() * depth_multiplier + m); - if(out_shift < 0) - { - acc.at(m) = saturating_doubling_high_mul(acc.at(m) * (1 << (-out_shift)), out_mul) + output_qoffset; - } - else - { - acc.at(m) = rounding_divide_by_exp2(saturating_doubling_high_mul(acc.at(m), out_mul), out_shift) + output_qoffset; - } - *(reinterpret_cast(output_it.ptr() + m * sizeof(T))) = static_cast(utility::clamp(acc.at(m))); - } + return (data.weights_dt == DataType::F32); + }, + REGISTER_FP32_NEON(neon_fp32_deptwiseconv2dnative) }, - input_it, weights_it, biases_it, output_it); -} - -template -void depthwise_loop_pow2_quantized_per_tensor(const ITensor *src, const ITensor *weights, const ITensor *biases, ITensor *dst, const PadStrideInfo &conv_info, - const Size2D &dilation, unsigned int depth_multiplier, std::vector output_multiplier, std::vector output_shift, const Window &window, bool has_biases) // NOLINT -{ - constexpr int half_vec = vector_size / 2; - - using AccType = int32_t; - using AccVectorType = typename wrapper::traits::neon_vector::type; - using AccVectorTagType = typename wrapper::traits::neon_vector::tag_type; - using TagType = typename wrapper::traits::neon_vector::tag_type; - - const auto run_info = DepthwiseConvolutionRunInfo(*src->info(), *weights->info(), conv_info, window, depth_multiplier); - - const auto input_qoffset_vec = wrapper::vreinterpret(wrapper::vmovl(wrapper::vdup_n(static_cast(src->info()->quantization_info().uniform().offset), TagType{}))); - const auto weights_qoffset_vec = wrapper::vreinterpret(wrapper::vmovl(wrapper::vdup_n(static_cast(weights->info()->quantization_info().uniform().offset), TagType{}))); - const auto output_qoffset_vec = wrapper::vdup_n(dst->info()->quantization_info().uniform().offset, arm_compute::wrapper::traits::vector_128_tag{}); - - const auto lower = wrapper::vdup_n(static_cast(std::numeric_limits::lowest()), AccVectorTagType{}); - const auto upper = wrapper::vdup_n(static_cast(std::numeric_limits::max()), AccVectorTagType{}); - const auto zero = wrapper::vdup_n(static_cast(0), AccVectorTagType{}); - - const auto out_mul = output_multiplier.at(0); - const auto out_shift = output_shift.at(0); - - Window execution_window = window; - execution_window.set(Window::DimX, Window::Dimension(0, run_info.input_depth, 1)); - - Window win_input = execution_window; - win_input.set(Window::DimY, dim_manual_loop); - win_input.set(Window::DimZ, dim_manual_loop); - - Window win_weights = window; - win_weights.set_dimension_step(Window::DimX, run_info.x_step); - win_weights.set(Window::DimY, dim_manual_loop); - win_weights.set(Window::DimZ, dim_manual_loop); - win_weights.set(Window::DimW, dim_manual_loop); - - Window win_output = window; - win_output.set_dimension_step(Window::DimX, run_info.x_step); - - Iterator input_it(src, win_input); - Iterator weights_it(weights, win_weights); - Iterator output_it(dst, win_output); - Iterator biases_it{}; - - if(has_biases) - { - biases_it = Iterator(biases, win_weights); - } - - std::vector acc0(depth_multiplier / vector_size); - std::vector acc1(depth_multiplier / vector_size); - - execute_window_loop(execution_window, [&](const Coordinates & id) { - std::fill(begin(acc0), end(acc0), zero); - std::fill(begin(acc1), end(acc1), zero); - - const int32_t input_y = id.y() * run_info.conv_stride_x - run_info.conv_pad_left; - const int32_t input_z = id.z() * run_info.conv_stride_y - run_info.conv_pad_top; - int64_t input_offset = input_y * run_info.input_stride_y + input_z * run_info.input_stride_z; - - auto weights_ptr = weights_it.ptr(); - for(size_t h = 0; h < run_info.weights_height; ++h) + "neon_qp8_qu8_deptwiseconv2dnative", + [](const DepthwiseConv2dNativeDataTypeISASelectorData & data) { - const int32_t current_h = input_z + h * dilation.y(); - if(current_h >= 0 && current_h < static_cast(run_info.input_height)) - { - int offs = input_offset; - for(size_t w = 0; w < run_info.weights_width; ++w) - { - const int32_t current_w = input_y + w * dilation.x(); - if(current_w >= 0 && current_w < static_cast(run_info.input_width)) - { - const auto input_8x8 = wrapper::vdup_n(*(reinterpret_cast(input_it.ptr() + std::min(static_cast(offs), run_info.input_max_offset))), TagType{}); - const auto input_s16x8 = wrapper::vreinterpret(wrapper::vmovl(input_8x8)); - const auto input_no_offs = wrapper::vsub(input_s16x8, input_qoffset_vec); - - for(size_t m = 0, i = 0; m < depth_multiplier; m += vector_size, ++i) - { - const auto weights_8x8 = wrapper::vload(reinterpret_cast(weights_ptr + m * sizeof(T) + w * run_info.weights_stride_y)); - const auto weights_s16x8 = wrapper::vreinterpret(wrapper::vmovl(weights_8x8)); - const auto weights_no_offs = wrapper::vsub(weights_s16x8, weights_qoffset_vec); - - acc0.at(i) = wrapper::vmlal(acc0.at(i), wrapper::vgetlow(input_no_offs), wrapper::vgetlow(weights_no_offs)); - acc1.at(i) = wrapper::vmlal(acc1.at(i), wrapper::vgethigh(input_no_offs), wrapper::vgethigh(weights_no_offs)); - } - } - - offs += dilation.x() * run_info.input_stride_y; - } - } - - weights_ptr += run_info.weights_stride_z; - input_offset += dilation.y() * run_info.input_stride_z; - } - - for(size_t m = 0, i = 0; m < depth_multiplier; m += vector_size, ++i) + return (data.weights_dt == DataType::QSYMM8_PER_CHANNEL && data.source_dt == DataType::QASYMM8); + }, + REGISTER_QASYMM8_NEON(neon_qp8_qu8_deptwiseconv2dnative) + }, + { + "neon_qp8_qs8_deptwiseconv2dnative", + [](const DepthwiseConv2dNativeDataTypeISASelectorData & data) { - if(has_biases) - { - const auto bias_val0 = wrapper::vloadq(reinterpret_cast(biases_it.ptr() + m * sizeof(int32_t))); - const auto bias_val1 = wrapper::vloadq(reinterpret_cast(biases_it.ptr() + (m + half_vec) * sizeof(int32_t))); - - acc0.at(i) = wrapper::vadd(acc0.at(i), bias_val0); - acc1.at(i) = wrapper::vadd(acc1.at(i), bias_val1); - } - - if(out_shift < 0) - { - acc0.at(i) = wrapper::vadd(saturating_doubling_high_mul(acc0.at(i) * (1 << (-out_shift)), out_mul), output_qoffset_vec); - acc1.at(i) = wrapper::vadd(saturating_doubling_high_mul(acc1.at(i) * (1 << (-out_shift)), out_mul), output_qoffset_vec); - } - else - { - acc0.at(i) = wrapper::vadd(rounding_divide_by_exp2(saturating_doubling_high_mul(acc0.at(i), out_mul), out_shift), output_qoffset_vec); - acc1.at(i) = wrapper::vadd(rounding_divide_by_exp2(saturating_doubling_high_mul(acc1.at(i), out_mul), out_shift), output_qoffset_vec); - } - - acc0.at(i) = wrapper::vmin(wrapper::vmax(acc0.at(i), lower), upper); - acc1.at(i) = wrapper::vmin(wrapper::vmax(acc1.at(i), lower), upper); - - const auto out_val = wrapper::vcombine(wrapper::vmovn(acc0.at(i)), - wrapper::vmovn(acc1.at(i))); - - if(std::is_same::value) - { - wrapper::vstore(reinterpret_cast(output_it.ptr() + m * sizeof(uint8_t)), wrapper::vqmovn(vreinterpretq_u16_s16(out_val))); - } - else - { - wrapper::vstore(reinterpret_cast(output_it.ptr() + m * sizeof(int8_t)), wrapper::vqmovn(out_val)); - } - } + return (data.weights_dt == DataType::QSYMM8_PER_CHANNEL && data.source_dt != DataType::QASYMM8); + }, + REGISTER_QASYMM8_SIGNED_NEON(neon_qp8_qs8_deptwiseconv2dnative) }, - input_it, weights_it, biases_it, output_it); -} +}; Status validate_arguments(const ITensorInfo *src, const ITensorInfo *weights, const ITensorInfo *biases, const ITensorInfo *dst, const ConvolutionInfo &info) { @@ -808,67 +148,13 @@ void CpuDepthwiseConv2dNativeKernel::configure(const ITensorInfo *src, const ITe ARM_COMPUTE_ERROR_ON_NULLPTR(src, weights, dst); ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(src, weights, (biases != nullptr) ? biases : nullptr, dst, info)); - _conv_info = info.pad_stride_info; - _depth_multiplier = info.depth_multiplier; - _dilation = info.dilation; - _has_biases = (biases != nullptr); - - if(is_data_type_quantized(src->data_type())) - { - const auto input_scale = src->quantization_info().uniform().scale; - const auto output_scale = dst->quantization_info().uniform().scale; - - auto weights_scale = weights->quantization_info().scale(); - if(!is_data_type_quantized_per_channel(weights->data_type())) - { - for(size_t i = 1; i < weights->dimension(channel_idx); ++i) - { - weights_scale.push_back(weights_scale.front()); - } - } - - for(const auto &s : weights_scale) - { - int32_t out_mult = 0; - int32_t out_shift = 0; - const float multiplier = input_scale * s / output_scale; - arm_compute::quantization::calculate_quantized_multiplier(multiplier, &out_mult, &out_shift); - - _output_multiplier.push_back(out_mult); - _output_shift.push_back(out_shift); - } - } + _has_biases = (biases != nullptr); + _conv_info = info; - switch(weights->data_type()) - { - case DataType::QASYMM8: - _func = &CpuDepthwiseConv2dNativeKernel::run_depthwise; - break; - case DataType::QASYMM8_SIGNED: - _func = &CpuDepthwiseConv2dNativeKernel::run_depthwise; - break; - case DataType::QSYMM8_PER_CHANNEL: - if(src->data_type() == DataType::QASYMM8) - { - _func = &CpuDepthwiseConv2dNativeKernel::run_depthwise; - } - else - { - _func = &CpuDepthwiseConv2dNativeKernel::run_depthwise; - } - break; -#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - case DataType::F16: - _func = &CpuDepthwiseConv2dNativeKernel::run_depthwise; - break; -#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - case DataType::F32: - _func = &CpuDepthwiseConv2dNativeKernel::run_depthwise; - break; - default: - ARM_COMPUTE_ERROR("Data type not supported"); - break; - } + const auto uk = CpuDepthwiseConv2dNativeKernel::get_implementation( + DepthwiseConv2dNativeDataTypeISASelectorData{ weights->data_type(), src->data_type(), CPUInfo::get().get_isa() }); + ARM_COMPUTE_ERROR_ON(uk == nullptr || uk->ukernel == nullptr); + _func = uk->ukernel; const TensorShape output_shape = misc::shape_calculator::compute_depthwise_convolution_shape(*src, *weights, info); auto_init_if_empty(*dst, src->clone()->set_is_resizable(true).reset_padding().set_tensor_shape(output_shape).set_quantization_info(dst->quantization_info())); @@ -883,50 +169,6 @@ Status CpuDepthwiseConv2dNativeKernel::validate(const ITensorInfo *src, const IT return Status{}; } -template > -void CpuDepthwiseConv2dNativeKernel::run_depthwise(const ITensor *src, const ITensor *weights, const ITensor *biases, - ITensor *dst, const Window &window, bool has_biases) -{ - ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); - ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICpuKernel::window(), window); - - if(_depth_multiplier == 1) - { - depthwise_loop_multiplier1_fp(src, weights, biases, dst, _conv_info, _dilation, window, has_biases); - } - else - { - depthwise_loop_generic_fp(src, weights, biases, dst, _conv_info, _dilation, _depth_multiplier, window, has_biases); - } -} - -template > -void CpuDepthwiseConv2dNativeKernel::run_depthwise(const ITensor *src, const ITensor *weights, const ITensor *biases, - ITensor *dst, const Window &window, bool has_biases) -{ - ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this); - ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(ICpuKernel::window(), window); - - if(_depth_multiplier == 1) - { - depthwise_loop_multiplier1_quantized(src, weights, biases, dst, _conv_info, _dilation, _output_multiplier, _output_shift, window, has_biases); - } - else - { - const bool is_pow2 = ((_depth_multiplier & (_depth_multiplier - 1)) == 0); - const bool is_quantized_per_tensor = !(is_data_type_quantized_per_channel(weights->info()->data_type())); - - if(is_pow2 && is_quantized_per_tensor && _depth_multiplier >= 8) - { - depthwise_loop_pow2_quantized_per_tensor(src, weights, biases, dst, _conv_info, _dilation, _depth_multiplier, _output_multiplier, _output_shift, window, has_biases); - } - else - { - depthwise_loop_generic_quantized(src, weights, biases, dst, _conv_info, _dilation, _depth_multiplier, _output_multiplier, _output_shift, window, has_biases); - } - } -} - void CpuDepthwiseConv2dNativeKernel::run_op(ITensorPack &tensors, const Window &window, const ThreadInfo &info) { ARM_COMPUTE_UNUSED(info); @@ -938,13 +180,18 @@ void CpuDepthwiseConv2dNativeKernel::run_op(ITensorPack &tensors, const Window & const auto weights = tensors.get_const_tensor(TensorType::ACL_SRC_1); const auto biases = tensors.get_const_tensor(TensorType::ACL_SRC_2); auto dst = tensors.get_tensor(TensorType::ACL_DST); - (this->*_func)(src, weights, biases, dst, window, _has_biases); + _func(src, weights, biases, dst, window, _has_biases, _conv_info); } const char *CpuDepthwiseConv2dNativeKernel::name() const { return "CpuDepthwiseConv2dNativeKernel"; } + +const std::vector &CpuDepthwiseConv2dNativeKernel::get_available_kernels() +{ + return available_kernels; +} } // namespace kernels } // namespace cpu } // namespace arm_compute -- cgit v1.2.1