From 4c30de056afe8680b42723b26a2241811715b989 Mon Sep 17 00:00:00 2001 From: Michael Tyler Date: Fri, 7 Jul 2023 12:01:32 +0100 Subject: Enable premultiplication for depthwise convolution with fp16 and quantized types Resolves: COMPMID-6337 Change-Id: I81542e51c9c0329f202ac8452f173b138e51a0f6 Signed-off-by: Michael Tyler Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/9883 Reviewed-by: Gunes Bayir Comments-Addressed: Arm Jenkins Tested-by: Arm Jenkins Benchmark: Arm Jenkins --- .../kernels/arm_conv/depthwise/depthwise_fp16.cpp | 54 ++++++++++++++++++---- .../kernels/arm_conv/depthwise/depthwise_fp32.cpp | 4 +- .../kernels/arm_conv/depthwise/depthwise_s8q.cpp | 29 +++++------- .../kernels/arm_conv/depthwise/depthwise_u8q.cpp | 35 +++++++------- .../arm_conv/depthwise/depthwise_u8s8u8q.cpp | 22 +++++---- .../depthwise/interleaves/a64_u8q_3x3_dot.cpp | 2 +- .../depthwise/interleaves/sve_s8q_3x3_dot.cpp | 2 +- .../depthwise/interleaves/sve_u8q_3x3_dot.cpp | 2 +- ...4_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp | 2 +- ...64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp | 2 +- ...ve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp | 2 +- ...e_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp | 2 +- ...ve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp | 2 +- .../kernels/arm_conv/depthwise/premultiply.cpp | 16 ++++++- 14 files changed, 113 insertions(+), 63 deletions(-) (limited to 'src/core/NEON/kernels') diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp index 3b76e52206..8fef6f8ae0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp @@ -66,9 +66,47 @@ namespace depthwise { namespace { +#if defined(__aarch64__) +#if defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) + bool prefer_premultiply(const DepthwiseArgs &args) { + if ((args.stride_rows != args.stride_cols) || (args.kernel_rows != args.kernel_cols)) + { + return false; + } + + unsigned int threshold; + + if (args.stride_rows == 1 && args.kernel_rows == 3) + { + threshold = 30; + } + else if (args.stride_rows == 1 && args.kernel_rows == 5) + { + threshold = 31; + } + else if (args.stride_rows == 2 && args.kernel_rows == 3) + { + threshold = 11; + } + else if (args.stride_rows == 2 && args.kernel_rows == 5) + { + threshold = 19; + } else + { + return false; + } + + return args.channel_multiplier <= threshold; + } + template unsigned int cycle_estimate(const DepthwiseArgs &args, const Nothing &) { + if (args.channel_multiplier > 1 && !prefer_premultiply(args)) + { + return std::numeric_limits::max(); + } + // First-pass: compute the number of output pixels which will be computed. return arm_gemm::roundup(args.output_rows, Strategy::output_rows) * arm_gemm::roundup(args.output_cols, Strategy::output_cols) * @@ -90,13 +128,18 @@ namespace ); } -#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) + unsigned int multiplier_cycle_estimate(const DepthwiseArgs &args, const Nothing &) + { + return prefer_premultiply(args)? std::numeric_limits::max() : 0; + } + unsigned int not_preferred(const DepthwiseArgs &, const Nothing &) __attribute__ ((unused)); unsigned int not_preferred(const DepthwiseArgs &, const Nothing &) { return std::numeric_limits::max(); } -#endif // defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) } static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = { @@ -108,7 +151,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -120,7 +162,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -132,7 +173,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -144,7 +184,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -156,7 +195,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -291,7 +329,7 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", constraint(cpu_has_fp16, has_channel_multiplier), - nullptr, + multiplier_cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { auto kern = new a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info); auto strat = new GenericDepthfirstMultiplierStrategy<__fp16>(kern, args); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp index b0f606332b..760328f3ba 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp @@ -116,7 +116,7 @@ namespace { if (args.channel_multiplier > 1 && !prefer_premultiply(args)) { - return UINT32_MAX; + return std::numeric_limits::max(); } // First-pass: compute the number of output pixels which will be computed. @@ -154,7 +154,7 @@ namespace unsigned int multiplier_cycle_estimate(const DepthwiseArgs &args, const Nothing &) { - return prefer_premultiply(args)? UINT32_MAX : 0; + return prefer_premultiply(args)? std::numeric_limits::max() : 0; } unsigned int not_preferred(const DepthwiseArgs &, const Nothing &) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_s8q.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_s8q.cpp index 2d03183c59..6ecdc36bf0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_s8q.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_s8q.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -75,6 +75,11 @@ bool qp_weights_are_symmetric(const DepthwiseArgs &, const void *_qp) const auto qp = static_cast(_qp); return qp->b_offset == 0; } + +uint64_t not_preferred(const DepthwiseArgs &, const Requantize32 &) +{ + return std::numeric_limits::max(); +} #endif // defined(__aarch64__) } @@ -139,7 +144,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, qp_weights_are_symmetric, cpu_has_sve2), @@ -153,7 +157,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -166,7 +169,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -179,7 +181,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -192,7 +193,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -208,7 +208,7 @@ static const DepthwiseImplementation depth qp_has_no_left_shift, has_channel_multiplier, cpu_has_sve2), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -221,7 +221,7 @@ static const DepthwiseImplementation depth qp_has_no_left_shift, has_channel_multiplier, cpu_has_sve2), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -232,7 +232,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_weights_are_symmetric, qp_has_no_left_shift, cpu_has_dot_product), @@ -246,7 +245,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_dot_product), nullptr, @@ -259,7 +257,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -271,7 +268,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -283,7 +279,6 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -294,7 +289,7 @@ static const DepthwiseImplementation depth { DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_generic_output3x3_mla_depthfirst", - constraint(has_no_channel_multiplier), + nullptr, nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto kernel = new a64_s8q_nhwc_generic_output9_mla_depthfirst(args.cpu_info); @@ -309,7 +304,7 @@ static const DepthwiseImplementation depth qp_has_no_left_shift, has_channel_multiplier, cpu_has_dot_product), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -322,7 +317,7 @@ static const DepthwiseImplementation depth qp_has_no_left_shift, has_channel_multiplier, cpu_has_dot_product), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -332,7 +327,7 @@ static const DepthwiseImplementation depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", constraint(has_channel_multiplier), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto kern = new a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info); auto strat = new GenericDepthfirstMultiplierStrategy(kern, args); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp index 9dbd89fb52..236930ee26 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -71,6 +71,16 @@ using arm_gemm::Requantize32; namespace arm_conv { namespace depthwise { +namespace +{ +#if defined(__aarch64__) +uint64_t not_preferred(const DepthwiseArgs &, const Requantize32 &) +{ + return std::numeric_limits::max(); +} +#endif // defined(__aarch64__) +} + static const DepthwiseImplementation depthwise_u8q_methods[] = { #if defined(__aarch64__) #if defined(ARM_COMPUTE_ENABLE_SVE) @@ -132,7 +142,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -145,7 +154,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -158,7 +166,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -171,7 +178,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -187,7 +193,7 @@ static const DepthwiseImplementation de qp_has_no_left_shift, has_channel_multiplier, cpu_has_sve2), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new sve_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -200,7 +206,7 @@ static const DepthwiseImplementation de qp_has_no_left_shift, has_channel_multiplier, cpu_has_sve2), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -212,7 +218,6 @@ static const DepthwiseImplementation de "a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint(is_supported, cpu_has_dot_product, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -225,7 +230,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_zero_a_offset, qp_has_no_left_shift), nullptr, @@ -238,7 +242,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_zero_a_offset, qp_has_no_left_shift), nullptr, @@ -251,7 +254,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_zero_a_offset, qp_has_no_left_shift), nullptr, @@ -265,7 +267,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -277,7 +278,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -289,7 +289,6 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -300,7 +299,7 @@ static const DepthwiseImplementation de { DepthwiseMethod::DEPTHFIRST, "a64_u8q_nhwc_generic_output3x3_mla_depthfirst", - constraint(has_no_channel_multiplier), + nullptr, nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto kernel = new a64_u8q_nhwc_generic_output9_mla_depthfirst(args.cpu_info); @@ -315,7 +314,7 @@ static const DepthwiseImplementation de cpu_has_dot_product, has_channel_multiplier, qp_has_no_left_shift), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -328,7 +327,7 @@ static const DepthwiseImplementation de cpu_has_dot_product, has_channel_multiplier, qp_has_no_left_shift), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto strat = new a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier(strat, args, qp); @@ -338,7 +337,7 @@ static const DepthwiseImplementation de DepthwiseMethod::DEPTHFIRST, "a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", constraint(has_channel_multiplier), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto kern = new a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info); auto strat = new GenericDepthfirstMultiplierStrategy(kern, args); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp index 0665c67fbb..a888958b76 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -58,6 +58,16 @@ using arm_gemm::Requantize32; namespace arm_conv { namespace depthwise { +namespace +{ +#if defined(__aarch64__) +uint64_t not_preferred(const DepthwiseArgs &, const Requantize32 &) +{ + return std::numeric_limits::max(); +} +#endif // defined(__aarch64__) +} + static const DepthwiseImplementation depthwise_u8q_methods[] = { #if defined(__aarch64__) #if defined(ARM_COMPUTE_ENABLE_SVE) @@ -119,7 +129,6 @@ static const DepthwiseImplementation dep DepthwiseMethod::DEPTHFIRST, "sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -132,7 +141,6 @@ static const DepthwiseImplementation dep DepthwiseMethod::DEPTHFIRST, "sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -145,7 +153,6 @@ static const DepthwiseImplementation dep DepthwiseMethod::DEPTHFIRST, "sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -159,7 +166,6 @@ static const DepthwiseImplementation dep DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -171,7 +177,6 @@ static const DepthwiseImplementation dep DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -183,7 +188,6 @@ static const DepthwiseImplementation dep DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { @@ -194,7 +198,7 @@ static const DepthwiseImplementation dep { DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_nhwc_generic_output3x3_mla_depthfirst", - constraint(has_no_channel_multiplier), + nullptr, nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto kernel = new a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst(args.cpu_info); @@ -206,7 +210,7 @@ static const DepthwiseImplementation dep DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", constraint(has_channel_multiplier), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon * { auto kern = new a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info); auto strat = new GenericDepthfirstMultiplierStrategy(kern, args); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp index 314f09a0c5..19264c9fce 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp @@ -42,7 +42,7 @@ size_t interleave_a64_u8q_3x3_dot::get_packed_size(const DepthwiseArgs &args) { // We store 7 vectors for every of channels. const unsigned int n = arm_gemm::roundup( - arm_gemm::iceildiv((long unsigned int) args.input_channels, + arm_gemm::iceildiv((long unsigned int) args.input_channels * args.channel_multiplier, get_vector_length(arm_gemm::VLType::None)), 4lu ); return n * 7 * get_vector_length(arm_gemm::VLType::None); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp index 3a4999296a..5d7b54f235 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp @@ -42,7 +42,7 @@ size_t interleave_sve_s8q_3x3_dot::get_packed_size(const DepthwiseArgs &args) { // We store 7 vectors for every of channels. const unsigned int n = arm_gemm::roundup( - arm_gemm::iceildiv((long unsigned int) args.input_channels, + arm_gemm::iceildiv((long unsigned int) args.input_channels * args.channel_multiplier, get_vector_length(arm_gemm::VLType::SVE)), 4lu ); return n * 7 * get_vector_length(arm_gemm::VLType::SVE); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp index 7c5d3c4904..c3da81448b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp @@ -42,7 +42,7 @@ size_t interleave_sve_u8q_3x3_dot::get_packed_size(const DepthwiseArgs &args) { // We store 7 vectors for every of channels. const unsigned int n = arm_gemm::roundup( - arm_gemm::iceildiv((long unsigned int) args.input_channels, + arm_gemm::iceildiv((long unsigned int) args.input_channels * args.channel_multiplier, get_vector_length(arm_gemm::VLType::SVE)), 4lu ); return n * 7 * get_vector_length(arm_gemm::VLType::SVE); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index fc83aaf5d2..bd2941b3d6 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -64,7 +64,7 @@ class a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirst ) const override { interleave_a64_s8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast(biases), reinterpret_cast(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index bea97a54b6..ed24f8fa3c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -64,7 +64,7 @@ class a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirstS ) const override { interleave_a64_u8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast(biases), reinterpret_cast(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index 32ea009e8a..04cf0d4036 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -64,7 +64,7 @@ class sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirstS ) const override { interleave_sve_s8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast(biases), reinterpret_cast(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index 1730574933..6799b10ed9 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -64,7 +64,7 @@ class sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirst ) const override { interleave_sve_s8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast(biases), reinterpret_cast(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index 9432cd7550..6b006e8d51 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -64,7 +64,7 @@ class sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirstS ) const override { interleave_sve_u8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast(biases), reinterpret_cast(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/premultiply.cpp b/src/core/NEON/kernels/arm_conv/depthwise/premultiply.cpp index ad4c821cfb..8a49c775d3 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/premultiply.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/premultiply.cpp @@ -45,7 +45,9 @@ void do_premultiply_float_6(const float *in_ptr, { const float *ip = ip2; float *op = op2; - for(unsigned int c = 0; c < input_channels; c += BLOCK_SIZE) + + unsigned int num_blocks = input_channels / BLOCK_SIZE; + for(unsigned int c = 0; c < num_blocks; c++) { float vals[BLOCK_SIZE]; for(unsigned int v = 0; v < BLOCK_SIZE; v++) @@ -63,6 +65,18 @@ void do_premultiply_float_6(const float *in_ptr, op += CHANNEL_MULTIPLIER; } } + + unsigned int rem = input_channels - num_blocks * BLOCK_SIZE; + for(unsigned int c = 0; c < rem; c++) + { + float val = ip[c]; + for(unsigned int r = 0; r < CHANNEL_MULTIPLIER; r++) + { + op[r] = val; + } + op += CHANNEL_MULTIPLIER; + } + ip2 += ld_col; op2 += out_ld_col; } -- cgit v1.2.1