diff options
author | Michael Tyler <michael.tyler@arm.com> | 2023-07-07 12:01:32 +0100 |
---|---|---|
committer | michael.tyler <michael.tyler@arm.com> | 2023-07-13 12:21:29 +0000 |
commit | 4c30de056afe8680b42723b26a2241811715b989 (patch) | |
tree | 4f522a816a5ea1b58b51226eb685c786096f30e3 /src | |
parent | c8e1617807ef1985a39d8f8f5f69c113b758494d (diff) | |
download | ComputeLibrary-4c30de056afe8680b42723b26a2241811715b989.tar.gz |
Enable premultiplication for depthwise convolution
with fp16 and quantized types
Resolves: COMPMID-6337
Change-Id: I81542e51c9c0329f202ac8452f173b138e51a0f6
Signed-off-by: Michael Tyler <michael.tyler@arm.com>
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/9883
Reviewed-by: Gunes Bayir <gunes.bayir@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Benchmark: Arm Jenkins <bsgcomp@arm.com>
Diffstat (limited to 'src')
14 files changed, 113 insertions, 63 deletions
diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp index 3b76e52206..8fef6f8ae0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp @@ -66,9 +66,47 @@ namespace depthwise { namespace { +#if defined(__aarch64__) +#if defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) + bool prefer_premultiply(const DepthwiseArgs &args) { + if ((args.stride_rows != args.stride_cols) || (args.kernel_rows != args.kernel_cols)) + { + return false; + } + + unsigned int threshold; + + if (args.stride_rows == 1 && args.kernel_rows == 3) + { + threshold = 30; + } + else if (args.stride_rows == 1 && args.kernel_rows == 5) + { + threshold = 31; + } + else if (args.stride_rows == 2 && args.kernel_rows == 3) + { + threshold = 11; + } + else if (args.stride_rows == 2 && args.kernel_rows == 5) + { + threshold = 19; + } else + { + return false; + } + + return args.channel_multiplier <= threshold; + } + template <class Strategy> unsigned int cycle_estimate(const DepthwiseArgs &args, const Nothing &) { + if (args.channel_multiplier > 1 && !prefer_premultiply(args)) + { + return std::numeric_limits<unsigned int>::max(); + } + // First-pass: compute the number of output pixels which will be computed. return arm_gemm::roundup(args.output_rows, Strategy::output_rows) * arm_gemm::roundup(args.output_cols, Strategy::output_cols) * @@ -90,13 +128,18 @@ namespace ); } -#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) + unsigned int multiplier_cycle_estimate(const DepthwiseArgs &args, const Nothing &) + { + return prefer_premultiply(args)? std::numeric_limits<unsigned int>::max() : 0; + } + unsigned int not_preferred(const DepthwiseArgs &, const Nothing &) __attribute__ ((unused)); unsigned int not_preferred(const DepthwiseArgs &, const Nothing &) { return std::numeric_limits<unsigned int>::max(); } -#endif // defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(ENABLE_FP16_KERNELS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) } static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = { @@ -108,7 +151,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst", constraint(is_supported<sme2_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst>, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate<sme2_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst>, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -120,7 +162,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst", constraint(is_supported<sme2_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst>, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate<sme2_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst>, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -132,7 +173,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported<sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate<sme2_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst>, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -144,7 +184,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported<sme2_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst>, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate<sme2_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst>, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -156,7 +195,6 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "sme2_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported<sme2_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, cpu_has_sme2), cycle_estimate<sme2_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst>, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { @@ -291,7 +329,7 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", constraint(cpu_has_fp16, has_channel_multiplier), - nullptr, + multiplier_cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { auto kern = new a64_fp16_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info); auto strat = new GenericDepthfirstMultiplierStrategy<__fp16>(kern, args); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp index b0f606332b..760328f3ba 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp @@ -116,7 +116,7 @@ namespace { if (args.channel_multiplier > 1 && !prefer_premultiply(args)) { - return UINT32_MAX; + return std::numeric_limits<unsigned int>::max(); } // First-pass: compute the number of output pixels which will be computed. @@ -154,7 +154,7 @@ namespace unsigned int multiplier_cycle_estimate(const DepthwiseArgs &args, const Nothing &) { - return prefer_premultiply(args)? UINT32_MAX : 0; + return prefer_premultiply(args)? std::numeric_limits<unsigned int>::max() : 0; } unsigned int not_preferred(const DepthwiseArgs &, const Nothing &) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_s8q.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_s8q.cpp index 2d03183c59..6ecdc36bf0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_s8q.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_s8q.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -75,6 +75,11 @@ bool qp_weights_are_symmetric(const DepthwiseArgs &, const void *_qp) const auto qp = static_cast<const arm_gemm::Requantize32 *>(_qp); return qp->b_offset == 0; } + +uint64_t not_preferred(const DepthwiseArgs &, const Requantize32 &) +{ + return std::numeric_limits<uint64_t>::max(); +} #endif // defined(__aarch64__) } @@ -139,7 +144,6 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth DepthwiseMethod::DEPTHFIRST, "sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint<Requantize32>(is_supported<sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift, qp_weights_are_symmetric, cpu_has_sve2), @@ -153,7 +157,6 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth DepthwiseMethod::DEPTHFIRST, "sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint<Requantize32>(is_supported<sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -166,7 +169,6 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth DepthwiseMethod::DEPTHFIRST, "sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<sve_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -179,7 +181,6 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth DepthwiseMethod::DEPTHFIRST, "sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<sve_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -192,7 +193,6 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth DepthwiseMethod::DEPTHFIRST, "sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<sve_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -208,7 +208,7 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth qp_has_no_left_shift, has_channel_multiplier, cpu_has_sve2), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * { auto strat = new sve_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier<int8_t, int8_t, int8_t, int32_t, false>(strat, args, qp); @@ -221,7 +221,7 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth qp_has_no_left_shift, has_channel_multiplier, cpu_has_sve2), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * { auto strat = new sve_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier<int8_t, int8_t, int8_t, int32_t, false>(strat, args, qp); @@ -232,7 +232,6 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth DepthwiseMethod::DEPTHFIRST, "a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint<Requantize32>(is_supported<a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst>, - has_no_channel_multiplier, qp_weights_are_symmetric, qp_has_no_left_shift, cpu_has_dot_product), @@ -246,7 +245,6 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint<Requantize32>(is_supported<a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_dot_product), nullptr, @@ -259,7 +257,6 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<a64_s8q_nhwc_3x3_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * { @@ -271,7 +268,6 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<a64_s8q_nhwc_3x3_s2_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * { @@ -283,7 +279,6 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<a64_s8q_nhwc_5x5_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * { @@ -294,7 +289,7 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth { DepthwiseMethod::DEPTHFIRST, "a64_s8q_nhwc_generic_output3x3_mla_depthfirst", - constraint<Requantize32>(has_no_channel_multiplier), + nullptr, nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * { auto kernel = new a64_s8q_nhwc_generic_output9_mla_depthfirst(args.cpu_info); @@ -309,7 +304,7 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth qp_has_no_left_shift, has_channel_multiplier, cpu_has_dot_product), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * { auto strat = new a64_s8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier<int8_t, int8_t, int8_t, int32_t, false>(strat, args, qp); @@ -322,7 +317,7 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth qp_has_no_left_shift, has_channel_multiplier, cpu_has_dot_product), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * { auto strat = new a64_s8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier<int8_t, int8_t, int8_t, int32_t, false>(strat, args, qp); @@ -332,7 +327,7 @@ static const DepthwiseImplementation<int8_t, int8_t, int8_t, Requantize32> depth DepthwiseMethod::DEPTHFIRST, "a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", constraint<Requantize32>(has_channel_multiplier), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<int8_t, int8_t, int8_t> * { auto kern = new a64_s8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info); auto strat = new GenericDepthfirstMultiplierStrategy<int8_t>(kern, args); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp index 9dbd89fb52..236930ee26 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8q.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -71,6 +71,16 @@ using arm_gemm::Requantize32; namespace arm_conv { namespace depthwise { +namespace +{ +#if defined(__aarch64__) +uint64_t not_preferred(const DepthwiseArgs &, const Requantize32 &) +{ + return std::numeric_limits<uint64_t>::max(); +} +#endif // defined(__aarch64__) +} + static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> depthwise_u8q_methods[] = { #if defined(__aarch64__) #if defined(ARM_COMPUTE_ENABLE_SVE) @@ -132,7 +142,6 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de DepthwiseMethod::DEPTHFIRST, "sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint<Requantize32>(is_supported<sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -145,7 +154,6 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de DepthwiseMethod::DEPTHFIRST, "sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -158,7 +166,6 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de DepthwiseMethod::DEPTHFIRST, "sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -171,7 +178,6 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de DepthwiseMethod::DEPTHFIRST, "sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -187,7 +193,7 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de qp_has_no_left_shift, has_channel_multiplier, cpu_has_sve2), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, uint8_t, uint8_t> * { auto strat = new sve_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier<uint8_t, uint8_t, uint8_t, int32_t, false>(strat, args, qp); @@ -200,7 +206,7 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de qp_has_no_left_shift, has_channel_multiplier, cpu_has_sve2), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, uint8_t, uint8_t> * { auto strat = new sve_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier<uint8_t, uint8_t, uint8_t, int32_t, false>(strat, args, qp); @@ -212,7 +218,6 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de "a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst", constraint<Requantize32>(is_supported<a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst>, cpu_has_dot_product, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, uint8_t, uint8_t> * { @@ -225,7 +230,6 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de DepthwiseMethod::DEPTHFIRST, "a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<a64_u8qa_nhwc_3x3_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_zero_a_offset, qp_has_no_left_shift), nullptr, @@ -238,7 +242,6 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de DepthwiseMethod::DEPTHFIRST, "a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<a64_u8qa_nhwc_3x3_s2_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_zero_a_offset, qp_has_no_left_shift), nullptr, @@ -251,7 +254,6 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de DepthwiseMethod::DEPTHFIRST, "a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<a64_u8qa_nhwc_5x5_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_zero_a_offset, qp_has_no_left_shift), nullptr, @@ -265,7 +267,6 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de DepthwiseMethod::DEPTHFIRST, "a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, uint8_t, uint8_t> * { @@ -277,7 +278,6 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de DepthwiseMethod::DEPTHFIRST, "a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, uint8_t, uint8_t> * { @@ -289,7 +289,6 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de DepthwiseMethod::DEPTHFIRST, "a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, uint8_t, uint8_t> * { @@ -300,7 +299,7 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de { DepthwiseMethod::DEPTHFIRST, "a64_u8q_nhwc_generic_output3x3_mla_depthfirst", - constraint<Requantize32>(has_no_channel_multiplier), + nullptr, nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, uint8_t, uint8_t> * { auto kernel = new a64_u8q_nhwc_generic_output9_mla_depthfirst(args.cpu_info); @@ -315,7 +314,7 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de cpu_has_dot_product, has_channel_multiplier, qp_has_no_left_shift), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, uint8_t, uint8_t> * { auto strat = new a64_u8q_packed_to_nhwc_3x3_s2_with_multiplier_output2x4_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier<uint8_t, uint8_t, uint8_t, int32_t, false>(strat, args, qp); @@ -328,7 +327,7 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de cpu_has_dot_product, has_channel_multiplier, qp_has_no_left_shift), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, uint8_t, uint8_t> * { auto strat = new a64_u8q_packed_to_nhwc_5x5_s1_with_multiplier_output4x2_dot_depthfirst(args.cpu_info); return new DepthwiseDepthfirstMultiplier<uint8_t, uint8_t, uint8_t, int32_t, false>(strat, args, qp); @@ -338,7 +337,7 @@ static const DepthwiseImplementation<uint8_t, uint8_t, uint8_t, Requantize32> de DepthwiseMethod::DEPTHFIRST, "a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", constraint<Requantize32>(has_channel_multiplier), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, uint8_t, uint8_t> * { auto kern = new a64_u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info); auto strat = new GenericDepthfirstMultiplierStrategy<uint8_t>(kern, args); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp index 0665c67fbb..a888958b76 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_u8s8u8q.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021-2022 Arm Limited. + * Copyright (c) 2021-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -58,6 +58,16 @@ using arm_gemm::Requantize32; namespace arm_conv { namespace depthwise { +namespace +{ +#if defined(__aarch64__) +uint64_t not_preferred(const DepthwiseArgs &, const Requantize32 &) +{ + return std::numeric_limits<uint64_t>::max(); +} +#endif // defined(__aarch64__) +} + static const DepthwiseImplementation<uint8_t, int8_t, uint8_t, Requantize32> depthwise_u8q_methods[] = { #if defined(__aarch64__) #if defined(ARM_COMPUTE_ENABLE_SVE) @@ -119,7 +129,6 @@ static const DepthwiseImplementation<uint8_t, int8_t, uint8_t, Requantize32> dep DepthwiseMethod::DEPTHFIRST, "sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<sve_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -132,7 +141,6 @@ static const DepthwiseImplementation<uint8_t, int8_t, uint8_t, Requantize32> dep DepthwiseMethod::DEPTHFIRST, "sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<sve_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -145,7 +153,6 @@ static const DepthwiseImplementation<uint8_t, int8_t, uint8_t, Requantize32> dep DepthwiseMethod::DEPTHFIRST, "sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<sve_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift, cpu_has_sve2), nullptr, @@ -159,7 +166,6 @@ static const DepthwiseImplementation<uint8_t, int8_t, uint8_t, Requantize32> dep DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<a64_u8s8u8q_nhwc_3x3_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * { @@ -171,7 +177,6 @@ static const DepthwiseImplementation<uint8_t, int8_t, uint8_t, Requantize32> dep DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<a64_u8s8u8q_nhwc_3x3_s2_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * { @@ -183,7 +188,6 @@ static const DepthwiseImplementation<uint8_t, int8_t, uint8_t, Requantize32> dep DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint<Requantize32>(is_supported<a64_u8s8u8q_nhwc_5x5_s1_output2x2_mla_depthfirst>, - has_no_channel_multiplier, qp_has_no_left_shift), nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * { @@ -194,7 +198,7 @@ static const DepthwiseImplementation<uint8_t, int8_t, uint8_t, Requantize32> dep { DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_nhwc_generic_output3x3_mla_depthfirst", - constraint<Requantize32>(has_no_channel_multiplier), + nullptr, nullptr, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * { auto kernel = new a64_u8s8u8q_nhwc_generic_output9_mla_depthfirst(args.cpu_info); @@ -206,7 +210,7 @@ static const DepthwiseImplementation<uint8_t, int8_t, uint8_t, Requantize32> dep DepthwiseMethod::DEPTHFIRST, "a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", constraint<Requantize32>(has_channel_multiplier), - nullptr, + not_preferred, [] (const DepthwiseArgs &args, const Requantize32 &qp) -> DepthwiseCommon<uint8_t, int8_t, uint8_t> * { auto kern = new a64_u8s8u8q_packed_to_nhwc_generic_with_multiplier_output2x8_mla_depthfirst(args.cpu_info); auto strat = new GenericDepthfirstMultiplierStrategy<uint8_t, int8_t>(kern, args); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp index 314f09a0c5..19264c9fce 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp @@ -42,7 +42,7 @@ size_t interleave_a64_u8q_3x3_dot::get_packed_size(const DepthwiseArgs &args) { // We store 7 vectors for every <vector_of_ints> of channels. const unsigned int n = arm_gemm::roundup( - arm_gemm::iceildiv((long unsigned int) args.input_channels, + arm_gemm::iceildiv((long unsigned int) args.input_channels * args.channel_multiplier, get_vector_length<int32_t>(arm_gemm::VLType::None)), 4lu ); return n * 7 * get_vector_length<uint8_t>(arm_gemm::VLType::None); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp index 3a4999296a..5d7b54f235 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp @@ -42,7 +42,7 @@ size_t interleave_sve_s8q_3x3_dot::get_packed_size(const DepthwiseArgs &args) { // We store 7 vectors for every <vector_of_ints> of channels. const unsigned int n = arm_gemm::roundup( - arm_gemm::iceildiv((long unsigned int) args.input_channels, + arm_gemm::iceildiv((long unsigned int) args.input_channels * args.channel_multiplier, get_vector_length<int32_t>(arm_gemm::VLType::SVE)), 4lu ); return n * 7 * get_vector_length<int8_t>(arm_gemm::VLType::SVE); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp index 7c5d3c4904..c3da81448b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp @@ -42,7 +42,7 @@ size_t interleave_sve_u8q_3x3_dot::get_packed_size(const DepthwiseArgs &args) { // We store 7 vectors for every <vector_of_ints> of channels. const unsigned int n = arm_gemm::roundup( - arm_gemm::iceildiv((long unsigned int) args.input_channels, + arm_gemm::iceildiv((long unsigned int) args.input_channels * args.channel_multiplier, get_vector_length<int32_t>(arm_gemm::VLType::SVE)), 4lu ); return n * 7 * get_vector_length<uint8_t>(arm_gemm::VLType::SVE); diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index fc83aaf5d2..bd2941b3d6 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -64,7 +64,7 @@ class a64_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirst ) const override { interleave_a64_s8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast<const int32_t *>(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast<const int32_t *>(biases), reinterpret_cast<const int8_t *>(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index bea97a54b6..ed24f8fa3c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -64,7 +64,7 @@ class a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirstS ) const override { interleave_a64_u8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast<const int32_t *>(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast<const int32_t *>(biases), reinterpret_cast<const uint8_t *>(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index 32ea009e8a..04cf0d4036 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -64,7 +64,7 @@ class sve_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirstS ) const override { interleave_sve_s8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast<const int32_t *>(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast<const int32_t *>(biases), reinterpret_cast<const int8_t *>(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index 1730574933..6799b10ed9 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -64,7 +64,7 @@ class sve_s8qs_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirst ) const override { interleave_sve_s8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast<const int32_t *>(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast<const int32_t *>(biases), reinterpret_cast<const int8_t *>(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp index 9432cd7550..6b006e8d51 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst.hpp @@ -64,7 +64,7 @@ class sve_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst : public DepthwiseDepthfirstS ) const override { interleave_sve_u8q_3x3_dot::pack_parameters( - args.input_channels, buffer, reinterpret_cast<const int32_t *>(biases), + args.input_channels * args.channel_multiplier, buffer, reinterpret_cast<const int32_t *>(biases), reinterpret_cast<const uint8_t *>(weights), qp, ld_weight_col, ld_weight_row ); } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/premultiply.cpp b/src/core/NEON/kernels/arm_conv/depthwise/premultiply.cpp index ad4c821cfb..8a49c775d3 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/premultiply.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/premultiply.cpp @@ -45,7 +45,9 @@ void do_premultiply_float_6(const float *in_ptr, { const float *ip = ip2; float *op = op2; - for(unsigned int c = 0; c < input_channels; c += BLOCK_SIZE) + + unsigned int num_blocks = input_channels / BLOCK_SIZE; + for(unsigned int c = 0; c < num_blocks; c++) { float vals[BLOCK_SIZE]; for(unsigned int v = 0; v < BLOCK_SIZE; v++) @@ -63,6 +65,18 @@ void do_premultiply_float_6(const float *in_ptr, op += CHANNEL_MULTIPLIER; } } + + unsigned int rem = input_channels - num_blocks * BLOCK_SIZE; + for(unsigned int c = 0; c < rem; c++) + { + float val = ip[c]; + for(unsigned int r = 0; r < CHANNEL_MULTIPLIER; r++) + { + op[r] = val; + } + op += CHANNEL_MULTIPLIER; + } + ip2 += ld_col; op2 += out_ld_col; } |