diff options
Diffstat (limited to 'src/core/utils/quantization/AsymmHelpers.cpp')
| -rw-r--r-- | src/core/utils/quantization/AsymmHelpers.cpp | 140 |
1 files changed, 90 insertions, 50 deletions
diff --git a/src/core/utils/quantization/AsymmHelpers.cpp b/src/core/utils/quantization/AsymmHelpers.cpp index 49e39f663f..f8b74a985d 100644 --- a/src/core/utils/quantization/AsymmHelpers.cpp +++ b/src/core/utils/quantization/AsymmHelpers.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2020 Arm Limited. + * Copyright (c) 2017-2024 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -22,7 +22,11 @@ * SOFTWARE. */ #include "arm_compute/core/utils/quantization/AsymmHelpers.h" + #include "arm_compute/core/Helpers.h" +#include "arm_compute/function_info/ActivationLayerInfo.h" + +#include "src/core/utils/quantization/AsymmHelpers.h" #include "support/ToolchainSupport.h" #include <cmath> @@ -38,7 +42,7 @@ constexpr float epsilon = 0.00001f; Status calculate_quantized_multiplier(float multiplier, int32_t *quant_multiplier, int32_t *shift, bool ignore_epsilon) { - if(multiplier >= 1.f) + if (multiplier >= 1.f) { Status status = calculate_quantized_multiplier_greater_than_one(multiplier, quant_multiplier, shift); *shift *= -1; @@ -61,25 +65,19 @@ Status calculate_quantized_multiplier_less_than_one(float multiplier, ARM_COMPUTE_RETURN_ERROR_ON(right_shift == nullptr); ARM_COMPUTE_RETURN_ERROR_ON(multiplier < -internal_epsilon); ARM_COMPUTE_RETURN_ERROR_ON(multiplier > 1.0f + internal_epsilon); - if(std::fabs(0.0f - multiplier) < internal_epsilon) - { - *quant_multiplier = 0; - *right_shift = 0; - return Status{}; - } int shift_exp = 0; const double q = std::frexp(multiplier, &shift_exp); *right_shift = -1 * shift_exp; auto q_fixed = static_cast<int64_t>(support::cpp11::round(q * fixed_point_one_Q0)); ARM_COMPUTE_RETURN_ERROR_ON(q_fixed > fixed_point_one_Q0); - if(q_fixed == fixed_point_one_Q0) + if (q_fixed == fixed_point_one_Q0) { q_fixed /= 2; --*right_shift; } - if(ignore_epsilon && *right_shift > 31) + if (ignore_epsilon && *right_shift > 31) { *right_shift = 0; q_fixed = 0; @@ -92,9 +90,8 @@ Status calculate_quantized_multiplier_less_than_one(float multiplier, return Status{}; } -Status calculate_quantized_multiplier_greater_than_one(float multiplier, - int32_t *quantized_multiplier, - int32_t *left_shift) +Status +calculate_quantized_multiplier_greater_than_one(float multiplier, int32_t *quantized_multiplier, int32_t *left_shift) { ARM_COMPUTE_RETURN_ERROR_ON(quantized_multiplier == nullptr); ARM_COMPUTE_RETURN_ERROR_ON(left_shift == nullptr); @@ -105,7 +102,7 @@ Status calculate_quantized_multiplier_greater_than_one(float multiplier, *left_shift = shift_exp; auto q_fixed = static_cast<int64_t>(support::cpp11::round(q * fixed_point_one_Q0)); ARM_COMPUTE_RETURN_ERROR_ON(q_fixed > fixed_point_one_Q0); - if(q_fixed == fixed_point_one_Q0) + if (q_fixed == fixed_point_one_Q0) { q_fixed /= 2; ++*left_shift; @@ -117,27 +114,27 @@ Status calculate_quantized_multiplier_greater_than_one(float multiplier, return Status{}; } -arm_compute::Status calculate_quantized_multipliers(const QuantizationInfo &iq_info, - const QuantizationInfo &wq_info, - const QuantizationInfo &oq_info, +arm_compute::Status calculate_quantized_multipliers(const QuantizationInfo &iq_info, + const QuantizationInfo &wq_info, + const QuantizationInfo &oq_info, GEMMLowpOutputStageInfo &stage_info) { ARM_COMPUTE_RETURN_ERROR_ON(iq_info.scale().empty()); ARM_COMPUTE_RETURN_ERROR_ON(wq_info.scale().empty()); ARM_COMPUTE_RETURN_ERROR_ON(oq_info.scale().empty()); - - const unsigned int size = wq_info.scale().size(); - - auto &quant_multipliers = stage_info.gemmlowp_multipliers; - auto &quant_shifts = stage_info.gemmlowp_shifts; - quant_multipliers.resize(size); - quant_shifts.resize(size); + constexpr unsigned int padding_elems = 32; // assembly kernels assume the shifts and multipliers buffers are padded + const unsigned int size = wq_info.scale().size(); + const size_t padded_size = (size == 1) ? 1 : size + padding_elems; + auto &quant_multipliers = stage_info.gemmlowp_multipliers; + auto &quant_shifts = stage_info.gemmlowp_shifts; + quant_multipliers.resize(padded_size); + quant_shifts.resize(padded_size); const auto &w_scales = wq_info.scale(); const float i_scale = iq_info.scale().at(0); const float o_scale = oq_info.scale().at(0); - for(unsigned int i = 0; i < size; ++i) + for (unsigned int i = 0; i < size; ++i) { const float multiplier = i_scale * w_scales[i] / o_scale; int32_t quant_multiplier = 0; @@ -158,7 +155,7 @@ std::pair<int, int> get_min_max_values_from_quantized_data_type(DataType data_ty { int min_quant_val = 0; int max_quant_val = 0; - switch(data_type) + switch (data_type) { case DataType::QASYMM8: min_quant_val = std::numeric_limits<uint8_t>::min(); @@ -182,20 +179,60 @@ std::pair<int, int> get_min_max_values_from_quantized_data_type(DataType data_ty } return std::make_pair(min_quant_val, max_quant_val); } + +std::tuple<int32_t, int32_t> get_quantized_asymmetric_output_min_max(const QuantizationInfo &q_info, + const ActivationLayerInfo &act_info, + DataType data_type) +{ + ARM_COMPUTE_ERROR_ON(data_type != DataType::QASYMM8 && data_type != DataType::QASYMM8_SIGNED); + + const auto min_max = get_min_max(data_type); + + int32_t type_min = std::get<0>(min_max).get<int32_t>(); + int32_t type_max = std::get<1>(min_max).get<int32_t>(); + + const UniformQuantizationInfo q_unif = q_info.uniform(); + + if (act_info.enabled()) + { + switch (act_info.activation()) + { + case ActivationLayerInfo::ActivationFunction::RELU: + type_min = q_unif.offset; + break; + case ActivationLayerInfo::ActivationFunction::BOUNDED_RELU: + type_min = q_unif.offset; + type_max = (data_type == DataType::QASYMM8) ? quantize_qasymm8(act_info.a(), q_info) + : quantize_qasymm8_signed(act_info.a(), q_info); + break; + case ActivationLayerInfo::ActivationFunction::LU_BOUNDED_RELU: + type_min = (data_type == DataType::QASYMM8) ? quantize_qasymm8(act_info.b(), q_info) + : quantize_qasymm8_signed(act_info.b(), q_info); + type_max = (data_type == DataType::QASYMM8) ? quantize_qasymm8(act_info.a(), q_info) + : quantize_qasymm8_signed(act_info.a(), q_info); + break; + default: + ARM_COMPUTE_ERROR("Activation function not supported."); + break; + } + } + + return std::make_tuple(type_min, type_max); +} + void compute_quantized_multipliers_and_shifts(const ITensorInfo *input, const ITensorInfo *weights, const ITensorInfo *output, - unsigned int idx_ofms, int32_t *output_multipliers_ptr, int32_t *output_shifts_ptr) { - const unsigned int num_filters = is_data_type_quantized_per_channel(weights->data_type()) ? weights->dimension(idx_ofms) : 1; - const UniformQuantizationInfo iq_info = input->quantization_info().uniform(); const QuantizationInfo wq_info = weights->quantization_info(); const UniformQuantizationInfo oq_info = output->quantization_info().uniform(); - for(unsigned int i = 0; i < num_filters; ++i) + const unsigned int num_filters = wq_info.scale().size(); + + for (unsigned int i = 0; i < num_filters; ++i) { int32_t output_multiplier = 0; int32_t output_shift = 0; @@ -209,13 +246,14 @@ void compute_quantized_multipliers_and_shifts(const ITensorInfo *input, int32_t saturating_rounding_doubling_highmul(int32_t a, int32_t b) { - bool overflow = a == b && a == std::numeric_limits<int32_t>::min(); - int64_t a_64(a); - int64_t b_64(b); - int64_t ab_64 = a_64 * b_64; - bool is_positive_or_zero = a == 0 || b == 0 || (std::signbit(a) == std::signbit(b)); - int32_t nudge = is_positive_or_zero ? (1 << 30) : (1 - (1 << 30)); - int32_t ab_x2_high32 = static_cast<int32_t>((ab_64 + nudge) / (1ll << 31)); + bool overflow = a == b && a == std::numeric_limits<int32_t>::min(); + int64_t a_64(a); + int64_t b_64(b); + int64_t ab_64 = a_64 * b_64; + const bool is_positive_or_zero = + a == 0 || b == 0 || (std::signbit(static_cast<double>(a)) == std::signbit(static_cast<double>(b))); + int32_t nudge = is_positive_or_zero ? (1 << 30) : (1 - (1 << 30)); + int32_t ab_x2_high32 = static_cast<int32_t>((ab_64 + nudge) / (1ll << 31)); return overflow ? std::numeric_limits<int32_t>::max() : ab_x2_high32; } @@ -235,11 +273,11 @@ int32_t multiply_by_quantized_multiplier(int32_t input, int32_t qmul, int32_t sh int32_t saturating_rounding_multiply_by_pow2(int32_t exponent, int32_t v) { - if(exponent == 0) + if (exponent == 0) { return v; } - else if(exponent < 0) + else if (exponent < 0) { return rounding_divide_by_pow2(v, -exponent); } @@ -259,11 +297,14 @@ int32_t saturating_rounding_multiply_by_pow2(int32_t exponent, int32_t v) } } -void get_invsqrt_quantized_multiplier_exp(int32_t input, int32_t reverse_shift, int32_t &output_inv_sqrt, int32_t &output_shift) +void get_invsqrt_quantized_multiplier_exp(int32_t input, + int32_t reverse_shift, + int32_t &output_inv_sqrt, + int32_t &output_shift) { ARM_COMPUTE_ERROR_ON(input < 0); - if(input <= 1) + if (input <= 1) { // dealing the inputs (0 and 1) separately to avoid overflow output_inv_sqrt = std::numeric_limits<std::int32_t>::max(); @@ -273,7 +314,7 @@ void get_invsqrt_quantized_multiplier_exp(int32_t input, int32_t reverse_shift, // prepare input for fixed point operation and compute shift value output_shift = 11; - while(input >= (1 << 29)) + while (input >= (1 << 29)) { input /= 4; ++output_shift; @@ -302,9 +343,7 @@ void get_invsqrt_quantized_multiplier_exp(int32_t input, int32_t reverse_shift, // multiplication of two fixed point numbers, defined for readability auto fixed_point_mul = [](FixedPointRawType a, FixedPointRawType b) -> FixedPointRawType - { - return saturating_rounding_doubling_highmul(a, b); - }; + { return saturating_rounding_doubling_highmul(a, b); }; // rescaling of fixed point to have dst_bit integer bits, defined for readability auto fixed_point_rescale = [](FixedPointRawType a, uint32_t src_bit, uint32_t dst_bit) -> FixedPointRawType @@ -315,17 +354,18 @@ void get_invsqrt_quantized_multiplier_exp(int32_t input, int32_t reverse_shift, // 5 iterations of Newton-Raphson method for inverse square root - 1.5 * x_n = input/2 * (x_n)^3 constexpr int32_t num_iteration = 5; - for(int32_t i = 0; i < num_iteration; ++i) + for (int32_t i = 0; i < num_iteration; ++i) { const auto x3 = fixed_point_rescale(fixed_point_mul(fixed_point_mul(x, x), x), 9, fixedpoint_position); - x = fixed_point_rescale(fixed_point_mul(fixedpoint_half_three, x) - fixed_point_mul(fixedpoint_half_input, x3), 6, fixedpoint_position); + x = fixed_point_rescale(fixed_point_mul(fixedpoint_half_three, x) - fixed_point_mul(fixedpoint_half_input, x3), + 6, fixedpoint_position); } // fixed point representation of sqrt(1/2) const FixedPoint0 fixedpoint_half_sqrt_2 = 1518500250; x = fixed_point_mul(fixedpoint_half_sqrt_2, x); output_inv_sqrt = x; - if(output_shift < 0) + if (output_shift < 0) { output_inv_sqrt <<= -output_shift; output_shift = 0; @@ -333,5 +373,5 @@ void get_invsqrt_quantized_multiplier_exp(int32_t input, int32_t reverse_shift, // convert right shift to left shift output_shift *= reverse_shift; } -} // quantization -} // arm_compute +} // namespace quantization +} // namespace arm_compute |