From 0d008f77b0085619c446d0ab5dc1228a80776706 Mon Sep 17 00:00:00 2001 From: Sang-Hoon Park Date: Fri, 13 Mar 2020 14:56:05 +0000 Subject: COMPMID-3281: Implement QSYMM16 Layer Normalization for NEON QLSTM - Reference kernel is modified to use the same algorithm as NEON kernel. - NEON kernel is implemented. - Tests for validation and run are added. Change-Id: I3533bc2bd12c6e9cc75d837ecf193f74ceddf796 Signed-off-by: Sang-Hoon Park Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/2948 Comments-Addressed: Arm Jenkins Tested-by: Arm Jenkins Reviewed-by: Michele Di Giorgio --- arm_compute/core/NEON/NEKernels.h | 1 + arm_compute/core/NEON/NESymm.h | 2 +- .../NEON/kernels/NEQLSTMLayerNormalizationKernel.h | 135 +++++++++++++++++++++ arm_compute/core/NEON/wrapper/intrinsics/getlane.h | 17 ++- arm_compute/core/utils/quantization/AsymmHelpers.h | 15 ++- 5 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 arm_compute/core/NEON/kernels/NEQLSTMLayerNormalizationKernel.h (limited to 'arm_compute') diff --git a/arm_compute/core/NEON/NEKernels.h b/arm_compute/core/NEON/NEKernels.h index d9f8f00c0b..38701f434a 100644 --- a/arm_compute/core/NEON/NEKernels.h +++ b/arm_compute/core/NEON/NEKernels.h @@ -120,6 +120,7 @@ #include "arm_compute/core/NEON/kernels/NEPixelWiseMultiplicationKernel.h" #include "arm_compute/core/NEON/kernels/NEPoolingLayerKernel.h" #include "arm_compute/core/NEON/kernels/NEPriorBoxLayerKernel.h" +#include "arm_compute/core/NEON/kernels/NEQLSTMLayerNormalizationKernel.h" #include "arm_compute/core/NEON/kernels/NEQuantizationLayerKernel.h" #include "arm_compute/core/NEON/kernels/NEROIAlignLayerKernel.h" #include "arm_compute/core/NEON/kernels/NEROIPoolingLayerKernel.h" diff --git a/arm_compute/core/NEON/NESymm.h b/arm_compute/core/NEON/NESymm.h index 0cc2a963cf..d6c5a7073a 100644 --- a/arm_compute/core/NEON/NESymm.h +++ b/arm_compute/core/NEON/NESymm.h @@ -239,7 +239,7 @@ inline qsymm16x8x2_t vquantize_qsymm16(const float32x4x4_t &qv, const UniformQua * * @return A neon vector holding the multiplied value */ -inline int32x4x2_t multiply_by_quantized_multipler_2row(int32x4x2_t input, int32_t qmul, int32_t shift) +inline int32x4x2_t multiply_by_quantized_multiplier_2row(int32x4x2_t input, int32_t qmul, int32_t shift) { const auto left_shift = shift > 0 ? shift : 0; const auto right_shift = shift > 0 ? 0 : -shift; diff --git a/arm_compute/core/NEON/kernels/NEQLSTMLayerNormalizationKernel.h b/arm_compute/core/NEON/kernels/NEQLSTMLayerNormalizationKernel.h new file mode 100644 index 0000000000..631de66cc2 --- /dev/null +++ b/arm_compute/core/NEON/kernels/NEQLSTMLayerNormalizationKernel.h @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2020 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_NEQLSTMLAYERNORMALIZATIONKERNEL_H +#define ARM_COMPUTE_NEQLSTMLAYERNORMALIZATIONKERNEL_H + +#include "arm_compute/core/NEON/INEKernel.h" +#include + +namespace arm_compute +{ +class ITensor; + +/** NEON kernel to perform layer normalization */ +class NEQLSTMLayerNormalizationKernel : public INEKernel +{ +public: + const char *name() const override + { + return "NEQLSTMLayerNormalizationKernel"; + } + /** Default constructor */ + NEQLSTMLayerNormalizationKernel() = default; + /** Prevent instances of this class from being copied (As this class contains pointers) */ + NEQLSTMLayerNormalizationKernel(const NEQLSTMLayerNormalizationKernel &) = delete; + /** Prevent instances of this class from being copied (As this class contains pointers) */ + NEQLSTMLayerNormalizationKernel &operator=(const NEQLSTMLayerNormalizationKernel &) = delete; + /** Default Move Constructor. */ + NEQLSTMLayerNormalizationKernel(NEQLSTMLayerNormalizationKernel &&) = default; + /** Default move assignment operator */ + NEQLSTMLayerNormalizationKernel &operator=(NEQLSTMLayerNormalizationKernel &&) = default; + /** Default destructor */ + ~NEQLSTMLayerNormalizationKernel() = default; + + /** Set the input and output tensors. + * + * @param[in] input Source tensor. Data types supported: QSYMM16. + * @param[out] output Destination tensor. Data types supported: Same as @p input. + * @param[in] weight Weight tensor. Data types supported: Same as @p input. + * @param[in] bias Bias tensor. Data types supported: S32 + */ + void configure(const ITensor *input, ITensor *output, const ITensor *weight, const ITensor *bias); + /** Static function to check if given info will lead to a valid configuration of @ref NEQLSTMLayerNormalizationKernel + * + * @param[in] input Source tensor info. Data types supported: QSYMM16. + * @param[in] output Destination tensor info. Data types supported: Same as @p input. + * @param[in] weight Weight tensor info. Data types supported: Same as @p input. + * @param[in] bias Bias tensor info. Data types supported: S32 + * + * @return a status + */ + static Status validate(const ITensorInfo *input, const ITensorInfo *output, const ITensorInfo *weight, const ITensorInfo *bias); + // Inherited methods overridden: + void run(const Window &window, const ThreadInfo &info) override; + +private: + // constants + static constexpr uint32_t max_input_dimension{ 2 }; /**< The maximum input dimension supported */ + static constexpr uint32_t max_weight_dimension{ 1 }; /**< The maximum weight dimension supported */ + static constexpr uint32_t max_bias_dimension{ 1 }; /**< The maximum bias dimension supported */ + static constexpr uint32_t vector_size_byte{ 16 }; /**< Computation vector size in byte */ + + using ComputeFuncType = std::function; + + ComputeFuncType _fn{}; /**< Function pointer to computation function */ + + const ITensor *_input{ nullptr }; /**< Input tensor */ + const ITensor *_weight{ nullptr }; /**< Weight tensor */ + const ITensor *_bias{ nullptr }; /**< Bias tensor */ + ITensor *_output{ nullptr }; /**< Output tensor */ + + int32_t _output_multiplier{}; /**< Multiplier for output values */ + int32_t _output_shift{}; /**< Shift value for output values */ + + int32_t _window_start_x{}; /**< The beginning of x-axis iteration */ + int32_t _window_end_x{}; /**< The end of x-axis iteration */ + int32_t _window_step_x{}; /**< The size of x-axis iteration's step */ + + Window _inout_window{}; /**< Window for input and output tensor */ + Window _weight_window{}; /**< Window for weight and bias tensor */ + + /** Function to configure initial windows for destination of computation + * + * @param[in] Target destination tensor to use for output window + * + * @return configured window + */ + Window configure_window(ITensor *target); + // Function to compute for data type QSYMM16 + void compute_qsymm16(); + /** Function to compute summation and summation of squared input of the given input pointer + * + * @param[in] Input_ptr pointer to input array + * + */ + std::pair sum_qsymm16(const int16_t *input_ptr); + /** Function to normalize values using computed mean and standard deviation + * + * @param[in] input_ptr Pointer to input array + * @param[in] output_ptr Pointer to output array + * @param[in] weight_ptr Pointer to weight array + * @param[in] bias_ptr Pointer to bias array + * @param[in] mean Mean value + * @param[in] inv_std_mul Quantized multiplier for standard deviation + * @param[in] inv_std_shift Shift for standard deviation + * + */ + void normalize_qasymm16(const int16_t *input_ptr, + int16_t *output_ptr, + const int16_t *weight_ptr, + const int32_t *bias_ptr, + int32_t mean, int32_t inv_std_mul, int32_t inv_std_shift); +}; +} // namespace arm_compute +#endif /* ARM_COMPUTE_NEQLSTMLAYERNORMALIZATIONKERNEL_H */ diff --git a/arm_compute/core/NEON/wrapper/intrinsics/getlane.h b/arm_compute/core/NEON/wrapper/intrinsics/getlane.h index 5cd390fee4..533bf63603 100644 --- a/arm_compute/core/NEON/wrapper/intrinsics/getlane.h +++ b/arm_compute/core/NEON/wrapper/intrinsics/getlane.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019 ARM Limited. + * Copyright (c) 2018-2020 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -185,6 +185,20 @@ VGETLANE_IMPL_4(float16_t, float16x4_t, f16) } \ } +#define VGETQLANE_IMPL_2(stype, vtype, postfix) \ + inline stype vgetlane(const vtype vector, const unsigned int lane) \ + { \ + switch(lane) \ + { \ + case 0: \ + return vgetq_lane_##postfix(vector, 0); \ + case 1: \ + return vgetq_lane_##postfix(vector, 1); \ + default: \ + ARM_COMPUTE_ERROR("Invalid lane"); \ + } \ + } + VGETQLANE_IMPL_16(uint8_t, uint8x16_t, u8) VGETQLANE_IMPL_16(int8_t, int8x16_t, s8) VGETQLANE_IMPL_8(uint16_t, uint16x8_t, u16) @@ -192,6 +206,7 @@ VGETQLANE_IMPL_8(int16_t, int16x8_t, s16) VGETQLANE_IMPL_4(uint32_t, uint32x4_t, u32) VGETQLANE_IMPL_4(int32_t, int32x4_t, s32) VGETQLANE_IMPL_4(float, float32x4_t, f32) +VGETQLANE_IMPL_2(int64_t, int64x2_t, s64) #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC VGETQLANE_IMPL_8(float16_t, float16x8_t, f16) #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC diff --git a/arm_compute/core/utils/quantization/AsymmHelpers.h b/arm_compute/core/utils/quantization/AsymmHelpers.h index 0f0ec72b60..a7bbf9b137 100644 --- a/arm_compute/core/utils/quantization/AsymmHelpers.h +++ b/arm_compute/core/utils/quantization/AsymmHelpers.h @@ -128,7 +128,7 @@ int32_t saturating_rounding_doubling_highmul(int32_t a, int32_t b); * * @return The multiplied value */ -int32_t multiply_by_quantized_multipler(int32_t input, int32_t qmul, int32_t shift); +int32_t multiply_by_quantized_multiplier(int32_t input, int32_t qmul, int32_t shift); /** Compute the value multiplied the power-of-two * @@ -137,7 +137,18 @@ int32_t multiply_by_quantized_multipler(int32_t input, int32_t qmul, int32_t shi * * @return The multiplied value */ -int32_t saturating_rounding_multiply_by_pow2(int exponent, int32_t v); +int32_t saturating_rounding_multiply_by_pow2(int32_t exponent, int32_t v); + +/** Compute quantized multiplier and shift for the inverse square root of input. + * Using 3-bit fixed point and 5 iteration of Newton-Raphson method. + * + * @param[in] input Input to use + * @param[in] reverse_shift -1 to reverse the shift direction + * @param[out] output_inv_sqrt Quantized multiplier for inverse square root + * @param[out] output_shift Shift for inverse square root + * + */ +void get_invsqrt_quantized_multiplier_exp(int32_t input, int32_t reverse_shift, int32_t &output_inv_sqrt, int32_t &output_shift); } // namespace quantization } // namespace arm_compute -- cgit v1.2.1