diff options
author | Sang-Hoon Park <sang-hoon.park@arm.com> | 2020-10-13 23:34:09 +0100 |
---|---|---|
committer | Sang-Hoon Park <sang-hoon.park@arm.com> | 2020-10-16 08:43:24 +0000 |
commit | e7280585b317f695a932be5981895377e174946d (patch) | |
tree | 3d5662a779d2c3bee9e45a2db27a27cd0275c9d5 /src | |
parent | cf343e3798d2a8c2ad2fcac488e4b78e2b5c968d (diff) | |
download | ComputeLibrary-e7280585b317f695a932be5981895377e174946d.tar.gz |
COMPMID-3805: Fix SQRT non-zero output for zero input
- For AArch64, NEActivationLayerKernel uses vsqrt rather than
vinvsqrt.
- For non-AArch64, it masks values to ensure zero input
results in zero output without producing NaN.
- Test cases for FP16 and FP32's positive boundary values
are added.
Change-Id: Ic0104ee5d7045059c2e9bd052616a4a3b43a315d
Signed-off-by: Sang-Hoon Park <sang-hoon.park@arm.com>
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/4150
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Georgios Pinitas <georgios.pinitas@arm.com>
Diffstat (limited to 'src')
-rw-r--r-- | src/core/NEON/kernels/NEActivationLayerKernel.cpp | 39 | ||||
-rw-r--r-- | src/core/NEON/wrapper/intrinsics/intrinsics.h | 1 | ||||
-rw-r--r-- | src/core/NEON/wrapper/intrinsics/sqrt.h | 56 |
3 files changed, 89 insertions, 7 deletions
diff --git a/src/core/NEON/kernels/NEActivationLayerKernel.cpp b/src/core/NEON/kernels/NEActivationLayerKernel.cpp index 621af51f3c..d80aab7069 100644 --- a/src/core/NEON/kernels/NEActivationLayerKernel.cpp +++ b/src/core/NEON/kernels/NEActivationLayerKernel.cpp @@ -108,6 +108,23 @@ std::pair<Status, Window> validate_and_configure_window(const ITensorInfo *input return std::make_pair(Status{}, win); } + +#ifndef __aarch64__ +inline float32x4_t mask_float_vector(const float32x4_t &in, const uint32x4_t &mask) +{ + auto int_in = vreinterpretq_u32_f32(in); + return vreinterpretq_f32_u32(wrapper::vand(int_in, mask)); +} + +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +inline float16x8_t mask_float_vector(const float16x8_t &in, const uint16x8_t &mask) +{ + auto int_in = vreinterpretq_u16_f16(in); + return vreinterpretq_f16_u16(wrapper::vand(int_in, mask)); +} +#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ +#endif /* __arch64__ */ + } // namespace NEActivationLayerKernel::NEActivationLayerKernel() @@ -252,12 +269,12 @@ NEActivationLayerKernel::activation(const ITensor *src, ITensor *dst, const Wind Iterator input(src, win_collapsed); Iterator output(dst, win_collapsed); - // A small delta added to the input to prevent NAN values caused by zeros in inputs to SQRT -#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC - const auto delta = wrapper::vdup_n(static_cast<T>(1e-7), ExactTagType {}); -#else /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ - const auto delta = wrapper::vdup_n(static_cast<T>(1e-24), ExactTagType {}); -#endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC */ + // In case of non-aarch64, a small delta value is added to the input + // to prevent NAN values caused by zeros in inputs to SQRT. + // In case of aarh64, we call vsqrt directly, so we don't use delta. +#ifndef __aarch64__ + const auto delta = wrapper::vdup_n(static_cast<T>((src->info()->data_type() == DataType::F32 ? 1e-24 : 1e-7)), ExactTagType {}); +#endif /* __aarch64 */ const auto const_1 = wrapper::vdup_n(static_cast<T>(1.f), ExactTagType {}); const auto const_0 = wrapper::vdup_n(static_cast<T>(0.f), ExactTagType{}); const auto const_6 = wrapper::vdup_n(static_cast<T>(6.f), ExactTagType{}); @@ -310,7 +327,15 @@ NEActivationLayerKernel::activation(const ITensor *src, ITensor *dst, const Wind tmp = wrapper::vbsl(wrapper::vcge(vin, const_0), vin, wrapper::vmul(va, wrapper::vsub(wrapper::vexpq(vin), const_1))); break; case ActivationFunction::SQRT: - tmp = wrapper::vinv(wrapper::vinvsqrt(wrapper::vadd(vin, delta))); +#ifdef __aarch64__ + tmp = wrapper::vsqrt(vin); +#else /* aarch64 */ + { + const auto bitmask = wrapper::vceq(vin, wrapper::vdup_n(T(0), ExactTagType{})); + tmp = wrapper::vinv(wrapper::vinvsqrt(wrapper::vadd(vin, mask_float_vector(delta, bitmask)))); + tmp = mask_float_vector(tmp, wrapper::vnot(bitmask)); + } +#endif /* aarch64 */ break; case ActivationFunction::SQUARE: tmp = wrapper::vmul(vin, vin); diff --git a/src/core/NEON/wrapper/intrinsics/intrinsics.h b/src/core/NEON/wrapper/intrinsics/intrinsics.h index 495321a6a1..070f3c7065 100644 --- a/src/core/NEON/wrapper/intrinsics/intrinsics.h +++ b/src/core/NEON/wrapper/intrinsics/intrinsics.h @@ -66,6 +66,7 @@ #include "src/core/NEON/wrapper/intrinsics/round.h" #include "src/core/NEON/wrapper/intrinsics/setlane.h" #include "src/core/NEON/wrapper/intrinsics/sin.h" +#include "src/core/NEON/wrapper/intrinsics/sqrt.h" #include "src/core/NEON/wrapper/intrinsics/store.h" #include "src/core/NEON/wrapper/intrinsics/sub.h" #include "src/core/NEON/wrapper/intrinsics/tanh.h" diff --git a/src/core/NEON/wrapper/intrinsics/sqrt.h b/src/core/NEON/wrapper/intrinsics/sqrt.h new file mode 100644 index 0000000000..11954cf6c9 --- /dev/null +++ b/src/core/NEON/wrapper/intrinsics/sqrt.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2020 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_WRAPPER_SQRT_H +#define ARM_COMPUTE_WRAPPER_SQRT_H + +#ifdef __aarch64__ + +#include <arm_neon.h> + +namespace arm_compute +{ +namespace wrapper +{ +#define VSQRT_IMPL(type, prefix, postfix) \ + inline type vsqrt(const type &a) \ + { \ + return prefix##_##postfix(a); \ + } + +VSQRT_IMPL(float32x2_t, vsqrt, f32) +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +VSQRT_IMPL(float16x4_t, vsqrt, f16) +#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + +VSQRT_IMPL(float32x4_t, vsqrtq, f32) +#ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC +VSQRT_IMPL(float16x8_t, vsqrtq, f16) +#endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC + +} // namespace wrapper +} // namespace arm_compute + +#endif // __aarch64__ + +#endif /* ARM_COMPUTE_WRAPPER_SQRT_H */
\ No newline at end of file |