From ea9bd8f99cf35986fc1ad13992bc1e3ae689c6d0 Mon Sep 17 00:00:00 2001 From: Pablo Marquez Tello Date: Mon, 4 Sep 2023 14:45:13 +0100 Subject: Changes to ElementwiseOp to enable fp16 in armv8a multi_isa builds * Code guarded with __ARM_FEATURE_FP16_VECTOR_ARITHMETIC needs to be moved to an fp16.cpp file to allow compilation with -march=armv8.2-a+fp16 * Partially resolves MLCE-1102 Change-Id: I5ecfc8f6c0d84f92d80bec2cde6e7338794b9788 Signed-off-by: Pablo Marquez Tello Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/10240 Tested-by: Arm Jenkins Reviewed-by: Viet-Hoa Do Comments-Addressed: Arm Jenkins Benchmark: Arm Jenkins --- Android.bp | 1 - filelist.json | 1 - src/BUILD.bazel | 1 - src/CMakeLists.txt | 1 - .../elementwise_unary/generic/neon/impl.cpp | 286 --------------------- .../kernels/elementwise_unary/generic/neon/impl.h | 253 +++++++++++++++++- 6 files changed, 250 insertions(+), 293 deletions(-) delete mode 100644 src/cpu/kernels/elementwise_unary/generic/neon/impl.cpp diff --git a/Android.bp b/Android.bp index b5a0ef9f9a..274d6e7f4d 100644 --- a/Android.bp +++ b/Android.bp @@ -501,7 +501,6 @@ cc_library_static { "src/cpu/kernels/elementwise_binary/generic/neon/qasymm8_signed.cpp", "src/cpu/kernels/elementwise_unary/generic/neon/fp16.cpp", "src/cpu/kernels/elementwise_unary/generic/neon/fp32.cpp", - "src/cpu/kernels/elementwise_unary/generic/neon/impl.cpp", "src/cpu/kernels/elementwise_unary/generic/neon/integer.cpp", "src/cpu/kernels/elementwise_unary/generic/neon/q8.cpp", "src/cpu/kernels/elementwise_unary/generic/neon/qasymm8.cpp", diff --git a/filelist.json b/filelist.json index a3f49d2bae..9a87e857ef 100644 --- a/filelist.json +++ b/filelist.json @@ -1445,7 +1445,6 @@ "src/runtime/NEON/functions/NEElementwiseUnaryLayer.cpp" ], "neon": { - "common":["src/cpu/kernels/elementwise_unary/generic/neon/impl.cpp"], "integer": ["src/cpu/kernels/elementwise_unary/generic/neon/integer.cpp"], "fp32": ["src/cpu/kernels/elementwise_unary/generic/neon/fp32.cpp"], "fp16": ["src/cpu/kernels/elementwise_unary/generic/neon/fp16.cpp"], diff --git a/src/BUILD.bazel b/src/BUILD.bazel index 0eb27c22ef..27251e67f2 100644 --- a/src/BUILD.bazel +++ b/src/BUILD.bazel @@ -753,7 +753,6 @@ filegroup( "cpu/kernels/elementwise_binary/generic/neon/qasymm8_signed.cpp", "cpu/kernels/elementwise_unary/generic/neon/fp16.cpp", "cpu/kernels/elementwise_unary/generic/neon/fp32.cpp", - "cpu/kernels/elementwise_unary/generic/neon/impl.cpp", "cpu/kernels/elementwise_unary/generic/neon/integer.cpp", "cpu/kernels/elementwise_unary/generic/neon/q8.cpp", "cpu/kernels/elementwise_unary/generic/neon/qasymm8.cpp", diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8fffa965b9..cf91a2c0a1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -745,7 +745,6 @@ target_sources( cpu/kernels/elementwise_binary/generic/neon/qasymm8_signed.cpp cpu/kernels/elementwise_unary/generic/neon/fp16.cpp cpu/kernels/elementwise_unary/generic/neon/fp32.cpp - cpu/kernels/elementwise_unary/generic/neon/impl.cpp cpu/kernels/elementwise_unary/generic/neon/integer.cpp cpu/kernels/elementwise_unary/generic/neon/q8.cpp cpu/kernels/elementwise_unary/generic/neon/qasymm8.cpp diff --git a/src/cpu/kernels/elementwise_unary/generic/neon/impl.cpp b/src/cpu/kernels/elementwise_unary/generic/neon/impl.cpp deleted file mode 100644 index 1b449340ba..0000000000 --- a/src/cpu/kernels/elementwise_unary/generic/neon/impl.cpp +++ /dev/null @@ -1,286 +0,0 @@ -/* - * Copyright (c) 2018-2023 Arm Limited. - * - * SPDX-License-Identifier: MIT - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to - * deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or - * sell copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -#include "src/cpu/kernels/elementwise_unary/generic/neon/impl.h" -#include "arm_compute/core/Helpers.h" -#include "src/core/NEON/NEAsymm.h" - -namespace arm_compute -{ -namespace cpu -{ -template -inline ScalarType elementwise_op_scalar_imp(ElementWiseUnary op, const ScalarType &a) -{ - switch(op) - { - case ElementWiseUnary::RSQRT: - return 1 / sqrt(a); - case ElementWiseUnary::EXP: - return std::exp(a); - case ElementWiseUnary::NEG: - return -a; - case ElementWiseUnary::LOG: - return std::log(a); - case ElementWiseUnary::ABS: - return std::abs(a); - case ElementWiseUnary::ROUND: - return support::cpp11::nearbyint(a); - case ElementWiseUnary::SIN: - return std::sin(a); - default: - ARM_COMPUTE_ERROR("NOT_SUPPORTED!"); - } -} - -template -inline VectorType elementwise_op_imp(ElementWiseUnary op, const VectorType &a) -{ - switch(op) - { - case ElementWiseUnary::RSQRT: - return wrapper::vinvsqrt(a); - case ElementWiseUnary::EXP: - return wrapper::vexpq(a); - case ElementWiseUnary::NEG: - return wrapper::vneg(a); - case ElementWiseUnary::LOG: - return wrapper::vlog(a); - case ElementWiseUnary::ABS: - return wrapper::vabs(a); - case ElementWiseUnary::ROUND: - return wrapper::vround(a); - case ElementWiseUnary::SIN: - return wrapper::vsin(a); - default: - ARM_COMPUTE_ERROR("NOT_SUPPORTED!"); - } -} - -template -void elementwise_op(const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op) -{ - const int window_step_x = 16 / sizeof(ScalarType); - const auto window_start_x = static_cast(window.x().start()); - const auto window_end_x = static_cast(window.x().end()); - - Window win = window; - win.set(Window::DimX, Window::Dimension(0, 1, 1)); - - Iterator input(in, win); - Iterator output(out, win); - - execute_window_loop(win, [&](const Coordinates &) - { - auto output_ptr = reinterpret_cast(output.ptr()); - const auto input_ptr = reinterpret_cast(input.ptr()); - - int x = window_start_x; - for(; x <= window_end_x - window_step_x; x += window_step_x) - { - wrapper::vstore(output_ptr + x, elementwise_op_imp(op, wrapper::vloadq(input_ptr + x))); - } - for(; x < window_end_x; ++x) - { - *(output_ptr + x) = elementwise_op_scalar_imp(op, *(input_ptr + x)); - } - }, - input, output); -} -#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) -template void elementwise_op<__fp16>(const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op); -#endif //defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(ENABLE_FP16_KERNELS) -template void elementwise_op(const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op); -template void elementwise_op(const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op); - -template <> -void elementwise_op(const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op) -{ - const int window_step_x = 16; - const auto window_start_x = static_cast(window.x().start()); - const auto window_end_x = static_cast(window.x().end()); - const UniformQuantizationInfo qi_in = in->info()->quantization_info().uniform(); - const UniformQuantizationInfo qi_out = out->info()->quantization_info().uniform(); - const auto min_clamped_value = vdupq_n_f32((-128 - qi_out.offset) * qi_out.scale); - const auto max_clamped_value = vdupq_n_f32((127 - qi_out.offset) * qi_out.scale); - Window win = window; - win.set(Window::DimX, Window::Dimension(0, 1, 1)); - - Iterator input(in, win); - Iterator output(out, win); - - execute_window_loop(win, [&](const Coordinates &) - { - int8x16_t vout; - auto output_ptr = reinterpret_cast(output.ptr()); - const auto input_ptr = reinterpret_cast(input.ptr()); - const auto vconst_0_f32 = vdupq_n_f32(0); - auto clamped_value = (op == ElementWiseUnary::LOG) ? min_clamped_value : max_clamped_value; - - int x = window_start_x; - for(; x <= window_end_x - window_step_x; x += window_step_x) - { - const auto vin = wrapper::vloadq(input_ptr + x); - - // De-quantize - const auto vin_deq = vdequantize(vin, qi_in); - - // Perform activation - float32x4x4_t vtmp_deq = - { - { - elementwise_op_imp(op, vin_deq.val[0]), - elementwise_op_imp(op, vin_deq.val[1]), - elementwise_op_imp(op, vin_deq.val[2]), - elementwise_op_imp(op, vin_deq.val[3]), - } - }; - - if((op == ElementWiseUnary::LOG) || (op == ElementWiseUnary::RSQRT)) - { - vtmp_deq.val[0] = vbslq_f32(vcleq_f32(vin_deq.val[0], vconst_0_f32), clamped_value, vtmp_deq.val[0]); - vtmp_deq.val[1] = vbslq_f32(vcleq_f32(vin_deq.val[1], vconst_0_f32), clamped_value, vtmp_deq.val[1]); - vtmp_deq.val[2] = vbslq_f32(vcleq_f32(vin_deq.val[2], vconst_0_f32), clamped_value, vtmp_deq.val[2]); - vtmp_deq.val[3] = vbslq_f32(vcleq_f32(vin_deq.val[3], vconst_0_f32), clamped_value, vtmp_deq.val[3]); - } - - // Re-quantize to new output space - vout = vquantize_signed(vtmp_deq, qi_out); - wrapper::vstore(output_ptr + x, vout); - } - for(; x < window_end_x; ++x) - { - qasymm8_signed_t in = *(reinterpret_cast(input_ptr + x)); - qasymm8_signed_t tmp = 0; - float tmp_f = dequantize_qasymm8_signed(in, qi_in); - if(tmp_f <= 0.0) - { - if(op == ElementWiseUnary::LOG) - { - tmp_f = (-128 - qi_out.offset) * qi_out.scale; - } - else if(op == ElementWiseUnary::RSQRT) - { - tmp_f = (127 - qi_out.offset) * qi_out.scale; - } - else - { - tmp_f = elementwise_op_scalar_imp(op, tmp_f); - } - } - else - { - tmp_f = elementwise_op_scalar_imp(op, tmp_f); - } - tmp = quantize_qasymm8_signed(tmp_f, qi_out, RoundingPolicy::TO_ZERO); // Set rounding policy TO_ZERO to be compatible with vquantize_signed() used above that follow same policy for armv7a. - // For aarch64 LUT is used and rounding to nearest is used - *(output_ptr + x) = tmp; - } - }, - input, output); -} -template <> -void elementwise_op(const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op) -{ - const int window_step_x = 16; - const auto window_start_x = static_cast(window.x().start()); - const auto window_end_x = static_cast(window.x().end()); - const UniformQuantizationInfo qi_in = in->info()->quantization_info().uniform(); - const UniformQuantizationInfo qi_out = out->info()->quantization_info().uniform(); - const auto vconst_0_f32 = vdupq_n_f32(0); - const auto min_clamped_value = vdupq_n_f32((0 - qi_out.offset) * qi_out.scale); - const auto max_clamped_value = vdupq_n_f32((255 - qi_out.offset) * qi_out.scale); - Window win = window; - win.set(Window::DimX, Window::Dimension(0, 1, 1)); - - Iterator input(in, win); - Iterator output(out, win); - - execute_window_loop(win, [&](const Coordinates &) - { - uint8x16_t vout; - auto clamped_value = (op == ElementWiseUnary::LOG) ? min_clamped_value : max_clamped_value; - auto output_ptr = reinterpret_cast(output.ptr()); - const auto input_ptr = reinterpret_cast(input.ptr()); - int x = window_start_x; - for(; x <= window_end_x - window_step_x; x += window_step_x) - { - const auto vin = wrapper::vloadq(input_ptr + x); - - // De-quantize - const auto vin_deq = vdequantize(vin, qi_in); - - // Perform activation - float32x4x4_t vtmp_deq = - { - { - elementwise_op_imp(op, vin_deq.val[0]), - elementwise_op_imp(op, vin_deq.val[1]), - elementwise_op_imp(op, vin_deq.val[2]), - elementwise_op_imp(op, vin_deq.val[3]), - } - }; - if((op == ElementWiseUnary::LOG) || (op == ElementWiseUnary::RSQRT)) - { - vtmp_deq.val[0] = vbslq_f32(vcleq_f32(vin_deq.val[0], vconst_0_f32), clamped_value, vtmp_deq.val[0]); - vtmp_deq.val[1] = vbslq_f32(vcleq_f32(vin_deq.val[1], vconst_0_f32), clamped_value, vtmp_deq.val[1]); - vtmp_deq.val[2] = vbslq_f32(vcleq_f32(vin_deq.val[2], vconst_0_f32), clamped_value, vtmp_deq.val[2]); - vtmp_deq.val[3] = vbslq_f32(vcleq_f32(vin_deq.val[3], vconst_0_f32), clamped_value, vtmp_deq.val[3]); - } - - // Re-quantize to new output space - vout = vquantize(vtmp_deq, qi_out); - wrapper::vstore(output_ptr + x, vout); - } - for(; x < window_end_x; ++x) - { - qasymm8_t in = *(reinterpret_cast(input_ptr + x)); - qasymm8_t tmp = 0; - float tmp_f = dequantize_qasymm8(in, qi_in); - if(tmp_f <= 0.0) - { - if(op == ElementWiseUnary::LOG) - { - tmp_f = (0 - qi_out.offset) * qi_out.scale; - } - else if(op == ElementWiseUnary::RSQRT) - { - tmp_f = (255 - qi_out.offset) * qi_out.scale; - } - else - { - tmp_f = elementwise_op_scalar_imp(op, tmp_f); - } - } - else - { - tmp_f = elementwise_op_scalar_imp(op, tmp_f); - } - tmp = quantize_qasymm8(tmp_f, qi_out, RoundingPolicy::TO_ZERO); - *(output_ptr + x) = tmp; - } - }, - input, output); -} -} // namespace cpu -} // namespace arm_compute diff --git a/src/cpu/kernels/elementwise_unary/generic/neon/impl.h b/src/cpu/kernels/elementwise_unary/generic/neon/impl.h index 66b8b5fe45..dbc1dde4fa 100644 --- a/src/cpu/kernels/elementwise_unary/generic/neon/impl.h +++ b/src/cpu/kernels/elementwise_unary/generic/neon/impl.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2022 Arm Limited. + * Copyright (c) 2018-2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -26,6 +26,7 @@ #include "arm_compute/core/Helpers.h" #include "arm_compute/core/Types.h" +#include "src/core/NEON/NEAsymm.h" #include "src/core/NEON/wrapper/intrinsics/intrinsics.h" namespace arm_compute @@ -33,9 +34,255 @@ namespace arm_compute namespace cpu { template -void elementwise_op(const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op); +inline ScalarType elementwise_op_scalar_imp(ElementWiseUnary op, const ScalarType &a) +{ + switch(op) + { + case ElementWiseUnary::RSQRT: + return 1 / sqrt(a); + case ElementWiseUnary::EXP: + return std::exp(a); + case ElementWiseUnary::NEG: + return -a; + case ElementWiseUnary::LOG: + return std::log(a); + case ElementWiseUnary::ABS: + return std::abs(a); + case ElementWiseUnary::ROUND: + return support::cpp11::nearbyint(a); + case ElementWiseUnary::SIN: + return std::sin(a); + default: + ARM_COMPUTE_ERROR("NOT_SUPPORTED!"); + } +} + +template +inline VectorType elementwise_op_imp(ElementWiseUnary op, const VectorType &a) +{ + switch(op) + { + case ElementWiseUnary::RSQRT: + return wrapper::vinvsqrt(a); + case ElementWiseUnary::EXP: + return wrapper::vexpq(a); + case ElementWiseUnary::NEG: + return wrapper::vneg(a); + case ElementWiseUnary::LOG: + return wrapper::vlog(a); + case ElementWiseUnary::ABS: + return wrapper::vabs(a); + case ElementWiseUnary::ROUND: + return wrapper::vround(a); + case ElementWiseUnary::SIN: + return wrapper::vsin(a); + default: + ARM_COMPUTE_ERROR("NOT_SUPPORTED!"); + } +} + +template +inline void elementwise_op(const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op) +{ + const int window_step_x = 16 / sizeof(ScalarType); + const auto window_start_x = static_cast(window.x().start()); + const auto window_end_x = static_cast(window.x().end()); + + Window win = window; + win.set(Window::DimX, Window::Dimension(0, 1, 1)); + + Iterator input(in, win); + Iterator output(out, win); + + execute_window_loop(win, [&](const Coordinates &) + { + auto output_ptr = reinterpret_cast(output.ptr()); + const auto input_ptr = reinterpret_cast(input.ptr()); + + int x = window_start_x; + for(; x <= window_end_x - window_step_x; x += window_step_x) + { + wrapper::vstore(output_ptr + x, elementwise_op_imp(op, wrapper::vloadq(input_ptr + x))); + } + for(; x < window_end_x; ++x) + { + *(output_ptr + x) = elementwise_op_scalar_imp(op, *(input_ptr + x)); + } + }, + input, output); +} + +template <> +inline void elementwise_op(const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op) +{ + const int window_step_x = 16; + const auto window_start_x = static_cast(window.x().start()); + const auto window_end_x = static_cast(window.x().end()); + const UniformQuantizationInfo qi_in = in->info()->quantization_info().uniform(); + const UniformQuantizationInfo qi_out = out->info()->quantization_info().uniform(); + const auto min_clamped_value = vdupq_n_f32((-128 - qi_out.offset) * qi_out.scale); + const auto max_clamped_value = vdupq_n_f32((127 - qi_out.offset) * qi_out.scale); + Window win = window; + win.set(Window::DimX, Window::Dimension(0, 1, 1)); + + Iterator input(in, win); + Iterator output(out, win); + + execute_window_loop(win, [&](const Coordinates &) + { + int8x16_t vout; + auto output_ptr = reinterpret_cast(output.ptr()); + const auto input_ptr = reinterpret_cast(input.ptr()); + const auto vconst_0_f32 = vdupq_n_f32(0); + auto clamped_value = (op == ElementWiseUnary::LOG) ? min_clamped_value : max_clamped_value; + + int x = window_start_x; + for(; x <= window_end_x - window_step_x; x += window_step_x) + { + const auto vin = wrapper::vloadq(input_ptr + x); + + // De-quantize + const auto vin_deq = vdequantize(vin, qi_in); + + // Perform activation + float32x4x4_t vtmp_deq = + { + { + elementwise_op_imp(op, vin_deq.val[0]), + elementwise_op_imp(op, vin_deq.val[1]), + elementwise_op_imp(op, vin_deq.val[2]), + elementwise_op_imp(op, vin_deq.val[3]), + } + }; + + if((op == ElementWiseUnary::LOG) || (op == ElementWiseUnary::RSQRT)) + { + vtmp_deq.val[0] = vbslq_f32(vcleq_f32(vin_deq.val[0], vconst_0_f32), clamped_value, vtmp_deq.val[0]); + vtmp_deq.val[1] = vbslq_f32(vcleq_f32(vin_deq.val[1], vconst_0_f32), clamped_value, vtmp_deq.val[1]); + vtmp_deq.val[2] = vbslq_f32(vcleq_f32(vin_deq.val[2], vconst_0_f32), clamped_value, vtmp_deq.val[2]); + vtmp_deq.val[3] = vbslq_f32(vcleq_f32(vin_deq.val[3], vconst_0_f32), clamped_value, vtmp_deq.val[3]); + } + + // Re-quantize to new output space + vout = vquantize_signed(vtmp_deq, qi_out); + wrapper::vstore(output_ptr + x, vout); + } + for(; x < window_end_x; ++x) + { + qasymm8_signed_t in = *(reinterpret_cast(input_ptr + x)); + qasymm8_signed_t tmp = 0; + float tmp_f = dequantize_qasymm8_signed(in, qi_in); + if(tmp_f <= 0.0) + { + if(op == ElementWiseUnary::LOG) + { + tmp_f = (-128 - qi_out.offset) * qi_out.scale; + } + else if(op == ElementWiseUnary::RSQRT) + { + tmp_f = (127 - qi_out.offset) * qi_out.scale; + } + else + { + tmp_f = elementwise_op_scalar_imp(op, tmp_f); + } + } + else + { + tmp_f = elementwise_op_scalar_imp(op, tmp_f); + } + tmp = quantize_qasymm8_signed(tmp_f, qi_out, RoundingPolicy::TO_ZERO); // Set rounding policy TO_ZERO to be compatible with vquantize_signed() used above that follow same policy for armv7a. + // For aarch64 LUT is used and rounding to nearest is used + *(output_ptr + x) = tmp; + } + }, + input, output); +} +template <> +inline void elementwise_op(const ITensor *in, ITensor *out, const Window &window, ElementWiseUnary op) +{ + const int window_step_x = 16; + const auto window_start_x = static_cast(window.x().start()); + const auto window_end_x = static_cast(window.x().end()); + const UniformQuantizationInfo qi_in = in->info()->quantization_info().uniform(); + const UniformQuantizationInfo qi_out = out->info()->quantization_info().uniform(); + const auto vconst_0_f32 = vdupq_n_f32(0); + const auto min_clamped_value = vdupq_n_f32((0 - qi_out.offset) * qi_out.scale); + const auto max_clamped_value = vdupq_n_f32((255 - qi_out.offset) * qi_out.scale); + Window win = window; + win.set(Window::DimX, Window::Dimension(0, 1, 1)); + + Iterator input(in, win); + Iterator output(out, win); + + execute_window_loop(win, [&](const Coordinates &) + { + uint8x16_t vout; + auto clamped_value = (op == ElementWiseUnary::LOG) ? min_clamped_value : max_clamped_value; + auto output_ptr = reinterpret_cast(output.ptr()); + const auto input_ptr = reinterpret_cast(input.ptr()); + int x = window_start_x; + for(; x <= window_end_x - window_step_x; x += window_step_x) + { + const auto vin = wrapper::vloadq(input_ptr + x); + + // De-quantize + const auto vin_deq = vdequantize(vin, qi_in); + + // Perform activation + float32x4x4_t vtmp_deq = + { + { + elementwise_op_imp(op, vin_deq.val[0]), + elementwise_op_imp(op, vin_deq.val[1]), + elementwise_op_imp(op, vin_deq.val[2]), + elementwise_op_imp(op, vin_deq.val[3]), + } + }; + if((op == ElementWiseUnary::LOG) || (op == ElementWiseUnary::RSQRT)) + { + vtmp_deq.val[0] = vbslq_f32(vcleq_f32(vin_deq.val[0], vconst_0_f32), clamped_value, vtmp_deq.val[0]); + vtmp_deq.val[1] = vbslq_f32(vcleq_f32(vin_deq.val[1], vconst_0_f32), clamped_value, vtmp_deq.val[1]); + vtmp_deq.val[2] = vbslq_f32(vcleq_f32(vin_deq.val[2], vconst_0_f32), clamped_value, vtmp_deq.val[2]); + vtmp_deq.val[3] = vbslq_f32(vcleq_f32(vin_deq.val[3], vconst_0_f32), clamped_value, vtmp_deq.val[3]); + } + + // Re-quantize to new output space + vout = vquantize(vtmp_deq, qi_out); + wrapper::vstore(output_ptr + x, vout); + } + for(; x < window_end_x; ++x) + { + qasymm8_t in = *(reinterpret_cast(input_ptr + x)); + qasymm8_t tmp = 0; + float tmp_f = dequantize_qasymm8(in, qi_in); + if(tmp_f <= 0.0) + { + if(op == ElementWiseUnary::LOG) + { + tmp_f = (0 - qi_out.offset) * qi_out.scale; + } + else if(op == ElementWiseUnary::RSQRT) + { + tmp_f = (255 - qi_out.offset) * qi_out.scale; + } + else + { + tmp_f = elementwise_op_scalar_imp(op, tmp_f); + } + } + else + { + tmp_f = elementwise_op_scalar_imp(op, tmp_f); + } + tmp = quantize_qasymm8(tmp_f, qi_out, RoundingPolicy::TO_ZERO); + *(output_ptr + x) = tmp; + } + }, + input, output); +} } // namespace cpu } // namespace arm_compute -#endif // SRC_CORE_NEON_KERNELS_ELEMENTWISE_UNARY_LIST_H \ No newline at end of file +#endif // SRC_CORE_NEON_KERNELS_ELEMENTWISE_UNARY_LIST_H -- cgit v1.2.1