From 9d7b690fd886bcc514a6aa8d17d53e25e3500a33 Mon Sep 17 00:00:00 2001 From: Pablo Marquez Tello Date: Mon, 19 Dec 2022 17:12:53 +0000 Subject: Fixed various mismatches in CpuCastKernel * Fixes various mismatches when converting FP32 to BF16 and BF16 to FP32 * Fixed segfault when trying logging=1 and trying to log BF16 * Resolves MLCE-979 Change-Id: Ie517d0b7411b4e3a7fecdee588f0e073d290625a Signed-off-by: Pablo Marquez Tello Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/8830 Comments-Addressed: Arm Jenkins Reviewed-by: Viet-Hoa Do Tested-by: Arm Jenkins Benchmark: Arm Jenkins --- src/core/NEON/wrapper/intrinsics/cvt.h | 20 ------ src/cpu/kernels/cast/generic/neon/bfloat16.cpp | 92 +++++++++----------------- utils/TypePrinter.h | 2 +- 3 files changed, 34 insertions(+), 80 deletions(-) diff --git a/src/core/NEON/wrapper/intrinsics/cvt.h b/src/core/NEON/wrapper/intrinsics/cvt.h index baad1319b2..c75d43dbf2 100644 --- a/src/core/NEON/wrapper/intrinsics/cvt.h +++ b/src/core/NEON/wrapper/intrinsics/cvt.h @@ -87,26 +87,6 @@ vcvta(const float32x4_t &a) return vcvtaq_s32_f32(a); } #endif //__aarch64__ - -#if defined(ARM_COMPUTE_ENABLE_BF16) -/** Convert 2x128-bit floating point vectors into 1x128-bit bfloat16 vector - * - * @param[in] inptr Pointer to the input memory to load values from - * @param[in,out] outptr Pointer to the output memory to store values to - */ -inline void vcvt_bf16_f32(const float *inptr, uint16_t *outptr) -{ - __asm __volatile( - "ldp q0, q1, [%[inptr]]\n" - ".inst 0xea16800\n" // BFCVTN v0, v0 - ".inst 0x4ea16820\n" // BFCVTN2 v0, v1 - "str q0, [%[outptr]]\n" - : [inptr] "+r"(inptr) - : [outptr] "r"(outptr) - : "v0", "v1", "memory"); -} -#endif /* defined(ARM_COMPUTE_ENABLE_BF16) */ - } // namespace wrapper } // namespace arm_compute #endif /* ARM_COMPUTE_WRAPPER_CVT_H */ diff --git a/src/cpu/kernels/cast/generic/neon/bfloat16.cpp b/src/cpu/kernels/cast/generic/neon/bfloat16.cpp index eed537039f..942bdfae61 100644 --- a/src/cpu/kernels/cast/generic/neon/bfloat16.cpp +++ b/src/cpu/kernels/cast/generic/neon/bfloat16.cpp @@ -24,10 +24,8 @@ #if defined(ARM_COMPUTE_ENABLE_BF16) #include "arm_compute/core/TensorInfo.h" -#include "src/core/NEON/wrapper/wrapper.h" #include "src/cpu/kernels/CpuCastKernel.h" #include "src/cpu/kernels/cast/list.h" -#include "support/SaturateCast.h" namespace arm_compute { @@ -38,9 +36,9 @@ void neon_fp32_to_bfloat16_cast(const ITensor *_src, ITensor *_dst, const Thread ARM_COMPUTE_UNUSED(info); ARM_COMPUTE_UNUSED(_policy); - const auto window_start_x = static_cast(window.x().start()); - const auto window_end_x = static_cast(window.x().end()); - const int window_step_x = 16; + const auto window_start_x = static_cast(window.x().start()); + const auto window_end_x = static_cast(window.x().end()); + constexpr int window_step_x = 8; ARM_COMPUTE_ERROR_ON_NULLPTR(_src, _dst); ARM_COMPUTE_ERROR_ON(_src == _dst); @@ -52,25 +50,23 @@ void neon_fp32_to_bfloat16_cast(const ITensor *_src, ITensor *_dst, const Thread Iterator src(_src, win); Iterator dst(_dst, win); - /* Down-conversion F32 -> BFLOAT16 */ execute_window_loop(win, [&](const Coordinates &) { - const auto src_ptr = reinterpret_cast(src.ptr()); - const auto dst_ptr = reinterpret_cast(dst.ptr()); - - int x = window_start_x; - for(; x <= (window_end_x - window_step_x); x += window_step_x) + const auto src_ptr = reinterpret_cast(src.ptr()); + const auto dst_ptr = reinterpret_cast(dst.ptr()); + int x = window_start_x; + const int right_bound = (window_end_x - window_step_x); + for(; x <= right_bound; x += window_step_x) { - wrapper::vcvt_bf16_f32(reinterpret_cast(src.ptr()), - reinterpret_cast(dst.ptr())); - wrapper::vcvt_bf16_f32(reinterpret_cast(src.ptr()) + 8, - reinterpret_cast(dst.ptr()) + 8); + const auto vbf16_0 = vcombine_bf16( + vcvt_bf16_f32(vld1q_f32(src_ptr + x)), + vcvt_bf16_f32(vld1q_f32(src_ptr + x + 4))); + vst1q_bf16(dst_ptr + x, vbf16_0); } - for(; x < window_end_x; ++x) { - *(dst_ptr + x) = *(src_ptr + x); + *(reinterpret_cast(dst.ptr()) + x) = *(src_ptr + x); } }, src, dst); @@ -81,9 +77,9 @@ void neon_bfloat16_to_fp32_cast(const ITensor *_src, ITensor *_dst, const Thread ARM_COMPUTE_UNUSED(info); ARM_COMPUTE_UNUSED(_policy); - const auto window_start_x = static_cast(window.x().start()); - const auto window_end_x = static_cast(window.x().end()); - const int window_step_x = 16; + const auto window_start_x = static_cast(window.x().start()); + const auto window_end_x = static_cast(window.x().end()); + constexpr int window_step_x = 8; ARM_COMPUTE_ERROR_ON_NULLPTR(_src, _dst); ARM_COMPUTE_ERROR_ON(_src == _dst); @@ -95,48 +91,26 @@ void neon_bfloat16_to_fp32_cast(const ITensor *_src, ITensor *_dst, const Thread Iterator src(_src, win); Iterator dst(_dst, win); - switch(_dst->info()->data_type()) + /* Up-conversion BFLOAT16 -> F32 */ + execute_window_loop(win, [&](const Coordinates &) { - case DataType::F32: + const auto src_ptr = reinterpret_cast(src.ptr()); + const auto dst_ptr = reinterpret_cast(dst.ptr()); + + int x = window_start_x; + const int right_bound(window_end_x - window_step_x); + for(; x <= right_bound; x += window_step_x) { - /* Up-conversion BFLOAT16 -> F32 */ - execute_window_loop(win, [&](const Coordinates &) - { - const auto src_ptr = reinterpret_cast(src.ptr()); - const auto dst_ptr = reinterpret_cast(dst.ptr()); - - int x = window_start_x; - for(; x <= (window_end_x - window_step_x); x += window_step_x) - { - const uint16x8x2_t texels = - { - { - vld1q_u16(reinterpret_cast(src.ptr())), - vld1q_u16(reinterpret_cast(src.ptr()) + 8) - } - }; - - vst1q_f32(reinterpret_cast(dst.ptr()), - vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(vget_low_u16(texels.val[0])), 16))); - vst1q_f32(reinterpret_cast(dst.ptr()) + 4, - vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(vget_high_u16(texels.val[0])), 16))); - vst1q_f32(reinterpret_cast(dst.ptr()) + 8, - vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(vget_low_u16(texels.val[1])), 16))); - vst1q_f32(reinterpret_cast(dst.ptr()) + 12, - vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(vget_high_u16(texels.val[1])), 16))); - } - - for(; x < window_end_x; ++x) - { - *(dst_ptr + x) = float(*(src_ptr + x)); - } - }, - src, dst); - break; + const bfloat16x8_t vinput = vld1q_bf16(src_ptr + x); + vst1q_f32(dst_ptr + x, vcvt_f32_bf16(vget_low_bf16(vinput))); + vst1q_f32(dst_ptr + x + 4, vcvt_f32_bf16(vget_high_bf16(vinput))); } - default: - ARM_COMPUTE_ERROR("dst data type unsupported"); - } + for(; x < window_end_x; ++x) + { + *(dst_ptr + x) = float(*(reinterpret_cast(src_ptr) + x)); + } + }, + src, dst); } } // namespace cpu diff --git a/utils/TypePrinter.h b/utils/TypePrinter.h index 9f20b38b96..d4265bfdbd 100644 --- a/utils/TypePrinter.h +++ b/utils/TypePrinter.h @@ -480,7 +480,7 @@ inline ::std::ostream &operator<<(::std::ostream &os, const BoundingBoxTransform inline ::std::ostream &operator<<(::std::ostream &os, const bfloat16 &v) { std::stringstream str; - str << v; + str << static_cast(v); os << str.str(); return os; } -- cgit v1.2.1