From 44827be1377e2ba635599596b903b47ac99764ac Mon Sep 17 00:00:00 2001 From: Jerry Ge Date: Fri, 15 Dec 2023 00:05:37 +0000 Subject: Fix Cast FP32 to Int32 Overflow - With input of 2147483648.00, the output overflows to -2147483648 - The root cause is the following: - std::rint still returns float, the existing implementation is forcing a cast from that float to int32_t - when the input is over INT32_MAX, the output right after rint will overflow which casues the clipplings later to be ineffective - Instead, perform the range check before rint Signed-off-by: Jerry Ge Change-Id: Ib5a8cfd98aea17e326f8b11097beeb2d2b3efac9 --- reference_model/src/ops/type_conversion.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/reference_model/src/ops/type_conversion.cc b/reference_model/src/ops/type_conversion.cc index 0135d1b..17abaf7 100644 --- a/reference_model/src/ops/type_conversion.cc +++ b/reference_model/src/ops/type_conversion.cc @@ -506,9 +506,13 @@ CastHelper::CastHelper() { // fp32 data converted to integer fcn = [](float in) -> OutEigenType { + if (in >= float(OutMax)) + return OutMax; + + if (in <= float(OutMin)) + return OutMin; + OutEigenType out = std::rint(in); - out = std::max(out, OutMin); - out = std::min(out, OutMax); return out; }; } -- cgit v1.2.1