From 750d27da447205ce82056d8b6b473e7a6ec051c6 Mon Sep 17 00:00:00 2001 From: Eric Kunze Date: Thu, 30 Jun 2022 21:37:09 +0000 Subject: Modify input limits for apply_scale_32 Aligns with change in specification, as the limit to value being between (-1 << shift-2) and (1 << shift-2) was overly constraining. Signed-off-by: Eric Kunze Change-Id: I72cc5743344d4036920c3df366e0a3930701a0cb --- reference_model/src/quant_util.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'reference_model') diff --git a/reference_model/src/quant_util.h b/reference_model/src/quant_util.h index 3b7674d..2e5c2e5 100644 --- a/reference_model/src/quant_util.h +++ b/reference_model/src/quant_util.h @@ -55,14 +55,13 @@ public: "apply_scale_32(): shift value should stay within [2, 62] but is " + std::to_string(shift); throw desc; } - int64_t low_val = -1L << (shift-2); - int64_t high_val = 1L << (shift-2); + int64_t low_val = -1L << (shift - 1); + int64_t high_val = 1L << (shift - 1); if (value < low_val || value >= high_val) { - std::string desc = - "apply_scale_32(): value should stay within [" + - std::to_string(low_val) + ", " + std::to_string(high_val-1) + - "] but is " + std::to_string(value) + " using shift of " + std::to_string(shift); + std::string desc = "apply_scale_32(): value should stay within [" + std::to_string(low_val) + ", " + + std::to_string(high_val - 1) + "] but is " + std::to_string(value) + " using shift of " + + std::to_string(shift); throw desc; } int64_t round = 1L << (shift - 1); -- cgit v1.2.1