From d1a08ce27ef8d0f6cf77e1b864610aade06edc5c Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Wed, 18 Oct 2023 17:22:21 +0100 Subject: Compliance mode testing for CONV2D Added CONV2D data generation. Updated verify dot product check to latest specification. Updated test generator and python datagenerator library to create const files during test generation. Add support for compliance test sets to conformance test_select. Signed-off-by: Jeremy Johnson Change-Id: I5be3b761a1e3ef259c058e493877cd5a89d5778b --- reference_model/src/verify/verify_dot_product.cc | 52 ++++++++++++------------ reference_model/src/verify/verify_utils.cc | 7 ++++ reference_model/src/verify/verify_utils.h | 36 +++++++++++++++- 3 files changed, 68 insertions(+), 27 deletions(-) (limited to 'reference_model/src/verify') diff --git a/reference_model/src/verify/verify_dot_product.cc b/reference_model/src/verify/verify_dot_product.cc index 2a1d273..233c072 100644 --- a/reference_model/src/verify/verify_dot_product.cc +++ b/reference_model/src/verify/verify_dot_product.cc @@ -14,6 +14,7 @@ #include "func_debug.h" #include "verifiers.h" +#include "verify_utils.h" #include #include @@ -24,22 +25,9 @@ namespace TosaReference { namespace { - -// Accumulator precision -template -struct AccPrecision; -#define two_m42 1.0 / (double)(((int64_t)1) << 42) // 2^-42 -template <> -struct AccPrecision -{ - static constexpr double precision = (double)(1 << 24); - static constexpr double min_normal = two_m42 * two_m42 * two_m42; // 2^-126 -}; -#undef two_m42 - // Generic element validation function template , int> = 0> -std::optional validateElement(double ref, double bnd, AccType imp, size_t KS) +std::optional validateElement(size_t index, double ref, double bnd, AccType imp, size_t KS) { double err = 0.0; bool is_valid = true; @@ -47,7 +35,11 @@ std::optional validateElement(double ref, double bnd, AccType imp, size_ if (bnd == 0.0) { is_valid = (ref == 0.0) && (imp == 0.0); - err = 0.0; + if (!is_valid) + { + WARNING("[Verifier][DP] index %d - bound is zero, but ref (%g) or imp (%f) is not.", index, ref, imp); + } + err = 0.0; } else if (std::isinf(static_cast(bnd))) { @@ -58,11 +50,15 @@ std::optional validateElement(double ref, double bnd, AccType imp, size_ else { // 0.0 < bnd < infinity - const double bnd_norm = std::max(bnd, AccPrecision::min_normal); - const double imp_fp64 = static_cast(imp); - const double acc_prec_fp64 = AccPrecision::precision; - err = (imp_fp64 - ref) * acc_prec_fp64 / bnd_norm; - is_valid = std::abs(err) <= KS; + const double out_err_bnd = + std::max(bnd * exp2(-1 - AccPrecision::normal_frac), AccPrecision::normal_min); + const double imp_fp64 = static_cast(imp); + err = (imp_fp64 - ref) / out_err_bnd; + is_valid = std::abs(err) <= KS; + if (!is_valid) + { + WARNING("[Verifier][DP] index %d - out_err (%g) is not within KS (%d).", index, err, KS); + } } return is_valid ? std::optional(err) : std::nullopt; @@ -73,7 +69,8 @@ template 0) ? (KS + 1) : KS + // NOTE: KS in the compliance config MUST have already been updated to (KS + 1) if the bias + // tensor is non-zero const int32_t KS = cfg.ks; double out_err_sum = 0.0; @@ -81,7 +78,7 @@ bool validateData(const double* ref, const double* bnd, const AccType* imp, size for (size_t i = 0; i < T; ++i) { - auto out_err = validateElement(ref[i], bnd[i], imp[i], KS); + auto out_err = validateElement(i, ref[i], bnd[i], imp[i], KS); TOSA_REF_REQUIRE(out_err, "[DP] Data required to be zero or error within range"); out_err_sum += out_err.value(); out_err_sumsq += out_err.value() * out_err.value(); @@ -89,11 +86,16 @@ bool validateData(const double* ref, const double* bnd, const AccType* imp, size if (S >= 3 && S <= 5) { + const double max_bias = 2 * sqrt(KS * T); + out_err_sum = std::abs(out_err_sum); // Check error bias magnitude for data sets S which are not positive biased - TOSA_REF_REQUIRE(std::abs(out_err_sum) <= 2 * sqrt(KS * T), "[DP] Bias magnitude is out of range"); + TOSA_REF_REQUIRE(out_err_sum <= max_bias, "[DP] Bias magnitude (%g) is out of range (%g)", out_err_sum, + max_bias); } // Check error variance magnitude - TOSA_REF_REQUIRE(out_err_sumsq <= 0.4 * KS * T, "[DP] Error variance magnitude is out of range"); + const double max_error = 0.4 * KS * T; + TOSA_REF_REQUIRE(out_err_sumsq <= max_error, "[DP] Error variance magnitude (%g) is out of range (%g)", + out_err_sumsq, max_error); return true; } } // namespace @@ -107,7 +109,7 @@ bool verifyDotProduct(const CTensor* ref, const CTensor* refBnd, const CTensor* // Get number of dot-product elements const int64_t T = numElements(std::vector(ref->shape, ref->shape + ref->num_dims)); - TOSA_REF_REQUIRE(T > 0, "invalid shape for reference tensor"); + TOSA_REF_REQUIRE(T > 0, "[DP] Invalid shape for reference tensor"); const double* refData = reinterpret_cast(ref->data); const double* refBndData = reinterpret_cast(refBnd->data); diff --git a/reference_model/src/verify/verify_utils.cc b/reference_model/src/verify/verify_utils.cc index ee11c41..43ecbe7 100644 --- a/reference_model/src/verify/verify_utils.cc +++ b/reference_model/src/verify/verify_utils.cc @@ -140,4 +140,11 @@ DType mapToDType(tosa_datatype_t dataType) return DType_UNKNOWN; } + +// Like const_exp2 but for use during runtime +double exp2(int32_t n) +{ + TOSA_REF_REQUIRE(-1022 <= n && n <= 1023, " Invalid exponent value (%d)", n); + return const_exp2(n); +} } // namespace TosaReference diff --git a/reference_model/src/verify/verify_utils.h b/reference_model/src/verify/verify_utils.h index bbe4b4e..486ce19 100644 --- a/reference_model/src/verify/verify_utils.h +++ b/reference_model/src/verify/verify_utils.h @@ -23,10 +23,10 @@ #include #include -#define TOSA_REF_REQUIRE(COND, MESSAGE) \ +#define TOSA_REF_REQUIRE(COND, MESSAGE, ...) \ if (!(COND)) \ { \ - WARNING("[Verifier]" MESSAGE "."); \ + WARNING("[Verifier]" MESSAGE ".", ##__VA_ARGS__); \ return false; \ } @@ -95,6 +95,38 @@ int64_t numElements(const std::vector& shape); /// \brief Map API data-type to DType DType mapToDType(tosa_datatype_t dataType); +/// \brief Raise a value by the power of N or -N +// For use during compile time - as no range check +constexpr double const_exp2(int32_t n) +{ + double v = 1.0; + while (n > 0) + { + v = v * 2.0; + n--; + } + while (n < 0) + { + v = v / 2.0; + n++; + } + return v; +} + +/// \brief Same as const_exp2 but with runtime range check of N +double exp2(int32_t n); + +/// \brief Accuracy precision information +template +struct AccPrecision; +template <> +struct AccPrecision +{ + static constexpr double normal_min = const_exp2(-126); + static constexpr double normal_max = const_exp2(128) - const_exp2(127 - 23); + static constexpr int32_t normal_frac = 23; +}; + }; // namespace TosaReference #endif // VERIFY_UTILS_H_ -- cgit v1.2.1