From 08965d35f728d93d8b215753b4b270a422fe39c9 Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Mon, 19 Feb 2024 13:57:21 +0000 Subject: Verifier - change to output largest error deviance Add general validateData function used by ABS_ERROR, ULP, RELATIVE and REDUCE_PRODUCT to find and output largest deviance from the error bounds. Clean up naming inconsistencies bewteen verify modes. Signed-off-by: Jeremy Johnson Change-Id: Ib903faf36f784cacae91edab61d8e489461a727c --- reference_model/src/verify/verifiers.h | 49 ++++++------ reference_model/src/verify/verify_abs_error.cc | 61 +++++++-------- reference_model/src/verify/verify_dot_product.cc | 41 +++++----- .../src/verify/verify_reduce_product.cc | 33 +++----- reference_model/src/verify/verify_relative.cc | 30 +++---- reference_model/src/verify/verify_ulp.cc | 60 ++++---------- reference_model/src/verify/verify_utils.cc | 91 ++++++++++++++++++++-- reference_model/src/verify/verify_utils.h | 39 +++++++++- 8 files changed, 230 insertions(+), 174 deletions(-) diff --git a/reference_model/src/verify/verifiers.h b/reference_model/src/verify/verifiers.h index 80b6e19..e5f9df1 100644 --- a/reference_model/src/verify/verifiers.h +++ b/reference_model/src/verify/verifiers.h @@ -22,30 +22,30 @@ namespace TosaReference { /// \brief Perform dot-product based verification /// -/// \param ref Reference tensor -/// \param refBnd Reference tensor when ran on abs(input) -/// \param imp Implementation resulting tensor -/// \param dpInfo Dot-product verification meta-data +/// \param referenceTensor Reference tensor +/// \param boundsTensor Reference tensor when ran on abs(input) +/// \param implementationTensor Implementation resulting tensor +/// \param dpInfo Dot-product verification meta-data /// /// \return True if compliant else false -bool verifyDotProduct(const CTensor* ref, - const CTensor* refBnd, - const CTensor* imp, +bool verifyDotProduct(const CTensor* referenceTensor, + const CTensor* boundsTensor, + const CTensor* implementationTensor, const DotProductVerifyInfo& dpInfo); /// \brief Perform exact result verification /// -/// \param referenceTensor Reference tensor -/// \param implementationTensor Implementation resulting tensor +/// \param referenceTensor Reference tensor +/// \param implementationTensor Implementation resulting tensor /// /// \return True if compliant else false bool verifyExact(const CTensor* referenceTensor, const CTensor* implementationTensor); /// \brief Perform reduce product result verification /// -/// \param referenceTensor Reference tensor -/// \param implementationTensor Implementation resulting tensor -/// \param rpInfo Reduce-product verification meta-data +/// \param referenceTensor Reference tensor +/// \param implementationTensor Implementation resulting tensor +/// \param rpInfo Reduce-product verification meta-data /// /// \return True if compliant else false bool verifyReduceProduct(const CTensor* referenceTensor, @@ -54,28 +54,31 @@ bool verifyReduceProduct(const CTensor* referenceTensor, /// \brief Perform ULP result verification /// -/// \param referenceTensor Reference tensor -/// \param implementationTensor Implementation resulting tensor -/// \param ulpInfo The ULP tolerence info for the comparison of the two tensors +/// \param referenceTensor Reference tensor +/// \param implementationTensor Implementation resulting tensor +/// \param ulpInfo ULP tolerence info for the comparison /// /// \return True if compliant else false bool verifyULP(const CTensor* referenceTensor, const CTensor* implementationTensor, const UlpVerifyInfo& ulpInfo); /// \brief Perform abs-error based verification /// -/// \param ref Reference tensor -/// \param refBnd Reference bounds tensor (according to op) -/// \param imp Implementation resulting tensor -/// \param aeInfo Abs-error verification meta-data +/// \param referenceTensor Reference tensor +/// \param boundsTensor Reference bounds tensor (according to op) +/// \param implementationTensor Implementation resulting tensor +/// \param aeInfo Abs-error verification meta-data /// /// \return True if compliant else false -bool verifyAbsError(const CTensor* ref, const CTensor* refBnd, const CTensor* imp, const AbsErrorVerifyInfo& aeInfo); +bool verifyAbsError(const CTensor* referenceTensor, + const CTensor* boundsTensor, + const CTensor* implementationTensor, + const AbsErrorVerifyInfo& aeInfo); /// \brief Perform relative result verification /// -/// \param referenceTensor Reference tensor -/// \param implementationTensor Implementation resulting tensor -/// \param rInfo Relative verification meta-data +/// \param referenceTensor Reference tensor +/// \param implementationTensor Implementation resulting tensor +/// \param rInfo Relative verification meta-data /// /// \return True if compliant else false bool verifyRelative(const CTensor* referenceTensor, diff --git a/reference_model/src/verify/verify_abs_error.cc b/reference_model/src/verify/verify_abs_error.cc index 5005dcf..a7b7bc2 100644 --- a/reference_model/src/verify/verify_abs_error.cc +++ b/reference_model/src/verify/verify_abs_error.cc @@ -26,59 +26,50 @@ namespace TosaReference namespace { -template -bool validateData(const double* ref, - const double* bnd, - const OutDtype* imp, - const std::vector& shape, - const AbsErrorVerifyInfo& cfg) +template +double calcErrorBound(double referenceValue, double boundsValue, const void* cfgPtr) { - const size_t T = static_cast(numElements(shape)); - TOSA_REF_REQUIRE(T > 0, "[AE] Invalid shape for reference tensor"); + const auto cfg = reinterpret_cast(cfgPtr); - for (size_t i = 0; i < T; ++i) + double valBound = std::abs(referenceValue) * boundsValue; + if (cfg->lowerBound > 0) { - double valBound = std::abs(ref[i]) * bnd[i]; - if (cfg.lowerBound > 0) - { - valBound = std::max(cfg.lowerBound, valBound); - } - double errBound = exp2(-AccPrecision::normal_frac) * valBound; - bool valid = tosaCheckFloatBound(imp[i], ref[i], errBound); - if (!valid) - { - auto pos = indexToPosition(i, shape); - WARNING("[Verifier][AE] Location %s", positionToString(pos).c_str()); - return false; - } + valBound = std::max(cfg->lowerBound, valBound); } - return true; + return exp2(-AccPrecision::normal_frac) * valBound; } } // namespace -bool verifyAbsError(const CTensor* ref, const CTensor* refBnd, const CTensor* imp, const AbsErrorVerifyInfo& aeInfo) + +bool verifyAbsError(const CTensor* referenceTensor, + const CTensor* boundsTensor, + const CTensor* implementationTensor, + const AbsErrorVerifyInfo& aeInfo) { // Validate that tensors are provided - TOSA_REF_REQUIRE(ref != nullptr, "[AE] Reference tensor is missing"); - TOSA_REF_REQUIRE(refBnd != nullptr, "[AE] Reference bounds tensor is missing"); - TOSA_REF_REQUIRE(imp != nullptr, "[AE] Implementation tensor is missing"); + TOSA_REF_REQUIRE(referenceTensor != nullptr, "[AE] Reference tensor is missing"); + TOSA_REF_REQUIRE(boundsTensor != nullptr, "[AE] Reference bounds tensor is missing"); + TOSA_REF_REQUIRE(implementationTensor != nullptr, "[AE] Implementation tensor is missing"); - const std::vector refShape(ref->shape, ref->shape + ref->num_dims); + const std::vector refShape(referenceTensor->shape, referenceTensor->shape + referenceTensor->num_dims); - const double* refData = reinterpret_cast(ref->data); - const double* refBndData = reinterpret_cast(refBnd->data); + const double* refData = reinterpret_cast(referenceTensor->data); + const double* refBndData = reinterpret_cast(boundsTensor->data); TOSA_REF_REQUIRE(refData != nullptr && refBndData != nullptr, "[AE] Missing data for reference or bounds tensors"); - switch (imp->data_type) + const std::string modeStr = "AE"; + + switch (implementationTensor->data_type) { case tosa_datatype_fp32_t: { - const auto* impData = reinterpret_cast(imp->data); + const auto* impData = reinterpret_cast(implementationTensor->data); TOSA_REF_REQUIRE(impData != nullptr, "[AE] Missing data for implementation"); - return validateData(refData, refBndData, impData, refShape, aeInfo); + return validateData(refData, refBndData, impData, refShape, modeStr, &aeInfo, &calcErrorBound); } case tosa_datatype_fp16_t: { - const auto* impData = reinterpret_cast(imp->data); + const auto* impData = reinterpret_cast(implementationTensor->data); TOSA_REF_REQUIRE(impData != nullptr, "[AE] Missing data for implementation"); - return validateData(refData, refBndData, impData, refShape, aeInfo); + return validateData(refData, refBndData, impData, refShape, modeStr, &aeInfo, + &calcErrorBound); } default: WARNING("[Verifier][AE] Data-type not supported."); diff --git a/reference_model/src/verify/verify_dot_product.cc b/reference_model/src/verify/verify_dot_product.cc index ea50573..3f82c1e 100644 --- a/reference_model/src/verify/verify_dot_product.cc +++ b/reference_model/src/verify/verify_dot_product.cc @@ -66,13 +66,13 @@ std::optional validateElement(size_t index, double ref, double bnd, AccT return is_valid ? std::optional(err) : std::nullopt; } -// Generic data validation function +// Dot Product data validation function template -bool validateData(const double* ref, - const double* bnd, - const AccType* imp, - const std::vector& shape, - const DotProductVerifyInfo& cfg) +bool validateDataDP(const double* referenceData, + const double* boundsData, + const AccType* implementationData, + const std::vector& shape, + const DotProductVerifyInfo& cfg) { const size_t T = static_cast(numElements(shape)); TOSA_REF_REQUIRE(T > 0, "[DP] Invalid shape for reference tensor"); @@ -87,7 +87,7 @@ bool validateData(const double* ref, for (size_t i = 0; i < T; ++i) { - auto out_err = validateElement(i, ref[i], bnd[i], imp[i], KS); + auto out_err = validateElement(i, referenceData[i], boundsData[i], implementationData[i], KS); if (!out_err) { auto pos = indexToPosition(i, shape); @@ -113,31 +113,34 @@ bool validateData(const double* ref, } } // namespace -bool verifyDotProduct(const CTensor* ref, const CTensor* refBnd, const CTensor* imp, const DotProductVerifyInfo& dpInfo) +bool verifyDotProduct(const CTensor* referenceTensor, + const CTensor* boundsTensor, + const CTensor* implementationTensor, + const DotProductVerifyInfo& dpInfo) { // Validate that tensors are provided - TOSA_REF_REQUIRE(ref != nullptr, "[DP] Reference tensor is missing"); - TOSA_REF_REQUIRE(refBnd != nullptr, "[DP] Reference bounds tensor is missing"); - TOSA_REF_REQUIRE(imp != nullptr, "[DP] Implementation tensor is missing"); + TOSA_REF_REQUIRE(referenceTensor != nullptr, "[DP] Reference tensor is missing"); + TOSA_REF_REQUIRE(boundsTensor != nullptr, "[DP] Reference bounds tensor is missing"); + TOSA_REF_REQUIRE(implementationTensor != nullptr, "[DP] Implementation tensor is missing"); - const std::vector refShape(ref->shape, ref->shape + ref->num_dims); + const std::vector refShape(referenceTensor->shape, referenceTensor->shape + referenceTensor->num_dims); - const double* refData = reinterpret_cast(ref->data); - const double* refBndData = reinterpret_cast(refBnd->data); + const double* refData = reinterpret_cast(referenceTensor->data); + const double* refBndData = reinterpret_cast(boundsTensor->data); TOSA_REF_REQUIRE(refData != nullptr && refBndData != nullptr, "[DP] Missing data for reference or bounds tensors"); - switch (imp->data_type) + switch (implementationTensor->data_type) { case tosa_datatype_fp32_t: { - const float* impData = reinterpret_cast(imp->data); + const float* impData = reinterpret_cast(implementationTensor->data); TOSA_REF_REQUIRE(impData != nullptr, "[DP] Missing data for implementation"); - return validateData(refData, refBndData, impData, refShape, dpInfo); + return validateDataDP(refData, refBndData, impData, refShape, dpInfo); break; } case tosa_datatype_fp16_t: { - const half_float::half* impData = reinterpret_cast(imp->data); + const half_float::half* impData = reinterpret_cast(implementationTensor->data); TOSA_REF_REQUIRE(impData != nullptr, "[DP] Missing data for implementation"); - return validateData(refData, refBndData, impData, refShape, dpInfo); + return validateDataDP(refData, refBndData, impData, refShape, dpInfo); break; } default: { diff --git a/reference_model/src/verify/verify_reduce_product.cc b/reference_model/src/verify/verify_reduce_product.cc index 0e58892..a8aaa53 100644 --- a/reference_model/src/verify/verify_reduce_product.cc +++ b/reference_model/src/verify/verify_reduce_product.cc @@ -21,31 +21,15 @@ namespace TosaReference { - namespace { -template -bool validateData(const double* ref, - const OutDtype* imp, - const std::vector& shape, - const ReduceProductVerifyInfo& cfg) +template +double calcErrorBound(double referenceValue, double boundsValue, const void* cfgPtr) { - const size_t T = static_cast(numElements(shape)); - TOSA_REF_REQUIRE(T > 0, "[RP] Invalid shape for reference tensor"); + const auto cfg = reinterpret_cast(cfgPtr); + unused(boundsValue); - for (size_t i = 0; i < T; ++i) - { - double errBound = - std::abs(ref[i]) * (std::pow(1 + std::pow(2, -AccPrecision::normal_frac - 1), cfg.n) - 1); - bool valid = tosaCheckFloatBound(imp[i], ref[i], errBound); - if (!valid) - { - auto pos = indexToPosition(i, shape); - WARNING("[Verifier][RP] Location %s", positionToString(pos).c_str()); - return false; - } - } - return true; + return std::abs(referenceValue) * (std::pow(1 + std::pow(2, -AccPrecision::normal_frac - 1), cfg->n) - 1); } } // namespace @@ -62,17 +46,20 @@ bool verifyReduceProduct(const CTensor* referenceTensor, const double* refData = reinterpret_cast(referenceTensor->data); TOSA_REF_REQUIRE(refData != nullptr, "[RP] Missing data for reference"); + const std::string modeStr = "RP"; + switch (implementationTensor->data_type) { case tosa_datatype_fp32_t: { const auto* impData = reinterpret_cast(implementationTensor->data); TOSA_REF_REQUIRE(impData != nullptr, "[RP] Missing data for implementation"); - return validateData(refData, impData, refShape, rpInfo); + return validateData(refData, nullptr, impData, refShape, modeStr, &rpInfo, &calcErrorBound); } case tosa_datatype_fp16_t: { const auto* impData = reinterpret_cast(implementationTensor->data); TOSA_REF_REQUIRE(impData != nullptr, "[RP] Missing data for implementation"); - return validateData(refData, impData, refShape, rpInfo); + return validateData(refData, nullptr, impData, refShape, modeStr, &rpInfo, + &calcErrorBound); } default: WARNING("[Verifier][RP] Data-type not supported."); diff --git a/reference_model/src/verify/verify_relative.cc b/reference_model/src/verify/verify_relative.cc index b12daf7..5085fd9 100644 --- a/reference_model/src/verify/verify_relative.cc +++ b/reference_model/src/verify/verify_relative.cc @@ -24,27 +24,13 @@ namespace TosaReference namespace { -template -bool validateData(const double* ref, - const OutDtype* imp, - const std::vector& shape, - const RelativeVerifyInfo& cfg) +double calcErrorBound(double referenceValue, double boundsValue, const void* cfgPtr) { - const size_t T = static_cast(numElements(shape)); - TOSA_REF_REQUIRE(T > 0, "[R] Invalid shape for reference tensor"); + const auto cfg = reinterpret_cast(cfgPtr); + unused(referenceValue); + unused(boundsValue); - double errBound = cfg.max * cfg.scale; - for (size_t i = 0; i < T; ++i) - { - bool valid = tosaCheckFloatBound(imp[i], ref[i], errBound); - if (!valid) - { - auto pos = indexToPosition(i, shape); - WARNING("[Verifier][RP] Location %s", positionToString(pos).c_str()); - return false; - } - } - return true; + return cfg->max * cfg->scale; } } // namespace @@ -61,17 +47,19 @@ bool verifyRelative(const CTensor* referenceTensor, const double* refData = reinterpret_cast(referenceTensor->data); TOSA_REF_REQUIRE(refData != nullptr, "[R] Missing data for reference"); + const std::string modeStr = "R"; + switch (implementationTensor->data_type) { case tosa_datatype_fp32_t: { const auto* impData = reinterpret_cast(implementationTensor->data); TOSA_REF_REQUIRE(impData != nullptr, "[R] Missing data for implementation"); - return validateData(refData, impData, refShape, rInfo); + return validateData(refData, nullptr, impData, refShape, modeStr, &rInfo, &calcErrorBound); } case tosa_datatype_fp16_t: { const auto* impData = reinterpret_cast(implementationTensor->data); TOSA_REF_REQUIRE(impData != nullptr, "[R] Missing data for implementation"); - return validateData(refData, impData, refShape, rInfo); + return validateData(refData, nullptr, impData, refShape, modeStr, &rInfo, &calcErrorBound); } default: WARNING("[Verifier][R] Data-type not supported."); diff --git a/reference_model/src/verify/verify_ulp.cc b/reference_model/src/verify/verify_ulp.cc index 13bf0a9..8bae6e6 100644 --- a/reference_model/src/verify/verify_ulp.cc +++ b/reference_model/src/verify/verify_ulp.cc @@ -27,24 +27,27 @@ namespace TosaReference namespace { template -bool tosaCheckULP(OutType testValue, double referenceValue, double ulpNum) +double calcErrorBound(double referenceValue, double boundsValue, const void* cfgPtr) { - double errorBound = 0.0; + const auto cfg = reinterpret_cast(cfgPtr); + unused(boundsValue); + + double errBound = 0.0; if (std::isfinite(referenceValue) && std::abs(referenceValue) != 0.0) { // Find the exponent of the reference value. - int32_t referenceExponent = ilog2(std::abs(referenceValue)); + int32_t refExponent = ilog2(std::abs(referenceValue)); // Work out the values magnitude - by raising 2 to the power of the // exponent and taking the normalized minimum for denormal values - const double referencePower2 = std::max(exp2(referenceExponent), AccPrecision::normal_min); + const double refPower2 = std::max(exp2(refExponent), AccPrecision::normal_min); // Get the value of changing the last bit - by shifting the least significant bit to this magnitude // i.e. the ULP. - double ulpValue = referencePower2 * exp2(-AccPrecision::normal_frac); + double ulpValue = refPower2 * exp2(-AccPrecision::normal_frac); - errorBound = ulpValue * ulpNum; + errBound = ulpValue * cfg->ulp; } - return tosaCheckFloatBound(testValue, referenceValue, errorBound); + return errBound; } } // namespace @@ -54,56 +57,25 @@ bool verifyULP(const CTensor* referenceTensor, const CTensor* implementationTens TOSA_REF_REQUIRE(referenceTensor != nullptr, "[ULP] Reference tensor is missing"); TOSA_REF_REQUIRE(implementationTensor != nullptr, "[ULP] Implementation tensor is missing"); - // Get number of elements const std::vector refShape(referenceTensor->shape, referenceTensor->shape + referenceTensor->num_dims); - const auto elementCount = numElements(refShape); - TOSA_REF_REQUIRE(elementCount > 0, "[ULP] Invalid shape for reference tensor"); - const double ulp = ulpInfo.ulp; const auto* refData = reinterpret_cast(referenceTensor->data); TOSA_REF_REQUIRE(refData != nullptr, "[ULP] Missing data for reference"); - const auto* refDataEnd = std::next(refData, elementCount); + + const std::string modeStr = "ULP"; + switch (implementationTensor->data_type) { case tosa_datatype_fp32_t: { const auto* impData = reinterpret_cast(implementationTensor->data); TOSA_REF_REQUIRE(impData != nullptr, "[ULP] Missing data for implementation"); - // Use mismatch to get the location of the first unequal value - auto pair = std::mismatch(refData, refDataEnd, impData, std::next(impData, elementCount), - [ulp](const auto& referenceValue, const auto& implementationValue) { - return tosaCheckULP(implementationValue, referenceValue, ulp); - }); - if (std::get<0>(pair) == refDataEnd) - { - // No mismatch found - return true; - } - else - { - auto pos = indexToPosition(std::get<0>(pair) - refData, refShape); - WARNING("[Verfier][ULP] Location %s", positionToString(pos).c_str()); - return false; - } + return validateData(refData, nullptr, impData, refShape, modeStr, &ulpInfo, &calcErrorBound); } case tosa_datatype_fp16_t: { const auto* impData = reinterpret_cast(implementationTensor->data); TOSA_REF_REQUIRE(impData != nullptr, "[ULP] Missing data for implementation"); - // Use mismatch to get the location of the first unequal value - auto pair = std::mismatch(refData, refDataEnd, impData, std::next(impData, elementCount), - [ulp](const auto& referenceValue, const auto& implementationValue) { - return tosaCheckULP(implementationValue, referenceValue, ulp); - }); - if (std::get<0>(pair) == refDataEnd) - { - // No mismatch found - return true; - } - else - { - auto pos = indexToPosition(std::get<0>(pair) - refData, refShape); - WARNING("[Verfier][ULP] Location %s", positionToString(pos).c_str()); - return false; - } + return validateData(refData, nullptr, impData, refShape, modeStr, &ulpInfo, + &calcErrorBound); } default: WARNING("[Verifier][ULP] Data-type not supported."); diff --git a/reference_model/src/verify/verify_utils.cc b/reference_model/src/verify/verify_utils.cc index 4ae245b..57cb50a 100644 --- a/reference_model/src/verify/verify_utils.cc +++ b/reference_model/src/verify/verify_utils.cc @@ -20,6 +20,7 @@ #include #include #include +#include namespace tosa { @@ -232,16 +233,22 @@ static_assert(std::numeric_limits::is_iec559, "verification is invalid"); template -bool tosaCheckFloatBound(OutType testValue, double referenceValue, double errorBound) +bool tosaCheckFloatBound( + OutType testValue, double referenceValue, double errorBound, double& resultDifference, std::string& resultWarning) { // Both must be NaNs to be correct if (std::isnan(referenceValue) || std::isnan(testValue)) { if (std::isnan(referenceValue) && std::isnan(testValue)) { + resultDifference = 0.0; return true; } - WARNING("[Verifier][Bound] Non-matching NaN values - ref (%g) versus test (%g).", referenceValue, testValue); + char buff[200]; + snprintf(buff, 200, "Non-matching NaN values - ref (%g) versus test (%g).", referenceValue, + static_cast(testValue)); + resultWarning.assign(buff); + resultDifference = std::numeric_limits::quiet_NaN(); return false; } @@ -307,16 +314,86 @@ bool tosaCheckFloatBound(OutType testValue, double referenceValue, double errorB // And finally... Do the comparison. double testValue64 = static_cast(testValue); bool withinBound = testValue64 >= referenceMin && testValue64 <= referenceMax; + resultDifference = testValue64 - referenceValue; if (!withinBound) { - WARNING("[Verifier][Bound] value %.*g is not in error bound %.*g range (%.*g <= ref %.*g <= %.*g).", DBL_DIG, - testValue64, DBL_DIG, errorBound, DBL_DIG, referenceMin, DBL_DIG, referenceValue, DBL_DIG, - referenceMax); + char buff[300]; + snprintf(buff, 300, + "value %.*g has a difference of %.*g compared to an error bound of +/- %.*g (range: %.*g <= ref %.*g " + "<= %.*g).", + DBL_DIG, testValue64, DBL_DIG, resultDifference, DBL_DIG, errorBound, DBL_DIG, referenceMin, DBL_DIG, + referenceValue, DBL_DIG, referenceMax); + resultWarning.assign(buff); } return withinBound; } +template +bool validateData(const double* referenceData, + const double* boundsData, + const OutType* implementationData, + const std::vector& shape, + const std::string& modeStr, + const void* cfgPtr, + double (*calcErrorBound)(double referenceValue, double boundsValue, const void* cfgPtr)) +{ + const size_t T = static_cast(numElements(shape)); + TOSA_REF_REQUIRE(T > 0, "Invalid shape for reference tensor"); + TOSA_REF_REQUIRE(referenceData != nullptr, "Missing data for reference tensor"); + TOSA_REF_REQUIRE(implementationData != nullptr, "Missing data for implementation tensor"); + // NOTE: Bounds data tensor is allowed to be null as it may not be needed + TOSA_REF_REQUIRE(cfgPtr != nullptr, "Missing config for validation"); + TOSA_REF_REQUIRE(calcErrorBound != nullptr, "Missing error bound function validation"); + + std::string warning, worstWarning; + double difference, worstDifference = 0.0; + size_t worstPosition; + bool compliant = true; + + for (size_t i = 0; i < T; ++i) + { + double boundVal = (boundsData == nullptr) ? 0.0 : boundsData[i]; + double errBound = calcErrorBound(referenceData[i], boundVal, cfgPtr); + bool valid = tosaCheckFloatBound(implementationData[i], referenceData[i], errBound, difference, warning); + if (!valid) + { + compliant = false; + if (std::isnan(difference) || std::abs(difference) > std::abs(worstDifference)) + { + worstPosition = i; + worstDifference = difference; + worstWarning.assign(warning); + if (std::isnan(difference)) + { + // Worst case is difference in NaN + break; + } + } + } + } + if (!compliant) + { + auto pos = indexToPosition(worstPosition, shape); + WARNING("[Verifier][%s] Largest deviance at location %s: %s", modeStr.c_str(), positionToString(pos).c_str(), + worstWarning.c_str()); + } + return compliant; +} + // Instantiate the needed check functions -template bool tosaCheckFloatBound(float testValue, double referenceValue, double errorBound); -template bool tosaCheckFloatBound(half_float::half testValue, double referenceValue, double errorBound); +template bool validateData(const double* referenceData, + const double* boundsData, + const float* implementationData, + const std::vector& shape, + const std::string& modeStr, + const void* cfgPtr, + double (*calcErrorBound)(double referenceValue, double boundsValue, const void* cfgPtr)); +template bool validateData(const double* referenceData, + const double* boundsData, + const half_float::half* implementationData, + const std::vector& shape, + const std::string& modeStr, + const void* cfgPtr, + double (*calcErrorBound)(double referenceValue, double boundsValue, const void* cfgPtr)); + } // namespace TosaReference diff --git a/reference_model/src/verify/verify_utils.h b/reference_model/src/verify/verify_utils.h index 341bd90..0d7bf47 100644 --- a/reference_model/src/verify/verify_utils.h +++ b/reference_model/src/verify/verify_utils.h @@ -22,6 +22,7 @@ #include #include +#include #include #define TOSA_REF_REQUIRE(COND, MESSAGE, ...) \ @@ -164,9 +165,43 @@ struct AccPrecision static constexpr int32_t normal_frac = 7; }; -/// \brief Error bounds check for ULP and ABS_ERROR modes +/// \brief Single value error bounds check for ULP, ABS_ERROR and other compliance modes +/// +/// \param testValue Implementation value +/// \param referenceValue Reference value +/// \param errorBound Positive error bound value +/// \param resultDifference Return: Difference between reference value and implementation value +/// \param resultWarning Return: Warning message if implementation is outside error bounds +/// +/// \return True if compliant else false template -bool tosaCheckFloatBound(OutType testValue, double referenceValue, double errorBound); +bool tosaCheckFloatBound( + OutType testValue, double referenceValue, double errorBound, double& resultDifference, std::string& resultWarning); + +/// \brief Whole tensor checker for values inside error bounds +/// +/// \param referenceData Reference output tensor data +/// \param boundsData Optional reference bounds tensor data +/// \param implementationData Implementation output tensor data +/// \param shape Tensor shape - all tensors must be this shape +/// \param modeStr Short string indicating which compliance mode we are testing +/// \param cfgPtr Pointer to this mode's configuration data, passed to the calcErrorBound() +/// \param calcErrorBound Pointer to a function that can calculate the error bound per ref value +/// +/// \return True if compliant else false +template +bool validateData(const double* referenceData, + const double* boundsData, + const OutType* implementationData, + const std::vector& shape, + const std::string& modeStr, + const void* cfgPtr, + double (*calcErrorBound)(double referenceValue, double boundsValue, const void* cfgPtr)); + +// Unused arguments helper function +template +inline void unused(Args&&...) +{} }; // namespace TosaReference #endif // VERIFY_UTILS_H_ -- cgit v1.2.1