diff options
author | Jeremy Johnson <jeremy.johnson@arm.com> | 2024-02-01 15:54:07 +0000 |
---|---|---|
committer | Jeremy Johnson <jeremy.johnson@arm.com> | 2024-02-08 11:14:04 +0000 |
commit | 32d0b5af61d978d9932ac5576b42203e57881168 (patch) | |
tree | eb9f3d6d845edc0f53da06285db1e4736282b4c3 /reference_model | |
parent | 6f57e6e665094959aed40c0e388ac81fbd118720 (diff) | |
download | reference_model-32d0b5af61d978d9932ac5576b42203e57881168.tar.gz |
Main Compliance: Add RESIZE support
Add RELATIVE verify mode for RESIZE.
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: I4fe352579507211dae7a048bf080c24426ce42a2
Diffstat (limited to 'reference_model')
-rw-r--r-- | reference_model/CMakeLists.txt | 2 | ||||
-rw-r--r-- | reference_model/src/generate/generate_utils.cc | 5 | ||||
-rw-r--r-- | reference_model/src/verify/verifiers.h | 11 | ||||
-rw-r--r-- | reference_model/src/verify/verify_entry.cc | 3 | ||||
-rw-r--r-- | reference_model/src/verify/verify_relative.cc | 83 | ||||
-rw-r--r-- | reference_model/src/verify/verify_utils.cc | 11 | ||||
-rw-r--r-- | reference_model/src/verify/verify_utils.h | 13 | ||||
-rw-r--r-- | reference_model/test/verify_tests.cpp | 84 |
8 files changed, 208 insertions, 4 deletions
diff --git a/reference_model/CMakeLists.txt b/reference_model/CMakeLists.txt index d2dce3c..0f806fc 100644 --- a/reference_model/CMakeLists.txt +++ b/reference_model/CMakeLists.txt @@ -81,6 +81,7 @@ set(CXX_SOURCE src/verify/verify_entry.cc src/verify/verify_exact.cc src/verify/verify_reduce_product.cc + src/verify/verify_relative.cc src/verify/verify_ulp.cc src/verify/verify_utils.cc src/ops/op_factory.cc @@ -157,6 +158,7 @@ add_library(tosa_reference_verify_lib SHARED src/verify/verify_entry.cc src/verify/verify_exact.cc src/verify/verify_reduce_product.cc + src/verify/verify_relative.cc src/verify/verify_ulp.cc src/verify/verify_utils.cc src/verify/verify_config.cc diff --git a/reference_model/src/generate/generate_utils.cc b/reference_model/src/generate/generate_utils.cc index cf5308b..8b16e97 100644 --- a/reference_model/src/generate/generate_utils.cc +++ b/reference_model/src/generate/generate_utils.cc @@ -71,14 +71,15 @@ NLOHMANN_JSON_SERIALIZE_ENUM(Op, { Op::Op_PAD, "PAD" }, { Op::Op_POW, "POW" }, { Op::Op_RECIPROCAL, "RECIPROCAL" }, - { Op::Op_RESHAPE, "RESHAPE" }, - { Op::Op_RSQRT, "RSQRT" }, { Op::Op_REDUCE_MAX, "REDUCE_MAX" }, { Op::Op_REDUCE_MIN, "REDUCE_MIN" }, { Op::Op_REDUCE_PRODUCT, "REDUCE_PRODUCT" }, { Op::Op_REDUCE_SUM, "REDUCE_SUM" }, + { Op::Op_RESHAPE, "RESHAPE" }, + { Op::Op_RESIZE, "RESIZE" }, { Op::Op_REVERSE, "REVERSE" }, { Op::Op_RFFT2D, "RFFT2D" }, + { Op::Op_RSQRT, "RSQRT" }, { Op::Op_SCATTER, "SCATTER" }, { Op::Op_SELECT, "SELECT" }, { Op::Op_SIGMOID, "SIGMOID" }, diff --git a/reference_model/src/verify/verifiers.h b/reference_model/src/verify/verifiers.h index 6830115..80b6e19 100644 --- a/reference_model/src/verify/verifiers.h +++ b/reference_model/src/verify/verifiers.h @@ -71,6 +71,17 @@ bool verifyULP(const CTensor* referenceTensor, const CTensor* implementationTens /// \return True if compliant else false bool verifyAbsError(const CTensor* ref, const CTensor* refBnd, const CTensor* imp, const AbsErrorVerifyInfo& aeInfo); +/// \brief Perform relative result verification +/// +/// \param referenceTensor Reference tensor +/// \param implementationTensor Implementation resulting tensor +/// \param rInfo Relative verification meta-data +/// +/// \return True if compliant else false +bool verifyRelative(const CTensor* referenceTensor, + const CTensor* implementationTensor, + const RelativeVerifyInfo& rInfo); + }; // namespace TosaReference #endif // VERIFIERS_H_ diff --git a/reference_model/src/verify/verify_entry.cc b/reference_model/src/verify/verify_entry.cc index afc5916..9702c36 100644 --- a/reference_model/src/verify/verify_entry.cc +++ b/reference_model/src/verify/verify_entry.cc @@ -43,6 +43,9 @@ bool verify(const CTensor* ref, const CTensor* refBnd, const CTensor* imp, const case VerifyMode::AbsError: { return verifyAbsError(ref, refBnd, imp, cfg.absErrorInfo); } + case VerifyMode::Relative: { + return verifyRelative(ref, imp, cfg.relativeInfo); + } default: { WARNING("[Verifier] Unsupported verification mode."); break; diff --git a/reference_model/src/verify/verify_relative.cc b/reference_model/src/verify/verify_relative.cc new file mode 100644 index 0000000..b12daf7 --- /dev/null +++ b/reference_model/src/verify/verify_relative.cc @@ -0,0 +1,83 @@ + +// Copyright (c) 2024, ARM Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cmath> +#include <vector> + +#include "verifiers.h" +#include "verify/verify_utils.h" + +namespace TosaReference +{ + +namespace +{ +template <typename OutDtype> +bool validateData(const double* ref, + const OutDtype* imp, + const std::vector<int32_t>& shape, + const RelativeVerifyInfo& cfg) +{ + const size_t T = static_cast<size_t>(numElements(shape)); + TOSA_REF_REQUIRE(T > 0, "[R] Invalid shape for reference tensor"); + + double errBound = cfg.max * cfg.scale; + for (size_t i = 0; i < T; ++i) + { + bool valid = tosaCheckFloatBound(imp[i], ref[i], errBound); + if (!valid) + { + auto pos = indexToPosition(i, shape); + WARNING("[Verifier][RP] Location %s", positionToString(pos).c_str()); + return false; + } + } + return true; +} +} // namespace + +bool verifyRelative(const CTensor* referenceTensor, + const CTensor* implementationTensor, + const RelativeVerifyInfo& rInfo) +{ + // Validate that tensors are provided + TOSA_REF_REQUIRE(referenceTensor != nullptr, "[R] Reference tensor is missing"); + TOSA_REF_REQUIRE(implementationTensor != nullptr, "[R] Implementation tensor is missing"); + + const std::vector<int32_t> refShape(referenceTensor->shape, referenceTensor->shape + referenceTensor->num_dims); + + const double* refData = reinterpret_cast<const double*>(referenceTensor->data); + TOSA_REF_REQUIRE(refData != nullptr, "[R] Missing data for reference"); + + switch (implementationTensor->data_type) + { + case tosa_datatype_fp32_t: { + const auto* impData = reinterpret_cast<const float*>(implementationTensor->data); + TOSA_REF_REQUIRE(impData != nullptr, "[R] Missing data for implementation"); + return validateData(refData, impData, refShape, rInfo); + } + case tosa_datatype_fp16_t: { + const auto* impData = reinterpret_cast<const half_float::half*>(implementationTensor->data); + TOSA_REF_REQUIRE(impData != nullptr, "[R] Missing data for implementation"); + return validateData(refData, impData, refShape, rInfo); + } + default: + WARNING("[Verifier][R] Data-type not supported."); + break; + } + + return false; +} +} // namespace TosaReference diff --git a/reference_model/src/verify/verify_utils.cc b/reference_model/src/verify/verify_utils.cc index abb55eb..14bc6f1 100644 --- a/reference_model/src/verify/verify_utils.cc +++ b/reference_model/src/verify/verify_utils.cc @@ -52,6 +52,7 @@ NLOHMANN_JSON_SERIALIZE_ENUM(VerifyMode, { VerifyMode::FpSpecial, "FP_SPECIAL" }, { VerifyMode::ReduceProduct, "REDUCE_PRODUCT" }, { VerifyMode::AbsError, "ABS_ERROR" }, + { VerifyMode::Relative, "RELATIVE" }, }) void from_json(const nlohmann::json& j, UlpVerifyInfo& ulpInfo) @@ -78,6 +79,12 @@ void from_json(const nlohmann::json& j, AbsErrorVerifyInfo& absErrorInfo) } } +void from_json(const nlohmann::json& j, RelativeVerifyInfo& rInfo) +{ + j.at("max").get_to(rInfo.max); + j.at("scale").get_to(rInfo.scale); +} + void from_json(const nlohmann::json& j, VerifyConfig& cfg) { j.at("mode").get_to(cfg.mode); @@ -100,6 +107,10 @@ void from_json(const nlohmann::json& j, VerifyConfig& cfg) { j.at("abs_error_info").get_to(cfg.absErrorInfo); } + if (j.contains("relative_info")) + { + j.at("relative_info").get_to(cfg.relativeInfo); + } } std::optional<VerifyConfig> parseVerifyConfig(const char* tensorName, const char* json) diff --git a/reference_model/src/verify/verify_utils.h b/reference_model/src/verify/verify_utils.h index 0fc68fb..341bd90 100644 --- a/reference_model/src/verify/verify_utils.h +++ b/reference_model/src/verify/verify_utils.h @@ -46,7 +46,8 @@ enum class VerifyMode DotProduct, FpSpecial, ReduceProduct, - AbsError + AbsError, + Relative }; /// \brief ULP verification meta-data @@ -83,6 +84,15 @@ struct AbsErrorVerifyInfo double lowerBound; }; +/// \brief relative verification meta-data +struct RelativeVerifyInfo +{ + RelativeVerifyInfo() = default; + + double max; + double scale; +}; + /// \brief Verification meta-data struct VerifyConfig { @@ -94,6 +104,7 @@ struct VerifyConfig DotProductVerifyInfo dotProductInfo; ReduceProductVerifyInfo reduceProductInfo; AbsErrorVerifyInfo absErrorInfo; + RelativeVerifyInfo relativeInfo; }; /// \brief Parse the verification config for a tensor when given in JSON form diff --git a/reference_model/test/verify_tests.cpp b/reference_model/test/verify_tests.cpp index ba18af1..a85546e 100644 --- a/reference_model/test/verify_tests.cpp +++ b/reference_model/test/verify_tests.cpp @@ -29,6 +29,16 @@ namespace { +void update_json_template(std::string& str, const std::string& find, const std::string& change) +{ + // Update the 'str' by looking for instances of 'find' and replacing them with 'change' + auto pos = str.find(find); + while (pos != std::string::npos) + { + str.replace(pos, find.length(), change); + pos = str.find(find); + } +} class TosaTensor { @@ -472,7 +482,7 @@ TEST_CASE("positive - abs error") SUBCASE("outside") { - // Generate some data that exceeds a specified number of ULP for each value in the tensor. + // Generate some data that exceeds a requirements for each value in the tensor. auto otherData_fp32 = data_fp32; std::for_each(std::begin(otherData_fp32), std::end(otherData_fp32), [outsideErrBound](auto& value) { if (std::abs(value) != 0.0 && !std::isinf(value) && !std::isnan(value)) @@ -489,4 +499,76 @@ TEST_CASE("positive - abs error") jsonCfg.c_str())); } } + +TEST_CASE("positive - relative") +{ + std::string templateJsonCfg = R"({ + "tensors" : { + "out1" : { + "mode": "RELATIVE", + "data_type": "FP32", + "relative_info": { + "max": _MAXIMUM_, + "scale": _SCALE_ + } + } + } + })"; + + const auto shape = std::vector<int32_t>{ 3, 3, 3 }; + const auto elementCount = std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies<>()); + + // Generate some random floats using the full range of fp32. + auto data_fp32 = generateRandomTensorData<float>(elementCount, true); + std::vector<double> data_fp64(data_fp32.begin(), data_fp32.end()); + + float scale = 0.0006; + float max = 0.0; + std::for_each(std::begin(data_fp32), std::end(data_fp32), [&max](auto& value) { + if (!std::isinf(value) && !std::isnan(value)) + { + max = std::max(max, std::abs(value)); + } + }); + std::string jsonCfg = templateJsonCfg; + update_json_template(jsonCfg, "_MAXIMUM_", std::to_string(max)); + update_json_template(jsonCfg, "_SCALE_", std::to_string(scale)); + + float errBound = max * scale; + // Use 10% error margin to test due to using v.large values in our random data + float insideErrBound = errBound * 0.9; + float outsideErrBound = errBound * 1.1; + + SUBCASE("inside") + { + // Generate some data that meets the requirements of the result. + auto otherData_fp32 = data_fp32; + std::for_each(std::begin(otherData_fp32), std::end(otherData_fp32), [insideErrBound](auto& value) { + if (std::abs(value) != 0.0 && !std::isinf(value) && !std::isnan(value)) + value += insideErrBound; + }); + const auto referenceTensor = + TosaTensor("out1", tosa_datatype_fp64_t, shape, reinterpret_cast<uint8_t*>(data_fp64.data())); + const auto implementationTensor = + TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast<uint8_t*>(otherData_fp32.data())); + REQUIRE(tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str())); + } + + SUBCASE("outside") + { + // Generate some data that exceeds the requirements for each value in the tensor. + auto otherData_fp32 = data_fp32; + std::for_each(std::begin(otherData_fp32), std::end(otherData_fp32), [outsideErrBound](auto& value) { + if (std::abs(value) != 0.0 && !std::isinf(value) && !std::isnan(value)) + value += outsideErrBound; + }); + + const auto referenceTensor = + TosaTensor("out1", tosa_datatype_fp64_t, shape, reinterpret_cast<uint8_t*>(data_fp64.data())); + const auto implementationTensor = + TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast<uint8_t*>(otherData_fp32.data())); + REQUIRE_FALSE( + tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), jsonCfg.c_str())); + } +} TEST_SUITE_END(); // verify |