diff options
author | Jack Frankland <jack.frankland@arm.com> | 2023-09-13 11:03:50 +0100 |
---|---|---|
committer | Eric Kunze <eric.kunze@arm.com> | 2023-09-28 18:28:07 +0000 |
commit | aafc8508398180835781605dd9229d4a6087af1f (patch) | |
tree | 57a65c17c83ff1b4eb8d07446e3abd0537e4d947 /reference_model | |
parent | f0348ea4206a7e02497515ffb6d88546e0121cc7 (diff) | |
download | reference_model-aafc8508398180835781605dd9229d4a6087af1f.tar.gz |
feat: Add exact verifier
Add a verifier to check that tensor results match exactly.
Add a unit test to check the behavior of this new verifier.
Change-Id: I9b80a6d57640fec67c6be80a97b3af484aeb935e
Signed-off-by: Jack Frankland <jack.frankland@arm.com>
Diffstat (limited to 'reference_model')
-rw-r--r-- | reference_model/CMakeLists.txt | 4 | ||||
-rw-r--r-- | reference_model/src/verify/verifiers.h | 8 | ||||
-rw-r--r-- | reference_model/src/verify/verify_dot_product.cc | 9 | ||||
-rw-r--r-- | reference_model/src/verify/verify_entry.cc | 4 | ||||
-rw-r--r-- | reference_model/src/verify/verify_exact.cc | 53 | ||||
-rw-r--r-- | reference_model/src/verify/verify_utils.h | 7 | ||||
-rw-r--r-- | reference_model/test/verify_tests.cpp | 116 |
7 files changed, 188 insertions, 13 deletions
diff --git a/reference_model/CMakeLists.txt b/reference_model/CMakeLists.txt index 88192bf..5facfe1 100644 --- a/reference_model/CMakeLists.txt +++ b/reference_model/CMakeLists.txt @@ -73,6 +73,7 @@ set(CXX_SOURCE src/tensor.cc src/verify/verify_dot_product.cc src/verify/verify_entry.cc + src/verify/verify_exact.cc src/verify/verify_utils.cc src/ops/op_factory.cc src/ops/tensor_ops.cc @@ -141,6 +142,7 @@ set_target_properties(tosa_reference_model_lib PROPERTIES PUBLIC_HEADER "${PUBLI add_library(tosa_reference_verify_lib SHARED src/verify/verify_dot_product.cc src/verify/verify_entry.cc + src/verify/verify_exact.cc src/verify/verify_utils.cc src/verify/verify_config.cc src/func_debug.cc @@ -248,4 +250,4 @@ install(EXPORT TosaReferenceModelLibTargets FILE TosaReferenceModelLibTargets.cmake NAMESPACE TosaReference:: DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/tosa_reference_model_lib" -)
\ No newline at end of file +) diff --git a/reference_model/src/verify/verifiers.h b/reference_model/src/verify/verifiers.h index afd50bf..177eeaf 100644 --- a/reference_model/src/verify/verifiers.h +++ b/reference_model/src/verify/verifiers.h @@ -33,6 +33,14 @@ bool verifyDotProduct(const CTensor* ref, const CTensor* imp, const DotProductVerifyInfo& dpInfo); +/// \brief Perform exact result verification +/// +/// \param referenceTensor Reference tensor +/// \param implementationTensor Implementation resulting tensor +/// +/// \return True if compliant else false +bool verifyExact(const CTensor* referenceTensor, const CTensor* implementationTensor); + }; // namespace TosaReference #endif // VERIFIERS_H_ diff --git a/reference_model/src/verify/verify_dot_product.cc b/reference_model/src/verify/verify_dot_product.cc index 0b05c92..a96befa 100644 --- a/reference_model/src/verify/verify_dot_product.cc +++ b/reference_model/src/verify/verify_dot_product.cc @@ -20,13 +20,6 @@ #include <optional> #include <type_traits> -#define TOSA_REF_REQUIRE(COND, MESSAGE) \ - if (!(COND)) \ - { \ - WARNING(MESSAGE); \ - return false; \ - } - namespace TosaReference { namespace @@ -139,5 +132,3 @@ bool verifyDotProduct(const CTensor* ref, const CTensor* refBnd, const CTensor* } } // namespace TosaReference - -#undef TOSA_REF_REQUIRE
\ No newline at end of file diff --git a/reference_model/src/verify/verify_entry.cc b/reference_model/src/verify/verify_entry.cc index 80ca916..1c48354 100644 --- a/reference_model/src/verify/verify_entry.cc +++ b/reference_model/src/verify/verify_entry.cc @@ -30,7 +30,9 @@ bool verify(const CTensor* ref, const CTensor* refBnd, const CTensor* imp, const { case VerifyMode::DotProduct: { return verifyDotProduct(ref, refBnd, imp, cfg.dotProductInfo); - break; + } + case VerifyMode::Exact: { + return verifyExact(ref, imp); } default: { WARNING("unsupported verification mode."); diff --git a/reference_model/src/verify/verify_exact.cc b/reference_model/src/verify/verify_exact.cc new file mode 100644 index 0000000..7fde5bb --- /dev/null +++ b/reference_model/src/verify/verify_exact.cc @@ -0,0 +1,53 @@ +// Copyright (c) 2023, ARM Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "func_debug.h" +#include "verifiers.h" +#include <cmath> + +namespace TosaReference +{ + +bool verifyExact(const CTensor* referenceTensor, const CTensor* implementationTensor) +{ + // Validate that tensors are provided + TOSA_REF_REQUIRE(referenceTensor != nullptr, "reference tensor is missing"); + TOSA_REF_REQUIRE(implementationTensor != nullptr, "implementation tensor is missing"); + + // Get number of elements + const auto elementCount = + numElements(std::vector<int32_t>(referenceTensor->shape, referenceTensor->shape + referenceTensor->num_dims)); + TOSA_REF_REQUIRE(elementCount > 0, "invalid shape for reference tensor"); + + switch (implementationTensor->data_type) + { + case tosa_datatype_fp32_t: { + const auto* refData = reinterpret_cast<const float*>(referenceTensor->data); + TOSA_REF_REQUIRE(refData != nullptr, "missing data for reference"); + const auto* impData = reinterpret_cast<const float*>(implementationTensor->data); + TOSA_REF_REQUIRE(impData != nullptr, "missing data for implementation"); + return std::equal(refData, std::next(refData, elementCount), impData, std::next(impData, elementCount), + [](const auto& referenceValue, const auto& implementationValue) { + return std::isnan(referenceValue) ? std::isnan(implementationValue) + : (referenceValue == implementationValue); + }); + } + default: + WARNING("data-type not supported."); + break; + } + + return false; +} +} // namespace TosaReference diff --git a/reference_model/src/verify/verify_utils.h b/reference_model/src/verify/verify_utils.h index 6e51e3e..3a527da 100644 --- a/reference_model/src/verify/verify_utils.h +++ b/reference_model/src/verify/verify_utils.h @@ -23,6 +23,13 @@ #include <optional> #include <vector> +#define TOSA_REF_REQUIRE(COND, MESSAGE) \ + if (!(COND)) \ + { \ + WARNING(MESSAGE); \ + return false; \ + } + namespace TosaReference { diff --git a/reference_model/test/verify_tests.cpp b/reference_model/test/verify_tests.cpp index 81f3e8d..731a808 100644 --- a/reference_model/test/verify_tests.cpp +++ b/reference_model/test/verify_tests.cpp @@ -13,10 +13,16 @@ // limitations under the License. #include "verify.h" +#include <algorithm> #include <doctest.h> #include <array> +#include <iterator> +#include <limits> +#include <numeric> +#include <random> #include <string> +#include <type_traits> #include <vector> namespace @@ -25,7 +31,7 @@ namespace class TosaTensor { public: - TosaTensor(std::string name, tosa_datatype_t dataType, std::vector<int32_t> shape) + TosaTensor(std::string name, tosa_datatype_t dataType, std::vector<int32_t> shape, uint8_t* data = nullptr) : _name(std::move(name)) , _shape(std::move(shape)) { @@ -33,6 +39,9 @@ public: _tensor.data_type = dataType; _tensor.num_dims = _shape.size(); _tensor.shape = _shape.data(); + _tensor.data = data; + _tensor.size = + std::accumulate(_tensor.shape, std::next(_tensor.shape, _tensor.num_dims), 1, std::multiplies<>()); }; const tosa_tensor_t* cTensor() const @@ -46,6 +55,66 @@ private: tosa_tensor_t _tensor; }; +auto& getRandomGenerator() +{ + static std::mt19937 gen(0); + return gen; +} + +template <typename FP> +std::enable_if_t<std::is_floating_point_v<FP>, std::add_lvalue_reference_t<std::uniform_real_distribution<FP>>> + getUniformRealDist() +{ + // Uniform real distribution generates real values in the range [a, b) + // and requires that b - a <= std::numeric_limits<FP>::max() so here + // we choose some arbitrary values that satisfy that condition. + constexpr auto min = std::numeric_limits<FP>::lowest() / 2; + constexpr auto max = std::numeric_limits<FP>::max() / 2; + static_assert(max <= std::numeric_limits<FP>::max() + min); + + static std::uniform_real_distribution<FP> dis(min, max); + return dis; +} + +template <typename FP> +std::enable_if_t<std::is_floating_point_v<FP>, FP> getRandomUniformFloat() +{ + return getUniformRealDist<FP>()(getRandomGenerator()); +} + +template <typename FP> +std::enable_if_t<std::is_floating_point_v<FP>, std::vector<FP>> generateRandomTensorData(size_t elementCount, + bool includeNans = false) +{ + // Generate some random floats using the full range of fp32. + auto data = std::vector<FP>(elementCount); + std::generate(std::begin(data), std::end(data), []() { return getRandomUniformFloat<FP>(); }); + + // Include some edge cases. + auto edgeCases = std::vector<float>{ +0.0f, -0.0f, std::numeric_limits<float>::infinity(), + -std::numeric_limits<float>::infinity() }; + if (includeNans) + { + static const auto nans = + std::vector<float>{ std::numeric_limits<float>::quiet_NaN(), std::numeric_limits<float>::signaling_NaN() }; + + std::copy(std::begin(nans), std::end(nans), std::back_inserter(edgeCases)); + } + + if (elementCount >= edgeCases.size()) + { + // Evenly distribute the edge cases throughout the data, this way for operations like reductions all edge cases won't + // end up in the same row/column over which a reduction happens. + const auto stride = (data.size() + (edgeCases.size() - 1)) / edgeCases.size(); + for (unsigned i = 0; i < edgeCases.size(); ++i) + { + data[i * stride] = edgeCases[i]; + } + } + + return data; +} + } // namespace TEST_SUITE_BEGIN("verify"); @@ -115,4 +184,47 @@ TEST_CASE("negative - api") } } -TEST_SUITE_END(); // verify
\ No newline at end of file +TEST_CASE("positive - exact") +{ + std::string json_cfg = R"({ + "tensors" : { + "out1" : { + "mode": "EXACT" + } + } + })"; + + const auto shape = std::vector<int32_t>{ 8, 8, 8 }; + const auto elementCount = std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies<>()); + + // Generate some random floats using the full range of fp32. + auto data = generateRandomTensorData<float>(elementCount); + SUBCASE("same") + { + const auto referenceTensor = + TosaTensor("out1", tosa_datatype_fp64_t, shape, reinterpret_cast<uint8_t*>(data.data())); + const auto implementationTensor = + TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast<uint8_t*>(data.data())); + REQUIRE(tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), json_cfg.c_str())); + } + + SUBCASE("different") + { + // Generate some mismatched tensors by setting every other value to an incrementing counter. + // In theory this could be the same, but the probability is tiny. + auto otherData = std::vector<float>(elementCount); + std::generate(std::begin(otherData), std::end(otherData), [&, i = 0]() mutable { + auto oldIndex = i++; + return oldIndex % 2 ? data[oldIndex] : static_cast<float>(oldIndex); + }); + + const auto referenceTensor = + TosaTensor("out1", tosa_datatype_fp64_t, shape, reinterpret_cast<uint8_t*>(data.data())); + const auto implementationTensor = + TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast<uint8_t*>(otherData.data())); + REQUIRE_FALSE( + tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), json_cfg.c_str())); + } +} + +TEST_SUITE_END(); // verify |