diff options
author | Jack Frankland <jack.frankland@arm.com> | 2023-09-13 11:03:50 +0100 |
---|---|---|
committer | Eric Kunze <eric.kunze@arm.com> | 2023-09-28 18:28:07 +0000 |
commit | aafc8508398180835781605dd9229d4a6087af1f (patch) | |
tree | 57a65c17c83ff1b4eb8d07446e3abd0537e4d947 /reference_model/test | |
parent | f0348ea4206a7e02497515ffb6d88546e0121cc7 (diff) | |
download | reference_model-aafc8508398180835781605dd9229d4a6087af1f.tar.gz |
feat: Add exact verifier
Add a verifier to check that tensor results match exactly.
Add a unit test to check the behavior of this new verifier.
Change-Id: I9b80a6d57640fec67c6be80a97b3af484aeb935e
Signed-off-by: Jack Frankland <jack.frankland@arm.com>
Diffstat (limited to 'reference_model/test')
-rw-r--r-- | reference_model/test/verify_tests.cpp | 116 |
1 files changed, 114 insertions, 2 deletions
diff --git a/reference_model/test/verify_tests.cpp b/reference_model/test/verify_tests.cpp index 81f3e8d..731a808 100644 --- a/reference_model/test/verify_tests.cpp +++ b/reference_model/test/verify_tests.cpp @@ -13,10 +13,16 @@ // limitations under the License. #include "verify.h" +#include <algorithm> #include <doctest.h> #include <array> +#include <iterator> +#include <limits> +#include <numeric> +#include <random> #include <string> +#include <type_traits> #include <vector> namespace @@ -25,7 +31,7 @@ namespace class TosaTensor { public: - TosaTensor(std::string name, tosa_datatype_t dataType, std::vector<int32_t> shape) + TosaTensor(std::string name, tosa_datatype_t dataType, std::vector<int32_t> shape, uint8_t* data = nullptr) : _name(std::move(name)) , _shape(std::move(shape)) { @@ -33,6 +39,9 @@ public: _tensor.data_type = dataType; _tensor.num_dims = _shape.size(); _tensor.shape = _shape.data(); + _tensor.data = data; + _tensor.size = + std::accumulate(_tensor.shape, std::next(_tensor.shape, _tensor.num_dims), 1, std::multiplies<>()); }; const tosa_tensor_t* cTensor() const @@ -46,6 +55,66 @@ private: tosa_tensor_t _tensor; }; +auto& getRandomGenerator() +{ + static std::mt19937 gen(0); + return gen; +} + +template <typename FP> +std::enable_if_t<std::is_floating_point_v<FP>, std::add_lvalue_reference_t<std::uniform_real_distribution<FP>>> + getUniformRealDist() +{ + // Uniform real distribution generates real values in the range [a, b) + // and requires that b - a <= std::numeric_limits<FP>::max() so here + // we choose some arbitrary values that satisfy that condition. + constexpr auto min = std::numeric_limits<FP>::lowest() / 2; + constexpr auto max = std::numeric_limits<FP>::max() / 2; + static_assert(max <= std::numeric_limits<FP>::max() + min); + + static std::uniform_real_distribution<FP> dis(min, max); + return dis; +} + +template <typename FP> +std::enable_if_t<std::is_floating_point_v<FP>, FP> getRandomUniformFloat() +{ + return getUniformRealDist<FP>()(getRandomGenerator()); +} + +template <typename FP> +std::enable_if_t<std::is_floating_point_v<FP>, std::vector<FP>> generateRandomTensorData(size_t elementCount, + bool includeNans = false) +{ + // Generate some random floats using the full range of fp32. + auto data = std::vector<FP>(elementCount); + std::generate(std::begin(data), std::end(data), []() { return getRandomUniformFloat<FP>(); }); + + // Include some edge cases. + auto edgeCases = std::vector<float>{ +0.0f, -0.0f, std::numeric_limits<float>::infinity(), + -std::numeric_limits<float>::infinity() }; + if (includeNans) + { + static const auto nans = + std::vector<float>{ std::numeric_limits<float>::quiet_NaN(), std::numeric_limits<float>::signaling_NaN() }; + + std::copy(std::begin(nans), std::end(nans), std::back_inserter(edgeCases)); + } + + if (elementCount >= edgeCases.size()) + { + // Evenly distribute the edge cases throughout the data, this way for operations like reductions all edge cases won't + // end up in the same row/column over which a reduction happens. + const auto stride = (data.size() + (edgeCases.size() - 1)) / edgeCases.size(); + for (unsigned i = 0; i < edgeCases.size(); ++i) + { + data[i * stride] = edgeCases[i]; + } + } + + return data; +} + } // namespace TEST_SUITE_BEGIN("verify"); @@ -115,4 +184,47 @@ TEST_CASE("negative - api") } } -TEST_SUITE_END(); // verify
\ No newline at end of file +TEST_CASE("positive - exact") +{ + std::string json_cfg = R"({ + "tensors" : { + "out1" : { + "mode": "EXACT" + } + } + })"; + + const auto shape = std::vector<int32_t>{ 8, 8, 8 }; + const auto elementCount = std::accumulate(std::begin(shape), std::end(shape), 1, std::multiplies<>()); + + // Generate some random floats using the full range of fp32. + auto data = generateRandomTensorData<float>(elementCount); + SUBCASE("same") + { + const auto referenceTensor = + TosaTensor("out1", tosa_datatype_fp64_t, shape, reinterpret_cast<uint8_t*>(data.data())); + const auto implementationTensor = + TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast<uint8_t*>(data.data())); + REQUIRE(tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), json_cfg.c_str())); + } + + SUBCASE("different") + { + // Generate some mismatched tensors by setting every other value to an incrementing counter. + // In theory this could be the same, but the probability is tiny. + auto otherData = std::vector<float>(elementCount); + std::generate(std::begin(otherData), std::end(otherData), [&, i = 0]() mutable { + auto oldIndex = i++; + return oldIndex % 2 ? data[oldIndex] : static_cast<float>(oldIndex); + }); + + const auto referenceTensor = + TosaTensor("out1", tosa_datatype_fp64_t, shape, reinterpret_cast<uint8_t*>(data.data())); + const auto implementationTensor = + TosaTensor("out1", tosa_datatype_fp32_t, shape, reinterpret_cast<uint8_t*>(otherData.data())); + REQUIRE_FALSE( + tvf_verify_data(referenceTensor.cTensor(), nullptr, implementationTensor.cTensor(), json_cfg.c_str())); + } +} + +TEST_SUITE_END(); // verify |