diff options
Diffstat (limited to 'tests/validation/Validation.h')
| -rw-r--r-- | tests/validation/Validation.h | 276 |
1 files changed, 53 insertions, 223 deletions
diff --git a/tests/validation/Validation.h b/tests/validation/Validation.h index fa3386901e..289aca4d08 100644 --- a/tests/validation/Validation.h +++ b/tests/validation/Validation.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 ARM Limited. + * Copyright (c) 2017-2022 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -45,6 +45,17 @@ namespace test { namespace validation { +namespace +{ +// Compare if 2 values are both infinities and if they are "equal" (has the same sign) +template <typename T> +inline bool are_equal_infs(T val0, T val1) +{ + const auto same_sign = support::cpp11::signbit(val0) == support::cpp11::signbit(val1); + return (!support::cpp11::isfinite(val0)) && (!support::cpp11::isfinite(val1)) && same_sign; +} +} // namespace + /** Class reprensenting an absolute tolerance value. */ template <typename T> class AbsoluteTolerance @@ -140,7 +151,7 @@ bool compare_dimensions(const Dimensions<T> &dimensions1, const Dimensions<T> &d { ARM_COMPUTE_ERROR_ON(data_layout == DataLayout::UNKNOWN); - if(data_layout == DataLayout::NCHW) + if(data_layout != DataLayout::NHWC) { if(dimensions1.num_dimensions() != dimensions2.num_dimensions()) { @@ -157,24 +168,22 @@ bool compare_dimensions(const Dimensions<T> &dimensions1, const Dimensions<T> &d } else { - // In case a 2D shape becomes 3D after permutation, the permuted tensor will have one dimension more and the first value will be 1 - if((dimensions1.num_dimensions() != dimensions2.num_dimensions()) && ((dimensions1.num_dimensions() != (dimensions2.num_dimensions() + 1)) || (dimensions1.x() != 1))) + // In case a 1D/2D shape becomes 3D after permutation, the permuted tensor will have two/one dimension(s) more and the first (two) value(s) will be 1 + // clang-format off + const auto max_dims = std::max(dimensions1.num_dimensions(), dimensions2.num_dimensions()); + for(unsigned int i = 3; i < max_dims; ++i) { - return false; + if(dimensions1[i] != dimensions2[i]) + { + return false; + } } + // clang-format on if((dimensions1[0] != dimensions2[2]) || (dimensions1[1] != dimensions2[0]) || (dimensions1[2] != dimensions2[1])) { return false; } - - for(unsigned int i = 3; i < dimensions1.num_dimensions(); ++i) - { - if(dimensions1[i] != dimensions2[i]) - { - return false; - } - } } return true; @@ -267,16 +276,6 @@ void validate(std::vector<unsigned int> classified_labels, std::vector<unsigned template <typename T, typename U = AbsoluteTolerance<T>> bool validate(T target, T reference, U tolerance = AbsoluteTolerance<T>()); -/** Validate key points. */ -template <typename T, typename U, typename V = AbsoluteTolerance<float>> -void validate_keypoints(T target_first, T target_last, U reference_first, U reference_last, V tolerance = AbsoluteTolerance<float>(), - float allowed_missing_percentage = 5.f, float allowed_mismatch_percentage = 5.f); - -/** Validate detection windows. */ -template <typename T, typename U, typename V = AbsoluteTolerance<float>> -void validate_detection_windows(T target_first, T target_last, U reference_first, U reference_last, V tolerance = AbsoluteTolerance<float>(), - float allowed_missing_percentage = 5.f, float allowed_mismatch_percentage = 5.f); - template <typename T> struct compare_base { @@ -308,9 +307,9 @@ struct compare<AbsoluteTolerance<U>> : public compare_base<AbsoluteTolerance<U>> /** Perform comparison */ operator bool() const { - if(!support::cpp11::isfinite(this->_target) || !support::cpp11::isfinite(this->_reference)) + if(are_equal_infs(this->_target, this->_reference)) { - return false; + return true; } else if(this->_target == this->_reference) { @@ -334,9 +333,9 @@ struct compare<RelativeTolerance<U>> : public compare_base<RelativeTolerance<U>> /** Perform comparison */ operator bool() const { - if(!support::cpp11::isfinite(this->_target) || !support::cpp11::isfinite(this->_reference)) + if(are_equal_infs(this->_target, this->_reference)) { - return false; + return true; } else if(this->_target == this->_reference) { @@ -380,6 +379,11 @@ void validate_wrap(const IAccessor &tensor, const SimpleTensor<T> &reference, U template <typename T, typename U> void validate(const IAccessor &tensor, const SimpleTensor<T> &reference, const ValidRegion &valid_region, U tolerance_value, float tolerance_number, float absolute_tolerance_value) { + if(framework::Framework::get().configure_only() && framework::Framework::get().new_fixture_call()) + { + return; + } + uint64_t num_mismatches = 0; uint64_t num_elements = 0; @@ -455,6 +459,11 @@ void validate(const IAccessor &tensor, const SimpleTensor<T> &reference, const V template <typename T, typename U, typename = typename std::enable_if<std::is_integral<T>::value>::type> void validate_wrap(const IAccessor &tensor, const SimpleTensor<T> &reference, const ValidRegion &valid_region, U tolerance_value, float tolerance_number) { + if(framework::Framework::get().configure_only() && framework::Framework::get().new_fixture_call()) + { + return; + } + uint64_t num_mismatches = 0; uint64_t num_elements = 0; @@ -496,9 +505,9 @@ void validate_wrap(const IAccessor &tensor, const SimpleTensor<T> &reference, co // check for wrapping if(!equal) { - if(!support::cpp11::isfinite(target_value) || !support::cpp11::isfinite(reference_value)) + if(are_equal_infs(target_value, reference_value)) { - equal = false; + equal = true; } else { @@ -543,6 +552,11 @@ void validate_wrap(const IAccessor &tensor, const SimpleTensor<T> &reference, co template <typename T, typename U> void validate(const IAccessor &tensor, const SimpleTensor<T> &reference, const SimpleTensor<T> &valid_mask, U tolerance_value, float tolerance_number, float absolute_tolerance_value) { + if(framework::Framework::get().configure_only() && framework::Framework::get().new_fixture_call()) + { + return; + } + uint64_t num_mismatches = 0; uint64_t num_elements = 0; @@ -622,6 +636,11 @@ void validate(const IAccessor &tensor, const SimpleTensor<T> &reference, const S template <typename T, typename U> bool validate(T target, T reference, U tolerance) { + if(framework::Framework::get().configure_only() && framework::Framework::get().new_fixture_call()) + { + return true; + } + ARM_COMPUTE_TEST_INFO("reference = " << std::setprecision(5) << framework::make_printable(reference)); ARM_COMPUTE_TEST_INFO("target = " << std::setprecision(5) << framework::make_printable(target)); ARM_COMPUTE_TEST_INFO("tolerance = " << std::setprecision(5) << framework::make_printable(static_cast<typename U::value_type>(tolerance))); @@ -636,6 +655,11 @@ bool validate(T target, T reference, U tolerance) template <typename T, typename U> void validate_min_max_loc(const MinMaxLocationValues<T> &target, const MinMaxLocationValues<U> &reference) { + if(framework::Framework::get().configure_only() && framework::Framework::get().new_fixture_call()) + { + return; + } + ARM_COMPUTE_EXPECT_EQUAL(target.min, reference.min, framework::LogLevel::ERRORS); ARM_COMPUTE_EXPECT_EQUAL(target.max, reference.max, framework::LogLevel::ERRORS); @@ -662,200 +686,6 @@ void validate_min_max_loc(const MinMaxLocationValues<T> &target, const MinMaxLoc ARM_COMPUTE_EXPECT(same_coords != reference.max_loc.end(), framework::LogLevel::ERRORS); } } - -/** Check which keypoints from [first1, last1) are missing in [first2, last2) */ -template <typename T, typename U, typename V> -std::pair<int64_t, int64_t> compare_keypoints(T first1, T last1, U first2, U last2, V tolerance, bool check_mismatches = true) -{ - /* Keypoint (x,y) should have similar strength (within tolerance) and other properties in both reference and target */ - const auto compare_props_eq = [&](const KeyPoint & lhs, const KeyPoint & rhs) - { - return compare<V>(lhs.strength, rhs.strength, tolerance) - && lhs.tracking_status == rhs.tracking_status - && lhs.scale == rhs.scale - && lhs.orientation == rhs.orientation - && lhs.error == rhs.error; - }; - - /* Used to sort KeyPoints by coordinates (x, y) */ - const auto compare_coords_lt = [](const KeyPoint & lhs, const KeyPoint & rhs) - { - return std::tie(lhs.x, lhs.y) < std::tie(rhs.x, rhs.y); - }; - - std::sort(first1, last1, compare_coords_lt); - std::sort(first2, last2, compare_coords_lt); - - if(check_mismatches) - { - ARM_COMPUTE_TEST_INFO("Checking for mismatches: ref count = " << std::distance(first1, last1) << " target count = " << std::distance(first2, last2)); - } - - int64_t num_missing = 0; - int64_t num_mismatches = 0; - bool rest_missing = false; - - while(first1 != last1) - { - if(first2 == last2) - { - rest_missing = true; - break; - } - - if(compare_coords_lt(*first1, *first2)) - { - ++num_missing; - ARM_COMPUTE_TEST_INFO("Key point not found"); - ARM_COMPUTE_TEST_INFO("keypoint1 = " << *first1++); - framework::ARM_COMPUTE_PRINT_INFO(); - } - else - { - if(!compare_coords_lt(*first2, *first1)) // Equal coordinates - { - if(check_mismatches && !compare_props_eq(*first1, *first2)) // Check other properties - { - ++num_mismatches; - ARM_COMPUTE_TEST_INFO("Mismatching keypoint"); - ARM_COMPUTE_TEST_INFO("keypoint1 [ref] = " << *first1); - ARM_COMPUTE_TEST_INFO("keypoint2 [tgt] = " << *first2); - framework::ARM_COMPUTE_PRINT_INFO(); - } - ++first1; - } - ++first2; - } - } - - if(rest_missing) - { - while(first1 != last1) - { - ++num_missing; - ARM_COMPUTE_TEST_INFO("Key point not found"); - ARM_COMPUTE_TEST_INFO("keypoint1 = " << *first1++); - framework::ARM_COMPUTE_PRINT_INFO(); - } - } - - return std::make_pair(num_missing, num_mismatches); -} - -template <typename T, typename U, typename V> -void validate_keypoints(T target_first, T target_last, U reference_first, U reference_last, V tolerance, float allowed_missing_percentage, float allowed_mismatch_percentage) -{ - const int64_t num_elements_target = std::distance(target_first, target_last); - const int64_t num_elements_reference = std::distance(reference_first, reference_last); - - int64_t num_missing = 0; - int64_t num_mismatches = 0; - - if(num_elements_reference > 0) - { - std::tie(num_missing, num_mismatches) = compare_keypoints(reference_first, reference_last, target_first, target_last, tolerance); - - const float percent_missing = static_cast<float>(num_missing) / num_elements_reference * 100.f; - const float percent_mismatches = static_cast<float>(num_mismatches) / num_elements_reference * 100.f; - - ARM_COMPUTE_TEST_INFO(num_missing << " keypoints (" << std::fixed << std::setprecision(2) << percent_missing << "%) in ref are missing from target"); - ARM_COMPUTE_TEST_INFO("Missing (not in tgt): " << num_missing << "/" << num_elements_reference << " = " << std::fixed << std::setprecision(2) << percent_missing - << "% \tMax allowed: " << allowed_missing_percentage << "%"); - ARM_COMPUTE_EXPECT(percent_missing <= allowed_missing_percentage, framework::LogLevel::ERRORS); - - ARM_COMPUTE_TEST_INFO(num_mismatches << " keypoints (" << std::fixed << std::setprecision(2) << percent_mismatches << "%) mismatched"); - ARM_COMPUTE_TEST_INFO("Mismatched keypoints: " << num_mismatches << "/" << num_elements_reference << " = " << std::fixed << std::setprecision(2) << percent_mismatches - << "% \tMax allowed: " << allowed_mismatch_percentage << "%"); - ARM_COMPUTE_EXPECT(percent_mismatches <= allowed_mismatch_percentage, framework::LogLevel::ERRORS); - } - - if(num_elements_target > 0) - { - // Note: no need to check for mismatches a second time (last argument is 'false') - std::tie(num_missing, num_mismatches) = compare_keypoints(target_first, target_last, reference_first, reference_last, tolerance, false); - - const float percent_missing = static_cast<float>(num_missing) / num_elements_target * 100.f; - - ARM_COMPUTE_TEST_INFO(num_missing << " keypoints (" << std::fixed << std::setprecision(2) << percent_missing << "%) in target are missing from ref"); - ARM_COMPUTE_TEST_INFO("Missing (not in ref): " << num_missing << "/" << num_elements_target << " = " << std::fixed << std::setprecision(2) << percent_missing - << "% \tMax allowed: " << allowed_missing_percentage << "%"); - ARM_COMPUTE_EXPECT(percent_missing <= allowed_missing_percentage, framework::LogLevel::ERRORS); - } -} - -/** Check which detection windows from [first1, last1) are missing in [first2, last2) */ -template <typename T, typename U, typename V> -std::pair<int64_t, int64_t> compare_detection_windows(T first1, T last1, U first2, U last2, V tolerance) -{ - int64_t num_missing = 0; - int64_t num_mismatches = 0; - - while(first1 != last1) - { - const auto window = std::find_if(first2, last2, [&](DetectionWindow window) - { - return window.x == first1->x && window.y == first1->y && window.width == first1->width && window.height == first1->height && window.idx_class == first1->idx_class; - }); - - if(window == last2) - { - ++num_missing; - ARM_COMPUTE_TEST_INFO("Detection window not found " << *first1) - framework::ARM_COMPUTE_PRINT_INFO(); - } - else - { - if(!compare<V>(window->score, first1->score, tolerance)) - { - ++num_mismatches; - ARM_COMPUTE_TEST_INFO("Mismatching detection window") - ARM_COMPUTE_TEST_INFO("detection window 1= " << *first1) - ARM_COMPUTE_TEST_INFO("detection window 2= " << *window) - framework::ARM_COMPUTE_PRINT_INFO(); - } - } - - ++first1; - } - - return std::make_pair(num_missing, num_mismatches); -} - -template <typename T, typename U, typename V> -void validate_detection_windows(T target_first, T target_last, U reference_first, U reference_last, V tolerance, - float allowed_missing_percentage, float allowed_mismatch_percentage) -{ - const int64_t num_elements_target = std::distance(target_first, target_last); - const int64_t num_elements_reference = std::distance(reference_first, reference_last); - - int64_t num_missing = 0; - int64_t num_mismatches = 0; - - if(num_elements_reference > 0) - { - std::tie(num_missing, num_mismatches) = compare_detection_windows(reference_first, reference_last, target_first, target_last, tolerance); - - const float percent_missing = static_cast<float>(num_missing) / num_elements_reference * 100.f; - const float percent_mismatches = static_cast<float>(num_mismatches) / num_elements_reference * 100.f; - - ARM_COMPUTE_TEST_INFO(num_missing << " detection windows (" << std::fixed << std::setprecision(2) << percent_missing << "%) are missing in target"); - ARM_COMPUTE_EXPECT(percent_missing <= allowed_missing_percentage, framework::LogLevel::ERRORS); - - ARM_COMPUTE_TEST_INFO(num_mismatches << " detection windows (" << std::fixed << std::setprecision(2) << percent_mismatches << "%) mismatched"); - ARM_COMPUTE_EXPECT(percent_mismatches <= allowed_mismatch_percentage, framework::LogLevel::ERRORS); - } - - if(num_elements_target > 0) - { - std::tie(num_missing, num_mismatches) = compare_detection_windows(target_first, target_last, reference_first, reference_last, tolerance); - - const float percent_missing = static_cast<float>(num_missing) / num_elements_target * 100.f; - - ARM_COMPUTE_TEST_INFO(num_missing << " detection windows (" << std::fixed << std::setprecision(2) << percent_missing << "%) are not part of target"); - ARM_COMPUTE_EXPECT(percent_missing <= allowed_missing_percentage, framework::LogLevel::ERRORS); - } -} - } // namespace validation } // namespace test } // namespace arm_compute |