From b33f28b8e362a8b36fbb03ea62cce1de338ae126 Mon Sep 17 00:00:00 2001 From: Jan Eilers Date: Wed, 18 Nov 2020 10:36:46 +0000 Subject: IVGCVSW-5545 Fix delegate Comparison failures on CpuAcc/GpuAcc * Create backend test suite structure * Add special compare function for boolean values Signed-off-by: Jan Eilers Change-Id: I55a2ae1ac6ad21cdcdd5ae99ef56ed00fa24776f --- delegate/src/test/ComparisonTest.cpp | 360 ++++++++++++++++++----------- delegate/src/test/ComparisonTestHelper.hpp | 11 +- delegate/src/test/TestUtils.cpp | 20 ++ delegate/src/test/TestUtils.hpp | 20 +- 4 files changed, 266 insertions(+), 145 deletions(-) diff --git a/delegate/src/test/ComparisonTest.cpp b/delegate/src/test/ComparisonTest.cpp index 0826535c8f..95bfe21d27 100644 --- a/delegate/src/test/ComparisonTest.cpp +++ b/delegate/src/test/ComparisonTest.cpp @@ -497,258 +497,348 @@ void LessEqualInt32Test(std::vector& backends) expectedOutputValues); } -TEST_SUITE("ComparisonTest") +TEST_SUITE("Comparison_CpuRefTests") { -TEST_CASE ("EQUAL_FP32_GpuAcc_Test") +TEST_CASE ("EQUAL_FP32_CpuRef_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; + std::vector backends = { armnn::Compute::CpuRef }; EqualFP32Test(backends); } -TEST_CASE ("EQUAL_FP32_CpuAcc_Test") +TEST_CASE ("EQUAL_Broadcast_CpuRef_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - EqualFP32Test(backends); + std::vector backends = { armnn::Compute::CpuRef }; + EqualBroadcastTest(backends); } -TEST_CASE ("EQUAL_Broadcast_GpuAcc_Test") +TEST_CASE ("EQUAL_INT32_CpuRef_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; - EqualBroadcastTest(backends); + std::vector backends = { armnn::Compute::CpuRef }; + EqualInt32Test(backends); } -TEST_CASE ("EQUAL_Broadcast_CpuAcc_Test") +TEST_CASE ("NOT_EQUAL_FP32_CpuRef_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - EqualBroadcastTest(backends); + std::vector backends = { armnn::Compute::CpuRef }; + NotEqualFP32Test(backends); } -TEST_CASE ("EQUAL_INT32_GpuAcc_Test") +TEST_CASE ("NOT_EQUAL_Broadcast_CpuRef_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; - EqualInt32Test(backends); + std::vector backends = { armnn::Compute::CpuRef }; + NotEqualBroadcastTest(backends); } -TEST_CASE ("EQUAL_INT32_CpuAcc_Test") +TEST_CASE ("NOT_EQUAL_INT32_CpuRef_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - EqualInt32Test(backends); + std::vector backends = { armnn::Compute::CpuRef }; + NotEqualInt32Test(backends); } -TEST_CASE ("NOT_EQUAL_FP32_GpuAcc_Test") +TEST_CASE ("GREATER_FP32_CpuRef_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; - NotEqualFP32Test(backends); + std::vector backends = { armnn::Compute::CpuRef }; + GreaterFP32Test(backends); } -TEST_CASE ("NOT_EQUAL_FP32_CpuAcc_Test") +TEST_CASE ("GREATER_Broadcast_CpuRef_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - NotEqualFP32Test(backends); + std::vector backends = { armnn::Compute::CpuRef }; + GreaterBroadcastTest(backends); } -TEST_CASE ("NOT_EQUAL_Broadcast_GpuAcc_Test") +TEST_CASE ("GREATER_INT32_CpuRef_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; - NotEqualBroadcastTest(backends); + std::vector backends = { armnn::Compute::CpuRef }; + GreaterInt32Test(backends); } -TEST_CASE ("NOT_EQUAL_Broadcast_CpuAcc_Test") +TEST_CASE ("GREATER_EQUAL_FP32_CpuRef_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - NotEqualBroadcastTest(backends); + std::vector backends = { armnn::Compute::CpuRef }; + GreaterEqualFP32Test(backends); } -TEST_CASE ("NOT_EQUAL_INT32_GpuAcc_Test") +TEST_CASE ("GREATER_EQUAL_Broadcast_CpuRef_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; - NotEqualInt32Test(backends); + std::vector backends = { armnn::Compute::CpuRef }; + GreaterEqualBroadcastTest(backends); } -TEST_CASE ("NOT_EQUAL_INT32_CpuAcc_Test") +TEST_CASE ("GREATER_EQUAL_INT32_CpuRef_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - NotEqualInt32Test(backends); + std::vector backends = { armnn::Compute::CpuRef }; + GreaterEqualInt32Test(backends); } -TEST_CASE ("GREATER_FP32_GpuAcc_Test") +TEST_CASE ("LESS_FP32_CpuRef_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; - GreaterFP32Test(backends); + std::vector backends = { armnn::Compute::CpuRef }; + LessFP32Test(backends); } -TEST_CASE ("GREATER_FP32_CpuAcc_Test") +TEST_CASE ("LESS_Broadcast_CpuRef_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - GreaterFP32Test(backends); + std::vector backends = { armnn::Compute::CpuRef }; + LessBroadcastTest(backends); } -TEST_CASE ("GREATER_Broadcast_GpuAcc_Test") +TEST_CASE ("LESS_INT32_CpuRef_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; - GreaterBroadcastTest(backends); + std::vector backends = { armnn::Compute::CpuRef }; + LessInt32Test(backends); } -TEST_CASE ("GREATER_Broadcast_CpuAcc_Test") +TEST_CASE ("LESS_EQUAL_FP32_CpuRef_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - GreaterBroadcastTest(backends); + std::vector backends = { armnn::Compute::CpuRef }; + LessEqualFP32Test(backends); } -TEST_CASE ("GREATER_INT32_GpuAcc_Test") +TEST_CASE ("LESS_EQUAL_Broadcast_CpuRef_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; - GreaterInt32Test(backends); + std::vector backends = { armnn::Compute::CpuRef }; + LessEqualBroadcastTest(backends); } -TEST_CASE ("GREATER_INT32_CpuAcc_Test") +TEST_CASE ("LESS_EQUAL_INT32_CpuRef_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - GreaterInt32Test(backends); + std::vector backends = { armnn::Compute::CpuRef }; + LessEqualInt32Test(backends); } -TEST_CASE ("GREATER_EQUAL_FP32_GpuAcc_Test") +} // End TEST_SUITE("Comparison_CpuRefTests") + + + +TEST_SUITE("Comparison_GpuAccTests") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; - GreaterEqualFP32Test(backends); + +TEST_CASE ("EQUAL_FP32_GpuAcc_Test") +{ + std::vector backends = { armnn::Compute::GpuAcc }; + EqualFP32Test(backends); } -TEST_CASE ("GREATER_EQUAL_FP32_CpuAcc_Test") +TEST_CASE ("EQUAL_Broadcast_GpuAcc_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - GreaterEqualFP32Test(backends); + std::vector backends = { armnn::Compute::GpuAcc }; + EqualBroadcastTest(backends); } -TEST_CASE ("GREATER_EQUAL_Broadcast_GpuAcc_Test") +TEST_CASE ("EQUAL_INT32_GpuAcc_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; - GreaterEqualBroadcastTest(backends); + std::vector backends = { armnn::Compute::GpuAcc }; + EqualInt32Test(backends); } -TEST_CASE ("GREATER_EQUAL_Broadcast_CpuAcc_Test") +TEST_CASE ("NOT_EQUAL_FP32_GpuAcc_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - GreaterEqualBroadcastTest(backends); + std::vector backends = { armnn::Compute::GpuAcc }; + NotEqualFP32Test(backends); } -TEST_CASE ("GREATER_EQUAL_INT32_GpuAcc_Test") +TEST_CASE ("NOT_EQUAL_Broadcast_GpuAcc_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; - GreaterEqualInt32Test(backends); + std::vector backends = { armnn::Compute::GpuAcc }; + NotEqualBroadcastTest(backends); } -TEST_CASE ("GREATER_EQUAL_INT32_CpuAcc_Test") +TEST_CASE ("NOT_EQUAL_INT32_GpuAcc_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - GreaterEqualInt32Test(backends); + std::vector backends = { armnn::Compute::GpuAcc }; + NotEqualInt32Test(backends); } -TEST_CASE ("LESS_FP32_GpuAcc_Test") + +TEST_CASE ("GREATER_FP32_GpuAcc_Test") { std::vector backends = { armnn::Compute::GpuAcc, armnn::Compute::CpuRef }; - LessFP32Test(backends); + GreaterFP32Test(backends); } -TEST_CASE ("LESS_FP32_CpuAcc_Test") +TEST_CASE ("GREATER_Broadcast_GpuAcc_Test") { - std::vector backends = { armnn::Compute::CpuAcc, + std::vector backends = { armnn::Compute::GpuAcc, armnn::Compute::CpuRef }; - LessFP32Test(backends); + GreaterBroadcastTest(backends); } -TEST_CASE ("LESS_Broadcast_GpuAcc_Test") +TEST_CASE ("GREATER_INT32_GpuAcc_Test") { std::vector backends = { armnn::Compute::GpuAcc, armnn::Compute::CpuRef }; - LessBroadcastTest(backends); + GreaterInt32Test(backends); } -TEST_CASE ("LESS_Broadcast_CpuAcc_Test") +TEST_CASE ("GREATER_EQUAL_FP32_GpuAcc_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - LessBroadcastTest(backends); + std::vector backends = { armnn::Compute::GpuAcc }; + GreaterEqualFP32Test(backends); } -TEST_CASE ("LESS_INT32_GpuAcc_Test") +TEST_CASE ("GREATER_EQUAL_Broadcast_GpuAcc_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; - LessInt32Test(backends); + std::vector backends = { armnn::Compute::GpuAcc }; + GreaterEqualBroadcastTest(backends); } -TEST_CASE ("LESS_INT32_CpuAcc_Test") +TEST_CASE ("GREATER_EQUAL_INT32_GpuAcc_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - LessInt32Test(backends); + std::vector backends = { armnn::Compute::GpuAcc }; + GreaterEqualInt32Test(backends); } -TEST_CASE ("LESS_EQUAL_FP32_GpuAcc_Test") + +TEST_CASE ("LESS_FP32_GpuAcc_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; - LessEqualFP32Test(backends); + std::vector backends = { armnn::Compute::GpuAcc }; + LessFP32Test(backends); } -TEST_CASE ("LESS_EQUAL_FP32_CpuAcc_Test") +TEST_CASE ("LESS_Broadcast_GpuAcc_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; - LessEqualFP32Test(backends); + std::vector backends = { armnn::Compute::GpuAcc }; + LessBroadcastTest(backends); } -TEST_CASE ("LESS_EQUAL_Broadcast_GpuAcc_Test") +TEST_CASE ("LESS_INT32_GpuAcc_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; - LessEqualBroadcastTest(backends); + std::vector backends = { armnn::Compute::GpuAcc }; + LessInt32Test(backends); } -TEST_CASE ("LESS_EQUAL_Broadcast_CpuAcc_Test") +TEST_CASE ("LESS_EQUAL_FP32_GpuAcc_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; + std::vector backends = { armnn::Compute::GpuAcc }; + LessEqualFP32Test(backends); +} + +TEST_CASE ("LESS_EQUAL_Broadcast_GpuAcc_Test") +{ + std::vector backends = { armnn::Compute::GpuAcc }; LessEqualBroadcastTest(backends); } TEST_CASE ("LESS_EQUAL_INT32_GpuAcc_Test") { - std::vector backends = { armnn::Compute::GpuAcc, - armnn::Compute::CpuRef }; + std::vector backends = { armnn::Compute::GpuAcc }; LessEqualInt32Test(backends); } +} // End TEST_SUITE("Comparison_GpuAccTests") + + +TEST_SUITE("Comparison_CpuAccTests") +{ + +TEST_CASE ("EQUAL_FP32_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + EqualFP32Test(backends); +} + +TEST_CASE ("EQUAL_Broadcast_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + EqualBroadcastTest(backends); +} + +TEST_CASE ("EQUAL_INT32_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + EqualInt32Test(backends); +} + +TEST_CASE ("NOT_EQUAL_FP32_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + NotEqualFP32Test(backends); +} + +TEST_CASE ("NOT_EQUAL_Broadcast_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + NotEqualBroadcastTest(backends); +} + +TEST_CASE ("NOT_EQUAL_INT32_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + NotEqualInt32Test(backends); +} + +TEST_CASE ("GREATER_FP32_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + GreaterFP32Test(backends); +} + +TEST_CASE ("GREATER_Broadcast_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + GreaterBroadcastTest(backends); +} + +TEST_CASE ("GREATER_INT32_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + GreaterInt32Test(backends); +} + +TEST_CASE ("GREATER_EQUAL_FP32_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + GreaterEqualFP32Test(backends); +} + +TEST_CASE ("GREATER_EQUAL_Broadcast_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + GreaterEqualBroadcastTest(backends); +} + +TEST_CASE ("GREATER_EQUAL_INT32_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + GreaterEqualInt32Test(backends); +} + +TEST_CASE ("LESS_FP32_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + LessFP32Test(backends); +} + +TEST_CASE ("LESS_Broadcast_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + LessBroadcastTest(backends); +} + +TEST_CASE ("LESS_INT32_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + LessInt32Test(backends); +} + +TEST_CASE ("LESS_EQUAL_FP32_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + LessEqualFP32Test(backends); +} + +TEST_CASE ("LESS_EQUAL_Broadcast_CpuAcc_Test") +{ + std::vector backends = { armnn::Compute::CpuAcc }; + LessEqualBroadcastTest(backends); +} + TEST_CASE ("LESS_EQUAL_INT32_CpuAcc_Test") { - std::vector backends = { armnn::Compute::CpuAcc, - armnn::Compute::CpuRef }; + std::vector backends = { armnn::Compute::CpuAcc }; LessEqualInt32Test(backends); } -} // End TEST_SUITE("ComparisonTest") +} // End TEST_SUITE("Comparison_CpuAccTests") } // namespace armnnDelegate \ No newline at end of file diff --git a/delegate/src/test/ComparisonTestHelper.hpp b/delegate/src/test/ComparisonTestHelper.hpp index 0011c763a0..21fc3a83a4 100644 --- a/delegate/src/test/ComparisonTestHelper.hpp +++ b/delegate/src/test/ComparisonTestHelper.hpp @@ -5,6 +5,8 @@ #pragma once +#include "TestUtils.hpp" + #include #include @@ -225,12 +227,9 @@ void ComparisonTest(tflite::BuiltinOperator comparisonOperatorCode, auto armnnDelegateOutputId = armnnDelegateInterpreter->outputs()[0]; auto armnnDelegateOutputData = armnnDelegateInterpreter->typed_tensor(armnnDelegateOutputId); - for (size_t i = 0; i < expectedOutputValues.size(); i++) - { - CHECK(expectedOutputValues[i] == armnnDelegateOutputData[i]); - CHECK(tfLiteDelageOutputData[i] == expectedOutputValues[i]); - CHECK(tfLiteDelageOutputData[i] == armnnDelegateOutputData[i]); - } + armnnDelegate::CompareData(expectedOutputValues , armnnDelegateOutputData, expectedOutputValues.size()); + armnnDelegate::CompareData(expectedOutputValues , tfLiteDelageOutputData , expectedOutputValues.size()); + armnnDelegate::CompareData(tfLiteDelageOutputData, armnnDelegateOutputData, expectedOutputValues.size()); } } // anonymous namespace \ No newline at end of file diff --git a/delegate/src/test/TestUtils.cpp b/delegate/src/test/TestUtils.cpp index cf3e1fee6b..31c05a678d 100644 --- a/delegate/src/test/TestUtils.cpp +++ b/delegate/src/test/TestUtils.cpp @@ -8,6 +8,26 @@ namespace armnnDelegate { + + +void CompareData(bool tensor1[], bool tensor2[], size_t tensorSize) +{ + auto compareBool = [](auto a, auto b) {return (((a == 0) && (b == 0)) || ((a != 0) && (b != 0)));}; + for (size_t i = 0; i < tensorSize; i++) + { + CHECK(compareBool(tensor1[i], tensor2[i])); + } +} + +void CompareData(std::vector& tensor1, bool tensor2[], size_t tensorSize) +{ + auto compareBool = [](auto a, auto b) {return (((a == 0) && (b == 0)) || ((a != 0) && (b != 0)));}; + for (size_t i = 0; i < tensorSize; i++) + { + CHECK(compareBool(tensor1[i], tensor2[i])); + } +} + void CompareData(float tensor1[], float tensor2[], size_t tensorSize) { for (size_t i = 0; i < tensorSize; i++) diff --git a/delegate/src/test/TestUtils.hpp b/delegate/src/test/TestUtils.hpp index d805f7092e..57ae3ce6fe 100644 --- a/delegate/src/test/TestUtils.hpp +++ b/delegate/src/test/TestUtils.hpp @@ -25,16 +25,28 @@ void FillInput(std::unique_ptr& interpreter, int inputIndex } } -// Can be used to compare data with a tolerance depending on their data type +/// Can be used to compare bool data coming from a tflite interpreter +/// Boolean types get converted to a bit representation in a vector. vector.data() returns a void pointer +/// instead of a pointer to bool. Therefore a special function to compare to vector of bool is required +void CompareData(std::vector& tensor1, bool tensor2[], size_t tensorSize); +void CompareData(bool tensor1[], bool tensor2[], size_t tensorSize); + +/// Can be used to compare float data coming from a tflite interpreter with a tolerance of limit_of_float*100 void CompareData(float tensor1[], float tensor2[], size_t tensorSize); + +/// Can be used to compare int8_t data coming from a tflite interpreter with a tolerance of 1 void CompareData(int8_t tensor1[], int8_t tensor2[], size_t tensorSize); + +/// Can be used to compare uint8_t data coming from a tflite interpreter with a tolerance of 1 void CompareData(uint8_t tensor1[], uint8_t tensor2[], size_t tensorSize); + +/// Can be used to compare int16_t data coming from a tflite interpreter with a tolerance of 1 void CompareData(int16_t tensor1[], int16_t tensor2[], size_t tensorSize); -// Can be used to compare the output tensor shape and values -// from armnnDelegateInterpreter and tfLiteInterpreter. -// Example usage can be found in ControlTestHelper.hpp +/// Can be used to compare the output tensor shape and values +/// from armnnDelegateInterpreter and tfLiteInterpreter. +/// Example usage can be found in ControlTestHelper.hpp template void CompareOutputData(std::unique_ptr& tfLiteInterpreter, std::unique_ptr& armnnDelegateInterpreter, -- cgit v1.2.1