aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--CMakeLists.txt6
-rw-r--r--tests/ExecuteNetwork/ArmNNExecutor.cpp40
-rw-r--r--tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp12
-rw-r--r--tests/ExecuteNetwork/TfliteExecutor.cpp44
-rw-r--r--tests/NetworkExecutionUtils/NetworkExecutionUtils.cpp18
-rw-r--r--tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp28
-rw-r--r--tests/NetworkExecutionUtils/test/NetworkExecutionUtilsTests.cpp37
7 files changed, 77 insertions, 108 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 9eab90afef..3b788cd056 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -835,6 +835,12 @@ if(BUILD_UNIT_TESTS)
)
endif()
+ if(BUILD_TESTS)
+ list(APPEND unittest_sources
+ ./tests/NetworkExecutionUtils/NetworkExecutionUtils.cpp
+ ./tests/NetworkExecutionUtils/test/NetworkExecutionUtilsTests.cpp)
+ endif()
+
foreach(lib ${armnnUnitTestLibraries})
message(STATUS "Adding object library dependency to UnitTests: ${lib}")
list(APPEND unittest_sources $<TARGET_OBJECTS:${lib}>)
diff --git a/tests/ExecuteNetwork/ArmNNExecutor.cpp b/tests/ExecuteNetwork/ArmNNExecutor.cpp
index 730c072836..29ef4c5186 100644
--- a/tests/ExecuteNetwork/ArmNNExecutor.cpp
+++ b/tests/ExecuteNetwork/ArmNNExecutor.cpp
@@ -707,48 +707,14 @@ void ArmNNExecutor::PrintOutputTensors(const armnn::OutputTensors* outputTensors
void ArmNNExecutor::CompareAndPrintResult(std::vector<const void*> otherOutput)
{
unsigned int index = 0;
-
+ std::string typeString;
for (const auto& outputTensors: m_OutputTensorsVec)
{
for (const auto& outputTensor: outputTensors)
{
- float result = 0;
size_t size = outputTensor.second.GetNumBytes();
-
- switch (outputTensor.second.GetDataType())
- {
- case armnn::DataType::Float32:
- {
- result = ComputeRMSE<float>(outputTensor.second.GetMemoryArea(), otherOutput[index++], size);
- break;
- }
- case armnn::DataType::Signed32:
- {
- result = ComputeRMSE<int32_t>(outputTensor.second.GetMemoryArea(), otherOutput[index++], size);
- break;
- }
- case armnn::DataType::QSymmS16:
- {
- result = ComputeRMSE<int16_t>(outputTensor.second.GetMemoryArea(), otherOutput[index++], size);
- break;
- }
- case armnn::DataType::QSymmS8:
- case armnn::DataType::QAsymmS8:
- {
- result = ComputeRMSE<int8_t>(outputTensor.second.GetMemoryArea(), otherOutput[index++], size);
- break;
- }
- case armnn::DataType::QAsymmU8:
- {
- result = ComputeRMSE<uint8_t>(outputTensor.second.GetMemoryArea(), otherOutput[index++], size);
- break;
- }
- default:
- {
- LogAndThrow("Unexpected DataType");
- }
- }
- std::cout << "RMSE: of " << result << "\n";
+ double result = ComputeByteLevelRMSE(outputTensor.second.GetMemoryArea(), otherOutput[index++], size);
+ std::cout << "Byte level root mean square error: " << result << "\n";
}
}
}
diff --git a/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp b/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp
index cba6748b45..007f81890e 100644
--- a/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp
+++ b/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp
@@ -359,17 +359,19 @@ ProgramOptions::ProgramOptions() : m_CxxOptions{"ExecuteNetwork",
cxxopts::value<std::string>()->default_value("parser"))
("C, compare-output",
- "Compare the output of the network with an output file that has been previously "
- "produced by running a network through ExecuteNetwork. See --write-outputs-to-file "
- "to produce an output file for an execution.",
+ "Perform a per byte root mean square error calculation of the inference output with an output"
+ " file that has been previously produced by running a network through ExecuteNetwork."
+ " See --write-outputs-to-file to produce an output file for an execution.",
cxxopts::value<std::string>(m_ExNetParams.m_ComparisonFile))
("B, compare-output-with-backend",
- "Compare the output of the network with a different backend.",
+ "Perform a per byte root mean square error calculation of the output of the inference with a"
+ " different backend.",
cxxopts::value<std::vector<std::string>>())
("A, compare-with-tflite",
- "Compare the output of the network with the tflite ref model.",
+ "Perform an per byte root mean square error calculation of the output of the inference with"
+ " the tflite ref model.",
cxxopts::value<bool>(m_ExNetParams.m_CompareWithTflite)->default_value("false")
->implicit_value("true"));
diff --git a/tests/ExecuteNetwork/TfliteExecutor.cpp b/tests/ExecuteNetwork/TfliteExecutor.cpp
index fc9c21a559..f365623d62 100644
--- a/tests/ExecuteNetwork/TfliteExecutor.cpp
+++ b/tests/ExecuteNetwork/TfliteExecutor.cpp
@@ -230,45 +230,9 @@ void TfLiteExecutor::CompareAndPrintResult(std::vector<const void*> otherOutput)
for (unsigned int outputIndex = 0; outputIndex < m_TfLiteInterpreter->outputs().size(); ++outputIndex)
{
auto tfLiteDelegateOutputId = m_TfLiteInterpreter->outputs()[outputIndex];
- float result = 0;
- switch (m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->type)
- {
- case kTfLiteFloat32:
- {
- result = ComputeRMSE<float>(m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->allocation,
- otherOutput[outputIndex],
- m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->bytes);
-
- break;
- }
- case kTfLiteInt32:
- {
- result = ComputeRMSE<int32_t>(m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->allocation,
- otherOutput[outputIndex],
- m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->bytes);
- break;
- }
- case kTfLiteUInt8:
- {
- result = ComputeRMSE<uint8_t>(m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->allocation,
- otherOutput[outputIndex],
- m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->bytes);
- break;
- }
- case kTfLiteInt8:
- {
- result = ComputeRMSE<int8_t>(m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->allocation,
- otherOutput[outputIndex],
- m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->bytes);
- break;
- }
- default:
- {
- }
- }
-
- std::cout << "RMSE of "
- << m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->name
- << ": " << result << std::endl;
+ size_t size = m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->bytes;
+ double result = ComputeByteLevelRMSE(m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->allocation,
+ otherOutput[outputIndex], size);
+ std::cout << "Byte level root mean square error: " << result << "\n";
}
};
diff --git a/tests/NetworkExecutionUtils/NetworkExecutionUtils.cpp b/tests/NetworkExecutionUtils/NetworkExecutionUtils.cpp
index e3c95d9312..6f9cdf87bc 100644
--- a/tests/NetworkExecutionUtils/NetworkExecutionUtils.cpp
+++ b/tests/NetworkExecutionUtils/NetworkExecutionUtils.cpp
@@ -78,3 +78,21 @@ void LogAndThrow(std::string eMsg)
throw armnn::Exception(eMsg);
}
+/// Compute the root-mean-square error (RMSE) at a byte level between two tensors of the same size.
+/// @param expected
+/// @param actual
+/// @param size size of the tensor in bytes.
+/// @return float the RMSE
+double ComputeByteLevelRMSE(const void* expected, const void* actual, const size_t size)
+{
+ const uint8_t* byteExpected = reinterpret_cast<const uint8_t*>(expected);
+ const uint8_t* byteActual = reinterpret_cast<const uint8_t*>(actual);
+
+ double errorSum = 0;
+ for (unsigned int i = 0; i < size; i++)
+ {
+ int difference = byteExpected[i] - byteActual[i];
+ errorSum += std::pow(difference, 2);
+ }
+ return std::sqrt(errorSum/armnn::numeric_cast<double>(size));
+}
diff --git a/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp b/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp
index 8c97238432..2136c446fb 100644
--- a/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp
+++ b/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp
@@ -73,6 +73,8 @@ std::vector<unsigned int> ParseArray(std::istream& stream);
/// Splits a given string at every accurance of delimiter into a vector of string
std::vector<std::string> ParseStringList(const std::string& inputString, const char* delimiter);
+double ComputeByteLevelRMSE(const void* expected, const void* actual, const size_t size);
+
/// Dequantize an array of a given type
/// @param array Type erased array to dequantize
/// @param numElements Elements in the array
@@ -285,29 +287,3 @@ std::vector<T> ParseArrayImpl(std::istream& stream, TParseElementFunc parseEleme
return result;
}
-
-/// Compute the root-mean-square error (RMSE)
-/// @param expected
-/// @param actual
-/// @param size size of the tensor
-/// @return float the RMSE
-template<typename T>
-float ComputeRMSE(const void* expected, const void* actual, const size_t size)
-{
- auto typedExpected = reinterpret_cast<const T*>(expected);
- auto typedActual = reinterpret_cast<const T*>(actual);
-
- T errorSum = 0;
-
- for (unsigned int i = 0; i < size; i++)
- {
- if (std::abs(typedExpected[i] - typedActual[i]) != 0)
- {
- std::cout << "";
- }
- errorSum += std::pow(std::abs(typedExpected[i] - typedActual[i]), 2);
- }
-
- float rmse = std::sqrt(armnn::numeric_cast<float>(errorSum) / armnn::numeric_cast<float>(size / sizeof(T)));
- return rmse;
-} \ No newline at end of file
diff --git a/tests/NetworkExecutionUtils/test/NetworkExecutionUtilsTests.cpp b/tests/NetworkExecutionUtils/test/NetworkExecutionUtilsTests.cpp
new file mode 100644
index 0000000000..d11fe571b0
--- /dev/null
+++ b/tests/NetworkExecutionUtils/test/NetworkExecutionUtilsTests.cpp
@@ -0,0 +1,37 @@
+//
+// Copyright © 2023 Arm Ltd and Contributors. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#include "../NetworkExecutionUtils.hpp"
+
+#include <doctest/doctest.h>
+
+namespace
+{
+
+TEST_SUITE("NetworkExecutionUtilsTests")
+{
+
+TEST_CASE ("ComputeByteLevelRMSE")
+{
+ // Bytes.
+ const uint8_t expected[] = {1, 128, 255};
+ const uint8_t actual[] = {0, 127, 254};
+
+ CHECK(ComputeByteLevelRMSE(expected, expected, 3) == 0);
+ CHECK(ComputeByteLevelRMSE(expected, actual, 3) == 1.0);
+
+ // Floats.
+ const float expectedFloat[] =
+ {55.20419, 24.58061, 67.76520, 47.31617, 55.58102, 44.64565, 105.76307, 54.65538, 80.41088, 66.05208};
+ const float actualFloat[] =
+ {13.87187, 14.16160, 49.28846, 25.89192, 97.70659, 91.30055, 15.88831, 4.79960, 102.99205, 51.28290};
+ const double expectedResult = 74.059098023; // Calculated manually.
+ CHECK(ComputeByteLevelRMSE(expectedFloat, expectedFloat, sizeof(float) * 10) == 0);
+ CHECK(ComputeByteLevelRMSE(expectedFloat, actualFloat, sizeof(float) * 10) == doctest::Approx(expectedResult));
+}
+
+} // End of TEST_SUITE("NetworkExecutionUtilsTests")
+
+} // anonymous namespace \ No newline at end of file
generated by cgit v1.2.1 (git 2.23.0) at 2024-07-27 20:59:01 +0000