Fixing compare output feature in ExecuteNetwork

The -A -B -C options in execute network were attempting to calculate
the RMS error over output tensors. However, the calculation was mixing
tensor elements and bytes when doing the calculation. This patch
changes the calculation to use a per byte RMS error calculation.

Signed-off-by: Colm Donelan <colm.donelan@arm.com>
Change-Id: If30230a16cfed1a8804b4d54ed1abcd371f26664

6 files changed, 71 insertions, 108 deletions

diff --git a/tests/ExecuteNetwork/ArmNNExecutor.cpp b/tests/ExecuteNetwork/ArmNNExecutor.cpp
index 730c072836..29ef4c5186 100644
--- a/tests/ExecuteNetwork/ArmNNExecutor.cpp
+++ b/tests/ExecuteNetwork/ArmNNExecutor.cpp
@@ -707,48 +707,14 @@ void ArmNNExecutor::PrintOutputTensors(const armnn::OutputTensors* outputTensors
 void ArmNNExecutor::CompareAndPrintResult(std::vector<const void*> otherOutput)
 {
     unsigned int index = 0;
-
+    std::string typeString;
     for (const auto& outputTensors: m_OutputTensorsVec)
     {
         for (const auto& outputTensor: outputTensors)
         {
-            float result = 0;
             size_t size = outputTensor.second.GetNumBytes();
-
-            switch (outputTensor.second.GetDataType())
-            {
-                case armnn::DataType::Float32:
-                {
-                    result = ComputeRMSE<float>(outputTensor.second.GetMemoryArea(), otherOutput[index++], size);
-                    break;
-                }
-                case armnn::DataType::Signed32:
-                {
-                    result = ComputeRMSE<int32_t>(outputTensor.second.GetMemoryArea(), otherOutput[index++], size);
-                    break;
-                }
-                case armnn::DataType::QSymmS16:
-                {
-                    result = ComputeRMSE<int16_t>(outputTensor.second.GetMemoryArea(), otherOutput[index++], size);
-                    break;
-                }
-                case armnn::DataType::QSymmS8:
-                case armnn::DataType::QAsymmS8:
-                {
-                    result = ComputeRMSE<int8_t>(outputTensor.second.GetMemoryArea(), otherOutput[index++], size);
-                    break;
-                }
-                case armnn::DataType::QAsymmU8:
-                {
-                    result = ComputeRMSE<uint8_t>(outputTensor.second.GetMemoryArea(), otherOutput[index++], size);
-                    break;
-                }
-                default:
-                {
-                    LogAndThrow("Unexpected DataType");
-                }
-            }
-            std::cout << "RMSE: of " << result << "\n";
+            double result = ComputeByteLevelRMSE(outputTensor.second.GetMemoryArea(), otherOutput[index++], size);
+            std::cout << "Byte level root mean square error: " << result << "\n";
         }
     }
 }
diff --git a/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp b/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp
index cba6748b45..007f81890e 100644
--- a/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp
+++ b/tests/ExecuteNetwork/ExecuteNetworkProgramOptions.cpp
@@ -359,17 +359,19 @@ ProgramOptions::ProgramOptions() : m_CxxOptions{"ExecuteNetwork",
                  cxxopts::value<std::string>()->default_value("parser"))
 
                 ("C, compare-output",
-                 "Compare the output of the network with an output file that has been previously "
-                 "produced by running a network through ExecuteNetwork. See --write-outputs-to-file "
-                 "to produce an output file for an execution.",
+                 "Perform a per byte root mean square error calculation of the inference output with an output"
+                 " file that has been previously produced by running a network through ExecuteNetwork."
+                 " See --write-outputs-to-file to produce an output file for an execution.",
                  cxxopts::value<std::string>(m_ExNetParams.m_ComparisonFile))
 
                 ("B, compare-output-with-backend",
-                 "Compare the output of the network with a different backend.",
+                 "Perform a per byte root mean square error calculation of the output of the inference with a"
+                 " different backend.",
                  cxxopts::value<std::vector<std::string>>())
 
                 ("A, compare-with-tflite",
-                 "Compare the output of the network with the tflite ref model.",
+                 "Perform an per byte root mean square error calculation of the output of the inference with"
+                 " the tflite ref model.",
                  cxxopts::value<bool>(m_ExNetParams.m_CompareWithTflite)->default_value("false")
                          ->implicit_value("true"));
 
diff --git a/tests/ExecuteNetwork/TfliteExecutor.cpp b/tests/ExecuteNetwork/TfliteExecutor.cpp
index fc9c21a559..f365623d62 100644
--- a/tests/ExecuteNetwork/TfliteExecutor.cpp
+++ b/tests/ExecuteNetwork/TfliteExecutor.cpp
@@ -230,45 +230,9 @@ void TfLiteExecutor::CompareAndPrintResult(std::vector<const void*> otherOutput)
     for (unsigned int outputIndex = 0; outputIndex < m_TfLiteInterpreter->outputs().size(); ++outputIndex)
     {
         auto tfLiteDelegateOutputId = m_TfLiteInterpreter->outputs()[outputIndex];
-        float result = 0;
-        switch (m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->type)
-        {
-            case kTfLiteFloat32:
-            {
-                result =  ComputeRMSE<float>(m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->allocation,
-                                             otherOutput[outputIndex],
-                                             m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->bytes);
-
-                break;
-            }
-            case kTfLiteInt32:
-            {
-                result =  ComputeRMSE<int32_t>(m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->allocation,
-                                                    otherOutput[outputIndex],
-                                                    m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->bytes);
-                break;
-            }
-            case kTfLiteUInt8:
-            {
-                result =  ComputeRMSE<uint8_t>(m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->allocation,
-                                                    otherOutput[outputIndex],
-                                                    m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->bytes);
-                break;
-            }
-            case kTfLiteInt8:
-            {
-                result =  ComputeRMSE<int8_t>(m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->allocation,
-                                                    otherOutput[outputIndex],
-                                                    m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->bytes);
-                break;
-            }
-            default:
-            {
-            }
-        }
-
-        std::cout << "RMSE of "
-                  << m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->name
-                  << ": " << result << std::endl;
+        size_t size = m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->bytes;
+        double result = ComputeByteLevelRMSE(m_TfLiteInterpreter->tensor(tfLiteDelegateOutputId)->allocation,
+                                             otherOutput[outputIndex], size);
+        std::cout << "Byte level root mean square error: " << result << "\n";
     }
 };
diff --git a/tests/NetworkExecutionUtils/NetworkExecutionUtils.cpp b/tests/NetworkExecutionUtils/NetworkExecutionUtils.cpp
index e3c95d9312..6f9cdf87bc 100644
--- a/tests/NetworkExecutionUtils/NetworkExecutionUtils.cpp
+++ b/tests/NetworkExecutionUtils/NetworkExecutionUtils.cpp
@@ -78,3 +78,21 @@ void LogAndThrow(std::string eMsg)
     throw armnn::Exception(eMsg);
 }
 
+/// Compute the root-mean-square error (RMSE) at a byte level between two tensors of the same size.
+/// @param expected
+/// @param actual
+/// @param size size of the tensor in bytes.
+/// @return float the RMSE
+double ComputeByteLevelRMSE(const void* expected, const void* actual, const size_t size)
+{
+    const uint8_t* byteExpected = reinterpret_cast<const uint8_t*>(expected);
+    const uint8_t* byteActual = reinterpret_cast<const uint8_t*>(actual);
+
+    double errorSum = 0;
+    for (unsigned int i = 0; i < size; i++)
+    {
+        int difference = byteExpected[i] - byteActual[i];
+        errorSum += std::pow(difference, 2);
+    }
+    return std::sqrt(errorSum/armnn::numeric_cast<double>(size));
+}
diff --git a/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp b/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp
index 8c97238432..2136c446fb 100644
--- a/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp
+++ b/tests/NetworkExecutionUtils/NetworkExecutionUtils.hpp
@@ -73,6 +73,8 @@ std::vector<unsigned int> ParseArray(std::istream& stream);
 /// Splits a given string at every accurance of delimiter into a vector of string
 std::vector<std::string> ParseStringList(const std::string& inputString, const char* delimiter);
 
+double ComputeByteLevelRMSE(const void* expected, const void* actual, const size_t size);
+
 /// Dequantize an array of a given type
 /// @param array Type erased array to dequantize
 /// @param numElements Elements in the array
@@ -285,29 +287,3 @@ std::vector<T> ParseArrayImpl(std::istream& stream, TParseElementFunc parseEleme
 
     return result;
 }
-
-/// Compute the root-mean-square error (RMSE)
-/// @param expected
-/// @param actual
-/// @param size size of the tensor
-/// @return float the RMSE
-template<typename T>
-float ComputeRMSE(const void* expected, const void* actual, const size_t size)
-{
-    auto typedExpected = reinterpret_cast<const T*>(expected);
-    auto typedActual = reinterpret_cast<const T*>(actual);
-
-    T errorSum = 0;
-
-    for (unsigned int i = 0; i < size; i++)
-    {
-        if (std::abs(typedExpected[i] - typedActual[i]) != 0)
-        {
-            std::cout << "";
-        }
-        errorSum += std::pow(std::abs(typedExpected[i] - typedActual[i]), 2);
-    }
-
-    float rmse = std::sqrt(armnn::numeric_cast<float>(errorSum) / armnn::numeric_cast<float>(size / sizeof(T)));
-    return rmse;
-}
\ No newline at end of file
diff --git a/tests/NetworkExecutionUtils/test/NetworkExecutionUtilsTests.cpp b/tests/NetworkExecutionUtils/test/NetworkExecutionUtilsTests.cpp
new file mode 100644
index 0000000000..d11fe571b0
--- /dev/null
+++ b/tests/NetworkExecutionUtils/test/NetworkExecutionUtilsTests.cpp
@@ -0,0 +1,37 @@
+//
+// Copyright © 2023 Arm Ltd and Contributors. All rights reserved.
+// SPDX-License-Identifier: MIT
+//
+
+#include "../NetworkExecutionUtils.hpp"
+
+#include <doctest/doctest.h>
+
+namespace
+{
+
+TEST_SUITE("NetworkExecutionUtilsTests")
+{
+
+TEST_CASE ("ComputeByteLevelRMSE")
+{
+    // Bytes.
+    const uint8_t expected[] = {1, 128, 255};
+    const uint8_t actual[] = {0, 127, 254};
+
+    CHECK(ComputeByteLevelRMSE(expected, expected, 3) == 0);
+    CHECK(ComputeByteLevelRMSE(expected, actual, 3) == 1.0);
+
+    // Floats.
+    const float expectedFloat[] =
+        {55.20419, 24.58061, 67.76520, 47.31617, 55.58102, 44.64565, 105.76307, 54.65538, 80.41088, 66.05208};
+    const float actualFloat[] =
+        {13.87187, 14.16160, 49.28846, 25.89192, 97.70659, 91.30055, 15.88831, 4.79960, 102.99205, 51.28290};
+    const double expectedResult = 74.059098023; // Calculated manually.
+    CHECK(ComputeByteLevelRMSE(expectedFloat, expectedFloat, sizeof(float) * 10) == 0);
+    CHECK(ComputeByteLevelRMSE(expectedFloat, actualFloat, sizeof(float) * 10) == doctest::Approx(expectedResult));
+}
+
+} // End of TEST_SUITE("NetworkExecutionUtilsTests")
+
+} // anonymous namespace
\ No newline at end of file