COMPMID-417: Fix validation

Change-Id: I7a745037136bc6e02d177f65fe4f4cd43873b98e
Reviewed-on: http://mpd-gerrit.cambridge.arm.com/87406
Tested-by: Kaizen <jeremy.johnson+kaizengerrit@arm.com>
Reviewed-by: Anthony Barbier <anthony.barbier@arm.com>

5 files changed, 64 insertions, 25 deletions

diff --git a/tests/framework/printers/JSONPrinter.cpp b/tests/framework/printers/JSONPrinter.cpp
index ae19cae67c..4f17e6277b 100644
--- a/tests/framework/printers/JSONPrinter.cpp
+++ b/tests/framework/printers/JSONPrinter.cpp
@@ -46,6 +46,31 @@ void JSONPrinter::print_separator(bool &flag)
     }
 }
 
+template <typename T>
+void JSONPrinter::print_strings(T &&first, T &&last)
+{
+    bool              first_entry = true;
+    std::stringstream log;
+
+    while(first != last)
+    {
+        print_separator(first_entry);
+
+        *_stream << R"(")";
+
+        log.str(*first);
+
+        for(std::string line; !std::getline(log, line).fail();)
+        {
+            *_stream << line << "; ";
+        }
+
+        *_stream << R"(")";
+
+        ++first;
+    }
+}
+
 void JSONPrinter::print_entry(const std::string &name, const std::string &value)
 {
     print_separator(_first_entry);
@@ -90,38 +115,43 @@ void JSONPrinter::print_test_footer()
 
 void JSONPrinter::print_errors_header()
 {
-    print_separator(_first_test_entry);
-
-    _first_error = true;
-    *_stream << R"("errors" : [)";
+    _errors.clear();
+    _expected_errors.clear();
+    _infos.clear();
 }
 
 void JSONPrinter::print_errors_footer()
 {
+    print_separator(_first_test_entry);
+
+    *_stream << R"("errors" : [)";
+    print_strings(_errors.begin(), _errors.end());
+    *_stream << "]";
+
+    *_stream << R"(, "expected_errors" : [)";
+    print_strings(_expected_errors.begin(), _expected_errors.end());
+    *_stream << "]";
+
+    *_stream << R"(, "infos" : [)";
+    print_strings(_infos.begin(), _infos.end());
     *_stream << "]";
 }
 
-void JSONPrinter::print_error(const std::exception &error)
+void JSONPrinter::print_error(const std::exception &error, bool expected)
 {
-    std::stringstream error_log;
-    error_log.str(error.what());
-
-    for(std::string line; !std::getline(error_log, line).fail();)
+    if(expected)
     {
-        print_separator(_first_error);
-
-        *_stream << R"(")" << line << R"(")";
+        _expected_errors.emplace_back(error.what());
+    }
+    else
+    {
+        _errors.emplace_back(error.what());
     }
 }
 
 void JSONPrinter::print_info(const std::string &info)
 {
-    std::istringstream iss(info);
-    for(std::string line; !std::getline(iss, line).fail();)
-    {
-        print_separator(_first_error);
-        *_stream << R"(")" << line << R"(")";
-    }
+    _infos.push_back(info);
 }
 
 void JSONPrinter::print_measurements(const Profiler::MeasurementsMap &measurements)
diff --git a/tests/framework/printers/JSONPrinter.h b/tests/framework/printers/JSONPrinter.h
index 18bd4380b0..a2811ea41b 100644
--- a/tests/framework/printers/JSONPrinter.h
+++ b/tests/framework/printers/JSONPrinter.h
@@ -26,6 +26,8 @@
 
 #include "Printer.h"
 
+#include <list>
+
 namespace arm_compute
 {
 namespace test
@@ -47,17 +49,22 @@ public:
     void print_test_footer() override;
     void print_errors_header() override;
     void print_errors_footer() override;
-    void print_error(const std::exception &error) override;
+    void print_error(const std::exception &error, bool expected) override;
     void print_info(const std::string &info) override;
     void print_measurements(const Profiler::MeasurementsMap &measurements) override;
 
 private:
     void print_separator(bool &flag);
+    template <typename T>
+    void print_strings(T &&first, T &&last);
+
+    std::list<std::string> _infos{};
+    std::list<std::string> _errors{};
+    std::list<std::string> _expected_errors{};
 
     bool _first_entry{ true };
     bool _first_test{ true };
     bool _first_test_entry{ true };
-    bool _first_error{ true };
 };
 } // namespace framework
 } // namespace test
diff --git a/tests/framework/printers/PrettyPrinter.cpp b/tests/framework/printers/PrettyPrinter.cpp
index 2f7df1837a..5eec72a2fe 100644
--- a/tests/framework/printers/PrettyPrinter.cpp
+++ b/tests/framework/printers/PrettyPrinter.cpp
@@ -102,9 +102,10 @@ void PrettyPrinter::print_info(const std::string &info)
     *_stream << begin_color("1") << "INFO: " << info << end_color() << "\n";
 }
 
-void PrettyPrinter::print_error(const std::exception &error)
+void PrettyPrinter::print_error(const std::exception &error, bool expected)
 {
-    *_stream << begin_color("1") << "ERROR: " << error.what() << end_color() << "\n";
+    std::string prefix = expected ? "EXPECTED ERROR: " : "ERROR: ";
+    *_stream << begin_color("1") << prefix << error.what() << end_color() << "\n";
 }
 
 void PrettyPrinter::print_measurements(const Profiler::MeasurementsMap &measurements)
diff --git a/tests/framework/printers/PrettyPrinter.h b/tests/framework/printers/PrettyPrinter.h
index 3e2bebd1f7..f72a613868 100644
--- a/tests/framework/printers/PrettyPrinter.h
+++ b/tests/framework/printers/PrettyPrinter.h
@@ -53,7 +53,7 @@ public:
     void print_test_footer() override;
     void print_errors_header() override;
     void print_errors_footer() override;
-    void print_error(const std::exception &error) override;
+    void print_error(const std::exception &error, bool expected) override;
     void print_info(const std::string &info) override;
     void print_measurements(const Profiler::MeasurementsMap &measurements) override;
 
diff --git a/tests/framework/printers/Printer.h b/tests/framework/printers/Printer.h
index 16a4170d7a..c2a44240ba 100644
--- a/tests/framework/printers/Printer.h
+++ b/tests/framework/printers/Printer.h
@@ -104,9 +104,10 @@ public:
 
     /** Print test error.
      *
-     * @param[in] error Description of the error.
+     * @param[in] error    Description of the error.
+     * @param[in] expected Whether the error was expected or not.
      */
-    virtual void print_error(const std::exception &error) = 0;
+    virtual void print_error(const std::exception &error, bool expected) = 0;
 
     /** Print test log info.
      *