COMPMID-415: Use printer for errors

Change-Id: Idc2fc1dfd5706580d15c2bbfffe2830d41075a4b
Reviewed-on: http://mpd-gerrit.cambridge.arm.com/82908
Reviewed-by: Anthony Barbier <anthony.barbier@arm.com>
Tested-by: Kaizen <jeremy.johnson+kaizengerrit@arm.com>

11 files changed, 139 insertions, 51 deletions

diff --git a/framework/Asserts.h b/framework/Asserts.h
index 4fd82abb97..b545a9ebba 100644
--- a/framework/Asserts.h
+++ b/framework/Asserts.h
@@ -36,8 +36,6 @@ namespace test
 {
 namespace framework
 {
-namespace detail
-{
 // Cast char values to int so that their numeric value are printed.
 inline int make_printable(int8_t value)
 {
@@ -51,7 +49,7 @@ inline unsigned int make_printable(uint8_t value)
 
 // Everything else can be printed as its own type.
 template <typename T>
-inline T &&make_printable(T &&value)
+inline T make_printable(T &&value)
 {
     return value;
 }
@@ -63,6 +61,8 @@ inline T &&make_printable(T &&value)
         arm_compute::test::framework::Framework::get().add_test_info(info.str()); \
     }
 
+namespace detail
+{
 #define ARM_COMPUTE_TEST_COMP_FACTORY(SEVERITY, SEVERITY_NAME, COMP, COMP_NAME, ERROR_CALL)                                            \
     template <typename T, typename U>                                                                                                  \
     void ARM_COMPUTE_##SEVERITY##_##COMP_NAME##_IMPL(T &&x, U &&y, const std::string &x_str, const std::string &y_str, LogLevel level) \
@@ -71,9 +71,9 @@ inline T &&make_printable(T &&value)
         {                                                                                                                              \
             std::stringstream msg;                                                                                                     \
             msg << #SEVERITY_NAME " '" << x_str << " " #COMP " " << y_str << "' failed. ["                                             \
-                << std::boolalpha << arm_compute::test::framework::detail::make_printable(x)                                           \
+                << std::boolalpha << arm_compute::test::framework::make_printable(x)                                                   \
                 << " " #COMP " "                                                                                                       \
-                << std::boolalpha << arm_compute::test::framework::detail::make_printable(y)                                           \
+                << std::boolalpha << arm_compute::test::framework::make_printable(y)                                                   \
                 << "]\n";                                                                                                              \
             arm_compute::test::framework::Framework::get().print_test_info(msg);                                                       \
             ERROR_CALL                                                                                                                 \
@@ -81,10 +81,11 @@ inline T &&make_printable(T &&value)
         arm_compute::test::framework::Framework::get().clear_test_info();                                                              \
     }
 
-ARM_COMPUTE_TEST_COMP_FACTORY(EXPECT, Expectation, ==, EQUAL, arm_compute::test::framework::Framework::get().log_failed_expectation(msg.str(), level);)
-ARM_COMPUTE_TEST_COMP_FACTORY(EXPECT, Expectation, !=, NOT_EQUAL, arm_compute::test::framework::Framework::get().log_failed_expectation(msg.str(), level);)
+ARM_COMPUTE_TEST_COMP_FACTORY(EXPECT, Expectation, ==, EQUAL, arm_compute::test::framework::Framework::get().log_failed_expectation(arm_compute::test::framework::TestError(msg.str(), level));)
+ARM_COMPUTE_TEST_COMP_FACTORY(EXPECT, Expectation, !=, NOT_EQUAL, arm_compute::test::framework::Framework::get().log_failed_expectation(arm_compute::test::framework::TestError(msg.str(), level));)
 ARM_COMPUTE_TEST_COMP_FACTORY(ASSERT, Assertion, ==, EQUAL, throw arm_compute::test::framework::TestError(msg.str(), level);)
 ARM_COMPUTE_TEST_COMP_FACTORY(ASSERT, Assertion, !=, NOT_EQUAL, throw arm_compute::test::framework::TestError(msg.str(), level);)
+} // namespace detail
 
 #define ARM_COMPUTE_ASSERT_NOT_EQUAL(X, Y) \
     arm_compute::test::framework::detail::ARM_COMPUTE_ASSERT_NOT_EQUAL_IMPL(X, Y, #X, #Y, LogLevel::ERRORS)
@@ -112,20 +113,19 @@ ARM_COMPUTE_TEST_COMP_FACTORY(ASSERT, Assertion, !=, NOT_EQUAL, throw arm_comput
         arm_compute::test::framework::Framework::get().clear_test_info();                                             \
     } while(false)
 
-#define ARM_COMPUTE_EXPECT(X, LEVEL)                                                                 \
-    do                                                                                               \
-    {                                                                                                \
-        const auto &x = X;                                                                           \
-        if(!x)                                                                                       \
-        {                                                                                            \
-            std::stringstream msg;                                                                   \
-            msg << "Expectation '" #X "' failed.\n";                                                 \
-            arm_compute::test::framework::Framework::get().print_test_info(msg);                     \
-            arm_compute::test::framework::Framework::get().log_failed_expectation(msg.str(), LEVEL); \
-        }                                                                                            \
-        arm_compute::test::framework::Framework::get().clear_test_info();                            \
+#define ARM_COMPUTE_EXPECT(X, LEVEL)                                                                                                          \
+    do                                                                                                                                        \
+    {                                                                                                                                         \
+        const auto &x = X;                                                                                                                    \
+        if(!x)                                                                                                                                \
+        {                                                                                                                                     \
+            std::stringstream msg;                                                                                                            \
+            msg << "Expectation '" #X "' failed.\n";                                                                                          \
+            arm_compute::test::framework::Framework::get().print_test_info(msg);                                                              \
+            arm_compute::test::framework::Framework::get().log_failed_expectation(arm_compute::test::framework::TestError(msg.str(), LEVEL)); \
+        }                                                                                                                                     \
+        arm_compute::test::framework::Framework::get().clear_test_info();                                                                     \
     } while(false)
-} // namespace detail
 } // namespace framework
 } // namespace test
 } // namespace arm_compute
diff --git a/framework/Exceptions.cpp b/framework/Exceptions.cpp
index d573fe91fa..46b1eac974 100644
--- a/framework/Exceptions.cpp
+++ b/framework/Exceptions.cpp
@@ -102,15 +102,25 @@ std::string to_string(LogLevel level)
     return stream.str();
 }
 
-TestError::TestError(const std::string &msg, LogLevel level)
-    : runtime_error{ msg }, _level{ level }
+TestError::TestError(const std::string &msg, LogLevel level, std::string context)
+    : std::runtime_error{ msg }, _level{ level }, _msg{ msg }, _context{ std::move(context) }, _combined{ "ERROR: " + msg }
 {
+    if(!_context.empty())
+    {
+        _combined += "\nCONTEXT:\n" + _context;
+    }
 }
 
 LogLevel TestError::level() const
 {
     return _level;
 }
+
+const char *TestError::what() const noexcept
+{
+    return _combined.c_str();
+}
+
 } // namespace framework
 } // namespace test
 } // namespace arm_compute
diff --git a/framework/Exceptions.h b/framework/Exceptions.h
index de617602dc..edb0ed92c9 100644
--- a/framework/Exceptions.h
+++ b/framework/Exceptions.h
@@ -71,10 +71,11 @@ public:
 
     /** Construct error with severity.
      *
-     * @param[in] msg   Error message.
-     * @param[in] level Severity level.
+     * @param[in] msg     Error message.
+     * @param[in] level   Severity level.
+     * @param[in] context Context.
      */
-    TestError(const std::string &msg, LogLevel level);
+    TestError(const std::string &msg, LogLevel level, std::string context = "");
 
     /** Severity of the error.
      *
@@ -82,8 +83,13 @@ public:
      */
     LogLevel level() const;
 
+    const char *what() const noexcept override;
+
 private:
-    LogLevel _level{ LogLevel::ERRORS };
+    LogLevel    _level{ LogLevel::ERRORS };
+    std::string _msg{};
+    std::string _context{};
+    std::string _combined{};
 };
 } // namespace framework
 } // namespace test
diff --git a/framework/Framework.cpp b/framework/Framework.cpp
index 5a2b02fb21..315f8ebea7 100644
--- a/framework/Framework.cpp
+++ b/framework/Framework.cpp
@@ -166,11 +166,11 @@ void Framework::log_test_end(const TestInfo &info)
     }
 }
 
-void Framework::log_failed_expectation(const std::string &msg, LogLevel level)
+void Framework::log_failed_expectation(const TestError &error)
 {
-    if(_log_level >= level)
+    if(_log_level >= error.level() && _printer != nullptr)
     {
-        std::cerr << "ERROR: " << msg << "\n";
+        _printer->print_error(error);
     }
 
     if(_current_test_result != nullptr)
@@ -225,6 +225,11 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
 
     _current_test_result = &result;
 
+    if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
+    {
+        _printer->print_errors_header();
+    }
+
     try
     {
         std::unique_ptr<TestCase> test_case = test_factory.make();
@@ -256,9 +261,9 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
         }
         catch(const TestError &error)
         {
-            if(_log_level >= error.level())
+            if(_log_level >= error.level() && _printer != nullptr)
             {
-                std::cerr << "FATAL ERROR: " << error.what() << "\n";
+                _printer->print_error(error);
             }
 
             result.status = TestResult::Status::FAILED;
@@ -271,9 +276,11 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
 #ifdef ARM_COMPUTE_CL
         catch(const ::cl::Error &error)
         {
-            if(_log_level >= LogLevel::ERRORS)
+            if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
             {
-                std::cerr << "FATAL CL ERROR: " << error.what() << " with code " << error.err() << "\n";
+                std::stringstream stream;
+                stream << "Error code: " << error.err();
+                _printer->print_error(TestError(error.what(), LogLevel::ERRORS, stream.str()));
             }
 
             result.status = TestResult::Status::FAILED;
@@ -286,9 +293,9 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
 #endif /* ARM_COMPUTE_CL */
         catch(const std::exception &error)
         {
-            if(_log_level >= LogLevel::ERRORS)
+            if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
             {
-                std::cerr << "FATAL ERROR: Received unhandled error: '" << error.what() << "'\n";
+                _printer->print_error(error);
             }
 
             result.status = TestResult::Status::CRASHED;
@@ -300,9 +307,9 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
         }
         catch(...)
         {
-            if(_log_level >= LogLevel::ERRORS)
+            if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
             {
-                std::cerr << "FATAL ERROR: Received unhandled exception\n";
+                _printer->print_error(TestError("Received unknown exception"));
             }
 
             result.status = TestResult::Status::CRASHED;
@@ -315,9 +322,9 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
     }
     catch(const std::exception &error)
     {
-        if(_log_level >= LogLevel::ERRORS)
+        if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
         {
-            std::cerr << "FATAL ERROR: Received unhandled error during fixture creation: '" << error.what() << "'\n";
+            _printer->print_error(error);
         }
 
         result.status = TestResult::Status::CRASHED;
@@ -329,9 +336,9 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
     }
     catch(...)
     {
-        if(_log_level >= LogLevel::ERRORS)
+        if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
         {
-            std::cerr << "FATAL ERROR: Received unhandled exception during fixture creation\n";
+            _printer->print_error(TestError("Received unknown exception"));
         }
 
         result.status = TestResult::Status::CRASHED;
@@ -342,6 +349,11 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
         }
     }
 
+    if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
+    {
+        _printer->print_errors_footer();
+    }
+
     _current_test_result = nullptr;
 
     if(result.status == TestResult::Status::FAILED)
diff --git a/framework/Framework.h b/framework/Framework.h
index 3526eee82f..055392cdae 100644
--- a/framework/Framework.h
+++ b/framework/Framework.h
@@ -187,10 +187,9 @@ public:
 
     /** Tell the framework that the currently running test case failed a non-fatal expectation.
      *
-     * @param[in] msg   Description of the failure.
-     * @param[in] level Severity of the failed expectation.
+     * @param[in] error Description of the error.
      */
-    void log_failed_expectation(const std::string &msg, LogLevel level = LogLevel::ERRORS);
+    void log_failed_expectation(const TestError &error);
 
     /** Number of iterations per test case.
      *
diff --git a/framework/printers/JSONPrinter.cpp b/framework/printers/JSONPrinter.cpp
index 7806644418..3408174b48 100644
--- a/framework/printers/JSONPrinter.cpp
+++ b/framework/printers/JSONPrinter.cpp
@@ -78,6 +78,7 @@ void JSONPrinter::print_test_header(const TestInfo &info)
 {
     print_separator(_first_test);
 
+    _first_test_entry = true;
     *_stream << R"(")" << info.name << R"(" : {)";
 }
 
@@ -86,8 +87,32 @@ void JSONPrinter::print_test_footer()
     *_stream << "}";
 }
 
+void JSONPrinter::print_errors_header()
+{
+    print_separator(_first_test_entry);
+
+    _first_error = true;
+    *_stream << R"("errors" : [)";
+}
+
+void JSONPrinter::print_errors_footer()
+{
+    *_stream << "]";
+}
+
+void JSONPrinter::print_error(const std::exception &error)
+{
+    print_separator(_first_error);
+
+    *_stream << R"(")" << error.what() << R"(")";
+}
+
 void JSONPrinter::print_measurements(const Profiler::MeasurementsMap &measurements)
 {
+    print_separator(_first_test_entry);
+
+    *_stream << R"("measurements" : {)";
+
     for(auto i_it = measurements.cbegin(), i_end = measurements.cend(); i_it != i_end;)
     {
         *_stream << R"(")" << i_it->first << R"(" : {)";
@@ -129,6 +154,8 @@ void JSONPrinter::print_measurements(const Profiler::MeasurementsMap &measuremen
             *_stream << ",";
         }
     }
+
+    *_stream << "}";
 }
 } // namespace framework
 } // namespace test
diff --git a/framework/printers/JSONPrinter.h b/framework/printers/JSONPrinter.h
index 7b34941ca1..14c8b35cb9 100644
--- a/framework/printers/JSONPrinter.h
+++ b/framework/printers/JSONPrinter.h
@@ -45,13 +45,18 @@ public:
     void print_run_footer() override;
     void print_test_header(const TestInfo &info) override;
     void print_test_footer() override;
+    void print_errors_header() override;
+    void print_errors_footer() override;
+    void print_error(const std::exception &error) override;
     void print_measurements(const Profiler::MeasurementsMap &measurements) override;
 
 private:
     void print_separator(bool &flag);
 
-    bool _first_test{ true };
     bool _first_entry{ true };
+    bool _first_test{ true };
+    bool _first_test_entry{ true };
+    bool _first_error{ true };
 };
 } // namespace framework
 } // namespace test
diff --git a/framework/printers/PrettyPrinter.cpp b/framework/printers/PrettyPrinter.cpp
index fd90401693..631d96917a 100644
--- a/framework/printers/PrettyPrinter.cpp
+++ b/framework/printers/PrettyPrinter.cpp
@@ -88,6 +88,19 @@ void PrettyPrinter::print_test_footer()
 {
 }
 
+void PrettyPrinter::print_errors_header()
+{
+}
+
+void PrettyPrinter::print_errors_footer()
+{
+}
+
+void PrettyPrinter::print_error(const std::exception &error)
+{
+    *_stream << begin_color("1") << error.what() << end_color() << "\n";
+}
+
 void PrettyPrinter::print_measurements(const Profiler::MeasurementsMap &measurements)
 {
     for(const auto &instrument : measurements)
diff --git a/framework/printers/PrettyPrinter.h b/framework/printers/PrettyPrinter.h
index 893b1fadd2..fa7b7b2c59 100644
--- a/framework/printers/PrettyPrinter.h
+++ b/framework/printers/PrettyPrinter.h
@@ -51,6 +51,9 @@ public:
     void print_run_footer() override;
     void print_test_header(const TestInfo &info) override;
     void print_test_footer() override;
+    void print_errors_header() override;
+    void print_errors_footer() override;
+    void print_error(const std::exception &error) override;
     void print_measurements(const Profiler::MeasurementsMap &measurements) override;
 
 private:
diff --git a/framework/printers/Printer.h b/framework/printers/Printer.h
index 7d8af12416..85b7a570c8 100644
--- a/framework/printers/Printer.h
+++ b/framework/printers/Printer.h
@@ -29,6 +29,7 @@
 #include <fstream>
 #include <iostream>
 #include <ostream>
+#include <stdexcept>
 
 namespace arm_compute
 {
@@ -95,6 +96,18 @@ public:
     /** Print footer after a test. */
     virtual void print_test_footer() = 0;
 
+    /** Print header before errors. */
+    virtual void print_errors_header() = 0;
+
+    /** Print footer after errors. */
+    virtual void print_errors_footer() = 0;
+
+    /** Print test error.
+     *
+     * @param[in] error Description of the error.
+     */
+    virtual void print_error(const std::exception &error) = 0;
+
     /** Print measurements for a test.
      *
      * @param[in] measurements Measurements as collected by a @ref Profiler.
diff --git a/tests/validation_new/Validation.h b/tests/validation_new/Validation.h
index b21d12932a..0766636c78 100644
--- a/tests/validation_new/Validation.h
+++ b/tests/validation_new/Validation.h
@@ -308,9 +308,9 @@ void validate(const IAccessor &tensor, const SimpleTensor<T> &reference, const V
                 {
                     ARM_COMPUTE_TEST_INFO("id = " << id);
                     ARM_COMPUTE_TEST_INFO("channel = " << c);
-                    ARM_COMPUTE_TEST_INFO("target = " << std::setprecision(5) << target_value);
-                    ARM_COMPUTE_TEST_INFO("reference = " << std::setprecision(5) << reference_value);
-                    ARM_COMPUTE_TEST_INFO("tolerance = " << std::setprecision(5) << tolerance_value);
+                    ARM_COMPUTE_TEST_INFO("target = " << std::setprecision(5) << framework::make_printable(target_value));
+                    ARM_COMPUTE_TEST_INFO("reference = " << std::setprecision(5) << framework::make_printable(reference_value));
+                    ARM_COMPUTE_TEST_INFO("tolerance = " << std::setprecision(5) << framework::make_printable(static_cast<typename U::value_type>(tolerance_value)));
                     ARM_COMPUTE_EXPECT_EQUAL(target_value, reference_value, framework::LogLevel::DEBUG);
 
                     ++num_mismatches;
@@ -335,9 +335,9 @@ void validate(const IAccessor &tensor, const SimpleTensor<T> &reference, const V
 template <typename T, typename U>
 void validate(T target, T reference, U tolerance)
 {
-    ARM_COMPUTE_TEST_INFO("reference = " << std::setprecision(5) << reference);
-    ARM_COMPUTE_TEST_INFO("target = " << std::setprecision(5) << target);
-    ARM_COMPUTE_TEST_INFO("tolerance = " << std::setprecision(5) << tolerance);
+    ARM_COMPUTE_TEST_INFO("reference = " << std::setprecision(5) << framework::make_printable(reference));
+    ARM_COMPUTE_TEST_INFO("target = " << std::setprecision(5) << framework::make_printable(target));
+    ARM_COMPUTE_TEST_INFO("tolerance = " << std::setprecision(5) << framework::make_printable(static_cast<typename U::value_type>(tolerance)));
     ARM_COMPUTE_EXPECT((compare<U, typename U::value_type>(target, reference, tolerance)), framework::LogLevel::ERRORS);
 }
 } // namespace validation