From 5b61fd3fbaf41031232296abde56258d12ba3340 Mon Sep 17 00:00:00 2001 From: Moritz Pflanzer Date: Tue, 12 Sep 2017 15:51:33 +0100 Subject: COMPMID-417: Fix validation Change-Id: I7a745037136bc6e02d177f65fe4f4cd43873b98e Reviewed-on: http://mpd-gerrit.cambridge.arm.com/87406 Tested-by: Kaizen Reviewed-by: Anthony Barbier --- tests/framework/Framework.cpp | 18 ++++---- tests/framework/printers/JSONPrinter.cpp | 66 ++++++++++++++++++++++-------- tests/framework/printers/JSONPrinter.h | 11 ++++- tests/framework/printers/PrettyPrinter.cpp | 5 ++- tests/framework/printers/PrettyPrinter.h | 2 +- tests/framework/printers/Printer.h | 5 ++- 6 files changed, 75 insertions(+), 32 deletions(-) (limited to 'tests/framework') diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp index 343b7a8561..31e524338b 100644 --- a/tests/framework/Framework.cpp +++ b/tests/framework/Framework.cpp @@ -164,7 +164,8 @@ void Framework::log_failed_expectation(const TestError &error) { if(_log_level >= error.level() && _printer != nullptr) { - _printer->print_error(error); + constexpr bool expected_error = true; + _printer->print_error(error, expected_error); } if(_current_test_result != nullptr) @@ -232,6 +233,8 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) _printer->print_errors_header(); } + const bool is_expected_failure = test_factory.status() == TestCaseFactory::Status::EXPECTED_FAILURE; + try { std::unique_ptr test_case = test_factory.make(); @@ -265,7 +268,7 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) { if(_log_level >= error.level() && _printer != nullptr) { - _printer->print_error(error); + _printer->print_error(error, is_expected_failure); } result.status = TestResult::Status::FAILED; @@ -282,7 +285,8 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) { std::stringstream stream; stream << "Error code: " << error.err(); - _printer->print_error(TestError(error.what(), LogLevel::ERRORS, stream.str())); + TestError test_error(error.what(), LogLevel::ERRORS, stream.str()); + _printer->print_error(test_error, is_expected_failure); } result.status = TestResult::Status::FAILED; @@ -297,7 +301,7 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) { if(_log_level >= LogLevel::ERRORS && _printer != nullptr) { - _printer->print_error(error); + _printer->print_error(error, is_expected_failure); } result.status = TestResult::Status::CRASHED; @@ -311,7 +315,7 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) { if(_log_level >= LogLevel::ERRORS && _printer != nullptr) { - _printer->print_error(TestError("Received unknown exception")); + _printer->print_error(TestError("Received unknown exception"), is_expected_failure); } result.status = TestResult::Status::CRASHED; @@ -326,7 +330,7 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) { if(_log_level >= LogLevel::ERRORS && _printer != nullptr) { - _printer->print_error(error); + _printer->print_error(error, is_expected_failure); } result.status = TestResult::Status::CRASHED; @@ -340,7 +344,7 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) { if(_log_level >= LogLevel::ERRORS && _printer != nullptr) { - _printer->print_error(TestError("Received unknown exception")); + _printer->print_error(TestError("Received unknown exception"), is_expected_failure); } result.status = TestResult::Status::CRASHED; diff --git a/tests/framework/printers/JSONPrinter.cpp b/tests/framework/printers/JSONPrinter.cpp index ae19cae67c..4f17e6277b 100644 --- a/tests/framework/printers/JSONPrinter.cpp +++ b/tests/framework/printers/JSONPrinter.cpp @@ -46,6 +46,31 @@ void JSONPrinter::print_separator(bool &flag) } } +template +void JSONPrinter::print_strings(T &&first, T &&last) +{ + bool first_entry = true; + std::stringstream log; + + while(first != last) + { + print_separator(first_entry); + + *_stream << R"(")"; + + log.str(*first); + + for(std::string line; !std::getline(log, line).fail();) + { + *_stream << line << "; "; + } + + *_stream << R"(")"; + + ++first; + } +} + void JSONPrinter::print_entry(const std::string &name, const std::string &value) { print_separator(_first_entry); @@ -90,38 +115,43 @@ void JSONPrinter::print_test_footer() void JSONPrinter::print_errors_header() { - print_separator(_first_test_entry); - - _first_error = true; - *_stream << R"("errors" : [)"; + _errors.clear(); + _expected_errors.clear(); + _infos.clear(); } void JSONPrinter::print_errors_footer() { + print_separator(_first_test_entry); + + *_stream << R"("errors" : [)"; + print_strings(_errors.begin(), _errors.end()); + *_stream << "]"; + + *_stream << R"(, "expected_errors" : [)"; + print_strings(_expected_errors.begin(), _expected_errors.end()); + *_stream << "]"; + + *_stream << R"(, "infos" : [)"; + print_strings(_infos.begin(), _infos.end()); *_stream << "]"; } -void JSONPrinter::print_error(const std::exception &error) +void JSONPrinter::print_error(const std::exception &error, bool expected) { - std::stringstream error_log; - error_log.str(error.what()); - - for(std::string line; !std::getline(error_log, line).fail();) + if(expected) { - print_separator(_first_error); - - *_stream << R"(")" << line << R"(")"; + _expected_errors.emplace_back(error.what()); + } + else + { + _errors.emplace_back(error.what()); } } void JSONPrinter::print_info(const std::string &info) { - std::istringstream iss(info); - for(std::string line; !std::getline(iss, line).fail();) - { - print_separator(_first_error); - *_stream << R"(")" << line << R"(")"; - } + _infos.push_back(info); } void JSONPrinter::print_measurements(const Profiler::MeasurementsMap &measurements) diff --git a/tests/framework/printers/JSONPrinter.h b/tests/framework/printers/JSONPrinter.h index 18bd4380b0..a2811ea41b 100644 --- a/tests/framework/printers/JSONPrinter.h +++ b/tests/framework/printers/JSONPrinter.h @@ -26,6 +26,8 @@ #include "Printer.h" +#include + namespace arm_compute { namespace test @@ -47,17 +49,22 @@ public: void print_test_footer() override; void print_errors_header() override; void print_errors_footer() override; - void print_error(const std::exception &error) override; + void print_error(const std::exception &error, bool expected) override; void print_info(const std::string &info) override; void print_measurements(const Profiler::MeasurementsMap &measurements) override; private: void print_separator(bool &flag); + template + void print_strings(T &&first, T &&last); + + std::list _infos{}; + std::list _errors{}; + std::list _expected_errors{}; bool _first_entry{ true }; bool _first_test{ true }; bool _first_test_entry{ true }; - bool _first_error{ true }; }; } // namespace framework } // namespace test diff --git a/tests/framework/printers/PrettyPrinter.cpp b/tests/framework/printers/PrettyPrinter.cpp index 2f7df1837a..5eec72a2fe 100644 --- a/tests/framework/printers/PrettyPrinter.cpp +++ b/tests/framework/printers/PrettyPrinter.cpp @@ -102,9 +102,10 @@ void PrettyPrinter::print_info(const std::string &info) *_stream << begin_color("1") << "INFO: " << info << end_color() << "\n"; } -void PrettyPrinter::print_error(const std::exception &error) +void PrettyPrinter::print_error(const std::exception &error, bool expected) { - *_stream << begin_color("1") << "ERROR: " << error.what() << end_color() << "\n"; + std::string prefix = expected ? "EXPECTED ERROR: " : "ERROR: "; + *_stream << begin_color("1") << prefix << error.what() << end_color() << "\n"; } void PrettyPrinter::print_measurements(const Profiler::MeasurementsMap &measurements) diff --git a/tests/framework/printers/PrettyPrinter.h b/tests/framework/printers/PrettyPrinter.h index 3e2bebd1f7..f72a613868 100644 --- a/tests/framework/printers/PrettyPrinter.h +++ b/tests/framework/printers/PrettyPrinter.h @@ -53,7 +53,7 @@ public: void print_test_footer() override; void print_errors_header() override; void print_errors_footer() override; - void print_error(const std::exception &error) override; + void print_error(const std::exception &error, bool expected) override; void print_info(const std::string &info) override; void print_measurements(const Profiler::MeasurementsMap &measurements) override; diff --git a/tests/framework/printers/Printer.h b/tests/framework/printers/Printer.h index 16a4170d7a..c2a44240ba 100644 --- a/tests/framework/printers/Printer.h +++ b/tests/framework/printers/Printer.h @@ -104,9 +104,10 @@ public: /** Print test error. * - * @param[in] error Description of the error. + * @param[in] error Description of the error. + * @param[in] expected Whether the error was expected or not. */ - virtual void print_error(const std::exception &error) = 0; + virtual void print_error(const std::exception &error, bool expected) = 0; /** Print test log info. * -- cgit v1.2.1