From 542002cf3d9acc2858e2311966cd9b060d70b5a6 Mon Sep 17 00:00:00 2001 From: Moritz Pflanzer Date: Wed, 26 Jul 2017 16:03:58 +0100 Subject: COMPMID-415: Print test id Change-Id: Ie8521aac33361a9296529eeea831f195319e0db7 Reviewed-on: http://mpd-gerrit.cambridge.arm.com/81726 Tested-by: Kaizen Reviewed-by: Anthony Barbier --- framework/Framework.cpp | 58 ++++++++++++++---------------- framework/Framework.h | 70 +++++++++++++++++++----------------- framework/printers/JSONPrinter.cpp | 6 ++-- framework/printers/JSONPrinter.h | 2 +- framework/printers/PrettyPrinter.cpp | 6 ++-- framework/printers/PrettyPrinter.h | 2 +- framework/printers/Printer.h | 8 +++-- 7 files changed, 79 insertions(+), 73 deletions(-) diff --git a/framework/Framework.cpp b/framework/Framework.cpp index 8f605e3e8b..056ee59860 100644 --- a/framework/Framework.cpp +++ b/framework/Framework.cpp @@ -136,26 +136,26 @@ void Framework::print_test_info(std::ostream &os) const } } -void Framework::log_test_start(const std::string &test_name) +void Framework::log_test_start(const TestInfo &info) { if(_printer != nullptr && _log_level >= LogLevel::TESTS) { - _printer->print_test_header(test_name); + _printer->print_test_header(info); } } -void Framework::log_test_skipped(const std::string &test_name) +void Framework::log_test_skipped(const TestInfo &info) { - static_cast(test_name); + static_cast(info); } -void Framework::log_test_end(const std::string &test_name) +void Framework::log_test_end(const TestInfo &info) { if(_printer != nullptr) { if(_log_level >= LogLevel::MEASUREMENTS) { - _printer->print_measurements(_test_results.at(test_name).measurements); + _printer->print_measurements(_test_results.at(info).measurements); } if(_log_level >= LogLevel::TESTS) @@ -218,18 +218,16 @@ bool Framework::is_selected(const TestInfo &info) const return true; } -void Framework::run_test(TestCaseFactory &test_factory) +void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) { - const std::string test_case_name = test_factory.name(); - if(test_factory.status() == TestCaseFactory::Status::DISABLED) { - log_test_skipped(test_case_name); - set_test_result(test_case_name, TestResult(TestResult::Status::DISABLED)); + log_test_skipped(info); + set_test_result(info, TestResult(TestResult::Status::DISABLED)); return; } - log_test_start(test_case_name); + log_test_start(info); Profiler profiler = get_profiler(); TestResult result(TestResult::Status::SUCCESS); @@ -354,22 +352,21 @@ void Framework::run_test(TestCaseFactory &test_factory) result.measurements = profiler.measurements(); - set_test_result(test_case_name, result); - log_test_end(test_case_name); + set_test_result(info, result); + log_test_end(info); } bool Framework::run() { // Clear old test results _test_results.clear(); - _runtime = std::chrono::seconds{ 0 }; if(_printer != nullptr && _log_level >= LogLevel::TESTS) { _printer->print_run_header(); } - const auto start = std::chrono::high_resolution_clock::now(); + const std::chrono::time_point start = std::chrono::high_resolution_clock::now(); int id = 0; @@ -380,41 +377,40 @@ bool Framework::run() if(is_selected(test_info)) { - run_test(*test_factory); + run_test(test_info, *test_factory); } ++id; } - const auto end = std::chrono::high_resolution_clock::now(); + const std::chrono::time_point end = std::chrono::high_resolution_clock::now(); if(_printer != nullptr && _log_level >= LogLevel::TESTS) { _printer->print_run_footer(); } - _runtime = std::chrono::duration_cast(end - start); - - auto test_results = count_test_results(); + auto runtime = std::chrono::duration_cast(end - start); + std::map results = count_test_results(); if(_log_level > LogLevel::NONE) { - std::cout << "Executed " << _test_results.size() << " test(s) (" - << test_results[TestResult::Status::SUCCESS] << " passed, " - << test_results[TestResult::Status::EXPECTED_FAILURE] << " expected failures, " - << test_results[TestResult::Status::FAILED] << " failed, " - << test_results[TestResult::Status::CRASHED] << " crashed, " - << test_results[TestResult::Status::DISABLED] << " disabled) in " << _runtime.count() << " second(s)\n"; + std::cout << "Executed " << results.size() << " test(s) (" + << results[TestResult::Status::SUCCESS] << " passed, " + << results[TestResult::Status::EXPECTED_FAILURE] << " expected failures, " + << results[TestResult::Status::FAILED] << " failed, " + << results[TestResult::Status::CRASHED] << " crashed, " + << results[TestResult::Status::DISABLED] << " disabled) in " << runtime.count() << " second(s)\n"; } - int num_successful_tests = test_results[TestResult::Status::SUCCESS] + test_results[TestResult::Status::EXPECTED_FAILURE]; + int num_successful_tests = results[TestResult::Status::SUCCESS] + results[TestResult::Status::EXPECTED_FAILURE]; return (static_cast(num_successful_tests) == _test_results.size()); } -void Framework::set_test_result(std::string test_case_name, TestResult result) +void Framework::set_test_result(TestInfo info, TestResult result) { - _test_results.emplace(std::move(test_case_name), std::move(result)); + _test_results.emplace(std::move(info), std::move(result)); } void Framework::print_test_results(Printer &printer) const @@ -451,7 +447,7 @@ void Framework::set_printer(Printer *printer) _printer = printer; } -std::vector Framework::test_infos() const +std::vector Framework::test_infos() const { std::vector ids; diff --git a/framework/Framework.h b/framework/Framework.h index 5f52b4fbe8..0929b4ab20 100644 --- a/framework/Framework.h +++ b/framework/Framework.h @@ -53,6 +53,28 @@ namespace test { namespace framework { +/** Information about a test case. + * + * A test can be identified either via its id or via its name. Additionally + * each test is tagged with the data set mode in which it will be used and + * its status. + * + * @note The mapping between test id and test name is not guaranteed to be + * stable. It is subject to change as new test are added. + */ +struct TestInfo +{ + int id; + std::string name; + DatasetMode mode; + TestCaseFactory::Status status; +}; + +inline bool operator<(const TestInfo &lhs, const TestInfo &rhs) +{ + return lhs.id < rhs.id; +} + /** Main framework class. * * Keeps track of the global state, owns all test cases and collects results. @@ -60,23 +82,6 @@ namespace framework class Framework final { public: - /** Information about a test case. - * - * A test can be identified either via its id or via its name. Additionally - * each test is tagged with the data set mode in which it will be used and - * its status. - * - * @note The mapping between test id and test name is not guaranteed to be - * stable. It is subject to change as new test are added. - */ - struct TestInfo - { - int id; - std::string name; - DatasetMode mode; - TestCaseFactory::Status status; - }; - /** Access to the singleton. * * @return Unique instance of the framework class. @@ -161,21 +166,21 @@ public: /** Tell the framework that execution of a test starts. * - * @param[in] test_name Name of the started test case. + * @param[in] info Test info. */ - void log_test_start(const std::string &test_name); + void log_test_start(const TestInfo &info); /** Tell the framework that a test case is skipped. * - * @param[in] test_name Name of the skipped test case. + * @param[in] info Test info. */ - void log_test_skipped(const std::string &test_name); + void log_test_skipped(const TestInfo &info); /** Tell the framework that a test case finished. * - * @param[in] test_name Name of the finished test case. + * @param[in] info Test info. */ - void log_test_end(const std::string &test_name); + void log_test_end(const TestInfo &info); /** Tell the framework that the currently running test case failed a non-fatal expectation. * @@ -224,10 +229,10 @@ public: /** Set the result for an executed test case. * - * @param[in] test_case_name Name of the executed test case. - * @param[in] result Execution result. + * @param[in] info Test info. + * @param[in] result Execution result. */ - void set_test_result(std::string test_case_name, TestResult result); + void set_test_result(TestInfo info, TestResult result); /** Use the specified printer to output test results from the last run. * @@ -257,7 +262,7 @@ public: * * @return Vector with all test ids. */ - std::vector test_infos() const; + std::vector test_infos() const; private: Framework(); @@ -266,7 +271,7 @@ private: Framework(const Framework &) = delete; Framework &operator=(const Framework &) = delete; - void run_test(TestCaseFactory &test_factory); + void run_test(const TestInfo &info, TestCaseFactory &test_factory); std::map count_test_results() const; /** Returns the current test suite name. @@ -281,11 +286,10 @@ private: std::vector _test_suite_name{}; std::vector> _test_factories{}; - std::map _test_results{}; - std::chrono::seconds _runtime{ 0 }; - int _num_iterations{ 1 }; - bool _throw_errors{ false }; - Printer *_printer{ nullptr }; + std::map _test_results{}; + int _num_iterations{ 1 }; + bool _throw_errors{ false }; + Printer *_printer{ nullptr }; using create_function = std::unique_ptr(); std::map _available_instruments{}; diff --git a/framework/printers/JSONPrinter.cpp b/framework/printers/JSONPrinter.cpp index 66973bb1ab..7806644418 100644 --- a/framework/printers/JSONPrinter.cpp +++ b/framework/printers/JSONPrinter.cpp @@ -23,6 +23,8 @@ */ #include "JSONPrinter.h" +#include "framework/Framework.h" + #include namespace arm_compute @@ -72,11 +74,11 @@ void JSONPrinter::print_run_footer() *_stream << "}"; } -void JSONPrinter::print_test_header(const std::string &name) +void JSONPrinter::print_test_header(const TestInfo &info) { print_separator(_first_test); - *_stream << R"(")" << name << R"(" : {)"; + *_stream << R"(")" << info.name << R"(" : {)"; } void JSONPrinter::print_test_footer() diff --git a/framework/printers/JSONPrinter.h b/framework/printers/JSONPrinter.h index 3cc5578ecd..7b34941ca1 100644 --- a/framework/printers/JSONPrinter.h +++ b/framework/printers/JSONPrinter.h @@ -43,7 +43,7 @@ public: void print_global_footer() override; void print_run_header() override; void print_run_footer() override; - void print_test_header(const std::string &name) override; + void print_test_header(const TestInfo &info) override; void print_test_footer() override; void print_measurements(const Profiler::MeasurementsMap &measurements) override; diff --git a/framework/printers/PrettyPrinter.cpp b/framework/printers/PrettyPrinter.cpp index 77fb6404a8..fd90401693 100644 --- a/framework/printers/PrettyPrinter.cpp +++ b/framework/printers/PrettyPrinter.cpp @@ -23,6 +23,8 @@ */ #include "PrettyPrinter.h" +#include "framework/Framework.h" + #include namespace arm_compute @@ -77,9 +79,9 @@ void PrettyPrinter::print_run_footer() { } -void PrettyPrinter::print_test_header(const std::string &name) +void PrettyPrinter::print_test_header(const TestInfo &info) { - *_stream << begin_color("2") << "Running '" << name << "'" << end_color() << "\n"; + *_stream << begin_color("2") << "Running [" << info.id << "] '" << info.name << "'" << end_color() << "\n"; } void PrettyPrinter::print_test_footer() diff --git a/framework/printers/PrettyPrinter.h b/framework/printers/PrettyPrinter.h index bf3f12943d..893b1fadd2 100644 --- a/framework/printers/PrettyPrinter.h +++ b/framework/printers/PrettyPrinter.h @@ -49,7 +49,7 @@ public: void print_global_footer() override; void print_run_header() override; void print_run_footer() override; - void print_test_header(const std::string &name) override; + void print_test_header(const TestInfo &info) override; void print_test_footer() override; void print_measurements(const Profiler::MeasurementsMap &measurements) override; diff --git a/framework/printers/Printer.h b/framework/printers/Printer.h index 7e1936413b..7d8af12416 100644 --- a/framework/printers/Printer.h +++ b/framework/printers/Printer.h @@ -24,7 +24,7 @@ #ifndef ARM_COMPUTE_TEST_PRINTER #define ARM_COMPUTE_TEST_PRINTER -#include "../Profiler.h" +#include "framework/Profiler.h" #include #include @@ -36,6 +36,8 @@ namespace test { namespace framework { +struct TestInfo; + /** Abstract printer class used by the @ref Framework to present output. */ class Printer { @@ -86,9 +88,9 @@ public: /** Print header before a test. * - * @param[in] name test_name. + * @param[in] info Test info. */ - virtual void print_test_header(const std::string &name) = 0; + virtual void print_test_header(const TestInfo &info) = 0; /** Print footer after a test. */ virtual void print_test_footer() = 0; -- cgit v1.2.1