COMPMID-415: Print test id

Change-Id: Ie8521aac33361a9296529eeea831f195319e0db7
Reviewed-on: http://mpd-gerrit.cambridge.arm.com/81726
Tested-by: Kaizen <jeremy.johnson+kaizengerrit@arm.com>
Reviewed-by: Anthony Barbier <anthony.barbier@arm.com>

7 files changed, 79 insertions, 73 deletions

diff --git a/framework/Framework.cpp b/framework/Framework.cpp
index 8f605e3e8b..056ee59860 100644
--- a/framework/Framework.cpp
+++ b/framework/Framework.cpp
@@ -136,26 +136,26 @@ void Framework::print_test_info(std::ostream &os) const
     }
 }
 
-void Framework::log_test_start(const std::string &test_name)
+void Framework::log_test_start(const TestInfo &info)
 {
     if(_printer != nullptr && _log_level >= LogLevel::TESTS)
     {
-        _printer->print_test_header(test_name);
+        _printer->print_test_header(info);
     }
 }
 
-void Framework::log_test_skipped(const std::string &test_name)
+void Framework::log_test_skipped(const TestInfo &info)
 {
-    static_cast<void>(test_name);
+    static_cast<void>(info);
 }
 
-void Framework::log_test_end(const std::string &test_name)
+void Framework::log_test_end(const TestInfo &info)
 {
     if(_printer != nullptr)
     {
         if(_log_level >= LogLevel::MEASUREMENTS)
         {
-            _printer->print_measurements(_test_results.at(test_name).measurements);
+            _printer->print_measurements(_test_results.at(info).measurements);
         }
 
         if(_log_level >= LogLevel::TESTS)
@@ -218,18 +218,16 @@ bool Framework::is_selected(const TestInfo &info) const
     return true;
 }
 
-void Framework::run_test(TestCaseFactory &test_factory)
+void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
 {
-    const std::string test_case_name = test_factory.name();
-
     if(test_factory.status() == TestCaseFactory::Status::DISABLED)
     {
-        log_test_skipped(test_case_name);
-        set_test_result(test_case_name, TestResult(TestResult::Status::DISABLED));
+        log_test_skipped(info);
+        set_test_result(info, TestResult(TestResult::Status::DISABLED));
         return;
     }
 
-    log_test_start(test_case_name);
+    log_test_start(info);
 
     Profiler   profiler = get_profiler();
     TestResult result(TestResult::Status::SUCCESS);
@@ -354,22 +352,21 @@ void Framework::run_test(TestCaseFactory &test_factory)
 
     result.measurements = profiler.measurements();
 
-    set_test_result(test_case_name, result);
-    log_test_end(test_case_name);
+    set_test_result(info, result);
+    log_test_end(info);
 }
 
 bool Framework::run()
 {
     // Clear old test results
     _test_results.clear();
-    _runtime = std::chrono::seconds{ 0 };
 
     if(_printer != nullptr && _log_level >= LogLevel::TESTS)
     {
         _printer->print_run_header();
     }
 
-    const auto start = std::chrono::high_resolution_clock::now();
+    const std::chrono::time_point<std::chrono::high_resolution_clock> start = std::chrono::high_resolution_clock::now();
 
     int id = 0;
 
@@ -380,41 +377,40 @@ bool Framework::run()
 
         if(is_selected(test_info))
         {
-            run_test(*test_factory);
+            run_test(test_info, *test_factory);
         }
 
         ++id;
     }
 
-    const auto end = std::chrono::high_resolution_clock::now();
+    const std::chrono::time_point<std::chrono::high_resolution_clock> end = std::chrono::high_resolution_clock::now();
 
     if(_printer != nullptr && _log_level >= LogLevel::TESTS)
     {
         _printer->print_run_footer();
     }
 
-    _runtime = std::chrono::duration_cast<std::chrono::seconds>(end - start);
-
-    auto test_results = count_test_results();
+    auto runtime = std::chrono::duration_cast<std::chrono::seconds>(end - start);
+    std::map<TestResult::Status, int> results = count_test_results();
 
     if(_log_level > LogLevel::NONE)
     {
-        std::cout << "Executed " << _test_results.size() << " test(s) ("
-                  << test_results[TestResult::Status::SUCCESS] << " passed, "
-                  << test_results[TestResult::Status::EXPECTED_FAILURE] << " expected failures, "
-                  << test_results[TestResult::Status::FAILED] << " failed, "
-                  << test_results[TestResult::Status::CRASHED] << " crashed, "
-                  << test_results[TestResult::Status::DISABLED] << " disabled) in " << _runtime.count() << " second(s)\n";
+        std::cout << "Executed " << results.size() << " test(s) ("
+                  << results[TestResult::Status::SUCCESS] << " passed, "
+                  << results[TestResult::Status::EXPECTED_FAILURE] << " expected failures, "
+                  << results[TestResult::Status::FAILED] << " failed, "
+                  << results[TestResult::Status::CRASHED] << " crashed, "
+                  << results[TestResult::Status::DISABLED] << " disabled) in " << runtime.count() << " second(s)\n";
     }
 
-    int num_successful_tests = test_results[TestResult::Status::SUCCESS] + test_results[TestResult::Status::EXPECTED_FAILURE];
+    int num_successful_tests = results[TestResult::Status::SUCCESS] + results[TestResult::Status::EXPECTED_FAILURE];
 
     return (static_cast<unsigned int>(num_successful_tests) == _test_results.size());
 }
 
-void Framework::set_test_result(std::string test_case_name, TestResult result)
+void Framework::set_test_result(TestInfo info, TestResult result)
 {
-    _test_results.emplace(std::move(test_case_name), std::move(result));
+    _test_results.emplace(std::move(info), std::move(result));
 }
 
 void Framework::print_test_results(Printer &printer) const
@@ -451,7 +447,7 @@ void Framework::set_printer(Printer *printer)
     _printer = printer;
 }
 
-std::vector<Framework::TestInfo> Framework::test_infos() const
+std::vector<TestInfo> Framework::test_infos() const
 {
     std::vector<TestInfo> ids;
 
diff --git a/framework/Framework.h b/framework/Framework.h
index 5f52b4fbe8..0929b4ab20 100644
--- a/framework/Framework.h
+++ b/framework/Framework.h
@@ -53,6 +53,28 @@ namespace test
 {
 namespace framework
 {
+/** Information about a test case.
+ *
+ * A test can be identified either via its id or via its name. Additionally
+ * each test is tagged with the data set mode in which it will be used and
+ * its status.
+ *
+ * @note The mapping between test id and test name is not guaranteed to be
+ * stable. It is subject to change as new test are added.
+ */
+struct TestInfo
+{
+    int                     id;
+    std::string             name;
+    DatasetMode             mode;
+    TestCaseFactory::Status status;
+};
+
+inline bool operator<(const TestInfo &lhs, const TestInfo &rhs)
+{
+    return lhs.id < rhs.id;
+}
+
 /** Main framework class.
  *
  * Keeps track of the global state, owns all test cases and collects results.
@@ -60,23 +82,6 @@ namespace framework
 class Framework final
 {
 public:
-    /** Information about a test case.
-     *
-     * A test can be identified either via its id or via its name. Additionally
-     * each test is tagged with the data set mode in which it will be used and
-     * its status.
-     *
-     * @note The mapping between test id and test name is not guaranteed to be
-     * stable. It is subject to change as new test are added.
-     */
-    struct TestInfo
-    {
-        int                     id;
-        std::string             name;
-        DatasetMode             mode;
-        TestCaseFactory::Status status;
-    };
-
     /** Access to the singleton.
      *
      * @return Unique instance of the framework class.
@@ -161,21 +166,21 @@ public:
 
     /** Tell the framework that execution of a test starts.
      *
-     * @param[in] test_name Name of the started test case.
+     * @param[in] info Test info.
      */
-    void log_test_start(const std::string &test_name);
+    void log_test_start(const TestInfo &info);
 
     /** Tell the framework that a test case is skipped.
      *
-     * @param[in] test_name Name of the skipped test case.
+     * @param[in] info Test info.
      */
-    void log_test_skipped(const std::string &test_name);
+    void log_test_skipped(const TestInfo &info);
 
     /** Tell the framework that a test case finished.
      *
-     * @param[in] test_name Name of the finished test case.
+     * @param[in] info Test info.
      */
-    void log_test_end(const std::string &test_name);
+    void log_test_end(const TestInfo &info);
 
     /** Tell the framework that the currently running test case failed a non-fatal expectation.
      *
@@ -224,10 +229,10 @@ public:
 
     /** Set the result for an executed test case.
      *
-     * @param[in] test_case_name Name of the executed test case.
-     * @param[in] result         Execution result.
+     * @param[in] info   Test info.
+     * @param[in] result Execution result.
      */
-    void set_test_result(std::string test_case_name, TestResult result);
+    void set_test_result(TestInfo info, TestResult result);
 
     /** Use the specified printer to output test results from the last run.
      *
@@ -257,7 +262,7 @@ public:
      *
      * @return Vector with all test ids.
      */
-    std::vector<Framework::TestInfo> test_infos() const;
+    std::vector<TestInfo> test_infos() const;
 
 private:
     Framework();
@@ -266,7 +271,7 @@ private:
     Framework(const Framework &) = delete;
     Framework &operator=(const Framework &) = delete;
 
-    void run_test(TestCaseFactory &test_factory);
+    void run_test(const TestInfo &info, TestCaseFactory &test_factory);
     std::map<TestResult::Status, int> count_test_results() const;
 
     /** Returns the current test suite name.
@@ -281,11 +286,10 @@ private:
 
     std::vector<std::string>                      _test_suite_name{};
     std::vector<std::unique_ptr<TestCaseFactory>> _test_factories{};
-    std::map<std::string, TestResult> _test_results{};
-    std::chrono::seconds _runtime{ 0 };
-    int                  _num_iterations{ 1 };
-    bool                 _throw_errors{ false };
-    Printer             *_printer{ nullptr };
+    std::map<TestInfo, TestResult> _test_results{};
+    int      _num_iterations{ 1 };
+    bool     _throw_errors{ false };
+    Printer *_printer{ nullptr };
 
     using create_function = std::unique_ptr<Instrument>();
     std::map<InstrumentType, create_function *> _available_instruments{};
diff --git a/framework/printers/JSONPrinter.cpp b/framework/printers/JSONPrinter.cpp
index 66973bb1ab..7806644418 100644
--- a/framework/printers/JSONPrinter.cpp
+++ b/framework/printers/JSONPrinter.cpp
@@ -23,6 +23,8 @@
  */
 #include "JSONPrinter.h"
 
+#include "framework/Framework.h"
+
 #include <algorithm>
 
 namespace arm_compute
@@ -72,11 +74,11 @@ void JSONPrinter::print_run_footer()
     *_stream << "}";
 }
 
-void JSONPrinter::print_test_header(const std::string &name)
+void JSONPrinter::print_test_header(const TestInfo &info)
 {
     print_separator(_first_test);
 
-    *_stream << R"(")" << name << R"(" : {)";
+    *_stream << R"(")" << info.name << R"(" : {)";
 }
 
 void JSONPrinter::print_test_footer()
diff --git a/framework/printers/JSONPrinter.h b/framework/printers/JSONPrinter.h
index 3cc5578ecd..7b34941ca1 100644
--- a/framework/printers/JSONPrinter.h
+++ b/framework/printers/JSONPrinter.h
@@ -43,7 +43,7 @@ public:
     void print_global_footer() override;
     void print_run_header() override;
     void print_run_footer() override;
-    void print_test_header(const std::string &name) override;
+    void print_test_header(const TestInfo &info) override;
     void print_test_footer() override;
     void print_measurements(const Profiler::MeasurementsMap &measurements) override;
 
diff --git a/framework/printers/PrettyPrinter.cpp b/framework/printers/PrettyPrinter.cpp
index 77fb6404a8..fd90401693 100644
--- a/framework/printers/PrettyPrinter.cpp
+++ b/framework/printers/PrettyPrinter.cpp
@@ -23,6 +23,8 @@
  */
 #include "PrettyPrinter.h"
 
+#include "framework/Framework.h"
+
 #include <algorithm>
 
 namespace arm_compute
@@ -77,9 +79,9 @@ void PrettyPrinter::print_run_footer()
 {
 }
 
-void PrettyPrinter::print_test_header(const std::string &name)
+void PrettyPrinter::print_test_header(const TestInfo &info)
 {
-    *_stream << begin_color("2") << "Running '" << name << "'" << end_color() << "\n";
+    *_stream << begin_color("2") << "Running [" << info.id << "] '" << info.name << "'" << end_color() << "\n";
 }
 
 void PrettyPrinter::print_test_footer()
diff --git a/framework/printers/PrettyPrinter.h b/framework/printers/PrettyPrinter.h
index bf3f12943d..893b1fadd2 100644
--- a/framework/printers/PrettyPrinter.h
+++ b/framework/printers/PrettyPrinter.h
@@ -49,7 +49,7 @@ public:
     void print_global_footer() override;
     void print_run_header() override;
     void print_run_footer() override;
-    void print_test_header(const std::string &name) override;
+    void print_test_header(const TestInfo &info) override;
     void print_test_footer() override;
     void print_measurements(const Profiler::MeasurementsMap &measurements) override;
 
diff --git a/framework/printers/Printer.h b/framework/printers/Printer.h
index 7e1936413b..7d8af12416 100644
--- a/framework/printers/Printer.h
+++ b/framework/printers/Printer.h
@@ -24,7 +24,7 @@
 #ifndef ARM_COMPUTE_TEST_PRINTER
 #define ARM_COMPUTE_TEST_PRINTER
 
-#include "../Profiler.h"
+#include "framework/Profiler.h"
 
 #include <fstream>
 #include <iostream>
@@ -36,6 +36,8 @@ namespace test
 {
 namespace framework
 {
+struct TestInfo;
+
 /** Abstract printer class used by the @ref Framework to present output. */
 class Printer
 {
@@ -86,9 +88,9 @@ public:
 
     /** Print header before a test.
      *
-     * @param[in] name test_name.
+     * @param[in] info Test info.
      */
-    virtual void print_test_header(const std::string &name) = 0;
+    virtual void print_test_header(const TestInfo &info) = 0;
 
     /** Print footer after a test. */
     virtual void print_test_footer() = 0;