Printing out the rerun command of each failed testcase

- After a testcase fail, print the arm_compute_validation command
  with filter specified and seed to rerun.

Resolves: COMPMID-5916

Signed-off-by: Ramy Elgammal <ramy.elgammal@arm.com>
Change-Id: I0c8214e7d3b61f01ce16f50499c29bfe08b6a885
Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/9673
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Pablo Marquez Tello <pablo.tello@arm.com>
Reviewed-by: Viet-Hoa Do <viet-hoa.do@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>
Benchmark: Arm Jenkins <bsgcomp@arm.com>

3 files changed, 32 insertions, 19 deletions

diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp
index a3dee07862..bfb955c525 100644
--- a/tests/framework/Framework.cpp
+++ b/tests/framework/Framework.cpp
@@ -130,10 +130,12 @@ Framework &Framework::get()
 void Framework::init(const FrameworkConfig &config)
 {
     _test_filter.reset(new TestFilter(config.mode, config.name_filter, config.id_filter));
-    _num_iterations = config.num_iterations;
-    _log_level      = config.log_level;
-    _cooldown_sec   = config.cooldown_sec;
-    _configure_only = config.configure_only;
+    _num_iterations  = config.num_iterations;
+    _log_level       = config.log_level;
+    _cooldown_sec    = config.cooldown_sec;
+    _configure_only  = config.configure_only;
+    _print_rerun_cmd = config.print_rerun_cmd;
+    _seed            = config.seed;
 
     _instruments = std::set<framework::InstrumentsDescription>(std::begin(config.instruments), std::end(config.instruments));
 }
@@ -292,13 +294,13 @@ bool Framework::error_on_missing_assets() const
     return _error_on_missing_assets;
 }
 
-void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
+TestResult::Status Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
 {
     if(test_factory.status() == TestCaseFactory::Status::DISABLED)
     {
         log_test_skipped(info);
         set_test_result(info, TestResult(TestResult::Status::DISABLED));
-        return;
+        return TestResult::Status::DISABLED;
     }
 
     log_test_start(info);
@@ -538,6 +540,7 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
 
     set_test_result(info, result);
     log_test_end(info);
+    return result.status;
 }
 
 bool Framework::run()
@@ -581,9 +584,11 @@ bool Framework::run()
                 CLScheduler::get().set_queue(new_queue);
             }
 #endif // ARM_COMPUTE_CL
-
-            run_test(test_info, *test_factory);
-
+            TestResult::Status result = run_test(test_info, *test_factory);
+            if((_print_rerun_cmd) && (result == TestResult::Status::CRASHED || result == TestResult::Status::FAILED))
+            {
+                std::cout << "Rerun command: ./arm_compute_validation --filter='^" << test_info.name << "$' --seed=" << _seed << std::endl;
+            }
             ++id_run_test;
 
             // Run test delay
diff --git a/tests/framework/Framework.h b/tests/framework/Framework.h
index 274f03a922..72ec7484d7 100644
--- a/tests/framework/Framework.h
+++ b/tests/framework/Framework.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2021 Arm Limited.
+ * Copyright (c) 2017-2021, 2023 Arm Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -64,6 +64,8 @@ struct FrameworkConfig
     float                                          cooldown_sec{ -1.f };        /**< Delay between tests in seconds. */
     LogLevel                                       log_level{ LogLevel::NONE }; /**< Verbosity of the output. */
     bool                                           configure_only{ false };     /**< Only configure kernels */
+    bool                                           print_rerun_cmd{ false };    /**< Print the command to rerun the failed testcase */
+    unsigned int                                   seed{0};                     /**< The seed that is used to fill tensors with random values.*/
 };
 
 /** Information about a test case.
@@ -330,7 +332,7 @@ private:
     Framework(const Framework &) = delete;
     Framework &operator=(const Framework &) = delete;
 
-    void run_test(const TestInfo &info, TestCaseFactory &test_factory);
+    TestResult::Status run_test(const TestInfo &info, TestCaseFactory &test_factory);
     std::map<TestResult::Status, int> count_test_results() const;
 
     /** Returns the current test suite name.
@@ -358,6 +360,8 @@ private:
     std::vector<Printer *> _printers{};
     bool                   _configure_only{ false };
     bool                   _new_fixture_call{ false };
+    bool                   _print_rerun_cmd{ false };
+    unsigned int           _seed {0};
 
     using create_function = std::unique_ptr<Instrument>();
     std::map<InstrumentsDescription, create_function *> _available_instruments{};
diff --git a/tests/main.cpp b/tests/main.cpp
index 58347cad42..bd79e572e3 100644
--- a/tests/main.cpp
+++ b/tests/main.cpp
@@ -127,6 +127,8 @@ int main(int argc, char **argv)
     error_on_missing_assets->set_help("Mark a test as failed instead of skipping it when assets are missing");
     auto assets = parser.add_positional_option<utils::SimpleOption<std::string>>("assets");
     assets->set_help("Path to the assets directory");
+    auto print_rerun_command = parser.add_option<utils::ToggleOption>("rerun-cmd");
+    print_rerun_command->set_help("Print out the command to rerun the exact failed testcase");
 #ifdef ARM_COMPUTE_CL
     auto enable_tuner = parser.add_option<utils::ToggleOption>("enable-tuner");
     enable_tuner->set_help("Enable OpenCL dynamic tuner");
@@ -270,14 +272,16 @@ int main(int argc, char **argv)
 
         // Initialize framework
         framework::FrameworkConfig fconfig;
-        fconfig.instruments    = options.instruments->value();
-        fconfig.name_filter    = filter->value();
-        fconfig.id_filter      = filter_id->value();
-        fconfig.num_iterations = options.iterations->value();
-        fconfig.mode           = dataset_mode->value();
-        fconfig.log_level      = options.log_level->value();
-        fconfig.cooldown_sec   = cooldown_sec->value();
-        fconfig.configure_only = configure_only->value();
+        fconfig.instruments     = options.instruments->value();
+        fconfig.name_filter     = filter->value();
+        fconfig.id_filter       = filter_id->value();
+        fconfig.num_iterations  = options.iterations->value();
+        fconfig.mode            = dataset_mode->value();
+        fconfig.log_level       = options.log_level->value();
+        fconfig.cooldown_sec    = cooldown_sec->value();
+        fconfig.configure_only  = configure_only->value();
+        fconfig.print_rerun_cmd = print_rerun_command->value();
+        fconfig.seed            = seed->value();
         framework.init(fconfig);
 
         for(auto &p : printers)