From 1355ec4797cd77060af51c8b27d99ea1d25c08da Mon Sep 17 00:00:00 2001 From: Ramy Elgammal Date: Mon, 15 May 2023 13:50:46 +0100 Subject: Printing out the rerun command of each failed testcase - After a testcase fail, print the arm_compute_validation command with filter specified and seed to rerun. Resolves: COMPMID-5916 Signed-off-by: Ramy Elgammal Change-Id: I0c8214e7d3b61f01ce16f50499c29bfe08b6a885 Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/9673 Tested-by: Arm Jenkins Reviewed-by: Pablo Marquez Tello Reviewed-by: Viet-Hoa Do Comments-Addressed: Arm Jenkins Benchmark: Arm Jenkins --- tests/framework/Framework.cpp | 23 ++++++++++++++--------- tests/framework/Framework.h | 8 ++++++-- tests/main.cpp | 20 ++++++++++++-------- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp index a3dee07862..bfb955c525 100644 --- a/tests/framework/Framework.cpp +++ b/tests/framework/Framework.cpp @@ -130,10 +130,12 @@ Framework &Framework::get() void Framework::init(const FrameworkConfig &config) { _test_filter.reset(new TestFilter(config.mode, config.name_filter, config.id_filter)); - _num_iterations = config.num_iterations; - _log_level = config.log_level; - _cooldown_sec = config.cooldown_sec; - _configure_only = config.configure_only; + _num_iterations = config.num_iterations; + _log_level = config.log_level; + _cooldown_sec = config.cooldown_sec; + _configure_only = config.configure_only; + _print_rerun_cmd = config.print_rerun_cmd; + _seed = config.seed; _instruments = std::set(std::begin(config.instruments), std::end(config.instruments)); } @@ -292,13 +294,13 @@ bool Framework::error_on_missing_assets() const return _error_on_missing_assets; } -void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) +TestResult::Status Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) { if(test_factory.status() == TestCaseFactory::Status::DISABLED) { log_test_skipped(info); set_test_result(info, TestResult(TestResult::Status::DISABLED)); - return; + return TestResult::Status::DISABLED; } log_test_start(info); @@ -538,6 +540,7 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) set_test_result(info, result); log_test_end(info); + return result.status; } bool Framework::run() @@ -581,9 +584,11 @@ bool Framework::run() CLScheduler::get().set_queue(new_queue); } #endif // ARM_COMPUTE_CL - - run_test(test_info, *test_factory); - + TestResult::Status result = run_test(test_info, *test_factory); + if((_print_rerun_cmd) && (result == TestResult::Status::CRASHED || result == TestResult::Status::FAILED)) + { + std::cout << "Rerun command: ./arm_compute_validation --filter='^" << test_info.name << "$' --seed=" << _seed << std::endl; + } ++id_run_test; // Run test delay diff --git a/tests/framework/Framework.h b/tests/framework/Framework.h index 274f03a922..72ec7484d7 100644 --- a/tests/framework/Framework.h +++ b/tests/framework/Framework.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2021 Arm Limited. + * Copyright (c) 2017-2021, 2023 Arm Limited. * * SPDX-License-Identifier: MIT * @@ -64,6 +64,8 @@ struct FrameworkConfig float cooldown_sec{ -1.f }; /**< Delay between tests in seconds. */ LogLevel log_level{ LogLevel::NONE }; /**< Verbosity of the output. */ bool configure_only{ false }; /**< Only configure kernels */ + bool print_rerun_cmd{ false }; /**< Print the command to rerun the failed testcase */ + unsigned int seed{0}; /**< The seed that is used to fill tensors with random values.*/ }; /** Information about a test case. @@ -330,7 +332,7 @@ private: Framework(const Framework &) = delete; Framework &operator=(const Framework &) = delete; - void run_test(const TestInfo &info, TestCaseFactory &test_factory); + TestResult::Status run_test(const TestInfo &info, TestCaseFactory &test_factory); std::map count_test_results() const; /** Returns the current test suite name. @@ -358,6 +360,8 @@ private: std::vector _printers{}; bool _configure_only{ false }; bool _new_fixture_call{ false }; + bool _print_rerun_cmd{ false }; + unsigned int _seed {0}; using create_function = std::unique_ptr(); std::map _available_instruments{}; diff --git a/tests/main.cpp b/tests/main.cpp index 58347cad42..bd79e572e3 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -127,6 +127,8 @@ int main(int argc, char **argv) error_on_missing_assets->set_help("Mark a test as failed instead of skipping it when assets are missing"); auto assets = parser.add_positional_option>("assets"); assets->set_help("Path to the assets directory"); + auto print_rerun_command = parser.add_option("rerun-cmd"); + print_rerun_command->set_help("Print out the command to rerun the exact failed testcase"); #ifdef ARM_COMPUTE_CL auto enable_tuner = parser.add_option("enable-tuner"); enable_tuner->set_help("Enable OpenCL dynamic tuner"); @@ -270,14 +272,16 @@ int main(int argc, char **argv) // Initialize framework framework::FrameworkConfig fconfig; - fconfig.instruments = options.instruments->value(); - fconfig.name_filter = filter->value(); - fconfig.id_filter = filter_id->value(); - fconfig.num_iterations = options.iterations->value(); - fconfig.mode = dataset_mode->value(); - fconfig.log_level = options.log_level->value(); - fconfig.cooldown_sec = cooldown_sec->value(); - fconfig.configure_only = configure_only->value(); + fconfig.instruments = options.instruments->value(); + fconfig.name_filter = filter->value(); + fconfig.id_filter = filter_id->value(); + fconfig.num_iterations = options.iterations->value(); + fconfig.mode = dataset_mode->value(); + fconfig.log_level = options.log_level->value(); + fconfig.cooldown_sec = cooldown_sec->value(); + fconfig.configure_only = configure_only->value(); + fconfig.print_rerun_cmd = print_rerun_command->value(); + fconfig.seed = seed->value(); framework.init(fconfig); for(auto &p : printers) -- cgit v1.2.1