From fa811656a2bd39bcabfc7f52270f1f5d655e0537 Mon Sep 17 00:00:00 2001
From: Moritz Pflanzer <moritz.pflanzer@arm.com>
Date: Wed, 26 Jul 2017 17:00:37 +0100
Subject: COMPMID-415: Add --stop-on-error

Change-Id: I557934e8f065d8877344c945980b33139c2a5c97
Reviewed-on: http://mpd-gerrit.cambridge.arm.com/81831
Tested-by: Kaizen <jeremy.johnson+kaizengerrit@arm.com>
Reviewed-by: Anthony Barbier <anthony.barbier@arm.com>
---
 framework/Framework.cpp | 21 +++++++++++++++++++--
 framework/Framework.h   | 13 +++++++++++++
 tests/main.cpp          |  5 ++++-
 3 files changed, 36 insertions(+), 3 deletions(-)

diff --git a/framework/Framework.cpp b/framework/Framework.cpp
index 056ee59860..ac7248c43c 100644
--- a/framework/Framework.cpp
+++ b/framework/Framework.cpp
@@ -198,6 +198,16 @@ bool Framework::throw_errors() const
     return _throw_errors;
 }
 
+void Framework::set_stop_on_error(bool stop_on_error)
+{
+    _stop_on_error = stop_on_error;
+}
+
+bool Framework::stop_on_error() const
+{
+    return _stop_on_error;
+}
+
 bool Framework::is_selected(const TestInfo &info) const
 {
     if((info.mode & _dataset_mode) == DatasetMode::DISABLED)
@@ -345,9 +355,16 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
 
     _current_test_result = nullptr;
 
-    if(test_factory.status() == TestCaseFactory::Status::EXPECTED_FAILURE && result.status == TestResult::Status::FAILED)
+    if(result.status == TestResult::Status::FAILED)
     {
-        result.status = TestResult::Status::EXPECTED_FAILURE;
+        if(info.status == TestCaseFactory::Status::EXPECTED_FAILURE)
+        {
+            result.status = TestResult::Status::EXPECTED_FAILURE;
+        }
+        else if(_stop_on_error)
+        {
+            throw std::runtime_error("Abort on first error.");
+        }
     }
 
     result.measurements = profiler.measurements();
diff --git a/framework/Framework.h b/framework/Framework.h
index 0929b4ab20..671e4e435f 100644
--- a/framework/Framework.h
+++ b/framework/Framework.h
@@ -213,6 +213,18 @@ public:
      */
     void set_throw_errors(bool throw_errors);
 
+    /** Indicates if test execution is stopped after the first failed test.
+     *
+     * @return True if the execution is going to be aborted after the first failed test.
+     */
+    bool stop_on_error() const;
+
+    /** Set whether to abort execution after the first failed test.
+     *
+     * @param[in] stop_on_error True if execution is going to be aborted after first failed test.
+     */
+    void set_stop_on_error(bool stop_on_error);
+
     /** Check if a test case is selected to be executed.
      *
      * @param[in] info Test case info.
@@ -289,6 +301,7 @@ private:
     std::map<TestInfo, TestResult> _test_results{};
     int      _num_iterations{ 1 };
     bool     _throw_errors{ false };
+    bool     _stop_on_error{ false };
     Printer *_printer{ nullptr };
 
     using create_function = std::unique_ptr<Instrument>();
diff --git a/tests/main.cpp b/tests/main.cpp
index 4f17685098..7a1265cbbb 100644
--- a/tests/main.cpp
+++ b/tests/main.cpp
@@ -122,7 +122,9 @@ int main(int argc, char **argv)
     auto log_level = parser.add_option<framework::EnumOption<framework::LogLevel>>("log-level", supported_log_levels, framework::LogLevel::ALL);
     log_file->set_help("Verbosity of the output");
     auto throw_errors = parser.add_option<framework::ToggleOption>("throw-errors");
-    throw_errors->set_help("Don't catch errors (useful for debugging)");
+    throw_errors->set_help("Don't catch fatal errors (useful for debugging)");
+    auto stop_on_error = parser.add_option<framework::ToggleOption>("stop-on-error");
+    throw_errors->set_help("Abort execution after the first failed test (useful for debugging)");
     auto seed = parser.add_option<framework::SimpleOption<std::random_device::result_type>>("seed", std::random_device()());
     seed->set_help("Global seed for random number generation");
     auto color_output = parser.add_option<framework::ToggleOption>("color-output", true);
@@ -193,6 +195,7 @@ int main(int argc, char **argv)
         framework.init(instruments->value(), iterations->value(), dataset_mode->value(), filter->value(), filter_id->value(), log_level->value());
         framework.set_printer(printer.get());
         framework.set_throw_errors(throw_errors->value());
+        framework.set_stop_on_error(stop_on_error->value());
 
         bool success = true;
 
-- 
cgit v1.2.1