COMPMID-417 Allow the tests to run even when assets are not present

Change-Id: Ief165b1d583a70cbe35aae93f05ddfe962196323
Reviewed-on: http://mpd-gerrit.cambridge.arm.com/89503
Reviewed-by: Georgios Pinitas <georgios.pinitas@arm.com>
Tested-by: Kaizen <jeremy.johnson+kaizengerrit@arm.com>

4 files changed, 66 insertions, 0 deletions

diff --git a/tests/framework/Exceptions.cpp b/tests/framework/Exceptions.cpp
index 3d6c65c181..0ca86a8589 100644
--- a/tests/framework/Exceptions.cpp
+++ b/tests/framework/Exceptions.cpp
@@ -104,6 +104,11 @@ std::string to_string(LogLevel level)
     return stream.str();
 }
 
+FileNotFound::FileNotFound(const std::string &msg)
+    : std::runtime_error{ msg }
+{
+}
+
 TestError::TestError(const std::string &msg, LogLevel level, std::string context)
     : std::runtime_error{ msg }, _level{ level }, _msg{ msg }, _context{ std::move(context) }, _combined{ "ERROR: " + msg }
 {
diff --git a/tests/framework/Exceptions.h b/tests/framework/Exceptions.h
index edb0ed92c9..f35c35020c 100644
--- a/tests/framework/Exceptions.h
+++ b/tests/framework/Exceptions.h
@@ -63,6 +63,17 @@ LogLevel log_level_from_name(const std::string &name);
 ::std::ostream &operator<<(::std::ostream &stream, LogLevel level);
 std::string to_string(LogLevel level);
 
+/** Error class for when some external assets are missing */
+class FileNotFound : public std::runtime_error
+{
+public:
+    /** Construct error with message
+     *
+     * @param[in] msg Error message
+     */
+    FileNotFound(const std::string &msg);
+};
+
 /** Error class for failures during test execution. */
 class TestError : public std::runtime_error
 {
diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp
index 9a67cca184..318f9b1d4c 100644
--- a/tests/framework/Framework.cpp
+++ b/tests/framework/Framework.cpp
@@ -216,6 +216,16 @@ bool Framework::stop_on_error() const
     return _stop_on_error;
 }
 
+void Framework::set_error_on_missing_assets(bool error_on_missing_assets)
+{
+    _error_on_missing_assets = error_on_missing_assets;
+}
+
+bool Framework::error_on_missing_assets() const
+{
+    return _error_on_missing_assets;
+}
+
 void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
 {
     if(test_factory.status() == TestCaseFactory::Status::DISABLED)
@@ -269,6 +279,33 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
                 result.status = TestResult::Status::SUCCESS;
             }
         }
+        catch(const FileNotFound &error)
+        {
+            if(_error_on_missing_assets)
+            {
+                if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
+                {
+                    TestError test_error(error.what(), LogLevel::ERRORS);
+                    _printer->print_error(test_error, is_expected_failure);
+                }
+
+                result.status = TestResult::Status::FAILED;
+
+                if(_throw_errors)
+                {
+                    throw;
+                }
+            }
+            else
+            {
+                if(_log_level >= LogLevel::DEBUG && _printer != nullptr)
+                {
+                    _printer->print_info(error.what());
+                }
+
+                result.status = TestResult::Status::NOT_RUN;
+            }
+        }
         catch(const TestError &error)
         {
             if(_log_level >= error.level() && _printer != nullptr)
diff --git a/tests/framework/Framework.h b/tests/framework/Framework.h
index 3741e44694..063824cbdb 100644
--- a/tests/framework/Framework.h
+++ b/tests/framework/Framework.h
@@ -233,6 +233,18 @@ public:
      */
     void set_stop_on_error(bool stop_on_error);
 
+    /** Indicates if a test should be marked as failed when its assets are missing.
+     *
+     * @return True if a test should be marked as failed when its assets are missing.
+     */
+    bool error_on_missing_assets() const;
+
+    /** Set whether a test should be considered as failed if its assets cannot be found.
+     *
+     * @param[in] error_on_missing_assets True if a test should be marked as failed when its assets are missing.
+     */
+    void set_error_on_missing_assets(bool error_on_missing_assets);
+
     /** Run all enabled test cases.
      *
      * @return True if all test cases executed successful.
@@ -308,6 +320,7 @@ private:
     int      _num_iterations{ 1 };
     bool     _throw_errors{ false };
     bool     _stop_on_error{ false };
+    bool     _error_on_missing_assets{ false };
     Printer *_printer{ nullptr };
 
     using create_function = std::unique_ptr<Instrument>();