COMPMID-417 Allow the tests to run even when assets are not present

Change-Id: Ief165b1d583a70cbe35aae93f05ddfe962196323
Reviewed-on: http://mpd-gerrit.cambridge.arm.com/89503
Reviewed-by: Georgios Pinitas <georgios.pinitas@arm.com>
Tested-by: Kaizen <jeremy.johnson+kaizengerrit@arm.com>

8 files changed, 80 insertions, 8 deletions

diff --git a/tests/AssetsLibrary.cpp b/tests/AssetsLibrary.cpp
index 529a2a2d95..7e380fa8f3 100644
--- a/tests/AssetsLibrary.cpp
+++ b/tests/AssetsLibrary.cpp
@@ -153,7 +153,7 @@ RawTensor load_ppm(const std::string &path)
 
     if(!file.good())
     {
-        throw std::runtime_error("Could not load PPM image: " + path);
+        throw framework::FileNotFound("Could not load PPM image: " + path);
     }
 
     unsigned int width  = 0;
diff --git a/tests/AssetsLibrary.h b/tests/AssetsLibrary.h
index bd1d8b4758..6199a09012 100644
--- a/tests/AssetsLibrary.h
+++ b/tests/AssetsLibrary.h
@@ -35,6 +35,7 @@
 #include "tests/RawTensor.h"
 #include "tests/TensorCache.h"
 #include "tests/Utils.h"
+#include "tests/framework/Exceptions.h"
 
 #include <algorithm>
 #include <cstddef>
@@ -663,7 +664,10 @@ void AssetsLibrary::fill_layer_data(T &&tensor, std::string name) const
 
     // Open file
     std::ifstream stream(path, std::ios::in | std::ios::binary);
-    ARM_COMPUTE_ERROR_ON_MSG(!stream.good(), "Failed to load binary data");
+    if(!stream.good())
+    {
+        throw framework::FileNotFound("Could not load npy file: " + path);
+    }
     // Check magic bytes and version number
     unsigned char v_major = 0;
     unsigned char v_minor = 0;
diff --git a/tests/framework/Exceptions.cpp b/tests/framework/Exceptions.cpp
index 3d6c65c181..0ca86a8589 100644
--- a/tests/framework/Exceptions.cpp
+++ b/tests/framework/Exceptions.cpp
@@ -104,6 +104,11 @@ std::string to_string(LogLevel level)
     return stream.str();
 }
 
+FileNotFound::FileNotFound(const std::string &msg)
+    : std::runtime_error{ msg }
+{
+}
+
 TestError::TestError(const std::string &msg, LogLevel level, std::string context)
     : std::runtime_error{ msg }, _level{ level }, _msg{ msg }, _context{ std::move(context) }, _combined{ "ERROR: " + msg }
 {
diff --git a/tests/framework/Exceptions.h b/tests/framework/Exceptions.h
index edb0ed92c9..f35c35020c 100644
--- a/tests/framework/Exceptions.h
+++ b/tests/framework/Exceptions.h
@@ -63,6 +63,17 @@ LogLevel log_level_from_name(const std::string &name);
 ::std::ostream &operator<<(::std::ostream &stream, LogLevel level);
 std::string to_string(LogLevel level);
 
+/** Error class for when some external assets are missing */
+class FileNotFound : public std::runtime_error
+{
+public:
+    /** Construct error with message
+     *
+     * @param[in] msg Error message
+     */
+    FileNotFound(const std::string &msg);
+};
+
 /** Error class for failures during test execution. */
 class TestError : public std::runtime_error
 {
diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp
index 9a67cca184..318f9b1d4c 100644
--- a/tests/framework/Framework.cpp
+++ b/tests/framework/Framework.cpp
@@ -216,6 +216,16 @@ bool Framework::stop_on_error() const
     return _stop_on_error;
 }
 
+void Framework::set_error_on_missing_assets(bool error_on_missing_assets)
+{
+    _error_on_missing_assets = error_on_missing_assets;
+}
+
+bool Framework::error_on_missing_assets() const
+{
+    return _error_on_missing_assets;
+}
+
 void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
 {
     if(test_factory.status() == TestCaseFactory::Status::DISABLED)
@@ -269,6 +279,33 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
                 result.status = TestResult::Status::SUCCESS;
             }
         }
+        catch(const FileNotFound &error)
+        {
+            if(_error_on_missing_assets)
+            {
+                if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
+                {
+                    TestError test_error(error.what(), LogLevel::ERRORS);
+                    _printer->print_error(test_error, is_expected_failure);
+                }
+
+                result.status = TestResult::Status::FAILED;
+
+                if(_throw_errors)
+                {
+                    throw;
+                }
+            }
+            else
+            {
+                if(_log_level >= LogLevel::DEBUG && _printer != nullptr)
+                {
+                    _printer->print_info(error.what());
+                }
+
+                result.status = TestResult::Status::NOT_RUN;
+            }
+        }
         catch(const TestError &error)
         {
             if(_log_level >= error.level() && _printer != nullptr)
diff --git a/tests/framework/Framework.h b/tests/framework/Framework.h
index 3741e44694..063824cbdb 100644
--- a/tests/framework/Framework.h
+++ b/tests/framework/Framework.h
@@ -233,6 +233,18 @@ public:
      */
     void set_stop_on_error(bool stop_on_error);
 
+    /** Indicates if a test should be marked as failed when its assets are missing.
+     *
+     * @return True if a test should be marked as failed when its assets are missing.
+     */
+    bool error_on_missing_assets() const;
+
+    /** Set whether a test should be considered as failed if its assets cannot be found.
+     *
+     * @param[in] error_on_missing_assets True if a test should be marked as failed when its assets are missing.
+     */
+    void set_error_on_missing_assets(bool error_on_missing_assets);
+
     /** Run all enabled test cases.
      *
      * @return True if all test cases executed successful.
@@ -308,6 +320,7 @@ private:
     int      _num_iterations{ 1 };
     bool     _throw_errors{ false };
     bool     _stop_on_error{ false };
+    bool     _error_on_missing_assets{ false };
     Printer *_printer{ nullptr };
 
     using create_function = std::unique_ptr<Instrument>();
diff --git a/tests/main.cpp b/tests/main.cpp
index 9cd2895e1b..ee12a38d1a 100644
--- a/tests/main.cpp
+++ b/tests/main.cpp
@@ -120,20 +120,21 @@ int main(int argc, char **argv)
     auto log_file = parser.add_option<framework::SimpleOption<std::string>>("log-file");
     log_file->set_help("Write output to file instead of to the console");
     auto log_level = parser.add_option<framework::EnumOption<framework::LogLevel>>("log-level", supported_log_levels, framework::LogLevel::ALL);
-    log_file->set_help("Verbosity of the output");
+    log_level->set_help("Verbosity of the output");
     auto throw_errors = parser.add_option<framework::ToggleOption>("throw-errors");
     throw_errors->set_help("Don't catch fatal errors (useful for debugging)");
     auto stop_on_error = parser.add_option<framework::ToggleOption>("stop-on-error");
-    throw_errors->set_help("Abort execution after the first failed test (useful for debugging)");
+    stop_on_error->set_help("Abort execution after the first failed test (useful for debugging)");
     auto seed = parser.add_option<framework::SimpleOption<std::random_device::result_type>>("seed", std::random_device()());
     seed->set_help("Global seed for random number generation");
     auto color_output = parser.add_option<framework::ToggleOption>("color-output", true);
     color_output->set_help("Produce colored output on the console");
     auto list_tests = parser.add_option<framework::ToggleOption>("list-tests", false);
     list_tests->set_help("List all test names");
+    auto error_on_missing_assets = parser.add_option<framework::ToggleOption>("error-on-missing-assets", false);
+    error_on_missing_assets->set_help("Mark a test as failed instead of skipping it when assets are missing");
     auto assets = parser.add_positional_option<framework::SimpleOption<std::string>>("assets");
     assets->set_help("Path to the assets directory");
-    assets->set_required(true);
 
     try
     {
@@ -196,6 +197,7 @@ int main(int argc, char **argv)
         framework.set_printer(printer.get());
         framework.set_throw_errors(throw_errors->value());
         framework.set_stop_on_error(stop_on_error->value());
+        framework.set_error_on_missing_assets(error_on_missing_assets->value());
 
         bool success = true;
 
diff --git a/tests/validation_old/UNIT/FixedPoint.cpp b/tests/validation_old/UNIT/FixedPoint.cpp
index 6a92cfb963..26cf905abf 100644
--- a/tests/validation_old/UNIT/FixedPoint.cpp
+++ b/tests/validation_old/UNIT/FixedPoint.cpp
@@ -57,7 +57,7 @@ BOOST_DATA_TEST_CASE(FixedPointQS8Inputs, boost::unit_test::data::make(func_name
     std::ifstream     inputs_file{ base_file_name + ".in", std::ios::binary | std::ios::in };
 
     BOOST_TEST_INFO(base_file_name + ".in");
-    BOOST_TEST_REQUIRE(inputs_file.good());
+    BOOST_TEST_REQUIRE(inputs_file.good()); //FIXME: When moving to new framework: throw a FileNotFound exception
 
     float float_val = 0.f;
 
@@ -90,9 +90,9 @@ BOOST_DATA_TEST_CASE(FixedPointQS8Outputs, (boost::unit_test::data::make(func_na
     std::ifstream     reference_file{ base_file_name + ".out", std::ios::binary | std::ios::in };
 
     BOOST_TEST_INFO(base_file_name + ".in");
-    BOOST_TEST_REQUIRE(inputs_file.good());
+    BOOST_TEST_REQUIRE(inputs_file.good()); //FIXME: When moving to new framework: throw a FileNotFound exception
     BOOST_TEST_INFO(base_file_name + ".out");
-    BOOST_TEST_REQUIRE(reference_file.good());
+    BOOST_TEST_REQUIRE(reference_file.good()); //FIXME: When moving to new framework: throw a FileNotFound exception
 
     const float step_size = std::pow(2.f, -frac_bits);