diff options
-rw-r--r-- | tests/AssetsLibrary.cpp | 2 | ||||
-rw-r--r-- | tests/AssetsLibrary.h | 6 | ||||
-rw-r--r-- | tests/framework/Exceptions.cpp | 5 | ||||
-rw-r--r-- | tests/framework/Exceptions.h | 11 | ||||
-rw-r--r-- | tests/framework/Framework.cpp | 37 | ||||
-rw-r--r-- | tests/framework/Framework.h | 13 | ||||
-rw-r--r-- | tests/main.cpp | 8 | ||||
-rw-r--r-- | tests/validation_old/UNIT/FixedPoint.cpp | 6 |
8 files changed, 80 insertions, 8 deletions
diff --git a/tests/AssetsLibrary.cpp b/tests/AssetsLibrary.cpp index 529a2a2d95..7e380fa8f3 100644 --- a/tests/AssetsLibrary.cpp +++ b/tests/AssetsLibrary.cpp @@ -153,7 +153,7 @@ RawTensor load_ppm(const std::string &path) if(!file.good()) { - throw std::runtime_error("Could not load PPM image: " + path); + throw framework::FileNotFound("Could not load PPM image: " + path); } unsigned int width = 0; diff --git a/tests/AssetsLibrary.h b/tests/AssetsLibrary.h index bd1d8b4758..6199a09012 100644 --- a/tests/AssetsLibrary.h +++ b/tests/AssetsLibrary.h @@ -35,6 +35,7 @@ #include "tests/RawTensor.h" #include "tests/TensorCache.h" #include "tests/Utils.h" +#include "tests/framework/Exceptions.h" #include <algorithm> #include <cstddef> @@ -663,7 +664,10 @@ void AssetsLibrary::fill_layer_data(T &&tensor, std::string name) const // Open file std::ifstream stream(path, std::ios::in | std::ios::binary); - ARM_COMPUTE_ERROR_ON_MSG(!stream.good(), "Failed to load binary data"); + if(!stream.good()) + { + throw framework::FileNotFound("Could not load npy file: " + path); + } // Check magic bytes and version number unsigned char v_major = 0; unsigned char v_minor = 0; diff --git a/tests/framework/Exceptions.cpp b/tests/framework/Exceptions.cpp index 3d6c65c181..0ca86a8589 100644 --- a/tests/framework/Exceptions.cpp +++ b/tests/framework/Exceptions.cpp @@ -104,6 +104,11 @@ std::string to_string(LogLevel level) return stream.str(); } +FileNotFound::FileNotFound(const std::string &msg) + : std::runtime_error{ msg } +{ +} + TestError::TestError(const std::string &msg, LogLevel level, std::string context) : std::runtime_error{ msg }, _level{ level }, _msg{ msg }, _context{ std::move(context) }, _combined{ "ERROR: " + msg } { diff --git a/tests/framework/Exceptions.h b/tests/framework/Exceptions.h index edb0ed92c9..f35c35020c 100644 --- a/tests/framework/Exceptions.h +++ b/tests/framework/Exceptions.h @@ -63,6 +63,17 @@ LogLevel log_level_from_name(const std::string &name); ::std::ostream &operator<<(::std::ostream &stream, LogLevel level); std::string to_string(LogLevel level); +/** Error class for when some external assets are missing */ +class FileNotFound : public std::runtime_error +{ +public: + /** Construct error with message + * + * @param[in] msg Error message + */ + FileNotFound(const std::string &msg); +}; + /** Error class for failures during test execution. */ class TestError : public std::runtime_error { diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp index 9a67cca184..318f9b1d4c 100644 --- a/tests/framework/Framework.cpp +++ b/tests/framework/Framework.cpp @@ -216,6 +216,16 @@ bool Framework::stop_on_error() const return _stop_on_error; } +void Framework::set_error_on_missing_assets(bool error_on_missing_assets) +{ + _error_on_missing_assets = error_on_missing_assets; +} + +bool Framework::error_on_missing_assets() const +{ + return _error_on_missing_assets; +} + void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) { if(test_factory.status() == TestCaseFactory::Status::DISABLED) @@ -269,6 +279,33 @@ void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) result.status = TestResult::Status::SUCCESS; } } + catch(const FileNotFound &error) + { + if(_error_on_missing_assets) + { + if(_log_level >= LogLevel::ERRORS && _printer != nullptr) + { + TestError test_error(error.what(), LogLevel::ERRORS); + _printer->print_error(test_error, is_expected_failure); + } + + result.status = TestResult::Status::FAILED; + + if(_throw_errors) + { + throw; + } + } + else + { + if(_log_level >= LogLevel::DEBUG && _printer != nullptr) + { + _printer->print_info(error.what()); + } + + result.status = TestResult::Status::NOT_RUN; + } + } catch(const TestError &error) { if(_log_level >= error.level() && _printer != nullptr) diff --git a/tests/framework/Framework.h b/tests/framework/Framework.h index 3741e44694..063824cbdb 100644 --- a/tests/framework/Framework.h +++ b/tests/framework/Framework.h @@ -233,6 +233,18 @@ public: */ void set_stop_on_error(bool stop_on_error); + /** Indicates if a test should be marked as failed when its assets are missing. + * + * @return True if a test should be marked as failed when its assets are missing. + */ + bool error_on_missing_assets() const; + + /** Set whether a test should be considered as failed if its assets cannot be found. + * + * @param[in] error_on_missing_assets True if a test should be marked as failed when its assets are missing. + */ + void set_error_on_missing_assets(bool error_on_missing_assets); + /** Run all enabled test cases. * * @return True if all test cases executed successful. @@ -308,6 +320,7 @@ private: int _num_iterations{ 1 }; bool _throw_errors{ false }; bool _stop_on_error{ false }; + bool _error_on_missing_assets{ false }; Printer *_printer{ nullptr }; using create_function = std::unique_ptr<Instrument>(); diff --git a/tests/main.cpp b/tests/main.cpp index 9cd2895e1b..ee12a38d1a 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -120,20 +120,21 @@ int main(int argc, char **argv) auto log_file = parser.add_option<framework::SimpleOption<std::string>>("log-file"); log_file->set_help("Write output to file instead of to the console"); auto log_level = parser.add_option<framework::EnumOption<framework::LogLevel>>("log-level", supported_log_levels, framework::LogLevel::ALL); - log_file->set_help("Verbosity of the output"); + log_level->set_help("Verbosity of the output"); auto throw_errors = parser.add_option<framework::ToggleOption>("throw-errors"); throw_errors->set_help("Don't catch fatal errors (useful for debugging)"); auto stop_on_error = parser.add_option<framework::ToggleOption>("stop-on-error"); - throw_errors->set_help("Abort execution after the first failed test (useful for debugging)"); + stop_on_error->set_help("Abort execution after the first failed test (useful for debugging)"); auto seed = parser.add_option<framework::SimpleOption<std::random_device::result_type>>("seed", std::random_device()()); seed->set_help("Global seed for random number generation"); auto color_output = parser.add_option<framework::ToggleOption>("color-output", true); color_output->set_help("Produce colored output on the console"); auto list_tests = parser.add_option<framework::ToggleOption>("list-tests", false); list_tests->set_help("List all test names"); + auto error_on_missing_assets = parser.add_option<framework::ToggleOption>("error-on-missing-assets", false); + error_on_missing_assets->set_help("Mark a test as failed instead of skipping it when assets are missing"); auto assets = parser.add_positional_option<framework::SimpleOption<std::string>>("assets"); assets->set_help("Path to the assets directory"); - assets->set_required(true); try { @@ -196,6 +197,7 @@ int main(int argc, char **argv) framework.set_printer(printer.get()); framework.set_throw_errors(throw_errors->value()); framework.set_stop_on_error(stop_on_error->value()); + framework.set_error_on_missing_assets(error_on_missing_assets->value()); bool success = true; diff --git a/tests/validation_old/UNIT/FixedPoint.cpp b/tests/validation_old/UNIT/FixedPoint.cpp index 6a92cfb963..26cf905abf 100644 --- a/tests/validation_old/UNIT/FixedPoint.cpp +++ b/tests/validation_old/UNIT/FixedPoint.cpp @@ -57,7 +57,7 @@ BOOST_DATA_TEST_CASE(FixedPointQS8Inputs, boost::unit_test::data::make(func_name std::ifstream inputs_file{ base_file_name + ".in", std::ios::binary | std::ios::in }; BOOST_TEST_INFO(base_file_name + ".in"); - BOOST_TEST_REQUIRE(inputs_file.good()); + BOOST_TEST_REQUIRE(inputs_file.good()); //FIXME: When moving to new framework: throw a FileNotFound exception float float_val = 0.f; @@ -90,9 +90,9 @@ BOOST_DATA_TEST_CASE(FixedPointQS8Outputs, (boost::unit_test::data::make(func_na std::ifstream reference_file{ base_file_name + ".out", std::ios::binary | std::ios::in }; BOOST_TEST_INFO(base_file_name + ".in"); - BOOST_TEST_REQUIRE(inputs_file.good()); + BOOST_TEST_REQUIRE(inputs_file.good()); //FIXME: When moving to new framework: throw a FileNotFound exception BOOST_TEST_INFO(base_file_name + ".out"); - BOOST_TEST_REQUIRE(reference_file.good()); + BOOST_TEST_REQUIRE(reference_file.good()); //FIXME: When moving to new framework: throw a FileNotFound exception const float step_size = std::pow(2.f, -frac_bits); |