diff options
Diffstat (limited to 'framework')
-rw-r--r-- | framework/Asserts.h | 100 | ||||
-rw-r--r-- | framework/Exceptions.cpp | 116 | ||||
-rw-r--r-- | framework/Exceptions.h | 47 | ||||
-rw-r--r-- | framework/Framework.cpp | 78 | ||||
-rw-r--r-- | framework/Framework.h | 10 |
5 files changed, 277 insertions, 74 deletions
diff --git a/framework/Asserts.h b/framework/Asserts.h index fd745d10e0..4fd82abb97 100644 --- a/framework/Asserts.h +++ b/framework/Asserts.h @@ -63,67 +63,67 @@ inline T &&make_printable(T &&value) arm_compute::test::framework::Framework::get().add_test_info(info.str()); \ } -#define ARM_COMPUTE_TEST_COMP_FACTORY(SEVERITY, SEVERITY_NAME, COMP, COMP_NAME, ERROR_CALL) \ - template <typename T, typename U> \ - void ARM_COMPUTE_##SEVERITY##_##COMP_NAME##_IMPL(T &&x, U &&y, const std::string &x_str, const std::string &y_str) \ - { \ - if(!(x COMP y)) \ - { \ - std::stringstream msg; \ - msg << #SEVERITY_NAME " '" << x_str << " " #COMP " " << y_str << "' failed. [" \ - << std::boolalpha << arm_compute::test::framework::detail::make_printable(x) \ - << " " #COMP " " \ - << std::boolalpha << arm_compute::test::framework::detail::make_printable(y) \ - << "]\n"; \ - arm_compute::test::framework::Framework::get().print_test_info(msg); \ - ERROR_CALL \ - } \ - arm_compute::test::framework::Framework::get().clear_test_info(); \ +#define ARM_COMPUTE_TEST_COMP_FACTORY(SEVERITY, SEVERITY_NAME, COMP, COMP_NAME, ERROR_CALL) \ + template <typename T, typename U> \ + void ARM_COMPUTE_##SEVERITY##_##COMP_NAME##_IMPL(T &&x, U &&y, const std::string &x_str, const std::string &y_str, LogLevel level) \ + { \ + if(!(x COMP y)) \ + { \ + std::stringstream msg; \ + msg << #SEVERITY_NAME " '" << x_str << " " #COMP " " << y_str << "' failed. [" \ + << std::boolalpha << arm_compute::test::framework::detail::make_printable(x) \ + << " " #COMP " " \ + << std::boolalpha << arm_compute::test::framework::detail::make_printable(y) \ + << "]\n"; \ + arm_compute::test::framework::Framework::get().print_test_info(msg); \ + ERROR_CALL \ + } \ + arm_compute::test::framework::Framework::get().clear_test_info(); \ } -ARM_COMPUTE_TEST_COMP_FACTORY(EXPECT, Expectation, ==, EQUAL, arm_compute::test::framework::Framework::get().log_failed_expectation(msg.str());) -ARM_COMPUTE_TEST_COMP_FACTORY(EXPECT, Expectation, !=, NOT_EQUAL, arm_compute::test::framework::Framework::get().log_failed_expectation(msg.str());) -ARM_COMPUTE_TEST_COMP_FACTORY(ASSERT, Assertion, ==, EQUAL, throw arm_compute::test::framework::TestError(msg.str());) -ARM_COMPUTE_TEST_COMP_FACTORY(ASSERT, Assertion, !=, NOT_EQUAL, throw arm_compute::test::framework::TestError(msg.str());) +ARM_COMPUTE_TEST_COMP_FACTORY(EXPECT, Expectation, ==, EQUAL, arm_compute::test::framework::Framework::get().log_failed_expectation(msg.str(), level);) +ARM_COMPUTE_TEST_COMP_FACTORY(EXPECT, Expectation, !=, NOT_EQUAL, arm_compute::test::framework::Framework::get().log_failed_expectation(msg.str(), level);) +ARM_COMPUTE_TEST_COMP_FACTORY(ASSERT, Assertion, ==, EQUAL, throw arm_compute::test::framework::TestError(msg.str(), level);) +ARM_COMPUTE_TEST_COMP_FACTORY(ASSERT, Assertion, !=, NOT_EQUAL, throw arm_compute::test::framework::TestError(msg.str(), level);) #define ARM_COMPUTE_ASSERT_NOT_EQUAL(X, Y) \ - arm_compute::test::framework::detail::ARM_COMPUTE_ASSERT_NOT_EQUAL_IMPL(X, Y, #X, #Y) + arm_compute::test::framework::detail::ARM_COMPUTE_ASSERT_NOT_EQUAL_IMPL(X, Y, #X, #Y, LogLevel::ERRORS) #define ARM_COMPUTE_ASSERT_EQUAL(X, Y) \ - arm_compute::test::framework::detail::ARM_COMPUTE_ASSERT_EQUAL_IMPL(X, Y, #X, #Y) + arm_compute::test::framework::detail::ARM_COMPUTE_ASSERT_EQUAL_IMPL(X, Y, #X, #Y, LogLevel::ERRORS) -#define ARM_COMPUTE_EXPECT_EQUAL(X, Y) \ - arm_compute::test::framework::detail::ARM_COMPUTE_EXPECT_EQUAL_IMPL(X, Y, #X, #Y) +#define ARM_COMPUTE_EXPECT_EQUAL(X, Y, LEVEL) \ + arm_compute::test::framework::detail::ARM_COMPUTE_EXPECT_EQUAL_IMPL(X, Y, #X, #Y, LEVEL) -#define ARM_COMPUTE_EXPECT_NOT_EQUAL(X, Y) \ - arm_compute::test::framework::detail::ARM_COMPUTE_EXPECT_NOT_EQUAL_IMPL(X, Y, #X, #Y) +#define ARM_COMPUTE_EXPECT_NOT_EQUAL(X, Y, LEVEL) \ + arm_compute::test::framework::detail::ARM_COMPUTE_EXPECT_NOT_EQUAL_IMPL(X, Y, #X, #Y, LEVEL) -#define ARM_COMPUTE_ASSERT(X) \ - do \ - { \ - const auto &x = X; \ - if(!x) \ - { \ - std::stringstream msg; \ - msg << "Assertion '" #X "' failed.\n"; \ - arm_compute::test::framework::Framework::get().print_test_info(msg); \ - throw arm_compute::test::framework::TestError(msg.str()); \ - } \ - arm_compute::test::framework::Framework::get().clear_test_info(); \ +#define ARM_COMPUTE_ASSERT(X) \ + do \ + { \ + const auto &x = X; \ + if(!x) \ + { \ + std::stringstream msg; \ + msg << "Assertion '" #X "' failed.\n"; \ + arm_compute::test::framework::Framework::get().print_test_info(msg); \ + throw arm_compute::test::framework::TestError(msg.str(), arm_compute::test::framework::LogLevel::ERRORS); \ + } \ + arm_compute::test::framework::Framework::get().clear_test_info(); \ } while(false) -#define ARM_COMPUTE_EXPECT(X) \ - do \ - { \ - const auto &x = X; \ - if(!x) \ - { \ - std::stringstream msg; \ - msg << "Expectation '" #X "' failed.\n"; \ - arm_compute::test::framework::Framework::get().print_test_info(msg); \ - arm_compute::test::framework::Framework::get().log_failed_expectation(msg.str()); \ - } \ - arm_compute::test::framework::Framework::get().clear_test_info(); \ +#define ARM_COMPUTE_EXPECT(X, LEVEL) \ + do \ + { \ + const auto &x = X; \ + if(!x) \ + { \ + std::stringstream msg; \ + msg << "Expectation '" #X "' failed.\n"; \ + arm_compute::test::framework::Framework::get().print_test_info(msg); \ + arm_compute::test::framework::Framework::get().log_failed_expectation(msg.str(), LEVEL); \ + } \ + arm_compute::test::framework::Framework::get().clear_test_info(); \ } while(false) } // namespace detail } // namespace framework diff --git a/framework/Exceptions.cpp b/framework/Exceptions.cpp new file mode 100644 index 0000000000..d573fe91fa --- /dev/null +++ b/framework/Exceptions.cpp @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "Exceptions.h" + +#include <map> +#include <sstream> + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +LogLevel log_level_from_name(const std::string &name) +{ + static const std::map<std::string, LogLevel> levels = + { + { "none", LogLevel::NONE }, + { "config", LogLevel::CONFIG }, + { "tests", LogLevel::TESTS }, + { "errors", LogLevel::ERRORS }, + { "debug", LogLevel::DEBUG }, + { "measurements", LogLevel::MEASUREMENTS }, + { "all", LogLevel::ALL }, + }; + + try + { + return levels.at(name); + } + catch(const std::out_of_range &) + { + throw std::invalid_argument(name); + } +} + +::std::istream &operator>>(::std::istream &stream, LogLevel &level) +{ + std::string value; + stream >> value; + level = log_level_from_name(value); + return stream; +} + +::std::ostream &operator<<(::std::ostream &stream, LogLevel level) +{ + switch(level) + { + case LogLevel::NONE: + stream << "NONE"; + break; + case LogLevel::CONFIG: + stream << "CONFIG"; + break; + case LogLevel::TESTS: + stream << "TESTS"; + break; + case LogLevel::ERRORS: + stream << "ERRORS"; + break; + case LogLevel::DEBUG: + stream << "DEBUG"; + break; + case LogLevel::MEASUREMENTS: + stream << "MEASUREMENTS"; + break; + case LogLevel::ALL: + stream << "ALL"; + break; + default: + throw std::invalid_argument("Unsupported log level"); + } + + return stream; +} + +std::string to_string(LogLevel level) +{ + std::stringstream stream; + stream << level; + return stream.str(); +} + +TestError::TestError(const std::string &msg, LogLevel level) + : runtime_error{ msg }, _level{ level } +{ +} + +LogLevel TestError::level() const +{ + return _level; +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/framework/Exceptions.h b/framework/Exceptions.h index 4e42971cd0..de617602dc 100644 --- a/framework/Exceptions.h +++ b/framework/Exceptions.h @@ -24,7 +24,10 @@ #ifndef ARM_COMPUTE_TEST_EXCEPTIONS #define ARM_COMPUTE_TEST_EXCEPTIONS +#include <istream> +#include <ostream> #include <stdexcept> +#include <string> namespace arm_compute { @@ -32,11 +35,55 @@ namespace test { namespace framework { +/** Severity of the information. + * + * Each category includes the ones above it. + * + * NONE == Only for filtering. Not used to tag information. + * CONFIG == Configuration info. + * TESTS == Information about the tests. + * ERRORS == Violated assertions/expectations. + * DEBUG == More violated assertions/expectations. + * MEASUREMENTS == Information about measurements. + * ALL == Only for filtering. Not used to tag information. + */ +enum class LogLevel +{ + NONE, + CONFIG, + TESTS, + ERRORS, + DEBUG, + MEASUREMENTS, + ALL, +}; + +LogLevel log_level_from_name(const std::string &name); +::std::istream &operator>>(::std::istream &stream, LogLevel &level); +::std::ostream &operator<<(::std::ostream &stream, LogLevel level); +std::string to_string(LogLevel level); + /** Error class for failures during test execution. */ class TestError : public std::runtime_error { public: using std::runtime_error::runtime_error; + + /** Construct error with severity. + * + * @param[in] msg Error message. + * @param[in] level Severity level. + */ + TestError(const std::string &msg, LogLevel level); + + /** Severity of the error. + * + * @return Severity. + */ + LogLevel level() const; + +private: + LogLevel _level{ LogLevel::ERRORS }; }; } // namespace framework } // namespace test diff --git a/framework/Framework.cpp b/framework/Framework.cpp index 73e2e8cf7a..8f605e3e8b 100644 --- a/framework/Framework.cpp +++ b/framework/Framework.cpp @@ -23,7 +23,6 @@ */ #include "Framework.h" -#include "Exceptions.h" #include "support/ToolchainSupport.h" #ifdef ARM_COMPUTE_CL @@ -81,12 +80,13 @@ Framework &Framework::get() return instance; } -void Framework::init(const std::vector<InstrumentType> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, int64_t id_filter) +void Framework::init(const std::vector<InstrumentType> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, int64_t id_filter, LogLevel log_level) { _test_name_filter = std::regex{ name_filter }; _test_id_filter = id_filter; _num_iterations = num_iterations; _dataset_mode = mode; + _log_level = log_level; _instruments = InstrumentType::NONE; @@ -138,7 +138,7 @@ void Framework::print_test_info(std::ostream &os) const void Framework::log_test_start(const std::string &test_name) { - if(_printer != nullptr) + if(_printer != nullptr && _log_level >= LogLevel::TESTS) { _printer->print_test_header(test_name); } @@ -153,14 +153,24 @@ void Framework::log_test_end(const std::string &test_name) { if(_printer != nullptr) { - _printer->print_measurements(_test_results.at(test_name).measurements); - _printer->print_test_footer(); + if(_log_level >= LogLevel::MEASUREMENTS) + { + _printer->print_measurements(_test_results.at(test_name).measurements); + } + + if(_log_level >= LogLevel::TESTS) + { + _printer->print_test_footer(); + } } } -void Framework::log_failed_expectation(const std::string &msg) +void Framework::log_failed_expectation(const std::string &msg, LogLevel level) { - std::cerr << "ERROR: " << msg << "\n"; + if(_log_level >= level) + { + std::cerr << "ERROR: " << msg << "\n"; + } if(_current_test_result != nullptr) { @@ -251,7 +261,11 @@ void Framework::run_test(TestCaseFactory &test_factory) } catch(const TestError &error) { - std::cerr << "FATAL ERROR: " << error.what() << "\n"; + if(_log_level >= error.level()) + { + std::cerr << "FATAL ERROR: " << error.what() << "\n"; + } + result.status = TestResult::Status::FAILED; if(_throw_errors) @@ -262,7 +276,11 @@ void Framework::run_test(TestCaseFactory &test_factory) #ifdef ARM_COMPUTE_CL catch(const ::cl::Error &error) { - std::cerr << "FATAL CL ERROR: " << error.what() << " with code " << error.err() << "\n"; + if(_log_level >= LogLevel::ERRORS) + { + std::cerr << "FATAL CL ERROR: " << error.what() << " with code " << error.err() << "\n"; + } + result.status = TestResult::Status::FAILED; if(_throw_errors) @@ -273,7 +291,11 @@ void Framework::run_test(TestCaseFactory &test_factory) #endif /* ARM_COMPUTE_CL */ catch(const std::exception &error) { - std::cerr << "FATAL ERROR: Received unhandled error: '" << error.what() << "'\n"; + if(_log_level >= LogLevel::ERRORS) + { + std::cerr << "FATAL ERROR: Received unhandled error: '" << error.what() << "'\n"; + } + result.status = TestResult::Status::CRASHED; if(_throw_errors) @@ -283,7 +305,11 @@ void Framework::run_test(TestCaseFactory &test_factory) } catch(...) { - std::cerr << "FATAL ERROR: Received unhandled exception\n"; + if(_log_level >= LogLevel::ERRORS) + { + std::cerr << "FATAL ERROR: Received unhandled exception\n"; + } + result.status = TestResult::Status::CRASHED; if(_throw_errors) @@ -294,7 +320,10 @@ void Framework::run_test(TestCaseFactory &test_factory) } catch(const std::exception &error) { - std::cerr << "FATAL ERROR: Received unhandled error during fixture creation: '" << error.what() << "'\n"; + if(_log_level >= LogLevel::ERRORS) + { + std::cerr << "FATAL ERROR: Received unhandled error during fixture creation: '" << error.what() << "'\n"; + } if(_throw_errors) { @@ -303,7 +332,11 @@ void Framework::run_test(TestCaseFactory &test_factory) } catch(...) { - std::cerr << "FATAL ERROR: Received unhandled exception during fixture creation\n"; + if(_log_level >= LogLevel::ERRORS) + { + std::cerr << "FATAL ERROR: Received unhandled exception during fixture creation\n"; + } + result.status = TestResult::Status::CRASHED; if(_throw_errors) @@ -331,7 +364,7 @@ bool Framework::run() _test_results.clear(); _runtime = std::chrono::seconds{ 0 }; - if(_printer != nullptr) + if(_printer != nullptr && _log_level >= LogLevel::TESTS) { _printer->print_run_header(); } @@ -355,7 +388,7 @@ bool Framework::run() const auto end = std::chrono::high_resolution_clock::now(); - if(_printer != nullptr) + if(_printer != nullptr && _log_level >= LogLevel::TESTS) { _printer->print_run_footer(); } @@ -364,12 +397,15 @@ bool Framework::run() auto test_results = count_test_results(); - std::cout << "Executed " << _test_results.size() << " test(s) (" - << test_results[TestResult::Status::SUCCESS] << " passed, " - << test_results[TestResult::Status::EXPECTED_FAILURE] << " expected failures, " - << test_results[TestResult::Status::FAILED] << " failed, " - << test_results[TestResult::Status::CRASHED] << " crashed, " - << test_results[TestResult::Status::DISABLED] << " disabled) in " << _runtime.count() << " second(s)\n"; + if(_log_level > LogLevel::NONE) + { + std::cout << "Executed " << _test_results.size() << " test(s) (" + << test_results[TestResult::Status::SUCCESS] << " passed, " + << test_results[TestResult::Status::EXPECTED_FAILURE] << " expected failures, " + << test_results[TestResult::Status::FAILED] << " failed, " + << test_results[TestResult::Status::CRASHED] << " crashed, " + << test_results[TestResult::Status::DISABLED] << " disabled) in " << _runtime.count() << " second(s)\n"; + } int num_successful_tests = test_results[TestResult::Status::SUCCESS] + test_results[TestResult::Status::EXPECTED_FAILURE]; diff --git a/framework/Framework.h b/framework/Framework.h index 6bf3f18f3f..5f52b4fbe8 100644 --- a/framework/Framework.h +++ b/framework/Framework.h @@ -25,6 +25,7 @@ #define ARM_COMPUTE_TEST_FRAMEWORK #include "DatasetModes.h" +#include "Exceptions.h" #include "Profiler.h" #include "TestCase.h" #include "TestCaseFactory.h" @@ -95,8 +96,9 @@ public: * @param[in] mode Dataset mode. * @param[in] name_filter Regular expression to filter tests by name. Only matching tests will be executed. * @param[in] id_filter Test id. Only this test will be executed. + * @param[in] log_level Verbosity of the output. */ - void init(const std::vector<InstrumentType> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, int64_t id_filter); + void init(const std::vector<InstrumentType> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, int64_t id_filter, LogLevel log_level); /** Add a new test suite. * @@ -177,9 +179,10 @@ public: /** Tell the framework that the currently running test case failed a non-fatal expectation. * - * @param[in] msg Description of the failure. + * @param[in] msg Description of the failure. + * @param[in] level Severity of the failed expectation. */ - void log_failed_expectation(const std::string &msg); + void log_failed_expectation(const std::string &msg, LogLevel level = LogLevel::ERRORS); /** Number of iterations per test case. * @@ -291,6 +294,7 @@ private: std::regex _test_name_filter{ ".*" }; int64_t _test_id_filter{ -1 }; DatasetMode _dataset_mode{ DatasetMode::ALL }; + LogLevel _log_level{ LogLevel::ALL }; TestResult *_current_test_result{ nullptr }; std::vector<std::string> _test_info{}; }; |