diff options
| -rw-r--r-- | framework/Framework.cpp | 81 | ||||
| -rw-r--r-- | framework/Framework.h | 41 | ||||
| -rw-r--r-- | framework/Macros.h | 74 | ||||
| -rw-r--r-- | framework/Registrars.h | 14 | ||||
| -rw-r--r-- | framework/TestCaseFactory.h | 60 | ||||
| -rw-r--r-- | framework/TestResult.h | 3 | ||||
| -rw-r--r-- | tests/main.cpp | 4 |
7 files changed, 186 insertions, 91 deletions
diff --git a/framework/Framework.cpp b/framework/Framework.cpp index f2ecff98f4..73e2e8cf7a 100644 --- a/framework/Framework.cpp +++ b/framework/Framework.cpp @@ -63,32 +63,16 @@ std::set<InstrumentType> Framework::available_instruments() const return types; } -std::tuple<int, int, int> Framework::count_test_results() const +std::map<TestResult::Status, int> Framework::count_test_results() const { - int passed = 0; - int failed = 0; - int crashed = 0; + std::map<TestResult::Status, int> counts; for(const auto &test : _test_results) { - switch(test.second.status) - { - case TestResult::Status::SUCCESS: - ++passed; - break; - case TestResult::Status::FAILED: - ++failed; - break; - case TestResult::Status::CRASHED: - ++crashed; - break; - default: - // Do nothing - break; - } + ++counts[test.second.status]; } - return std::make_tuple(passed, failed, crashed); + return counts; } Framework &Framework::get() @@ -204,24 +188,19 @@ bool Framework::throw_errors() const return _throw_errors; } -bool Framework::is_enabled(const TestId &id) const +bool Framework::is_selected(const TestInfo &info) const { - int test_id = 0; - std::string name; - DatasetMode mode = DatasetMode::ALL; - std::tie(test_id, name, mode) = id; - - if((mode & _dataset_mode) == DatasetMode::DISABLED) + if((info.mode & _dataset_mode) == DatasetMode::DISABLED) { return false; } - if(_test_id_filter > -1 && _test_id_filter != test_id) + if(_test_id_filter > -1 && _test_id_filter != info.id) { return false; } - if(!std::regex_search(name, _test_name_filter)) + if(!std::regex_search(info.name, _test_name_filter)) { return false; } @@ -233,6 +212,13 @@ void Framework::run_test(TestCaseFactory &test_factory) { const std::string test_case_name = test_factory.name(); + if(test_factory.status() == TestCaseFactory::Status::DISABLED) + { + log_test_skipped(test_case_name); + set_test_result(test_case_name, TestResult(TestResult::Status::DISABLED)); + return; + } + log_test_start(test_case_name); Profiler profiler = get_profiler(); @@ -328,6 +314,11 @@ void Framework::run_test(TestCaseFactory &test_factory) _current_test_result = nullptr; + if(test_factory.status() == TestCaseFactory::Status::EXPECTED_FAILURE && result.status == TestResult::Status::FAILED) + { + result.status = TestResult::Status::EXPECTED_FAILURE; + } + result.measurements = profiler.measurements(); set_test_result(test_case_name, result); @@ -352,12 +343,9 @@ bool Framework::run() for(auto &test_factory : _test_factories) { const std::string test_case_name = test_factory->name(); + const TestInfo test_info{ id, test_case_name, test_factory->mode(), test_factory->status() }; - if(!is_enabled(TestId(id, test_case_name, test_factory->mode()))) - { - log_test_skipped(test_case_name); - } - else + if(is_selected(test_info)) { run_test(*test_factory); } @@ -374,15 +362,18 @@ bool Framework::run() _runtime = std::chrono::duration_cast<std::chrono::seconds>(end - start); - int passed = 0; - int failed = 0; - int crashed = 0; + auto test_results = count_test_results(); - std::tie(passed, failed, crashed) = count_test_results(); + std::cout << "Executed " << _test_results.size() << " test(s) (" + << test_results[TestResult::Status::SUCCESS] << " passed, " + << test_results[TestResult::Status::EXPECTED_FAILURE] << " expected failures, " + << test_results[TestResult::Status::FAILED] << " failed, " + << test_results[TestResult::Status::CRASHED] << " crashed, " + << test_results[TestResult::Status::DISABLED] << " disabled) in " << _runtime.count() << " second(s)\n"; - std::cout << "Executed " << _test_results.size() << " test(s) (" << passed << " passed, " << failed << " failed, " << crashed << " crashed) in " << _runtime.count() << " second(s)\n"; + int num_successful_tests = test_results[TestResult::Status::SUCCESS] + test_results[TestResult::Status::EXPECTED_FAILURE]; - return (static_cast<unsigned int>(passed) == _test_results.size()); + return (static_cast<unsigned int>(num_successful_tests) == _test_results.size()); } void Framework::set_test_result(std::string test_case_name, TestResult result) @@ -424,17 +415,19 @@ void Framework::set_printer(Printer *printer) _printer = printer; } -std::vector<Framework::TestId> Framework::test_ids() const +std::vector<Framework::TestInfo> Framework::test_infos() const { - std::vector<TestId> ids; + std::vector<TestInfo> ids; int id = 0; for(const auto &factory : _test_factories) { - if(is_enabled(TestId(id, factory->name(), factory->mode()))) + TestInfo test_info{ id, factory->name(), factory->mode(), factory->status() }; + + if(is_selected(test_info)) { - ids.emplace_back(id, factory->name(), factory->mode()); + ids.emplace_back(std::move(test_info)); } ++id; diff --git a/framework/Framework.h b/framework/Framework.h index 4f60fed721..6bf3f18f3f 100644 --- a/framework/Framework.h +++ b/framework/Framework.h @@ -59,15 +59,22 @@ namespace framework class Framework final { public: - /** Type of a test identifier. + /** Information about a test case. * * A test can be identified either via its id or via its name. Additionally - * each test is tagged with the data set mode in which it will be used. + * each test is tagged with the data set mode in which it will be used and + * its status. * * @note The mapping between test id and test name is not guaranteed to be * stable. It is subject to change as new test are added. */ - using TestId = std::tuple<int, std::string, DatasetMode>; + struct TestInfo + { + int id; + std::string name; + DatasetMode mode; + TestCaseFactory::Status status; + }; /** Access to the singleton. * @@ -113,19 +120,21 @@ public: * * @param[in] test_name Name of the new test case. * @param[in] mode Mode in which to include the test. + * @param[in] status Status of the test case. */ template <typename T> - void add_test_case(std::string test_name, DatasetMode mode); + void add_test_case(std::string test_name, DatasetMode mode, TestCaseFactory::Status status); /** Add a data test case to the framework. * * @param[in] test_name Name of the new test case. * @param[in] mode Mode in which to include the test. + * @param[in] status Status of the test case. * @param[in] description Description of @p data. * @param[in] data Data that will be used as input to the test. */ template <typename T, typename D> - void add_data_test_case(std::string test_name, DatasetMode mode, std::string description, D &&data); + void add_data_test_case(std::string test_name, DatasetMode mode, TestCaseFactory::Status status, std::string description, D &&data); /** Add info string for the next expectation/assertion. * @@ -196,13 +205,13 @@ public: */ void set_throw_errors(bool throw_errors); - /** Check if a test case would be executed. + /** Check if a test case is selected to be executed. * - * @param[in] id Id of the test case. + * @param[in] info Test case info. * - * @return True if the test case would be executed. + * @return True if the test case is selected to be executed. */ - bool is_enabled(const TestId &id) const; + bool is_selected(const TestInfo &info) const; /** Run all enabled test cases. * @@ -241,11 +250,11 @@ public: */ void set_printer(Printer *printer); - /** List of @ref TestId's. + /** List of @ref TestInfo's. * * @return Vector with all test ids. */ - std::vector<Framework::TestId> test_ids() const; + std::vector<Framework::TestInfo> test_infos() const; private: Framework(); @@ -255,7 +264,7 @@ private: Framework &operator=(const Framework &) = delete; void run_test(TestCaseFactory &test_factory); - std::tuple<int, int, int> count_test_results() const; + std::map<TestResult::Status, int> count_test_results() const; /** Returns the current test suite name. * @@ -287,18 +296,18 @@ private: }; template <typename T> -inline void Framework::add_test_case(std::string test_name, DatasetMode mode) +inline void Framework::add_test_case(std::string test_name, DatasetMode mode, TestCaseFactory::Status status) { - _test_factories.emplace_back(support::cpp14::make_unique<SimpleTestCaseFactory<T>>(current_suite_name(), std::move(test_name), mode)); + _test_factories.emplace_back(support::cpp14::make_unique<SimpleTestCaseFactory<T>>(current_suite_name(), std::move(test_name), mode, status)); } template <typename T, typename D> -inline void Framework::add_data_test_case(std::string test_name, DatasetMode mode, std::string description, D &&data) +inline void Framework::add_data_test_case(std::string test_name, DatasetMode mode, TestCaseFactory::Status status, std::string description, D &&data) { // WORKAROUND for GCC 4.9 // The function should get *it which is tuple but that seems to trigger a // bug in the compiler. - auto tmp = std::unique_ptr<DataTestCaseFactory<T, decltype(*std::declval<D>())>>(new DataTestCaseFactory<T, decltype(*std::declval<D>())>(current_suite_name(), std::move(test_name), mode, + auto tmp = std::unique_ptr<DataTestCaseFactory<T, decltype(*std::declval<D>())>>(new DataTestCaseFactory<T, decltype(*std::declval<D>())>(current_suite_name(), std::move(test_name), mode, status, std::move(description), *data)); _test_factories.emplace_back(std::move(tmp)); } diff --git a/framework/Macros.h b/framework/Macros.h index f5b22b461a..7aabb75cfc 100644 --- a/framework/Macros.h +++ b/framework/Macros.h @@ -119,28 +119,35 @@ { \ FIXTURE::teardown(); \ } -#define TEST_REGISTRAR(TEST_NAME, MODE) \ +#define TEST_REGISTRAR(TEST_NAME, MODE, STATUS) \ static arm_compute::test::framework::detail::TestCaseRegistrar<TEST_NAME> TEST_NAME##_reg \ { \ - #TEST_NAME, MODE \ + #TEST_NAME, MODE, STATUS \ } -#define DATA_TEST_REGISTRAR(TEST_NAME, MODE, DATASET) \ +#define DATA_TEST_REGISTRAR(TEST_NAME, MODE, STATUS, DATASET) \ static arm_compute::test::framework::detail::TestCaseRegistrar<TEST_NAME<decltype(DATASET)::type>> TEST_NAME##_reg \ { \ - #TEST_NAME, MODE, DATASET \ + #TEST_NAME, MODE, STATUS, DATASET \ } -#define TEST_CASE(TEST_NAME, MODE) \ +#define TEST_CASE_IMPL(TEST_NAME, MODE, STATUS) \ class TEST_NAME : public arm_compute::test::framework::TestCase \ { \ public: \ TEST_CASE_CONSTRUCTOR(TEST_NAME) \ void do_run() override; \ }; \ - TEST_REGISTRAR(TEST_NAME, MODE); \ + TEST_REGISTRAR(TEST_NAME, MODE, STATUS); \ void TEST_NAME::do_run() -#define DATA_TEST_CASE(TEST_NAME, MODE, DATASET, ...) \ +#define TEST_CASE(TEST_NAME, MODE) \ + TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE) +#define EXPECTED_FAILURE_TEST_CASE(TEST_NAME, MODE) \ + TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE) +#define DISABLED_TEST_CASE(TEST_NAME, MODE) \ + TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED) + +#define DATA_TEST_CASE_IMPL(TEST_NAME, MODE, STATUS, DATASET, ...) \ template <typename T> \ class TEST_NAME; \ template <typename... As> \ @@ -155,12 +162,19 @@ template <MAKE_TYPE_PARAMS(__VA_ARGS__)> \ void run(MAKE_ARG_PARAMS(__VA_ARGS__)); \ }; \ - DATA_TEST_REGISTRAR(TEST_NAME, MODE, DATASET); \ + DATA_TEST_REGISTRAR(TEST_NAME, MODE, STATUS, DATASET); \ template <typename... As> \ template <MAKE_TYPE_PARAMS(__VA_ARGS__)> \ void TEST_NAME<std::tuple<As...>>::run(MAKE_ARG_PARAMS(__VA_ARGS__)) -#define FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \ +#define DATA_TEST_CASE(TEST_NAME, MODE, DATASET, ...) \ + DATA_TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE, DATASET, __VA_ARGS__) +#define EXPECTED_FAILURE_DATA_TEST_CASE(TEST_NAME, MODE, DATASET, ...) \ + DATA_TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE, DATASET, __VA_ARGS__) +#define DISABLED_DATA_TEST_CASE(TEST_NAME, MODE, DATASET, ...) \ + DATA_TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED, DATASET, __VA_ARGS__) + +#define FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, STATUS) \ class TEST_NAME : public arm_compute::test::framework::TestCase, public FIXTURE \ { \ public: \ @@ -169,10 +183,17 @@ void do_run() override; \ FIXTURE_TEARDOWN(FIXTURE) \ }; \ - TEST_REGISTRAR(TEST_NAME, MODE); \ + TEST_REGISTRAR(TEST_NAME, MODE, STATUS); \ void TEST_NAME::do_run() -#define FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \ +#define FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \ + FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE) +#define EXPECTED_FAILURE_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \ + FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE) +#define DISABLED_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \ + FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED) + +#define FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, STATUS, DATASET) \ template <typename T> \ class TEST_NAME; \ template <typename... As> \ @@ -184,11 +205,18 @@ void do_run() override; \ FIXTURE_TEARDOWN(FIXTURE) \ }; \ - DATA_TEST_REGISTRAR(TEST_NAME, MODE, DATASET); \ + DATA_TEST_REGISTRAR(TEST_NAME, MODE, STATUS, DATASET); \ template <typename... As> \ void TEST_NAME<std::tuple<As...>>::do_run() -#define REGISTER_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \ +#define FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \ + FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE, DATASET) +#define EXPECTED_FAILURE_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \ + FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE, DATASET) +#define DISABLED_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \ + FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED, DATASET) + +#define REGISTER_FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, STATUS) \ class TEST_NAME : public arm_compute::test::framework::TestCase, public FIXTURE \ { \ public: \ @@ -197,9 +225,16 @@ FIXTURE_RUN(FIXTURE) \ FIXTURE_TEARDOWN(FIXTURE) \ }; \ - TEST_REGISTRAR(TEST_NAME, MODE) + TEST_REGISTRAR(TEST_NAME, MODE, STATUS) + +#define REGISTER_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \ + REGISTER_FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE) +#define EXPECTED_FAILURE_REGISTER_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \ + REGISTER_FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE) +#define DISABLED_REGISTER_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \ + REGISTER_FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED) -#define REGISTER_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \ +#define REGISTER_FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, STATUS, DATASET) \ template <typename T> \ class TEST_NAME; \ template <typename... As> \ @@ -211,7 +246,14 @@ FIXTURE_RUN(FIXTURE) \ FIXTURE_TEARDOWN(FIXTURE) \ }; \ - DATA_TEST_REGISTRAR(TEST_NAME, MODE, DATASET) + DATA_TEST_REGISTRAR(TEST_NAME, MODE, STATUS, DATASET) + +#define REGISTER_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \ + REGISTER_FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE, DATASET) +#define EXPECTED_FAILURE_REGISTER_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \ + REGISTER_FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE, DATASET) +#define DISABLED_REGISTER_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \ + REGISTER_FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED, DATASET) // // TEST CASE MACROS END // diff --git a/framework/Registrars.h b/framework/Registrars.h index ddbffabee4..ca23edf0de 100644 --- a/framework/Registrars.h +++ b/framework/Registrars.h @@ -47,17 +47,19 @@ public: * * @param[in] test_name Name of the test case. * @param[in] mode Mode in which the test should be activated. + * @param[in] status Status of the test case. */ - TestCaseRegistrar(std::string test_name, DatasetMode mode); + TestCaseRegistrar(std::string test_name, DatasetMode mode, TestCaseFactory::Status status); /** Add a new data test case with the given name to the framework. * * @param[in] test_name Name of the test case. * @param[in] mode Mode in which the test should be activated. + * @param[in] status Status of the test case. * @param[in] dataset Dataset used as input for the test case. */ template <typename D> - TestCaseRegistrar(std::string test_name, DatasetMode mode, D &&dataset); + TestCaseRegistrar(std::string test_name, DatasetMode mode, TestCaseFactory::Status status, D &&dataset); }; /** Helper class to statically begin and end a test suite. */ @@ -75,14 +77,14 @@ public: }; template <typename T> -inline TestCaseRegistrar<T>::TestCaseRegistrar(std::string test_name, DatasetMode mode) +inline TestCaseRegistrar<T>::TestCaseRegistrar(std::string test_name, DatasetMode mode, TestCaseFactory::Status status) { - Framework::get().add_test_case<T>(std::move(test_name), mode); + Framework::get().add_test_case<T>(std::move(test_name), mode, status); } template <typename T> template <typename D> -inline TestCaseRegistrar<T>::TestCaseRegistrar(std::string test_name, DatasetMode mode, D &&dataset) +inline TestCaseRegistrar<T>::TestCaseRegistrar(std::string test_name, DatasetMode mode, TestCaseFactory::Status status, D &&dataset) { auto it = dataset.begin(); @@ -91,7 +93,7 @@ inline TestCaseRegistrar<T>::TestCaseRegistrar(std::string test_name, DatasetMod // WORKAROUND for GCC 4.9 // The last argument should be *it to pass just the data and not the // iterator. - Framework::get().add_data_test_case<T>(test_name, mode, it.description(), it); + Framework::get().add_data_test_case<T>(test_name, mode, status, it.description(), it); } } diff --git a/framework/TestCaseFactory.h b/framework/TestCaseFactory.h index e275e298d6..b8c8cdbeb0 100644 --- a/framework/TestCaseFactory.h +++ b/framework/TestCaseFactory.h @@ -41,14 +41,28 @@ namespace framework class TestCaseFactory { public: + /** Test case status. + * + * ACTIVE == Test is run and result is validated. Failure on failed validation. + * EXPECTED_FAILURE == Test is run and result is validated. Failure on successful validation. + * DISABLED == Test is not run. + */ + enum class Status + { + ACTIVE, + EXPECTED_FAILURE, + DISABLED + }; + /** Constructor. * * @param[in] suite_name Name of the test suite to which the test case has been added. * @param[in] name Name of the test case. * @param[in] mode Datset mode of the test case. + * @param[in] status Status of the test case. * @param[in] description Description of data arguments. */ - TestCaseFactory(std::string suite_name, std::string name, DatasetMode mode, std::string description = ""); + TestCaseFactory(std::string suite_name, std::string name, DatasetMode mode, Status status, std::string description = ""); /** Default destructor. */ virtual ~TestCaseFactory() = default; @@ -65,6 +79,12 @@ public: */ DatasetMode mode() const; + /** Get the status of the test case. + * + * @return Status of the test case. + */ + Status status() const; + /** Factory function to create the test case * * @return Unique pointer to a newly created test case. @@ -76,6 +96,7 @@ private: const std::string _test_name; const std::string _data_description; const DatasetMode _mode{ DatasetMode::ALL }; + const Status _status{ Status::ACTIVE }; }; /** Implementation of a test case factory to create non-data test cases. */ @@ -98,10 +119,11 @@ public: * @param[in] suite_name Name of the test suite to which the test case has been added. * @param[in] test_name Name of the test case. * @param[in] mode Mode in which the test case is enabled. + * @param[in] status Status of the test case. * @param[in] description Description of data arguments. * @param[in] data Input data for the test case. */ - DataTestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, std::string description, const D &data); + DataTestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, Status status, std::string description, const D &data); std::unique_ptr<TestCase> make() const override; @@ -109,8 +131,9 @@ private: D _data; }; -inline TestCaseFactory::TestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, std::string description) - : _suite_name{ std::move(suite_name) }, _test_name{ std::move(test_name) }, _data_description{ std::move(description) }, _mode{ mode } +inline TestCaseFactory::TestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, Status status, std::string description) + : _suite_name{ std::move(suite_name) }, _test_name{ std::move(test_name) }, _data_description{ std::move(description) }, _mode{ mode }, _status{ status } + { } @@ -131,6 +154,31 @@ inline DatasetMode TestCaseFactory::mode() const return _mode; } +inline TestCaseFactory::Status TestCaseFactory::status() const +{ + return _status; +} + +inline ::std::ostream &operator<<(::std::ostream &stream, TestCaseFactory::Status status) +{ + switch(status) + { + case TestCaseFactory::Status::ACTIVE: + stream << "ACTIVE"; + break; + case TestCaseFactory::Status::EXPECTED_FAILURE: + stream << "EXPECTED_FAILURE"; + break; + case TestCaseFactory::Status::DISABLED: + stream << "DISABLED"; + break; + default: + throw std::invalid_argument("Unsupported test case factory status"); + } + + return stream; +} + template <typename T> inline std::unique_ptr<TestCase> SimpleTestCaseFactory<T>::make() const { @@ -138,8 +186,8 @@ inline std::unique_ptr<TestCase> SimpleTestCaseFactory<T>::make() const } template <typename T, typename D> -inline DataTestCaseFactory<T, D>::DataTestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, std::string description, const D &data) - : TestCaseFactory{ std::move(suite_name), std::move(test_name), mode, std::move(description) }, _data{ data } +inline DataTestCaseFactory<T, D>::DataTestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, Status status, std::string description, const D &data) + : TestCaseFactory{ std::move(suite_name), std::move(test_name), mode, status, std::move(description) }, _data{ data } { } diff --git a/framework/TestResult.h b/framework/TestResult.h index 747c2defb4..e71ef95112 100644 --- a/framework/TestResult.h +++ b/framework/TestResult.h @@ -45,7 +45,8 @@ struct TestResult SUCCESS, EXPECTED_FAILURE, FAILED, - CRASHED + CRASHED, + DISABLED }; /** Default constructor. */ diff --git a/tests/main.cpp b/tests/main.cpp index 95c2d949cf..e60aad4d86 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -176,9 +176,9 @@ int main(int argc, char **argv) if(list_tests->value()) { - for(const auto &id : framework.test_ids()) + for(const auto &info : framework.test_infos()) { - std::cout << "[" << std::get<0>(id) << ", " << std::get<2>(id) << "] " << std::get<1>(id) << "\n"; + std::cout << "[" << info.id << ", " << info.mode << ", " << info.status << "] " << info.name << "\n"; } return 0; |