diff options
author | Moritz Pflanzer <moritz.pflanzer@arm.com> | 2017-07-05 11:07:07 +0100 |
---|---|---|
committer | Anthony Barbier <anthony.barbier@arm.com> | 2018-09-17 14:15:39 +0100 |
commit | fc95ed2b9900471922d93c963b263f1f506da167 (patch) | |
tree | 765f6a94e965ed47b0c0215a725f869e0619a611 /framework | |
parent | 7d323a6adca97c130a0fc7c6299c75d581906edd (diff) | |
download | ComputeLibrary-fc95ed2b9900471922d93c963b263f1f506da167.tar.gz |
COMPMID-415: New framework - base framework [1/5]
Change-Id: Icfbfb43321c3bbe6e2aa511dd03a613eed7734a5
Reviewed-on: http://mpd-gerrit.cambridge.arm.com/79760
Tested-by: Kaizen <jeremy.johnson+kaizengerrit@arm.com>
Reviewed-by: Anthony Barbier <anthony.barbier@arm.com>
Diffstat (limited to 'framework')
-rw-r--r-- | framework/Exceptions.h | 44 | ||||
-rw-r--r-- | framework/Fixture.h | 63 | ||||
-rw-r--r-- | framework/Framework.cpp | 286 | ||||
-rw-r--r-- | framework/Framework.h | 240 | ||||
-rw-r--r-- | framework/Macros.h | 188 | ||||
-rw-r--r-- | framework/Registrars.h | 108 | ||||
-rw-r--r-- | framework/SConscript | 68 | ||||
-rw-r--r-- | framework/TestCase.h | 70 | ||||
-rw-r--r-- | framework/TestCaseFactory.h | 139 | ||||
-rw-r--r-- | framework/TestResult.h | 66 | ||||
-rw-r--r-- | framework/Utils.h | 143 |
11 files changed, 1415 insertions, 0 deletions
diff --git a/framework/Exceptions.h b/framework/Exceptions.h new file mode 100644 index 0000000000..4e42971cd0 --- /dev/null +++ b/framework/Exceptions.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_EXCEPTIONS +#define ARM_COMPUTE_TEST_EXCEPTIONS + +#include <stdexcept> + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Error class for failures during test execution. */ +class TestError : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_EXCEPTIONS */ diff --git a/framework/Fixture.h b/framework/Fixture.h new file mode 100644 index 0000000000..916dcc7fef --- /dev/null +++ b/framework/Fixture.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_FIXTURE +#define ARM_COMPUTE_TEST_FIXTURE + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Abstract fixture class. + * + * All custom fixtures have to inherit from this class. + */ +class Fixture +{ +public: + /** Setup function. + * + * This function is only invoked by non-data fixture test cases. Fixture + * data test cases implement a setup function with arguments matching the + * dataset. + * + * The function is called before the test case is executed. + */ + void setup() {}; + + /** Teardown function. + * + * The function is called after the test case finished. + */ + void teardown() {}; + +protected: + Fixture() = default; + virtual ~Fixture() = default; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_FIXTURE */ diff --git a/framework/Framework.cpp b/framework/Framework.cpp new file mode 100644 index 0000000000..b54c0c75b6 --- /dev/null +++ b/framework/Framework.cpp @@ -0,0 +1,286 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "Framework.h" + +#include "Exceptions.h" +#include "support/ToolchainSupport.h" + +#include <chrono> +#include <iostream> +#include <sstream> +#include <type_traits> + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +std::tuple<int, int, int> Framework::count_test_results() const +{ + int passed = 0; + int failed = 0; + int crashed = 0; + + for(const auto &test : _test_results) + { + switch(test.second.status) + { + case TestResult::Status::SUCCESS: + ++passed; + break; + case TestResult::Status::FAILED: + ++failed; + break; + case TestResult::Status::CRASHED: + ++crashed; + break; + default: + // Do nothing + break; + } + } + + return std::make_tuple(passed, failed, crashed); +} + +Framework &Framework::get() +{ + static Framework instance; + return instance; +} + +void Framework::init(int num_iterations, const std::string &name_filter, const std::string &id_filter) +{ + _test_name_filter = std::regex{ name_filter }; + _test_id_filter = std::regex{ id_filter }; + _num_iterations = num_iterations; +} + +std::string Framework::current_suite_name() const +{ + return join(_test_suite_name.cbegin(), _test_suite_name.cend(), "/"); +} + +void Framework::push_suite(std::string name) +{ + _test_suite_name.emplace_back(std::move(name)); +} + +void Framework::pop_suite() +{ + _test_suite_name.pop_back(); +} + +void Framework::log_test_start(const std::string &test_name) +{ + static_cast<void>(test_name); +} + +void Framework::log_test_skipped(const std::string &test_name) +{ + static_cast<void>(test_name); +} + +void Framework::log_test_end(const std::string &test_name) +{ + static_cast<void>(test_name); +} + +void Framework::log_failed_expectation(const std::string &msg) +{ + std::cerr << "ERROR: " << msg << "\n"; +} + +int Framework::num_iterations() const +{ + return _num_iterations; +} + +void Framework::set_num_iterations(int num_iterations) +{ + _num_iterations = num_iterations; +} + +void Framework::set_throw_errors(bool throw_errors) +{ + _throw_errors = throw_errors; +} + +bool Framework::throw_errors() const +{ + return _throw_errors; +} + +bool Framework::is_enabled(const TestId &id) const +{ + return (std::regex_search(support::cpp11::to_string(id.first), _test_id_filter) && std::regex_search(id.second, _test_name_filter)); +} + +void Framework::run_test(TestCaseFactory &test_factory) +{ + const std::string test_case_name = test_factory.name(); + + log_test_start(test_case_name); + + TestResult result; + + try + { + std::unique_ptr<TestCase> test_case = test_factory.make(); + + try + { + test_case->do_setup(); + + for(int i = 0; i < _num_iterations; ++i) + { + test_case->do_run(); + } + + test_case->do_teardown(); + + result.status = TestResult::Status::SUCCESS; + } + catch(const TestError &error) + { + std::cerr << "FATAL ERROR: " << error.what() << "\n"; + result.status = TestResult::Status::FAILED; + + if(_throw_errors) + { + throw; + } + } + catch(const std::exception &error) + { + std::cerr << "FATAL ERROR: Received unhandled error: '" << error.what() << "'\n"; + result.status = TestResult::Status::CRASHED; + + if(_throw_errors) + { + throw; + } + } + catch(...) + { + std::cerr << "FATAL ERROR: Received unhandled exception\n"; + result.status = TestResult::Status::CRASHED; + + if(_throw_errors) + { + throw; + } + } + } + catch(const std::exception &error) + { + std::cerr << "FATAL ERROR: Received unhandled error during fixture creation: '" << error.what() << "'\n"; + + if(_throw_errors) + { + throw; + } + } + catch(...) + { + std::cerr << "FATAL ERROR: Received unhandled exception during fixture creation\n"; + + if(_throw_errors) + { + throw; + } + } + + set_test_result(test_case_name, result); + log_test_end(test_case_name); +} + +bool Framework::run() +{ + // Clear old test results + _test_results.clear(); + _runtime = std::chrono::seconds{ 0 }; + + const auto start = std::chrono::high_resolution_clock::now(); + + int id = 0; + + for(auto &test_factory : _test_factories) + { + const std::string test_case_name = test_factory->name(); + + if(!is_enabled(TestId(id, test_case_name))) + { + log_test_skipped(test_case_name); + } + else + { + run_test(*test_factory); + } + + ++id; + } + + const auto end = std::chrono::high_resolution_clock::now(); + + _runtime = std::chrono::duration_cast<std::chrono::seconds>(end - start); + + int passed = 0; + int failed = 0; + int crashed = 0; + + std::tie(passed, failed, crashed) = count_test_results(); + + std::cout << "Executed " << _test_results.size() << " test(s) (" << passed << " passed, " << failed << " failed, " << crashed << " crashed) in " << _runtime.count() << " second(s)\n"; + + return (static_cast<unsigned int>(passed) == _test_results.size()); +} + +void Framework::set_test_result(std::string test_case_name, TestResult result) +{ + _test_results.emplace(std::move(test_case_name), result); +} + +std::vector<Framework::TestId> Framework::test_ids() const +{ + std::vector<TestId> ids; + + int id = 0; + + for(const auto &factory : _test_factories) + { + if(is_enabled(TestId(id, factory->name()))) + { + ids.emplace_back(id, factory->name()); + } + + ++id; + } + + return ids; +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/framework/Framework.h b/framework/Framework.h new file mode 100644 index 0000000000..e9beafd6eb --- /dev/null +++ b/framework/Framework.h @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_FRAMEWORK +#define ARM_COMPUTE_TEST_FRAMEWORK + +#include "TestCase.h" +#include "TestCaseFactory.h" +#include "TestResult.h" +#include "Utils.h" + +#include <algorithm> +#include <chrono> +#include <map> +#include <memory> +#include <numeric> +#include <ostream> +#include <regex> +#include <set> +#include <sstream> +#include <string> +#include <tuple> +#include <vector> + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Main framework class. + * + * Keeps track of the global state, owns all test cases and collects results. + */ +class Framework final +{ +public: + /** Type of a test identifier. + * + * A test can be identified either via its id or via its name. + * + * @note The mapping between test id and test name is not guaranteed to be + * stable. It is subject to change as new test are added. + */ + using TestId = std::pair<int, std::string>; + + /** Access to the singleton. + * + * @return Unique instance of the framework class. + */ + static Framework &get(); + + /** Init the framework. + * + * @param[in] num_iterations Number of iterations per test. + * @param[in] name_filter Regular expression to filter tests by name. Only matching tests will be executed. + * @param[in] id_filter Regular expression to filter tests by id. Only matching tests will be executed. + */ + void init(int num_iterations, const std::string &name_filter, const std::string &id_filter); + + /** Add a new test suite. + * + * @warning Cannot be used at execution time. It can only be used for + * registering test cases. + * + * @param[in] name Name of the added test suite. + * + * @return Name of the current test suite. + */ + void push_suite(std::string name); + + /** Remove innermost test suite. + * + * @warning Cannot be used at execution time. It can only be used for + * registering test cases. + */ + void pop_suite(); + + /** Add a test case to the framework. + * + * @param[in] test_name Name of the new test case. + */ + template <typename T> + void add_test_case(std::string test_name); + + /** Add a data test case to the framework. + * + * @param[in] test_name Name of the new test case. + * @param[in] description Description of @p data. + * @param[in] data Data that will be used as input to the test. + */ + template <typename T, typename D> + void add_data_test_case(std::string test_name, std::string description, D &&data); + + /** Tell the framework that execution of a test starts. + * + * @param[in] test_name Name of the started test case. + */ + void log_test_start(const std::string &test_name); + + /** Tell the framework that a test case is skipped. + * + * @param[in] test_name Name of the skipped test case. + */ + void log_test_skipped(const std::string &test_name); + + /** Tell the framework that a test case finished. + * + * @param[in] test_name Name of the finished test case. + */ + void log_test_end(const std::string &test_name); + + /** Tell the framework that the currently running test case failed a non-fatal expectation. + * + * @param[in] msg Description of the failure. + */ + void log_failed_expectation(const std::string &msg); + + /** Number of iterations per test case. + * + * @return Number of iterations per test case. + */ + int num_iterations() const; + + /** Set number of iterations per test case. + * + * @param[in] num_iterations Number of iterations per test case. + */ + void set_num_iterations(int num_iterations); + + /** Should errors be caught or thrown by the framework. + * + * @return True if errors are thrown. + */ + bool throw_errors() const; + + /** Set whether errors are caught or thrown by the framework. + * + * @param[in] throw_errors True if errors should be thrown. + */ + void set_throw_errors(bool throw_errors); + + /** Check if a test case would be executed. + * + * @param[in] id Id of the test case. + * + * @return True if the test case would be executed. + */ + bool is_enabled(const TestId &id) const; + + /** Run all enabled test cases. + * + * @return True if all test cases executed successful. + */ + bool run(); + + /** Set the result for an executed test case. + * + * @param[in] test_case_name Name of the executed test case. + * @param[in] result Execution result. + */ + void set_test_result(std::string test_case_name, TestResult result); + + /** List of @ref TestId's. + * + * @return Vector with all test ids. + */ + std::vector<Framework::TestId> test_ids() const; + +private: + Framework() = default; + ~Framework() = default; + + Framework(const Framework &) = delete; + Framework &operator=(const Framework &) = delete; + + void run_test(TestCaseFactory &test_factory); + std::tuple<int, int, int> count_test_results() const; + + /** Returns the current test suite name. + * + * @warning Cannot be used at execution time to get the test suite of the + * currently executed test case. It can only be used for registering test + * cases. + * + * @return Name of the current test suite. + */ + std::string current_suite_name() const; + + std::vector<std::string> _test_suite_name{}; + std::vector<std::unique_ptr<TestCaseFactory>> _test_factories{}; + std::map<std::string, TestResult> _test_results{}; + std::chrono::seconds _runtime{ 0 }; + int _num_iterations{ 1 }; + bool _throw_errors{ false }; + + std::regex _test_name_filter{ ".*" }; + std::regex _test_id_filter{ ".*" }; +}; + +template <typename T> +inline void Framework::add_test_case(std::string test_name) +{ + _test_factories.emplace_back(support::cpp14::make_unique<SimpleTestCaseFactory<T>>(current_suite_name(), std::move(test_name))); +} + +template <typename T, typename D> +inline void Framework::add_data_test_case(std::string test_name, std::string description, D &&data) +{ + // WORKAROUND for GCC 4.9 + // The function should get *it which is tuple but that seems to trigger a + // bug in the compiler. + auto tmp = std::unique_ptr<DataTestCaseFactory<T, decltype(*std::declval<D>())>>(new DataTestCaseFactory<T, decltype(*std::declval<D>())>(current_suite_name(), std::move(test_name), + std::move(description), *data)); + _test_factories.emplace_back(std::move(tmp)); +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_FRAMEWORK */ diff --git a/framework/Macros.h b/framework/Macros.h new file mode 100644 index 0000000000..e3ef71ac56 --- /dev/null +++ b/framework/Macros.h @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_FRAMEWORK_MACROS +#define ARM_COMPUTE_TEST_FRAMEWORK_MACROS + +#include "Exceptions.h" +#include "Framework.h" +#include "Registrars.h" +#include "TestCase.h" + +#include <sstream> + +// +// TEST SUITE MACROS +// +#define TEST_SUITE(SUITE_NAME) \ + namespace SUITE_NAME##Suite \ + { \ + static arm_compute::test::framework::detail::TestSuiteRegistrar SUITE_NAME##Suite_reg{ #SUITE_NAME }; + +#define TEST_SUITE_END() \ + static arm_compute::test::framework::detail::TestSuiteRegistrar Suite_end; \ + } +// +// TEST SUITE MACROS END +// + +// +// TEST CASE MACROS +// +#define TEST_CASE_CONSTRUCTOR(TEST_NAME) \ + TEST_NAME() = default; +#define DATA_TEST_CASE_CONSTRUCTOR(TEST_NAME, DATASET) \ + template <typename D> \ + explicit TEST_NAME(D &&data) : DataTestCase{ std::forward<D>(data) } \ + { \ + } +#define FIXTURE_SETUP(FIXTURE) \ + void do_setup() override \ + { \ + FIXTURE::setup(); \ + } +#define FIXTURE_DATA_SETUP(FIXTURE) \ + void do_setup() override \ + { \ + apply(this, &FIXTURE::setup, _data); \ + } +#define FIXTURE_RUN(FIXTURE) \ + void do_run() override \ + { \ + FIXTURE::run(); \ + } +#define FIXTURE_TEARDOWN(FIXTURE) \ + void do_teardown() override \ + { \ + FIXTURE::teardown(); \ + } +#define TEST_REGISTRAR(TEST_NAME) \ + static arm_compute::test::framework::detail::TestCaseRegistrar<TEST_NAME> TEST_NAME##_reg \ + { \ + #TEST_NAME \ + } +#define DATA_TEST_REGISTRAR(TEST_NAME, DATASET) \ + static arm_compute::test::framework::detail::TestCaseRegistrar<TEST_NAME> TEST_NAME##_reg \ + { \ + #TEST_NAME, DATASET \ + } + +#define TEST_CASE(TEST_NAME) \ + class TEST_NAME : public arm_compute::test::framework::TestCase \ + { \ + public: \ + TEST_CASE_CONSTRUCTOR(TEST_NAME) \ + void do_run() override; \ + }; \ + TEST_REGISTRAR(TEST_NAME); \ + void TEST_NAME::do_run() + +#define DATA_TEST_CASE(TEST_NAME, DATASET, ...) \ + class TEST_NAME : public arm_compute::test::framework::DataTestCase<decltype(DATASET)::type> \ + { \ + public: \ + DATA_TEST_CASE_CONSTRUCTOR(TEST_NAME, DATASET) \ + void do_run() override \ + { \ + arm_compute::test::framework::apply(this, &TEST_NAME::run, _data); \ + } \ + void run(__VA_ARGS__); \ + }; \ + DATA_TEST_REGISTRAR(TEST_NAME, DATASET); \ + void TEST_NAME::run(__VA_ARGS__) + +#define FIXTURE_TEST_CASE(TEST_NAME, FIXTURE) \ + class TEST_NAME : public arm_compute::test::framework::TestCase, public FIXTURE \ + { \ + public: \ + TEST_CASE_CONSTRUCTOR(TEST_NAME) \ + FIXTURE_SETUP(FIXTURE) \ + void do_run() override; \ + FIXTURE_TEARDOWN(FIXTURE) \ + }; \ + TEST_REGISTRAR(TEST_NAME); \ + void TEST_NAME::do_run() + +#define FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, DATASET) \ + class TEST_NAME : public arm_compute::test::framework::DataTestCase<decltype(DATASET)::type>, public FIXTURE \ + { \ + public: \ + DATA_TEST_CASE_CONSTRUCTOR(TEST_NAME, DATASET) \ + FIXTURE_DATA_SETUP(FIXTURE) \ + void do_run() override; \ + FIXTURE_TEARDOWN(FIXTURE) \ + }; \ + DATA_TEST_REGISTRAR(TEST_NAME, DATASET); \ + void TEST_NAME::do_run() + +#define REGISTER_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE) \ + class TEST_NAME : public arm_compute::test::framework::TestCase, public FIXTURE \ + { \ + public: \ + TEST_CASE_CONSTRUCTOR(TEST_NAME) \ + FIXTURE_SETUP(FIXTURE) \ + FIXTURE_RUN(FIXTURE) \ + FIXTURE_TEARDOWN(FIXTURE) \ + }; \ + TEST_REGISTRAR(TEST_NAME) + +#define REGISTER_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, DATASET) \ + class TEST_NAME : public arm_compute::test::framework::DataTestCase<decltype(DATASET)::type>, public FIXTURE \ + { \ + public: \ + DATA_TEST_CASE_CONSTRUCTOR(TEST_NAME, DATASET) \ + FIXTURE_DATA_SETUP(FIXTURE) \ + FIXTURE_RUN(FIXTURE) \ + FIXTURE_TEARDOWN(FIXTURE) \ + }; \ + DATA_TEST_REGISTRAR(TEST_NAME, DATASET) +// +// TEST CASE MACROS END +// + +#define ARM_COMPUTE_ASSERT_EQUAL(x, y) \ + do \ + { \ + const auto &_x = (x); \ + const auto &_y = (y); \ + if(_x != _y) \ + { \ + std::stringstream msg; \ + msg << "Assertion " << _x << " != " << _y << " failed."; \ + throw arm_compute::test::framework::TestError(msg.str()); \ + } \ + } while(false) + +#define ARM_COMPUTE_EXPECT_EQUAL(x, y) \ + do \ + { \ + const auto &_x = (x); \ + const auto &_y = (y); \ + if(_x != _y) \ + { \ + std::stringstream msg; \ + msg << "Expectation " << _x << " != " << _y << " failed."; \ + arm_compute::test::framework::Framework::get().log_failed_expectation(msg.str()); \ + } \ + } while(false) +#endif /* ARM_COMPUTE_TEST_FRAMEWORK_MACROS */ diff --git a/framework/Registrars.h b/framework/Registrars.h new file mode 100644 index 0000000000..19064c07f5 --- /dev/null +++ b/framework/Registrars.h @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_FRAMEWORK_REGISTRARS +#define ARM_COMPUTE_TEST_FRAMEWORK_REGISTRARS + +#include "Framework.h" + +#include <string> +#include <utility> + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +namespace detail +{ +/** Helper class to statically register a test case. */ +template <typename T> +class TestCaseRegistrar final +{ +public: + /** Add a new test case with the given name to the framework. + * + * @param[in] test_name Name of the test case. + */ + TestCaseRegistrar(std::string test_name); + + /** Add a new data test case with the given name to the framework. + * + * @param[in] test_name Name of the test case. + * @param[in] dataset Dataset used as input for the test case. + */ + template <typename D> + TestCaseRegistrar(std::string test_name, D &&dataset); +}; + +/** Helper class to statically begin and end a test suite. */ +class TestSuiteRegistrar final +{ +public: + /** Remove the last added test suite from the framework. */ + TestSuiteRegistrar(); + + /** Add a new test suite with the given name to the framework. + * + * @param[in] name Name of the test suite. + */ + TestSuiteRegistrar(std::string name); +}; + +template <typename T> +inline TestCaseRegistrar<T>::TestCaseRegistrar(std::string test_name) +{ + Framework::get().add_test_case<T>(std::move(test_name)); +} + +template <typename T> +template <typename D> +inline TestCaseRegistrar<T>::TestCaseRegistrar(std::string test_name, D &&dataset) +{ + auto it = dataset.begin(); + + for(int i = 0; i < dataset.size(); ++i, ++it) + { + // WORKAROUND for GCC 4.9 + // The last argument should be *it to pass just the data and not the + // iterator. + Framework::get().add_data_test_case<T>(test_name, it.description(), it); + } +} + +inline TestSuiteRegistrar::TestSuiteRegistrar() +{ + Framework::get().pop_suite(); +} + +inline TestSuiteRegistrar::TestSuiteRegistrar(std::string name) +{ + Framework::get().push_suite(std::move(name)); +} +} // namespace detail +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_FRAMEWORK_REGISTRARS */ diff --git a/framework/SConscript b/framework/SConscript new file mode 100644 index 0000000000..bf98241ca9 --- /dev/null +++ b/framework/SConscript @@ -0,0 +1,68 @@ +# Copyright (c) 2017 ARM Limited. +# +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +import SCons +import os.path + +Import('env') +Import('vars') + +# vars is imported from arm_compute: +variables = [ + BoolVariable("pmu", "Enable PMU counters", False) +] + +# We need a separate set of Variables for the Help message (Otherwise the global variables will get displayed twice) +new_options = Variables('scons') + +for v in variables: + new_options.Add(v) + vars.Add(v) + +# Clone the environment to make sure we're not polluting the arm_compute one: +framework_env = env.Clone() +vars.Update(framework_env) + +Help(new_options.GenerateHelpText(framework_env)) + +if env['os'] == 'android' and framework_env['pmu']: + print("pmu=1 is not supported for os=android") + Exit(1) + +framework_env.Append(CPPPATH = ["."]) +framework_env.Append(CPPFLAGS=['-Wno-overloaded-virtual']) + +files = Glob('*.cpp') +files += Glob('command_line/*.cpp') +files += Glob('printers/*.cpp') +files += Glob('datasets/*.cpp') +files += Glob('instruments/*.cpp') + +if not framework_env['pmu']: + # Remove PMU files + files = [f for f in files if "PMU" not in os.path.basename(str(f))] +else: + framework_env.Append(CPPDEFINES = ['PMU_ENABLED']) + +arm_compute_test_framework = framework_env.StaticLibrary('arm_compute_test_framework', files) + +Default(arm_compute_test_framework) +Export('arm_compute_test_framework') diff --git a/framework/TestCase.h b/framework/TestCase.h new file mode 100644 index 0000000000..43750b1d1b --- /dev/null +++ b/framework/TestCase.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_TESTCASE +#define ARM_COMPUTE_TEST_TESTCASE + +#include <string> +#include <utility> + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Abstract test case class. + * + * All test cases have to inherit from this class. + */ +class TestCase +{ +public: + virtual void do_setup() {}; + virtual void do_run() {}; + virtual void do_teardown() {}; + + /** Default destructor. */ + virtual ~TestCase() = default; + +protected: + TestCase() = default; + + friend class TestCaseFactory; +}; + +template <typename T> +class DataTestCase : public TestCase +{ +protected: + explicit DataTestCase(T data) + : _data{ data } + { + } + + T _data; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_TESTCASE */ diff --git a/framework/TestCaseFactory.h b/framework/TestCaseFactory.h new file mode 100644 index 0000000000..09e9d198d6 --- /dev/null +++ b/framework/TestCaseFactory.h @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_TEST_CASE_FACTORY +#define ARM_COMPUTE_TEST_TEST_CASE_FACTORY + +#include "TestCase.h" +#include "support/ToolchainSupport.h" + +#include <memory> +#include <string> + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Abstract factory class to create test cases. */ +class TestCaseFactory +{ +public: + /** Constructor. + * + * @param[in] suite_name Name of the test suite to which the test case has been added. + * @param[in] name Name of the test case. + * @param[in] description Description of data arguments. + */ + TestCaseFactory(std::string suite_name, std::string name, std::string description = ""); + + /** Default destructor. */ + virtual ~TestCaseFactory() = default; + + /** Name of the test case. + * + * @return Name of the test case. + */ + std::string name() const; + + /** Factory function to create the test case + * + * @return Unique pointer to a newly created test case. + */ + virtual std::unique_ptr<TestCase> make() const = 0; + +private: + const std::string _suite_name; + const std::string _test_name; + const std::string _data_description; +}; + +/** Implementation of a test case factory to create non-data test cases. */ +template <typename T> +class SimpleTestCaseFactory final : public TestCaseFactory +{ +public: + /** Default constructor. */ + using TestCaseFactory::TestCaseFactory; + + std::unique_ptr<TestCase> make() const override; +}; + +template <typename T, typename D> +class DataTestCaseFactory final : public TestCaseFactory +{ +public: + /** Constructor. + * + * @param[in] suite_name Name of the test suite to which the test case has been added. + * @param[in] test_name Name of the test case. + * @param[in] description Description of data arguments. + * @param[in] data Input data for the test case. + */ + DataTestCaseFactory(std::string suite_name, std::string test_name, std::string description, const D &data); + + std::unique_ptr<TestCase> make() const override; + +private: + D _data; +}; + +inline TestCaseFactory::TestCaseFactory(std::string suite_name, std::string test_name, std::string description) + : _suite_name{ std::move(suite_name) }, _test_name{ std::move(test_name) }, _data_description{ std::move(description) } +{ +} + +inline std::string TestCaseFactory::name() const +{ + std::string name = _suite_name + "/" + _test_name; + + if(!_data_description.empty()) + { + name += "@" + _data_description; + } + + return name; +} + +template <typename T> +inline std::unique_ptr<TestCase> SimpleTestCaseFactory<T>::make() const +{ + return support::cpp14::make_unique<T>(); +} + +template <typename T, typename D> +inline DataTestCaseFactory<T, D>::DataTestCaseFactory(std::string suite_name, std::string test_name, std::string description, const D &data) + : TestCaseFactory{ std::move(suite_name), std::move(test_name), std::move(description) }, _data{ data } +{ +} + +template <typename T, typename D> +inline std::unique_ptr<TestCase> DataTestCaseFactory<T, D>::make() const +{ + return support::cpp14::make_unique<T>(_data); +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_TEST_CASE_FACTORY */ diff --git a/framework/TestResult.h b/framework/TestResult.h new file mode 100644 index 0000000000..c860cbc22c --- /dev/null +++ b/framework/TestResult.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_TESTRESULT +#define ARM_COMPUTE_TEST_TESTRESULT + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Class to store results of a test. + * + * Currently the execution status and profiling information are stored. + */ +struct TestResult +{ + /** Execution status of a test. */ + enum class Status + { + NOT_RUN, + SUCCESS, + EXPECTED_FAILURE, + FAILED, + CRASHED + }; + + /** Default constructor. */ + TestResult() = default; + + /** Initialise the result with a status. + * + * @param[in] status Execution status. + */ + TestResult(Status status) + : status{ status } + { + } + + Status status{ Status::NOT_RUN }; //< Execution status +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_TESTRESULT */ diff --git a/framework/Utils.h b/framework/Utils.h new file mode 100644 index 0000000000..e9298618e8 --- /dev/null +++ b/framework/Utils.h @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_UTILS +#define ARM_COMPUTE_TEST_UTILS + +#include "support/ToolchainSupport.h" + +#include <algorithm> +#include <cmath> +#include <cstddef> +#include <limits> +#include <memory> +#include <numeric> +#include <sstream> +#include <string> +#include <type_traits> + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** @cond */ +namespace detail +{ +template <int...> +struct sequence +{ +}; + +template <int N, int... Ns> +struct sequence_generator; + +template <int... Ns> +struct sequence_generator<0, Ns...> +{ + using type = sequence<Ns...>; +}; + +template <int N, int... Ns> +struct sequence_generator : sequence_generator < N - 1, N - 1, Ns... > +{ +}; + +template <int N> +using sequence_t = typename sequence_generator<N>::type; +/** @endcond */ + +template <typename O, typename F, typename... As, int... S> +void apply_impl(O *obj, F &&func, const std::tuple<As...> &args, detail::sequence<S...>) +{ + (obj->*func)(std::get<S>(args)...); +} +} // namespace + +template <typename O, typename F, typename... As> +void apply(O *obj, F &&func, const std::tuple<As...> &args) +{ + detail::apply_impl(obj, std::forward<F>(func), args, detail::sequence_t<sizeof...(As)>()); +} + +/** Helper function to concatenate multiple strings. + * + * @param[in] first Iterator pointing to the first element to be concatenated. + * @param[in] last Iterator pointing behind the last element to be concatenated. + * @param[in] separator String used to join the elements. + * + * @return String containing all elements joined by @p separator. + */ +template <typename T, typename std::enable_if<std::is_same<typename T::value_type, std::string>::value, int>::type = 0> +std::string join(T first, T last, const std::string &separator) +{ + return std::accumulate(std::next(first), last, *first, [&separator](const std::string & base, const std::string & suffix) + { + return base + separator + suffix; + }); +} + +/** Helper function to concatenate multiple values. + * + * All values are converted to std::string using the provided operation before + * being joined. + * + * The signature of op has to be equivalent to + * std::string op(const T::value_type &val). + * + * @param[in] first Iterator pointing to the first element to be concatenated. + * @param[in] last Iterator pointing behind the last element to be concatenated. + * @param[in] separator String used to join the elements. + * @param[in] op Conversion function. + * + * @return String containing all elements joined by @p separator. + */ +template <typename T, typename UnaryOp> +std::string join(T &&first, T &&last, const std::string &separator, UnaryOp &&op) +{ + return std::accumulate(std::next(first), last, op(*first), [&separator, &op](const std::string & base, const typename T::value_type & suffix) + { + return base + separator + op(suffix); + }); +} + +/** Helper function to concatenate multiple values. + * + * All values are converted to std::string using std::to_string before being joined. + * + * @param[in] first Iterator pointing to the first element to be concatenated. + * @param[in] last Iterator pointing behind the last element to be concatenated. + * @param[in] separator String used to join the elements. + * + * @return String containing all elements joined by @p separator. + */ +template <typename T, typename std::enable_if<std::is_arithmetic<typename T::value_type>::value, int>::type = 0> +std::string join(T && first, T && last, const std::string &separator) +{ + return join(std::forward<T>(first), std::forward<T>(last), separator, support::cpp11::to_string); +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_UTILS */ |