From a09de0c8b2ed0f1481502d3b023375609362d9e3 Mon Sep 17 00:00:00 2001 From: Moritz Pflanzer Date: Fri, 1 Sep 2017 20:41:12 +0100 Subject: COMPMID-415: Rename and move tests The boost validation is now "standalone" in validation_old and builds as arm_compute_validation_old. The new validation builds now as arm_compute_validation. Change-Id: Ib93ba848a25680ac60afb92b461d574a0757150d Reviewed-on: http://mpd-gerrit.cambridge.arm.com/86187 Tested-by: Kaizen Reviewed-by: Anthony Barbier --- tests/framework/Asserts.h | 132 ++++++ tests/framework/DatasetModes.cpp | 56 +++ tests/framework/DatasetModes.h | 106 +++++ tests/framework/Exceptions.cpp | 128 ++++++ tests/framework/Exceptions.h | 97 ++++ tests/framework/Fixture.h | 63 +++ tests/framework/Framework.cpp | 494 +++++++++++++++++++++ tests/framework/Framework.h | 330 ++++++++++++++ tests/framework/Macros.h | 260 +++++++++++ tests/framework/Profiler.cpp | 67 +++ tests/framework/Profiler.h | 75 ++++ tests/framework/Registrars.h | 113 +++++ tests/framework/SConscript | 71 +++ tests/framework/TestCase.h | 70 +++ tests/framework/TestCaseFactory.h | 202 +++++++++ tests/framework/TestFilter.cpp | 143 ++++++ tests/framework/TestFilter.h | 84 ++++ tests/framework/TestResult.h | 80 ++++ tests/framework/Utils.h | 158 +++++++ tests/framework/command_line/CommandLineOptions.h | 33 ++ tests/framework/command_line/CommandLineParser.cpp | 149 +++++++ tests/framework/command_line/CommandLineParser.h | 117 +++++ tests/framework/command_line/EnumListOption.h | 148 ++++++ tests/framework/command_line/EnumOption.h | 134 ++++++ tests/framework/command_line/ListOption.h | 116 +++++ tests/framework/command_line/Option.cpp | 68 +++ tests/framework/command_line/Option.h | 109 +++++ tests/framework/command_line/SimpleOption.h | 105 +++++ tests/framework/command_line/ToggleOption.cpp | 64 +++ tests/framework/command_line/ToggleOption.h | 56 +++ tests/framework/datasets/CartesianProductDataset.h | 165 +++++++ tests/framework/datasets/ContainerDataset.h | 148 ++++++ tests/framework/datasets/Dataset.h | 86 ++++ tests/framework/datasets/Datasets.h | 35 ++ tests/framework/datasets/InitializerListDataset.h | 135 ++++++ tests/framework/datasets/JoinDataset.h | 148 ++++++ tests/framework/datasets/RangeDataset.h | 141 ++++++ tests/framework/datasets/SingletonDataset.h | 138 ++++++ tests/framework/datasets/ZipDataset.h | 139 ++++++ tests/framework/instruments/Instrument.h | 98 ++++ tests/framework/instruments/Instruments.cpp | 59 +++ tests/framework/instruments/Instruments.h | 105 +++++ tests/framework/instruments/PMUCounter.cpp | 140 ++++++ tests/framework/instruments/PMUCounter.h | 71 +++ tests/framework/instruments/WallClockTimer.cpp | 57 +++ tests/framework/instruments/WallClockTimer.h | 53 +++ tests/framework/printers/JSONPrinter.cpp | 168 +++++++ tests/framework/printers/JSONPrinter.h | 64 +++ tests/framework/printers/PrettyPrinter.cpp | 140 ++++++ tests/framework/printers/PrettyPrinter.h | 69 +++ tests/framework/printers/Printer.cpp | 48 ++ tests/framework/printers/Printer.h | 129 ++++++ tests/framework/printers/Printers.cpp | 57 +++ tests/framework/printers/Printers.h | 75 ++++ 54 files changed, 6296 insertions(+) create mode 100644 tests/framework/Asserts.h create mode 100644 tests/framework/DatasetModes.cpp create mode 100644 tests/framework/DatasetModes.h create mode 100644 tests/framework/Exceptions.cpp create mode 100644 tests/framework/Exceptions.h create mode 100644 tests/framework/Fixture.h create mode 100644 tests/framework/Framework.cpp create mode 100644 tests/framework/Framework.h create mode 100644 tests/framework/Macros.h create mode 100644 tests/framework/Profiler.cpp create mode 100644 tests/framework/Profiler.h create mode 100644 tests/framework/Registrars.h create mode 100644 tests/framework/SConscript create mode 100644 tests/framework/TestCase.h create mode 100644 tests/framework/TestCaseFactory.h create mode 100644 tests/framework/TestFilter.cpp create mode 100644 tests/framework/TestFilter.h create mode 100644 tests/framework/TestResult.h create mode 100644 tests/framework/Utils.h create mode 100644 tests/framework/command_line/CommandLineOptions.h create mode 100644 tests/framework/command_line/CommandLineParser.cpp create mode 100644 tests/framework/command_line/CommandLineParser.h create mode 100644 tests/framework/command_line/EnumListOption.h create mode 100644 tests/framework/command_line/EnumOption.h create mode 100644 tests/framework/command_line/ListOption.h create mode 100644 tests/framework/command_line/Option.cpp create mode 100644 tests/framework/command_line/Option.h create mode 100644 tests/framework/command_line/SimpleOption.h create mode 100644 tests/framework/command_line/ToggleOption.cpp create mode 100644 tests/framework/command_line/ToggleOption.h create mode 100644 tests/framework/datasets/CartesianProductDataset.h create mode 100644 tests/framework/datasets/ContainerDataset.h create mode 100644 tests/framework/datasets/Dataset.h create mode 100644 tests/framework/datasets/Datasets.h create mode 100644 tests/framework/datasets/InitializerListDataset.h create mode 100644 tests/framework/datasets/JoinDataset.h create mode 100644 tests/framework/datasets/RangeDataset.h create mode 100644 tests/framework/datasets/SingletonDataset.h create mode 100644 tests/framework/datasets/ZipDataset.h create mode 100644 tests/framework/instruments/Instrument.h create mode 100644 tests/framework/instruments/Instruments.cpp create mode 100644 tests/framework/instruments/Instruments.h create mode 100644 tests/framework/instruments/PMUCounter.cpp create mode 100644 tests/framework/instruments/PMUCounter.h create mode 100644 tests/framework/instruments/WallClockTimer.cpp create mode 100644 tests/framework/instruments/WallClockTimer.h create mode 100644 tests/framework/printers/JSONPrinter.cpp create mode 100644 tests/framework/printers/JSONPrinter.h create mode 100644 tests/framework/printers/PrettyPrinter.cpp create mode 100644 tests/framework/printers/PrettyPrinter.h create mode 100644 tests/framework/printers/Printer.cpp create mode 100644 tests/framework/printers/Printer.h create mode 100644 tests/framework/printers/Printers.cpp create mode 100644 tests/framework/printers/Printers.h (limited to 'tests/framework') diff --git a/tests/framework/Asserts.h b/tests/framework/Asserts.h new file mode 100644 index 0000000000..b545a9ebba --- /dev/null +++ b/tests/framework/Asserts.h @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_FRAMEWORK_ASSERTS +#define ARM_COMPUTE_TEST_FRAMEWORK_ASSERTS + +#include "Exceptions.h" +#include "Framework.h" + +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +// Cast char values to int so that their numeric value are printed. +inline int make_printable(int8_t value) +{ + return value; +} + +inline unsigned int make_printable(uint8_t value) +{ + return value; +} + +// Everything else can be printed as its own type. +template +inline T make_printable(T &&value) +{ + return value; +} + +#define ARM_COMPUTE_TEST_INFO(INFO) \ + { \ + std::stringstream info; \ + info << INFO; \ + arm_compute::test::framework::Framework::get().add_test_info(info.str()); \ + } + +namespace detail +{ +#define ARM_COMPUTE_TEST_COMP_FACTORY(SEVERITY, SEVERITY_NAME, COMP, COMP_NAME, ERROR_CALL) \ + template \ + void ARM_COMPUTE_##SEVERITY##_##COMP_NAME##_IMPL(T &&x, U &&y, const std::string &x_str, const std::string &y_str, LogLevel level) \ + { \ + if(!(x COMP y)) \ + { \ + std::stringstream msg; \ + msg << #SEVERITY_NAME " '" << x_str << " " #COMP " " << y_str << "' failed. [" \ + << std::boolalpha << arm_compute::test::framework::make_printable(x) \ + << " " #COMP " " \ + << std::boolalpha << arm_compute::test::framework::make_printable(y) \ + << "]\n"; \ + arm_compute::test::framework::Framework::get().print_test_info(msg); \ + ERROR_CALL \ + } \ + arm_compute::test::framework::Framework::get().clear_test_info(); \ + } + +ARM_COMPUTE_TEST_COMP_FACTORY(EXPECT, Expectation, ==, EQUAL, arm_compute::test::framework::Framework::get().log_failed_expectation(arm_compute::test::framework::TestError(msg.str(), level));) +ARM_COMPUTE_TEST_COMP_FACTORY(EXPECT, Expectation, !=, NOT_EQUAL, arm_compute::test::framework::Framework::get().log_failed_expectation(arm_compute::test::framework::TestError(msg.str(), level));) +ARM_COMPUTE_TEST_COMP_FACTORY(ASSERT, Assertion, ==, EQUAL, throw arm_compute::test::framework::TestError(msg.str(), level);) +ARM_COMPUTE_TEST_COMP_FACTORY(ASSERT, Assertion, !=, NOT_EQUAL, throw arm_compute::test::framework::TestError(msg.str(), level);) +} // namespace detail + +#define ARM_COMPUTE_ASSERT_NOT_EQUAL(X, Y) \ + arm_compute::test::framework::detail::ARM_COMPUTE_ASSERT_NOT_EQUAL_IMPL(X, Y, #X, #Y, LogLevel::ERRORS) + +#define ARM_COMPUTE_ASSERT_EQUAL(X, Y) \ + arm_compute::test::framework::detail::ARM_COMPUTE_ASSERT_EQUAL_IMPL(X, Y, #X, #Y, LogLevel::ERRORS) + +#define ARM_COMPUTE_EXPECT_EQUAL(X, Y, LEVEL) \ + arm_compute::test::framework::detail::ARM_COMPUTE_EXPECT_EQUAL_IMPL(X, Y, #X, #Y, LEVEL) + +#define ARM_COMPUTE_EXPECT_NOT_EQUAL(X, Y, LEVEL) \ + arm_compute::test::framework::detail::ARM_COMPUTE_EXPECT_NOT_EQUAL_IMPL(X, Y, #X, #Y, LEVEL) + +#define ARM_COMPUTE_ASSERT(X) \ + do \ + { \ + const auto &x = X; \ + if(!x) \ + { \ + std::stringstream msg; \ + msg << "Assertion '" #X "' failed.\n"; \ + arm_compute::test::framework::Framework::get().print_test_info(msg); \ + throw arm_compute::test::framework::TestError(msg.str(), arm_compute::test::framework::LogLevel::ERRORS); \ + } \ + arm_compute::test::framework::Framework::get().clear_test_info(); \ + } while(false) + +#define ARM_COMPUTE_EXPECT(X, LEVEL) \ + do \ + { \ + const auto &x = X; \ + if(!x) \ + { \ + std::stringstream msg; \ + msg << "Expectation '" #X "' failed.\n"; \ + arm_compute::test::framework::Framework::get().print_test_info(msg); \ + arm_compute::test::framework::Framework::get().log_failed_expectation(arm_compute::test::framework::TestError(msg.str(), LEVEL)); \ + } \ + arm_compute::test::framework::Framework::get().clear_test_info(); \ + } while(false) +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_FRAMEWORK_ASSERTS */ diff --git a/tests/framework/DatasetModes.cpp b/tests/framework/DatasetModes.cpp new file mode 100644 index 0000000000..3f747df747 --- /dev/null +++ b/tests/framework/DatasetModes.cpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "DatasetModes.h" + +#include "Utils.h" + +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +DatasetMode dataset_mode_from_name(const std::string &name) +{ + static const std::map modes = + { + { "all", DatasetMode::ALL }, + { "precommit", DatasetMode::PRECOMMIT }, + { "nightly", DatasetMode::NIGHTLY }, + }; + + try + { + return modes.at(tolower(name)); + } + catch(const std::out_of_range &) + { + throw std::invalid_argument(name); + } +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/DatasetModes.h b/tests/framework/DatasetModes.h new file mode 100644 index 0000000000..27638b0504 --- /dev/null +++ b/tests/framework/DatasetModes.h @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_DATASET_MODES +#define ARM_COMPUTE_TEST_DATASET_MODES + +#include +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Possible dataset modes. */ +enum class DatasetMode : unsigned int +{ + ALL = ~0U, + DISABLED = 0, + PRECOMMIT = 1, + NIGHTLY = 2 +}; + +inline DatasetMode operator&(DatasetMode t1, DatasetMode t2) +{ + using type = std::underlying_type::type; + return static_cast(static_cast(t1) & static_cast(t2)); +} + +inline DatasetMode operator|(DatasetMode t1, DatasetMode t2) +{ + using type = std::underlying_type::type; + return static_cast(static_cast(t1) | static_cast(t2)); +} + +inline DatasetMode &operator|=(DatasetMode &t1, DatasetMode t2) +{ + using type = std::underlying_type::type; + t1 = static_cast(static_cast(t1) | static_cast(t2)); + return t1; +} + +DatasetMode dataset_mode_from_name(const std::string &name); + +inline ::std::istream &operator>>(::std::istream &stream, DatasetMode &mode) +{ + std::string value; + stream >> value; + mode = dataset_mode_from_name(value); + return stream; +} + +inline ::std::ostream &operator<<(::std::ostream &stream, DatasetMode mode) +{ + switch(mode) + { + case DatasetMode::PRECOMMIT: + stream << "PRECOMMIT"; + break; + case DatasetMode::NIGHTLY: + stream << "NIGHTLY"; + break; + case DatasetMode::ALL: + stream << "ALL"; + break; + default: + throw std::invalid_argument("Unsupported dataset mode"); + } + + return stream; +} + +inline std::string to_string(DatasetMode mode) +{ + std::stringstream stream; + stream << mode; + return stream.str(); +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_DATASET_MODES */ diff --git a/tests/framework/Exceptions.cpp b/tests/framework/Exceptions.cpp new file mode 100644 index 0000000000..3d6c65c181 --- /dev/null +++ b/tests/framework/Exceptions.cpp @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "Exceptions.h" + +#include "Utils.h" + +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +LogLevel log_level_from_name(const std::string &name) +{ + static const std::map levels = + { + { "none", LogLevel::NONE }, + { "config", LogLevel::CONFIG }, + { "tests", LogLevel::TESTS }, + { "errors", LogLevel::ERRORS }, + { "debug", LogLevel::DEBUG }, + { "measurements", LogLevel::MEASUREMENTS }, + { "all", LogLevel::ALL }, + }; + + try + { + return levels.at(tolower(name)); + } + catch(const std::out_of_range &) + { + throw std::invalid_argument(name); + } +} + +::std::istream &operator>>(::std::istream &stream, LogLevel &level) +{ + std::string value; + stream >> value; + level = log_level_from_name(value); + return stream; +} + +::std::ostream &operator<<(::std::ostream &stream, LogLevel level) +{ + switch(level) + { + case LogLevel::NONE: + stream << "NONE"; + break; + case LogLevel::CONFIG: + stream << "CONFIG"; + break; + case LogLevel::TESTS: + stream << "TESTS"; + break; + case LogLevel::ERRORS: + stream << "ERRORS"; + break; + case LogLevel::DEBUG: + stream << "DEBUG"; + break; + case LogLevel::MEASUREMENTS: + stream << "MEASUREMENTS"; + break; + case LogLevel::ALL: + stream << "ALL"; + break; + default: + throw std::invalid_argument("Unsupported log level"); + } + + return stream; +} + +std::string to_string(LogLevel level) +{ + std::stringstream stream; + stream << level; + return stream.str(); +} + +TestError::TestError(const std::string &msg, LogLevel level, std::string context) + : std::runtime_error{ msg }, _level{ level }, _msg{ msg }, _context{ std::move(context) }, _combined{ "ERROR: " + msg } +{ + if(!_context.empty()) + { + _combined += "\nCONTEXT:\n" + _context; + } +} + +LogLevel TestError::level() const +{ + return _level; +} + +const char *TestError::what() const noexcept +{ + return _combined.c_str(); +} + +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/Exceptions.h b/tests/framework/Exceptions.h new file mode 100644 index 0000000000..edb0ed92c9 --- /dev/null +++ b/tests/framework/Exceptions.h @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_EXCEPTIONS +#define ARM_COMPUTE_TEST_EXCEPTIONS + +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Severity of the information. + * + * Each category includes the ones above it. + * + * NONE == Only for filtering. Not used to tag information. + * CONFIG == Configuration info. + * TESTS == Information about the tests. + * ERRORS == Violated assertions/expectations. + * DEBUG == More violated assertions/expectations. + * MEASUREMENTS == Information about measurements. + * ALL == Only for filtering. Not used to tag information. + */ +enum class LogLevel +{ + NONE, + CONFIG, + TESTS, + ERRORS, + DEBUG, + MEASUREMENTS, + ALL, +}; + +LogLevel log_level_from_name(const std::string &name); +::std::istream &operator>>(::std::istream &stream, LogLevel &level); +::std::ostream &operator<<(::std::ostream &stream, LogLevel level); +std::string to_string(LogLevel level); + +/** Error class for failures during test execution. */ +class TestError : public std::runtime_error +{ +public: + using std::runtime_error::runtime_error; + + /** Construct error with severity. + * + * @param[in] msg Error message. + * @param[in] level Severity level. + * @param[in] context Context. + */ + TestError(const std::string &msg, LogLevel level, std::string context = ""); + + /** Severity of the error. + * + * @return Severity. + */ + LogLevel level() const; + + const char *what() const noexcept override; + +private: + LogLevel _level{ LogLevel::ERRORS }; + std::string _msg{}; + std::string _context{}; + std::string _combined{}; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_EXCEPTIONS */ diff --git a/tests/framework/Fixture.h b/tests/framework/Fixture.h new file mode 100644 index 0000000000..916dcc7fef --- /dev/null +++ b/tests/framework/Fixture.h @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_FIXTURE +#define ARM_COMPUTE_TEST_FIXTURE + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Abstract fixture class. + * + * All custom fixtures have to inherit from this class. + */ +class Fixture +{ +public: + /** Setup function. + * + * This function is only invoked by non-data fixture test cases. Fixture + * data test cases implement a setup function with arguments matching the + * dataset. + * + * The function is called before the test case is executed. + */ + void setup() {}; + + /** Teardown function. + * + * The function is called after the test case finished. + */ + void teardown() {}; + +protected: + Fixture() = default; + virtual ~Fixture() = default; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_FIXTURE */ diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp new file mode 100644 index 0000000000..315f8ebea7 --- /dev/null +++ b/tests/framework/Framework.cpp @@ -0,0 +1,494 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "Framework.h" + +#include "support/ToolchainSupport.h" + +#ifdef ARM_COMPUTE_CL +#include "arm_compute/core/CL/OpenCL.h" +#include "arm_compute/runtime/CL/CLScheduler.h" +#endif /* ARM_COMPUTE_CL */ + +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +Framework::Framework() +{ + _available_instruments.emplace(InstrumentType::WALL_CLOCK_TIMER, Instrument::make_instrument); +#ifdef PMU_ENABLED + _available_instruments.emplace(InstrumentType::PMU_CYCLE_COUNTER, Instrument::make_instrument); + _available_instruments.emplace(InstrumentType::PMU_INSTRUCTION_COUNTER, Instrument::make_instrument); +#endif /* PMU_ENABLED */ +} + +std::set Framework::available_instruments() const +{ + std::set types; + + for(const auto &instrument : _available_instruments) + { + types.emplace(instrument.first); + } + + return types; +} + +std::map Framework::count_test_results() const +{ + std::map counts; + + for(const auto &test : _test_results) + { + ++counts[test.second.status]; + } + + return counts; +} + +Framework &Framework::get() +{ + static Framework instance; + return instance; +} + +void Framework::init(const std::vector &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter, LogLevel log_level) +{ + _test_filter = TestFilter(mode, name_filter, id_filter); + _num_iterations = num_iterations; + _log_level = log_level; + + _instruments = InstrumentType::NONE; + + for(const auto &instrument : instruments) + { + _instruments |= instrument; + } +} + +std::string Framework::current_suite_name() const +{ + return join(_test_suite_name.cbegin(), _test_suite_name.cend(), "/"); +} + +void Framework::push_suite(std::string name) +{ + _test_suite_name.emplace_back(std::move(name)); +} + +void Framework::pop_suite() +{ + _test_suite_name.pop_back(); +} + +void Framework::add_test_info(std::string info) +{ + _test_info.emplace_back(std::move(info)); +} + +void Framework::clear_test_info() +{ + _test_info.clear(); +} + +bool Framework::has_test_info() const +{ + return !_test_info.empty(); +} + +void Framework::print_test_info(std::ostream &os) const +{ + if(!_test_info.empty()) + { + os << "CONTEXT:\n"; + + for(const auto &str : _test_info) + { + os << " " << str << "\n"; + } + } +} + +void Framework::log_test_start(const TestInfo &info) +{ + if(_printer != nullptr && _log_level >= LogLevel::TESTS) + { + _printer->print_test_header(info); + } +} + +void Framework::log_test_skipped(const TestInfo &info) +{ + static_cast(info); +} + +void Framework::log_test_end(const TestInfo &info) +{ + if(_printer != nullptr) + { + if(_log_level >= LogLevel::MEASUREMENTS) + { + _printer->print_measurements(_test_results.at(info).measurements); + } + + if(_log_level >= LogLevel::TESTS) + { + _printer->print_test_footer(); + } + } +} + +void Framework::log_failed_expectation(const TestError &error) +{ + if(_log_level >= error.level() && _printer != nullptr) + { + _printer->print_error(error); + } + + if(_current_test_result != nullptr) + { + _current_test_result->status = TestResult::Status::FAILED; + } +} + +int Framework::num_iterations() const +{ + return _num_iterations; +} + +void Framework::set_num_iterations(int num_iterations) +{ + _num_iterations = num_iterations; +} + +void Framework::set_throw_errors(bool throw_errors) +{ + _throw_errors = throw_errors; +} + +bool Framework::throw_errors() const +{ + return _throw_errors; +} + +void Framework::set_stop_on_error(bool stop_on_error) +{ + _stop_on_error = stop_on_error; +} + +bool Framework::stop_on_error() const +{ + return _stop_on_error; +} + +void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) +{ + if(test_factory.status() == TestCaseFactory::Status::DISABLED) + { + log_test_skipped(info); + set_test_result(info, TestResult(TestResult::Status::DISABLED)); + return; + } + + log_test_start(info); + + Profiler profiler = get_profiler(); + TestResult result(TestResult::Status::NOT_RUN); + + _current_test_result = &result; + + if(_log_level >= LogLevel::ERRORS && _printer != nullptr) + { + _printer->print_errors_header(); + } + + try + { + std::unique_ptr test_case = test_factory.make(); + + try + { + test_case->do_setup(); + + for(int i = 0; i < _num_iterations; ++i) + { + profiler.start(); + test_case->do_run(); +#ifdef ARM_COMPUTE_CL + if(opencl_is_available()) + { + CLScheduler::get().sync(); + } +#endif /* ARM_COMPUTE_CL */ + profiler.stop(); + } + + test_case->do_teardown(); + + // Change status to success if no error has happend + if(result.status == TestResult::Status::NOT_RUN) + { + result.status = TestResult::Status::SUCCESS; + } + } + catch(const TestError &error) + { + if(_log_level >= error.level() && _printer != nullptr) + { + _printer->print_error(error); + } + + result.status = TestResult::Status::FAILED; + + if(_throw_errors) + { + throw; + } + } +#ifdef ARM_COMPUTE_CL + catch(const ::cl::Error &error) + { + if(_log_level >= LogLevel::ERRORS && _printer != nullptr) + { + std::stringstream stream; + stream << "Error code: " << error.err(); + _printer->print_error(TestError(error.what(), LogLevel::ERRORS, stream.str())); + } + + result.status = TestResult::Status::FAILED; + + if(_throw_errors) + { + throw; + } + } +#endif /* ARM_COMPUTE_CL */ + catch(const std::exception &error) + { + if(_log_level >= LogLevel::ERRORS && _printer != nullptr) + { + _printer->print_error(error); + } + + result.status = TestResult::Status::CRASHED; + + if(_throw_errors) + { + throw; + } + } + catch(...) + { + if(_log_level >= LogLevel::ERRORS && _printer != nullptr) + { + _printer->print_error(TestError("Received unknown exception")); + } + + result.status = TestResult::Status::CRASHED; + + if(_throw_errors) + { + throw; + } + } + } + catch(const std::exception &error) + { + if(_log_level >= LogLevel::ERRORS && _printer != nullptr) + { + _printer->print_error(error); + } + + result.status = TestResult::Status::CRASHED; + + if(_throw_errors) + { + throw; + } + } + catch(...) + { + if(_log_level >= LogLevel::ERRORS && _printer != nullptr) + { + _printer->print_error(TestError("Received unknown exception")); + } + + result.status = TestResult::Status::CRASHED; + + if(_throw_errors) + { + throw; + } + } + + if(_log_level >= LogLevel::ERRORS && _printer != nullptr) + { + _printer->print_errors_footer(); + } + + _current_test_result = nullptr; + + if(result.status == TestResult::Status::FAILED) + { + if(info.status == TestCaseFactory::Status::EXPECTED_FAILURE) + { + result.status = TestResult::Status::EXPECTED_FAILURE; + } + } + + if(result.status == TestResult::Status::FAILED || result.status == TestResult::Status::CRASHED) + { + if(_stop_on_error) + { + throw std::runtime_error("Abort on first error."); + } + } + + result.measurements = profiler.measurements(); + + set_test_result(info, result); + log_test_end(info); +} + +bool Framework::run() +{ + // Clear old test results + _test_results.clear(); + + if(_printer != nullptr && _log_level >= LogLevel::TESTS) + { + _printer->print_run_header(); + } + + const std::chrono::time_point start = std::chrono::high_resolution_clock::now(); + + int id = 0; + + for(auto &test_factory : _test_factories) + { + const std::string test_case_name = test_factory->name(); + const TestInfo test_info{ id, test_case_name, test_factory->mode(), test_factory->status() }; + + if(_test_filter.is_selected(test_info)) + { + run_test(test_info, *test_factory); + } + + ++id; + } + + const std::chrono::time_point end = std::chrono::high_resolution_clock::now(); + + if(_printer != nullptr && _log_level >= LogLevel::TESTS) + { + _printer->print_run_footer(); + } + + auto runtime = std::chrono::duration_cast(end - start); + std::map results = count_test_results(); + + if(_log_level > LogLevel::NONE) + { + std::cout << "Executed " << _test_results.size() << " test(s) (" + << results[TestResult::Status::SUCCESS] << " passed, " + << results[TestResult::Status::EXPECTED_FAILURE] << " expected failures, " + << results[TestResult::Status::FAILED] << " failed, " + << results[TestResult::Status::CRASHED] << " crashed, " + << results[TestResult::Status::DISABLED] << " disabled) in " << runtime.count() << " second(s)\n"; + } + + int num_successful_tests = results[TestResult::Status::SUCCESS] + results[TestResult::Status::EXPECTED_FAILURE]; + + return (static_cast(num_successful_tests) == _test_results.size()); +} + +void Framework::set_test_result(TestInfo info, TestResult result) +{ + _test_results.emplace(std::move(info), std::move(result)); +} + +void Framework::print_test_results(Printer &printer) const +{ + printer.print_run_header(); + + for(const auto &test : _test_results) + { + printer.print_test_header(test.first); + printer.print_measurements(test.second.measurements); + printer.print_test_footer(); + } + + printer.print_run_footer(); +} + +Profiler Framework::get_profiler() const +{ + Profiler profiler; + + for(const auto &instrument : _available_instruments) + { + if((instrument.first & _instruments) != InstrumentType::NONE) + { + profiler.add(instrument.second()); + } + } + + return profiler; +} + +void Framework::set_printer(Printer *printer) +{ + _printer = printer; +} + +std::vector Framework::test_infos() const +{ + std::vector ids; + + int id = 0; + + for(const auto &factory : _test_factories) + { + TestInfo test_info{ id, factory->name(), factory->mode(), factory->status() }; + + if(_test_filter.is_selected(test_info)) + { + ids.emplace_back(std::move(test_info)); + } + + ++id; + } + + return ids; +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/Framework.h b/tests/framework/Framework.h new file mode 100644 index 0000000000..055392cdae --- /dev/null +++ b/tests/framework/Framework.h @@ -0,0 +1,330 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_FRAMEWORK +#define ARM_COMPUTE_TEST_FRAMEWORK + +#include "DatasetModes.h" +#include "Exceptions.h" +#include "Profiler.h" +#include "TestCase.h" +#include "TestCaseFactory.h" +#include "TestFilter.h" +#include "TestResult.h" +#include "Utils.h" +#include "instruments/Instruments.h" +#include "printers/Printer.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Information about a test case. + * + * A test can be identified either via its id or via its name. Additionally + * each test is tagged with the data set mode in which it will be used and + * its status. + * + * @note The mapping between test id and test name is not guaranteed to be + * stable. It is subject to change as new test are added. + */ +struct TestInfo +{ + int id; + std::string name; + DatasetMode mode; + TestCaseFactory::Status status; +}; + +inline bool operator<(const TestInfo &lhs, const TestInfo &rhs) +{ + return lhs.id < rhs.id; +} + +/** Main framework class. + * + * Keeps track of the global state, owns all test cases and collects results. + */ +class Framework final +{ +public: + /** Access to the singleton. + * + * @return Unique instance of the framework class. + */ + static Framework &get(); + + /** Supported instrument types for benchmarking. + * + * @return Set of all available instrument types. + */ + std::set available_instruments() const; + + /** Init the framework. + * + * @see TestFilter::TestFilter for the format of the string to filter ids. + * + * @param[in] instruments Instrument types that will be used for benchmarking. + * @param[in] num_iterations Number of iterations per test. + * @param[in] mode Dataset mode. + * @param[in] name_filter Regular expression to filter tests by name. Only matching tests will be executed. + * @param[in] id_filter String to match selected test ids. Only matching tests will be executed. + * @param[in] log_level Verbosity of the output. + */ + void init(const std::vector &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter, LogLevel log_level); + + /** Add a new test suite. + * + * @warning Cannot be used at execution time. It can only be used for + * registering test cases. + * + * @param[in] name Name of the added test suite. + * + * @return Name of the current test suite. + */ + void push_suite(std::string name); + + /** Remove innermost test suite. + * + * @warning Cannot be used at execution time. It can only be used for + * registering test cases. + */ + void pop_suite(); + + /** Add a test case to the framework. + * + * @param[in] test_name Name of the new test case. + * @param[in] mode Mode in which to include the test. + * @param[in] status Status of the test case. + */ + template + void add_test_case(std::string test_name, DatasetMode mode, TestCaseFactory::Status status); + + /** Add a data test case to the framework. + * + * @param[in] test_name Name of the new test case. + * @param[in] mode Mode in which to include the test. + * @param[in] status Status of the test case. + * @param[in] description Description of @p data. + * @param[in] data Data that will be used as input to the test. + */ + template + void add_data_test_case(std::string test_name, DatasetMode mode, TestCaseFactory::Status status, std::string description, D &&data); + + /** Add info string for the next expectation/assertion. + * + * @param[in] info Info string. + */ + void add_test_info(std::string info); + + /** Clear the collected test info. */ + void clear_test_info(); + + /** Check if any info has been registered. + * + * @return True if there is test info. + */ + bool has_test_info() const; + + /** Print test info. + * + * @param[out] os Output stream. + */ + void print_test_info(std::ostream &os) const; + + /** Tell the framework that execution of a test starts. + * + * @param[in] info Test info. + */ + void log_test_start(const TestInfo &info); + + /** Tell the framework that a test case is skipped. + * + * @param[in] info Test info. + */ + void log_test_skipped(const TestInfo &info); + + /** Tell the framework that a test case finished. + * + * @param[in] info Test info. + */ + void log_test_end(const TestInfo &info); + + /** Tell the framework that the currently running test case failed a non-fatal expectation. + * + * @param[in] error Description of the error. + */ + void log_failed_expectation(const TestError &error); + + /** Number of iterations per test case. + * + * @return Number of iterations per test case. + */ + int num_iterations() const; + + /** Set number of iterations per test case. + * + * @param[in] num_iterations Number of iterations per test case. + */ + void set_num_iterations(int num_iterations); + + /** Should errors be caught or thrown by the framework. + * + * @return True if errors are thrown. + */ + bool throw_errors() const; + + /** Set whether errors are caught or thrown by the framework. + * + * @param[in] throw_errors True if errors should be thrown. + */ + void set_throw_errors(bool throw_errors); + + /** Indicates if test execution is stopped after the first failed test. + * + * @return True if the execution is going to be aborted after the first failed test. + */ + bool stop_on_error() const; + + /** Set whether to abort execution after the first failed test. + * + * @param[in] stop_on_error True if execution is going to be aborted after first failed test. + */ + void set_stop_on_error(bool stop_on_error); + + /** Run all enabled test cases. + * + * @return True if all test cases executed successful. + */ + bool run(); + + /** Set the result for an executed test case. + * + * @param[in] info Test info. + * @param[in] result Execution result. + */ + void set_test_result(TestInfo info, TestResult result); + + /** Use the specified printer to output test results from the last run. + * + * This method can be used if the test results need to be obtained using a + * different printer than the one managed by the framework. + * + * @param[in] printer Printer used to output results. + */ + void print_test_results(Printer &printer) const; + + /** Factory method to obtain a configured profiler. + * + * The profiler enables all instruments that have been passed to the @ref + * init method. + * + * @return Configured profiler to collect benchmark results. + */ + Profiler get_profiler() const; + + /** Set the printer used for the output of test results. + * + * @param[in] printer Pointer to a printer. + */ + void set_printer(Printer *printer); + + /** List of @ref TestInfo's. + * + * @return Vector with all test ids. + */ + std::vector test_infos() const; + +private: + Framework(); + ~Framework() = default; + + Framework(const Framework &) = delete; + Framework &operator=(const Framework &) = delete; + + void run_test(const TestInfo &info, TestCaseFactory &test_factory); + std::map count_test_results() const; + + /** Returns the current test suite name. + * + * @warning Cannot be used at execution time to get the test suite of the + * currently executed test case. It can only be used for registering test + * cases. + * + * @return Name of the current test suite. + */ + std::string current_suite_name() const; + + std::vector _test_suite_name{}; + std::vector> _test_factories{}; + std::map _test_results{}; + int _num_iterations{ 1 }; + bool _throw_errors{ false }; + bool _stop_on_error{ false }; + Printer *_printer{ nullptr }; + + using create_function = std::unique_ptr(); + std::map _available_instruments{}; + + InstrumentType _instruments{ InstrumentType::NONE }; + TestFilter _test_filter{}; + LogLevel _log_level{ LogLevel::ALL }; + TestResult *_current_test_result{ nullptr }; + std::vector _test_info{}; +}; + +template +inline void Framework::add_test_case(std::string test_name, DatasetMode mode, TestCaseFactory::Status status) +{ + _test_factories.emplace_back(support::cpp14::make_unique>(current_suite_name(), std::move(test_name), mode, status)); +} + +template +inline void Framework::add_data_test_case(std::string test_name, DatasetMode mode, TestCaseFactory::Status status, std::string description, D &&data) +{ + // WORKAROUND for GCC 4.9 + // The function should get *it which is tuple but that seems to trigger a + // bug in the compiler. + auto tmp = std::unique_ptr())>>(new DataTestCaseFactory())>(current_suite_name(), std::move(test_name), mode, status, + std::move(description), *data)); + _test_factories.emplace_back(std::move(tmp)); +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_FRAMEWORK */ diff --git a/tests/framework/Macros.h b/tests/framework/Macros.h new file mode 100644 index 0000000000..7aabb75cfc --- /dev/null +++ b/tests/framework/Macros.h @@ -0,0 +1,260 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_FRAMEWORK_MACROS +#define ARM_COMPUTE_TEST_FRAMEWORK_MACROS + +#include "Framework.h" +#include "Registrars.h" +#include "TestCase.h" + +// +// TEST SUITE MACROS +// +#define TEST_SUITE(SUITE_NAME) \ + namespace SUITE_NAME##Suite \ + { \ + static arm_compute::test::framework::detail::TestSuiteRegistrar SUITE_NAME##Suite_reg{ #SUITE_NAME }; + +#define TEST_SUITE_END() \ + static arm_compute::test::framework::detail::TestSuiteRegistrar Suite_end; \ + } +// +// TEST SUITE MACROS END +// + +// +// HELPER MACROS +// + +#define CONCAT(ARG0, ARG1) ARG0##ARG1 + +#define VARIADIC_SIZE_IMPL(e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, size, ...) size +#define VARIADIC_SIZE(...) VARIADIC_SIZE_IMPL(__VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) + +#define JOIN_PARAM1(OP, param) OP(0, param) +#define JOIN_PARAM2(OP, param, ...) \ + OP(1, param) \ + , JOIN_PARAM1(OP, __VA_ARGS__) +#define JOIN_PARAM3(OP, param, ...) \ + OP(2, param) \ + , JOIN_PARAM2(OP, __VA_ARGS__) +#define JOIN_PARAM4(OP, param, ...) \ + OP(3, param) \ + , JOIN_PARAM3(OP, __VA_ARGS__) +#define JOIN_PARAM5(OP, param, ...) \ + OP(4, param) \ + , JOIN_PARAM4(OP, __VA_ARGS__) +#define JOIN_PARAM6(OP, param, ...) \ + OP(5, param) \ + , JOIN_PARAM5(OP, __VA_ARGS__) +#define JOIN_PARAM7(OP, param, ...) \ + OP(6, param) \ + , JOIN_PARAM6(OP, __VA_ARGS__) +#define JOIN_PARAM8(OP, param, ...) \ + OP(7, param) \ + , JOIN_PARAM7(OP, __VA_ARGS__) +#define JOIN_PARAM9(OP, param, ...) \ + OP(8, param) \ + , JOIN_PARAM8(OP, __VA_ARGS__) +#define JOIN_PARAM10(OP, param, ...) \ + OP(9, param) \ + , JOIN_PARAM9(OP, __VA_ARGS__) +#define JOIN_PARAM(OP, NUM, ...) \ + CONCAT(JOIN_PARAM, NUM) \ + (OP, __VA_ARGS__) + +#define MAKE_TYPE_PARAM(i, name) typename T##i +#define MAKE_ARG_PARAM(i, name) const T##i &name +#define MAKE_TYPE_PARAMS(...) JOIN_PARAM(MAKE_TYPE_PARAM, VARIADIC_SIZE(__VA_ARGS__), __VA_ARGS__) +#define MAKE_ARG_PARAMS(...) JOIN_PARAM(MAKE_ARG_PARAM, VARIADIC_SIZE(__VA_ARGS__), __VA_ARGS__) + +// +// TEST CASE MACROS +// +#define TEST_CASE_CONSTRUCTOR(TEST_NAME) \ + TEST_NAME() = default; +#define DATA_TEST_CASE_CONSTRUCTOR(TEST_NAME, DATASET) \ + template \ + explicit TEST_NAME(D &&data) : DataTestCase{ std::forward(data) } \ + { \ + } +#define FIXTURE_SETUP(FIXTURE) \ + void do_setup() override \ + { \ + FIXTURE::setup(); \ + } +#define FIXTURE_DATA_SETUP(FIXTURE) \ + void do_setup() override \ + { \ + apply(this, &FIXTURE::setup, _data); \ + } +#define FIXTURE_RUN(FIXTURE) \ + void do_run() override \ + { \ + FIXTURE::run(); \ + } +#define FIXTURE_TEARDOWN(FIXTURE) \ + void do_teardown() override \ + { \ + FIXTURE::teardown(); \ + } +#define TEST_REGISTRAR(TEST_NAME, MODE, STATUS) \ + static arm_compute::test::framework::detail::TestCaseRegistrar TEST_NAME##_reg \ + { \ + #TEST_NAME, MODE, STATUS \ + } +#define DATA_TEST_REGISTRAR(TEST_NAME, MODE, STATUS, DATASET) \ + static arm_compute::test::framework::detail::TestCaseRegistrar> TEST_NAME##_reg \ + { \ + #TEST_NAME, MODE, STATUS, DATASET \ + } + +#define TEST_CASE_IMPL(TEST_NAME, MODE, STATUS) \ + class TEST_NAME : public arm_compute::test::framework::TestCase \ + { \ + public: \ + TEST_CASE_CONSTRUCTOR(TEST_NAME) \ + void do_run() override; \ + }; \ + TEST_REGISTRAR(TEST_NAME, MODE, STATUS); \ + void TEST_NAME::do_run() + +#define TEST_CASE(TEST_NAME, MODE) \ + TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE) +#define EXPECTED_FAILURE_TEST_CASE(TEST_NAME, MODE) \ + TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE) +#define DISABLED_TEST_CASE(TEST_NAME, MODE) \ + TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED) + +#define DATA_TEST_CASE_IMPL(TEST_NAME, MODE, STATUS, DATASET, ...) \ + template \ + class TEST_NAME; \ + template \ + class TEST_NAME> : public arm_compute::test::framework::DataTestCase \ + { \ + public: \ + DATA_TEST_CASE_CONSTRUCTOR(TEST_NAME, DATASET) \ + void do_run() override \ + { \ + arm_compute::test::framework::apply(this, &TEST_NAME::run, _data); \ + } \ + template \ + void run(MAKE_ARG_PARAMS(__VA_ARGS__)); \ + }; \ + DATA_TEST_REGISTRAR(TEST_NAME, MODE, STATUS, DATASET); \ + template \ + template \ + void TEST_NAME>::run(MAKE_ARG_PARAMS(__VA_ARGS__)) + +#define DATA_TEST_CASE(TEST_NAME, MODE, DATASET, ...) \ + DATA_TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE, DATASET, __VA_ARGS__) +#define EXPECTED_FAILURE_DATA_TEST_CASE(TEST_NAME, MODE, DATASET, ...) \ + DATA_TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE, DATASET, __VA_ARGS__) +#define DISABLED_DATA_TEST_CASE(TEST_NAME, MODE, DATASET, ...) \ + DATA_TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED, DATASET, __VA_ARGS__) + +#define FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, STATUS) \ + class TEST_NAME : public arm_compute::test::framework::TestCase, public FIXTURE \ + { \ + public: \ + TEST_CASE_CONSTRUCTOR(TEST_NAME) \ + FIXTURE_SETUP(FIXTURE) \ + void do_run() override; \ + FIXTURE_TEARDOWN(FIXTURE) \ + }; \ + TEST_REGISTRAR(TEST_NAME, MODE, STATUS); \ + void TEST_NAME::do_run() + +#define FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \ + FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE) +#define EXPECTED_FAILURE_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \ + FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE) +#define DISABLED_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \ + FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED) + +#define FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, STATUS, DATASET) \ + template \ + class TEST_NAME; \ + template \ + class TEST_NAME> : public arm_compute::test::framework::DataTestCase, public FIXTURE \ + { \ + public: \ + DATA_TEST_CASE_CONSTRUCTOR(TEST_NAME, DATASET) \ + FIXTURE_DATA_SETUP(FIXTURE) \ + void do_run() override; \ + FIXTURE_TEARDOWN(FIXTURE) \ + }; \ + DATA_TEST_REGISTRAR(TEST_NAME, MODE, STATUS, DATASET); \ + template \ + void TEST_NAME>::do_run() + +#define FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \ + FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE, DATASET) +#define EXPECTED_FAILURE_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \ + FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE, DATASET) +#define DISABLED_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \ + FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED, DATASET) + +#define REGISTER_FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, STATUS) \ + class TEST_NAME : public arm_compute::test::framework::TestCase, public FIXTURE \ + { \ + public: \ + TEST_CASE_CONSTRUCTOR(TEST_NAME) \ + FIXTURE_SETUP(FIXTURE) \ + FIXTURE_RUN(FIXTURE) \ + FIXTURE_TEARDOWN(FIXTURE) \ + }; \ + TEST_REGISTRAR(TEST_NAME, MODE, STATUS) + +#define REGISTER_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \ + REGISTER_FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE) +#define EXPECTED_FAILURE_REGISTER_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \ + REGISTER_FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE) +#define DISABLED_REGISTER_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \ + REGISTER_FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED) + +#define REGISTER_FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, STATUS, DATASET) \ + template \ + class TEST_NAME; \ + template \ + class TEST_NAME> : public arm_compute::test::framework::DataTestCase, public FIXTURE \ + { \ + public: \ + DATA_TEST_CASE_CONSTRUCTOR(TEST_NAME, DATASET) \ + FIXTURE_DATA_SETUP(FIXTURE) \ + FIXTURE_RUN(FIXTURE) \ + FIXTURE_TEARDOWN(FIXTURE) \ + }; \ + DATA_TEST_REGISTRAR(TEST_NAME, MODE, STATUS, DATASET) + +#define REGISTER_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \ + REGISTER_FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE, DATASET) +#define EXPECTED_FAILURE_REGISTER_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \ + REGISTER_FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE, DATASET) +#define DISABLED_REGISTER_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \ + REGISTER_FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED, DATASET) +// +// TEST CASE MACROS END +// +#endif /* ARM_COMPUTE_TEST_FRAMEWORK_MACROS */ diff --git a/tests/framework/Profiler.cpp b/tests/framework/Profiler.cpp new file mode 100644 index 0000000000..76de9a818f --- /dev/null +++ b/tests/framework/Profiler.cpp @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "Profiler.h" + +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +void Profiler::add(std::unique_ptr instrument) +{ + _instruments.emplace_back(std::move(instrument)); +} + +void Profiler::start() +{ + for(auto &instrument : _instruments) + { + instrument->start(); + } +} + +void Profiler::stop() +{ + for(auto &instrument : _instruments) + { + instrument->stop(); + } + + for(const auto &instrument : _instruments) + { + _measurements[instrument->id()].push_back(instrument->measurement()); + } +} + +const Profiler::MeasurementsMap &Profiler::measurements() const +{ + return _measurements; +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/Profiler.h b/tests/framework/Profiler.h new file mode 100644 index 0000000000..1454c0f875 --- /dev/null +++ b/tests/framework/Profiler.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_PROFILER +#define ARM_COMPUTE_TEST_PROFILER + +#include "instruments/Instrument.h" + +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Profiler class to collect benchmark numbers. + * + * A profiler manages multiple instruments that can collect different types of benchmarking numbers. + */ +class Profiler +{ +public: + /** Mapping from instrument ids to their measurements. */ + using MeasurementsMap = std::map>; + + /** Add @p instrument to the performance monitor. + * + * All added instruments will be used when @ref start or @ref stop are + * called to make measurements. + * + * @param[in] instrument Instrument to be used to measure performance. + */ + void add(std::unique_ptr instrument); + + /** Start all added instruments to measure performance. */ + void start(); + + /** Stop all added instruments. */ + void stop(); + + /** Return measurements for all instruments. */ + const MeasurementsMap &measurements() const; + +private: + std::vector> _instruments{}; + MeasurementsMap _measurements{}; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_PROFILER */ diff --git a/tests/framework/Registrars.h b/tests/framework/Registrars.h new file mode 100644 index 0000000000..ca23edf0de --- /dev/null +++ b/tests/framework/Registrars.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_FRAMEWORK_REGISTRARS +#define ARM_COMPUTE_TEST_FRAMEWORK_REGISTRARS + +#include "DatasetModes.h" +#include "Framework.h" + +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +namespace detail +{ +/** Helper class to statically register a test case. */ +template +class TestCaseRegistrar final +{ +public: + /** Add a new test case with the given name to the framework. + * + * @param[in] test_name Name of the test case. + * @param[in] mode Mode in which the test should be activated. + * @param[in] status Status of the test case. + */ + TestCaseRegistrar(std::string test_name, DatasetMode mode, TestCaseFactory::Status status); + + /** Add a new data test case with the given name to the framework. + * + * @param[in] test_name Name of the test case. + * @param[in] mode Mode in which the test should be activated. + * @param[in] status Status of the test case. + * @param[in] dataset Dataset used as input for the test case. + */ + template + TestCaseRegistrar(std::string test_name, DatasetMode mode, TestCaseFactory::Status status, D &&dataset); +}; + +/** Helper class to statically begin and end a test suite. */ +class TestSuiteRegistrar final +{ +public: + /** Remove the last added test suite from the framework. */ + TestSuiteRegistrar(); + + /** Add a new test suite with the given name to the framework. + * + * @param[in] name Name of the test suite. + */ + TestSuiteRegistrar(std::string name); +}; + +template +inline TestCaseRegistrar::TestCaseRegistrar(std::string test_name, DatasetMode mode, TestCaseFactory::Status status) +{ + Framework::get().add_test_case(std::move(test_name), mode, status); +} + +template +template +inline TestCaseRegistrar::TestCaseRegistrar(std::string test_name, DatasetMode mode, TestCaseFactory::Status status, D &&dataset) +{ + auto it = dataset.begin(); + + for(int i = 0; i < dataset.size(); ++i, ++it) + { + // WORKAROUND for GCC 4.9 + // The last argument should be *it to pass just the data and not the + // iterator. + Framework::get().add_data_test_case(test_name, mode, status, it.description(), it); + } +} + +inline TestSuiteRegistrar::TestSuiteRegistrar() +{ + Framework::get().pop_suite(); +} + +inline TestSuiteRegistrar::TestSuiteRegistrar(std::string name) +{ + Framework::get().push_suite(std::move(name)); +} +} // namespace detail +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_FRAMEWORK_REGISTRARS */ diff --git a/tests/framework/SConscript b/tests/framework/SConscript new file mode 100644 index 0000000000..36632515a3 --- /dev/null +++ b/tests/framework/SConscript @@ -0,0 +1,71 @@ +# Copyright (c) 2017 ARM Limited. +# +# SPDX-License-Identifier: MIT +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +import SCons +import os.path + +Import('env') +Import('vars') + +# vars is imported from arm_compute: +variables = [ + BoolVariable("pmu", "Enable PMU counters", False) +] + +# We need a separate set of Variables for the Help message (Otherwise the global variables will get displayed twice) +new_options = Variables('scons') + +for v in variables: + new_options.Add(v) + vars.Add(v) + +# Clone the environment to make sure we're not polluting the arm_compute one: +framework_env = env.Clone() +vars.Update(framework_env) + +Help(new_options.GenerateHelpText(framework_env)) + +if env['os'] == 'android' and framework_env['pmu']: + print("pmu=1 is not supported for os=android") + Exit(1) + +if(env['opencl']): + framework_env.Append(CPPDEFINES=['ARM_COMPUTE_CL']) + +framework_env.Append(CPPPATH = ["."]) +framework_env.Append(CPPFLAGS=['-Wno-overloaded-virtual']) + +files = Glob('*.cpp') +files += Glob('command_line/*.cpp') +files += Glob('printers/*.cpp') +files += Glob('datasets/*.cpp') +files += Glob('instruments/*.cpp') + +if not framework_env['pmu']: + # Remove PMU files + files = [f for f in files if "PMU" not in os.path.basename(str(f))] +else: + framework_env.Append(CPPDEFINES = ['PMU_ENABLED']) + +arm_compute_test_framework = framework_env.StaticLibrary('arm_compute_test_framework', files) + +Default(arm_compute_test_framework) +Export('arm_compute_test_framework') diff --git a/tests/framework/TestCase.h b/tests/framework/TestCase.h new file mode 100644 index 0000000000..dbb9312dee --- /dev/null +++ b/tests/framework/TestCase.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_TESTCASE +#define ARM_COMPUTE_TEST_TESTCASE + +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Abstract test case class. + * + * All test cases have to inherit from this class. + */ +class TestCase +{ +public: + virtual void do_setup() {}; + virtual void do_run() {}; + virtual void do_teardown() {}; + + /** Default destructor. */ + virtual ~TestCase() = default; + +protected: + TestCase() = default; + + friend class TestCaseFactory; +}; + +template +class DataTestCase : public TestCase +{ +protected: + explicit DataTestCase(T data) + : _data{ std::move(data) } + { + } + + T _data; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_TESTCASE */ diff --git a/tests/framework/TestCaseFactory.h b/tests/framework/TestCaseFactory.h new file mode 100644 index 0000000000..b8c8cdbeb0 --- /dev/null +++ b/tests/framework/TestCaseFactory.h @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_TEST_CASE_FACTORY +#define ARM_COMPUTE_TEST_TEST_CASE_FACTORY + +#include "DatasetModes.h" +#include "TestCase.h" +#include "support/ToolchainSupport.h" + +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Abstract factory class to create test cases. */ +class TestCaseFactory +{ +public: + /** Test case status. + * + * ACTIVE == Test is run and result is validated. Failure on failed validation. + * EXPECTED_FAILURE == Test is run and result is validated. Failure on successful validation. + * DISABLED == Test is not run. + */ + enum class Status + { + ACTIVE, + EXPECTED_FAILURE, + DISABLED + }; + + /** Constructor. + * + * @param[in] suite_name Name of the test suite to which the test case has been added. + * @param[in] name Name of the test case. + * @param[in] mode Datset mode of the test case. + * @param[in] status Status of the test case. + * @param[in] description Description of data arguments. + */ + TestCaseFactory(std::string suite_name, std::string name, DatasetMode mode, Status status, std::string description = ""); + + /** Default destructor. */ + virtual ~TestCaseFactory() = default; + + /** Name of the test case. + * + * @return Name of the test case. + */ + std::string name() const; + + /** Get the mode for which test case will be enabled. + * + * @return Dataset mode of the test case. + */ + DatasetMode mode() const; + + /** Get the status of the test case. + * + * @return Status of the test case. + */ + Status status() const; + + /** Factory function to create the test case + * + * @return Unique pointer to a newly created test case. + */ + virtual std::unique_ptr make() const = 0; + +private: + const std::string _suite_name; + const std::string _test_name; + const std::string _data_description; + const DatasetMode _mode{ DatasetMode::ALL }; + const Status _status{ Status::ACTIVE }; +}; + +/** Implementation of a test case factory to create non-data test cases. */ +template +class SimpleTestCaseFactory final : public TestCaseFactory +{ +public: + /** Default constructor. */ + using TestCaseFactory::TestCaseFactory; + + std::unique_ptr make() const override; +}; + +template +class DataTestCaseFactory final : public TestCaseFactory +{ +public: + /** Constructor. + * + * @param[in] suite_name Name of the test suite to which the test case has been added. + * @param[in] test_name Name of the test case. + * @param[in] mode Mode in which the test case is enabled. + * @param[in] status Status of the test case. + * @param[in] description Description of data arguments. + * @param[in] data Input data for the test case. + */ + DataTestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, Status status, std::string description, const D &data); + + std::unique_ptr make() const override; + +private: + D _data; +}; + +inline TestCaseFactory::TestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, Status status, std::string description) + : _suite_name{ std::move(suite_name) }, _test_name{ std::move(test_name) }, _data_description{ std::move(description) }, _mode{ mode }, _status{ status } + +{ +} + +inline std::string TestCaseFactory::name() const +{ + std::string name = _suite_name + "/" + _test_name; + + if(!_data_description.empty()) + { + name += "@" + _data_description; + } + + return name; +} + +inline DatasetMode TestCaseFactory::mode() const +{ + return _mode; +} + +inline TestCaseFactory::Status TestCaseFactory::status() const +{ + return _status; +} + +inline ::std::ostream &operator<<(::std::ostream &stream, TestCaseFactory::Status status) +{ + switch(status) + { + case TestCaseFactory::Status::ACTIVE: + stream << "ACTIVE"; + break; + case TestCaseFactory::Status::EXPECTED_FAILURE: + stream << "EXPECTED_FAILURE"; + break; + case TestCaseFactory::Status::DISABLED: + stream << "DISABLED"; + break; + default: + throw std::invalid_argument("Unsupported test case factory status"); + } + + return stream; +} + +template +inline std::unique_ptr SimpleTestCaseFactory::make() const +{ + return support::cpp14::make_unique(); +} + +template +inline DataTestCaseFactory::DataTestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, Status status, std::string description, const D &data) + : TestCaseFactory{ std::move(suite_name), std::move(test_name), mode, status, std::move(description) }, _data{ data } +{ +} + +template +inline std::unique_ptr DataTestCaseFactory::make() const +{ + return support::cpp14::make_unique(_data); +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_TEST_CASE_FACTORY */ diff --git a/tests/framework/TestFilter.cpp b/tests/framework/TestFilter.cpp new file mode 100644 index 0000000000..0af40c1717 --- /dev/null +++ b/tests/framework/TestFilter.cpp @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "TestFilter.h" + +#include "Framework.h" +#include "support/ToolchainSupport.h" + +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +TestFilter::TestFilter(DatasetMode mode, const std::string &name_filter, const std::string &id_filter) + : _dataset_mode{ mode }, _name_filter{ name_filter }, _id_filter{ parse_id_filter(id_filter) } +{ +} + +bool TestFilter::is_selected(const TestInfo &info) const +{ + if((info.mode & _dataset_mode) == DatasetMode::DISABLED) + { + return false; + } + + if(!std::regex_search(info.name, _name_filter)) + { + return false; + } + + if(!_id_filter.empty()) + { + bool found = false; + + for(const auto range : _id_filter) + { + if(range.first <= info.id && info.id <= range.second) + { + found = true; + break; + } + } + + if(!found) + { + return false; + } + } + + return true; +} + +TestFilter::Ranges TestFilter::parse_id_filter(const std::string &id_filter) const +{ + Ranges ranges; + std::string str; + bool in_range = false; + int value = 0; + int start = 0; + int end = std::numeric_limits::max(); + + std::stringstream stream(id_filter); + + // Get first value + std::getline(stream, str, ','); + + if(stream.fail()) + { + return ranges; + } + + if(str.find("...") != std::string::npos) + { + in_range = true; + } + else + { + start = support::cpp11::stoi(str); + end = start; + } + + while(!stream.eof()) + { + std::getline(stream, str, ','); + + if(stream.fail()) + { + break; + } + + if(str.find("...") != std::string::npos) + { + end = std::numeric_limits::max(); + in_range = true; + } + else + { + value = support::cpp11::stoi(str); + + if(in_range || end == value - 1) + { + end = value; + in_range = false; + } + else + { + ranges.emplace_back(start, end); + start = value; + end = start; + } + } + } + + ranges.emplace_back(start, end); + return ranges; +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/TestFilter.h b/tests/framework/TestFilter.h new file mode 100644 index 0000000000..f64e73a2ba --- /dev/null +++ b/tests/framework/TestFilter.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_TESTFILTER +#define ARM_COMPUTE_TEST_TESTFILTER + +#include "DatasetModes.h" + +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +struct TestInfo; + +/** Test filter class. + * + * Stores information about which test cases are selected for execution. Based + * on test name and test id. + */ +class TestFilter final +{ +public: + /** Default constructor. All tests selected. */ + TestFilter() = default; + + /** Constructor. + * + * The id_filter string has be a comma separated list of test ids. ... can + * be used to include a range of tests. For instance, "..., 15" means all + * test up to and including 15, "3, 6, ..., 10" means tests 3 and 6 to 10, + * and "15, ..." means test 15 and all following. + * + * @param[in] mode Dataset mode. + * @param[in] name_filter Regular expression to filter tests by name. Only matching tests will be executed. + * @param[in] id_filter String to match selected test ids. Only matching tests will be executed. + */ + TestFilter(DatasetMode mode, const std::string &name_filter, const std::string &id_filter); + + /** Check if a test case is selected to be executed. + * + * @param[in] info Test case info. + * + * @return True if the test case is selected to be executed. + */ + bool is_selected(const TestInfo &info) const; + +private: + using Ranges = std::vector>; + Ranges parse_id_filter(const std::string &id_filter) const; + + DatasetMode _dataset_mode{ DatasetMode::ALL }; + std::regex _name_filter{ ".*" }; + Ranges _id_filter{}; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_TESTFILTER */ diff --git a/tests/framework/TestResult.h b/tests/framework/TestResult.h new file mode 100644 index 0000000000..e71ef95112 --- /dev/null +++ b/tests/framework/TestResult.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_TESTRESULT +#define ARM_COMPUTE_TEST_TESTRESULT + +#include "Profiler.h" + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Class to store results of a test. + * + * Currently the execution status and profiling information are stored. + */ +struct TestResult +{ + /** Execution status of a test. */ + enum class Status + { + NOT_RUN, + SUCCESS, + EXPECTED_FAILURE, + FAILED, + CRASHED, + DISABLED + }; + + /** Default constructor. */ + TestResult() = default; + + /** Initialise the result with a status. + * + * @param[in] status Execution status. + */ + TestResult(Status status) + : status{ status } + { + } + + /** Initialise the result with a status and profiling information. + * + * @param[in] status Execution status. + * @param[in] measurements Profiling information. + */ + TestResult(Status status, const Profiler::MeasurementsMap &measurements) + : status{ status }, measurements{ measurements } + { + } + + Status status{ Status::NOT_RUN }; //< Execution status + Profiler::MeasurementsMap measurements{}; //< Profiling information +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_TESTRESULT */ diff --git a/tests/framework/Utils.h b/tests/framework/Utils.h new file mode 100644 index 0000000000..a9fe0dcaa3 --- /dev/null +++ b/tests/framework/Utils.h @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_UTILS +#define ARM_COMPUTE_TEST_UTILS + +#include "support/ToolchainSupport.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** @cond */ +namespace detail +{ +template +struct sequence +{ +}; + +template +struct sequence_generator; + +template +struct sequence_generator<0, Ns...> +{ + using type = sequence; +}; + +template +struct sequence_generator : sequence_generator < N - 1, N - 1, Ns... > +{ +}; + +template +using sequence_t = typename sequence_generator::type; +/** @endcond */ + +template +void apply_impl(O *obj, F &&func, const std::tuple &args, detail::sequence) +{ + (obj->*func)(std::get(args)...); +} +} // namespace + +template +void apply(O *obj, F &&func, const std::tuple &args) +{ + detail::apply_impl(obj, std::forward(func), args, detail::sequence_t()); +} + +/** Helper function to concatenate multiple strings. + * + * @param[in] first Iterator pointing to the first element to be concatenated. + * @param[in] last Iterator pointing behind the last element to be concatenated. + * @param[in] separator String used to join the elements. + * + * @return String containing all elements joined by @p separator. + */ +template ::value, int>::type = 0> +std::string join(T first, T last, const std::string &separator) +{ + return std::accumulate(std::next(first), last, *first, [&separator](const std::string & base, const std::string & suffix) + { + return base + separator + suffix; + }); +} + +/** Helper function to concatenate multiple values. + * + * All values are converted to std::string using the provided operation before + * being joined. + * + * The signature of op has to be equivalent to + * std::string op(const T::value_type &val). + * + * @param[in] first Iterator pointing to the first element to be concatenated. + * @param[in] last Iterator pointing behind the last element to be concatenated. + * @param[in] separator String used to join the elements. + * @param[in] op Conversion function. + * + * @return String containing all elements joined by @p separator. + */ +template +std::string join(T &&first, T &&last, const std::string &separator, UnaryOp &&op) +{ + return std::accumulate(std::next(first), last, op(*first), [&separator, &op](const std::string & base, const typename T::value_type & suffix) + { + return base + separator + op(suffix); + }); +} + +/** Helper function to concatenate multiple values. + * + * All values are converted to std::string using std::to_string before being joined. + * + * @param[in] first Iterator pointing to the first element to be concatenated. + * @param[in] last Iterator pointing behind the last element to be concatenated. + * @param[in] separator String used to join the elements. + * + * @return String containing all elements joined by @p separator. + */ +template ::value, int>::type = 0> +std::string join(T && first, T && last, const std::string &separator) +{ + return join(std::forward(first), std::forward(last), separator, support::cpp11::to_string); +} + +/** Convert string to lower case. + * + * @param[in] string To be converted string. + * + * @return Lower case string. + */ +inline std::string tolower(std::string string) +{ + std::transform(string.begin(), string.end(), string.begin(), [](unsigned char c) + { + return std::tolower(c); + }); + return string; +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_UTILS */ diff --git a/tests/framework/command_line/CommandLineOptions.h b/tests/framework/command_line/CommandLineOptions.h new file mode 100644 index 0000000000..cb4b794a3e --- /dev/null +++ b/tests/framework/command_line/CommandLineOptions.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_COMMANDLINEOPTIONS +#define ARM_COMPUTE_TEST_COMMANDLINEOPTIONS + +#include "EnumListOption.h" +#include "EnumOption.h" +#include "ListOption.h" +#include "Option.h" +#include "ToggleOption.h" + +#endif /* ARM_COMPUTE_TEST_COMMANDLINEOPTIONS */ diff --git a/tests/framework/command_line/CommandLineParser.cpp b/tests/framework/command_line/CommandLineParser.cpp new file mode 100644 index 0000000000..228b18d842 --- /dev/null +++ b/tests/framework/command_line/CommandLineParser.cpp @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "CommandLineParser.h" + +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +void CommandLineParser::parse(int argc, char **argv) +{ + const std::regex option_regex{ "--((?:no-)?)([^=]+)(?:=(.*))?" }; + + const auto set_option = [&](const std::string & option, const std::string & name, const std::string & value) + { + if(_options.find(name) == _options.end()) + { + _unknown_options.push_back(option); + return; + } + + const bool success = _options[name]->parse(value); + + if(!success) + { + _invalid_options.push_back(option); + } + }; + + unsigned int positional_index = 0; + + for(int i = 1; i < argc; ++i) + { + const std::string option{ argv[i] }; + std::smatch option_matches; + + if(std::regex_match(option, option_matches, option_regex)) + { + // Boolean option + if(option_matches.str(3).empty()) + { + set_option(option, option_matches.str(2), option_matches.str(1).empty() ? "true" : "false"); + } + else + { + // Can't have "no-" and a value + if(!option_matches.str(1).empty()) + { + _invalid_options.emplace_back(option); + } + else + { + set_option(option, option_matches.str(2), option_matches.str(3)); + } + } + } + else + { + if(positional_index >= _positional_options.size()) + { + _invalid_options.push_back(option); + } + else + { + _positional_options[positional_index]->parse(option); + ++positional_index; + } + } + } +} + +bool CommandLineParser::validate() const +{ + bool is_valid = true; + + for(const auto &option : _options) + { + if(option.second->is_required() && !option.second->is_set()) + { + is_valid = false; + std::cerr << "ERROR: Option '" << option.second->name() << "' is required but not given!\n"; + } + } + + for(const auto &option : _positional_options) + { + if(option->is_required() && !option->is_set()) + { + is_valid = false; + std::cerr << "ERROR: Option '" << option->name() << "' is required but not given!\n"; + } + } + + for(const auto &option : _unknown_options) + { + std::cerr << "WARNING: Skipping unknown option '" << option << "'!\n"; + } + + for(const auto &option : _invalid_options) + { + std::cerr << "WARNING: Skipping invalid option '" << option << "'!\n"; + } + + return is_valid; +} + +void CommandLineParser::print_help(const std::string &program_name) const +{ + std::cout << "usage: " << program_name << " \n"; + + for(const auto &option : _options) + { + std::cout << option.second->help() << "\n"; + } + + for(const auto &option : _positional_options) + { + //FIXME: Print help string as well + std::cout << option->name() << "\n"; + } +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/command_line/CommandLineParser.h b/tests/framework/command_line/CommandLineParser.h new file mode 100644 index 0000000000..adb5214e2f --- /dev/null +++ b/tests/framework/command_line/CommandLineParser.h @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_COMMANDLINEPARSER +#define ARM_COMPUTE_TEST_COMMANDLINEPARSER + +#include "../Utils.h" +#include "Option.h" + +#include +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Class to parse command line arguments. */ +class CommandLineParser final +{ +public: + /** Default constructor. */ + CommandLineParser() = default; + + /** Function to add a new option to the parser. + * + * @param[in] name Name of the option. Will be available under --name=VALUE. + * @param[in] args Option specific configuration arguments. + * + * @return Pointer to the option. The option is owned by the parser. + */ + template + T *add_option(const std::string &name, As &&... args); + + /** Function to add a new positional argument to the parser. + * + * @param[in] args Option specific configuration arguments. + * + * @return Pointer to the option. The option is owned by the parser. + */ + template + T *add_positional_option(As &&... args); + + /** Parses the command line arguments and updates the options accordingly. + * + * @param[in] argc Number of arguments. + * @param[in] argv Arguments. + */ + void parse(int argc, char **argv); + + /** Validates the previously parsed command line arguments. + * + * Validation fails if not all required options are provided. Additionally + * warnings are generated for options that have illegal values or unknown + * options. + * + * @return True if all required options have been provided. + */ + bool validate() const; + + /** Prints a help message for all configured options. + * + * @param[in] program_name Name of the program to be used in the help message. + */ + void print_help(const std::string &program_name) const; + +private: + using OptionsMap = std::map>; + using PositionalOptionsVector = std::vector>; + + OptionsMap _options{}; + PositionalOptionsVector _positional_options{}; + std::vector _unknown_options{}; + std::vector _invalid_options{}; +}; + +template +inline T *CommandLineParser::add_option(const std::string &name, As &&... args) +{ + auto result = _options.emplace(name, support::cpp14::make_unique(name, std::forward(args)...)); + return static_cast(result.first->second.get()); +} + +template +inline T *CommandLineParser::add_positional_option(As &&... args) +{ + _positional_options.emplace_back(support::cpp14::make_unique(std::forward(args)...)); + return static_cast(_positional_options.back().get()); +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_COMMANDLINEPARSER */ diff --git a/tests/framework/command_line/EnumListOption.h b/tests/framework/command_line/EnumListOption.h new file mode 100644 index 0000000000..d19bfbdc0f --- /dev/null +++ b/tests/framework/command_line/EnumListOption.h @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_ENUMLISTOPTION +#define ARM_COMPUTE_TEST_ENUMLISTOPTION + +#include "Option.h" + +#include +#include +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Implementation of an option that accepts any number of values from a fixed set. */ +template +class EnumListOption : public Option +{ +public: + /** Construct option with allowed values. + * + * @param[in] name Name of the option. + * @param[in] allowed_values Set of allowed values for the option. + */ + EnumListOption(std::string name, std::set allowed_values); + + /** Construct option with allowed values, a fixed number of accepted values and default values for the option. + * + * @param[in] name Name of the option. + * @param[in] allowed_values Set of allowed values for the option. + * @param[in] default_values Default values. + */ + EnumListOption(std::string name, std::set allowed_values, std::initializer_list &&default_values); + + bool parse(std::string value) override; + std::string help() const override; + const std::vector &value() const; + +private: + std::vector _values{}; + std::set _allowed_values{}; +}; + +template +inline EnumListOption::EnumListOption(std::string name, std::set allowed_values) + : Option{ std::move(name) }, _allowed_values{ std::move(allowed_values) } +{ +} + +template +inline EnumListOption::EnumListOption(std::string name, std::set allowed_values, std::initializer_list &&default_values) + : Option{ std::move(name), false, true }, _values{ std::forward>(default_values) }, _allowed_values{ std::move(allowed_values) } +{ +} + +template +bool EnumListOption::parse(std::string value) +{ + // Remove default values + _values.clear(); + _is_set = true; + + try + { + std::stringstream stream{ value }; + std::string item; + + while(!std::getline(stream, item, ',').fail()) + { + std::stringstream item_stream(item); + T typed_value{}; + + item_stream >> typed_value; + + if(!item_stream.fail()) + { + if(_allowed_values.count(typed_value) == 0) + { + _values.clear(); + return false; + } + + _values.emplace_back(typed_value); + } + + _is_set = _is_set && !item_stream.fail(); + } + + return _is_set; + } + catch(const std::invalid_argument &) + { + return false; + } +} + +template +std::string EnumListOption::help() const +{ + std::stringstream msg; + msg << "--" + name() + "={"; + + for(const auto &value : _allowed_values) + { + msg << value << ","; + } + + msg << "}[,{...}[,...]] - " << _help; + + return msg.str(); +} + +template +inline const std::vector &EnumListOption::value() const +{ + return _values; +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_ENUMLISTOPTION */ diff --git a/tests/framework/command_line/EnumOption.h b/tests/framework/command_line/EnumOption.h new file mode 100644 index 0000000000..1abba77b4b --- /dev/null +++ b/tests/framework/command_line/EnumOption.h @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_ENUMOPTION +#define ARM_COMPUTE_TEST_ENUMOPTION + +#include "SimpleOption.h" + +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Implementation of a simple option that accepts a value from a fixed set. */ +template +class EnumOption : public SimpleOption +{ +public: + /** Construct option with allowed values. + * + * @param[in] name Name of the option. + * @param[in] allowed_values Set of allowed values for the option. + */ + EnumOption(std::string name, std::set allowed_values); + + /** Construct option with allowed values, a fixed number of accepted values and default values for the option. + * + * @param[in] name Name of the option. + * @param[in] allowed_values Set of allowed values for the option. + * @param[in] default_value Default value. + */ + EnumOption(std::string name, std::set allowed_values, T default_value); + + bool parse(std::string value) override; + std::string help() const override; + const T &value() const; + +private: + std::set _allowed_values{}; +}; + +template +inline EnumOption::EnumOption(std::string name, std::set allowed_values) + : SimpleOption{ std::move(name) }, _allowed_values{ std::move(allowed_values) } +{ +} + +template +inline EnumOption::EnumOption(std::string name, std::set allowed_values, T default_value) + : SimpleOption{ std::move(name), std::move(default_value) }, _allowed_values{ std::move(allowed_values) } +{ +} + +template +bool EnumOption::parse(std::string value) +{ + try + { + std::stringstream stream{ value }; + T typed_value{}; + + stream >> typed_value; + + if(!stream.fail()) + { + if(_allowed_values.count(typed_value) == 0) + { + return false; + } + + this->_value = std::move(typed_value); + this->_is_set = true; + return true; + } + + return false; + } + catch(const std::invalid_argument &) + { + return false; + } +} + +template +std::string EnumOption::help() const +{ + std::stringstream msg; + msg << "--" + this->name() + "={"; + + for(const auto &value : _allowed_values) + { + msg << value << ","; + } + + msg << "} - " << this->_help; + + return msg.str(); +} + +template +inline const T &EnumOption::value() const +{ + return this->_value; +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_ENUMOPTION */ diff --git a/tests/framework/command_line/ListOption.h b/tests/framework/command_line/ListOption.h new file mode 100644 index 0000000000..8b1bb3d05a --- /dev/null +++ b/tests/framework/command_line/ListOption.h @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_LISTOPTION +#define ARM_COMPUTE_TEST_LISTOPTION + +#include "Option.h" + +#include +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Implementation of an option that accepts any number of values. */ +template +class ListOption : public Option +{ +public: + using Option::Option; + + /** Construct the option with the given default values. + * + * @param[in] name Name of the option. + * @param[in] default_values Default values. + */ + ListOption(std::string name, std::initializer_list &&default_values); + + bool parse(std::string value) override; + std::string help() const override; + const std::vector &value() const; + +private: + std::vector _values{}; +}; + +template +inline ListOption::ListOption(std::string name, std::initializer_list &&default_values) + : Option{ std::move(name), false, true }, _values{ std::forward>(default_values) } +{ +} + +template +bool ListOption::parse(std::string value) +{ + _is_set = true; + + try + { + std::stringstream stream{ value }; + std::string item; + + while(!std::getline(stream, item, ',').fail()) + { + std::stringstream item_stream(item); + T typed_value{}; + + item_stream >> typed_value; + + if(!item_stream.fail()) + { + _values.emplace_back(typed_value); + } + + _is_set = _is_set && !item_stream.fail(); + } + + return _is_set; + } + catch(const std::invalid_argument &) + { + return false; + } +} + +template +inline std::string ListOption::help() const +{ + return "--" + name() + "=VALUE[,VALUE[,...]] - " + _help; +} + +template +inline const std::vector &ListOption::value() const +{ + return _values; +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_LISTOPTION */ diff --git a/tests/framework/command_line/Option.cpp b/tests/framework/command_line/Option.cpp new file mode 100644 index 0000000000..d60c35a698 --- /dev/null +++ b/tests/framework/command_line/Option.cpp @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "Option.h" + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +Option::Option(std::string name) + : _name{ std::move(name) } +{ +} + +Option::Option(std::string name, bool is_required, bool is_set) + : _name{ std::move(name) }, _is_required{ is_required }, _is_set{ is_set } +{ +} + +std::string Option::name() const +{ + return _name; +} + +void Option::set_required(bool is_required) +{ + _is_required = is_required; +} + +void Option::set_help(std::string help) +{ + _help = std::move(help); +} + +bool Option::is_required() const +{ + return _is_required; +} + +bool Option::is_set() const +{ + return _is_set; +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/command_line/Option.h b/tests/framework/command_line/Option.h new file mode 100644 index 0000000000..25cf492b86 --- /dev/null +++ b/tests/framework/command_line/Option.h @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_OPTIONBASE +#define ARM_COMPUTE_TEST_OPTIONBASE + +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Abstract base class for a command line option. */ +class Option +{ +public: + /** Constructor. + * + * @param[in] name Name of the option. + */ + Option(std::string name); + + /** Constructor. + * + * @param[in] name Name of the option. + * @param[in] is_required Is the option required? + * @param[in] is_set Has a value been assigned to the option? + */ + Option(std::string name, bool is_required, bool is_set); + + /** Default destructor. */ + virtual ~Option() = default; + + /** Parses the given string. + * + * @param[in] value String representation as passed on the command line. + * + * @return True if the value could be parsed by the specific subclass. + */ + virtual bool parse(std::string value) = 0; + + /** Help message for the option. + * + * @return String representing the help message for the specific subclass. + */ + virtual std::string help() const = 0; + + /** Name of the option. + * + * @return Name of the option. + */ + std::string name() const; + + /** Set whether the option is required. + * + * @param[in] is_required Pass true if the option is required. + */ + void set_required(bool is_required); + + /** Set the help message for the option. + * + * @param[in] help Option specific help message. + */ + void set_help(std::string help); + + /** Is the option required? + * + * @return True if the option is required. + */ + bool is_required() const; + + /** Has a value been assigned to the option? + * + * @return True if a value has been set. + */ + bool is_set() const; + +protected: + std::string _name; + bool _is_required{ false }; + bool _is_set{ false }; + std::string _help{}; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_OPTIONBASE */ diff --git a/tests/framework/command_line/SimpleOption.h b/tests/framework/command_line/SimpleOption.h new file mode 100644 index 0000000000..e6e8045840 --- /dev/null +++ b/tests/framework/command_line/SimpleOption.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_SIMPLEOPTION +#define ARM_COMPUTE_TEST_SIMPLEOPTION + +#include "Option.h" + +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Implementation of an option that accepts a single value. */ +template +class SimpleOption : public Option +{ +public: + using Option::Option; + + /** Construct the option with the given default value. + * + * @param[in] name Name of the option. + * @param[in] default_value Default value. + */ + SimpleOption(std::string name, T default_value); + + bool parse(std::string value) override; + std::string help() const override; + const T &value() const; + +protected: + T _value{}; +}; + +template +inline SimpleOption::SimpleOption(std::string name, T default_value) + : Option{ std::move(name), false, true }, _value{ std::move(default_value) } +{ +} + +template +bool SimpleOption::parse(std::string value) +{ + try + { + std::stringstream stream{ std::move(value) }; + stream >> _value; + _is_set = !stream.fail(); + return _is_set; + } + catch(const std::invalid_argument &) + { + return false; + } +} + +template <> +inline bool SimpleOption::parse(std::string value) +{ + _value = std::move(value); + _is_set = true; + return true; +} + +template +inline std::string SimpleOption::help() const +{ + return "--" + name() + "=VALUE - " + _help; +} + +template +inline const T &SimpleOption::value() const +{ + return _value; +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_SIMPLEOPTION */ diff --git a/tests/framework/command_line/ToggleOption.cpp b/tests/framework/command_line/ToggleOption.cpp new file mode 100644 index 0000000000..df5b1f813b --- /dev/null +++ b/tests/framework/command_line/ToggleOption.cpp @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "ToggleOption.h" + +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +ToggleOption::ToggleOption(std::string name, bool default_value) + : SimpleOption +{ + std::move(name), default_value +} +{ +} + +bool ToggleOption::parse(std::string value) +{ + if(value == "true") + { + _value = true; + _is_set = true; + } + else if(value == "false") + { + _value = false; + _is_set = true; + } + + return _is_set; +} + +std::string ToggleOption::help() const +{ + return "--" + name() + ", --no-" + name() + " - " + _help; +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/command_line/ToggleOption.h b/tests/framework/command_line/ToggleOption.h new file mode 100644 index 0000000000..c440c0ee87 --- /dev/null +++ b/tests/framework/command_line/ToggleOption.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_TOGGLEOPTION +#define ARM_COMPUTE_TEST_TOGGLEOPTION + +#include "SimpleOption.h" + +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Implementation of an option that can be either true or false. */ +class ToggleOption : public SimpleOption +{ +public: + using SimpleOption::SimpleOption; + + /** Construct the option with the given default value. + * + * @param[in] name Name of the option. + * @param[in] default_value Default value. + */ + ToggleOption(std::string name, bool default_value); + + bool parse(std::string value) override; + std::string help() const override; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_TOGGLEOPTION */ diff --git a/tests/framework/datasets/CartesianProductDataset.h b/tests/framework/datasets/CartesianProductDataset.h new file mode 100644 index 0000000000..f6e45ddc12 --- /dev/null +++ b/tests/framework/datasets/CartesianProductDataset.h @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_DATASET_CARTESIAN_PRODUCT +#define ARM_COMPUTE_TEST_DATASET_CARTESIAN_PRODUCT + +#include "Dataset.h" + +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +namespace dataset +{ +/** Implementation of a dataset representing all combinations of values of the input datasets. + * + * For example, for the inputs {1, 2} and {3, 4} this dataset virtually + * represents the values {(1, 3), (1, 4), (2, 3), (2, 4)}. + */ +template +class CartesianProductDataset : public Dataset +{ +private: + using T_noref = typename std::remove_reference::type; + using U_noref = typename std::remove_reference::type; + using iter1_type = typename T_noref::iterator; + using iter2_type = typename U_noref::iterator; + +public: + /** Construct dataset from the given datasets. + * + * @param[in] dataset1 First dataset. + * @param[in] dataset2 Second dataset. + */ + CartesianProductDataset(T &&dataset1, U &&dataset2) + : _dataset1{ std::forward(dataset1) }, + _dataset2{ std::forward(dataset2) } + { + } + + CartesianProductDataset(CartesianProductDataset &&) = default; + + /** Type of the dataset. */ + using type = decltype(std::tuple_cat(*std::declval(), *std::declval())); + + /** Iterator for the dataset. */ + struct iterator + { + iterator(const T_noref *dataset1, const U_noref *dataset2) + : _iter1{ dataset1->begin() }, + _dataset2{ dataset2 }, + _iter2{ dataset2->begin() } + { + } + + iterator(const iterator &) = default; + iterator &operator=(const iterator &) = default; + iterator(iterator &&) = default; + iterator &operator=(iterator &&) = default; + + ~iterator() = default; + + std::string description() const + { + return _iter1.description() + ":" + _iter2.description(); + } + + CartesianProductDataset::type operator*() const + { + return std::tuple_cat(*_iter1, *_iter2); + } + + iterator &operator++() + { + ++_second_pos; + + if(_second_pos < _dataset2->size()) + { + ++_iter2; + } + else + { + _second_pos = 0; + _iter2 = _dataset2->begin(); + + ++_iter1; + } + + return *this; + } + + private: + iter1_type _iter1; + const U_noref *_dataset2; + iter2_type _iter2; + int _first_pos{ 0 }; + int _second_pos{ 0 }; + }; + + /** Iterator pointing at the begin of the dataset. + * + * @return Iterator for the dataset. + */ + iterator begin() const + { + return iterator(&_dataset1, &_dataset2); + } + + /** Size of the dataset. + * + * @return Number of values in the dataset. + */ + int size() const + { + return _dataset1.size() * _dataset2.size(); + } + +private: + T _dataset1; + U _dataset2; +}; + +/** Helper function to create a @ref CartesianProductDataset. + * + * @param[in] dataset1 First dataset. + * @param[in] dataset2 Second dataset. + * + * @return A grid dataset. + */ +template +CartesianProductDataset combine(T &&dataset1, U &&dataset2) +{ + return CartesianProductDataset(std::forward(dataset1), std::forward(dataset2)); +} +} // namespace dataset +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_DATASET_CARTESIAN_PRODUCT */ diff --git a/tests/framework/datasets/ContainerDataset.h b/tests/framework/datasets/ContainerDataset.h new file mode 100644 index 0000000000..bdca97cbac --- /dev/null +++ b/tests/framework/datasets/ContainerDataset.h @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_DATASET_CONTAINER +#define ARM_COMPUTE_TEST_DATASET_CONTAINER + +#include "Dataset.h" +#include "support/ToolchainSupport.h" + +#include +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +namespace dataset +{ +/** Base case. Nothing is a container. */ +template +struct is_container : public std::false_type +{ +}; + +/** Vector is considered a container. */ +template +struct is_container> : public std::true_type +{ +}; + +/** Implementation of a dataset created from a container. */ +template +class ContainerDataset : public NamedDataset +{ +private: + using container_value_type = typename T::value_type; + using container_const_iterator = typename T::const_iterator; + +public: + /** Construct dataset with given name and values from the container. + * + * @param[in] name Description of the values. + * @param[in] container Values for the dataset. + */ + ContainerDataset(std::string name, T &&container) + : NamedDataset{ std::move(name) }, _container(std::forward(container)) + { + } + + ContainerDataset(ContainerDataset &&) = default; + + /** Type of the dataset. */ + using type = std::tuple; + + /** Iterator for the dataset. */ + struct iterator + { + iterator(std::string name, container_const_iterator iterator) + : _name{ name }, _iterator{ iterator } + { + } + + std::string description() const + { + using support::cpp11::to_string; + return _name + "=" + to_string(*_iterator); + } + + ContainerDataset::type operator*() const + { + return std::make_tuple(*_iterator); + } + + iterator &operator++() + { + ++_iterator; + return *this; + } + + private: + std::string _name; + container_const_iterator _iterator; + }; + + /** Iterator pointing at the begin of the dataset. + * + * @return Iterator for the dataset. + */ + iterator begin() const + { + return iterator(name(), _container.cbegin()); + } + + /** Size of the dataset. + * + * @return Number of values in the dataset. + */ + int size() const + { + return _container.size(); + } + +private: + T _container; +}; + +/** Helper function to create a @ref ContainerDataset. + * + * @param[in] name Name of the dataset. + * @param[in] values Container. + * + * @return A container dataset. + */ +template +typename std::enable_if::value, ContainerDataset>::type make(std::string name, T &&values) +{ + return ContainerDataset(std::move(name), std::forward(values)); +} +} // namespace dataset +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_DATASET_CONTAINER */ diff --git a/tests/framework/datasets/Dataset.h b/tests/framework/datasets/Dataset.h new file mode 100644 index 0000000000..d91673073a --- /dev/null +++ b/tests/framework/datasets/Dataset.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_DATASET +#define ARM_COMPUTE_TEST_DATASET + +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +namespace dataset +{ +/** Abstract dataset base class. */ +class Dataset +{ +protected: + Dataset() = default; + ~Dataset() = default; + +public: + Dataset(Dataset &&) = default; +}; + +/** Abstract implementation of a named dataset. + * + * The name should describe the values of the dataset. + */ +class NamedDataset : public Dataset +{ +protected: + /** Construct the dataset with the given name. + * + * @param[in] name Description of the values. + */ + explicit NamedDataset(std::string name) + : _name{ std::move(name) } + { + } + + ~NamedDataset() = default; + +public: + NamedDataset(NamedDataset &&) = default; + + /** Return name of the dataset. + * + * @return Description of the values. + */ + std::string name() const + { + return _name; + } + +protected: + const std::string _name; +}; +} // namespace dataset +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_DATASET */ diff --git a/tests/framework/datasets/Datasets.h b/tests/framework/datasets/Datasets.h new file mode 100644 index 0000000000..c0e5822e17 --- /dev/null +++ b/tests/framework/datasets/Datasets.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_DATASETS +#define ARM_COMPUTE_TEST_DATASETS + +#include "CartesianProductDataset.h" +#include "ContainerDataset.h" +#include "InitializerListDataset.h" +#include "JoinDataset.h" +#include "RangeDataset.h" +#include "SingletonDataset.h" +#include "ZipDataset.h" + +#endif /* ARM_COMPUTE_TEST_DATASETS */ diff --git a/tests/framework/datasets/InitializerListDataset.h b/tests/framework/datasets/InitializerListDataset.h new file mode 100644 index 0000000000..7d32234fab --- /dev/null +++ b/tests/framework/datasets/InitializerListDataset.h @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_DATASET_LIST +#define ARM_COMPUTE_TEST_DATASET_LIST + +#include "Dataset.h" +#include "support/ToolchainSupport.h" + +#include +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +namespace dataset +{ +/** Implementation of a dataset created from an initializer list. */ +template +class InitializerListDataset final : public NamedDataset +{ +private: + using data_const_iterator = typename std::vector::const_iterator; + +public: + /** Construct dataset with given name and values from the container. + * + * @param[in] name Description of the values. + * @param[in] list Values for the dataset. + */ + InitializerListDataset(std::string name, std::initializer_list &&list) + : NamedDataset{ std::move(name) }, _data(std::forward>(list)) + { + } + + InitializerListDataset(InitializerListDataset &&) = default; + + /** Type of the dataset. */ + using type = std::tuple; + + /** Iterator for the dataset. */ + struct iterator + { + iterator(std::string name, data_const_iterator iterator) + : _name{ name }, _iterator{ iterator } + { + } + + std::string description() const + { + using support::cpp11::to_string; + return _name + "=" + to_string(*_iterator); + } + + InitializerListDataset::type operator*() const + { + return std::make_tuple(*_iterator); + } + + iterator &operator++() + { + ++_iterator; + return *this; + } + + private: + std::string _name; + data_const_iterator _iterator; + }; + + /** Iterator pointing at the begin of the dataset. + * + * @return Iterator for the dataset. + */ + iterator begin() const + { + return iterator(name(), _data.cbegin()); + } + + /** Size of the dataset. + * + * @return Number of values in the dataset. + */ + int size() const + { + return _data.size(); + } + +private: + std::vector _data; +}; + +/** Helper function to create an @ref InitializerListDataset. + * + * @param[in] name Name of the dataset. + * @param[in] list Initializer list. + * + * @return An initializer list dataset. + */ +template +InitializerListDataset make(std::string name, std::initializer_list &&list) +{ + return InitializerListDataset(std::move(name), std::forward>(list)); +} +} // namespace dataset +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_DATASET_LIST */ diff --git a/tests/framework/datasets/JoinDataset.h b/tests/framework/datasets/JoinDataset.h new file mode 100644 index 0000000000..eded6e0259 --- /dev/null +++ b/tests/framework/datasets/JoinDataset.h @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_DATASET_JOIN +#define ARM_COMPUTE_TEST_DATASET_JOIN + +#include "Dataset.h" + +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +namespace dataset +{ +/** Implementation of a dataset representing the concatenation of the input datasets. + * + * For example, for the inputs {1, 2} and {3, 4} this dataset virtually + * represents the values {1, 2, 3, 4}. + */ +template +class JoinDataset : public Dataset +{ +private: + using iter1_type = typename T::iterator; + using iter2_type = typename U::iterator; + +public: + /** Construct dataset from the given datasets. + * + * @param[in] dataset1 First dataset. + * @param[in] dataset2 Second dataset. + */ + JoinDataset(T &&dataset1, U &&dataset2) + : _dataset1{ std::forward(dataset1) }, + _dataset2{ std::forward(dataset2) } + { + } + + JoinDataset(JoinDataset &&) = default; + + /** Type of the dataset. */ + using type = typename T::type; + + /** Iterator for the dataset. */ + struct iterator + { + iterator(const T *dataset1, const U *dataset2) + : _iter1{ dataset1->begin() }, _iter2{ dataset2->begin() }, _first_size{ dataset1->size() } + { + } + + std::string description() const + { + return _first_size > 0 ? _iter1.description() : _iter2.description(); + } + + JoinDataset::type operator*() const + { + return _first_size > 0 ? *_iter1 : *_iter2; + } + + iterator &operator++() + { + if(_first_size > 0) + { + --_first_size; + ++_iter1; + } + else + { + ++_iter2; + } + + return *this; + } + + private: + iter1_type _iter1; + iter2_type _iter2; + int _first_size; + }; + + /** Iterator pointing at the begin of the dataset. + * + * @return Iterator for the dataset. + */ + iterator begin() const + { + return iterator(&_dataset1, &_dataset2); + } + + /** Size of the dataset. + * + * @return Number of values in the dataset. + */ + int size() const + { + return _dataset1.size() + _dataset2.size(); + } + +private: + T _dataset1; + U _dataset2; +}; + +/** Helper function to create a @ref JoinDataset. + * + * @param[in] dataset1 First dataset. + * @param[in] dataset2 Second dataset. + * + * @return A join dataset. + */ +template +JoinDataset concat(T &&dataset1, U &&dataset2) +{ + return JoinDataset(std::forward(dataset1), std::forward(dataset2)); +} +} // namespace dataset +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_DATASET_JOIN */ diff --git a/tests/framework/datasets/RangeDataset.h b/tests/framework/datasets/RangeDataset.h new file mode 100644 index 0000000000..637abe0a83 --- /dev/null +++ b/tests/framework/datasets/RangeDataset.h @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_DATASET_RANGE +#define ARM_COMPUTE_TEST_DATASET_RANGE + +#include "Dataset.h" +#include "support/ToolchainSupport.h" + +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +namespace dataset +{ +/** Implementation of a dataset created from a range of values. + * + * The range is inclusive of the first value but exclusive of the last, i.e. [start, end). + */ +template +class RangeDataset final : public NamedDataset +{ +public: + /** Construct dataset with given name and values in the specified range. + * + * @param[in] name Description of the values. + * @param[in] start Begin of the range. + * @param[in] end End of the range. + * @param[in] step Step size. + */ + RangeDataset(std::string name, T start, T end, T step = 1) + : NamedDataset{ std::move(name) }, _start{ start }, _end{ end }, _step{ step } + { + } + + RangeDataset(RangeDataset &&) = default; + + /** Type of the dataset. */ + using type = std::tuple; + + /** Iterator for the dataset. */ + struct iterator + { + iterator(std::string name, T start, T step) + : _name{ name }, _value{ start }, _step{ step } + { + } + + std::string description() const + { + using support::cpp11::to_string; + return _name + "=" + to_string(_value); + } + + RangeDataset::type operator*() const + { + return std::make_tuple(_value); + } + + iterator &operator++() + { + _value += _step; + return *this; + } + + private: + std::string _name; + T _value; + T _step; + }; + + /** Iterator pointing at the begin of the dataset. + * + * @return Iterator for the dataset. + */ + iterator begin() const + { + return iterator(name(), _start, _step); + } + + /** Size of the dataset. + * + * @return Number of values in the dataset. + */ + int size() const + { + return (_end - _start) / std::abs(_step); + } + +private: + T _start; + T _end; + T _step; +}; + +/** Helper function to create a @ref RangeDataset. + * + * @param[in] name Name of the dataset. + * @param[in] start Begin of the range. + * @param[in] end End of the range. + * @param[in] step Step size. + * + * @return A range dataset. + */ +template +RangeDataset make(std::string name, T start, T end, T step = 1) +{ + return RangeDataset(std::move(name), start, end, step); +} +} // namespace dataset +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_DATASET_RANGE */ diff --git a/tests/framework/datasets/SingletonDataset.h b/tests/framework/datasets/SingletonDataset.h new file mode 100644 index 0000000000..1acb5765e5 --- /dev/null +++ b/tests/framework/datasets/SingletonDataset.h @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_DATASET_SINGLETON +#define ARM_COMPUTE_TEST_DATASET_SINGLETON + +#include "ContainerDataset.h" +#include "Dataset.h" +#include "support/ToolchainSupport.h" + +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +namespace dataset +{ +/** Implementation of a dataset holding a single value. */ +template +class SingletonDataset : public NamedDataset +{ +public: + /** Construct dataset with given name and value. + * + * @param[in] name Description of the value. + * @param[in] value Value for the dataset. + */ + SingletonDataset(std::string name, T &&value) + : NamedDataset{ std::move(name) }, _value{ std::forward(value) } + { + } + + SingletonDataset(SingletonDataset &&) = default; + + /** Type of the dataset. */ + using type = std::tuple; + + /** Iterator for the dataset. */ + struct iterator + { + iterator(std::string name, const T *value) + : _name{ name }, _value{ value } + { + } + + ~iterator() = default; + + iterator(const iterator &) = default; + iterator &operator=(const iterator &) = default; + iterator(iterator &&) = default; + iterator &operator=(iterator &&) = default; + + std::string description() const + { + using support::cpp11::to_string; + return _name + "=" + to_string(*_value); + } + + SingletonDataset::type operator*() const + { + return std::make_tuple(*_value); + } + + iterator &operator++() + { + return *this; + } + + private: + std::string _name; + const T *_value; + }; + + /** Iterator pointing at the begin of the dataset. + * + * @return Iterator for the dataset. + */ + iterator begin() const + { + return iterator(name(), &_value); + } + + /** Size of the dataset. + * + * @return Number of values in the dataset. + */ + int size() const + { + return 1; + } + +private: + T _value; +}; + +/** Helper function to create a @ref SingletonDataset. + * + * @param[in] name Name of the dataset. + * @param[in] value Value. + * + * @return A singleton dataset. + */ +template +typename std::enable_if < !is_container::value, SingletonDataset>::type make(std::string name, T &&value) +{ + return SingletonDataset(std::move(name), std::forward(value)); +} +} // namespace dataset +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_DATASET_SINGLETON */ diff --git a/tests/framework/datasets/ZipDataset.h b/tests/framework/datasets/ZipDataset.h new file mode 100644 index 0000000000..b95b7209a7 --- /dev/null +++ b/tests/framework/datasets/ZipDataset.h @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_DATASET_ZIP +#define ARM_COMPUTE_TEST_DATASET_ZIP + +#include "Dataset.h" + +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +namespace dataset +{ +/** Implementation of a dataset representing pairs of values of the input datasets. + * + * For example, for the inputs {1, 2} and {3, 4} this dataset virtually + * represents the values {(1, 3), (1, 4)}. + */ +template +class ZipDataset : public Dataset +{ +private: + using iter1_type = typename T::iterator; + using iter2_type = typename U::iterator; + +public: + /** Construct dataset from the given datasets. + * + * @param[in] dataset1 First dataset. + * @param[in] dataset2 Second dataset. + */ + ZipDataset(T &&dataset1, U &&dataset2) + : _dataset1{ std::forward(dataset1) }, + _dataset2{ std::forward(dataset2) } + { + } + + ZipDataset(ZipDataset &&) = default; + + /** Type of the dataset. */ + using type = decltype(std::tuple_cat(*std::declval(), *std::declval())); + + /** Iterator for the dataset. */ + struct iterator + { + iterator(iter1_type iter1, iter2_type iter2) + : _iter1{ std::move(iter1) }, _iter2{ std::move(iter2) } + { + } + + std::string description() const + { + return _iter1.description() + ":" + _iter2.description(); + } + + ZipDataset::type operator*() const + { + return std::tuple_cat(*_iter1, *_iter2); + } + + iterator &operator++() + { + ++_iter1; + ++_iter2; + return *this; + } + + private: + iter1_type _iter1; + iter2_type _iter2; + }; + + /** Iterator pointing at the begin of the dataset. + * + * @return Iterator for the dataset. + */ + iterator begin() const + { + return iterator(_dataset1.begin(), _dataset2.begin()); + } + + /** Size of the dataset. + * + * @return Number of values in the dataset. + */ + int size() const + { + return std::min(_dataset1.size(), _dataset2.size()); + } + +private: + T _dataset1; + U _dataset2; +}; + +/** Helper function to create a @ref ZipDataset. + * + * @param[in] dataset1 First dataset. + * @param[in] dataset2 Second dataset. + * + * @return A zip dataset. + */ +template +ZipDataset zip(T &&dataset1, U &&dataset2) +{ + return ZipDataset(std::forward(dataset1), std::forward(dataset2)); +} +} // namespace dataset +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_DATASET_ZIP */ diff --git a/tests/framework/instruments/Instrument.h b/tests/framework/instruments/Instrument.h new file mode 100644 index 0000000000..895a64738c --- /dev/null +++ b/tests/framework/instruments/Instrument.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_INSTRUMENT +#define ARM_COMPUTE_TEST_INSTRUMENT + +#include "../Utils.h" + +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Interface for classes that can be used to measure performance. */ +class Instrument +{ +public: + /** Helper function to create an instrument of the given type. + * + * @return Instance of an instrument of the given type. + */ + template + static std::unique_ptr make_instrument(); + + /** Struct representing measurement consisting of value and unit. */ + struct Measurement final + { + Measurement(double value, std::string unit) + : value{ value }, unit{ std::move(unit) } + { + } + + friend std::ostream &operator<<(std::ostream &os, const Measurement &measurement); + + double value; + std::string unit; + }; + + Instrument() = default; + Instrument(const Instrument &) = default; + Instrument(Instrument &&) = default; + Instrument &operator=(const Instrument &) = default; + Instrument &operator=(Instrument &&) = default; + virtual ~Instrument() = default; + + /** Identifier for the instrument */ + virtual std::string id() const = 0; + + /** Start measuring. */ + virtual void start() = 0; + + /** Stop measuring. */ + virtual void stop() = 0; + + /** Return the latest measurement. */ + virtual Measurement measurement() const = 0; +}; + +inline std::ostream &operator<<(std::ostream &os, const Instrument::Measurement &measurement) +{ + os << measurement.value << measurement.unit; + return os; +} + +template +inline std::unique_ptr Instrument::make_instrument() +{ + return support::cpp14::make_unique(); +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_INSTRUMENT */ diff --git a/tests/framework/instruments/Instruments.cpp b/tests/framework/instruments/Instruments.cpp new file mode 100644 index 0000000000..797a7242ae --- /dev/null +++ b/tests/framework/instruments/Instruments.cpp @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "Instruments.h" + +#include "../Utils.h" + +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +InstrumentType instrument_type_from_name(const std::string &name) +{ + static const std::map types = + { + { "all", InstrumentType::ALL }, + { "none", InstrumentType::NONE }, + { "wall_clock", InstrumentType::WALL_CLOCK_TIMER }, + { "cycles", InstrumentType::PMU_CYCLE_COUNTER }, + { "instructions", InstrumentType::PMU_INSTRUCTION_COUNTER }, + }; + + try + { + return types.at(tolower(name)); + } + catch(const std::out_of_range &) + { + throw std::invalid_argument(name); + } +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/instruments/Instruments.h b/tests/framework/instruments/Instruments.h new file mode 100644 index 0000000000..034fa168f5 --- /dev/null +++ b/tests/framework/instruments/Instruments.h @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_INSTRUMENTS +#define ARM_COMPUTE_TEST_INSTRUMENTS + +#include "PMUCounter.h" +#include "WallClockTimer.h" + +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +enum class InstrumentType : unsigned int +{ + ALL = ~0U, + NONE = 0, + WALL_CLOCK_TIMER = 1, + PMU_CYCLE_COUNTER = 2, + PMU_INSTRUCTION_COUNTER = 4 +}; + +InstrumentType instrument_type_from_name(const std::string &name); + +inline InstrumentType operator&(InstrumentType t1, InstrumentType t2) +{ + using type = std::underlying_type::type; + return static_cast(static_cast(t1) & static_cast(t2)); +} + +inline InstrumentType operator|(InstrumentType t1, InstrumentType t2) +{ + using type = std::underlying_type::type; + return static_cast(static_cast(t1) | static_cast(t2)); +} + +inline InstrumentType &operator|=(InstrumentType &t1, InstrumentType t2) +{ + using type = std::underlying_type::type; + t1 = static_cast(static_cast(t1) | static_cast(t2)); + return t1; +} + +inline ::std::stringstream &operator>>(::std::stringstream &stream, InstrumentType &instrument) +{ + std::string value; + stream >> value; + instrument = instrument_type_from_name(value); + return stream; +} + +inline ::std::stringstream &operator<<(::std::stringstream &stream, InstrumentType instrument) +{ + switch(instrument) + { + case InstrumentType::WALL_CLOCK_TIMER: + stream << "WALL_CLOCK_TIMER"; + break; + case InstrumentType::PMU_CYCLE_COUNTER: + stream << "PMU_CYCLE_COUNTER"; + break; + case InstrumentType::PMU_INSTRUCTION_COUNTER: + stream << "PMU_INSTRUCTION_COUNTER"; + break; + case InstrumentType::ALL: + stream << "ALL"; + break; + case InstrumentType::NONE: + stream << "NONE"; + break; + default: + throw std::invalid_argument("Unsupported instrument type"); + } + + return stream; +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_INSTRUMENTS */ diff --git a/tests/framework/instruments/PMUCounter.cpp b/tests/framework/instruments/PMUCounter.cpp new file mode 100644 index 0000000000..7994a15862 --- /dev/null +++ b/tests/framework/instruments/PMUCounter.cpp @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "PMUCounter.h" + +#define _GNU_SOURCE 1 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#undef _GNU_SOURCE + +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +CycleCounter::CycleCounter() +{ + const pid_t pid = getpid(); + + struct perf_event_attr perf_config; //NOLINT + memset(&perf_config, 0, sizeof(struct perf_event_attr)); + + perf_config.config = PERF_COUNT_HW_CPU_CYCLES; + perf_config.size = sizeof(struct perf_event_attr); + perf_config.type = PERF_TYPE_HARDWARE; + // The inherit bit specifies that this counter should count events of child + // tasks as well as the task specified + perf_config.inherit = 1; + // Enables saving of event counts on context switch for inherited tasks + perf_config.inherit_stat = 1; + + _fd = syscall(__NR_perf_event_open, &perf_config, pid, -1, -1, 0); + + if(_fd < 0) + { + throw std::runtime_error("perf_event_open for cycles failed"); + } +} + +std::string CycleCounter::id() const +{ + return "Cycle Counter"; +} + +void CycleCounter::start() +{ + ioctl(_fd, PERF_EVENT_IOC_RESET, 0); + ioctl(_fd, PERF_EVENT_IOC_ENABLE, 0); +} + +void CycleCounter::stop() +{ + ioctl(_fd, PERF_EVENT_IOC_DISABLE, 0); + read(_fd, &_cycles, sizeof(_cycles)); +} + +Instrument::Measurement CycleCounter::measurement() const +{ + return Measurement(_cycles, "cycles"); +} + +InstructionCounter::InstructionCounter() +{ + const pid_t pid = getpid(); + + struct perf_event_attr perf_config; //NOLINT + memset(&perf_config, 0, sizeof(struct perf_event_attr)); + + perf_config.config = PERF_COUNT_HW_INSTRUCTIONS; + perf_config.size = sizeof(struct perf_event_attr); + perf_config.type = PERF_TYPE_HARDWARE; + // The inherit bit specifies that this counter should count events of child + // tasks as well as the task specified + perf_config.inherit = 1; + // Enables saving of event counts on context switch for inherited tasks + perf_config.inherit_stat = 1; + + _fd = syscall(__NR_perf_event_open, &perf_config, pid, -1, -1, 0); + + if(_fd < 0) + { + throw std::runtime_error("perf_event_open for instructions failed"); + } +} + +std::string InstructionCounter::id() const +{ + return "Instruction Counter"; +} + +void InstructionCounter::start() +{ + ioctl(_fd, PERF_EVENT_IOC_RESET, 0); + ioctl(_fd, PERF_EVENT_IOC_ENABLE, 0); +} + +void InstructionCounter::stop() +{ + ioctl(_fd, PERF_EVENT_IOC_DISABLE, 0); + read(_fd, &_instructions, sizeof(_instructions)); +} + +Instrument::Measurement InstructionCounter::measurement() const +{ + return Measurement(_instructions, "instructions"); +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/instruments/PMUCounter.h b/tests/framework/instruments/PMUCounter.h new file mode 100644 index 0000000000..f407be602f --- /dev/null +++ b/tests/framework/instruments/PMUCounter.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_PMU_COUNTER +#define ARM_COMPUTE_TEST_PMU_COUNTER + +#include "Instrument.h" + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Implementation of an instrument to count CPU cycles. */ +class CycleCounter : public Instrument +{ +public: + /** Initialise the cycle counter. */ + CycleCounter(); + + std::string id() const override; + void start() override; + void stop() override; + Measurement measurement() const override; + +private: + long _fd{ -1 }; + long long _cycles{ 0 }; +}; + +/** Implementation of an instrument to count executed CPU instructions. */ +class InstructionCounter : public Instrument +{ +public: + /** Initialise the instruction counter. */ + InstructionCounter(); + + std::string id() const override; + void start() override; + void stop() override; + Measurement measurement() const override; + +private: + long _fd{ -1 }; + long long _instructions{ 0 }; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_PMU_COUNTER */ diff --git a/tests/framework/instruments/WallClockTimer.cpp b/tests/framework/instruments/WallClockTimer.cpp new file mode 100644 index 0000000000..37db0c7f05 --- /dev/null +++ b/tests/framework/instruments/WallClockTimer.cpp @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "WallClockTimer.h" + +#include "../Framework.h" +#include "../Utils.h" + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +std::string WallClockTimer::id() const +{ + return "Wall clock"; +} + +void WallClockTimer::start() +{ + _start = std::chrono::high_resolution_clock::now(); +} + +void WallClockTimer::stop() +{ + _stop = std::chrono::high_resolution_clock::now(); +} + +Instrument::Measurement WallClockTimer::measurement() const +{ + const auto delta = std::chrono::duration_cast(_stop - _start); + return Instrument::Measurement(delta.count(), "us"); +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/instruments/WallClockTimer.h b/tests/framework/instruments/WallClockTimer.h new file mode 100644 index 0000000000..b7c390f691 --- /dev/null +++ b/tests/framework/instruments/WallClockTimer.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_WALL_CLOCK_TIMER +#define ARM_COMPUTE_TEST_WALL_CLOCK_TIMER + +#include "Instrument.h" + +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Implementation of an instrument to measure elapsed wall-clock time in milliseconds. */ +class WallClockTimer : public Instrument +{ +public: + std::string id() const override; + void start() override; + void stop() override; + Measurement measurement() const override; + +private: + std::chrono::high_resolution_clock::time_point _start{}; + std::chrono::high_resolution_clock::time_point _stop{}; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_WALL_CLOCK_TIMER */ diff --git a/tests/framework/printers/JSONPrinter.cpp b/tests/framework/printers/JSONPrinter.cpp new file mode 100644 index 0000000000..5b30389eca --- /dev/null +++ b/tests/framework/printers/JSONPrinter.cpp @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "JSONPrinter.h" + +#include "tests/framework/Framework.h" + +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +void JSONPrinter::print_separator(bool &flag) +{ + if(flag) + { + flag = false; + } + else + { + *_stream << ","; + } +} + +void JSONPrinter::print_entry(const std::string &name, const std::string &value) +{ + print_separator(_first_entry); + + *_stream << R"(")" << name << R"(" : ")" << value << R"(")"; +} + +void JSONPrinter::print_global_header() +{ + *_stream << "{"; +} + +void JSONPrinter::print_global_footer() +{ + *_stream << "}\n"; +} + +void JSONPrinter::print_run_header() +{ + print_separator(_first_entry); + + *_stream << R"("tests" : {)"; +} + +void JSONPrinter::print_run_footer() +{ + *_stream << "}"; +} + +void JSONPrinter::print_test_header(const TestInfo &info) +{ + print_separator(_first_test); + + _first_test_entry = true; + *_stream << R"(")" << info.name << R"(" : {)"; +} + +void JSONPrinter::print_test_footer() +{ + *_stream << "}"; +} + +void JSONPrinter::print_errors_header() +{ + print_separator(_first_test_entry); + + _first_error = true; + *_stream << R"("errors" : [)"; +} + +void JSONPrinter::print_errors_footer() +{ + *_stream << "]"; +} + +void JSONPrinter::print_error(const std::exception &error) +{ + std::stringstream error_log; + error_log.str(error.what()); + + for(std::string line; !std::getline(error_log, line).fail();) + { + print_separator(_first_error); + + *_stream << R"(")" << line << R"(")"; + } +} + +void JSONPrinter::print_measurements(const Profiler::MeasurementsMap &measurements) +{ + print_separator(_first_test_entry); + + *_stream << R"("measurements" : {)"; + + for(auto i_it = measurements.cbegin(), i_end = measurements.cend(); i_it != i_end;) + { + *_stream << R"(")" << i_it->first << R"(" : {)"; + + auto add_measurements = [](double a, const Instrument::Measurement & b) + { + return a + b.value; + }; + + auto cmp_measurements = [](const Instrument::Measurement & a, const Instrument::Measurement & b) + { + return a.value < b.value; + }; + + double sum_values = std::accumulate(i_it->second.cbegin(), i_it->second.cend(), 0., add_measurements); + int num_values = i_it->second.size(); + const auto minmax_values = std::minmax_element(i_it->second.begin(), i_it->second.end(), cmp_measurements); + + if(num_values > 2) + { + sum_values -= minmax_values.first->value + minmax_values.second->value; + num_values -= 2; + } + + auto measurement_to_string = [](const Instrument::Measurement & measurement) + { + return support::cpp11::to_string(measurement.value); + }; + + *_stream << R"("avg" : )" << (sum_values / num_values) << ","; + *_stream << R"("min" : )" << minmax_values.first->value << ","; + *_stream << R"("max" : )" << minmax_values.second->value << ","; + *_stream << R"("raw" : [)" << join(i_it->second.begin(), i_it->second.end(), ",", measurement_to_string) << "],"; + *_stream << R"("unit" : ")" << minmax_values.first->unit << R"(")"; + *_stream << "}"; + + if(++i_it != i_end) + { + *_stream << ","; + } + } + + *_stream << "}"; +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/printers/JSONPrinter.h b/tests/framework/printers/JSONPrinter.h new file mode 100644 index 0000000000..14c8b35cb9 --- /dev/null +++ b/tests/framework/printers/JSONPrinter.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_JSONPRINTER +#define ARM_COMPUTE_TEST_JSONPRINTER + +#include "Printer.h" + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Implementation of a @ref Printer that produces JSON output. */ +class JSONPrinter : public Printer +{ +public: + using Printer::Printer; + + void print_entry(const std::string &name, const std::string &value) override; + void print_global_header() override; + void print_global_footer() override; + void print_run_header() override; + void print_run_footer() override; + void print_test_header(const TestInfo &info) override; + void print_test_footer() override; + void print_errors_header() override; + void print_errors_footer() override; + void print_error(const std::exception &error) override; + void print_measurements(const Profiler::MeasurementsMap &measurements) override; + +private: + void print_separator(bool &flag); + + bool _first_entry{ true }; + bool _first_test{ true }; + bool _first_test_entry{ true }; + bool _first_error{ true }; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_JSONPRINTER */ diff --git a/tests/framework/printers/PrettyPrinter.cpp b/tests/framework/printers/PrettyPrinter.cpp new file mode 100644 index 0000000000..ec32e5296e --- /dev/null +++ b/tests/framework/printers/PrettyPrinter.cpp @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "PrettyPrinter.h" + +#include "tests/framework/Framework.h" + +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +std::string PrettyPrinter::begin_color(const std::string &color) const +{ + if(!_color_output) + { + return ""; + } + + return "\033[0;3" + color + "m"; +} + +std::string PrettyPrinter::end_color() const +{ + if(!_color_output) + { + return ""; + } + + return "\033[m"; +} + +void PrettyPrinter::set_color_output(bool color_output) +{ + _color_output = color_output; +} + +void PrettyPrinter::print_entry(const std::string &name, const std::string &value) +{ + *_stream << begin_color("4") << name << " = " << value << end_color() << "\n"; +} + +void PrettyPrinter::print_global_header() +{ +} + +void PrettyPrinter::print_global_footer() +{ +} + +void PrettyPrinter::print_run_header() +{ +} + +void PrettyPrinter::print_run_footer() +{ +} + +void PrettyPrinter::print_test_header(const TestInfo &info) +{ + *_stream << begin_color("2") << "Running [" << info.id << "] '" << info.name << "'" << end_color() << "\n"; +} + +void PrettyPrinter::print_test_footer() +{ +} + +void PrettyPrinter::print_errors_header() +{ +} + +void PrettyPrinter::print_errors_footer() +{ +} + +void PrettyPrinter::print_error(const std::exception &error) +{ + *_stream << begin_color("1") << "ERROR: " << error.what() << end_color() << "\n"; +} + +void PrettyPrinter::print_measurements(const Profiler::MeasurementsMap &measurements) +{ + for(const auto &instrument : measurements) + { + *_stream << begin_color("3") << " " << instrument.first << ":"; + + auto add_measurements = [](double a, const Instrument::Measurement & b) + { + return a + b.value; + }; + + auto cmp_measurements = [](const Instrument::Measurement & a, const Instrument::Measurement & b) + { + return a.value < b.value; + }; + + double sum_values = std::accumulate(instrument.second.begin(), instrument.second.end(), 0., add_measurements); + int num_values = instrument.second.size(); + const auto minmax_values = std::minmax_element(instrument.second.begin(), instrument.second.end(), cmp_measurements); + + if(num_values > 2) + { + sum_values -= minmax_values.first->value + minmax_values.second->value; + num_values -= 2; + } + + Instrument::Measurement avg{ sum_values / num_values, minmax_values.first->unit }; + + *_stream << " "; + *_stream << "AVG=" << avg << ", "; + *_stream << "MIN=" << *minmax_values.first << ", "; + *_stream << "MAX=" << *minmax_values.second << end_color() << "\n"; + } +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/printers/PrettyPrinter.h b/tests/framework/printers/PrettyPrinter.h new file mode 100644 index 0000000000..fa7b7b2c59 --- /dev/null +++ b/tests/framework/printers/PrettyPrinter.h @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_PRETTYPRINTER +#define ARM_COMPUTE_TEST_PRETTYPRINTER + +#include "Printer.h" + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +/** Implementation of a @ref Printer that produces human readable output. */ +class PrettyPrinter : public Printer +{ +public: + using Printer::Printer; + + /** Set if the output is colored. + * + * @param[in] color_output True if the output is colored. + */ + void set_color_output(bool color_output); + + void print_entry(const std::string &name, const std::string &value) override; + void print_global_header() override; + void print_global_footer() override; + void print_run_header() override; + void print_run_footer() override; + void print_test_header(const TestInfo &info) override; + void print_test_footer() override; + void print_errors_header() override; + void print_errors_footer() override; + void print_error(const std::exception &error) override; + void print_measurements(const Profiler::MeasurementsMap &measurements) override; + +private: + std::string begin_color(const std::string &color) const; + std::string end_color() const; + + bool _color_output{ true }; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute + +#endif /* ARM_COMPUTE_TEST_PRETTYPRINTER */ diff --git a/tests/framework/printers/Printer.cpp b/tests/framework/printers/Printer.cpp new file mode 100644 index 0000000000..e034c2ed43 --- /dev/null +++ b/tests/framework/printers/Printer.cpp @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "Printer.h" + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +Printer::Printer(std::ostream &stream) + : _stream{ &stream } +{ +} + +void Printer::print(const std::string &str) +{ + *_stream << str; +} + +void Printer::set_stream(std::ostream &stream) +{ + _stream = &stream; +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/printers/Printer.h b/tests/framework/printers/Printer.h new file mode 100644 index 0000000000..198d84d466 --- /dev/null +++ b/tests/framework/printers/Printer.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_PRINTER +#define ARM_COMPUTE_TEST_PRINTER + +#include "tests/framework/Profiler.h" + +#include +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +struct TestInfo; + +/** Abstract printer class used by the @ref Framework to present output. */ +class Printer +{ +public: + /** Default constructor. + * + * Prints values to std::cout. + * */ + Printer() = default; + + /** Construct printer with given output stream. + * + * @param[out] stream Output stream. + */ + Printer(std::ostream &stream); + + Printer(const Printer &) = delete; + Printer &operator=(const Printer &) = delete; + Printer(Printer &&) = default; + Printer &operator=(Printer &&) = default; + + virtual ~Printer() = default; + + /** Print given string. + * + * @param[in] str String. + */ + void print(const std::string &str); + + /** Print an entry consisting of a (name, value) pair. + * + * @param[in] name Description of the value. + * @param[in] value Value. + */ + virtual void print_entry(const std::string &name, const std::string &value) = 0; + + /** Print global header. */ + virtual void print_global_header() = 0; + + /** Print global footer. */ + virtual void print_global_footer() = 0; + + /** Print header before running all tests. */ + virtual void print_run_header() = 0; + + /** Print footer after running all tests. */ + virtual void print_run_footer() = 0; + + /** Print header before a test. + * + * @param[in] info Test info. + */ + virtual void print_test_header(const TestInfo &info) = 0; + + /** Print footer after a test. */ + virtual void print_test_footer() = 0; + + /** Print header before errors. */ + virtual void print_errors_header() = 0; + + /** Print footer after errors. */ + virtual void print_errors_footer() = 0; + + /** Print test error. + * + * @param[in] error Description of the error. + */ + virtual void print_error(const std::exception &error) = 0; + + /** Print measurements for a test. + * + * @param[in] measurements Measurements as collected by a @ref Profiler. + */ + virtual void print_measurements(const Profiler::MeasurementsMap &measurements) = 0; + + /** Set the output stream. + * + * @param[out] stream Output stream. + */ + void set_stream(std::ostream &stream); + +protected: + std::ostream *_stream{ &std::cout }; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_PRINTER */ diff --git a/tests/framework/printers/Printers.cpp b/tests/framework/printers/Printers.cpp new file mode 100644 index 0000000000..6e11b63a9a --- /dev/null +++ b/tests/framework/printers/Printers.cpp @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "Printers.h" + +#include "../Utils.h" + +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +LogFormat log_format_from_name(const std::string &name) +{ + static const std::map formats = + { + { "pretty", LogFormat::PRETTY }, + { "none", LogFormat::NONE }, + { "json", LogFormat::JSON }, + }; + + try + { + return formats.at(tolower(name)); + } + catch(const std::out_of_range &) + { + throw std::invalid_argument(name); + } +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/printers/Printers.h b/tests/framework/printers/Printers.h new file mode 100644 index 0000000000..53867e2dff --- /dev/null +++ b/tests/framework/printers/Printers.h @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_PRINTERS +#define ARM_COMPUTE_TEST_PRINTERS + +#include "JSONPrinter.h" +#include "PrettyPrinter.h" + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +enum class LogFormat +{ + NONE, + JSON, + PRETTY +}; + +LogFormat log_format_from_name(const std::string &name); + +inline ::std::stringstream &operator>>(::std::stringstream &stream, LogFormat &format) +{ + std::string value; + stream >> value; + format = log_format_from_name(value); + return stream; +} + +inline ::std::stringstream &operator<<(::std::stringstream &stream, LogFormat format) +{ + switch(format) + { + case LogFormat::PRETTY: + stream << "PRETTY"; + break; + case LogFormat::NONE: + stream << "NONE"; + break; + case LogFormat::JSON: + stream << "JSON"; + break; + default: + throw std::invalid_argument("Unsupported log format"); + } + + return stream; +} +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_PRINTERS */ -- cgit v1.2.1