aboutsummaryrefslogtreecommitdiff
path: root/tests/framework
diff options
context:
space:
mode:
authorMoritz Pflanzer <moritz.pflanzer@arm.com>2017-09-01 20:41:12 +0100
committerAnthony Barbier <anthony.barbier@arm.com>2018-11-02 16:35:24 +0000
commita09de0c8b2ed0f1481502d3b023375609362d9e3 (patch)
treee34b56d9ca69b025d7d9b943cc4df59cd458f6cb /tests/framework
parent5280071b336d53aff94ca3a6c70ebbe6bf03f4c3 (diff)
downloadComputeLibrary-a09de0c8b2ed0f1481502d3b023375609362d9e3.tar.gz
COMPMID-415: Rename and move tests
The boost validation is now "standalone" in validation_old and builds as arm_compute_validation_old. The new validation builds now as arm_compute_validation. Change-Id: Ib93ba848a25680ac60afb92b461d574a0757150d Reviewed-on: http://mpd-gerrit.cambridge.arm.com/86187 Tested-by: Kaizen <jeremy.johnson+kaizengerrit@arm.com> Reviewed-by: Anthony Barbier <anthony.barbier@arm.com>
Diffstat (limited to 'tests/framework')
-rw-r--r--tests/framework/Asserts.h132
-rw-r--r--tests/framework/DatasetModes.cpp56
-rw-r--r--tests/framework/DatasetModes.h106
-rw-r--r--tests/framework/Exceptions.cpp128
-rw-r--r--tests/framework/Exceptions.h97
-rw-r--r--tests/framework/Fixture.h63
-rw-r--r--tests/framework/Framework.cpp494
-rw-r--r--tests/framework/Framework.h330
-rw-r--r--tests/framework/Macros.h260
-rw-r--r--tests/framework/Profiler.cpp67
-rw-r--r--tests/framework/Profiler.h75
-rw-r--r--tests/framework/Registrars.h113
-rw-r--r--tests/framework/SConscript71
-rw-r--r--tests/framework/TestCase.h70
-rw-r--r--tests/framework/TestCaseFactory.h202
-rw-r--r--tests/framework/TestFilter.cpp143
-rw-r--r--tests/framework/TestFilter.h84
-rw-r--r--tests/framework/TestResult.h80
-rw-r--r--tests/framework/Utils.h158
-rw-r--r--tests/framework/command_line/CommandLineOptions.h33
-rw-r--r--tests/framework/command_line/CommandLineParser.cpp149
-rw-r--r--tests/framework/command_line/CommandLineParser.h117
-rw-r--r--tests/framework/command_line/EnumListOption.h148
-rw-r--r--tests/framework/command_line/EnumOption.h134
-rw-r--r--tests/framework/command_line/ListOption.h116
-rw-r--r--tests/framework/command_line/Option.cpp68
-rw-r--r--tests/framework/command_line/Option.h109
-rw-r--r--tests/framework/command_line/SimpleOption.h105
-rw-r--r--tests/framework/command_line/ToggleOption.cpp64
-rw-r--r--tests/framework/command_line/ToggleOption.h56
-rw-r--r--tests/framework/datasets/CartesianProductDataset.h165
-rw-r--r--tests/framework/datasets/ContainerDataset.h148
-rw-r--r--tests/framework/datasets/Dataset.h86
-rw-r--r--tests/framework/datasets/Datasets.h35
-rw-r--r--tests/framework/datasets/InitializerListDataset.h135
-rw-r--r--tests/framework/datasets/JoinDataset.h148
-rw-r--r--tests/framework/datasets/RangeDataset.h141
-rw-r--r--tests/framework/datasets/SingletonDataset.h138
-rw-r--r--tests/framework/datasets/ZipDataset.h139
-rw-r--r--tests/framework/instruments/Instrument.h98
-rw-r--r--tests/framework/instruments/Instruments.cpp59
-rw-r--r--tests/framework/instruments/Instruments.h105
-rw-r--r--tests/framework/instruments/PMUCounter.cpp140
-rw-r--r--tests/framework/instruments/PMUCounter.h71
-rw-r--r--tests/framework/instruments/WallClockTimer.cpp57
-rw-r--r--tests/framework/instruments/WallClockTimer.h53
-rw-r--r--tests/framework/printers/JSONPrinter.cpp168
-rw-r--r--tests/framework/printers/JSONPrinter.h64
-rw-r--r--tests/framework/printers/PrettyPrinter.cpp140
-rw-r--r--tests/framework/printers/PrettyPrinter.h69
-rw-r--r--tests/framework/printers/Printer.cpp48
-rw-r--r--tests/framework/printers/Printer.h129
-rw-r--r--tests/framework/printers/Printers.cpp57
-rw-r--r--tests/framework/printers/Printers.h75
54 files changed, 6296 insertions, 0 deletions
diff --git a/tests/framework/Asserts.h b/tests/framework/Asserts.h
new file mode 100644
index 0000000000..b545a9ebba
--- /dev/null
+++ b/tests/framework/Asserts.h
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_FRAMEWORK_ASSERTS
+#define ARM_COMPUTE_TEST_FRAMEWORK_ASSERTS
+
+#include "Exceptions.h"
+#include "Framework.h"
+
+#include <sstream>
+#include <type_traits>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+// Cast char values to int so that their numeric value are printed.
+inline int make_printable(int8_t value)
+{
+ return value;
+}
+
+inline unsigned int make_printable(uint8_t value)
+{
+ return value;
+}
+
+// Everything else can be printed as its own type.
+template <typename T>
+inline T make_printable(T &&value)
+{
+ return value;
+}
+
+#define ARM_COMPUTE_TEST_INFO(INFO) \
+ { \
+ std::stringstream info; \
+ info << INFO; \
+ arm_compute::test::framework::Framework::get().add_test_info(info.str()); \
+ }
+
+namespace detail
+{
+#define ARM_COMPUTE_TEST_COMP_FACTORY(SEVERITY, SEVERITY_NAME, COMP, COMP_NAME, ERROR_CALL) \
+ template <typename T, typename U> \
+ void ARM_COMPUTE_##SEVERITY##_##COMP_NAME##_IMPL(T &&x, U &&y, const std::string &x_str, const std::string &y_str, LogLevel level) \
+ { \
+ if(!(x COMP y)) \
+ { \
+ std::stringstream msg; \
+ msg << #SEVERITY_NAME " '" << x_str << " " #COMP " " << y_str << "' failed. [" \
+ << std::boolalpha << arm_compute::test::framework::make_printable(x) \
+ << " " #COMP " " \
+ << std::boolalpha << arm_compute::test::framework::make_printable(y) \
+ << "]\n"; \
+ arm_compute::test::framework::Framework::get().print_test_info(msg); \
+ ERROR_CALL \
+ } \
+ arm_compute::test::framework::Framework::get().clear_test_info(); \
+ }
+
+ARM_COMPUTE_TEST_COMP_FACTORY(EXPECT, Expectation, ==, EQUAL, arm_compute::test::framework::Framework::get().log_failed_expectation(arm_compute::test::framework::TestError(msg.str(), level));)
+ARM_COMPUTE_TEST_COMP_FACTORY(EXPECT, Expectation, !=, NOT_EQUAL, arm_compute::test::framework::Framework::get().log_failed_expectation(arm_compute::test::framework::TestError(msg.str(), level));)
+ARM_COMPUTE_TEST_COMP_FACTORY(ASSERT, Assertion, ==, EQUAL, throw arm_compute::test::framework::TestError(msg.str(), level);)
+ARM_COMPUTE_TEST_COMP_FACTORY(ASSERT, Assertion, !=, NOT_EQUAL, throw arm_compute::test::framework::TestError(msg.str(), level);)
+} // namespace detail
+
+#define ARM_COMPUTE_ASSERT_NOT_EQUAL(X, Y) \
+ arm_compute::test::framework::detail::ARM_COMPUTE_ASSERT_NOT_EQUAL_IMPL(X, Y, #X, #Y, LogLevel::ERRORS)
+
+#define ARM_COMPUTE_ASSERT_EQUAL(X, Y) \
+ arm_compute::test::framework::detail::ARM_COMPUTE_ASSERT_EQUAL_IMPL(X, Y, #X, #Y, LogLevel::ERRORS)
+
+#define ARM_COMPUTE_EXPECT_EQUAL(X, Y, LEVEL) \
+ arm_compute::test::framework::detail::ARM_COMPUTE_EXPECT_EQUAL_IMPL(X, Y, #X, #Y, LEVEL)
+
+#define ARM_COMPUTE_EXPECT_NOT_EQUAL(X, Y, LEVEL) \
+ arm_compute::test::framework::detail::ARM_COMPUTE_EXPECT_NOT_EQUAL_IMPL(X, Y, #X, #Y, LEVEL)
+
+#define ARM_COMPUTE_ASSERT(X) \
+ do \
+ { \
+ const auto &x = X; \
+ if(!x) \
+ { \
+ std::stringstream msg; \
+ msg << "Assertion '" #X "' failed.\n"; \
+ arm_compute::test::framework::Framework::get().print_test_info(msg); \
+ throw arm_compute::test::framework::TestError(msg.str(), arm_compute::test::framework::LogLevel::ERRORS); \
+ } \
+ arm_compute::test::framework::Framework::get().clear_test_info(); \
+ } while(false)
+
+#define ARM_COMPUTE_EXPECT(X, LEVEL) \
+ do \
+ { \
+ const auto &x = X; \
+ if(!x) \
+ { \
+ std::stringstream msg; \
+ msg << "Expectation '" #X "' failed.\n"; \
+ arm_compute::test::framework::Framework::get().print_test_info(msg); \
+ arm_compute::test::framework::Framework::get().log_failed_expectation(arm_compute::test::framework::TestError(msg.str(), LEVEL)); \
+ } \
+ arm_compute::test::framework::Framework::get().clear_test_info(); \
+ } while(false)
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_FRAMEWORK_ASSERTS */
diff --git a/tests/framework/DatasetModes.cpp b/tests/framework/DatasetModes.cpp
new file mode 100644
index 0000000000..3f747df747
--- /dev/null
+++ b/tests/framework/DatasetModes.cpp
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "DatasetModes.h"
+
+#include "Utils.h"
+
+#include <map>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+DatasetMode dataset_mode_from_name(const std::string &name)
+{
+ static const std::map<std::string, DatasetMode> modes =
+ {
+ { "all", DatasetMode::ALL },
+ { "precommit", DatasetMode::PRECOMMIT },
+ { "nightly", DatasetMode::NIGHTLY },
+ };
+
+ try
+ {
+ return modes.at(tolower(name));
+ }
+ catch(const std::out_of_range &)
+ {
+ throw std::invalid_argument(name);
+ }
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/DatasetModes.h b/tests/framework/DatasetModes.h
new file mode 100644
index 0000000000..27638b0504
--- /dev/null
+++ b/tests/framework/DatasetModes.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_DATASET_MODES
+#define ARM_COMPUTE_TEST_DATASET_MODES
+
+#include <istream>
+#include <ostream>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Possible dataset modes. */
+enum class DatasetMode : unsigned int
+{
+ ALL = ~0U,
+ DISABLED = 0,
+ PRECOMMIT = 1,
+ NIGHTLY = 2
+};
+
+inline DatasetMode operator&(DatasetMode t1, DatasetMode t2)
+{
+ using type = std::underlying_type<DatasetMode>::type;
+ return static_cast<DatasetMode>(static_cast<type>(t1) & static_cast<type>(t2));
+}
+
+inline DatasetMode operator|(DatasetMode t1, DatasetMode t2)
+{
+ using type = std::underlying_type<DatasetMode>::type;
+ return static_cast<DatasetMode>(static_cast<type>(t1) | static_cast<type>(t2));
+}
+
+inline DatasetMode &operator|=(DatasetMode &t1, DatasetMode t2)
+{
+ using type = std::underlying_type<DatasetMode>::type;
+ t1 = static_cast<DatasetMode>(static_cast<type>(t1) | static_cast<type>(t2));
+ return t1;
+}
+
+DatasetMode dataset_mode_from_name(const std::string &name);
+
+inline ::std::istream &operator>>(::std::istream &stream, DatasetMode &mode)
+{
+ std::string value;
+ stream >> value;
+ mode = dataset_mode_from_name(value);
+ return stream;
+}
+
+inline ::std::ostream &operator<<(::std::ostream &stream, DatasetMode mode)
+{
+ switch(mode)
+ {
+ case DatasetMode::PRECOMMIT:
+ stream << "PRECOMMIT";
+ break;
+ case DatasetMode::NIGHTLY:
+ stream << "NIGHTLY";
+ break;
+ case DatasetMode::ALL:
+ stream << "ALL";
+ break;
+ default:
+ throw std::invalid_argument("Unsupported dataset mode");
+ }
+
+ return stream;
+}
+
+inline std::string to_string(DatasetMode mode)
+{
+ std::stringstream stream;
+ stream << mode;
+ return stream.str();
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_DATASET_MODES */
diff --git a/tests/framework/Exceptions.cpp b/tests/framework/Exceptions.cpp
new file mode 100644
index 0000000000..3d6c65c181
--- /dev/null
+++ b/tests/framework/Exceptions.cpp
@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "Exceptions.h"
+
+#include "Utils.h"
+
+#include <map>
+#include <sstream>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+LogLevel log_level_from_name(const std::string &name)
+{
+ static const std::map<std::string, LogLevel> levels =
+ {
+ { "none", LogLevel::NONE },
+ { "config", LogLevel::CONFIG },
+ { "tests", LogLevel::TESTS },
+ { "errors", LogLevel::ERRORS },
+ { "debug", LogLevel::DEBUG },
+ { "measurements", LogLevel::MEASUREMENTS },
+ { "all", LogLevel::ALL },
+ };
+
+ try
+ {
+ return levels.at(tolower(name));
+ }
+ catch(const std::out_of_range &)
+ {
+ throw std::invalid_argument(name);
+ }
+}
+
+::std::istream &operator>>(::std::istream &stream, LogLevel &level)
+{
+ std::string value;
+ stream >> value;
+ level = log_level_from_name(value);
+ return stream;
+}
+
+::std::ostream &operator<<(::std::ostream &stream, LogLevel level)
+{
+ switch(level)
+ {
+ case LogLevel::NONE:
+ stream << "NONE";
+ break;
+ case LogLevel::CONFIG:
+ stream << "CONFIG";
+ break;
+ case LogLevel::TESTS:
+ stream << "TESTS";
+ break;
+ case LogLevel::ERRORS:
+ stream << "ERRORS";
+ break;
+ case LogLevel::DEBUG:
+ stream << "DEBUG";
+ break;
+ case LogLevel::MEASUREMENTS:
+ stream << "MEASUREMENTS";
+ break;
+ case LogLevel::ALL:
+ stream << "ALL";
+ break;
+ default:
+ throw std::invalid_argument("Unsupported log level");
+ }
+
+ return stream;
+}
+
+std::string to_string(LogLevel level)
+{
+ std::stringstream stream;
+ stream << level;
+ return stream.str();
+}
+
+TestError::TestError(const std::string &msg, LogLevel level, std::string context)
+ : std::runtime_error{ msg }, _level{ level }, _msg{ msg }, _context{ std::move(context) }, _combined{ "ERROR: " + msg }
+{
+ if(!_context.empty())
+ {
+ _combined += "\nCONTEXT:\n" + _context;
+ }
+}
+
+LogLevel TestError::level() const
+{
+ return _level;
+}
+
+const char *TestError::what() const noexcept
+{
+ return _combined.c_str();
+}
+
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/Exceptions.h b/tests/framework/Exceptions.h
new file mode 100644
index 0000000000..edb0ed92c9
--- /dev/null
+++ b/tests/framework/Exceptions.h
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_EXCEPTIONS
+#define ARM_COMPUTE_TEST_EXCEPTIONS
+
+#include <istream>
+#include <ostream>
+#include <stdexcept>
+#include <string>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Severity of the information.
+ *
+ * Each category includes the ones above it.
+ *
+ * NONE == Only for filtering. Not used to tag information.
+ * CONFIG == Configuration info.
+ * TESTS == Information about the tests.
+ * ERRORS == Violated assertions/expectations.
+ * DEBUG == More violated assertions/expectations.
+ * MEASUREMENTS == Information about measurements.
+ * ALL == Only for filtering. Not used to tag information.
+ */
+enum class LogLevel
+{
+ NONE,
+ CONFIG,
+ TESTS,
+ ERRORS,
+ DEBUG,
+ MEASUREMENTS,
+ ALL,
+};
+
+LogLevel log_level_from_name(const std::string &name);
+::std::istream &operator>>(::std::istream &stream, LogLevel &level);
+::std::ostream &operator<<(::std::ostream &stream, LogLevel level);
+std::string to_string(LogLevel level);
+
+/** Error class for failures during test execution. */
+class TestError : public std::runtime_error
+{
+public:
+ using std::runtime_error::runtime_error;
+
+ /** Construct error with severity.
+ *
+ * @param[in] msg Error message.
+ * @param[in] level Severity level.
+ * @param[in] context Context.
+ */
+ TestError(const std::string &msg, LogLevel level, std::string context = "");
+
+ /** Severity of the error.
+ *
+ * @return Severity.
+ */
+ LogLevel level() const;
+
+ const char *what() const noexcept override;
+
+private:
+ LogLevel _level{ LogLevel::ERRORS };
+ std::string _msg{};
+ std::string _context{};
+ std::string _combined{};
+};
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_EXCEPTIONS */
diff --git a/tests/framework/Fixture.h b/tests/framework/Fixture.h
new file mode 100644
index 0000000000..916dcc7fef
--- /dev/null
+++ b/tests/framework/Fixture.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_FIXTURE
+#define ARM_COMPUTE_TEST_FIXTURE
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Abstract fixture class.
+ *
+ * All custom fixtures have to inherit from this class.
+ */
+class Fixture
+{
+public:
+ /** Setup function.
+ *
+ * This function is only invoked by non-data fixture test cases. Fixture
+ * data test cases implement a setup function with arguments matching the
+ * dataset.
+ *
+ * The function is called before the test case is executed.
+ */
+ void setup() {};
+
+ /** Teardown function.
+ *
+ * The function is called after the test case finished.
+ */
+ void teardown() {};
+
+protected:
+ Fixture() = default;
+ virtual ~Fixture() = default;
+};
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_FIXTURE */
diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp
new file mode 100644
index 0000000000..315f8ebea7
--- /dev/null
+++ b/tests/framework/Framework.cpp
@@ -0,0 +1,494 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "Framework.h"
+
+#include "support/ToolchainSupport.h"
+
+#ifdef ARM_COMPUTE_CL
+#include "arm_compute/core/CL/OpenCL.h"
+#include "arm_compute/runtime/CL/CLScheduler.h"
+#endif /* ARM_COMPUTE_CL */
+
+#include <chrono>
+#include <iostream>
+#include <sstream>
+#include <type_traits>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+Framework::Framework()
+{
+ _available_instruments.emplace(InstrumentType::WALL_CLOCK_TIMER, Instrument::make_instrument<WallClockTimer>);
+#ifdef PMU_ENABLED
+ _available_instruments.emplace(InstrumentType::PMU_CYCLE_COUNTER, Instrument::make_instrument<CycleCounter>);
+ _available_instruments.emplace(InstrumentType::PMU_INSTRUCTION_COUNTER, Instrument::make_instrument<InstructionCounter>);
+#endif /* PMU_ENABLED */
+}
+
+std::set<InstrumentType> Framework::available_instruments() const
+{
+ std::set<InstrumentType> types;
+
+ for(const auto &instrument : _available_instruments)
+ {
+ types.emplace(instrument.first);
+ }
+
+ return types;
+}
+
+std::map<TestResult::Status, int> Framework::count_test_results() const
+{
+ std::map<TestResult::Status, int> counts;
+
+ for(const auto &test : _test_results)
+ {
+ ++counts[test.second.status];
+ }
+
+ return counts;
+}
+
+Framework &Framework::get()
+{
+ static Framework instance;
+ return instance;
+}
+
+void Framework::init(const std::vector<InstrumentType> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter, LogLevel log_level)
+{
+ _test_filter = TestFilter(mode, name_filter, id_filter);
+ _num_iterations = num_iterations;
+ _log_level = log_level;
+
+ _instruments = InstrumentType::NONE;
+
+ for(const auto &instrument : instruments)
+ {
+ _instruments |= instrument;
+ }
+}
+
+std::string Framework::current_suite_name() const
+{
+ return join(_test_suite_name.cbegin(), _test_suite_name.cend(), "/");
+}
+
+void Framework::push_suite(std::string name)
+{
+ _test_suite_name.emplace_back(std::move(name));
+}
+
+void Framework::pop_suite()
+{
+ _test_suite_name.pop_back();
+}
+
+void Framework::add_test_info(std::string info)
+{
+ _test_info.emplace_back(std::move(info));
+}
+
+void Framework::clear_test_info()
+{
+ _test_info.clear();
+}
+
+bool Framework::has_test_info() const
+{
+ return !_test_info.empty();
+}
+
+void Framework::print_test_info(std::ostream &os) const
+{
+ if(!_test_info.empty())
+ {
+ os << "CONTEXT:\n";
+
+ for(const auto &str : _test_info)
+ {
+ os << " " << str << "\n";
+ }
+ }
+}
+
+void Framework::log_test_start(const TestInfo &info)
+{
+ if(_printer != nullptr && _log_level >= LogLevel::TESTS)
+ {
+ _printer->print_test_header(info);
+ }
+}
+
+void Framework::log_test_skipped(const TestInfo &info)
+{
+ static_cast<void>(info);
+}
+
+void Framework::log_test_end(const TestInfo &info)
+{
+ if(_printer != nullptr)
+ {
+ if(_log_level >= LogLevel::MEASUREMENTS)
+ {
+ _printer->print_measurements(_test_results.at(info).measurements);
+ }
+
+ if(_log_level >= LogLevel::TESTS)
+ {
+ _printer->print_test_footer();
+ }
+ }
+}
+
+void Framework::log_failed_expectation(const TestError &error)
+{
+ if(_log_level >= error.level() && _printer != nullptr)
+ {
+ _printer->print_error(error);
+ }
+
+ if(_current_test_result != nullptr)
+ {
+ _current_test_result->status = TestResult::Status::FAILED;
+ }
+}
+
+int Framework::num_iterations() const
+{
+ return _num_iterations;
+}
+
+void Framework::set_num_iterations(int num_iterations)
+{
+ _num_iterations = num_iterations;
+}
+
+void Framework::set_throw_errors(bool throw_errors)
+{
+ _throw_errors = throw_errors;
+}
+
+bool Framework::throw_errors() const
+{
+ return _throw_errors;
+}
+
+void Framework::set_stop_on_error(bool stop_on_error)
+{
+ _stop_on_error = stop_on_error;
+}
+
+bool Framework::stop_on_error() const
+{
+ return _stop_on_error;
+}
+
+void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
+{
+ if(test_factory.status() == TestCaseFactory::Status::DISABLED)
+ {
+ log_test_skipped(info);
+ set_test_result(info, TestResult(TestResult::Status::DISABLED));
+ return;
+ }
+
+ log_test_start(info);
+
+ Profiler profiler = get_profiler();
+ TestResult result(TestResult::Status::NOT_RUN);
+
+ _current_test_result = &result;
+
+ if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
+ {
+ _printer->print_errors_header();
+ }
+
+ try
+ {
+ std::unique_ptr<TestCase> test_case = test_factory.make();
+
+ try
+ {
+ test_case->do_setup();
+
+ for(int i = 0; i < _num_iterations; ++i)
+ {
+ profiler.start();
+ test_case->do_run();
+#ifdef ARM_COMPUTE_CL
+ if(opencl_is_available())
+ {
+ CLScheduler::get().sync();
+ }
+#endif /* ARM_COMPUTE_CL */
+ profiler.stop();
+ }
+
+ test_case->do_teardown();
+
+ // Change status to success if no error has happend
+ if(result.status == TestResult::Status::NOT_RUN)
+ {
+ result.status = TestResult::Status::SUCCESS;
+ }
+ }
+ catch(const TestError &error)
+ {
+ if(_log_level >= error.level() && _printer != nullptr)
+ {
+ _printer->print_error(error);
+ }
+
+ result.status = TestResult::Status::FAILED;
+
+ if(_throw_errors)
+ {
+ throw;
+ }
+ }
+#ifdef ARM_COMPUTE_CL
+ catch(const ::cl::Error &error)
+ {
+ if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
+ {
+ std::stringstream stream;
+ stream << "Error code: " << error.err();
+ _printer->print_error(TestError(error.what(), LogLevel::ERRORS, stream.str()));
+ }
+
+ result.status = TestResult::Status::FAILED;
+
+ if(_throw_errors)
+ {
+ throw;
+ }
+ }
+#endif /* ARM_COMPUTE_CL */
+ catch(const std::exception &error)
+ {
+ if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
+ {
+ _printer->print_error(error);
+ }
+
+ result.status = TestResult::Status::CRASHED;
+
+ if(_throw_errors)
+ {
+ throw;
+ }
+ }
+ catch(...)
+ {
+ if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
+ {
+ _printer->print_error(TestError("Received unknown exception"));
+ }
+
+ result.status = TestResult::Status::CRASHED;
+
+ if(_throw_errors)
+ {
+ throw;
+ }
+ }
+ }
+ catch(const std::exception &error)
+ {
+ if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
+ {
+ _printer->print_error(error);
+ }
+
+ result.status = TestResult::Status::CRASHED;
+
+ if(_throw_errors)
+ {
+ throw;
+ }
+ }
+ catch(...)
+ {
+ if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
+ {
+ _printer->print_error(TestError("Received unknown exception"));
+ }
+
+ result.status = TestResult::Status::CRASHED;
+
+ if(_throw_errors)
+ {
+ throw;
+ }
+ }
+
+ if(_log_level >= LogLevel::ERRORS && _printer != nullptr)
+ {
+ _printer->print_errors_footer();
+ }
+
+ _current_test_result = nullptr;
+
+ if(result.status == TestResult::Status::FAILED)
+ {
+ if(info.status == TestCaseFactory::Status::EXPECTED_FAILURE)
+ {
+ result.status = TestResult::Status::EXPECTED_FAILURE;
+ }
+ }
+
+ if(result.status == TestResult::Status::FAILED || result.status == TestResult::Status::CRASHED)
+ {
+ if(_stop_on_error)
+ {
+ throw std::runtime_error("Abort on first error.");
+ }
+ }
+
+ result.measurements = profiler.measurements();
+
+ set_test_result(info, result);
+ log_test_end(info);
+}
+
+bool Framework::run()
+{
+ // Clear old test results
+ _test_results.clear();
+
+ if(_printer != nullptr && _log_level >= LogLevel::TESTS)
+ {
+ _printer->print_run_header();
+ }
+
+ const std::chrono::time_point<std::chrono::high_resolution_clock> start = std::chrono::high_resolution_clock::now();
+
+ int id = 0;
+
+ for(auto &test_factory : _test_factories)
+ {
+ const std::string test_case_name = test_factory->name();
+ const TestInfo test_info{ id, test_case_name, test_factory->mode(), test_factory->status() };
+
+ if(_test_filter.is_selected(test_info))
+ {
+ run_test(test_info, *test_factory);
+ }
+
+ ++id;
+ }
+
+ const std::chrono::time_point<std::chrono::high_resolution_clock> end = std::chrono::high_resolution_clock::now();
+
+ if(_printer != nullptr && _log_level >= LogLevel::TESTS)
+ {
+ _printer->print_run_footer();
+ }
+
+ auto runtime = std::chrono::duration_cast<std::chrono::seconds>(end - start);
+ std::map<TestResult::Status, int> results = count_test_results();
+
+ if(_log_level > LogLevel::NONE)
+ {
+ std::cout << "Executed " << _test_results.size() << " test(s) ("
+ << results[TestResult::Status::SUCCESS] << " passed, "
+ << results[TestResult::Status::EXPECTED_FAILURE] << " expected failures, "
+ << results[TestResult::Status::FAILED] << " failed, "
+ << results[TestResult::Status::CRASHED] << " crashed, "
+ << results[TestResult::Status::DISABLED] << " disabled) in " << runtime.count() << " second(s)\n";
+ }
+
+ int num_successful_tests = results[TestResult::Status::SUCCESS] + results[TestResult::Status::EXPECTED_FAILURE];
+
+ return (static_cast<unsigned int>(num_successful_tests) == _test_results.size());
+}
+
+void Framework::set_test_result(TestInfo info, TestResult result)
+{
+ _test_results.emplace(std::move(info), std::move(result));
+}
+
+void Framework::print_test_results(Printer &printer) const
+{
+ printer.print_run_header();
+
+ for(const auto &test : _test_results)
+ {
+ printer.print_test_header(test.first);
+ printer.print_measurements(test.second.measurements);
+ printer.print_test_footer();
+ }
+
+ printer.print_run_footer();
+}
+
+Profiler Framework::get_profiler() const
+{
+ Profiler profiler;
+
+ for(const auto &instrument : _available_instruments)
+ {
+ if((instrument.first & _instruments) != InstrumentType::NONE)
+ {
+ profiler.add(instrument.second());
+ }
+ }
+
+ return profiler;
+}
+
+void Framework::set_printer(Printer *printer)
+{
+ _printer = printer;
+}
+
+std::vector<TestInfo> Framework::test_infos() const
+{
+ std::vector<TestInfo> ids;
+
+ int id = 0;
+
+ for(const auto &factory : _test_factories)
+ {
+ TestInfo test_info{ id, factory->name(), factory->mode(), factory->status() };
+
+ if(_test_filter.is_selected(test_info))
+ {
+ ids.emplace_back(std::move(test_info));
+ }
+
+ ++id;
+ }
+
+ return ids;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/Framework.h b/tests/framework/Framework.h
new file mode 100644
index 0000000000..055392cdae
--- /dev/null
+++ b/tests/framework/Framework.h
@@ -0,0 +1,330 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_FRAMEWORK
+#define ARM_COMPUTE_TEST_FRAMEWORK
+
+#include "DatasetModes.h"
+#include "Exceptions.h"
+#include "Profiler.h"
+#include "TestCase.h"
+#include "TestCaseFactory.h"
+#include "TestFilter.h"
+#include "TestResult.h"
+#include "Utils.h"
+#include "instruments/Instruments.h"
+#include "printers/Printer.h"
+
+#include <algorithm>
+#include <chrono>
+#include <map>
+#include <memory>
+#include <numeric>
+#include <ostream>
+#include <regex>
+#include <set>
+#include <sstream>
+#include <string>
+#include <tuple>
+#include <vector>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Information about a test case.
+ *
+ * A test can be identified either via its id or via its name. Additionally
+ * each test is tagged with the data set mode in which it will be used and
+ * its status.
+ *
+ * @note The mapping between test id and test name is not guaranteed to be
+ * stable. It is subject to change as new test are added.
+ */
+struct TestInfo
+{
+ int id;
+ std::string name;
+ DatasetMode mode;
+ TestCaseFactory::Status status;
+};
+
+inline bool operator<(const TestInfo &lhs, const TestInfo &rhs)
+{
+ return lhs.id < rhs.id;
+}
+
+/** Main framework class.
+ *
+ * Keeps track of the global state, owns all test cases and collects results.
+ */
+class Framework final
+{
+public:
+ /** Access to the singleton.
+ *
+ * @return Unique instance of the framework class.
+ */
+ static Framework &get();
+
+ /** Supported instrument types for benchmarking.
+ *
+ * @return Set of all available instrument types.
+ */
+ std::set<InstrumentType> available_instruments() const;
+
+ /** Init the framework.
+ *
+ * @see TestFilter::TestFilter for the format of the string to filter ids.
+ *
+ * @param[in] instruments Instrument types that will be used for benchmarking.
+ * @param[in] num_iterations Number of iterations per test.
+ * @param[in] mode Dataset mode.
+ * @param[in] name_filter Regular expression to filter tests by name. Only matching tests will be executed.
+ * @param[in] id_filter String to match selected test ids. Only matching tests will be executed.
+ * @param[in] log_level Verbosity of the output.
+ */
+ void init(const std::vector<InstrumentType> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter, LogLevel log_level);
+
+ /** Add a new test suite.
+ *
+ * @warning Cannot be used at execution time. It can only be used for
+ * registering test cases.
+ *
+ * @param[in] name Name of the added test suite.
+ *
+ * @return Name of the current test suite.
+ */
+ void push_suite(std::string name);
+
+ /** Remove innermost test suite.
+ *
+ * @warning Cannot be used at execution time. It can only be used for
+ * registering test cases.
+ */
+ void pop_suite();
+
+ /** Add a test case to the framework.
+ *
+ * @param[in] test_name Name of the new test case.
+ * @param[in] mode Mode in which to include the test.
+ * @param[in] status Status of the test case.
+ */
+ template <typename T>
+ void add_test_case(std::string test_name, DatasetMode mode, TestCaseFactory::Status status);
+
+ /** Add a data test case to the framework.
+ *
+ * @param[in] test_name Name of the new test case.
+ * @param[in] mode Mode in which to include the test.
+ * @param[in] status Status of the test case.
+ * @param[in] description Description of @p data.
+ * @param[in] data Data that will be used as input to the test.
+ */
+ template <typename T, typename D>
+ void add_data_test_case(std::string test_name, DatasetMode mode, TestCaseFactory::Status status, std::string description, D &&data);
+
+ /** Add info string for the next expectation/assertion.
+ *
+ * @param[in] info Info string.
+ */
+ void add_test_info(std::string info);
+
+ /** Clear the collected test info. */
+ void clear_test_info();
+
+ /** Check if any info has been registered.
+ *
+ * @return True if there is test info.
+ */
+ bool has_test_info() const;
+
+ /** Print test info.
+ *
+ * @param[out] os Output stream.
+ */
+ void print_test_info(std::ostream &os) const;
+
+ /** Tell the framework that execution of a test starts.
+ *
+ * @param[in] info Test info.
+ */
+ void log_test_start(const TestInfo &info);
+
+ /** Tell the framework that a test case is skipped.
+ *
+ * @param[in] info Test info.
+ */
+ void log_test_skipped(const TestInfo &info);
+
+ /** Tell the framework that a test case finished.
+ *
+ * @param[in] info Test info.
+ */
+ void log_test_end(const TestInfo &info);
+
+ /** Tell the framework that the currently running test case failed a non-fatal expectation.
+ *
+ * @param[in] error Description of the error.
+ */
+ void log_failed_expectation(const TestError &error);
+
+ /** Number of iterations per test case.
+ *
+ * @return Number of iterations per test case.
+ */
+ int num_iterations() const;
+
+ /** Set number of iterations per test case.
+ *
+ * @param[in] num_iterations Number of iterations per test case.
+ */
+ void set_num_iterations(int num_iterations);
+
+ /** Should errors be caught or thrown by the framework.
+ *
+ * @return True if errors are thrown.
+ */
+ bool throw_errors() const;
+
+ /** Set whether errors are caught or thrown by the framework.
+ *
+ * @param[in] throw_errors True if errors should be thrown.
+ */
+ void set_throw_errors(bool throw_errors);
+
+ /** Indicates if test execution is stopped after the first failed test.
+ *
+ * @return True if the execution is going to be aborted after the first failed test.
+ */
+ bool stop_on_error() const;
+
+ /** Set whether to abort execution after the first failed test.
+ *
+ * @param[in] stop_on_error True if execution is going to be aborted after first failed test.
+ */
+ void set_stop_on_error(bool stop_on_error);
+
+ /** Run all enabled test cases.
+ *
+ * @return True if all test cases executed successful.
+ */
+ bool run();
+
+ /** Set the result for an executed test case.
+ *
+ * @param[in] info Test info.
+ * @param[in] result Execution result.
+ */
+ void set_test_result(TestInfo info, TestResult result);
+
+ /** Use the specified printer to output test results from the last run.
+ *
+ * This method can be used if the test results need to be obtained using a
+ * different printer than the one managed by the framework.
+ *
+ * @param[in] printer Printer used to output results.
+ */
+ void print_test_results(Printer &printer) const;
+
+ /** Factory method to obtain a configured profiler.
+ *
+ * The profiler enables all instruments that have been passed to the @ref
+ * init method.
+ *
+ * @return Configured profiler to collect benchmark results.
+ */
+ Profiler get_profiler() const;
+
+ /** Set the printer used for the output of test results.
+ *
+ * @param[in] printer Pointer to a printer.
+ */
+ void set_printer(Printer *printer);
+
+ /** List of @ref TestInfo's.
+ *
+ * @return Vector with all test ids.
+ */
+ std::vector<TestInfo> test_infos() const;
+
+private:
+ Framework();
+ ~Framework() = default;
+
+ Framework(const Framework &) = delete;
+ Framework &operator=(const Framework &) = delete;
+
+ void run_test(const TestInfo &info, TestCaseFactory &test_factory);
+ std::map<TestResult::Status, int> count_test_results() const;
+
+ /** Returns the current test suite name.
+ *
+ * @warning Cannot be used at execution time to get the test suite of the
+ * currently executed test case. It can only be used for registering test
+ * cases.
+ *
+ * @return Name of the current test suite.
+ */
+ std::string current_suite_name() const;
+
+ std::vector<std::string> _test_suite_name{};
+ std::vector<std::unique_ptr<TestCaseFactory>> _test_factories{};
+ std::map<TestInfo, TestResult> _test_results{};
+ int _num_iterations{ 1 };
+ bool _throw_errors{ false };
+ bool _stop_on_error{ false };
+ Printer *_printer{ nullptr };
+
+ using create_function = std::unique_ptr<Instrument>();
+ std::map<InstrumentType, create_function *> _available_instruments{};
+
+ InstrumentType _instruments{ InstrumentType::NONE };
+ TestFilter _test_filter{};
+ LogLevel _log_level{ LogLevel::ALL };
+ TestResult *_current_test_result{ nullptr };
+ std::vector<std::string> _test_info{};
+};
+
+template <typename T>
+inline void Framework::add_test_case(std::string test_name, DatasetMode mode, TestCaseFactory::Status status)
+{
+ _test_factories.emplace_back(support::cpp14::make_unique<SimpleTestCaseFactory<T>>(current_suite_name(), std::move(test_name), mode, status));
+}
+
+template <typename T, typename D>
+inline void Framework::add_data_test_case(std::string test_name, DatasetMode mode, TestCaseFactory::Status status, std::string description, D &&data)
+{
+ // WORKAROUND for GCC 4.9
+ // The function should get *it which is tuple but that seems to trigger a
+ // bug in the compiler.
+ auto tmp = std::unique_ptr<DataTestCaseFactory<T, decltype(*std::declval<D>())>>(new DataTestCaseFactory<T, decltype(*std::declval<D>())>(current_suite_name(), std::move(test_name), mode, status,
+ std::move(description), *data));
+ _test_factories.emplace_back(std::move(tmp));
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_FRAMEWORK */
diff --git a/tests/framework/Macros.h b/tests/framework/Macros.h
new file mode 100644
index 0000000000..7aabb75cfc
--- /dev/null
+++ b/tests/framework/Macros.h
@@ -0,0 +1,260 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_FRAMEWORK_MACROS
+#define ARM_COMPUTE_TEST_FRAMEWORK_MACROS
+
+#include "Framework.h"
+#include "Registrars.h"
+#include "TestCase.h"
+
+//
+// TEST SUITE MACROS
+//
+#define TEST_SUITE(SUITE_NAME) \
+ namespace SUITE_NAME##Suite \
+ { \
+ static arm_compute::test::framework::detail::TestSuiteRegistrar SUITE_NAME##Suite_reg{ #SUITE_NAME };
+
+#define TEST_SUITE_END() \
+ static arm_compute::test::framework::detail::TestSuiteRegistrar Suite_end; \
+ }
+//
+// TEST SUITE MACROS END
+//
+
+//
+// HELPER MACROS
+//
+
+#define CONCAT(ARG0, ARG1) ARG0##ARG1
+
+#define VARIADIC_SIZE_IMPL(e0, e1, e2, e3, e4, e5, e6, e7, e8, e9, size, ...) size
+#define VARIADIC_SIZE(...) VARIADIC_SIZE_IMPL(__VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
+
+#define JOIN_PARAM1(OP, param) OP(0, param)
+#define JOIN_PARAM2(OP, param, ...) \
+ OP(1, param) \
+ , JOIN_PARAM1(OP, __VA_ARGS__)
+#define JOIN_PARAM3(OP, param, ...) \
+ OP(2, param) \
+ , JOIN_PARAM2(OP, __VA_ARGS__)
+#define JOIN_PARAM4(OP, param, ...) \
+ OP(3, param) \
+ , JOIN_PARAM3(OP, __VA_ARGS__)
+#define JOIN_PARAM5(OP, param, ...) \
+ OP(4, param) \
+ , JOIN_PARAM4(OP, __VA_ARGS__)
+#define JOIN_PARAM6(OP, param, ...) \
+ OP(5, param) \
+ , JOIN_PARAM5(OP, __VA_ARGS__)
+#define JOIN_PARAM7(OP, param, ...) \
+ OP(6, param) \
+ , JOIN_PARAM6(OP, __VA_ARGS__)
+#define JOIN_PARAM8(OP, param, ...) \
+ OP(7, param) \
+ , JOIN_PARAM7(OP, __VA_ARGS__)
+#define JOIN_PARAM9(OP, param, ...) \
+ OP(8, param) \
+ , JOIN_PARAM8(OP, __VA_ARGS__)
+#define JOIN_PARAM10(OP, param, ...) \
+ OP(9, param) \
+ , JOIN_PARAM9(OP, __VA_ARGS__)
+#define JOIN_PARAM(OP, NUM, ...) \
+ CONCAT(JOIN_PARAM, NUM) \
+ (OP, __VA_ARGS__)
+
+#define MAKE_TYPE_PARAM(i, name) typename T##i
+#define MAKE_ARG_PARAM(i, name) const T##i &name
+#define MAKE_TYPE_PARAMS(...) JOIN_PARAM(MAKE_TYPE_PARAM, VARIADIC_SIZE(__VA_ARGS__), __VA_ARGS__)
+#define MAKE_ARG_PARAMS(...) JOIN_PARAM(MAKE_ARG_PARAM, VARIADIC_SIZE(__VA_ARGS__), __VA_ARGS__)
+
+//
+// TEST CASE MACROS
+//
+#define TEST_CASE_CONSTRUCTOR(TEST_NAME) \
+ TEST_NAME() = default;
+#define DATA_TEST_CASE_CONSTRUCTOR(TEST_NAME, DATASET) \
+ template <typename D> \
+ explicit TEST_NAME(D &&data) : DataTestCase{ std::forward<D>(data) } \
+ { \
+ }
+#define FIXTURE_SETUP(FIXTURE) \
+ void do_setup() override \
+ { \
+ FIXTURE::setup(); \
+ }
+#define FIXTURE_DATA_SETUP(FIXTURE) \
+ void do_setup() override \
+ { \
+ apply(this, &FIXTURE::setup<As...>, _data); \
+ }
+#define FIXTURE_RUN(FIXTURE) \
+ void do_run() override \
+ { \
+ FIXTURE::run(); \
+ }
+#define FIXTURE_TEARDOWN(FIXTURE) \
+ void do_teardown() override \
+ { \
+ FIXTURE::teardown(); \
+ }
+#define TEST_REGISTRAR(TEST_NAME, MODE, STATUS) \
+ static arm_compute::test::framework::detail::TestCaseRegistrar<TEST_NAME> TEST_NAME##_reg \
+ { \
+ #TEST_NAME, MODE, STATUS \
+ }
+#define DATA_TEST_REGISTRAR(TEST_NAME, MODE, STATUS, DATASET) \
+ static arm_compute::test::framework::detail::TestCaseRegistrar<TEST_NAME<decltype(DATASET)::type>> TEST_NAME##_reg \
+ { \
+ #TEST_NAME, MODE, STATUS, DATASET \
+ }
+
+#define TEST_CASE_IMPL(TEST_NAME, MODE, STATUS) \
+ class TEST_NAME : public arm_compute::test::framework::TestCase \
+ { \
+ public: \
+ TEST_CASE_CONSTRUCTOR(TEST_NAME) \
+ void do_run() override; \
+ }; \
+ TEST_REGISTRAR(TEST_NAME, MODE, STATUS); \
+ void TEST_NAME::do_run()
+
+#define TEST_CASE(TEST_NAME, MODE) \
+ TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE)
+#define EXPECTED_FAILURE_TEST_CASE(TEST_NAME, MODE) \
+ TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE)
+#define DISABLED_TEST_CASE(TEST_NAME, MODE) \
+ TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED)
+
+#define DATA_TEST_CASE_IMPL(TEST_NAME, MODE, STATUS, DATASET, ...) \
+ template <typename T> \
+ class TEST_NAME; \
+ template <typename... As> \
+ class TEST_NAME<std::tuple<As...>> : public arm_compute::test::framework::DataTestCase<decltype(DATASET)::type> \
+ { \
+ public: \
+ DATA_TEST_CASE_CONSTRUCTOR(TEST_NAME, DATASET) \
+ void do_run() override \
+ { \
+ arm_compute::test::framework::apply(this, &TEST_NAME::run<As...>, _data); \
+ } \
+ template <MAKE_TYPE_PARAMS(__VA_ARGS__)> \
+ void run(MAKE_ARG_PARAMS(__VA_ARGS__)); \
+ }; \
+ DATA_TEST_REGISTRAR(TEST_NAME, MODE, STATUS, DATASET); \
+ template <typename... As> \
+ template <MAKE_TYPE_PARAMS(__VA_ARGS__)> \
+ void TEST_NAME<std::tuple<As...>>::run(MAKE_ARG_PARAMS(__VA_ARGS__))
+
+#define DATA_TEST_CASE(TEST_NAME, MODE, DATASET, ...) \
+ DATA_TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE, DATASET, __VA_ARGS__)
+#define EXPECTED_FAILURE_DATA_TEST_CASE(TEST_NAME, MODE, DATASET, ...) \
+ DATA_TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE, DATASET, __VA_ARGS__)
+#define DISABLED_DATA_TEST_CASE(TEST_NAME, MODE, DATASET, ...) \
+ DATA_TEST_CASE_IMPL(TEST_NAME, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED, DATASET, __VA_ARGS__)
+
+#define FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, STATUS) \
+ class TEST_NAME : public arm_compute::test::framework::TestCase, public FIXTURE \
+ { \
+ public: \
+ TEST_CASE_CONSTRUCTOR(TEST_NAME) \
+ FIXTURE_SETUP(FIXTURE) \
+ void do_run() override; \
+ FIXTURE_TEARDOWN(FIXTURE) \
+ }; \
+ TEST_REGISTRAR(TEST_NAME, MODE, STATUS); \
+ void TEST_NAME::do_run()
+
+#define FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \
+ FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE)
+#define EXPECTED_FAILURE_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \
+ FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE)
+#define DISABLED_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \
+ FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED)
+
+#define FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, STATUS, DATASET) \
+ template <typename T> \
+ class TEST_NAME; \
+ template <typename... As> \
+ class TEST_NAME<std::tuple<As...>> : public arm_compute::test::framework::DataTestCase<decltype(DATASET)::type>, public FIXTURE \
+ { \
+ public: \
+ DATA_TEST_CASE_CONSTRUCTOR(TEST_NAME, DATASET) \
+ FIXTURE_DATA_SETUP(FIXTURE) \
+ void do_run() override; \
+ FIXTURE_TEARDOWN(FIXTURE) \
+ }; \
+ DATA_TEST_REGISTRAR(TEST_NAME, MODE, STATUS, DATASET); \
+ template <typename... As> \
+ void TEST_NAME<std::tuple<As...>>::do_run()
+
+#define FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \
+ FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE, DATASET)
+#define EXPECTED_FAILURE_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \
+ FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE, DATASET)
+#define DISABLED_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \
+ FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED, DATASET)
+
+#define REGISTER_FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, STATUS) \
+ class TEST_NAME : public arm_compute::test::framework::TestCase, public FIXTURE \
+ { \
+ public: \
+ TEST_CASE_CONSTRUCTOR(TEST_NAME) \
+ FIXTURE_SETUP(FIXTURE) \
+ FIXTURE_RUN(FIXTURE) \
+ FIXTURE_TEARDOWN(FIXTURE) \
+ }; \
+ TEST_REGISTRAR(TEST_NAME, MODE, STATUS)
+
+#define REGISTER_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \
+ REGISTER_FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE)
+#define EXPECTED_FAILURE_REGISTER_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \
+ REGISTER_FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE)
+#define DISABLED_REGISTER_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE) \
+ REGISTER_FIXTURE_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED)
+
+#define REGISTER_FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, STATUS, DATASET) \
+ template <typename T> \
+ class TEST_NAME; \
+ template <typename... As> \
+ class TEST_NAME<std::tuple<As...>> : public arm_compute::test::framework::DataTestCase<decltype(DATASET)::type>, public FIXTURE \
+ { \
+ public: \
+ DATA_TEST_CASE_CONSTRUCTOR(TEST_NAME, DATASET) \
+ FIXTURE_DATA_SETUP(FIXTURE) \
+ FIXTURE_RUN(FIXTURE) \
+ FIXTURE_TEARDOWN(FIXTURE) \
+ }; \
+ DATA_TEST_REGISTRAR(TEST_NAME, MODE, STATUS, DATASET)
+
+#define REGISTER_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \
+ REGISTER_FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::ACTIVE, DATASET)
+#define EXPECTED_FAILURE_REGISTER_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \
+ REGISTER_FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::EXPECTED_FAILURE, DATASET)
+#define DISABLED_REGISTER_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET) \
+ REGISTER_FIXTURE_DATA_TEST_CASE_IMPL(TEST_NAME, FIXTURE, MODE, arm_compute::test::framework::TestCaseFactory::Status::DISABLED, DATASET)
+//
+// TEST CASE MACROS END
+//
+#endif /* ARM_COMPUTE_TEST_FRAMEWORK_MACROS */
diff --git a/tests/framework/Profiler.cpp b/tests/framework/Profiler.cpp
new file mode 100644
index 0000000000..76de9a818f
--- /dev/null
+++ b/tests/framework/Profiler.cpp
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "Profiler.h"
+
+#include <iostream>
+#include <utility>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+void Profiler::add(std::unique_ptr<Instrument> instrument)
+{
+ _instruments.emplace_back(std::move(instrument));
+}
+
+void Profiler::start()
+{
+ for(auto &instrument : _instruments)
+ {
+ instrument->start();
+ }
+}
+
+void Profiler::stop()
+{
+ for(auto &instrument : _instruments)
+ {
+ instrument->stop();
+ }
+
+ for(const auto &instrument : _instruments)
+ {
+ _measurements[instrument->id()].push_back(instrument->measurement());
+ }
+}
+
+const Profiler::MeasurementsMap &Profiler::measurements() const
+{
+ return _measurements;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/Profiler.h b/tests/framework/Profiler.h
new file mode 100644
index 0000000000..1454c0f875
--- /dev/null
+++ b/tests/framework/Profiler.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_PROFILER
+#define ARM_COMPUTE_TEST_PROFILER
+
+#include "instruments/Instrument.h"
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Profiler class to collect benchmark numbers.
+ *
+ * A profiler manages multiple instruments that can collect different types of benchmarking numbers.
+ */
+class Profiler
+{
+public:
+ /** Mapping from instrument ids to their measurements. */
+ using MeasurementsMap = std::map<std::string, std::vector<Instrument::Measurement>>;
+
+ /** Add @p instrument to the performance monitor.
+ *
+ * All added instruments will be used when @ref start or @ref stop are
+ * called to make measurements.
+ *
+ * @param[in] instrument Instrument to be used to measure performance.
+ */
+ void add(std::unique_ptr<Instrument> instrument);
+
+ /** Start all added instruments to measure performance. */
+ void start();
+
+ /** Stop all added instruments. */
+ void stop();
+
+ /** Return measurements for all instruments. */
+ const MeasurementsMap &measurements() const;
+
+private:
+ std::vector<std::unique_ptr<Instrument>> _instruments{};
+ MeasurementsMap _measurements{};
+};
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_PROFILER */
diff --git a/tests/framework/Registrars.h b/tests/framework/Registrars.h
new file mode 100644
index 0000000000..ca23edf0de
--- /dev/null
+++ b/tests/framework/Registrars.h
@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_FRAMEWORK_REGISTRARS
+#define ARM_COMPUTE_TEST_FRAMEWORK_REGISTRARS
+
+#include "DatasetModes.h"
+#include "Framework.h"
+
+#include <string>
+#include <utility>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+namespace detail
+{
+/** Helper class to statically register a test case. */
+template <typename T>
+class TestCaseRegistrar final
+{
+public:
+ /** Add a new test case with the given name to the framework.
+ *
+ * @param[in] test_name Name of the test case.
+ * @param[in] mode Mode in which the test should be activated.
+ * @param[in] status Status of the test case.
+ */
+ TestCaseRegistrar(std::string test_name, DatasetMode mode, TestCaseFactory::Status status);
+
+ /** Add a new data test case with the given name to the framework.
+ *
+ * @param[in] test_name Name of the test case.
+ * @param[in] mode Mode in which the test should be activated.
+ * @param[in] status Status of the test case.
+ * @param[in] dataset Dataset used as input for the test case.
+ */
+ template <typename D>
+ TestCaseRegistrar(std::string test_name, DatasetMode mode, TestCaseFactory::Status status, D &&dataset);
+};
+
+/** Helper class to statically begin and end a test suite. */
+class TestSuiteRegistrar final
+{
+public:
+ /** Remove the last added test suite from the framework. */
+ TestSuiteRegistrar();
+
+ /** Add a new test suite with the given name to the framework.
+ *
+ * @param[in] name Name of the test suite.
+ */
+ TestSuiteRegistrar(std::string name);
+};
+
+template <typename T>
+inline TestCaseRegistrar<T>::TestCaseRegistrar(std::string test_name, DatasetMode mode, TestCaseFactory::Status status)
+{
+ Framework::get().add_test_case<T>(std::move(test_name), mode, status);
+}
+
+template <typename T>
+template <typename D>
+inline TestCaseRegistrar<T>::TestCaseRegistrar(std::string test_name, DatasetMode mode, TestCaseFactory::Status status, D &&dataset)
+{
+ auto it = dataset.begin();
+
+ for(int i = 0; i < dataset.size(); ++i, ++it)
+ {
+ // WORKAROUND for GCC 4.9
+ // The last argument should be *it to pass just the data and not the
+ // iterator.
+ Framework::get().add_data_test_case<T>(test_name, mode, status, it.description(), it);
+ }
+}
+
+inline TestSuiteRegistrar::TestSuiteRegistrar()
+{
+ Framework::get().pop_suite();
+}
+
+inline TestSuiteRegistrar::TestSuiteRegistrar(std::string name)
+{
+ Framework::get().push_suite(std::move(name));
+}
+} // namespace detail
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_FRAMEWORK_REGISTRARS */
diff --git a/tests/framework/SConscript b/tests/framework/SConscript
new file mode 100644
index 0000000000..36632515a3
--- /dev/null
+++ b/tests/framework/SConscript
@@ -0,0 +1,71 @@
+# Copyright (c) 2017 ARM Limited.
+#
+# SPDX-License-Identifier: MIT
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to
+# deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+import SCons
+import os.path
+
+Import('env')
+Import('vars')
+
+# vars is imported from arm_compute:
+variables = [
+ BoolVariable("pmu", "Enable PMU counters", False)
+]
+
+# We need a separate set of Variables for the Help message (Otherwise the global variables will get displayed twice)
+new_options = Variables('scons')
+
+for v in variables:
+ new_options.Add(v)
+ vars.Add(v)
+
+# Clone the environment to make sure we're not polluting the arm_compute one:
+framework_env = env.Clone()
+vars.Update(framework_env)
+
+Help(new_options.GenerateHelpText(framework_env))
+
+if env['os'] == 'android' and framework_env['pmu']:
+ print("pmu=1 is not supported for os=android")
+ Exit(1)
+
+if(env['opencl']):
+ framework_env.Append(CPPDEFINES=['ARM_COMPUTE_CL'])
+
+framework_env.Append(CPPPATH = ["."])
+framework_env.Append(CPPFLAGS=['-Wno-overloaded-virtual'])
+
+files = Glob('*.cpp')
+files += Glob('command_line/*.cpp')
+files += Glob('printers/*.cpp')
+files += Glob('datasets/*.cpp')
+files += Glob('instruments/*.cpp')
+
+if not framework_env['pmu']:
+ # Remove PMU files
+ files = [f for f in files if "PMU" not in os.path.basename(str(f))]
+else:
+ framework_env.Append(CPPDEFINES = ['PMU_ENABLED'])
+
+arm_compute_test_framework = framework_env.StaticLibrary('arm_compute_test_framework', files)
+
+Default(arm_compute_test_framework)
+Export('arm_compute_test_framework')
diff --git a/tests/framework/TestCase.h b/tests/framework/TestCase.h
new file mode 100644
index 0000000000..dbb9312dee
--- /dev/null
+++ b/tests/framework/TestCase.h
@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_TESTCASE
+#define ARM_COMPUTE_TEST_TESTCASE
+
+#include <string>
+#include <utility>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Abstract test case class.
+ *
+ * All test cases have to inherit from this class.
+ */
+class TestCase
+{
+public:
+ virtual void do_setup() {};
+ virtual void do_run() {};
+ virtual void do_teardown() {};
+
+ /** Default destructor. */
+ virtual ~TestCase() = default;
+
+protected:
+ TestCase() = default;
+
+ friend class TestCaseFactory;
+};
+
+template <typename T>
+class DataTestCase : public TestCase
+{
+protected:
+ explicit DataTestCase(T data)
+ : _data{ std::move(data) }
+ {
+ }
+
+ T _data;
+};
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_TESTCASE */
diff --git a/tests/framework/TestCaseFactory.h b/tests/framework/TestCaseFactory.h
new file mode 100644
index 0000000000..b8c8cdbeb0
--- /dev/null
+++ b/tests/framework/TestCaseFactory.h
@@ -0,0 +1,202 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_TEST_CASE_FACTORY
+#define ARM_COMPUTE_TEST_TEST_CASE_FACTORY
+
+#include "DatasetModes.h"
+#include "TestCase.h"
+#include "support/ToolchainSupport.h"
+
+#include <memory>
+#include <string>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Abstract factory class to create test cases. */
+class TestCaseFactory
+{
+public:
+ /** Test case status.
+ *
+ * ACTIVE == Test is run and result is validated. Failure on failed validation.
+ * EXPECTED_FAILURE == Test is run and result is validated. Failure on successful validation.
+ * DISABLED == Test is not run.
+ */
+ enum class Status
+ {
+ ACTIVE,
+ EXPECTED_FAILURE,
+ DISABLED
+ };
+
+ /** Constructor.
+ *
+ * @param[in] suite_name Name of the test suite to which the test case has been added.
+ * @param[in] name Name of the test case.
+ * @param[in] mode Datset mode of the test case.
+ * @param[in] status Status of the test case.
+ * @param[in] description Description of data arguments.
+ */
+ TestCaseFactory(std::string suite_name, std::string name, DatasetMode mode, Status status, std::string description = "");
+
+ /** Default destructor. */
+ virtual ~TestCaseFactory() = default;
+
+ /** Name of the test case.
+ *
+ * @return Name of the test case.
+ */
+ std::string name() const;
+
+ /** Get the mode for which test case will be enabled.
+ *
+ * @return Dataset mode of the test case.
+ */
+ DatasetMode mode() const;
+
+ /** Get the status of the test case.
+ *
+ * @return Status of the test case.
+ */
+ Status status() const;
+
+ /** Factory function to create the test case
+ *
+ * @return Unique pointer to a newly created test case.
+ */
+ virtual std::unique_ptr<TestCase> make() const = 0;
+
+private:
+ const std::string _suite_name;
+ const std::string _test_name;
+ const std::string _data_description;
+ const DatasetMode _mode{ DatasetMode::ALL };
+ const Status _status{ Status::ACTIVE };
+};
+
+/** Implementation of a test case factory to create non-data test cases. */
+template <typename T>
+class SimpleTestCaseFactory final : public TestCaseFactory
+{
+public:
+ /** Default constructor. */
+ using TestCaseFactory::TestCaseFactory;
+
+ std::unique_ptr<TestCase> make() const override;
+};
+
+template <typename T, typename D>
+class DataTestCaseFactory final : public TestCaseFactory
+{
+public:
+ /** Constructor.
+ *
+ * @param[in] suite_name Name of the test suite to which the test case has been added.
+ * @param[in] test_name Name of the test case.
+ * @param[in] mode Mode in which the test case is enabled.
+ * @param[in] status Status of the test case.
+ * @param[in] description Description of data arguments.
+ * @param[in] data Input data for the test case.
+ */
+ DataTestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, Status status, std::string description, const D &data);
+
+ std::unique_ptr<TestCase> make() const override;
+
+private:
+ D _data;
+};
+
+inline TestCaseFactory::TestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, Status status, std::string description)
+ : _suite_name{ std::move(suite_name) }, _test_name{ std::move(test_name) }, _data_description{ std::move(description) }, _mode{ mode }, _status{ status }
+
+{
+}
+
+inline std::string TestCaseFactory::name() const
+{
+ std::string name = _suite_name + "/" + _test_name;
+
+ if(!_data_description.empty())
+ {
+ name += "@" + _data_description;
+ }
+
+ return name;
+}
+
+inline DatasetMode TestCaseFactory::mode() const
+{
+ return _mode;
+}
+
+inline TestCaseFactory::Status TestCaseFactory::status() const
+{
+ return _status;
+}
+
+inline ::std::ostream &operator<<(::std::ostream &stream, TestCaseFactory::Status status)
+{
+ switch(status)
+ {
+ case TestCaseFactory::Status::ACTIVE:
+ stream << "ACTIVE";
+ break;
+ case TestCaseFactory::Status::EXPECTED_FAILURE:
+ stream << "EXPECTED_FAILURE";
+ break;
+ case TestCaseFactory::Status::DISABLED:
+ stream << "DISABLED";
+ break;
+ default:
+ throw std::invalid_argument("Unsupported test case factory status");
+ }
+
+ return stream;
+}
+
+template <typename T>
+inline std::unique_ptr<TestCase> SimpleTestCaseFactory<T>::make() const
+{
+ return support::cpp14::make_unique<T>();
+}
+
+template <typename T, typename D>
+inline DataTestCaseFactory<T, D>::DataTestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, Status status, std::string description, const D &data)
+ : TestCaseFactory{ std::move(suite_name), std::move(test_name), mode, status, std::move(description) }, _data{ data }
+{
+}
+
+template <typename T, typename D>
+inline std::unique_ptr<TestCase> DataTestCaseFactory<T, D>::make() const
+{
+ return support::cpp14::make_unique<T>(_data);
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_TEST_CASE_FACTORY */
diff --git a/tests/framework/TestFilter.cpp b/tests/framework/TestFilter.cpp
new file mode 100644
index 0000000000..0af40c1717
--- /dev/null
+++ b/tests/framework/TestFilter.cpp
@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "TestFilter.h"
+
+#include "Framework.h"
+#include "support/ToolchainSupport.h"
+
+#include <sstream>
+#include <string>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+TestFilter::TestFilter(DatasetMode mode, const std::string &name_filter, const std::string &id_filter)
+ : _dataset_mode{ mode }, _name_filter{ name_filter }, _id_filter{ parse_id_filter(id_filter) }
+{
+}
+
+bool TestFilter::is_selected(const TestInfo &info) const
+{
+ if((info.mode & _dataset_mode) == DatasetMode::DISABLED)
+ {
+ return false;
+ }
+
+ if(!std::regex_search(info.name, _name_filter))
+ {
+ return false;
+ }
+
+ if(!_id_filter.empty())
+ {
+ bool found = false;
+
+ for(const auto range : _id_filter)
+ {
+ if(range.first <= info.id && info.id <= range.second)
+ {
+ found = true;
+ break;
+ }
+ }
+
+ if(!found)
+ {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+TestFilter::Ranges TestFilter::parse_id_filter(const std::string &id_filter) const
+{
+ Ranges ranges;
+ std::string str;
+ bool in_range = false;
+ int value = 0;
+ int start = 0;
+ int end = std::numeric_limits<int>::max();
+
+ std::stringstream stream(id_filter);
+
+ // Get first value
+ std::getline(stream, str, ',');
+
+ if(stream.fail())
+ {
+ return ranges;
+ }
+
+ if(str.find("...") != std::string::npos)
+ {
+ in_range = true;
+ }
+ else
+ {
+ start = support::cpp11::stoi(str);
+ end = start;
+ }
+
+ while(!stream.eof())
+ {
+ std::getline(stream, str, ',');
+
+ if(stream.fail())
+ {
+ break;
+ }
+
+ if(str.find("...") != std::string::npos)
+ {
+ end = std::numeric_limits<int>::max();
+ in_range = true;
+ }
+ else
+ {
+ value = support::cpp11::stoi(str);
+
+ if(in_range || end == value - 1)
+ {
+ end = value;
+ in_range = false;
+ }
+ else
+ {
+ ranges.emplace_back(start, end);
+ start = value;
+ end = start;
+ }
+ }
+ }
+
+ ranges.emplace_back(start, end);
+ return ranges;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/TestFilter.h b/tests/framework/TestFilter.h
new file mode 100644
index 0000000000..f64e73a2ba
--- /dev/null
+++ b/tests/framework/TestFilter.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_TESTFILTER
+#define ARM_COMPUTE_TEST_TESTFILTER
+
+#include "DatasetModes.h"
+
+#include <regex>
+#include <utility>
+#include <vector>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+struct TestInfo;
+
+/** Test filter class.
+ *
+ * Stores information about which test cases are selected for execution. Based
+ * on test name and test id.
+ */
+class TestFilter final
+{
+public:
+ /** Default constructor. All tests selected. */
+ TestFilter() = default;
+
+ /** Constructor.
+ *
+ * The id_filter string has be a comma separated list of test ids. ... can
+ * be used to include a range of tests. For instance, "..., 15" means all
+ * test up to and including 15, "3, 6, ..., 10" means tests 3 and 6 to 10,
+ * and "15, ..." means test 15 and all following.
+ *
+ * @param[in] mode Dataset mode.
+ * @param[in] name_filter Regular expression to filter tests by name. Only matching tests will be executed.
+ * @param[in] id_filter String to match selected test ids. Only matching tests will be executed.
+ */
+ TestFilter(DatasetMode mode, const std::string &name_filter, const std::string &id_filter);
+
+ /** Check if a test case is selected to be executed.
+ *
+ * @param[in] info Test case info.
+ *
+ * @return True if the test case is selected to be executed.
+ */
+ bool is_selected(const TestInfo &info) const;
+
+private:
+ using Ranges = std::vector<std::pair<int, int>>;
+ Ranges parse_id_filter(const std::string &id_filter) const;
+
+ DatasetMode _dataset_mode{ DatasetMode::ALL };
+ std::regex _name_filter{ ".*" };
+ Ranges _id_filter{};
+};
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_TESTFILTER */
diff --git a/tests/framework/TestResult.h b/tests/framework/TestResult.h
new file mode 100644
index 0000000000..e71ef95112
--- /dev/null
+++ b/tests/framework/TestResult.h
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_TESTRESULT
+#define ARM_COMPUTE_TEST_TESTRESULT
+
+#include "Profiler.h"
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Class to store results of a test.
+ *
+ * Currently the execution status and profiling information are stored.
+ */
+struct TestResult
+{
+ /** Execution status of a test. */
+ enum class Status
+ {
+ NOT_RUN,
+ SUCCESS,
+ EXPECTED_FAILURE,
+ FAILED,
+ CRASHED,
+ DISABLED
+ };
+
+ /** Default constructor. */
+ TestResult() = default;
+
+ /** Initialise the result with a status.
+ *
+ * @param[in] status Execution status.
+ */
+ TestResult(Status status)
+ : status{ status }
+ {
+ }
+
+ /** Initialise the result with a status and profiling information.
+ *
+ * @param[in] status Execution status.
+ * @param[in] measurements Profiling information.
+ */
+ TestResult(Status status, const Profiler::MeasurementsMap &measurements)
+ : status{ status }, measurements{ measurements }
+ {
+ }
+
+ Status status{ Status::NOT_RUN }; //< Execution status
+ Profiler::MeasurementsMap measurements{}; //< Profiling information
+};
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_TESTRESULT */
diff --git a/tests/framework/Utils.h b/tests/framework/Utils.h
new file mode 100644
index 0000000000..a9fe0dcaa3
--- /dev/null
+++ b/tests/framework/Utils.h
@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_UTILS
+#define ARM_COMPUTE_TEST_UTILS
+
+#include "support/ToolchainSupport.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstddef>
+#include <limits>
+#include <memory>
+#include <numeric>
+#include <sstream>
+#include <string>
+#include <type_traits>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** @cond */
+namespace detail
+{
+template <int...>
+struct sequence
+{
+};
+
+template <int N, int... Ns>
+struct sequence_generator;
+
+template <int... Ns>
+struct sequence_generator<0, Ns...>
+{
+ using type = sequence<Ns...>;
+};
+
+template <int N, int... Ns>
+struct sequence_generator : sequence_generator < N - 1, N - 1, Ns... >
+{
+};
+
+template <int N>
+using sequence_t = typename sequence_generator<N>::type;
+/** @endcond */
+
+template <typename O, typename F, typename... As, int... S>
+void apply_impl(O *obj, F &&func, const std::tuple<As...> &args, detail::sequence<S...>)
+{
+ (obj->*func)(std::get<S>(args)...);
+}
+} // namespace
+
+template <typename O, typename F, typename... As>
+void apply(O *obj, F &&func, const std::tuple<As...> &args)
+{
+ detail::apply_impl(obj, std::forward<F>(func), args, detail::sequence_t<sizeof...(As)>());
+}
+
+/** Helper function to concatenate multiple strings.
+ *
+ * @param[in] first Iterator pointing to the first element to be concatenated.
+ * @param[in] last Iterator pointing behind the last element to be concatenated.
+ * @param[in] separator String used to join the elements.
+ *
+ * @return String containing all elements joined by @p separator.
+ */
+template <typename T, typename std::enable_if<std::is_same<typename T::value_type, std::string>::value, int>::type = 0>
+std::string join(T first, T last, const std::string &separator)
+{
+ return std::accumulate(std::next(first), last, *first, [&separator](const std::string & base, const std::string & suffix)
+ {
+ return base + separator + suffix;
+ });
+}
+
+/** Helper function to concatenate multiple values.
+ *
+ * All values are converted to std::string using the provided operation before
+ * being joined.
+ *
+ * The signature of op has to be equivalent to
+ * std::string op(const T::value_type &val).
+ *
+ * @param[in] first Iterator pointing to the first element to be concatenated.
+ * @param[in] last Iterator pointing behind the last element to be concatenated.
+ * @param[in] separator String used to join the elements.
+ * @param[in] op Conversion function.
+ *
+ * @return String containing all elements joined by @p separator.
+ */
+template <typename T, typename UnaryOp>
+std::string join(T &&first, T &&last, const std::string &separator, UnaryOp &&op)
+{
+ return std::accumulate(std::next(first), last, op(*first), [&separator, &op](const std::string & base, const typename T::value_type & suffix)
+ {
+ return base + separator + op(suffix);
+ });
+}
+
+/** Helper function to concatenate multiple values.
+ *
+ * All values are converted to std::string using std::to_string before being joined.
+ *
+ * @param[in] first Iterator pointing to the first element to be concatenated.
+ * @param[in] last Iterator pointing behind the last element to be concatenated.
+ * @param[in] separator String used to join the elements.
+ *
+ * @return String containing all elements joined by @p separator.
+ */
+template <typename T, typename std::enable_if<std::is_arithmetic<typename T::value_type>::value, int>::type = 0>
+std::string join(T && first, T && last, const std::string &separator)
+{
+ return join(std::forward<T>(first), std::forward<T>(last), separator, support::cpp11::to_string);
+}
+
+/** Convert string to lower case.
+ *
+ * @param[in] string To be converted string.
+ *
+ * @return Lower case string.
+ */
+inline std::string tolower(std::string string)
+{
+ std::transform(string.begin(), string.end(), string.begin(), [](unsigned char c)
+ {
+ return std::tolower(c);
+ });
+ return string;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_UTILS */
diff --git a/tests/framework/command_line/CommandLineOptions.h b/tests/framework/command_line/CommandLineOptions.h
new file mode 100644
index 0000000000..cb4b794a3e
--- /dev/null
+++ b/tests/framework/command_line/CommandLineOptions.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_COMMANDLINEOPTIONS
+#define ARM_COMPUTE_TEST_COMMANDLINEOPTIONS
+
+#include "EnumListOption.h"
+#include "EnumOption.h"
+#include "ListOption.h"
+#include "Option.h"
+#include "ToggleOption.h"
+
+#endif /* ARM_COMPUTE_TEST_COMMANDLINEOPTIONS */
diff --git a/tests/framework/command_line/CommandLineParser.cpp b/tests/framework/command_line/CommandLineParser.cpp
new file mode 100644
index 0000000000..228b18d842
--- /dev/null
+++ b/tests/framework/command_line/CommandLineParser.cpp
@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "CommandLineParser.h"
+
+#include <iostream>
+#include <regex>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+void CommandLineParser::parse(int argc, char **argv)
+{
+ const std::regex option_regex{ "--((?:no-)?)([^=]+)(?:=(.*))?" };
+
+ const auto set_option = [&](const std::string & option, const std::string & name, const std::string & value)
+ {
+ if(_options.find(name) == _options.end())
+ {
+ _unknown_options.push_back(option);
+ return;
+ }
+
+ const bool success = _options[name]->parse(value);
+
+ if(!success)
+ {
+ _invalid_options.push_back(option);
+ }
+ };
+
+ unsigned int positional_index = 0;
+
+ for(int i = 1; i < argc; ++i)
+ {
+ const std::string option{ argv[i] };
+ std::smatch option_matches;
+
+ if(std::regex_match(option, option_matches, option_regex))
+ {
+ // Boolean option
+ if(option_matches.str(3).empty())
+ {
+ set_option(option, option_matches.str(2), option_matches.str(1).empty() ? "true" : "false");
+ }
+ else
+ {
+ // Can't have "no-" and a value
+ if(!option_matches.str(1).empty())
+ {
+ _invalid_options.emplace_back(option);
+ }
+ else
+ {
+ set_option(option, option_matches.str(2), option_matches.str(3));
+ }
+ }
+ }
+ else
+ {
+ if(positional_index >= _positional_options.size())
+ {
+ _invalid_options.push_back(option);
+ }
+ else
+ {
+ _positional_options[positional_index]->parse(option);
+ ++positional_index;
+ }
+ }
+ }
+}
+
+bool CommandLineParser::validate() const
+{
+ bool is_valid = true;
+
+ for(const auto &option : _options)
+ {
+ if(option.second->is_required() && !option.second->is_set())
+ {
+ is_valid = false;
+ std::cerr << "ERROR: Option '" << option.second->name() << "' is required but not given!\n";
+ }
+ }
+
+ for(const auto &option : _positional_options)
+ {
+ if(option->is_required() && !option->is_set())
+ {
+ is_valid = false;
+ std::cerr << "ERROR: Option '" << option->name() << "' is required but not given!\n";
+ }
+ }
+
+ for(const auto &option : _unknown_options)
+ {
+ std::cerr << "WARNING: Skipping unknown option '" << option << "'!\n";
+ }
+
+ for(const auto &option : _invalid_options)
+ {
+ std::cerr << "WARNING: Skipping invalid option '" << option << "'!\n";
+ }
+
+ return is_valid;
+}
+
+void CommandLineParser::print_help(const std::string &program_name) const
+{
+ std::cout << "usage: " << program_name << " \n";
+
+ for(const auto &option : _options)
+ {
+ std::cout << option.second->help() << "\n";
+ }
+
+ for(const auto &option : _positional_options)
+ {
+ //FIXME: Print help string as well
+ std::cout << option->name() << "\n";
+ }
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/command_line/CommandLineParser.h b/tests/framework/command_line/CommandLineParser.h
new file mode 100644
index 0000000000..adb5214e2f
--- /dev/null
+++ b/tests/framework/command_line/CommandLineParser.h
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_COMMANDLINEPARSER
+#define ARM_COMPUTE_TEST_COMMANDLINEPARSER
+
+#include "../Utils.h"
+#include "Option.h"
+
+#include <map>
+#include <memory>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Class to parse command line arguments. */
+class CommandLineParser final
+{
+public:
+ /** Default constructor. */
+ CommandLineParser() = default;
+
+ /** Function to add a new option to the parser.
+ *
+ * @param[in] name Name of the option. Will be available under --name=VALUE.
+ * @param[in] args Option specific configuration arguments.
+ *
+ * @return Pointer to the option. The option is owned by the parser.
+ */
+ template <typename T, typename... As>
+ T *add_option(const std::string &name, As &&... args);
+
+ /** Function to add a new positional argument to the parser.
+ *
+ * @param[in] args Option specific configuration arguments.
+ *
+ * @return Pointer to the option. The option is owned by the parser.
+ */
+ template <typename T, typename... As>
+ T *add_positional_option(As &&... args);
+
+ /** Parses the command line arguments and updates the options accordingly.
+ *
+ * @param[in] argc Number of arguments.
+ * @param[in] argv Arguments.
+ */
+ void parse(int argc, char **argv);
+
+ /** Validates the previously parsed command line arguments.
+ *
+ * Validation fails if not all required options are provided. Additionally
+ * warnings are generated for options that have illegal values or unknown
+ * options.
+ *
+ * @return True if all required options have been provided.
+ */
+ bool validate() const;
+
+ /** Prints a help message for all configured options.
+ *
+ * @param[in] program_name Name of the program to be used in the help message.
+ */
+ void print_help(const std::string &program_name) const;
+
+private:
+ using OptionsMap = std::map<std::string, std::unique_ptr<Option>>;
+ using PositionalOptionsVector = std::vector<std::unique_ptr<Option>>;
+
+ OptionsMap _options{};
+ PositionalOptionsVector _positional_options{};
+ std::vector<std::string> _unknown_options{};
+ std::vector<std::string> _invalid_options{};
+};
+
+template <typename T, typename... As>
+inline T *CommandLineParser::add_option(const std::string &name, As &&... args)
+{
+ auto result = _options.emplace(name, support::cpp14::make_unique<T>(name, std::forward<As>(args)...));
+ return static_cast<T *>(result.first->second.get());
+}
+
+template <typename T, typename... As>
+inline T *CommandLineParser::add_positional_option(As &&... args)
+{
+ _positional_options.emplace_back(support::cpp14::make_unique<T>(std::forward<As>(args)...));
+ return static_cast<T *>(_positional_options.back().get());
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_COMMANDLINEPARSER */
diff --git a/tests/framework/command_line/EnumListOption.h b/tests/framework/command_line/EnumListOption.h
new file mode 100644
index 0000000000..d19bfbdc0f
--- /dev/null
+++ b/tests/framework/command_line/EnumListOption.h
@@ -0,0 +1,148 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_ENUMLISTOPTION
+#define ARM_COMPUTE_TEST_ENUMLISTOPTION
+
+#include "Option.h"
+
+#include <initializer_list>
+#include <set>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Implementation of an option that accepts any number of values from a fixed set. */
+template <typename T>
+class EnumListOption : public Option
+{
+public:
+ /** Construct option with allowed values.
+ *
+ * @param[in] name Name of the option.
+ * @param[in] allowed_values Set of allowed values for the option.
+ */
+ EnumListOption(std::string name, std::set<T> allowed_values);
+
+ /** Construct option with allowed values, a fixed number of accepted values and default values for the option.
+ *
+ * @param[in] name Name of the option.
+ * @param[in] allowed_values Set of allowed values for the option.
+ * @param[in] default_values Default values.
+ */
+ EnumListOption(std::string name, std::set<T> allowed_values, std::initializer_list<T> &&default_values);
+
+ bool parse(std::string value) override;
+ std::string help() const override;
+ const std::vector<T> &value() const;
+
+private:
+ std::vector<T> _values{};
+ std::set<T> _allowed_values{};
+};
+
+template <typename T>
+inline EnumListOption<T>::EnumListOption(std::string name, std::set<T> allowed_values)
+ : Option{ std::move(name) }, _allowed_values{ std::move(allowed_values) }
+{
+}
+
+template <typename T>
+inline EnumListOption<T>::EnumListOption(std::string name, std::set<T> allowed_values, std::initializer_list<T> &&default_values)
+ : Option{ std::move(name), false, true }, _values{ std::forward<std::initializer_list<T>>(default_values) }, _allowed_values{ std::move(allowed_values) }
+{
+}
+
+template <typename T>
+bool EnumListOption<T>::parse(std::string value)
+{
+ // Remove default values
+ _values.clear();
+ _is_set = true;
+
+ try
+ {
+ std::stringstream stream{ value };
+ std::string item;
+
+ while(!std::getline(stream, item, ',').fail())
+ {
+ std::stringstream item_stream(item);
+ T typed_value{};
+
+ item_stream >> typed_value;
+
+ if(!item_stream.fail())
+ {
+ if(_allowed_values.count(typed_value) == 0)
+ {
+ _values.clear();
+ return false;
+ }
+
+ _values.emplace_back(typed_value);
+ }
+
+ _is_set = _is_set && !item_stream.fail();
+ }
+
+ return _is_set;
+ }
+ catch(const std::invalid_argument &)
+ {
+ return false;
+ }
+}
+
+template <typename T>
+std::string EnumListOption<T>::help() const
+{
+ std::stringstream msg;
+ msg << "--" + name() + "={";
+
+ for(const auto &value : _allowed_values)
+ {
+ msg << value << ",";
+ }
+
+ msg << "}[,{...}[,...]] - " << _help;
+
+ return msg.str();
+}
+
+template <typename T>
+inline const std::vector<T> &EnumListOption<T>::value() const
+{
+ return _values;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_ENUMLISTOPTION */
diff --git a/tests/framework/command_line/EnumOption.h b/tests/framework/command_line/EnumOption.h
new file mode 100644
index 0000000000..1abba77b4b
--- /dev/null
+++ b/tests/framework/command_line/EnumOption.h
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_ENUMOPTION
+#define ARM_COMPUTE_TEST_ENUMOPTION
+
+#include "SimpleOption.h"
+
+#include <set>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Implementation of a simple option that accepts a value from a fixed set. */
+template <typename T>
+class EnumOption : public SimpleOption<T>
+{
+public:
+ /** Construct option with allowed values.
+ *
+ * @param[in] name Name of the option.
+ * @param[in] allowed_values Set of allowed values for the option.
+ */
+ EnumOption(std::string name, std::set<T> allowed_values);
+
+ /** Construct option with allowed values, a fixed number of accepted values and default values for the option.
+ *
+ * @param[in] name Name of the option.
+ * @param[in] allowed_values Set of allowed values for the option.
+ * @param[in] default_value Default value.
+ */
+ EnumOption(std::string name, std::set<T> allowed_values, T default_value);
+
+ bool parse(std::string value) override;
+ std::string help() const override;
+ const T &value() const;
+
+private:
+ std::set<T> _allowed_values{};
+};
+
+template <typename T>
+inline EnumOption<T>::EnumOption(std::string name, std::set<T> allowed_values)
+ : SimpleOption<T>{ std::move(name) }, _allowed_values{ std::move(allowed_values) }
+{
+}
+
+template <typename T>
+inline EnumOption<T>::EnumOption(std::string name, std::set<T> allowed_values, T default_value)
+ : SimpleOption<T>{ std::move(name), std::move(default_value) }, _allowed_values{ std::move(allowed_values) }
+{
+}
+
+template <typename T>
+bool EnumOption<T>::parse(std::string value)
+{
+ try
+ {
+ std::stringstream stream{ value };
+ T typed_value{};
+
+ stream >> typed_value;
+
+ if(!stream.fail())
+ {
+ if(_allowed_values.count(typed_value) == 0)
+ {
+ return false;
+ }
+
+ this->_value = std::move(typed_value);
+ this->_is_set = true;
+ return true;
+ }
+
+ return false;
+ }
+ catch(const std::invalid_argument &)
+ {
+ return false;
+ }
+}
+
+template <typename T>
+std::string EnumOption<T>::help() const
+{
+ std::stringstream msg;
+ msg << "--" + this->name() + "={";
+
+ for(const auto &value : _allowed_values)
+ {
+ msg << value << ",";
+ }
+
+ msg << "} - " << this->_help;
+
+ return msg.str();
+}
+
+template <typename T>
+inline const T &EnumOption<T>::value() const
+{
+ return this->_value;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_ENUMOPTION */
diff --git a/tests/framework/command_line/ListOption.h b/tests/framework/command_line/ListOption.h
new file mode 100644
index 0000000000..8b1bb3d05a
--- /dev/null
+++ b/tests/framework/command_line/ListOption.h
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_LISTOPTION
+#define ARM_COMPUTE_TEST_LISTOPTION
+
+#include "Option.h"
+
+#include <initializer_list>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Implementation of an option that accepts any number of values. */
+template <typename T>
+class ListOption : public Option
+{
+public:
+ using Option::Option;
+
+ /** Construct the option with the given default values.
+ *
+ * @param[in] name Name of the option.
+ * @param[in] default_values Default values.
+ */
+ ListOption(std::string name, std::initializer_list<T> &&default_values);
+
+ bool parse(std::string value) override;
+ std::string help() const override;
+ const std::vector<T> &value() const;
+
+private:
+ std::vector<T> _values{};
+};
+
+template <typename T>
+inline ListOption<T>::ListOption(std::string name, std::initializer_list<T> &&default_values)
+ : Option{ std::move(name), false, true }, _values{ std::forward<std::initializer_list<T>>(default_values) }
+{
+}
+
+template <typename T>
+bool ListOption<T>::parse(std::string value)
+{
+ _is_set = true;
+
+ try
+ {
+ std::stringstream stream{ value };
+ std::string item;
+
+ while(!std::getline(stream, item, ',').fail())
+ {
+ std::stringstream item_stream(item);
+ T typed_value{};
+
+ item_stream >> typed_value;
+
+ if(!item_stream.fail())
+ {
+ _values.emplace_back(typed_value);
+ }
+
+ _is_set = _is_set && !item_stream.fail();
+ }
+
+ return _is_set;
+ }
+ catch(const std::invalid_argument &)
+ {
+ return false;
+ }
+}
+
+template <typename T>
+inline std::string ListOption<T>::help() const
+{
+ return "--" + name() + "=VALUE[,VALUE[,...]] - " + _help;
+}
+
+template <typename T>
+inline const std::vector<T> &ListOption<T>::value() const
+{
+ return _values;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_LISTOPTION */
diff --git a/tests/framework/command_line/Option.cpp b/tests/framework/command_line/Option.cpp
new file mode 100644
index 0000000000..d60c35a698
--- /dev/null
+++ b/tests/framework/command_line/Option.cpp
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "Option.h"
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+Option::Option(std::string name)
+ : _name{ std::move(name) }
+{
+}
+
+Option::Option(std::string name, bool is_required, bool is_set)
+ : _name{ std::move(name) }, _is_required{ is_required }, _is_set{ is_set }
+{
+}
+
+std::string Option::name() const
+{
+ return _name;
+}
+
+void Option::set_required(bool is_required)
+{
+ _is_required = is_required;
+}
+
+void Option::set_help(std::string help)
+{
+ _help = std::move(help);
+}
+
+bool Option::is_required() const
+{
+ return _is_required;
+}
+
+bool Option::is_set() const
+{
+ return _is_set;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/command_line/Option.h b/tests/framework/command_line/Option.h
new file mode 100644
index 0000000000..25cf492b86
--- /dev/null
+++ b/tests/framework/command_line/Option.h
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_OPTIONBASE
+#define ARM_COMPUTE_TEST_OPTIONBASE
+
+#include <string>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Abstract base class for a command line option. */
+class Option
+{
+public:
+ /** Constructor.
+ *
+ * @param[in] name Name of the option.
+ */
+ Option(std::string name);
+
+ /** Constructor.
+ *
+ * @param[in] name Name of the option.
+ * @param[in] is_required Is the option required?
+ * @param[in] is_set Has a value been assigned to the option?
+ */
+ Option(std::string name, bool is_required, bool is_set);
+
+ /** Default destructor. */
+ virtual ~Option() = default;
+
+ /** Parses the given string.
+ *
+ * @param[in] value String representation as passed on the command line.
+ *
+ * @return True if the value could be parsed by the specific subclass.
+ */
+ virtual bool parse(std::string value) = 0;
+
+ /** Help message for the option.
+ *
+ * @return String representing the help message for the specific subclass.
+ */
+ virtual std::string help() const = 0;
+
+ /** Name of the option.
+ *
+ * @return Name of the option.
+ */
+ std::string name() const;
+
+ /** Set whether the option is required.
+ *
+ * @param[in] is_required Pass true if the option is required.
+ */
+ void set_required(bool is_required);
+
+ /** Set the help message for the option.
+ *
+ * @param[in] help Option specific help message.
+ */
+ void set_help(std::string help);
+
+ /** Is the option required?
+ *
+ * @return True if the option is required.
+ */
+ bool is_required() const;
+
+ /** Has a value been assigned to the option?
+ *
+ * @return True if a value has been set.
+ */
+ bool is_set() const;
+
+protected:
+ std::string _name;
+ bool _is_required{ false };
+ bool _is_set{ false };
+ std::string _help{};
+};
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_OPTIONBASE */
diff --git a/tests/framework/command_line/SimpleOption.h b/tests/framework/command_line/SimpleOption.h
new file mode 100644
index 0000000000..e6e8045840
--- /dev/null
+++ b/tests/framework/command_line/SimpleOption.h
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_SIMPLEOPTION
+#define ARM_COMPUTE_TEST_SIMPLEOPTION
+
+#include "Option.h"
+
+#include <sstream>
+#include <stdexcept>
+#include <string>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Implementation of an option that accepts a single value. */
+template <typename T>
+class SimpleOption : public Option
+{
+public:
+ using Option::Option;
+
+ /** Construct the option with the given default value.
+ *
+ * @param[in] name Name of the option.
+ * @param[in] default_value Default value.
+ */
+ SimpleOption(std::string name, T default_value);
+
+ bool parse(std::string value) override;
+ std::string help() const override;
+ const T &value() const;
+
+protected:
+ T _value{};
+};
+
+template <typename T>
+inline SimpleOption<T>::SimpleOption(std::string name, T default_value)
+ : Option{ std::move(name), false, true }, _value{ std::move(default_value) }
+{
+}
+
+template <typename T>
+bool SimpleOption<T>::parse(std::string value)
+{
+ try
+ {
+ std::stringstream stream{ std::move(value) };
+ stream >> _value;
+ _is_set = !stream.fail();
+ return _is_set;
+ }
+ catch(const std::invalid_argument &)
+ {
+ return false;
+ }
+}
+
+template <>
+inline bool SimpleOption<std::string>::parse(std::string value)
+{
+ _value = std::move(value);
+ _is_set = true;
+ return true;
+}
+
+template <typename T>
+inline std::string SimpleOption<T>::help() const
+{
+ return "--" + name() + "=VALUE - " + _help;
+}
+
+template <typename T>
+inline const T &SimpleOption<T>::value() const
+{
+ return _value;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_SIMPLEOPTION */
diff --git a/tests/framework/command_line/ToggleOption.cpp b/tests/framework/command_line/ToggleOption.cpp
new file mode 100644
index 0000000000..df5b1f813b
--- /dev/null
+++ b/tests/framework/command_line/ToggleOption.cpp
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "ToggleOption.h"
+
+#include <utility>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+ToggleOption::ToggleOption(std::string name, bool default_value)
+ : SimpleOption<bool>
+{
+ std::move(name), default_value
+}
+{
+}
+
+bool ToggleOption::parse(std::string value)
+{
+ if(value == "true")
+ {
+ _value = true;
+ _is_set = true;
+ }
+ else if(value == "false")
+ {
+ _value = false;
+ _is_set = true;
+ }
+
+ return _is_set;
+}
+
+std::string ToggleOption::help() const
+{
+ return "--" + name() + ", --no-" + name() + " - " + _help;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/command_line/ToggleOption.h b/tests/framework/command_line/ToggleOption.h
new file mode 100644
index 0000000000..c440c0ee87
--- /dev/null
+++ b/tests/framework/command_line/ToggleOption.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_TOGGLEOPTION
+#define ARM_COMPUTE_TEST_TOGGLEOPTION
+
+#include "SimpleOption.h"
+
+#include <string>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Implementation of an option that can be either true or false. */
+class ToggleOption : public SimpleOption<bool>
+{
+public:
+ using SimpleOption::SimpleOption;
+
+ /** Construct the option with the given default value.
+ *
+ * @param[in] name Name of the option.
+ * @param[in] default_value Default value.
+ */
+ ToggleOption(std::string name, bool default_value);
+
+ bool parse(std::string value) override;
+ std::string help() const override;
+};
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_TOGGLEOPTION */
diff --git a/tests/framework/datasets/CartesianProductDataset.h b/tests/framework/datasets/CartesianProductDataset.h
new file mode 100644
index 0000000000..f6e45ddc12
--- /dev/null
+++ b/tests/framework/datasets/CartesianProductDataset.h
@@ -0,0 +1,165 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_DATASET_CARTESIAN_PRODUCT
+#define ARM_COMPUTE_TEST_DATASET_CARTESIAN_PRODUCT
+
+#include "Dataset.h"
+
+#include <string>
+#include <tuple>
+#include <utility>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+namespace dataset
+{
+/** Implementation of a dataset representing all combinations of values of the input datasets.
+ *
+ * For example, for the inputs {1, 2} and {3, 4} this dataset virtually
+ * represents the values {(1, 3), (1, 4), (2, 3), (2, 4)}.
+ */
+template <typename T, typename U>
+class CartesianProductDataset : public Dataset
+{
+private:
+ using T_noref = typename std::remove_reference<T>::type;
+ using U_noref = typename std::remove_reference<U>::type;
+ using iter1_type = typename T_noref::iterator;
+ using iter2_type = typename U_noref::iterator;
+
+public:
+ /** Construct dataset from the given datasets.
+ *
+ * @param[in] dataset1 First dataset.
+ * @param[in] dataset2 Second dataset.
+ */
+ CartesianProductDataset(T &&dataset1, U &&dataset2)
+ : _dataset1{ std::forward<T>(dataset1) },
+ _dataset2{ std::forward<U>(dataset2) }
+ {
+ }
+
+ CartesianProductDataset(CartesianProductDataset &&) = default;
+
+ /** Type of the dataset. */
+ using type = decltype(std::tuple_cat(*std::declval<iter1_type>(), *std::declval<iter2_type>()));
+
+ /** Iterator for the dataset. */
+ struct iterator
+ {
+ iterator(const T_noref *dataset1, const U_noref *dataset2)
+ : _iter1{ dataset1->begin() },
+ _dataset2{ dataset2 },
+ _iter2{ dataset2->begin() }
+ {
+ }
+
+ iterator(const iterator &) = default;
+ iterator &operator=(const iterator &) = default;
+ iterator(iterator &&) = default;
+ iterator &operator=(iterator &&) = default;
+
+ ~iterator() = default;
+
+ std::string description() const
+ {
+ return _iter1.description() + ":" + _iter2.description();
+ }
+
+ CartesianProductDataset::type operator*() const
+ {
+ return std::tuple_cat(*_iter1, *_iter2);
+ }
+
+ iterator &operator++()
+ {
+ ++_second_pos;
+
+ if(_second_pos < _dataset2->size())
+ {
+ ++_iter2;
+ }
+ else
+ {
+ _second_pos = 0;
+ _iter2 = _dataset2->begin();
+
+ ++_iter1;
+ }
+
+ return *this;
+ }
+
+ private:
+ iter1_type _iter1;
+ const U_noref *_dataset2;
+ iter2_type _iter2;
+ int _first_pos{ 0 };
+ int _second_pos{ 0 };
+ };
+
+ /** Iterator pointing at the begin of the dataset.
+ *
+ * @return Iterator for the dataset.
+ */
+ iterator begin() const
+ {
+ return iterator(&_dataset1, &_dataset2);
+ }
+
+ /** Size of the dataset.
+ *
+ * @return Number of values in the dataset.
+ */
+ int size() const
+ {
+ return _dataset1.size() * _dataset2.size();
+ }
+
+private:
+ T _dataset1;
+ U _dataset2;
+};
+
+/** Helper function to create a @ref CartesianProductDataset.
+ *
+ * @param[in] dataset1 First dataset.
+ * @param[in] dataset2 Second dataset.
+ *
+ * @return A grid dataset.
+ */
+template <typename T, typename U>
+CartesianProductDataset<T, U> combine(T &&dataset1, U &&dataset2)
+{
+ return CartesianProductDataset<T, U>(std::forward<T>(dataset1), std::forward<U>(dataset2));
+}
+} // namespace dataset
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_DATASET_CARTESIAN_PRODUCT */
diff --git a/tests/framework/datasets/ContainerDataset.h b/tests/framework/datasets/ContainerDataset.h
new file mode 100644
index 0000000000..bdca97cbac
--- /dev/null
+++ b/tests/framework/datasets/ContainerDataset.h
@@ -0,0 +1,148 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_DATASET_CONTAINER
+#define ARM_COMPUTE_TEST_DATASET_CONTAINER
+
+#include "Dataset.h"
+#include "support/ToolchainSupport.h"
+
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+namespace dataset
+{
+/** Base case. Nothing is a container. */
+template <typename T>
+struct is_container : public std::false_type
+{
+};
+
+/** Vector is considered a container. */
+template <typename V, typename A>
+struct is_container<std::vector<V, A>> : public std::true_type
+{
+};
+
+/** Implementation of a dataset created from a container. */
+template <typename T>
+class ContainerDataset : public NamedDataset
+{
+private:
+ using container_value_type = typename T::value_type;
+ using container_const_iterator = typename T::const_iterator;
+
+public:
+ /** Construct dataset with given name and values from the container.
+ *
+ * @param[in] name Description of the values.
+ * @param[in] container Values for the dataset.
+ */
+ ContainerDataset(std::string name, T &&container)
+ : NamedDataset{ std::move(name) }, _container(std::forward<T>(container))
+ {
+ }
+
+ ContainerDataset(ContainerDataset &&) = default;
+
+ /** Type of the dataset. */
+ using type = std::tuple<container_value_type>;
+
+ /** Iterator for the dataset. */
+ struct iterator
+ {
+ iterator(std::string name, container_const_iterator iterator)
+ : _name{ name }, _iterator{ iterator }
+ {
+ }
+
+ std::string description() const
+ {
+ using support::cpp11::to_string;
+ return _name + "=" + to_string(*_iterator);
+ }
+
+ ContainerDataset::type operator*() const
+ {
+ return std::make_tuple(*_iterator);
+ }
+
+ iterator &operator++()
+ {
+ ++_iterator;
+ return *this;
+ }
+
+ private:
+ std::string _name;
+ container_const_iterator _iterator;
+ };
+
+ /** Iterator pointing at the begin of the dataset.
+ *
+ * @return Iterator for the dataset.
+ */
+ iterator begin() const
+ {
+ return iterator(name(), _container.cbegin());
+ }
+
+ /** Size of the dataset.
+ *
+ * @return Number of values in the dataset.
+ */
+ int size() const
+ {
+ return _container.size();
+ }
+
+private:
+ T _container;
+};
+
+/** Helper function to create a @ref ContainerDataset.
+ *
+ * @param[in] name Name of the dataset.
+ * @param[in] values Container.
+ *
+ * @return A container dataset.
+ */
+template <typename T>
+typename std::enable_if<is_container<T>::value, ContainerDataset<T>>::type make(std::string name, T &&values)
+{
+ return ContainerDataset<T>(std::move(name), std::forward<T>(values));
+}
+} // namespace dataset
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_DATASET_CONTAINER */
diff --git a/tests/framework/datasets/Dataset.h b/tests/framework/datasets/Dataset.h
new file mode 100644
index 0000000000..d91673073a
--- /dev/null
+++ b/tests/framework/datasets/Dataset.h
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_DATASET
+#define ARM_COMPUTE_TEST_DATASET
+
+#include <string>
+#include <utility>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+namespace dataset
+{
+/** Abstract dataset base class. */
+class Dataset
+{
+protected:
+ Dataset() = default;
+ ~Dataset() = default;
+
+public:
+ Dataset(Dataset &&) = default;
+};
+
+/** Abstract implementation of a named dataset.
+ *
+ * The name should describe the values of the dataset.
+ */
+class NamedDataset : public Dataset
+{
+protected:
+ /** Construct the dataset with the given name.
+ *
+ * @param[in] name Description of the values.
+ */
+ explicit NamedDataset(std::string name)
+ : _name{ std::move(name) }
+ {
+ }
+
+ ~NamedDataset() = default;
+
+public:
+ NamedDataset(NamedDataset &&) = default;
+
+ /** Return name of the dataset.
+ *
+ * @return Description of the values.
+ */
+ std::string name() const
+ {
+ return _name;
+ }
+
+protected:
+ const std::string _name;
+};
+} // namespace dataset
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_DATASET */
diff --git a/tests/framework/datasets/Datasets.h b/tests/framework/datasets/Datasets.h
new file mode 100644
index 0000000000..c0e5822e17
--- /dev/null
+++ b/tests/framework/datasets/Datasets.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_DATASETS
+#define ARM_COMPUTE_TEST_DATASETS
+
+#include "CartesianProductDataset.h"
+#include "ContainerDataset.h"
+#include "InitializerListDataset.h"
+#include "JoinDataset.h"
+#include "RangeDataset.h"
+#include "SingletonDataset.h"
+#include "ZipDataset.h"
+
+#endif /* ARM_COMPUTE_TEST_DATASETS */
diff --git a/tests/framework/datasets/InitializerListDataset.h b/tests/framework/datasets/InitializerListDataset.h
new file mode 100644
index 0000000000..7d32234fab
--- /dev/null
+++ b/tests/framework/datasets/InitializerListDataset.h
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_DATASET_LIST
+#define ARM_COMPUTE_TEST_DATASET_LIST
+
+#include "Dataset.h"
+#include "support/ToolchainSupport.h"
+
+#include <initializer_list>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+namespace dataset
+{
+/** Implementation of a dataset created from an initializer list. */
+template <typename T>
+class InitializerListDataset final : public NamedDataset
+{
+private:
+ using data_const_iterator = typename std::vector<T>::const_iterator;
+
+public:
+ /** Construct dataset with given name and values from the container.
+ *
+ * @param[in] name Description of the values.
+ * @param[in] list Values for the dataset.
+ */
+ InitializerListDataset(std::string name, std::initializer_list<T> &&list)
+ : NamedDataset{ std::move(name) }, _data(std::forward<std::initializer_list<T>>(list))
+ {
+ }
+
+ InitializerListDataset(InitializerListDataset &&) = default;
+
+ /** Type of the dataset. */
+ using type = std::tuple<T>;
+
+ /** Iterator for the dataset. */
+ struct iterator
+ {
+ iterator(std::string name, data_const_iterator iterator)
+ : _name{ name }, _iterator{ iterator }
+ {
+ }
+
+ std::string description() const
+ {
+ using support::cpp11::to_string;
+ return _name + "=" + to_string(*_iterator);
+ }
+
+ InitializerListDataset::type operator*() const
+ {
+ return std::make_tuple(*_iterator);
+ }
+
+ iterator &operator++()
+ {
+ ++_iterator;
+ return *this;
+ }
+
+ private:
+ std::string _name;
+ data_const_iterator _iterator;
+ };
+
+ /** Iterator pointing at the begin of the dataset.
+ *
+ * @return Iterator for the dataset.
+ */
+ iterator begin() const
+ {
+ return iterator(name(), _data.cbegin());
+ }
+
+ /** Size of the dataset.
+ *
+ * @return Number of values in the dataset.
+ */
+ int size() const
+ {
+ return _data.size();
+ }
+
+private:
+ std::vector<T> _data;
+};
+
+/** Helper function to create an @ref InitializerListDataset.
+ *
+ * @param[in] name Name of the dataset.
+ * @param[in] list Initializer list.
+ *
+ * @return An initializer list dataset.
+ */
+template <typename T>
+InitializerListDataset<T> make(std::string name, std::initializer_list<T> &&list)
+{
+ return InitializerListDataset<T>(std::move(name), std::forward<std::initializer_list<T>>(list));
+}
+} // namespace dataset
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_DATASET_LIST */
diff --git a/tests/framework/datasets/JoinDataset.h b/tests/framework/datasets/JoinDataset.h
new file mode 100644
index 0000000000..eded6e0259
--- /dev/null
+++ b/tests/framework/datasets/JoinDataset.h
@@ -0,0 +1,148 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_DATASET_JOIN
+#define ARM_COMPUTE_TEST_DATASET_JOIN
+
+#include "Dataset.h"
+
+#include <string>
+#include <tuple>
+#include <utility>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+namespace dataset
+{
+/** Implementation of a dataset representing the concatenation of the input datasets.
+ *
+ * For example, for the inputs {1, 2} and {3, 4} this dataset virtually
+ * represents the values {1, 2, 3, 4}.
+ */
+template <typename T, typename U>
+class JoinDataset : public Dataset
+{
+private:
+ using iter1_type = typename T::iterator;
+ using iter2_type = typename U::iterator;
+
+public:
+ /** Construct dataset from the given datasets.
+ *
+ * @param[in] dataset1 First dataset.
+ * @param[in] dataset2 Second dataset.
+ */
+ JoinDataset(T &&dataset1, U &&dataset2)
+ : _dataset1{ std::forward<T>(dataset1) },
+ _dataset2{ std::forward<U>(dataset2) }
+ {
+ }
+
+ JoinDataset(JoinDataset &&) = default;
+
+ /** Type of the dataset. */
+ using type = typename T::type;
+
+ /** Iterator for the dataset. */
+ struct iterator
+ {
+ iterator(const T *dataset1, const U *dataset2)
+ : _iter1{ dataset1->begin() }, _iter2{ dataset2->begin() }, _first_size{ dataset1->size() }
+ {
+ }
+
+ std::string description() const
+ {
+ return _first_size > 0 ? _iter1.description() : _iter2.description();
+ }
+
+ JoinDataset::type operator*() const
+ {
+ return _first_size > 0 ? *_iter1 : *_iter2;
+ }
+
+ iterator &operator++()
+ {
+ if(_first_size > 0)
+ {
+ --_first_size;
+ ++_iter1;
+ }
+ else
+ {
+ ++_iter2;
+ }
+
+ return *this;
+ }
+
+ private:
+ iter1_type _iter1;
+ iter2_type _iter2;
+ int _first_size;
+ };
+
+ /** Iterator pointing at the begin of the dataset.
+ *
+ * @return Iterator for the dataset.
+ */
+ iterator begin() const
+ {
+ return iterator(&_dataset1, &_dataset2);
+ }
+
+ /** Size of the dataset.
+ *
+ * @return Number of values in the dataset.
+ */
+ int size() const
+ {
+ return _dataset1.size() + _dataset2.size();
+ }
+
+private:
+ T _dataset1;
+ U _dataset2;
+};
+
+/** Helper function to create a @ref JoinDataset.
+ *
+ * @param[in] dataset1 First dataset.
+ * @param[in] dataset2 Second dataset.
+ *
+ * @return A join dataset.
+ */
+template <typename T, typename U>
+JoinDataset<T, U> concat(T &&dataset1, U &&dataset2)
+{
+ return JoinDataset<T, U>(std::forward<T>(dataset1), std::forward<U>(dataset2));
+}
+} // namespace dataset
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_DATASET_JOIN */
diff --git a/tests/framework/datasets/RangeDataset.h b/tests/framework/datasets/RangeDataset.h
new file mode 100644
index 0000000000..637abe0a83
--- /dev/null
+++ b/tests/framework/datasets/RangeDataset.h
@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_DATASET_RANGE
+#define ARM_COMPUTE_TEST_DATASET_RANGE
+
+#include "Dataset.h"
+#include "support/ToolchainSupport.h"
+
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+namespace dataset
+{
+/** Implementation of a dataset created from a range of values.
+ *
+ * The range is inclusive of the first value but exclusive of the last, i.e. [start, end).
+ */
+template <typename T>
+class RangeDataset final : public NamedDataset
+{
+public:
+ /** Construct dataset with given name and values in the specified range.
+ *
+ * @param[in] name Description of the values.
+ * @param[in] start Begin of the range.
+ * @param[in] end End of the range.
+ * @param[in] step Step size.
+ */
+ RangeDataset(std::string name, T start, T end, T step = 1)
+ : NamedDataset{ std::move(name) }, _start{ start }, _end{ end }, _step{ step }
+ {
+ }
+
+ RangeDataset(RangeDataset &&) = default;
+
+ /** Type of the dataset. */
+ using type = std::tuple<T>;
+
+ /** Iterator for the dataset. */
+ struct iterator
+ {
+ iterator(std::string name, T start, T step)
+ : _name{ name }, _value{ start }, _step{ step }
+ {
+ }
+
+ std::string description() const
+ {
+ using support::cpp11::to_string;
+ return _name + "=" + to_string(_value);
+ }
+
+ RangeDataset::type operator*() const
+ {
+ return std::make_tuple(_value);
+ }
+
+ iterator &operator++()
+ {
+ _value += _step;
+ return *this;
+ }
+
+ private:
+ std::string _name;
+ T _value;
+ T _step;
+ };
+
+ /** Iterator pointing at the begin of the dataset.
+ *
+ * @return Iterator for the dataset.
+ */
+ iterator begin() const
+ {
+ return iterator(name(), _start, _step);
+ }
+
+ /** Size of the dataset.
+ *
+ * @return Number of values in the dataset.
+ */
+ int size() const
+ {
+ return (_end - _start) / std::abs(_step);
+ }
+
+private:
+ T _start;
+ T _end;
+ T _step;
+};
+
+/** Helper function to create a @ref RangeDataset.
+ *
+ * @param[in] name Name of the dataset.
+ * @param[in] start Begin of the range.
+ * @param[in] end End of the range.
+ * @param[in] step Step size.
+ *
+ * @return A range dataset.
+ */
+template <typename T>
+RangeDataset<T> make(std::string name, T start, T end, T step = 1)
+{
+ return RangeDataset<T>(std::move(name), start, end, step);
+}
+} // namespace dataset
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_DATASET_RANGE */
diff --git a/tests/framework/datasets/SingletonDataset.h b/tests/framework/datasets/SingletonDataset.h
new file mode 100644
index 0000000000..1acb5765e5
--- /dev/null
+++ b/tests/framework/datasets/SingletonDataset.h
@@ -0,0 +1,138 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_DATASET_SINGLETON
+#define ARM_COMPUTE_TEST_DATASET_SINGLETON
+
+#include "ContainerDataset.h"
+#include "Dataset.h"
+#include "support/ToolchainSupport.h"
+
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+namespace dataset
+{
+/** Implementation of a dataset holding a single value. */
+template <typename T>
+class SingletonDataset : public NamedDataset
+{
+public:
+ /** Construct dataset with given name and value.
+ *
+ * @param[in] name Description of the value.
+ * @param[in] value Value for the dataset.
+ */
+ SingletonDataset(std::string name, T &&value)
+ : NamedDataset{ std::move(name) }, _value{ std::forward<T>(value) }
+ {
+ }
+
+ SingletonDataset(SingletonDataset &&) = default;
+
+ /** Type of the dataset. */
+ using type = std::tuple<T>;
+
+ /** Iterator for the dataset. */
+ struct iterator
+ {
+ iterator(std::string name, const T *value)
+ : _name{ name }, _value{ value }
+ {
+ }
+
+ ~iterator() = default;
+
+ iterator(const iterator &) = default;
+ iterator &operator=(const iterator &) = default;
+ iterator(iterator &&) = default;
+ iterator &operator=(iterator &&) = default;
+
+ std::string description() const
+ {
+ using support::cpp11::to_string;
+ return _name + "=" + to_string(*_value);
+ }
+
+ SingletonDataset::type operator*() const
+ {
+ return std::make_tuple(*_value);
+ }
+
+ iterator &operator++()
+ {
+ return *this;
+ }
+
+ private:
+ std::string _name;
+ const T *_value;
+ };
+
+ /** Iterator pointing at the begin of the dataset.
+ *
+ * @return Iterator for the dataset.
+ */
+ iterator begin() const
+ {
+ return iterator(name(), &_value);
+ }
+
+ /** Size of the dataset.
+ *
+ * @return Number of values in the dataset.
+ */
+ int size() const
+ {
+ return 1;
+ }
+
+private:
+ T _value;
+};
+
+/** Helper function to create a @ref SingletonDataset.
+ *
+ * @param[in] name Name of the dataset.
+ * @param[in] value Value.
+ *
+ * @return A singleton dataset.
+ */
+template <typename T>
+typename std::enable_if < !is_container<T>::value, SingletonDataset<T >>::type make(std::string name, T &&value)
+{
+ return SingletonDataset<T>(std::move(name), std::forward<T>(value));
+}
+} // namespace dataset
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_DATASET_SINGLETON */
diff --git a/tests/framework/datasets/ZipDataset.h b/tests/framework/datasets/ZipDataset.h
new file mode 100644
index 0000000000..b95b7209a7
--- /dev/null
+++ b/tests/framework/datasets/ZipDataset.h
@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_DATASET_ZIP
+#define ARM_COMPUTE_TEST_DATASET_ZIP
+
+#include "Dataset.h"
+
+#include <string>
+#include <tuple>
+#include <utility>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+namespace dataset
+{
+/** Implementation of a dataset representing pairs of values of the input datasets.
+ *
+ * For example, for the inputs {1, 2} and {3, 4} this dataset virtually
+ * represents the values {(1, 3), (1, 4)}.
+ */
+template <typename T, typename U>
+class ZipDataset : public Dataset
+{
+private:
+ using iter1_type = typename T::iterator;
+ using iter2_type = typename U::iterator;
+
+public:
+ /** Construct dataset from the given datasets.
+ *
+ * @param[in] dataset1 First dataset.
+ * @param[in] dataset2 Second dataset.
+ */
+ ZipDataset(T &&dataset1, U &&dataset2)
+ : _dataset1{ std::forward<T>(dataset1) },
+ _dataset2{ std::forward<U>(dataset2) }
+ {
+ }
+
+ ZipDataset(ZipDataset &&) = default;
+
+ /** Type of the dataset. */
+ using type = decltype(std::tuple_cat(*std::declval<iter1_type>(), *std::declval<iter2_type>()));
+
+ /** Iterator for the dataset. */
+ struct iterator
+ {
+ iterator(iter1_type iter1, iter2_type iter2)
+ : _iter1{ std::move(iter1) }, _iter2{ std::move(iter2) }
+ {
+ }
+
+ std::string description() const
+ {
+ return _iter1.description() + ":" + _iter2.description();
+ }
+
+ ZipDataset::type operator*() const
+ {
+ return std::tuple_cat(*_iter1, *_iter2);
+ }
+
+ iterator &operator++()
+ {
+ ++_iter1;
+ ++_iter2;
+ return *this;
+ }
+
+ private:
+ iter1_type _iter1;
+ iter2_type _iter2;
+ };
+
+ /** Iterator pointing at the begin of the dataset.
+ *
+ * @return Iterator for the dataset.
+ */
+ iterator begin() const
+ {
+ return iterator(_dataset1.begin(), _dataset2.begin());
+ }
+
+ /** Size of the dataset.
+ *
+ * @return Number of values in the dataset.
+ */
+ int size() const
+ {
+ return std::min(_dataset1.size(), _dataset2.size());
+ }
+
+private:
+ T _dataset1;
+ U _dataset2;
+};
+
+/** Helper function to create a @ref ZipDataset.
+ *
+ * @param[in] dataset1 First dataset.
+ * @param[in] dataset2 Second dataset.
+ *
+ * @return A zip dataset.
+ */
+template <typename T, typename U>
+ZipDataset<T, U> zip(T &&dataset1, U &&dataset2)
+{
+ return ZipDataset<T, U>(std::forward<T>(dataset1), std::forward<U>(dataset2));
+}
+} // namespace dataset
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_DATASET_ZIP */
diff --git a/tests/framework/instruments/Instrument.h b/tests/framework/instruments/Instrument.h
new file mode 100644
index 0000000000..895a64738c
--- /dev/null
+++ b/tests/framework/instruments/Instrument.h
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_INSTRUMENT
+#define ARM_COMPUTE_TEST_INSTRUMENT
+
+#include "../Utils.h"
+
+#include <memory>
+#include <ostream>
+#include <string>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Interface for classes that can be used to measure performance. */
+class Instrument
+{
+public:
+ /** Helper function to create an instrument of the given type.
+ *
+ * @return Instance of an instrument of the given type.
+ */
+ template <typename T>
+ static std::unique_ptr<Instrument> make_instrument();
+
+ /** Struct representing measurement consisting of value and unit. */
+ struct Measurement final
+ {
+ Measurement(double value, std::string unit)
+ : value{ value }, unit{ std::move(unit) }
+ {
+ }
+
+ friend std::ostream &operator<<(std::ostream &os, const Measurement &measurement);
+
+ double value;
+ std::string unit;
+ };
+
+ Instrument() = default;
+ Instrument(const Instrument &) = default;
+ Instrument(Instrument &&) = default;
+ Instrument &operator=(const Instrument &) = default;
+ Instrument &operator=(Instrument &&) = default;
+ virtual ~Instrument() = default;
+
+ /** Identifier for the instrument */
+ virtual std::string id() const = 0;
+
+ /** Start measuring. */
+ virtual void start() = 0;
+
+ /** Stop measuring. */
+ virtual void stop() = 0;
+
+ /** Return the latest measurement. */
+ virtual Measurement measurement() const = 0;
+};
+
+inline std::ostream &operator<<(std::ostream &os, const Instrument::Measurement &measurement)
+{
+ os << measurement.value << measurement.unit;
+ return os;
+}
+
+template <typename T>
+inline std::unique_ptr<Instrument> Instrument::make_instrument()
+{
+ return support::cpp14::make_unique<T>();
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_INSTRUMENT */
diff --git a/tests/framework/instruments/Instruments.cpp b/tests/framework/instruments/Instruments.cpp
new file mode 100644
index 0000000000..797a7242ae
--- /dev/null
+++ b/tests/framework/instruments/Instruments.cpp
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "Instruments.h"
+
+#include "../Utils.h"
+
+#include <map>
+#include <stdexcept>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+InstrumentType instrument_type_from_name(const std::string &name)
+{
+ static const std::map<std::string, InstrumentType> types =
+ {
+ { "all", InstrumentType::ALL },
+ { "none", InstrumentType::NONE },
+ { "wall_clock", InstrumentType::WALL_CLOCK_TIMER },
+ { "cycles", InstrumentType::PMU_CYCLE_COUNTER },
+ { "instructions", InstrumentType::PMU_INSTRUCTION_COUNTER },
+ };
+
+ try
+ {
+ return types.at(tolower(name));
+ }
+ catch(const std::out_of_range &)
+ {
+ throw std::invalid_argument(name);
+ }
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/instruments/Instruments.h b/tests/framework/instruments/Instruments.h
new file mode 100644
index 0000000000..034fa168f5
--- /dev/null
+++ b/tests/framework/instruments/Instruments.h
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_INSTRUMENTS
+#define ARM_COMPUTE_TEST_INSTRUMENTS
+
+#include "PMUCounter.h"
+#include "WallClockTimer.h"
+
+#include <sstream>
+#include <string>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+enum class InstrumentType : unsigned int
+{
+ ALL = ~0U,
+ NONE = 0,
+ WALL_CLOCK_TIMER = 1,
+ PMU_CYCLE_COUNTER = 2,
+ PMU_INSTRUCTION_COUNTER = 4
+};
+
+InstrumentType instrument_type_from_name(const std::string &name);
+
+inline InstrumentType operator&(InstrumentType t1, InstrumentType t2)
+{
+ using type = std::underlying_type<InstrumentType>::type;
+ return static_cast<InstrumentType>(static_cast<type>(t1) & static_cast<type>(t2));
+}
+
+inline InstrumentType operator|(InstrumentType t1, InstrumentType t2)
+{
+ using type = std::underlying_type<InstrumentType>::type;
+ return static_cast<InstrumentType>(static_cast<type>(t1) | static_cast<type>(t2));
+}
+
+inline InstrumentType &operator|=(InstrumentType &t1, InstrumentType t2)
+{
+ using type = std::underlying_type<InstrumentType>::type;
+ t1 = static_cast<InstrumentType>(static_cast<type>(t1) | static_cast<type>(t2));
+ return t1;
+}
+
+inline ::std::stringstream &operator>>(::std::stringstream &stream, InstrumentType &instrument)
+{
+ std::string value;
+ stream >> value;
+ instrument = instrument_type_from_name(value);
+ return stream;
+}
+
+inline ::std::stringstream &operator<<(::std::stringstream &stream, InstrumentType instrument)
+{
+ switch(instrument)
+ {
+ case InstrumentType::WALL_CLOCK_TIMER:
+ stream << "WALL_CLOCK_TIMER";
+ break;
+ case InstrumentType::PMU_CYCLE_COUNTER:
+ stream << "PMU_CYCLE_COUNTER";
+ break;
+ case InstrumentType::PMU_INSTRUCTION_COUNTER:
+ stream << "PMU_INSTRUCTION_COUNTER";
+ break;
+ case InstrumentType::ALL:
+ stream << "ALL";
+ break;
+ case InstrumentType::NONE:
+ stream << "NONE";
+ break;
+ default:
+ throw std::invalid_argument("Unsupported instrument type");
+ }
+
+ return stream;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_INSTRUMENTS */
diff --git a/tests/framework/instruments/PMUCounter.cpp b/tests/framework/instruments/PMUCounter.cpp
new file mode 100644
index 0000000000..7994a15862
--- /dev/null
+++ b/tests/framework/instruments/PMUCounter.cpp
@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "PMUCounter.h"
+
+#define _GNU_SOURCE 1
+#include <asm/unistd.h>
+#include <csignal>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <fcntl.h>
+#include <linux/hw_breakpoint.h>
+#include <linux/perf_event.h>
+#include <sys/ioctl.h>
+#include <unistd.h>
+#undef _GNU_SOURCE
+
+#include <stdexcept>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+CycleCounter::CycleCounter()
+{
+ const pid_t pid = getpid();
+
+ struct perf_event_attr perf_config; //NOLINT
+ memset(&perf_config, 0, sizeof(struct perf_event_attr));
+
+ perf_config.config = PERF_COUNT_HW_CPU_CYCLES;
+ perf_config.size = sizeof(struct perf_event_attr);
+ perf_config.type = PERF_TYPE_HARDWARE;
+ // The inherit bit specifies that this counter should count events of child
+ // tasks as well as the task specified
+ perf_config.inherit = 1;
+ // Enables saving of event counts on context switch for inherited tasks
+ perf_config.inherit_stat = 1;
+
+ _fd = syscall(__NR_perf_event_open, &perf_config, pid, -1, -1, 0);
+
+ if(_fd < 0)
+ {
+ throw std::runtime_error("perf_event_open for cycles failed");
+ }
+}
+
+std::string CycleCounter::id() const
+{
+ return "Cycle Counter";
+}
+
+void CycleCounter::start()
+{
+ ioctl(_fd, PERF_EVENT_IOC_RESET, 0);
+ ioctl(_fd, PERF_EVENT_IOC_ENABLE, 0);
+}
+
+void CycleCounter::stop()
+{
+ ioctl(_fd, PERF_EVENT_IOC_DISABLE, 0);
+ read(_fd, &_cycles, sizeof(_cycles));
+}
+
+Instrument::Measurement CycleCounter::measurement() const
+{
+ return Measurement(_cycles, "cycles");
+}
+
+InstructionCounter::InstructionCounter()
+{
+ const pid_t pid = getpid();
+
+ struct perf_event_attr perf_config; //NOLINT
+ memset(&perf_config, 0, sizeof(struct perf_event_attr));
+
+ perf_config.config = PERF_COUNT_HW_INSTRUCTIONS;
+ perf_config.size = sizeof(struct perf_event_attr);
+ perf_config.type = PERF_TYPE_HARDWARE;
+ // The inherit bit specifies that this counter should count events of child
+ // tasks as well as the task specified
+ perf_config.inherit = 1;
+ // Enables saving of event counts on context switch for inherited tasks
+ perf_config.inherit_stat = 1;
+
+ _fd = syscall(__NR_perf_event_open, &perf_config, pid, -1, -1, 0);
+
+ if(_fd < 0)
+ {
+ throw std::runtime_error("perf_event_open for instructions failed");
+ }
+}
+
+std::string InstructionCounter::id() const
+{
+ return "Instruction Counter";
+}
+
+void InstructionCounter::start()
+{
+ ioctl(_fd, PERF_EVENT_IOC_RESET, 0);
+ ioctl(_fd, PERF_EVENT_IOC_ENABLE, 0);
+}
+
+void InstructionCounter::stop()
+{
+ ioctl(_fd, PERF_EVENT_IOC_DISABLE, 0);
+ read(_fd, &_instructions, sizeof(_instructions));
+}
+
+Instrument::Measurement InstructionCounter::measurement() const
+{
+ return Measurement(_instructions, "instructions");
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/instruments/PMUCounter.h b/tests/framework/instruments/PMUCounter.h
new file mode 100644
index 0000000000..f407be602f
--- /dev/null
+++ b/tests/framework/instruments/PMUCounter.h
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_PMU_COUNTER
+#define ARM_COMPUTE_TEST_PMU_COUNTER
+
+#include "Instrument.h"
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Implementation of an instrument to count CPU cycles. */
+class CycleCounter : public Instrument
+{
+public:
+ /** Initialise the cycle counter. */
+ CycleCounter();
+
+ std::string id() const override;
+ void start() override;
+ void stop() override;
+ Measurement measurement() const override;
+
+private:
+ long _fd{ -1 };
+ long long _cycles{ 0 };
+};
+
+/** Implementation of an instrument to count executed CPU instructions. */
+class InstructionCounter : public Instrument
+{
+public:
+ /** Initialise the instruction counter. */
+ InstructionCounter();
+
+ std::string id() const override;
+ void start() override;
+ void stop() override;
+ Measurement measurement() const override;
+
+private:
+ long _fd{ -1 };
+ long long _instructions{ 0 };
+};
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_PMU_COUNTER */
diff --git a/tests/framework/instruments/WallClockTimer.cpp b/tests/framework/instruments/WallClockTimer.cpp
new file mode 100644
index 0000000000..37db0c7f05
--- /dev/null
+++ b/tests/framework/instruments/WallClockTimer.cpp
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "WallClockTimer.h"
+
+#include "../Framework.h"
+#include "../Utils.h"
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+std::string WallClockTimer::id() const
+{
+ return "Wall clock";
+}
+
+void WallClockTimer::start()
+{
+ _start = std::chrono::high_resolution_clock::now();
+}
+
+void WallClockTimer::stop()
+{
+ _stop = std::chrono::high_resolution_clock::now();
+}
+
+Instrument::Measurement WallClockTimer::measurement() const
+{
+ const auto delta = std::chrono::duration_cast<std::chrono::microseconds>(_stop - _start);
+ return Instrument::Measurement(delta.count(), "us");
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/instruments/WallClockTimer.h b/tests/framework/instruments/WallClockTimer.h
new file mode 100644
index 0000000000..b7c390f691
--- /dev/null
+++ b/tests/framework/instruments/WallClockTimer.h
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_WALL_CLOCK_TIMER
+#define ARM_COMPUTE_TEST_WALL_CLOCK_TIMER
+
+#include "Instrument.h"
+
+#include <chrono>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Implementation of an instrument to measure elapsed wall-clock time in milliseconds. */
+class WallClockTimer : public Instrument
+{
+public:
+ std::string id() const override;
+ void start() override;
+ void stop() override;
+ Measurement measurement() const override;
+
+private:
+ std::chrono::high_resolution_clock::time_point _start{};
+ std::chrono::high_resolution_clock::time_point _stop{};
+};
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_WALL_CLOCK_TIMER */
diff --git a/tests/framework/printers/JSONPrinter.cpp b/tests/framework/printers/JSONPrinter.cpp
new file mode 100644
index 0000000000..5b30389eca
--- /dev/null
+++ b/tests/framework/printers/JSONPrinter.cpp
@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "JSONPrinter.h"
+
+#include "tests/framework/Framework.h"
+
+#include <algorithm>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+void JSONPrinter::print_separator(bool &flag)
+{
+ if(flag)
+ {
+ flag = false;
+ }
+ else
+ {
+ *_stream << ",";
+ }
+}
+
+void JSONPrinter::print_entry(const std::string &name, const std::string &value)
+{
+ print_separator(_first_entry);
+
+ *_stream << R"(")" << name << R"(" : ")" << value << R"(")";
+}
+
+void JSONPrinter::print_global_header()
+{
+ *_stream << "{";
+}
+
+void JSONPrinter::print_global_footer()
+{
+ *_stream << "}\n";
+}
+
+void JSONPrinter::print_run_header()
+{
+ print_separator(_first_entry);
+
+ *_stream << R"("tests" : {)";
+}
+
+void JSONPrinter::print_run_footer()
+{
+ *_stream << "}";
+}
+
+void JSONPrinter::print_test_header(const TestInfo &info)
+{
+ print_separator(_first_test);
+
+ _first_test_entry = true;
+ *_stream << R"(")" << info.name << R"(" : {)";
+}
+
+void JSONPrinter::print_test_footer()
+{
+ *_stream << "}";
+}
+
+void JSONPrinter::print_errors_header()
+{
+ print_separator(_first_test_entry);
+
+ _first_error = true;
+ *_stream << R"("errors" : [)";
+}
+
+void JSONPrinter::print_errors_footer()
+{
+ *_stream << "]";
+}
+
+void JSONPrinter::print_error(const std::exception &error)
+{
+ std::stringstream error_log;
+ error_log.str(error.what());
+
+ for(std::string line; !std::getline(error_log, line).fail();)
+ {
+ print_separator(_first_error);
+
+ *_stream << R"(")" << line << R"(")";
+ }
+}
+
+void JSONPrinter::print_measurements(const Profiler::MeasurementsMap &measurements)
+{
+ print_separator(_first_test_entry);
+
+ *_stream << R"("measurements" : {)";
+
+ for(auto i_it = measurements.cbegin(), i_end = measurements.cend(); i_it != i_end;)
+ {
+ *_stream << R"(")" << i_it->first << R"(" : {)";
+
+ auto add_measurements = [](double a, const Instrument::Measurement & b)
+ {
+ return a + b.value;
+ };
+
+ auto cmp_measurements = [](const Instrument::Measurement & a, const Instrument::Measurement & b)
+ {
+ return a.value < b.value;
+ };
+
+ double sum_values = std::accumulate(i_it->second.cbegin(), i_it->second.cend(), 0., add_measurements);
+ int num_values = i_it->second.size();
+ const auto minmax_values = std::minmax_element(i_it->second.begin(), i_it->second.end(), cmp_measurements);
+
+ if(num_values > 2)
+ {
+ sum_values -= minmax_values.first->value + minmax_values.second->value;
+ num_values -= 2;
+ }
+
+ auto measurement_to_string = [](const Instrument::Measurement & measurement)
+ {
+ return support::cpp11::to_string(measurement.value);
+ };
+
+ *_stream << R"("avg" : )" << (sum_values / num_values) << ",";
+ *_stream << R"("min" : )" << minmax_values.first->value << ",";
+ *_stream << R"("max" : )" << minmax_values.second->value << ",";
+ *_stream << R"("raw" : [)" << join(i_it->second.begin(), i_it->second.end(), ",", measurement_to_string) << "],";
+ *_stream << R"("unit" : ")" << minmax_values.first->unit << R"(")";
+ *_stream << "}";
+
+ if(++i_it != i_end)
+ {
+ *_stream << ",";
+ }
+ }
+
+ *_stream << "}";
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/printers/JSONPrinter.h b/tests/framework/printers/JSONPrinter.h
new file mode 100644
index 0000000000..14c8b35cb9
--- /dev/null
+++ b/tests/framework/printers/JSONPrinter.h
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_JSONPRINTER
+#define ARM_COMPUTE_TEST_JSONPRINTER
+
+#include "Printer.h"
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Implementation of a @ref Printer that produces JSON output. */
+class JSONPrinter : public Printer
+{
+public:
+ using Printer::Printer;
+
+ void print_entry(const std::string &name, const std::string &value) override;
+ void print_global_header() override;
+ void print_global_footer() override;
+ void print_run_header() override;
+ void print_run_footer() override;
+ void print_test_header(const TestInfo &info) override;
+ void print_test_footer() override;
+ void print_errors_header() override;
+ void print_errors_footer() override;
+ void print_error(const std::exception &error) override;
+ void print_measurements(const Profiler::MeasurementsMap &measurements) override;
+
+private:
+ void print_separator(bool &flag);
+
+ bool _first_entry{ true };
+ bool _first_test{ true };
+ bool _first_test_entry{ true };
+ bool _first_error{ true };
+};
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_JSONPRINTER */
diff --git a/tests/framework/printers/PrettyPrinter.cpp b/tests/framework/printers/PrettyPrinter.cpp
new file mode 100644
index 0000000000..ec32e5296e
--- /dev/null
+++ b/tests/framework/printers/PrettyPrinter.cpp
@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "PrettyPrinter.h"
+
+#include "tests/framework/Framework.h"
+
+#include <algorithm>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+std::string PrettyPrinter::begin_color(const std::string &color) const
+{
+ if(!_color_output)
+ {
+ return "";
+ }
+
+ return "\033[0;3" + color + "m";
+}
+
+std::string PrettyPrinter::end_color() const
+{
+ if(!_color_output)
+ {
+ return "";
+ }
+
+ return "\033[m";
+}
+
+void PrettyPrinter::set_color_output(bool color_output)
+{
+ _color_output = color_output;
+}
+
+void PrettyPrinter::print_entry(const std::string &name, const std::string &value)
+{
+ *_stream << begin_color("4") << name << " = " << value << end_color() << "\n";
+}
+
+void PrettyPrinter::print_global_header()
+{
+}
+
+void PrettyPrinter::print_global_footer()
+{
+}
+
+void PrettyPrinter::print_run_header()
+{
+}
+
+void PrettyPrinter::print_run_footer()
+{
+}
+
+void PrettyPrinter::print_test_header(const TestInfo &info)
+{
+ *_stream << begin_color("2") << "Running [" << info.id << "] '" << info.name << "'" << end_color() << "\n";
+}
+
+void PrettyPrinter::print_test_footer()
+{
+}
+
+void PrettyPrinter::print_errors_header()
+{
+}
+
+void PrettyPrinter::print_errors_footer()
+{
+}
+
+void PrettyPrinter::print_error(const std::exception &error)
+{
+ *_stream << begin_color("1") << "ERROR: " << error.what() << end_color() << "\n";
+}
+
+void PrettyPrinter::print_measurements(const Profiler::MeasurementsMap &measurements)
+{
+ for(const auto &instrument : measurements)
+ {
+ *_stream << begin_color("3") << " " << instrument.first << ":";
+
+ auto add_measurements = [](double a, const Instrument::Measurement & b)
+ {
+ return a + b.value;
+ };
+
+ auto cmp_measurements = [](const Instrument::Measurement & a, const Instrument::Measurement & b)
+ {
+ return a.value < b.value;
+ };
+
+ double sum_values = std::accumulate(instrument.second.begin(), instrument.second.end(), 0., add_measurements);
+ int num_values = instrument.second.size();
+ const auto minmax_values = std::minmax_element(instrument.second.begin(), instrument.second.end(), cmp_measurements);
+
+ if(num_values > 2)
+ {
+ sum_values -= minmax_values.first->value + minmax_values.second->value;
+ num_values -= 2;
+ }
+
+ Instrument::Measurement avg{ sum_values / num_values, minmax_values.first->unit };
+
+ *_stream << " ";
+ *_stream << "AVG=" << avg << ", ";
+ *_stream << "MIN=" << *minmax_values.first << ", ";
+ *_stream << "MAX=" << *minmax_values.second << end_color() << "\n";
+ }
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/printers/PrettyPrinter.h b/tests/framework/printers/PrettyPrinter.h
new file mode 100644
index 0000000000..fa7b7b2c59
--- /dev/null
+++ b/tests/framework/printers/PrettyPrinter.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_PRETTYPRINTER
+#define ARM_COMPUTE_TEST_PRETTYPRINTER
+
+#include "Printer.h"
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+/** Implementation of a @ref Printer that produces human readable output. */
+class PrettyPrinter : public Printer
+{
+public:
+ using Printer::Printer;
+
+ /** Set if the output is colored.
+ *
+ * @param[in] color_output True if the output is colored.
+ */
+ void set_color_output(bool color_output);
+
+ void print_entry(const std::string &name, const std::string &value) override;
+ void print_global_header() override;
+ void print_global_footer() override;
+ void print_run_header() override;
+ void print_run_footer() override;
+ void print_test_header(const TestInfo &info) override;
+ void print_test_footer() override;
+ void print_errors_header() override;
+ void print_errors_footer() override;
+ void print_error(const std::exception &error) override;
+ void print_measurements(const Profiler::MeasurementsMap &measurements) override;
+
+private:
+ std::string begin_color(const std::string &color) const;
+ std::string end_color() const;
+
+ bool _color_output{ true };
+};
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+
+#endif /* ARM_COMPUTE_TEST_PRETTYPRINTER */
diff --git a/tests/framework/printers/Printer.cpp b/tests/framework/printers/Printer.cpp
new file mode 100644
index 0000000000..e034c2ed43
--- /dev/null
+++ b/tests/framework/printers/Printer.cpp
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "Printer.h"
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+Printer::Printer(std::ostream &stream)
+ : _stream{ &stream }
+{
+}
+
+void Printer::print(const std::string &str)
+{
+ *_stream << str;
+}
+
+void Printer::set_stream(std::ostream &stream)
+{
+ _stream = &stream;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/printers/Printer.h b/tests/framework/printers/Printer.h
new file mode 100644
index 0000000000..198d84d466
--- /dev/null
+++ b/tests/framework/printers/Printer.h
@@ -0,0 +1,129 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_PRINTER
+#define ARM_COMPUTE_TEST_PRINTER
+
+#include "tests/framework/Profiler.h"
+
+#include <fstream>
+#include <iostream>
+#include <ostream>
+#include <stdexcept>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+struct TestInfo;
+
+/** Abstract printer class used by the @ref Framework to present output. */
+class Printer
+{
+public:
+ /** Default constructor.
+ *
+ * Prints values to std::cout.
+ * */
+ Printer() = default;
+
+ /** Construct printer with given output stream.
+ *
+ * @param[out] stream Output stream.
+ */
+ Printer(std::ostream &stream);
+
+ Printer(const Printer &) = delete;
+ Printer &operator=(const Printer &) = delete;
+ Printer(Printer &&) = default;
+ Printer &operator=(Printer &&) = default;
+
+ virtual ~Printer() = default;
+
+ /** Print given string.
+ *
+ * @param[in] str String.
+ */
+ void print(const std::string &str);
+
+ /** Print an entry consisting of a (name, value) pair.
+ *
+ * @param[in] name Description of the value.
+ * @param[in] value Value.
+ */
+ virtual void print_entry(const std::string &name, const std::string &value) = 0;
+
+ /** Print global header. */
+ virtual void print_global_header() = 0;
+
+ /** Print global footer. */
+ virtual void print_global_footer() = 0;
+
+ /** Print header before running all tests. */
+ virtual void print_run_header() = 0;
+
+ /** Print footer after running all tests. */
+ virtual void print_run_footer() = 0;
+
+ /** Print header before a test.
+ *
+ * @param[in] info Test info.
+ */
+ virtual void print_test_header(const TestInfo &info) = 0;
+
+ /** Print footer after a test. */
+ virtual void print_test_footer() = 0;
+
+ /** Print header before errors. */
+ virtual void print_errors_header() = 0;
+
+ /** Print footer after errors. */
+ virtual void print_errors_footer() = 0;
+
+ /** Print test error.
+ *
+ * @param[in] error Description of the error.
+ */
+ virtual void print_error(const std::exception &error) = 0;
+
+ /** Print measurements for a test.
+ *
+ * @param[in] measurements Measurements as collected by a @ref Profiler.
+ */
+ virtual void print_measurements(const Profiler::MeasurementsMap &measurements) = 0;
+
+ /** Set the output stream.
+ *
+ * @param[out] stream Output stream.
+ */
+ void set_stream(std::ostream &stream);
+
+protected:
+ std::ostream *_stream{ &std::cout };
+};
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_PRINTER */
diff --git a/tests/framework/printers/Printers.cpp b/tests/framework/printers/Printers.cpp
new file mode 100644
index 0000000000..6e11b63a9a
--- /dev/null
+++ b/tests/framework/printers/Printers.cpp
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "Printers.h"
+
+#include "../Utils.h"
+
+#include <map>
+#include <stdexcept>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+LogFormat log_format_from_name(const std::string &name)
+{
+ static const std::map<std::string, LogFormat> formats =
+ {
+ { "pretty", LogFormat::PRETTY },
+ { "none", LogFormat::NONE },
+ { "json", LogFormat::JSON },
+ };
+
+ try
+ {
+ return formats.at(tolower(name));
+ }
+ catch(const std::out_of_range &)
+ {
+ throw std::invalid_argument(name);
+ }
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/printers/Printers.h b/tests/framework/printers/Printers.h
new file mode 100644
index 0000000000..53867e2dff
--- /dev/null
+++ b/tests/framework/printers/Printers.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_PRINTERS
+#define ARM_COMPUTE_TEST_PRINTERS
+
+#include "JSONPrinter.h"
+#include "PrettyPrinter.h"
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+enum class LogFormat
+{
+ NONE,
+ JSON,
+ PRETTY
+};
+
+LogFormat log_format_from_name(const std::string &name);
+
+inline ::std::stringstream &operator>>(::std::stringstream &stream, LogFormat &format)
+{
+ std::string value;
+ stream >> value;
+ format = log_format_from_name(value);
+ return stream;
+}
+
+inline ::std::stringstream &operator<<(::std::stringstream &stream, LogFormat format)
+{
+ switch(format)
+ {
+ case LogFormat::PRETTY:
+ stream << "PRETTY";
+ break;
+ case LogFormat::NONE:
+ stream << "NONE";
+ break;
+ case LogFormat::JSON:
+ stream << "JSON";
+ break;
+ default:
+ throw std::invalid_argument("Unsupported log format");
+ }
+
+ return stream;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_PRINTERS */