COMPMID-415: Fix dataset modes

Change-Id: I266e8a22890c914edb3335104f073e79d2bf0ad9
Reviewed-on: http://mpd-gerrit.cambridge.arm.com/80766
Tested-by: Kaizen <jeremy.johnson+kaizengerrit@arm.com>
Reviewed-by: Anthony Barbier <anthony.barbier@arm.com>

26 files changed, 222 insertions, 201 deletions

diff --git a/framework/DatasetModes.cpp b/framework/DatasetModes.cpp
new file mode 100644
index 0000000000..0a9e92a38b
--- /dev/null
+++ b/framework/DatasetModes.cpp
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "DatasetModes.h"
+
+#include <map>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+DatasetMode dataset_mode_from_name(const std::string &name)
+{
+    static const std::map<std::string, DatasetMode> modes =
+    {
+        { "all", DatasetMode::ALL },
+        { "precommit", DatasetMode::PRECOMMIT },
+        { "nightly", DatasetMode::NIGHTLY },
+    };
+
+    try
+    {
+        return modes.at(name);
+    }
+    catch(const std::out_of_range &)
+    {
+        throw std::invalid_argument(name);
+    }
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/DatasetManager.h b/framework/DatasetModes.h
index 2080203286..27638b0504 100644
--- a/tests/DatasetManager.h
+++ b/framework/DatasetModes.h
@@ -21,12 +21,11 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
-#ifndef ARM_COMPUTE_TEST_DATASETMANAGER
-#define ARM_COMPUTE_TEST_DATASETMANAGER
-
-#include "arm_compute/core/TensorShape.h"
-#include "framework/datasets/Datasets.h"
+#ifndef ARM_COMPUTE_TEST_DATASET_MODES
+#define ARM_COMPUTE_TEST_DATASET_MODES
 
+#include <istream>
+#include <ostream>
 #include <sstream>
 #include <stdexcept>
 #include <string>
@@ -35,34 +34,39 @@ namespace arm_compute
 {
 namespace test
 {
-class DatasetManager final
+namespace framework
 {
-public:
-    enum class DatasetMode : unsigned int
-    {
-        ALL       = 0,
-        PRECOMMIT = 1,
-        NIGHTLY   = 2
-    };
-
-    using ShapesDataset = framework::dataset::RangeDataset<std::vector<arm_compute::TensorShape>::const_iterator>;
-
-    static DatasetManager &get();
-
-    void set_mode(DatasetMode mode);
+/** Possible dataset modes. */
+enum class DatasetMode : unsigned int
+{
+    ALL       = ~0U,
+    DISABLED  = 0,
+    PRECOMMIT = 1,
+    NIGHTLY   = 2
+};
 
-    ShapesDataset shapesDataset() const;
+inline DatasetMode operator&(DatasetMode t1, DatasetMode t2)
+{
+    using type = std::underlying_type<DatasetMode>::type;
+    return static_cast<DatasetMode>(static_cast<type>(t1) & static_cast<type>(t2));
+}
 
-private:
-    DatasetManager()  = default;
-    ~DatasetManager() = default;
+inline DatasetMode operator|(DatasetMode t1, DatasetMode t2)
+{
+    using type = std::underlying_type<DatasetMode>::type;
+    return static_cast<DatasetMode>(static_cast<type>(t1) | static_cast<type>(t2));
+}
 
-    DatasetMode _mode{ DatasetMode::ALL };
-};
+inline DatasetMode &operator|=(DatasetMode &t1, DatasetMode t2)
+{
+    using type = std::underlying_type<DatasetMode>::type;
+    t1         = static_cast<DatasetMode>(static_cast<type>(t1) | static_cast<type>(t2));
+    return t1;
+}
 
-DatasetManager::DatasetMode dataset_mode_from_name(const std::string &name);
+DatasetMode dataset_mode_from_name(const std::string &name);
 
-inline ::std::stringstream &operator>>(::std::stringstream &stream, DatasetManager::DatasetMode &mode)
+inline ::std::istream &operator>>(::std::istream &stream, DatasetMode &mode)
 {
     std::string value;
     stream >> value;
@@ -70,17 +74,17 @@ inline ::std::stringstream &operator>>(::std::stringstream &stream, DatasetManag
     return stream;
 }
 
-inline ::std::stringstream &operator<<(::std::stringstream &stream, DatasetManager::DatasetMode mode)
+inline ::std::ostream &operator<<(::std::ostream &stream, DatasetMode mode)
 {
     switch(mode)
     {
-        case DatasetManager::DatasetMode::PRECOMMIT:
+        case DatasetMode::PRECOMMIT:
             stream << "PRECOMMIT";
             break;
-        case DatasetManager::DatasetMode::NIGHTLY:
+        case DatasetMode::NIGHTLY:
             stream << "NIGHTLY";
             break;
-        case DatasetManager::DatasetMode::ALL:
+        case DatasetMode::ALL:
             stream << "ALL";
             break;
         default:
@@ -90,12 +94,13 @@ inline ::std::stringstream &operator<<(::std::stringstream &stream, DatasetManag
     return stream;
 }
 
-inline std::string to_string(const DatasetManager::DatasetMode &mode)
+inline std::string to_string(DatasetMode mode)
 {
     std::stringstream stream;
     stream << mode;
     return stream.str();
 }
+} // namespace framework
 } // namespace test
 } // namespace arm_compute
-#endif /* ARM_COMPUTE_TEST_DATASETMANAGER */
+#endif /* ARM_COMPUTE_TEST_DATASET_MODES */
diff --git a/framework/Framework.cpp b/framework/Framework.cpp
index 72fa1981fc..c25b3e5d4c 100644
--- a/framework/Framework.cpp
+++ b/framework/Framework.cpp
@@ -92,11 +92,12 @@ Framework &Framework::get()
     return instance;
 }
 
-void Framework::init(const std::vector<InstrumentType> &instruments, int num_iterations, const std::string &name_filter, const std::string &id_filter)
+void Framework::init(const std::vector<InstrumentType> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter)
 {
     _test_name_filter = std::regex{ name_filter };
     _test_id_filter   = std::regex{ id_filter };
     _num_iterations   = num_iterations;
+    _dataset_mode     = mode;
 
     _instruments = InstrumentType::NONE;
 
@@ -170,7 +171,27 @@ bool Framework::throw_errors() const
 
 bool Framework::is_enabled(const TestId &id) const
 {
-    return (std::regex_search(support::cpp11::to_string(id.first), _test_id_filter) && std::regex_search(id.second, _test_name_filter));
+    int         test_id = 0;
+    std::string name;
+    DatasetMode mode = DatasetMode::ALL;
+    std::tie(test_id, name, mode) = id;
+
+    if((mode & _dataset_mode) == DatasetMode::DISABLED)
+    {
+        return false;
+    }
+
+    if(!std::regex_search(support::cpp11::to_string(test_id), _test_id_filter))
+    {
+        return false;
+    }
+
+    if(!std::regex_search(name, _test_name_filter))
+    {
+        return false;
+    }
+
+    return true;
 }
 
 void Framework::run_test(TestCaseFactory &test_factory)
@@ -276,7 +297,7 @@ bool Framework::run()
     {
         const std::string test_case_name = test_factory->name();
 
-        if(!is_enabled(TestId(id, test_case_name)))
+        if(!is_enabled(TestId(id, test_case_name, test_factory->mode())))
         {
             log_test_skipped(test_case_name);
         }
@@ -355,9 +376,9 @@ std::vector<Framework::TestId> Framework::test_ids() const
 
     for(const auto &factory : _test_factories)
     {
-        if(is_enabled(TestId(id, factory->name())))
+        if(is_enabled(TestId(id, factory->name(), factory->mode())))
         {
-            ids.emplace_back(id, factory->name());
+            ids.emplace_back(id, factory->name(), factory->mode());
         }
 
         ++id;
diff --git a/framework/Framework.h b/framework/Framework.h
index fbb6b9c04c..a7d8a15541 100644
--- a/framework/Framework.h
+++ b/framework/Framework.h
@@ -24,6 +24,7 @@
 #ifndef ARM_COMPUTE_TEST_FRAMEWORK
 #define ARM_COMPUTE_TEST_FRAMEWORK
 
+#include "DatasetModes.h"
 #include "Profiler.h"
 #include "TestCase.h"
 #include "TestCaseFactory.h"
@@ -60,12 +61,13 @@ class Framework final
 public:
     /** Type of a test identifier.
      *
-     * A test can be identified either via its id or via its name.
+     * A test can be identified either via its id or via its name. Additionally
+     * each test is tagged with the data set mode in which it will be used.
      *
      * @note The mapping between test id and test name is not guaranteed to be
      * stable. It is subject to change as new test are added.
      */
-    using TestId = std::pair<int, std::string>;
+    using TestId = std::tuple<int, std::string, DatasetMode>;
 
     /** Access to the singleton.
      *
@@ -83,10 +85,11 @@ public:
      *
      * @param[in] instruments    Instrument types that will be used for benchmarking.
      * @param[in] num_iterations Number of iterations per test.
+     * @param[in] mode           Dataset mode.
      * @param[in] name_filter    Regular expression to filter tests by name. Only matching tests will be executed.
      * @param[in] id_filter      Regular expression to filter tests by id. Only matching tests will be executed.
      */
-    void init(const std::vector<InstrumentType> &instruments, int num_iterations, const std::string &name_filter, const std::string &id_filter);
+    void init(const std::vector<InstrumentType> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter);
 
     /** Add a new test suite.
      *
@@ -109,18 +112,20 @@ public:
     /** Add a test case to the framework.
      *
      * @param[in] test_name Name of the new test case.
+     * @param[in] mode      Mode in which to include the test.
      */
     template <typename T>
-    void add_test_case(std::string test_name);
+    void add_test_case(std::string test_name, DatasetMode mode);
 
     /** Add a data test case to the framework.
      *
      * @param[in] test_name   Name of the new test case.
+     * @param[in] mode        Mode in which to include the test.
      * @param[in] description Description of @p data.
      * @param[in] data        Data that will be used as input to the test.
      */
     template <typename T, typename D>
-    void add_data_test_case(std::string test_name, std::string description, D &&data);
+    void add_data_test_case(std::string test_name, DatasetMode mode, std::string description, D &&data);
 
     /** Tell the framework that execution of a test starts.
      *
@@ -255,21 +260,22 @@ private:
     InstrumentType _instruments{ InstrumentType::NONE };
     std::regex     _test_name_filter{ ".*" };
     std::regex     _test_id_filter{ ".*" };
+    DatasetMode    _dataset_mode{ DatasetMode::ALL };
 };
 
 template <typename T>
-inline void Framework::add_test_case(std::string test_name)
+inline void Framework::add_test_case(std::string test_name, DatasetMode mode)
 {
-    _test_factories.emplace_back(support::cpp14::make_unique<SimpleTestCaseFactory<T>>(current_suite_name(), std::move(test_name)));
+    _test_factories.emplace_back(support::cpp14::make_unique<SimpleTestCaseFactory<T>>(current_suite_name(), std::move(test_name), mode));
 }
 
 template <typename T, typename D>
-inline void Framework::add_data_test_case(std::string test_name, std::string description, D &&data)
+inline void Framework::add_data_test_case(std::string test_name, DatasetMode mode, std::string description, D &&data)
 {
     // WORKAROUND for GCC 4.9
     // The function should get *it which is tuple but that seems to trigger a
     // bug in the compiler.
-    auto tmp = std::unique_ptr<DataTestCaseFactory<T, decltype(*std::declval<D>())>>(new DataTestCaseFactory<T, decltype(*std::declval<D>())>(current_suite_name(), std::move(test_name),
+    auto tmp = std::unique_ptr<DataTestCaseFactory<T, decltype(*std::declval<D>())>>(new DataTestCaseFactory<T, decltype(*std::declval<D>())>(current_suite_name(), std::move(test_name), mode,
                                                                                      std::move(description), *data));
     _test_factories.emplace_back(std::move(tmp));
 }
diff --git a/framework/Macros.h b/framework/Macros.h
index e3ef71ac56..38eb29c6a1 100644
--- a/framework/Macros.h
+++ b/framework/Macros.h
@@ -76,28 +76,28 @@
     {                             \
         FIXTURE::teardown();      \
     }
-#define TEST_REGISTRAR(TEST_NAME)                                                             \
+#define TEST_REGISTRAR(TEST_NAME, MODE)                                                       \
     static arm_compute::test::framework::detail::TestCaseRegistrar<TEST_NAME> TEST_NAME##_reg \
     {                                                                                         \
-        #TEST_NAME                                                                            \
+        #TEST_NAME, MODE                                                                      \
     }
-#define DATA_TEST_REGISTRAR(TEST_NAME, DATASET)                                               \
+#define DATA_TEST_REGISTRAR(TEST_NAME, MODE, DATASET)                                         \
     static arm_compute::test::framework::detail::TestCaseRegistrar<TEST_NAME> TEST_NAME##_reg \
     {                                                                                         \
-        #TEST_NAME, DATASET                                                                   \
+        #TEST_NAME, MODE, DATASET                                                             \
     }
 
-#define TEST_CASE(TEST_NAME)                                        \
+#define TEST_CASE(TEST_NAME, MODE)                                  \
     class TEST_NAME : public arm_compute::test::framework::TestCase \
     {                                                               \
     public:                                                     \
         TEST_CASE_CONSTRUCTOR(TEST_NAME)                            \
         void do_run() override;                                     \
     };                                                              \
-    TEST_REGISTRAR(TEST_NAME);                                      \
+    TEST_REGISTRAR(TEST_NAME, MODE);                                \
     void TEST_NAME::do_run()
 
-#define DATA_TEST_CASE(TEST_NAME, DATASET, ...)                                                  \
+#define DATA_TEST_CASE(TEST_NAME, MODE, DATASET, ...)                                            \
     class TEST_NAME : public arm_compute::test::framework::DataTestCase<decltype(DATASET)::type> \
     {                                                                                            \
     public:                                                                                  \
@@ -108,10 +108,10 @@
         }                                                                                        \
         void run(__VA_ARGS__);                                                                   \
     };                                                                                           \
-    DATA_TEST_REGISTRAR(TEST_NAME, DATASET);                                                     \
+    DATA_TEST_REGISTRAR(TEST_NAME, MODE, DATASET);                                               \
     void TEST_NAME::run(__VA_ARGS__)
 
-#define FIXTURE_TEST_CASE(TEST_NAME, FIXTURE)                                       \
+#define FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE)                                 \
     class TEST_NAME : public arm_compute::test::framework::TestCase, public FIXTURE \
     {                                                                               \
     public:                                                                     \
@@ -120,10 +120,10 @@
         void do_run() override;                                                     \
         FIXTURE_TEARDOWN(FIXTURE)                                                   \
     };                                                                              \
-    TEST_REGISTRAR(TEST_NAME);                                                      \
+    TEST_REGISTRAR(TEST_NAME, MODE);                                                \
     void TEST_NAME::do_run()
 
-#define FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, DATASET)                                                      \
+#define FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET)                                                \
     class TEST_NAME : public arm_compute::test::framework::DataTestCase<decltype(DATASET)::type>, public FIXTURE \
     {                                                                                                            \
     public:                                                                                                  \
@@ -132,10 +132,10 @@
         void do_run() override;                                                                                  \
         FIXTURE_TEARDOWN(FIXTURE)                                                                                \
     };                                                                                                           \
-    DATA_TEST_REGISTRAR(TEST_NAME, DATASET);                                                                     \
+    DATA_TEST_REGISTRAR(TEST_NAME, MODE, DATASET);                                                               \
     void TEST_NAME::do_run()
 
-#define REGISTER_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE)                              \
+#define REGISTER_FIXTURE_TEST_CASE(TEST_NAME, FIXTURE, MODE)                        \
     class TEST_NAME : public arm_compute::test::framework::TestCase, public FIXTURE \
     {                                                                               \
     public:                                                                     \
@@ -144,9 +144,9 @@
         FIXTURE_RUN(FIXTURE)                                                        \
         FIXTURE_TEARDOWN(FIXTURE)                                                   \
     };                                                                              \
-    TEST_REGISTRAR(TEST_NAME)
+    TEST_REGISTRAR(TEST_NAME, MODE)
 
-#define REGISTER_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, DATASET)                                             \
+#define REGISTER_FIXTURE_DATA_TEST_CASE(TEST_NAME, FIXTURE, MODE, DATASET)                                       \
     class TEST_NAME : public arm_compute::test::framework::DataTestCase<decltype(DATASET)::type>, public FIXTURE \
     {                                                                                                            \
     public:                                                                                                  \
@@ -155,7 +155,7 @@
         FIXTURE_RUN(FIXTURE)                                                                                     \
         FIXTURE_TEARDOWN(FIXTURE)                                                                                \
     };                                                                                                           \
-    DATA_TEST_REGISTRAR(TEST_NAME, DATASET)
+    DATA_TEST_REGISTRAR(TEST_NAME, MODE, DATASET)
 //
 // TEST CASE MACROS END
 //
diff --git a/framework/Registrars.h b/framework/Registrars.h
index 19064c07f5..ddbffabee4 100644
--- a/framework/Registrars.h
+++ b/framework/Registrars.h
@@ -24,6 +24,7 @@
 #ifndef ARM_COMPUTE_TEST_FRAMEWORK_REGISTRARS
 #define ARM_COMPUTE_TEST_FRAMEWORK_REGISTRARS
 
+#include "DatasetModes.h"
 #include "Framework.h"
 
 #include <string>
@@ -45,16 +46,18 @@ public:
     /** Add a new test case with the given name to the framework.
      *
      * @param[in] test_name Name of the test case.
+     * @param[in] mode      Mode in which the test should be activated.
      */
-    TestCaseRegistrar(std::string test_name);
+    TestCaseRegistrar(std::string test_name, DatasetMode mode);
 
     /** Add a new data test case with the given name to the framework.
      *
      * @param[in] test_name Name of the test case.
+     * @param[in] mode      Mode in which the test should be activated.
      * @param[in] dataset   Dataset used as input for the test case.
      */
     template <typename D>
-    TestCaseRegistrar(std::string test_name, D &&dataset);
+    TestCaseRegistrar(std::string test_name, DatasetMode mode, D &&dataset);
 };
 
 /** Helper class to statically begin and end a test suite. */
@@ -72,14 +75,14 @@ public:
 };
 
 template <typename T>
-inline TestCaseRegistrar<T>::TestCaseRegistrar(std::string test_name)
+inline TestCaseRegistrar<T>::TestCaseRegistrar(std::string test_name, DatasetMode mode)
 {
-    Framework::get().add_test_case<T>(std::move(test_name));
+    Framework::get().add_test_case<T>(std::move(test_name), mode);
 }
 
 template <typename T>
 template <typename D>
-inline TestCaseRegistrar<T>::TestCaseRegistrar(std::string test_name, D &&dataset)
+inline TestCaseRegistrar<T>::TestCaseRegistrar(std::string test_name, DatasetMode mode, D &&dataset)
 {
     auto it = dataset.begin();
 
@@ -88,7 +91,7 @@ inline TestCaseRegistrar<T>::TestCaseRegistrar(std::string test_name, D &&datase
         // WORKAROUND for GCC 4.9
         // The last argument should be *it to pass just the data and not the
         // iterator.
-        Framework::get().add_data_test_case<T>(test_name, it.description(), it);
+        Framework::get().add_data_test_case<T>(test_name, mode, it.description(), it);
     }
 }
 
diff --git a/framework/TestCaseFactory.h b/framework/TestCaseFactory.h
index 09e9d198d6..e275e298d6 100644
--- a/framework/TestCaseFactory.h
+++ b/framework/TestCaseFactory.h
@@ -24,6 +24,7 @@
 #ifndef ARM_COMPUTE_TEST_TEST_CASE_FACTORY
 #define ARM_COMPUTE_TEST_TEST_CASE_FACTORY
 
+#include "DatasetModes.h"
 #include "TestCase.h"
 #include "support/ToolchainSupport.h"
 
@@ -44,9 +45,10 @@ public:
      *
      * @param[in] suite_name  Name of the test suite to which the test case has been added.
      * @param[in] name        Name of the test case.
+     * @param[in] mode        Datset mode of the test case.
      * @param[in] description Description of data arguments.
      */
-    TestCaseFactory(std::string suite_name, std::string name, std::string description = "");
+    TestCaseFactory(std::string suite_name, std::string name, DatasetMode mode, std::string description = "");
 
     /** Default destructor. */
     virtual ~TestCaseFactory() = default;
@@ -57,6 +59,12 @@ public:
      */
     std::string name() const;
 
+    /** Get the mode for which test case will be enabled.
+     *
+     * @return Dataset mode of the test case.
+     */
+    DatasetMode mode() const;
+
     /** Factory function to create the test case
      *
      * @return Unique pointer to a newly created test case.
@@ -67,6 +75,7 @@ private:
     const std::string _suite_name;
     const std::string _test_name;
     const std::string _data_description;
+    const DatasetMode _mode{ DatasetMode::ALL };
 };
 
 /** Implementation of a test case factory to create non-data test cases. */
@@ -88,10 +97,11 @@ public:
      *
      * @param[in] suite_name  Name of the test suite to which the test case has been added.
      * @param[in] test_name   Name of the test case.
+     * @param[in] mode        Mode in which the test case is enabled.
      * @param[in] description Description of data arguments.
      * @param[in] data        Input data for the test case.
      */
-    DataTestCaseFactory(std::string suite_name, std::string test_name, std::string description, const D &data);
+    DataTestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, std::string description, const D &data);
 
     std::unique_ptr<TestCase> make() const override;
 
@@ -99,8 +109,8 @@ private:
     D _data;
 };
 
-inline TestCaseFactory::TestCaseFactory(std::string suite_name, std::string test_name, std::string description)
-    : _suite_name{ std::move(suite_name) }, _test_name{ std::move(test_name) }, _data_description{ std::move(description) }
+inline TestCaseFactory::TestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, std::string description)
+    : _suite_name{ std::move(suite_name) }, _test_name{ std::move(test_name) }, _data_description{ std::move(description) }, _mode{ mode }
 {
 }
 
@@ -116,6 +126,11 @@ inline std::string TestCaseFactory::name() const
     return name;
 }
 
+inline DatasetMode TestCaseFactory::mode() const
+{
+    return _mode;
+}
+
 template <typename T>
 inline std::unique_ptr<TestCase> SimpleTestCaseFactory<T>::make() const
 {
@@ -123,8 +138,8 @@ inline std::unique_ptr<TestCase> SimpleTestCaseFactory<T>::make() const
 }
 
 template <typename T, typename D>
-inline DataTestCaseFactory<T, D>::DataTestCaseFactory(std::string suite_name, std::string test_name, std::string description, const D &data)
-    : TestCaseFactory{ std::move(suite_name), std::move(test_name), std::move(description) }, _data{ data }
+inline DataTestCaseFactory<T, D>::DataTestCaseFactory(std::string suite_name, std::string test_name, DatasetMode mode, std::string description, const D &data)
+    : TestCaseFactory{ std::move(suite_name), std::move(test_name), mode, std::move(description) }, _data{ data }
 {
 }
 
diff --git a/tests/DatasetManager.cpp b/tests/DatasetManager.cpp
deleted file mode 100644
index fbc40e64cd..0000000000
--- a/tests/DatasetManager.cpp
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright (c) 2017 ARM Limited.
- *
- * SPDX-License-Identifier: MIT
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to
- * deal in the Software without restriction, including without limitation the
- * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
- * sell copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in all
- * copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#include "DatasetManager.h"
-
-#include <map>
-
-namespace arm_compute
-{
-namespace test
-{
-DatasetManager &DatasetManager::get()
-{
-    static DatasetManager instance;
-    return instance;
-}
-
-void DatasetManager::set_mode(DatasetMode mode)
-{
-    _mode = mode;
-}
-
-DatasetManager::ShapesDataset DatasetManager::shapesDataset() const
-{
-    static const std::string                           name = "Shape";
-    static const std::vector<arm_compute::TensorShape> shapes{ arm_compute::TensorShape(1U), arm_compute::TensorShape(2U), arm_compute::TensorShape(3U), arm_compute::TensorShape(10U), arm_compute::TensorShape(20U), arm_compute::TensorShape(30U) };
-
-    switch(_mode)
-    {
-        case DatasetManager::DatasetMode::PRECOMMIT:
-            return framework::dataset::make(name, shapes.cbegin(), shapes.cbegin() + 3);
-            break;
-        case DatasetManager::DatasetMode::NIGHTLY:
-            return framework::dataset::make(name, shapes.cbegin() + 3, shapes.cend());
-            break;
-        case DatasetManager::DatasetMode::ALL:
-        // Fallthrough
-        default:
-            return framework::dataset::make(name, shapes.cbegin(), shapes.cend());
-    }
-}
-
-DatasetManager::DatasetMode dataset_mode_from_name(const std::string &name)
-{
-    static const std::map<std::string, DatasetManager::DatasetMode> modes =
-    {
-        { "all", DatasetManager::DatasetMode::ALL },
-        { "precommit", DatasetManager::DatasetMode::PRECOMMIT },
-        { "nightly", DatasetManager::DatasetMode::NIGHTLY },
-    };
-
-    try
-    {
-        return modes.at(name);
-    }
-    catch(const std::out_of_range &)
-    {
-        throw std::invalid_argument(name);
-    }
-}
-} // namespace test
-} // namespace arm_compute
diff --git a/tests/benchmark_new/CL/ActivationLayer.cpp b/tests/benchmark_new/CL/ActivationLayer.cpp
index 2ab23d292f..7ce222925c 100644
--- a/tests/benchmark_new/CL/ActivationLayer.cpp
+++ b/tests/benchmark_new/CL/ActivationLayer.cpp
@@ -41,17 +41,17 @@ using CLActivationLayerFixture = ActivationLayerFixture<CLTensor, CLActivationLa
 
 TEST_SUITE(CL)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetActivationLayer, CLActivationLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetActivationLayer, CLActivationLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::AlexNetActivationLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5ActivationLayer, CLActivationLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5ActivationLayer, CLActivationLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::LeNet5ActivationLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetActivationLayer, CLActivationLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetActivationLayer, CLActivationLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::GoogLeNetActivationLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
diff --git a/tests/benchmark_new/CL/ConvolutionLayer.cpp b/tests/benchmark_new/CL/ConvolutionLayer.cpp
index b0faf4ac0f..88eb2fe6ab 100644
--- a/tests/benchmark_new/CL/ConvolutionLayer.cpp
+++ b/tests/benchmark_new/CL/ConvolutionLayer.cpp
@@ -43,17 +43,17 @@ using CLConvolutionLayerFixture = ConvolutionLayerFixture<CLTensor, CLConvolutio
 
 TEST_SUITE(CL)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetConvolutionLayer, CLConvolutionLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetConvolutionLayer, CLConvolutionLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::AlexNetConvolutionLayerDataset(),
                                                                                         framework::dataset::make("Data type", { DataType::F32 })),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5ConvolutionLayer, CLConvolutionLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5ConvolutionLayer, CLConvolutionLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::LeNet5ConvolutionLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetConvolutionLayer, CLConvolutionLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetConvolutionLayer, CLConvolutionLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::GoogLeNetConvolutionLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
diff --git a/tests/benchmark_new/CL/FullyConnectedLayer.cpp b/tests/benchmark_new/CL/FullyConnectedLayer.cpp
index 3c8d450498..f2ada4d47a 100644
--- a/tests/benchmark_new/CL/FullyConnectedLayer.cpp
+++ b/tests/benchmark_new/CL/FullyConnectedLayer.cpp
@@ -43,17 +43,17 @@ using CLFullyConnectedLayerFixture = FullyConnectedLayerFixture<CLTensor, CLFull
 
 TEST_SUITE(CL)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetFullyConnectedLayer, CLFullyConnectedLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetFullyConnectedLayer, CLFullyConnectedLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::AlexNetFullyConnectedLayerDataset(),
                                                                                         framework::dataset::make("Data type", { DataType::F32 })),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5FullyConnectedLayer, CLFullyConnectedLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5FullyConnectedLayer, CLFullyConnectedLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::LeNet5FullyConnectedLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetFullyConnectedLayer, CLFullyConnectedLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetFullyConnectedLayer, CLFullyConnectedLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::GoogLeNetFullyConnectedLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
diff --git a/tests/benchmark_new/CL/GEMM.cpp b/tests/benchmark_new/CL/GEMM.cpp
index d75db500a0..e46175590e 100644
--- a/tests/benchmark_new/CL/GEMM.cpp
+++ b/tests/benchmark_new/CL/GEMM.cpp
@@ -50,7 +50,7 @@ using CLGEMMFixture = GEMMFixture<CLTensor, CLGEMM>;
 
 TEST_SUITE(CL)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetGEMM, CLGEMMFixture, framework::dataset::combine(datasets::GoogLeNetGEMMDataset(), std::move(data_types)));
+REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetGEMM, CLGEMMFixture, framework::DatasetMode::ALL, framework::dataset::combine(datasets::GoogLeNetGEMMDataset(), std::move(data_types)));
 
 TEST_SUITE_END()
 } // namespace test
diff --git a/tests/benchmark_new/CL/NormalizationLayer.cpp b/tests/benchmark_new/CL/NormalizationLayer.cpp
index 95e78c4a18..088d8739d0 100644
--- a/tests/benchmark_new/CL/NormalizationLayer.cpp
+++ b/tests/benchmark_new/CL/NormalizationLayer.cpp
@@ -41,12 +41,12 @@ using CLNormalizationLayerFixture = NormalizationLayerFixture<CLTensor, CLNormal
 
 TEST_SUITE(CL)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetNormalizationLayer, CLNormalizationLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetNormalizationLayer, CLNormalizationLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::AlexNetNormalizationLayerDataset(),
                                                                                         framework::dataset::make("Data type", { DataType::F32 })),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetNormalizationLayer, CLNormalizationLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetNormalizationLayer, CLNormalizationLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::GoogLeNetNormalizationLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
diff --git a/tests/benchmark_new/CL/PoolingLayer.cpp b/tests/benchmark_new/CL/PoolingLayer.cpp
index 1f2229cb10..2dc0951d38 100644
--- a/tests/benchmark_new/CL/PoolingLayer.cpp
+++ b/tests/benchmark_new/CL/PoolingLayer.cpp
@@ -43,17 +43,17 @@ using CLPoolingLayerFixture = PoolingLayerFixture<CLTensor, CLPoolingLayer, cl::
 
 TEST_SUITE(CL)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetPoolingLayer, CLPoolingLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetPoolingLayer, CLPoolingLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::AlexNetPoolingLayerDataset(),
                                                                                         framework::dataset::make("Data type", { DataType::F32 })),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5PoolingLayer, CLPoolingLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5PoolingLayer, CLPoolingLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::LeNet5PoolingLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetPoolingLayer, CLPoolingLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetPoolingLayer, CLPoolingLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::GoogLeNetPoolingLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
diff --git a/tests/benchmark_new/CL/SYSTEM/AlexNet.cpp b/tests/benchmark_new/CL/SYSTEM/AlexNet.cpp
index a0673b94df..90c15c536a 100644
--- a/tests/benchmark_new/CL/SYSTEM/AlexNet.cpp
+++ b/tests/benchmark_new/CL/SYSTEM/AlexNet.cpp
@@ -57,7 +57,7 @@ using CLAlexNetFixture = AlexNetFixture<ICLTensor,
 TEST_SUITE(SYSTEM_TEST)
 TEST_SUITE(CL)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(AlexNet, CLAlexNetFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(AlexNet, CLAlexNetFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::make("Data type", DataType::F32),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
diff --git a/tests/benchmark_new/CL/SYSTEM/LeNet5.cpp b/tests/benchmark_new/CL/SYSTEM/LeNet5.cpp
index 0ee7c0177a..b25fb685bf 100644
--- a/tests/benchmark_new/CL/SYSTEM/LeNet5.cpp
+++ b/tests/benchmark_new/CL/SYSTEM/LeNet5.cpp
@@ -51,7 +51,7 @@ using CLLeNet5Fixture = LeNet5Fixture<CLTensor,
 TEST_SUITE(SYSTEM_TEST)
 TEST_SUITE(CL)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5, CLLeNet5Fixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5, CLLeNet5Fixture, framework::DatasetMode::ALL,
                                 framework::dataset::make("Batches", { 1, 4, 8 }));
 
 TEST_SUITE_END()
diff --git a/tests/benchmark_new/NEON/ActivationLayer.cpp b/tests/benchmark_new/NEON/ActivationLayer.cpp
index 75e98c4ccb..ee8d6633aa 100644
--- a/tests/benchmark_new/NEON/ActivationLayer.cpp
+++ b/tests/benchmark_new/NEON/ActivationLayer.cpp
@@ -41,17 +41,17 @@ using NEActivationLayerFixture = ActivationLayerFixture<Tensor, NEActivationLaye
 
 TEST_SUITE(NEON)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetActivationLayer, NEActivationLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetActivationLayer, NEActivationLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::AlexNetActivationLayerDataset(),
                                                                                         framework::dataset::make("Data type", { DataType::F32, DataType::QS8 })),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5ActivationLayer, NEActivationLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5ActivationLayer, NEActivationLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::LeNet5ActivationLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetActivationLayer, NEActivationLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetActivationLayer, NEActivationLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::GoogLeNetActivationLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
diff --git a/tests/benchmark_new/NEON/ConvolutionLayer.cpp b/tests/benchmark_new/NEON/ConvolutionLayer.cpp
index 57d046e9a5..3d9b2a3acf 100644
--- a/tests/benchmark_new/NEON/ConvolutionLayer.cpp
+++ b/tests/benchmark_new/NEON/ConvolutionLayer.cpp
@@ -43,17 +43,17 @@ using NEConvolutionLayerFixture = ConvolutionLayerFixture<Tensor, NEConvolutionL
 
 TEST_SUITE(NEON)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetConvolutionLayer, NEConvolutionLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetConvolutionLayer, NEConvolutionLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::AlexNetConvolutionLayerDataset(),
                                                                                         framework::dataset::make("Data type", { DataType::F32, DataType::QS8 })),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5ConvolutionLayer, NEConvolutionLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5ConvolutionLayer, NEConvolutionLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::LeNet5ConvolutionLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetConvolutionLayer, NEConvolutionLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetConvolutionLayer, NEConvolutionLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::GoogLeNetConvolutionLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
diff --git a/tests/benchmark_new/NEON/DirectConvolutionLayer.cpp b/tests/benchmark_new/NEON/DirectConvolutionLayer.cpp
index a63cbae8e5..6610fd3ec9 100644
--- a/tests/benchmark_new/NEON/DirectConvolutionLayer.cpp
+++ b/tests/benchmark_new/NEON/DirectConvolutionLayer.cpp
@@ -41,7 +41,7 @@ using NEDirectConvolutionLayerFixture = ConvolutionLayerFixture<Tensor, NEDirect
 
 TEST_SUITE(NEON)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(DirectConvolutionLayer, NEDirectConvolutionLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(DirectConvolutionLayer, NEDirectConvolutionLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::DirectConvolutionLayerDataset(),
                                                                                         framework::dataset::make("Data type", { DataType::F32, DataType::QS8 })),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
diff --git a/tests/benchmark_new/NEON/FullyConnectedLayer.cpp b/tests/benchmark_new/NEON/FullyConnectedLayer.cpp
index 75815ad363..4a72b27665 100644
--- a/tests/benchmark_new/NEON/FullyConnectedLayer.cpp
+++ b/tests/benchmark_new/NEON/FullyConnectedLayer.cpp
@@ -43,17 +43,17 @@ using NEFullyConnectedLayerFixture = FullyConnectedLayerFixture<Tensor, NEFullyC
 
 TEST_SUITE(NEON)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetFullyConnectedLayer, NEFullyConnectedLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetFullyConnectedLayer, NEFullyConnectedLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::AlexNetFullyConnectedLayerDataset(),
                                                                                         framework::dataset::make("Data type", { DataType::F32, DataType::QS8 })),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5FullyConnectedLayer, NEFullyConnectedLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5FullyConnectedLayer, NEFullyConnectedLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::LeNet5FullyConnectedLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetFullyConnectedLayer, NEFullyConnectedLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetFullyConnectedLayer, NEFullyConnectedLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::GoogLeNetFullyConnectedLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
diff --git a/tests/benchmark_new/NEON/GEMM.cpp b/tests/benchmark_new/NEON/GEMM.cpp
index 2ba9a71664..724b514490 100644
--- a/tests/benchmark_new/NEON/GEMM.cpp
+++ b/tests/benchmark_new/NEON/GEMM.cpp
@@ -52,7 +52,7 @@ using NEGEMMFixture = GEMMFixture<Tensor, NEGEMM>;
 
 TEST_SUITE(NEON)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetGEMM, NEGEMMFixture, framework::dataset::combine(datasets::GoogLeNetGEMMDataset(), std::move(data_types)));
+REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetGEMM, NEGEMMFixture, framework::DatasetMode::ALL, framework::dataset::combine(datasets::GoogLeNetGEMMDataset(), std::move(data_types)));
 
 TEST_SUITE_END()
 } // namespace test
diff --git a/tests/benchmark_new/NEON/NormalizationLayer.cpp b/tests/benchmark_new/NEON/NormalizationLayer.cpp
index d9f1323620..8c1f37df0a 100644
--- a/tests/benchmark_new/NEON/NormalizationLayer.cpp
+++ b/tests/benchmark_new/NEON/NormalizationLayer.cpp
@@ -41,12 +41,12 @@ using NENormalizationLayerFixture = NormalizationLayerFixture<Tensor, NENormaliz
 
 TEST_SUITE(NEON)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetNormalizationLayer, NENormalizationLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetNormalizationLayer, NENormalizationLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::AlexNetNormalizationLayerDataset(),
                                                                                         framework::dataset::make("Data type", { DataType::F32, DataType::QS8 })),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetNormalizationLayer, NENormalizationLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetNormalizationLayer, NENormalizationLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::GoogLeNetNormalizationLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
diff --git a/tests/benchmark_new/NEON/PoolingLayer.cpp b/tests/benchmark_new/NEON/PoolingLayer.cpp
index fdaadd0493..6100f40653 100644
--- a/tests/benchmark_new/NEON/PoolingLayer.cpp
+++ b/tests/benchmark_new/NEON/PoolingLayer.cpp
@@ -43,17 +43,17 @@ using NEPoolingLayerFixture = PoolingLayerFixture<Tensor, NEPoolingLayer, neon::
 
 TEST_SUITE(NEON)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetPoolingLayer, NEPoolingLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(AlexNetPoolingLayer, NEPoolingLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::AlexNetPoolingLayerDataset(),
                                                                                         framework::dataset::make("Data type", { DataType::F32, DataType::QS8 })),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5PoolingLayer, NEPoolingLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5PoolingLayer, NEPoolingLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::LeNet5PoolingLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
-REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetPoolingLayer, NEPoolingLayerFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(GoogLeNetPoolingLayer, NEPoolingLayerFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::combine(datasets::GoogLeNetPoolingLayerDataset(),
                                                                                         framework::dataset::make("Data type", DataType::F32)),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
diff --git a/tests/benchmark_new/NEON/SYSTEM/AlexNet.cpp b/tests/benchmark_new/NEON/SYSTEM/AlexNet.cpp
index 2a50bc9d85..8ed70f7a6e 100644
--- a/tests/benchmark_new/NEON/SYSTEM/AlexNet.cpp
+++ b/tests/benchmark_new/NEON/SYSTEM/AlexNet.cpp
@@ -57,7 +57,7 @@ using NEAlexNetFixture = AlexNetFixture<ITensor,
 TEST_SUITE(SYSTEM_TEST)
 TEST_SUITE(NEON)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(AlexNet, NEAlexNetFixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(AlexNet, NEAlexNetFixture, framework::DatasetMode::ALL,
                                 framework::dataset::combine(framework::dataset::make("Data type", { DataType::F32, DataType::QS8 }),
                                                             framework::dataset::make("Batches", { 1, 4, 8 })));
 
diff --git a/tests/benchmark_new/NEON/SYSTEM/LeNet5.cpp b/tests/benchmark_new/NEON/SYSTEM/LeNet5.cpp
index 3957ce76b9..9a5c49e975 100644
--- a/tests/benchmark_new/NEON/SYSTEM/LeNet5.cpp
+++ b/tests/benchmark_new/NEON/SYSTEM/LeNet5.cpp
@@ -51,7 +51,7 @@ using NELeNet5Fixture = LeNet5Fixture<Tensor,
 TEST_SUITE(SYSTEM_TEST)
 TEST_SUITE(NEON)
 
-REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5, NELeNet5Fixture,
+REGISTER_FIXTURE_DATA_TEST_CASE(LeNet5, NELeNet5Fixture, framework::DatasetMode::ALL,
                                 framework::dataset::make("Batches", { 1, 4, 8 }));
 
 TEST_SUITE_END()
diff --git a/tests/benchmark_new/main.cpp b/tests/benchmark_new/main.cpp
index 2db5fc8cef..4c6811e372 100644
--- a/tests/benchmark_new/main.cpp
+++ b/tests/benchmark_new/main.cpp
@@ -21,13 +21,13 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */
+#include "framework/DatasetModes.h"
 #include "framework/Macros.h"
 #include "framework/command_line/CommandLineOptions.h"
 #include "framework/command_line/CommandLineParser.h"
 #include "framework/instruments/Instruments.h"
 #include "framework/printers/Printers.h"
 #include "support/ToolchainSupport.h"
-#include "tests/DatasetManager.h"
 #include "tests/TensorLibrary.h"
 
 #ifdef OPENCL
@@ -74,11 +74,11 @@ int main(int argc, char **argv)
         allowed_instruments.insert(type);
     }
 
-    std::set<DatasetManager::DatasetMode> allowed_modes
+    std::set<framework::DatasetMode> allowed_modes
     {
-        DatasetManager::DatasetMode::PRECOMMIT,
-        DatasetManager::DatasetMode::NIGHTLY,
-        DatasetManager::DatasetMode::ALL
+        framework::DatasetMode::PRECOMMIT,
+        framework::DatasetMode::NIGHTLY,
+        framework::DatasetMode::ALL
     };
 
     std::set<framework::LogFormat> supported_log_formats
@@ -90,7 +90,7 @@ int main(int argc, char **argv)
 
     auto help = parser.add_option<framework::ToggleOption>("help");
     help->set_help("Show this help message");
-    auto dataset_mode = parser.add_option<framework::EnumOption<DatasetManager::DatasetMode>>("mode", allowed_modes, DatasetManager::DatasetMode::ALL);
+    auto dataset_mode = parser.add_option<framework::EnumOption<framework::DatasetMode>>("mode", allowed_modes, framework::DatasetMode::ALL);
     dataset_mode->set_help("For managed datasets select which group to use");
     auto instruments = parser.add_option<framework::EnumListOption<framework::InstrumentType>>("instruments", allowed_instruments, std::initializer_list<framework::InstrumentType> { framework::InstrumentType::ALL });
     instruments->set_help("Set the profiling instruments to use");
@@ -162,7 +162,6 @@ int main(int argc, char **argv)
             }
         }
 
-        DatasetManager::get().set_mode(dataset_mode->value());
         library = support::cpp14::make_unique<TensorLibrary>(assets->value(), seed->value());
         Scheduler::get().set_num_threads(threads->value());
 
@@ -175,7 +174,7 @@ int main(int argc, char **argv)
             printer->print_entry("Dataset mode", to_string(dataset_mode->value()));
         }
 
-        framework.init(instruments->value(), iterations->value(), filter->value(), filter_id->value());
+        framework.init(instruments->value(), iterations->value(), dataset_mode->value(), filter->value(), filter_id->value());
         framework.set_printer(printer.get());
         framework.set_throw_errors(throw_errors->value());
 
@@ -185,7 +184,7 @@ int main(int argc, char **argv)
         {
             for(const auto &id : framework.test_ids())
             {
-                std::cout << "[" << id.first << "] " << id.second << "\n";
+                std::cout << "[" << std::get<0>(id) << ", " << std::get<2>(id) << "] " << std::get<1>(id) << "\n";
             }
         }
         else