COMPMID-2986: Extend test framework options.

Add the following  option to the test framework:
- delay: Allows run delay between tests in seconds.

Signed-off-by: Georgios Pinitas <georgios.pinitas@arm.com>
Change-Id: Ib1b7a1d1ad83c9a99b23f353b1800db075b32bce
Reviewed-on: https://review.mlplatform.org/c/2525
Reviewed-by: Michele Di Giorgio <michele.digiorgio@arm.com>
Tested-by: Arm Jenkins <bsgcomp@arm.com>
Reviewed-by: Gian Marco Iodice <gianmarco.iodice@arm.com>
Comments-Addressed: Arm Jenkins <bsgcomp@arm.com>

7 files changed, 113 insertions, 27 deletions

diff --git a/tests/benchmark_examples/RunExample.cpp b/tests/benchmark_examples/RunExample.cpp
index f3de308bef..613e985707 100644
--- a/tests/benchmark_examples/RunExample.cpp
+++ b/tests/benchmark_examples/RunExample.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-2019 ARM Limited.
+ * Copyright (c) 2018-2020 ARM Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -156,7 +156,13 @@ int run_example(int argc, char **argv, std::unique_ptr<Example> example)
         }
     }
 
-    framework.init(options.instruments->value(), options.iterations->value(), framework::DatasetMode::ALL, "", "", options.log_level->value());
+    // Initialize framework
+    framework::FrameworkConfig fconfig;
+    fconfig.instruments    = options.instruments->value();
+    fconfig.num_iterations = options.iterations->value();
+    fconfig.log_level      = options.log_level->value();
+    framework.init(fconfig);
+
     for(auto &p : printers)
     {
         framework.add_printer(p.get());
diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp
index 44887a9125..4d66b14d67 100644
--- a/tests/framework/Framework.cpp
+++ b/tests/framework/Framework.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2019 ARM Limited.
+ * Copyright (c) 2017-2020 ARM Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -125,14 +125,14 @@ Framework &Framework::get()
     return instance;
 }
 
-void Framework::init(const std::vector<framework::InstrumentsDescription> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter,
-                     LogLevel log_level)
+void Framework::init(const FrameworkConfig &config)
 {
-    _test_filter    = TestFilter(mode, name_filter, id_filter);
-    _num_iterations = num_iterations;
-    _log_level      = log_level;
+    _test_filter    = TestFilter(config.mode, config.name_filter, config.id_filter);
+    _num_iterations = config.num_iterations;
+    _log_level      = config.log_level;
+    _cooldown_sec   = config.cooldown_sec;
 
-    _instruments = std::set<framework::InstrumentsDescription>(instruments.begin(), instruments.end());
+    _instruments = std::set<framework::InstrumentsDescription>(std::begin(config.instruments), std::end(config.instruments));
 }
 
 std::string Framework::current_suite_name() const
@@ -579,6 +579,9 @@ bool Framework::run()
             run_test(test_info, *test_factory);
 
             ++id_run_test;
+
+            // Run test delay
+            sleep_in_seconds(_cooldown_sec);
         }
 
         ++id;
diff --git a/tests/framework/Framework.h b/tests/framework/Framework.h
index c02416f9b5..fb52fd8ffe 100644
--- a/tests/framework/Framework.h
+++ b/tests/framework/Framework.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2019 ARM Limited.
+ * Copyright (c) 2017-2020 ARM Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -54,6 +54,18 @@ namespace test
 {
 namespace framework
 {
+/** Framework configuration structure */
+struct FrameworkConfig
+{
+    std::vector<framework::InstrumentsDescription> instruments{};               /**< Instrument types that will be used for benchmarking. */
+    std::string                                    name_filter{};               /**< Regular expression to filter tests by name. Only matching tests will be executed. */
+    std::string                                    id_filter{};                 /**< String to match selected test ids. Only matching tests will be executed. */
+    DatasetMode                                    mode{ DatasetMode::ALL };    /**< Dataset mode. */
+    int                                            num_iterations{ 1 };         /**< Number of iterations per test. */
+    float                                          cooldown_sec{ -1.f };        /**< Delay between tests in seconds. */
+    LogLevel                                       log_level{ LogLevel::NONE }; /**< Verbosity of the output. */
+};
+
 /** Information about a test case.
  *
  * A test can be identified either via its id or via its name. Additionally
@@ -99,14 +111,9 @@ public:
      *
      * @see TestFilter::TestFilter for the format of the string to filter ids.
      *
-     * @param[in] instruments    Instrument types that will be used for benchmarking.
-     * @param[in] num_iterations Number of iterations per test.
-     * @param[in] mode           Dataset mode.
-     * @param[in] name_filter    Regular expression to filter tests by name. Only matching tests will be executed.
-     * @param[in] id_filter      String to match selected test ids. Only matching tests will be executed.
-     * @param[in] log_level      Verbosity of the output.
+     * @param[in] config Framework configuration meta-data.
      */
-    void init(const std::vector<framework::InstrumentsDescription> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter, LogLevel log_level);
+    void init(const FrameworkConfig &config);
 
     /** Add a new test suite.
      *
@@ -329,6 +336,7 @@ private:
     std::vector<std::unique_ptr<TestCaseFactory>> _test_factories{};
     std::map<TestInfo, TestResult> _test_results{};
     int                    _num_iterations{ 1 };
+    float                  _cooldown_sec{ -1.f };
     bool                   _throw_errors{ false };
     bool                   _stop_on_error{ false };
     bool                   _error_on_missing_assets{ false };
diff --git a/tests/framework/Utils.cpp b/tests/framework/Utils.cpp
new file mode 100644
index 0000000000..589d27493c
--- /dev/null
+++ b/tests/framework/Utils.cpp
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2019-2020 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "Utils.h"
+
+#ifndef NO_MULTI_THREADING
+#include <chrono>
+#include <thread>
+#endif /* NO_MULTI_THREADING */
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+void sleep_in_seconds(float seconds)
+{
+    // Early return on non-positive input
+    if(seconds <= 0.f)
+    {
+        return;
+    }
+
+#ifndef NO_MULTI_THREADING
+    const int64_t us = static_cast<int64_t>(seconds * 1e6);
+    std::this_thread::sleep_for(std::chrono::microseconds(us));
+#endif /* NO_MULTI_THREADING */
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/tests/framework/Utils.h b/tests/framework/Utils.h
index c0442cf9af..0280daaae9 100644
--- a/tests/framework/Utils.h
+++ b/tests/framework/Utils.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017 ARM Limited.
+ * Copyright (c) 2017-2020 ARM Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -170,6 +170,11 @@ inline std::string arithmetic_to_string(T val, int decimal_places = 0)
     return ss.str();
 }
 
+/**  Makes the calling thread to sleep for a specified number of seconds
+ *
+ * @param[in] seconds Amount of seconds to sleep. Will return immediately if less or equal to zero.
+ */
+void sleep_in_seconds(float seconds);
 } // namespace framework
 } // namespace test
 } // namespace arm_compute
diff --git a/tests/main.cpp b/tests/main.cpp
index 604a51fc0f..441e28b2a8 100644
--- a/tests/main.cpp
+++ b/tests/main.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017-2019 ARM Limited.
+ * Copyright (c) 2017-2020 ARM Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -142,6 +142,8 @@ int main(int argc, char **argv)
 #endif /* ARM_COMPUTE_CL */
     auto threads = parser.add_option<utils::SimpleOption<int>>("threads", 1);
     threads->set_help("Number of threads to use");
+    auto cooldown_sec = parser.add_option<utils::SimpleOption<float>>("delay", -1.f);
+    cooldown_sec->set_help("Delay to add between test executions in seconds");
 
     try
     {
@@ -261,12 +263,15 @@ int main(int argc, char **argv)
         framework.set_instruments_info(instruments_info);
 
         // Initialize framework
-        framework.init(options.instruments->value(),
-                       options.iterations->value(),
-                       dataset_mode->value(),
-                       filter->value(),
-                       filter_id->value(),
-                       options.log_level->value());
+        framework::FrameworkConfig fconfig;
+        fconfig.instruments    = options.instruments->value();
+        fconfig.name_filter    = filter->value();
+        fconfig.id_filter      = filter_id->value();
+        fconfig.num_iterations = options.iterations->value();
+        fconfig.mode           = dataset_mode->value();
+        fconfig.log_level      = options.log_level->value();
+        fconfig.cooldown_sec   = cooldown_sec->value();
+        framework.init(fconfig);
 
         for(auto &p : printers)
         {
diff --git a/tests/validate_examples/RunExample.cpp b/tests/validate_examples/RunExample.cpp
index f00460a643..5d5291abfb 100644
--- a/tests/validate_examples/RunExample.cpp
+++ b/tests/validate_examples/RunExample.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018-2019 ARM Limited.
+ * Copyright (c) 2018-2020 ARM Limited.
  *
  * SPDX-License-Identifier: MIT
  *
@@ -172,7 +172,13 @@ int run_example(int argc, char **argv, std::unique_ptr<ValidateExample> example)
         }
     }
 
-    framework.init(options.instruments->value(), options.iterations->value(), framework::DatasetMode::ALL, "", "", options.log_level->value());
+    // Initialize framework
+    framework::FrameworkConfig fconfig;
+    fconfig.instruments    = options.instruments->value();
+    fconfig.num_iterations = options.iterations->value();
+    fconfig.log_level      = options.log_level->value();
+    framework.init(fconfig);
+
     for(auto &p : printers)
     {
         framework.add_printer(p.get());