From 7f15251e2fd6c729a1d8f4f524a4bb902c30cdad Mon Sep 17 00:00:00 2001 From: Georgios Pinitas Date: Mon, 16 Dec 2019 19:59:52 +0000 Subject: COMPMID-2986: Extend test framework options. Add the following option to the test framework: - delay: Allows run delay between tests in seconds. Signed-off-by: Georgios Pinitas Change-Id: Ib1b7a1d1ad83c9a99b23f353b1800db075b32bce Reviewed-on: https://review.mlplatform.org/c/2525 Reviewed-by: Michele Di Giorgio Tested-by: Arm Jenkins Reviewed-by: Gian Marco Iodice Comments-Addressed: Arm Jenkins --- tests/benchmark_examples/RunExample.cpp | 10 +++++-- tests/framework/Framework.cpp | 17 ++++++----- tests/framework/Framework.h | 24 ++++++++++----- tests/framework/Utils.cpp | 53 +++++++++++++++++++++++++++++++++ tests/framework/Utils.h | 7 ++++- tests/main.cpp | 19 +++++++----- tests/validate_examples/RunExample.cpp | 10 +++++-- 7 files changed, 113 insertions(+), 27 deletions(-) create mode 100644 tests/framework/Utils.cpp diff --git a/tests/benchmark_examples/RunExample.cpp b/tests/benchmark_examples/RunExample.cpp index f3de308bef..613e985707 100644 --- a/tests/benchmark_examples/RunExample.cpp +++ b/tests/benchmark_examples/RunExample.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019 ARM Limited. + * Copyright (c) 2018-2020 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -156,7 +156,13 @@ int run_example(int argc, char **argv, std::unique_ptr example) } } - framework.init(options.instruments->value(), options.iterations->value(), framework::DatasetMode::ALL, "", "", options.log_level->value()); + // Initialize framework + framework::FrameworkConfig fconfig; + fconfig.instruments = options.instruments->value(); + fconfig.num_iterations = options.iterations->value(); + fconfig.log_level = options.log_level->value(); + framework.init(fconfig); + for(auto &p : printers) { framework.add_printer(p.get()); diff --git a/tests/framework/Framework.cpp b/tests/framework/Framework.cpp index 44887a9125..4d66b14d67 100644 --- a/tests/framework/Framework.cpp +++ b/tests/framework/Framework.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 ARM Limited. + * Copyright (c) 2017-2020 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -125,14 +125,14 @@ Framework &Framework::get() return instance; } -void Framework::init(const std::vector &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter, - LogLevel log_level) +void Framework::init(const FrameworkConfig &config) { - _test_filter = TestFilter(mode, name_filter, id_filter); - _num_iterations = num_iterations; - _log_level = log_level; + _test_filter = TestFilter(config.mode, config.name_filter, config.id_filter); + _num_iterations = config.num_iterations; + _log_level = config.log_level; + _cooldown_sec = config.cooldown_sec; - _instruments = std::set(instruments.begin(), instruments.end()); + _instruments = std::set(std::begin(config.instruments), std::end(config.instruments)); } std::string Framework::current_suite_name() const @@ -579,6 +579,9 @@ bool Framework::run() run_test(test_info, *test_factory); ++id_run_test; + + // Run test delay + sleep_in_seconds(_cooldown_sec); } ++id; diff --git a/tests/framework/Framework.h b/tests/framework/Framework.h index c02416f9b5..fb52fd8ffe 100644 --- a/tests/framework/Framework.h +++ b/tests/framework/Framework.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 ARM Limited. + * Copyright (c) 2017-2020 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -54,6 +54,18 @@ namespace test { namespace framework { +/** Framework configuration structure */ +struct FrameworkConfig +{ + std::vector instruments{}; /**< Instrument types that will be used for benchmarking. */ + std::string name_filter{}; /**< Regular expression to filter tests by name. Only matching tests will be executed. */ + std::string id_filter{}; /**< String to match selected test ids. Only matching tests will be executed. */ + DatasetMode mode{ DatasetMode::ALL }; /**< Dataset mode. */ + int num_iterations{ 1 }; /**< Number of iterations per test. */ + float cooldown_sec{ -1.f }; /**< Delay between tests in seconds. */ + LogLevel log_level{ LogLevel::NONE }; /**< Verbosity of the output. */ +}; + /** Information about a test case. * * A test can be identified either via its id or via its name. Additionally @@ -99,14 +111,9 @@ public: * * @see TestFilter::TestFilter for the format of the string to filter ids. * - * @param[in] instruments Instrument types that will be used for benchmarking. - * @param[in] num_iterations Number of iterations per test. - * @param[in] mode Dataset mode. - * @param[in] name_filter Regular expression to filter tests by name. Only matching tests will be executed. - * @param[in] id_filter String to match selected test ids. Only matching tests will be executed. - * @param[in] log_level Verbosity of the output. + * @param[in] config Framework configuration meta-data. */ - void init(const std::vector &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter, LogLevel log_level); + void init(const FrameworkConfig &config); /** Add a new test suite. * @@ -329,6 +336,7 @@ private: std::vector> _test_factories{}; std::map _test_results{}; int _num_iterations{ 1 }; + float _cooldown_sec{ -1.f }; bool _throw_errors{ false }; bool _stop_on_error{ false }; bool _error_on_missing_assets{ false }; diff --git a/tests/framework/Utils.cpp b/tests/framework/Utils.cpp new file mode 100644 index 0000000000..589d27493c --- /dev/null +++ b/tests/framework/Utils.cpp @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2019-2020 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "Utils.h" + +#ifndef NO_MULTI_THREADING +#include +#include +#endif /* NO_MULTI_THREADING */ + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +void sleep_in_seconds(float seconds) +{ + // Early return on non-positive input + if(seconds <= 0.f) + { + return; + } + +#ifndef NO_MULTI_THREADING + const int64_t us = static_cast(seconds * 1e6); + std::this_thread::sleep_for(std::chrono::microseconds(us)); +#endif /* NO_MULTI_THREADING */ +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/tests/framework/Utils.h b/tests/framework/Utils.h index c0442cf9af..0280daaae9 100644 --- a/tests/framework/Utils.h +++ b/tests/framework/Utils.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017 ARM Limited. + * Copyright (c) 2017-2020 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -170,6 +170,11 @@ inline std::string arithmetic_to_string(T val, int decimal_places = 0) return ss.str(); } +/** Makes the calling thread to sleep for a specified number of seconds + * + * @param[in] seconds Amount of seconds to sleep. Will return immediately if less or equal to zero. + */ +void sleep_in_seconds(float seconds); } // namespace framework } // namespace test } // namespace arm_compute diff --git a/tests/main.cpp b/tests/main.cpp index 604a51fc0f..441e28b2a8 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017-2019 ARM Limited. + * Copyright (c) 2017-2020 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -142,6 +142,8 @@ int main(int argc, char **argv) #endif /* ARM_COMPUTE_CL */ auto threads = parser.add_option>("threads", 1); threads->set_help("Number of threads to use"); + auto cooldown_sec = parser.add_option>("delay", -1.f); + cooldown_sec->set_help("Delay to add between test executions in seconds"); try { @@ -261,12 +263,15 @@ int main(int argc, char **argv) framework.set_instruments_info(instruments_info); // Initialize framework - framework.init(options.instruments->value(), - options.iterations->value(), - dataset_mode->value(), - filter->value(), - filter_id->value(), - options.log_level->value()); + framework::FrameworkConfig fconfig; + fconfig.instruments = options.instruments->value(); + fconfig.name_filter = filter->value(); + fconfig.id_filter = filter_id->value(); + fconfig.num_iterations = options.iterations->value(); + fconfig.mode = dataset_mode->value(); + fconfig.log_level = options.log_level->value(); + fconfig.cooldown_sec = cooldown_sec->value(); + framework.init(fconfig); for(auto &p : printers) { diff --git a/tests/validate_examples/RunExample.cpp b/tests/validate_examples/RunExample.cpp index f00460a643..5d5291abfb 100644 --- a/tests/validate_examples/RunExample.cpp +++ b/tests/validate_examples/RunExample.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018-2019 ARM Limited. + * Copyright (c) 2018-2020 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -172,7 +172,13 @@ int run_example(int argc, char **argv, std::unique_ptr example) } } - framework.init(options.instruments->value(), options.iterations->value(), framework::DatasetMode::ALL, "", "", options.log_level->value()); + // Initialize framework + framework::FrameworkConfig fconfig; + fconfig.instruments = options.instruments->value(); + fconfig.num_iterations = options.iterations->value(); + fconfig.log_level = options.log_level->value(); + framework.init(fconfig); + for(auto &p : printers) { framework.add_printer(p.get()); -- cgit v1.2.1