From ec2de0f0784ece788e6d9aa9d2af1f9a8c4175b1 Mon Sep 17 00:00:00 2001 From: Moritz Pflanzer Date: Thu, 27 Jul 2017 14:43:46 +0100 Subject: COMPMID-415: Allow multiple ids for filter-id Change-Id: Id2e4ab6094438105fec17b8ea0ad74057968571b Reviewed-on: http://mpd-gerrit.cambridge.arm.com/81859 Reviewed-by: Anthony Barbier Tested-by: Kaizen --- framework/Framework.cpp | 34 ++--------- framework/Framework.h | 19 ++----- framework/TestFilter.cpp | 143 +++++++++++++++++++++++++++++++++++++++++++++++ framework/TestFilter.h | 84 ++++++++++++++++++++++++++++ tests/main.cpp | 4 +- 5 files changed, 241 insertions(+), 43 deletions(-) create mode 100644 framework/TestFilter.cpp create mode 100644 framework/TestFilter.h diff --git a/framework/Framework.cpp b/framework/Framework.cpp index ac7248c43c..bdf5319ebc 100644 --- a/framework/Framework.cpp +++ b/framework/Framework.cpp @@ -80,13 +80,11 @@ Framework &Framework::get() return instance; } -void Framework::init(const std::vector &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, int64_t id_filter, LogLevel log_level) +void Framework::init(const std::vector &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter, LogLevel log_level) { - _test_name_filter = std::regex{ name_filter }; - _test_id_filter = id_filter; - _num_iterations = num_iterations; - _dataset_mode = mode; - _log_level = log_level; + _test_filter = TestFilter(mode, name_filter, id_filter); + _num_iterations = num_iterations; + _log_level = log_level; _instruments = InstrumentType::NONE; @@ -208,26 +206,6 @@ bool Framework::stop_on_error() const return _stop_on_error; } -bool Framework::is_selected(const TestInfo &info) const -{ - if((info.mode & _dataset_mode) == DatasetMode::DISABLED) - { - return false; - } - - if(_test_id_filter > -1 && _test_id_filter != info.id) - { - return false; - } - - if(!std::regex_search(info.name, _test_name_filter)) - { - return false; - } - - return true; -} - void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory) { if(test_factory.status() == TestCaseFactory::Status::DISABLED) @@ -392,7 +370,7 @@ bool Framework::run() const std::string test_case_name = test_factory->name(); const TestInfo test_info{ id, test_case_name, test_factory->mode(), test_factory->status() }; - if(is_selected(test_info)) + if(_test_filter.is_selected(test_info)) { run_test(test_info, *test_factory); } @@ -474,7 +452,7 @@ std::vector Framework::test_infos() const { TestInfo test_info{ id, factory->name(), factory->mode(), factory->status() }; - if(is_selected(test_info)) + if(_test_filter.is_selected(test_info)) { ids.emplace_back(std::move(test_info)); } diff --git a/framework/Framework.h b/framework/Framework.h index 671e4e435f..3526eee82f 100644 --- a/framework/Framework.h +++ b/framework/Framework.h @@ -29,6 +29,7 @@ #include "Profiler.h" #include "TestCase.h" #include "TestCaseFactory.h" +#include "TestFilter.h" #include "TestResult.h" #include "Utils.h" #include "instruments/Instruments.h" @@ -95,15 +96,17 @@ public: std::set available_instruments() const; /** Init the framework. + * + * @see TestFilter::TestFilter for the format of the string to filter ids. * * @param[in] instruments Instrument types that will be used for benchmarking. * @param[in] num_iterations Number of iterations per test. * @param[in] mode Dataset mode. * @param[in] name_filter Regular expression to filter tests by name. Only matching tests will be executed. - * @param[in] id_filter Test id. Only this test will be executed. + * @param[in] id_filter String to match selected test ids. Only matching tests will be executed. * @param[in] log_level Verbosity of the output. */ - void init(const std::vector &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, int64_t id_filter, LogLevel log_level); + void init(const std::vector &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter, LogLevel log_level); /** Add a new test suite. * @@ -225,14 +228,6 @@ public: */ void set_stop_on_error(bool stop_on_error); - /** Check if a test case is selected to be executed. - * - * @param[in] info Test case info. - * - * @return True if the test case is selected to be executed. - */ - bool is_selected(const TestInfo &info) const; - /** Run all enabled test cases. * * @return True if all test cases executed successful. @@ -308,9 +303,7 @@ private: std::map _available_instruments{}; InstrumentType _instruments{ InstrumentType::NONE }; - std::regex _test_name_filter{ ".*" }; - int64_t _test_id_filter{ -1 }; - DatasetMode _dataset_mode{ DatasetMode::ALL }; + TestFilter _test_filter{}; LogLevel _log_level{ LogLevel::ALL }; TestResult *_current_test_result{ nullptr }; std::vector _test_info{}; diff --git a/framework/TestFilter.cpp b/framework/TestFilter.cpp new file mode 100644 index 0000000000..0af40c1717 --- /dev/null +++ b/framework/TestFilter.cpp @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include "TestFilter.h" + +#include "Framework.h" +#include "support/ToolchainSupport.h" + +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +TestFilter::TestFilter(DatasetMode mode, const std::string &name_filter, const std::string &id_filter) + : _dataset_mode{ mode }, _name_filter{ name_filter }, _id_filter{ parse_id_filter(id_filter) } +{ +} + +bool TestFilter::is_selected(const TestInfo &info) const +{ + if((info.mode & _dataset_mode) == DatasetMode::DISABLED) + { + return false; + } + + if(!std::regex_search(info.name, _name_filter)) + { + return false; + } + + if(!_id_filter.empty()) + { + bool found = false; + + for(const auto range : _id_filter) + { + if(range.first <= info.id && info.id <= range.second) + { + found = true; + break; + } + } + + if(!found) + { + return false; + } + } + + return true; +} + +TestFilter::Ranges TestFilter::parse_id_filter(const std::string &id_filter) const +{ + Ranges ranges; + std::string str; + bool in_range = false; + int value = 0; + int start = 0; + int end = std::numeric_limits::max(); + + std::stringstream stream(id_filter); + + // Get first value + std::getline(stream, str, ','); + + if(stream.fail()) + { + return ranges; + } + + if(str.find("...") != std::string::npos) + { + in_range = true; + } + else + { + start = support::cpp11::stoi(str); + end = start; + } + + while(!stream.eof()) + { + std::getline(stream, str, ','); + + if(stream.fail()) + { + break; + } + + if(str.find("...") != std::string::npos) + { + end = std::numeric_limits::max(); + in_range = true; + } + else + { + value = support::cpp11::stoi(str); + + if(in_range || end == value - 1) + { + end = value; + in_range = false; + } + else + { + ranges.emplace_back(start, end); + start = value; + end = start; + } + } + } + + ranges.emplace_back(start, end); + return ranges; +} +} // namespace framework +} // namespace test +} // namespace arm_compute diff --git a/framework/TestFilter.h b/framework/TestFilter.h new file mode 100644 index 0000000000..f64e73a2ba --- /dev/null +++ b/framework/TestFilter.h @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2017 ARM Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#ifndef ARM_COMPUTE_TEST_TESTFILTER +#define ARM_COMPUTE_TEST_TESTFILTER + +#include "DatasetModes.h" + +#include +#include +#include + +namespace arm_compute +{ +namespace test +{ +namespace framework +{ +struct TestInfo; + +/** Test filter class. + * + * Stores information about which test cases are selected for execution. Based + * on test name and test id. + */ +class TestFilter final +{ +public: + /** Default constructor. All tests selected. */ + TestFilter() = default; + + /** Constructor. + * + * The id_filter string has be a comma separated list of test ids. ... can + * be used to include a range of tests. For instance, "..., 15" means all + * test up to and including 15, "3, 6, ..., 10" means tests 3 and 6 to 10, + * and "15, ..." means test 15 and all following. + * + * @param[in] mode Dataset mode. + * @param[in] name_filter Regular expression to filter tests by name. Only matching tests will be executed. + * @param[in] id_filter String to match selected test ids. Only matching tests will be executed. + */ + TestFilter(DatasetMode mode, const std::string &name_filter, const std::string &id_filter); + + /** Check if a test case is selected to be executed. + * + * @param[in] info Test case info. + * + * @return True if the test case is selected to be executed. + */ + bool is_selected(const TestInfo &info) const; + +private: + using Ranges = std::vector>; + Ranges parse_id_filter(const std::string &id_filter) const; + + DatasetMode _dataset_mode{ DatasetMode::ALL }; + std::regex _name_filter{ ".*" }; + Ranges _id_filter{}; +}; +} // namespace framework +} // namespace test +} // namespace arm_compute +#endif /* ARM_COMPUTE_TEST_TESTFILTER */ diff --git a/tests/main.cpp b/tests/main.cpp index 7a1265cbbb..0b01185fc9 100644 --- a/tests/main.cpp +++ b/tests/main.cpp @@ -115,8 +115,8 @@ int main(int argc, char **argv) log_format->set_help("Output format for measurements and failures"); auto filter = parser.add_option>("filter", ".*"); filter->set_help("Regular expression to select test cases"); - auto filter_id = parser.add_option>("filter-id", -1); - filter_id->set_help("Test id. Only this test will be executed."); + auto filter_id = parser.add_option>("filter-id"); + filter_id->set_help("List of test ids. ... can be used to define a range."); auto log_file = parser.add_option>("log-file"); log_file->set_help("Write output to file instead of to the console"); auto log_level = parser.add_option>("log-level", supported_log_levels, framework::LogLevel::ALL); -- cgit v1.2.1