COMPMID-415: Allow multiple ids for filter-id

Change-Id: Id2e4ab6094438105fec17b8ea0ad74057968571b
Reviewed-on: http://mpd-gerrit.cambridge.arm.com/81859
Reviewed-by: Anthony Barbier <anthony.barbier@arm.com>
Tested-by: Kaizen <jeremy.johnson+kaizengerrit@arm.com>

5 files changed, 241 insertions, 43 deletions

diff --git a/framework/Framework.cpp b/framework/Framework.cpp
index ac7248c43c..bdf5319ebc 100644
--- a/framework/Framework.cpp
+++ b/framework/Framework.cpp
@@ -80,13 +80,11 @@ Framework &Framework::get()
     return instance;
 }
 
-void Framework::init(const std::vector<InstrumentType> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, int64_t id_filter, LogLevel log_level)
+void Framework::init(const std::vector<InstrumentType> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter, LogLevel log_level)
 {
-    _test_name_filter = std::regex{ name_filter };
-    _test_id_filter   = id_filter;
-    _num_iterations   = num_iterations;
-    _dataset_mode     = mode;
-    _log_level        = log_level;
+    _test_filter    = TestFilter(mode, name_filter, id_filter);
+    _num_iterations = num_iterations;
+    _log_level      = log_level;
 
     _instruments = InstrumentType::NONE;
 
@@ -208,26 +206,6 @@ bool Framework::stop_on_error() const
     return _stop_on_error;
 }
 
-bool Framework::is_selected(const TestInfo &info) const
-{
-    if((info.mode & _dataset_mode) == DatasetMode::DISABLED)
-    {
-        return false;
-    }
-
-    if(_test_id_filter > -1 && _test_id_filter != info.id)
-    {
-        return false;
-    }
-
-    if(!std::regex_search(info.name, _test_name_filter))
-    {
-        return false;
-    }
-
-    return true;
-}
-
 void Framework::run_test(const TestInfo &info, TestCaseFactory &test_factory)
 {
     if(test_factory.status() == TestCaseFactory::Status::DISABLED)
@@ -392,7 +370,7 @@ bool Framework::run()
         const std::string test_case_name = test_factory->name();
         const TestInfo    test_info{ id, test_case_name, test_factory->mode(), test_factory->status() };
 
-        if(is_selected(test_info))
+        if(_test_filter.is_selected(test_info))
         {
             run_test(test_info, *test_factory);
         }
@@ -474,7 +452,7 @@ std::vector<TestInfo> Framework::test_infos() const
     {
         TestInfo test_info{ id, factory->name(), factory->mode(), factory->status() };
 
-        if(is_selected(test_info))
+        if(_test_filter.is_selected(test_info))
         {
             ids.emplace_back(std::move(test_info));
         }
diff --git a/framework/Framework.h b/framework/Framework.h
index 671e4e435f..3526eee82f 100644
--- a/framework/Framework.h
+++ b/framework/Framework.h
@@ -29,6 +29,7 @@
 #include "Profiler.h"
 #include "TestCase.h"
 #include "TestCaseFactory.h"
+#include "TestFilter.h"
 #include "TestResult.h"
 #include "Utils.h"
 #include "instruments/Instruments.h"
@@ -96,14 +97,16 @@ public:
 
     /** Init the framework.
      *
+     * @see TestFilter::TestFilter for the format of the string to filter ids.
+     *
      * @param[in] instruments    Instrument types that will be used for benchmarking.
      * @param[in] num_iterations Number of iterations per test.
      * @param[in] mode           Dataset mode.
      * @param[in] name_filter    Regular expression to filter tests by name. Only matching tests will be executed.
-     * @param[in] id_filter      Test id. Only this test will be executed.
+     * @param[in] id_filter      String to match selected test ids. Only matching tests will be executed.
      * @param[in] log_level      Verbosity of the output.
      */
-    void init(const std::vector<InstrumentType> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, int64_t id_filter, LogLevel log_level);
+    void init(const std::vector<InstrumentType> &instruments, int num_iterations, DatasetMode mode, const std::string &name_filter, const std::string &id_filter, LogLevel log_level);
 
     /** Add a new test suite.
      *
@@ -225,14 +228,6 @@ public:
      */
     void set_stop_on_error(bool stop_on_error);
 
-    /** Check if a test case is selected to be executed.
-     *
-     * @param[in] info Test case info.
-     *
-     * @return True if the test case is selected to be executed.
-     */
-    bool is_selected(const TestInfo &info) const;
-
     /** Run all enabled test cases.
      *
      * @return True if all test cases executed successful.
@@ -308,9 +303,7 @@ private:
     std::map<InstrumentType, create_function *> _available_instruments{};
 
     InstrumentType           _instruments{ InstrumentType::NONE };
-    std::regex               _test_name_filter{ ".*" };
-    int64_t                  _test_id_filter{ -1 };
-    DatasetMode              _dataset_mode{ DatasetMode::ALL };
+    TestFilter               _test_filter{};
     LogLevel                 _log_level{ LogLevel::ALL };
     TestResult              *_current_test_result{ nullptr };
     std::vector<std::string> _test_info{};
diff --git a/framework/TestFilter.cpp b/framework/TestFilter.cpp
new file mode 100644
index 0000000000..0af40c1717
--- /dev/null
+++ b/framework/TestFilter.cpp
@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "TestFilter.h"
+
+#include "Framework.h"
+#include "support/ToolchainSupport.h"
+
+#include <sstream>
+#include <string>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+TestFilter::TestFilter(DatasetMode mode, const std::string &name_filter, const std::string &id_filter)
+    : _dataset_mode{ mode }, _name_filter{ name_filter }, _id_filter{ parse_id_filter(id_filter) }
+{
+}
+
+bool TestFilter::is_selected(const TestInfo &info) const
+{
+    if((info.mode & _dataset_mode) == DatasetMode::DISABLED)
+    {
+        return false;
+    }
+
+    if(!std::regex_search(info.name, _name_filter))
+    {
+        return false;
+    }
+
+    if(!_id_filter.empty())
+    {
+        bool found = false;
+
+        for(const auto range : _id_filter)
+        {
+            if(range.first <= info.id && info.id <= range.second)
+            {
+                found = true;
+                break;
+            }
+        }
+
+        if(!found)
+        {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+TestFilter::Ranges TestFilter::parse_id_filter(const std::string &id_filter) const
+{
+    Ranges      ranges;
+    std::string str;
+    bool        in_range = false;
+    int         value    = 0;
+    int         start    = 0;
+    int         end      = std::numeric_limits<int>::max();
+
+    std::stringstream stream(id_filter);
+
+    // Get first value
+    std::getline(stream, str, ',');
+
+    if(stream.fail())
+    {
+        return ranges;
+    }
+
+    if(str.find("...") != std::string::npos)
+    {
+        in_range = true;
+    }
+    else
+    {
+        start = support::cpp11::stoi(str);
+        end   = start;
+    }
+
+    while(!stream.eof())
+    {
+        std::getline(stream, str, ',');
+
+        if(stream.fail())
+        {
+            break;
+        }
+
+        if(str.find("...") != std::string::npos)
+        {
+            end      = std::numeric_limits<int>::max();
+            in_range = true;
+        }
+        else
+        {
+            value = support::cpp11::stoi(str);
+
+            if(in_range || end == value - 1)
+            {
+                end      = value;
+                in_range = false;
+            }
+            else
+            {
+                ranges.emplace_back(start, end);
+                start = value;
+                end   = start;
+            }
+        }
+    }
+
+    ranges.emplace_back(start, end);
+    return ranges;
+}
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
diff --git a/framework/TestFilter.h b/framework/TestFilter.h
new file mode 100644
index 0000000000..f64e73a2ba
--- /dev/null
+++ b/framework/TestFilter.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2017 ARM Limited.
+ *
+ * SPDX-License-Identifier: MIT
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to
+ * deal in the Software without restriction, including without limitation the
+ * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
+ * sell copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef ARM_COMPUTE_TEST_TESTFILTER
+#define ARM_COMPUTE_TEST_TESTFILTER
+
+#include "DatasetModes.h"
+
+#include <regex>
+#include <utility>
+#include <vector>
+
+namespace arm_compute
+{
+namespace test
+{
+namespace framework
+{
+struct TestInfo;
+
+/** Test filter class.
+ *
+ * Stores information about which test cases are selected for execution. Based
+ * on test name and test id.
+ */
+class TestFilter final
+{
+public:
+    /** Default constructor. All tests selected. */
+    TestFilter() = default;
+
+    /** Constructor.
+     *
+     * The id_filter string has be a comma separated list of test ids. ... can
+     * be used to include a range of tests. For instance, "..., 15" means all
+     * test up to and including 15, "3, 6, ..., 10" means tests 3 and 6 to 10,
+     * and "15, ..." means test 15 and all following.
+     *
+     * @param[in] mode        Dataset mode.
+     * @param[in] name_filter Regular expression to filter tests by name. Only matching tests will be executed.
+     * @param[in] id_filter   String to match selected test ids. Only matching tests will be executed.
+     */
+    TestFilter(DatasetMode mode, const std::string &name_filter, const std::string &id_filter);
+
+    /** Check if a test case is selected to be executed.
+     *
+     * @param[in] info Test case info.
+     *
+     * @return True if the test case is selected to be executed.
+     */
+    bool is_selected(const TestInfo &info) const;
+
+private:
+    using Ranges = std::vector<std::pair<int, int>>;
+    Ranges parse_id_filter(const std::string &id_filter) const;
+
+    DatasetMode _dataset_mode{ DatasetMode::ALL };
+    std::regex  _name_filter{ ".*" };
+    Ranges      _id_filter{};
+};
+} // namespace framework
+} // namespace test
+} // namespace arm_compute
+#endif /* ARM_COMPUTE_TEST_TESTFILTER */
diff --git a/tests/main.cpp b/tests/main.cpp
index 7a1265cbbb..0b01185fc9 100644
--- a/tests/main.cpp
+++ b/tests/main.cpp
@@ -115,8 +115,8 @@ int main(int argc, char **argv)
     log_format->set_help("Output format for measurements and failures");
     auto filter = parser.add_option<framework::SimpleOption<std::string>>("filter", ".*");
     filter->set_help("Regular expression to select test cases");
-    auto filter_id = parser.add_option<framework::SimpleOption<int64_t>>("filter-id", -1);
-    filter_id->set_help("Test id. Only this test will be executed.");
+    auto filter_id = parser.add_option<framework::SimpleOption<std::string>>("filter-id");
+    filter_id->set_help("List of test ids. ... can be used to define a range.");
     auto log_file = parser.add_option<framework::SimpleOption<std::string>>("log-file");
     log_file->set_help("Write output to file instead of to the console");
     auto log_level = parser.add_option<framework::EnumOption<framework::LogLevel>>("log-level", supported_log_levels, framework::LogLevel::ALL);