Add profile to all created conformance tests.

Add supported profiles for each test in convert2conformance and
tosa_verif_conformance_generator.
Enable filtering of profile tests on running in tosa_verif_run_tests.
Reorganize arguments in conformance_generator to have more important
ones first.

Change-Id: Ie6e5b68727adb3c39b04aa482dd6433788f7bcc9
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>

2 files changed, 19 insertions, 7 deletions

diff --git a/verif/runner/tosa_test_runner.py b/verif/runner/tosa_test_runner.py
index d653a94..65931d8 100644
--- a/verif/runner/tosa_test_runner.py
+++ b/verif/runner/tosa_test_runner.py
@@ -76,13 +76,17 @@ class TosaTestRunner:
         self.descFile = str(descFilePath)
 
     def skipTest(self):
-        """Check if the test is skipped due to test type selection."""
+        """Check if the test is skipped due to test type or profile selection."""
         expectedFailure = self.testDesc["expected_failure"]
         if self.args.test_type == "negative" and not expectedFailure:
-            return True
+            return True, "non-negative type"
         elif self.args.test_type == "positive" and expectedFailure:
-            return True
-        return False
+            return True, "non-positive type"
+        if self.args.profile:
+            profile = self.testDesc["profile"] if "profile" in self.testDesc else []
+            if self.args.profile not in profile:
+                return True, "non-{} profile".format(self.args.profile)
+        return False, ""
 
     def runTestGraph(self):
         """Override with function that calls system under test."""
diff --git a/verif/runner/tosa_verif_run_tests.py b/verif/runner/tosa_verif_run_tests.py
index b400d76..77394cc 100644
--- a/verif/runner/tosa_verif_run_tests.py
+++ b/verif/runner/tosa_verif_run_tests.py
@@ -117,7 +117,7 @@ def parseArgs(argv):
         type=str,
         default="both",
         choices=["positive", "negative", "both"],
-        help="Filter tests based on expected failure status (positive, negative or both)",
+        help="Filter tests based on expected failure status",
     )
     parser.add_argument(
         "--no-color",
@@ -126,6 +126,13 @@ def parseArgs(argv):
         action="store_true",
         help="Disable color output",
     )
+    parser.add_argument(
+        "--profile",
+        dest="profile",
+        type=str,
+        choices=["tosa-bi", "tosa-mi"],
+        help="Filter tests based on profile",
+    )
 
     args = parser.parse_args(argv)
 
@@ -171,8 +178,9 @@ def workerThread(task_queue, runnerList, args, result_queue):
                 runnerName = runnerModule.__name__
                 runner = runnerModule.TosaSUTRunner(args, runnerArgs, test)
 
-                if runner.skipTest():
-                    msg = "Skipping non-{} test".format(args.test_type)
+                skip, reason = runner.skipTest()
+                if skip:
+                    msg = "Skipping {} test".format(reason)
                     print("{} {}".format(msg, test))
                     rc = TosaTestRunner.Result.SKIPPED
                 else: