From 88588624e513cead1b977d94087733157ed5df3b Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Tue, 12 Jul 2022 16:42:29 +0100 Subject: Add profile to all created conformance tests. Add supported profiles for each test in convert2conformance and tosa_verif_conformance_generator. Enable filtering of profile tests on running in tosa_verif_run_tests. Reorganize arguments in conformance_generator to have more important ones first. Change-Id: Ie6e5b68727adb3c39b04aa482dd6433788f7bcc9 Signed-off-by: Jeremy Johnson --- scripts/convert2conformance/convert2conformance.py | 30 ++- .../tosa_base_profile_framework_ops_info.json | 76 +++++++ verif/conformance/tosa_base_profile_ops_info.json | 224 +++++++++++++++++++++ .../tosa_verif_conformance_generator.py | 124 +++++++----- verif/runner/tosa_test_runner.py | 12 +- verif/runner/tosa_verif_run_tests.py | 14 +- 6 files changed, 416 insertions(+), 64 deletions(-) diff --git a/scripts/convert2conformance/convert2conformance.py b/scripts/convert2conformance/convert2conformance.py index aa41ff1..06726ba 100755 --- a/scripts/convert2conformance/convert2conformance.py +++ b/scripts/convert2conformance/convert2conformance.py @@ -30,6 +30,8 @@ NAME_DESC_FILENAME = "desc.json" NAME_CONFORMANCE_RESULT_PREFIX = "Conformance-" NAME_REFMODEL_RUN_RESULT_SUFFIX = ".runner.tosa_refmodel_sut_run.npy" +PROFILES_LIST = ["tosa-bi", "tosa-mi"] + def parse_args(argv): """Parse the arguments.""" @@ -68,10 +70,19 @@ def parse_args(argv): type=Path, help="Framework schema needed to convert framework models", ) + parser.add_argument( + "--profile", + dest="profile", + choices=PROFILES_LIST, + action="append", + required=True, + help="Profiles this test is suitable for. May be repeated", + ) parser.add_argument( "-v", "--verbose", dest="verbose", action="store_true", help="Verbose operation" ) args = parser.parse_args(argv) + return args @@ -138,7 +149,11 @@ def convert_numpy_file(n_file: Path, output: Path, outname: Optional[str] = None def update_desc_json( - test_dir: Path, test_desc, output_dir: Optional[Path] = None, create_result=True + test_dir: Path, + test_desc, + output_dir: Optional[Path] = None, + create_result=True, + profiles=None, ): """Update the desc.json format for conformance and optionally create result.""" ofm_files = [] @@ -171,6 +186,13 @@ def update_desc_json( if not test_desc["expected_failure"]: # Output expected result file for conformance if expected pass test_desc["expected_result_file"] = cfm_files + + # Add supported profiles + if profiles is None: + # Assume base profile + profiles = [PROFILES_LIST[0]] + test_desc["profile"] = profiles + return test_desc @@ -289,7 +311,11 @@ def main(argv=None): # Update desc.json and convert result files to JSON test_desc = update_desc_json( - desc_filename.parent, test_desc, output_dir=args.output_dir, create_result=True + desc_filename.parent, + test_desc, + output_dir=args.output_dir, + create_result=True, + profiles=args.profile, ) if not test_desc: # Error from conversion/update diff --git a/verif/conformance/tosa_base_profile_framework_ops_info.json b/verif/conformance/tosa_base_profile_framework_ops_info.json index 7beb2dd..c8bcb00 100644 --- a/verif/conformance/tosa_base_profile_framework_ops_info.json +++ b/verif/conformance/tosa_base_profile_framework_ops_info.json @@ -3,6 +3,10 @@ "tests": [ "add_1_qi16", "add_1x32x32x8_qi8" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "average_pool_2d": { @@ -15,6 +19,10 @@ "average_pool_2d_1x7x7x9_qi8_st22_padSAME_kern11", "average_pool_2d_1x32x32x8_qu8_st12_padVALID_kern12", "average_pool_2d_1x8x4x17_qu8_st21_padVALID_kern21" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "concatenation": { @@ -31,6 +39,10 @@ "alternate_names": [ "concat", "concatv2" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "conv_2d": { @@ -51,6 +63,10 @@ "conv_2d_relu_n1_to_1_5x5_1x7x7x9_qi8", "conv_2d_tanh_3x3_1x7x7x9_qi8", "conv_2d_tanh_5x5_1x32x32x8_qu8" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "depthwise_conv_2d": { @@ -63,6 +79,10 @@ "depthwise_conv_2d_5x5_1x32x32x8_qi8_st11_padSAME_dilat12", "depthwise_conv_2d_bias_1x1_1x4x4x4_qu8_st11_padSAME_dilat11", "depthwise_conv_2d_bias_3x3_1x32x32x8_qu8_st11_padVALID_dilat22" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "fully_connected": { @@ -73,6 +93,10 @@ "fully_connected_1x4x4x4_qi8", "fully_connected_13x21x3_qu8", "fully_connected_14x19_qi8" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "leaky_relu": { @@ -81,6 +105,10 @@ "leaky_relu_14x19_qi8_0", "leaky_relu_13x21x3_qu8_0", "leaky_relu_1x4x8x19_qi16_0" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "logistic": { @@ -91,6 +119,10 @@ "logistic_13x21x3_qu8", "logistic_1x8x4x17_qi8", "logistic_64_qi16" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "max_pool_2d": { @@ -103,6 +135,10 @@ "max_pool_2d_1x4x8x19_qu8_st21_padVALID_kern22", "max_pool_2d_1x8x4x17_qu8_st21_padVALID_kern21", "max_pool_2d_1x32x32x8_qi8_st12_padVALID_kern22" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "minimum": { @@ -112,12 +148,20 @@ "tests": [ "minimum_1x4x4x4_i32", "minimum_64_i32" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "mul": { "tests": [ "mul_13x21x3_qi16", "mul_1x4x8x19_qu8" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "pack": { @@ -129,6 +173,10 @@ "pack_14x19_i32_axis1", "pack_1x7x7x9_i32_axis2", "pack_1x8x4x17_i32_axis4" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "reshape": { @@ -136,12 +184,20 @@ "reshape_14x19_i32_rank1", "reshape_13x21x3_i32_rank3", "reshape_1x32x32x8_i32_rank2" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "resize_bilinear": { "tests": [ "resize_bilinear_1x32x32x8_qi16", "resize_bilinear_1x4x4x4_qi8" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "resize_nearest_neighbor": { @@ -151,6 +207,10 @@ "tests": [ "resize_nearest_neighbor_1x4x8x19_qu8", "resize_nearest_neighbor_1x8x4x17_qi8" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "softmax": { @@ -158,6 +218,10 @@ "softmax_1_qi8", "softmax_14x19_qi16", "softmax_1x32x32x8_qu8" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "split": { @@ -166,6 +230,10 @@ "split_1x32x32x8_i32_split4_axis2", "split_1x4x8x19_i32_splitv_axis3", "split_64_i32_split2_axis0" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "strided_slice": { @@ -173,6 +241,10 @@ "strided_slice_13x21x3_i32_perm0", "strided_slice_14x19_i32_perm1", "strided_slice_64_i32_perm3" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "tanh": { @@ -180,6 +252,10 @@ "tanh_13x21x3_qu8", "tanh_14x19_qi8", "tanh_1x8x4x17_qi16" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] } } \ No newline at end of file diff --git a/verif/conformance/tosa_base_profile_ops_info.json b/verif/conformance/tosa_base_profile_ops_info.json index 79e50ff..b9f392c 100644 --- a/verif/conformance/tosa_base_profile_ops_info.json +++ b/verif/conformance/tosa_base_profile_ops_info.json @@ -37,6 +37,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "add": { @@ -77,6 +81,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "argmax": { @@ -125,6 +133,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "arithmetic_right_shift": { @@ -162,6 +174,10 @@ "shape", "type", "rounding" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "avg_pool2d": { @@ -198,6 +214,10 @@ "shape", "type", "pad" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "bitwise_and": { @@ -236,6 +256,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "bitwise_not": { @@ -274,6 +298,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "bitwise_or": { @@ -312,6 +340,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "bitwise_xor": { @@ -350,6 +382,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "cast": { @@ -412,6 +448,10 @@ "shape", "type", "output_type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "clamp": { @@ -448,6 +488,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "clz": { @@ -488,6 +532,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "concat": { @@ -546,6 +594,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "cond_if": { @@ -573,6 +625,10 @@ "variant", "shape", "cond" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "const": { @@ -624,6 +680,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "conv2d": { @@ -665,6 +725,10 @@ "shape", "type", "pad" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "conv3d": { @@ -711,6 +775,10 @@ "type", "pad", "stride" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "depthwise_conv2d": { @@ -752,6 +820,10 @@ "shape", "type", "pad" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "equal": { @@ -792,6 +864,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "fully_connected": { @@ -828,6 +904,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "gather": { @@ -856,6 +936,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "greater": { @@ -902,6 +986,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "greater_equal": { @@ -942,6 +1030,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "identity": { @@ -995,6 +1087,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "intdiv": { @@ -1035,6 +1131,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "logical_and": { @@ -1069,6 +1169,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "logical_left_shift": { @@ -1117,6 +1221,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "logical_not": { @@ -1151,6 +1259,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "logical_or": { @@ -1185,6 +1297,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "logical_right_shift": { @@ -1233,6 +1349,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "logical_xor": { @@ -1267,6 +1387,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "matmul": { @@ -1303,6 +1427,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "maximum": { @@ -1343,6 +1471,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "max_pool2d": { @@ -1381,6 +1513,10 @@ "shape", "type", "pad" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "minimum": { @@ -1421,6 +1557,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "mul": { @@ -1471,6 +1611,10 @@ "type", "perm", "shift" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "negate": { @@ -1519,6 +1663,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "pad": { @@ -1585,6 +1733,10 @@ "type": "i8", "pad": "pad00" } + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "reduce_all": { @@ -1606,6 +1758,10 @@ "shape", "type", "axis" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "reduce_any": { @@ -1627,6 +1783,10 @@ "shape", "type", "axis" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "reduce_max": { @@ -1656,6 +1816,10 @@ "shape", "type", "axis" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "reduce_min": { @@ -1685,6 +1849,10 @@ "shape", "type", "axis" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "reduce_sum": { @@ -1710,6 +1878,10 @@ "shape", "type", "axis" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "rescale": { @@ -1771,6 +1943,10 @@ "shape", "type", "output_type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "reshape": { @@ -1805,6 +1981,10 @@ "type", "rank", "perm" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "resize": { @@ -1827,6 +2007,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "reverse": { @@ -1857,6 +2041,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "scatter": { @@ -1885,6 +2073,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "select": { @@ -1939,6 +2131,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "slice": { @@ -1996,6 +2192,10 @@ "shape", "type", "perm" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "sub": { @@ -2036,6 +2236,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "table": { @@ -2072,6 +2276,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "tile": { @@ -2098,6 +2306,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "transpose": { @@ -2151,6 +2363,10 @@ "shape", "type", "perm" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "transpose_conv2d": { @@ -2193,6 +2409,10 @@ "type", "stride", "pad" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] }, "while_loop": { @@ -2215,6 +2435,10 @@ "permutes": [ "shape", "type" + ], + "profile": [ + "tosa-bi", + "tosa-mi" ] } } \ No newline at end of file diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py index 836c639..c057b73 100644 --- a/verif/conformance/tosa_verif_conformance_generator.py +++ b/verif/conformance/tosa_verif_conformance_generator.py @@ -31,7 +31,7 @@ logger = logging.getLogger("tosa_verif_conformance_generator") # Configuration for each TOSA profile PROFILE_OPS_INFO = { - "base": { + "tosa-bi": { "operator_test_params": "tosa_base_profile_ops_info.json", "framework_tests": "tosa_base_profile_framework_ops_info.json", "exclude_types": ["float"], @@ -219,6 +219,7 @@ def convert_tests( operator, op_build_dir, output_dir, + profiles, tests=None, group=None, trim_op_subdir=False, @@ -230,6 +231,8 @@ def convert_tests( output_dir = output_dir / group ref_cmd_base = ["--ref-model-directory", str(ref_model_dir)] + for profile in profiles: + ref_cmd_base.extend(["--profile", profile]) if args.framework_schema: ref_cmd_base.extend(["--framework-schema", str(args.framework_schema)]) ref_cmd_base.append("--output-directory") @@ -388,12 +391,37 @@ def get_framework_tests_selection(args, operator, test_picks, op_build_dir): def parse_args(argv=None): """Parse the arguments.""" parser = argparse.ArgumentParser() + profiles = list(PROFILE_OPS_INFO.keys()) + parser.add_argument( + "--profile", + dest="profile", + choices=profiles, + default=profiles[0], + type=str, + help=f"TOSA profile (default is {profiles[0]})", + ) parser.add_argument( "--operators", type=str, nargs="*", help="The operator(s) to create tests for, if not supplied all tests will be created", ) + parser.add_argument( + "--unit-tests", + dest="unit_tests", + choices=["operator", "framework", "both"], + default="operator", + type=str, + help="Which unit tests are produced (default is operator)", + ) + parser.add_argument( + "--test-type", + dest="test_type", + choices=["positive", "negative", "both"], + default="both", + type=str, + help="Type of tests produced (default is both)", + ) parser.add_argument( "--ref-model-directory", dest="ref_model_dir", @@ -401,13 +429,40 @@ def parse_args(argv=None): required=True, help="Reference Model directory (must be pre-built)", ) + parser.add_argument( + "--framework-tests-directory", + dest="framework_tests_dir", + type=Path, + default=Path.cwd() / "tests", + help="The pre-built framework tests directory (default is tests)", + ) + parser.add_argument( + "--framework-schema", + dest="framework_schema", + type=Path, + help="Framework flatbuffers schema needed to convert framework models", + ) + parser.add_argument( + "--build-directory", + dest="build_dir", + type=Path, + default=Path.cwd() / "conformance_build", + help="Temporary build directory for files created during this process (default is conformance_build)", + ) + parser.add_argument( + "--output-directory", + dest="output_dir", + type=Path, + default=Path.cwd() / "conformance", + help="Output directory (default is conformance)", + ) script_dir = Path(__file__).parent.absolute() parser.add_argument( "--test-param-json-directory", dest="param_json_dir", type=Path, default=script_dir, - help="Test parameters (ops info) JSON file directory", + help=f"Test parameters (ops info) JSON file directory (default is {script_dir})", ) parser.add_argument( "--convert-all-tests", @@ -424,20 +479,6 @@ def parse_args(argv=None): action="store_true", help="Prints output of running sh commands", ) - parser.add_argument( - "--build-directory", - dest="build_dir", - type=Path, - default=Path.cwd() / "conformance_build", - help="Temporary build directory for files created during this process (default is conformance_build)", - ) - parser.add_argument( - "--output-directory", - dest="output_dir", - type=Path, - default=Path.cwd() / "conformance", - help="Output directory (default is conformance)", - ) parser.add_argument( "-j", dest="num_cores", @@ -452,44 +493,6 @@ def parse_args(argv=None): default=0, help="Verbosity (can be used multiple times for more details)", ) - parser.add_argument( - "--unit-tests", - dest="unit_tests", - choices=["operator", "framework", "both"], - default="operator", - type=str, - help="Which unit tests are produced: operator, framework, or both", - ) - parser.add_argument( - "--test-type", - dest="test_type", - choices=["positive", "negative", "both"], - default="both", - type=str, - help="Type of tests produced: positive, negative, or both", - ) - profiles = list(PROFILE_OPS_INFO.keys()) - parser.add_argument( - "--profile", - dest="profile", - choices=profiles, - default=profiles[0], - type=str, - help="TOSA profile", - ) - parser.add_argument( - "--framework-tests-directory", - dest="framework_tests_dir", - type=Path, - default=Path.cwd() / "tests", - help="The pre-built framework tests directory (default is tests)", - ) - parser.add_argument( - "--framework-schema", - dest="framework_schema", - type=Path, - help="Framework flatbuffers schema needed to convert framework models", - ) args = parser.parse_args(argv) return args @@ -567,6 +570,7 @@ def main(): logger.debug(f"Copying and renaming {op}") framework_test_dir = copy_rename_framework_tests(args, op, test_picks) + profiles = test_picks[op]["profile"] if args.convert_all_tests: logger.debug("Running and converting all framework tests") convert_tests( @@ -574,6 +578,7 @@ def main(): op, framework_test_dir, root_output_dir, + profiles, trim_op_subdir=True, ) else: @@ -585,6 +590,7 @@ def main(): op, framework_test_dir, root_output_dir, + profiles, tests=framework_tests, trim_op_subdir=True, ) @@ -629,11 +635,17 @@ def main(): operator_group = test_params[op]["group"] root_output_dir = args.output_dir / "operators" + profiles = test_params[op]["profile"] if args.convert_all_tests: logger.debug(f"Running and converting all {op} tests") generate_results(args, op, op_build_dir) output_dir = convert_tests( - args, op, op_build_dir, root_output_dir, group=operator_group + args, + op, + op_build_dir, + root_output_dir, + profiles, + group=operator_group, ) else: if args.test_type in ["positive", "both"]: @@ -646,6 +658,7 @@ def main(): op, op_build_dir, root_output_dir, + profiles, tests=tests_gen2, group=operator_group, ) @@ -661,6 +674,7 @@ def main(): op, op_build_dir, root_output_dir, + profiles, tests=negative_tests, group=operator_group, ) diff --git a/verif/runner/tosa_test_runner.py b/verif/runner/tosa_test_runner.py index d653a94..65931d8 100644 --- a/verif/runner/tosa_test_runner.py +++ b/verif/runner/tosa_test_runner.py @@ -76,13 +76,17 @@ class TosaTestRunner: self.descFile = str(descFilePath) def skipTest(self): - """Check if the test is skipped due to test type selection.""" + """Check if the test is skipped due to test type or profile selection.""" expectedFailure = self.testDesc["expected_failure"] if self.args.test_type == "negative" and not expectedFailure: - return True + return True, "non-negative type" elif self.args.test_type == "positive" and expectedFailure: - return True - return False + return True, "non-positive type" + if self.args.profile: + profile = self.testDesc["profile"] if "profile" in self.testDesc else [] + if self.args.profile not in profile: + return True, "non-{} profile".format(self.args.profile) + return False, "" def runTestGraph(self): """Override with function that calls system under test.""" diff --git a/verif/runner/tosa_verif_run_tests.py b/verif/runner/tosa_verif_run_tests.py index b400d76..77394cc 100644 --- a/verif/runner/tosa_verif_run_tests.py +++ b/verif/runner/tosa_verif_run_tests.py @@ -117,7 +117,7 @@ def parseArgs(argv): type=str, default="both", choices=["positive", "negative", "both"], - help="Filter tests based on expected failure status (positive, negative or both)", + help="Filter tests based on expected failure status", ) parser.add_argument( "--no-color", @@ -126,6 +126,13 @@ def parseArgs(argv): action="store_true", help="Disable color output", ) + parser.add_argument( + "--profile", + dest="profile", + type=str, + choices=["tosa-bi", "tosa-mi"], + help="Filter tests based on profile", + ) args = parser.parse_args(argv) @@ -171,8 +178,9 @@ def workerThread(task_queue, runnerList, args, result_queue): runnerName = runnerModule.__name__ runner = runnerModule.TosaSUTRunner(args, runnerArgs, test) - if runner.skipTest(): - msg = "Skipping non-{} test".format(args.test_type) + skip, reason = runner.skipTest() + if skip: + msg = "Skipping {} test".format(reason) print("{} {}".format(msg, test)) rc = TosaTestRunner.Result.SKIPPED else: -- cgit v1.2.1