diff options
Diffstat (limited to 'verif/conformance')
-rw-r--r-- | verif/conformance/README.md | 15 | ||||
-rw-r--r-- | verif/conformance/tosa_main_profile_ops_info.json | 86 | ||||
-rw-r--r-- | verif/conformance/tosa_verif_conformance_generator.py | 274 |
3 files changed, 210 insertions, 165 deletions
diff --git a/verif/conformance/README.md b/verif/conformance/README.md index 0869ab6..5bededd 100644 --- a/verif/conformance/README.md +++ b/verif/conformance/README.md @@ -17,8 +17,10 @@ Each operator entry contains: * "group" - name of the group this operator is in, in the spec * "profile" - list of profiles that this operator covers -* "support_for" - optional list of supported creation modes out of: lazy_data_gen (data generation just before test run) * "gen_filter" - optional filter string for op to give to tosa_verif_build_tests - defaults to "^opname$" +* "support_for" - optional list of supported creation modes out of: + * lazy_data_gen - data generation just before test run + * generator_select - use generator selector instead of conformance test_select * "generation" - dictionary of test generation details - see below * "selection" - dictionary of test selection details - see below @@ -42,7 +44,16 @@ Each selection criteria is a dictionary that contains: * "all": "true" - to select all tests (and not use test_select) -or (more information for each entry in `test_select.py`): +or for operators that have "support_for" "generator_select": + +* "permutes" - optional list of parameters whose values are to be permuted, the default is ["rank", "dtype"] +* "maximum" - optional number - at most "maximum" tests (not including specific tests) will be captured per permuted "permutes" value, effects "full_params" as well +* "full_params" - optional list of parameter names used to select tests covering a full range of values for these params up to "maximum" +* "specifics" - optional dictionary of params with lists of values, tests that meet any of these "specifics" will be selected and kept (even using "post_sparsity") +* "groups" - optional list of parameters that should be considered as a grouping of tests and treated as one test for "sparsity" and "specifics" +* "num_errorifs" - optional value of error_if tests to keep per error_if case, the default is 1 + +or for other operators it defaults to the old test select (more information for each entry in `test_select.py`): * "params" - optional dictionary with mappings of parameter names to the values to select * "permutes" - optional list of parameter names to be permuted diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json index 202e7b7..63a2a9c 100644 --- a/verif/conformance/tosa_main_profile_ops_info.json +++ b/verif/conformance/tosa_main_profile_ops_info.json @@ -723,7 +723,7 @@ "profile": [ "tosa-mi" ], - "support_for": [ "lazy_data_gen" ], + "support_for": [ "lazy_data_gen", "generator_select" ], "gen_filter": "^conv2d", "generation": { "standard": { @@ -784,31 +784,11 @@ }, "selection": { "default": { - "params": { - "shape": [], - "type": [], - "kernel": [], - "stride": [], - "pad": [], - "dilation": [] - }, - "permutes": [ - "kernel", - "shape", - "type", - "pad" - ], - "preselected": [ - { - "shape": "1x34x19x27", - "type": "f32xf32", - "kernel": "3x1", - "pad": "pad0000", - "accum_type": "accf32", - "stride": "st11", - "dilation": "dilat11" - } - ] + "permutes": [ "rank", "dtype", "kernel", "acc_type" ], + "full_params": [ "stride", "dilation" ], + "specifics": { "pad": [ "(0, 0, 0, 0)" ] }, + "groups": [ "s" ], + "maximum": 3 } } }, @@ -1901,7 +1881,7 @@ "profile": [ "tosa-mi" ], - "support_for": [ "lazy_data_gen" ], + "support_for": [ "lazy_data_gen", "generator_select" ], "generation": { "standard": { "generator_args": [ @@ -1925,23 +1905,21 @@ ], [ "--target-dtype", + "fp32", + "--target-dtype", "fp16", - "--fp-values-range", - "-max,max", - "--tensor-dim-range", - "1,17", - "--target-rank", - "4" - ], - [ "--target-dtype", "bf16", "--fp-values-range", "-max,max", "--tensor-dim-range", - "1,16", + "1,11", "--target-rank", - "5" + "4", + "--target-rank", + "5", + "--target-rank", + "6" ], [ "--target-dtype", @@ -1980,30 +1958,16 @@ }, "selection": { "default": { - "params": {}, - "permutes": [ - "shape", - "type" - ], - "preselected": [ - { - "shape": "50", - "type": "bf16", - "pad": "pad11" - }, - { - "shape": "63x46", - "type": "bf16", - "pad": "pad1010" - }, - { - "shape": "6", - "type": "f16", - "pad": "pad01" - } - ], - "sparsity": { - "pad": 21 + "maximum": 5, + "specifics": { + "pad": [ + "[[0 0]]", + "[[0 0], [0 0]]", + "[[0 0], [0 0], [0 0]]", + "[[0 0], [0 0], [0 0], [0 0]]", + "[[0 0], [0 0], [0 0], [0 0], [0 0]]", + "[[0 0], [0 0], [0 0], [0 0], [0 0], [0 0]]" + ] } } } diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py index 97aba13..7c82f31 100644 --- a/verif/conformance/tosa_verif_conformance_generator.py +++ b/verif/conformance/tosa_verif_conformance_generator.py @@ -70,9 +70,12 @@ class GenConformanceError(Exception): def _run_sh_command(args, cwd, full_cmd): """Run an external command and capture stdout/stderr.""" # Quote the command line for printing - full_cmd_esc = [shlex.quote(x) for x in full_cmd] + try: + full_cmd_esc = [shlex.quote(x) for x in full_cmd] + except Exception as e: + raise Exception(f"Error quoting command: {e}") if args.capture_output: - logger.debug(f"Command: {full_cmd_esc}") + logger.info(f"Command: {full_cmd_esc}") rc = subprocess.run( full_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd @@ -80,7 +83,7 @@ def _run_sh_command(args, cwd, full_cmd): if args.capture_output: stdout = rc.stdout.decode("utf-8") - logger.debug(f"stdout: \n{stdout}") + logger.info(f"stdout: \n{stdout}") if rc.returncode != 0: raise Exception( @@ -101,6 +104,7 @@ def build_op_tests( gen_neg_dim_range, supports=[], gen_filter=None, + selector_info=None, ): """Build tests for a given operator. @@ -126,9 +130,31 @@ def build_op_tests( str(args.random_seed), ] + if args.verbosity: + build_cmd_base.append("-" + ("v" * args.verbosity)) + + if args.tests_list_file is not None: + build_cmd_base.append("--list-tests") + if "lazy_data_gen" in supports and args.lazy_data_generation: build_cmd_base.append("--lazy-data-generation") + if "generator_select" in supports: + if selector_info is None: + logger.error( + "build_op_tests error: generator_select mode without selector information" + ) + raise (GenConformanceError()) + selector_config, selector_name = selector_info + build_cmd_base.extend( + [ + "--test-selection-config", + str(selector_config), + "--test-selection-criteria", + selector_name, + ] + ) + build_cmds_list = [] if test_type in ["positive", "both"]: @@ -159,14 +185,19 @@ def build_op_tests( build_cmd_neg_test.extend(target_dtypes_args) build_cmds_list.append(build_cmd_neg_test) - logger.debug(f"Creating {operator} tests with {len(build_cmds_list)} parameter(s)") + logger.info(f"Creating {operator} tests in {len(build_cmds_list)} batch(es)") error = False for i, cmd in enumerate(build_cmds_list): try: - _run_sh_command(args, args.ref_model_path.parent, cmd) + raw_stdout, _ = _run_sh_command(args, args.ref_model_path.parent, cmd) logger.info( f"{operator} test batch {(i+1)}/{len(build_cmds_list)} created successfully" ) + + if args.tests_list_file is not None: + with args.tests_list_file.open("a") as fd: + fd.write(raw_stdout.decode("utf-8")) + except Exception as e: logger.error( f"{operator} test batch {(i+1)}/{len(build_cmds_list)} unsuccessful, skipping" @@ -179,20 +210,20 @@ def build_op_tests( return op_build_dir -def _check_to_include_test(profile, test_name, exclude_negative_tests=False): - """Check test name for exclusions, return False to indicate excluded.""" - excludes = ["ERRORIF"] if exclude_negative_tests else [] +def _check_to_include_test(test_type, test_name): + """Check test name for inclusion based on test_type, returns True to include.""" - for exclusion in excludes: - if f"_{exclusion}_" in test_name: - return False - return True + if test_type == "both": + return True + else: + error_test = "_ERRORIF_" in test_name + return (error_test and test_type == "negative") or ( + not error_test and test_type == "positive" + ) -def _get_all_tests_list( - profile, test_root_dir, operator, exclude_negative_tests=False, include_all=False -): - """Create test list based on tests in the test_dir.""" +def _get_all_tests_list(test_type, test_root_dir, operator): + """Create test list from tests in the test_dir based on chosen type.""" test_dir = test_root_dir / operator if not test_dir.is_dir(): # Tests are split into multiple dirs, for example: conv2d_1x1, conv2d_3x3 @@ -209,8 +240,7 @@ def _get_all_tests_list( [ test for test in tdir.glob("*") - if include_all - or _check_to_include_test(profile, test.name, exclude_negative_tests) + if _check_to_include_test(test_type, test.name) ] ) return tests @@ -240,16 +270,16 @@ def generate_results(args, profile, operator, op_build_dir, supports=[], tests=N if not tests: # Do not need to run ERRORIF tests as they don't have result files - tests = _get_all_tests_list( - profile, op_build_dir, operator, exclude_negative_tests=True - ) + tests = _get_all_tests_list("positive", op_build_dir, operator) + skipped = 0 for test in tests: desc = test / "desc.json" with desc.open("r") as fd: test_desc = json.load(fd) if "meta" in test_desc and "compliance" in test_desc["meta"]: - logger.info( + skipped += 1 + logger.debug( f"Skipping generating results for new compliance test - {str(test)}" ) continue @@ -257,6 +287,9 @@ def generate_results(args, profile, operator, op_build_dir, supports=[], tests=N ref_cmd.append(str(test.absolute())) ref_cmds.append(ref_cmd) + if skipped: + logger.info(f"{skipped} new compliance tests skipped for results generation") + fail_string = "UNEXPECTED_FAILURE" failed_counter = 0 @@ -272,7 +305,7 @@ def generate_results(args, profile, operator, op_build_dir, supports=[], tests=N logger.error(f"Test {i+1}/{len(ref_cmds)}: {ref_cmds[i][-1]} failed.") failed_counter += 1 else: - logger.info(f"Test {i+1}/{len(ref_cmds)}: {ref_cmds[i][-1]} passed.") + logger.debug(f"Test {i+1}/{len(ref_cmds)}: {ref_cmds[i][-1]} passed.") logger.info(f"{len(ref_cmds)-failed_counter}/{len(ref_cmds)} tests passed") logger.info("Ran tests on model and saved results of passing tests") @@ -280,6 +313,7 @@ def generate_results(args, profile, operator, op_build_dir, supports=[], tests=N def convert_tests( args, + test_type, profile, operator, op_build_dir, @@ -315,13 +349,13 @@ def convert_tests( c2c_args_list = [] if not tests: - tests = _get_all_tests_list(profile, op_build_dir, operator) - logger.info(f"Converting all {profile} profile tests") + tests = _get_all_tests_list(test_type, op_build_dir, operator) + logger.info(f"Converting all {profile} profile tests of type {test_type}") # Controls if we copy the tests in their operator sub-directory or not output_dir_relative_pos = -1 if trim_op_subdir else -2 for test in tests: - logger.info(f"Test chosen: {test}") + logger.debug(f"Test chosen: {test}") c2c_args = c2c_args_base.copy() full_output_directory = output_dir / test.relative_to( *test.parts[:output_dir_relative_pos] @@ -350,7 +384,7 @@ def convert_tests( ) failed_counter += 1 else: - logger.info( + logger.debug( f"test {i+1}/{len(c2c_args_list)}: {c2c_args_list[i][-1]} converted" ) logger.info( @@ -394,7 +428,8 @@ def check_op_tests(args, profile, operator, output_dir): """Move test folders than contain files larger than 30MB to new directory.""" destination_dir = str(args.output_dir) + "_large_files" - tests = _get_all_tests_list(profile, output_dir, operator, include_all=True) + # Include all tests - both positive and negative + tests = _get_all_tests_list("both", output_dir, operator) if not tests: logger.error( f"Couldn't find any tests to size check for {operator} in {output_dir}" @@ -617,6 +652,12 @@ def parse_args(argv=None): help="Converts all tests instead of those picked by test_select", ) parser.add_argument( + "--list-tests-to-file", + dest="tests_list_file", + type=Path, + help="Lists out the tests to be generated to a file instead of generating them", + ) + parser.add_argument( "--keep-large-files", action="store_true", help="Keeps tests that contain files larger than 30MB in output directory", @@ -642,45 +683,6 @@ def parse_args(argv=None): ) args = parser.parse_args(argv) - return args - - -def in_version(test_version, gen_dict): - """Check if the selected test_version is compatible with the tests.""" - - def version_string_to_numbers(verstr): - # Turn the "vM.mm.pp" string into Major, Minor, Patch versions - if verstr == TEST_VERSION_LATEST: - return (TOSA_VERSION[0], TOSA_VERSION[1], TOSA_VERSION[2]) - else: - match = re.match(REGEX_VERSION, verstr) - if match is None: - raise KeyError(f"Invalid version string {verstr}") - return (int(v) for v in match.groups()) - - if "from_version" in gen_dict: - selected_version = version_string_to_numbers(test_version) - from_version = version_string_to_numbers(gen_dict["from_version"]) - - # Check the Major version is compatible, then Minor, and lastly Patch - # Unless the versions match, we can exit early due to obvious precedence - for sel, fro in zip(selected_version, from_version): - if sel < fro: - # From version is later than selected version - return False - elif sel > fro: - # From version is earlier than selected version - return True - # If we get here, the version numbers match exactly - return True - else: - # No specific version info - return True - - -def main(): - args = parse_args() - if args.ref_model_dir is not None: # Assume the ref model exe path based on the ref model directory args.ref_model_path = cmf.find_tosa_file( @@ -690,7 +692,7 @@ def main(): logger.error( f"Missing reference model binary (--ref-model-path): {args.ref_model_path}" ) - return 2 + return None args.ref_model_path = args.ref_model_path.absolute() if args.generate_lib_path is None: @@ -701,7 +703,7 @@ def main(): logger.error( f"Missing TOSA generate data library (--generate-lib-path): {args.generate_lib_path}" ) - return 2 + return None args.generate_lib_path = args.generate_lib_path.absolute() if args.schema_path is None: @@ -712,7 +714,7 @@ def main(): logger.error( f"Missing reference model schema (--schema-path): {args.schema_path}" ) - return 2 + return None args.schema_path = args.schema_path.absolute() if args.flatc_path is None: @@ -721,9 +723,11 @@ def main(): ) if not args.flatc_path.is_file(): logger.error(f"Missing flatc binary (--flatc-path): {args.flatc_path}") - return 2 + return None args.flatc_path = args.flatc_path.absolute() + args.param_json_dir = args.param_json_dir.absolute() + if args.unit_tests in ["framework", "both"]: logger.warning( "DEPRECATION - Framework tests are not part of TOSA conformance testing" @@ -732,17 +736,65 @@ def main(): logger.error( "Need to supply location of Framework flatbuffers schema via --framework-schema" ) - return 2 + return None if not args.framework_tests_dir.is_dir(): logger.error( f"Missing or invalid framework tests directory: {args.framework_tests_dir}" ) - return 2 + return None + + return args + + +def in_version(test_version, gen_dict): + """Check if the selected test_version is compatible with the tests.""" + + def version_string_to_numbers(verstr): + # Turn the "vM.mm.pp" string into Major, Minor, Patch versions + if verstr == TEST_VERSION_LATEST: + return (TOSA_VERSION[0], TOSA_VERSION[1], TOSA_VERSION[2]) + else: + match = re.match(REGEX_VERSION, verstr) + if match is None: + raise KeyError(f"Invalid version string {verstr}") + return (int(v) for v in match.groups()) + + if "from_version" in gen_dict: + selected_version = version_string_to_numbers(test_version) + from_version = version_string_to_numbers(gen_dict["from_version"]) + + # Check the Major version is compatible, then Minor, and lastly Patch + # Unless the versions match, we can exit early due to obvious precedence + for sel, fro in zip(selected_version, from_version): + if sel < fro: + # From version is later than selected version + return False + elif sel > fro: + # From version is earlier than selected version + return True + # If we get here, the version numbers match exactly + return True + else: + # No specific version info + return True + +def _get_log_level(verbosity): loglevels = (logging.WARNING, logging.INFO, logging.DEBUG) - loglevel = loglevels[min(args.verbosity, len(loglevels) - 1)] + verbosity = max(verbosity, 0) + return loglevels[min(verbosity, len(loglevels) - 1)] + + +def main(): + args = parse_args() + if args is None: + # Argument processing error + return 2 + + loglevel = _get_log_level(args.verbosity) logger.setLevel(loglevel) - # Set other loggers the same + # Set other loggers to a quieter level + loglevel = _get_log_level(args.verbosity - 1) logging.getLogger("test_select").setLevel(loglevel) logging.getLogger("convert2conformance").setLevel(loglevel) @@ -757,6 +809,11 @@ def main(): logger.debug(f"Creating build directory: {args.build_dir}") args.build_dir.mkdir(parents=True, exist_ok=True) + if args.tests_list_file is not None: + # Try creating tests list file + with args.tests_list_file.open("w") as fd: + fd.write("") + # TODO: For tosa-mi should really generate tosa-bi profile as well # - for now leave it as subset instead of as superset (for testing) if args.profile == PROFILES_ALL: @@ -822,6 +879,7 @@ def main(): ) convert_tests( args, + "positive", profile, op, framework_test_dir, @@ -846,6 +904,7 @@ def main(): f"Couldn't load operator test params - {test_params_file}: {e}" ) return 1 + logger.debug(f"Using config file: {str(test_params_file)}") operators = args.operators if not operators: @@ -913,23 +972,6 @@ def main(): else None ) - ignore_missing = gen_name != STANDARD_GENERATOR_GROUP - tags = ( - [gen_name] if gen_name != STANDARD_GENERATOR_GROUP else None - ) - - op_build_dir = build_op_tests( - args, - test_type, - profile, - op, - gen_name, - gen_dict["generator_args"], - gen_neg_dim_range, - supports=supports, - gen_filter=gen_filter, - ) - # Work out which selection criteria we are using if "selector" in gen_dict: selector_name = gen_dict["selector"] @@ -946,19 +988,38 @@ def main(): ) raise (GenConformanceError()) - # Selection criteria - selection_config = test_params[op]["selection"][selector_name] + op_build_dir = build_op_tests( + args, + test_type, + profile, + op, + gen_name, + gen_dict["generator_args"], + gen_neg_dim_range, + supports=supports, + gen_filter=gen_filter, + selector_info=(test_params_file, selector_name), + ) + + if args.tests_list_file is not None: + logger.info("Tests list file extended") + continue - if args.convert_all_tests: - logger.debug(f"Running and converting all {op} tests") - generate_results( - args, profile, op, op_build_dir, supports=supports - ) + if args.convert_all_tests or "generator_select" in supports: + if test_type in ["positive", "both"]: + logger.info(f"Running and converting all {op} tests") + generate_results( + args, profile, op, op_build_dir, supports=supports + ) operator_test_list = None else: - logger.debug( + logger.info( f"Running and converting selection of {op} tests" ) + # Selection criteria + selection_config = test_params[op]["selection"][ + selector_name + ] if test_type in ["positive", "both"]: if ( "all" in selection_config @@ -967,13 +1028,16 @@ def main(): # Just get all the positive tests tests_gen, tests_gen2 = tee( _get_all_tests_list( - profile, + "positive", op_build_dir, op, - exclude_negative_tests=True, ) ) else: + ignore_missing = ( + gen_name != STANDARD_GENERATOR_GROUP + ) + # Get a selection of positive tests tests_gen, tests_gen2 = tee( get_op_tests_selection( @@ -1007,8 +1071,14 @@ def main(): negative=True, ) ) + + tags = ( + [gen_name] if gen_name != STANDARD_GENERATOR_GROUP else None + ) + output_dir = convert_tests( args, + test_type, profile, op, op_build_dir, |