From 1271c44bd2c9e670e132db491a053a0e6603798f Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Tue, 5 Sep 2023 11:39:26 +0100 Subject: Initial lazy data-gen and compliance test build support Add initial support for compliance and lazy data-gen meta data added to desc.json for MATMUL. Signed-off-by: Jeremy Johnson Change-Id: I00c047814134a96d7c98d890e93b5884e25b8e64 --- verif/conformance/README.md | 3 ++ verif/conformance/tosa_main_profile_ops_info.json | 7 +--- .../tosa_verif_conformance_generator.py | 44 ++++++++++++++++++++-- 3 files changed, 45 insertions(+), 9 deletions(-) (limited to 'verif/conformance') diff --git a/verif/conformance/README.md b/verif/conformance/README.md index a7ad789..8fbf4fd 100644 --- a/verif/conformance/README.md +++ b/verif/conformance/README.md @@ -17,6 +17,7 @@ Each operator entry contains: * "group" - name of the group this operator is in, in the spec * "profile" - list of profiles that this operator covers +* "support_for" - optional list of supported creation modes out of: lazy_data_gen (data generation just before test run) * "generation" - dictionary of test generation details - see below * "selection" - dictionary of test selection details - see below @@ -49,6 +50,8 @@ before permutes are applied) ### Framework ops +DEPRECATED - not supported for conformance testing. + NOTE: Currently assumed all framework ops will be TFLite. Naming: `tosa_PPP_profile_framework_ops_info.json` diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json index 7388835..0d3df72 100644 --- a/verif/conformance/tosa_main_profile_ops_info.json +++ b/verif/conformance/tosa_main_profile_ops_info.json @@ -1417,6 +1417,7 @@ "profile": [ "tosa-mi" ], + "support_for": [ "lazy_data_gen" ], "generation": { "standard": { "negative_dim_range": "1,10", @@ -1460,11 +1461,7 @@ }, "selection": { "default": { - "params": {}, - "permutes": [ - "shape", - "type" - ] + "all": "true" } } }, diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py index 0fb5500..c2ea4ec 100644 --- a/verif/conformance/tosa_verif_conformance_generator.py +++ b/verif/conformance/tosa_verif_conformance_generator.py @@ -84,7 +84,14 @@ def _run_sh_command(args, cwd, full_cmd): def build_op_tests( - args, test_type, profile, operator, group, gen_args_list, gen_neg_dim_range + args, + test_type, + profile, + operator, + group, + gen_args_list, + gen_neg_dim_range, + supports=[], ): """Build tests for a given operator. @@ -105,6 +112,9 @@ def build_op_tests( str(args.random_seed), ] + if "lazy_data_gen" in supports and args.lazy_data_generation: + build_cmd_base.append("--lazy-data-generation") + build_cmds_list = [] if test_type in ["positive", "both"]: @@ -198,8 +208,12 @@ def _get_all_tests_list( return tests -def generate_results(args, profile, operator, op_build_dir, tests=None): +def generate_results(args, profile, operator, op_build_dir, supports=[], tests=None): """Run tests on reference model and save result to the test directory.""" + if "lazy_data_gen" in supports and args.lazy_data_generation: + logger.info("Skipping running tests due to lazy data gen") + return + num_cores = args.num_cores run_tests_cmd = "tosa_verif_run_tests" @@ -254,6 +268,7 @@ def convert_tests( op_build_dir, output_dir, op_profiles_list, + supports=[], tests=None, group=None, trim_op_subdir=False, @@ -275,6 +290,8 @@ def convert_tests( c2c_args_base.extend(["--tag", tag]) if args.framework_schema: c2c_args_base.extend(["--framework-schema", str(args.framework_schema)]) + if "lazy_data_gen" in supports and args.lazy_data_generation: + c2c_args_base.append("--lazy-data-generation") c2c_args_base.append("--output-directory") c2c_args_list = [] @@ -473,6 +490,11 @@ def parse_args(argv=None): type=str, help="Type of tests produced (default is both)", ) + parser.add_argument( + "--lazy-data-generation", + action="store_true", + help="Enable lazy data generation (only for tosa-mi)", + ) parser.add_argument( "--ref-model-directory", dest="ref_model_dir", @@ -718,6 +740,11 @@ def main(): operator_group = test_params[op]["group"] root_output_dir = args.output_dir / "operators" + supports = ( + test_params[op]["support_for"] + if "support_for" in test_params[op] + else [] + ) # Iterate through the generation groups selecting tests from each for gen_name, gen_dict in test_params[op]["generation"].items(): @@ -756,6 +783,7 @@ def main(): gen_name, gen_dict["generator_args"], gen_neg_dim_range, + supports=supports, ) # Work out which selection criteria we are using @@ -782,7 +810,9 @@ def main(): and selection_config["all"] == "true" ): logger.debug(f"Running and converting all {op} tests") - generate_results(args, profile, op, op_build_dir) + generate_results( + args, profile, op, op_build_dir, supports=supports + ) operator_test_list = None else: logger.debug( @@ -800,7 +830,12 @@ def main(): ) ) generate_results( - args, profile, op, op_build_dir, tests_gen + args, + profile, + op, + op_build_dir, + supports=supports, + tests=tests_gen, ) operator_test_list = list(tests_gen2) else: @@ -823,6 +858,7 @@ def main(): op_build_dir, root_output_dir, op_profiles_list, + supports=supports, tests=operator_test_list, group=operator_group, tags=tags, -- cgit v1.2.1