diff options
author | Jeremy Johnson <jeremy.johnson@arm.com> | 2022-12-12 14:18:10 +0000 |
---|---|---|
committer | Jeremy Johnson <jeremy.johnson@arm.com> | 2023-01-04 17:14:28 +0000 |
commit | dd8d9c251db0fece6453d86116052ad7f3e2d697 (patch) | |
tree | 272f92e603f2d6158ea8b44ce3d11b34978b6b0d | |
parent | 6ffb7c8fb88d3a5d0e94f3e604dc3548deea95d7 (diff) | |
download | reference_model-dd8d9c251db0fece6453d86116052ad7f3e2d697.tar.gz |
Create MI tests for EW Binary: ADD, SUB, MUL
Add option to control negative tests dim range to stop ERROR_IF
test clashes between profiles.
Change-Id: I1f806934480cc32b95e80613c66fecb1e6ff5426
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
-rwxr-xr-x | scripts/convert2conformance/convert2conformance.py | 13 | ||||
-rw-r--r-- | verif/conformance/tosa_main_profile_ops_info.json | 160 | ||||
-rw-r--r-- | verif/conformance/tosa_verif_conformance_generator.py | 65 |
3 files changed, 207 insertions, 31 deletions
diff --git a/scripts/convert2conformance/convert2conformance.py b/scripts/convert2conformance/convert2conformance.py index 06726ba..9cafed3 100755 --- a/scripts/convert2conformance/convert2conformance.py +++ b/scripts/convert2conformance/convert2conformance.py @@ -79,6 +79,12 @@ def parse_args(argv): help="Profiles this test is suitable for. May be repeated", ) parser.add_argument( + "--strict", + dest="strict", + action="store_true", + help="Output directory must not contain the same test directory", + ) + parser.add_argument( "-v", "--verbose", dest="verbose", action="store_true", help="Verbose operation" ) args = parser.parse_args(argv) @@ -269,9 +275,12 @@ def main(argv=None): # Make the output directory if needed try: - args.output_dir.mkdir(parents=True, exist_ok=True) + args.output_dir.mkdir(parents=True, exist_ok=(not args.strict)) except FileExistsError: - logger.error(f"{args.output_dir} is not a directory") + if args.strict: + logger.error(f"{args.output_dir} already exists") + else: + logger.error(f"{args.output_dir} is not a directory") return 2 # Convert the TOSA flatbuffer binary diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json index da04d85..a175d3e 100644 --- a/verif/conformance/tosa_main_profile_ops_info.json +++ b/verif/conformance/tosa_main_profile_ops_info.json @@ -57,6 +57,59 @@ "tosa-mi" ] }, + "add": { + "group": "ew_binary", + "generator_args": [ + [ + "--target-dtype", + "fp32", + "--target-dtype", + "fp16", + "--target-dtype", + "bf16", + "--fp-values-range", + "-2.0,2.0", + "--tensor-dim-range", + "16,64", + "--target-rank", + "1", + "--target-rank", + "2", + "--target-rank", + "3" + ], + [ + "--target-dtype", + "fp32", + "--fp-values-range", + "-2.0,2.0", + "--tensor-dim-range", + "1,16", + "--target-rank", + "4", + "--target-rank", + "5" + ], + [ + "--target-dtype", + "fp16", + "--fp-values-range", + "-2.0,2.0", + "--target-shape", + "2,1,1,65532", + "--target-shape", + "65542,2,1,1,1" + ] + ], + "params": {}, + "permutes": [ + "shape", + "type" + ], + "profile": [ + "tosa-mi" + ] + }, "argmax": { "group": "tensor", "generator_args": [ @@ -1106,6 +1159,60 @@ "tosa-mi" ] }, + "mul": { + "group": "ew_binary", + "generator_negative_dim_range": "1,10", + "generator_args": [ + [ + "--target-dtype", + "fp32", + "--target-dtype", + "fp16", + "--target-dtype", + "bf16", + "--fp-values-range", + "-2.0,2.0", + "--tensor-dim-range", + "16,64", + "--target-rank", + "1", + "--target-rank", + "2", + "--target-rank", + "3" + ], + [ + "--target-dtype", + "fp16", + "--fp-values-range", + "-2.0,2.0", + "--tensor-dim-range", + "1,16", + "--target-rank", + "4", + "--target-rank", + "5" + ], + [ + "--target-dtype", + "bf16", + "--fp-values-range", + "-2.0,2.0", + "--target-shape", + "1,1,3,65534", + "--target-shape", + "1,2,1,65541,1" + ] + ], + "params": {}, + "permutes": [ + "shape", + "type" + ], + "profile": [ + "tosa-mi" + ] + }, "reduce_max": { "group": "reduction", "generator_args": [ @@ -1335,6 +1442,59 @@ "tosa-mi" ] }, + "sub": { + "group": "ew_binary", + "generator_args": [ + [ + "--target-dtype", + "fp32", + "--target-dtype", + "fp16", + "--target-dtype", + "bf16", + "--fp-values-range", + "-2.0,2.0", + "--tensor-dim-range", + "16,64", + "--target-rank", + "1", + "--target-rank", + "2", + "--target-rank", + "3" + ], + [ + "--target-dtype", + "bf16", + "--fp-values-range", + "-2.0,2.0", + "--tensor-dim-range", + "1,16", + "--target-rank", + "4", + "--target-rank", + "5" + ], + [ + "--target-dtype", + "fp32", + "--fp-values-range", + "-2.0,2.0", + "--target-shape", + "1,1,65533,3", + "--target-shape", + "1,1,2,1,65545" + ] + ], + "params": {}, + "permutes": [ + "shape", + "type" + ], + "profile": [ + "tosa-mi" + ] + }, "tanh": { "group": "activation", "generator_args": [ diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py index 7182d9c..4b57f63 100644 --- a/verif/conformance/tosa_verif_conformance_generator.py +++ b/verif/conformance/tosa_verif_conformance_generator.py @@ -92,7 +92,7 @@ def build_op_tests(args, profile, operator, test_params): build_tests_cmd = "tosa_verif_build_tests" op_build_dir = args.build_dir / profile - ref_cmd_base = [ + build_cmd_base = [ build_tests_cmd, "--filter", operator, @@ -102,15 +102,15 @@ def build_op_tests(args, profile, operator, test_params): str(args.random_seed), ] - ref_cmds = [] + build_cmds_list = [] if args.test_type in ["positive", "both"]: # Append extra parameters and run test generator for each set of parameters. for arglist in test_params[operator]["generator_args"]: - ref_cmd_pos_test = ref_cmd_base.copy() - ref_cmd_pos_test.extend(["--test-type", "positive"]) - ref_cmd_pos_test.extend(arglist) - ref_cmds.append(ref_cmd_pos_test) + build_cmd_pos_test = build_cmd_base.copy() + build_cmd_pos_test.extend(["--test-type", "positive"]) + build_cmd_pos_test.extend(arglist) + build_cmds_list.append(build_cmd_pos_test) if args.test_type in ["negative", "both"]: # Get target-dtypes options only to limit tests to those needed @@ -123,24 +123,29 @@ def build_op_tests(args, profile, operator, test_params): target_dtypes_args.extend(arglist[idx : idx + 2]) idx += 1 # skip over option (and then argument below) idx += 1 - ref_cmd_neg_test = ref_cmd_base.copy() - ref_cmd_neg_test.extend(["--test-type", "negative"]) + build_cmd_neg_test = build_cmd_base.copy() + build_cmd_neg_test.extend(["--test-type", "negative"]) # Limit sizes of negative tests - ref_cmd_neg_test.extend(["--tensor-dim-range", "1,16"]) - ref_cmd_neg_test.extend(target_dtypes_args) - ref_cmds.append(ref_cmd_neg_test) + dim_range = ( + test_params[operator]["generator_negative_dim_range"] + if "generator_negative_dim_range" in test_params[operator] + else "1,16" + ) + build_cmd_neg_test.extend(["--tensor-dim-range", dim_range]) + build_cmd_neg_test.extend(target_dtypes_args) + build_cmds_list.append(build_cmd_neg_test) - logger.debug(f"Creating {operator} tests with {len(ref_cmds)} parameter(s)") + logger.debug(f"Creating {operator} tests with {len(build_cmds_list)} parameter(s)") error = False - for i, cmd in enumerate(ref_cmds): + for i, cmd in enumerate(build_cmds_list): try: _run_sh_command(args, args.ref_model_dir.absolute(), cmd) logger.info( - f"{operator} test batch {(i+1)}/{len(ref_cmds)} created successfully" + f"{operator} test batch {(i+1)}/{len(build_cmds_list)} created successfully" ) except Exception as e: logger.error( - f"{operator} test batch {(i+1)}/{len(ref_cmds)} unsuccessful, skipping" + f"{operator} test batch {(i+1)}/{len(build_cmds_list)} unsuccessful, skipping" ) logger.error(f" build_op_tests error: {e} ") error = True @@ -254,16 +259,16 @@ def convert_tests( if group: output_dir = output_dir / group - ref_cmd_base = ["--ref-model-directory", str(ref_model_dir)] + c2c_args_base = ["--strict", "--ref-model-directory", str(ref_model_dir)] # This op maybe in more than one profile - e.g. tosa_bi and tosa_mi # even if we are only producing tests for tosa_mi for op_profile in op_profiles_list: - ref_cmd_base.extend(["--profile", op_profile]) + c2c_args_base.extend(["--profile", op_profile]) if args.framework_schema: - ref_cmd_base.extend(["--framework-schema", str(args.framework_schema)]) - ref_cmd_base.append("--output-directory") + c2c_args_base.extend(["--framework-schema", str(args.framework_schema)]) + c2c_args_base.append("--output-directory") - ref_cmds = [] + c2c_args_list = [] if not tests: tests = _get_all_tests_list(profile, op_build_dir, operator) @@ -273,21 +278,21 @@ def convert_tests( output_dir_relative_pos = -1 if trim_op_subdir else -2 for test in tests: logger.info(f"Test chosen: {test}") - ref_cmd = ref_cmd_base.copy() + c2c_args = c2c_args_base.copy() full_output_directory = output_dir / test.relative_to( *test.parts[:output_dir_relative_pos] ) - ref_cmd.append(str(full_output_directory)) - ref_cmd.append(str(test)) - ref_cmds.append(ref_cmd) + c2c_args.append(str(full_output_directory)) + c2c_args.append(str(test)) + c2c_args_list.append(c2c_args) - if len(ref_cmds) == 0: + if len(c2c_args_list) == 0: logger.warning("No tests found. Nothing to convert") return job_pool = mp.Pool(args.num_cores) - pool_results = job_pool.map(c2c_main, ref_cmds) + pool_results = job_pool.map(c2c_main, c2c_args_list) job_pool.close() job_pool.join() @@ -295,13 +300,15 @@ def convert_tests( for i, result in enumerate(pool_results): if result != 0: logger.error( - f"test {i+1}/{len(ref_cmds)}: {ref_cmds[i][-1]} failed to convert." + f"test {i+1}/{len(c2c_args_list)}: {c2c_args_list[i][-1]} failed to convert." ) failed_counter += 1 else: - logger.info(f"test {i+1}/{len(ref_cmds)}: {ref_cmds[i][-1]} converted") + logger.info( + f"test {i+1}/{len(c2c_args_list)}: {c2c_args_list[i][-1]} converted" + ) logger.info( - f"{len(ref_cmds)-failed_counter}/{len(ref_cmds)} tests successfully converted" + f"{len(c2c_args_list)-failed_counter}/{len(c2c_args_list)} tests successfully converted" ) if failed_counter > 0: |