Create MI tests for EW Binary: ADD, SUB, MUL

Add option to control negative tests dim range to stop ERROR_IF
test clashes between profiles.

Change-Id: I1f806934480cc32b95e80613c66fecb1e6ff5426
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>

3 files changed, 207 insertions, 31 deletions

diff --git a/scripts/convert2conformance/convert2conformance.py b/scripts/convert2conformance/convert2conformance.py
index 06726ba..9cafed3 100755
--- a/scripts/convert2conformance/convert2conformance.py
+++ b/scripts/convert2conformance/convert2conformance.py
@@ -79,6 +79,12 @@ def parse_args(argv):
         help="Profiles this test is suitable for. May be repeated",
     )
     parser.add_argument(
+        "--strict",
+        dest="strict",
+        action="store_true",
+        help="Output directory must not contain the same test directory",
+    )
+    parser.add_argument(
         "-v", "--verbose", dest="verbose", action="store_true", help="Verbose operation"
     )
     args = parser.parse_args(argv)
@@ -269,9 +275,12 @@ def main(argv=None):
 
     # Make the output directory if needed
     try:
-        args.output_dir.mkdir(parents=True, exist_ok=True)
+        args.output_dir.mkdir(parents=True, exist_ok=(not args.strict))
     except FileExistsError:
-        logger.error(f"{args.output_dir} is not a directory")
+        if args.strict:
+            logger.error(f"{args.output_dir} already exists")
+        else:
+            logger.error(f"{args.output_dir} is not a directory")
         return 2
 
     # Convert the TOSA flatbuffer binary
diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json
index da04d85..a175d3e 100644
--- a/verif/conformance/tosa_main_profile_ops_info.json
+++ b/verif/conformance/tosa_main_profile_ops_info.json
@@ -57,6 +57,59 @@
             "tosa-mi"
         ]
     },
+    "add": {
+        "group": "ew_binary",
+        "generator_args": [
+            [
+                "--target-dtype",
+                "fp32",
+                "--target-dtype",
+                "fp16",
+                "--target-dtype",
+                "bf16",
+                "--fp-values-range",
+                "-2.0,2.0",
+                "--tensor-dim-range",
+                "16,64",
+                "--target-rank",
+                "1",
+                "--target-rank",
+                "2",
+                "--target-rank",
+                "3"
+            ],
+            [
+                "--target-dtype",
+                "fp32",
+                "--fp-values-range",
+                "-2.0,2.0",
+                "--tensor-dim-range",
+                "1,16",
+                "--target-rank",
+                "4",
+                "--target-rank",
+                "5"
+            ],
+            [
+                "--target-dtype",
+                "fp16",
+                "--fp-values-range",
+                "-2.0,2.0",
+                "--target-shape",
+                "2,1,1,65532",
+                "--target-shape",
+                "65542,2,1,1,1"
+            ]
+        ],
+        "params": {},
+        "permutes": [
+            "shape",
+            "type"
+        ],
+        "profile": [
+            "tosa-mi"
+        ]
+    },
     "argmax": {
         "group": "tensor",
         "generator_args": [
@@ -1106,6 +1159,60 @@
             "tosa-mi"
         ]
     },
+    "mul": {
+        "group": "ew_binary",
+        "generator_negative_dim_range": "1,10",
+        "generator_args": [
+            [
+                "--target-dtype",
+                "fp32",
+                "--target-dtype",
+                "fp16",
+                "--target-dtype",
+                "bf16",
+                "--fp-values-range",
+                "-2.0,2.0",
+                "--tensor-dim-range",
+                "16,64",
+                "--target-rank",
+                "1",
+                "--target-rank",
+                "2",
+                "--target-rank",
+                "3"
+            ],
+            [
+                "--target-dtype",
+                "fp16",
+                "--fp-values-range",
+                "-2.0,2.0",
+                "--tensor-dim-range",
+                "1,16",
+                "--target-rank",
+                "4",
+                "--target-rank",
+                "5"
+            ],
+            [
+                "--target-dtype",
+                "bf16",
+                "--fp-values-range",
+                "-2.0,2.0",
+                "--target-shape",
+                "1,1,3,65534",
+                "--target-shape",
+                "1,2,1,65541,1"
+            ]
+        ],
+        "params": {},
+        "permutes": [
+            "shape",
+            "type"
+        ],
+        "profile": [
+            "tosa-mi"
+        ]
+    },
     "reduce_max": {
         "group": "reduction",
         "generator_args": [
@@ -1335,6 +1442,59 @@
             "tosa-mi"
         ]
     },
+    "sub": {
+        "group": "ew_binary",
+        "generator_args": [
+            [
+                "--target-dtype",
+                "fp32",
+                "--target-dtype",
+                "fp16",
+                "--target-dtype",
+                "bf16",
+                "--fp-values-range",
+                "-2.0,2.0",
+                "--tensor-dim-range",
+                "16,64",
+                "--target-rank",
+                "1",
+                "--target-rank",
+                "2",
+                "--target-rank",
+                "3"
+            ],
+            [
+                "--target-dtype",
+                "bf16",
+                "--fp-values-range",
+                "-2.0,2.0",
+                "--tensor-dim-range",
+                "1,16",
+                "--target-rank",
+                "4",
+                "--target-rank",
+                "5"
+            ],
+            [
+                "--target-dtype",
+                "fp32",
+                "--fp-values-range",
+                "-2.0,2.0",
+                "--target-shape",
+                "1,1,65533,3",
+                "--target-shape",
+                "1,1,2,1,65545"
+            ]
+        ],
+        "params": {},
+        "permutes": [
+            "shape",
+            "type"
+        ],
+        "profile": [
+            "tosa-mi"
+        ]
+    },
     "tanh": {
         "group": "activation",
         "generator_args": [
diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py
index 7182d9c..4b57f63 100644
--- a/verif/conformance/tosa_verif_conformance_generator.py
+++ b/verif/conformance/tosa_verif_conformance_generator.py
@@ -92,7 +92,7 @@ def build_op_tests(args, profile, operator, test_params):
     build_tests_cmd = "tosa_verif_build_tests"
     op_build_dir = args.build_dir / profile
 
-    ref_cmd_base = [
+    build_cmd_base = [
         build_tests_cmd,
         "--filter",
         operator,
@@ -102,15 +102,15 @@ def build_op_tests(args, profile, operator, test_params):
         str(args.random_seed),
     ]
 
-    ref_cmds = []
+    build_cmds_list = []
 
     if args.test_type in ["positive", "both"]:
         # Append extra parameters and run test generator for each set of parameters.
         for arglist in test_params[operator]["generator_args"]:
-            ref_cmd_pos_test = ref_cmd_base.copy()
-            ref_cmd_pos_test.extend(["--test-type", "positive"])
-            ref_cmd_pos_test.extend(arglist)
-            ref_cmds.append(ref_cmd_pos_test)
+            build_cmd_pos_test = build_cmd_base.copy()
+            build_cmd_pos_test.extend(["--test-type", "positive"])
+            build_cmd_pos_test.extend(arglist)
+            build_cmds_list.append(build_cmd_pos_test)
 
     if args.test_type in ["negative", "both"]:
         # Get target-dtypes options only to limit tests to those needed
@@ -123,24 +123,29 @@ def build_op_tests(args, profile, operator, test_params):
                         target_dtypes_args.extend(arglist[idx : idx + 2])
                     idx += 1  # skip over option (and then argument below)
                 idx += 1
-        ref_cmd_neg_test = ref_cmd_base.copy()
-        ref_cmd_neg_test.extend(["--test-type", "negative"])
+        build_cmd_neg_test = build_cmd_base.copy()
+        build_cmd_neg_test.extend(["--test-type", "negative"])
         # Limit sizes of negative tests
-        ref_cmd_neg_test.extend(["--tensor-dim-range", "1,16"])
-        ref_cmd_neg_test.extend(target_dtypes_args)
-        ref_cmds.append(ref_cmd_neg_test)
+        dim_range = (
+            test_params[operator]["generator_negative_dim_range"]
+            if "generator_negative_dim_range" in test_params[operator]
+            else "1,16"
+        )
+        build_cmd_neg_test.extend(["--tensor-dim-range", dim_range])
+        build_cmd_neg_test.extend(target_dtypes_args)
+        build_cmds_list.append(build_cmd_neg_test)
 
-    logger.debug(f"Creating {operator} tests with {len(ref_cmds)} parameter(s)")
+    logger.debug(f"Creating {operator} tests with {len(build_cmds_list)} parameter(s)")
     error = False
-    for i, cmd in enumerate(ref_cmds):
+    for i, cmd in enumerate(build_cmds_list):
         try:
             _run_sh_command(args, args.ref_model_dir.absolute(), cmd)
             logger.info(
-                f"{operator} test batch {(i+1)}/{len(ref_cmds)} created successfully"
+                f"{operator} test batch {(i+1)}/{len(build_cmds_list)} created successfully"
             )
         except Exception as e:
             logger.error(
-                f"{operator} test batch {(i+1)}/{len(ref_cmds)} unsuccessful, skipping"
+                f"{operator} test batch {(i+1)}/{len(build_cmds_list)} unsuccessful, skipping"
             )
             logger.error(f" build_op_tests error: {e} ")
             error = True
@@ -254,16 +259,16 @@ def convert_tests(
     if group:
         output_dir = output_dir / group
 
-    ref_cmd_base = ["--ref-model-directory", str(ref_model_dir)]
+    c2c_args_base = ["--strict", "--ref-model-directory", str(ref_model_dir)]
     # This op maybe in more than one profile - e.g. tosa_bi and tosa_mi
     # even if we are only producing tests for tosa_mi
     for op_profile in op_profiles_list:
-        ref_cmd_base.extend(["--profile", op_profile])
+        c2c_args_base.extend(["--profile", op_profile])
     if args.framework_schema:
-        ref_cmd_base.extend(["--framework-schema", str(args.framework_schema)])
-    ref_cmd_base.append("--output-directory")
+        c2c_args_base.extend(["--framework-schema", str(args.framework_schema)])
+    c2c_args_base.append("--output-directory")
 
-    ref_cmds = []
+    c2c_args_list = []
 
     if not tests:
         tests = _get_all_tests_list(profile, op_build_dir, operator)
@@ -273,21 +278,21 @@ def convert_tests(
     output_dir_relative_pos = -1 if trim_op_subdir else -2
     for test in tests:
         logger.info(f"Test chosen: {test}")
-        ref_cmd = ref_cmd_base.copy()
+        c2c_args = c2c_args_base.copy()
         full_output_directory = output_dir / test.relative_to(
             *test.parts[:output_dir_relative_pos]
         )
-        ref_cmd.append(str(full_output_directory))
-        ref_cmd.append(str(test))
-        ref_cmds.append(ref_cmd)
+        c2c_args.append(str(full_output_directory))
+        c2c_args.append(str(test))
+        c2c_args_list.append(c2c_args)
 
-    if len(ref_cmds) == 0:
+    if len(c2c_args_list) == 0:
         logger.warning("No tests found. Nothing to convert")
         return
 
     job_pool = mp.Pool(args.num_cores)
 
-    pool_results = job_pool.map(c2c_main, ref_cmds)
+    pool_results = job_pool.map(c2c_main, c2c_args_list)
     job_pool.close()
     job_pool.join()
 
@@ -295,13 +300,15 @@ def convert_tests(
     for i, result in enumerate(pool_results):
         if result != 0:
             logger.error(
-                f"test {i+1}/{len(ref_cmds)}: {ref_cmds[i][-1]} failed to convert."
+                f"test {i+1}/{len(c2c_args_list)}: {c2c_args_list[i][-1]} failed to convert."
             )
             failed_counter += 1
         else:
-            logger.info(f"test {i+1}/{len(ref_cmds)}: {ref_cmds[i][-1]} converted")
+            logger.info(
+                f"test {i+1}/{len(c2c_args_list)}: {c2c_args_list[i][-1]} converted"
+            )
     logger.info(
-        f"{len(ref_cmds)-failed_counter}/{len(ref_cmds)} tests successfully converted"
+        f"{len(c2c_args_list)-failed_counter}/{len(c2c_args_list)} tests successfully converted"
     )
 
     if failed_counter > 0: