Fix missing/broken ERROR_IF tests

Fix CONV2D WrongOutputType FP32 & Pad/Stride/DilationSmallerZero issues.
Fix PAD WrongInputType.

Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: I57fc57c43e63685e05bf5e3d562c3167411fd57b

7 files changed, 141 insertions, 46 deletions

diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py
index 5402c21..3b7c2df 100644
--- a/verif/conformance/tosa_verif_conformance_generator.py
+++ b/verif/conformance/tosa_verif_conformance_generator.py
@@ -82,7 +82,9 @@ def _run_sh_command(args, cwd, full_cmd):
     )
 
     if args.capture_output:
+        stderr = rc.stderr.decode("utf-8")
         stdout = rc.stdout.decode("utf-8")
+        logger.info(f"stderr: \n{stderr}")
         logger.info(f"stdout: \n{stdout}")
     if rc.returncode != 0:
 
diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py
index ffa3683..4878708 100644
--- a/verif/generator/tosa_arg_gen.py
+++ b/verif/generator/tosa_arg_gen.py
@@ -1976,6 +1976,9 @@ class TosaArgGen:
             sparsity += 1
         return sparsity
 
+    # Maximum number of error_if variants to produce
+    MAX_CONV_ERROR_IFS = 3
+
     @staticmethod
     def agConv(testGen, rng, opName, shapeList, dtypes, error_name=None):
         # Used by CONV2D, CONV3D and DEPTHWISE_CONV2D
@@ -2017,17 +2020,60 @@ class TosaArgGen:
             k_size *= ifm_shape[-1]
 
         if not testGen.args.level8k:
-            # Generate comprehensive argument lists
-            # - except for named errors, which use specific invalid value(s)
-            if error_name == ErrorIf.PadSmallerZero:
-                p_vals = [rng.choice(range(-5, 0))]
+            if error_name in (
+                ErrorIf.PadSmallerZero,
+                ErrorIf.StrideSmallerOne,
+                ErrorIf.DilationSmallerOne,
+            ):
+                # Use specific invalid value(s)
+                if error_name == ErrorIf.PadSmallerZero:
+                    # Create negative paddings but with positive opposite paddings
+                    neg_pad = rng.choice(range(-5, 0))
+                    p_vals = [neg_pad, abs(neg_pad)]
+                else:
+                    p_vals = [0, 0]
+                if error_name == ErrorIf.StrideSmallerOne:
+                    # Can't use stride=0, as it is used to derive output shape, as a divisor
+                    s_vals = [rng.choice(range(-5, 0))]
+                else:
+                    s_vals = [1]
+                if error_name == ErrorIf.DilationSmallerOne:
+                    d_vals = [rng.choice(range(-5, 1))]
+                else:
+                    d_vals = [1]
+                p = p_vals * k_rank
+                s = s_vals * k_rank
+                d = d_vals * k_rank
+
+                # Fix values to produce valid error_if
+                for index in range(k_rank):
+                    pad_offset = index * 2
+                    fixed = False
+                    while not fixed:
+                        partial = (
+                            ifm_shape[index + 1]
+                            - 1
+                            + p[pad_offset]
+                            + p[pad_offset + 1]
+                            - (k_shape[index] - 1) * d[index]
+                        )
+                        remainder = partial % s[index]
+                        if partial <= 0:
+                            p[pad_offset + 1] += abs(partial) + 1
+                        elif remainder:
+                            # Stride will be negative for StrideSmallerOne
+                            assert remainder < 0
+                            p[pad_offset + 1] += abs(remainder)
+                        else:
+                            fixed = True
+                paddings = {tuple(p)}
+                strides = {tuple(s)}
+                dilations = {tuple(d)}
+                logger.debug(f"agConv: error pad={p} stride={s} dilation={d}")
             else:
+                # Generate comprehensive argument lists
                 p_vals = [x for x in range(0, testGen.args.max_conv_padding + 1)]
-            paddings = {x for x in itertools.product(*([p_vals] * k_rank * 2))}
-            if error_name == ErrorIf.StrideSmallerOne:
-                # Can't use stride=0, as it is used to derive output shape, as a divisor
-                s_vals = [rng.choice(range(-5, 0))]
-            else:
+                paddings = {x for x in itertools.product(*([p_vals] * k_rank * 2))}
                 # Stride must be greater than 1 to force non-integer error
                 startStride = (
                     1 if error_name != ErrorIf.ConvOutputShapeNonInteger else 2
@@ -2035,12 +2081,10 @@ class TosaArgGen:
                 s_vals = [
                     x for x in range(startStride, testGen.args.max_conv_stride + 1)
                 ]
-            strides = {x for x in itertools.product(*([s_vals] * k_rank))}
-            if error_name == ErrorIf.DilationSmallerOne:
-                d_vals = [rng.choice(range(-5, 1))]
-            else:
                 d_vals = [x for x in range(1, testGen.args.max_conv_dilation + 1)]
-            dilations = {x for x in itertools.product(*([d_vals] * k_rank))}
+
+                strides = {x for x in itertools.product(*([s_vals] * k_rank))}
+                dilations = {x for x in itertools.product(*([d_vals] * k_rank))}
 
             if not error_name and testGen.args.oversize:
                 # add some oversize argument values
@@ -2064,12 +2108,15 @@ class TosaArgGen:
                 )
             max_dim_size = None
 
-            # There are too many parameter combinations, so generate them sparsely,
-            # very sparse for negative tests
-            sparsity_factor = 2 if error_name else 120
-            sparsity = TosaArgGen._calculate_sparsity(
-                len(paddings) * len(strides) * len(dilations), sparsity_factor
-            )
+            if error_name:
+                # Cycle through all error_if tests but we only keep the first few
+                sparsity = 1
+            else:
+                # There are too many parameter combinations, so generate them sparsely,
+                sparsity_factor = 120
+                sparsity = TosaArgGen._calculate_sparsity(
+                    len(paddings) * len(strides) * len(dilations), sparsity_factor
+                )
         else:
             # Only test 8k levels boundaries
             bigStride = testGen.TOSA_8K_LEVEL_MAX_STRIDE
@@ -2114,13 +2161,15 @@ class TosaArgGen:
             # Currently allow all combinations that are reasonable size
             sparsity = 1
 
+        more_tests = True
         n = 0
         for a in accum_dtypes:
             for s in sorted(list(strides)):
                 for p in sorted(list(paddings)):
                     for d in sorted(list(dilations)):
                         if (
-                            n % sparsity == 0
+                            more_tests
+                            and n % sparsity == 0
                             # the padded shape must exceed the dilation * kernel to get a positive
                             # sized output shape
                             and (ifm_shape[1] - 1 + p[0] + p[1])
@@ -2199,6 +2248,15 @@ class TosaArgGen:
                                         args_dict,
                                     )
                                 )
+                                if (
+                                    error_name
+                                    and len(arg_list) >= TosaArgGen.MAX_CONV_ERROR_IFS
+                                ):
+                                    # Found enough errors
+                                    logger.debug(
+                                        f"Skipping creating more conv error tests for {error_name}"
+                                    )
+                                    more_tests = False
                         n += 1
 
         arg_list = TosaArgGen._add_data_generators(
@@ -2482,7 +2540,9 @@ class TosaArgGen:
             pad_const_int = 0
             pad_const_fp = rng.randNumberDType(dtype)
         else:
-            return []
+            assert error_name == ErrorIf.WrongInputType
+            pad_const_int = 0
+            pad_const_fp = 0
 
         list_shape_pad_values = list(shape_pad_values)
         # If we are producing tests for rank 6 or greater use sparsity
@@ -2523,7 +2583,9 @@ class TosaArgGen:
                 arg_list.append((name, args_dict))
 
         if error_name == ErrorIf.PadSmallerZero and len(arg_list) == 0:
-            logger.info(f"No ErrorIf test created for input shape: {shapeList[0]}")
+            logger.debug(
+                f"agPad: No PadSmallerZero ErrorIf test created for input shape: {shapeList[0]}"
+            )
 
         arg_list = TosaArgGen._add_data_generators(
             testGen,
@@ -3106,8 +3168,8 @@ class TosaArgGen:
         # Find new shapes up to the number of permutations asked for
         # This code is NOT fast.  Fortunately, the numbers are fairly small.
         for p in range(testGen.args.num_rand_permutations):
-            # Rank from 1 to TOSA_TENSOR_MAX_RANK
-            newRank = rng.randInt(1, (testGen.TOSA_TENSOR_MAX_RANK + 1))
+            # Rank from 1 to MAX_TENSOR_RANK
+            newRank = rng.randInt(1, (gtu.MAX_TENSOR_RANK + 1))
             if len(factors) < newRank:
                 continue
 
diff --git a/verif/generator/tosa_error_if.py b/verif/generator/tosa_error_if.py
index 916b4f9..1b6b044 100644
--- a/verif/generator/tosa_error_if.py
+++ b/verif/generator/tosa_error_if.py
@@ -3,11 +3,8 @@
 import logging
 import math
 
+import generator.tosa_utils as gtu
 import numpy as np
-from generator.tosa_utils import MAX_RESIZE_DIMENSION
-from generator.tosa_utils import product
-from generator.tosa_utils import usableDTypes
-from generator.tosa_utils import valueToName
 from tosa.DType import DType
 from tosa.Op import Op
 from tosa.ResizeMode import ResizeMode
@@ -287,7 +284,7 @@ class TosaErrorIfArgGen:
         max_dim and max_items.
         """
         new_shape = [min(d, max_dim) for d in shape] if max(shape) > max_dim else shape
-        while product(new_shape) > max_items:
+        while gtu.product(new_shape) > max_items:
             new_shape = [max(d - 1, 1) for d in new_shape]
         return new_shape
 
@@ -389,12 +386,12 @@ class TosaErrorValidator:
                 serializer.setExpectedReturnCode(2, True, desc=error_reason)
             elif error_result:  # and not expected_result
                 logger.error(
-                    f"Unexpected ERROR_IF: Op: {valueToName(Op, kwargs['op']['op'])}"
+                    f"Unexpected ERROR_IF: Op: {gtu.valueToName(Op, kwargs['op']['op'])}"
                     f" Expected: {error_name}, Got: {validator_name}"
                 )
             elif not expected_result:  # and not error_result
                 logger.error(
-                    f"Missed ERROR_IF: Op: {valueToName(Op, kwargs['op']['op'])}"
+                    f"Missed ERROR_IF: Op: {gtu.valueToName(Op, kwargs['op']['op'])}"
                     f" Expected: {error_name}"
                 )
 
@@ -402,7 +399,7 @@ class TosaErrorValidator:
                 for k, v in sorted(kwargs.items()):
                     if k != "op":
                         if k.endswith("dtype"):
-                            v = valueToName(DType, v)
+                            v = gtu.valueToName(DType, v)
                         logger.error(f"  {k} = {v}")
 
         return overall_result
@@ -417,7 +414,8 @@ class TosaErrorValidator:
         allowed_input_dtypes = {
             t[0] if isinstance(t, list) else t for t in input_dtypes
         }
-        wrong_input_dtypes = list(usableDTypes(excludes=allowed_input_dtypes))
+        wrong_input_dtypes = list(gtu.usableDTypes(excludes=allowed_input_dtypes))
+        assert len(wrong_input_dtypes) > 0
 
         # Turn the wrong dtypes into required list of types
         if op["op"] in [
@@ -682,7 +680,8 @@ class TosaErrorValidator:
 
     @staticmethod
     def evWrongRank(check=False, **kwargs):
-        all_ranks = (1, 2, 3, 4, 5)
+        # From 1 to MAX_TENSOR_RANK+1 inclusively
+        all_ranks = tuple(range(1, gtu.MAX_TENSOR_RANK + 2))
 
         # Make a list of incorrect ranks
         assert "op" in kwargs
@@ -784,16 +783,16 @@ class TosaErrorValidator:
             "shape": [[1, 16584, 5, 1], [1, 2, 16499, 4]],
         }
         error_result = False
-        error_reason = f"At least one maximum dimension is greater than or equal to {MAX_RESIZE_DIMENSION}"
+        error_reason = f"At least one maximum dimension is greater than or equal to {gtu.MAX_RESIZE_DIMENSION}"
 
         if check:
             input_shape = kwargs["input_shape"]
             output_shape = kwargs["output_shape"]
             if (
-                (input_shape[1] >= MAX_RESIZE_DIMENSION)
-                or (input_shape[2] >= MAX_RESIZE_DIMENSION)
-                or (output_shape[1] >= MAX_RESIZE_DIMENSION)
-                or (output_shape[2] >= MAX_RESIZE_DIMENSION)
+                (input_shape[1] >= gtu.MAX_RESIZE_DIMENSION)
+                or (input_shape[2] >= gtu.MAX_RESIZE_DIMENSION)
+                or (output_shape[1] >= gtu.MAX_RESIZE_DIMENSION)
+                or (output_shape[2] >= gtu.MAX_RESIZE_DIMENSION)
             ):
                 error_result = True
 
@@ -1075,8 +1074,8 @@ class TosaErrorValidator:
 
             if (
                 params_valid
-                and max(output_shape) < MAX_RESIZE_DIMENSION
-                and max(input_shape) < MAX_RESIZE_DIMENSION
+                and max(output_shape) < gtu.MAX_RESIZE_DIMENSION
+                and max(input_shape) < gtu.MAX_RESIZE_DIMENSION
             ):
                 output_y = (
                     (input_shape[1] - 1) * scale[0] - offset[0] + border[0]
diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py
index c867070..28b3d28 100644
--- a/verif/generator/tosa_test_gen.py
+++ b/verif/generator/tosa_test_gen.py
@@ -36,9 +36,7 @@ logger = logging.getLogger("tosa_verif_build_tests")
 
 
 class TosaTestGen:
-    # Maximum rank of tensor supported by test generator.
     # This currently matches the 8K level defined in the specification.
-    TOSA_TENSOR_MAX_RANK = 6
     TOSA_8K_LEVEL_MAX_SCALE = 64
     TOSA_8K_LEVEL_MAX_KERNEL = 8192
     TOSA_8K_LEVEL_MAX_STRIDE = 8192
@@ -2941,8 +2939,10 @@ class TosaTestGen:
         testList = []
         if testType == "negative" and "error_if_validators" in op:
             error_if_validators = op["error_if_validators"]
+            num_error_types_created = 0
         else:
             error_if_validators = [None]
+            num_error_types_created = None
 
         for validator in error_if_validators:
             if validator is not None:
@@ -3020,6 +3020,21 @@ class TosaTestGen:
                             testList.append(
                                 (opName, testStr, t, error_name, shapeList, args)
                             )
+            if error_name is not None:
+                # Check the last test is of the error we wanted
+                if len(testList) == 0 or testList[-1][3] != error_name:
+                    if self.args.level8k:
+                        logger.info(f"Missing {error_name} tests due to level8k mode")
+                    else:
+                        logger.error(f"ERROR: Failed to create any {error_name} tests")
+                        logger.debug(
+                            "Last test created: {}".format(
+                                testList[-1] if testList else None
+                            )
+                        )
+                else:
+                    # Successfully created at least one ERRROR_IF test
+                    num_error_types_created += 1
 
         if testType == "positive":
             # Remove tests which are expected to fail but don't correlate to a ERROR_IF statement
@@ -3039,6 +3054,15 @@ class TosaTestGen:
                     if not remove_test:
                         clean_testList.append(test)
                 testList = clean_testList
+        else:
+            if num_error_types_created is not None and not self.args.level8k:
+                remaining_error_types = (
+                    len(error_if_validators) - num_error_types_created
+                )
+                if remaining_error_types:
+                    raise Exception(
+                        f"Failed to create {remaining_error_types} error types for {opName}"
+                    )
 
         return testList
 
@@ -3141,9 +3165,11 @@ class TosaTestGen:
                 if compliance:
                     tensMeta["compliance"] = compliance
             self.serialize("test", tensMeta)
+            return True
         else:
             # The test is not valid
             logger.error(f"Invalid ERROR_IF test created: {opName} {testStr}")
+            return False
 
     def createDynamicOpLists(self):
 
@@ -3305,7 +3331,7 @@ class TosaTestGen:
         [DType.FP8E5M2, DType.FP8E5M2, DType.FP16],
     ]
 
-    DEFAULT_RANK_RANGE = (1, TOSA_TENSOR_MAX_RANK)
+    DEFAULT_RANK_RANGE = (1, gtu.MAX_TENSOR_RANK)
 
     TOSA_OP_LIST = {
         # Tensor operators
diff --git a/verif/generator/tosa_test_select.py b/verif/generator/tosa_test_select.py
index c6e1d0d..904d90f 100644
--- a/verif/generator/tosa_test_select.py
+++ b/verif/generator/tosa_test_select.py
@@ -185,7 +185,7 @@ class TestOpList:
         # Add a test to this op group and set up the permutations/group for it
         assert test.opName.startswith(self.opName)
         if str(test) in self.testStrings:
-            logger.info(f"Skipping duplicate test: {str(test)}")
+            logger.debug(f"Skipping duplicate test: {str(test)}")
             return
 
         self.tests.append(test)
diff --git a/verif/generator/tosa_utils.py b/verif/generator/tosa_utils.py
index 4a4f6bb..a8e321e 100644
--- a/verif/generator/tosa_utils.py
+++ b/verif/generator/tosa_utils.py
@@ -10,6 +10,9 @@ from tosa.DType import DType
 # Maximum dimension size for output and inputs for RESIZE
 MAX_RESIZE_DIMENSION = 16384
 
+# Maximum rank of tensor supported by test generator.
+MAX_TENSOR_RANK = 6
+
 # Data type information dictionary
 # - str: filename abbreviation
 # - width: number of bytes needed for type
diff --git a/verif/generator/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py
index 83c06d7..0839cec 100644
--- a/verif/generator/tosa_verif_build_tests.py
+++ b/verif/generator/tosa_verif_build_tests.py
@@ -428,6 +428,9 @@ def main(argv=None):
             logger.error(f"INTERNAL ERROR: Failure creating test output for {opName}")
             raise e
 
+    if results.count(False):
+        raise Exception(f"Failed to create {results.count(False)} tests")
+
     if not args.list_tests:
         print(f"Done creating {len(results)} tests")
     return 0