Compliance testing support for MAX_POOL2D & PAD

Added Pseudo Random number generator in generate library.
Enabled MAX_POOL2D, PAD FP32 tests to use new generator and compliance.
Fixed verify library exact mode to expect reference data as FP64.
Simplified tosa_verif_build_tests internal interfaces for new tests.

Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: Icc0ffa924cf38107c3a212efd452c47a650c9d98

3 files changed, 198 insertions, 107 deletions

diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py
index 475f062..f7837a0 100644
--- a/verif/generator/tosa_arg_gen.py
+++ b/verif/generator/tosa_arg_gen.py
@@ -635,7 +635,11 @@ class TosaTensorValuesGen:
         # Variable inputs versus constants
         pCount, cCount = testGen.TOSA_OP_LIST[opName]["operands"]
 
-        if error_name is not None or not gtu.dtypeIsSupportedByCompliance(dtypeList[0]):
+        if (
+            error_name is not None
+            or not gtu.dtypeIsSupportedByCompliance(dtypeList[0])
+            or opName in ("avg_pool2d",)
+        ):
             # Fall back to original path when dealing with unsupported types
 
             # First turn off lazy data gen so we always produce data
@@ -678,7 +682,7 @@ class TosaTensorValuesGen:
             if dg_type == gtu.DataGenType.PSEUDO_RANDOM:
                 info = {}
                 # TODO - generate seed for this generator based on test
-                info["rng_seed"] = -1
+                info["rng_seed"] = 42
                 info["range"] = [
                     str(v)
                     for v in testGen.getDTypeRange(dtypeList[idx], high_inclusive=True)
@@ -1107,7 +1111,7 @@ class TosaArgGen:
         pass
 
     @staticmethod
-    def _add_data_generators(testGen, opName, dtype, arg_list, error_name, **kwargs):
+    def _add_data_generators(testGen, opName, dtype, arg_list, error_name):
         """Add extra tests for each type of data generator for this op."""
         if (
             error_name is None
@@ -1125,32 +1129,28 @@ class TosaArgGen:
         # Expand arg list with other data generator types
         new_arg_list = []
         for dg_type in dataGenTypesList:
-            for arg_str, arg_attrs in arg_list:
-                arg_dict = arg_attrs[0]
-                arg_dict["dg_type"] = dg_type
-
+            for arg_str, args_dict in arg_list:
+                args_dict["dg_type"] = dg_type
                 if dg_type == gtu.DataGenType.PSEUDO_RANDOM:
                     # Default test
-                    new_arg_list.append((arg_str, [arg_dict]))
+                    new_arg_list.append((arg_str, args_dict))
 
                 elif dg_type == gtu.DataGenType.DOT_PRODUCT:
                     # Extra tests for each dot product test set
-                    dot_products = kwargs["dot_products"]
+                    dot_products = args_dict["dot_products"]
                     if dot_products < testGen.TOSA_MI_DOT_PRODUCT_MIN:
                         print(
                             f"Skipping {opName} dot product test as too few calculations {dot_products} < {testGen.TOSA_MI_DOT_PRODUCT_MIN}"
                         )
                         continue
-                    arg_dict["ks"] = kwargs["ks"]
-                    for key in gtu.DG_DOT_PRODUCT_OPTIONAL_INFO:
-                        if key in kwargs:
-                            arg_dict[key] = kwargs[key]
+                    # KS is required by all dot product generators
+                    assert "ks" in args_dict
 
                     for s in testGen.TOSA_MI_DOT_PRODUCT_TEST_SETS:
                         new_arg_str = f"{arg_str}_s{s}"
-                        new_arg_dict = arg_dict.copy()
-                        new_arg_dict["s"] = s
-                        new_arg_list.append((new_arg_str, [new_arg_dict]))
+                        new_args_dict = args_dict.copy()
+                        new_args_dict["s"] = s
+                        new_arg_list.append((new_arg_str, new_args_dict))
 
         return new_arg_list
 
@@ -1421,9 +1421,21 @@ class TosaArgGen:
             # Pick some potentially correct output dtype if input type is incorrect
             accum_dtypes = [DType.INT32]
 
-        arg_list = [
-            (f"acc{testGen.typeStr(a)}", [{"acc_type": a}]) for a in accum_dtypes
-        ]
+        # Set up compliance info
+        args_dict = {
+            "ks": int(shapeList[0][2]),  # Set KS = C, from input A (N,H,C)
+            # Set dot_products = N*H*W
+            "dot_products": gtu.product(
+                (shapeList[0][0], shapeList[0][1], shapeList[1][2])
+            ),
+        }
+
+        # Create arg tuple of string and dict
+        arg_list = []
+        for a in accum_dtypes:
+            d = args_dict.copy()
+            d["acc_type"] = a
+            arg_list.append((f"acc{testGen.typeStr(a)}", d))
 
         arg_list = TosaArgGen._add_data_generators(
             testGen,
@@ -1431,12 +1443,8 @@ class TosaArgGen:
             dtype,
             arg_list,
             error_name,
-            ks=int(shapeList[0][2]),  # Set KS = C, from input A (N,H,C)
-            # Set dot_products = N*H*W
-            dot_products=gtu.product(
-                (shapeList[0][0], shapeList[0][1], shapeList[1][2])
-            ),
         )
+        # Return list of tuples: (arg_str, args_dict)
         return arg_list
 
     @staticmethod
@@ -1574,7 +1582,6 @@ class TosaArgGen:
 
     @staticmethod
     def agPad(testGen, opName, shapeList, dtype, error_name=None):
-        arg_list = []
         rank = len(shapeList[0])
 
         # Exhaustively test combinations of padding on each side of each dimension
@@ -1606,6 +1613,8 @@ class TosaArgGen:
         else:
             sparsity = 1
 
+        # Build arg list
+        arg_list = []
         for n, paddings in enumerate(list_shape_pad_values):
             paddings = list(paddings)
             args_valid = True
@@ -1625,13 +1634,25 @@ class TosaArgGen:
                 for r in range(rank):
                     before, after = paddings[r]
                     name = f"{name}{before}{after}"
-                arg_list.append(
-                    (name, [np.array(paddings), pad_const_int, pad_const_fp])
-                )
+                    args_dict = {
+                        "pad": np.array(paddings),
+                        "pad_const_int": pad_const_int,
+                        "pad_const_fp": pad_const_fp,
+                    }
+                arg_list.append((name, args_dict))
 
         if error_name == ErrorIf.PadSmallerZero and len(arg_list) == 0:
             warnings.warn(f"No ErrorIf test created for input shape: {shapeList[0]}")
 
+        arg_list = TosaArgGen._add_data_generators(
+            testGen,
+            opName,
+            dtype,
+            arg_list,
+            error_name,
+        )
+
+        # Return list of tuples: (arg_str, args_dict)
         return arg_list
 
     @staticmethod
@@ -1735,9 +1756,9 @@ class TosaArgGen:
             else "st{}_kern{}_pad{}"
         )
 
-        def get_arg_list_element(accum, stride, pad, kern):
+        def get_arg_list_element(accum, stride, pad, kern, dot_products=0):
             # Return tuple containing the formatted argument string and
-            # the corresponding argument values
+            # the corresponding argument values in a dictionary
 
             # Support for larger values than 9 needs different delimiter
             delim = "" if max(stride + kern + pad) <= 9 else "x"
@@ -1746,13 +1767,18 @@ class TosaArgGen:
                 delim.join([str(x) for x in kern]),
                 delim.join([str(x) for x in pad]),
             ]
-            # Note: different order to string
-            arg_val_elems = [stride, pad, kern]
+            args_dict = {
+                "stride": stride,
+                "pad": pad,
+                "kernel": kern,
+                "dot_products": dot_products,  # Ignored for error tests
+                "ks": gtu.product(kern),  # avg_pool2d: KS = KX*KY
+            }
 
             if accum is not None:
                 arg_str_elems.insert(0, testGen.typeStr(accum))
-                arg_val_elems.insert(0, accum)
-            return (arg_str.format(*arg_str_elems), arg_val_elems)
+                args_dict["acc_type"] = accum
+            return (arg_str.format(*arg_str_elems), args_dict)
 
         n = 0
         for a in accum_dtypes:
@@ -1769,8 +1795,9 @@ class TosaArgGen:
                                 testGen, error_name, s, p, k
                             )
                             if None not in [sNew, pNew, kNew] and n % sparsity == 0:
-                                arg_vals = [a, sNew, pNew, kNew]
-                                arg_list.append(get_arg_list_element(*arg_vals))
+                                arg_list.append(
+                                    get_arg_list_element(a, sNew, pNew, kNew)
+                                )
                         elif (
                             n % sparsity == 0
                             # padding must not exceed the kernel size
@@ -1804,10 +1831,23 @@ class TosaArgGen:
                                 ):
                                     # Test will consume too much memory - skip it
                                     continue
-                                arg_vals = [a, s, p, k]
-                                arg_list.append(get_arg_list_element(*arg_vals))
+                                # Dot products = N*OH*OW*C
+                                dp = gtu.product(
+                                    (shape[0], output_h, output_w, shape[3])
+                                )
+                                arg_list.append(get_arg_list_element(a, s, p, k, dp))
                         n += 1
 
+        # Now add data generator types
+        arg_list = TosaArgGen._add_data_generators(
+            testGen,
+            opName,
+            dtype,
+            arg_list,
+            error_name,
+        )
+
+        # Return list of tuples: (arg_str, args_dict)
         return arg_list
 
     @staticmethod
diff --git a/verif/generator/tosa_error_if.py b/verif/generator/tosa_error_if.py
index d490cf2..ed1a941 100644
--- a/verif/generator/tosa_error_if.py
+++ b/verif/generator/tosa_error_if.py
@@ -2653,16 +2653,28 @@ class TosaInvalidValidator:
 
         args = kwargs["args"]
 
-        # Skip accum_dtype arg (apart from MaxPool2D that doesn't have one)
-        stride_idx, pad_idx = (1, 2) if opName != "max_pool2d" else (0, 1)
+        if isinstance(args, dict):
+            args_dict = args
+        else:
+            # Create args_dict from list elements
+            # TODO - Remove this once all NWHC operators agFunctions have been
+            # converted to args_dict output
+
+            # Skip accum_dtype arg (apart from MaxPool2D that doesn't have one)
+            stride_idx, pad_idx = (1, 2) if opName != "max_pool2d" else (0, 1)
+            args_dict = {"stride": args[stride_idx], "pad": args[pad_idx]}
+            # Alias different info for each op
+            args_dict["kernel"] = args[pad_idx + 1]
+            args_dict["out_shape"] = args[pad_idx + 1]
+            args_dict["dilation"] = args[pad_idx + 1]
 
         # Common info for all ops
-        strides = args[stride_idx]
-        padding = args[pad_idx]
+        strides = args_dict["stride"]
+        padding = args_dict["pad"]
 
         if opName.endswith("pool2d"):
             # avg_pool2d, max_pool2d
-            kernel_shape = args[pad_idx + 1]
+            kernel_shape = args_dict["kernel"]
             h = (
                 input_shape[1] + padding[0] + padding[1] + strides[0] - kernel_shape[0]
             ) // strides[0]
@@ -2674,7 +2686,7 @@ class TosaInvalidValidator:
 
         if opName.startswith("transpose_conv2d"):
             # transpose_conv2d
-            output_shape = args[pad_idx + 1]
+            output_shape = args_dict["out_shape"]
             filter_shape = inputShapes[1]
             kernel_shape = filter_shape[1:-1]
 
@@ -2703,7 +2715,7 @@ class TosaInvalidValidator:
 
         if "conv2d" in opName or "conv3d" in opName:
             # conv2d, conv3d, depthwise_conv2d
-            dilations = args[pad_idx + 1]
+            dilations = args_dict["dilation"]
             filter_shape = inputShapes[1]
             kernel_shape = (
                 filter_shape[0:2]
diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py
index 8fcea29..17cbd8f 100644
--- a/verif/generator/tosa_test_gen.py
+++ b/verif/generator/tosa_test_gen.py
@@ -658,15 +658,22 @@ class TosaTestGen:
     def build_pool2d(
         self,
         op,
-        input,
-        accum_dtype,
-        stride,
-        pad,
-        kernel,
+        inputs,
+        args_dict,
         validator_fcns=None,
         error_name=None,
         qinfo=None,
     ):
+        assert len(inputs) == 1
+        input = inputs[0]
+        # max_pool has no accum_dtype
+        accum_dtype = (
+            args_dict["acc_type"] if "acc_type" in args_dict else DType.UNKNOWN
+        )
+        stride = args_dict["stride"]
+        pad = args_dict["pad"]
+        kernel = args_dict["kernel"]
+
         result_tens = OutputShaper.pool2dOp(
             self.ser, self.rng, input, kernel, stride, pad, error_name
         )
@@ -720,27 +727,28 @@ class TosaTestGen:
     def build_maxpool2d(
         self,
         op,
-        input,
-        stride,
-        pad,
-        kernel,
+        inputs,
+        args_dict,
         validator_fcns=None,
         error_name=None,
         qinfo=None,
     ):
-        # Same as build_pool2d but manually sets accum_dtype value
-        # (maxpool has no accum_dtype)
-        return self.build_pool2d(
+        result_tensor = self.build_pool2d(
             op,
-            input,
-            DType.UNKNOWN,
-            stride,
-            pad,
-            kernel,
+            inputs,
+            args_dict,
             validator_fcns,
             error_name,
             qinfo,
         )
+        if gtu.dtypeIsSupportedByCompliance(inputs[0].dtype):
+            compliance = self.tensorComplianceMetaData(
+                op, args_dict, result_tensor, error_name
+            )
+        else:
+            compliance = None
+
+        return TosaTestGen.BuildInfo(result_tensor, compliance)
 
     def build_conv2d(
         self,
@@ -1070,8 +1078,10 @@ class TosaTestGen:
         return result_tens
 
     def build_matmul(
-        self, op, a, b, args_dict, validator_fcns=None, error_name=None, qinfo=None
+        self, op, inputs, args_dict, validator_fcns=None, error_name=None, qinfo=None
     ):
+        assert len(inputs) == 2
+        a, b = inputs
         accum_dtype = args_dict["acc_type"]
         result_tensor = OutputShaper.matmulOp(
             self.ser, self.rng, a, b, accum_dtype, error_name
@@ -1372,15 +1382,19 @@ class TosaTestGen:
     def build_pad(
         self,
         op,
-        a,
-        padding,
-        pad_const_int,
-        pad_const_float,
+        inputs,
+        args_dict,
         validator_fcns=None,
         error_name=None,
         qinfo=None,
     ):
-        result_tens = OutputShaper.padOp(self.ser, self.rng, a, padding, error_name)
+        assert len(inputs) == 1
+        a = inputs[0]
+        padding = args_dict["pad"]
+        pad_const_int = args_dict["pad_const_int"]
+        pad_const_float = args_dict["pad_const_fp"]
+
+        result_tensor = OutputShaper.padOp(self.ser, self.rng, a, padding, error_name)
 
         attr = ts.TosaSerializerAttribute()
         attr.PadAttribute(
@@ -1389,7 +1403,7 @@ class TosaTestGen:
 
         # Invalidate Input/Output list for error if checks.
         input_list = [a.name]
-        output_list = [result_tens.name]
+        output_list = [result_tensor.name]
         pCount, cCount = op["operands"]
         num_operands = pCount + cCount
         input_list, output_list = TosaErrorIfArgGen.eiInvalidateInputOutputList(
@@ -1402,12 +1416,12 @@ class TosaTestGen:
             error_name,
             op=op,
             input_shape=a.shape,
-            output_shape=result_tens.shape,
+            output_shape=result_tensor.shape,
             input_dtype=a.dtype,
-            output_dtype=result_tens.dtype,
+            output_dtype=result_tensor.dtype,
             pad=padding,
             qinfo=qinfo,
-            result_tensors=[result_tens],
+            result_tensors=[result_tensor],
             input_list=input_list,
             output_list=output_list,
             num_operands=num_operands,
@@ -1416,7 +1430,15 @@ class TosaTestGen:
             return None
 
         self.ser.addOperator(op["op"], input_list, output_list, attr)
-        return result_tens
+
+        if gtu.dtypeIsSupportedByCompliance(a.dtype):
+            compliance = self.tensorComplianceMetaData(
+                op, args_dict, result_tensor, error_name
+            )
+        else:
+            compliance = None
+
+        return TosaTestGen.BuildInfo(result_tensor, compliance)
 
     def build_dim(
         self,
@@ -2609,8 +2631,9 @@ class TosaTestGen:
         tensMeta = {}
 
         # Check we are using the new testArgs interface with an argsDict dictionary
-        if len(testArgs) == 1 and isinstance(testArgs[0], dict):
-            argsDict = testArgs[0]
+        if isinstance(testArgs, dict):
+            # New interface with args info in dictionary
+            argsDict = testArgs
             assert "dg_type" in argsDict
             tvgInfo = tvgen_fcn(
                 self, opName, dtypeList, shapeList, argsDict, error_name
@@ -2618,38 +2641,49 @@ class TosaTestGen:
             if tvgInfo.dataGenDict:
                 tensMeta["data_gen"] = tvgInfo.dataGenDict
             tens = tvgInfo.tensorList
+
+            result = build_fcn(
+                self,
+                op,
+                tens,
+                argsDict,
+                validator_fcns=error_if_validators,
+                error_name=error_name,
+                qinfo=qinfo,
+            )
         else:
+            # Old interface with args info in a list
             tens = tvgen_fcn(self, op, dtypeList, shapeList, testArgs, error_name)
 
-        try:
-            if error_if_validators is None:
-                if qinfo is not None:
-                    result = build_fcn(self, op, *tens, *testArgs, qinfo)
-                else:
-                    result = build_fcn(self, op, *tens, *testArgs)
-            else:
-                if qinfo is not None:
-                    result = build_fcn(
-                        self,
-                        op,
-                        *tens,
-                        *testArgs,
-                        validator_fcns=error_if_validators,
-                        error_name=error_name,
-                        qinfo=qinfo,
-                    )
+            try:
+                if error_if_validators is None:
+                    if qinfo is not None:
+                        result = build_fcn(self, op, *tens, *testArgs, qinfo)
+                    else:
+                        result = build_fcn(self, op, *tens, *testArgs)
                 else:
-                    result = build_fcn(
-                        self,
-                        op,
-                        *tens,
-                        *testArgs,
-                        validator_fcns=error_if_validators,
-                        error_name=error_name,
-                    )
-        except TypeError as e:
-            print(f"build_fcn: {build_fcn}\nTensors: {tens}\nArgs: {testArgs}\n")
-            raise e
+                    if qinfo is not None:
+                        result = build_fcn(
+                            self,
+                            op,
+                            *tens,
+                            *testArgs,
+                            validator_fcns=error_if_validators,
+                            error_name=error_name,
+                            qinfo=qinfo,
+                        )
+                    else:
+                        result = build_fcn(
+                            self,
+                            op,
+                            *tens,
+                            *testArgs,
+                            validator_fcns=error_if_validators,
+                            error_name=error_name,
+                        )
+            except TypeError as e:
+                print(f"build_fcn: {build_fcn}\nTensors: {tens}\nArgs: {testArgs}\n")
+                raise e
 
         if result:
             # The test is valid, serialize it
@@ -2847,7 +2881,7 @@ class TosaTestGen:
             "build_fcn": (
                 build_pool2d,
                 TosaTensorGen.tgNHWC,
-                TosaTensorValuesGen.tvgDefault,
+                TosaTensorValuesGen.tvgLazyGenDefault,
                 TosaArgGen.agPooling,
             ),
             "qgen": TosaQuantGen.qgUnary,
@@ -3004,7 +3038,6 @@ class TosaTestGen:
             ),
             "data_gen": {
                 "fp": (gtu.DataGenType.DOT_PRODUCT,),
-                "int": (gtu.DataGenType.PSEUDO_RANDOM,),
             },
         },
         "max_pool2d": {
@@ -3014,7 +3047,7 @@ class TosaTestGen:
             "build_fcn": (
                 build_maxpool2d,
                 TosaTensorGen.tgNHWC,
-                TosaTensorValuesGen.tvgDefault,
+                TosaTensorValuesGen.tvgLazyGenDefault,
                 TosaArgGen.agPooling,
             ),
             "types": TYPE_NARROW_INT_FP,
@@ -3032,6 +3065,9 @@ class TosaTestGen:
                 TosaErrorValidator.evPoolingOutputShapeMismatch,
                 TosaErrorValidator.evPoolingOutputShapeNonInteger,
             ),
+            "data_gen": {
+                "fp": (gtu.DataGenType.PSEUDO_RANDOM,),
+            },
         },
         # Templated operator.  Filled in by createDynamicOpLists
         "transpose_conv2d_TEMPLATE": {
@@ -3909,7 +3945,7 @@ class TosaTestGen:
             "build_fcn": (
                 build_pad,
                 TosaTensorGen.tgBasic,
-                TosaTensorValuesGen.tvgDefault,
+                TosaTensorValuesGen.tvgLazyGenDefault,
                 TosaArgGen.agPad,
             ),
             "types": TYPE_FIB,
@@ -3923,6 +3959,9 @@ class TosaTestGen:
                 TosaErrorValidator.evRankMismatch,
                 TosaErrorValidator.evWrongRank,
             ),
+            "data_gen": {
+                "fp": (gtu.DataGenType.PSEUDO_RANDOM,),
+            },
         },
         "dim": {
             "op": Op.DIM,