Initial lazy data-gen and compliance test build support

Add initial support for compliance and lazy data-gen meta data
added to desc.json for MATMUL.

Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: I00c047814134a96d7c98d890e93b5884e25b8e64

8 files changed, 540 insertions, 186 deletions

diff --git a/scripts/convert2conformance/convert2conformance.py b/scripts/convert2conformance/convert2conformance.py
index b1f8d0e..f915070 100755
--- a/scripts/convert2conformance/convert2conformance.py
+++ b/scripts/convert2conformance/convert2conformance.py
@@ -11,11 +11,13 @@ import argparse
 import json
 import logging
 import os
+import shutil
 from pathlib import Path
 from typing import Optional
 
 from json2fbbin.json2fbbin import fbbin_to_json
 from json2numpy.json2numpy import npy_to_json
+from schemavalidation.schemavalidation import TestDescSchemaValidator
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger("convert2conformance")
@@ -35,7 +37,8 @@ PROFILES_LIST = ["tosa-bi", "tosa-mi"]
 
 def parse_args(argv):
     """Parse the arguments."""
-    parser = argparse.ArgumentParser()
+    # Set prog for when we are called via tosa_verif_conformance_generator
+    parser = argparse.ArgumentParser(prog="convert2conformance")
     parser.add_argument(
         "test_dir",
         default=Path.cwd(),
@@ -92,6 +95,11 @@ def parse_args(argv):
         help="Output directory must not contain the same test directory",
     )
     parser.add_argument(
+        "--lazy-data-generation",
+        action="store_true",
+        help="Enable lazy data generation (only for tosa-mi)",
+    )
+    parser.add_argument(
         "-v", "--verbose", dest="verbose", action="store_true", help="Verbose operation"
     )
     args = parser.parse_args(argv)
@@ -191,14 +199,15 @@ def update_desc_json(
                 else:
                     logger.error(f"Missing result file {ofm_path}")
                     return None
-            cfm_files.append(cfm + ".npy")
+                cfm_files.append(cfm + ".npy")
         # Remove path and "ref-"/"ref_model_" from output filenames
         ofm_files.append(strip_ref_output_name(ofm_path.name))
 
     # Rewrite output file names as they can be relative, but keep them npys
     test_desc["ofm_file"] = ofm_files
-    if not test_desc["expected_failure"]:
-        # Output expected result file for conformance if expected pass
+    if not test_desc["expected_failure"] and cfm_files:
+        # Output expected result file for conformance if expected pass and we
+        # have some files!
         test_desc["expected_result_file"] = cfm_files
 
     # Add supported profiles
@@ -319,23 +328,32 @@ def main(argv=None):
     # Convert input files to JSON
     ifm_files = []
     for file in test_desc["ifm_file"]:
-        if file is None:
-            ifm_files.append(None)
-        else:
+        if file:
             path = desc_filename.parent / file
-            convert_numpy_file(path, args.output_dir)
             ifm_files.append(path.name)
+            if path.is_file():
+                convert_numpy_file(path, args.output_dir)
+            else:
+                if not args.lazy_data_generation:
+                    logger.error(f"Missing input file {path.name}")
+                    return 1
+
     # Rewrite input file names to make sure the paths are correct,
     # but keep them numpys as the test runner will convert them back
     # before giving them to the SUT
     test_desc["ifm_file"] = ifm_files
 
+    # Check for cpp files for data-generator/verifier
+    cpp_files = args.test_dir.glob("*.cpp")
+    for cpp in cpp_files:
+        shutil.copy(str(cpp), str(args.output_dir))
+
     # Update desc.json and convert result files to JSON
     test_desc = update_desc_json(
         desc_filename.parent,
         test_desc,
         output_dir=args.output_dir,
-        create_result=True,
+        create_result=(not args.lazy_data_generation),
         profiles=args.profile,
         tags=args.tag,
     )
@@ -343,6 +361,9 @@ def main(argv=None):
         # Error from conversion/update
         return 1
 
+    # Validate the desc.json schema
+    TestDescSchemaValidator().validate_config(test_desc)
+
     # Output new desc.json
     new_desc_filename = args.output_dir / NAME_DESC_FILENAME
     with open(new_desc_filename, "w") as fd:
diff --git a/verif/conformance/README.md b/verif/conformance/README.md
index a7ad789..8fbf4fd 100644
--- a/verif/conformance/README.md
+++ b/verif/conformance/README.md
@@ -17,6 +17,7 @@ Each operator entry contains:
 
 * "group" - name of the group this operator is in, in the spec
 * "profile" - list of profiles that this operator covers
+* "support_for" - optional list of supported creation modes out of: lazy_data_gen (data generation just before test run)
 * "generation" - dictionary of test generation details - see below
 * "selection" - dictionary of test selection details - see below
 
@@ -49,6 +50,8 @@ before permutes are applied)
 
 ### Framework ops
 
+DEPRECATED - not supported for conformance testing.
+
 NOTE: Currently assumed all framework ops will be TFLite.
 
 Naming: `tosa_PPP_profile_framework_ops_info.json`
diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json
index 7388835..0d3df72 100644
--- a/verif/conformance/tosa_main_profile_ops_info.json
+++ b/verif/conformance/tosa_main_profile_ops_info.json
@@ -1417,6 +1417,7 @@
         "profile": [
             "tosa-mi"
         ],
+        "support_for": [ "lazy_data_gen" ],
         "generation": {
             "standard": {
                 "negative_dim_range": "1,10",
@@ -1460,11 +1461,7 @@
         },
         "selection": {
             "default": {
-                "params": {},
-                "permutes": [
-                    "shape",
-                    "type"
-                ]
+                "all": "true"
             }
         }
     },
diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py
index 0fb5500..c2ea4ec 100644
--- a/verif/conformance/tosa_verif_conformance_generator.py
+++ b/verif/conformance/tosa_verif_conformance_generator.py
@@ -84,7 +84,14 @@ def _run_sh_command(args, cwd, full_cmd):
 
 
 def build_op_tests(
-    args, test_type, profile, operator, group, gen_args_list, gen_neg_dim_range
+    args,
+    test_type,
+    profile,
+    operator,
+    group,
+    gen_args_list,
+    gen_neg_dim_range,
+    supports=[],
 ):
     """Build tests for a given operator.
 
@@ -105,6 +112,9 @@ def build_op_tests(
         str(args.random_seed),
     ]
 
+    if "lazy_data_gen" in supports and args.lazy_data_generation:
+        build_cmd_base.append("--lazy-data-generation")
+
     build_cmds_list = []
 
     if test_type in ["positive", "both"]:
@@ -198,8 +208,12 @@ def _get_all_tests_list(
     return tests
 
 
-def generate_results(args, profile, operator, op_build_dir, tests=None):
+def generate_results(args, profile, operator, op_build_dir, supports=[], tests=None):
     """Run tests on reference model and save result to the test directory."""
+    if "lazy_data_gen" in supports and args.lazy_data_generation:
+        logger.info("Skipping running tests due to lazy data gen")
+        return
+
     num_cores = args.num_cores
     run_tests_cmd = "tosa_verif_run_tests"
 
@@ -254,6 +268,7 @@ def convert_tests(
     op_build_dir,
     output_dir,
     op_profiles_list,
+    supports=[],
     tests=None,
     group=None,
     trim_op_subdir=False,
@@ -275,6 +290,8 @@ def convert_tests(
             c2c_args_base.extend(["--tag", tag])
     if args.framework_schema:
         c2c_args_base.extend(["--framework-schema", str(args.framework_schema)])
+    if "lazy_data_gen" in supports and args.lazy_data_generation:
+        c2c_args_base.append("--lazy-data-generation")
     c2c_args_base.append("--output-directory")
 
     c2c_args_list = []
@@ -474,6 +491,11 @@ def parse_args(argv=None):
         help="Type of tests produced (default is both)",
     )
     parser.add_argument(
+        "--lazy-data-generation",
+        action="store_true",
+        help="Enable lazy data generation (only for tosa-mi)",
+    )
+    parser.add_argument(
         "--ref-model-directory",
         dest="ref_model_dir",
         type=Path,
@@ -718,6 +740,11 @@ def main():
 
                     operator_group = test_params[op]["group"]
                     root_output_dir = args.output_dir / "operators"
+                    supports = (
+                        test_params[op]["support_for"]
+                        if "support_for" in test_params[op]
+                        else []
+                    )
 
                     # Iterate through the generation groups selecting tests from each
                     for gen_name, gen_dict in test_params[op]["generation"].items():
@@ -756,6 +783,7 @@ def main():
                             gen_name,
                             gen_dict["generator_args"],
                             gen_neg_dim_range,
+                            supports=supports,
                         )
 
                         # Work out which selection criteria we are using
@@ -782,7 +810,9 @@ def main():
                             and selection_config["all"] == "true"
                         ):
                             logger.debug(f"Running and converting all {op} tests")
-                            generate_results(args, profile, op, op_build_dir)
+                            generate_results(
+                                args, profile, op, op_build_dir, supports=supports
+                            )
                             operator_test_list = None
                         else:
                             logger.debug(
@@ -800,7 +830,12 @@ def main():
                                     )
                                 )
                                 generate_results(
-                                    args, profile, op, op_build_dir, tests_gen
+                                    args,
+                                    profile,
+                                    op,
+                                    op_build_dir,
+                                    supports=supports,
+                                    tests=tests_gen,
                                 )
                                 operator_test_list = list(tests_gen2)
                             else:
@@ -823,6 +858,7 @@ def main():
                             op_build_dir,
                             root_output_dir,
                             op_profiles_list,
+                            supports=supports,
                             tests=operator_test_list,
                             group=operator_group,
                             tags=tags,
diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py
index 97ff237..8d96090 100644
--- a/verif/generator/tosa_arg_gen.py
+++ b/verif/generator/tosa_arg_gen.py
@@ -4,12 +4,10 @@ import itertools
 import math
 import warnings
 
+import generator.tosa_utils as gtu
 import numpy as np
 from generator.tosa_error_if import ErrorIf
 from generator.tosa_error_if import TosaErrorIfArgGen
-from generator.tosa_utils import get_accum_dtype_from_tgTypes
-from generator.tosa_utils import get_wrong_output_type
-from generator.tosa_utils import MAX_RESIZE_DIMENSION
 from serializer.tosa_serializer import DTypeNames
 from tosa.DType import DType
 from tosa.Op import Op
@@ -606,11 +604,18 @@ class TosaTensorGen:
 
 
 class TosaTensorValuesGen:
-    """Tensor Value generators create the random data for each test."""
+    """Tensor Value generators create the random data for each tensor in each test."""
 
     def __init__(self):
         pass
 
+    class TVGInfo:
+        """Enhanced tensor values information including data gen dict."""
+
+        def __init__(self, tensorList, dataGenDict):
+            self.tensorList = tensorList
+            self.dataGenDict = dataGenDict
+
     @staticmethod
     def tvgDefault(testGen, op, dtypeList, shapeList, testArgs, error_name=None):
         pCount, cCount = op["operands"]
@@ -624,6 +629,87 @@ class TosaTensorValuesGen:
         return tens
 
     @staticmethod
+    def tvgLazyGenDefault(
+        testGen, opName, dtypeList, shapeList, argsDict, error_name=None
+    ):
+        # Variable inputs versus constants
+        pCount, cCount = testGen.TOSA_OP_LIST[opName]["operands"]
+
+        overrideLazy = False
+        if not gtu.dtypeIsFloat(dtypeList[0]) and testGen.args.lazy_data_gen:
+            # TEMPORARY OVERRIDE for integer types
+            overrideLazy = True
+            testGen.args.lazy_data_gen = False
+
+        # TODO - Change to generation of data using library!
+        # For now - we fall back to original path (or when dealing with non-floats)
+        if not testGen.args.lazy_data_gen:
+            tens_ser_list = TosaTensorValuesGen.tvgDefault(
+                testGen,
+                testGen.TOSA_OP_LIST[opName],
+                dtypeList,
+                shapeList,
+                [],
+                error_name,
+            )
+            if overrideLazy:
+                # Return to lazy mode
+                testGen.args.lazy_data_gen = True
+            return TosaTensorValuesGen.TVGInfo(tens_ser_list, None)
+
+        # Create data generator meta-data
+        dg_type = argsDict["dg_type"]
+        dg_tens_meta = {}
+        tens_ser_list = []
+        for idx, shape in enumerate(shapeList):
+
+            tens_meta = {}
+            tens_meta["generator"] = gtu.DataGenType(dg_type).name
+            tens_meta["data_type"] = gtu.DTYPE_ATTRIBUTES[dtypeList[idx]]["json"]
+            tens_meta["shape"] = [int(i) for i in shape]
+            tens_meta["input_pos"] = idx
+            tens_meta["op"] = opName
+
+            if idx < pCount:
+                tens_meta["input_type"] = "variable"
+                tens = testGen.ser.addPlaceholder(shape, dtypeList[idx], None)
+            else:
+                tens_meta["input_type"] = "constant"
+                tens = testGen.ser.addConst(shape, dtypeList[idx], None)
+            tens_ser_list.append(tens)
+
+            if dg_type == gtu.DataGenType.PSEUDO_RANDOM:
+                info = {}
+                # TODO - generate seed for this generator based on test
+                info["rng_seed"] = -1
+                info["range"] = [
+                    str(v)
+                    for v in testGen.getDTypeRange(dtypeList[idx], high_inclusive=True)
+                ]
+                tens_meta["pseudo_random_info"] = info
+            elif dg_type == gtu.DataGenType.DOT_PRODUCT:
+                info = {}
+                info["s"] = argsDict["s"]
+                info["ks"] = argsDict["ks"]
+                for key in gtu.DG_DOT_PRODUCT_OPTIONAL_INFO:
+                    if key in argsDict:
+                        if key.endswith("_type"):
+                            info[key] = gtu.DTYPE_ATTRIBUTES[argsDict[key]]["json"]
+                        else:
+                            info[key] = argsDict[key]
+                tens_meta["dot_product_info"] = info
+            else:
+                # TODO - other data gen type
+                assert False, "TODO: support other data gen types"
+            dg_tens_meta[tens.name] = tens_meta
+
+        tens_data = {
+            "version": "0.1",
+            "tensors": dg_tens_meta,
+        }
+        return TosaTensorValuesGen.TVGInfo(tens_ser_list, tens_data)
+
+    @staticmethod
     def tvgNegate(testGen, op, dtypeList, shapeList, testArgs, error_name=None):
         if dtypeList[0] == DType.INT32 and error_name is None:
             pCount, cCount = op["operands"]
@@ -1024,6 +1110,50 @@ class TosaArgGen:
         pass
 
     @staticmethod
+    def _add_data_generators(testGen, opName, dtype, arg_list, error_name, **kwargs):
+        """Add extra tests for each type of data generator for this op."""
+        if error_name is None and "data_gen" in testGen.TOSA_OP_LIST[opName]:
+            if dtype in [DType.FP16, DType.FP32, DType.BF16]:
+                dataGenTypesList = testGen.TOSA_OP_LIST[opName]["data_gen"]["fp"]
+            else:
+                dataGenTypesList = testGen.TOSA_OP_LIST[opName]["data_gen"]["int"]
+        else:
+            # Error test or No data generator types listed - assume random
+            dataGenTypesList = (gtu.DataGenType.PSEUDO_RANDOM,)
+
+        # Expand arg list with other data generator types
+        new_arg_list = []
+        for dg_type in dataGenTypesList:
+            for arg_str, arg_attrs in arg_list:
+                arg_dict = arg_attrs[0]
+                arg_dict["dg_type"] = dg_type
+
+                if dg_type == gtu.DataGenType.PSEUDO_RANDOM:
+                    # Default test
+                    new_arg_list.append((arg_str, [arg_dict]))
+
+                elif dg_type == gtu.DataGenType.DOT_PRODUCT:
+                    # Extra tests for each dot product test set
+                    dot_products = kwargs["dot_products"]
+                    if dot_products < testGen.TOSA_MI_DOT_PRODUCT_MIN:
+                        print(
+                            f"Skipping dot product test as too few calculations {dot_products} < {testGen.TOSA_MI_DOT_PRODUCT_MIN}"
+                        )
+                        continue
+                    arg_dict["ks"] = kwargs["ks"]
+                    for key in gtu.DG_DOT_PRODUCT_OPTIONAL_INFO:
+                        if key in kwargs:
+                            arg_dict[key] = kwargs[key]
+
+                    for s in testGen.TOSA_MI_DOT_PRODUCT_TEST_SETS:
+                        new_arg_str = f"{arg_str}_s{s}"
+                        new_arg_dict = arg_dict.copy()
+                        new_arg_dict["s"] = s
+                        new_arg_list.append((new_arg_str, [new_arg_dict]))
+
+        return new_arg_list
+
+    @staticmethod
     def agNone(testGen, opName, shapeList, dtype, error_name=None):
         """A trivial argument generator for operators that don't take any
         non-tensor arguments"""
@@ -1073,7 +1203,7 @@ class TosaArgGen:
         # Shape: (OFM channels), (KD), KH, KW, IFM channels
         filter_shape = shapeList[1]
 
-        accum_dtype = get_accum_dtype_from_tgTypes(dtypes)
+        accum_dtype = gtu.get_accum_dtype_from_tgTypes(dtypes)
 
         # Check the rank
         conv3d = opName.startswith("conv3d")
@@ -1258,12 +1388,12 @@ class TosaArgGen:
         input_dtype = dtypes[0]
 
         if error_name == ErrorIf.WrongOutputType:
-            accum_dtype = get_wrong_output_type(opName, testGen.rng, input_dtype)
+            accum_dtype = gtu.get_wrong_output_type(opName, testGen.rng, input_dtype)
         elif error_name == ErrorIf.WrongInputType:
             # Pick some potentially correct output dtype if input type is incorrect
             accum_dtype = DType.INT32
         else:
-            accum_dtype = get_accum_dtype_from_tgTypes(dtypes)
+            accum_dtype = gtu.get_accum_dtype_from_tgTypes(dtypes)
 
         return [(f"acc{testGen.typeStr(accum_dtype)}", [accum_dtype])]
 
@@ -1285,12 +1415,28 @@ class TosaArgGen:
 
         if error_name == ErrorIf.WrongOutputType:
             # Get incorrect output dtype for ErrorIf case
-            accum_dtypes = [get_wrong_output_type(opName, testGen.rng, dtype)]
+            accum_dtypes = [gtu.get_wrong_output_type(opName, testGen.rng, dtype)]
         elif error_name == ErrorIf.WrongInputType:
             # Pick some potentially correct output dtype if input type is incorrect
             accum_dtypes = [DType.INT32]
 
-        return [(f"acc{testGen.typeStr(a)}", [a]) for a in accum_dtypes]
+        arg_list = [
+            (f"acc{testGen.typeStr(a)}", [{"acc_type": a}]) for a in accum_dtypes
+        ]
+
+        arg_list = TosaArgGen._add_data_generators(
+            testGen,
+            opName,
+            dtype,
+            arg_list,
+            error_name,
+            ks=int(shapeList[0][2]),  # Set KS = C, from input A (N,H,C)
+            # Set dot_products = N*H*W
+            dot_products=gtu.product(
+                (shapeList[0][0], shapeList[0][1], shapeList[1][2])
+            ),
+        )
+        return arg_list
 
     @staticmethod
     def agTransposeConv2D(testGen, opName, shapeList, dtypes, error_name=None):
@@ -1303,7 +1449,7 @@ class TosaArgGen:
         ifm_shape = shapeList[0]
         filter_shape = shapeList[1]
 
-        accum_dtype = get_accum_dtype_from_tgTypes(dtypes)
+        accum_dtype = gtu.get_accum_dtype_from_tgTypes(dtypes)
 
         # Must be rank 4
         if error_name != ErrorIf.WrongRank:
@@ -2288,9 +2434,9 @@ class TosaArgGen:
 
                     if (
                         output_y <= 0
-                        or output_y >= MAX_RESIZE_DIMENSION
+                        or output_y >= gtu.MAX_RESIZE_DIMENSION
                         or output_x <= 0
-                        or output_x >= MAX_RESIZE_DIMENSION
+                        or output_x >= gtu.MAX_RESIZE_DIMENSION
                     ):
                         # Output dimensions out of scope
                         if error_name is not None and perm > 0:
@@ -2301,11 +2447,11 @@ class TosaArgGen:
 
                     if error_name == ErrorIf.ResizeOutputShapeMismatch and (
                         (
-                            output_y + scale_y_d >= MAX_RESIZE_DIMENSION
+                            output_y + scale_y_d >= gtu.MAX_RESIZE_DIMENSION
                             and output_y - scale_y_d < 1
                         )
                         or (
-                            output_x + scale_x_d >= MAX_RESIZE_DIMENSION
+                            output_x + scale_x_d >= gtu.MAX_RESIZE_DIMENSION
                             and output_x - scale_x_d < 1
                         )
                     ):
diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py
index 3014c81..d15f785 100644
--- a/verif/generator/tosa_test_gen.py
+++ b/verif/generator/tosa_test_gen.py
@@ -1,8 +1,12 @@
 # Copyright (c) 2020-2023, ARM Limited.
 # SPDX-License-Identifier: Apache-2.0
+import json
 import os
 from copy import deepcopy
+from datetime import datetime
+from pathlib import Path
 
+import generator.tosa_utils as gtu
 import numpy as np
 import serializer.tosa_serializer as ts
 from generator.tosa_arg_gen import TosaArgGen
@@ -13,15 +17,15 @@ from generator.tosa_error_if import ErrorIf
 from generator.tosa_error_if import TosaErrorIfArgGen
 from generator.tosa_error_if import TosaErrorValidator
 from generator.tosa_error_if import TosaInvalidValidator
-from generator.tosa_utils import DTYPE_ATTRIBUTES
-from generator.tosa_utils import get_rank_mismatch_shape
-from generator.tosa_utils import get_wrong_output_type
-from generator.tosa_utils import MAX_RESIZE_DIMENSION
-from generator.tosa_utils import usableDTypes
-from generator.tosa_utils import vect_f32_to_bf16
+from schemavalidation.schemavalidation import TestDescSchemaValidator
 from tosa.DType import DType
 from tosa.Op import Op
 
+TOSA_AUTOGENERATED_HEADER = f"""// Copyright (c) {datetime.today().year}, ARM Limited
+// SPDX-License-Identifier: Apache-2.0
+// AUTO-GENERATED FILE CREATED BY tosa_verif_build_tests
+"""
+
 
 class TosaTestGen:
     # Maximum rank of tensor supported by test generator.
@@ -31,6 +35,10 @@ class TosaTestGen:
     TOSA_8K_LEVEL_MAX_KERNEL = 8192
     TOSA_8K_LEVEL_MAX_STRIDE = 8192
 
+    # Main compliance dot product statistical test range
+    TOSA_MI_DOT_PRODUCT_TEST_SETS = range(0, 6)
+    TOSA_MI_DOT_PRODUCT_MIN = 1000
+
     def __init__(self, args):
         self.args = args
         self.basePath = args.output_dir
@@ -45,6 +53,8 @@ class TosaTestGen:
         # Work out floating point range
         self.random_fp_low = min(args.tensor_fp_value_range)
         self.random_fp_high = max(args.tensor_fp_value_range)
+        # JSON schema validation
+        self.descSchemaValidator = TestDescSchemaValidator()
 
     def createSerializer(self, opName, testPath):
         self.testPath = os.path.join(opName, testPath)
@@ -53,81 +63,131 @@ class TosaTestGen:
         os.makedirs(fullPath, exist_ok=True)
         # Embed const data in the flatbuffer
         constMode = ts.ConstMode.EMBED
-        if self.args.dump_consts:
+        if self.args.lazy_data_gen:
+            # Lazy data generation - so make constants files
+            constMode = ts.ConstMode.INPUTS
+        elif self.args.dump_consts:
             constMode = ts.ConstMode.EMBED_DUMP
         self.ser = ts.TosaSerializer(fullPath, constMode)
 
     def getSerializer(self):
         return self.ser
 
-    def serialize(self, testName):
-        with open(
-            os.path.join(self.basePath, self.testPath, "{}.tosa".format(testName)), "wb"
-        ) as fd:
+    def serialize(self, testName, metaData=None):
+        path = Path(self.basePath) / self.testPath
+
+        # Write out TOSA flatbuffer binary
+        path_fb = path / f"{testName}.tosa"
+        with path_fb.open("wb") as fd:
             fd.write(self.ser.serialize())
 
-        with open(os.path.join(self.basePath, self.testPath, "desc.json"), "w") as fd:
-            fd.write(self.ser.writeJson("{}.tosa".format(testName)))
+        # Get JSON descriptor from serializer
+        desc = json.loads(self.ser.writeJson(f"{testName}.tosa"))
+
+        if metaData:
+            # Add extra meta data to desc.json
+            desc["meta"] = metaData
+
+        # Validate desc.json before we output it
+        self.descSchemaValidator.validate_config(desc)
+
+        if metaData:
+            if self.args.lazy_data_gen and "data_gen" in metaData:
+                # Output datagen meta data as CPP data
+                path_md = path / f"{testName}_meta_data_gen.cpp"
+                with path_md.open("w") as fd:
+                    fd.write(TOSA_AUTOGENERATED_HEADER)
+                    fd.write("// Test meta data for data generation setup\n\n")
+                    fd.write(f'const char* json_tdg_config_{path.stem} = R"(')
+                    json.dump(metaData["data_gen"], fd)
+                    fd.write(')";\n\n')
+            if "compliance" in metaData:
+                # Output datagen meta data as CPP data
+                path_md = path / f"{testName}_meta_compliance.cpp"
+                with path_md.open("w") as fd:
+                    fd.write(TOSA_AUTOGENERATED_HEADER)
+                    fd.write("// Test meta data for compliance validation\n\n")
+                    fd.write(f'const char* json_tvf_config_{path.stem} = R"(')
+                    json.dump(metaData["compliance"], fd)
+                    fd.write(')";\n\n')
+
+        # Write desc.json
+        path_desc = path / "desc.json"
+        with path_desc.open("w") as fd:
+            json.dump(desc, fd, indent=1)
 
     def resetRNG(self, seed=None):
         if seed is None:
             seed = self.random_seed + 1
         self.rng = np.random.default_rng(seed)
 
-    def getRandTensor(self, shape, dtype):
-        if dtype == DType.BOOL:
-            return np.bool_(self.rng.choice(a=[False, True], size=shape))
-        # TOSA specific INT4 weight range from -7 to 7
+    def getDTypeRange(self, dtype, high_inclusive=False):
+        # Returns dtype value range boundaries (low, high)
+        # The high boundary is excluded in the range
+        # unless high_inclusive is True
+
+        if dtype in (DType.FP32, DType.FP16, DType.BF16):
+            return (self.random_fp_low, self.random_fp_high)
+        elif dtype == DType.BOOL:
+            rng = (0, 2)
+        elif dtype == DType.UINT8:
+            rng = (0, 256)
+        elif dtype == DType.UINT16:
+            rng = (0, 65536)
         elif dtype == DType.INT4:
-            return np.int32(self.rng.integers(low=-7, high=8, size=shape))
+            # TOSA specific INT4 weight range from -7 to 7
+            rng = (-7, 8)
         elif dtype == DType.INT8:
-            return np.int32(self.rng.integers(low=-128, high=128, size=shape))
-        elif dtype == DType.UINT8:
-            return np.int32(self.rng.integers(low=0, high=256, size=shape))
+            rng = (-128, 128)
         elif dtype == DType.INT16:
-            return np.int32(self.rng.integers(low=-32768, high=32768, size=shape))
-        elif dtype == DType.UINT16:
-            return np.int32(self.rng.integers(low=0, high=65536, size=shape))
-        elif (
-            dtype == DType.INT32 or dtype == DType.SHAPE
-        ):  # restricting too large value for SHAPE
-            return np.int32(
-                self.rng.integers(low=-(1 << 31), high=(1 << 31), size=shape)
-            )
+            rng = (-32768, 32768)
+        elif dtype in (DType.INT32, DType.SHAPE):
+            # restricting too large value for SHAPE
+            rng = (-(1 << 31), (1 << 31))
         elif dtype == DType.INT48:
-            return np.int64(
-                self.rng.integers(low=-(1 << 47), high=(1 << 47), size=shape)
-            )
-        elif dtype == DType.FP16:
-            return np.float16(
-                self.rng.uniform(
-                    low=self.random_fp_low, high=self.random_fp_high, size=shape
-                )
-            )
-        elif dtype == DType.BF16:
-            f32_tensor = np.float32(
-                self.rng.uniform(
-                    low=self.random_fp_low, high=self.random_fp_high, size=shape
-                )
-            )
-            # Floor the last 16 bits of each f32 value
-            return np.float32(vect_f32_to_bf16(f32_tensor))
-        elif dtype == DType.FP32:
-            return np.float32(
-                self.rng.uniform(
-                    low=self.random_fp_low, high=self.random_fp_high, size=shape
-                )
-            )
+            rng = (-(1 << 47), (1 << 47))
+        else:
+            raise Exception("Unknown dtype: {}".format(dtype))
+
+        if not high_inclusive:
+            # Exclusive high: low <= range < high
+            return rng
         else:
-            raise Exception("Unrecognized Dtype: {}".format(dtype))
+            # Inclusive range: low <= range <= high
+            return (rng[0], rng[1] - 1)
+
+    def getRandTensor(self, shape, dtype):
+        low, high = self.getDTypeRange(dtype)
+
+        if dtype == DType.BOOL:
+            return np.bool_(self.rng.choice(a=[False, True], size=shape))
+        elif dtype == DType.INT48:
+            return np.int64(self.rng.integers(low=low, high=high, size=shape))
+        elif dtype in (DType.FP16, DType.BF16, DType.FP32):
+            f_tensor = self.rng.uniform(low=low, high=high, size=shape)
+
+            if dtype == DType.FP16:
+                return np.float16(f_tensor)
+            else:
+                f32_tensor = np.float32(f_tensor)
+                if dtype == DType.BF16:
+                    # Floor the last 16 bits of each f32 value
+                    return np.float32(gtu.vect_f32_to_bf16(f32_tensor))
+                else:
+                    return f32_tensor
+        else:
+            # All other integer types
+            return np.int32(self.rng.integers(low=low, high=high, size=shape))
 
     def buildPlaceholderTensors(self, shape_list, dtype_list):
         placeholders = []
 
         assert len(shape_list) == len(dtype_list)
 
+        arr = None
         for idx, shape in enumerate(shape_list):
-            arr = self.getRandTensor(shape, dtype_list[idx])
+            if not self.args.lazy_data_gen:
+                arr = self.getRandTensor(shape, dtype_list[idx])
             placeholders.append(self.ser.addPlaceholder(shape, dtype_list[idx], arr))
 
         return placeholders
@@ -137,8 +197,10 @@ class TosaTestGen:
 
         assert len(shape_list) == len(dtype_list)
 
+        arr = None
         for idx, shape in enumerate(shape_list):
-            arr = self.getRandTensor(shape, dtype_list[idx])
+            if not self.args.lazy_data_gen:
+                arr = self.getRandTensor(shape, dtype_list[idx])
             consts.append(self.ser.addConst(shape, dtype_list[idx], arr))
 
         return consts
@@ -161,38 +223,20 @@ class TosaTestGen:
         return np.int32(self.rng.integers(low=low, high=high, size=1))[0]
 
     def getRandNumberDType(self, dtype):
+        low, high = self.getDTypeRange(dtype)
+
         if dtype == DType.FP32:
-            return np.float32(
-                self.rng.uniform(low=self.random_fp_low, high=self.random_fp_high)
-            )
+            return np.float32(self.rng.uniform(low=low, high=high))
         elif dtype == DType.FP16:
-            return np.float16(
-                self.rng.uniform(low=self.random_fp_low, high=self.random_fp_high)
-            )
+            return np.float16(self.rng.uniform(low=low, high=high))
         elif dtype == DType.BF16:
-            rand_f32 = np.float32(
-                self.rng.uniform(low=self.random_fp_low, high=self.random_fp_high)
-            )
-            return vect_f32_to_bf16(rand_f32)
+            rand_f32 = np.float32(self.rng.uniform(low=low, high=high))
+            return gtu.vect_f32_to_bf16(rand_f32)
         elif dtype == DType.BOOL:
             return self.rng.choice([False, True])
-        # TOSA specific INT4 weight range from -7 to 7
-        elif dtype == DType.INT4:
-            low, high = (-7, 8)
-        elif dtype == DType.INT8:
-            low, high = (-128, 128)
-        elif dtype == DType.INT16:
-            low, high = (-32768, 32768)
-        elif (
-            dtype == DType.INT32 or dtype == DType.SHAPE
-        ):  # restricting too large value for SHAPE
-            low, high = (-(1 << 31), (1 << 31))
         elif dtype == DType.INT48:
-            low, high = (-(1 << 47), (1 << 47))
             # Special size
             return np.int64(self.rng.integers(low, high, size=1))[0]
-        else:
-            raise Exception("Unknown dtype: {}".format(dtype))
 
         return np.int32(self.rng.integers(low, high, size=1))[0]
 
@@ -212,8 +256,8 @@ class TosaTestGen:
             # Limit types to the first 2 as the 3rd is the accumulator
             return "x".join(strs[:2])
         else:
-            if dtype in DTYPE_ATTRIBUTES:
-                return DTYPE_ATTRIBUTES[dtype]["str"]
+            if dtype in gtu.DTYPE_ATTRIBUTES:
+                return gtu.DTYPE_ATTRIBUTES[dtype]["str"]
             else:
                 raise Exception(
                     "Unknown dtype, cannot convert to string: {}".format(dtype)
@@ -221,8 +265,8 @@ class TosaTestGen:
 
     def typeWidth(self, dtype):
         """Get the datatype width for data types"""
-        if dtype in DTYPE_ATTRIBUTES:
-            return DTYPE_ATTRIBUTES[dtype]["width"]
+        if dtype in gtu.DTYPE_ATTRIBUTES:
+            return gtu.DTYPE_ATTRIBUTES[dtype]["width"]
         else:
             raise Exception(f"Unknown dtype, cannot determine width: {dtype}")
 
@@ -237,11 +281,44 @@ class TosaTestGen:
             low=self.args.tensor_shape_range[0], high=self.args.tensor_shape_range[1]
         )
 
-    # Argument generators
-    # Returns a list of tuples (stringDescriptor, [build_fcn_arg_list])
-    # Where the string descriptor is used to generate the test name and
-    # The build_fcn_arg_list is expanded and passed to the operator test
-    # build function
+    def tensorComplianceMetaData(self, op, argsDict, outputTensor, errorName):
+        if errorName:
+            # No compliance for error tests
+            return None
+        # Create compliance meta data for expected output tensor
+        compliance_tens = {"mode": None}
+        if argsDict["dg_type"] == gtu.DataGenType.DOT_PRODUCT:
+            mode = gtu.ComplianceMode.DOT_PRODUCT
+            compliance_tens["dot_product_info"] = {
+                "s": argsDict["s"],
+                "ks": argsDict["ks"],
+                "data_type": gtu.DTYPE_ATTRIBUTES[outputTensor.dtype]["json"],
+            }
+        elif argsDict["dg_type"] == gtu.DataGenType.OP_SPECIAL:
+            mode = gtu.ComplianceMode.FP_SPECIAL
+        elif "compliance" in op and "ulp" in op["compliance"]:
+            mode = gtu.ComplianceMode.ULP
+            compliance_tens["ulp_info"] = {"ulp": op["compliance"]["ulp"]}
+        elif op["op"] == Op.REDUCE_PRODUCT:
+            mode = gtu.ComplianceMode.REDUCE_PRODUCT
+        else:
+            mode = gtu.ComplianceMode.EXACT
+        compliance_tens["mode"] = gtu.ComplianceMode(mode).name
+
+        return compliance_tens
+
+    # Build Op functions
+    # Create the output tensor (calling OutputShaper as needed)
+    # Do final tweaks to attributes (if necessary for errorIf)
+    # Add Op into graph
+    # Return resulting tensor information or BuildInfo
+
+    class BuildInfo:
+        """Enhanced build information containing result tensor and associated compliance dict."""
+
+        def __init__(self, resultTensor, complianceDict):
+            self.resultTensor = resultTensor
+            self.complianceDict = complianceDict
 
     def build_unary(self, op, a, validator_fcns=None, error_name=None, qinfo=None):
         result_tens = OutputShaper.unaryOp(self.ser, self.rng, a, error_name)
@@ -975,15 +1052,16 @@ class TosaTestGen:
         return result_tens
 
     def build_matmul(
-        self, op, a, b, accum_dtype, validator_fcns=None, error_name=None, qinfo=None
+        self, op, a, b, args_dict, validator_fcns=None, error_name=None, qinfo=None
     ):
-        result_tens = OutputShaper.matmulOp(
+        accum_dtype = args_dict["acc_type"]
+        result_tensor = OutputShaper.matmulOp(
             self.ser, self.rng, a, b, accum_dtype, error_name
         )
 
         # Invalidate Input/Output list for error if checks.
         input_list = [a.name, b.name]
-        output_list = [result_tens.name]
+        output_list = [result_tensor.name]
         pCount, cCount = op["operands"]
         num_operands = pCount + cCount
         input_list, output_list = TosaErrorIfArgGen.eiInvalidateInputOutputList(
@@ -999,10 +1077,10 @@ class TosaTestGen:
             input_dtype=a.dtype,
             input2_shape=b.shape,
             input2_dtype=b.dtype,
-            output_shape=result_tens.shape,
-            output_dtype=result_tens.dtype,
+            output_shape=result_tensor.shape,
+            output_dtype=result_tensor.dtype,
             qinfo=qinfo,
-            result_tensors=[result_tens],
+            result_tensors=[result_tensor],
             input_list=input_list,
             output_list=output_list,
             num_operands=num_operands,
@@ -1014,7 +1092,12 @@ class TosaTestGen:
         attr.MatMulAttribute(qinfo[0], qinfo[1])
 
         self.ser.addOperator(op["op"], input_list, output_list, attr)
-        return result_tens
+
+        compliance = self.tensorComplianceMetaData(
+            op, args_dict, result_tensor, error_name
+        )
+
+        return TosaTestGen.BuildInfo(result_tensor, compliance)
 
     def build_reduce(self, op, a, axis, validator_fcns, error_name=None):
         result_tens = OutputShaper.reduceOp(self.ser, self.rng, a, axis, error_name)
@@ -1895,7 +1978,7 @@ class TosaTestGen:
 
     def _get_condition_tensor(self, op, cond, error_name):
         if error_name == ErrorIf.CondIfCondNotMatchingBool:
-            cond_type = get_wrong_output_type(op, self.rng, DType.BOOL)
+            cond_type = gtu.get_wrong_output_type(op, self.rng, DType.BOOL)
         else:
             cond_type = DType.BOOL
         if error_name == ErrorIf.CondIfCondShapeNotSizeOne:
@@ -2357,7 +2440,7 @@ class TosaTestGen:
         # Initialize a new random number generator
         self.rng = np.random.default_rng(self.random_seed)
 
-        build_fcn, tgen_fcn, tvgen_fcn, agen_fcn = op["build_fcn"]
+        _, tgen_fcn, _, agen_fcn = op["build_fcn"]
 
         # Test list consists of a tuple of:
         # (opName, testNameStr, dtype, shapeList, argumentsList)
@@ -2461,7 +2544,7 @@ class TosaTestGen:
         # Create a serializer
         self.createSerializer(opName, testStr)
 
-        build_fcn, tgen_fcn, tvgen_fcn, agen_fcn = op["build_fcn"]
+        build_fcn, _, tvgen_fcn, _ = op["build_fcn"]
         if "error_if_validators" in op:
             error_if_validators = op["error_if_validators"]
         else:
@@ -2495,24 +2578,37 @@ class TosaTestGen:
             qgen = None
 
         # Build the random tensor operands and the test
-        tens = []
 
         if qgen is not None:
             qinfo = qgen(self, op, dtype_or_dtypeList, error_name)
         else:
             qinfo = None
 
-        tens = tvgen_fcn(self, op, dtypeList, shapeList, testArgs, error_name)
+        # Extra meta data for the desc.json
+        tensMeta = {}
+
+        # Check we are using the new testArgs interface with an argsDict dictionary
+        if len(testArgs) == 1 and isinstance(testArgs[0], dict):
+            argsDict = testArgs[0]
+            assert "dg_type" in argsDict
+            tvgInfo = tvgen_fcn(
+                self, opName, dtypeList, shapeList, argsDict, error_name
+            )
+            if tvgInfo.dataGenDict:
+                tensMeta["data_gen"] = tvgInfo.dataGenDict
+            tens = tvgInfo.tensorList
+        else:
+            tens = tvgen_fcn(self, op, dtypeList, shapeList, testArgs, error_name)
 
         try:
             if error_if_validators is None:
                 if qinfo is not None:
-                    resultName = build_fcn(self, op, *tens, *testArgs, qinfo)
+                    result = build_fcn(self, op, *tens, *testArgs, qinfo)
                 else:
-                    resultName = build_fcn(self, op, *tens, *testArgs)
+                    result = build_fcn(self, op, *tens, *testArgs)
             else:
                 if qinfo is not None:
-                    resultName = build_fcn(
+                    result = build_fcn(
                         self,
                         op,
                         *tens,
@@ -2522,7 +2618,7 @@ class TosaTestGen:
                         qinfo=qinfo,
                     )
                 else:
-                    resultName = build_fcn(
+                    result = build_fcn(
                         self,
                         op,
                         *tens,
@@ -2534,9 +2630,16 @@ class TosaTestGen:
             print(f"build_fcn: {build_fcn}\nTensors: {tens}\nArgs: {testArgs}\n")
             raise e
 
-        if resultName:
+        if result:
             # The test is valid, serialize it
-            self.serialize("test")
+            if isinstance(result, TosaTestGen.BuildInfo) and result.complianceDict:
+                # Add the compliance meta data
+                # NOTE: This currently expects only one result output
+                tensMeta["compliance"] = {
+                    "version": "0.1",
+                    "tensors": {result.resultTensor.name: result.complianceDict},
+                }
+            self.serialize("test", tensMeta)
         else:
             # The test is not valid
             print(f"Invalid ERROR_IF test created: {opName} {testStr}")
@@ -2865,7 +2968,7 @@ class TosaTestGen:
             "build_fcn": (
                 build_matmul,
                 TosaTensorGen.tgMatmul,
-                TosaTensorValuesGen.tvgDefault,
+                TosaTensorValuesGen.tvgLazyGenDefault,
                 TosaArgGen.agMatMul,
             ),
             "qgen": TosaQuantGen.qgMatmul,
@@ -2878,6 +2981,10 @@ class TosaTestGen:
                 TosaErrorValidator.evWrongInputList,
                 TosaErrorValidator.evWrongOutputList,
             ),
+            "data_gen": {
+                "fp": (gtu.DataGenType.DOT_PRODUCT,),
+                "int": (gtu.DataGenType.PSEUDO_RANDOM,),
+            },
         },
         "max_pool2d": {
             "op": Op.MAX_POOL2D,
@@ -4446,7 +4553,7 @@ class OutputShaper:
                 excludes = [DType.FP16, DType.FP32]
             else:
                 excludes = [out_dtype]
-            wrong_dtypes = list(usableDTypes(excludes=excludes))
+            wrong_dtypes = list(gtu.usableDTypes(excludes=excludes))
             out_dtype = rng.choice(wrong_dtypes)
 
         return ser.addOutput(ofm_shape, out_dtype)
@@ -4508,7 +4615,7 @@ class OutputShaper:
                 excludes = [DType.FP16, DType.FP32]
             else:
                 excludes = [out_dtype]
-            wrong_dtypes = list(usableDTypes(excludes=excludes))
+            wrong_dtypes = list(gtu.usableDTypes(excludes=excludes))
             out_dtype = rng.choice(wrong_dtypes)
 
         return ser.addOutput(ofm_shape, out_dtype)
@@ -4559,7 +4666,7 @@ class OutputShaper:
                 excludes = [DType.FP16, DType.FP32]
             else:
                 excludes = [out_dtype]
-            wrong_dtypes = list(usableDTypes(excludes=excludes))
+            wrong_dtypes = list(gtu.usableDTypes(excludes=excludes))
             out_dtype = rng.choice(wrong_dtypes)
 
         return ser.addOutput(ofm_shape, out_dtype)
@@ -4711,7 +4818,7 @@ class OutputShaper:
             bad_dim = rng.choice(range(len(output_shape)))
             output_shape[bad_dim] -= rng.choice([1, 2])
         elif error_name == ErrorIf.RankMismatch:
-            output_shape = get_rank_mismatch_shape(rng, output_shape)
+            output_shape = gtu.get_rank_mismatch_shape(rng, output_shape)
 
         if error_name == ErrorIf.WrongOutputType:
             all_dtypes = [
@@ -4806,7 +4913,7 @@ class OutputShaper:
         elif error_name == ErrorIf.InputSizeStartLengthMismatch:
             output_shape = input.shape.copy()
         elif error_name == ErrorIf.RankMismatch:
-            output_shape = get_rank_mismatch_shape(rng, output_shape)
+            output_shape = gtu.get_rank_mismatch_shape(rng, output_shape)
 
         return ser.addOutput(output_shape, outputDType)
 
@@ -4820,7 +4927,7 @@ class OutputShaper:
             output_shape[i] = a.shape[i] * multiples[i]
 
         if error_name == ErrorIf.RankMismatch:
-            output_shape = get_rank_mismatch_shape(rng, output_shape)
+            output_shape = gtu.get_rank_mismatch_shape(rng, output_shape)
 
         if error_name == ErrorIf.WrongOutputType:
             all_dtypes = [
@@ -4853,7 +4960,7 @@ class OutputShaper:
             for i in range(len(output_shape)):
                 output_shape[i] += rng.integers(1, 10)
         elif error_name == ErrorIf.RankMismatch:
-            output_shape = get_rank_mismatch_shape(rng, output_shape)
+            output_shape = gtu.get_rank_mismatch_shape(rng, output_shape)
 
         if error_name == ErrorIf.WrongOutputType:
             all_dtypes = [
@@ -4980,21 +5087,21 @@ class OutputShaper:
             oh = max(oh, 1)
             ow = max(ow, 1)
             if error_name != ErrorIf.MaxDimExceeded:
-                oh = min(oh, MAX_RESIZE_DIMENSION - 1)
-                ow = min(ow, MAX_RESIZE_DIMENSION - 1)
+                oh = min(oh, gtu.MAX_RESIZE_DIMENSION - 1)
+                ow = min(ow, gtu.MAX_RESIZE_DIMENSION - 1)
 
         if error_name == ErrorIf.ResizeOutputShapeMismatch:
             choices = [1, 2, 3]
             change = rng.choice(choices)
             # increment in multiples of scale_y/x_d so we don't hit non-integer error case
             if change in [1, 3]:
-                if oh + scale_y_d >= MAX_RESIZE_DIMENSION:
+                if oh + scale_y_d >= gtu.MAX_RESIZE_DIMENSION:
                     oh -= scale_y_d
                     assert oh > 0  # Should have been caught in agResize
                 else:
                     oh += scale_y_d
             if change in [2, 3]:
-                if ow + scale_x_d >= MAX_RESIZE_DIMENSION:
+                if ow + scale_x_d >= gtu.MAX_RESIZE_DIMENSION:
                     ow -= scale_x_d
                     assert ow > 0  # Should have been caught in agResize
                 else:
@@ -5051,7 +5158,7 @@ class OutputShaper:
                 excludes = [DType.FP16, DType.FP32]
             else:
                 excludes = [out_dtype]
-            wrong_dtypes = list(usableDTypes(excludes=excludes))
+            wrong_dtypes = list(gtu.usableDTypes(excludes=excludes))
             out_dtype = rng.choice(wrong_dtypes)
 
         return ser.addOutput(output_shape, out_dtype)
@@ -5075,7 +5182,7 @@ class OutputShaper:
 
         if error_name == ErrorIf.WrongOutputType:
             excludes = [DType.FP32]
-            wrong_dtypes = list(usableDTypes(excludes=excludes))
+            wrong_dtypes = list(gtu.usableDTypes(excludes=excludes))
             output_dtype = rng.choice(wrong_dtypes)
         elif error_name == ErrorIf.BatchMismatch:
             output_shape[0] += rng.integers(1, 10)
@@ -5100,7 +5207,7 @@ class OutputShaper:
         output_dtype = value.dtype
         if error_name == ErrorIf.WrongOutputType:
             excludes = [DType.FP32]
-            wrong_dtypes = list(usableDTypes(excludes=excludes))
+            wrong_dtypes = list(gtu.usableDTypes(excludes=excludes))
             output_dtype = rng.choice(wrong_dtypes)
         elif error_name == ErrorIf.BatchMismatch:
             output_shape[0] += rng.integers(1, 10)
diff --git a/verif/generator/tosa_utils.py b/verif/generator/tosa_utils.py
index 3cd0370..75a0df5 100644
--- a/verif/generator/tosa_utils.py
+++ b/verif/generator/tosa_utils.py
@@ -1,7 +1,8 @@
-# Copyright (c) 2021-2022, ARM Limited.
+# Copyright (c) 2021-2023, ARM Limited.
 # SPDX-License-Identifier: Apache-2.0
 import struct
 import sys
+from enum import IntEnum
 
 import numpy as np
 from tosa.DType import DType
@@ -9,22 +10,54 @@ from tosa.DType import DType
 # Maximum dimension size for output and inputs for RESIZE
 MAX_RESIZE_DIMENSION = 16384
 
+# Data type information dictionary
+# - str: filename abbreviation
+# - width: number of bytes needed for type
+# - json: JSON type string
 DTYPE_ATTRIBUTES = {
-    DType.BOOL: {"str": "b", "width": 1},
-    DType.INT4: {"str": "i4", "width": 4},
-    DType.INT8: {"str": "i8", "width": 8},
-    DType.UINT8: {"str": "u8", "width": 8},
-    DType.INT16: {"str": "i16", "width": 16},
-    DType.UINT16: {"str": "u16", "width": 16},
-    DType.INT32: {"str": "i32", "width": 32},
-    DType.INT48: {"str": "i48", "width": 48},
-    DType.SHAPE: {"str": "i64", "width": 64},
-    DType.FP16: {"str": "f16", "width": 16},
-    DType.BF16: {"str": "bf16", "width": 16},
-    DType.FP32: {"str": "f32", "width": 32},
+    DType.BOOL: {"str": "b", "width": 1, "json": "BOOL"},
+    DType.INT4: {"str": "i4", "width": 4, "json": "INT4"},
+    DType.INT8: {"str": "i8", "width": 8, "json": "INT8"},
+    DType.UINT8: {"str": "u8", "width": 8, "json": "UINT8"},
+    DType.INT16: {"str": "i16", "width": 16, "json": "INT16"},
+    DType.UINT16: {"str": "u16", "width": 16, "json": "UINT16"},
+    DType.INT32: {"str": "i32", "width": 32, "json": "INT32"},
+    DType.INT48: {"str": "i48", "width": 48, "json": "INT48"},
+    DType.SHAPE: {"str": "s", "width": 64, "json": "SHAPE"},
+    DType.FP16: {"str": "f16", "width": 16, "json": "FP16"},
+    DType.BF16: {"str": "bf16", "width": 16, "json": "BF16"},
+    DType.FP32: {"str": "f32", "width": 32, "json": "FP32"},
 }
 
 
+class ComplianceMode(IntEnum):
+    """Compliance mode types."""
+
+    EXACT = 0
+    DOT_PRODUCT = 1
+    ULP = 2
+    FP_SPECIAL = 3
+    REDUCE_PRODUCT = 4
+
+
+class DataGenType(IntEnum):
+    """Data generator types."""
+
+    PSEUDO_RANDOM = 0
+    DOT_PRODUCT = 1
+    OP_BOUNDARY = 2
+    OP_FULLSET = 3
+    OP_SPECIAL = 4
+
+
+# Additional (optional) data for dot product data generator
+DG_DOT_PRODUCT_OPTIONAL_INFO = ("acc_type", "kernel", "axis")
+
+
+def dtypeIsFloat(dtype):
+    return dtype in (DType.FP16, DType.BF16, DType.FP32)
+
+
 def valueToName(item, value):
     """Get the name of an attribute with the given value.
 
diff --git a/verif/generator/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py
index 4d12bb0..0d532c0 100644
--- a/verif/generator/tosa_verif_build_tests.py
+++ b/verif/generator/tosa_verif_build_tests.py
@@ -51,6 +51,10 @@ def parseArgs(argv):
         argv = new_argv
 
     parser = argparse.ArgumentParser()
+
+    ops_group = parser.add_argument_group("operator options")
+    tens_group = parser.add_argument_group("tensor options")
+
     parser.add_argument(
         "-o", dest="output_dir", type=str, default="vtest", help="Test output directory"
     )
@@ -75,8 +79,15 @@ def parseArgs(argv):
         "-v", "--verbose", dest="verbose", action="count", help="Verbose operation"
     )
 
-    # Constraints on tests
     parser.add_argument(
+        "--lazy-data-generation",
+        dest="lazy_data_gen",
+        action="store_true",
+        help="Tensor data generation is delayed til test running",
+    )
+
+    # Constraints on tests
+    tens_group.add_argument(
         "--tensor-dim-range",
         dest="tensor_shape_range",
         default="1,64",
@@ -84,7 +95,7 @@ def parseArgs(argv):
         help="Min,Max range of tensor shapes",
     )
 
-    parser.add_argument(
+    tens_group.add_argument(
         OPTION_FP_VALUES_RANGE,
         dest="tensor_fp_value_range",
         default="0.0,1.0",
@@ -92,7 +103,7 @@ def parseArgs(argv):
         help="Min,Max range of floating point tensor values",
     )
 
-    parser.add_argument(
+    ops_group.add_argument(
         "--max-batch-size",
         dest="max_batch_size",
         default=1,
@@ -100,7 +111,7 @@ def parseArgs(argv):
         help="Maximum batch size for NHWC tests",
     )
 
-    parser.add_argument(
+    ops_group.add_argument(
         "--max-conv-padding",
         dest="max_conv_padding",
         default=1,
@@ -108,7 +119,7 @@ def parseArgs(argv):
         help="Maximum padding for Conv tests",
     )
 
-    parser.add_argument(
+    ops_group.add_argument(
         "--max-conv-dilation",
         dest="max_conv_dilation",
         default=2,
@@ -116,7 +127,7 @@ def parseArgs(argv):
         help="Maximum dilation for Conv tests",
     )
 
-    parser.add_argument(
+    ops_group.add_argument(
         "--max-conv-stride",
         dest="max_conv_stride",
         default=2,
@@ -124,7 +135,7 @@ def parseArgs(argv):
         help="Maximum stride for Conv tests",
     )
 
-    parser.add_argument(
+    ops_group.add_argument(
         "--max-pooling-padding",
         dest="max_pooling_padding",
         default=1,
@@ -132,7 +143,7 @@ def parseArgs(argv):
         help="Maximum padding for pooling tests",
     )
 
-    parser.add_argument(
+    ops_group.add_argument(
         "--max-pooling-stride",
         dest="max_pooling_stride",
         default=2,
@@ -140,7 +151,7 @@ def parseArgs(argv):
         help="Maximum stride for pooling tests",
     )
 
-    parser.add_argument(
+    ops_group.add_argument(
         "--max-pooling-kernel",
         dest="max_pooling_kernel",
         default=3,
@@ -148,7 +159,7 @@ def parseArgs(argv):
         help="Maximum kernel for pooling tests",
     )
 
-    parser.add_argument(
+    ops_group.add_argument(
         "--num-rand-permutations",
         dest="num_rand_permutations",
         default=6,
@@ -156,7 +167,7 @@ def parseArgs(argv):
         help="Number of random permutations for a given shape/rank for randomly-sampled parameter spaces",
     )
 
-    parser.add_argument(
+    ops_group.add_argument(
         "--max-resize-output-dim",
         dest="max_resize_output_dim",
         default=1000,
@@ -165,7 +176,7 @@ def parseArgs(argv):
     )
 
     # Targeting a specific shape/rank/dtype
-    parser.add_argument(
+    tens_group.add_argument(
         "--target-shape",
         dest="target_shapes",
         action="append",
@@ -174,7 +185,7 @@ def parseArgs(argv):
         help="Create tests with a particular input tensor shape, e.g., 1,4,4,8 (may be repeated for tests that require multiple input shapes)",
     )
 
-    parser.add_argument(
+    tens_group.add_argument(
         "--target-rank",
         dest="target_ranks",
         action="append",
@@ -184,7 +195,7 @@ def parseArgs(argv):
     )
 
     # Used for parsing a comma-separated list of integers in a string
-    parser.add_argument(
+    tens_group.add_argument(
         "--target-dtype",
         dest="target_dtypes",
         action="append",
@@ -193,7 +204,7 @@ def parseArgs(argv):
         help=f"Create test with a particular DType: [{', '.join([d.lower() for d in DTypeNames[1:]])}] (may be repeated)",
     )
 
-    parser.add_argument(
+    ops_group.add_argument(
         "--num-const-inputs-concat",
         dest="num_const_inputs_concat",
         default=0,
@@ -211,14 +222,14 @@ def parseArgs(argv):
         help="type of tests produced, positive, negative, or both",
     )
 
-    parser.add_argument(
+    ops_group.add_argument(
         "--allow-pooling-and-conv-oversizes",
         dest="oversize",
         action="store_true",
         help="allow oversize padding, stride and kernel tests",
     )
 
-    parser.add_argument(
+    ops_group.add_argument(
         "--zero-point",
         dest="zeropoint",
         default=None,
@@ -233,11 +244,11 @@ def parseArgs(argv):
         help="output const tensors as numpy files for inspection",
     )
 
-    parser.add_argument(
+    ops_group.add_argument(
         "--level-8k-sizes",
         dest="level8k",
         action="store_true",
-        help="create level 8k size tests (RESIZE)",
+        help="create level 8k size tests",
     )
 
     args = parser.parse_args(argv)