diff options
-rwxr-xr-x | scripts/convert2conformance/convert2conformance.py | 39 | ||||
-rw-r--r-- | verif/conformance/README.md | 3 | ||||
-rw-r--r-- | verif/conformance/tosa_main_profile_ops_info.json | 7 | ||||
-rw-r--r-- | verif/conformance/tosa_verif_conformance_generator.py | 44 | ||||
-rw-r--r-- | verif/generator/tosa_arg_gen.py | 174 | ||||
-rw-r--r-- | verif/generator/tosa_test_gen.py | 351 | ||||
-rw-r--r-- | verif/generator/tosa_utils.py | 59 | ||||
-rw-r--r-- | verif/generator/tosa_verif_build_tests.py | 49 |
8 files changed, 540 insertions, 186 deletions
diff --git a/scripts/convert2conformance/convert2conformance.py b/scripts/convert2conformance/convert2conformance.py index b1f8d0e..f915070 100755 --- a/scripts/convert2conformance/convert2conformance.py +++ b/scripts/convert2conformance/convert2conformance.py @@ -11,11 +11,13 @@ import argparse import json import logging import os +import shutil from pathlib import Path from typing import Optional from json2fbbin.json2fbbin import fbbin_to_json from json2numpy.json2numpy import npy_to_json +from schemavalidation.schemavalidation import TestDescSchemaValidator logging.basicConfig(level=logging.INFO) logger = logging.getLogger("convert2conformance") @@ -35,7 +37,8 @@ PROFILES_LIST = ["tosa-bi", "tosa-mi"] def parse_args(argv): """Parse the arguments.""" - parser = argparse.ArgumentParser() + # Set prog for when we are called via tosa_verif_conformance_generator + parser = argparse.ArgumentParser(prog="convert2conformance") parser.add_argument( "test_dir", default=Path.cwd(), @@ -92,6 +95,11 @@ def parse_args(argv): help="Output directory must not contain the same test directory", ) parser.add_argument( + "--lazy-data-generation", + action="store_true", + help="Enable lazy data generation (only for tosa-mi)", + ) + parser.add_argument( "-v", "--verbose", dest="verbose", action="store_true", help="Verbose operation" ) args = parser.parse_args(argv) @@ -191,14 +199,15 @@ def update_desc_json( else: logger.error(f"Missing result file {ofm_path}") return None - cfm_files.append(cfm + ".npy") + cfm_files.append(cfm + ".npy") # Remove path and "ref-"/"ref_model_" from output filenames ofm_files.append(strip_ref_output_name(ofm_path.name)) # Rewrite output file names as they can be relative, but keep them npys test_desc["ofm_file"] = ofm_files - if not test_desc["expected_failure"]: - # Output expected result file for conformance if expected pass + if not test_desc["expected_failure"] and cfm_files: + # Output expected result file for conformance if expected pass and we + # have some files! test_desc["expected_result_file"] = cfm_files # Add supported profiles @@ -319,23 +328,32 @@ def main(argv=None): # Convert input files to JSON ifm_files = [] for file in test_desc["ifm_file"]: - if file is None: - ifm_files.append(None) - else: + if file: path = desc_filename.parent / file - convert_numpy_file(path, args.output_dir) ifm_files.append(path.name) + if path.is_file(): + convert_numpy_file(path, args.output_dir) + else: + if not args.lazy_data_generation: + logger.error(f"Missing input file {path.name}") + return 1 + # Rewrite input file names to make sure the paths are correct, # but keep them numpys as the test runner will convert them back # before giving them to the SUT test_desc["ifm_file"] = ifm_files + # Check for cpp files for data-generator/verifier + cpp_files = args.test_dir.glob("*.cpp") + for cpp in cpp_files: + shutil.copy(str(cpp), str(args.output_dir)) + # Update desc.json and convert result files to JSON test_desc = update_desc_json( desc_filename.parent, test_desc, output_dir=args.output_dir, - create_result=True, + create_result=(not args.lazy_data_generation), profiles=args.profile, tags=args.tag, ) @@ -343,6 +361,9 @@ def main(argv=None): # Error from conversion/update return 1 + # Validate the desc.json schema + TestDescSchemaValidator().validate_config(test_desc) + # Output new desc.json new_desc_filename = args.output_dir / NAME_DESC_FILENAME with open(new_desc_filename, "w") as fd: diff --git a/verif/conformance/README.md b/verif/conformance/README.md index a7ad789..8fbf4fd 100644 --- a/verif/conformance/README.md +++ b/verif/conformance/README.md @@ -17,6 +17,7 @@ Each operator entry contains: * "group" - name of the group this operator is in, in the spec * "profile" - list of profiles that this operator covers +* "support_for" - optional list of supported creation modes out of: lazy_data_gen (data generation just before test run) * "generation" - dictionary of test generation details - see below * "selection" - dictionary of test selection details - see below @@ -49,6 +50,8 @@ before permutes are applied) ### Framework ops +DEPRECATED - not supported for conformance testing. + NOTE: Currently assumed all framework ops will be TFLite. Naming: `tosa_PPP_profile_framework_ops_info.json` diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json index 7388835..0d3df72 100644 --- a/verif/conformance/tosa_main_profile_ops_info.json +++ b/verif/conformance/tosa_main_profile_ops_info.json @@ -1417,6 +1417,7 @@ "profile": [ "tosa-mi" ], + "support_for": [ "lazy_data_gen" ], "generation": { "standard": { "negative_dim_range": "1,10", @@ -1460,11 +1461,7 @@ }, "selection": { "default": { - "params": {}, - "permutes": [ - "shape", - "type" - ] + "all": "true" } } }, diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py index 0fb5500..c2ea4ec 100644 --- a/verif/conformance/tosa_verif_conformance_generator.py +++ b/verif/conformance/tosa_verif_conformance_generator.py @@ -84,7 +84,14 @@ def _run_sh_command(args, cwd, full_cmd): def build_op_tests( - args, test_type, profile, operator, group, gen_args_list, gen_neg_dim_range + args, + test_type, + profile, + operator, + group, + gen_args_list, + gen_neg_dim_range, + supports=[], ): """Build tests for a given operator. @@ -105,6 +112,9 @@ def build_op_tests( str(args.random_seed), ] + if "lazy_data_gen" in supports and args.lazy_data_generation: + build_cmd_base.append("--lazy-data-generation") + build_cmds_list = [] if test_type in ["positive", "both"]: @@ -198,8 +208,12 @@ def _get_all_tests_list( return tests -def generate_results(args, profile, operator, op_build_dir, tests=None): +def generate_results(args, profile, operator, op_build_dir, supports=[], tests=None): """Run tests on reference model and save result to the test directory.""" + if "lazy_data_gen" in supports and args.lazy_data_generation: + logger.info("Skipping running tests due to lazy data gen") + return + num_cores = args.num_cores run_tests_cmd = "tosa_verif_run_tests" @@ -254,6 +268,7 @@ def convert_tests( op_build_dir, output_dir, op_profiles_list, + supports=[], tests=None, group=None, trim_op_subdir=False, @@ -275,6 +290,8 @@ def convert_tests( c2c_args_base.extend(["--tag", tag]) if args.framework_schema: c2c_args_base.extend(["--framework-schema", str(args.framework_schema)]) + if "lazy_data_gen" in supports and args.lazy_data_generation: + c2c_args_base.append("--lazy-data-generation") c2c_args_base.append("--output-directory") c2c_args_list = [] @@ -474,6 +491,11 @@ def parse_args(argv=None): help="Type of tests produced (default is both)", ) parser.add_argument( + "--lazy-data-generation", + action="store_true", + help="Enable lazy data generation (only for tosa-mi)", + ) + parser.add_argument( "--ref-model-directory", dest="ref_model_dir", type=Path, @@ -718,6 +740,11 @@ def main(): operator_group = test_params[op]["group"] root_output_dir = args.output_dir / "operators" + supports = ( + test_params[op]["support_for"] + if "support_for" in test_params[op] + else [] + ) # Iterate through the generation groups selecting tests from each for gen_name, gen_dict in test_params[op]["generation"].items(): @@ -756,6 +783,7 @@ def main(): gen_name, gen_dict["generator_args"], gen_neg_dim_range, + supports=supports, ) # Work out which selection criteria we are using @@ -782,7 +810,9 @@ def main(): and selection_config["all"] == "true" ): logger.debug(f"Running and converting all {op} tests") - generate_results(args, profile, op, op_build_dir) + generate_results( + args, profile, op, op_build_dir, supports=supports + ) operator_test_list = None else: logger.debug( @@ -800,7 +830,12 @@ def main(): ) ) generate_results( - args, profile, op, op_build_dir, tests_gen + args, + profile, + op, + op_build_dir, + supports=supports, + tests=tests_gen, ) operator_test_list = list(tests_gen2) else: @@ -823,6 +858,7 @@ def main(): op_build_dir, root_output_dir, op_profiles_list, + supports=supports, tests=operator_test_list, group=operator_group, tags=tags, diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py index 97ff237..8d96090 100644 --- a/verif/generator/tosa_arg_gen.py +++ b/verif/generator/tosa_arg_gen.py @@ -4,12 +4,10 @@ import itertools import math import warnings +import generator.tosa_utils as gtu import numpy as np from generator.tosa_error_if import ErrorIf from generator.tosa_error_if import TosaErrorIfArgGen -from generator.tosa_utils import get_accum_dtype_from_tgTypes -from generator.tosa_utils import get_wrong_output_type -from generator.tosa_utils import MAX_RESIZE_DIMENSION from serializer.tosa_serializer import DTypeNames from tosa.DType import DType from tosa.Op import Op @@ -606,11 +604,18 @@ class TosaTensorGen: class TosaTensorValuesGen: - """Tensor Value generators create the random data for each test.""" + """Tensor Value generators create the random data for each tensor in each test.""" def __init__(self): pass + class TVGInfo: + """Enhanced tensor values information including data gen dict.""" + + def __init__(self, tensorList, dataGenDict): + self.tensorList = tensorList + self.dataGenDict = dataGenDict + @staticmethod def tvgDefault(testGen, op, dtypeList, shapeList, testArgs, error_name=None): pCount, cCount = op["operands"] @@ -624,6 +629,87 @@ class TosaTensorValuesGen: return tens @staticmethod + def tvgLazyGenDefault( + testGen, opName, dtypeList, shapeList, argsDict, error_name=None + ): + # Variable inputs versus constants + pCount, cCount = testGen.TOSA_OP_LIST[opName]["operands"] + + overrideLazy = False + if not gtu.dtypeIsFloat(dtypeList[0]) and testGen.args.lazy_data_gen: + # TEMPORARY OVERRIDE for integer types + overrideLazy = True + testGen.args.lazy_data_gen = False + + # TODO - Change to generation of data using library! + # For now - we fall back to original path (or when dealing with non-floats) + if not testGen.args.lazy_data_gen: + tens_ser_list = TosaTensorValuesGen.tvgDefault( + testGen, + testGen.TOSA_OP_LIST[opName], + dtypeList, + shapeList, + [], + error_name, + ) + if overrideLazy: + # Return to lazy mode + testGen.args.lazy_data_gen = True + return TosaTensorValuesGen.TVGInfo(tens_ser_list, None) + + # Create data generator meta-data + dg_type = argsDict["dg_type"] + dg_tens_meta = {} + tens_ser_list = [] + for idx, shape in enumerate(shapeList): + + tens_meta = {} + tens_meta["generator"] = gtu.DataGenType(dg_type).name + tens_meta["data_type"] = gtu.DTYPE_ATTRIBUTES[dtypeList[idx]]["json"] + tens_meta["shape"] = [int(i) for i in shape] + tens_meta["input_pos"] = idx + tens_meta["op"] = opName + + if idx < pCount: + tens_meta["input_type"] = "variable" + tens = testGen.ser.addPlaceholder(shape, dtypeList[idx], None) + else: + tens_meta["input_type"] = "constant" + tens = testGen.ser.addConst(shape, dtypeList[idx], None) + tens_ser_list.append(tens) + + if dg_type == gtu.DataGenType.PSEUDO_RANDOM: + info = {} + # TODO - generate seed for this generator based on test + info["rng_seed"] = -1 + info["range"] = [ + str(v) + for v in testGen.getDTypeRange(dtypeList[idx], high_inclusive=True) + ] + tens_meta["pseudo_random_info"] = info + elif dg_type == gtu.DataGenType.DOT_PRODUCT: + info = {} + info["s"] = argsDict["s"] + info["ks"] = argsDict["ks"] + for key in gtu.DG_DOT_PRODUCT_OPTIONAL_INFO: + if key in argsDict: + if key.endswith("_type"): + info[key] = gtu.DTYPE_ATTRIBUTES[argsDict[key]]["json"] + else: + info[key] = argsDict[key] + tens_meta["dot_product_info"] = info + else: + # TODO - other data gen type + assert False, "TODO: support other data gen types" + dg_tens_meta[tens.name] = tens_meta + + tens_data = { + "version": "0.1", + "tensors": dg_tens_meta, + } + return TosaTensorValuesGen.TVGInfo(tens_ser_list, tens_data) + + @staticmethod def tvgNegate(testGen, op, dtypeList, shapeList, testArgs, error_name=None): if dtypeList[0] == DType.INT32 and error_name is None: pCount, cCount = op["operands"] @@ -1024,6 +1110,50 @@ class TosaArgGen: pass @staticmethod + def _add_data_generators(testGen, opName, dtype, arg_list, error_name, **kwargs): + """Add extra tests for each type of data generator for this op.""" + if error_name is None and "data_gen" in testGen.TOSA_OP_LIST[opName]: + if dtype in [DType.FP16, DType.FP32, DType.BF16]: + dataGenTypesList = testGen.TOSA_OP_LIST[opName]["data_gen"]["fp"] + else: + dataGenTypesList = testGen.TOSA_OP_LIST[opName]["data_gen"]["int"] + else: + # Error test or No data generator types listed - assume random + dataGenTypesList = (gtu.DataGenType.PSEUDO_RANDOM,) + + # Expand arg list with other data generator types + new_arg_list = [] + for dg_type in dataGenTypesList: + for arg_str, arg_attrs in arg_list: + arg_dict = arg_attrs[0] + arg_dict["dg_type"] = dg_type + + if dg_type == gtu.DataGenType.PSEUDO_RANDOM: + # Default test + new_arg_list.append((arg_str, [arg_dict])) + + elif dg_type == gtu.DataGenType.DOT_PRODUCT: + # Extra tests for each dot product test set + dot_products = kwargs["dot_products"] + if dot_products < testGen.TOSA_MI_DOT_PRODUCT_MIN: + print( + f"Skipping dot product test as too few calculations {dot_products} < {testGen.TOSA_MI_DOT_PRODUCT_MIN}" + ) + continue + arg_dict["ks"] = kwargs["ks"] + for key in gtu.DG_DOT_PRODUCT_OPTIONAL_INFO: + if key in kwargs: + arg_dict[key] = kwargs[key] + + for s in testGen.TOSA_MI_DOT_PRODUCT_TEST_SETS: + new_arg_str = f"{arg_str}_s{s}" + new_arg_dict = arg_dict.copy() + new_arg_dict["s"] = s + new_arg_list.append((new_arg_str, [new_arg_dict])) + + return new_arg_list + + @staticmethod def agNone(testGen, opName, shapeList, dtype, error_name=None): """A trivial argument generator for operators that don't take any non-tensor arguments""" @@ -1073,7 +1203,7 @@ class TosaArgGen: # Shape: (OFM channels), (KD), KH, KW, IFM channels filter_shape = shapeList[1] - accum_dtype = get_accum_dtype_from_tgTypes(dtypes) + accum_dtype = gtu.get_accum_dtype_from_tgTypes(dtypes) # Check the rank conv3d = opName.startswith("conv3d") @@ -1258,12 +1388,12 @@ class TosaArgGen: input_dtype = dtypes[0] if error_name == ErrorIf.WrongOutputType: - accum_dtype = get_wrong_output_type(opName, testGen.rng, input_dtype) + accum_dtype = gtu.get_wrong_output_type(opName, testGen.rng, input_dtype) elif error_name == ErrorIf.WrongInputType: # Pick some potentially correct output dtype if input type is incorrect accum_dtype = DType.INT32 else: - accum_dtype = get_accum_dtype_from_tgTypes(dtypes) + accum_dtype = gtu.get_accum_dtype_from_tgTypes(dtypes) return [(f"acc{testGen.typeStr(accum_dtype)}", [accum_dtype])] @@ -1285,12 +1415,28 @@ class TosaArgGen: if error_name == ErrorIf.WrongOutputType: # Get incorrect output dtype for ErrorIf case - accum_dtypes = [get_wrong_output_type(opName, testGen.rng, dtype)] + accum_dtypes = [gtu.get_wrong_output_type(opName, testGen.rng, dtype)] elif error_name == ErrorIf.WrongInputType: # Pick some potentially correct output dtype if input type is incorrect accum_dtypes = [DType.INT32] - return [(f"acc{testGen.typeStr(a)}", [a]) for a in accum_dtypes] + arg_list = [ + (f"acc{testGen.typeStr(a)}", [{"acc_type": a}]) for a in accum_dtypes + ] + + arg_list = TosaArgGen._add_data_generators( + testGen, + opName, + dtype, + arg_list, + error_name, + ks=int(shapeList[0][2]), # Set KS = C, from input A (N,H,C) + # Set dot_products = N*H*W + dot_products=gtu.product( + (shapeList[0][0], shapeList[0][1], shapeList[1][2]) + ), + ) + return arg_list @staticmethod def agTransposeConv2D(testGen, opName, shapeList, dtypes, error_name=None): @@ -1303,7 +1449,7 @@ class TosaArgGen: ifm_shape = shapeList[0] filter_shape = shapeList[1] - accum_dtype = get_accum_dtype_from_tgTypes(dtypes) + accum_dtype = gtu.get_accum_dtype_from_tgTypes(dtypes) # Must be rank 4 if error_name != ErrorIf.WrongRank: @@ -2288,9 +2434,9 @@ class TosaArgGen: if ( output_y <= 0 - or output_y >= MAX_RESIZE_DIMENSION + or output_y >= gtu.MAX_RESIZE_DIMENSION or output_x <= 0 - or output_x >= MAX_RESIZE_DIMENSION + or output_x >= gtu.MAX_RESIZE_DIMENSION ): # Output dimensions out of scope if error_name is not None and perm > 0: @@ -2301,11 +2447,11 @@ class TosaArgGen: if error_name == ErrorIf.ResizeOutputShapeMismatch and ( ( - output_y + scale_y_d >= MAX_RESIZE_DIMENSION + output_y + scale_y_d >= gtu.MAX_RESIZE_DIMENSION and output_y - scale_y_d < 1 ) or ( - output_x + scale_x_d >= MAX_RESIZE_DIMENSION + output_x + scale_x_d >= gtu.MAX_RESIZE_DIMENSION and output_x - scale_x_d < 1 ) ): diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py index 3014c81..d15f785 100644 --- a/verif/generator/tosa_test_gen.py +++ b/verif/generator/tosa_test_gen.py @@ -1,8 +1,12 @@ # Copyright (c) 2020-2023, ARM Limited. # SPDX-License-Identifier: Apache-2.0 +import json import os from copy import deepcopy +from datetime import datetime +from pathlib import Path +import generator.tosa_utils as gtu import numpy as np import serializer.tosa_serializer as ts from generator.tosa_arg_gen import TosaArgGen @@ -13,15 +17,15 @@ from generator.tosa_error_if import ErrorIf from generator.tosa_error_if import TosaErrorIfArgGen from generator.tosa_error_if import TosaErrorValidator from generator.tosa_error_if import TosaInvalidValidator -from generator.tosa_utils import DTYPE_ATTRIBUTES -from generator.tosa_utils import get_rank_mismatch_shape -from generator.tosa_utils import get_wrong_output_type -from generator.tosa_utils import MAX_RESIZE_DIMENSION -from generator.tosa_utils import usableDTypes -from generator.tosa_utils import vect_f32_to_bf16 +from schemavalidation.schemavalidation import TestDescSchemaValidator from tosa.DType import DType from tosa.Op import Op +TOSA_AUTOGENERATED_HEADER = f"""// Copyright (c) {datetime.today().year}, ARM Limited +// SPDX-License-Identifier: Apache-2.0 +// AUTO-GENERATED FILE CREATED BY tosa_verif_build_tests +""" + class TosaTestGen: # Maximum rank of tensor supported by test generator. @@ -31,6 +35,10 @@ class TosaTestGen: TOSA_8K_LEVEL_MAX_KERNEL = 8192 TOSA_8K_LEVEL_MAX_STRIDE = 8192 + # Main compliance dot product statistical test range + TOSA_MI_DOT_PRODUCT_TEST_SETS = range(0, 6) + TOSA_MI_DOT_PRODUCT_MIN = 1000 + def __init__(self, args): self.args = args self.basePath = args.output_dir @@ -45,6 +53,8 @@ class TosaTestGen: # Work out floating point range self.random_fp_low = min(args.tensor_fp_value_range) self.random_fp_high = max(args.tensor_fp_value_range) + # JSON schema validation + self.descSchemaValidator = TestDescSchemaValidator() def createSerializer(self, opName, testPath): self.testPath = os.path.join(opName, testPath) @@ -53,81 +63,131 @@ class TosaTestGen: os.makedirs(fullPath, exist_ok=True) # Embed const data in the flatbuffer constMode = ts.ConstMode.EMBED - if self.args.dump_consts: + if self.args.lazy_data_gen: + # Lazy data generation - so make constants files + constMode = ts.ConstMode.INPUTS + elif self.args.dump_consts: constMode = ts.ConstMode.EMBED_DUMP self.ser = ts.TosaSerializer(fullPath, constMode) def getSerializer(self): return self.ser - def serialize(self, testName): - with open( - os.path.join(self.basePath, self.testPath, "{}.tosa".format(testName)), "wb" - ) as fd: + def serialize(self, testName, metaData=None): + path = Path(self.basePath) / self.testPath + + # Write out TOSA flatbuffer binary + path_fb = path / f"{testName}.tosa" + with path_fb.open("wb") as fd: fd.write(self.ser.serialize()) - with open(os.path.join(self.basePath, self.testPath, "desc.json"), "w") as fd: - fd.write(self.ser.writeJson("{}.tosa".format(testName))) + # Get JSON descriptor from serializer + desc = json.loads(self.ser.writeJson(f"{testName}.tosa")) + + if metaData: + # Add extra meta data to desc.json + desc["meta"] = metaData + + # Validate desc.json before we output it + self.descSchemaValidator.validate_config(desc) + + if metaData: + if self.args.lazy_data_gen and "data_gen" in metaData: + # Output datagen meta data as CPP data + path_md = path / f"{testName}_meta_data_gen.cpp" + with path_md.open("w") as fd: + fd.write(TOSA_AUTOGENERATED_HEADER) + fd.write("// Test meta data for data generation setup\n\n") + fd.write(f'const char* json_tdg_config_{path.stem} = R"(') + json.dump(metaData["data_gen"], fd) + fd.write(')";\n\n') + if "compliance" in metaData: + # Output datagen meta data as CPP data + path_md = path / f"{testName}_meta_compliance.cpp" + with path_md.open("w") as fd: + fd.write(TOSA_AUTOGENERATED_HEADER) + fd.write("// Test meta data for compliance validation\n\n") + fd.write(f'const char* json_tvf_config_{path.stem} = R"(') + json.dump(metaData["compliance"], fd) + fd.write(')";\n\n') + + # Write desc.json + path_desc = path / "desc.json" + with path_desc.open("w") as fd: + json.dump(desc, fd, indent=1) def resetRNG(self, seed=None): if seed is None: seed = self.random_seed + 1 self.rng = np.random.default_rng(seed) - def getRandTensor(self, shape, dtype): - if dtype == DType.BOOL: - return np.bool_(self.rng.choice(a=[False, True], size=shape)) - # TOSA specific INT4 weight range from -7 to 7 + def getDTypeRange(self, dtype, high_inclusive=False): + # Returns dtype value range boundaries (low, high) + # The high boundary is excluded in the range + # unless high_inclusive is True + + if dtype in (DType.FP32, DType.FP16, DType.BF16): + return (self.random_fp_low, self.random_fp_high) + elif dtype == DType.BOOL: + rng = (0, 2) + elif dtype == DType.UINT8: + rng = (0, 256) + elif dtype == DType.UINT16: + rng = (0, 65536) elif dtype == DType.INT4: - return np.int32(self.rng.integers(low=-7, high=8, size=shape)) + # TOSA specific INT4 weight range from -7 to 7 + rng = (-7, 8) elif dtype == DType.INT8: - return np.int32(self.rng.integers(low=-128, high=128, size=shape)) - elif dtype == DType.UINT8: - return np.int32(self.rng.integers(low=0, high=256, size=shape)) + rng = (-128, 128) elif dtype == DType.INT16: - return np.int32(self.rng.integers(low=-32768, high=32768, size=shape)) - elif dtype == DType.UINT16: - return np.int32(self.rng.integers(low=0, high=65536, size=shape)) - elif ( - dtype == DType.INT32 or dtype == DType.SHAPE - ): # restricting too large value for SHAPE - return np.int32( - self.rng.integers(low=-(1 << 31), high=(1 << 31), size=shape) - ) + rng = (-32768, 32768) + elif dtype in (DType.INT32, DType.SHAPE): + # restricting too large value for SHAPE + rng = (-(1 << 31), (1 << 31)) elif dtype == DType.INT48: - return np.int64( - self.rng.integers(low=-(1 << 47), high=(1 << 47), size=shape) - ) - elif dtype == DType.FP16: - return np.float16( - self.rng.uniform( - low=self.random_fp_low, high=self.random_fp_high, size=shape - ) - ) - elif dtype == DType.BF16: - f32_tensor = np.float32( - self.rng.uniform( - low=self.random_fp_low, high=self.random_fp_high, size=shape - ) - ) - # Floor the last 16 bits of each f32 value - return np.float32(vect_f32_to_bf16(f32_tensor)) - elif dtype == DType.FP32: - return np.float32( - self.rng.uniform( - low=self.random_fp_low, high=self.random_fp_high, size=shape - ) - ) + rng = (-(1 << 47), (1 << 47)) + else: + raise Exception("Unknown dtype: {}".format(dtype)) + + if not high_inclusive: + # Exclusive high: low <= range < high + return rng else: - raise Exception("Unrecognized Dtype: {}".format(dtype)) + # Inclusive range: low <= range <= high + return (rng[0], rng[1] - 1) + + def getRandTensor(self, shape, dtype): + low, high = self.getDTypeRange(dtype) + + if dtype == DType.BOOL: + return np.bool_(self.rng.choice(a=[False, True], size=shape)) + elif dtype == DType.INT48: + return np.int64(self.rng.integers(low=low, high=high, size=shape)) + elif dtype in (DType.FP16, DType.BF16, DType.FP32): + f_tensor = self.rng.uniform(low=low, high=high, size=shape) + + if dtype == DType.FP16: + return np.float16(f_tensor) + else: + f32_tensor = np.float32(f_tensor) + if dtype == DType.BF16: + # Floor the last 16 bits of each f32 value + return np.float32(gtu.vect_f32_to_bf16(f32_tensor)) + else: + return f32_tensor + else: + # All other integer types + return np.int32(self.rng.integers(low=low, high=high, size=shape)) def buildPlaceholderTensors(self, shape_list, dtype_list): placeholders = [] assert len(shape_list) == len(dtype_list) + arr = None for idx, shape in enumerate(shape_list): - arr = self.getRandTensor(shape, dtype_list[idx]) + if not self.args.lazy_data_gen: + arr = self.getRandTensor(shape, dtype_list[idx]) placeholders.append(self.ser.addPlaceholder(shape, dtype_list[idx], arr)) return placeholders @@ -137,8 +197,10 @@ class TosaTestGen: assert len(shape_list) == len(dtype_list) + arr = None for idx, shape in enumerate(shape_list): - arr = self.getRandTensor(shape, dtype_list[idx]) + if not self.args.lazy_data_gen: + arr = self.getRandTensor(shape, dtype_list[idx]) consts.append(self.ser.addConst(shape, dtype_list[idx], arr)) return consts @@ -161,38 +223,20 @@ class TosaTestGen: return np.int32(self.rng.integers(low=low, high=high, size=1))[0] def getRandNumberDType(self, dtype): + low, high = self.getDTypeRange(dtype) + if dtype == DType.FP32: - return np.float32( - self.rng.uniform(low=self.random_fp_low, high=self.random_fp_high) - ) + return np.float32(self.rng.uniform(low=low, high=high)) elif dtype == DType.FP16: - return np.float16( - self.rng.uniform(low=self.random_fp_low, high=self.random_fp_high) - ) + return np.float16(self.rng.uniform(low=low, high=high)) elif dtype == DType.BF16: - rand_f32 = np.float32( - self.rng.uniform(low=self.random_fp_low, high=self.random_fp_high) - ) - return vect_f32_to_bf16(rand_f32) + rand_f32 = np.float32(self.rng.uniform(low=low, high=high)) + return gtu.vect_f32_to_bf16(rand_f32) elif dtype == DType.BOOL: return self.rng.choice([False, True]) - # TOSA specific INT4 weight range from -7 to 7 - elif dtype == DType.INT4: - low, high = (-7, 8) - elif dtype == DType.INT8: - low, high = (-128, 128) - elif dtype == DType.INT16: - low, high = (-32768, 32768) - elif ( - dtype == DType.INT32 or dtype == DType.SHAPE - ): # restricting too large value for SHAPE - low, high = (-(1 << 31), (1 << 31)) elif dtype == DType.INT48: - low, high = (-(1 << 47), (1 << 47)) # Special size return np.int64(self.rng.integers(low, high, size=1))[0] - else: - raise Exception("Unknown dtype: {}".format(dtype)) return np.int32(self.rng.integers(low, high, size=1))[0] @@ -212,8 +256,8 @@ class TosaTestGen: # Limit types to the first 2 as the 3rd is the accumulator return "x".join(strs[:2]) else: - if dtype in DTYPE_ATTRIBUTES: - return DTYPE_ATTRIBUTES[dtype]["str"] + if dtype in gtu.DTYPE_ATTRIBUTES: + return gtu.DTYPE_ATTRIBUTES[dtype]["str"] else: raise Exception( "Unknown dtype, cannot convert to string: {}".format(dtype) @@ -221,8 +265,8 @@ class TosaTestGen: def typeWidth(self, dtype): """Get the datatype width for data types""" - if dtype in DTYPE_ATTRIBUTES: - return DTYPE_ATTRIBUTES[dtype]["width"] + if dtype in gtu.DTYPE_ATTRIBUTES: + return gtu.DTYPE_ATTRIBUTES[dtype]["width"] else: raise Exception(f"Unknown dtype, cannot determine width: {dtype}") @@ -237,11 +281,44 @@ class TosaTestGen: low=self.args.tensor_shape_range[0], high=self.args.tensor_shape_range[1] ) - # Argument generators - # Returns a list of tuples (stringDescriptor, [build_fcn_arg_list]) - # Where the string descriptor is used to generate the test name and - # The build_fcn_arg_list is expanded and passed to the operator test - # build function + def tensorComplianceMetaData(self, op, argsDict, outputTensor, errorName): + if errorName: + # No compliance for error tests + return None + # Create compliance meta data for expected output tensor + compliance_tens = {"mode": None} + if argsDict["dg_type"] == gtu.DataGenType.DOT_PRODUCT: + mode = gtu.ComplianceMode.DOT_PRODUCT + compliance_tens["dot_product_info"] = { + "s": argsDict["s"], + "ks": argsDict["ks"], + "data_type": gtu.DTYPE_ATTRIBUTES[outputTensor.dtype]["json"], + } + elif argsDict["dg_type"] == gtu.DataGenType.OP_SPECIAL: + mode = gtu.ComplianceMode.FP_SPECIAL + elif "compliance" in op and "ulp" in op["compliance"]: + mode = gtu.ComplianceMode.ULP + compliance_tens["ulp_info"] = {"ulp": op["compliance"]["ulp"]} + elif op["op"] == Op.REDUCE_PRODUCT: + mode = gtu.ComplianceMode.REDUCE_PRODUCT + else: + mode = gtu.ComplianceMode.EXACT + compliance_tens["mode"] = gtu.ComplianceMode(mode).name + + return compliance_tens + + # Build Op functions + # Create the output tensor (calling OutputShaper as needed) + # Do final tweaks to attributes (if necessary for errorIf) + # Add Op into graph + # Return resulting tensor information or BuildInfo + + class BuildInfo: + """Enhanced build information containing result tensor and associated compliance dict.""" + + def __init__(self, resultTensor, complianceDict): + self.resultTensor = resultTensor + self.complianceDict = complianceDict def build_unary(self, op, a, validator_fcns=None, error_name=None, qinfo=None): result_tens = OutputShaper.unaryOp(self.ser, self.rng, a, error_name) @@ -975,15 +1052,16 @@ class TosaTestGen: return result_tens def build_matmul( - self, op, a, b, accum_dtype, validator_fcns=None, error_name=None, qinfo=None + self, op, a, b, args_dict, validator_fcns=None, error_name=None, qinfo=None ): - result_tens = OutputShaper.matmulOp( + accum_dtype = args_dict["acc_type"] + result_tensor = OutputShaper.matmulOp( self.ser, self.rng, a, b, accum_dtype, error_name ) # Invalidate Input/Output list for error if checks. input_list = [a.name, b.name] - output_list = [result_tens.name] + output_list = [result_tensor.name] pCount, cCount = op["operands"] num_operands = pCount + cCount input_list, output_list = TosaErrorIfArgGen.eiInvalidateInputOutputList( @@ -999,10 +1077,10 @@ class TosaTestGen: input_dtype=a.dtype, input2_shape=b.shape, input2_dtype=b.dtype, - output_shape=result_tens.shape, - output_dtype=result_tens.dtype, + output_shape=result_tensor.shape, + output_dtype=result_tensor.dtype, qinfo=qinfo, - result_tensors=[result_tens], + result_tensors=[result_tensor], input_list=input_list, output_list=output_list, num_operands=num_operands, @@ -1014,7 +1092,12 @@ class TosaTestGen: attr.MatMulAttribute(qinfo[0], qinfo[1]) self.ser.addOperator(op["op"], input_list, output_list, attr) - return result_tens + + compliance = self.tensorComplianceMetaData( + op, args_dict, result_tensor, error_name + ) + + return TosaTestGen.BuildInfo(result_tensor, compliance) def build_reduce(self, op, a, axis, validator_fcns, error_name=None): result_tens = OutputShaper.reduceOp(self.ser, self.rng, a, axis, error_name) @@ -1895,7 +1978,7 @@ class TosaTestGen: def _get_condition_tensor(self, op, cond, error_name): if error_name == ErrorIf.CondIfCondNotMatchingBool: - cond_type = get_wrong_output_type(op, self.rng, DType.BOOL) + cond_type = gtu.get_wrong_output_type(op, self.rng, DType.BOOL) else: cond_type = DType.BOOL if error_name == ErrorIf.CondIfCondShapeNotSizeOne: @@ -2357,7 +2440,7 @@ class TosaTestGen: # Initialize a new random number generator self.rng = np.random.default_rng(self.random_seed) - build_fcn, tgen_fcn, tvgen_fcn, agen_fcn = op["build_fcn"] + _, tgen_fcn, _, agen_fcn = op["build_fcn"] # Test list consists of a tuple of: # (opName, testNameStr, dtype, shapeList, argumentsList) @@ -2461,7 +2544,7 @@ class TosaTestGen: # Create a serializer self.createSerializer(opName, testStr) - build_fcn, tgen_fcn, tvgen_fcn, agen_fcn = op["build_fcn"] + build_fcn, _, tvgen_fcn, _ = op["build_fcn"] if "error_if_validators" in op: error_if_validators = op["error_if_validators"] else: @@ -2495,24 +2578,37 @@ class TosaTestGen: qgen = None # Build the random tensor operands and the test - tens = [] if qgen is not None: qinfo = qgen(self, op, dtype_or_dtypeList, error_name) else: qinfo = None - tens = tvgen_fcn(self, op, dtypeList, shapeList, testArgs, error_name) + # Extra meta data for the desc.json + tensMeta = {} + + # Check we are using the new testArgs interface with an argsDict dictionary + if len(testArgs) == 1 and isinstance(testArgs[0], dict): + argsDict = testArgs[0] + assert "dg_type" in argsDict + tvgInfo = tvgen_fcn( + self, opName, dtypeList, shapeList, argsDict, error_name + ) + if tvgInfo.dataGenDict: + tensMeta["data_gen"] = tvgInfo.dataGenDict + tens = tvgInfo.tensorList + else: + tens = tvgen_fcn(self, op, dtypeList, shapeList, testArgs, error_name) try: if error_if_validators is None: if qinfo is not None: - resultName = build_fcn(self, op, *tens, *testArgs, qinfo) + result = build_fcn(self, op, *tens, *testArgs, qinfo) else: - resultName = build_fcn(self, op, *tens, *testArgs) + result = build_fcn(self, op, *tens, *testArgs) else: if qinfo is not None: - resultName = build_fcn( + result = build_fcn( self, op, *tens, @@ -2522,7 +2618,7 @@ class TosaTestGen: qinfo=qinfo, ) else: - resultName = build_fcn( + result = build_fcn( self, op, *tens, @@ -2534,9 +2630,16 @@ class TosaTestGen: print(f"build_fcn: {build_fcn}\nTensors: {tens}\nArgs: {testArgs}\n") raise e - if resultName: + if result: # The test is valid, serialize it - self.serialize("test") + if isinstance(result, TosaTestGen.BuildInfo) and result.complianceDict: + # Add the compliance meta data + # NOTE: This currently expects only one result output + tensMeta["compliance"] = { + "version": "0.1", + "tensors": {result.resultTensor.name: result.complianceDict}, + } + self.serialize("test", tensMeta) else: # The test is not valid print(f"Invalid ERROR_IF test created: {opName} {testStr}") @@ -2865,7 +2968,7 @@ class TosaTestGen: "build_fcn": ( build_matmul, TosaTensorGen.tgMatmul, - TosaTensorValuesGen.tvgDefault, + TosaTensorValuesGen.tvgLazyGenDefault, TosaArgGen.agMatMul, ), "qgen": TosaQuantGen.qgMatmul, @@ -2878,6 +2981,10 @@ class TosaTestGen: TosaErrorValidator.evWrongInputList, TosaErrorValidator.evWrongOutputList, ), + "data_gen": { + "fp": (gtu.DataGenType.DOT_PRODUCT,), + "int": (gtu.DataGenType.PSEUDO_RANDOM,), + }, }, "max_pool2d": { "op": Op.MAX_POOL2D, @@ -4446,7 +4553,7 @@ class OutputShaper: excludes = [DType.FP16, DType.FP32] else: excludes = [out_dtype] - wrong_dtypes = list(usableDTypes(excludes=excludes)) + wrong_dtypes = list(gtu.usableDTypes(excludes=excludes)) out_dtype = rng.choice(wrong_dtypes) return ser.addOutput(ofm_shape, out_dtype) @@ -4508,7 +4615,7 @@ class OutputShaper: excludes = [DType.FP16, DType.FP32] else: excludes = [out_dtype] - wrong_dtypes = list(usableDTypes(excludes=excludes)) + wrong_dtypes = list(gtu.usableDTypes(excludes=excludes)) out_dtype = rng.choice(wrong_dtypes) return ser.addOutput(ofm_shape, out_dtype) @@ -4559,7 +4666,7 @@ class OutputShaper: excludes = [DType.FP16, DType.FP32] else: excludes = [out_dtype] - wrong_dtypes = list(usableDTypes(excludes=excludes)) + wrong_dtypes = list(gtu.usableDTypes(excludes=excludes)) out_dtype = rng.choice(wrong_dtypes) return ser.addOutput(ofm_shape, out_dtype) @@ -4711,7 +4818,7 @@ class OutputShaper: bad_dim = rng.choice(range(len(output_shape))) output_shape[bad_dim] -= rng.choice([1, 2]) elif error_name == ErrorIf.RankMismatch: - output_shape = get_rank_mismatch_shape(rng, output_shape) + output_shape = gtu.get_rank_mismatch_shape(rng, output_shape) if error_name == ErrorIf.WrongOutputType: all_dtypes = [ @@ -4806,7 +4913,7 @@ class OutputShaper: elif error_name == ErrorIf.InputSizeStartLengthMismatch: output_shape = input.shape.copy() elif error_name == ErrorIf.RankMismatch: - output_shape = get_rank_mismatch_shape(rng, output_shape) + output_shape = gtu.get_rank_mismatch_shape(rng, output_shape) return ser.addOutput(output_shape, outputDType) @@ -4820,7 +4927,7 @@ class OutputShaper: output_shape[i] = a.shape[i] * multiples[i] if error_name == ErrorIf.RankMismatch: - output_shape = get_rank_mismatch_shape(rng, output_shape) + output_shape = gtu.get_rank_mismatch_shape(rng, output_shape) if error_name == ErrorIf.WrongOutputType: all_dtypes = [ @@ -4853,7 +4960,7 @@ class OutputShaper: for i in range(len(output_shape)): output_shape[i] += rng.integers(1, 10) elif error_name == ErrorIf.RankMismatch: - output_shape = get_rank_mismatch_shape(rng, output_shape) + output_shape = gtu.get_rank_mismatch_shape(rng, output_shape) if error_name == ErrorIf.WrongOutputType: all_dtypes = [ @@ -4980,21 +5087,21 @@ class OutputShaper: oh = max(oh, 1) ow = max(ow, 1) if error_name != ErrorIf.MaxDimExceeded: - oh = min(oh, MAX_RESIZE_DIMENSION - 1) - ow = min(ow, MAX_RESIZE_DIMENSION - 1) + oh = min(oh, gtu.MAX_RESIZE_DIMENSION - 1) + ow = min(ow, gtu.MAX_RESIZE_DIMENSION - 1) if error_name == ErrorIf.ResizeOutputShapeMismatch: choices = [1, 2, 3] change = rng.choice(choices) # increment in multiples of scale_y/x_d so we don't hit non-integer error case if change in [1, 3]: - if oh + scale_y_d >= MAX_RESIZE_DIMENSION: + if oh + scale_y_d >= gtu.MAX_RESIZE_DIMENSION: oh -= scale_y_d assert oh > 0 # Should have been caught in agResize else: oh += scale_y_d if change in [2, 3]: - if ow + scale_x_d >= MAX_RESIZE_DIMENSION: + if ow + scale_x_d >= gtu.MAX_RESIZE_DIMENSION: ow -= scale_x_d assert ow > 0 # Should have been caught in agResize else: @@ -5051,7 +5158,7 @@ class OutputShaper: excludes = [DType.FP16, DType.FP32] else: excludes = [out_dtype] - wrong_dtypes = list(usableDTypes(excludes=excludes)) + wrong_dtypes = list(gtu.usableDTypes(excludes=excludes)) out_dtype = rng.choice(wrong_dtypes) return ser.addOutput(output_shape, out_dtype) @@ -5075,7 +5182,7 @@ class OutputShaper: if error_name == ErrorIf.WrongOutputType: excludes = [DType.FP32] - wrong_dtypes = list(usableDTypes(excludes=excludes)) + wrong_dtypes = list(gtu.usableDTypes(excludes=excludes)) output_dtype = rng.choice(wrong_dtypes) elif error_name == ErrorIf.BatchMismatch: output_shape[0] += rng.integers(1, 10) @@ -5100,7 +5207,7 @@ class OutputShaper: output_dtype = value.dtype if error_name == ErrorIf.WrongOutputType: excludes = [DType.FP32] - wrong_dtypes = list(usableDTypes(excludes=excludes)) + wrong_dtypes = list(gtu.usableDTypes(excludes=excludes)) output_dtype = rng.choice(wrong_dtypes) elif error_name == ErrorIf.BatchMismatch: output_shape[0] += rng.integers(1, 10) diff --git a/verif/generator/tosa_utils.py b/verif/generator/tosa_utils.py index 3cd0370..75a0df5 100644 --- a/verif/generator/tosa_utils.py +++ b/verif/generator/tosa_utils.py @@ -1,7 +1,8 @@ -# Copyright (c) 2021-2022, ARM Limited. +# Copyright (c) 2021-2023, ARM Limited. # SPDX-License-Identifier: Apache-2.0 import struct import sys +from enum import IntEnum import numpy as np from tosa.DType import DType @@ -9,22 +10,54 @@ from tosa.DType import DType # Maximum dimension size for output and inputs for RESIZE MAX_RESIZE_DIMENSION = 16384 +# Data type information dictionary +# - str: filename abbreviation +# - width: number of bytes needed for type +# - json: JSON type string DTYPE_ATTRIBUTES = { - DType.BOOL: {"str": "b", "width": 1}, - DType.INT4: {"str": "i4", "width": 4}, - DType.INT8: {"str": "i8", "width": 8}, - DType.UINT8: {"str": "u8", "width": 8}, - DType.INT16: {"str": "i16", "width": 16}, - DType.UINT16: {"str": "u16", "width": 16}, - DType.INT32: {"str": "i32", "width": 32}, - DType.INT48: {"str": "i48", "width": 48}, - DType.SHAPE: {"str": "i64", "width": 64}, - DType.FP16: {"str": "f16", "width": 16}, - DType.BF16: {"str": "bf16", "width": 16}, - DType.FP32: {"str": "f32", "width": 32}, + DType.BOOL: {"str": "b", "width": 1, "json": "BOOL"}, + DType.INT4: {"str": "i4", "width": 4, "json": "INT4"}, + DType.INT8: {"str": "i8", "width": 8, "json": "INT8"}, + DType.UINT8: {"str": "u8", "width": 8, "json": "UINT8"}, + DType.INT16: {"str": "i16", "width": 16, "json": "INT16"}, + DType.UINT16: {"str": "u16", "width": 16, "json": "UINT16"}, + DType.INT32: {"str": "i32", "width": 32, "json": "INT32"}, + DType.INT48: {"str": "i48", "width": 48, "json": "INT48"}, + DType.SHAPE: {"str": "s", "width": 64, "json": "SHAPE"}, + DType.FP16: {"str": "f16", "width": 16, "json": "FP16"}, + DType.BF16: {"str": "bf16", "width": 16, "json": "BF16"}, + DType.FP32: {"str": "f32", "width": 32, "json": "FP32"}, } +class ComplianceMode(IntEnum): + """Compliance mode types.""" + + EXACT = 0 + DOT_PRODUCT = 1 + ULP = 2 + FP_SPECIAL = 3 + REDUCE_PRODUCT = 4 + + +class DataGenType(IntEnum): + """Data generator types.""" + + PSEUDO_RANDOM = 0 + DOT_PRODUCT = 1 + OP_BOUNDARY = 2 + OP_FULLSET = 3 + OP_SPECIAL = 4 + + +# Additional (optional) data for dot product data generator +DG_DOT_PRODUCT_OPTIONAL_INFO = ("acc_type", "kernel", "axis") + + +def dtypeIsFloat(dtype): + return dtype in (DType.FP16, DType.BF16, DType.FP32) + + def valueToName(item, value): """Get the name of an attribute with the given value. diff --git a/verif/generator/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py index 4d12bb0..0d532c0 100644 --- a/verif/generator/tosa_verif_build_tests.py +++ b/verif/generator/tosa_verif_build_tests.py @@ -51,6 +51,10 @@ def parseArgs(argv): argv = new_argv parser = argparse.ArgumentParser() + + ops_group = parser.add_argument_group("operator options") + tens_group = parser.add_argument_group("tensor options") + parser.add_argument( "-o", dest="output_dir", type=str, default="vtest", help="Test output directory" ) @@ -75,8 +79,15 @@ def parseArgs(argv): "-v", "--verbose", dest="verbose", action="count", help="Verbose operation" ) - # Constraints on tests parser.add_argument( + "--lazy-data-generation", + dest="lazy_data_gen", + action="store_true", + help="Tensor data generation is delayed til test running", + ) + + # Constraints on tests + tens_group.add_argument( "--tensor-dim-range", dest="tensor_shape_range", default="1,64", @@ -84,7 +95,7 @@ def parseArgs(argv): help="Min,Max range of tensor shapes", ) - parser.add_argument( + tens_group.add_argument( OPTION_FP_VALUES_RANGE, dest="tensor_fp_value_range", default="0.0,1.0", @@ -92,7 +103,7 @@ def parseArgs(argv): help="Min,Max range of floating point tensor values", ) - parser.add_argument( + ops_group.add_argument( "--max-batch-size", dest="max_batch_size", default=1, @@ -100,7 +111,7 @@ def parseArgs(argv): help="Maximum batch size for NHWC tests", ) - parser.add_argument( + ops_group.add_argument( "--max-conv-padding", dest="max_conv_padding", default=1, @@ -108,7 +119,7 @@ def parseArgs(argv): help="Maximum padding for Conv tests", ) - parser.add_argument( + ops_group.add_argument( "--max-conv-dilation", dest="max_conv_dilation", default=2, @@ -116,7 +127,7 @@ def parseArgs(argv): help="Maximum dilation for Conv tests", ) - parser.add_argument( + ops_group.add_argument( "--max-conv-stride", dest="max_conv_stride", default=2, @@ -124,7 +135,7 @@ def parseArgs(argv): help="Maximum stride for Conv tests", ) - parser.add_argument( + ops_group.add_argument( "--max-pooling-padding", dest="max_pooling_padding", default=1, @@ -132,7 +143,7 @@ def parseArgs(argv): help="Maximum padding for pooling tests", ) - parser.add_argument( + ops_group.add_argument( "--max-pooling-stride", dest="max_pooling_stride", default=2, @@ -140,7 +151,7 @@ def parseArgs(argv): help="Maximum stride for pooling tests", ) - parser.add_argument( + ops_group.add_argument( "--max-pooling-kernel", dest="max_pooling_kernel", default=3, @@ -148,7 +159,7 @@ def parseArgs(argv): help="Maximum kernel for pooling tests", ) - parser.add_argument( + ops_group.add_argument( "--num-rand-permutations", dest="num_rand_permutations", default=6, @@ -156,7 +167,7 @@ def parseArgs(argv): help="Number of random permutations for a given shape/rank for randomly-sampled parameter spaces", ) - parser.add_argument( + ops_group.add_argument( "--max-resize-output-dim", dest="max_resize_output_dim", default=1000, @@ -165,7 +176,7 @@ def parseArgs(argv): ) # Targeting a specific shape/rank/dtype - parser.add_argument( + tens_group.add_argument( "--target-shape", dest="target_shapes", action="append", @@ -174,7 +185,7 @@ def parseArgs(argv): help="Create tests with a particular input tensor shape, e.g., 1,4,4,8 (may be repeated for tests that require multiple input shapes)", ) - parser.add_argument( + tens_group.add_argument( "--target-rank", dest="target_ranks", action="append", @@ -184,7 +195,7 @@ def parseArgs(argv): ) # Used for parsing a comma-separated list of integers in a string - parser.add_argument( + tens_group.add_argument( "--target-dtype", dest="target_dtypes", action="append", @@ -193,7 +204,7 @@ def parseArgs(argv): help=f"Create test with a particular DType: [{', '.join([d.lower() for d in DTypeNames[1:]])}] (may be repeated)", ) - parser.add_argument( + ops_group.add_argument( "--num-const-inputs-concat", dest="num_const_inputs_concat", default=0, @@ -211,14 +222,14 @@ def parseArgs(argv): help="type of tests produced, positive, negative, or both", ) - parser.add_argument( + ops_group.add_argument( "--allow-pooling-and-conv-oversizes", dest="oversize", action="store_true", help="allow oversize padding, stride and kernel tests", ) - parser.add_argument( + ops_group.add_argument( "--zero-point", dest="zeropoint", default=None, @@ -233,11 +244,11 @@ def parseArgs(argv): help="output const tensors as numpy files for inspection", ) - parser.add_argument( + ops_group.add_argument( "--level-8k-sizes", dest="level8k", action="store_true", - help="create level 8k size tests (RESIZE)", + help="create level 8k size tests", ) args = parser.parse_args(argv) |