diff options
Diffstat (limited to 'verif/generator')
| -rw-r--r-- | verif/generator/tosa_arg_gen.py | 142 | ||||
| -rw-r--r-- | verif/generator/tosa_test_gen.py | 74 | ||||
| -rw-r--r-- | verif/generator/tosa_verif_build_tests.py | 7 |
3 files changed, 150 insertions, 73 deletions
diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py index 41b0936..a71419f 100644 --- a/verif/generator/tosa_arg_gen.py +++ b/verif/generator/tosa_arg_gen.py @@ -732,6 +732,12 @@ class TosaTensorValuesGen: tens_ser_list = [] + # Retrieve any fixed data tensors + fixed_data_tensors = argsDict.get("fixed_data", [None] * len(shapeList)) + assert len(fixed_data_tensors) == len( + shapeList + ), "Fixed data list must match shapes list" + if ( error_name is not None or not gtu.dtypeIsSupportedByCompliance(dtypeList[0]) @@ -759,15 +765,15 @@ class TosaTensorValuesGen: data_range = (data_range[0], data_range[1] + 1) # Ignore lazy data gen option and create data array using any range limits - if "fixed_data" in argsDict and argsDict["fixed_data"][idx] is not None: + if fixed_data_tensors[idx] is not None: if dtype == DType.SHAPE: - arr = np.int64(argsDict["fixed_data"][idx]) + arr = np.int64(fixed_data_tensors[idx]) elif dtype == DType.INT8: - arr = np.int8(argsDict["fixed_data"][idx]) + arr = np.int8(fixed_data_tensors[idx]) elif dtype == DType.INT16: - arr = np.int16(argsDict["fixed_data"][idx]) + arr = np.int16(fixed_data_tensors[idx]) elif dtype == DType.INT32: - arr = np.int32(argsDict["fixed_data"][idx]) + arr = np.int32(fixed_data_tensors[idx]) else: assert False, "Unsupported fixed_data type" else: @@ -782,7 +788,6 @@ class TosaTensorValuesGen: return TosaTensorValuesGen.TVGInfo(tens_ser_list, None) # Create data generator meta-data - dg_type = argsDict["dg_type"] tens_data = { "version": "0.1", "tensors": {}, @@ -795,13 +800,12 @@ class TosaTensorValuesGen: for idx, shape in enumerate(shapeList): tens_meta = {} - if "fixed_data" in argsDict and argsDict["fixed_data"][idx] is not None: - tens_meta["generator"] = gtu.DataGenType( - gtu.DataGenType.FIXED_DATA - ).name + if fixed_data_tensors[idx] is not None: + dg_type = gtu.DataGenType.FIXED_DATA else: - tens_meta["generator"] = gtu.DataGenType(dg_type).name + dg_type = argsDict["dg_type"] + tens_meta["generator"] = gtu.DataGenType(dg_type).name tens_meta["data_type"] = gtu.DTYPE_ATTRIBUTES[dtypeList[idx]]["json"] tens_meta["shape"] = [int(i) for i in shape] tens_meta["input_pos"] = idx @@ -822,29 +826,28 @@ class TosaTensorValuesGen: else: tens_meta["input_type"] = "CONSTANT" - if dg_type == gtu.DataGenType.PSEUDO_RANDOM: + if dg_type == gtu.DataGenType.FIXED_DATA: info = {} - if ( - tens_meta["generator"] - == gtu.DataGenType(gtu.DataGenType.FIXED_DATA).name - ): - info["data"] = [int(i) for i in argsDict["fixed_data"][idx]] - tens_meta["fixed_data_info"] = info - else: - info["rng_seed"] = rng.seed - - data_range = None - if "data_range_list" in argsDict: - data_range = argsDict["data_range_list"][idx]["range"] - if "round" in argsDict["data_range_list"][idx]: - info["round"] = argsDict["data_range_list"][idx]["round"] - elif "data_range" in argsDict: - data_range = argsDict["data_range"] - - if data_range is None: - data_range = rng.dTypeRange(dtypeList[idx], high_inclusive=True) - info["range"] = [str(v) for v in data_range] - tens_meta["pseudo_random_info"] = info + info["data"] = [int(i) for i in fixed_data_tensors[idx]] + tens_meta["fixed_data_info"] = info + + elif dg_type == gtu.DataGenType.PSEUDO_RANDOM: + info = {} + info["rng_seed"] = rng.seed + + data_range = None + if "data_range_list" in argsDict: + data_range = argsDict["data_range_list"][idx]["range"] + if "round" in argsDict["data_range_list"][idx]: + info["round"] = argsDict["data_range_list"][idx]["round"] + elif "data_range" in argsDict: + data_range = argsDict["data_range"] + + if data_range is None: + data_range = rng.dTypeRange(dtypeList[idx], high_inclusive=True) + info["range"] = [str(v) for v in data_range] + tens_meta["pseudo_random_info"] = info + elif dg_type == gtu.DataGenType.DOT_PRODUCT: info = {} info["s"] = argsDict["s"] @@ -859,14 +862,17 @@ class TosaTensorValuesGen: if "axis" in argsDict: info["axis"] = int(argsDict["axis"]) tens_meta["dot_product_info"] = info + elif dg_type == gtu.DataGenType.FULL_RANGE: info = {} info["start_val"] = int( rng.randInt(0, gtu.DTYPE_ATTRIBUTES[dtypeList[idx]]["fullset"]) ) tens_meta["full_range_info"] = info + elif dg_type == gtu.DataGenType.FP_SPECIAL: tens_meta["fp_special_info"] = fp_special_info + else: # TODO - other data gen type assert False, "TODO: support other data gen types" @@ -1506,6 +1512,21 @@ class TosaTensorValuesGen: ) @staticmethod + def tvgTable(testGen, rng, opName, dtypeList, shapeList, argsDict, error_name=None): + # Use supported type for table data on ERROR_IF + dtypeList[1] = ( + dtypeList[0] if error_name != ErrorIf.WrongInputType else DType.INT8 + ) + + table_values = argsDict["table"] + shapeList[1] = [len(table_values)] + argsDict["fixed_data"] = [None, table_values] + + return TosaTensorValuesGen.tvgLazyGenDefault( + testGen, rng, opName, dtypeList, shapeList, argsDict, error_name + ) + + @staticmethod def tvgResize( testGen, rng, opName, dtypeList, shapeList, argsDict, error_name=None ): @@ -1875,14 +1896,35 @@ class TosaArgGen: dataGenTypesList = testGen.TOSA_OP_LIST[opName]["data_gen"].get( dtype, (gtu.DataGenType.PSEUDO_RANDOM,) ) - else: # Error test or No data generator types listed - assume random dataGenTypesList = (gtu.DataGenType.PSEUDO_RANDOM,) + def check_min_size(opName, shape, min_size, reason): + # Check tensor size meets minimum requirements + tensor_size = gtu.product(shape) + if tensor_size < min_size: + shape_info = " ({})".format(shape) + logger.info( + f"Skipping {opName}{shape_info} as tensor data size too small for {reason} values {tensor_size} < {min_size}" + ) + return False + return True + + def update_data_gen(testGen, opName, dtype, dgt_remove): + # Remove special data generator to limit number of tests + assert "data_gen" in testGen.TOSA_OP_LIST[opName] + assert dtype in testGen.TOSA_OP_LIST[opName]["data_gen"] + data_gen = testGen.TOSA_OP_LIST[opName]["data_gen"].copy() + dgt_list = list(data_gen[dtype]) + dgt_list.remove(dgt_remove) + data_gen[dtype] = tuple(dgt_list) + testGen.TOSA_OP_LIST[opName]["data_gen"] = data_gen + # Expand arg list with other data generator types new_arg_list = [] for dg_type in dataGenTypesList: + only_one = False # Check for only one dg type test for arg_str, args_dict in arg_list: gen_args_dict = args_dict.copy() # Only create one test by default - no sets of tests @@ -1912,20 +1954,33 @@ class TosaArgGen: num_test_sets = testGen.TOSA_MI_DOT_PRODUCT_TEST_SETS elif dg_type == gtu.DataGenType.FULL_RANGE: - tensor_size = gtu.product(shapeList[0]) - if tensor_size < gtu.DTYPE_ATTRIBUTES[dtype]["fullset"]: - shape_info = " ({})".format(shapeList[0]) - logger.info( - f"Skipping {opName}{shape_info} as tensor data size too small for full range of values {tensor_size} < {gtu.DTYPE_ATTRIBUTES[dtype]['fullset']}" - ) + if testGen.args.no_special_tests: + continue + if not check_min_size( + opName, + shapeList[0], + gtu.DTYPE_ATTRIBUTES[dtype]["fullset"], + "full range of", + ): continue # Large enough tensor data size for full range, add full test arg_str = f"{arg_str}_full" if arg_str else "full" gen_args_dict["tags"] = args_dict.get("tags", []) + [ "non_finite_fp_data" ] + # Create only one special test per data type + only_one = True elif dg_type == gtu.DataGenType.FP_SPECIAL: + if testGen.args.no_special_tests: + continue + if not check_min_size( + opName, + shapeList[0], + testGen.TOSA_MI_FP_SPECIAL_MIN_SIZE, + "FP special", + ): + continue shapes_set = {tuple(x) for x in shapeList} if len(shapes_set) != 1: logger.info( @@ -1938,6 +1993,8 @@ class TosaArgGen: gen_args_dict["tags"] = args_dict.get("tags", []) + [ "non_finite_fp_data" ] + # Create only one special test per data type + only_one = True gen_args_dict["dg_type"] = dg_type if num_test_sets > 0: @@ -1950,6 +2007,11 @@ class TosaArgGen: # Default is a single test new_arg_list.append((arg_str, gen_args_dict)) + if only_one: + # Skip all remaining tests and remove this data generator + update_data_gen(testGen, opName, dtype, dg_type) + break + return new_arg_list @staticmethod diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py index 88dd17a..2db094a 100644 --- a/verif/generator/tosa_test_gen.py +++ b/verif/generator/tosa_test_gen.py @@ -44,6 +44,9 @@ class TosaTestGen: TOSA_MI_DOT_PRODUCT_TEST_SETS = 6 TOSA_MI_DOT_PRODUCT_MIN = 1000 + # Minimum tensor size for the FP special tests + TOSA_MI_FP_SPECIAL_MIN_SIZE = 20 + def __init__(self, args): self.args = args self.basePath = args.output_dir @@ -260,6 +263,11 @@ class TosaTestGen: # Data type is needed for all FP runs, as refmodel precise mode produces FP64 "data_type": gtu.DTYPE_ATTRIBUTES[outputTensor.dtype]["json"], } + + op_compliance = op.get("compliance", {}) + mode = None + + # Check what data generation we have done if argsDict["dg_type"] == gtu.DataGenType.DOT_PRODUCT: mode = gtu.ComplianceMode.DOT_PRODUCT compliance_tens["dot_product_info"] = { @@ -268,12 +276,10 @@ class TosaTestGen: int(argsDict["ksb"]) if "ksb" in argsDict else int(argsDict["ks"]) ), } - elif argsDict["dg_type"] == gtu.DataGenType.FP_SPECIAL: - mode = gtu.ComplianceMode.FP_SPECIAL - elif "compliance" in op and "ulp" in op["compliance"]: + elif "ulp" in op_compliance: mode = gtu.ComplianceMode.ULP compliance_tens["ulp_info"] = {"ulp": op["compliance"]["ulp"]} - elif "compliance" in op and "relative" in op["compliance"]: + elif "relative" in op_compliance: mode = gtu.ComplianceMode.RELATIVE compliance_tens["relative_info"] = { "max": argsDict["max_abs_value"], @@ -284,26 +290,30 @@ class TosaTestGen: compliance_tens["reduce_product_info"] = {"n": argsDict["n"]} elif op["op"] in (Op.EXP, Op.POW, Op.TANH, Op.SIGMOID): mode = gtu.ComplianceMode.ABS_ERROR - if "compliance" in op and "abs_error_lower_bound" in op["compliance"]: + if "abs_error_lower_bound" in op_compliance: compliance_tens["abs_error_info"] = { "lower_bound": op["compliance"]["abs_error_lower_bound"] } elif op["op"] in (Op.SIN, Op.COS): mode = gtu.ComplianceMode.ABS_ERROR - if "compliance" in op: - normal_divisor = op["compliance"].get("abs_error_normal_divisor", 1) - bound_addition = op["compliance"].get("abs_error_bound_addition", 0) - else: - normal_divisor = 1 - bound_addition = 0 + normal_divisor = op_compliance.get("abs_error_normal_divisor", 1) + bound_addition = op_compliance.get("abs_error_bound_addition", 0) compliance_tens["abs_error_info"] = { "normal_divisor": normal_divisor, "bound_as_magnitude": True, "bound_addition": bound_addition, } + elif argsDict["dg_type"] == gtu.DataGenType.FP_SPECIAL: + if gtu.ComplianceMode.DOT_PRODUCT in op["data_gen"][inputType]: + # Use special mode that only checks for matching inf/nan/zeroes + # as normal values need statistical analysis + mode = gtu.ComplianceMode.FP_SPECIAL + else: + mode = gtu.ComplianceMode.EXACT else: mode = gtu.ComplianceMode.EXACT + compliance_tens["mode"] = gtu.ComplianceMode(mode).name return compliance_tens @@ -569,16 +579,13 @@ class TosaTestGen: error_name=None, qinfo=None, ): - assert len(inputs) == 1 + assert len(inputs) == 2 a = inputs[0] - table = args_dict["table"] + table = inputs[1] result_tensor = OutputShaper.tableOp(self.ser, rng, a, error_name) - attr = ts.TosaSerializerAttribute() - attr.TableAttribute(table) - # Invalidate Input/Output list for error if checks. - input_list = [a.name] + input_list = [a.name, table.name] output_list = [result_tensor.name] pCount, cCount = op["operands"] num_operands = pCount + cCount @@ -601,7 +608,7 @@ class TosaTestGen: ): return None - self.ser.addOperator(op["op"], input_list, output_list, attr) + self.ser.addOperator(op["op"], input_list, output_list) compliance = self.tensorComplianceMetaData( op, a.dtype, args_dict, result_tensor, error_name @@ -3360,6 +3367,7 @@ class TosaTestGen: } EW_UNARY_DATAGEN = { DType.FP16: (gtu.DataGenType.PSEUDO_RANDOM, gtu.DataGenType.FULL_RANGE), + DType.FP32: (gtu.DataGenType.PSEUDO_RANDOM, gtu.DataGenType.FP_SPECIAL), } PR_FS_DATAGEN = { DType.FP16: (gtu.DataGenType.PSEUDO_RANDOM, gtu.DataGenType.FP_SPECIAL), @@ -3647,7 +3655,7 @@ class TosaTestGen: TosaErrorValidator.evWrongInputList, TosaErrorValidator.evWrongOutputList, ), - "data_gen": PSEUDO_RANDOM_DATAGEN, + "data_gen": PR_FS_DATAGEN, }, "sigmoid": { "op": Op.SIGMOID, @@ -3665,7 +3673,7 @@ class TosaTestGen: TosaErrorValidator.evWrongInputList, TosaErrorValidator.evWrongOutputList, ), - "data_gen": PSEUDO_RANDOM_DATAGEN, + "data_gen": PR_FS_DATAGEN, }, "tanh": { "op": Op.TANH, @@ -3683,7 +3691,7 @@ class TosaTestGen: TosaErrorValidator.evWrongInputList, TosaErrorValidator.evWrongOutputList, ), - "data_gen": PSEUDO_RANDOM_DATAGEN, + "data_gen": PR_FS_DATAGEN, "compliance": { "abs_error_lower_bound": 0.5, }, @@ -3704,7 +3712,7 @@ class TosaTestGen: TosaErrorValidator.evWrongInputList, TosaErrorValidator.evWrongOutputList, ), - "data_gen": PSEUDO_RANDOM_DATAGEN, + "data_gen": PR_FS_DATAGEN, "compliance": {"ulp": 5}, }, # Elementwise Binary Operators @@ -4042,11 +4050,11 @@ class TosaTestGen: # Use the automatic generation functions to create the input array # but create the table tensor in the build function, as it may be # a different type from the input - "operands": (1, 0), + "operands": (2, 0), "build_fcn": ( build_table, TosaTensorGen.tgBasic, - TosaTensorValuesGen.tvgLazyGenDefault, + TosaTensorValuesGen.tvgTable, TosaArgGen.agTable, ), "types": [DType.INT8, DType.INT16], @@ -4145,7 +4153,7 @@ class TosaTestGen: TosaErrorValidator.evWrongInputList, TosaErrorValidator.evWrongOutputList, ), - "data_gen": PSEUDO_RANDOM_DATAGEN, + "data_gen": EW_UNARY_DATAGEN, "compliance": { "abs_error_normal_divisor": 2, "abs_error_bound_addition": 1, @@ -4299,7 +4307,7 @@ class TosaTestGen: TosaErrorValidator.evWrongInputList, TosaErrorValidator.evWrongOutputList, ), - "data_gen": PSEUDO_RANDOM_DATAGEN, + "data_gen": EW_UNARY_DATAGEN, "compliance": {"abs_error_normal_divisor": 2}, }, # Elementwise Ternary operators @@ -4540,7 +4548,7 @@ class TosaTestGen: TosaErrorValidator.evWrongOutputType, TosaErrorValidator.evWrongOutputList, ), - "data_gen": PSEUDO_RANDOM_DATAGEN, + "data_gen": PR_FS_DATAGEN, }, "pad": { "op": Op.PAD, @@ -4563,7 +4571,7 @@ class TosaTestGen: TosaErrorValidator.evRankMismatch, TosaErrorValidator.evWrongRank, ), - "data_gen": PSEUDO_RANDOM_DATAGEN, + "data_gen": PR_FS_DATAGEN, }, "dim": { "op": Op.DIM, @@ -4603,7 +4611,7 @@ class TosaTestGen: TosaErrorValidator.evWrongInputList, TosaErrorValidator.evWrongOutputList, ), - "data_gen": PSEUDO_RANDOM_DATAGEN, + "data_gen": PR_FS_DATAGEN, }, "reverse": { "op": Op.REVERSE, @@ -4623,7 +4631,7 @@ class TosaTestGen: TosaErrorValidator.evWrongInputList, TosaErrorValidator.evWrongOutputList, ), - "data_gen": PSEUDO_RANDOM_DATAGEN, + "data_gen": PR_FS_DATAGEN, }, "slice": { "op": Op.SLICE, @@ -4653,7 +4661,7 @@ class TosaTestGen: TosaErrorValidator.evWrongOutputList, TosaErrorValidator.evRankMismatch, ), - "data_gen": PSEUDO_RANDOM_DATAGEN, + "data_gen": PR_FS_DATAGEN, }, "tile": { "op": Op.TILE, @@ -4674,7 +4682,7 @@ class TosaTestGen: TosaErrorValidator.evRankMismatch, TosaErrorValidator.evWrongRank, ), - "data_gen": PSEUDO_RANDOM_DATAGEN, + "data_gen": PR_FS_DATAGEN, }, "transpose": { "op": Op.TRANSPOSE, @@ -4698,7 +4706,7 @@ class TosaTestGen: TosaErrorValidator.evRankMismatch, TosaErrorValidator.evTensorSizeInputOutputMismatch, ), - "data_gen": PSEUDO_RANDOM_DATAGEN, + "data_gen": PR_FS_DATAGEN, }, # Data nodes "const": { diff --git a/verif/generator/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py index a46b061..dbd46b5 100644 --- a/verif/generator/tosa_verif_build_tests.py +++ b/verif/generator/tosa_verif_build_tests.py @@ -287,6 +287,13 @@ def parseArgs(argv): help="enables test selection, this is the selection criteria to use from the selection config", ) + filter_group.add_argument( + "--no-special-tests", + dest="no_special_tests", + action="store_true", + help="Do not produce special 'full range' or 'FP special' tests", + ) + parser.add_argument( "--list-tests", dest="list_tests", |