From dd975b842f5220b57090aacf1845f4ed669182b7 Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Wed, 28 Feb 2024 17:29:13 +0000 Subject: Fix missing/broken ERROR_IF tests Fix CONV2D WrongOutputType FP32 & Pad/Stride/DilationSmallerZero issues. Fix PAD WrongInputType. Signed-off-by: Jeremy Johnson Change-Id: I57fc57c43e63685e05bf5e3d562c3167411fd57b --- .../tosa_verif_conformance_generator.py | 2 + verif/generator/tosa_arg_gen.py | 112 ++++++++++++++++----- verif/generator/tosa_error_if.py | 33 +++--- verif/generator/tosa_test_gen.py | 32 +++++- verif/generator/tosa_test_select.py | 2 +- verif/generator/tosa_utils.py | 3 + verif/generator/tosa_verif_build_tests.py | 3 + 7 files changed, 141 insertions(+), 46 deletions(-) diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py index 5402c21..3b7c2df 100644 --- a/verif/conformance/tosa_verif_conformance_generator.py +++ b/verif/conformance/tosa_verif_conformance_generator.py @@ -82,7 +82,9 @@ def _run_sh_command(args, cwd, full_cmd): ) if args.capture_output: + stderr = rc.stderr.decode("utf-8") stdout = rc.stdout.decode("utf-8") + logger.info(f"stderr: \n{stderr}") logger.info(f"stdout: \n{stdout}") if rc.returncode != 0: diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py index ffa3683..4878708 100644 --- a/verif/generator/tosa_arg_gen.py +++ b/verif/generator/tosa_arg_gen.py @@ -1976,6 +1976,9 @@ class TosaArgGen: sparsity += 1 return sparsity + # Maximum number of error_if variants to produce + MAX_CONV_ERROR_IFS = 3 + @staticmethod def agConv(testGen, rng, opName, shapeList, dtypes, error_name=None): # Used by CONV2D, CONV3D and DEPTHWISE_CONV2D @@ -2017,17 +2020,60 @@ class TosaArgGen: k_size *= ifm_shape[-1] if not testGen.args.level8k: - # Generate comprehensive argument lists - # - except for named errors, which use specific invalid value(s) - if error_name == ErrorIf.PadSmallerZero: - p_vals = [rng.choice(range(-5, 0))] + if error_name in ( + ErrorIf.PadSmallerZero, + ErrorIf.StrideSmallerOne, + ErrorIf.DilationSmallerOne, + ): + # Use specific invalid value(s) + if error_name == ErrorIf.PadSmallerZero: + # Create negative paddings but with positive opposite paddings + neg_pad = rng.choice(range(-5, 0)) + p_vals = [neg_pad, abs(neg_pad)] + else: + p_vals = [0, 0] + if error_name == ErrorIf.StrideSmallerOne: + # Can't use stride=0, as it is used to derive output shape, as a divisor + s_vals = [rng.choice(range(-5, 0))] + else: + s_vals = [1] + if error_name == ErrorIf.DilationSmallerOne: + d_vals = [rng.choice(range(-5, 1))] + else: + d_vals = [1] + p = p_vals * k_rank + s = s_vals * k_rank + d = d_vals * k_rank + + # Fix values to produce valid error_if + for index in range(k_rank): + pad_offset = index * 2 + fixed = False + while not fixed: + partial = ( + ifm_shape[index + 1] + - 1 + + p[pad_offset] + + p[pad_offset + 1] + - (k_shape[index] - 1) * d[index] + ) + remainder = partial % s[index] + if partial <= 0: + p[pad_offset + 1] += abs(partial) + 1 + elif remainder: + # Stride will be negative for StrideSmallerOne + assert remainder < 0 + p[pad_offset + 1] += abs(remainder) + else: + fixed = True + paddings = {tuple(p)} + strides = {tuple(s)} + dilations = {tuple(d)} + logger.debug(f"agConv: error pad={p} stride={s} dilation={d}") else: + # Generate comprehensive argument lists p_vals = [x for x in range(0, testGen.args.max_conv_padding + 1)] - paddings = {x for x in itertools.product(*([p_vals] * k_rank * 2))} - if error_name == ErrorIf.StrideSmallerOne: - # Can't use stride=0, as it is used to derive output shape, as a divisor - s_vals = [rng.choice(range(-5, 0))] - else: + paddings = {x for x in itertools.product(*([p_vals] * k_rank * 2))} # Stride must be greater than 1 to force non-integer error startStride = ( 1 if error_name != ErrorIf.ConvOutputShapeNonInteger else 2 @@ -2035,12 +2081,10 @@ class TosaArgGen: s_vals = [ x for x in range(startStride, testGen.args.max_conv_stride + 1) ] - strides = {x for x in itertools.product(*([s_vals] * k_rank))} - if error_name == ErrorIf.DilationSmallerOne: - d_vals = [rng.choice(range(-5, 1))] - else: d_vals = [x for x in range(1, testGen.args.max_conv_dilation + 1)] - dilations = {x for x in itertools.product(*([d_vals] * k_rank))} + + strides = {x for x in itertools.product(*([s_vals] * k_rank))} + dilations = {x for x in itertools.product(*([d_vals] * k_rank))} if not error_name and testGen.args.oversize: # add some oversize argument values @@ -2064,12 +2108,15 @@ class TosaArgGen: ) max_dim_size = None - # There are too many parameter combinations, so generate them sparsely, - # very sparse for negative tests - sparsity_factor = 2 if error_name else 120 - sparsity = TosaArgGen._calculate_sparsity( - len(paddings) * len(strides) * len(dilations), sparsity_factor - ) + if error_name: + # Cycle through all error_if tests but we only keep the first few + sparsity = 1 + else: + # There are too many parameter combinations, so generate them sparsely, + sparsity_factor = 120 + sparsity = TosaArgGen._calculate_sparsity( + len(paddings) * len(strides) * len(dilations), sparsity_factor + ) else: # Only test 8k levels boundaries bigStride = testGen.TOSA_8K_LEVEL_MAX_STRIDE @@ -2114,13 +2161,15 @@ class TosaArgGen: # Currently allow all combinations that are reasonable size sparsity = 1 + more_tests = True n = 0 for a in accum_dtypes: for s in sorted(list(strides)): for p in sorted(list(paddings)): for d in sorted(list(dilations)): if ( - n % sparsity == 0 + more_tests + and n % sparsity == 0 # the padded shape must exceed the dilation * kernel to get a positive # sized output shape and (ifm_shape[1] - 1 + p[0] + p[1]) @@ -2199,6 +2248,15 @@ class TosaArgGen: args_dict, ) ) + if ( + error_name + and len(arg_list) >= TosaArgGen.MAX_CONV_ERROR_IFS + ): + # Found enough errors + logger.debug( + f"Skipping creating more conv error tests for {error_name}" + ) + more_tests = False n += 1 arg_list = TosaArgGen._add_data_generators( @@ -2482,7 +2540,9 @@ class TosaArgGen: pad_const_int = 0 pad_const_fp = rng.randNumberDType(dtype) else: - return [] + assert error_name == ErrorIf.WrongInputType + pad_const_int = 0 + pad_const_fp = 0 list_shape_pad_values = list(shape_pad_values) # If we are producing tests for rank 6 or greater use sparsity @@ -2523,7 +2583,9 @@ class TosaArgGen: arg_list.append((name, args_dict)) if error_name == ErrorIf.PadSmallerZero and len(arg_list) == 0: - logger.info(f"No ErrorIf test created for input shape: {shapeList[0]}") + logger.debug( + f"agPad: No PadSmallerZero ErrorIf test created for input shape: {shapeList[0]}" + ) arg_list = TosaArgGen._add_data_generators( testGen, @@ -3106,8 +3168,8 @@ class TosaArgGen: # Find new shapes up to the number of permutations asked for # This code is NOT fast. Fortunately, the numbers are fairly small. for p in range(testGen.args.num_rand_permutations): - # Rank from 1 to TOSA_TENSOR_MAX_RANK - newRank = rng.randInt(1, (testGen.TOSA_TENSOR_MAX_RANK + 1)) + # Rank from 1 to MAX_TENSOR_RANK + newRank = rng.randInt(1, (gtu.MAX_TENSOR_RANK + 1)) if len(factors) < newRank: continue diff --git a/verif/generator/tosa_error_if.py b/verif/generator/tosa_error_if.py index 916b4f9..1b6b044 100644 --- a/verif/generator/tosa_error_if.py +++ b/verif/generator/tosa_error_if.py @@ -3,11 +3,8 @@ import logging import math +import generator.tosa_utils as gtu import numpy as np -from generator.tosa_utils import MAX_RESIZE_DIMENSION -from generator.tosa_utils import product -from generator.tosa_utils import usableDTypes -from generator.tosa_utils import valueToName from tosa.DType import DType from tosa.Op import Op from tosa.ResizeMode import ResizeMode @@ -287,7 +284,7 @@ class TosaErrorIfArgGen: max_dim and max_items. """ new_shape = [min(d, max_dim) for d in shape] if max(shape) > max_dim else shape - while product(new_shape) > max_items: + while gtu.product(new_shape) > max_items: new_shape = [max(d - 1, 1) for d in new_shape] return new_shape @@ -389,12 +386,12 @@ class TosaErrorValidator: serializer.setExpectedReturnCode(2, True, desc=error_reason) elif error_result: # and not expected_result logger.error( - f"Unexpected ERROR_IF: Op: {valueToName(Op, kwargs['op']['op'])}" + f"Unexpected ERROR_IF: Op: {gtu.valueToName(Op, kwargs['op']['op'])}" f" Expected: {error_name}, Got: {validator_name}" ) elif not expected_result: # and not error_result logger.error( - f"Missed ERROR_IF: Op: {valueToName(Op, kwargs['op']['op'])}" + f"Missed ERROR_IF: Op: {gtu.valueToName(Op, kwargs['op']['op'])}" f" Expected: {error_name}" ) @@ -402,7 +399,7 @@ class TosaErrorValidator: for k, v in sorted(kwargs.items()): if k != "op": if k.endswith("dtype"): - v = valueToName(DType, v) + v = gtu.valueToName(DType, v) logger.error(f" {k} = {v}") return overall_result @@ -417,7 +414,8 @@ class TosaErrorValidator: allowed_input_dtypes = { t[0] if isinstance(t, list) else t for t in input_dtypes } - wrong_input_dtypes = list(usableDTypes(excludes=allowed_input_dtypes)) + wrong_input_dtypes = list(gtu.usableDTypes(excludes=allowed_input_dtypes)) + assert len(wrong_input_dtypes) > 0 # Turn the wrong dtypes into required list of types if op["op"] in [ @@ -682,7 +680,8 @@ class TosaErrorValidator: @staticmethod def evWrongRank(check=False, **kwargs): - all_ranks = (1, 2, 3, 4, 5) + # From 1 to MAX_TENSOR_RANK+1 inclusively + all_ranks = tuple(range(1, gtu.MAX_TENSOR_RANK + 2)) # Make a list of incorrect ranks assert "op" in kwargs @@ -784,16 +783,16 @@ class TosaErrorValidator: "shape": [[1, 16584, 5, 1], [1, 2, 16499, 4]], } error_result = False - error_reason = f"At least one maximum dimension is greater than or equal to {MAX_RESIZE_DIMENSION}" + error_reason = f"At least one maximum dimension is greater than or equal to {gtu.MAX_RESIZE_DIMENSION}" if check: input_shape = kwargs["input_shape"] output_shape = kwargs["output_shape"] if ( - (input_shape[1] >= MAX_RESIZE_DIMENSION) - or (input_shape[2] >= MAX_RESIZE_DIMENSION) - or (output_shape[1] >= MAX_RESIZE_DIMENSION) - or (output_shape[2] >= MAX_RESIZE_DIMENSION) + (input_shape[1] >= gtu.MAX_RESIZE_DIMENSION) + or (input_shape[2] >= gtu.MAX_RESIZE_DIMENSION) + or (output_shape[1] >= gtu.MAX_RESIZE_DIMENSION) + or (output_shape[2] >= gtu.MAX_RESIZE_DIMENSION) ): error_result = True @@ -1075,8 +1074,8 @@ class TosaErrorValidator: if ( params_valid - and max(output_shape) < MAX_RESIZE_DIMENSION - and max(input_shape) < MAX_RESIZE_DIMENSION + and max(output_shape) < gtu.MAX_RESIZE_DIMENSION + and max(input_shape) < gtu.MAX_RESIZE_DIMENSION ): output_y = ( (input_shape[1] - 1) * scale[0] - offset[0] + border[0] diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py index c867070..28b3d28 100644 --- a/verif/generator/tosa_test_gen.py +++ b/verif/generator/tosa_test_gen.py @@ -36,9 +36,7 @@ logger = logging.getLogger("tosa_verif_build_tests") class TosaTestGen: - # Maximum rank of tensor supported by test generator. # This currently matches the 8K level defined in the specification. - TOSA_TENSOR_MAX_RANK = 6 TOSA_8K_LEVEL_MAX_SCALE = 64 TOSA_8K_LEVEL_MAX_KERNEL = 8192 TOSA_8K_LEVEL_MAX_STRIDE = 8192 @@ -2941,8 +2939,10 @@ class TosaTestGen: testList = [] if testType == "negative" and "error_if_validators" in op: error_if_validators = op["error_if_validators"] + num_error_types_created = 0 else: error_if_validators = [None] + num_error_types_created = None for validator in error_if_validators: if validator is not None: @@ -3020,6 +3020,21 @@ class TosaTestGen: testList.append( (opName, testStr, t, error_name, shapeList, args) ) + if error_name is not None: + # Check the last test is of the error we wanted + if len(testList) == 0 or testList[-1][3] != error_name: + if self.args.level8k: + logger.info(f"Missing {error_name} tests due to level8k mode") + else: + logger.error(f"ERROR: Failed to create any {error_name} tests") + logger.debug( + "Last test created: {}".format( + testList[-1] if testList else None + ) + ) + else: + # Successfully created at least one ERRROR_IF test + num_error_types_created += 1 if testType == "positive": # Remove tests which are expected to fail but don't correlate to a ERROR_IF statement @@ -3039,6 +3054,15 @@ class TosaTestGen: if not remove_test: clean_testList.append(test) testList = clean_testList + else: + if num_error_types_created is not None and not self.args.level8k: + remaining_error_types = ( + len(error_if_validators) - num_error_types_created + ) + if remaining_error_types: + raise Exception( + f"Failed to create {remaining_error_types} error types for {opName}" + ) return testList @@ -3141,9 +3165,11 @@ class TosaTestGen: if compliance: tensMeta["compliance"] = compliance self.serialize("test", tensMeta) + return True else: # The test is not valid logger.error(f"Invalid ERROR_IF test created: {opName} {testStr}") + return False def createDynamicOpLists(self): @@ -3305,7 +3331,7 @@ class TosaTestGen: [DType.FP8E5M2, DType.FP8E5M2, DType.FP16], ] - DEFAULT_RANK_RANGE = (1, TOSA_TENSOR_MAX_RANK) + DEFAULT_RANK_RANGE = (1, gtu.MAX_TENSOR_RANK) TOSA_OP_LIST = { # Tensor operators diff --git a/verif/generator/tosa_test_select.py b/verif/generator/tosa_test_select.py index c6e1d0d..904d90f 100644 --- a/verif/generator/tosa_test_select.py +++ b/verif/generator/tosa_test_select.py @@ -185,7 +185,7 @@ class TestOpList: # Add a test to this op group and set up the permutations/group for it assert test.opName.startswith(self.opName) if str(test) in self.testStrings: - logger.info(f"Skipping duplicate test: {str(test)}") + logger.debug(f"Skipping duplicate test: {str(test)}") return self.tests.append(test) diff --git a/verif/generator/tosa_utils.py b/verif/generator/tosa_utils.py index 4a4f6bb..a8e321e 100644 --- a/verif/generator/tosa_utils.py +++ b/verif/generator/tosa_utils.py @@ -10,6 +10,9 @@ from tosa.DType import DType # Maximum dimension size for output and inputs for RESIZE MAX_RESIZE_DIMENSION = 16384 +# Maximum rank of tensor supported by test generator. +MAX_TENSOR_RANK = 6 + # Data type information dictionary # - str: filename abbreviation # - width: number of bytes needed for type diff --git a/verif/generator/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py index 83c06d7..0839cec 100644 --- a/verif/generator/tosa_verif_build_tests.py +++ b/verif/generator/tosa_verif_build_tests.py @@ -428,6 +428,9 @@ def main(argv=None): logger.error(f"INTERNAL ERROR: Failure creating test output for {opName}") raise e + if results.count(False): + raise Exception(f"Failed to create {results.count(False)} tests") + if not args.list_tests: print(f"Done creating {len(results)} tests") return 0 -- cgit v1.2.1