From 5e36bde3bcfb0c55e61a80cde460ae3955968af3 Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Thu, 14 Mar 2024 16:56:10 +0000 Subject: Improve CONV2D coverage for conformance Enable testing of broadcastable bias channel. Change convolution arguments function to calculate workable convolutions by adjusting PAD size as needed for FP and errors. Add --conv-kernel build_tests option to allow different sized convolution kernels Improve coverage with batch size, specific shapes and kernels of 7x7 and 9x9 Signed-off-by: Jeremy Johnson Change-Id: I59bb737f398f4363cc606b24fea7812b2059bc01 --- verif/conformance/tosa_main_profile_ops_info.json | 26 +- verif/generator/tosa_arg_gen.py | 292 ++++++++++++---------- verif/generator/tosa_test_gen.py | 107 ++++---- verif/generator/tosa_test_select.py | 3 - verif/generator/tosa_verif_build_tests.py | 9 + 5 files changed, 241 insertions(+), 196 deletions(-) diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json index fbf5a82..d5d08ba 100644 --- a/verif/conformance/tosa_main_profile_ops_info.json +++ b/verif/conformance/tosa_main_profile_ops_info.json @@ -739,9 +739,9 @@ "--fp-values-range", "-max,max", "--target-shape", - "1,34,19,27", + "2,34,19,27", "--target-shape", - "1,16,51,7", + "3,16,51,7", "--tensor-dim-range", "1,16", "--allow-pooling-and-conv-oversizes" @@ -760,6 +760,28 @@ "--max-conv-dilation", "1", "--allow-pooling-and-conv-oversizes" + ], + [ + "--target-dtype", + "fp32", + "--target-dtype", + "fp16", + "--target-dtype", + "bf16", + "--fp-values-range", + "-max,max", + "--conv-kernel", + "7,7", + "--conv-kernel", + "9,9", + "--target-shape", + "1,127,127,1", + "--target-shape", + "1,256,256,1", + "--target-shape", + "1,1080,720,1", + "--tensor-dim-range", + "1,8" ] ] }, diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py index 41ef4df..3a85961 100644 --- a/verif/generator/tosa_arg_gen.py +++ b/verif/generator/tosa_arg_gen.py @@ -339,7 +339,13 @@ class TosaTensorGen: # The filter dimensions are OHWI filter_shape = np.asarray([ofm_depth, filter_hw[0], filter_hw[1], ifm_shape[3]]) - # The bias is OC + # The bias is OC or 1 if broadcastable + try: + if op["broadcastable_bias"]: + if rng.choice([True, False]): + ofm_depth = 1 + except KeyError: + pass bias_shape = np.asarray([ofm_depth]) return [ifm_shape, filter_shape, bias_shape] @@ -1868,7 +1874,7 @@ class TosaArgGen: else "" ) logger.info( - f"Skipping {opName}{shape_info} dot product test as too few calculations {dot_products} < {testGen.TOSA_MI_DOT_PRODUCT_MIN}" + f"Skipping {opName}{shape_info} {gtu.DTYPE_ATTRIBUTES[dtype]['json']} dot product test as too few calculations {dot_products} < {testGen.TOSA_MI_DOT_PRODUCT_MIN}" ) continue # KS and acc_type is required by all dot product generators @@ -1986,10 +1992,6 @@ class TosaArgGen: # Used by CONV2D, CONV3D and DEPTHWISE_CONV2D arg_list = [] - if testGen.args.level8k and error_name is not None: - # Don't produce negative large tests - return arg_list - # Shape: Batches, (Depth), Height, Width, Channels ifm_shape = shapeList[0] # Shape: (OFM channels), (KD), KH, KW, IFM channels @@ -2002,124 +2004,116 @@ class TosaArgGen: [DType.BF16] if gtu.dtypeIsFloat(dtypes[0]) else [DType.INT16] ) - # Op type checks - conv3d = opName.startswith("conv3d") - depthwise = opName.startswith("depthwise") + # For op type checks + op = testGen.TOSA_OP_LIST[opName] # Check the rank - rank = 5 if conv3d else 4 + rank = 5 if op["op"] == Op.CONV3D else 4 if error_name != ErrorIf.WrongRank: assert len(ifm_shape) == rank assert len(filter_shape) == rank # kernel rank omits channels k_rank = rank - 2 - k_pos = 0 if depthwise else 1 + k_pos = 0 if op["op"] == Op.DEPTHWISE_CONV2D else 1 k_shape = tuple(filter_shape[k_pos : (k_pos + k_rank)]) # compliance size - KS k_size = gtu.product(k_shape) - if not depthwise: + if not op["op"] == Op.DEPTHWISE_CONV2D: k_size *= ifm_shape[-1] - if not testGen.args.level8k: - if error_name in ( - ErrorIf.PadSmallerZero, - ErrorIf.StrideSmallerOne, - ErrorIf.DilationSmallerOne, - ): - # Use specific invalid value(s) - if error_name == ErrorIf.PadSmallerZero: - # Create negative paddings but with positive opposite paddings - neg_pad = rng.choice(range(-5, 0)) - p_vals = [neg_pad, abs(neg_pad)] - else: - p_vals = [0, 0] - if error_name == ErrorIf.StrideSmallerOne: - # Can't use stride=0, as it is used to derive output shape, as a divisor - s_vals = [rng.choice(range(-5, 0))] - else: - s_vals = [1] - if error_name == ErrorIf.DilationSmallerOne: - d_vals = [rng.choice(range(-5, 1))] - else: - d_vals = [1] - p = p_vals * k_rank - s = s_vals * k_rank - d = d_vals * k_rank - - # Fix values to produce valid error_if - for index in range(k_rank): - pad_offset = index * 2 - fixed = False - while not fixed: - partial = ( - ifm_shape[index + 1] - - 1 - + p[pad_offset] - + p[pad_offset + 1] - - (k_shape[index] - 1) * d[index] - ) - remainder = partial % s[index] - if partial <= 0: - p[pad_offset + 1] += abs(partial) + 1 - elif remainder: + def get_conv_output_info(p, s, d, fix_up_padding=False): + # Work out remainders and output dimensions with an + # option to adjust paddings to create a valid operation + nonlocal ifm_shape, k_shape, error_name, k_rank + if fix_up_padding: + p = list(p) # Make paddings editable + outputs_no_stride = [] + remainders = [] + outputs = [] + for index in range(k_rank): + pad_offset = index * 2 + fixed = False + # Fix up pad values to produce valid conv2d + while not fixed: + # Output dimension without being adjusted for stride + output_no_stride = ( + ifm_shape[index + 1] + - 1 + + p[pad_offset] + + p[pad_offset + 1] + - (k_shape[index] - 1) * d[index] + ) + # Tensor left over after applying striding + remainder = output_no_stride % s[index] + if not fix_up_padding: + # Just want remainders and outputs + break + if output_no_stride <= 0: + p[pad_offset + 1] += abs(output_no_stride) + 1 + continue + if error_name == ErrorIf.ConvOutputShapeNonInteger: + if remainder: + # Conditions to trigger the test + fixed = True + else: + p[pad_offset + 1] += 1 + else: + if remainder: # Stride will be negative for StrideSmallerOne - assert remainder < 0 + assert remainder > 0 or ( + error_name == ErrorIf.StrideSmallerOne and remainder < 0 + ) p[pad_offset + 1] += abs(remainder) else: fixed = True - paddings = {tuple(p)} - strides = {tuple(s)} - dilations = {tuple(d)} - logger.debug(f"agConv: error pad={p} stride={s} dilation={d}") + outputs_no_stride.append(output_no_stride) + remainders.append(remainder) + # Output dimension taking in to account stride + outputs.append((output_no_stride // s[index]) + 1) + + if fix_up_padding: + p = tuple(p) # Make the paddings read-only + assert min(outputs_no_stride) > 0, "Fix up did not work!" + return p, remainders, outputs, outputs_no_stride + + # Only fix up padding for conv2d and float types currently + fix_up_padding = gtu.dtypeIsFloat(dtypes[0]) and op["op"] == Op.CONV2D + # Allow any size of output dimension + max_dim_size = None + # Include all tests by default + sparsity = 1 + + # Work out padding, strides and dilation ranges depending on + # error and arguments + if error_name in ( + ErrorIf.PadSmallerZero, + ErrorIf.StrideSmallerOne, + ErrorIf.DilationSmallerOne, + ): + # Use specific invalid value(s) + if error_name == ErrorIf.PadSmallerZero: + # Create negative paddings but with positive opposite paddings + neg_pad = rng.choice(range(-5, 0)) + p_vals = [neg_pad, abs(neg_pad)] else: - # Generate comprehensive argument lists - p_vals = [x for x in range(0, testGen.args.max_conv_padding + 1)] - paddings = {x for x in itertools.product(*([p_vals] * k_rank * 2))} - # Stride must be greater than 1 to force non-integer error - startStride = ( - 1 if error_name != ErrorIf.ConvOutputShapeNonInteger else 2 - ) - s_vals = [ - x for x in range(startStride, testGen.args.max_conv_stride + 1) - ] - d_vals = [x for x in range(1, testGen.args.max_conv_dilation + 1)] - - strides = {x for x in itertools.product(*([s_vals] * k_rank))} - dilations = {x for x in itertools.product(*([d_vals] * k_rank))} + p_vals = [0, 0] + if error_name == ErrorIf.StrideSmallerOne: + # Can't use stride=0, as it is used to derive output shape, as a divisor + s_vals = [rng.choice(range(-5, 0))] + else: + s_vals = [1] + if error_name == ErrorIf.DilationSmallerOne: + d_vals = [rng.choice(range(-5, 1))] + else: + d_vals = [1] + paddings = {tuple(p_vals) * k_rank} + strides = {tuple(s_vals) * k_rank} + dilations = {tuple(d_vals) * k_rank} - if not error_name and testGen.args.oversize: - # add some oversize argument values - if max(ifm_shape) < 64: - bigPadding = 9 - paddings.update( - { - x - for x in itertools.product( - *([[0, bigPadding]] * (k_rank * 2)) - ) - } - ) - bigStride = 8 - strides.update( - {x for x in itertools.product(*([[1, bigStride]] * k_rank))} - ) - bigDilation = 7 - dilations.update( - {x for x in itertools.product(*([[1, bigDilation]] * k_rank))} - ) - max_dim_size = None + fix_up_padding = True # Need to fix up paddings to be valid - if error_name: - # Cycle through all error_if tests but we only keep the first few - sparsity = 1 - else: - # There are too many parameter combinations, so generate them sparsely, - sparsity_factor = 120 - sparsity = TosaArgGen._calculate_sparsity( - len(paddings) * len(strides) * len(dilations), sparsity_factor - ) - else: + elif testGen.args.level8k and error_name is None: # Only test 8k levels boundaries bigStride = testGen.TOSA_8K_LEVEL_MAX_STRIDE bigKernel = testGen.TOSA_8K_LEVEL_MAX_KERNEL @@ -2127,7 +2121,7 @@ class TosaArgGen: dilation_shape = [1] * k_rank pad_shape = [0] * k_rank * 2 - if conv3d: + if op["op"] == Op.CONV3D: # Small stride apart from for big kernel (see below) to keep # tensor size/calculation small stride_shape = [1] * k_rank @@ -2162,43 +2156,66 @@ class TosaArgGen: # Currently allow all combinations that are reasonable size sparsity = 1 + else: + # Generate comprehensive argument lists + p_vals = [x for x in range(0, testGen.args.max_conv_padding + 1)] + paddings = {x for x in itertools.product(*([p_vals] * k_rank * 2))} + # Stride must be greater than 1 to force non-integer error + startStride = 1 if error_name != ErrorIf.ConvOutputShapeNonInteger else 2 + s_vals = [x for x in range(startStride, testGen.args.max_conv_stride + 1)] + d_vals = [x for x in range(1, testGen.args.max_conv_dilation + 1)] + + strides = {x for x in itertools.product(*([s_vals] * k_rank))} + dilations = {x for x in itertools.product(*([d_vals] * k_rank))} + + if error_name is None and testGen.args.oversize: + # add some oversize argument values + if max(ifm_shape) < 64: + bigPadding = 9 + paddings.update( + { + x + for x in itertools.product( + *([[0, bigPadding]] * (k_rank * 2)) + ) + } + ) + bigStride = 8 + strides.update( + {x for x in itertools.product(*([[1, bigStride]] * k_rank))} + ) + bigDilation = 7 + dilations.update( + {x for x in itertools.product(*([[1, bigDilation]] * k_rank))} + ) + if error_name is None: + # There are too many parameter combinations, so generate them sparsely, + sparsity_factor = 120 + sparsity = TosaArgGen._calculate_sparsity( + len(paddings) * len(strides) * len(dilations), sparsity_factor + ) + + # Run through all the argument options creating valid test cases more_tests = True n = 0 for a in accum_dtypes: for s in sorted(list(strides)): for p in sorted(list(paddings)): for d in sorted(list(dilations)): - if ( - more_tests - and n % sparsity == 0 - # the padded shape must exceed the dilation * kernel to get a positive - # sized output shape - and (ifm_shape[1] - 1 + p[0] + p[1]) - > d[0] * (k_shape[0] - 1) - and (ifm_shape[2] - 1 + p[2] + p[3]) - > d[1] * (k_shape[1] - 1) - and ( - k_rank < 3 - or ( - (ifm_shape[3] - 1 + p[4] + p[5]) - > d[2] * (k_shape[2] - 1) - ) - ) - ): - remainders = [] - outputs = [] - for index in range(k_rank): - pad_offset = index * 2 - partial = ( - ifm_shape[index + 1] - - 1 - + p[pad_offset] - + p[pad_offset + 1] - - (k_shape[index] - 1) * d[index] - ) - remainders.append(partial % s[index]) - outputs.append((partial // s[index]) + 1) + if more_tests and (n % sparsity == 0): + ( + p, + remainders, + outputs, + outputs_no_stride, + ) = get_conv_output_info(p, s, d, fix_up_padding) + # Following is like checking each dimension N: + # (ifm_shape[N+1] - 1 + p[N*2] + p[N*2+1]) > d[N] * (k_shape[N] - 1) + if min(outputs_no_stride) <= 0: + # Not a valid operation + n += 1 # Increment count of tests + continue if ( # the parameters must produce integer exact output @@ -2213,10 +2230,13 @@ class TosaArgGen: and max(outputs) >= max_dim_size ): # Test will consume too much memory - skip it + logger.debug( + "agConv: Convolution output too big - skipped" + ) continue # Compliance - number of dot product calculations - if depthwise: + if op["op"] == Op.DEPTHWISE_CONV2D: # N*OH*OW*C*M dots = gtu.product( (ifm_shape[0], *outputs, *filter_shape[2:]) diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py index 28b3d28..71d7fcc 100644 --- a/verif/generator/tosa_test_gen.py +++ b/verif/generator/tosa_test_gen.py @@ -3172,64 +3172,54 @@ class TosaTestGen: return False def createDynamicOpLists(self): - - if "conv2d_TEMPLATE" not in self.TOSA_OP_LIST: - # Already created these lists (can occur when class is initialized more than once) - return - - # Dynamically create op lists for convolutions with a list of kernel sizes - if not self.args.level8k: - KERNELS_2D = [[1, 1], [2, 2], [3, 3], [5, 5], [3, 1], [1, 3]] - KERNELS_3D = [[1, 1, 1], [2, 1, 1], [1, 2, 1], [1, 1, 2]] - else: - bigK = self.TOSA_8K_LEVEL_MAX_KERNEL - KERNELS_2D = [[1, bigK], [bigK, 2]] - KERNELS_3D = [[1, bigK, 1], [2, 2, bigK]] - - for k in KERNELS_2D: - testName = "conv2d_{}x{}".format(k[0], k[1]) - self.TOSA_OP_LIST[testName] = self.TOSA_OP_LIST["conv2d_TEMPLATE"].copy() - self.TOSA_OP_LIST[testName]["filter"] = k - self.TOSA_OP_LIST[testName]["template"] = False - self.TOSA_OP_LIST[testName]["real_name"] = "conv2d" - - testName = "depthwise_conv2d_{}x{}".format(k[0], k[1]) - self.TOSA_OP_LIST[testName] = self.TOSA_OP_LIST[ - "depthwise_conv2d_TEMPLATE" - ].copy() - self.TOSA_OP_LIST[testName]["filter"] = k - self.TOSA_OP_LIST[testName]["template"] = False - self.TOSA_OP_LIST[testName]["real_name"] = "depthwise_conv2d" - - testName = "transpose_conv2d_{}x{}".format(k[0], k[1]) - self.TOSA_OP_LIST[testName] = self.TOSA_OP_LIST[ - "transpose_conv2d_TEMPLATE" - ].copy() - self.TOSA_OP_LIST[testName]["filter"] = k - self.TOSA_OP_LIST[testName]["template"] = False - self.TOSA_OP_LIST[testName]["real_name"] = "transpose_conv2d" - - for k in KERNELS_3D: - testName = "conv3d_{}x{}x{}".format(k[0], k[1], k[2]) - self.TOSA_OP_LIST[testName] = self.TOSA_OP_LIST["conv3d_TEMPLATE"].copy() - self.TOSA_OP_LIST[testName]["filter"] = k - self.TOSA_OP_LIST[testName]["template"] = False - self.TOSA_OP_LIST[testName]["real_name"] = "conv3d" - - # Delete any templates after having created any dynamic ops - # This is a two-pass operation because it's bad practice to delete - # keys from dictionaries while iterating - keyList = [] - for k in self.TOSA_OP_LIST: + # Find all the ops marked as templates + templateKeys = [] + for opName in self.TOSA_OP_LIST: try: - if self.TOSA_OP_LIST[k]["template"]: - keyList.append(k) - continue + if self.TOSA_OP_LIST[opName]["template"]: + templateKeys.append(opName) except KeyError: pass - for k in keyList: - del self.TOSA_OP_LIST[k] + bigK = self.TOSA_8K_LEVEL_MAX_KERNEL + + # Add dynamic ops based on kernel sizes + for opName in templateKeys: + assert opName.endswith("_TEMPLATE"), "Found incorrect template" + realName = opName[: len(opName) - len("_TEMPLATE")] + template = self.TOSA_OP_LIST[opName] + k_rank = 3 if realName == "conv3d" else 2 + + # Choose kernels to build tests for from the template or args + if self.args.level8k: + if k_rank == 3: + kernels = [[1, bigK, 1], [2, 2, bigK]] + else: + kernels = [[1, bigK], [bigK, 2]] + else: + kernels = [] + if len(self.args.conv_kernels) > 0: + kernels = [k for k in self.args.conv_kernels if len(k) == k_rank] + if len(kernels) == 0: + logger.debug( + f"{realName} op using defaults as no rank {k_rank} kernels found in {self.args.conv_kernels}" + ) + if len(kernels) == 0: + # Fallback to use the defined template kernels + kernels = self.TOSA_OP_LIST[opName]["filter"] + + # Dynamically create ops for listed kernel sizes + for k in kernels: + kernelStr = "x".join([str(d) for d in k]) + testName = f"{realName}_{kernelStr}" + kernelOp = template.copy() + kernelOp["filter"] = k + kernelOp["template"] = False + kernelOp["real_name"] = realName + self.TOSA_OP_LIST[testName] = kernelOp + + # Delete the template after having created the dynamic ops + del self.TOSA_OP_LIST[opName] def initOpListDefaults(self): """Fill in default fields for ops if they aren't already specified. @@ -3333,6 +3323,9 @@ class TosaTestGen: DEFAULT_RANK_RANGE = (1, gtu.MAX_TENSOR_RANK) + KERNELS_2D = [[1, 1], [2, 2], [3, 3], [5, 5], [3, 1], [1, 3]] + KERNELS_3D = [[1, 1, 1], [2, 1, 1], [1, 2, 1], [1, 1, 2]] + TOSA_OP_LIST = { # Tensor operators "argmax": { @@ -3426,6 +3419,8 @@ class TosaTestGen: "data_gen": { "fp": (gtu.DataGenType.DOT_PRODUCT,), }, + "broadcastable_bias": True, + "filter": KERNELS_2D, "template": True, }, # Templated operator. Filled in by createDynamicOpLists @@ -3460,13 +3455,13 @@ class TosaTestGen: "data_gen": { "fp": (gtu.DataGenType.DOT_PRODUCT,), }, + "filter": KERNELS_3D, "template": True, }, # Templated operator. Filled in by createDynamicOpLists "depthwise_conv2d_TEMPLATE": { "op": Op.DEPTHWISE_CONV2D, "operands": (1, 2), - "filter": [1, 1], "rank": (4, 4), "build_fcn": ( build_depthwise_conv2d, @@ -3495,6 +3490,7 @@ class TosaTestGen: "data_gen": { "fp": (gtu.DataGenType.DOT_PRODUCT,), }, + "filter": KERNELS_2D, "template": True, }, "fully_connected": { @@ -3608,6 +3604,7 @@ class TosaTestGen: "data_gen": { "fp": (gtu.DataGenType.DOT_PRODUCT,), }, + "filter": KERNELS_2D, "template": True, }, # Activation functions diff --git a/verif/generator/tosa_test_select.py b/verif/generator/tosa_test_select.py index 904d90f..a894bf4 100644 --- a/verif/generator/tosa_test_select.py +++ b/verif/generator/tosa_test_select.py @@ -164,9 +164,6 @@ class TestOpList: op, selectionCriteria, "all", False ) - if self.paramArgs and self.maximumPerPermute > 1: - logger.warning(f"Unsupported - selection params AND maximum for {opName}") - self.tests = [] self.testStrings = set() self.shapes = set() diff --git a/verif/generator/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py index 0839cec..f746009 100644 --- a/verif/generator/tosa_verif_build_tests.py +++ b/verif/generator/tosa_verif_build_tests.py @@ -167,6 +167,15 @@ def parseArgs(argv): help="Maximum stride for Conv tests", ) + ops_group.add_argument( + "--conv-kernel", + dest="conv_kernels", + action="append", + default=[], + type=lambda x: str_to_list(x), + help="Create convolution tests with a particular kernel shape, e.g., 1,4 or 1,3,1 (only 2D kernel sizes will be used for 2D ops, etc.)", + ) + ops_group.add_argument( "--max-pooling-padding", dest="max_pooling_padding", -- cgit v1.2.1