From 18a379d99ad10002b3cf6eda086457179221cc22 Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Thu, 28 Mar 2024 15:53:21 +0000 Subject: Add rank 0 testing support Default test range is now rank 0 to 3 instead of 1 to 4 Signed-off-by: Jeremy Johnson Change-Id: Ibde66b60b58de9f4a3852a3807c01f8dae61206f --- .gitignore | 2 +- thirdparty/serialization_lib | 2 +- verif/conformance/tosa_base_profile_ops_info.json | 58 ++++++++++- verif/conformance/tosa_main_profile_ops_info.json | 36 ++++++- verif/generator/datagenerator.py | 6 +- verif/generator/tosa_arg_gen.py | 20 +++- verif/generator/tosa_test_gen.py | 116 +++++++++++++--------- 7 files changed, 177 insertions(+), 63 deletions(-) diff --git a/.gitignore b/.gitignore index dddbcc0..5433993 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ __pycache__/ build/ debug-build/ -conformance/ +/conformance/ conformance_build/ conformance_large_files/ .cache diff --git a/thirdparty/serialization_lib b/thirdparty/serialization_lib index ad78daa..8f9e284 160000 --- a/thirdparty/serialization_lib +++ b/thirdparty/serialization_lib @@ -1 +1 @@ -Subproject commit ad78daaf0fa1e41742cbed314459c3dbbb483c20 +Subproject commit 8f9e2842ce7d25645233ad4f6fa406be982346ae diff --git a/verif/conformance/tosa_base_profile_ops_info.json b/verif/conformance/tosa_base_profile_ops_info.json index 25a6076..4232851 100644 --- a/verif/conformance/tosa_base_profile_ops_info.json +++ b/verif/conformance/tosa_base_profile_ops_info.json @@ -1052,7 +1052,15 @@ "--target-dtype", "int16", "--target-dtype", - "int32" + "int32", + "--target-rank", + "1", + "--target-rank", + "2", + "--target-rank", + "3", + "--target-rank", + "4" ], [ "--target-dtype", @@ -3331,6 +3339,14 @@ "int32", "--target-dtype", "bool", + "--target-rank", + "1", + "--target-rank", + "2", + "--target-rank", + "3", + "--target-rank", + "4", "--num-rand-permutations", "2" ], @@ -3453,7 +3469,15 @@ "--target-dtype", "bool", "--tensor-dim-range", - "1,32" + "1,32", + "--target-rank", + "1", + "--target-rank", + "2", + "--target-rank", + "3", + "--target-rank", + "4" ], [ "--target-dtype", @@ -3896,13 +3920,29 @@ "--target-dtype", "int32", "--tensor-dim-range", - "4,32" + "4,32", + "--target-rank", + "1", + "--target-rank", + "2", + "--target-rank", + "3", + "--target-rank", + "4" ], [ "--target-dtype", "bool", "--tensor-dim-range", - "1,16" + "1,16", + "--target-rank", + "1", + "--target-rank", + "2", + "--target-rank", + "3", + "--target-rank", + "4" ], [ "--target-dtype", @@ -4114,7 +4154,15 @@ "generator_args": [ [ "--target-dtype", - "int32" + "int32", + "--target-rank", + "1", + "--target-rank", + "2", + "--target-rank", + "3", + "--target-rank", + "4" ], [ "--target-dtype", diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json index 83c1573..ac0f384 100644 --- a/verif/conformance/tosa_main_profile_ops_info.json +++ b/verif/conformance/tosa_main_profile_ops_info.json @@ -1970,6 +1970,14 @@ "bf16", "--fp-values-range", "-max,max", + "--target-rank", + "1", + "--target-rank", + "2", + "--target-rank", + "3", + "--target-rank", + "4", "--oversize" ] ] @@ -2160,6 +2168,14 @@ "bf16", "--fp-values-range", "-max,max", + "--target-rank", + "1", + "--target-rank", + "2", + "--target-rank", + "3", + "--target-rank", + "4", "--num-rand-permutations", "2" ], @@ -2233,7 +2249,15 @@ "--fp-values-range", "-max,max", "--tensor-dim-range", - "1,48" + "1,48", + "--target-rank", + "1", + "--target-rank", + "2", + "--target-rank", + "3", + "--target-rank", + "4" ], [ "--target-dtype", @@ -2555,7 +2579,15 @@ "--fp-values-range", "-max,max", "--tensor-dim-range", - "4,32" + "4,32", + "--target-rank", + "1", + "--target-rank", + "2", + "--target-rank", + "3", + "--target-rank", + "4" ], [ "--target-dtype", diff --git a/verif/generator/datagenerator.py b/verif/generator/datagenerator.py index c63a2d5..9f3d29d 100644 --- a/verif/generator/datagenerator.py +++ b/verif/generator/datagenerator.py @@ -68,7 +68,11 @@ class GenerateLibrary: def _create_buffer(self, dtype: str, shape: tuple): """Helper to create a buffer of the required type.""" - size = np.prod(shape) + if shape: + size = np.prod(shape) + else: + # Rank 0 + size = 1 if dtype == "FP32": # Create buffer and initialize to zero diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py index 670a3e4..8d6c8d7 100644 --- a/verif/generator/tosa_arg_gen.py +++ b/verif/generator/tosa_arg_gen.py @@ -259,11 +259,15 @@ class TosaTensorGen: @staticmethod def _get_broadcast_shapes(testGen, rng, num_shapes, rank, error_name=None): + if rank == 0: + # No broadcasting possible for rank 0 + return [[]] * num_shapes + shape = testGen.makeShape(rng, rank) shape_list = [] - # Choose one of the inputs to broadcast - # Note: Simplifies OutputShaper code if we don't change first shape for errors + # Choose any one of the inputs to broadcast + # Note for ERRORS: Simplifies OutputShaper code if we don't change first shape bcast_idx = rng.randInt(0 if error_name is None else 1, num_shapes) fuzz_idx = rng.randInt(0, rank) @@ -1304,10 +1308,14 @@ class TosaTensorValuesGen: else: # MUL with 3 inputs (3rd is shift) tens_ser_list.append( - testGen.ser.addPlaceholder(shapeList[0], dtypeList[0], a_arr) + testGen.ser.addPlaceholder( + shapeList[0], dtypeList[0], a_arr.astype(np.int32) + ) ) tens_ser_list.append( - testGen.ser.addPlaceholder(shapeList[1], dtypeList[1], b_arr) + testGen.ser.addPlaceholder( + shapeList[1], dtypeList[1], b_arr.astype(np.int32) + ) ) tens_ser_list.append( testGen.ser.addPlaceholder([1], DType.INT8, np.int8([shift])) @@ -3021,7 +3029,9 @@ class TosaArgGen: for double_round in [False, True]: if error_name == ErrorIf.ScaleNotTrue and not double_round: continue - for per_channel in [False, True]: + # Per_channel is only valid with rank > 0 + pc_options = (False, True) if len(shapeList[0]) > 0 else (False,) + for per_channel in pc_options: if ( inDtype == DType.INT48 diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py index 399fed6..c5ac0f9 100644 --- a/verif/generator/tosa_test_gen.py +++ b/verif/generator/tosa_test_gen.py @@ -186,25 +186,26 @@ class TosaTestGen: def makeShape(self, rng, rank): if self.targetted_shape: return np.int32(self.targetted_shape) - return np.int32( - rng.integers( - low=self.args.tensor_shape_range[0], - high=self.args.tensor_shape_range[1], - size=rank, + else: + return np.int32( + rng.integers( + low=self.args.tensor_shape_range[0], + high=self.args.tensor_shape_range[1], + size=rank, + ) ) - ) def setTargetShape(self, shape): self.targetted_shape = shape def shapeStr(self, shape): - - sStr = [] - # Convert to strings - for i in shape: - sStr.append(str(i)) - - return "x".join(sStr) + assert shape is not None + if len(shape) > 0: + # Rank > 0 + return "x".join([str(d) for d in shape]) + else: + # Rank 0 + return "0" def typeStr(self, dtype): if isinstance(dtype, list) or isinstance(dtype, tuple): @@ -2839,29 +2840,36 @@ class TosaTestGen: def create_filter_lists( self, op, shapeFilter, rankFilter, dtypeFilter, testType, validator=None ): - # Create a default testing rank range, 1-4 inclusive to keep test sizes reasonably small. - default_test_rank_range = range(1, 5) - if not shapeFilter: - shapeFilter = [None] + # Create a default testing rank range + if testType == "positive": + # 0-3 inclusive to keep test sizes reasonably small. + default_test_rank_range = range(0, 4) + else: + # Some errors do not work with rank 0, use 1-3 + default_test_rank_range = range(1, 4) # Calculate the filters based on what is requested and what the operator allows rmin, rmax = op["rank"] - if rankFilter is not None: - cleanRankFilter = [] - # Ensure rankFilter values are allowed by operator - for rank in rankFilter: - if rank >= rmin and rank <= rmax: - cleanRankFilter.append(rank) - elif rankFilter is None and shapeFilter[0] is None: - # Ensure default behaviour is bounded by default range or by operator, - # whichever is the smaller range of ranks. - opRankRange = range(rmin, rmax + 1) - cleanRankFilter = ( - opRankRange - if len(opRankRange) <= len(default_test_rank_range) - else default_test_rank_range - ) + + if shapeFilter: + # Specified shapes - ignore rank filter and default to op ranks below + rankFilter = None + ranksToCheck = [] + elif rankFilter is None: + # No set rank filter so ensure default behaviour is bounded + ranksToCheck = default_test_rank_range else: + ranksToCheck = rankFilter + + cleanRankFilter = [] + # Ensure rank values are allowed by operator + for rank in ranksToCheck: + if rank >= rmin and rank <= rmax: + cleanRankFilter.append(rank) + + if shapeFilter or (len(cleanRankFilter) == 0 and rankFilter is None): + # Shapes specified or default test ranks didn't meet + # op requirements - so just use op ranks cleanRankFilter = range(rmin, rmax + 1) dtypes = op["types"] @@ -2877,6 +2885,9 @@ class TosaTestGen: else: cleanDtypeFilter = dtypes + if not shapeFilter: + shapeFilter = [None] + if testType == "positive": filterDict = { "shapeFilter": shapeFilter, @@ -3326,7 +3337,7 @@ class TosaTestGen: [DType.FP8E5M2, DType.FP8E5M2, DType.FP16], ] - DEFAULT_RANK_RANGE = (1, gtu.MAX_TENSOR_RANK) + DEFAULT_RANK_RANGE = (0, gtu.MAX_TENSOR_RANK) KERNELS_2D = [[1, 1], [2, 2], [3, 3], [5, 5], [3, 1], [1, 3]] KERNELS_3D = [[1, 1, 1], [2, 1, 1], [1, 2, 1], [1, 1, 2]] @@ -3348,7 +3359,7 @@ class TosaTestGen: "argmax": { "op": Op.ARGMAX, "operands": (1, 0), - "rank": (1, 6), + "rank": (1, gtu.MAX_TENSOR_RANK), "build_fcn": ( build_argmax, TosaTensorGen.tgBasic, @@ -4519,6 +4530,7 @@ class TosaTestGen: "pad": { "op": Op.PAD, "operands": (2, 0), + "rank": (1, gtu.MAX_TENSOR_RANK), "build_fcn": ( build_pad, TosaTensorGen.tgBasic, @@ -4541,6 +4553,7 @@ class TosaTestGen: "dim": { "op": Op.DIM, "operands": (1, 0), + "rank": (1, gtu.MAX_TENSOR_RANK), "build_fcn": ( build_dim, TosaTensorGen.tgBasic, @@ -4560,6 +4573,7 @@ class TosaTestGen: "reshape": { "op": Op.RESHAPE, "operands": (2, 0), + "rank": (1, gtu.MAX_TENSOR_RANK), "build_fcn": ( build_reshape, TosaTensorGen.tgBasic, @@ -4599,7 +4613,7 @@ class TosaTestGen: "slice": { "op": Op.SLICE, "operands": (3, 0), - "rank": (1, 6), + "rank": (1, gtu.MAX_TENSOR_RANK), "build_fcn": ( build_slice, TosaTensorGen.tgBasic, @@ -4629,7 +4643,7 @@ class TosaTestGen: "tile": { "op": Op.TILE, "operands": (2, 0), - "rank": (1, 6), + "rank": (1, gtu.MAX_TENSOR_RANK), "build_fcn": ( build_tile, TosaTensorGen.tgBasic, @@ -4650,7 +4664,7 @@ class TosaTestGen: "transpose": { "op": Op.TRANSPOSE, "operands": (1, 0), - "rank": (1, 6), + "rank": (1, gtu.MAX_TENSOR_RANK), "build_fcn": ( build_transpose, TosaTensorGen.tgBasic, @@ -5047,6 +5061,7 @@ class OutputShaper: assert len(a.shape) == len(b.shape) assert a.dtype == b.dtype + # Work out broadcasted output shape (when not ERRORIF test) shape = [] for i in range(len(a.shape)): if a.shape[i] == 1 and error_name is None: @@ -5054,8 +5069,9 @@ class OutputShaper: else: shape.append(a.shape[i]) - fuzz_idx = rng.integers(0, len(a.shape)) - if error_name == ErrorIf.DimensionMismatch: + if len(shape) > 0 and error_name == ErrorIf.DimensionMismatch: + # Can only create this error for rank > 0 + fuzz_idx = rng.integers(0, len(shape)) shape[fuzz_idx] += 1 if error_name == ErrorIf.WrongOutputType: @@ -5112,6 +5128,7 @@ class OutputShaper: assert len(a.shape) == len(b.shape) and len(a.shape) == len(cond.shape) assert a.dtype == b.dtype + # Work out broadcasted output shape (when not ERRORIF test) shape = [] for i in range(len(cond.shape)): if cond.shape[i] == 1 and error_name is None: @@ -5119,8 +5136,9 @@ class OutputShaper: else: shape.append(cond.shape[i]) - fuzz_idx = rng.integers(0, len(a.shape)) - if error_name == ErrorIf.DimensionMismatch: + if len(shape) > 0 and error_name == ErrorIf.DimensionMismatch: + # Can only create this error for rank > 0 + fuzz_idx = rng.integers(0, len(shape)) shape[fuzz_idx] += 1 if error_name == ErrorIf.WrongOutputType: @@ -5146,7 +5164,7 @@ class OutputShaper: assert len(a.shape) == len(b.shape) assert a.dtype == b.dtype - # Do broadcast + # Work out broadcasted output shape shape = [] for i in range(len(a.shape)): if a.shape[i] == 1 and len(b.shape) > i: @@ -5154,8 +5172,9 @@ class OutputShaper: else: shape.append(a.shape[i]) - fuzz_idx = rng.integers(0, len(a.shape)) - if error_name == ErrorIf.DimensionMismatch: + if len(shape) > 0 and error_name == ErrorIf.DimensionMismatch: + # Can only create this error for rank > 0 + fuzz_idx = rng.integers(0, len(shape)) shape[fuzz_idx] += 1 if error_name == ErrorIf.WrongOutputType: @@ -5994,12 +6013,13 @@ class OutputShaper: assert len(a.shape) == len(b.shape) assert a.dtype == b.dtype - shape = [] - for i in range(len(a.shape)): - shape.append(a.shape[i]) + shape = a.shape.copy() - fuzz_idx = rng.integers(0, len(a.shape)) + # Do not expect rank 0 tests! + assert len(shape) > 0 if error_name == ErrorIf.DimensionMismatch: + # Can only create this error for rank > 0 + fuzz_idx = rng.integers(0, len(shape)) shape[fuzz_idx] += 1 if error_name == ErrorIf.WrongOutputType: -- cgit v1.2.1