From a015001dfbd0ed48caf54fd66b0509ee344a229e Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Wed, 15 Nov 2023 15:52:06 +0000 Subject: Main Compliance testing support for COMPARISON ops Signed-off-by: Jeremy Johnson Change-Id: Id6229cfaccad866b110630119eb045dbf6453bf5 --- .../src/generate/generate_pseudo_random.cc | 7 ++++ reference_model/src/generate/generate_utils.cc | 3 ++ verif/conformance/tosa_main_profile_ops_info.json | 21 ++++++----- verif/generator/tosa_arg_gen.py | 21 ++++++----- verif/generator/tosa_test_gen.py | 42 +++++++++++++++------- 5 files changed, 65 insertions(+), 29 deletions(-) diff --git a/reference_model/src/generate/generate_pseudo_random.cc b/reference_model/src/generate/generate_pseudo_random.cc index 78013eb..d8d2288 100644 --- a/reference_model/src/generate/generate_pseudo_random.cc +++ b/reference_model/src/generate/generate_pseudo_random.cc @@ -107,9 +107,16 @@ bool generateFP32(const TosaReference::GenerateConfig& cfg, void* data, size_t s float* a = reinterpret_cast(data); const auto T = TosaReference::numElementsFromShape(cfg.shape); + const bool comparisonOp = + (cfg.opType == Op::Op_EQUAL) || (cfg.opType == Op::Op_GREATER_EQUAL) || (cfg.opType == Op::Op_GREATER); for (auto t = 0; t < T; ++t) { a[t] = generator->getRandomFloat(); + if (comparisonOp && (t % 4 == 0)) + { + // Set every 4th value to 0 to enable better comparison testing + a[t] = 0.f; + } } return true; } diff --git a/reference_model/src/generate/generate_utils.cc b/reference_model/src/generate/generate_utils.cc index d2168c9..1edc79d 100644 --- a/reference_model/src/generate/generate_utils.cc +++ b/reference_model/src/generate/generate_utils.cc @@ -45,10 +45,13 @@ NLOHMANN_JSON_SERIALIZE_ENUM(Op, { Op::Op_CEIL, "CEIL" }, { Op::Op_CLAMP, "CLAMP" }, { Op::Op_CONV2D, "CONV2D" }, + { Op::Op_EQUAL, "EQUAL" }, { Op::Op_ERF, "ERF" }, { Op::Op_EXP, "EXP" }, { Op::Op_FLOOR, "FLOOR" }, { Op::Op_FULLY_CONNECTED, "FULLY_CONNECTED" }, + { Op::Op_GREATER, "GREATER" }, + { Op::Op_GREATER_EQUAL, "GREATER_EQUAL" }, { Op::Op_IDENTITY, "IDENTITY" }, { Op::Op_LOG, "LOG" }, { Op::Op_MATMUL, "MATMUL" }, diff --git a/verif/conformance/tosa_main_profile_ops_info.json b/verif/conformance/tosa_main_profile_ops_info.json index 35d72e8..0b20e4f 100644 --- a/verif/conformance/tosa_main_profile_ops_info.json +++ b/verif/conformance/tosa_main_profile_ops_info.json @@ -839,6 +839,7 @@ "profile": [ "tosa-mi" ], + "support_for": [ "lazy_data_gen" ], "generation": { "standard": { "generator_args": [ @@ -850,7 +851,7 @@ "--target-dtype", "bf16", "--fp-values-range", - "-2.0,2.0", + "-max,max", "--tensor-dim-range", "18,64", "--target-rank", @@ -864,7 +865,7 @@ "--target-dtype", "fp32", "--fp-values-range", - "-2.0,2.0", + "-max,max", "--tensor-dim-range", "1,18", "--target-rank", @@ -876,7 +877,7 @@ "--target-dtype", "fp16", "--fp-values-range", - "-2.0,2.0", + "-max,max", "--target-shape", "1,1,2,65550", "--target-shape", @@ -1175,6 +1176,7 @@ "profile": [ "tosa-mi" ], + "support_for": [ "lazy_data_gen" ], "generation": { "standard": { "generator_args": [ @@ -1188,7 +1190,7 @@ "--target-dtype", "bf16", "--fp-values-range", - "-2.0,2.0", + "-max,max", "--tensor-dim-range", "18,64", "--target-rank", @@ -1204,7 +1206,7 @@ "--target-dtype", "bf16", "--fp-values-range", - "-2.0,2.0", + "-max,max", "--tensor-dim-range", "1,18", "--target-rank", @@ -1218,7 +1220,7 @@ "--target-dtype", "fp32", "--fp-values-range", - "-2.0,2.0", + "-max,max", "--target-shape", "3,65551,1,1", "--target-shape", @@ -1242,6 +1244,7 @@ "profile": [ "tosa-mi" ], + "support_for": [ "lazy_data_gen" ], "generation": { "standard": { "generator_args": [ @@ -1253,7 +1256,7 @@ "--target-dtype", "bf16", "--fp-values-range", - "-2.0,2.0", + "-max,max", "--tensor-dim-range", "18,64", "--target-rank", @@ -1267,7 +1270,7 @@ "--target-dtype", "fp16", "--fp-values-range", - "-2.0,2.0", + "-max,max", "--tensor-dim-range", "1,18", "--target-rank", @@ -1279,7 +1282,7 @@ "--target-dtype", "bf16", "--fp-values-range", - "-2.0,2.0", + "-max,max", "--target-shape", "65552,1,1,4", "--target-shape", diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py index 6675025..9147605 100644 --- a/verif/generator/tosa_arg_gen.py +++ b/verif/generator/tosa_arg_gen.py @@ -1119,14 +1119,18 @@ class TosaTensorValuesGen: return TosaTensorValuesGen.TVGInfo(tens_ser_list, None) @staticmethod - def tvgEqual(testGen, op, dtypeList, shapeList, testArgs, error_name=None): - if error_name is None: + def tvgEqual(testGen, opName, dtypeList, shapeList, argsDict, error_name=None): + if error_name is None and not gtu.dtypeIsSupportedByCompliance(dtypeList[0]): + # Integer + op = testGen.TOSA_OP_LIST[opName] pCount, cCount = op["operands"] assert ( pCount == 2 and cCount == 0 ), "Op.EQUAL must have 2 placeholders, 0 consts" + a_arr = testGen.getRandTensor(shapeList[0], dtypeList[0]) b_arr = testGen.getRandTensor(shapeList[1], dtypeList[1]) + # Using random numbers means that it will be very unlikely that # there are any matching (equal) values, therefore force that # there are twice the number of matching values as the tensor rank @@ -1147,17 +1151,18 @@ class TosaTensorValuesGen: a_arr[tuple(a_index)] = b_arr[tuple(b_index)] - placeholders = [] - placeholders.append( + tens_ser_list = [] + tens_ser_list.append( testGen.ser.addPlaceholder(shapeList[0], dtypeList[0], a_arr) ) - placeholders.append( + tens_ser_list.append( testGen.ser.addPlaceholder(shapeList[1], dtypeList[1], b_arr) ) - return placeholders + return TosaTensorValuesGen.TVGInfo(tens_ser_list, None) else: - return TosaTensorValuesGen.tvgDefault( - testGen, op, dtypeList, shapeList, testArgs, error_name + # ERROR_IF or floating point test + return TosaTensorValuesGen.tvgLazyGenDefault( + testGen, opName, dtypeList, shapeList, argsDict, error_name ) @staticmethod diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py index 04093b8..7b44ced 100644 --- a/verif/generator/tosa_test_gen.py +++ b/verif/generator/tosa_test_gen.py @@ -622,14 +622,19 @@ class TosaTestGen: ) return result_tens - def build_comparison(self, op, a, b, validator_fcns=None, error_name=None): - result_tens = OutputShaper.binaryComparisonOp( + def build_comparison( + self, op, inputs, args_dict, validator_fcns=None, error_name=None, qinfo=None + ): + assert len(inputs) == 2 + a, b = inputs + + result_tensor = OutputShaper.binaryComparisonOp( self.ser, self.rng, a, b, error_name ) # Invalidate Input/Output list for error if checks. input_list = [a.name, b.name] - output_list = [result_tens.name] + output_list = [result_tensor.name] pCount, cCount = op["operands"] num_operands = pCount + cCount input_list, output_list = TosaErrorIfArgGen.eiInvalidateInputOutputList( @@ -645,9 +650,9 @@ class TosaTestGen: input2=b, input_shape=a.shape, input_dtype=a.dtype, - output_shape=result_tens.shape, - output_dtype=result_tens.dtype, - result_tensors=[result_tens], + output_shape=result_tensor.shape, + output_dtype=result_tensor.dtype, + result_tensors=[result_tensor], input_list=input_list, output_list=output_list, num_operands=num_operands, @@ -659,7 +664,11 @@ class TosaTestGen: input_list, output_list, ) - return result_tens + + compliance = self.tensorComplianceMetaData( + op, a.dtype, args_dict, result_tensor, error_name + ) + return TosaTestGen.BuildInfo(result_tensor, compliance) def build_argmax( self, op, inputs, args_dict, validator_fcns, error_name, qinfo=None @@ -3863,7 +3872,7 @@ class TosaTestGen: build_comparison, TosaTensorGen.tgBroadcastFuzz, TosaTensorValuesGen.tvgEqual, - None, + TosaArgGen.agNone, ), "types": TYPE_FI32, "error_if_validators": ( @@ -3875,6 +3884,9 @@ class TosaTestGen: TosaErrorValidator.evDimensionMismatch, TosaErrorValidator.evBroadcastShapesMismatch, ), + "data_gen": { + "fp": (gtu.DataGenType.PSEUDO_RANDOM,), + }, }, "greater_equal": { "op": Op.GREATER_EQUAL, @@ -3882,8 +3894,8 @@ class TosaTestGen: "build_fcn": ( build_comparison, TosaTensorGen.tgBroadcastFuzz, - TosaTensorValuesGen.tvgDefault, - None, + TosaTensorValuesGen.tvgLazyGenDefault, + TosaArgGen.agNone, ), "types": TYPE_FI32, "error_if_validators": ( @@ -3895,6 +3907,9 @@ class TosaTestGen: TosaErrorValidator.evDimensionMismatch, TosaErrorValidator.evBroadcastShapesMismatch, ), + "data_gen": { + "fp": (gtu.DataGenType.PSEUDO_RANDOM,), + }, }, "greater": { "op": Op.GREATER, @@ -3902,8 +3917,8 @@ class TosaTestGen: "build_fcn": ( build_comparison, TosaTensorGen.tgBroadcastFuzz, - TosaTensorValuesGen.tvgDefault, - None, + TosaTensorValuesGen.tvgLazyGenDefault, + TosaArgGen.agNone, ), "types": TYPE_FI32, "error_if_validators": ( @@ -3915,6 +3930,9 @@ class TosaTestGen: TosaErrorValidator.evDimensionMismatch, TosaErrorValidator.evBroadcastShapesMismatch, ), + "data_gen": { + "fp": (gtu.DataGenType.PSEUDO_RANDOM,), + }, }, # Reduction operators "reduce_all": { -- cgit v1.2.1