From 74567097e161ce9cdb0f09d474da6d9d36aa7476 Mon Sep 17 00:00:00 2001 From: Matthew Haddon Date: Fri, 16 Jul 2021 15:38:20 +0100 Subject: Allow user to specify test type generated * The option --test-type allows the user to select 'positive', 'negative', or 'both' types of tests produced by the test generator. * Reset RNG when looping through negative test generation (generation not implemented) Signed-off-by: Matthew Haddon Change-Id: I1bfcb3170e7380be0f98b36b3d4abc4779a05abe --- verif/tosa_test_gen.py | 101 +++++++++++++++++++++++----------------- verif/tosa_verif_build_tests.py | 16 ++++++- 2 files changed, 71 insertions(+), 46 deletions(-) diff --git a/verif/tosa_test_gen.py b/verif/tosa_test_gen.py index b5ddbd7..777c059 100644 --- a/verif/tosa_test_gen.py +++ b/verif/tosa_test_gen.py @@ -982,6 +982,11 @@ class TosaTestGen: with open(os.path.join(self.basePath, self.testPath, "desc.json"), "w") as fd: fd.write(self.ser.writeJson("{}.tosa".format(testName))) + def resetRNG(self, seed=None): + if seed == None: + seed = self.random_seed + 1 + self.rng = np.random.default_rng(seed) + def getRandTensor(self, shape, dtype): if dtype == DType.BOOL: np_dt = np.bool @@ -1694,7 +1699,7 @@ class TosaTestGen: return acc_out def genOpTestList( - self, opName, shapeFilter=[None], rankFilter=None, dtypeFilter=None + self, opName, shapeFilter=[None], rankFilter=None, dtypeFilter=None, testType='positive' ): try: @@ -1720,58 +1725,66 @@ class TosaTestGen: if not shapeFilter: shapeFilter = [None] - for r in range(rmin, rmax + 1): - - # Filter out the rank? - if rankFilter is not None and r not in rankFilter: - continue - if ( - rankFilter is None - and shapeFilter[0] is None - and r not in default_test_rank_range - ): - continue + # Positive test loop + if testType in ['positive', 'both']: + for r in range(rmin, rmax + 1): - for t in op["types"]: + # Filter out the rank? + if rankFilter is not None and r not in rankFilter: + continue + if ( + rankFilter is None + and shapeFilter[0] is None + and r not in default_test_rank_range + ): + continue - # Filter tests based on dtype? - if dtypeFilter is not None: - if not ( - t in dtypeFilter - or (isinstance(t, list) and t[0] in dtypeFilter) - ): - continue + for t in op["types"]: - # Create the placeholder and const tensors - for shape in shapeFilter: - # A None shape chooses a random shape of a given rank + # Filter tests based on dtype? + if dtypeFilter is not None: + if not ( + t in dtypeFilter + or (isinstance(t, list) and t[0] in dtypeFilter) + ): + continue - # Filter out by rank - if shape is not None and len(shape) != r: - continue + # Create the placeholder and const tensors + for shape in shapeFilter: + # A None shape chooses a random shape of a given rank - self.setTargetShape(shape) - shapeList = tgen_fcn(self, op, r) + # Filter out by rank + if shape is not None and len(shape) != r: + continue - shapeStr = self.shapeStr(shapeList[0]) - typeStr = self.typeStr(t) + self.setTargetShape(shape) + shapeList = tgen_fcn(self, op, r) - # Argument lists consists of tuples of the (str, []) string representation and the build function argument list - argList = [] - if agen_fcn: - argList = agen_fcn(self, opName, shapeList, t) - else: - argList = [("", [])] + shapeStr = self.shapeStr(shapeList[0]) + typeStr = self.typeStr(t) - for argStr, args in argList: - if argStr: - testStr = "{}_{}_{}_{}".format( - opName, shapeStr, typeStr, argStr - ) + # Argument lists consists of tuples of the (str, []) string representation and the build function argument list + argList = [] + if agen_fcn: + argList = agen_fcn(self, opName, shapeList, t) else: - testStr = "{}_{}_{}".format(opName, shapeStr, typeStr) - - testList.append((opName, testStr, t, shapeList, args)) + argList = [("", [])] + + for argStr, args in argList: + if argStr: + testStr = "{}_{}_{}_{}".format( + opName, shapeStr, typeStr, argStr + ) + else: + testStr = "{}_{}_{}".format(opName, shapeStr, typeStr) + + testList.append((opName, testStr, t, shapeList, args)) + + # Reset RNG so both positive and negative tests are reproducible + self.resetRNG() + # Negative test loop + if testType in ['negative', 'both']: + print("Negative tests unsupported") return testList diff --git a/verif/tosa_verif_build_tests.py b/verif/tosa_verif_build_tests.py index 343d8d4..02d1934 100755 --- a/verif/tosa_verif_build_tests.py +++ b/verif/tosa_verif_build_tests.py @@ -201,6 +201,14 @@ def parseArgs(): help="Allow constant input tensors for concat operator", ) + parser.add_argument( + "--test-type", + dest="test_type", + choices=['positive', 'negative', 'both'], + default="positive", + type=str, + help="type of tests produced, postive, negative, or both", + ) args = parser.parse_args() return args @@ -221,15 +229,19 @@ def main(): shapeFilter=args.target_shapes, rankFilter=args.target_ranks, dtypeFilter=args.target_dtypes, + testType=args.test_type ) ) print("{} matching tests".format(len(testList))) + results = [] for opName, testStr, dtype, shapeList, testArgs in testList: if args.verbose: print(testStr) - ttg.serializeTest(opName, testStr, dtype, shapeList, testArgs) - print("Done creating {} tests".format(len(testList))) + results.append(ttg.serializeTest(opName, testStr, dtype, shapeList, testArgs)) + + print(f"Done creating {len(results)} tests") + if __name__ == "__main__": -- cgit v1.2.1