diff options
Diffstat (limited to 'verif/generator')
| -rw-r--r-- | verif/generator/tosa_arg_gen.py | 27 | ||||
| -rw-r--r-- | verif/generator/tosa_error_if.py | 10 | ||||
| -rw-r--r-- | verif/generator/tosa_test_gen.py | 22 | ||||
| -rw-r--r-- | verif/generator/tosa_test_select.py | 348 | ||||
| -rw-r--r-- | verif/generator/tosa_verif_build_tests.py | 135 |
5 files changed, 497 insertions, 45 deletions
diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py index c596645..253e8ee 100644 --- a/verif/generator/tosa_arg_gen.py +++ b/verif/generator/tosa_arg_gen.py @@ -1,8 +1,8 @@ # Copyright (c) 2021-2024, ARM Limited. # SPDX-License-Identifier: Apache-2.0 import itertools +import logging import math -import warnings import generator.tosa_utils as gtu import numpy as np @@ -16,6 +16,9 @@ from tosa.ResizeMode import ResizeMode # DTypeNames, DType, Op and ResizeMode are convenience variables to the # flatc-generated types that should be enums, but aren't +logging.basicConfig() +logger = logging.getLogger("tosa_verif_build_tests") + class TosaQuantGen: """QuantizedInfo random generator helper functions. @@ -131,8 +134,9 @@ class TosaQuantGen: shift = shift + 1 shift = (-shift) + scaleBits - # print('scalefp {} scaleBits {} m {} mult {} shift {}'.format( - # scaleFp, scaleBits, m, multiplier, shift)) + logger.debug( + f"computeMultiplierAndShift: scalefp={scaleFp} scaleBits={scaleBits} m={m} mult={multiplier} shift={shift}" + ) # Adjust multiplier such that shift is in allowed value range. if shift == 0: @@ -690,8 +694,9 @@ class TosaTensorValuesGen: # Invalid data range from low to high created due to user # constraints revert to using internal ranges as they are # known to work - msg = f"Using safe data range ({low_val} to {high_val}) instead of supplied ({type_range[0]} to {type_range[1]})" - warnings.warn(msg) + logger.info( + f"Using safe data range ({low_val} to {high_val}) instead of supplied ({type_range[0]} to {type_range[1]})" + ) data_range = (low_val, high_val) return data_range return None @@ -1856,7 +1861,7 @@ class TosaArgGen: if "shape" in args_dict else "" ) - print( + logger.info( f"Skipping {opName}{shape_info} dot product test as too few calculations {dot_products} < {testGen.TOSA_MI_DOT_PRODUCT_MIN}" ) continue @@ -2503,7 +2508,7 @@ class TosaArgGen: arg_list.append((name, args_dict)) if error_name == ErrorIf.PadSmallerZero and len(arg_list) == 0: - warnings.warn(f"No ErrorIf test created for input shape: {shapeList[0]}") + logger.info(f"No ErrorIf test created for input shape: {shapeList[0]}") arg_list = TosaArgGen._add_data_generators( testGen, @@ -2683,7 +2688,9 @@ class TosaArgGen: remainder_w = partial_w % s[1] output_h = partial_h // s[0] + 1 output_w = partial_w // s[1] + 1 - # debug print(shape, remainder_h, remainder_w, "/", output_h, output_w) + logger.debug( + f"agPooling: {shape} remainder=({remainder_h}, {remainder_w}) output=({output_h}, {output_w})" + ) if ( # the parameters must produce integer exact output error_name != ErrorIf.PoolingOutputShapeNonInteger @@ -2920,7 +2927,9 @@ class TosaArgGen: # Cap the scaling at 2^15 - 1 for scale16 scale_arr = np.clip(scale_arr, 1.0 / (1 << 31), 32767.0) - # print('{} {} -> {}'.format(out_type_width, in_type_width, scale_arr)) + logger.debug( + f"agRescale: {out_type_width} {in_type_width} -> {scale_arr}" + ) multiplier_arr = np.int32(np.zeros(shape=[nc])) shift_arr = np.int32(np.zeros(shape=[nc])) diff --git a/verif/generator/tosa_error_if.py b/verif/generator/tosa_error_if.py index 7a4d0d6..3972edd 100644 --- a/verif/generator/tosa_error_if.py +++ b/verif/generator/tosa_error_if.py @@ -1,5 +1,6 @@ # Copyright (c) 2021-2024, ARM Limited. # SPDX-License-Identifier: Apache-2.0 +import logging import math import numpy as np @@ -11,6 +12,9 @@ from tosa.DType import DType from tosa.Op import Op from tosa.ResizeMode import ResizeMode +logging.basicConfig() +logger = logging.getLogger("tosa_verif_build_tests") + class ErrorIf(object): MaxDimExceeded = "MaxDimExceeded" @@ -386,12 +390,12 @@ class TosaErrorValidator: if expected_result and error_result: serializer.setExpectedReturnCode(2, True, desc=error_reason) elif error_result: # and not expected_result - print( + logger.error( f"Unexpected ERROR_IF: Op: {valueToName(Op, kwargs['op']['op'])}" f" Expected: {error_name}, Got: {validator_name}" ) elif not expected_result: # and not error_result - print( + logger.error( f"Missed ERROR_IF: Op: {valueToName(Op, kwargs['op']['op'])}" f" Expected: {error_name}" ) @@ -401,7 +405,7 @@ class TosaErrorValidator: if k != "op": if k.endswith("dtype"): v = valueToName(DType, v) - print(f" {k} = {v}") + logger.error(f" {k} = {v}") return overall_result diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py index 0f68999..e7704f1 100644 --- a/verif/generator/tosa_test_gen.py +++ b/verif/generator/tosa_test_gen.py @@ -1,6 +1,7 @@ # Copyright (c) 2020-2024, ARM Limited. # SPDX-License-Identifier: Apache-2.0 import json +import logging import os from copy import deepcopy from datetime import datetime @@ -27,6 +28,9 @@ TOSA_AUTOGENERATED_HEADER = f"""// Copyright (c) {datetime.today().year}, ARM Li // AUTO-GENERATED FILE CREATED BY tosa_verif_build_tests """ +logging.basicConfig() +logger = logging.getLogger("tosa_verif_build_tests") + class TosaTestGen: # Maximum rank of tensor supported by test generator. @@ -2134,6 +2138,7 @@ class TosaTestGen: double_round = args_dict["double_round"] per_channel = args_dict["per_channel"] shift_arr = args_dict["shift"] + multiplier_arr = args_dict["multiplier"] result_tensor = OutputShaper.typeConversionOp( self.ser, self.rng, val, out_dtype, error_name @@ -2203,7 +2208,9 @@ class TosaTestGen: min_shift_value_arr[i] = -1 << (shift_arr[i] - 1) max_shift_value_arr[i] = (1 << (shift_arr[i] - 1)) - 1 - # print('multiplier {} shift {} inzp {} outzp {}'.format(multiplier_arr, shift_arr, input_zp, output_zp)) + logger.debug( + f"build_rescale: multiplier={multiplier_arr} shift={shift_arr} inzp={input_zp} outzp={output_zp}" + ) if scale32 and error_name is None: # Make sure random values are within apply_scale_32 specification # REQUIRES(value >= (-1<<(shift-1)) && value < (1<<(shift-1)) @@ -2907,7 +2914,9 @@ class TosaTestGen: cleanRankFilter = filterDict["rankFilter"] cleanDtypeFilter = filterDict["dtypeFilter"] cleanShapeFilter = filterDict["shapeFilter"] - # print(f"Error: {error_name}, Filters: S {cleanShapeFilter}, R {cleanRankFilter}, T {cleanDtypeFilter}") + logger.debug( + f"genOpTestList: Error={error_name}, Filters S={cleanShapeFilter}, R={cleanRankFilter}, T={cleanDtypeFilter}" + ) for r in cleanRankFilter: for t in cleanDtypeFilter: @@ -2981,8 +2990,7 @@ class TosaTestGen: except KeyError: raise Exception("Cannot find op with name {}".format(opName)) - if self.args.verbose: - print(f"Creating {testStr}") + logger.info(f"Creating {testStr}") # Create a serializer self.createSerializer(opName, testStr) @@ -3062,7 +3070,7 @@ class TosaTestGen: self.serialize("test", tensMeta) else: # The test is not valid - print(f"Invalid ERROR_IF test created: {opName} {testStr}") + logger.error(f"Invalid ERROR_IF test created: {opName} {testStr}") def createDynamicOpLists(self): @@ -3084,6 +3092,7 @@ class TosaTestGen: self.TOSA_OP_LIST[testName] = self.TOSA_OP_LIST["conv2d_TEMPLATE"].copy() self.TOSA_OP_LIST[testName]["filter"] = k self.TOSA_OP_LIST[testName]["template"] = False + self.TOSA_OP_LIST[testName]["real_name"] = "conv2d" testName = "depthwise_conv2d_{}x{}".format(k[0], k[1]) self.TOSA_OP_LIST[testName] = self.TOSA_OP_LIST[ @@ -3091,6 +3100,7 @@ class TosaTestGen: ].copy() self.TOSA_OP_LIST[testName]["filter"] = k self.TOSA_OP_LIST[testName]["template"] = False + self.TOSA_OP_LIST[testName]["real_name"] = "depthwise_conv2d" testName = "transpose_conv2d_{}x{}".format(k[0], k[1]) self.TOSA_OP_LIST[testName] = self.TOSA_OP_LIST[ @@ -3098,12 +3108,14 @@ class TosaTestGen: ].copy() self.TOSA_OP_LIST[testName]["filter"] = k self.TOSA_OP_LIST[testName]["template"] = False + self.TOSA_OP_LIST[testName]["real_name"] = "transpose_conv2d" for k in KERNELS_3D: testName = "conv3d_{}x{}x{}".format(k[0], k[1], k[2]) self.TOSA_OP_LIST[testName] = self.TOSA_OP_LIST["conv3d_TEMPLATE"].copy() self.TOSA_OP_LIST[testName]["filter"] = k self.TOSA_OP_LIST[testName]["template"] = False + self.TOSA_OP_LIST[testName]["real_name"] = "conv3d" # Delete any templates after having created any dynamic ops # This is a two-pass operation because it's bad practice to delete diff --git a/verif/generator/tosa_test_select.py b/verif/generator/tosa_test_select.py new file mode 100644 index 0000000..5a13178 --- /dev/null +++ b/verif/generator/tosa_test_select.py @@ -0,0 +1,348 @@ +# Copyright (c) 2024, ARM Limited. +# SPDX-License-Identifier: Apache-2.0 +import copy +import logging + +logging.basicConfig() +logger = logging.getLogger("tosa_verif_build_tests") + + +class Test: + """Test container to allow group and permute selection.""" + + def __init__( + self, opName, testStr, dtype, error, shapeList, argsDict, testOpName=None + ): + self.opName = opName + self.testStr = testStr + self.dtype = dtype + self.error = error + self.shapeList = shapeList + self.argsDict = argsDict + # Given test op name used for look up in TOSA_OP_LIST for "conv2d_1x1" for example + self.testOpName = testOpName if testOpName is not None else opName + + self.key = None + self.groupKey = None + self.mark = False + + def __str__(self): + return self.testStr + + def __lt__(self, other): + return self.testStr < str(other) + + def getArg(self, param): + # Get parameter values (arguments) for this test + if param == "rank": + return len(self.shapeList[0]) + elif param == "dtype": + if isinstance(self.dtype, list): + return tuple(self.dtype) + return self.dtype + elif param == "shape" and "shape" not in self.argsDict: + return str(self.shapeList[0]) + + if param in self.argsDict: + # Turn other args into hashable string without newlines + val = str(self.argsDict[param]) + return ",".join(str(val).splitlines()) + else: + return None + + def setKey(self, keyParams): + if self.error is None: + # Create the main key based on primary parameters + key = [self.getArg(param) for param in keyParams] + self.key = tuple(key) + else: + # Use the error as the key + self.key = self.error + return self.key + + def getKey(self): + return self.key + + def setGroupKey(self, groupParams): + # Create the group key based on arguments that do not define the group + # Therefore this test will match other tests that have the same arguments + # that are NOT the group arguments (group arguments like test set number) + paramsList = sorted(["shape", "dtype"] + list(self.argsDict.keys())) + key = [] + for param in paramsList: + if param in groupParams: + continue + key.append(self.getArg(param)) + self.groupKey = tuple(key) + return self.groupKey + + def getGroupKey(self): + return self.groupKey + + def inGroup(self, groupKey): + return self.groupKey == groupKey + + def setMark(self): + # Marks the test as important + self.mark = True + + def getMark(self): + return self.mark + + def isError(self): + return self.error is not None + + +def _get_selection_info_from_op(op, selectionCriteria, item, default): + # Get selection info from the op + if ( + "selection" in op + and selectionCriteria in op["selection"] + and item in op["selection"][selectionCriteria] + ): + return op["selection"][selectionCriteria][item] + else: + return default + + +def _get_tests_by_group(tests): + # Create simple structures to record the tests in groups + groups = [] + group_tests = {} + + for test in tests: + key = test.getGroupKey() + if key in group_tests: + group_tests[key].append(test) + else: + group_tests[key] = [test] + groups.append(key) + + # Return list of test groups (group keys) and a dictionary with a list of tests + # associated with each group key + return groups, group_tests + + +def _get_specific_op_info(opName, opSelectionInfo, testOpName): + # Get the op specific section from the selection config + name = opName if opName in opSelectionInfo else testOpName + if name not in opSelectionInfo: + logger.info(f"No op entry found for {opName} in test selection config") + return {} + return opSelectionInfo[name] + + +class TestOpList: + """All the tests for one op grouped by permutations.""" + + def __init__(self, opName, opSelectionInfo, selectionCriteria, testOpName): + self.opName = opName + self.testOpName = testOpName + op = _get_specific_op_info(opName, opSelectionInfo, testOpName) + + # See verif/conformance/README.md for more information on + # these selection arguments + self.permuteArgs = _get_selection_info_from_op( + op, selectionCriteria, "permutes", ["rank", "dtype"] + ) + self.paramArgs = _get_selection_info_from_op( + op, selectionCriteria, "full_params", [] + ) + self.specificArgs = _get_selection_info_from_op( + op, selectionCriteria, "specifics", {} + ) + self.groupArgs = _get_selection_info_from_op( + op, selectionCriteria, "groups", ["s"] + ) + self.maximumPerPermute = _get_selection_info_from_op( + op, selectionCriteria, "maximum", None + ) + self.numErrorIfs = _get_selection_info_from_op( + op, selectionCriteria, "num_errorifs", 1 + ) + self.selectAll = _get_selection_info_from_op( + op, selectionCriteria, "all", False + ) + + if self.paramArgs and self.maximumPerPermute > 1: + logger.warning(f"Unsupported - selection params AND maximum for {opName}") + + self.tests = [] + self.testStrings = set() + self.shapes = set() + + self.permutes = set() + self.testsPerPermute = {} + self.paramsPerPermute = {} + self.specificsPerPermute = {} + + self.selectionDone = False + + def __len__(self): + return len(self.tests) + + def add(self, test): + # Add a test to this op group and set up the permutations/group for it + assert test.opName.startswith(self.opName) + if str(test) in self.testStrings: + logger.info(f"Skipping duplicate test: {str(test)}") + return + + self.tests.append(test) + self.testStrings.add(str(test)) + + self.shapes.add(test.getArg("shape")) + + # Work out the permutation key for this test + permute = test.setKey(self.permuteArgs) + # Set up the group key for the test (for pulling out groups during selection) + test.setGroupKey(self.groupArgs) + + if permute not in self.permutes: + # New permutation + self.permutes.add(permute) + # Set up area to record the selected tests + self.testsPerPermute[permute] = [] + if self.paramArgs: + # Set up area to record the unique test params found + self.paramsPerPermute[permute] = {} + for param in self.paramArgs: + self.paramsPerPermute[permute][param] = set() + # Set up copy of the specific test args for selecting these + self.specificsPerPermute[permute] = copy.deepcopy(self.specificArgs) + + def _init_select(self): + # Can only perform the selection process once as it alters the permute + # information set at init + assert not self.selectionDone + + # Count of non-specific tests added to each permute (not error) + if not self.selectAll: + countPerPermute = {permute: 0 for permute in self.permutes} + + # Go through each test looking for permutes, unique params & specifics + for test in self.tests: + permute = test.getKey() + append = False + possible_append = False + + if test.isError(): + # Error test, choose up to number of tests + if len(self.testsPerPermute[permute]) < self.numErrorIfs: + append = True + else: + if self.selectAll: + append = True + else: + # See if this is a specific test to add + for param, values in self.specificsPerPermute[permute].items(): + arg = test.getArg(param) + # Iterate over a copy of the values, so we can remove them from the original + if arg in values.copy(): + # Found a match, remove it, so we don't look for it later + values.remove(arg) + # Mark the test as special (and so shouldn't be removed) + test.setMark() + append = True + + if self.paramArgs: + # See if this test contains any new params we should keep + # Perform this check even if we have already selected the test + # so we can record the params found + for param in self.paramArgs: + arg = test.getArg(param) + if arg not in self.paramsPerPermute[permute][param]: + # We have found a new value for this arg, record it + self.paramsPerPermute[permute][param].add(arg) + possible_append = True + else: + # No params set, so possible test to add up to maximum + possible_append = True + + if (not append and possible_append) and ( + self.maximumPerPermute is None + or countPerPermute[permute] < self.maximumPerPermute + ): + # Not selected but could be added and we have space left if + # a maximum is set. + append = True + countPerPermute[permute] += 1 + + # Check for grouping with chosen tests + if not append: + # We will keep any tests together than form a group + key = test.getGroupKey() + for t in self.testsPerPermute[permute]: + if t.getGroupKey() == key: + if t.getMark(): + test.setMark() + append = True + + if append: + self.testsPerPermute[permute].append(test) + + self.selectionDone = True + + def select(self, rng=None): + # Create selection of tests with optional shuffle + if not self.selectionDone: + if rng: + rng.shuffle(self.tests) + + self._init_select() + + # Now create the full list of selected tests per permute + selection = [] + + for permute, tests in self.testsPerPermute.items(): + selection.extend(tests) + + return selection + + def all(self): + # Un-selected list of tests - i.e. all of them + return self.tests + + +class TestList: + """List of all tests grouped by operator.""" + + def __init__(self, opSelectionInfo, selectionCriteria="default"): + self.opLists = {} + self.opSelectionInfo = opSelectionInfo + self.selectionCriteria = selectionCriteria + + def __len__(self): + length = 0 + for opName in self.opLists.keys(): + length += len(self.opLists[opName]) + return length + + def add(self, test): + if test.opName not in self.opLists: + self.opLists[test.opName] = TestOpList( + test.opName, + self.opSelectionInfo, + self.selectionCriteria, + test.testOpName, + ) + self.opLists[test.opName].add(test) + + def _get_tests(self, selectMode, rng): + selection = [] + + for opList in self.opLists.values(): + if selectMode: + tests = opList.select(rng=rng) + else: + tests = opList.all() + selection.extend(tests) + + selection = sorted(selection) + return selection + + def select(self, rng=None): + return self._get_tests(True, rng) + + def all(self): + return self._get_tests(False, None) diff --git a/verif/generator/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py index 8012d93..c32993a 100644 --- a/verif/generator/tosa_verif_build_tests.py +++ b/verif/generator/tosa_verif_build_tests.py @@ -1,17 +1,23 @@ -# Copyright (c) 2020-2023, ARM Limited. +# Copyright (c) 2020-2024, ARM Limited. # SPDX-License-Identifier: Apache-2.0 import argparse +import json +import logging import re import sys from pathlib import Path import conformance.model_files as cmf +import generator.tosa_test_select as tts from generator.tosa_test_gen import TosaTestGen from serializer.tosa_serializer import dtype_str_to_val from serializer.tosa_serializer import DTypeNames OPTION_FP_VALUES_RANGE = "--fp-values-range" +logging.basicConfig() +logger = logging.getLogger("tosa_verif_build_tests") + # Used for parsing a comma-separated list of integers/floats in a string # to an actual list of integers/floats with special case max @@ -58,6 +64,7 @@ def parseArgs(argv): parser = argparse.ArgumentParser() + filter_group = parser.add_argument_group("test filter options") ops_group = parser.add_argument_group("operator options") tens_group = parser.add_argument_group("tensor options") @@ -73,7 +80,7 @@ def parseArgs(argv): help="Random seed for test generation", ) - parser.add_argument( + filter_group.add_argument( "--filter", dest="filter", default="", @@ -82,7 +89,12 @@ def parseArgs(argv): ) parser.add_argument( - "-v", "--verbose", dest="verbose", action="count", help="Verbose operation" + "-v", + "--verbose", + dest="verbose", + action="count", + default=0, + help="Verbose operation", ) parser.add_argument( @@ -226,7 +238,7 @@ def parseArgs(argv): help="Allow constant input tensors for concat operator", ) - parser.add_argument( + filter_group.add_argument( "--test-type", dest="test_type", choices=["positive", "negative", "both"], @@ -235,6 +247,26 @@ def parseArgs(argv): help="type of tests produced, positive, negative, or both", ) + filter_group.add_argument( + "--test-selection-config", + dest="selection_config", + type=Path, + help="enables test selection, this is the path to the JSON test selection config file, will use the default selection specified for each op unless --selection-criteria is supplied", + ) + + filter_group.add_argument( + "--test-selection-criteria", + dest="selection_criteria", + help="enables test selection, this is the selection criteria to use from the selection config", + ) + + parser.add_argument( + "--list-tests", + dest="list_tests", + action="store_true", + help="lists the tests that will be generated and then exits", + ) + ops_group.add_argument( "--allow-pooling-and-conv-oversizes", dest="oversize", @@ -281,6 +313,10 @@ def main(argv=None): args = parseArgs(argv) + loglevels = (logging.WARNING, logging.INFO, logging.DEBUG) + loglevel = loglevels[min(args.verbose, len(loglevels) - 1)] + logger.setLevel(loglevel) + if not args.lazy_data_gen: if args.generate_lib_path is None: args.generate_lib_path = cmf.find_tosa_file( @@ -290,55 +326,98 @@ def main(argv=None): print( f"Argument error: Generate library (--generate-lib-path) not found - {str(args.generate_lib_path)}" ) - exit(2) + return 2 ttg = TosaTestGen(args) + # Determine if test selection mode is enabled or not + selectionMode = ( + args.selection_config is not None or args.selection_criteria is not None + ) + selectionCriteria = ( + "default" if args.selection_criteria is None else args.selection_criteria + ) + if args.selection_config is not None: + # Try loading the selection config + if not args.generate_lib_path.is_file(): + print( + f"Argument error: Test selection config (--test-selection-config) not found {str(args.selection_config)}" + ) + return 2 + with args.selection_config.open("r") as fd: + selectionCfg = json.load(fd) + else: + # Fallback to using anything defined in the TosaTestGen list + # by default this will mean only selecting a tests using a + # permutation of rank by type for each op + selectionCfg = ttg.TOSA_OP_LIST + if args.test_type == "both": testType = ["positive", "negative"] else: testType = [args.test_type] + results = [] for test_type in testType: - testList = [] + testList = tts.TestList(selectionCfg, selectionCriteria=selectionCriteria) try: for opName in ttg.TOSA_OP_LIST: if re.match(args.filter + ".*", opName): - testList.extend( - ttg.genOpTestList( - opName, - shapeFilter=args.target_shapes, - rankFilter=args.target_ranks, - dtypeFilter=args.target_dtypes, - testType=test_type, - ) + tests = ttg.genOpTestList( + opName, + shapeFilter=args.target_shapes, + rankFilter=args.target_ranks, + dtypeFilter=args.target_dtypes, + testType=test_type, ) + for testOpName, testStr, dtype, error, shapeList, argsDict in tests: + if "real_name" in ttg.TOSA_OP_LIST[testOpName]: + name = ttg.TOSA_OP_LIST[testOpName]["real_name"] + else: + name = testOpName + test = tts.Test( + name, testStr, dtype, error, shapeList, argsDict, testOpName + ) + testList.add(test) except Exception as e: - print(f"INTERNAL ERROR: Failure generating test lists for {opName}") + logger.error(f"INTERNAL ERROR: Failure generating test lists for {opName}") raise e - print("{} matching {} tests".format(len(testList), test_type)) + if not selectionMode: + # Allow all tests to be selected + tests = testList.all() + else: + # Use the random number generator to shuffle the test list + # and select the per op tests from it + tests = testList.select(ttg.rng) - testStrings = [] - try: - for opName, testStr, dtype, error, shapeList, argsDict in testList: - # Check for and skip duplicate tests - if testStr in testStrings: - print(f"Skipping duplicate test: {testStr}") - continue - else: - testStrings.append(testStr) + if args.list_tests: + for test in tests: + print(test) + continue + + print(f"{len(tests)} matching {test_type} tests") + try: + for test in tests: + opName = test.testOpName results.append( ttg.serializeTest( - opName, testStr, dtype, error, shapeList, argsDict + opName, + str(test), + test.dtype, + test.error, + test.shapeList, + test.argsDict, ) ) except Exception as e: - print(f"INTERNAL ERROR: Failure creating test output for {opName}") + logger.error(f"INTERNAL ERROR: Failure creating test output for {opName}") raise e - print(f"Done creating {len(results)} tests") + if not args.list_tests: + print(f"Done creating {len(results)} tests") + return 0 if __name__ == "__main__": |