1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
|
# Copyright (c) 2024, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
import copy
import logging
logging.basicConfig()
logger = logging.getLogger("tosa_verif_build_tests")
class Test:
"""Test container to allow group and permute selection."""
def __init__(
self, opName, testStr, dtype, error, shapeList, argsDict, testOpName=None
):
self.opName = opName
self.testStr = testStr
self.dtype = dtype
self.error = error
self.shapeList = shapeList
self.argsDict = argsDict
# Given test op name used for look up in TOSA_OP_LIST for "conv2d_1x1" for example
self.testOpName = testOpName if testOpName is not None else opName
self.key = None
self.groupKey = None
self.mark = False
def __str__(self):
return self.testStr
def __lt__(self, other):
return self.testStr < str(other)
def getArg(self, param):
# Get parameter values (arguments) for this test
if param == "rank":
return len(self.shapeList[0])
elif param == "dtype":
if isinstance(self.dtype, list):
return tuple(self.dtype)
return self.dtype
elif param == "shape" and "shape" not in self.argsDict:
return str(self.shapeList[0])
if param in self.argsDict:
# Turn other args into hashable string without newlines
val = str(self.argsDict[param])
return ",".join(str(val).splitlines())
else:
return None
def setKey(self, keyParams):
if self.error is None:
# Create the main key based on primary parameters
key = [self.getArg(param) for param in keyParams]
self.key = tuple(key)
else:
# Use the error as the key
self.key = self.error
return self.key
def getKey(self):
return self.key
def setGroupKey(self, groupParams):
# Create the group key based on arguments that do not define the group
# Therefore this test will match other tests that have the same arguments
# that are NOT the group arguments (group arguments like test set number)
paramsList = sorted(["shape", "dtype"] + list(self.argsDict.keys()))
key = []
for param in paramsList:
if param in groupParams:
continue
key.append(self.getArg(param))
self.groupKey = tuple(key)
return self.groupKey
def getGroupKey(self):
return self.groupKey
def inGroup(self, groupKey):
return self.groupKey == groupKey
def setMark(self):
# Marks the test as important
self.mark = True
def getMark(self):
return self.mark
def isError(self):
return self.error is not None
def _get_selection_info_from_op(op, selectionCriteria, item, default):
# Get selection info from the op
if (
"selection" in op
and selectionCriteria in op["selection"]
and item in op["selection"][selectionCriteria]
):
return op["selection"][selectionCriteria][item]
else:
return default
def _get_tests_by_group(tests):
# Create simple structures to record the tests in groups
groups = []
group_tests = {}
for test in tests:
key = test.getGroupKey()
if key in group_tests:
group_tests[key].append(test)
else:
group_tests[key] = [test]
groups.append(key)
# Return list of test groups (group keys) and a dictionary with a list of tests
# associated with each group key
return groups, group_tests
def _get_specific_op_info(opName, opSelectionInfo, testOpName):
# Get the op specific section from the selection config
name = opName if opName in opSelectionInfo else testOpName
if name not in opSelectionInfo:
logger.info(f"No op entry found for {opName} in test selection config")
return {}
return opSelectionInfo[name]
class TestOpList:
"""All the tests for one op grouped by permutations."""
def __init__(self, opName, opSelectionInfo, selectionCriteria, testOpName):
self.opName = opName
self.testOpName = testOpName
op = _get_specific_op_info(opName, opSelectionInfo, testOpName)
# See verif/conformance/README.md for more information on
# these selection arguments
self.permuteArgs = _get_selection_info_from_op(
op, selectionCriteria, "permutes", ["rank", "dtype"]
)
self.paramArgs = _get_selection_info_from_op(
op, selectionCriteria, "full_params", []
)
self.specificArgs = _get_selection_info_from_op(
op, selectionCriteria, "specifics", {}
)
self.groupArgs = _get_selection_info_from_op(
op, selectionCriteria, "groups", ["s"]
)
self.maximumPerPermute = _get_selection_info_from_op(
op, selectionCriteria, "maximum", None
)
self.numErrorIfs = _get_selection_info_from_op(
op, selectionCriteria, "num_errorifs", 1
)
self.selectAll = _get_selection_info_from_op(
op, selectionCriteria, "all", False
)
if self.paramArgs and self.maximumPerPermute > 1:
logger.warning(f"Unsupported - selection params AND maximum for {opName}")
self.tests = []
self.testStrings = set()
self.shapes = set()
self.permutes = set()
self.testsPerPermute = {}
self.paramsPerPermute = {}
self.specificsPerPermute = {}
self.selectionDone = False
def __len__(self):
return len(self.tests)
def add(self, test):
# Add a test to this op group and set up the permutations/group for it
assert test.opName.startswith(self.opName)
if str(test) in self.testStrings:
logger.info(f"Skipping duplicate test: {str(test)}")
return
self.tests.append(test)
self.testStrings.add(str(test))
self.shapes.add(test.getArg("shape"))
# Work out the permutation key for this test
permute = test.setKey(self.permuteArgs)
# Set up the group key for the test (for pulling out groups during selection)
test.setGroupKey(self.groupArgs)
if permute not in self.permutes:
# New permutation
self.permutes.add(permute)
# Set up area to record the selected tests
self.testsPerPermute[permute] = []
if self.paramArgs:
# Set up area to record the unique test params found
self.paramsPerPermute[permute] = {}
for param in self.paramArgs:
self.paramsPerPermute[permute][param] = set()
# Set up copy of the specific test args for selecting these
self.specificsPerPermute[permute] = copy.deepcopy(self.specificArgs)
def _init_select(self):
# Can only perform the selection process once as it alters the permute
# information set at init
assert not self.selectionDone
# Count of non-specific tests added to each permute (not error)
if not self.selectAll:
countPerPermute = {permute: 0 for permute in self.permutes}
# Go through each test looking for permutes, unique params & specifics
for test in self.tests:
permute = test.getKey()
append = False
possible_append = False
if test.isError():
# Error test, choose up to number of tests
if len(self.testsPerPermute[permute]) < self.numErrorIfs:
append = True
else:
if self.selectAll:
append = True
else:
# See if this is a specific test to add
for param, values in self.specificsPerPermute[permute].items():
arg = test.getArg(param)
# Iterate over a copy of the values, so we can remove them from the original
if arg in values.copy():
# Found a match, remove it, so we don't look for it later
values.remove(arg)
# Mark the test as special (and so shouldn't be removed)
test.setMark()
append = True
if self.paramArgs:
# See if this test contains any new params we should keep
# Perform this check even if we have already selected the test
# so we can record the params found
for param in self.paramArgs:
arg = test.getArg(param)
if arg not in self.paramsPerPermute[permute][param]:
# We have found a new value for this arg, record it
self.paramsPerPermute[permute][param].add(arg)
possible_append = True
else:
# No params set, so possible test to add up to maximum
possible_append = True
if (not append and possible_append) and (
self.maximumPerPermute is None
or countPerPermute[permute] < self.maximumPerPermute
):
# Not selected but could be added and we have space left if
# a maximum is set.
append = True
countPerPermute[permute] += 1
# Check for grouping with chosen tests
if not append:
# We will keep any tests together than form a group
key = test.getGroupKey()
for t in self.testsPerPermute[permute]:
if t.getGroupKey() == key:
if t.getMark():
test.setMark()
append = True
if append:
self.testsPerPermute[permute].append(test)
self.selectionDone = True
def select(self, rng=None):
# Create selection of tests with optional shuffle
if not self.selectionDone:
if rng:
rng.shuffle(self.tests)
self._init_select()
# Now create the full list of selected tests per permute
selection = []
for permute, tests in self.testsPerPermute.items():
selection.extend(tests)
return selection
def all(self):
# Un-selected list of tests - i.e. all of them
return self.tests
class TestList:
"""List of all tests grouped by operator."""
def __init__(self, opSelectionInfo, selectionCriteria="default"):
self.opLists = {}
self.opSelectionInfo = opSelectionInfo
self.selectionCriteria = selectionCriteria
def __len__(self):
length = 0
for opName in self.opLists.keys():
length += len(self.opLists[opName])
return length
def add(self, test):
if test.opName not in self.opLists:
self.opLists[test.opName] = TestOpList(
test.opName,
self.opSelectionInfo,
self.selectionCriteria,
test.testOpName,
)
self.opLists[test.opName].add(test)
def _get_tests(self, selectMode, rng):
selection = []
for opList in self.opLists.values():
if selectMode:
tests = opList.select(rng=rng)
else:
tests = opList.all()
selection.extend(tests)
if selectMode:
selection = sorted(selection)
return selection
def select(self, rng=None):
return self._get_tests(True, rng)
def all(self):
return self._get_tests(False, None)
|