1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
|
# Copyright (c) 2023, ARM Limited.
# SPDX-License-Identifier: Apache-2.0
"""Calls the data generation library to create the test data."""
import ctypes as ct
import json
from pathlib import Path
import numpy as np
import schemavalidation.schemavalidation as sch
class GenerateError(Exception):
"""Exception raised for errors performing data generation."""
class GenerateLibrary:
"""Python interface to the C generate library.
Simple usage to write out all input files:
set_config(test_desc)
write_numpy_files(test_path)
To get data buffers (for const data):
get_tensor_data(tensor_name)
"""
def __init__(self, generate_lib_path):
"""Find the library and set up the interface."""
self.lib_path = generate_lib_path
if self.lib_path is None or not self.lib_path.is_file():
raise GenerateError(f"Could not find generate library - {self.lib_path}")
self.schema_validator = sch.TestDescSchemaValidator()
self.test_desc = None
self.json_config = None
self.lib = ct.cdll.LoadLibrary(self.lib_path)
self.tgd_generate_data = self.lib.tgd_generate_data
self.tgd_generate_data.argtypes = [
ct.c_char_p,
ct.c_char_p,
ct.c_void_p,
ct.c_size_t,
]
self.tgd_generate_data.restype = ct.c_bool
def check_config(self, test_desc: dict):
"""Quick check that the config supports data generation."""
return ("meta" in test_desc) and ("data_gen" in test_desc["meta"])
def set_config(self, test_desc: dict):
"""Set the test config in the library.
test_desc - the test desc.json file
"""
self.test_desc = None
self.json_config = None
if not self.check_config(test_desc):
raise GenerateError("No meta/data_gen section found in desc.json")
# Validate the config versus the schema
self.schema_validator.validate_config(test_desc)
self.test_desc = test_desc
self.json_config = test_desc["meta"]["data_gen"]
def _create_buffer(self, dtype: str, shape: tuple):
"""Helper to create a buffer of the required type."""
size = np.prod(shape)
if dtype == "FP32":
# Create buffer and initialize to zero
buffer = (ct.c_float * size)(0)
size_bytes = size * 4
elif dtype == "FP16":
size_bytes = size * 2
# Create buffer of bytes and initialize to zero
buffer = (ct.c_ubyte * size_bytes)(0)
elif dtype == "INT32" or dtype == "SHAPE":
# Create buffer and initialize to zero
buffer = (ct.c_int32 * size)(0)
size_bytes = size * 4
else:
raise GenerateError(f"Unsupported data type {dtype}")
return buffer, size_bytes
def _convert_buffer(self, buffer, dtype: str, shape: tuple):
"""Helper to convert a buffer to a numpy array."""
arr = np.ctypeslib.as_array(buffer)
if dtype == "FP16":
# Convert from bytes back to FP16
arr = np.frombuffer(arr, np.float16)
arr = np.reshape(arr, shape)
return arr
def _data_gen_array(self, json_config: str, tensor_name: str):
"""Generate the named tensor data and return a numpy array."""
try:
tensor = json_config["tensors"][tensor_name]
dtype = tensor["data_type"]
shape = tuple(tensor["shape"])
except KeyError as e:
raise GenerateError(
f"Missing data in json config for input {tensor_name} - {repr(e)}"
)
buffer, size_bytes = self._create_buffer(dtype, shape)
buffer_ptr = ct.cast(buffer, ct.c_void_p)
json_bytes = bytes(json.dumps(json_config), "utf8")
result = self.tgd_generate_data(
ct.c_char_p(json_bytes),
ct.c_char_p(bytes(tensor_name, "utf8")),
buffer_ptr,
ct.c_size_t(size_bytes),
)
if not result:
raise GenerateError("Data generate failed")
arr = self._convert_buffer(buffer, dtype, shape)
return arr
def _data_gen_write(
self, test_path: Path, json_config: str, ifm_name: str, ifm_file: str
):
"""Generate the named tensor data and save it in numpy format."""
arr = self._data_gen_array(json_config, ifm_name)
file_name = test_path / ifm_file
np.save(file_name, arr)
def write_numpy_files(self, test_path: Path):
"""Write out all the desc.json input tensors to numpy data files."""
if self.test_desc is None or self.json_config is None:
raise GenerateError("Cannot write numpy files as no config set up")
try:
ifm_names = self.test_desc["ifm_name"]
ifm_files = self.test_desc["ifm_file"]
except KeyError as e:
raise GenerateError(f"Missing data in desc.json - {repr(e)}")
failures = []
for iname, ifile in zip(ifm_names, ifm_files):
try:
self._data_gen_write(test_path, self.json_config, iname, ifile)
except GenerateError as e:
failures.append(
f"ERROR: Failed to create data for tensor {iname} - {repr(e)}"
)
if len(failures) > 0:
raise GenerateError("\n".join(failures))
def get_tensor_data(self, tensor_name: str, json_config=None):
"""Get a numpy array for a named tensor in the data_gen meta data."""
if json_config is None:
if self.json_config is None:
raise GenerateError("Cannot get tensor data as no config set up")
json_config = self.json_config
else:
# Validate the given config
self.schema_validator.validate_config(
json_config, schema_type=sch.TD_SCHEMA_DATA_GEN
)
return self._data_gen_array(json_config, tensor_name)
def main(argv=None):
"""Simple command line interface for the data generator."""
import argparse
import conformance.model_files as cmf
parser = argparse.ArgumentParser()
parser.add_argument(
"--generate-lib-path",
type=Path,
help="Path to TOSA generate lib",
)
parser.add_argument(
"path", type=Path, help="the path to the test directory to generate data for"
)
args = parser.parse_args(argv)
test_path = args.path
if args.generate_lib_path is None:
# Try to work out ref model directory and find the verify library
# but this default only works for the python developer environment
# i.e. when using the scripts/py-dev-env.* scripts
# otherwise use the command line option --generate-lib-path to specify path
ref_model_dir = Path(__file__).absolute().parents[2]
args.generate_lib_path = cmf.find_tosa_file(
cmf.TosaFileType.GENERATE_LIBRARY, ref_model_dir, False
)
if not test_path.is_dir():
print(f"ERROR: Invalid directory - {test_path}")
return 2
test_desc_path = test_path / "desc.json"
if not test_desc_path.is_file():
print(f"ERROR: No test description found: {test_desc_path}")
return 2
# Load the JSON desc.json
try:
with test_desc_path.open("r") as fd:
test_desc = json.load(fd)
except Exception as e:
print(f"ERROR: Loading {test_desc_path} - {repr(e)}")
return 2
try:
dgl = GenerateLibrary(args.generate_lib_path)
if not dgl.check_config(test_desc):
print(f"WARNING: No data generation supported for {test_path}")
return 2
dgl.set_config(test_desc)
except GenerateError as e:
print(f"ERROR: Initializing generate library - {repr(e)}")
return 1
try:
dgl.write_numpy_files(test_path)
except GenerateError as e:
print(f"ERROR: Writing out data files to {test_path}\n{repr(e)}")
return 1
if __name__ == "__main__":
exit(main())
|
