From be1a9408eb53871d96a022f59664f016926a8cf4 Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Wed, 15 Dec 2021 17:14:56 +0000 Subject: Update tosa_verif_run_ref Rename to tosa_verif_run_tests to match build_tests Improve output and system under test support Improve xunit support Add results checker Add utilities json2numpy and json2fbbin Add set of python tests Update README.md Signed-off-by: Jeremy Johnson Change-Id: Ia09f8e6fd126579b3ba1c1cda95c1326802417ca --- scripts/json2fbbin/__init__.py | 3 + scripts/json2fbbin/json2fbbin.py | 105 +++++++++++++++++++++++ scripts/json2numpy/__init__.py | 3 + scripts/json2numpy/json2numpy.py | 180 +++++++++++++++++++++++++++++++++++++++ scripts/xunit/xunit.py | 97 ++++++++++++--------- 5 files changed, 350 insertions(+), 38 deletions(-) create mode 100644 scripts/json2fbbin/__init__.py create mode 100644 scripts/json2fbbin/json2fbbin.py create mode 100644 scripts/json2numpy/__init__.py create mode 100644 scripts/json2numpy/json2numpy.py (limited to 'scripts') diff --git a/scripts/json2fbbin/__init__.py b/scripts/json2fbbin/__init__.py new file mode 100644 index 0000000..39e9ecc --- /dev/null +++ b/scripts/json2fbbin/__init__.py @@ -0,0 +1,3 @@ +"""Namespace.""" +# Copyright (c) 2021-2022 Arm Limited. +# SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/json2fbbin/json2fbbin.py b/scripts/json2fbbin/json2fbbin.py new file mode 100644 index 0000000..957acb1 --- /dev/null +++ b/scripts/json2fbbin/json2fbbin.py @@ -0,0 +1,105 @@ +"""Conversion utility from flatbuffer JSON files to binary and the reverse.""" +# Copyright (c) 2021-2022, ARM Limited. +# SPDX-License-Identifier: Apache-2.0 +from pathlib import Path +from typing import Optional + +from runner.run_command import run_sh_command, RunShCommandError + + +def fbbin_to_json(flatc: Path, fbs: Path, t_path: Path, o_path: Optional[Path] = None): + """Convert the binary flatbuffer to JSON. + + flatc: the Path to the flatc compiler program + fbs: the Path to the fbs (flatbuffer schema) file + t_path: the Path to the binary flatbuffer file + o_path: the output Path where JSON file will be put, if None, it is same as t_path + """ + if o_path is None: + o_path = t_path.parent + cmd = [ + str(flatc.absolute()), + "-o", + str(o_path.absolute()), + "--json", + "--defaults-json", + "--raw-binary", + str(fbs.absolute()), + "--", + str(t_path.absolute()), + ] + run_sh_command(verbose=False, full_cmd=cmd) + + +def json_to_fbbin(flatc: Path, fbs: Path, j_path: Path, o_path: Optional[Path] = None): + """Convert JSON flatbuffer to binary. + + flatc: the Path to the flatc compiler program + fbs: the Path to the fbs (flatbuffer schema) file + j_path: the Path to the JSON flatbuffer file + o_path: the output Path where JSON file will be put, if None, it is same as j_path + """ + if o_path is None: + o_path = j_path.parent + cmd = [ + str(flatc.absolute()), + "-o", + str(o_path.absolute()), + "--binary", + str(fbs.absolute()), + str(j_path.absolute()), + ] + run_sh_command(verbose=False, full_cmd=cmd) + + +# ------------------------------------------------------------------------------ + + +def main(argv=None): + """Load and convert supplied file based on file suffix.""" + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--flatc", + type=Path, + default="reference_model/build/thirdparty/serialization_lib/third_party/flatbuffers/flatc", + help="the path to the flatc compiler program", + ) + parser.add_argument( + "--fbs", + type=Path, + default="conformance_tests/third_party/serialization_lib/schema/tosa.fbs", + help="the path to the flatbuffer schema", + ) + parser.add_argument("path", type=Path, help="the path to the file to convert") + args = parser.parse_args(argv) + path = args.path + + if not path.is_file(): + print(f"Invalid file to convert - {path}") + return 2 + + if not args.flatc.is_file(): + print(f"Invalid flatc compiler - {args.flatc}") + return 2 + + if not args.fbs.is_file(): + print(f"Invalid flatbuffer schema - {args.fbs}") + return 2 + + try: + if path.suffix == ".json": + json_to_fbbin(args.flatc, args.fbs, path) + else: + # Have to assume this is a binary flatbuffer file as could have any suffix + fbbin_to_json(args.flatc, args.fbs, path) + except RunShCommandError as e: + print(e) + return 1 + + return 0 + + +if __name__ == "__main__": + exit(main()) diff --git a/scripts/json2numpy/__init__.py b/scripts/json2numpy/__init__.py new file mode 100644 index 0000000..39e9ecc --- /dev/null +++ b/scripts/json2numpy/__init__.py @@ -0,0 +1,3 @@ +"""Namespace.""" +# Copyright (c) 2021-2022 Arm Limited. +# SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/json2numpy/json2numpy.py b/scripts/json2numpy/json2numpy.py new file mode 100644 index 0000000..21b1acd --- /dev/null +++ b/scripts/json2numpy/json2numpy.py @@ -0,0 +1,180 @@ +"""Conversion utility from binary numpy files to JSON and the reverse.""" +# Copyright (c) 2021-2022, ARM Limited. +# SPDX-License-Identifier: Apache-2.0 +import json +from pathlib import Path +from typing import Optional +from typing import Union + +import numpy as np + + +class NumpyArrayEncoder(json.JSONEncoder): + """A JSON encoder for Numpy data types.""" + + def default(self, obj): + """Encode default operation.""" + if isinstance(obj, np.integer): + return int(obj) + elif isinstance(obj, np.floating): + return float(obj) + elif isinstance(obj, np.ndarray): + return obj.tolist() + return super(NumpyArrayEncoder, self).default(obj) + + +def get_shape(t: Union[list, tuple]): + """Get the shape of an N-Dimensional tensor.""" + # TODO: validate shape is consistent for all rows and ccolumns + if isinstance(t, (list, tuple)) and t: + return [len(t)] + get_shape(t[0]) + return [] + + +def npy_to_json(n_path: Path, j_path: Optional[Path] = None): + """Load a numpy data file and save it as a JSON file. + + n_path: the Path to the numpy file + j_path: the Path to the JSON file, if None, it is derived from n_path + """ + if not j_path: + j_path = n_path.parent / (n_path.stem + ".json") + with open(n_path, "rb") as fd: + data = np.load(fd) + jdata = { + "type": data.dtype.name, + "data": data.tolist(), + } + with open(j_path, "w") as fp: + json.dump(jdata, fp, indent=2) + + +def json_to_npy(j_path: Path, n_path: Optional[Path] = None): + """Load a JSON file and save it as a numpy data file. + + j_path: the Path to the JSON file + n_path: the Path to the numpy file, if None, it is derived from j_path + """ + if not n_path: + n_path = j_path.parent / (j_path.stem + ".npy") + with open(j_path, "rb") as fd: + jdata = json.load(fd) + raw_data = jdata["data"] + raw_type = jdata["type"] + shape = get_shape(raw_data) + data = np.asarray(raw_data).reshape(shape).astype(raw_type) + with open(n_path, "wb") as fd: + np.save(fd, data) + + +# ------------------------------------------------------------------------------ + + +def test(): + """Test conversion routines.""" + shape = [2, 3, 4] + elements = 1 + for i in shape: + elements *= i + + # file names + n_path = Path("data.npy") + j_path = Path("data.json") + j2n_path = Path("data_j2n.npy") + + datatypes = [ + np.bool_, + np.int8, + np.int16, + np.int32, + np.int64, + np.uint8, + np.uint16, + np.uint32, + np.uint64, + np.float16, + np.float32, + np.float64, + # np.float128, + # np.complex64, + # np.complex128, + # np.complex256, + # np.datetime64, + # np.str, + ] + + for data_type in datatypes: + dt = np.dtype(data_type) + print(data_type, dt, dt.char, dt.num, dt.name, dt.str) + + # create a tensor of the given shape + tensor = np.arange(elements).reshape(shape).astype(data_type) + # print(tensor) + + # save the tensor in a binary numpy file + with open(n_path, "wb") as fd: + np.save(fd, tensor) + + # read back the numpy file for verification + with open(n_path, "rb") as fd: + tensor1 = np.load(fd) + + # confirm the loaded tensor matches the original + assert tensor.shape == tensor1.shape + assert tensor.dtype == tensor1.dtype + assert (tensor == tensor1).all() + + # convert the numpy file to json + npy_to_json(n_path, j_path) + + # convert the json file to numpy + json_to_npy(j_path, j2n_path) + + # read back the json-to-numpy file for verification + with open(j2n_path, "rb") as fd: + tensor1 = np.load(fd) + + # confirm the loaded tensor matches the original + assert tensor.shape == tensor1.shape + assert tensor.dtype == tensor1.dtype + assert (tensor == tensor1).all() + + # delete the files, if no problems were found + # they are left for debugging if any of the asserts failed + n_path.unlink() + j_path.unlink() + j2n_path.unlink() + return 0 + + +def main(argv=None): + """Load and convert supplied file based on file suffix.""" + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "path", type=Path, help="the path to the file to convert, or 'test'" + ) + args = parser.parse_args(argv) + path = args.path + if str(path) == "test": + print("test") + return test() + + if not path.is_file(): + print(f"Invalid file - {path}") + return 2 + + if path.suffix == ".npy": + npy_to_json(path) + elif path.suffix == ".json": + json_to_npy(path) + else: + print("Unknown file type - {path.suffix}") + return 2 + + return 0 + + +if __name__ == "__main__": + exit(main()) diff --git a/scripts/xunit/xunit.py b/scripts/xunit/xunit.py index c636136..540ef56 100644 --- a/scripts/xunit/xunit.py +++ b/scripts/xunit/xunit.py @@ -1,91 +1,112 @@ +"""Simple xunit results file creator utility.""" +# Copyright (c) 2020-2022, ARM Limited. +# SPDX-License-Identifier: Apache-2.0 +import xml.etree.ElementTree as ET +from xml.dom import minidom -# Copyright (c) 2020, ARM Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function -import xml.etree.ElementTree as ET +class xunit_results: + """Xunit results writer.""" -class xunit_results(): - def __init__(self, name='Testsuites'): - self.name = name + def __init__(self): + """Initialize results.""" + self.name = "testsuites" self.suites = [] + def create_suite(self, name): + """Create xunit suite for results.""" s = xunit_suite(name) self.suites.append(s) return s + def write_results(self, filename): + """Write the results to the appropriate suites.""" suites = ET.Element(self.name) tree = ET.ElementTree(suites) for s in self.suites: - testsuite = ET.SubElement(suites, 'testsuite', {'name' : s.name, 'errors' : '0'}) + testsuite = ET.SubElement( + suites, "testsuite", {"name": s.name, "errors": "0"} + ) tests = 0 failures = 0 skip = 0 for t in s.tests: - test = ET.SubElement(testsuite, 'testcase', {'name' : t.name, 'classname' : t.classname, 'time' : t.time}) + test = ET.SubElement( + testsuite, + "testcase", + {"name": t.name, "classname": t.classname, "time": t.time}, + ) tests += 1 if t.skip: skip += 1 - ET.SubElement(test, 'skipped', {'type' : 'Skipped test'}) + ET.SubElement(test, "skipped", {"type": "Skipped test"}) if t.fail: failures += 1 - fail = ET.SubElement(test, 'failure', {'type' : 'Test failed'}) + fail = ET.SubElement(test, "failure", {"type": "Test failed"}) fail.text = t.fail if t.sysout: - sysout = ET.SubElement(test, 'system-out') + sysout = ET.SubElement(test, "system-out") sysout.text = t.sysout if t.syserr: - syserr = ET.SubElement(test, 'system-err') + syserr = ET.SubElement(test, "system-err") syserr.text = t.syserr - testsuite.attrib['tests'] = str(tests) - testsuite.attrib['failures'] = str(failures) - testsuite.attrib['skip'] = str(skip) - tree.write(filename, 'UTF-8', True) + testsuite.attrib["tests"] = str(tests) + testsuite.attrib["failures"] = str(failures) + testsuite.attrib["skip"] = str(skip) + xmlstr = minidom.parseString(ET.tostring(tree.getroot())).toprettyxml( + indent=" " + ) + with open(filename, "w") as f: + f.write(xmlstr) + +class xunit_suite: + """Xunit suite for test results.""" -class xunit_suite(): def __init__(self, name): + """Initialize suite.""" self.name = name self.tests = [] -class xunit_test(): + +# classname should be of the form suite.class/subclass/subclass2/... +# You can have an unlimited number of subclasses in this manner + + +class xunit_test: + """Xunit test result.""" + def __init__(self, name, classname=None): + """Initialize test.""" self.name = name if classname: self.classname = classname else: self.classname = name - self.time = '0.000' + self.time = "0.000" self.fail = None self.skip = False self.sysout = None self.syserr = None + def failed(self, text): + """Set test failed information.""" self.fail = text + def skipped(self): + """Set test as skipped.""" self.skip = True -if __name__ == '__main__': +if __name__ == "__main__": + # Simple test r = xunit_results() - s = r.create_suite('selftest') - for i in range(0,10): - t = xunit_test('atest' + str(i)) + s = r.create_suite("selftest") + for i in range(0, 10): + t = xunit_test("atest" + str(i), "selftest") if i == 3: - t.failed('Unknown failure foo') + t.failed("Unknown failure foo") if i == 7: t.skipped() s.tests.append(t) - r.write_results('foo.xml') + r.write_results("foo.xml") -- cgit v1.2.1