Update tosa_verif_run_ref

Rename to tosa_verif_run_tests to match build_tests
Improve output and system under test support
Improve xunit support
Add results checker
Add utilities json2numpy and json2fbbin
Add set of python tests
Update README.md

Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: Ia09f8e6fd126579b3ba1c1cda95c1326802417ca

5 files changed, 350 insertions, 38 deletions

diff --git a/scripts/json2fbbin/__init__.py b/scripts/json2fbbin/__init__.py
new file mode 100644
index 0000000..39e9ecc
--- /dev/null
+++ b/scripts/json2fbbin/__init__.py
@@ -0,0 +1,3 @@
+"""Namespace."""
+# Copyright (c) 2021-2022 Arm Limited.
+# SPDX-License-Identifier: Apache-2.0
diff --git a/scripts/json2fbbin/json2fbbin.py b/scripts/json2fbbin/json2fbbin.py
new file mode 100644
index 0000000..957acb1
--- /dev/null
+++ b/scripts/json2fbbin/json2fbbin.py
@@ -0,0 +1,105 @@
+"""Conversion utility from flatbuffer JSON files to binary and the reverse."""
+# Copyright (c) 2021-2022, ARM Limited.
+# SPDX-License-Identifier: Apache-2.0
+from pathlib import Path
+from typing import Optional
+
+from runner.run_command import run_sh_command, RunShCommandError
+
+
+def fbbin_to_json(flatc: Path, fbs: Path, t_path: Path, o_path: Optional[Path] = None):
+    """Convert the binary flatbuffer to JSON.
+
+    flatc: the Path to the flatc compiler program
+    fbs: the Path to the fbs (flatbuffer schema) file
+    t_path: the Path to the binary flatbuffer file
+    o_path: the output Path where JSON file will be put, if None, it is same as t_path
+    """
+    if o_path is None:
+        o_path = t_path.parent
+    cmd = [
+        str(flatc.absolute()),
+        "-o",
+        str(o_path.absolute()),
+        "--json",
+        "--defaults-json",
+        "--raw-binary",
+        str(fbs.absolute()),
+        "--",
+        str(t_path.absolute()),
+    ]
+    run_sh_command(verbose=False, full_cmd=cmd)
+
+
+def json_to_fbbin(flatc: Path, fbs: Path, j_path: Path, o_path: Optional[Path] = None):
+    """Convert JSON flatbuffer to binary.
+
+    flatc: the Path to the flatc compiler program
+    fbs: the Path to the fbs (flatbuffer schema) file
+    j_path: the Path to the JSON flatbuffer file
+    o_path: the output Path where JSON file will be put, if None, it is same as j_path
+    """
+    if o_path is None:
+        o_path = j_path.parent
+    cmd = [
+        str(flatc.absolute()),
+        "-o",
+        str(o_path.absolute()),
+        "--binary",
+        str(fbs.absolute()),
+        str(j_path.absolute()),
+    ]
+    run_sh_command(verbose=False, full_cmd=cmd)
+
+
+# ------------------------------------------------------------------------------
+
+
+def main(argv=None):
+    """Load and convert supplied file based on file suffix."""
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--flatc",
+        type=Path,
+        default="reference_model/build/thirdparty/serialization_lib/third_party/flatbuffers/flatc",
+        help="the path to the flatc compiler program",
+    )
+    parser.add_argument(
+        "--fbs",
+        type=Path,
+        default="conformance_tests/third_party/serialization_lib/schema/tosa.fbs",
+        help="the path to the flatbuffer schema",
+    )
+    parser.add_argument("path", type=Path, help="the path to the file to convert")
+    args = parser.parse_args(argv)
+    path = args.path
+
+    if not path.is_file():
+        print(f"Invalid file to convert - {path}")
+        return 2
+
+    if not args.flatc.is_file():
+        print(f"Invalid flatc compiler - {args.flatc}")
+        return 2
+
+    if not args.fbs.is_file():
+        print(f"Invalid flatbuffer schema - {args.fbs}")
+        return 2
+
+    try:
+        if path.suffix == ".json":
+            json_to_fbbin(args.flatc, args.fbs, path)
+        else:
+            # Have to assume this is a binary flatbuffer file as could have any suffix
+            fbbin_to_json(args.flatc, args.fbs, path)
+    except RunShCommandError as e:
+        print(e)
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    exit(main())
diff --git a/scripts/json2numpy/__init__.py b/scripts/json2numpy/__init__.py
new file mode 100644
index 0000000..39e9ecc
--- /dev/null
+++ b/scripts/json2numpy/__init__.py
@@ -0,0 +1,3 @@
+"""Namespace."""
+# Copyright (c) 2021-2022 Arm Limited.
+# SPDX-License-Identifier: Apache-2.0
diff --git a/scripts/json2numpy/json2numpy.py b/scripts/json2numpy/json2numpy.py
new file mode 100644
index 0000000..21b1acd
--- /dev/null
+++ b/scripts/json2numpy/json2numpy.py
@@ -0,0 +1,180 @@
+"""Conversion utility from binary numpy files to JSON and the reverse."""
+# Copyright (c) 2021-2022, ARM Limited.
+# SPDX-License-Identifier: Apache-2.0
+import json
+from pathlib import Path
+from typing import Optional
+from typing import Union
+
+import numpy as np
+
+
+class NumpyArrayEncoder(json.JSONEncoder):
+    """A JSON encoder for Numpy data types."""
+
+    def default(self, obj):
+        """Encode default operation."""
+        if isinstance(obj, np.integer):
+            return int(obj)
+        elif isinstance(obj, np.floating):
+            return float(obj)
+        elif isinstance(obj, np.ndarray):
+            return obj.tolist()
+        return super(NumpyArrayEncoder, self).default(obj)
+
+
+def get_shape(t: Union[list, tuple]):
+    """Get the shape of an N-Dimensional tensor."""
+    # TODO: validate shape is consistent for all rows and ccolumns
+    if isinstance(t, (list, tuple)) and t:
+        return [len(t)] + get_shape(t[0])
+    return []
+
+
+def npy_to_json(n_path: Path, j_path: Optional[Path] = None):
+    """Load a numpy data file and save it as a JSON file.
+
+    n_path: the Path to the numpy file
+    j_path: the Path to the JSON file, if None, it is derived from n_path
+    """
+    if not j_path:
+        j_path = n_path.parent / (n_path.stem + ".json")
+    with open(n_path, "rb") as fd:
+        data = np.load(fd)
+    jdata = {
+        "type": data.dtype.name,
+        "data": data.tolist(),
+    }
+    with open(j_path, "w") as fp:
+        json.dump(jdata, fp, indent=2)
+
+
+def json_to_npy(j_path: Path, n_path: Optional[Path] = None):
+    """Load a JSON file and save it as a numpy data file.
+
+    j_path: the Path to the JSON file
+    n_path: the Path to the numpy file, if None, it is derived from j_path
+    """
+    if not n_path:
+        n_path = j_path.parent / (j_path.stem + ".npy")
+    with open(j_path, "rb") as fd:
+        jdata = json.load(fd)
+    raw_data = jdata["data"]
+    raw_type = jdata["type"]
+    shape = get_shape(raw_data)
+    data = np.asarray(raw_data).reshape(shape).astype(raw_type)
+    with open(n_path, "wb") as fd:
+        np.save(fd, data)
+
+
+# ------------------------------------------------------------------------------
+
+
+def test():
+    """Test conversion routines."""
+    shape = [2, 3, 4]
+    elements = 1
+    for i in shape:
+        elements *= i
+
+    # file names
+    n_path = Path("data.npy")
+    j_path = Path("data.json")
+    j2n_path = Path("data_j2n.npy")
+
+    datatypes = [
+        np.bool_,
+        np.int8,
+        np.int16,
+        np.int32,
+        np.int64,
+        np.uint8,
+        np.uint16,
+        np.uint32,
+        np.uint64,
+        np.float16,
+        np.float32,
+        np.float64,
+        # np.float128,
+        # np.complex64,
+        # np.complex128,
+        # np.complex256,
+        # np.datetime64,
+        # np.str,
+    ]
+
+    for data_type in datatypes:
+        dt = np.dtype(data_type)
+        print(data_type, dt, dt.char, dt.num, dt.name, dt.str)
+
+        # create a tensor of the given shape
+        tensor = np.arange(elements).reshape(shape).astype(data_type)
+        # print(tensor)
+
+        # save the tensor in a binary numpy file
+        with open(n_path, "wb") as fd:
+            np.save(fd, tensor)
+
+        # read back the numpy file for verification
+        with open(n_path, "rb") as fd:
+            tensor1 = np.load(fd)
+
+        # confirm the loaded tensor matches the original
+        assert tensor.shape == tensor1.shape
+        assert tensor.dtype == tensor1.dtype
+        assert (tensor == tensor1).all()
+
+        # convert the numpy file to json
+        npy_to_json(n_path, j_path)
+
+        # convert the json file to numpy
+        json_to_npy(j_path, j2n_path)
+
+        # read back the json-to-numpy file for verification
+        with open(j2n_path, "rb") as fd:
+            tensor1 = np.load(fd)
+
+        # confirm the loaded tensor matches the original
+        assert tensor.shape == tensor1.shape
+        assert tensor.dtype == tensor1.dtype
+        assert (tensor == tensor1).all()
+
+    # delete the files, if no problems were found
+    # they are left for debugging if any of the asserts failed
+    n_path.unlink()
+    j_path.unlink()
+    j2n_path.unlink()
+    return 0
+
+
+def main(argv=None):
+    """Load and convert supplied file based on file suffix."""
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "path", type=Path, help="the path to the file to convert, or 'test'"
+    )
+    args = parser.parse_args(argv)
+    path = args.path
+    if str(path) == "test":
+        print("test")
+        return test()
+
+    if not path.is_file():
+        print(f"Invalid file - {path}")
+        return 2
+
+    if path.suffix == ".npy":
+        npy_to_json(path)
+    elif path.suffix == ".json":
+        json_to_npy(path)
+    else:
+        print("Unknown file type - {path.suffix}")
+        return 2
+
+    return 0
+
+
+if __name__ == "__main__":
+    exit(main())
diff --git a/scripts/xunit/xunit.py b/scripts/xunit/xunit.py
index c636136..540ef56 100644
--- a/scripts/xunit/xunit.py
+++ b/scripts/xunit/xunit.py
@@ -1,91 +1,112 @@
+"""Simple xunit results file creator utility."""
+# Copyright (c) 2020-2022, ARM Limited.
+# SPDX-License-Identifier: Apache-2.0
+import xml.etree.ElementTree as ET
+from xml.dom import minidom
 
 
-# Copyright (c) 2020, ARM Limited.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-
-from __future__ import print_function
-import xml.etree.ElementTree as ET
+class xunit_results:
+    """Xunit results writer."""
 
-class xunit_results():
-    def __init__(self, name='Testsuites'):
-        self.name = name
+    def __init__(self):
+        """Initialize results."""
+        self.name = "testsuites"
         self.suites = []
+
     def create_suite(self, name):
+        """Create xunit suite for results."""
         s = xunit_suite(name)
         self.suites.append(s)
         return s
+
     def write_results(self, filename):
+        """Write the results to the appropriate suites."""
         suites = ET.Element(self.name)
         tree = ET.ElementTree(suites)
         for s in self.suites:
-            testsuite = ET.SubElement(suites, 'testsuite', {'name' : s.name, 'errors' : '0'})
+            testsuite = ET.SubElement(
+                suites, "testsuite", {"name": s.name, "errors": "0"}
+            )
             tests = 0
             failures = 0
             skip = 0
             for t in s.tests:
-                test = ET.SubElement(testsuite, 'testcase', {'name' : t.name, 'classname' : t.classname, 'time' : t.time})
+                test = ET.SubElement(
+                    testsuite,
+                    "testcase",
+                    {"name": t.name, "classname": t.classname, "time": t.time},
+                )
                 tests += 1
                 if t.skip:
                     skip += 1
-                    ET.SubElement(test, 'skipped', {'type' : 'Skipped test'})
+                    ET.SubElement(test, "skipped", {"type": "Skipped test"})
                 if t.fail:
                     failures += 1
-                    fail = ET.SubElement(test, 'failure', {'type' : 'Test failed'})
+                    fail = ET.SubElement(test, "failure", {"type": "Test failed"})
                     fail.text = t.fail
                 if t.sysout:
-                    sysout = ET.SubElement(test, 'system-out')
+                    sysout = ET.SubElement(test, "system-out")
                     sysout.text = t.sysout
                 if t.syserr:
-                    syserr = ET.SubElement(test, 'system-err')
+                    syserr = ET.SubElement(test, "system-err")
                     syserr.text = t.syserr
-            testsuite.attrib['tests'] = str(tests)
-            testsuite.attrib['failures'] = str(failures)
-            testsuite.attrib['skip'] = str(skip)
-        tree.write(filename, 'UTF-8', True)
+            testsuite.attrib["tests"] = str(tests)
+            testsuite.attrib["failures"] = str(failures)
+            testsuite.attrib["skip"] = str(skip)
+        xmlstr = minidom.parseString(ET.tostring(tree.getroot())).toprettyxml(
+            indent="  "
+        )
+        with open(filename, "w") as f:
+            f.write(xmlstr)
+
 
+class xunit_suite:
+    """Xunit suite for test results."""
 
-class xunit_suite():
     def __init__(self, name):
+        """Initialize suite."""
         self.name = name
         self.tests = []
 
-class xunit_test():
+
+# classname should be of the form suite.class/subclass/subclass2/...
+# You can have an unlimited number of subclasses in this manner
+
+
+class xunit_test:
+    """Xunit test result."""
+
     def __init__(self, name, classname=None):
+        """Initialize test."""
         self.name = name
         if classname:
             self.classname = classname
         else:
             self.classname = name
-        self.time = '0.000'
+        self.time = "0.000"
         self.fail = None
         self.skip = False
         self.sysout = None
         self.syserr = None
+
     def failed(self, text):
+        """Set test failed information."""
         self.fail = text
+
     def skipped(self):
+        """Set test as skipped."""
         self.skip = True
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
+    # Simple test
     r = xunit_results()
-    s = r.create_suite('selftest')
-    for i in range(0,10):
-        t = xunit_test('atest' + str(i))
+    s = r.create_suite("selftest")
+    for i in range(0, 10):
+        t = xunit_test("atest" + str(i), "selftest")
         if i == 3:
-            t.failed('Unknown failure foo')
+            t.failed("Unknown failure foo")
         if i == 7:
             t.skipped()
         s.tests.append(t)
-    r.write_results('foo.xml')
+    r.write_results("foo.xml")