Update tosa_verif_run_ref

Rename to tosa_verif_run_tests to match build_tests
Improve output and system under test support
Improve xunit support
Add results checker
Add utilities json2numpy and json2fbbin
Add set of python tests
Update README.md

Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: Ia09f8e6fd126579b3ba1c1cda95c1326802417ca

1 files changed, 73 insertions, 0 deletions

diff --git a/verif/runner/tosa_refmodel_sut_run.py b/verif/runner/tosa_refmodel_sut_run.py
new file mode 100644
index 0000000..b9a9575
--- /dev/null
+++ b/verif/runner/tosa_refmodel_sut_run.py
@@ -0,0 +1,73 @@
+"""TOSA test runner module for the Reference Model."""
+# Copyright (c) 2020-2022, ARM Limited.
+# SPDX-License-Identifier: Apache-2.0
+from enum import IntEnum
+from enum import unique
+
+from runner.run_command import run_sh_command
+from runner.run_command import RunShCommandError
+from runner.tosa_test_runner import TosaTestRunner
+
+
+@unique
+class TosaRefReturnCode(IntEnum):
+    """Return codes from the Tosa Reference Model."""
+
+    VALID = 0
+    UNPREDICTABLE = 1
+    ERROR = 2
+
+
+class TosaSUTRunner(TosaTestRunner):
+    """TOSA Reference Model runner."""
+
+    def __init__(self, args, runnerArgs, testDir):
+        """Initialize using the given test details."""
+        super().__init__(args, runnerArgs, testDir)
+
+    def runTestGraph(self):
+        """Run the test on the reference model."""
+        # Build up the TOSA reference command line
+        # Uses arguments from the argParser args, not the runnerArgs
+        args = self.args
+
+        # Call Reference model with description file to provide all file details
+        cmd = [
+            args.ref_model_path,
+            "-Coperator_fbs={}".format(args.operator_fbs),
+            "-Ctest_desc={}".format(self.descFile),
+        ]
+
+        # Specific debug options for reference model
+        if args.ref_debug:
+            cmd.extend(["-dALL", "-l{}".format(args.ref_debug)])
+
+        if args.ref_intermediates:
+            cmd.extend(["-Ddump_intermediates=1"])
+
+        # Run command and interpret tosa graph result via process return codes
+        graphMessage = None
+        try:
+            run_sh_command(cmd, self.args.verbose, capture_output=True)
+            graphResult = TosaTestRunner.TosaGraphResult.TOSA_VALID
+        except RunShCommandError as e:
+            graphMessage = e.stderr
+            if e.return_code == TosaRefReturnCode.ERROR:
+                graphResult = TosaTestRunner.TosaGraphResult.TOSA_ERROR
+            elif e.return_code == TosaRefReturnCode.UNPREDICTABLE:
+                graphResult = TosaTestRunner.TosaGraphResult.TOSA_UNPREDICTABLE
+            else:
+                graphResult = TosaTestRunner.TosaGraphResult.OTHER_ERROR
+            if (
+                self.args.verbose
+                or graphResult == TosaTestRunner.TosaGraphResult.OTHER_ERROR
+            ):
+                print(e)
+
+        except Exception as e:
+            print(e)
+            graphMessage = str(e)
+            graphResult = TosaTestRunner.TosaGraphResult.OTHER_ERROR
+
+        # Return graph result and message
+        return graphResult, graphMessage