Add simple post commit reference model testing against Numpy

Only perform testing of this after build of ref model using postcommit pytest mark.

Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: I771a18d2c9cd4051fecafad3e6079b44f2ed62fa

7 files changed, 227 insertions, 5 deletions

diff --git a/.gitignore b/.gitignore
index 1ef9759..6674c8c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,4 +6,8 @@ build/
 dist/
 *.egg-info
 verif/tests/test-result-*.tests.tosa_mock_sut_run.npy
+verif/tests/_pytest_vtest_*
 .coverage
+result.xml
+vtest/
+examples/*/*/out.npy
\ No newline at end of file
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f2ccc3f..b253ddd 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -25,6 +25,6 @@ repos:
         name: pytest
         stages: [commit]
         language: system
-        entry: pytest
+        entry: pytest -m "not postcommit"
         types: [python]
         pass_filenames: false
diff --git a/setup.cfg b/setup.cfg
index 1e4c55d..a9dc3ab 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -60,6 +60,8 @@ console_scripts =
 
 [tool:pytest]
 testpaths=verif/tests
+markers =
+    postcommit: marks tests for post commit testing on CI as part of the review
 
 [flake8]
 ignore = D213, E203, E266, E501, W503
diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py
index ef84762..a65e220 100644
--- a/verif/generator/tosa_arg_gen.py
+++ b/verif/generator/tosa_arg_gen.py
@@ -29,8 +29,12 @@ class TosaQuantGen:
     def getZeroPoint(testGen, dtype, error_name=None):
 
         if dtype == DType.INT8:
+            if testGen.args.zeropoint is not None:
+                return min(127, max(-128, testGen.args.zeropoint))
             return testGen.randInt(-128, 128)
         elif dtype == DType.UINT8:
+            if testGen.args.zeropoint is not None:
+                return min(255, max(0, testGen.args.zeropoint))
             return testGen.randInt(0, 256)
         elif error_name in [
             ErrorIf.InputZeroPointNotZero,
diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py
index 56a34e5..fe05b57 100644
--- a/verif/generator/tosa_test_gen.py
+++ b/verif/generator/tosa_test_gen.py
@@ -2252,6 +2252,10 @@ class TosaTestGen:
 
     def createDynamicOpLists(self):
 
+        if "conv2d_TEMPLATE" not in self.TOSA_OP_LIST:
+            # Already created these lists (can occur when class is initialized more than once)
+            return
+
         # Dynamically create op lists for convolutions with a list of kernel sizes
         KERNELS_2D = [[1, 1], [2, 2], [3, 3], [5, 5], [3, 1], [1, 3]]
 
diff --git a/verif/generator/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py
index 6ee873f..fee551b 100644
--- a/verif/generator/tosa_verif_build_tests.py
+++ b/verif/generator/tosa_verif_build_tests.py
@@ -23,7 +23,7 @@ def auto_int(x):
     return int(x, 0)
 
 
-def parseArgs():
+def parseArgs(argv):
 
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -184,14 +184,22 @@ def parseArgs():
         help="allow oversize padding, stride and kernel tests",
     )
 
-    args = parser.parse_args()
+    parser.add_argument(
+        "--zero-point",
+        dest="zeropoint",
+        default=None,
+        type=int,
+        help="set a particular zero point for all valid positive tests",
+    )
+
+    args = parser.parse_args(argv)
 
     return args
 
 
-def main():
+def main(argv=None):
 
-    args = parseArgs()
+    args = parseArgs(argv)
 
     ttg = TosaTestGen(args)
 
diff --git a/verif/tests/test_tosa_refmodel.py b/verif/tests/test_tosa_refmodel.py
new file mode 100644
index 0000000..46c197a
--- /dev/null
+++ b/verif/tests/test_tosa_refmodel.py
@@ -0,0 +1,200 @@
+"""Tests for tosa_reference_model."""
+# Copyright (c) 2022, ARM Limited.
+# SPDX-License-Identifier: Apache-2.0
+import json
+from pathlib import Path
+from shutil import rmtree
+
+import numpy as np
+import pytest
+from checker.tosa_result_checker import test_check as tosa_check
+from checker.tosa_result_checker import TestResult as TosaResult
+from generator.tosa_verif_build_tests import main as tosa_builder
+from runner.run_command import run_sh_command
+from runner.run_command import RunShCommandError
+
+# Note: Must rename imports so that pytest doesn't assume its a test function/class
+
+# Set this to False if you want ot preserve the test directories after running
+CLEAN_UP_TESTS = True
+
+# Location of reference model binary
+REF_MODEL_PATH = Path(__file__).resolve().parents[2] / "build" / "reference_model"
+REF_MODEL_EXE = "tosa_reference_model"
+REF_MODEL = REF_MODEL_PATH / REF_MODEL_EXE
+
+# Default tensor shape information
+SHAPE_LIST = ["10", "5"]
+SHAPE_ARG = ",".join(SHAPE_LIST)
+SHAPE_OUT = "x".join(SHAPE_LIST)
+
+# Output file information
+OUTPUT_DIR_PREFIX = "_pytest_vtest"
+OUTPUT_OFM_FILE = "result_refmodel_pytest.npy"
+OUTPUT_RESULT_FILE = "result_numpy_pytest.npy"
+
+TEST_DESC_FILENAME = "desc.json"
+
+# Conversion from refmodel type into the type abbreviation used in the test output
+REF_MODEL_TYPE_TO_OUT = {
+    "int8": "i8",
+    "uint8": "u8",
+    "int16": "i16",
+    "int32": "i32",
+    "float": "float",
+}
+
+
+@pytest.mark.postcommit
+def test_refmodel_built():
+    """First test to check the reference model has been built."""
+    assert REF_MODEL.is_file()
+
+
+class BuildTosaTest:
+    """Wrapper for managing lifecycle of TOSA unit tests."""
+
+    def __init__(self, op_name, ref_model_type):
+        self.op_name = op_name
+        self.ref_model_type = ref_model_type
+        self.output_dir = None
+        self.test_dir = None
+
+    def create_test(self):
+        """Helper to generate a TOSA unit test."""
+        if self.output_dir is not None:
+            # Already created
+            return self.test_dir
+
+        self.output_dir = (
+            Path(__file__).parent
+            / f"{OUTPUT_DIR_PREFIX}_{self.op_name}_{self.ref_model_type}"
+        )
+
+        # Generate test without any zero-point
+        build_args = [
+            "--filter",
+            self.op_name,
+            "--target-shape",
+            SHAPE_ARG,
+            "--target-dtype",
+            self.ref_model_type,
+            "--zero-point",
+            "0",
+            "-o",
+            str(self.output_dir),
+        ]
+        print(build_args)
+        tosa_builder(build_args)
+
+        # Find the created test
+        test_dir = self.output_dir / self.op_name
+        # Can't assume exact name due to broadcasting and other changes to shape
+        test_glob = f"{self.op_name}_*_{REF_MODEL_TYPE_TO_OUT[self.ref_model_type]}"
+        tests = sorted(test_dir.glob(test_glob))
+        assert len(tests) == 1
+        assert tests[0].is_dir()
+        self.test_dir = tests[0]
+
+        return self.test_dir
+
+    def remove_test(self):
+        if self.output_dir is not None and self.output_dir.is_dir():
+            # Delete directory
+            test_tree = self.output_dir.resolve()
+            if CLEAN_UP_TESTS:
+                print(f"Deleting {test_tree}")
+                rmtree(str(test_tree))
+                self.output_dir = None
+            else:
+                print(f"Skipped clean up of {test_tree}")
+
+
+# Tests - op_name, ref_model_type
+TEST_PARAMS = [
+    ("add", "int32"),
+    ("add", "float"),
+    ("abs", "int32"),
+    ("abs", "float"),
+    ("negate", "int8"),
+    ("negate", "int16"),
+    ("negate", "int32"),
+    ("negate", "float"),
+]
+
+
+def id_2_name(id):
+    """Convert test id to name - otherwise it will be tosaTestN."""
+    op_name, ref_model_type = id
+    return f"{op_name}-{ref_model_type}"
+
+
+@pytest.fixture(params=TEST_PARAMS, ids=id_2_name)
+def tosaTest(request):
+    """Fixture to generate the required test params and clean up."""
+    op_name, ref_model_type = request.param
+    tst = BuildTosaTest(op_name, ref_model_type)
+    yield tst
+    tst.remove_test()
+
+
+@pytest.mark.postcommit
+def test_refmodel_simple_op(tosaTest):
+    """Operator testing versus Numpy."""
+    op_name = tosaTest.op_name
+
+    # Generate a TOSA test
+    test_dir = tosaTest.create_test()
+
+    # Run ref model
+    desc_file = test_dir / TEST_DESC_FILENAME
+    assert desc_file.is_file()
+    refmodel_cmd = [
+        str(REF_MODEL),
+        "--test_desc",
+        str(desc_file),
+        "--ofm_file",
+        OUTPUT_OFM_FILE,
+    ]
+    try:
+        run_sh_command(refmodel_cmd, verbose=True, capture_output=True)
+    except RunShCommandError as err:
+        assert False, f"Unexpected exception {err}"
+
+    # Find output
+    ofm_file = test_dir / OUTPUT_OFM_FILE
+    assert ofm_file.is_file()
+
+    # Load inputs for Numpy
+    with desc_file.open("r") as fp:
+        test_desc = json.load(fp)
+    tensors = []
+    assert "ifm_file" in test_desc
+    for input_name in test_desc["ifm_file"]:
+        input_file = test_dir / input_name
+        assert input_file.is_file()
+        tensors.append(np.load(str(input_file)))
+
+    # Perform Numpy operation
+    if op_name == "abs":
+        assert len(tensors) == 1
+        result = np.abs(tensors[0])
+    elif op_name == "add":
+        assert len(tensors) == 2
+        result = np.add(tensors[0], tensors[1])
+    elif op_name == "negate":
+        assert len(tensors) == 1
+        result = np.negative(tensors[0])
+    else:
+        assert False, f"Unknown operation {op_name}"
+
+    # Save Numpy result
+    result_file = test_dir / OUTPUT_RESULT_FILE
+    np.save(str(result_file), result)
+    assert result_file.is_file()
+
+    # Check Numpy result versus refmodel
+    check_result, tolerance, msg = tosa_check(
+        str(result_file), str(ofm_file), test_name=test_dir.name
+    )
+    assert check_result == TosaResult.PASS