Add CONST testing to Numpy refmodel tests

Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: Id32f3b7c287c9e8b0fd42dd96922be5a255598e2

3 files changed, 115 insertions, 78 deletions

diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py
index fe05b57..b76b656 100644
--- a/verif/generator/tosa_test_gen.py
+++ b/verif/generator/tosa_test_gen.py
@@ -40,7 +40,7 @@ class TosaTestGen:
 
         fullPath = os.path.join(self.basePath, self.testPath)
         os.makedirs(fullPath, exist_ok=True)
-        self.ser = ts.TosaSerializer(fullPath)
+        self.ser = ts.TosaSerializer(fullPath, saveConstsToFile=self.args.dump_consts)
 
     def getSerializer(self):
         return self.ser
diff --git a/verif/generator/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py
index fee551b..2fafacb 100644
--- a/verif/generator/tosa_verif_build_tests.py
+++ b/verif/generator/tosa_verif_build_tests.py
@@ -192,6 +192,13 @@ def parseArgs(argv):
         help="set a particular zero point for all valid positive tests",
     )
 
+    parser.add_argument(
+        "--dump-const-tensors",
+        dest="dump_consts",
+        action="store_true",
+        help="output const tensors as numpy files for inspection",
+    )
+
     args = parser.parse_args(argv)
 
     return args
diff --git a/verif/tests/test_tosa_refmodel.py b/verif/tests/test_tosa_refmodel.py
index 46c197a..b2f33dd 100644
--- a/verif/tests/test_tosa_refmodel.py
+++ b/verif/tests/test_tosa_refmodel.py
@@ -2,6 +2,7 @@
 # Copyright (c) 2022, ARM Limited.
 # SPDX-License-Identifier: Apache-2.0
 import json
+import re
 from pathlib import Path
 from shutil import rmtree
 
@@ -25,6 +26,7 @@ REF_MODEL = REF_MODEL_PATH / REF_MODEL_EXE
 
 # Default tensor shape information
 SHAPE_LIST = ["10", "5"]
+SHAPE_DIMS = len(SHAPE_LIST)
 SHAPE_ARG = ",".join(SHAPE_LIST)
 SHAPE_OUT = "x".join(SHAPE_LIST)
 
@@ -32,11 +34,13 @@ SHAPE_OUT = "x".join(SHAPE_LIST)
 OUTPUT_DIR_PREFIX = "_pytest_vtest"
 OUTPUT_OFM_FILE = "result_refmodel_pytest.npy"
 OUTPUT_RESULT_FILE = "result_numpy_pytest.npy"
+OUTPUT_CONST_GLOB = "const-*.npy"
 
 TEST_DESC_FILENAME = "desc.json"
 
 # Conversion from refmodel type into the type abbreviation used in the test output
 REF_MODEL_TYPE_TO_OUT = {
+    "bool": "b",
     "int8": "i8",
     "uint8": "u8",
     "int16": "i16",
@@ -54,11 +58,12 @@ def test_refmodel_built():
 class BuildTosaTest:
     """Wrapper for managing lifecycle of TOSA unit tests."""
 
-    def __init__(self, op_name, ref_model_type):
+    def __init__(self, op_name, ref_model_type, num_expected_tests):
         self.op_name = op_name
         self.ref_model_type = ref_model_type
+        self.num_expected_tests = num_expected_tests
         self.output_dir = None
-        self.test_dir = None
+        self.test_dirs = None
 
     def create_test(self):
         """Helper to generate a TOSA unit test."""
@@ -71,7 +76,7 @@ class BuildTosaTest:
             / f"{OUTPUT_DIR_PREFIX}_{self.op_name}_{self.ref_model_type}"
         )
 
-        # Generate test without any zero-point
+        # Generate tests without any zero-point
         build_args = [
             "--filter",
             self.op_name,
@@ -81,22 +86,26 @@ class BuildTosaTest:
             self.ref_model_type,
             "--zero-point",
             "0",
+            "--num-const-inputs-concat",
+            "1",
+            "--dump-const-tensors",
             "-o",
             str(self.output_dir),
         ]
-        print(build_args)
+        print(f"### Building tests: tosa_verif_build_tests {' '.join(build_args)}")
         tosa_builder(build_args)
 
         # Find the created test
         test_dir = self.output_dir / self.op_name
         # Can't assume exact name due to broadcasting and other changes to shape
-        test_glob = f"{self.op_name}_*_{REF_MODEL_TYPE_TO_OUT[self.ref_model_type]}"
-        tests = sorted(test_dir.glob(test_glob))
-        assert len(tests) == 1
-        assert tests[0].is_dir()
-        self.test_dir = tests[0]
+        test_glob = f"{self.op_name}_*_{REF_MODEL_TYPE_TO_OUT[self.ref_model_type]}*"
+        test_dirs = sorted(test_dir.glob(test_glob))
+        assert len(test_dirs) == self.num_expected_tests
+        for test_dir in test_dirs:
+            assert test_dir.is_dir()
+        self.test_dirs = test_dirs
 
-        return self.test_dir
+        return self.test_dirs
 
     def remove_test(self):
         if self.output_dir is not None and self.output_dir.is_dir():
@@ -110,30 +119,36 @@ class BuildTosaTest:
                 print(f"Skipped clean up of {test_tree}")
 
 
-# Tests - op_name, ref_model_type
+# Tests - op_name, ref_model_type, num_expected_tests
 TEST_PARAMS = [
-    ("add", "int32"),
-    ("add", "float"),
-    ("abs", "int32"),
-    ("abs", "float"),
-    ("negate", "int8"),
-    ("negate", "int16"),
-    ("negate", "int32"),
-    ("negate", "float"),
+    ("add", "int32", 1),
+    ("add", "float", 1),
+    ("abs", "int32", 1),
+    ("abs", "float", 1),
+    ("negate", "int8", 1),
+    ("negate", "int16", 1),
+    ("negate", "int32", 1),
+    ("negate", "float", 1),
+    # One test per axis (shape dimensions)
+    ("concat", "bool", SHAPE_DIMS),
+    ("concat", "int8", SHAPE_DIMS),
+    ("concat", "int16", SHAPE_DIMS),
+    ("concat", "int32", SHAPE_DIMS),
+    ("concat", "float", SHAPE_DIMS),
 ]
 
 
 def id_2_name(id):
     """Convert test id to name - otherwise it will be tosaTestN."""
-    op_name, ref_model_type = id
+    op_name, ref_model_type, _ = id
     return f"{op_name}-{ref_model_type}"
 
 
 @pytest.fixture(params=TEST_PARAMS, ids=id_2_name)
 def tosaTest(request):
     """Fixture to generate the required test params and clean up."""
-    op_name, ref_model_type = request.param
-    tst = BuildTosaTest(op_name, ref_model_type)
+    op_name, ref_model_type, num_expected_tests = request.param
+    tst = BuildTosaTest(op_name, ref_model_type, num_expected_tests)
     yield tst
     tst.remove_test()
 
@@ -143,58 +158,73 @@ def test_refmodel_simple_op(tosaTest):
     """Operator testing versus Numpy."""
     op_name = tosaTest.op_name
 
-    # Generate a TOSA test
-    test_dir = tosaTest.create_test()
-
-    # Run ref model
-    desc_file = test_dir / TEST_DESC_FILENAME
-    assert desc_file.is_file()
-    refmodel_cmd = [
-        str(REF_MODEL),
-        "--test_desc",
-        str(desc_file),
-        "--ofm_file",
-        OUTPUT_OFM_FILE,
-    ]
-    try:
-        run_sh_command(refmodel_cmd, verbose=True, capture_output=True)
-    except RunShCommandError as err:
-        assert False, f"Unexpected exception {err}"
-
-    # Find output
-    ofm_file = test_dir / OUTPUT_OFM_FILE
-    assert ofm_file.is_file()
-
-    # Load inputs for Numpy
-    with desc_file.open("r") as fp:
-        test_desc = json.load(fp)
-    tensors = []
-    assert "ifm_file" in test_desc
-    for input_name in test_desc["ifm_file"]:
-        input_file = test_dir / input_name
-        assert input_file.is_file()
-        tensors.append(np.load(str(input_file)))
-
-    # Perform Numpy operation
-    if op_name == "abs":
-        assert len(tensors) == 1
-        result = np.abs(tensors[0])
-    elif op_name == "add":
-        assert len(tensors) == 2
-        result = np.add(tensors[0], tensors[1])
-    elif op_name == "negate":
-        assert len(tensors) == 1
-        result = np.negative(tensors[0])
-    else:
-        assert False, f"Unknown operation {op_name}"
-
-    # Save Numpy result
-    result_file = test_dir / OUTPUT_RESULT_FILE
-    np.save(str(result_file), result)
-    assert result_file.is_file()
-
-    # Check Numpy result versus refmodel
-    check_result, tolerance, msg = tosa_check(
-        str(result_file), str(ofm_file), test_name=test_dir.name
-    )
-    assert check_result == TosaResult.PASS
+    # Generate TOSA test(s) (mostly should be single test)
+    test_dirs = tosaTest.create_test()
+
+    for test_dir in test_dirs:
+        # Run ref model
+        desc_file = test_dir / TEST_DESC_FILENAME
+        assert desc_file.is_file()
+        refmodel_cmd = [
+            str(REF_MODEL),
+            "--test_desc",
+            str(desc_file),
+            "--ofm_file",
+            OUTPUT_OFM_FILE,
+        ]
+        try:
+            run_sh_command(refmodel_cmd, verbose=True, capture_output=True)
+        except RunShCommandError as err:
+            assert False, f"Unexpected exception {err}"
+
+        # Find output
+        ofm_file = test_dir / OUTPUT_OFM_FILE
+        assert ofm_file.is_file()
+
+        # Load inputs for Numpy
+        with desc_file.open("r") as fp:
+            test_desc = json.load(fp)
+        tensors = []
+        assert "ifm_file" in test_desc
+        for input_name in test_desc["ifm_file"]:
+            input_file = test_dir / input_name
+            assert input_file.is_file()
+            tensors.append(np.load(str(input_file)))
+
+        # Load constants for Numpy
+        const_files = sorted(test_dir.glob(OUTPUT_CONST_GLOB))
+        consts = []
+        for const_file in const_files:
+            assert const_file.is_file()
+            consts.append(np.load(str(const_file)))
+
+        # Perform Numpy operation
+        if op_name == "abs":
+            assert len(tensors) == 1
+            result = np.abs(tensors[0])
+        elif op_name == "add":
+            assert len(tensors) == 2
+            result = np.add(tensors[0], tensors[1])
+        elif op_name == "concat":
+            assert len(consts) == 1
+            # Get axis from test directory name
+            match = re.search(r"axis([0-9]+)", test_dir.name)
+            assert match is not None
+            axis = int(match.group(1))
+            result = np.concatenate((*tensors, consts[0]), axis=axis)
+        elif op_name == "negate":
+            assert len(tensors) == 1
+            result = np.negative(tensors[0])
+        else:
+            assert False, f"Unknown operation {op_name}"
+
+        # Save Numpy result
+        result_file = test_dir / OUTPUT_RESULT_FILE
+        np.save(str(result_file), result)
+        assert result_file.is_file()
+
+        # Check Numpy result versus refmodel
+        check_result, tolerance, msg = tosa_check(
+            str(result_file), str(ofm_file), test_name=test_dir.name
+        )
+        assert check_result == TosaResult.PASS