Build compliance verifier library with a python interface.

Build steps to create tosa_reference_verify_lib.so
New python class to handle loading and calling verifier with pytests.

Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: Id42c71c398ed7364742b4a039977412ef3081c4a

6 files changed, 348 insertions, 42 deletions

diff --git a/reference_model/CMakeLists.txt b/reference_model/CMakeLists.txt
index 227d19f..88192bf 100644
--- a/reference_model/CMakeLists.txt
+++ b/reference_model/CMakeLists.txt
@@ -91,14 +91,11 @@ set(CXX_SOURCE
     src/ops/control_flow.cc
 )
 
-# Build TOSA Reference Model library
-add_library(tosa_reference_model_lib ${CXX_SOURCE})
-
-target_include_directories(tosa_reference_model_lib
-  PUBLIC
+set(PUBLIC_INCLUDE_DIRS
     $<INSTALL_INTERFACE:include>
     $<BUILD_INTERFACE:${CMAKE_CURRENT_SRC_DIR}/include>
-  PRIVATE
+)
+set(PRIVATE_INCLUDE_DIRS
     ${CMAKE_CURRENT_SOURCE_DIR}/src
     ${FLATBUFFERS_DIR}/include
     ${EIGEN_DIR}
@@ -108,6 +105,16 @@ target_include_directories(tosa_reference_model_lib
     ${JSON_DIR}/single_include
 )
 
+# Build TOSA Reference Model library
+add_library(tosa_reference_model_lib STATIC ${CXX_SOURCE})
+
+target_include_directories(tosa_reference_model_lib
+  PUBLIC
+    ${PUBLIC_INCLUDE_DIRS}
+  PRIVATE
+    ${PRIVATE_INCLUDE_DIRS}
+)
+
 target_link_libraries(tosa_reference_model_lib
   PRIVATE
     ${SERIALIZATION_LIB}
@@ -130,6 +137,21 @@ list(APPEND PUBLIC_HEADERS
 
 set_target_properties(tosa_reference_model_lib PROPERTIES PUBLIC_HEADER "${PUBLIC_HEADERS}")
 
+# Build TOSA verification library
+add_library(tosa_reference_verify_lib SHARED
+  src/verify/verify_dot_product.cc
+  src/verify/verify_entry.cc
+  src/verify/verify_utils.cc
+  src/verify/verify_config.cc
+  src/func_debug.cc
+)
+target_include_directories(tosa_reference_verify_lib
+  PUBLIC
+    ${PUBLIC_INCLUDE_DIRS}
+  PRIVATE
+    ${PRIVATE_INCLUDE_DIRS}
+)
+
 # Build TOSA Refererence Model executable
 if(BUILD_TOSA_REFERENCE_MODEL_EXECUTABLE)
   set(CXX_SOURCE_EX src/main.cpp)
@@ -139,16 +161,9 @@ if(BUILD_TOSA_REFERENCE_MODEL_EXECUTABLE)
 
   target_include_directories(tosa_reference_model
     PUBLIC
-      $<INSTALL_INTERFACE:include>
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SRC_DIR}/include>
+      ${PUBLIC_INCLUDE_DIRS}
     PRIVATE
-      ${CMAKE_CURRENT_SOURCE_DIR}/src
-      ${FLATBUFFERS_DIR}/include
-      ${EIGEN_DIR}
-      ${EIGEN_DIR}/unsupported/
-      ${SERIALIZATION_DIR}/include
-      ${HALF_DIR}/include
-      ${JSON_DIR}/single_include
+      ${PRIVATE_INCLUDE_DIRS}
   )
 
   target_link_libraries(tosa_reference_model
@@ -183,17 +198,10 @@ if(BUILD_TOSA_REFERENCE_MODEL_TESTS)
 
   target_include_directories(unit_tests
     PUBLIC
-      $<INSTALL_INTERFACE:include>
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SRC_DIR}/include>
+      ${PUBLIC_INCLUDE_DIRS}
     PRIVATE
-      ${CMAKE_CURRENT_SOURCE_DIR}/src
-      ${FLATBUFFERS_DIR}/include
-      ${EIGEN_DIR}
-      ${EIGEN_DIR}/unsupported/
-      ${SERIALIZATION_DIR}/include
-      ${HALF_DIR}/include
+      ${PRIVATE_INCLUDE_DIRS}
       ${DOCTEST_DIR}
-      ${JSON_DIR}/single_include
     )
 
   target_link_libraries(unit_tests
@@ -217,15 +225,9 @@ if(BUILD_MODEL_RUNNER_SAMPLE)
 
   target_include_directories(model_runner_sample
     PUBLIC
-      $<INSTALL_INTERFACE:include>
-      $<BUILD_INTERFACE:${CMAKE_CURRENT_SRC_DIR}/include>
+      ${PUBLIC_INCLUDE_DIRS}
     PRIVATE
-      ${CMAKE_CURRENT_SOURCE_DIR}/src
-      ${FLATBUFFERS_DIR}/include
-      ${EIGEN_DIR}
-      ${EIGEN_DIR}/unsupported/
-      ${SERIALIZATION_DIR}/include
-      ${HALF_DIR}/include
+      ${PRIVATE_INCLUDE_DIRS}
   )
 
   target_link_libraries(model_runner_sample
diff --git a/reference_model/src/verify/verify_config.cc b/reference_model/src/verify/verify_config.cc
new file mode 100644
index 0000000..3bae9b9
--- /dev/null
+++ b/reference_model/src/verify/verify_config.cc
@@ -0,0 +1,19 @@
+// Copyright (c) 2023, ARM Limited.
+//
+//    Licensed under the Apache License, Version 2.0 (the "License");
+//    you may not use this file except in compliance with the License.
+//    You may obtain a copy of the License at
+//
+//         http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS,
+//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+//    See the License for the specific language governing permissions and
+//    limitations under the License.
+
+#include "model_common.h"
+
+// Global instantiation of configuration and debug objects for verify lib
+func_config_t g_func_config;
+func_debug_t g_func_debug;
diff --git a/reference_model/src/verify/verify_dot_product.cc b/reference_model/src/verify/verify_dot_product.cc
index a24f83f..0b05c92 100644
--- a/reference_model/src/verify/verify_dot_product.cc
+++ b/reference_model/src/verify/verify_dot_product.cc
@@ -89,7 +89,7 @@ bool validateData(const double* ref, const double* bnd, const AccType* imp, size
     for (size_t i = 0; i < T; ++i)
     {
         auto out_err = validateElement<AccType>(ref[i], bnd[i], imp[i], KS);
-        TOSA_REF_REQUIRE(out_err, "output error is 0");
+        TOSA_REF_REQUIRE(out_err, "data required to be zero or error within range");
         out_err_sum += out_err.value();
         out_err_sumsq += out_err.value() * out_err.value();
     }
diff --git a/verif/checker/verifier.py b/verif/checker/verifier.py
new file mode 100644
index 0000000..3a86bab
--- /dev/null
+++ b/verif/checker/verifier.py
@@ -0,0 +1,207 @@
+# Copyright (c) 2023, ARM Limited.
+# SPDX-License-Identifier: Apache-2.0
+"""Verfier library interface."""
+import ctypes as ct
+import json
+from pathlib import Path
+from typing import Optional
+
+import numpy as np
+import schemavalidation.schemavalidation as sch
+
+
+# Default library info
+SCRIPT = Path(__file__).absolute()
+# NOTE: This REFMODEL_DIR default only works for the python developer environment
+# i.e. when using the scripts/py-dev-env.* scripts
+# otherwise use the command line option --ref-model-directory to specify path
+REFMODEL_DIR = SCRIPT.parents[2]
+LIBRARY = "libtosa_reference_verify_lib.so"
+
+# Type conversion from numpy to tosa_datatype_t
+# "type" matches enum - see include/types.h
+# "size" is size in bytes per value of this datatype
+NUMPY_DATATYPE_TO_CLIENTTYPE = {
+    # tosa_datatype_int32_t (all integer types are this!)
+    np.dtype("int32"): {"type": 5, "size": 4},
+    # tosa_datatype_int48_t (or SHAPE)
+    np.dtype("int64"): {"type": 6, "size": 8},
+    # tosa_datatype_fp16_t
+    np.dtype("float16"): {"type": 2, "size": 2},
+    # tosa_datatype_fp32_t (bf16 stored as this)
+    np.dtype("float32"): {"type": 3, "size": 4},
+    # tosa_datatype_fp64_t (for precise refmodel data)
+    np.dtype("float64"): {"type": 99, "size": 8},
+    # tosa_datatype_bool_t
+    np.dtype("bool"): {"type": 1, "size": 1},
+}
+
+
+class TosaTensor(ct.Structure):
+    _fields_ = [
+        ("name", ct.c_char_p),
+        ("shape", ct.POINTER(ct.c_int32)),
+        ("num_dims", ct.c_int32),
+        ("data_type", ct.c_int),
+        ("data", ct.POINTER(ct.c_uint8)),
+        ("size", ct.c_size_t),
+    ]
+
+
+class VerifierError(Exception):
+    """Exception raised for errors performing data generation."""
+
+
+class VerifierLibrary:
+    """Python interface to the C verify library."""
+
+    def __init__(self, path: Optional[Path] = None):
+        """Find the library and set up the interface."""
+        if path is None:
+            path = REFMODEL_DIR
+        lib_paths = sorted(path.glob(f"**/{LIBRARY}"))
+
+        if len(lib_paths) < 1:
+            raise VerifierError(
+                f"Could not find {LIBRARY} - have you built the ref-model?"
+            )
+
+        self.lib_path = lib_paths[0]
+        self.lib = ct.cdll.LoadLibrary(self.lib_path)
+
+        self.tvf_verify_data = self.lib.tvf_verify_data
+        self.tvf_verify_data.argtypes = [
+            ct.POINTER(TosaTensor),  # ref
+            ct.POINTER(TosaTensor),  # ref_bnd
+            ct.POINTER(TosaTensor),  # imp
+            ct.c_char_p,  # config_json
+        ]
+        self.tvf_verify_data.restype = ct.c_bool
+
+    def _get_tensor_data(self, name, array):
+        """Set up tosa_tensor_t using the given a numpy array."""
+        shape = (ct.c_int32 * len(array.shape))(*array.shape)
+        size_in_bytes = array.size * NUMPY_DATATYPE_TO_CLIENTTYPE[array.dtype]["size"]
+
+        tensor = TosaTensor(
+            ct.c_char_p(bytes(name, "utf8")),
+            ct.cast(shape, ct.POINTER(ct.c_int32)),
+            ct.c_int32(len(array.shape)),
+            ct.c_int(NUMPY_DATATYPE_TO_CLIENTTYPE[array.dtype]["type"]),
+            ct.cast(np.ctypeslib.as_ctypes(array), ct.POINTER(ct.c_uint8)),
+            ct.c_size_t(size_in_bytes),
+        )
+        return tensor
+
+    def verify_data(
+        self,
+        output_name,
+        compliance_json_config,
+        imp_result_array,
+        ref_result_array,
+        bnd_result_array=None,
+    ):
+        """Verify the data using the verification library."""
+        sch.TestDescSchemaValidator().validate_config(
+            compliance_json_config, sch.TD_SCHEMA_COMPLIANCE
+        )
+        jsb = bytes(json.dumps(compliance_json_config), "utf8")
+
+        imp = self._get_tensor_data(output_name, imp_result_array)
+        ref = self._get_tensor_data(output_name, ref_result_array)
+        if bnd_result_array is not None:
+            ref_bnd = self._get_tensor_data(output_name, bnd_result_array)
+        else:
+            ref_bnd = None
+
+        result = self.tvf_verify_data(ref, ref_bnd, imp, ct.c_char_p(jsb))
+
+        return result
+
+
+def main(argv=None):
+    """Simple command line interface for the verifier library."""
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--ref-model-directory",
+        dest="ref_model_dir",
+        default=REFMODEL_DIR,
+        type=Path,
+        help="Path to pre-built reference model directory",
+    )
+    parser.add_argument(
+        "--test-desc",
+        type=Path,
+        help="Path to test description file: desc.json",
+    )
+    parser.add_argument(
+        "-n",
+        "--ofm-name",
+        dest="ofm_name",
+        type=str,
+        help="output tensor name to check (defaults to only ofm_name in desc.json)",
+    )
+    parser.add_argument(
+        "--bnd-result-path",
+        type=Path,
+        help="path to the reference bounds result numpy file",
+    )
+
+    parser.add_argument(
+        "ref_result_path", type=Path, help="path to the reference result numpy file"
+    )
+    parser.add_argument(
+        "imp_result_path",
+        type=Path,
+        help="path to the implementation result numpy file",
+    )
+    args = parser.parse_args(argv)
+
+    if args.test_desc:
+        json_path = args.test_desc
+    else:
+        # Assume its with the reference file
+        json_path = args.ref_result_path.parent / "desc.json"
+
+    print("Load test description")
+    with json_path.open("r") as fd:
+        test_desc = json.load(fd)
+
+    if args.ofm_name is None:
+        if len(test_desc["ofm_name"]) != 1:
+            print("ERROR: ambiguous output to check, please specify output tensor name")
+            return 2
+        output_name = test_desc["ofm_name"][0]
+    else:
+        output_name = args.ofm_name
+
+    if "meta" not in test_desc or "compliance" not in test_desc["meta"]:
+        print(f"ERROR: no compliance meta-data found in {str(json_path)}")
+        return 2
+
+    print("Load numpy data")
+    paths = [args.imp_result_path, args.ref_result_path, args.bnd_result_path]
+    arrays = [None, None, None]
+    for idx, path in enumerate(paths):
+        if path is not None:
+            array = np.load(path)
+        else:
+            array = None
+        arrays[idx] = array
+
+    print("Load verifier library")
+    vlib = VerifierLibrary(args.ref_model_dir)
+
+    print("Verify data")
+    if vlib.verify_data(output_name, test_desc["meta"]["compliance"], *arrays):
+        print("SUCCESS")
+        return 0
+    else:
+        print("FAILURE - NOT COMPLIANT")
+        return 1
+
+
+if __name__ == "__main__":
+    exit(main())
diff --git a/verif/tests/test_tosa_refmodel.py b/verif/tests/test_tosa_refmodel.py
index 79e6720..675a534 100644
--- a/verif/tests/test_tosa_refmodel.py
+++ b/verif/tests/test_tosa_refmodel.py
@@ -1,5 +1,5 @@
 """Tests for tosa_reference_model."""
-# Copyright (c) 2022, ARM Limited.
+# Copyright (c) 2022-2023, ARM Limited.
 # SPDX-License-Identifier: Apache-2.0
 import json
 import re
@@ -14,16 +14,16 @@ from generator.tosa_verif_build_tests import main as tosa_builder
 from runner.run_command import run_sh_command
 from runner.run_command import RunShCommandError
 
-# Note: Must rename imports so that pytest doesn't assume its a test function/class
+# Note: Must rename imports (like test_check) so that pytest doesn't assume its a test function/class
+
+# Location of reference model binaries
+REF_MODEL_BUILD_PATH = Path(__file__).resolve().parents[2] / "build" / "reference_model"
+REF_MODEL_EXE = "tosa_reference_model"
+REF_MODEL_EXE_PATH = REF_MODEL_BUILD_PATH / REF_MODEL_EXE
 
 # Set this to False if you want ot preserve the test directories after running
 CLEAN_UP_TESTS = True
 
-# Location of reference model binary
-REF_MODEL_PATH = Path(__file__).resolve().parents[2] / "build" / "reference_model"
-REF_MODEL_EXE = "tosa_reference_model"
-REF_MODEL = REF_MODEL_PATH / REF_MODEL_EXE
-
 # Default tensor shape information
 SHAPE_LIST = ["10", "5"]
 SHAPE_DIMS = len(SHAPE_LIST)
@@ -51,11 +51,13 @@ REF_MODEL_TYPE_TO_OUT = {
     "bf16": "bf16",
 }
 
+# NOTE: These tests are set to POST COMMIT - so will only run on the CI
+
 
 @pytest.mark.postcommit
 def test_refmodel_built():
     """First test to check the reference model has been built."""
-    assert REF_MODEL.is_file()
+    assert REF_MODEL_EXE_PATH.is_file()
 
 
 class BuildTosaTest:
@@ -178,7 +180,7 @@ def test_refmodel_simple_op(tosaTest):
         desc_file = test_dir / TEST_DESC_FILENAME
         assert desc_file.is_file()
         refmodel_cmd = [
-            str(REF_MODEL),
+            str(REF_MODEL_EXE_PATH),
             "--test_desc",
             str(desc_file),
             "--ofm_file",
diff --git a/verif/tests/test_tosa_verifier.py b/verif/tests/test_tosa_verifier.py
new file mode 100644
index 0000000..9524158
--- /dev/null
+++ b/verif/tests/test_tosa_verifier.py
@@ -0,0 +1,76 @@
+"""Tests for the python interface to the verifier library."""
+# Copyright (c) 2023, ARM Limited.
+# SPDX-License-Identifier: Apache-2.0
+from pathlib import Path
+
+import numpy as np
+import pytest
+from checker.verifier import VerifierError
+from checker.verifier import VerifierLibrary
+
+# NOTE: These tests are set to POST COMMIT - so will only run on the CI
+
+# Location of reference model binaries
+REF_MODEL_BUILD_PATH = Path(__file__).resolve().parents[2] / "build" / "reference_model"
+VERIFIER_LIB = "libtosa_reference_verify_lib.so"
+VERIFIER_LIB_PATH = REF_MODEL_BUILD_PATH / VERIFIER_LIB
+
+
+@pytest.mark.postcommit
+def test_verifier_lib_built():
+    """First test to check the library has been built."""
+    assert VERIFIER_LIB_PATH.is_file()
+
+
+@pytest.mark.postcommit
+def test_checker_verifier_load_fail():
+    with pytest.raises(VerifierError) as excinfo:
+        VerifierLibrary(Path("/place-that-does-not-exist"))
+    assert str(excinfo.value).startswith(f"Could not find {VERIFIER_LIB}")
+
+
+@pytest.mark.postcommit
+def test_checker_verifier_load():
+    vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent)
+    assert vlib
+
+
+JSON_COMPLIANCE_DOT_PRODUCT = {
+    "version": "0.1",
+    "tensors": {
+        "output1": {
+            "mode": "DOT_PRODUCT",
+            "dot_product_info": {"ks": 1000, "s": 0, "data_type": "FP32"},
+        }
+    },
+}
+
+
+@pytest.mark.postcommit
+def test_checker_verifier_dot_product_check():
+    vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent)
+    assert vlib
+
+    imp_arr = np.zeros((10, 10, 10), dtype=np.float32)
+    ref_arr = np.zeros((10, 10, 10), dtype=np.float64)
+    bnd_arr = np.zeros((10, 10, 10), dtype=np.float64)
+
+    json_config = JSON_COMPLIANCE_DOT_PRODUCT
+
+    ret = vlib.verify_data("output1", json_config, imp_arr, ref_arr, bnd_arr)
+    assert ret
+
+
+@pytest.mark.postcommit
+def test_checker_verifier_dot_product_check_fail():
+    vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent)
+    assert vlib
+
+    imp_arr = np.zeros((10, 10, 10), dtype=np.float32)
+    ref_arr = np.ones((10, 10, 10), dtype=np.float64)
+    bnd_arr = np.zeros((10, 10, 10), dtype=np.float64)
+
+    json_config = JSON_COMPLIANCE_DOT_PRODUCT
+
+    ret = vlib.verify_data("output1", json_config, imp_arr, ref_arr, bnd_arr)
+    assert not ret