diff options
-rw-r--r-- | reference_model/CMakeLists.txt | 66 | ||||
-rw-r--r-- | reference_model/src/verify/verify_config.cc | 19 | ||||
-rw-r--r-- | reference_model/src/verify/verify_dot_product.cc | 2 | ||||
-rw-r--r-- | verif/checker/verifier.py | 207 | ||||
-rw-r--r-- | verif/tests/test_tosa_refmodel.py | 20 | ||||
-rw-r--r-- | verif/tests/test_tosa_verifier.py | 76 |
6 files changed, 348 insertions, 42 deletions
diff --git a/reference_model/CMakeLists.txt b/reference_model/CMakeLists.txt index 227d19f..88192bf 100644 --- a/reference_model/CMakeLists.txt +++ b/reference_model/CMakeLists.txt @@ -91,14 +91,11 @@ set(CXX_SOURCE src/ops/control_flow.cc ) -# Build TOSA Reference Model library -add_library(tosa_reference_model_lib ${CXX_SOURCE}) - -target_include_directories(tosa_reference_model_lib - PUBLIC +set(PUBLIC_INCLUDE_DIRS $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SRC_DIR}/include> - PRIVATE +) +set(PRIVATE_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/src ${FLATBUFFERS_DIR}/include ${EIGEN_DIR} @@ -108,6 +105,16 @@ target_include_directories(tosa_reference_model_lib ${JSON_DIR}/single_include ) +# Build TOSA Reference Model library +add_library(tosa_reference_model_lib STATIC ${CXX_SOURCE}) + +target_include_directories(tosa_reference_model_lib + PUBLIC + ${PUBLIC_INCLUDE_DIRS} + PRIVATE + ${PRIVATE_INCLUDE_DIRS} +) + target_link_libraries(tosa_reference_model_lib PRIVATE ${SERIALIZATION_LIB} @@ -130,6 +137,21 @@ list(APPEND PUBLIC_HEADERS set_target_properties(tosa_reference_model_lib PROPERTIES PUBLIC_HEADER "${PUBLIC_HEADERS}") +# Build TOSA verification library +add_library(tosa_reference_verify_lib SHARED + src/verify/verify_dot_product.cc + src/verify/verify_entry.cc + src/verify/verify_utils.cc + src/verify/verify_config.cc + src/func_debug.cc +) +target_include_directories(tosa_reference_verify_lib + PUBLIC + ${PUBLIC_INCLUDE_DIRS} + PRIVATE + ${PRIVATE_INCLUDE_DIRS} +) + # Build TOSA Refererence Model executable if(BUILD_TOSA_REFERENCE_MODEL_EXECUTABLE) set(CXX_SOURCE_EX src/main.cpp) @@ -139,16 +161,9 @@ if(BUILD_TOSA_REFERENCE_MODEL_EXECUTABLE) target_include_directories(tosa_reference_model PUBLIC - $<INSTALL_INTERFACE:include> - $<BUILD_INTERFACE:${CMAKE_CURRENT_SRC_DIR}/include> + ${PUBLIC_INCLUDE_DIRS} PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/src - ${FLATBUFFERS_DIR}/include - ${EIGEN_DIR} - ${EIGEN_DIR}/unsupported/ - ${SERIALIZATION_DIR}/include - ${HALF_DIR}/include - ${JSON_DIR}/single_include + ${PRIVATE_INCLUDE_DIRS} ) target_link_libraries(tosa_reference_model @@ -183,17 +198,10 @@ if(BUILD_TOSA_REFERENCE_MODEL_TESTS) target_include_directories(unit_tests PUBLIC - $<INSTALL_INTERFACE:include> - $<BUILD_INTERFACE:${CMAKE_CURRENT_SRC_DIR}/include> + ${PUBLIC_INCLUDE_DIRS} PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/src - ${FLATBUFFERS_DIR}/include - ${EIGEN_DIR} - ${EIGEN_DIR}/unsupported/ - ${SERIALIZATION_DIR}/include - ${HALF_DIR}/include + ${PRIVATE_INCLUDE_DIRS} ${DOCTEST_DIR} - ${JSON_DIR}/single_include ) target_link_libraries(unit_tests @@ -217,15 +225,9 @@ if(BUILD_MODEL_RUNNER_SAMPLE) target_include_directories(model_runner_sample PUBLIC - $<INSTALL_INTERFACE:include> - $<BUILD_INTERFACE:${CMAKE_CURRENT_SRC_DIR}/include> + ${PUBLIC_INCLUDE_DIRS} PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR}/src - ${FLATBUFFERS_DIR}/include - ${EIGEN_DIR} - ${EIGEN_DIR}/unsupported/ - ${SERIALIZATION_DIR}/include - ${HALF_DIR}/include + ${PRIVATE_INCLUDE_DIRS} ) target_link_libraries(model_runner_sample diff --git a/reference_model/src/verify/verify_config.cc b/reference_model/src/verify/verify_config.cc new file mode 100644 index 0000000..3bae9b9 --- /dev/null +++ b/reference_model/src/verify/verify_config.cc @@ -0,0 +1,19 @@ +// Copyright (c) 2023, ARM Limited. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "model_common.h" + +// Global instantiation of configuration and debug objects for verify lib +func_config_t g_func_config; +func_debug_t g_func_debug; diff --git a/reference_model/src/verify/verify_dot_product.cc b/reference_model/src/verify/verify_dot_product.cc index a24f83f..0b05c92 100644 --- a/reference_model/src/verify/verify_dot_product.cc +++ b/reference_model/src/verify/verify_dot_product.cc @@ -89,7 +89,7 @@ bool validateData(const double* ref, const double* bnd, const AccType* imp, size for (size_t i = 0; i < T; ++i) { auto out_err = validateElement<AccType>(ref[i], bnd[i], imp[i], KS); - TOSA_REF_REQUIRE(out_err, "output error is 0"); + TOSA_REF_REQUIRE(out_err, "data required to be zero or error within range"); out_err_sum += out_err.value(); out_err_sumsq += out_err.value() * out_err.value(); } diff --git a/verif/checker/verifier.py b/verif/checker/verifier.py new file mode 100644 index 0000000..3a86bab --- /dev/null +++ b/verif/checker/verifier.py @@ -0,0 +1,207 @@ +# Copyright (c) 2023, ARM Limited. +# SPDX-License-Identifier: Apache-2.0 +"""Verfier library interface.""" +import ctypes as ct +import json +from pathlib import Path +from typing import Optional + +import numpy as np +import schemavalidation.schemavalidation as sch + + +# Default library info +SCRIPT = Path(__file__).absolute() +# NOTE: This REFMODEL_DIR default only works for the python developer environment +# i.e. when using the scripts/py-dev-env.* scripts +# otherwise use the command line option --ref-model-directory to specify path +REFMODEL_DIR = SCRIPT.parents[2] +LIBRARY = "libtosa_reference_verify_lib.so" + +# Type conversion from numpy to tosa_datatype_t +# "type" matches enum - see include/types.h +# "size" is size in bytes per value of this datatype +NUMPY_DATATYPE_TO_CLIENTTYPE = { + # tosa_datatype_int32_t (all integer types are this!) + np.dtype("int32"): {"type": 5, "size": 4}, + # tosa_datatype_int48_t (or SHAPE) + np.dtype("int64"): {"type": 6, "size": 8}, + # tosa_datatype_fp16_t + np.dtype("float16"): {"type": 2, "size": 2}, + # tosa_datatype_fp32_t (bf16 stored as this) + np.dtype("float32"): {"type": 3, "size": 4}, + # tosa_datatype_fp64_t (for precise refmodel data) + np.dtype("float64"): {"type": 99, "size": 8}, + # tosa_datatype_bool_t + np.dtype("bool"): {"type": 1, "size": 1}, +} + + +class TosaTensor(ct.Structure): + _fields_ = [ + ("name", ct.c_char_p), + ("shape", ct.POINTER(ct.c_int32)), + ("num_dims", ct.c_int32), + ("data_type", ct.c_int), + ("data", ct.POINTER(ct.c_uint8)), + ("size", ct.c_size_t), + ] + + +class VerifierError(Exception): + """Exception raised for errors performing data generation.""" + + +class VerifierLibrary: + """Python interface to the C verify library.""" + + def __init__(self, path: Optional[Path] = None): + """Find the library and set up the interface.""" + if path is None: + path = REFMODEL_DIR + lib_paths = sorted(path.glob(f"**/{LIBRARY}")) + + if len(lib_paths) < 1: + raise VerifierError( + f"Could not find {LIBRARY} - have you built the ref-model?" + ) + + self.lib_path = lib_paths[0] + self.lib = ct.cdll.LoadLibrary(self.lib_path) + + self.tvf_verify_data = self.lib.tvf_verify_data + self.tvf_verify_data.argtypes = [ + ct.POINTER(TosaTensor), # ref + ct.POINTER(TosaTensor), # ref_bnd + ct.POINTER(TosaTensor), # imp + ct.c_char_p, # config_json + ] + self.tvf_verify_data.restype = ct.c_bool + + def _get_tensor_data(self, name, array): + """Set up tosa_tensor_t using the given a numpy array.""" + shape = (ct.c_int32 * len(array.shape))(*array.shape) + size_in_bytes = array.size * NUMPY_DATATYPE_TO_CLIENTTYPE[array.dtype]["size"] + + tensor = TosaTensor( + ct.c_char_p(bytes(name, "utf8")), + ct.cast(shape, ct.POINTER(ct.c_int32)), + ct.c_int32(len(array.shape)), + ct.c_int(NUMPY_DATATYPE_TO_CLIENTTYPE[array.dtype]["type"]), + ct.cast(np.ctypeslib.as_ctypes(array), ct.POINTER(ct.c_uint8)), + ct.c_size_t(size_in_bytes), + ) + return tensor + + def verify_data( + self, + output_name, + compliance_json_config, + imp_result_array, + ref_result_array, + bnd_result_array=None, + ): + """Verify the data using the verification library.""" + sch.TestDescSchemaValidator().validate_config( + compliance_json_config, sch.TD_SCHEMA_COMPLIANCE + ) + jsb = bytes(json.dumps(compliance_json_config), "utf8") + + imp = self._get_tensor_data(output_name, imp_result_array) + ref = self._get_tensor_data(output_name, ref_result_array) + if bnd_result_array is not None: + ref_bnd = self._get_tensor_data(output_name, bnd_result_array) + else: + ref_bnd = None + + result = self.tvf_verify_data(ref, ref_bnd, imp, ct.c_char_p(jsb)) + + return result + + +def main(argv=None): + """Simple command line interface for the verifier library.""" + import argparse + + parser = argparse.ArgumentParser() + parser.add_argument( + "--ref-model-directory", + dest="ref_model_dir", + default=REFMODEL_DIR, + type=Path, + help="Path to pre-built reference model directory", + ) + parser.add_argument( + "--test-desc", + type=Path, + help="Path to test description file: desc.json", + ) + parser.add_argument( + "-n", + "--ofm-name", + dest="ofm_name", + type=str, + help="output tensor name to check (defaults to only ofm_name in desc.json)", + ) + parser.add_argument( + "--bnd-result-path", + type=Path, + help="path to the reference bounds result numpy file", + ) + + parser.add_argument( + "ref_result_path", type=Path, help="path to the reference result numpy file" + ) + parser.add_argument( + "imp_result_path", + type=Path, + help="path to the implementation result numpy file", + ) + args = parser.parse_args(argv) + + if args.test_desc: + json_path = args.test_desc + else: + # Assume its with the reference file + json_path = args.ref_result_path.parent / "desc.json" + + print("Load test description") + with json_path.open("r") as fd: + test_desc = json.load(fd) + + if args.ofm_name is None: + if len(test_desc["ofm_name"]) != 1: + print("ERROR: ambiguous output to check, please specify output tensor name") + return 2 + output_name = test_desc["ofm_name"][0] + else: + output_name = args.ofm_name + + if "meta" not in test_desc or "compliance" not in test_desc["meta"]: + print(f"ERROR: no compliance meta-data found in {str(json_path)}") + return 2 + + print("Load numpy data") + paths = [args.imp_result_path, args.ref_result_path, args.bnd_result_path] + arrays = [None, None, None] + for idx, path in enumerate(paths): + if path is not None: + array = np.load(path) + else: + array = None + arrays[idx] = array + + print("Load verifier library") + vlib = VerifierLibrary(args.ref_model_dir) + + print("Verify data") + if vlib.verify_data(output_name, test_desc["meta"]["compliance"], *arrays): + print("SUCCESS") + return 0 + else: + print("FAILURE - NOT COMPLIANT") + return 1 + + +if __name__ == "__main__": + exit(main()) diff --git a/verif/tests/test_tosa_refmodel.py b/verif/tests/test_tosa_refmodel.py index 79e6720..675a534 100644 --- a/verif/tests/test_tosa_refmodel.py +++ b/verif/tests/test_tosa_refmodel.py @@ -1,5 +1,5 @@ """Tests for tosa_reference_model.""" -# Copyright (c) 2022, ARM Limited. +# Copyright (c) 2022-2023, ARM Limited. # SPDX-License-Identifier: Apache-2.0 import json import re @@ -14,16 +14,16 @@ from generator.tosa_verif_build_tests import main as tosa_builder from runner.run_command import run_sh_command from runner.run_command import RunShCommandError -# Note: Must rename imports so that pytest doesn't assume its a test function/class +# Note: Must rename imports (like test_check) so that pytest doesn't assume its a test function/class + +# Location of reference model binaries +REF_MODEL_BUILD_PATH = Path(__file__).resolve().parents[2] / "build" / "reference_model" +REF_MODEL_EXE = "tosa_reference_model" +REF_MODEL_EXE_PATH = REF_MODEL_BUILD_PATH / REF_MODEL_EXE # Set this to False if you want ot preserve the test directories after running CLEAN_UP_TESTS = True -# Location of reference model binary -REF_MODEL_PATH = Path(__file__).resolve().parents[2] / "build" / "reference_model" -REF_MODEL_EXE = "tosa_reference_model" -REF_MODEL = REF_MODEL_PATH / REF_MODEL_EXE - # Default tensor shape information SHAPE_LIST = ["10", "5"] SHAPE_DIMS = len(SHAPE_LIST) @@ -51,11 +51,13 @@ REF_MODEL_TYPE_TO_OUT = { "bf16": "bf16", } +# NOTE: These tests are set to POST COMMIT - so will only run on the CI + @pytest.mark.postcommit def test_refmodel_built(): """First test to check the reference model has been built.""" - assert REF_MODEL.is_file() + assert REF_MODEL_EXE_PATH.is_file() class BuildTosaTest: @@ -178,7 +180,7 @@ def test_refmodel_simple_op(tosaTest): desc_file = test_dir / TEST_DESC_FILENAME assert desc_file.is_file() refmodel_cmd = [ - str(REF_MODEL), + str(REF_MODEL_EXE_PATH), "--test_desc", str(desc_file), "--ofm_file", diff --git a/verif/tests/test_tosa_verifier.py b/verif/tests/test_tosa_verifier.py new file mode 100644 index 0000000..9524158 --- /dev/null +++ b/verif/tests/test_tosa_verifier.py @@ -0,0 +1,76 @@ +"""Tests for the python interface to the verifier library.""" +# Copyright (c) 2023, ARM Limited. +# SPDX-License-Identifier: Apache-2.0 +from pathlib import Path + +import numpy as np +import pytest +from checker.verifier import VerifierError +from checker.verifier import VerifierLibrary + +# NOTE: These tests are set to POST COMMIT - so will only run on the CI + +# Location of reference model binaries +REF_MODEL_BUILD_PATH = Path(__file__).resolve().parents[2] / "build" / "reference_model" +VERIFIER_LIB = "libtosa_reference_verify_lib.so" +VERIFIER_LIB_PATH = REF_MODEL_BUILD_PATH / VERIFIER_LIB + + +@pytest.mark.postcommit +def test_verifier_lib_built(): + """First test to check the library has been built.""" + assert VERIFIER_LIB_PATH.is_file() + + +@pytest.mark.postcommit +def test_checker_verifier_load_fail(): + with pytest.raises(VerifierError) as excinfo: + VerifierLibrary(Path("/place-that-does-not-exist")) + assert str(excinfo.value).startswith(f"Could not find {VERIFIER_LIB}") + + +@pytest.mark.postcommit +def test_checker_verifier_load(): + vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent) + assert vlib + + +JSON_COMPLIANCE_DOT_PRODUCT = { + "version": "0.1", + "tensors": { + "output1": { + "mode": "DOT_PRODUCT", + "dot_product_info": {"ks": 1000, "s": 0, "data_type": "FP32"}, + } + }, +} + + +@pytest.mark.postcommit +def test_checker_verifier_dot_product_check(): + vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent) + assert vlib + + imp_arr = np.zeros((10, 10, 10), dtype=np.float32) + ref_arr = np.zeros((10, 10, 10), dtype=np.float64) + bnd_arr = np.zeros((10, 10, 10), dtype=np.float64) + + json_config = JSON_COMPLIANCE_DOT_PRODUCT + + ret = vlib.verify_data("output1", json_config, imp_arr, ref_arr, bnd_arr) + assert ret + + +@pytest.mark.postcommit +def test_checker_verifier_dot_product_check_fail(): + vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent) + assert vlib + + imp_arr = np.zeros((10, 10, 10), dtype=np.float32) + ref_arr = np.ones((10, 10, 10), dtype=np.float64) + bnd_arr = np.zeros((10, 10, 10), dtype=np.float64) + + json_config = JSON_COMPLIANCE_DOT_PRODUCT + + ret = vlib.verify_data("output1", json_config, imp_arr, ref_arr, bnd_arr) + assert not ret |