# Copyright (c) 2023, ARM Limited. # SPDX-License-Identifier: Apache-2.0 """Verfier library interface.""" import ctypes as ct import json from pathlib import Path import numpy as np import schemavalidation.schemavalidation as sch # Type conversion from numpy to tosa_datatype_t # "type" matches enum - see include/types.h # "size" is size in bytes per value of this datatype NUMPY_DATATYPE_TO_CLIENTTYPE = { # tosa_datatype_int32_t (all integer types are this!) np.dtype("int32"): {"type": 5, "size": 4}, # tosa_datatype_int48_t (or SHAPE) np.dtype("int64"): {"type": 6, "size": 8}, # tosa_datatype_fp16_t np.dtype("float16"): {"type": 2, "size": 2}, # tosa_datatype_fp32_t (bf16 stored as this) np.dtype("float32"): {"type": 3, "size": 4}, # tosa_datatype_fp64_t (for precise refmodel data) np.dtype("float64"): {"type": 99, "size": 8}, # tosa_datatype_bool_t np.dtype("bool"): {"type": 1, "size": 1}, } class TosaTensor(ct.Structure): _fields_ = [ ("name", ct.c_char_p), ("shape", ct.POINTER(ct.c_int32)), ("num_dims", ct.c_int32), ("data_type", ct.c_int), ("data", ct.POINTER(ct.c_uint8)), ("size", ct.c_size_t), ] class VerifierError(Exception): """Exception raised for errors performing data generation.""" class VerifierLibrary: """Python interface to the C verify library.""" def __init__(self, verify_lib_path): """Find the library and set up the interface.""" self.lib_path = verify_lib_path if self.lib_path is None or not self.lib_path.is_file(): raise VerifierError(f"Could not find verify library - {self.lib_path}") self.lib = ct.cdll.LoadLibrary(self.lib_path) self.tvf_verify_data = self.lib.tvf_verify_data self.tvf_verify_data.argtypes = [ ct.POINTER(TosaTensor), # ref ct.POINTER(TosaTensor), # ref_bnd ct.POINTER(TosaTensor), # imp ct.c_char_p, # config_json ] self.tvf_verify_data.restype = ct.c_bool def _get_tensor_data(self, name, array): """Set up tosa_tensor_t using the given a numpy array.""" shape = (ct.c_int32 * len(array.shape))(*array.shape) size_in_bytes = array.size * NUMPY_DATATYPE_TO_CLIENTTYPE[array.dtype]["size"] tensor = TosaTensor( ct.c_char_p(bytes(name, "utf8")), ct.cast(shape, ct.POINTER(ct.c_int32)), ct.c_int32(len(array.shape)), ct.c_int(NUMPY_DATATYPE_TO_CLIENTTYPE[array.dtype]["type"]), array.ctypes.data_as(ct.POINTER(ct.c_uint8)), ct.c_size_t(size_in_bytes), ) return tensor def verify_data( self, output_name, compliance_json_config, imp_result_array, ref_result_array, bnd_result_array=None, ): """Verify the data using the verification library.""" sch.TestDescSchemaValidator().validate_config( compliance_json_config, sch.TD_SCHEMA_COMPLIANCE ) jsb = bytes(json.dumps(compliance_json_config), "utf8") imp = self._get_tensor_data(output_name, imp_result_array) ref = self._get_tensor_data(output_name, ref_result_array) if bnd_result_array is not None: ref_bnd = self._get_tensor_data(output_name, bnd_result_array) else: ref_bnd = None result = self.tvf_verify_data(ref, ref_bnd, imp, ct.c_char_p(jsb)) return result def main(argv=None): """Simple command line interface for the verifier library.""" import argparse import conformance.model_files as cmf parser = argparse.ArgumentParser() parser.add_argument( "--verify-lib-path", type=Path, help="Path to TOSA verify lib", ) parser.add_argument( "--test-desc", type=Path, help="Path to test description file: desc.json", ) parser.add_argument( "-n", "--ofm-name", dest="ofm_name", type=str, help="output tensor name to check (defaults to only ofm_name in desc.json)", ) parser.add_argument( "--bnd-result-path", type=Path, help="path to the reference bounds result numpy file", ) parser.add_argument( "ref_result_path", type=Path, help="path to the reference result numpy file" ) parser.add_argument( "imp_result_path", type=Path, help="path to the implementation result numpy file", ) args = parser.parse_args(argv) if args.verify_lib_path is None: # Try to work out ref model directory and find the verify library # but this default only works for the python developer environment # i.e. when using the scripts/py-dev-env.* scripts # otherwise use the command line option --verify-lib-path to specify path ref_model_dir = Path(__file__).absolute().parents[2] args.verify_lib_path = cmf.find_tosa_file( cmf.TosaFileType.VERIFY_LIBRARY, ref_model_dir, False ) if args.test_desc: json_path = args.test_desc else: # Assume its with the reference file json_path = args.ref_result_path.parent / "desc.json" print("Load test description") with json_path.open("r") as fd: test_desc = json.load(fd) if args.ofm_name is None: if len(test_desc["ofm_name"]) != 1: print("ERROR: ambiguous output to check, please specify output tensor name") return 2 output_name = test_desc["ofm_name"][0] else: output_name = args.ofm_name if "meta" not in test_desc or "compliance" not in test_desc["meta"]: print(f"ERROR: no compliance meta-data found in {str(json_path)}") return 2 print("Load numpy data") paths = [args.imp_result_path, args.ref_result_path, args.bnd_result_path] arrays = [None, None, None] for idx, path in enumerate(paths): if path is not None: array = np.load(path) else: array = None arrays[idx] = array print("Load verifier library") vlib = VerifierLibrary(args.verify_lib_path) print("Verify data") if vlib.verify_data(output_name, test_desc["meta"]["compliance"], *arrays): print("SUCCESS") return 0 else: print("FAILURE - NOT COMPLIANT") return 1 if __name__ == "__main__": exit(main())