From f0348ea4206a7e02497515ffb6d88546e0121cc7 Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Wed, 27 Sep 2023 16:10:59 +0100 Subject: Refactor path arguments to tosa-tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tosa_verif_conformance_generator - Move to using ref-model-path instead of ref-model-dir - Add schema-path and flatc-path - Add model_files module to locate default places for files convert2conformance - Remove default paths verifier - Switch to using exact path of verifier library tosa_verif_run_tests - Use conformance model_files to locate defaults Change-Id: Ieca3b164670e2a7dcb047743667cc4e8317daa97 Signed-off-by: Josef Malmström Signed-off-by: Jeremy Johnson --- verif/checker/verifier.py | 44 +++++------- verif/conformance/model_files.py | 79 ++++++++++++++++++++ .../tosa_verif_conformance_generator.py | 83 ++++++++++++++++------ verif/runner/tosa_refmodel_sut_run.py | 10 +-- verif/runner/tosa_test_runner.py | 7 +- verif/runner/tosa_verif_run_tests.py | 54 ++++++++------ verif/tests/test_tosa_run_tests_args.py | 4 +- verif/tests/test_tosa_verifier.py | 8 +-- verif/tests/tosa_mock_sut_run.py | 4 +- 9 files changed, 210 insertions(+), 83 deletions(-) create mode 100644 verif/conformance/model_files.py (limited to 'verif') diff --git a/verif/checker/verifier.py b/verif/checker/verifier.py index 3a86bab..06ffcfb 100644 --- a/verif/checker/verifier.py +++ b/verif/checker/verifier.py @@ -4,20 +4,10 @@ import ctypes as ct import json from pathlib import Path -from typing import Optional import numpy as np import schemavalidation.schemavalidation as sch - -# Default library info -SCRIPT = Path(__file__).absolute() -# NOTE: This REFMODEL_DIR default only works for the python developer environment -# i.e. when using the scripts/py-dev-env.* scripts -# otherwise use the command line option --ref-model-directory to specify path -REFMODEL_DIR = SCRIPT.parents[2] -LIBRARY = "libtosa_reference_verify_lib.so" - # Type conversion from numpy to tosa_datatype_t # "type" matches enum - see include/types.h # "size" is size in bytes per value of this datatype @@ -55,18 +45,12 @@ class VerifierError(Exception): class VerifierLibrary: """Python interface to the C verify library.""" - def __init__(self, path: Optional[Path] = None): + def __init__(self, verify_lib_path): """Find the library and set up the interface.""" - if path is None: - path = REFMODEL_DIR - lib_paths = sorted(path.glob(f"**/{LIBRARY}")) - - if len(lib_paths) < 1: - raise VerifierError( - f"Could not find {LIBRARY} - have you built the ref-model?" - ) + self.lib_path = verify_lib_path + if not self.lib_path.is_file(): + raise VerifierError(f"Could not find verify library - {self.lib_path}") - self.lib_path = lib_paths[0] self.lib = ct.cdll.LoadLibrary(self.lib_path) self.tvf_verify_data = self.lib.tvf_verify_data @@ -122,14 +106,14 @@ class VerifierLibrary: def main(argv=None): """Simple command line interface for the verifier library.""" import argparse + import conformance.model_files as cmf parser = argparse.ArgumentParser() + parser.add_argument( - "--ref-model-directory", - dest="ref_model_dir", - default=REFMODEL_DIR, + "--verify-lib-path", type=Path, - help="Path to pre-built reference model directory", + help="Path to TOSA verify lib", ) parser.add_argument( "--test-desc", @@ -159,6 +143,16 @@ def main(argv=None): ) args = parser.parse_args(argv) + if args.verify_lib_path is None: + # Try to work out ref model directory and find the verify library + # but this default only works for the python developer environment + # i.e. when using the scripts/py-dev-env.* scripts + # otherwise use the command line option --verify-lib-path to specify path + ref_model_dir = Path(__file__).absolute().parents[2] + args.verify_lib_path = cmf.find_tosa_file( + cmf.TosaFileType.VERIFY_LIBRARY, ref_model_dir, False + ) + if args.test_desc: json_path = args.test_desc else: @@ -192,7 +186,7 @@ def main(argv=None): arrays[idx] = array print("Load verifier library") - vlib = VerifierLibrary(args.ref_model_dir) + vlib = VerifierLibrary(args.verify_lib_path) print("Verify data") if vlib.verify_data(output_name, test_desc["meta"]["compliance"], *arrays): diff --git a/verif/conformance/model_files.py b/verif/conformance/model_files.py new file mode 100644 index 0000000..a9e1db3 --- /dev/null +++ b/verif/conformance/model_files.py @@ -0,0 +1,79 @@ +# Copyright (c) 2023, ARM Limited. +# SPDX-License-Identifier: Apache-2.0 +"""Locate model files helper functions.""" +from enum import IntEnum +from pathlib import Path + +DEFAULT_REF_MODEL_SCHEMA_PATH = Path("thirdparty/serialization_lib/schema") +DEFAULT_REF_MODEL_BUILD_FLATC_PATH = Path( + "thirdparty/serialization_lib/third_party/flatbuffers" +) +DEFAULT_REF_MODEL_BUILD_EXE_PATH = Path("reference_model") +DEFAULT_BUILD_DIR = Path("build") + + +class TosaFileType(IntEnum): + """TOSA file types.""" + + REF_MODEL = 0 + SCHEMA = 1 + FLATC = 2 + VERIFY_LIBRARY = 3 + + +TOSA_FILE_TYPE_TO_DETAILS = { + TosaFileType.REF_MODEL: { + "name": "tosa_reference_model", + "location": DEFAULT_REF_MODEL_BUILD_EXE_PATH, + "build": True, + }, + TosaFileType.SCHEMA: { + "name": "tosa.fbs", + "location": DEFAULT_REF_MODEL_SCHEMA_PATH, + "build": False, + }, + TosaFileType.FLATC: { + "name": "flatc", + "location": DEFAULT_REF_MODEL_BUILD_FLATC_PATH, + "build": True, + }, + TosaFileType.VERIFY_LIBRARY: { + "name": "libtosa_reference_verify_lib.so", + "location": DEFAULT_REF_MODEL_BUILD_EXE_PATH, + "build": True, + }, +} + + +def find_tosa_file(file_type, ref_model_path, path_is_ref_model_exe=True): + """Return the possible path to the required tosa file type.""" + name = TOSA_FILE_TYPE_TO_DETAILS[file_type]["name"] + location = TOSA_FILE_TYPE_TO_DETAILS[file_type]["location"] + build = TOSA_FILE_TYPE_TO_DETAILS[file_type]["build"] + + if path_is_ref_model_exe: + # Given a path to the reference_model executable + + # Special case - return what we have been given! + if file_type == TosaFileType.REF_MODEL: + return ref_model_path + + try: + if build: + # Look in build directory + search_path = ref_model_path.parents[1] + else: + # Look in reference_model directory + search_path = ref_model_path.parents[2] + except IndexError: + search_path = ref_model_path.parent + else: + # Given a path to the reference_model directory + if build: + search_path = ref_model_path / DEFAULT_BUILD_DIR + else: + search_path = ref_model_path + + search_path = search_path / location / name + + return search_path diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py index 236f729..564617b 100644 --- a/verif/conformance/tosa_verif_conformance_generator.py +++ b/verif/conformance/tosa_verif_conformance_generator.py @@ -23,6 +23,7 @@ from functools import partial from itertools import tee from pathlib import Path +import conformance.model_files as cmf from conformance.test_select import Operator from convert2conformance.convert2conformance import main as c2c_main from distutils.dir_util import copy_tree @@ -44,8 +45,6 @@ PROFILE_OPS_INFO = { } PROFILES_ALL = "all" -LOCATION_REF_MODEL_BINARY = Path("build/reference_model/tosa_reference_model") - DEFAULT_SEED = 42 # When there is a dictionary of generator argument lists (groups) only the @@ -155,7 +154,7 @@ def build_op_tests( error = False for i, cmd in enumerate(build_cmds_list): try: - _run_sh_command(args, args.ref_model_dir.absolute(), cmd) + _run_sh_command(args, args.ref_model_path.parent.absolute(), cmd) logger.info( f"{operator} test batch {(i+1)}/{len(build_cmds_list)} created successfully" ) @@ -217,11 +216,10 @@ def generate_results(args, profile, operator, op_build_dir, supports=[], tests=N num_cores = args.num_cores run_tests_cmd = "tosa_verif_run_tests" - ref_model_path = args.ref_model_dir / LOCATION_REF_MODEL_BINARY ref_cmd_base = ref_cmd = [ run_tests_cmd, "--ref-model-path", - str(ref_model_path.absolute()), + str(args.ref_model_path.absolute()), "-j", str(num_cores), "-v", @@ -244,7 +242,7 @@ def generate_results(args, profile, operator, op_build_dir, supports=[], tests=N failed_counter = 0 job_pool = mp.Pool(args.num_cores) - sh_partial = partial(_run_sh_command, args, args.ref_model_dir.absolute()) + sh_partial = partial(_run_sh_command, args, args.ref_model_path.parent.absolute()) pool_results = job_pool.map(sh_partial, ref_cmds) job_pool.close() job_pool.join() @@ -275,12 +273,12 @@ def convert_tests( tags=None, ): """Convert tests to JSON and save to output directory.""" - ref_model_dir = args.ref_model_dir - if group: output_dir = output_dir / group - c2c_args_base = ["--strict", "--ref-model-directory", str(ref_model_dir)] + c2c_args_base = ["--strict"] + c2c_args_base.extend(["--schema-path", str(args.schema_path)]) + c2c_args_base.extend(["--flatc-path", str(args.flatc_path)]) # This op maybe in more than one profile - e.g. tosa_bi and tosa_mi # even if we are only producing tests for tosa_mi for op_profile in op_profiles_list: @@ -470,6 +468,7 @@ def parse_args(argv=None): ) parser.add_argument( "--operators", + "--op", type=str, nargs="*", help="The operator(s) to create tests for, if not supplied all tests will be created", @@ -495,12 +494,37 @@ def parse_args(argv=None): action="store_true", help="Enable lazy data generation (only for tosa-mi)", ) - parser.add_argument( + rm_group = parser.add_mutually_exclusive_group(required=True) + rm_group.add_argument( "--ref-model-directory", dest="ref_model_dir", type=Path, - required=True, - help="Reference Model directory (must be pre-built)", + help="(DEPRECATED - use ref-model-path) Reference Model directory - with build directory", + ) + rm_group.add_argument( + "--ref-model-path", + dest="ref_model_path", + type=Path, + help="Path to TOSA reference model executable", + ) + parser.add_argument( + "--schema-path", + "--operator-fbs", + dest="schema_path", + type=Path, + help=( + "Path to TOSA reference model flat buffer schema. Defaults to " + f"`{cmf.DEFAULT_REF_MODEL_SCHEMA_PATH}` in parents parent directory of `ref-model-path`" + ), + ) + parser.add_argument( + "--flatc-path", + dest="flatc_path", + type=Path, + help=( + "Path to flatc executable. Defaults to " + f"`{cmf.DEFAULT_REF_MODEL_BUILD_FLATC_PATH}` in parent directory of `ref-model-path`" + ), ) parser.add_argument( "--seed", @@ -581,18 +605,35 @@ def parse_args(argv=None): def main(): args = parse_args() - if not args.ref_model_dir.is_dir(): + if args.ref_model_dir is not None: + # Assume the ref model exe path based on the ref model directory + args.ref_model_path = cmf.find_tosa_file( + cmf.TosaFileType.REF_MODEL, args.ref_model_dir, False + ) + if not args.ref_model_path.is_file(): + logger.error( + f"Missing reference model binary (--ref-model-path): {args.ref_model_path}" + ) + return 2 + + if args.schema_path is None: + args.schema_path = cmf.find_tosa_file( + cmf.TosaFileType.SCHEMA, args.ref_model_path + ) + if not args.schema_path.is_file(): logger.error( - f"Missing or invalid reference model directory: {args.ref_model_dir}" + f"Missing reference model schema (--schema-path): {args.schema_path}" ) return 2 - else: - ref_model = args.ref_model_dir / LOCATION_REF_MODEL_BINARY - if not ref_model.is_file(): - logger.error( - f"{LOCATION_REF_MODEL_BINARY} not found in {args.ref_model_dir}\nHave you built the reference model?" - ) - return 2 + + if args.flatc_path is None: + args.flatc_path = cmf.find_tosa_file( + cmf.TosaFileType.FLATC, args.ref_model_path + ) + if not args.flatc_path.is_file(): + logger.error(f"Missing flatc binary (--flatc-path): {args.flatc_path}") + return 2 + if args.unit_tests in ["framework", "both"]: logger.warning( "DEPRECATION - Framework tests are not part of TOSA conformance testing" diff --git a/verif/runner/tosa_refmodel_sut_run.py b/verif/runner/tosa_refmodel_sut_run.py index 7b129da..419f87b 100644 --- a/verif/runner/tosa_refmodel_sut_run.py +++ b/verif/runner/tosa_refmodel_sut_run.py @@ -1,5 +1,5 @@ """TOSA test runner module for the Reference Model.""" -# Copyright (c) 2020-2022, ARM Limited. +# Copyright (c) 2020-2023, ARM Limited. # SPDX-License-Identifier: Apache-2.0 from enum import IntEnum from enum import unique @@ -33,10 +33,10 @@ class TosaSUTRunner(TosaTestRunner): # Call Reference model with description file to provide all file details cmd = [ - args.ref_model_path, - "--tosa_level={}".format(args.tosa_level), - "--operator_fbs={}".format(args.operator_fbs), - "--test_desc={}".format(self.descFile), + str(args.ref_model_path), + f"--tosa_level={args.tosa_level}", + f"--operator_fbs={str(args.schema_path)}", + f"--test_desc={self.descFile}", ] # Specific debug options for reference model diff --git a/verif/runner/tosa_test_runner.py b/verif/runner/tosa_test_runner.py index d8c2a87..579dd60 100644 --- a/verif/runner/tosa_test_runner.py +++ b/verif/runner/tosa_test_runner.py @@ -1,9 +1,8 @@ """Template test runner class for running TOSA tests.""" -# Copyright (c) 2020-2022, ARM Limited. +# Copyright (c) 2020-2023, ARM Limited. # SPDX-License-Identifier: Apache-2.0 import json from enum import IntEnum -from pathlib import Path from checker.tosa_result_checker import LogColors from checker.tosa_result_checker import print_color @@ -62,8 +61,8 @@ class TosaTestRunner: if args.binary and tosaFilePath.suffix == ".json": # Convert tosa JSON to binary json2fbbin.json_to_fbbin( - Path(args.flatc_path), - Path(args.operator_fbs), + args.flatc_path, + args.schema_path, tosaFilePath, testDirPath, ) diff --git a/verif/runner/tosa_verif_run_tests.py b/verif/runner/tosa_verif_run_tests.py index 814c864..722c0e7 100644 --- a/verif/runner/tosa_verif_run_tests.py +++ b/verif/runner/tosa_verif_run_tests.py @@ -1,5 +1,5 @@ """TOSA verification runner script.""" -# Copyright (c) 2020-2022, ARM Limited. +# Copyright (c) 2020-2023, ARM Limited. # SPDX-License-Identifier: Apache-2.0 import argparse import importlib @@ -10,6 +10,7 @@ import traceback from datetime import datetime from pathlib import Path +import conformance.model_files as cmf from json2numpy import json2numpy from runner.tosa_test_runner import TosaTestInvalid from runner.tosa_test_runner import TosaTestRunner @@ -45,26 +46,30 @@ def parseArgs(argv): action="store_true", help="Recursively search for tests", ) - parser.add_argument( - "--operator-fbs", - dest="operator_fbs", - default="conformance_tests/third_party/serialization_lib/schema/tosa.fbs", - type=str, - help="flat buffer syntax file", - ) parser.add_argument( "--ref-model-path", dest="ref_model_path", - default="reference_model/build/reference_model/tosa_reference_model", - type=str, - help="Path to reference model executable", + type=Path, + help="Path to TOSA reference model executable", + ) + parser.add_argument( + "--operator-fbs", + "--schema-path", + dest="schema_path", + type=Path, + help=( + "Path to TOSA reference model flat buffer schema. Defaults to " + f"`{cmf.DEFAULT_REF_MODEL_SCHEMA_PATH}` in parents parent directory of `ref-model-path`" + ), ) parser.add_argument( "--flatc-path", dest="flatc_path", - default="reference_model/build/thirdparty/serialization_lib/third_party/flatbuffers/flatc", - type=str, - help="Path to flatc compiler executable", + type=Path, + help=( + "Path to flatc executable. Defaults to " + f"`{cmf.DEFAULT_REF_MODEL_BUILD_FLATC_PATH}` in parent directory of `ref-model-path`" + ), ) parser.add_argument( "--ref-debug", @@ -339,13 +344,22 @@ def main(argv=None): """Start worker threads to do the testing and outputs the results.""" args = parseArgs(argv) - if ( - TOSA_REFMODEL_RUNNER in args.sut_module - and not Path(args.ref_model_path).is_file() - ): - print( - "Argument error: Reference Model not found ({})".format(args.ref_model_path) + # Set up some defaults + if args.ref_model_path is None: + args.ref_model_path = cmf.find_tosa_file( + cmf.TosaFileType.REF_MODEL, Path("reference_model"), False ) + if args.flatc_path is None: + args.flatc_path = cmf.find_tosa_file( + cmf.TosaFileType.FLATC, args.ref_model_path + ) + if args.schema_path is None: + args.schema_path = cmf.find_tosa_file( + cmf.TosaFileType.SCHEMA, args.ref_model_path + ) + + if TOSA_REFMODEL_RUNNER in args.sut_module and not args.ref_model_path.is_file(): + print(f"Argument error: Reference Model not found - {str(args.ref_model_path)}") exit(2) if args.test_list_file: diff --git a/verif/tests/test_tosa_run_tests_args.py b/verif/tests/test_tosa_run_tests_args.py index a0c3ed5..e6c5001 100644 --- a/verif/tests/test_tosa_run_tests_args.py +++ b/verif/tests/test_tosa_run_tests_args.py @@ -1,5 +1,5 @@ """Tests for tosa_verif_run_tests.py.""" -# Copyright (c) 2021-2022, ARM Limited. +# Copyright (c) 2021-2023, ARM Limited. # SPDX-License-Identifier: Apache-2.0 from runner.tosa_verif_run_tests import parseArgs @@ -15,7 +15,7 @@ def test_args_ref_model_path(): """Test arguments - ref_model_path.""" args = ["--ref-model-path", "ref_model_path", "-t", "test"] parsed_args = parseArgs(args) - assert parsed_args.ref_model_path == "ref_model_path" + assert str(parsed_args.ref_model_path) == "ref_model_path" def test_args_ref_debug(): diff --git a/verif/tests/test_tosa_verifier.py b/verif/tests/test_tosa_verifier.py index 9524158..996939c 100644 --- a/verif/tests/test_tosa_verifier.py +++ b/verif/tests/test_tosa_verifier.py @@ -26,12 +26,12 @@ def test_verifier_lib_built(): def test_checker_verifier_load_fail(): with pytest.raises(VerifierError) as excinfo: VerifierLibrary(Path("/place-that-does-not-exist")) - assert str(excinfo.value).startswith(f"Could not find {VERIFIER_LIB}") + assert str(excinfo.value).startswith("Could not find verify library") @pytest.mark.postcommit def test_checker_verifier_load(): - vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent) + vlib = VerifierLibrary(VERIFIER_LIB_PATH) assert vlib @@ -48,7 +48,7 @@ JSON_COMPLIANCE_DOT_PRODUCT = { @pytest.mark.postcommit def test_checker_verifier_dot_product_check(): - vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent) + vlib = VerifierLibrary(VERIFIER_LIB_PATH) assert vlib imp_arr = np.zeros((10, 10, 10), dtype=np.float32) @@ -63,7 +63,7 @@ def test_checker_verifier_dot_product_check(): @pytest.mark.postcommit def test_checker_verifier_dot_product_check_fail(): - vlib = VerifierLibrary(VERIFIER_LIB_PATH.parent) + vlib = VerifierLibrary(VERIFIER_LIB_PATH) assert vlib imp_arr = np.zeros((10, 10, 10), dtype=np.float32) diff --git a/verif/tests/tosa_mock_sut_run.py b/verif/tests/tosa_mock_sut_run.py index 9572618..295ee92 100644 --- a/verif/tests/tosa_mock_sut_run.py +++ b/verif/tests/tosa_mock_sut_run.py @@ -1,5 +1,5 @@ """TOSA test runner module for a mock System Under Test (SUT).""" -# Copyright (c) 2021, ARM Limited. +# Copyright (c) 2021-2023, ARM Limited. # SPDX-License-Identifier: Apache-2.0 import os @@ -33,7 +33,7 @@ class TosaSUTRunner(TosaTestRunner): print(f"MOCK SUT: Runner argument dictionary: {sutArgs}") # Useful meta data and arguments - tosaFlatbufferSchema = self.args.operator_fbs + tosaFlatbufferSchema = str(self.args.schema_path) tosaSubgraphFile = self.testDesc["tosa_file"] tosaTestDirectory = self.testDir tosaTestDescFile = self.descFile -- cgit v1.2.1