From 9c2fe6e129e4d176c3e14f172b92efe985af7c78 Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Wed, 4 Oct 2023 16:55:04 +0100 Subject: Conformance simple backward version and binary files support Add --test-version option to tosa_verif_conformance_generator to select the version for tests to output. Add --output-type to allow json, binary or both files to be created during conformance generation. Fix passing schema_path to test runner. Add explicit verify lib path arg to test runner. Change-Id: I5f1ad137d713fca408a98470ea77bddf8916c5f3 Signed-off-by: Jeremy Johnson --- .../tosa_verif_conformance_generator.py | 41 ++++++++++++++++++---- verif/runner/tosa_test_runner.py | 5 +-- verif/runner/tosa_verif_run_tests.py | 13 +++++++ 3 files changed, 48 insertions(+), 11 deletions(-) (limited to 'verif') diff --git a/verif/conformance/tosa_verif_conformance_generator.py b/verif/conformance/tosa_verif_conformance_generator.py index c9a0b3a..4281fc2 100644 --- a/verif/conformance/tosa_verif_conformance_generator.py +++ b/verif/conformance/tosa_verif_conformance_generator.py @@ -8,7 +8,7 @@ Steps: settings in the .json files. - Tests are selected to produce a good coverage. - Tests are run on the reference model to produce the correct output files. -- Tests are converted into JSON format and saved to desired output directory. +- Tests are converted to JSON and/or copied and saved to desired output directory. """ import argparse import copy @@ -26,6 +26,8 @@ from pathlib import Path import conformance.model_files as cmf from conformance.test_select import Operator from convert2conformance.convert2conformance import main as c2c_main +from convert2conformance.convert2conformance import OUTPUT_TYPE_DEFAULT +from convert2conformance.convert2conformance import OUTPUT_TYPES from distutils.dir_util import copy_tree logging.basicConfig() @@ -51,6 +53,10 @@ DEFAULT_SEED = 42 # standard group will have negative tests generated for it STANDARD_GENERATOR_GROUP = "standard" +TEST_VERSION_LATEST = "latest" +TEST_VERSION_V0_60_0 = "v0.60.0" +TEST_VERSIONS = (TEST_VERSION_LATEST, TEST_VERSION_V0_60_0) + class GenConformanceError(Exception): """Generation error reporting exception.""" @@ -214,12 +220,14 @@ def generate_results(args, profile, operator, op_build_dir, supports=[], tests=N return num_cores = args.num_cores - run_tests_cmd = "tosa_verif_run_tests" - ref_cmd_base = ref_cmd = [ - run_tests_cmd, + # Use the test runner + ref_cmd_base = [ + "tosa_verif_run_tests", "--ref-model-path", str(args.ref_model_path.absolute()), + "--schema-path", + str(args.schema_path.absolute()), "-j", str(num_cores), "-v", @@ -243,7 +251,7 @@ def generate_results(args, profile, operator, op_build_dir, supports=[], tests=N ) continue ref_cmd = ref_cmd_base.copy() - ref_cmd.append(str(test)) + ref_cmd.append(str(test.absolute())) ref_cmds.append(ref_cmd) fail_string = "UNEXPECTED_FAILURE" @@ -280,13 +288,14 @@ def convert_tests( trim_op_subdir=False, tags=None, ): - """Convert tests to JSON and save to output directory.""" + """Convert/copy tests to output directory.""" if group: output_dir = output_dir / group c2c_args_base = ["--strict"] c2c_args_base.extend(["--schema-path", str(args.schema_path)]) c2c_args_base.extend(["--flatc-path", str(args.flatc_path)]) + c2c_args_base.extend(["--output-type", args.output_type]) # This op maybe in more than one profile - e.g. tosa_bi and tosa_mi # even if we are only producing tests for tosa_mi for op_profile in op_profiles_list: @@ -349,7 +358,7 @@ def convert_tests( logger.error(f"Stopping due to {failed_counter} test conversion errors") raise (GenConformanceError()) - logger.info("Converted tests to JSON and saved to output directory") + logger.info("Converted/copied tests and saved to output directory") return output_dir @@ -534,6 +543,20 @@ def parse_args(argv=None): f"`{cmf.DEFAULT_REF_MODEL_BUILD_FLATC_PATH}` in parent directory of `ref-model-path`" ), ) + parser.add_argument( + "--test-version", + dest="test_version", + choices=TEST_VERSIONS, + default=TEST_VERSION_LATEST, + help=f"Version of the tests to produce (default is {TEST_VERSION_LATEST})", + ) + parser.add_argument( + "--output-type", + dest="output_type", + choices=OUTPUT_TYPES, + default=OUTPUT_TYPE_DEFAULT, + help=f"Output file type produced (default is {OUTPUT_TYPE_DEFAULT})", + ) parser.add_argument( "--seed", dest="random_seed", @@ -778,6 +801,10 @@ def main(): ) continue + if args.test_version == TEST_VERSION_V0_60_0 and op in ("dim",): + logger.warning(f"{op} is not in {args.test_version} - skipping") + continue + op_profiles_list = test_params[op]["profile"] if ( args.profile != PROFILES_ALL diff --git a/verif/runner/tosa_test_runner.py b/verif/runner/tosa_test_runner.py index 30a7168..b348f50 100644 --- a/verif/runner/tosa_test_runner.py +++ b/verif/runner/tosa_test_runner.py @@ -4,7 +4,6 @@ import json from enum import IntEnum -import conformance.model_files as cmf import schemavalidation.schemavalidation as sch from checker.color_print import LogColors from checker.color_print import print_color @@ -71,9 +70,7 @@ class TosaTestRunner: self.testDir = str(testDirPath) self.testDirPath = testDirPath self.testName = self.testDirPath.name - self.verify_lib_path = cmf.find_tosa_file( - cmf.TosaFileType.VERIFY_LIBRARY, args.ref_model_path - ) + self.verify_lib_path = args.verify_lib_path set_print_in_color(not args.no_color) # Stop the result checker printing anything - we will do it diff --git a/verif/runner/tosa_verif_run_tests.py b/verif/runner/tosa_verif_run_tests.py index d1755e6..54cb7b2 100644 --- a/verif/runner/tosa_verif_run_tests.py +++ b/verif/runner/tosa_verif_run_tests.py @@ -51,6 +51,15 @@ def parseArgs(argv): type=Path, help="Path to TOSA reference model executable", ) + parser.add_argument( + "--verify-lib-path", + dest="verify_lib_path", + type=Path, + help=( + "Path to TOSA verify library. Defaults to " + "the library in the directory of `ref-model-path`" + ), + ) parser.add_argument( "--operator-fbs", "--schema-path", @@ -365,6 +374,10 @@ def main(argv=None): args.ref_model_path = cmf.find_tosa_file( cmf.TosaFileType.REF_MODEL, Path("reference_model"), False ) + if args.verify_lib_path is None: + args.verify_lib_path = cmf.find_tosa_file( + cmf.TosaFileType.VERIFY_LIBRARY, args.ref_model_path + ) if args.flatc_path is None: args.flatc_path = cmf.find_tosa_file( cmf.TosaFileType.FLATC, args.ref_model_path -- cgit v1.2.1