From 6179c21ce3081b51913d5219354cccac813b69ac Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Thu, 13 Jan 2022 13:46:35 +0000 Subject: Add convert2conformance script Script converts framework and reference model unit tests into a suitable format for including into a conformance suite. Signed-off-by: Jeremy Johnson Change-Id: Ida1ec8a0a7ea31fd3a3f62c4cb52d7cc2bf0b439 --- README.md | 1 + scripts/convert2conformance/__init__.py | 3 + scripts/convert2conformance/convert2conformance.py | 305 +++++++++++++++++++++ setup.cfg | 1 + 4 files changed, 310 insertions(+) create mode 100644 scripts/convert2conformance/__init__.py create mode 100755 scripts/convert2conformance/convert2conformance.py diff --git a/README.md b/README.md index f22e4ff..d4aa5f9 100644 --- a/README.md +++ b/README.md @@ -383,6 +383,7 @@ Included in this repository are some support utilities used by the test runner: binary format or the reverse operation. This is dependent on the FlatBuffers command `flatc` - see the section on the FlatBuffers compiler below. * `tosa_verif_result_check` - compares two results files. +* `convert2conformance` - converts a unit test into a conformance suitable test. Please see the respective `--help` of each utility for more information on using them standalone. diff --git a/scripts/convert2conformance/__init__.py b/scripts/convert2conformance/__init__.py new file mode 100644 index 0000000..8792170 --- /dev/null +++ b/scripts/convert2conformance/__init__.py @@ -0,0 +1,3 @@ +"""Namespace.""" +# Copyright (c) 2022 Arm Limited. +# SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/convert2conformance/convert2conformance.py b/scripts/convert2conformance/convert2conformance.py new file mode 100755 index 0000000..71f263b --- /dev/null +++ b/scripts/convert2conformance/convert2conformance.py @@ -0,0 +1,305 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021-2022, ARM Limited. +# SPDX-License-Identifier: Apache-2.0 +"""This script converts generated tests into conformance tests. + +It can convert a framework unit test or a reference model unit test. +It expects the tests have been already run on the reference model +so it can capture the result as the expected result. +""" +import argparse +import json +import logging +import os +from pathlib import Path +from typing import Optional + +from json2fbbin.json2fbbin import fbbin_to_json +from json2numpy.json2numpy import npy_to_json + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("convert2conformance") + +LOCATION_REF_MODEL_SCHEMA = Path("thirdparty/serialization_lib/schema/tosa.fbs") +LOCATION_REF_MODEL_FLATC = Path( + "build/thirdparty/serialization_lib/third_party/flatbuffers/flatc" +) + +NAME_FLATBUFFER_DIR = ["flatbuffer-", "_FW_"] +NAME_DESC_FILENAME = "desc.json" +NAME_CONFORMANCE_RESULT_PREFIX = "Conformance-" +NAME_REFMODEL_RUN_RESULT_SUFFIX = ".runner.tosa_refmodel_sut_run.npy" + + +def parse_args(argv): + """Parse the arguments.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "test_dir", + default=Path.cwd(), + type=Path, + nargs="?", + help="The test directory to convert (default is CWD)", + ) + parser.add_argument( + "--ref-model-directory", + dest="ref_model_dir", + type=Path, + required=True, + help="Reference Model directory (must be pre-built)", + ) + parser.add_argument( + "--output-directory", + dest="output_dir", + type=Path, + default=Path.cwd() / "conformance", + help="Output directory (default is conformance in CWD)", + ) + parser.add_argument( + "--framework", + dest="framework", + choices=["tflite"], + default="tflite", + help="Framework to convert (default tflite)", + ) + parser.add_argument( + "--framework-schema", + dest="framework_schema", + type=Path, + help="Framework schema needed to convert framework models", + ) + parser.add_argument( + "-v", "--verbose", dest="verbose", action="store_true", help="Verbose operation" + ) + args = parser.parse_args(argv) + return args + + +def find_ref_model_artifacts(path: Path): + """Check the location of the flatc compiler and schema artifacts.""" + flatc = path / LOCATION_REF_MODEL_FLATC + schema = path / LOCATION_REF_MODEL_SCHEMA + if not flatc.is_file(): + raise Exception( + f"flatc not found in {flatc}\nHave you built the flatbuffers compiler?" + ) + if not schema.is_file(): + raise Exception( + f"TOSA schema not found at {schema}\nHave you checked out the submodules?" + ) + return flatc, schema + + +def find_framework_artifacts(framework: str, schema_path: Path, desc_file: Path): + """Check that any required schema has been supplied for conversion.""" + if framework == "tflite": + if not schema_path: + raise Exception("the following arguments are required: --framework-schema") + elif not schema_path.is_file(): + raise Exception(f"framework schema not found at {schema_path}") + model = desc_file.parent.parent / "model.tflite" + if not model.is_file(): + raise Exception(f"Model file not found at {model}") + return schema_path, model + return None, None + + +def get_framework_name(name_array: list, framework: str): + """Get the framework conversion directory name.""" + name = "" + for part in name_array: + if part == "_FW_": + part = framework + name = f"{name}{part}" + return name + + +def convert_flatbuffer_file(flatc: Path, schema: Path, model_file: Path, output: Path): + """Convert the flatbuffer binary into JSON.""" + try: + fbbin_to_json(flatc, schema, model_file, output) + except Exception as e: + logger.error(f"Failed to convert flatbuffer binary:\n{e}") + return None + + if model_file.name == "model.tflite": + file_name = "model-tflite.json" + os.rename(output / "model.json", output / file_name) + else: + file_name = model_file.stem + ".json" + return output / file_name + + +def convert_numpy_file(n_file: Path, output: Path, outname: Optional[str] = None): + """Convert a numpy file into a JSON file.""" + j_file = output / (outname if outname else (n_file.stem + ".json")) + npy_to_json(n_file, j_file) + return j_file + + +def update_desc_json( + test_dir: Path, test_desc, output_dir: Optional[Path] = None, create_result=True +): + """Update the desc.json format for conformance and optionally create result.""" + ofm_files = [] + cfm_files = [] + if not output_dir: + output_dir = test_dir + for index, ofm in enumerate(test_desc["ofm_file"]): + ofm_path = test_dir / ofm + if not test_desc["expected_failure"]: + cfm = NAME_CONFORMANCE_RESULT_PREFIX + test_desc["ofm_name"][index] + if create_result: + if ofm_path.is_file(): + # Use the desc.json name + ofm_refmodel = ofm_path + else: + # Adjust for renaming due to tosa_verif_run_tests + ofm_refmodel = ofm_path.with_suffix(NAME_REFMODEL_RUN_RESULT_SUFFIX) + # Create conformance result + if ofm_refmodel.is_file(): + convert_numpy_file(ofm_refmodel, output_dir, outname=cfm + ".json") + else: + logger.error(f"Missing result file {ofm_path}") + return None + cfm_files.append(cfm + ".npy") + # Remove path and "ref-"/"ref_model_" from output filenames + ofm_files.append(strip_ref_output_name(ofm_path.name)) + + # Rewrite output file names as they can be relative, but keep them npys + test_desc["ofm_file"] = ofm_files + if not test_desc["expected_failure"]: + # Output expected result file for conformance if expected pass + test_desc["expected_result_file"] = cfm_files + return test_desc + + +def strip_ref_output_name(name): + """Remove mentions of reference from output files.""" + if name.startswith("ref-"): + name = name[4:] + if name.startswith("ref_model_"): + name = name[10:] + return name + + +def main(argv=None): + """Convert the given directory to a conformance test.""" + args = parse_args(argv) + # Verbosity + if args.verbose: + logger.setLevel(logging.DEBUG) + + # Check we can get the files we need + try: + flatc, schema = find_ref_model_artifacts(args.ref_model_dir) + except Exception as err: + logger.error(err) + return 2 + + # Work out where the desc.json file is + desc_filename = args.test_dir / NAME_DESC_FILENAME + framework_conversion = False + if desc_filename.is_file(): + logger.info("Found reference model unit test") + else: + desc_filename = ( + args.test_dir + / get_framework_name(NAME_FLATBUFFER_DIR, args.framework) + / NAME_DESC_FILENAME + ) + if desc_filename.is_file(): + logger.info(f"Found framework unit test for {args.framework}") + framework_conversion = True + else: + logger.error(f"Could not find {NAME_DESC_FILENAME} in {args.test_dir}") + return 2 + logger.debug(f"desc.json file: {desc_filename}") + + # Check for required files for framework conversion + if framework_conversion: + try: + framework_schema, framework_filename = find_framework_artifacts( + args.framework, args.framework_schema, desc_filename + ) + except Exception as err: + logger.error(err) + return 2 + else: + framework_schema, framework_filename = None, None + + # Open the meta desc.json file + with open(desc_filename, mode="r") as fd: + test_desc = json.load(fd) + + if "tosa_file" not in test_desc: + logger.error(f"Unsupported desc.json file found {desc_filename}") + return 2 + + # Dictionary fix + if "ifm_name" not in test_desc: + logger.warn("Old format desc.json file found - attempting to fix up") + test_desc["ifm_name"] = test_desc["ifm_placeholder"] + del test_desc["ifm_placeholder"] + + # Make the output directory if needed + try: + args.output_dir.mkdir(parents=True, exist_ok=True) + except FileExistsError: + logger.error(f"{args.output_dir} is not a directory") + return 2 + + # Convert the TOSA flatbuffer binary + tosa_filename = desc_filename.parent / test_desc["tosa_file"] + tosa_filename = convert_flatbuffer_file( + flatc, schema, tosa_filename, args.output_dir + ) + if not tosa_filename: + # Failed to convert the file, json2fbbin will have printed an error + return 1 + else: + # Replace binary with JSON name + test_desc["tosa_file"] = tosa_filename.name + + if framework_conversion and framework_filename: + # Convert the framework flatbuffer binary + framework_filename = convert_flatbuffer_file( + flatc, framework_schema, framework_filename, args.output_dir + ) + if not framework_filename: + # Failed to convert the file, json2fbbin will have printed an error + return 1 + + # Convert input files to JSON + ifm_files = [] + for file in test_desc["ifm_file"]: + if file is None: + ifm_files.append(None) + else: + path = desc_filename.parent / file + convert_numpy_file(path, args.output_dir) + ifm_files.append(path.name) + # Rewrite input file names to make sure the paths are correct, + # but keep them numpys as the test runner will convert them back + # before giving them to the SUT + test_desc["ifm_file"] = ifm_files + + # Update desc.json and convert result files to JSON + test_desc = update_desc_json( + desc_filename.parent, test_desc, output_dir=args.output_dir, create_result=True + ) + if not test_desc: + # Error from conversion/update + return 1 + + # Output new desc.json + new_desc_filename = args.output_dir / NAME_DESC_FILENAME + with open(new_desc_filename, "w") as fd: + json.dump(test_desc, fd, indent=2) + + logger.info(f"Converted to {args.output_dir}") + return 0 + + +if __name__ == "__main__": + exit(main()) diff --git a/setup.cfg b/setup.cfg index 4e3dc10..7862af4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,6 +47,7 @@ console_scripts = json2numpy = json2numpy.json2numpy:main json2fbbin = json2fbbin.json2fbbin:main tosa_verif_result_check = checker.tosa_result_checker:main + convert2conformance = convert2conformance.convert2conformance:main [tool:pytest] testpaths=verif/tests -- cgit v1.2.1