Add convert2conformance script

Script converts framework and reference model unit tests into a
suitable format for including into a conformance suite.

Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: Ida1ec8a0a7ea31fd3a3f62c4cb52d7cc2bf0b439

4 files changed, 310 insertions, 0 deletions

diff --git a/README.md b/README.md
index f22e4ff..d4aa5f9 100644
--- a/README.md
+++ b/README.md
@@ -383,6 +383,7 @@ Included in this repository are some support utilities used by the test runner:
     binary format or the reverse operation. This is dependent on the FlatBuffers
     command `flatc` - see the section on the FlatBuffers compiler below.
 * `tosa_verif_result_check` - compares two results files.
+* `convert2conformance` - converts a unit test into a conformance suitable test.
 
 Please see the respective `--help` of each utility for more information on using
 them standalone.
diff --git a/scripts/convert2conformance/__init__.py b/scripts/convert2conformance/__init__.py
new file mode 100644
index 0000000..8792170
--- /dev/null
+++ b/scripts/convert2conformance/__init__.py
@@ -0,0 +1,3 @@
+"""Namespace."""
+# Copyright (c) 2022 Arm Limited.
+# SPDX-License-Identifier: Apache-2.0
diff --git a/scripts/convert2conformance/convert2conformance.py b/scripts/convert2conformance/convert2conformance.py
new file mode 100755
index 0000000..71f263b
--- /dev/null
+++ b/scripts/convert2conformance/convert2conformance.py
@@ -0,0 +1,305 @@
+#!/usr/bin/env python3
+# Copyright (c) 2021-2022, ARM Limited.
+# SPDX-License-Identifier: Apache-2.0
+"""This script converts generated tests into conformance tests.
+
+It can convert a framework unit test or a reference model unit test.
+It expects the tests have been already run on the reference model
+so it can capture the result as the expected result.
+"""
+import argparse
+import json
+import logging
+import os
+from pathlib import Path
+from typing import Optional
+
+from json2fbbin.json2fbbin import fbbin_to_json
+from json2numpy.json2numpy import npy_to_json
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger("convert2conformance")
+
+LOCATION_REF_MODEL_SCHEMA = Path("thirdparty/serialization_lib/schema/tosa.fbs")
+LOCATION_REF_MODEL_FLATC = Path(
+    "build/thirdparty/serialization_lib/third_party/flatbuffers/flatc"
+)
+
+NAME_FLATBUFFER_DIR = ["flatbuffer-", "_FW_"]
+NAME_DESC_FILENAME = "desc.json"
+NAME_CONFORMANCE_RESULT_PREFIX = "Conformance-"
+NAME_REFMODEL_RUN_RESULT_SUFFIX = ".runner.tosa_refmodel_sut_run.npy"
+
+
+def parse_args(argv):
+    """Parse the arguments."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "test_dir",
+        default=Path.cwd(),
+        type=Path,
+        nargs="?",
+        help="The test directory to convert (default is CWD)",
+    )
+    parser.add_argument(
+        "--ref-model-directory",
+        dest="ref_model_dir",
+        type=Path,
+        required=True,
+        help="Reference Model directory (must be pre-built)",
+    )
+    parser.add_argument(
+        "--output-directory",
+        dest="output_dir",
+        type=Path,
+        default=Path.cwd() / "conformance",
+        help="Output directory (default is conformance in CWD)",
+    )
+    parser.add_argument(
+        "--framework",
+        dest="framework",
+        choices=["tflite"],
+        default="tflite",
+        help="Framework to convert (default tflite)",
+    )
+    parser.add_argument(
+        "--framework-schema",
+        dest="framework_schema",
+        type=Path,
+        help="Framework schema needed to convert framework models",
+    )
+    parser.add_argument(
+        "-v", "--verbose", dest="verbose", action="store_true", help="Verbose operation"
+    )
+    args = parser.parse_args(argv)
+    return args
+
+
+def find_ref_model_artifacts(path: Path):
+    """Check the location of the flatc compiler and schema artifacts."""
+    flatc = path / LOCATION_REF_MODEL_FLATC
+    schema = path / LOCATION_REF_MODEL_SCHEMA
+    if not flatc.is_file():
+        raise Exception(
+            f"flatc not found in {flatc}\nHave you built the flatbuffers compiler?"
+        )
+    if not schema.is_file():
+        raise Exception(
+            f"TOSA schema not found at {schema}\nHave you checked out the submodules?"
+        )
+    return flatc, schema
+
+
+def find_framework_artifacts(framework: str, schema_path: Path, desc_file: Path):
+    """Check that any required schema has been supplied for conversion."""
+    if framework == "tflite":
+        if not schema_path:
+            raise Exception("the following arguments are required: --framework-schema")
+        elif not schema_path.is_file():
+            raise Exception(f"framework schema not found at {schema_path}")
+        model = desc_file.parent.parent / "model.tflite"
+        if not model.is_file():
+            raise Exception(f"Model file not found at {model}")
+        return schema_path, model
+    return None, None
+
+
+def get_framework_name(name_array: list, framework: str):
+    """Get the framework conversion directory name."""
+    name = ""
+    for part in name_array:
+        if part == "_FW_":
+            part = framework
+        name = f"{name}{part}"
+    return name
+
+
+def convert_flatbuffer_file(flatc: Path, schema: Path, model_file: Path, output: Path):
+    """Convert the flatbuffer binary into JSON."""
+    try:
+        fbbin_to_json(flatc, schema, model_file, output)
+    except Exception as e:
+        logger.error(f"Failed to convert flatbuffer binary:\n{e}")
+        return None
+
+    if model_file.name == "model.tflite":
+        file_name = "model-tflite.json"
+        os.rename(output / "model.json", output / file_name)
+    else:
+        file_name = model_file.stem + ".json"
+    return output / file_name
+
+
+def convert_numpy_file(n_file: Path, output: Path, outname: Optional[str] = None):
+    """Convert a numpy file into a JSON file."""
+    j_file = output / (outname if outname else (n_file.stem + ".json"))
+    npy_to_json(n_file, j_file)
+    return j_file
+
+
+def update_desc_json(
+    test_dir: Path, test_desc, output_dir: Optional[Path] = None, create_result=True
+):
+    """Update the desc.json format for conformance and optionally create result."""
+    ofm_files = []
+    cfm_files = []
+    if not output_dir:
+        output_dir = test_dir
+    for index, ofm in enumerate(test_desc["ofm_file"]):
+        ofm_path = test_dir / ofm
+        if not test_desc["expected_failure"]:
+            cfm = NAME_CONFORMANCE_RESULT_PREFIX + test_desc["ofm_name"][index]
+            if create_result:
+                if ofm_path.is_file():
+                    # Use the desc.json name
+                    ofm_refmodel = ofm_path
+                else:
+                    # Adjust for renaming due to tosa_verif_run_tests
+                    ofm_refmodel = ofm_path.with_suffix(NAME_REFMODEL_RUN_RESULT_SUFFIX)
+                # Create conformance result
+                if ofm_refmodel.is_file():
+                    convert_numpy_file(ofm_refmodel, output_dir, outname=cfm + ".json")
+                else:
+                    logger.error(f"Missing result file {ofm_path}")
+                    return None
+            cfm_files.append(cfm + ".npy")
+        # Remove path and "ref-"/"ref_model_" from output filenames
+        ofm_files.append(strip_ref_output_name(ofm_path.name))
+
+    # Rewrite output file names as they can be relative, but keep them npys
+    test_desc["ofm_file"] = ofm_files
+    if not test_desc["expected_failure"]:
+        # Output expected result file for conformance if expected pass
+        test_desc["expected_result_file"] = cfm_files
+    return test_desc
+
+
+def strip_ref_output_name(name):
+    """Remove mentions of reference from output files."""
+    if name.startswith("ref-"):
+        name = name[4:]
+    if name.startswith("ref_model_"):
+        name = name[10:]
+    return name
+
+
+def main(argv=None):
+    """Convert the given directory to a conformance test."""
+    args = parse_args(argv)
+    # Verbosity
+    if args.verbose:
+        logger.setLevel(logging.DEBUG)
+
+    # Check we can get the files we need
+    try:
+        flatc, schema = find_ref_model_artifacts(args.ref_model_dir)
+    except Exception as err:
+        logger.error(err)
+        return 2
+
+    # Work out where the desc.json file is
+    desc_filename = args.test_dir / NAME_DESC_FILENAME
+    framework_conversion = False
+    if desc_filename.is_file():
+        logger.info("Found reference model unit test")
+    else:
+        desc_filename = (
+            args.test_dir
+            / get_framework_name(NAME_FLATBUFFER_DIR, args.framework)
+            / NAME_DESC_FILENAME
+        )
+        if desc_filename.is_file():
+            logger.info(f"Found framework unit test for {args.framework}")
+            framework_conversion = True
+        else:
+            logger.error(f"Could not find {NAME_DESC_FILENAME} in {args.test_dir}")
+            return 2
+    logger.debug(f"desc.json file: {desc_filename}")
+
+    # Check for required files for framework conversion
+    if framework_conversion:
+        try:
+            framework_schema, framework_filename = find_framework_artifacts(
+                args.framework, args.framework_schema, desc_filename
+            )
+        except Exception as err:
+            logger.error(err)
+            return 2
+    else:
+        framework_schema, framework_filename = None, None
+
+    # Open the meta desc.json file
+    with open(desc_filename, mode="r") as fd:
+        test_desc = json.load(fd)
+
+    if "tosa_file" not in test_desc:
+        logger.error(f"Unsupported desc.json file found {desc_filename}")
+        return 2
+
+    # Dictionary fix
+    if "ifm_name" not in test_desc:
+        logger.warn("Old format desc.json file found - attempting to fix up")
+        test_desc["ifm_name"] = test_desc["ifm_placeholder"]
+        del test_desc["ifm_placeholder"]
+
+    # Make the output directory if needed
+    try:
+        args.output_dir.mkdir(parents=True, exist_ok=True)
+    except FileExistsError:
+        logger.error(f"{args.output_dir} is not a directory")
+        return 2
+
+    # Convert the TOSA flatbuffer binary
+    tosa_filename = desc_filename.parent / test_desc["tosa_file"]
+    tosa_filename = convert_flatbuffer_file(
+        flatc, schema, tosa_filename, args.output_dir
+    )
+    if not tosa_filename:
+        # Failed to convert the file, json2fbbin will have printed an error
+        return 1
+    else:
+        # Replace binary with JSON name
+        test_desc["tosa_file"] = tosa_filename.name
+
+    if framework_conversion and framework_filename:
+        # Convert the framework flatbuffer binary
+        framework_filename = convert_flatbuffer_file(
+            flatc, framework_schema, framework_filename, args.output_dir
+        )
+        if not framework_filename:
+            # Failed to convert the file, json2fbbin will have printed an error
+            return 1
+
+    # Convert input files to JSON
+    ifm_files = []
+    for file in test_desc["ifm_file"]:
+        if file is None:
+            ifm_files.append(None)
+        else:
+            path = desc_filename.parent / file
+            convert_numpy_file(path, args.output_dir)
+            ifm_files.append(path.name)
+    # Rewrite input file names to make sure the paths are correct,
+    # but keep them numpys as the test runner will convert them back
+    # before giving them to the SUT
+    test_desc["ifm_file"] = ifm_files
+
+    # Update desc.json and convert result files to JSON
+    test_desc = update_desc_json(
+        desc_filename.parent, test_desc, output_dir=args.output_dir, create_result=True
+    )
+    if not test_desc:
+        # Error from conversion/update
+        return 1
+
+    # Output new desc.json
+    new_desc_filename = args.output_dir / NAME_DESC_FILENAME
+    with open(new_desc_filename, "w") as fd:
+        json.dump(test_desc, fd, indent=2)
+
+    logger.info(f"Converted to {args.output_dir}")
+    return 0
+
+
+if __name__ == "__main__":
+    exit(main())
diff --git a/setup.cfg b/setup.cfg
index 4e3dc10..7862af4 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -47,6 +47,7 @@ console_scripts =
     json2numpy = json2numpy.json2numpy:main
     json2fbbin = json2fbbin.json2fbbin:main
     tosa_verif_result_check = checker.tosa_result_checker:main
+    convert2conformance = convert2conformance.convert2conformance:main
 
 [tool:pytest]
 testpaths=verif/tests