From 00423436471c23c70d89d434271de34bafd5986b Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Mon, 12 Sep 2022 17:27:37 +0100 Subject: Add simple post commit reference model testing against Numpy Only perform testing of this after build of ref model using postcommit pytest mark. Signed-off-by: Jeremy Johnson Change-Id: I771a18d2c9cd4051fecafad3e6079b44f2ed62fa --- .gitignore | 4 + .pre-commit-config.yaml | 2 +- setup.cfg | 2 + verif/generator/tosa_arg_gen.py | 4 + verif/generator/tosa_test_gen.py | 4 + verif/generator/tosa_verif_build_tests.py | 16 ++- verif/tests/test_tosa_refmodel.py | 200 ++++++++++++++++++++++++++++++ 7 files changed, 227 insertions(+), 5 deletions(-) create mode 100644 verif/tests/test_tosa_refmodel.py diff --git a/.gitignore b/.gitignore index 1ef9759..6674c8c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,8 @@ build/ dist/ *.egg-info verif/tests/test-result-*.tests.tosa_mock_sut_run.npy +verif/tests/_pytest_vtest_* .coverage +result.xml +vtest/ +examples/*/*/out.npy \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f2ccc3f..b253ddd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,6 +25,6 @@ repos: name: pytest stages: [commit] language: system - entry: pytest + entry: pytest -m "not postcommit" types: [python] pass_filenames: false diff --git a/setup.cfg b/setup.cfg index 1e4c55d..a9dc3ab 100644 --- a/setup.cfg +++ b/setup.cfg @@ -60,6 +60,8 @@ console_scripts = [tool:pytest] testpaths=verif/tests +markers = + postcommit: marks tests for post commit testing on CI as part of the review [flake8] ignore = D213, E203, E266, E501, W503 diff --git a/verif/generator/tosa_arg_gen.py b/verif/generator/tosa_arg_gen.py index ef84762..a65e220 100644 --- a/verif/generator/tosa_arg_gen.py +++ b/verif/generator/tosa_arg_gen.py @@ -29,8 +29,12 @@ class TosaQuantGen: def getZeroPoint(testGen, dtype, error_name=None): if dtype == DType.INT8: + if testGen.args.zeropoint is not None: + return min(127, max(-128, testGen.args.zeropoint)) return testGen.randInt(-128, 128) elif dtype == DType.UINT8: + if testGen.args.zeropoint is not None: + return min(255, max(0, testGen.args.zeropoint)) return testGen.randInt(0, 256) elif error_name in [ ErrorIf.InputZeroPointNotZero, diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py index 56a34e5..fe05b57 100644 --- a/verif/generator/tosa_test_gen.py +++ b/verif/generator/tosa_test_gen.py @@ -2252,6 +2252,10 @@ class TosaTestGen: def createDynamicOpLists(self): + if "conv2d_TEMPLATE" not in self.TOSA_OP_LIST: + # Already created these lists (can occur when class is initialized more than once) + return + # Dynamically create op lists for convolutions with a list of kernel sizes KERNELS_2D = [[1, 1], [2, 2], [3, 3], [5, 5], [3, 1], [1, 3]] diff --git a/verif/generator/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py index 6ee873f..fee551b 100644 --- a/verif/generator/tosa_verif_build_tests.py +++ b/verif/generator/tosa_verif_build_tests.py @@ -23,7 +23,7 @@ def auto_int(x): return int(x, 0) -def parseArgs(): +def parseArgs(argv): parser = argparse.ArgumentParser() parser.add_argument( @@ -184,14 +184,22 @@ def parseArgs(): help="allow oversize padding, stride and kernel tests", ) - args = parser.parse_args() + parser.add_argument( + "--zero-point", + dest="zeropoint", + default=None, + type=int, + help="set a particular zero point for all valid positive tests", + ) + + args = parser.parse_args(argv) return args -def main(): +def main(argv=None): - args = parseArgs() + args = parseArgs(argv) ttg = TosaTestGen(args) diff --git a/verif/tests/test_tosa_refmodel.py b/verif/tests/test_tosa_refmodel.py new file mode 100644 index 0000000..46c197a --- /dev/null +++ b/verif/tests/test_tosa_refmodel.py @@ -0,0 +1,200 @@ +"""Tests for tosa_reference_model.""" +# Copyright (c) 2022, ARM Limited. +# SPDX-License-Identifier: Apache-2.0 +import json +from pathlib import Path +from shutil import rmtree + +import numpy as np +import pytest +from checker.tosa_result_checker import test_check as tosa_check +from checker.tosa_result_checker import TestResult as TosaResult +from generator.tosa_verif_build_tests import main as tosa_builder +from runner.run_command import run_sh_command +from runner.run_command import RunShCommandError + +# Note: Must rename imports so that pytest doesn't assume its a test function/class + +# Set this to False if you want ot preserve the test directories after running +CLEAN_UP_TESTS = True + +# Location of reference model binary +REF_MODEL_PATH = Path(__file__).resolve().parents[2] / "build" / "reference_model" +REF_MODEL_EXE = "tosa_reference_model" +REF_MODEL = REF_MODEL_PATH / REF_MODEL_EXE + +# Default tensor shape information +SHAPE_LIST = ["10", "5"] +SHAPE_ARG = ",".join(SHAPE_LIST) +SHAPE_OUT = "x".join(SHAPE_LIST) + +# Output file information +OUTPUT_DIR_PREFIX = "_pytest_vtest" +OUTPUT_OFM_FILE = "result_refmodel_pytest.npy" +OUTPUT_RESULT_FILE = "result_numpy_pytest.npy" + +TEST_DESC_FILENAME = "desc.json" + +# Conversion from refmodel type into the type abbreviation used in the test output +REF_MODEL_TYPE_TO_OUT = { + "int8": "i8", + "uint8": "u8", + "int16": "i16", + "int32": "i32", + "float": "float", +} + + +@pytest.mark.postcommit +def test_refmodel_built(): + """First test to check the reference model has been built.""" + assert REF_MODEL.is_file() + + +class BuildTosaTest: + """Wrapper for managing lifecycle of TOSA unit tests.""" + + def __init__(self, op_name, ref_model_type): + self.op_name = op_name + self.ref_model_type = ref_model_type + self.output_dir = None + self.test_dir = None + + def create_test(self): + """Helper to generate a TOSA unit test.""" + if self.output_dir is not None: + # Already created + return self.test_dir + + self.output_dir = ( + Path(__file__).parent + / f"{OUTPUT_DIR_PREFIX}_{self.op_name}_{self.ref_model_type}" + ) + + # Generate test without any zero-point + build_args = [ + "--filter", + self.op_name, + "--target-shape", + SHAPE_ARG, + "--target-dtype", + self.ref_model_type, + "--zero-point", + "0", + "-o", + str(self.output_dir), + ] + print(build_args) + tosa_builder(build_args) + + # Find the created test + test_dir = self.output_dir / self.op_name + # Can't assume exact name due to broadcasting and other changes to shape + test_glob = f"{self.op_name}_*_{REF_MODEL_TYPE_TO_OUT[self.ref_model_type]}" + tests = sorted(test_dir.glob(test_glob)) + assert len(tests) == 1 + assert tests[0].is_dir() + self.test_dir = tests[0] + + return self.test_dir + + def remove_test(self): + if self.output_dir is not None and self.output_dir.is_dir(): + # Delete directory + test_tree = self.output_dir.resolve() + if CLEAN_UP_TESTS: + print(f"Deleting {test_tree}") + rmtree(str(test_tree)) + self.output_dir = None + else: + print(f"Skipped clean up of {test_tree}") + + +# Tests - op_name, ref_model_type +TEST_PARAMS = [ + ("add", "int32"), + ("add", "float"), + ("abs", "int32"), + ("abs", "float"), + ("negate", "int8"), + ("negate", "int16"), + ("negate", "int32"), + ("negate", "float"), +] + + +def id_2_name(id): + """Convert test id to name - otherwise it will be tosaTestN.""" + op_name, ref_model_type = id + return f"{op_name}-{ref_model_type}" + + +@pytest.fixture(params=TEST_PARAMS, ids=id_2_name) +def tosaTest(request): + """Fixture to generate the required test params and clean up.""" + op_name, ref_model_type = request.param + tst = BuildTosaTest(op_name, ref_model_type) + yield tst + tst.remove_test() + + +@pytest.mark.postcommit +def test_refmodel_simple_op(tosaTest): + """Operator testing versus Numpy.""" + op_name = tosaTest.op_name + + # Generate a TOSA test + test_dir = tosaTest.create_test() + + # Run ref model + desc_file = test_dir / TEST_DESC_FILENAME + assert desc_file.is_file() + refmodel_cmd = [ + str(REF_MODEL), + "--test_desc", + str(desc_file), + "--ofm_file", + OUTPUT_OFM_FILE, + ] + try: + run_sh_command(refmodel_cmd, verbose=True, capture_output=True) + except RunShCommandError as err: + assert False, f"Unexpected exception {err}" + + # Find output + ofm_file = test_dir / OUTPUT_OFM_FILE + assert ofm_file.is_file() + + # Load inputs for Numpy + with desc_file.open("r") as fp: + test_desc = json.load(fp) + tensors = [] + assert "ifm_file" in test_desc + for input_name in test_desc["ifm_file"]: + input_file = test_dir / input_name + assert input_file.is_file() + tensors.append(np.load(str(input_file))) + + # Perform Numpy operation + if op_name == "abs": + assert len(tensors) == 1 + result = np.abs(tensors[0]) + elif op_name == "add": + assert len(tensors) == 2 + result = np.add(tensors[0], tensors[1]) + elif op_name == "negate": + assert len(tensors) == 1 + result = np.negative(tensors[0]) + else: + assert False, f"Unknown operation {op_name}" + + # Save Numpy result + result_file = test_dir / OUTPUT_RESULT_FILE + np.save(str(result_file), result) + assert result_file.is_file() + + # Check Numpy result versus refmodel + check_result, tolerance, msg = tosa_check( + str(result_file), str(ofm_file), test_name=test_dir.name + ) + assert check_result == TosaResult.PASS -- cgit v1.2.1