From a0848c6edbf37034e280a670bdd2f990fdf796da Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Thu, 15 Sep 2022 15:01:30 +0100 Subject: Add CONST testing to Numpy refmodel tests Signed-off-by: Jeremy Johnson Change-Id: Id32f3b7c287c9e8b0fd42dd96922be5a255598e2 --- verif/generator/tosa_test_gen.py | 2 +- verif/generator/tosa_verif_build_tests.py | 7 ++ verif/tests/test_tosa_refmodel.py | 184 +++++++++++++++++------------- 3 files changed, 115 insertions(+), 78 deletions(-) diff --git a/verif/generator/tosa_test_gen.py b/verif/generator/tosa_test_gen.py index fe05b57..b76b656 100644 --- a/verif/generator/tosa_test_gen.py +++ b/verif/generator/tosa_test_gen.py @@ -40,7 +40,7 @@ class TosaTestGen: fullPath = os.path.join(self.basePath, self.testPath) os.makedirs(fullPath, exist_ok=True) - self.ser = ts.TosaSerializer(fullPath) + self.ser = ts.TosaSerializer(fullPath, saveConstsToFile=self.args.dump_consts) def getSerializer(self): return self.ser diff --git a/verif/generator/tosa_verif_build_tests.py b/verif/generator/tosa_verif_build_tests.py index fee551b..2fafacb 100644 --- a/verif/generator/tosa_verif_build_tests.py +++ b/verif/generator/tosa_verif_build_tests.py @@ -192,6 +192,13 @@ def parseArgs(argv): help="set a particular zero point for all valid positive tests", ) + parser.add_argument( + "--dump-const-tensors", + dest="dump_consts", + action="store_true", + help="output const tensors as numpy files for inspection", + ) + args = parser.parse_args(argv) return args diff --git a/verif/tests/test_tosa_refmodel.py b/verif/tests/test_tosa_refmodel.py index 46c197a..b2f33dd 100644 --- a/verif/tests/test_tosa_refmodel.py +++ b/verif/tests/test_tosa_refmodel.py @@ -2,6 +2,7 @@ # Copyright (c) 2022, ARM Limited. # SPDX-License-Identifier: Apache-2.0 import json +import re from pathlib import Path from shutil import rmtree @@ -25,6 +26,7 @@ REF_MODEL = REF_MODEL_PATH / REF_MODEL_EXE # Default tensor shape information SHAPE_LIST = ["10", "5"] +SHAPE_DIMS = len(SHAPE_LIST) SHAPE_ARG = ",".join(SHAPE_LIST) SHAPE_OUT = "x".join(SHAPE_LIST) @@ -32,11 +34,13 @@ SHAPE_OUT = "x".join(SHAPE_LIST) OUTPUT_DIR_PREFIX = "_pytest_vtest" OUTPUT_OFM_FILE = "result_refmodel_pytest.npy" OUTPUT_RESULT_FILE = "result_numpy_pytest.npy" +OUTPUT_CONST_GLOB = "const-*.npy" TEST_DESC_FILENAME = "desc.json" # Conversion from refmodel type into the type abbreviation used in the test output REF_MODEL_TYPE_TO_OUT = { + "bool": "b", "int8": "i8", "uint8": "u8", "int16": "i16", @@ -54,11 +58,12 @@ def test_refmodel_built(): class BuildTosaTest: """Wrapper for managing lifecycle of TOSA unit tests.""" - def __init__(self, op_name, ref_model_type): + def __init__(self, op_name, ref_model_type, num_expected_tests): self.op_name = op_name self.ref_model_type = ref_model_type + self.num_expected_tests = num_expected_tests self.output_dir = None - self.test_dir = None + self.test_dirs = None def create_test(self): """Helper to generate a TOSA unit test.""" @@ -71,7 +76,7 @@ class BuildTosaTest: / f"{OUTPUT_DIR_PREFIX}_{self.op_name}_{self.ref_model_type}" ) - # Generate test without any zero-point + # Generate tests without any zero-point build_args = [ "--filter", self.op_name, @@ -81,22 +86,26 @@ class BuildTosaTest: self.ref_model_type, "--zero-point", "0", + "--num-const-inputs-concat", + "1", + "--dump-const-tensors", "-o", str(self.output_dir), ] - print(build_args) + print(f"### Building tests: tosa_verif_build_tests {' '.join(build_args)}") tosa_builder(build_args) # Find the created test test_dir = self.output_dir / self.op_name # Can't assume exact name due to broadcasting and other changes to shape - test_glob = f"{self.op_name}_*_{REF_MODEL_TYPE_TO_OUT[self.ref_model_type]}" - tests = sorted(test_dir.glob(test_glob)) - assert len(tests) == 1 - assert tests[0].is_dir() - self.test_dir = tests[0] + test_glob = f"{self.op_name}_*_{REF_MODEL_TYPE_TO_OUT[self.ref_model_type]}*" + test_dirs = sorted(test_dir.glob(test_glob)) + assert len(test_dirs) == self.num_expected_tests + for test_dir in test_dirs: + assert test_dir.is_dir() + self.test_dirs = test_dirs - return self.test_dir + return self.test_dirs def remove_test(self): if self.output_dir is not None and self.output_dir.is_dir(): @@ -110,30 +119,36 @@ class BuildTosaTest: print(f"Skipped clean up of {test_tree}") -# Tests - op_name, ref_model_type +# Tests - op_name, ref_model_type, num_expected_tests TEST_PARAMS = [ - ("add", "int32"), - ("add", "float"), - ("abs", "int32"), - ("abs", "float"), - ("negate", "int8"), - ("negate", "int16"), - ("negate", "int32"), - ("negate", "float"), + ("add", "int32", 1), + ("add", "float", 1), + ("abs", "int32", 1), + ("abs", "float", 1), + ("negate", "int8", 1), + ("negate", "int16", 1), + ("negate", "int32", 1), + ("negate", "float", 1), + # One test per axis (shape dimensions) + ("concat", "bool", SHAPE_DIMS), + ("concat", "int8", SHAPE_DIMS), + ("concat", "int16", SHAPE_DIMS), + ("concat", "int32", SHAPE_DIMS), + ("concat", "float", SHAPE_DIMS), ] def id_2_name(id): """Convert test id to name - otherwise it will be tosaTestN.""" - op_name, ref_model_type = id + op_name, ref_model_type, _ = id return f"{op_name}-{ref_model_type}" @pytest.fixture(params=TEST_PARAMS, ids=id_2_name) def tosaTest(request): """Fixture to generate the required test params and clean up.""" - op_name, ref_model_type = request.param - tst = BuildTosaTest(op_name, ref_model_type) + op_name, ref_model_type, num_expected_tests = request.param + tst = BuildTosaTest(op_name, ref_model_type, num_expected_tests) yield tst tst.remove_test() @@ -143,58 +158,73 @@ def test_refmodel_simple_op(tosaTest): """Operator testing versus Numpy.""" op_name = tosaTest.op_name - # Generate a TOSA test - test_dir = tosaTest.create_test() - - # Run ref model - desc_file = test_dir / TEST_DESC_FILENAME - assert desc_file.is_file() - refmodel_cmd = [ - str(REF_MODEL), - "--test_desc", - str(desc_file), - "--ofm_file", - OUTPUT_OFM_FILE, - ] - try: - run_sh_command(refmodel_cmd, verbose=True, capture_output=True) - except RunShCommandError as err: - assert False, f"Unexpected exception {err}" - - # Find output - ofm_file = test_dir / OUTPUT_OFM_FILE - assert ofm_file.is_file() - - # Load inputs for Numpy - with desc_file.open("r") as fp: - test_desc = json.load(fp) - tensors = [] - assert "ifm_file" in test_desc - for input_name in test_desc["ifm_file"]: - input_file = test_dir / input_name - assert input_file.is_file() - tensors.append(np.load(str(input_file))) - - # Perform Numpy operation - if op_name == "abs": - assert len(tensors) == 1 - result = np.abs(tensors[0]) - elif op_name == "add": - assert len(tensors) == 2 - result = np.add(tensors[0], tensors[1]) - elif op_name == "negate": - assert len(tensors) == 1 - result = np.negative(tensors[0]) - else: - assert False, f"Unknown operation {op_name}" - - # Save Numpy result - result_file = test_dir / OUTPUT_RESULT_FILE - np.save(str(result_file), result) - assert result_file.is_file() - - # Check Numpy result versus refmodel - check_result, tolerance, msg = tosa_check( - str(result_file), str(ofm_file), test_name=test_dir.name - ) - assert check_result == TosaResult.PASS + # Generate TOSA test(s) (mostly should be single test) + test_dirs = tosaTest.create_test() + + for test_dir in test_dirs: + # Run ref model + desc_file = test_dir / TEST_DESC_FILENAME + assert desc_file.is_file() + refmodel_cmd = [ + str(REF_MODEL), + "--test_desc", + str(desc_file), + "--ofm_file", + OUTPUT_OFM_FILE, + ] + try: + run_sh_command(refmodel_cmd, verbose=True, capture_output=True) + except RunShCommandError as err: + assert False, f"Unexpected exception {err}" + + # Find output + ofm_file = test_dir / OUTPUT_OFM_FILE + assert ofm_file.is_file() + + # Load inputs for Numpy + with desc_file.open("r") as fp: + test_desc = json.load(fp) + tensors = [] + assert "ifm_file" in test_desc + for input_name in test_desc["ifm_file"]: + input_file = test_dir / input_name + assert input_file.is_file() + tensors.append(np.load(str(input_file))) + + # Load constants for Numpy + const_files = sorted(test_dir.glob(OUTPUT_CONST_GLOB)) + consts = [] + for const_file in const_files: + assert const_file.is_file() + consts.append(np.load(str(const_file))) + + # Perform Numpy operation + if op_name == "abs": + assert len(tensors) == 1 + result = np.abs(tensors[0]) + elif op_name == "add": + assert len(tensors) == 2 + result = np.add(tensors[0], tensors[1]) + elif op_name == "concat": + assert len(consts) == 1 + # Get axis from test directory name + match = re.search(r"axis([0-9]+)", test_dir.name) + assert match is not None + axis = int(match.group(1)) + result = np.concatenate((*tensors, consts[0]), axis=axis) + elif op_name == "negate": + assert len(tensors) == 1 + result = np.negative(tensors[0]) + else: + assert False, f"Unknown operation {op_name}" + + # Save Numpy result + result_file = test_dir / OUTPUT_RESULT_FILE + np.save(str(result_file), result) + assert result_file.is_file() + + # Check Numpy result versus refmodel + check_result, tolerance, msg = tosa_check( + str(result_file), str(ofm_file), test_name=test_dir.name + ) + assert check_result == TosaResult.PASS -- cgit v1.2.1