From be1a9408eb53871d96a022f59664f016926a8cf4 Mon Sep 17 00:00:00 2001 From: Jeremy Johnson Date: Wed, 15 Dec 2021 17:14:56 +0000 Subject: Update tosa_verif_run_ref Rename to tosa_verif_run_tests to match build_tests Improve output and system under test support Improve xunit support Add results checker Add utilities json2numpy and json2fbbin Add set of python tests Update README.md Signed-off-by: Jeremy Johnson Change-Id: Ia09f8e6fd126579b3ba1c1cda95c1326802417ca --- verif/runner/tosa_test_runner.py | 212 ++++++++++++++++++++++++++++++--------- 1 file changed, 167 insertions(+), 45 deletions(-) (limited to 'verif/runner/tosa_test_runner.py') diff --git a/verif/runner/tosa_test_runner.py b/verif/runner/tosa_test_runner.py index e8f921d..0fd7f13 100644 --- a/verif/runner/tosa_test_runner.py +++ b/verif/runner/tosa_test_runner.py @@ -1,68 +1,190 @@ -import os - -# Copyright (c) 2020, ARM Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - +"""Template test runner class for running TOSA tests.""" +# Copyright (c) 2020-2022, ARM Limited. +# SPDX-License-Identifier: Apache-2.0 import json -import shlex -import subprocess -from enum import IntEnum, unique - +from enum import IntEnum +from pathlib import Path -def run_sh_command(args, full_cmd, capture_output=False): - """Utility function to run an external command. Optionally return captured stdout/stderr""" +from checker.tosa_result_checker import LogColors +from checker.tosa_result_checker import print_color +from checker.tosa_result_checker import test_check +from json2fbbin import json2fbbin - # Quote the command line for printing - full_cmd_esc = [shlex.quote(x) for x in full_cmd] - if args.verbose: - print("### Running {}".format(" ".join(full_cmd_esc))) +class TosaTestInvalid(Exception): + """Exception raised for errors loading test description. - if capture_output: - rc = subprocess.run(full_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if rc.returncode != 0: - print(rc.stdout.decode("utf-8")) - print(rc.stderr.decode("utf-8")) - raise Exception( - "Error running command: {}.\n{}".format( - " ".join(full_cmd_esc), rc.stderr.decode("utf-8") - ) - ) - return (rc.stdout, rc.stderr) - else: - rc = subprocess.run(full_cmd) + Attributes: + path - full path to missing test description file + exception = underlying exception + """ - return rc.returncode + def __init__(self, path, exception): + """Initialize test not found error.""" + self.path = path + self.exception = exception + self.message = "Invalid test, could not read test description {}: {}".format( + self.path, str(self.exception) + ) + super().__init__(self.message) class TosaTestRunner: - def __init__(self, args, runnerArgs, testDir): + """TOSA Test Runner template class for systems under test.""" + def __init__(self, args, runnerArgs, testDir): + """Initialize and load JSON meta data file.""" self.args = args self.runnerArgs = runnerArgs self.testDir = testDir + self.testName = Path(self.testDir).name + + # Check if we want to run binary and if its already converted + descFilePath = Path(testDir, "desc.json") + descBinFilePath = Path(testDir, "desc_binary.json") + if args.binary: + if descBinFilePath.is_file(): + descFilePath = descBinFilePath + + try: + # Load the json test file + with open(descFilePath, "r") as fd: + self.testDesc = json.load(fd) + except Exception as e: + raise TosaTestInvalid(str(descFilePath), e) + + # Convert to binary if needed + tosaFilePath = Path(testDir, self.testDesc["tosa_file"]) + if args.binary and tosaFilePath.suffix == ".json": + # Convert tosa JSON to binary + json2fbbin.json_to_fbbin( + Path(args.flatc_path), + Path(args.operator_fbs), + tosaFilePath, + Path(testDir), + ) + # Write new desc_binary file + self.testDesc["tosa_file"] = tosaFilePath.stem + ".tosa" + with open(descBinFilePath, "w") as fd: + json.dump(self.testDesc, fd, indent=2) + descFilePath = descBinFilePath + + # Set location of desc.json (or desc_binary.json) file in use + self.descFile = str(descFilePath) - # Load the json test file - with open(os.path.join(testDir, "desc.json"), "r") as fd: - self.testDesc = json.load(fd) + def skipTest(self): + """Check if the test is skipped due to test type selection.""" + expectedFailure = self.testDesc["expected_failure"] + if self.args.test_type == "negative" and not expectedFailure: + return True + elif self.args.test_type == "positive" and expectedFailure: + return True + return False - def runModel(self): + def runTestGraph(self): + """Override with function that calls system under test.""" pass + def testResult(self, tosaGraphResult, graphMessage=None): + """Work out test result based on graph result and output files.""" + expectedFailure = self.testDesc["expected_failure"] + print_result_line = True + + if tosaGraphResult == TosaTestRunner.TosaGraphResult.TOSA_VALID: + if expectedFailure: + result = TosaTestRunner.Result.UNEXPECTED_PASS + resultMessage = "Expected failure test incorrectly passed" + else: + # Work through all the results produced by the testing, assuming success + # but overriding this with any failures found + result = TosaTestRunner.Result.EXPECTED_PASS + messages = [] + for resultNum, resultFileName in enumerate(self.testDesc["ofm_file"]): + if "expected_result_file" in self.testDesc: + try: + conformanceFile = Path( + self.testDir, + self.testDesc["expected_result_file"][resultNum], + ) + except IndexError: + result = TosaTestRunner.Result.INTERNAL_ERROR + msg = "Internal error: Missing expected_result_file {} in {}".format( + resultNum, self.descFile + ) + messages.append(msg) + print(msg) + break + else: + conformanceFile = None + resultFile = Path(self.testDir, resultFileName) + + if conformanceFile: + print_result_line = False # Checker will print one for us + chkResult, tolerance, msg = test_check( + str(conformanceFile), + str(resultFile), + test_name=self.testName, + ) + # Change EXPECTED_PASS assumption if we have any failures + if chkResult != 0: + result = TosaTestRunner.Result.UNEXPECTED_FAILURE + messages.append(msg) + if self.args.verbose: + print(msg) + else: + # No conformance file to verify, just check results file exists + if not resultFile.is_file(): + result = TosaTestRunner.Result.UNEXPECTED_FAILURE + msg = "Results file is missing: {}".format(resultFile) + messages.append(msg) + print(msg) + + if resultFile.is_file(): + # Move the resultFile to allow subsequent system under + # tests to create them and to test they have been created + resultFile = resultFile.rename( + resultFile.with_suffix( + ".{}{}".format(self.__module__, resultFile.suffix) + ) + ) + + resultMessage = "\n".join(messages) if len(messages) > 0 else None + else: + if ( + expectedFailure + and tosaGraphResult == TosaTestRunner.TosaGraphResult.TOSA_ERROR + ): + result = TosaTestRunner.Result.EXPECTED_FAILURE + resultMessage = None + else: + result = TosaTestRunner.Result.UNEXPECTED_FAILURE + resultMessage = graphMessage + + if print_result_line: + if ( + result == TosaTestRunner.Result.EXPECTED_FAILURE + or result == TosaTestRunner.Result.EXPECTED_PASS + ): + print_color(LogColors.GREEN, "Results PASS {}".format(self.testName)) + else: + print_color(LogColors.RED, "Results FAIL {}".format(self.testName)) + + return result, resultMessage + class Result(IntEnum): + """Test result codes.""" + EXPECTED_PASS = 0 EXPECTED_FAILURE = 1 UNEXPECTED_PASS = 2 UNEXPECTED_FAILURE = 3 INTERNAL_ERROR = 4 + SKIPPED = 5 + + class TosaGraphResult(IntEnum): + """The tosa_graph_result codes.""" + + TOSA_VALID = 0 + TOSA_UNPREDICTABLE = 1 + TOSA_ERROR = 2 + OTHER_ERROR = 3 -- cgit v1.2.1