diff options
author | Jeremy Johnson <jeremy.johnson@arm.com> | 2021-12-15 17:14:56 +0000 |
---|---|---|
committer | Jeremy Johnson <jeremy.johnson@arm.com> | 2022-01-06 11:40:12 +0000 |
commit | be1a9408eb53871d96a022f59664f016926a8cf4 (patch) | |
tree | 458e8a389c0c909fc6008dfb4cc577e1b0a895e5 /verif/tests/test_tosa_run_tests_mocksut.py | |
parent | 2ec3494060ffdafec072fe1b2099a8177b8eca6a (diff) | |
download | reference_model-be1a9408eb53871d96a022f59664f016926a8cf4.tar.gz |
Update tosa_verif_run_ref
Rename to tosa_verif_run_tests to match build_tests
Improve output and system under test support
Improve xunit support
Add results checker
Add utilities json2numpy and json2fbbin
Add set of python tests
Update README.md
Signed-off-by: Jeremy Johnson <jeremy.johnson@arm.com>
Change-Id: Ia09f8e6fd126579b3ba1c1cda95c1326802417ca
Diffstat (limited to 'verif/tests/test_tosa_run_tests_mocksut.py')
-rw-r--r-- | verif/tests/test_tosa_run_tests_mocksut.py | 241 |
1 files changed, 241 insertions, 0 deletions
diff --git a/verif/tests/test_tosa_run_tests_mocksut.py b/verif/tests/test_tosa_run_tests_mocksut.py new file mode 100644 index 0000000..98044e0 --- /dev/null +++ b/verif/tests/test_tosa_run_tests_mocksut.py @@ -0,0 +1,241 @@ +"""Tests for tosa_verif_run_tests.py.""" +# Copyright (c) 2021-2022, ARM Limited. +# SPDX-License-Identifier: Apache-2.0 +import json +from copy import deepcopy +from pathlib import Path +from xml.dom import minidom + +import pytest + +from runner.tosa_verif_run_tests import main + + +TEST_DESC = { + "tosa_file": "pytest.json", + "ifm_name": ["test-0", "test-1"], + "ifm_file": ["test-0.npy", "test-1.npy"], + "ofm_name": ["test-result-0"], + "ofm_file": ["test-result-0.npy"], + "expected_failure": False, +} +GRAPH_RESULT_VALID = "valid" +GRAPH_RESULT_ERROR = "error" + + +def _create_desc_json(json_object) -> Path: + """Create test desc.json.""" + file = Path(__file__).parent / "desc.json" + with open(file, "w") as fd: + json.dump(json_object, fd, indent=2) + return file + + +def _delete_desc_json(file: Path): + """Clean up desc.json.""" + binary_file = file.parent / "desc_binary.json" + if binary_file.exists(): + print(binary_file.read_text()) + binary_file.unlink() + else: + print(file.read_text()) + file.unlink() + + +@pytest.fixture +def testDir() -> str: + """Set up a mock expected pass test.""" + print("SET UP - testDir") + file = _create_desc_json(TEST_DESC) + yield file.parent + print("TEAR DOWN - testDir") + _delete_desc_json(file) + + +@pytest.fixture +def testDirExpectedFail() -> str: + """Set up a mock expected fail test.""" + print("SET UP - testDirExpectedFail") + fail = deepcopy(TEST_DESC) + fail["expected_failure"] = True + file = _create_desc_json(fail) + yield file.parent + print("TEAR DOWN - testDirExpectedFail") + _delete_desc_json(file) + + +@pytest.fixture +def testDirMultiOutputs() -> str: + """Set up a mock multiple results output test.""" + print("SET UP - testDirMultiOutputs") + out = deepcopy(TEST_DESC) + out["ofm_name"].append("tr1") + out["ofm_file"].append("test-result-1.npy") + file = _create_desc_json(out) + yield file.parent + print("TEAR DOWN - testDirMultiOutputs") + _delete_desc_json(file) + + +def _get_default_argv(testDir: Path, graphResult: str) -> list: + """Create default args based on test directory and graph result.""" + return [ + "--sut-module", + "tests.tosa_mock_sut_run", + "--test", + str(testDir), + "--xunit-file", + str(testDir / "result.xml"), + # Must be last argument to allow easy extension with extra args + "--sut-module-args", + f"tests.tosa_mock_sut_run:graph={graphResult}", + ] + + +def _get_xml_results(argv: list): + """Get XML results and remove file.""" + resultsFile = Path(argv[argv.index("--xunit-file") + 1]) + results = minidom.parse(str(resultsFile)) + resultsFile.unlink() + return results + + +def _get_xml_testsuites_from_results(results, expectedTestSuites: int): + """Get XML testcases from results.""" + testSuites = results.getElementsByTagName("testsuite") + assert len(testSuites) == expectedTestSuites + return testSuites + + +def _get_xml_testcases_from_results(results, expectedTestCases: int): + """Get XML testcases from results.""" + testCases = results.getElementsByTagName("testcase") + assert len(testCases) == expectedTestCases + return testCases + + +def _get_xml_failure(argv: list): + """Get the results and single testcase with the failure result entry if there is one.""" + results = _get_xml_results(argv) + testCases = _get_xml_testcases_from_results(results, 1) + fail = testCases[0].getElementsByTagName("failure") + if fail: + return fail[0].firstChild.data + return None + + +def test_mock_sut_expected_pass(testDir: Path): + """Run expected pass SUT test.""" + try: + argv = _get_default_argv(testDir, GRAPH_RESULT_VALID) + main(argv) + fail = _get_xml_failure(argv) + except Exception as e: + assert False, f"Unexpected exception {e}" + assert not fail + + +UNEXPECTED_PASS_PREFIX_STR = "UNEXPECTED_PASS" +UNEXPECTED_FAIL_PREFIX_STR = "UNEXPECTED_FAIL" + + +def test_mock_sut_unexpected_pass(testDirExpectedFail: Path): + """Run unexpected pass SUT test.""" + try: + argv = _get_default_argv(testDirExpectedFail, GRAPH_RESULT_VALID) + main(argv) + fail = _get_xml_failure(argv) + except Exception as e: + assert False, f"Unexpected exception {e}" + assert fail.startswith(UNEXPECTED_PASS_PREFIX_STR) + + +def test_mock_sut_expected_failure(testDirExpectedFail: Path): + """Run expected failure SUT test.""" + try: + argv = _get_default_argv(testDirExpectedFail, GRAPH_RESULT_ERROR) + main(argv) + fail = _get_xml_failure(argv) + except Exception as e: + assert False, f"Unexpected exception {e}" + assert not fail + + +def test_mock_sut_unexpected_failure(testDir: Path): + """Run unexpected failure SUT test.""" + try: + argv = _get_default_argv(testDir, GRAPH_RESULT_ERROR) + main(argv) + fail = _get_xml_failure(argv) + except Exception as e: + assert False, f"Unexpected exception {e}" + assert fail.startswith(UNEXPECTED_FAIL_PREFIX_STR) + + +def test_mock_sut_binary_conversion(testDir: Path): + """Run unexpected failure SUT test.""" + try: + argv = _get_default_argv(testDir, GRAPH_RESULT_VALID) + argv.extend(["--binary", "--flatc-path", str(testDir / "mock_flatc.py")]) + main(argv) + binary_desc = testDir / "desc_binary.json" + assert binary_desc.exists() + fail = _get_xml_failure(argv) + except Exception as e: + assert False, f"Unexpected exception {e}" + assert not fail + + +def test_mock_and_dummy_sut_results(testDir: Path): + """Run two SUTs and check they both return results.""" + try: + argv = _get_default_argv(testDir, GRAPH_RESULT_VALID) + # Override sut-module setting with both SUTs + argv.extend( + ["--sut-module", "tests.tosa_dummy_sut_run", "tests.tosa_mock_sut_run"] + ) + main(argv) + results = _get_xml_results(argv) + _get_xml_testsuites_from_results(results, 2) + _get_xml_testcases_from_results(results, 2) + except Exception as e: + assert False, f"Unexpected exception {e}" + + +def test_two_mock_suts(testDir: Path): + """Test that a duplicated SUT is ignored.""" + try: + argv = _get_default_argv(testDir, GRAPH_RESULT_VALID) + # Override sut-module setting with duplicated SUT + argv.extend( + ["--sut-module", "tests.tosa_mock_sut_run", "tests.tosa_mock_sut_run"] + ) + main(argv) + results = _get_xml_results(argv) + _get_xml_testsuites_from_results(results, 1) + _get_xml_testcases_from_results(results, 1) + except Exception as e: + assert False, f"Unexpected exception {e}" + + +def test_mock_sut_multi_outputs_expected_pass(testDirMultiOutputs: Path): + """Run expected pass SUT test with multiple outputs.""" + try: + argv = _get_default_argv(testDirMultiOutputs, GRAPH_RESULT_VALID) + main(argv) + fail = _get_xml_failure(argv) + except Exception as e: + assert False, f"Unexpected exception {e}" + assert not fail + + +def test_mock_sut_multi_outputs_unexpected_failure(testDirMultiOutputs: Path): + """Run SUT test which expects multiple outputs, but last one is missing.""" + try: + argv = _get_default_argv(testDirMultiOutputs, GRAPH_RESULT_VALID) + argv.append("tests.tosa_mock_sut_run:num_results=1") + main(argv) + fail = _get_xml_failure(argv) + except Exception as e: + assert False, f"Unexpected exception {e}" + assert fail.startswith(UNEXPECTED_FAIL_PREFIX_STR) |