From 4eb3fef8e5876c69dc6bac70fdc010805d5b97f2 Mon Sep 17 00:00:00 2001 From: Ruomei Yan Date: Tue, 13 Dec 2022 22:02:21 +0000 Subject: MLIA-741/2 Report test results - add version extraction function in compat.py - create Metadata, MLIAMetadata, TOSAMetadata and MetadataDisplay classes - update the reporting functions so tosa and mlia version will be displayed in output json - update unit test test_configure_and_get_tosa_advisor to mock the get_events function - update the copyright information of all changed/added files - handle exception and report to json when program crashes - write new context managers for capturing stderr and stdout - support reporting stderr to json output - support reporting model checksum and model name to json output - made changes in test_e2e.py handling {model_name} replacement in --output - add unit tests Change-Id: I6629fd1c5754378e6accd488217c83d87c7eb6f1 --- tests/test_backend_tosa_compat.py | 38 +++++++++++++++++---------- tests/test_target_tosa_advisor.py | 15 +++++++++-- tests/test_target_tosa_reporters.py | 52 +++++++++++++++++++++++++++++++++++++ tests/test_utils_logging.py | 15 ++++++++++- tests/test_utils_misc.py | 10 ++++++- 5 files changed, 112 insertions(+), 18 deletions(-) create mode 100644 tests/test_target_tosa_reporters.py (limited to 'tests') diff --git a/tests/test_backend_tosa_compat.py b/tests/test_backend_tosa_compat.py index 52aaa44..5a80b4b 100644 --- a/tests/test_backend_tosa_compat.py +++ b/tests/test_backend_tosa_compat.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for TOSA compatibility.""" from __future__ import annotations @@ -39,12 +39,13 @@ def test_compatibility_check_should_fail_if_checker_not_available( @pytest.mark.parametrize( - "is_tosa_compatible, operators, expected_result", + "is_tosa_compatible, operators, exception, expected_result", [ [ True, [], - TOSACompatibilityInfo(True, []), + None, + TOSACompatibilityInfo(True, [], None, None, None), ], [ True, @@ -55,18 +56,16 @@ def test_compatibility_check_should_fail_if_checker_not_available( is_tosa_compatible=True, ) ], - TOSACompatibilityInfo(True, [Operator("op_location", "op_name", True)]), + None, + TOSACompatibilityInfo( + True, [Operator("op_location", "op_name", True)], None, [], [] + ), ], [ False, - [ - SimpleNamespace( - location="op_location", - name="op_name", - is_tosa_compatible=False, - ) - ], - TOSACompatibilityInfo(False, [Operator("op_location", "op_name", False)]), + [], + ValueError("error_test"), + TOSACompatibilityInfo(False, [], ValueError("error_test"), [], []), ], ], ) @@ -75,6 +74,7 @@ def test_get_tosa_compatibility_info( test_tflite_model: Path, is_tosa_compatible: bool, operators: Any, + exception: Exception | None, expected_result: TOSACompatibilityInfo, ) -> None: """Test getting TOSA compatibility information.""" @@ -83,7 +83,17 @@ def test_get_tosa_compatibility_info( mock_checker._get_tosa_compatibility_for_ops.return_value = ( # pylint: disable=protected-access operators ) - + if exception: + mock_checker._get_tosa_compatibility_for_ops.side_effect = ( # pylint: disable=protected-access + exception + ) replace_get_tosa_checker_with_mock(monkeypatch, mock_checker) - assert get_tosa_compatibility_info(test_tflite_model) == expected_result + returned_compatibility_info = get_tosa_compatibility_info(test_tflite_model) + assert repr(returned_compatibility_info.exception) == repr( + expected_result.exception + ) + assert ( + returned_compatibility_info.tosa_compatible == expected_result.tosa_compatible + ) + assert returned_compatibility_info.operators == expected_result.operators diff --git a/tests/test_target_tosa_advisor.py b/tests/test_target_tosa_advisor.py index 32a6b77..9646c96 100644 --- a/tests/test_target_tosa_advisor.py +++ b/tests/test_target_tosa_advisor.py @@ -1,7 +1,10 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for TOSA advisor.""" from pathlib import Path +from unittest.mock import MagicMock + +import pytest from mlia.core.context import ExecutionContext from mlia.core.workflow import DefaultWorkflowExecutor @@ -9,15 +12,23 @@ from mlia.target.tosa.advisor import configure_and_get_tosa_advisor from mlia.target.tosa.advisor import TOSAInferenceAdvisor -def test_configure_and_get_tosa_advisor(test_tflite_model: Path) -> None: +def test_configure_and_get_tosa_advisor( + monkeypatch: pytest.MonkeyPatch, test_tflite_model: Path +) -> None: """Test TOSA advisor configuration.""" ctx = ExecutionContext() + get_events_mock = MagicMock() + monkeypatch.setattr( + "mlia.target.tosa.advisor.TOSAInferenceAdvisor.get_events", + MagicMock(return_value=get_events_mock), + ) advisor = configure_and_get_tosa_advisor(ctx, "tosa", test_tflite_model) workflow = advisor.configure(ctx) assert isinstance(advisor, TOSAInferenceAdvisor) + assert advisor.get_events(ctx) == get_events_mock assert ctx.event_handlers is not None assert ctx.config_parameters == { "tosa_inference_advisor": { diff --git a/tests/test_target_tosa_reporters.py b/tests/test_target_tosa_reporters.py new file mode 100644 index 0000000..59da270 --- /dev/null +++ b/tests/test_target_tosa_reporters.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright 2023, Arm Limited and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for tosa-checker reporters.""" +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from mlia.core.metadata import MLIAMetadata +from mlia.core.metadata import ModelMetadata +from mlia.core.reporting import Report +from mlia.target.tosa.config import TOSAConfiguration +from mlia.target.tosa.metadata import TOSAMetadata +from mlia.target.tosa.reporters import MetadataDisplay +from mlia.target.tosa.reporters import report_device +from mlia.target.tosa.reporters import tosa_formatters + + +def test_tosa_report_device() -> None: + """Test function report_device().""" + report = report_device(TOSAConfiguration("tosa")) + assert report.to_plain_text() + + +def test_tosa_formatters( + monkeypatch: pytest.MonkeyPatch, test_tflite_model: Path +) -> None: + """Test function tosa_formatters() with valid input.""" + mock_version = MagicMock() + monkeypatch.setattr( + "mlia.core.metadata.get_pkg_version", + MagicMock(return_value=mock_version), + ) + + data = MetadataDisplay( + TOSAMetadata("tosa-checker"), + MLIAMetadata("mlia"), + ModelMetadata(test_tflite_model), + ) + formatter = tosa_formatters(data) + report = formatter(data) + assert data.tosa_version == mock_version + assert isinstance(report, Report) + + +def test_tosa_formatters_invalid_data() -> None: + """Test tosa_formatters() with invalid input.""" + with pytest.raises( + Exception, + match=r"^Unable to find appropriate formatter for .*", + ): + tosa_formatters(12) diff --git a/tests/test_utils_logging.py b/tests/test_utils_logging.py index ac835c6..c02e8b0 100644 --- a/tests/test_utils_logging.py +++ b/tests/test_utils_logging.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for the logging utility functions.""" from __future__ import annotations @@ -13,6 +13,7 @@ from unittest.mock import MagicMock import pytest +from mlia.utils.logging import capture_raw_output from mlia.utils.logging import create_log_handler from mlia.utils.logging import redirect_output from mlia.utils.logging import redirect_raw_output @@ -84,3 +85,15 @@ def test_output_redirection(redirect_context_manager: Callable) -> None: print("after redirect") logger_mock.log.assert_called_once_with(logging.INFO, "output redirected") + + +def test_output_and_error_capture() -> None: + """Test output/error capturing.""" + with capture_raw_output(sys.stdout) as std_output, capture_raw_output( + sys.stderr + ) as stderr_output: + print("hello from stdout") + print("hello from stderr", file=sys.stderr) + + assert std_output == ["hello from stdout\n"] + assert stderr_output == ["hello from stderr\n"] diff --git a/tests/test_utils_misc.py b/tests/test_utils_misc.py index 011d09e..ae91850 100644 --- a/tests/test_utils_misc.py +++ b/tests/test_utils_misc.py @@ -1,10 +1,11 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for misc util functions.""" from unittest.mock import MagicMock import pytest +from mlia.utils.misc import get_pkg_version from mlia.utils.misc import yes @@ -23,3 +24,10 @@ def test_yes( """Test yes function.""" monkeypatch.setattr("builtins.input", MagicMock(return_value=response)) assert yes("some_prompt") == expected_result + + +@pytest.mark.parametrize("response", ["some version", FileNotFoundError()]) +def test_get_pkg_version(monkeypatch: pytest.MonkeyPatch, response: str) -> None: + """Test get_tosa_version.""" + monkeypatch.setattr("importlib.metadata.version", MagicMock(return_value=response)) + assert get_pkg_version("any name") == response -- cgit v1.2.1