MLIA-741/2 Report test results

- add version extraction function in compat.py
- create Metadata, MLIAMetadata, TOSAMetadata and MetadataDisplay classes
- update the reporting functions so tosa and mlia version will be displayed in output json
- update unit test test_configure_and_get_tosa_advisor to mock the get_events function
- update the copyright information of all changed/added files
- handle exception and report to json when program crashes
- write new context managers for capturing stderr and stdout
- support reporting stderr to json output
- support reporting model checksum and model name to json output
- made changes in test_e2e.py handling {model_name} replacement in --output
- add unit tests

Change-Id: I6629fd1c5754378e6accd488217c83d87c7eb6f1

5 files changed, 112 insertions, 18 deletions

diff --git a/tests/test_backend_tosa_compat.py b/tests/test_backend_tosa_compat.py
index 52aaa44..5a80b4b 100644
--- a/tests/test_backend_tosa_compat.py
+++ b/tests/test_backend_tosa_compat.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Tests for TOSA compatibility."""
 from __future__ import annotations
@@ -39,12 +39,13 @@ def test_compatibility_check_should_fail_if_checker_not_available(
 
 
 @pytest.mark.parametrize(
-    "is_tosa_compatible, operators, expected_result",
+    "is_tosa_compatible, operators, exception, expected_result",
     [
         [
             True,
             [],
-            TOSACompatibilityInfo(True, []),
+            None,
+            TOSACompatibilityInfo(True, [], None, None, None),
         ],
         [
             True,
@@ -55,18 +56,16 @@ def test_compatibility_check_should_fail_if_checker_not_available(
                     is_tosa_compatible=True,
                 )
             ],
-            TOSACompatibilityInfo(True, [Operator("op_location", "op_name", True)]),
+            None,
+            TOSACompatibilityInfo(
+                True, [Operator("op_location", "op_name", True)], None, [], []
+            ),
         ],
         [
             False,
-            [
-                SimpleNamespace(
-                    location="op_location",
-                    name="op_name",
-                    is_tosa_compatible=False,
-                )
-            ],
-            TOSACompatibilityInfo(False, [Operator("op_location", "op_name", False)]),
+            [],
+            ValueError("error_test"),
+            TOSACompatibilityInfo(False, [], ValueError("error_test"), [], []),
         ],
     ],
 )
@@ -75,6 +74,7 @@ def test_get_tosa_compatibility_info(
     test_tflite_model: Path,
     is_tosa_compatible: bool,
     operators: Any,
+    exception: Exception | None,
     expected_result: TOSACompatibilityInfo,
 ) -> None:
     """Test getting TOSA compatibility information."""
@@ -83,7 +83,17 @@ def test_get_tosa_compatibility_info(
     mock_checker._get_tosa_compatibility_for_ops.return_value = (  # pylint: disable=protected-access
         operators
     )
-
+    if exception:
+        mock_checker._get_tosa_compatibility_for_ops.side_effect = (  # pylint: disable=protected-access
+            exception
+        )
     replace_get_tosa_checker_with_mock(monkeypatch, mock_checker)
 
-    assert get_tosa_compatibility_info(test_tflite_model) == expected_result
+    returned_compatibility_info = get_tosa_compatibility_info(test_tflite_model)
+    assert repr(returned_compatibility_info.exception) == repr(
+        expected_result.exception
+    )
+    assert (
+        returned_compatibility_info.tosa_compatible == expected_result.tosa_compatible
+    )
+    assert returned_compatibility_info.operators == expected_result.operators
diff --git a/tests/test_target_tosa_advisor.py b/tests/test_target_tosa_advisor.py
index 32a6b77..9646c96 100644
--- a/tests/test_target_tosa_advisor.py
+++ b/tests/test_target_tosa_advisor.py
@@ -1,7 +1,10 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Tests for TOSA advisor."""
 from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
 
 from mlia.core.context import ExecutionContext
 from mlia.core.workflow import DefaultWorkflowExecutor
@@ -9,15 +12,23 @@ from mlia.target.tosa.advisor import configure_and_get_tosa_advisor
 from mlia.target.tosa.advisor import TOSAInferenceAdvisor
 
 
-def test_configure_and_get_tosa_advisor(test_tflite_model: Path) -> None:
+def test_configure_and_get_tosa_advisor(
+    monkeypatch: pytest.MonkeyPatch, test_tflite_model: Path
+) -> None:
     """Test TOSA advisor configuration."""
     ctx = ExecutionContext()
+    get_events_mock = MagicMock()
+    monkeypatch.setattr(
+        "mlia.target.tosa.advisor.TOSAInferenceAdvisor.get_events",
+        MagicMock(return_value=get_events_mock),
+    )
 
     advisor = configure_and_get_tosa_advisor(ctx, "tosa", test_tflite_model)
     workflow = advisor.configure(ctx)
 
     assert isinstance(advisor, TOSAInferenceAdvisor)
 
+    assert advisor.get_events(ctx) == get_events_mock
     assert ctx.event_handlers is not None
     assert ctx.config_parameters == {
         "tosa_inference_advisor": {
diff --git a/tests/test_target_tosa_reporters.py b/tests/test_target_tosa_reporters.py
new file mode 100644
index 0000000..59da270
--- /dev/null
+++ b/tests/test_target_tosa_reporters.py
@@ -0,0 +1,52 @@
+# SPDX-FileCopyrightText: Copyright 2023, Arm Limited and/or its affiliates.
+# SPDX-License-Identifier: Apache-2.0
+"""Tests for tosa-checker reporters."""
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+from mlia.core.metadata import MLIAMetadata
+from mlia.core.metadata import ModelMetadata
+from mlia.core.reporting import Report
+from mlia.target.tosa.config import TOSAConfiguration
+from mlia.target.tosa.metadata import TOSAMetadata
+from mlia.target.tosa.reporters import MetadataDisplay
+from mlia.target.tosa.reporters import report_device
+from mlia.target.tosa.reporters import tosa_formatters
+
+
+def test_tosa_report_device() -> None:
+    """Test function report_device()."""
+    report = report_device(TOSAConfiguration("tosa"))
+    assert report.to_plain_text()
+
+
+def test_tosa_formatters(
+    monkeypatch: pytest.MonkeyPatch, test_tflite_model: Path
+) -> None:
+    """Test function tosa_formatters() with valid input."""
+    mock_version = MagicMock()
+    monkeypatch.setattr(
+        "mlia.core.metadata.get_pkg_version",
+        MagicMock(return_value=mock_version),
+    )
+
+    data = MetadataDisplay(
+        TOSAMetadata("tosa-checker"),
+        MLIAMetadata("mlia"),
+        ModelMetadata(test_tflite_model),
+    )
+    formatter = tosa_formatters(data)
+    report = formatter(data)
+    assert data.tosa_version == mock_version
+    assert isinstance(report, Report)
+
+
+def test_tosa_formatters_invalid_data() -> None:
+    """Test tosa_formatters() with invalid input."""
+    with pytest.raises(
+        Exception,
+        match=r"^Unable to find appropriate formatter for .*",
+    ):
+        tosa_formatters(12)
diff --git a/tests/test_utils_logging.py b/tests/test_utils_logging.py
index ac835c6..c02e8b0 100644
--- a/tests/test_utils_logging.py
+++ b/tests/test_utils_logging.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Tests for the logging utility functions."""
 from __future__ import annotations
@@ -13,6 +13,7 @@ from unittest.mock import MagicMock
 
 import pytest
 
+from mlia.utils.logging import capture_raw_output
 from mlia.utils.logging import create_log_handler
 from mlia.utils.logging import redirect_output
 from mlia.utils.logging import redirect_raw_output
@@ -84,3 +85,15 @@ def test_output_redirection(redirect_context_manager: Callable) -> None:
     print("after redirect")
 
     logger_mock.log.assert_called_once_with(logging.INFO, "output redirected")
+
+
+def test_output_and_error_capture() -> None:
+    """Test output/error capturing."""
+    with capture_raw_output(sys.stdout) as std_output, capture_raw_output(
+        sys.stderr
+    ) as stderr_output:
+        print("hello from stdout")
+        print("hello from stderr", file=sys.stderr)
+
+    assert std_output == ["hello from stdout\n"]
+    assert stderr_output == ["hello from stderr\n"]
diff --git a/tests/test_utils_misc.py b/tests/test_utils_misc.py
index 011d09e..ae91850 100644
--- a/tests/test_utils_misc.py
+++ b/tests/test_utils_misc.py
@@ -1,10 +1,11 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Tests for misc util functions."""
 from unittest.mock import MagicMock
 
 import pytest
 
+from mlia.utils.misc import get_pkg_version
 from mlia.utils.misc import yes
 
 
@@ -23,3 +24,10 @@ def test_yes(
     """Test yes function."""
     monkeypatch.setattr("builtins.input", MagicMock(return_value=response))
     assert yes("some_prompt") == expected_result
+
+
+@pytest.mark.parametrize("response", ["some version", FileNotFoundError()])
+def test_get_pkg_version(monkeypatch: pytest.MonkeyPatch, response: str) -> None:
+    """Test get_tosa_version."""
+    monkeypatch.setattr("importlib.metadata.version", MagicMock(return_value=response))
+    assert get_pkg_version("any name") == response