MLIA-711 Extend TensorFlow Lite Compatibility Check

- Unify the TensorFlow Lite compatibility check across Cortex-A, TOSA
and Ethos-U targets
- Display tables/messages with parsed information
- Do not display raw TensorFlow Lite errors, and return with exit code 0

Change-Id: I9333fdb6cbe592f1ed7395d392412168492a1479

5 files changed, 135 insertions, 7 deletions

diff --git a/tests/test_target_cortex_a_advice_generation.py b/tests/test_target_cortex_a_advice_generation.py
index 9596d47..916bdc1 100644
--- a/tests/test_target_cortex_a_advice_generation.py
+++ b/tests/test_target_cortex_a_advice_generation.py
@@ -13,13 +13,13 @@ from mlia.core.common import AdviceCategory
 from mlia.core.common import DataItem
 from mlia.core.context import ExecutionContext
 from mlia.nn.tensorflow.tflite_graph import TFL_ACTIVATION_FUNCTION
+from mlia.target.common.reporters import ModelHasCustomOperators
+from mlia.target.common.reporters import ModelIsNotTFLiteCompatible
+from mlia.target.common.reporters import TFLiteCompatibilityCheckFailed
 from mlia.target.cortex_a.advice_generation import CortexAAdviceProducer
 from mlia.target.cortex_a.config import CortexAConfiguration
-from mlia.target.cortex_a.data_analysis import ModelHasCustomOperators
 from mlia.target.cortex_a.data_analysis import ModelIsCortexACompatible
 from mlia.target.cortex_a.data_analysis import ModelIsNotCortexACompatible
-from mlia.target.cortex_a.data_analysis import ModelIsNotTFLiteCompatible
-from mlia.target.cortex_a.data_analysis import TFLiteCompatibilityCheckFailed
 
 VERSION = CortexAConfiguration.load_profile("cortex-a").armnn_tflite_delegate_version
 BACKEND_INFO = (
diff --git a/tests/test_target_cortex_a_data_analysis.py b/tests/test_target_cortex_a_data_analysis.py
index 0a6b490..e033ef9 100644
--- a/tests/test_target_cortex_a_data_analysis.py
+++ b/tests/test_target_cortex_a_data_analysis.py
@@ -15,13 +15,13 @@ from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus
 from mlia.nn.tensorflow.tflite_compat import TFLiteConversionError
 from mlia.nn.tensorflow.tflite_compat import TFLiteConversionErrorCode
 from mlia.nn.tensorflow.tflite_graph import TFL_ACTIVATION_FUNCTION
+from mlia.target.common.reporters import ModelHasCustomOperators
+from mlia.target.common.reporters import ModelIsNotTFLiteCompatible
+from mlia.target.common.reporters import TFLiteCompatibilityCheckFailed
 from mlia.target.cortex_a.config import CortexAConfiguration
 from mlia.target.cortex_a.data_analysis import CortexADataAnalyzer
-from mlia.target.cortex_a.data_analysis import ModelHasCustomOperators
 from mlia.target.cortex_a.data_analysis import ModelIsCortexACompatible
 from mlia.target.cortex_a.data_analysis import ModelIsNotCortexACompatible
-from mlia.target.cortex_a.data_analysis import ModelIsNotTFLiteCompatible
-from mlia.target.cortex_a.data_analysis import TFLiteCompatibilityCheckFailed
 from mlia.target.cortex_a.operators import CortexACompatibilityInfo
 from mlia.target.cortex_a.operators import Operator
 
diff --git a/tests/test_target_ethos_u_data_analysis.py b/tests/test_target_ethos_u_data_analysis.py
index 8e63946..80f0603 100644
--- a/tests/test_target_ethos_u_data_analysis.py
+++ b/tests/test_target_ethos_u_data_analysis.py
@@ -13,6 +13,13 @@ from mlia.backend.vela.compat import Operators
 from mlia.core.common import DataItem
 from mlia.core.data_analysis import Fact
 from mlia.nn.tensorflow.optimizations.select import OptimizationSettings
+from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo
+from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus
+from mlia.nn.tensorflow.tflite_compat import TFLiteConversionError
+from mlia.nn.tensorflow.tflite_compat import TFLiteConversionErrorCode
+from mlia.target.common.reporters import ModelHasCustomOperators
+from mlia.target.common.reporters import ModelIsNotTFLiteCompatible
+from mlia.target.common.reporters import TFLiteCompatibilityCheckFailed
 from mlia.target.ethos_u.config import EthosUConfiguration
 from mlia.target.ethos_u.data_analysis import AllOperatorsSupportedOnNPU
 from mlia.target.ethos_u.data_analysis import EthosUDataAnalyzer
@@ -139,6 +146,51 @@ def test_perf_metrics_diff() -> None:
             ),
             [],
         ],
+        [
+            TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.COMPATIBLE),
+            [],
+        ],
+        [
+            TFLiteCompatibilityInfo(
+                status=TFLiteCompatibilityStatus.MODEL_WITH_CUSTOM_OP_ERROR
+            ),
+            [ModelHasCustomOperators()],
+        ],
+        [
+            TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.UNKNOWN_ERROR),
+            [TFLiteCompatibilityCheckFailed()],
+        ],
+        [
+            TFLiteCompatibilityInfo(
+                status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR
+            ),
+            [ModelIsNotTFLiteCompatible(custom_ops=[], flex_ops=[])],
+        ],
+        [
+            TFLiteCompatibilityInfo(
+                status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR,
+                conversion_errors=[
+                    TFLiteConversionError(
+                        "error",
+                        TFLiteConversionErrorCode.NEEDS_CUSTOM_OPS,
+                        "custom_op1",
+                        [],
+                    ),
+                    TFLiteConversionError(
+                        "error",
+                        TFLiteConversionErrorCode.NEEDS_FLEX_OPS,
+                        "flex_op1",
+                        [],
+                    ),
+                ],
+            ),
+            [
+                ModelIsNotTFLiteCompatible(
+                    custom_ops=["custom_op1"],
+                    flex_ops=["flex_op1"],
+                )
+            ],
+        ],
     ],
 )
 def test_ethos_u_data_analyzer(
diff --git a/tests/test_target_ethos_u_reporters.py b/tests/test_target_ethos_u_reporters.py
index b8014e4..debeeb2 100644
--- a/tests/test_target_ethos_u_reporters.py
+++ b/tests/test_target_ethos_u_reporters.py
@@ -3,6 +3,7 @@
 """Tests for reports module."""
 from __future__ import annotations
 
+from typing import Any
 from typing import cast
 
 import pytest
@@ -11,7 +12,10 @@ from mlia.backend.vela.compat import NpuSupported
 from mlia.backend.vela.compat import Operator
 from mlia.core.reporting import Report
 from mlia.core.reporting import Table
+from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo
+from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus
 from mlia.target.ethos_u.config import EthosUConfiguration
+from mlia.target.ethos_u.reporters import ethos_u_formatters
 from mlia.target.ethos_u.reporters import report_operators
 from mlia.target.ethos_u.reporters import report_target_details
 from mlia.target.registry import profile
@@ -231,3 +235,23 @@ def test_report_target_details(
 
     json_dict = report.to_json()
     assert json_dict == expected_json_dict
+
+
+@pytest.mark.parametrize(
+    "data",
+    (TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.COMPATIBLE),),
+)
+def test_ethos_u_formatters(data: Any) -> None:
+    """Test function ethos_u_formatters() with valid input."""
+    formatter = ethos_u_formatters(data)
+    report = formatter(data)
+    assert isinstance(report, Report)
+
+
+def test_ethos_u_formatters_invalid_data() -> None:
+    """Test function ethos_u_formatters() with invalid input."""
+    with pytest.raises(
+        Exception,
+        match=r"^Unable to find appropriate formatter for .*",
+    ):
+        ethos_u_formatters(200)
diff --git a/tests/test_target_tosa_data_analysis.py b/tests/test_target_tosa_data_analysis.py
index 41e977f..23adcc8 100644
--- a/tests/test_target_tosa_data_analysis.py
+++ b/tests/test_target_tosa_data_analysis.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Tests for TOSA data analysis module."""
 from __future__ import annotations
@@ -8,6 +8,13 @@ import pytest
 from mlia.backend.tosa_checker.compat import TOSACompatibilityInfo
 from mlia.core.common import DataItem
 from mlia.core.data_analysis import Fact
+from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo
+from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus
+from mlia.nn.tensorflow.tflite_compat import TFLiteConversionError
+from mlia.nn.tensorflow.tflite_compat import TFLiteConversionErrorCode
+from mlia.target.common.reporters import ModelHasCustomOperators
+from mlia.target.common.reporters import ModelIsNotTFLiteCompatible
+from mlia.target.common.reporters import TFLiteCompatibilityCheckFailed
 from mlia.target.tosa.data_analysis import ModelIsNotTOSACompatible
 from mlia.target.tosa.data_analysis import ModelIsTOSACompatible
 from mlia.target.tosa.data_analysis import TOSADataAnalyzer
@@ -24,6 +31,51 @@ from mlia.target.tosa.data_analysis import TOSADataAnalyzer
             TOSACompatibilityInfo(False, []),
             [ModelIsNotTOSACompatible()],
         ],
+        [
+            TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.COMPATIBLE),
+            [],
+        ],
+        [
+            TFLiteCompatibilityInfo(
+                status=TFLiteCompatibilityStatus.MODEL_WITH_CUSTOM_OP_ERROR
+            ),
+            [ModelHasCustomOperators()],
+        ],
+        [
+            TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.UNKNOWN_ERROR),
+            [TFLiteCompatibilityCheckFailed()],
+        ],
+        [
+            TFLiteCompatibilityInfo(
+                status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR
+            ),
+            [ModelIsNotTFLiteCompatible(custom_ops=[], flex_ops=[])],
+        ],
+        [
+            TFLiteCompatibilityInfo(
+                status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR,
+                conversion_errors=[
+                    TFLiteConversionError(
+                        "error",
+                        TFLiteConversionErrorCode.NEEDS_CUSTOM_OPS,
+                        "custom_op1",
+                        [],
+                    ),
+                    TFLiteConversionError(
+                        "error",
+                        TFLiteConversionErrorCode.NEEDS_FLEX_OPS,
+                        "flex_op1",
+                        [],
+                    ),
+                ],
+            ),
+            [
+                ModelIsNotTFLiteCompatible(
+                    custom_ops=["custom_op1"],
+                    flex_ops=["flex_op1"],
+                )
+            ],
+        ],
     ],
 )
 def test_tosa_data_analyzer(input_data: DataItem, expected_facts: list[Fact]) -> None: