MLIA-685 Warn about custom operators in SavedModel/Keras models

- Add new error types for the TensorFlow Lite compatibility
  check
- Try to detect custom operators in SavedModel/Keras models
- Add warning to the advice about models with custom operators

Change-Id: I2f65474eecf2788110acc43585fa300eda80e21b

4 files changed, 82 insertions, 12 deletions

diff --git a/tests/test_devices_cortexa_advice_generation.py b/tests/test_devices_cortexa_advice_generation.py
index 0446f38..4e6d892 100644
--- a/tests/test_devices_cortexa_advice_generation.py
+++ b/tests/test_devices_cortexa_advice_generation.py
@@ -10,9 +10,11 @@ from mlia.core.common import AdviceCategory
 from mlia.core.common import DataItem
 from mlia.core.context import ExecutionContext
 from mlia.devices.cortexa.advice_generation import CortexAAdviceProducer
+from mlia.devices.cortexa.data_analysis import ModelHasCustomOperators
 from mlia.devices.cortexa.data_analysis import ModelIsCortexACompatible
 from mlia.devices.cortexa.data_analysis import ModelIsNotCortexACompatible
 from mlia.devices.cortexa.data_analysis import ModelIsNotTFLiteCompatible
+from mlia.devices.cortexa.data_analysis import TFLiteCompatibilityCheckFailed
 from mlia.devices.cortexa.operator_compatibility import ARMNN_TFLITE_DELEGATE
 from mlia.nn.tensorflow.tflite_graph import TFL_ACTIVATION_FUNCTION
 
@@ -110,16 +112,26 @@ BACKEND_INFO = (
                     [
                         "The following operators are not natively "
                         "supported by TensorFlow Lite: flex_op1, flex_op2.",
+                        "Using select TensorFlow operators in TensorFlow Lite model "
+                        "requires special initialization of TFLiteConverter and "
+                        "TensorFlow Lite run-time.",
                         "Please refer to the TensorFlow documentation for "
                         "more details.",
+                        "Note, such models are not supported by "
+                        "the ML Inference Advisor.",
                     ]
                 ),
                 Advice(
                     [
-                        "The following operators are custom and not natively "
+                        "The following operators appears to be custom and not natively "
                         "supported by TensorFlow Lite: custom_op1, custom_op2.",
+                        "Using custom operators in TensorFlow Lite model "
+                        "requires special initialization of TFLiteConverter and "
+                        "TensorFlow Lite run-time.",
                         "Please refer to the TensorFlow documentation for "
                         "more details.",
+                        "Note, such models are not supported by "
+                        "the ML Inference Advisor.",
                     ]
                 ),
             ],
@@ -136,6 +148,30 @@ BACKEND_INFO = (
                 ),
             ],
         ],
+        [
+            ModelHasCustomOperators(),
+            AdviceCategory.OPERATORS,
+            [
+                Advice(
+                    [
+                        "Models with custom operators require special initialization "
+                        "and currently are not supported by the ML Inference Advisor.",
+                    ]
+                ),
+            ],
+        ],
+        [
+            TFLiteCompatibilityCheckFailed(),
+            AdviceCategory.OPERATORS,
+            [
+                Advice(
+                    [
+                        "Model could not be converted into TensorFlow Lite format.",
+                        "Please refer to the table for more details.",
+                    ]
+                ),
+            ],
+        ],
     ],
 )
 def test_cortex_a_advice_producer(
diff --git a/tests/test_devices_cortexa_data_analysis.py b/tests/test_devices_cortexa_data_analysis.py
index 4d98212..ed30b9a 100644
--- a/tests/test_devices_cortexa_data_analysis.py
+++ b/tests/test_devices_cortexa_data_analysis.py
@@ -8,13 +8,16 @@ import pytest
 from mlia.core.common import DataItem
 from mlia.core.data_analysis import Fact
 from mlia.devices.cortexa.data_analysis import CortexADataAnalyzer
+from mlia.devices.cortexa.data_analysis import ModelHasCustomOperators
 from mlia.devices.cortexa.data_analysis import ModelIsCortexACompatible
 from mlia.devices.cortexa.data_analysis import ModelIsNotCortexACompatible
 from mlia.devices.cortexa.data_analysis import ModelIsNotTFLiteCompatible
+from mlia.devices.cortexa.data_analysis import TFLiteCompatibilityCheckFailed
 from mlia.devices.cortexa.operator_compatibility import ARMNN_TFLITE_DELEGATE
 from mlia.devices.cortexa.operators import CortexACompatibilityInfo
 from mlia.devices.cortexa.operators import Operator
 from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo
+from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus
 from mlia.nn.tensorflow.tflite_compat import TFLiteConversionError
 from mlia.nn.tensorflow.tflite_compat import TFLiteConversionErrorCode
 from mlia.nn.tensorflow.tflite_graph import TFL_ACTIVATION_FUNCTION
@@ -104,16 +107,28 @@ BACKEND_INFO = (
             # pylint: enable=line-too-long
         ],
         [
-            TFLiteCompatibilityInfo(compatible=True),
+            TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.COMPATIBLE),
             [],
         ],
         [
-            TFLiteCompatibilityInfo(compatible=False),
+            TFLiteCompatibilityInfo(
+                status=TFLiteCompatibilityStatus.MODEL_WITH_CUSTOM_OP_ERROR
+            ),
+            [ModelHasCustomOperators()],
+        ],
+        [
+            TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.UNKNOWN_ERROR),
+            [TFLiteCompatibilityCheckFailed()],
+        ],
+        [
+            TFLiteCompatibilityInfo(
+                status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR
+            ),
             [ModelIsNotTFLiteCompatible(custom_ops=[], flex_ops=[])],
         ],
         [
             TFLiteCompatibilityInfo(
-                compatible=False,
+                status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR,
                 conversion_errors=[
                     TFLiteConversionError(
                         "error",
diff --git a/tests/test_devices_cortexa_reporters.py b/tests/test_devices_cortexa_reporters.py
index 4177b55..1110653 100644
--- a/tests/test_devices_cortexa_reporters.py
+++ b/tests/test_devices_cortexa_reporters.py
@@ -12,6 +12,7 @@ from mlia.devices.cortexa.operators import Operator
 from mlia.devices.cortexa.reporters import cortex_a_formatters
 from mlia.devices.cortexa.reporters import report_device
 from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo
+from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus
 from mlia.nn.tensorflow.tflite_graph import TFL_ACTIVATION_FUNCTION
 
 
@@ -25,7 +26,7 @@ def test_report_device() -> None:
     "data",
     (
         [Advice(["Sample", "Advice"])],
-        TFLiteCompatibilityInfo(compatible=True),
+        TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.COMPATIBLE),
         [
             Operator(
                 name="Test",
diff --git a/tests/test_nn_tensorflow_tflite_compat.py b/tests/test_nn_tensorflow_tflite_compat.py
index 1bd4c34..f203125 100644
--- a/tests/test_nn_tensorflow_tflite_compat.py
+++ b/tests/test_nn_tensorflow_tflite_compat.py
@@ -12,6 +12,7 @@ from tensorflow.lite.python import convert
 from mlia.nn.tensorflow.tflite_compat import converter_error_data_pb2
 from mlia.nn.tensorflow.tflite_compat import TFLiteChecker
 from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo
+from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus
 from mlia.nn.tensorflow.tflite_compat import TFLiteConversionError
 from mlia.nn.tensorflow.tflite_compat import TFLiteConversionErrorCode
 
@@ -87,11 +88,14 @@ def _get_tflite_conversion_error(
 @pytest.mark.parametrize(
     "conversion_error, expected_result",
     [
-        (None, TFLiteCompatibilityInfo(compatible=True)),
+        (
+            None,
+            TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.COMPATIBLE),
+        ),
         (
             err := _get_tflite_conversion_error(custom_op=True),
             TFLiteCompatibilityInfo(
-                compatible=False,
+                status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR,
                 conversion_exception=err,
                 conversion_errors=[
                     TFLiteConversionError(
@@ -106,7 +110,7 @@ def _get_tflite_conversion_error(
         (
             err := _get_tflite_conversion_error(flex_op=True),
             TFLiteCompatibilityInfo(
-                compatible=False,
+                status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR,
                 conversion_exception=err,
                 conversion_errors=[
                     TFLiteConversionError(
@@ -121,7 +125,7 @@ def _get_tflite_conversion_error(
         (
             err := _get_tflite_conversion_error(unknown_reason=True),
             TFLiteCompatibilityInfo(
-                compatible=False,
+                status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR,
                 conversion_exception=err,
                 conversion_errors=[
                     TFLiteConversionError(
@@ -141,7 +145,7 @@ def _get_tflite_conversion_error(
                 unsupported_flow_v1=True,
             ),
             TFLiteCompatibilityInfo(
-                compatible=False,
+                status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR,
                 conversion_exception=err,
                 conversion_errors=[
                     TFLiteConversionError(
@@ -174,7 +178,7 @@ def _get_tflite_conversion_error(
         (
             err := _get_tflite_conversion_error(),
             TFLiteCompatibilityInfo(
-                compatible=False,
+                status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR,
                 conversion_exception=err,
                 conversion_errors=[],
             ),
@@ -182,7 +186,21 @@ def _get_tflite_conversion_error(
         (
             err := ValueError("Some unknown issue"),
             TFLiteCompatibilityInfo(
-                compatible=False,
+                status=TFLiteCompatibilityStatus.UNKNOWN_ERROR,
+                conversion_exception=err,
+            ),
+        ),
+        (
+            err := ValueError("Unable to restore custom object"),
+            TFLiteCompatibilityInfo(
+                status=TFLiteCompatibilityStatus.MODEL_WITH_CUSTOM_OP_ERROR,
+                conversion_exception=err,
+            ),
+        ),
+        (
+            err := FileNotFoundError("Op type not registered"),
+            TFLiteCompatibilityInfo(
+                status=TFLiteCompatibilityStatus.MODEL_WITH_CUSTOM_OP_ERROR,
                 conversion_exception=err,
             ),
         ),