From ef73bb773df214f3f33f8e4ca7d276041106cad2 Mon Sep 17 00:00:00 2001 From: Dmitrii Agibov Date: Wed, 9 Nov 2022 11:23:50 +0000 Subject: MLIA-685 Warn about custom operators in SavedModel/Keras models - Add new error types for the TensorFlow Lite compatibility check - Try to detect custom operators in SavedModel/Keras models - Add warning to the advice about models with custom operators Change-Id: I2f65474eecf2788110acc43585fa300eda80e21b --- src/mlia/devices/cortexa/advice_generation.py | 38 ++++++++++- src/mlia/devices/cortexa/data_analysis.py | 32 ++++++---- src/mlia/nn/tensorflow/tflite_compat.py | 92 +++++++++++++++++++++++++-- 3 files changed, 143 insertions(+), 19 deletions(-) (limited to 'src') diff --git a/src/mlia/devices/cortexa/advice_generation.py b/src/mlia/devices/cortexa/advice_generation.py index 186f489..3d2f106 100644 --- a/src/mlia/devices/cortexa/advice_generation.py +++ b/src/mlia/devices/cortexa/advice_generation.py @@ -7,9 +7,11 @@ from mlia.core.advice_generation import advice_category from mlia.core.advice_generation import FactBasedAdviceProducer from mlia.core.common import AdviceCategory from mlia.core.common import DataItem +from mlia.devices.cortexa.data_analysis import ModelHasCustomOperators from mlia.devices.cortexa.data_analysis import ModelIsCortexACompatible from mlia.devices.cortexa.data_analysis import ModelIsNotCortexACompatible from mlia.devices.cortexa.data_analysis import ModelIsNotTFLiteCompatible +from mlia.devices.cortexa.data_analysis import TFLiteCompatibilityCheckFailed class CortexAAdviceProducer(FactBasedAdviceProducer): @@ -92,17 +94,25 @@ class CortexAAdviceProducer(FactBasedAdviceProducer): "The following operators are not natively " "supported by TensorFlow Lite: " f"{', '.join(data_item.flex_ops)}.", + "Using select TensorFlow operators in TensorFlow Lite model " + "requires special initialization of TFLiteConverter and " + "TensorFlow Lite run-time.", "Please refer to the TensorFlow documentation for more details.", + "Note, such models are not supported by the ML Inference Advisor.", ] ) if data_item.custom_ops: self.add_advice( [ - "The following operators are custom and not natively " + "The following operators appears to be custom and not natively " "supported by TensorFlow Lite: " f"{', '.join(data_item.custom_ops)}.", + "Using custom operators in TensorFlow Lite model " + "requires special initialization of TFLiteConverter and " + "TensorFlow Lite run-time.", "Please refer to the TensorFlow documentation for more details.", + "Note, such models are not supported by the ML Inference Advisor.", ] ) @@ -113,3 +123,29 @@ class CortexAAdviceProducer(FactBasedAdviceProducer): "Please refer to the table for more details.", ] ) + + @produce_advice.register + @advice_category(AdviceCategory.ALL, AdviceCategory.OPERATORS) + def handle_tflite_check_failed( + self, _data_item: TFLiteCompatibilityCheckFailed + ) -> None: + """Advice for the failed TensorFlow Lite compatibility checks.""" + self.add_advice( + [ + "Model could not be converted into TensorFlow Lite format.", + "Please refer to the table for more details.", + ] + ) + + @produce_advice.register + @advice_category(AdviceCategory.ALL, AdviceCategory.OPERATORS) + def handle_model_has_custom_operators( + self, _data_item: ModelHasCustomOperators + ) -> None: + """Advice for the models with custom operators.""" + self.add_advice( + [ + "Models with custom operators require special initialization " + "and currently are not supported by the ML Inference Advisor.", + ] + ) diff --git a/src/mlia/devices/cortexa/data_analysis.py b/src/mlia/devices/cortexa/data_analysis.py index 6a82dd0..04bc819 100644 --- a/src/mlia/devices/cortexa/data_analysis.py +++ b/src/mlia/devices/cortexa/data_analysis.py @@ -14,7 +14,6 @@ from mlia.core.data_analysis import FactExtractor from mlia.devices.cortexa.operators import CortexACompatibilityInfo from mlia.devices.cortexa.operators import Operator from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo -from mlia.nn.tensorflow.tflite_compat import TFLiteConversionErrorCode class CortexADataAnalyzer(FactExtractor): @@ -69,18 +68,19 @@ class CortexADataAnalyzer(FactExtractor): if data_item.compatible: return - custom_ops, flex_ops = [], [] - if data_item.conversion_errors: - custom_ops = data_item.unsupported_ops_by_code( - TFLiteConversionErrorCode.NEEDS_CUSTOM_OPS - ) - flex_ops = data_item.unsupported_ops_by_code( - TFLiteConversionErrorCode.NEEDS_FLEX_OPS + if data_item.conversion_failed_with_errors: + self.add_fact( + ModelIsNotTFLiteCompatible( + custom_ops=data_item.required_custom_ops, + flex_ops=data_item.required_flex_ops, + ) ) - self.add_fact( - ModelIsNotTFLiteCompatible(custom_ops=custom_ops, flex_ops=flex_ops) - ) + if data_item.check_failed_with_unknown_error: + self.add_fact(TFLiteCompatibilityCheckFailed()) + + if data_item.conversion_failed_for_model_with_custom_ops: + self.add_fact(ModelHasCustomOperators()) @dataclass @@ -116,3 +116,13 @@ class ModelIsNotTFLiteCompatible(Fact): custom_ops: list[str] | None = None flex_ops: list[str] | None = None + + +@dataclass +class TFLiteCompatibilityCheckFailed(Fact): + """TensorFlow Lite compatibility check failed by unknown reason.""" + + +@dataclass +class ModelHasCustomOperators(Fact): + """Model could not be loaded because it contains custom ops.""" diff --git a/src/mlia/nn/tensorflow/tflite_compat.py b/src/mlia/nn/tensorflow/tflite_compat.py index 6f183ca..2b29879 100644 --- a/src/mlia/nn/tensorflow/tflite_compat.py +++ b/src/mlia/nn/tensorflow/tflite_compat.py @@ -49,11 +49,20 @@ class TFLiteConversionError: location: list[str] +class TFLiteCompatibilityStatus(Enum): + """TensorFlow lite compatiblity status.""" + + COMPATIBLE = auto() + TFLITE_CONVERSION_ERROR = auto() + MODEL_WITH_CUSTOM_OP_ERROR = auto() + UNKNOWN_ERROR = auto() + + @dataclass class TFLiteCompatibilityInfo: """TensorFlow Lite compatibility information.""" - compatible: bool + status: TFLiteCompatibilityStatus conversion_exception: Exception | None = None conversion_errors: list[TFLiteConversionError] | None = None @@ -64,6 +73,36 @@ class TFLiteCompatibilityInfo: return [err.operator for err in self.conversion_errors if err.code == code] + @property + def compatible(self) -> bool: + """Return true if model compatible with the TensorFlow Lite format.""" + return self.status == TFLiteCompatibilityStatus.COMPATIBLE + + @property + def conversion_failed_with_errors(self) -> bool: + """Return true if conversion to TensorFlow Lite format failed.""" + return self.status == TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR + + @property + def conversion_failed_for_model_with_custom_ops(self) -> bool: + """Return true if conversion failed due to custom ops in the model.""" + return self.status == TFLiteCompatibilityStatus.MODEL_WITH_CUSTOM_OP_ERROR + + @property + def check_failed_with_unknown_error(self) -> bool: + """Return true if check failed with unknown error.""" + return self.status == TFLiteCompatibilityStatus.UNKNOWN_ERROR + + @property + def required_custom_ops(self) -> list[str]: + """Return list of the custom ops reported during conversion.""" + return self.unsupported_ops_by_code(TFLiteConversionErrorCode.NEEDS_CUSTOM_OPS) + + @property + def required_flex_ops(self) -> list[str]: + """Return list of the flex ops reported during conversion.""" + return self.unsupported_ops_by_code(TFLiteConversionErrorCode.NEEDS_FLEX_OPS) + class TFLiteChecker: """Class for checking TensorFlow Lite compatibility.""" @@ -86,13 +125,15 @@ class TFLiteChecker: ): converter.convert() except convert.ConverterError as err: - return self._process_exception(err) + return self._process_convert_error(err) except Exception as err: # pylint: disable=broad-except - return TFLiteCompatibilityInfo(compatible=False, conversion_exception=err) - else: - return TFLiteCompatibilityInfo(compatible=True) + return self._process_exception(err) - def _process_exception( + return TFLiteCompatibilityInfo( + status=TFLiteCompatibilityStatus.COMPATIBLE, + ) + + def _process_convert_error( self, err: convert.ConverterError ) -> TFLiteCompatibilityInfo: """Parse error details if possible.""" @@ -114,11 +155,48 @@ class TFLiteChecker: ] return TFLiteCompatibilityInfo( - compatible=False, + status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR, conversion_exception=err, conversion_errors=conversion_errors, ) + def _process_exception(self, err: Exception) -> TFLiteCompatibilityInfo: + """Process exception during conversion.""" + status = TFLiteCompatibilityStatus.UNKNOWN_ERROR + + if self._model_with_custom_op(err): + status = TFLiteCompatibilityStatus.MODEL_WITH_CUSTOM_OP_ERROR + + return TFLiteCompatibilityInfo( + status=status, + conversion_exception=err, + ) + + @staticmethod + def _model_with_custom_op(err: Exception) -> bool: + """Check if model could not be loaded because of custom ops.""" + exc_attrs = [ + ( + ValueError, + [ + "Unable to restore custom object", + "passed to the `custom_objects`", + ], + ), + ( + FileNotFoundError, + [ + "Op type not registered", + ], + ), + ] + + return any( + any(msg in str(err) for msg in messages) + for exc_type, messages in exc_attrs + if isinstance(err, exc_type) + ) + @staticmethod def _convert_error_code(code: int) -> TFLiteConversionErrorCode: """Convert internal error codes.""" -- cgit v1.2.1