From edf436c48029aa4e2b4ca5d17eee5a8f07ecbd6f Mon Sep 17 00:00:00 2001 From: Dhruv Chauhan Date: Mon, 20 Mar 2023 10:22:08 +0000 Subject: MLIA-711 Extend TensorFlow Lite Compatibility Check - Unify the TensorFlow Lite compatibility check across Cortex-A, TOSA and Ethos-U targets - Display tables/messages with parsed information - Do not display raw TensorFlow Lite errors, and return with exit code 0 Change-Id: I9333fdb6cbe592f1ed7395d392412168492a1479 --- tests/test_target_cortex_a_advice_generation.py | 6 +-- tests/test_target_cortex_a_data_analysis.py | 6 +-- tests/test_target_ethos_u_data_analysis.py | 52 ++++++++++++++++++++++++ tests/test_target_ethos_u_reporters.py | 24 +++++++++++ tests/test_target_tosa_data_analysis.py | 54 ++++++++++++++++++++++++- 5 files changed, 135 insertions(+), 7 deletions(-) (limited to 'tests') diff --git a/tests/test_target_cortex_a_advice_generation.py b/tests/test_target_cortex_a_advice_generation.py index 9596d47..916bdc1 100644 --- a/tests/test_target_cortex_a_advice_generation.py +++ b/tests/test_target_cortex_a_advice_generation.py @@ -13,13 +13,13 @@ from mlia.core.common import AdviceCategory from mlia.core.common import DataItem from mlia.core.context import ExecutionContext from mlia.nn.tensorflow.tflite_graph import TFL_ACTIVATION_FUNCTION +from mlia.target.common.reporters import ModelHasCustomOperators +from mlia.target.common.reporters import ModelIsNotTFLiteCompatible +from mlia.target.common.reporters import TFLiteCompatibilityCheckFailed from mlia.target.cortex_a.advice_generation import CortexAAdviceProducer from mlia.target.cortex_a.config import CortexAConfiguration -from mlia.target.cortex_a.data_analysis import ModelHasCustomOperators from mlia.target.cortex_a.data_analysis import ModelIsCortexACompatible from mlia.target.cortex_a.data_analysis import ModelIsNotCortexACompatible -from mlia.target.cortex_a.data_analysis import ModelIsNotTFLiteCompatible -from mlia.target.cortex_a.data_analysis import TFLiteCompatibilityCheckFailed VERSION = CortexAConfiguration.load_profile("cortex-a").armnn_tflite_delegate_version BACKEND_INFO = ( diff --git a/tests/test_target_cortex_a_data_analysis.py b/tests/test_target_cortex_a_data_analysis.py index 0a6b490..e033ef9 100644 --- a/tests/test_target_cortex_a_data_analysis.py +++ b/tests/test_target_cortex_a_data_analysis.py @@ -15,13 +15,13 @@ from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus from mlia.nn.tensorflow.tflite_compat import TFLiteConversionError from mlia.nn.tensorflow.tflite_compat import TFLiteConversionErrorCode from mlia.nn.tensorflow.tflite_graph import TFL_ACTIVATION_FUNCTION +from mlia.target.common.reporters import ModelHasCustomOperators +from mlia.target.common.reporters import ModelIsNotTFLiteCompatible +from mlia.target.common.reporters import TFLiteCompatibilityCheckFailed from mlia.target.cortex_a.config import CortexAConfiguration from mlia.target.cortex_a.data_analysis import CortexADataAnalyzer -from mlia.target.cortex_a.data_analysis import ModelHasCustomOperators from mlia.target.cortex_a.data_analysis import ModelIsCortexACompatible from mlia.target.cortex_a.data_analysis import ModelIsNotCortexACompatible -from mlia.target.cortex_a.data_analysis import ModelIsNotTFLiteCompatible -from mlia.target.cortex_a.data_analysis import TFLiteCompatibilityCheckFailed from mlia.target.cortex_a.operators import CortexACompatibilityInfo from mlia.target.cortex_a.operators import Operator diff --git a/tests/test_target_ethos_u_data_analysis.py b/tests/test_target_ethos_u_data_analysis.py index 8e63946..80f0603 100644 --- a/tests/test_target_ethos_u_data_analysis.py +++ b/tests/test_target_ethos_u_data_analysis.py @@ -13,6 +13,13 @@ from mlia.backend.vela.compat import Operators from mlia.core.common import DataItem from mlia.core.data_analysis import Fact from mlia.nn.tensorflow.optimizations.select import OptimizationSettings +from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo +from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus +from mlia.nn.tensorflow.tflite_compat import TFLiteConversionError +from mlia.nn.tensorflow.tflite_compat import TFLiteConversionErrorCode +from mlia.target.common.reporters import ModelHasCustomOperators +from mlia.target.common.reporters import ModelIsNotTFLiteCompatible +from mlia.target.common.reporters import TFLiteCompatibilityCheckFailed from mlia.target.ethos_u.config import EthosUConfiguration from mlia.target.ethos_u.data_analysis import AllOperatorsSupportedOnNPU from mlia.target.ethos_u.data_analysis import EthosUDataAnalyzer @@ -139,6 +146,51 @@ def test_perf_metrics_diff() -> None: ), [], ], + [ + TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.COMPATIBLE), + [], + ], + [ + TFLiteCompatibilityInfo( + status=TFLiteCompatibilityStatus.MODEL_WITH_CUSTOM_OP_ERROR + ), + [ModelHasCustomOperators()], + ], + [ + TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.UNKNOWN_ERROR), + [TFLiteCompatibilityCheckFailed()], + ], + [ + TFLiteCompatibilityInfo( + status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR + ), + [ModelIsNotTFLiteCompatible(custom_ops=[], flex_ops=[])], + ], + [ + TFLiteCompatibilityInfo( + status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR, + conversion_errors=[ + TFLiteConversionError( + "error", + TFLiteConversionErrorCode.NEEDS_CUSTOM_OPS, + "custom_op1", + [], + ), + TFLiteConversionError( + "error", + TFLiteConversionErrorCode.NEEDS_FLEX_OPS, + "flex_op1", + [], + ), + ], + ), + [ + ModelIsNotTFLiteCompatible( + custom_ops=["custom_op1"], + flex_ops=["flex_op1"], + ) + ], + ], ], ) def test_ethos_u_data_analyzer( diff --git a/tests/test_target_ethos_u_reporters.py b/tests/test_target_ethos_u_reporters.py index b8014e4..debeeb2 100644 --- a/tests/test_target_ethos_u_reporters.py +++ b/tests/test_target_ethos_u_reporters.py @@ -3,6 +3,7 @@ """Tests for reports module.""" from __future__ import annotations +from typing import Any from typing import cast import pytest @@ -11,7 +12,10 @@ from mlia.backend.vela.compat import NpuSupported from mlia.backend.vela.compat import Operator from mlia.core.reporting import Report from mlia.core.reporting import Table +from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo +from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus from mlia.target.ethos_u.config import EthosUConfiguration +from mlia.target.ethos_u.reporters import ethos_u_formatters from mlia.target.ethos_u.reporters import report_operators from mlia.target.ethos_u.reporters import report_target_details from mlia.target.registry import profile @@ -231,3 +235,23 @@ def test_report_target_details( json_dict = report.to_json() assert json_dict == expected_json_dict + + +@pytest.mark.parametrize( + "data", + (TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.COMPATIBLE),), +) +def test_ethos_u_formatters(data: Any) -> None: + """Test function ethos_u_formatters() with valid input.""" + formatter = ethos_u_formatters(data) + report = formatter(data) + assert isinstance(report, Report) + + +def test_ethos_u_formatters_invalid_data() -> None: + """Test function ethos_u_formatters() with invalid input.""" + with pytest.raises( + Exception, + match=r"^Unable to find appropriate formatter for .*", + ): + ethos_u_formatters(200) diff --git a/tests/test_target_tosa_data_analysis.py b/tests/test_target_tosa_data_analysis.py index 41e977f..23adcc8 100644 --- a/tests/test_target_tosa_data_analysis.py +++ b/tests/test_target_tosa_data_analysis.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for TOSA data analysis module.""" from __future__ import annotations @@ -8,6 +8,13 @@ import pytest from mlia.backend.tosa_checker.compat import TOSACompatibilityInfo from mlia.core.common import DataItem from mlia.core.data_analysis import Fact +from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo +from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus +from mlia.nn.tensorflow.tflite_compat import TFLiteConversionError +from mlia.nn.tensorflow.tflite_compat import TFLiteConversionErrorCode +from mlia.target.common.reporters import ModelHasCustomOperators +from mlia.target.common.reporters import ModelIsNotTFLiteCompatible +from mlia.target.common.reporters import TFLiteCompatibilityCheckFailed from mlia.target.tosa.data_analysis import ModelIsNotTOSACompatible from mlia.target.tosa.data_analysis import ModelIsTOSACompatible from mlia.target.tosa.data_analysis import TOSADataAnalyzer @@ -24,6 +31,51 @@ from mlia.target.tosa.data_analysis import TOSADataAnalyzer TOSACompatibilityInfo(False, []), [ModelIsNotTOSACompatible()], ], + [ + TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.COMPATIBLE), + [], + ], + [ + TFLiteCompatibilityInfo( + status=TFLiteCompatibilityStatus.MODEL_WITH_CUSTOM_OP_ERROR + ), + [ModelHasCustomOperators()], + ], + [ + TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.UNKNOWN_ERROR), + [TFLiteCompatibilityCheckFailed()], + ], + [ + TFLiteCompatibilityInfo( + status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR + ), + [ModelIsNotTFLiteCompatible(custom_ops=[], flex_ops=[])], + ], + [ + TFLiteCompatibilityInfo( + status=TFLiteCompatibilityStatus.TFLITE_CONVERSION_ERROR, + conversion_errors=[ + TFLiteConversionError( + "error", + TFLiteConversionErrorCode.NEEDS_CUSTOM_OPS, + "custom_op1", + [], + ), + TFLiteConversionError( + "error", + TFLiteConversionErrorCode.NEEDS_FLEX_OPS, + "flex_op1", + [], + ), + ], + ), + [ + ModelIsNotTFLiteCompatible( + custom_ops=["custom_op1"], + flex_ops=["flex_op1"], + ) + ], + ], ], ) def test_tosa_data_analyzer(input_data: DataItem, expected_facts: list[Fact]) -> None: -- cgit v1.2.1