From 49dfd03617ed28a4fcbd847dd958c27442c84556 Mon Sep 17 00:00:00 2001 From: Annie Tallund Date: Mon, 6 Feb 2023 14:56:31 +0100 Subject: MLIA-709 Update compatibility data for Cortex-A - Add operator compatibility data for Cortex-A via for ArmNN TensorFlow Lite delegate 22.11 - Extend the Cortex-A target profile to include the version of the ArmNN TensorFlow Lite delegate to be used. - Some re-factoring work to support multiple versions and the new target profile parameter. Change-Id: Iae91bb0757ea3909be975af68b34d0ca2be47c43 --- tests/test_target_cortex_a_advice_generation.py | 5 +++-- tests/test_target_cortex_a_data_analysis.py | 23 +++++++++-------------- tests/test_target_cortex_a_data_collection.py | 8 ++++++-- tests/test_target_cortex_a_operators.py | 24 +++++++++++++++--------- tests/test_target_cortex_a_reporters.py | 19 +++++++++++-------- 5 files changed, 44 insertions(+), 35 deletions(-) (limited to 'tests') diff --git a/tests/test_target_cortex_a_advice_generation.py b/tests/test_target_cortex_a_advice_generation.py index b9edbb5..9596d47 100644 --- a/tests/test_target_cortex_a_advice_generation.py +++ b/tests/test_target_cortex_a_advice_generation.py @@ -14,15 +14,16 @@ from mlia.core.common import DataItem from mlia.core.context import ExecutionContext from mlia.nn.tensorflow.tflite_graph import TFL_ACTIVATION_FUNCTION from mlia.target.cortex_a.advice_generation import CortexAAdviceProducer +from mlia.target.cortex_a.config import CortexAConfiguration from mlia.target.cortex_a.data_analysis import ModelHasCustomOperators from mlia.target.cortex_a.data_analysis import ModelIsCortexACompatible from mlia.target.cortex_a.data_analysis import ModelIsNotCortexACompatible from mlia.target.cortex_a.data_analysis import ModelIsNotTFLiteCompatible from mlia.target.cortex_a.data_analysis import TFLiteCompatibilityCheckFailed +VERSION = CortexAConfiguration.load_profile("cortex-a").armnn_tflite_delegate_version BACKEND_INFO = ( - f"{ARMNN_TFLITE_DELEGATE['metadata']['backend']} " - f"{ARMNN_TFLITE_DELEGATE['metadata']['version']}" + f"{ARMNN_TFLITE_DELEGATE['backend']} " f"{ARMNN_TFLITE_DELEGATE['ops'][VERSION]}" ) diff --git a/tests/test_target_cortex_a_data_analysis.py b/tests/test_target_cortex_a_data_analysis.py index e9fc8bc..0a6b490 100644 --- a/tests/test_target_cortex_a_data_analysis.py +++ b/tests/test_target_cortex_a_data_analysis.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for Cortex-A data analysis module.""" from __future__ import annotations @@ -15,6 +15,7 @@ from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus from mlia.nn.tensorflow.tflite_compat import TFLiteConversionError from mlia.nn.tensorflow.tflite_compat import TFLiteConversionErrorCode from mlia.nn.tensorflow.tflite_graph import TFL_ACTIVATION_FUNCTION +from mlia.target.cortex_a.config import CortexAConfiguration from mlia.target.cortex_a.data_analysis import CortexADataAnalyzer from mlia.target.cortex_a.data_analysis import ModelHasCustomOperators from mlia.target.cortex_a.data_analysis import ModelIsCortexACompatible @@ -24,65 +25,58 @@ from mlia.target.cortex_a.data_analysis import TFLiteCompatibilityCheckFailed from mlia.target.cortex_a.operators import CortexACompatibilityInfo from mlia.target.cortex_a.operators import Operator -BACKEND_INFO = ( - f"{ARMNN_TFLITE_DELEGATE['metadata']['backend']} " - f"{ARMNN_TFLITE_DELEGATE['metadata']['version']}" -) +VERSION = CortexAConfiguration.load_profile("cortex-a").armnn_tflite_delegate_version +BACKEND_INFO = f"{ARMNN_TFLITE_DELEGATE['backend']} {VERSION}" @pytest.mark.parametrize( "input_data, expected_facts", [ [ - CortexACompatibilityInfo(True, []), + CortexACompatibilityInfo([], VERSION), [ModelIsCortexACompatible(BACKEND_INFO)], ], [ CortexACompatibilityInfo( - True, [ Operator( "CONV_2D", "somewhere", - support_type=Operator.SupportType.COMPATIBLE, activation_func=TFL_ACTIVATION_FUNCTION.NONE, ), Operator( "CUSTOM", "somewhere else", - support_type=Operator.SupportType.COMPATIBLE, activation_func=TFL_ACTIVATION_FUNCTION.SIGN_BIT, custom_name="MaxPool3D", ), ], + VERSION, ), [ModelIsCortexACompatible(BACKEND_INFO)], ], [ # pylint: disable=line-too-long CortexACompatibilityInfo( - False, [ Operator( "UNSUPPORTED_OP", "somewhere", - support_type=Operator.SupportType.OP_NOT_SUPPORTED, activation_func=TFL_ACTIVATION_FUNCTION.NONE, ), Operator( "CUSTOM", "somewhere", - support_type=Operator.SupportType.OP_NOT_SUPPORTED, activation_func=TFL_ACTIVATION_FUNCTION.NONE, custom_name="UNSUPPORTED_OP", ), Operator( "CONV_2D", "somewhere else", - support_type=Operator.SupportType.ACTIVATION_NOT_SUPPORTED, activation_func=TFL_ACTIVATION_FUNCTION.SIGN_BIT, ), ], + VERSION, ), [ ModelIsNotCortexACompatible( @@ -161,4 +155,5 @@ def test_cortex_a_data_analyzer( """Test Cortex-A data analyzer.""" analyzer = CortexADataAnalyzer() analyzer.analyze_data(input_data) - assert analyzer.get_analyzed_data() == expected_facts + analyzed_data = analyzer.get_analyzed_data() + assert analyzed_data == expected_facts diff --git a/tests/test_target_cortex_a_data_collection.py b/tests/test_target_cortex_a_data_collection.py index d5f5a2d..6876f83 100644 --- a/tests/test_target_cortex_a_data_collection.py +++ b/tests/test_target_cortex_a_data_collection.py @@ -7,9 +7,13 @@ from unittest.mock import MagicMock import pytest from mlia.core.context import ExecutionContext +from mlia.target.cortex_a.config import CortexAConfiguration from mlia.target.cortex_a.data_collection import CortexAOperatorCompatibility from mlia.target.cortex_a.operators import CortexACompatibilityInfo +CORTEX_A_CONFIG = CortexAConfiguration.load_profile("cortex-a") +VERSION = CORTEX_A_CONFIG.armnn_tflite_delegate_version + def check_cortex_a_data_collection( monkeypatch: pytest.MonkeyPatch, model: Path, tmpdir: str @@ -19,11 +23,11 @@ def check_cortex_a_data_collection( monkeypatch.setattr( "mlia.target.cortex_a.data_collection.get_cortex_a_compatibility_info", - MagicMock(return_value=CortexACompatibilityInfo(True, [])), + MagicMock(return_value=CortexACompatibilityInfo([], VERSION)), ) context = ExecutionContext(output_dir=tmpdir) - collector = CortexAOperatorCompatibility(model) + collector = CortexAOperatorCompatibility(model, CORTEX_A_CONFIG) collector.set_context(context) data_item = collector.collect_data() diff --git a/tests/test_target_cortex_a_operators.py b/tests/test_target_cortex_a_operators.py index 262ebc8..8bc48e6 100644 --- a/tests/test_target_cortex_a_operators.py +++ b/tests/test_target_cortex_a_operators.py @@ -1,7 +1,8 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for Cortex-A operator compatibility.""" from pathlib import Path +from typing import cast import pytest import tensorflow as tf @@ -9,18 +10,19 @@ import tensorflow as tf from mlia.backend.armnn_tflite_delegate import compat from mlia.nn.tensorflow.tflite_graph import TFL_OP from mlia.nn.tensorflow.utils import convert_to_tflite +from mlia.target.cortex_a.config import CortexAConfiguration from mlia.target.cortex_a.operators import CortexACompatibilityInfo from mlia.target.cortex_a.operators import get_cortex_a_compatibility_info -from mlia.target.cortex_a.operators import Operator def test_compat_data() -> None: """Make sure all data contains the necessary items.""" builtin_tfl_ops = {op.name for op in TFL_OP} - for data in [compat.ARMNN_TFLITE_DELEGATE]: - assert "metadata" in data - assert "backend" in data["metadata"] - assert "version" in data["metadata"] + assert "backend" in compat.ARMNN_TFLITE_DELEGATE + assert "ops" in compat.ARMNN_TFLITE_DELEGATE + + ops = cast(dict, compat.ARMNN_TFLITE_DELEGATE["ops"]) + for data in ops.values(): assert "builtin_ops" in data for comp in data["builtin_ops"]: assert comp in builtin_tfl_ops @@ -32,14 +34,18 @@ def check_get_cortex_a_compatibility_info( expected_success: bool, ) -> None: """Check the function 'get_cortex_a_compatibility_info'.""" - compat_info = get_cortex_a_compatibility_info(model_path) + compat_info = get_cortex_a_compatibility_info( + model_path, CortexAConfiguration.load_profile("cortex-a") + ) assert isinstance(compat_info, CortexACompatibilityInfo) - assert expected_success == compat_info.cortex_a_compatible + assert expected_success == compat_info.is_cortex_a_compatible assert compat_info.operators for oper in compat_info.operators: assert oper.name assert oper.location - assert oper.support_type in Operator.SupportType + assert ( + compat_info.get_support_type(oper) in CortexACompatibilityInfo.SupportType + ) def test_get_cortex_a_compatibility_info_compatible( diff --git a/tests/test_target_cortex_a_reporters.py b/tests/test_target_cortex_a_reporters.py index 6866396..7ed0996 100644 --- a/tests/test_target_cortex_a_reporters.py +++ b/tests/test_target_cortex_a_reporters.py @@ -11,6 +11,7 @@ from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus from mlia.nn.tensorflow.tflite_graph import TFL_ACTIVATION_FUNCTION from mlia.target.cortex_a.config import CortexAConfiguration +from mlia.target.cortex_a.operators import CortexACompatibilityInfo from mlia.target.cortex_a.operators import Operator from mlia.target.cortex_a.reporters import cortex_a_formatters from mlia.target.cortex_a.reporters import report_target @@ -27,14 +28,16 @@ def test_report_target() -> None: ( [Advice(["Sample", "Advice"])], TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.COMPATIBLE), - [ - Operator( - name="Test", - location="loc", - support_type=Operator.SupportType.OP_NOT_SUPPORTED, - activation_func=TFL_ACTIVATION_FUNCTION.NONE, - ) - ], + CortexACompatibilityInfo( + [ + Operator( + name="Test", + location="loc", + activation_func=TFL_ACTIVATION_FUNCTION.NONE, + ) + ], + CortexAConfiguration.load_profile("cortex-a").armnn_tflite_delegate_version, + ), ), ) def test_cortex_a_formatters(data: Any) -> None: -- cgit v1.2.1