From 49dfd03617ed28a4fcbd847dd958c27442c84556 Mon Sep 17 00:00:00 2001 From: Annie Tallund Date: Mon, 6 Feb 2023 14:56:31 +0100 Subject: MLIA-709 Update compatibility data for Cortex-A - Add operator compatibility data for Cortex-A via for ArmNN TensorFlow Lite delegate 22.11 - Extend the Cortex-A target profile to include the version of the ArmNN TensorFlow Lite delegate to be used. - Some re-factoring work to support multiple versions and the new target profile parameter. Change-Id: Iae91bb0757ea3909be975af68b34d0ca2be47c43 --- src/mlia/backend/armnn_tflite_delegate/__init__.py | 4 +- src/mlia/backend/armnn_tflite_delegate/compat.py | 523 ++++++++++++++------- src/mlia/resources/target_profiles/cortex-a.toml | 3 + src/mlia/target/cortex_a/advisor.py | 4 +- src/mlia/target/cortex_a/config.py | 15 + src/mlia/target/cortex_a/data_analysis.py | 13 +- src/mlia/target/cortex_a/data_collection.py | 9 +- src/mlia/target/cortex_a/handlers.py | 2 +- src/mlia/target/cortex_a/operators.py | 135 +++--- src/mlia/target/cortex_a/reporters.py | 14 +- tests/test_target_cortex_a_advice_generation.py | 5 +- tests/test_target_cortex_a_data_analysis.py | 23 +- tests/test_target_cortex_a_data_collection.py | 8 +- tests/test_target_cortex_a_operators.py | 24 +- tests/test_target_cortex_a_reporters.py | 19 +- 15 files changed, 503 insertions(+), 298 deletions(-) diff --git a/src/mlia/backend/armnn_tflite_delegate/__init__.py b/src/mlia/backend/armnn_tflite_delegate/__init__.py index 4fe8639..66e5a2a 100644 --- a/src/mlia/backend/armnn_tflite_delegate/__init__.py +++ b/src/mlia/backend/armnn_tflite_delegate/__init__.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Arm NN TensorFlow Lite delegate backend module.""" +from typing import cast + from mlia.backend.armnn_tflite_delegate.compat import ARMNN_TFLITE_DELEGATE from mlia.backend.config import BackendConfiguration from mlia.backend.config import BackendType @@ -14,5 +16,5 @@ registry.register( supported_systems=None, backend_type=BackendType.BUILTIN, ), - pretty_name=ARMNN_TFLITE_DELEGATE["metadata"]["backend"], + pretty_name=cast(str, ARMNN_TFLITE_DELEGATE["backend"]), ) diff --git a/src/mlia/backend/armnn_tflite_delegate/compat.py b/src/mlia/backend/armnn_tflite_delegate/compat.py index c474e75..e2650be 100644 --- a/src/mlia/backend/armnn_tflite_delegate/compat.py +++ b/src/mlia/backend/armnn_tflite_delegate/compat.py @@ -1,184 +1,355 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Collection of Cortex-A operator compatibility information.""" from __future__ import annotations -from typing import Any - -ARMNN_TFLITE_DELEGATE: dict[str, dict[str, Any]] = { - "metadata": { - "backend": "Arm NN TensorFlow Lite delegate", - "version": "22.08", - }, - # BUILTIN OPERATORS - "builtin_ops": { - "ABS": {}, - "ADD": {}, - "ARG_MAX": {}, - "ARG_MIN": {}, - "AVERAGE_POOL_2D": { - "supported_fused_activation": [ - "RELU", - "RELU6", - "RELU_N1_TO_1", - "SIGMOID", - "TANH", - "NONE", - ] - }, - "BATCH_TO_SPACE_ND": {}, - "CAST": {}, - "CONCATENATION": { - "supported_fused_activation": [ - "RELU", - "RELU6", - "RELU_N1_TO_1", - "SIGMOID", - "TANH", - "NONE", - ] - }, - "CONV_2D": { - "supported_fused_activation": [ - "RELU", - "RELU6", - "RELU_N1_TO_1", - "SIGMOID", - "TANH", - "NONE", - ] - }, - "CONV_3D": { - "supported_fused_activation": [ - "RELU", - "RELU6", - "RELU_N1_TO_1", - "SIGMOID", - "TANH", - "NONE", - ] - }, - "DEPTH_TO_SPACE": {}, - "DEPTHWISE_CONV_2D": { - "supported_fused_activation": [ - "RELU", - "RELU6", - "RELU_N1_TO_1", - "SIGMOID", - "TANH", - "NONE", - ] - }, - "DEQUANTIZE": {}, - "DIV": {}, - "EQUAL": {}, - "ELU": {}, - "EXP": {}, - "EXPAND_DIMS": {}, - "FILL": {}, - "FLOOR": {}, - "FLOOR_DIV": {}, - "FULLY_CONNECTED": { - "supported_fused_activation": [ - "RELU", - "RELU6", - "RELU_N1_TO_1", - "SIGMOID", - "TANH", - "NONE", - ] - }, - "GATHER": {}, - "GATHER_ND": {}, - "GREATER": {}, - "GREATER_EQUAL": {}, - "HARD_SWISH": {}, - "L2_NORMALIZATION": {}, - "L2_POOL_2D": {}, - "LESS": {}, - "LESS_EQUAL": {}, - "LOCAL_RESPONSE_NORMALIZATION": {}, - "LOG": {}, - "LOGICAL_AND": {}, - "LOGICAL_NOT": {}, - "LOGICAL_OR": {}, - "LOGISTIC": {}, - "LOG_SOFTMAX": {}, - "LSTM": {}, - "MAXIMUM": {}, - "MAX_POOL_2D": { - "supported_fused_activation": [ - "RELU", - "RELU6", - "RELU_N1_TO_1", - "SIGMOID", - "TANH", - "NONE", - ] - }, - "MEAN": {}, - "MINIMUM": {}, - "MIRROR_PAD": {}, - "MUL": {}, - "NEG": {}, - "NOT_EQUAL": {}, - "PACK": {}, - "PAD": {}, - "PADV2": {}, - "PRELU": {}, - "QUANTIZE": {}, - "RANK": {}, - "REDUCE_MAX": {}, - "REDUCE_MIN": {}, - "REDUCE_PROD": {}, - "RELU": {}, - "RELU6": {}, - "RELU_N1_TO_1": {}, - "RESHAPE": {}, - "RESIZE_BILINEAR": {}, - "RESIZE_NEAREST_NEIGHBOR": {}, - "RSQRT": {}, - "SHAPE": {}, - "SIN": {}, - "SOFTMAX": {}, - "SPACE_TO_BATCH_ND": {}, - "SPACE_TO_DEPTH": {}, - "SPLIT": {}, - "SPLIT_V": {}, - "SQRT": {}, - "SQUEEZE": {}, - "STRIDED_SLICE": {}, - "SUB": {}, - "SUM": {}, - "TANH": {}, - "TRANSPOSE": {}, - "TRANSPOSE_CONV": {}, - "UNIDIRECTIONAL_SEQUENCE_LSTM": {}, - "UNPACK": {}, - }, - # CUSTOM OPERATORS - "custom_ops": { - "AveragePool3D": { - "supported_fused_activation": [ - "RELU", - "RELU6", - "RELU_N1_TO_1", - "SIGMOID", - "SIGN_BIT", - "TANH", - "NONE", - ] +ARMNN_TFLITE_DELEGATE: dict = { + "backend": "Arm NN TensorFlow Lite delegate", + "ops": { + "22.08": { + "builtin_ops": { + "ABS": {}, + "ADD": {}, + "ARG_MAX": {}, + "ARG_MIN": {}, + "AVERAGE_POOL_2D": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "TANH", + "NONE", + ] + }, + "BATCH_TO_SPACE_ND": {}, + "CAST": {}, + "CONCATENATION": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "TANH", + "NONE", + ] + }, + "CONV_2D": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "TANH", + "NONE", + ] + }, + "CONV_3D": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "TANH", + "NONE", + ] + }, + "DEPTH_TO_SPACE": {}, + "DEPTHWISE_CONV_2D": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "TANH", + "NONE", + ] + }, + "DEQUANTIZE": {}, + "DIV": {}, + "ELU": {}, + "EQUAL": {}, + "EXP": {}, + "EXPAND_DIMS": {}, + "FILL": {}, + "FLOOR": {}, + "FLOOR_DIV": {}, + "FULLY_CONNECTED": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "TANH", + "NONE", + ] + }, + "GATHER": {}, + "GATHER_ND": {}, + "GREATER": {}, + "GREATER_EQUAL": {}, + "HARD_SWISH": {}, + "L2_NORMALIZATION": {}, + "L2_POOL_2D": {}, + "LESS": {}, + "LESS_EQUAL": {}, + "LOCAL_RESPONSE_NORMALIZATION": {}, + "LOG": {}, + "LOGICAL_AND": {}, + "LOGICAL_NOT": {}, + "LOGICAL_OR": {}, + "LOGISTIC": {}, + "LOG_SOFTMAX": {}, + "LSTM": {}, + "MAX_POOL_2D": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "TANH", + "NONE", + ] + }, + "MAXIMUM": {}, + "MEAN": {}, + "MINIMUM": {}, + "MIRROR_PAD": {}, + "MUL": {}, + "NEG": {}, + "NOT_EQUAL": {}, + "PACK": {}, + "PAD": {}, + "PADV2": {}, + "PRELU": {}, + "QUANTIZE": {}, + "RANK": {}, + "REDUCE_MAX": {}, + "REDUCE_MIN": {}, + "REDUCE_PROD": {}, + "RELU": {}, + "RELU_N1_TO_1": {}, + "RELU6": {}, + "RESHAPE": {}, + "RESIZE_BILINEAR": {}, + "RESIZE_NEAREST_NEIGHBOR": {}, + "RSQRT": {}, + "SHAPE": {}, + "SIN": {}, + "SOFTMAX": {}, + "SPACE_TO_BATCH_ND": {}, + "SPACE_TO_DEPTH": {}, + "SPLIT": {}, + "SPLIT_V": {}, + "SQRT": {}, + "SQUEEZE": {}, + "STRIDED_SLICE": {}, + "SUB": {}, + "SUM": {}, + "TANH": {}, + "TRANSPOSE": {}, + "TRANSPOSE_CONV": {}, + "UNIDIRECTIONAL_SEQUENCE_LSTM": {}, + "UNPACK": {}, + }, + # CUSTOM OPERATORS + "custom_ops": { + "AveragePool3D": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "SIGN_BIT", + "TANH", + "NONE", + ] + }, + "MaxPool3D": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "SIGN_BIT", + "TANH", + "NONE", + ] + }, + }, }, - "MaxPool3D": { - "supported_fused_activation": [ - "RELU", - "RELU6", - "RELU_N1_TO_1", - "SIGMOID", - "SIGN_BIT", - "TANH", - "NONE", - ] + "22.11": { + "builtin_ops": { + "ABS": {}, + "ADD": {}, + "ARG_MAX": {}, + "ARG_MIN": {}, + "AVERAGE_POOL_2D": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "TANH", + "NONE", + ] + }, + "BATCH_MATMUL": {}, + "BATCH_TO_SPACE_ND": {}, + "CAST": {}, + "CONCATENATION": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "TANH", + "NONE", + ] + }, + "CONV_2D": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "TANH", + "NONE", + ] + }, + "CONV_3D": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "TANH", + "NONE", + ] + }, + "DEPTH_TO_SPACE": {}, + "DEPTHWISE_CONV_2D": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "TANH", + "NONE", + ] + }, + "DEQUANTIZE": {}, + "DIV": {}, + "ELU": {}, + "EQUAL": {}, + "EXP": {}, + "EXPAND_DIMS": {}, + "FILL": {}, + "FLOOR": {}, + "FLOOR_DIV": {}, + "FULLY_CONNECTED": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "TANH", + "NONE", + ] + }, + "GATHER": {}, + "GATHER_ND": {}, + "GREATER": {}, + "GREATER_EQUAL": {}, + "HARD_SWISH": {}, + "L2_NORMALIZATION": {}, + "L2_POOL_2D": {}, + "LESS": {}, + "LESS_EQUAL": {}, + "LOCAL_RESPONSE_NORMALIZATION": {}, + "LOG": {}, + "LOGICAL_AND": {}, + "LOGICAL_NOT": {}, + "LOGICAL_OR": {}, + "LOGISTIC": {}, + "LOG_SOFTMAX": {}, + "LSTM": {}, + "MAX_POOL_2D": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "TANH", + "NONE", + ] + }, + "MAXIMUM": {}, + "MEAN": {}, + "MINIMUM": {}, + "MIRROR_PAD": {}, + "MUL": {}, + "NEG": {}, + "NOT_EQUAL": {}, + "PACK": {}, + "PAD": {}, + "PADV2": {}, + "PRELU": {}, + "QUANTIZE": {}, + "RANK": {}, + "REDUCE_MAX": {}, + "REDUCE_MIN": {}, + "REDUCE_PROD": {}, + "RELU": {}, + "RELU_N1_TO_1": {}, + "RELU6": {}, + "RESHAPE": {}, + "RESIZE_BILINEAR": {}, + "RESIZE_NEAREST_NEIGHBOR": {}, + "RSQRT": {}, + "SHAPE": {}, + "SIN": {}, + "SOFTMAX": {}, + "SPACE_TO_BATCH_ND": {}, + "SPACE_TO_DEPTH": {}, + "SPLIT": {}, + "SPLIT_V": {}, + "SQRT": {}, + "SQUEEZE": {}, + "STRIDED_SLICE": {}, + "SUB": {}, + "SUM": {}, + "TANH": {}, + "TRANSPOSE": {}, + "TRANSPOSE_CONV": {}, + "UNIDIRECTIONAL_SEQUENCE_LSTM": {}, + "UNPACK": {}, + }, + # CUSTOM OPERATORS + "custom_ops": { + "AveragePool3D": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "SIGN_BIT", + "TANH", + "NONE", + ] + }, + "MaxPool3D": { + "supported_fused_activation": [ + "RELU", + "RELU6", + "RELU_N1_TO_1", + "SIGMOID", + "SIGN_BIT", + "TANH", + "NONE", + ] + }, + }, }, }, } diff --git a/src/mlia/resources/target_profiles/cortex-a.toml b/src/mlia/resources/target_profiles/cortex-a.toml index 9de9cee..e8c76bf 100644 --- a/src/mlia/resources/target_profiles/cortex-a.toml +++ b/src/mlia/resources/target_profiles/cortex-a.toml @@ -2,3 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 target="cortex-a" + +[backend.armnn-tflite-delegate] +version="22.11" diff --git a/src/mlia/target/cortex_a/advisor.py b/src/mlia/target/cortex_a/advisor.py index a093784..5c077fb 100644 --- a/src/mlia/target/cortex_a/advisor.py +++ b/src/mlia/target/cortex_a/advisor.py @@ -36,11 +36,13 @@ class CortexAInferenceAdvisor(DefaultInferenceAdvisor): def get_collectors(self, context: Context) -> list[DataCollector]: """Return list of the data collectors.""" model = self.get_model(context) + target_profile = self.get_target_profile(context) + target_config = cast(CortexAConfiguration, profile(target_profile)) collectors: list[DataCollector] = [] if context.category_enabled(AdviceCategory.COMPATIBILITY): - collectors.append(CortexAOperatorCompatibility(model)) + collectors.append(CortexAOperatorCompatibility(model, target_config)) return collectors diff --git a/src/mlia/target/cortex_a/config.py b/src/mlia/target/cortex_a/config.py index fd39e0a..f91031e 100644 --- a/src/mlia/target/cortex_a/config.py +++ b/src/mlia/target/cortex_a/config.py @@ -5,6 +5,7 @@ from __future__ import annotations from typing import Any +from mlia.backend.armnn_tflite_delegate.compat import ARMNN_TFLITE_DELEGATE from mlia.target.config import TargetProfile @@ -16,8 +17,22 @@ class CortexAConfiguration(TargetProfile): target = kwargs["target"] super().__init__(target) + self.backend_config = kwargs.get("backend") + self.armnn_tflite_delegate_version = kwargs["backend"]["armnn-tflite-delegate"][ + "version" + ] + def verify(self) -> None: """Check the parameters.""" super().verify() if self.target != "cortex-a": raise ValueError(f"Wrong target {self.target} for Cortex-A configuration.") + + if not self.armnn_tflite_delegate_version: + raise ValueError("No version for ArmNN TensorFlow Lite delegate specified.") + if self.armnn_tflite_delegate_version not in ARMNN_TFLITE_DELEGATE["ops"]: + raise ValueError( + f"Version '{self.armnn_tflite_delegate_version}' of " + f"backend {ARMNN_TFLITE_DELEGATE['backend']} is not supported. " + f"Choose from: {', '.join(ARMNN_TFLITE_DELEGATE['ops'])}" + ) diff --git a/src/mlia/target/cortex_a/data_analysis.py b/src/mlia/target/cortex_a/data_analysis.py index 4a3a068..089c1a2 100644 --- a/src/mlia/target/cortex_a/data_analysis.py +++ b/src/mlia/target/cortex_a/data_analysis.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Cortex-A data analysis module.""" from __future__ import annotations @@ -13,7 +13,6 @@ from mlia.core.data_analysis import Fact from mlia.core.data_analysis import FactExtractor from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo from mlia.target.cortex_a.operators import CortexACompatibilityInfo -from mlia.target.cortex_a.operators import Operator class CortexADataAnalyzer(FactExtractor): @@ -28,7 +27,7 @@ class CortexADataAnalyzer(FactExtractor): self, data_item: CortexACompatibilityInfo ) -> None: """Analyse operator compatibility information.""" - if data_item.cortex_a_compatible: + if data_item.is_cortex_a_compatible: self.add_fact(ModelIsCortexACompatible(data_item.backend_info)) else: unsupported_ops = set() @@ -36,17 +35,17 @@ class CortexADataAnalyzer(FactExtractor): str, ModelIsNotCortexACompatible.ActivationFunctionSupport ] = defaultdict(ModelIsNotCortexACompatible.ActivationFunctionSupport) for oper in data_item.operators: - if oper.support_type == Operator.SupportType.OP_NOT_SUPPORTED: + support_type = data_item.get_support_type(oper) + if support_type == data_item.SupportType.OP_NOT_SUPPORTED: unsupported_ops.add(oper.full_name) - - if oper.support_type == Operator.SupportType.ACTIVATION_NOT_SUPPORTED: + elif support_type == data_item.SupportType.ACTIVATION_NOT_SUPPORTED: # Add used but unsupported actication functions activation_func_support[oper.full_name].used_unsupported.add( oper.activation_func.name ) # Add supported activation functions activation_func_support[oper.full_name].supported.update( - oper.supported_activation_functions + data_item.supported_activation_functions(oper) ) assert ( diff --git a/src/mlia/target/cortex_a/data_collection.py b/src/mlia/target/cortex_a/data_collection.py index 3ec63e2..cf1268f 100644 --- a/src/mlia/target/cortex_a/data_collection.py +++ b/src/mlia/target/cortex_a/data_collection.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Data collection module for Cortex-A.""" from __future__ import annotations @@ -11,6 +11,7 @@ from mlia.nn.tensorflow.config import get_tflite_model from mlia.nn.tensorflow.tflite_compat import TFLiteChecker from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo from mlia.nn.tensorflow.utils import is_tflite_model +from mlia.target.cortex_a.config import CortexAConfiguration from mlia.target.cortex_a.operators import CortexACompatibilityInfo from mlia.target.cortex_a.operators import get_cortex_a_compatibility_info from mlia.utils.logging import log_action @@ -22,9 +23,10 @@ logger = logging.getLogger(__name__) class CortexAOperatorCompatibility(ContextAwareDataCollector): """Collect operator compatibility information.""" - def __init__(self, model: Path) -> None: + def __init__(self, model: Path, target_config: CortexAConfiguration) -> None: """Init operator compatibility data collector.""" self.model = model + self.target_config = target_config def collect_data(self) -> TFLiteCompatibilityInfo | CortexACompatibilityInfo | None: """Collect operator compatibility information.""" @@ -41,7 +43,8 @@ class CortexAOperatorCompatibility(ContextAwareDataCollector): with log_action("Checking operator compatibility ..."): return ( get_cortex_a_compatibility_info( # pylint: disable=assignment-from-none - Path(tflite_model.model_path) + Path(tflite_model.model_path), + self.target_config, ) ) diff --git a/src/mlia/target/cortex_a/handlers.py b/src/mlia/target/cortex_a/handlers.py index a952c39..cc2c6b4 100644 --- a/src/mlia/target/cortex_a/handlers.py +++ b/src/mlia/target/cortex_a/handlers.py @@ -28,7 +28,7 @@ class CortexAEventHandler(WorkflowEventsHandler, CortexAAdvisorEventHandler): data_item = event.data_item if isinstance(data_item, CortexACompatibilityInfo): - self.reporter.submit(data_item.operators, delay_print=True) + self.reporter.submit(data_item, delay_print=True) if isinstance(data_item, TFLiteCompatibilityInfo) and not data_item.compatible: self.reporter.submit(data_item, delay_print=True) diff --git a/src/mlia/target/cortex_a/operators.py b/src/mlia/target/cortex_a/operators.py index ae611e5..cd92f31 100644 --- a/src/mlia/target/cortex_a/operators.py +++ b/src/mlia/target/cortex_a/operators.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Cortex-A tools module.""" from __future__ import annotations @@ -7,7 +7,7 @@ from dataclasses import dataclass from enum import Enum from pathlib import Path from typing import Any -from typing import ClassVar +from typing import cast from mlia.backend.armnn_tflite_delegate.compat import ( ARMNN_TFLITE_DELEGATE as TFLITE_DELEGATE_COMPAT, @@ -15,33 +15,18 @@ from mlia.backend.armnn_tflite_delegate.compat import ( from mlia.nn.tensorflow.tflite_graph import Op from mlia.nn.tensorflow.tflite_graph import parse_subgraphs from mlia.nn.tensorflow.tflite_graph import TFL_ACTIVATION_FUNCTION +from mlia.target.cortex_a.config import CortexAConfiguration @dataclass class Operator: """Cortex-A compatibility information of the operator.""" - BUILTIN_COMPATIBILITY = TFLITE_DELEGATE_COMPAT["builtin_ops"] - CUSTOM_COMPATIBILITY = TFLITE_DELEGATE_COMPAT["custom_ops"] - - class SupportType(Enum): - """Type of operator support.""" - - COMPATIBLE = "Compatible" - OP_NOT_SUPPORTED = "Operator not supported" - ACTIVATION_NOT_SUPPORTED = "Activation not supported" - name: str location: str - support_type: SupportType activation_func: TFL_ACTIVATION_FUNCTION custom_name: str | None = None - @property - def is_cortex_a_compatible(self) -> bool: - """Check if this operator is compatible.""" - return self.support_type == Operator.SupportType.COMPATIBLE - @property def full_name(self) -> str: """Returun the full name including the custom name if applicable.""" @@ -52,27 +37,11 @@ class Operator: """Check if this is a custom operator.""" return bool(self.custom_name) - @property - def compatibility_data(self) -> dict[str, dict[str, Any]]: - """Get the compatibility data (builtin or custom ops).""" - return ( - Operator.CUSTOM_COMPATIBILITY - if self.is_custom - else Operator.BUILTIN_COMPATIBILITY - ) - - @property - def supported_activation_functions(self) -> list[str]: - """Return a list of fused activation functions supported by this op.""" - op_name = self.custom_name if self.custom_name else self.name - return self.compatibility_data[op_name].get("supported_fused_activation", []) - @classmethod def from_tflite_op(cls, tfl_op: Op, location: str) -> Operator: """Create a new instance from TensorFlow Lite operator and location.""" - support_type = cls._get_support_type(tfl_op) activation_func = ( - tfl_op.builtin_options["fused_activation_function"] + TFL_ACTIVATION_FUNCTION[tfl_op.builtin_options["fused_activation_function"]] if ( tfl_op.builtin_options and "fused_activation_function" in tfl_op.builtin_options @@ -82,50 +51,81 @@ class Operator: return Operator( tfl_op.type, location, - support_type, activation_func=activation_func, custom_name=(tfl_op.custom_type if tfl_op.is_custom else None), ) - @staticmethod - def _get_support_type(tfl_op: Op) -> Operator.SupportType: - """Get the support type from the TensorFlow Lite operator.""" - compat_data = ( - Operator.CUSTOM_COMPATIBILITY - if tfl_op.is_custom - else Operator.BUILTIN_COMPATIBILITY + +class CortexACompatibilityInfo: + """Model's operators.""" + + class SupportType(Enum): + """Type of operator support.""" + + COMPATIBLE = "Compatible" + OP_NOT_SUPPORTED = "Operator not supported" + ACTIVATION_NOT_SUPPORTED = "Activation not supported" + + def __init__(self, ops: list[Operator], armnn_tfl_delegate_version: str) -> None: + """Create a new collection of op compatibility information.""" + compat_data = TFLITE_DELEGATE_COMPAT["ops"][armnn_tfl_delegate_version] + self._builtin_compatibility = compat_data["builtin_ops"] + self._custom_compatibility = compat_data["custom_ops"] + + self.backend_info = ( + f"{TFLITE_DELEGATE_COMPAT['backend']} {armnn_tfl_delegate_version}" ) - op_type = tfl_op.custom_type if tfl_op.is_custom else tfl_op.type - if op_type not in compat_data: - return Operator.SupportType.OP_NOT_SUPPORTED + self.operators = ops + + @property + def is_cortex_a_compatible(self) -> bool: + """Check if all operators are compatible.""" + return all(self.is_op_compatible(oper) for oper in self.operators) + + def is_op_compatible(self, operator: Operator) -> bool: + """Check if the given operator is compatible.""" + return self.get_support_type(operator) == self.SupportType.COMPATIBLE - compat_op = compat_data[op_type] + def compatibility_data(self, operator: Operator) -> dict[str, dict[str, Any]]: + """Get the compatibility data (builtin or custom ops).""" + return ( + cast(dict, self._custom_compatibility) + if operator.is_custom + else cast(dict, self._builtin_compatibility) + ) + + def supported_activation_functions(self, operator: Operator) -> list[str]: + """Return a list of fused activation functions supported by this op.""" + op_name = operator.custom_name if operator.custom_name else operator.name + return self.compatibility_data(operator)[op_name].get( + "supported_fused_activation", [] + ) + + def get_support_type( + self, operator: Operator + ) -> CortexACompatibilityInfo.SupportType: + """Get the support type from the TensorFlow Lite operator.""" + compat_data = self.compatibility_data(operator) + op_name = operator.custom_name if operator.is_custom else operator.name + + if op_name not in compat_data: + return CortexACompatibilityInfo.SupportType.OP_NOT_SUPPORTED + + compat_op = compat_data[op_name] if "supported_fused_activation" in compat_op: - assert tfl_op.builtin_options - assert "fused_activation_function" in tfl_op.builtin_options if ( - tfl_op.builtin_options["fused_activation_function"] + operator.activation_func.name not in compat_op["supported_fused_activation"] ): - return Operator.SupportType.ACTIVATION_NOT_SUPPORTED + return CortexACompatibilityInfo.SupportType.ACTIVATION_NOT_SUPPORTED - return Operator.SupportType.COMPATIBLE + return CortexACompatibilityInfo.SupportType.COMPATIBLE -@dataclass -class CortexACompatibilityInfo: - """Model's operators.""" - - cortex_a_compatible: bool - operators: list[Operator] - backend_info: ClassVar[str] = ( - f"{TFLITE_DELEGATE_COMPAT['metadata']['backend']} " - f"{TFLITE_DELEGATE_COMPAT['metadata']['version']}" - ) - - -def get_cortex_a_compatibility_info(model_path: Path) -> CortexACompatibilityInfo: +def get_cortex_a_compatibility_info( + model_path: Path, target_config: CortexAConfiguration +) -> CortexACompatibilityInfo: """Return list of model's operators.""" model = parse_subgraphs(model_path) @@ -134,8 +134,9 @@ def get_cortex_a_compatibility_info(model_path: Path) -> CortexACompatibilityInf for g_idx, g in enumerate(model) for op_idx, oper in enumerate(g) ] - all_compatible = all(oper.is_cortex_a_compatible for oper in op_list) - compat_info = CortexACompatibilityInfo(all_compatible, op_list) + compat_info = CortexACompatibilityInfo( + op_list, target_config.armnn_tflite_delegate_version + ) return compat_info diff --git a/src/mlia/target/cortex_a/reporters.py b/src/mlia/target/cortex_a/reporters.py index d214b09..65d7906 100644 --- a/src/mlia/target/cortex_a/reporters.py +++ b/src/mlia/target/cortex_a/reporters.py @@ -18,7 +18,7 @@ from mlia.core.reporting import ReportItem from mlia.core.reporting import Table from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo from mlia.target.cortex_a.config import CortexAConfiguration -from mlia.target.cortex_a.operators import Operator +from mlia.target.cortex_a.operators import CortexACompatibilityInfo from mlia.utils.console import style_improvement from mlia.utils.types import is_list_of @@ -85,7 +85,7 @@ def report_tflite_compatiblity(compat_info: TFLiteCompatibilityInfo) -> Report: ) -def report_cortex_a_operators(ops: list[Operator]) -> Report: +def report_cortex_a_operators(op_compat: CortexACompatibilityInfo) -> Report: """Generate report for the operators.""" return Table( [ @@ -108,15 +108,15 @@ def report_cortex_a_operators(ops: list[Operator]) -> Report: op.location, op.full_name, Cell( - op.support_type, + op_compat.get_support_type(op), Format( wrap_width=30, - style=style_improvement(op.is_cortex_a_compatible), + style=style_improvement(op_compat.is_op_compatible(op)), str_fmt=lambda v: cast(str, v.value), ), ), ) - for index, op in enumerate(ops) + for index, op in enumerate(op_compat.operators) ], name="Operators", alias="operators", @@ -134,7 +134,7 @@ def cortex_a_formatters(data: Any) -> Callable[[Any], Report]: if isinstance(data, TFLiteCompatibilityInfo): return report_tflite_compatiblity - if is_list_of(data, Operator): + if isinstance(data, CortexACompatibilityInfo): return report_cortex_a_operators - raise Exception(f"Unable to find appropriate formatter for {data}") + raise Exception(f"Unable to find appropriate formatter for {data}.") diff --git a/tests/test_target_cortex_a_advice_generation.py b/tests/test_target_cortex_a_advice_generation.py index b9edbb5..9596d47 100644 --- a/tests/test_target_cortex_a_advice_generation.py +++ b/tests/test_target_cortex_a_advice_generation.py @@ -14,15 +14,16 @@ from mlia.core.common import DataItem from mlia.core.context import ExecutionContext from mlia.nn.tensorflow.tflite_graph import TFL_ACTIVATION_FUNCTION from mlia.target.cortex_a.advice_generation import CortexAAdviceProducer +from mlia.target.cortex_a.config import CortexAConfiguration from mlia.target.cortex_a.data_analysis import ModelHasCustomOperators from mlia.target.cortex_a.data_analysis import ModelIsCortexACompatible from mlia.target.cortex_a.data_analysis import ModelIsNotCortexACompatible from mlia.target.cortex_a.data_analysis import ModelIsNotTFLiteCompatible from mlia.target.cortex_a.data_analysis import TFLiteCompatibilityCheckFailed +VERSION = CortexAConfiguration.load_profile("cortex-a").armnn_tflite_delegate_version BACKEND_INFO = ( - f"{ARMNN_TFLITE_DELEGATE['metadata']['backend']} " - f"{ARMNN_TFLITE_DELEGATE['metadata']['version']}" + f"{ARMNN_TFLITE_DELEGATE['backend']} " f"{ARMNN_TFLITE_DELEGATE['ops'][VERSION]}" ) diff --git a/tests/test_target_cortex_a_data_analysis.py b/tests/test_target_cortex_a_data_analysis.py index e9fc8bc..0a6b490 100644 --- a/tests/test_target_cortex_a_data_analysis.py +++ b/tests/test_target_cortex_a_data_analysis.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for Cortex-A data analysis module.""" from __future__ import annotations @@ -15,6 +15,7 @@ from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus from mlia.nn.tensorflow.tflite_compat import TFLiteConversionError from mlia.nn.tensorflow.tflite_compat import TFLiteConversionErrorCode from mlia.nn.tensorflow.tflite_graph import TFL_ACTIVATION_FUNCTION +from mlia.target.cortex_a.config import CortexAConfiguration from mlia.target.cortex_a.data_analysis import CortexADataAnalyzer from mlia.target.cortex_a.data_analysis import ModelHasCustomOperators from mlia.target.cortex_a.data_analysis import ModelIsCortexACompatible @@ -24,65 +25,58 @@ from mlia.target.cortex_a.data_analysis import TFLiteCompatibilityCheckFailed from mlia.target.cortex_a.operators import CortexACompatibilityInfo from mlia.target.cortex_a.operators import Operator -BACKEND_INFO = ( - f"{ARMNN_TFLITE_DELEGATE['metadata']['backend']} " - f"{ARMNN_TFLITE_DELEGATE['metadata']['version']}" -) +VERSION = CortexAConfiguration.load_profile("cortex-a").armnn_tflite_delegate_version +BACKEND_INFO = f"{ARMNN_TFLITE_DELEGATE['backend']} {VERSION}" @pytest.mark.parametrize( "input_data, expected_facts", [ [ - CortexACompatibilityInfo(True, []), + CortexACompatibilityInfo([], VERSION), [ModelIsCortexACompatible(BACKEND_INFO)], ], [ CortexACompatibilityInfo( - True, [ Operator( "CONV_2D", "somewhere", - support_type=Operator.SupportType.COMPATIBLE, activation_func=TFL_ACTIVATION_FUNCTION.NONE, ), Operator( "CUSTOM", "somewhere else", - support_type=Operator.SupportType.COMPATIBLE, activation_func=TFL_ACTIVATION_FUNCTION.SIGN_BIT, custom_name="MaxPool3D", ), ], + VERSION, ), [ModelIsCortexACompatible(BACKEND_INFO)], ], [ # pylint: disable=line-too-long CortexACompatibilityInfo( - False, [ Operator( "UNSUPPORTED_OP", "somewhere", - support_type=Operator.SupportType.OP_NOT_SUPPORTED, activation_func=TFL_ACTIVATION_FUNCTION.NONE, ), Operator( "CUSTOM", "somewhere", - support_type=Operator.SupportType.OP_NOT_SUPPORTED, activation_func=TFL_ACTIVATION_FUNCTION.NONE, custom_name="UNSUPPORTED_OP", ), Operator( "CONV_2D", "somewhere else", - support_type=Operator.SupportType.ACTIVATION_NOT_SUPPORTED, activation_func=TFL_ACTIVATION_FUNCTION.SIGN_BIT, ), ], + VERSION, ), [ ModelIsNotCortexACompatible( @@ -161,4 +155,5 @@ def test_cortex_a_data_analyzer( """Test Cortex-A data analyzer.""" analyzer = CortexADataAnalyzer() analyzer.analyze_data(input_data) - assert analyzer.get_analyzed_data() == expected_facts + analyzed_data = analyzer.get_analyzed_data() + assert analyzed_data == expected_facts diff --git a/tests/test_target_cortex_a_data_collection.py b/tests/test_target_cortex_a_data_collection.py index d5f5a2d..6876f83 100644 --- a/tests/test_target_cortex_a_data_collection.py +++ b/tests/test_target_cortex_a_data_collection.py @@ -7,9 +7,13 @@ from unittest.mock import MagicMock import pytest from mlia.core.context import ExecutionContext +from mlia.target.cortex_a.config import CortexAConfiguration from mlia.target.cortex_a.data_collection import CortexAOperatorCompatibility from mlia.target.cortex_a.operators import CortexACompatibilityInfo +CORTEX_A_CONFIG = CortexAConfiguration.load_profile("cortex-a") +VERSION = CORTEX_A_CONFIG.armnn_tflite_delegate_version + def check_cortex_a_data_collection( monkeypatch: pytest.MonkeyPatch, model: Path, tmpdir: str @@ -19,11 +23,11 @@ def check_cortex_a_data_collection( monkeypatch.setattr( "mlia.target.cortex_a.data_collection.get_cortex_a_compatibility_info", - MagicMock(return_value=CortexACompatibilityInfo(True, [])), + MagicMock(return_value=CortexACompatibilityInfo([], VERSION)), ) context = ExecutionContext(output_dir=tmpdir) - collector = CortexAOperatorCompatibility(model) + collector = CortexAOperatorCompatibility(model, CORTEX_A_CONFIG) collector.set_context(context) data_item = collector.collect_data() diff --git a/tests/test_target_cortex_a_operators.py b/tests/test_target_cortex_a_operators.py index 262ebc8..8bc48e6 100644 --- a/tests/test_target_cortex_a_operators.py +++ b/tests/test_target_cortex_a_operators.py @@ -1,7 +1,8 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for Cortex-A operator compatibility.""" from pathlib import Path +from typing import cast import pytest import tensorflow as tf @@ -9,18 +10,19 @@ import tensorflow as tf from mlia.backend.armnn_tflite_delegate import compat from mlia.nn.tensorflow.tflite_graph import TFL_OP from mlia.nn.tensorflow.utils import convert_to_tflite +from mlia.target.cortex_a.config import CortexAConfiguration from mlia.target.cortex_a.operators import CortexACompatibilityInfo from mlia.target.cortex_a.operators import get_cortex_a_compatibility_info -from mlia.target.cortex_a.operators import Operator def test_compat_data() -> None: """Make sure all data contains the necessary items.""" builtin_tfl_ops = {op.name for op in TFL_OP} - for data in [compat.ARMNN_TFLITE_DELEGATE]: - assert "metadata" in data - assert "backend" in data["metadata"] - assert "version" in data["metadata"] + assert "backend" in compat.ARMNN_TFLITE_DELEGATE + assert "ops" in compat.ARMNN_TFLITE_DELEGATE + + ops = cast(dict, compat.ARMNN_TFLITE_DELEGATE["ops"]) + for data in ops.values(): assert "builtin_ops" in data for comp in data["builtin_ops"]: assert comp in builtin_tfl_ops @@ -32,14 +34,18 @@ def check_get_cortex_a_compatibility_info( expected_success: bool, ) -> None: """Check the function 'get_cortex_a_compatibility_info'.""" - compat_info = get_cortex_a_compatibility_info(model_path) + compat_info = get_cortex_a_compatibility_info( + model_path, CortexAConfiguration.load_profile("cortex-a") + ) assert isinstance(compat_info, CortexACompatibilityInfo) - assert expected_success == compat_info.cortex_a_compatible + assert expected_success == compat_info.is_cortex_a_compatible assert compat_info.operators for oper in compat_info.operators: assert oper.name assert oper.location - assert oper.support_type in Operator.SupportType + assert ( + compat_info.get_support_type(oper) in CortexACompatibilityInfo.SupportType + ) def test_get_cortex_a_compatibility_info_compatible( diff --git a/tests/test_target_cortex_a_reporters.py b/tests/test_target_cortex_a_reporters.py index 6866396..7ed0996 100644 --- a/tests/test_target_cortex_a_reporters.py +++ b/tests/test_target_cortex_a_reporters.py @@ -11,6 +11,7 @@ from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityStatus from mlia.nn.tensorflow.tflite_graph import TFL_ACTIVATION_FUNCTION from mlia.target.cortex_a.config import CortexAConfiguration +from mlia.target.cortex_a.operators import CortexACompatibilityInfo from mlia.target.cortex_a.operators import Operator from mlia.target.cortex_a.reporters import cortex_a_formatters from mlia.target.cortex_a.reporters import report_target @@ -27,14 +28,16 @@ def test_report_target() -> None: ( [Advice(["Sample", "Advice"])], TFLiteCompatibilityInfo(status=TFLiteCompatibilityStatus.COMPATIBLE), - [ - Operator( - name="Test", - location="loc", - support_type=Operator.SupportType.OP_NOT_SUPPORTED, - activation_func=TFL_ACTIVATION_FUNCTION.NONE, - ) - ], + CortexACompatibilityInfo( + [ + Operator( + name="Test", + location="loc", + activation_func=TFL_ACTIVATION_FUNCTION.NONE, + ) + ], + CortexAConfiguration.load_profile("cortex-a").armnn_tflite_delegate_version, + ), ), ) def test_cortex_a_formatters(data: Any) -> None: -- cgit v1.2.1