From 836efd40317a397761ec8b66e3f4398faac43ad0 Mon Sep 17 00:00:00 2001 From: Annie Tallund Date: Thu, 12 Jan 2023 07:49:06 +0100 Subject: MLIA-770 List all available backends - Rely on target and backend registry for support information - Make above information less Ethos(TM)-U specific Change-Id: I8dbfb84401016412a3d719a84eb592f21d79c46b --- src/mlia/backend/corstone/performance.py | 4 ++-- src/mlia/backend/install.py | 24 +++++------------------- src/mlia/backend/registry.py | 14 +++++++++++++- src/mlia/cli/command_validators.py | 4 ++-- src/mlia/cli/config.py | 25 +++++++++++++------------ src/mlia/target/ethos_u/performance.py | 10 +++++----- src/mlia/target/registry.py | 24 ++++++++++++++++++++++++ 7 files changed, 64 insertions(+), 41 deletions(-) (limited to 'src/mlia') diff --git a/src/mlia/backend/corstone/performance.py b/src/mlia/backend/corstone/performance.py index 5aabfa5..531f0cd 100644 --- a/src/mlia/backend/corstone/performance.py +++ b/src/mlia/backend/corstone/performance.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Module for backend integration.""" from __future__ import annotations @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) class DeviceInfo: """Device information.""" - device_type: Literal["ethos-u55", "ethos-u65"] + device_type: Literal["Ethos-U55", "Ethos-U65", "ethos-u55", "ethos-u65"] mac: int diff --git a/src/mlia/backend/install.py b/src/mlia/backend/install.py index eea3403..37a277b 100644 --- a/src/mlia/backend/install.py +++ b/src/mlia/backend/install.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Module for installation process.""" from __future__ import annotations @@ -26,17 +26,20 @@ from mlia.utils.filesystem import temp_directory from mlia.utils.filesystem import working_directory from mlia.utils.py_manager import get_package_manager - logger = logging.getLogger(__name__) # Mapping backend -> device_type -> system_name _SUPPORTED_SYSTEMS = { "Corstone-300": { + "Ethos-U55": "Corstone-300: Cortex-M55+Ethos-U55", + "Ethos-U65": "Corstone-300: Cortex-M55+Ethos-U65", "ethos-u55": "Corstone-300: Cortex-M55+Ethos-U55", "ethos-u65": "Corstone-300: Cortex-M55+Ethos-U65", }, "Corstone-310": { + "Ethos-U55": "Corstone-310: Cortex-M85+Ethos-U55", + "Ethos-U65": "Corstone-310: Cortex-M85+Ethos-U65", "ethos-u55": "Corstone-310: Cortex-M85+Ethos-U55", "ethos-u65": "Corstone-310: Cortex-M85+Ethos-U65", }, @@ -61,23 +64,6 @@ def get_application_name(system_name: str) -> str: return _SYSTEM_TO_APP_MAP[system_name] -def is_supported(backend: str, device_type: str | None = None) -> bool: - """Check if the backend (and optionally device type) is supported.""" - if device_type is None: - return backend in _SUPPORTED_SYSTEMS - - try: - get_system_name(backend, device_type) - return True - except KeyError: - return False - - -def supported_backends() -> list[str]: - """Get a list of all backends supported by the backend manager.""" - return list(_SUPPORTED_SYSTEMS.keys()) - - def get_all_system_names(backend: str) -> list[str]: """Get all systems supported by the backend.""" return list(_SUPPORTED_SYSTEMS.get(backend, {}).values()) diff --git a/src/mlia/backend/registry.py b/src/mlia/backend/registry.py index 6a0da74..988c8c3 100644 --- a/src/mlia/backend/registry.py +++ b/src/mlia/backend/registry.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Backend module.""" from mlia.backend.config import BackendConfiguration @@ -6,3 +6,15 @@ from mlia.utils.registry import Registry # All supported targets are required to be registered here. registry = Registry[BackendConfiguration]() + + +def get_supported_backends() -> list: + """Get a list of all backends supported by the backend manager.""" + return sorted(list(registry.items.keys())) + + +def get_supported_systems() -> dict: + """Get a list of all systems supported by the backend manager.""" + return { + backend: config.supported_systems for backend, config in registry.items.items() + } diff --git a/src/mlia/cli/command_validators.py b/src/mlia/cli/command_validators.py index eb04192..8eb966b 100644 --- a/src/mlia/cli/command_validators.py +++ b/src/mlia/cli/command_validators.py @@ -7,7 +7,7 @@ import argparse import logging import sys -from mlia.cli.config import get_default_backends +from mlia.cli.config import get_default_backends_dict from mlia.target.config import get_target from mlia.target.registry import supported_backends @@ -32,7 +32,7 @@ def validate_backend( target = get_target(target_profile) if not backend: - return get_default_backends()[target] + return get_default_backends_dict()[target] compatible_backends = supported_backends(target_map[target]) diff --git a/src/mlia/cli/config.py b/src/mlia/cli/config.py index 680b4b6..0dac3e8 100644 --- a/src/mlia/cli/config.py +++ b/src/mlia/cli/config.py @@ -10,9 +10,9 @@ from typing import Optional from typing import TypedDict from mlia.backend.corstone.install import get_corstone_installations -from mlia.backend.install import supported_backends from mlia.backend.manager import DefaultInstallationManager from mlia.backend.manager import InstallationManager +from mlia.backend.registry import get_supported_backends from mlia.backend.tosa_checker.install import get_tosa_backend_installation logger = logging.getLogger(__name__) @@ -32,25 +32,28 @@ def get_installation_manager(noninteractive: bool = False) -> InstallationManage @lru_cache def get_available_backends() -> list[str]: """Return list of the available backends.""" - available_backends = ["Vela", "tosa-checker", "armnn-tflitedelegate"] - # Add backends using backend manager manager = get_installation_manager() - available_backends.extend( + available_backends = [ backend - for backend in supported_backends() + for backend in get_supported_backends() if manager.backend_installed(backend) - ) + ] return available_backends # List of mutually exclusive Corstone backends ordered by priority _CORSTONE_EXCLUSIVE_PRIORITY = ("Corstone-310", "Corstone-300") -_NON_ETHOS_U_BACKENDS = ("tosa-checker", "armnn-tflitedelegate") +_NON_ETHOS_U_BACKENDS = ("TOSA-Checker", "ArmNNTFLiteDelegate") + + +def get_ethos_u_default_backends(backends: list[str]) -> list[str]: + """Get Ethos-U default backends for evaluation.""" + return [x for x in backends if x not in _NON_ETHOS_U_BACKENDS] -def get_ethos_u_default_backends() -> list[str]: +def get_default_backends() -> list[str]: """Get default backends for evaluation.""" backends = get_available_backends() @@ -64,8 +67,6 @@ def get_ethos_u_default_backends() -> list[str]: ] break - # Filter out non ethos-u backends - backends = [x for x in backends if x not in _NON_ETHOS_U_BACKENDS] return backends @@ -86,9 +87,9 @@ BackendCompatibility = TypedDict( ) -def get_default_backends() -> dict[str, list[str]]: +def get_default_backends_dict() -> dict[str, list[str]]: """Return default backends for all targets.""" - ethos_u_defaults = get_ethos_u_default_backends() + ethos_u_defaults = get_ethos_u_default_backends(get_default_backends()) return { "ethos-u55": ethos_u_defaults, "ethos-u65": ethos_u_defaults, diff --git a/src/mlia/target/ethos_u/performance.py b/src/mlia/target/ethos_u/performance.py index e39f4d9..0d791a1 100644 --- a/src/mlia/target/ethos_u/performance.py +++ b/src/mlia/target/ethos_u/performance.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Performance estimation.""" from __future__ import annotations @@ -14,14 +14,14 @@ import mlia.backend.vela.performance as vela_perf from mlia.backend.corstone.performance import DeviceInfo from mlia.backend.corstone.performance import estimate_performance from mlia.backend.corstone.performance import ModelInfo -from mlia.backend.install import is_supported -from mlia.backend.install import supported_backends +from mlia.backend.registry import get_supported_backends from mlia.core.context import Context from mlia.core.performance import PerformanceEstimator from mlia.nn.tensorflow.config import get_tflite_model from mlia.nn.tensorflow.config import ModelConfiguration from mlia.nn.tensorflow.optimizations.select import OptimizationSettings from mlia.target.ethos_u.config import EthosUConfiguration +from mlia.target.registry import is_supported from mlia.utils.logging import log_action @@ -226,7 +226,7 @@ class EthosUPerformanceEstimator( if backend != "Vela" and not is_supported(backend): raise ValueError( f"Unsupported backend '{backend}'. " - f"Only 'Vela' and {supported_backends()} " + f"Only 'Vela' and {get_supported_backends()} " "are supported." ) self.backends = set(backends) @@ -246,7 +246,7 @@ class EthosUPerformanceEstimator( if backend == "Vela": vela_estimator = VelaPerformanceEstimator(self.context, self.device) memory_usage = vela_estimator.estimate(tflite_model) - elif backend in supported_backends(): + elif backend in get_supported_backends(): corstone_estimator = CorstonePerformanceEstimator( self.context, self.device, backend ) diff --git a/src/mlia/target/registry.py b/src/mlia/target/registry.py index 2d29f1b..325dd04 100644 --- a/src/mlia/target/registry.py +++ b/src/mlia/target/registry.py @@ -32,6 +32,30 @@ def supported_backends(target: str) -> list[str]: return registry.items[target].filter_supported_backends(check_system=False) +def get_backend_to_supported_targets() -> dict[str, list]: + """Get a dict that maps a list of supported targets given backend.""" + targets = dict(registry.items) + supported_backends_dict: dict[str, list] = {} + for target, info in targets.items(): + target_backends = info.supported_backends + for backend in target_backends: + supported_backends_dict.setdefault(backend, []).append(target) + return supported_backends_dict + + +def is_supported(backend: str, target: str | None = None) -> bool: + """Check if the backend (and optionally target) is supported.""" + backends = get_backend_to_supported_targets() + if target is None: + if backend in backends: + return True + return False + try: + return target in backends[backend] + except KeyError: + return False + + def supported_targets(advice: AdviceCategory) -> list[str]: """Get a list of all targets supporting the given advice category.""" return [ -- cgit v1.2.1