From 718277eaece76902c4950f18d428907b39a18ef1 Mon Sep 17 00:00:00 2001 From: Benjamin Klimczak Date: Fri, 10 Feb 2023 13:12:57 +0000 Subject: MLIA-769 Add "pretty names" for targets / backends - Provide "pretty names" to print information for targets and backends. - Use 'target_config' instead of 'target' if a target profile is used. - Fix minor issue in output regarding the output directory. Change-Id: Ib38231f30b4d609a0d1e8f9c52b2fb547c69cb6a --- README.md | 4 +-- src/mlia/backend/armnn_tflite_delegate/__init__.py | 4 ++- src/mlia/backend/corstone/__init__.py | 8 +++-- src/mlia/backend/corstone/install.py | 4 +-- src/mlia/backend/corstone/performance.py | 14 ++++---- src/mlia/backend/tosa_checker/__init__.py | 1 + src/mlia/backend/vela/__init__.py | 3 +- src/mlia/cli/main.py | 7 ++-- src/mlia/target/cortex_a/__init__.py | 5 +-- src/mlia/target/cortex_a/events.py | 2 +- src/mlia/target/cortex_a/handlers.py | 2 +- src/mlia/target/cortex_a/reporters.py | 4 +-- src/mlia/target/ethos_u/__init__.py | 10 ++++-- src/mlia/target/ethos_u/advisor.py | 16 ++++----- src/mlia/target/ethos_u/data_collection.py | 16 ++++----- src/mlia/target/ethos_u/events.py | 2 +- src/mlia/target/ethos_u/handlers.py | 2 +- src/mlia/target/ethos_u/performance.py | 42 ++++++++++++---------- src/mlia/target/ethos_u/reporters.py | 8 ++--- src/mlia/target/registry.py | 20 +++++------ src/mlia/target/tosa/__init__.py | 1 + src/mlia/target/tosa/reporters.py | 4 +-- src/mlia/utils/registry.py | 14 ++++++-- tests/conftest.py | 6 ++-- tests/test_backend_corstone.py | 4 +-- tests/test_backend_corstone_performance.py | 4 +-- tests/test_backend_registry.py | 16 ++++----- tests/test_backend_vela_compat.py | 6 ++-- tests/test_backend_vela_compiler.py | 6 ++-- tests/test_backend_vela_performance.py | 22 +++++++----- tests/test_cli_command_validators.py | 14 ++++---- tests/test_cli_main.py | 2 +- tests/test_target_registry.py | 34 +++++++++--------- 33 files changed, 170 insertions(+), 137 deletions(-) diff --git a/README.md b/README.md index 37ba6fc..1abe0d0 100644 --- a/README.md +++ b/README.md @@ -149,8 +149,8 @@ mlia check ~/models/mobilenet_v1_1.0_224_quant.tflite \ mlia check ~/models/ds_cnn_large_fully_quantized_int8.tflite \ --target-profile ethos-u65-512 \ --performance \ - --backend "Vela" \ - --backend "Corstone-300" + --backend "vela" \ + --backend "corstone-300" # Get help and further information mlia check --help diff --git a/src/mlia/backend/armnn_tflite_delegate/__init__.py b/src/mlia/backend/armnn_tflite_delegate/__init__.py index ccb7e38..4fe8639 100644 --- a/src/mlia/backend/armnn_tflite_delegate/__init__.py +++ b/src/mlia/backend/armnn_tflite_delegate/__init__.py @@ -1,16 +1,18 @@ # SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Arm NN TensorFlow Lite delegate backend module.""" +from mlia.backend.armnn_tflite_delegate.compat import ARMNN_TFLITE_DELEGATE from mlia.backend.config import BackendConfiguration from mlia.backend.config import BackendType from mlia.backend.registry import registry from mlia.core.common import AdviceCategory registry.register( - "ArmNNTFLiteDelegate", + "armnn-tflite-delegate", BackendConfiguration( supported_advice=[AdviceCategory.COMPATIBILITY], supported_systems=None, backend_type=BackendType.BUILTIN, ), + pretty_name=ARMNN_TFLITE_DELEGATE["metadata"]["backend"], ) diff --git a/src/mlia/backend/corstone/__init__.py b/src/mlia/backend/corstone/__init__.py index b59ab65..5aa688b 100644 --- a/src/mlia/backend/corstone/__init__.py +++ b/src/mlia/backend/corstone/__init__.py @@ -12,15 +12,19 @@ CORSTONE_PRIORITY = ("Corstone-310", "Corstone-300") for corstone_name in CORSTONE_PRIORITY: registry.register( - corstone_name, + corstone_name.lower(), BackendConfiguration( supported_advice=[AdviceCategory.PERFORMANCE, AdviceCategory.OPTIMIZATION], supported_systems=[System.LINUX_AMD64], backend_type=BackendType.CUSTOM, ), + pretty_name=corstone_name, ) def is_corstone_backend(backend_name: str) -> bool: """Check if backend belongs to Corstone.""" - return backend_name in CORSTONE_PRIORITY + return any( + name in CORSTONE_PRIORITY + for name in (backend_name, registry.pretty_name(backend_name)) + ) diff --git a/src/mlia/backend/corstone/install.py b/src/mlia/backend/corstone/install.py index c57a47b..35976cf 100644 --- a/src/mlia/backend/corstone/install.py +++ b/src/mlia/backend/corstone/install.py @@ -62,7 +62,7 @@ def get_corstone_300_installation() -> Installation: """Get Corstone-300 installation.""" corstone_300 = BackendInstallation( # pylint: disable=line-too-long - name="Corstone-300", + name="corstone-300", description="Corstone-300 FVP", fvp_dir_name="corstone_300", download_artifact=DownloadArtifact( @@ -102,7 +102,7 @@ def get_corstone_300_installation() -> Installation: def get_corstone_310_installation() -> Installation: """Get Corstone-310 installation.""" corstone_310 = BackendInstallation( - name="Corstone-310", + name="corstone-310", description="Corstone-310 FVP", fvp_dir_name="corstone_310", download_artifact=None, diff --git a/src/mlia/backend/corstone/performance.py b/src/mlia/backend/corstone/performance.py index 8fd3e40..5012821 100644 --- a/src/mlia/backend/corstone/performance.py +++ b/src/mlia/backend/corstone/performance.py @@ -95,7 +95,7 @@ def get_generic_inference_app_path(fvp: str, target: str) -> Path: """Return path to the generic inference runner binary.""" apps_path = get_mlia_resources() / "backends/applications" - fvp_mapping = {"Corstone-300": "300", "Corstone-310": "310"} + fvp_mapping = {"corstone-300": "300", "corstone-310": "310"} target_mapping = {"ethos-u55": "U55", "ethos-u65": "U65"} fvp_version = f"sse-{fvp_mapping[fvp]}" @@ -108,12 +108,12 @@ def get_generic_inference_app_path(fvp: str, target: str) -> Path: def get_executable_name(fvp: str, profile: str, target: str) -> str: """Return name of the executable for selected FVP and profile.""" executable_name_mapping = { - ("Corstone-300", "AVH", "ethos-u55"): "VHT_Corstone_SSE-300_Ethos-U55", - ("Corstone-300", "AVH", "ethos-u65"): "VHT_Corstone_SSE-300_Ethos-U65", - ("Corstone-300", "default", "ethos-u55"): "FVP_Corstone_SSE-300_Ethos-U55", - ("Corstone-300", "default", "ethos-u65"): "FVP_Corstone_SSE-300_Ethos-U65", - ("Corstone-310", "AVH", "ethos-u55"): "VHT_Corstone_SSE-310", - ("Corstone-310", "AVH", "ethos-u65"): "VHT_Corstone_SSE-310_Ethos-U65", + ("corstone-300", "AVH", "ethos-u55"): "VHT_Corstone_SSE-300_Ethos-U55", + ("corstone-300", "AVH", "ethos-u65"): "VHT_Corstone_SSE-300_Ethos-U65", + ("corstone-300", "default", "ethos-u55"): "FVP_Corstone_SSE-300_Ethos-U55", + ("corstone-300", "default", "ethos-u65"): "FVP_Corstone_SSE-300_Ethos-U65", + ("corstone-310", "AVH", "ethos-u55"): "VHT_Corstone_SSE-310", + ("corstone-310", "AVH", "ethos-u65"): "VHT_Corstone_SSE-310_Ethos-U65", } return executable_name_mapping[(fvp, profile, target)] diff --git a/src/mlia/backend/tosa_checker/__init__.py b/src/mlia/backend/tosa_checker/__init__.py index e11034f..d2364cb 100644 --- a/src/mlia/backend/tosa_checker/__init__.py +++ b/src/mlia/backend/tosa_checker/__init__.py @@ -14,4 +14,5 @@ registry.register( supported_systems=[System.LINUX_AMD64], backend_type=BackendType.WHEEL, ), + pretty_name="TOSA Checker", ) diff --git a/src/mlia/backend/vela/__init__.py b/src/mlia/backend/vela/__init__.py index 68fbcba..7325630 100644 --- a/src/mlia/backend/vela/__init__.py +++ b/src/mlia/backend/vela/__init__.py @@ -8,7 +8,7 @@ from mlia.backend.registry import registry from mlia.core.common import AdviceCategory registry.register( - "Vela", + "vela", BackendConfiguration( supported_advice=[ AdviceCategory.COMPATIBILITY, @@ -23,4 +23,5 @@ registry.register( ], backend_type=BackendType.BUILTIN, ), + pretty_name="Vela", ) diff --git a/src/mlia/cli/main.py b/src/mlia/cli/main.py index b3a9d4c..cc97494 100644 --- a/src/mlia/cli/main.py +++ b/src/mlia/cli/main.py @@ -84,7 +84,7 @@ def get_commands() -> list[CommandInfo]: ), partial( add_backend_options, - backends_to_skip=["tosa-checker", "ArmNNTFLiteDelegate"], + backends_to_skip=["tosa-checker", "armnn-tflite-delegate"], ), add_multi_optimization_options, add_output_options, @@ -197,12 +197,9 @@ def run_command(args: argparse.Namespace) -> int: try: logger.info("ML Inference Advisor %s", __version__) - logger.info( - "\nThis execution of MLIA uses output directory: %s", ctx.output_dir - ) if copy_profile_file(ctx, func_args): logger.info( - "Target profile information copied to %s/target_profile.toml", + "\nThe target profile (.toml) is copied to the output directory: %s", ctx.output_dir, ) args.func(**func_args) diff --git a/src/mlia/target/cortex_a/__init__.py b/src/mlia/target/cortex_a/__init__.py index 87f268a..729dd30 100644 --- a/src/mlia/target/cortex_a/__init__.py +++ b/src/mlia/target/cortex_a/__init__.py @@ -9,9 +9,10 @@ from mlia.target.registry import TargetInfo registry.register( "cortex-a", TargetInfo( - supported_backends=["ArmNNTFLiteDelegate"], - default_backends=["ArmNNTFLiteDelegate"], + supported_backends=["armnn-tflite-delegate"], + default_backends=["armnn-tflite-delegate"], advisor_factory_func=configure_and_get_cortexa_advisor, target_profile_cls=CortexAConfiguration, ), + pretty_name="Cortex-A", ) diff --git a/src/mlia/target/cortex_a/events.py b/src/mlia/target/cortex_a/events.py index 76f17ba..675335d 100644 --- a/src/mlia/target/cortex_a/events.py +++ b/src/mlia/target/cortex_a/events.py @@ -14,7 +14,7 @@ class CortexAAdvisorStartedEvent(Event): """Event with Cortex-A advisor parameters.""" model: Path - target: CortexAConfiguration + target_config: CortexAConfiguration class CortexAAdvisorEventHandler(EventDispatcher): diff --git a/src/mlia/target/cortex_a/handlers.py b/src/mlia/target/cortex_a/handlers.py index d46197c..a952c39 100644 --- a/src/mlia/target/cortex_a/handlers.py +++ b/src/mlia/target/cortex_a/handlers.py @@ -35,4 +35,4 @@ class CortexAEventHandler(WorkflowEventsHandler, CortexAAdvisorEventHandler): def on_cortex_a_advisor_started(self, event: CortexAAdvisorStartedEvent) -> None: """Handle CortexAAdvisorStarted event.""" - self.reporter.submit(event.target) + self.reporter.submit(event.target_config) diff --git a/src/mlia/target/cortex_a/reporters.py b/src/mlia/target/cortex_a/reporters.py index e23bf4d..d214b09 100644 --- a/src/mlia/target/cortex_a/reporters.py +++ b/src/mlia/target/cortex_a/reporters.py @@ -23,13 +23,13 @@ from mlia.utils.console import style_improvement from mlia.utils.types import is_list_of -def report_target(target: CortexAConfiguration) -> Report: +def report_target(target_config: CortexAConfiguration) -> Report: """Generate report for the target.""" return NestedReport( "Target information", "target", [ - ReportItem("Target", alias="target", value=target.target), + ReportItem("Target", alias="target", value=target_config.target), ], ) diff --git a/src/mlia/target/ethos_u/__init__.py b/src/mlia/target/ethos_u/__init__.py index 6b6777d..7fa8af1 100644 --- a/src/mlia/target/ethos_u/__init__.py +++ b/src/mlia/target/ethos_u/__init__.py @@ -8,16 +8,20 @@ from mlia.target.ethos_u.config import get_default_ethos_u_backends from mlia.target.registry import registry from mlia.target.registry import TargetInfo -SUPPORTED_BACKENDS_PRIORITY = ["Vela", *CORSTONE_PRIORITY] +SUPPORTED_BACKENDS_PRIORITY = [ + "vela", + *(corstone.lower() for corstone in CORSTONE_PRIORITY), +] -for ethos_u in ("ethos-u55", "ethos-u65"): +for name in ("Ethos-U55", "Ethos-U65"): registry.register( - ethos_u, + name.lower(), TargetInfo( supported_backends=SUPPORTED_BACKENDS_PRIORITY, default_backends=get_default_ethos_u_backends(SUPPORTED_BACKENDS_PRIORITY), advisor_factory_func=configure_and_get_ethosu_advisor, target_profile_cls=EthosUConfiguration, ), + pretty_name=name, ) diff --git a/src/mlia/target/ethos_u/advisor.py b/src/mlia/target/ethos_u/advisor.py index b34d1e0..0eec6aa 100644 --- a/src/mlia/target/ethos_u/advisor.py +++ b/src/mlia/target/ethos_u/advisor.py @@ -41,13 +41,13 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor): def get_collectors(self, context: Context) -> list[DataCollector]: """Return list of the data collectors.""" model = self.get_model(context) - target = self._get_target_cfg(context) + target_config = self._get_target_config(context) backends = self._get_backends(context) collectors: list[DataCollector] = [] if context.category_enabled(AdviceCategory.COMPATIBILITY): - collectors.append(EthosUOperatorCompatibility(model, target)) + collectors.append(EthosUOperatorCompatibility(model, target_config)) # Performance and optimization are mutually exclusive. # Decide which one to use (taking into account the model format). @@ -58,18 +58,18 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor): "Command 'optimization' is not supported for TensorFlow Lite files." ) if context.category_enabled(AdviceCategory.PERFORMANCE): - collectors.append(EthosUPerformance(model, target, backends)) + collectors.append(EthosUPerformance(model, target_config, backends)) else: # Keras/SavedModel: Prefer optimization if context.category_enabled(AdviceCategory.OPTIMIZATION): optimization_settings = self._get_optimization_settings(context) collectors.append( EthosUOptimizationPerformance( - model, target, optimization_settings, backends + model, target_config, optimization_settings, backends ) ) elif context.category_enabled(AdviceCategory.PERFORMANCE): - collectors.append(EthosUPerformance(model, target, backends)) + collectors.append(EthosUPerformance(model, target_config, backends)) return collectors @@ -89,13 +89,13 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor): def get_events(self, context: Context) -> list[Event]: """Return list of the startup events.""" model = self.get_model(context) - target = self._get_target_cfg(context) + target_config = self._get_target_config(context) return [ - EthosUAdvisorStartedEvent(target=target, model=model), + EthosUAdvisorStartedEvent(target_config=target_config, model=model), ] - def _get_target_cfg(self, context: Context) -> EthosUConfiguration: + def _get_target_config(self, context: Context) -> EthosUConfiguration: """Get target configuration.""" target_profile = self.get_target_profile(context) return cast(EthosUConfiguration, profile(target_profile)) diff --git a/src/mlia/target/ethos_u/data_collection.py b/src/mlia/target/ethos_u/data_collection.py index 96fe240..4fdfe96 100644 --- a/src/mlia/target/ethos_u/data_collection.py +++ b/src/mlia/target/ethos_u/data_collection.py @@ -31,10 +31,10 @@ logger = logging.getLogger(__name__) class EthosUOperatorCompatibility(ContextAwareDataCollector): """Collect operator compatibility information.""" - def __init__(self, model: Path, target: EthosUConfiguration) -> None: + def __init__(self, model: Path, target_config: EthosUConfiguration) -> None: """Init operator compatibility data collector.""" self.model = model - self.target = target + self.target_config = target_config def collect_data(self) -> Operators: """Collect operator compatibility information.""" @@ -42,7 +42,7 @@ class EthosUOperatorCompatibility(ContextAwareDataCollector): with log_action("Checking operator compatibility ..."): return supported_operators( - Path(tflite_model.model_path), self.target.compiler_options + Path(tflite_model.model_path), self.target_config.compiler_options ) @classmethod @@ -57,12 +57,12 @@ class EthosUPerformance(ContextAwareDataCollector): def __init__( self, model: Path, - target: EthosUConfiguration, + target_config: EthosUConfiguration, backends: list[str] | None = None, ) -> None: """Init performance data collector.""" self.model = model - self.target = target + self.target_config = target_config self.backends = backends def collect_data(self) -> PerformanceMetrics: @@ -70,7 +70,7 @@ class EthosUPerformance(ContextAwareDataCollector): tflite_model = get_tflite_model(self.model, self.context) estimator = EthosUPerformanceEstimator( self.context, - self.target, + self.target_config, self.backends, ) @@ -113,13 +113,13 @@ class EthosUOptimizationPerformance(ContextAwareDataCollector): def __init__( self, model: Path, - target: EthosUConfiguration, + target_config: EthosUConfiguration, optimizations: list[list[dict]], backends: list[str] | None = None, ) -> None: """Init performance optimizations data collector.""" self.model = model - self.target = target + self.target = target_config self.optimizations = optimizations self.backends = backends diff --git a/src/mlia/target/ethos_u/events.py b/src/mlia/target/ethos_u/events.py index 8060382..17e5e7f 100644 --- a/src/mlia/target/ethos_u/events.py +++ b/src/mlia/target/ethos_u/events.py @@ -14,7 +14,7 @@ class EthosUAdvisorStartedEvent(Event): """Event with Ethos-U advisor parameters.""" model: Path - target: EthosUConfiguration + target_config: EthosUConfiguration class EthosUAdvisorEventHandler(EventDispatcher): diff --git a/src/mlia/target/ethos_u/handlers.py b/src/mlia/target/ethos_u/handlers.py index c9d0dc7..b9c89e8 100644 --- a/src/mlia/target/ethos_u/handlers.py +++ b/src/mlia/target/ethos_u/handlers.py @@ -51,4 +51,4 @@ class EthosUEventHandler(WorkflowEventsHandler, EthosUAdvisorEventHandler): def on_ethos_u_advisor_started(self, event: EthosUAdvisorStartedEvent) -> None: """Handle EthosUAdvisorStarted event.""" - self.reporter.submit(event.target) + self.reporter.submit(event.target_config) diff --git a/src/mlia/target/ethos_u/performance.py b/src/mlia/target/ethos_u/performance.py index 5bcafab..f7f9a8c 100644 --- a/src/mlia/target/ethos_u/performance.py +++ b/src/mlia/target/ethos_u/performance.py @@ -92,17 +92,19 @@ class MemoryUsage: class PerformanceMetrics: """Performance metrics.""" - target: EthosUConfiguration + target_config: EthosUConfiguration npu_cycles: NPUCycles | None memory_usage: MemoryUsage | None def in_kilobytes(self) -> PerformanceMetrics: """Return metrics with memory usage in KiB.""" if self.memory_usage is None: - return PerformanceMetrics(self.target, self.npu_cycles, self.memory_usage) + return PerformanceMetrics( + self.target_config, self.npu_cycles, self.memory_usage + ) return PerformanceMetrics( - self.target, self.npu_cycles, self.memory_usage.in_kilobytes() + self.target_config, self.npu_cycles, self.memory_usage.in_kilobytes() ) @@ -121,10 +123,10 @@ class VelaPerformanceEstimator( ): """Vela based performance estimator.""" - def __init__(self, context: Context, target: EthosUConfiguration) -> None: + def __init__(self, context: Context, target_config: EthosUConfiguration) -> None: """Init Vela based performance estimator.""" self.context = context - self.target = target + self.target = target_config def estimate(self, model: Path | ModelConfiguration) -> MemoryUsage: """Estimate performance.""" @@ -154,11 +156,11 @@ class CorstonePerformanceEstimator( """Corstone-based performance estimator.""" def __init__( - self, context: Context, target: EthosUConfiguration, backend: str + self, context: Context, target_config: EthosUConfiguration, backend: str ) -> None: """Init Corstone-based performance estimator.""" self.context = context - self.target = target + self.target_config = target_config self.backend = backend def estimate(self, model: Path | ModelConfiguration) -> NPUCycles: @@ -180,12 +182,12 @@ class CorstonePerformanceEstimator( ) vela_comp.optimize_model( - model_path, self.target.compiler_options, optimized_model_path + model_path, self.target_config.compiler_options, optimized_model_path ) corstone_perf_metrics = estimate_performance( - self.target.target, - self.target.mac, + self.target_config.target, + self.target_config.mac, optimized_model_path, self.backend, ) @@ -208,17 +210,17 @@ class EthosUPerformanceEstimator( def __init__( self, context: Context, - target: EthosUConfiguration, + target_config: EthosUConfiguration, backends: list[str] | None = None, ) -> None: """Init performance estimator.""" self.context = context - self.target = target + self.target_config = target_config if backends is None: - backends = ["Vela"] # Only Vela is always available as default - ethos_u_backends = supported_backends(target.target) + backends = ["vela"] # Only Vela is always available as default + ethos_u_backends = supported_backends(target_config.target) for backend in backends: - if backend != "Vela" and backend not in ethos_u_backends: + if backend != "vela" and backend not in ethos_u_backends: raise ValueError( f"Unsupported backend '{backend}'. " f"Only 'Vela' and {ethos_u_backends} " @@ -237,12 +239,14 @@ class EthosUPerformanceEstimator( memory_usage = None npu_cycles = None for backend in self.backends: - if backend == "Vela": - vela_estimator = VelaPerformanceEstimator(self.context, self.target) + if backend == "vela": + vela_estimator = VelaPerformanceEstimator( + self.context, self.target_config + ) memory_usage = vela_estimator.estimate(tflite_model) elif is_corstone_backend(backend): corstone_estimator = CorstonePerformanceEstimator( - self.context, self.target, backend + self.context, self.target_config, backend ) npu_cycles = corstone_estimator.estimate(tflite_model) else: @@ -252,4 +256,4 @@ class EthosUPerformanceEstimator( backend, ) - return PerformanceMetrics(self.target, npu_cycles, memory_usage) + return PerformanceMetrics(self.target_config, npu_cycles, memory_usage) diff --git a/src/mlia/target/ethos_u/reporters.py b/src/mlia/target/ethos_u/reporters.py index 4de60bb..2a5b5d3 100644 --- a/src/mlia/target/ethos_u/reporters.py +++ b/src/mlia/target/ethos_u/reporters.py @@ -110,9 +110,9 @@ def report_operators(ops: list[Operator]) -> Report: return Table(columns, rows, name="Operators", alias="operators") -def report_target_details(target: EthosUConfiguration) -> Report: +def report_target_details(target_config: EthosUConfiguration) -> Report: """Return table representation for the target.""" - compiler_config = target.resolved_compiler_config + compiler_config = target_config.resolved_compiler_config memory_settings = [ ReportItem( @@ -211,8 +211,8 @@ def report_target_details(target: EthosUConfiguration) -> Report: "Target information", "target", [ - ReportItem("Target", alias="target", value=target.target), - ReportItem("MAC", alias="mac", value=target.mac), + ReportItem("Target", alias="target", value=target_config.target), + ReportItem("MAC", alias="mac", value=target_config.mac), ReportItem( "Memory mode", alias="memory_mode", diff --git a/src/mlia/target/registry.py b/src/mlia/target/registry.py index 9fccecb..b7b6193 100644 --- a/src/mlia/target/registry.py +++ b/src/mlia/target/registry.py @@ -25,12 +25,14 @@ from mlia.utils.registry import Registry class TargetRegistry(Registry[TargetInfo]): """Registry for targets.""" - def register(self, name: str, item: TargetInfo) -> bool: + def register( + self, name: str, item: TargetInfo, pretty_name: str | None = None + ) -> bool: """Register an item: returns `False` if already registered.""" assert all( backend in backend_registry.items for backend in item.supported_backends ) - return super().register(name, item) + return super().register(name, item, pretty_name) # All supported targets are required to be registered here. @@ -159,15 +161,12 @@ def table() -> Table: rows = [ ( - name, - Table( - columns=[Column("Backend"), Column("Status")], - rows=[ - (backend, get_status(backend)) - for backend in info.supported_backends - ], - name="Backends", + f"{registry.pretty_name(name)}\n<{name}>", + "\n".join( + f"{backend_registry.pretty_name(backend)}\n<{backend}>" + for backend in info.supported_backends ), + "\n\n".join(get_status(backend) for backend in info.supported_backends), "/".join(get_advice(name)), ) for name, info in registry.items.items() @@ -177,6 +176,7 @@ def table() -> Table: columns=[ Column("Target"), Column("Backend(s)"), + Column("Status"), Column("Advice: comp/perf/opt"), ], rows=rows, diff --git a/src/mlia/target/tosa/__init__.py b/src/mlia/target/tosa/__init__.py index 3830ce5..12077bd 100644 --- a/src/mlia/target/tosa/__init__.py +++ b/src/mlia/target/tosa/__init__.py @@ -14,4 +14,5 @@ registry.register( advisor_factory_func=configure_and_get_tosa_advisor, target_profile_cls=TOSAConfiguration, ), + pretty_name="TOSA", ) diff --git a/src/mlia/target/tosa/reporters.py b/src/mlia/target/tosa/reporters.py index 5c015ff..9575978 100644 --- a/src/mlia/target/tosa/reporters.py +++ b/src/mlia/target/tosa/reporters.py @@ -42,13 +42,13 @@ class MetadataDisplay: # pylint: disable=too-few-public-methods self.model_name = model_meta.model_name -def report_target(target: TOSAConfiguration) -> Report: +def report_target(target_config: TOSAConfiguration) -> Report: """Generate report for the target.""" return NestedReport( "Target information", "target", [ - ReportItem("Target", alias="target", value=target.target), + ReportItem("Target", alias="target", value=target_config.target), ], ) diff --git a/src/mlia/utils/registry.py b/src/mlia/utils/registry.py index 9b25a81..a886376 100644 --- a/src/mlia/utils/registry.py +++ b/src/mlia/utils/registry.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Generic registry class.""" from __future__ import annotations @@ -15,17 +15,25 @@ class Registry(Generic[T]): def __init__(self) -> None: """Create an empty registry.""" self.items: dict[str, T] = {} + self.pretty_names: dict[str, str] = {} def __str__(self) -> str: """List all registered items.""" return "\n".join( - f"- {name}: {item}" + f"- {self.pretty_names[name] if name in self.pretty_names else name}: " + f"{item}" for name, item in sorted(self.items.items(), key=lambda v: v[0]) ) - def register(self, name: str, item: T) -> bool: + def register(self, name: str, item: T, pretty_name: str | None = None) -> bool: """Register an item: returns `False` if already registered.""" if name in self.items: return False # already registered self.items[name] = item + if pretty_name: + self.pretty_names[name] = pretty_name return True + + def pretty_name(self, name: str) -> str: + """Get the pretty name (if available) or return the name as is otherwise.""" + return self.pretty_names[name] if name in self.pretty_names else name diff --git a/tests/conftest.py b/tests/conftest.py index 55b296f..30889ca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -84,9 +84,11 @@ def fixture_test_models_path( tflite_vela_model = tmp_path / "test_model_vela.tflite" - target_profile = EthosUConfiguration.load_profile("ethos-u55-256") + target_config = EthosUConfiguration.load_profile("ethos-u55-256") optimize_model( - tflite_model_path, target_profile.compiler_options, tflite_vela_model + tflite_model_path, + target_config.compiler_options, + tflite_vela_model, ) tf.saved_model.save(keras_model, str(tmp_path / "tf_model_test_model")) diff --git a/tests/test_backend_corstone.py b/tests/test_backend_corstone.py index 29ef084..190931a 100644 --- a/tests/test_backend_corstone.py +++ b/tests/test_backend_corstone.py @@ -6,6 +6,6 @@ from mlia.backend.corstone import is_corstone_backend def test_is_corstone_backend() -> None: """Test function is_corstone_backend.""" - assert is_corstone_backend("Corstone-300") is True - assert is_corstone_backend("Corstone-310") is True + assert is_corstone_backend("corstone-300") is True + assert is_corstone_backend("corstone-310") is True assert is_corstone_backend("New backend") is False diff --git a/tests/test_backend_corstone_performance.py b/tests/test_backend_corstone_performance.py index 2d5b196..d3d378e 100644 --- a/tests/test_backend_corstone_performance.py +++ b/tests/test_backend_corstone_performance.py @@ -81,7 +81,7 @@ def test_generic_inference_output_parser_failure(wrong_fvp_output: list[str]) -> [ [ Path("backend_path"), - "Corstone-300", + "corstone-300", "ethos-u55", 256, Path("model.tflite"), @@ -163,7 +163,7 @@ def test_estimate_performance(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr("mlia.utils.proc.command_output", command_output_mock) result = estimate_performance( - "ethos-u55", 256, Path("model.tflite"), "Corstone-300" + "ethos-u55", 256, Path("model.tflite"), "corstone-300" ) assert result == PerformanceMetrics(1, 2, 3, 4, 5, 6) diff --git a/tests/test_backend_registry.py b/tests/test_backend_registry.py index e3e2da2..cc05632 100644 --- a/tests/test_backend_registry.py +++ b/tests/test_backend_registry.py @@ -19,19 +19,19 @@ from mlia.core.common import AdviceCategory ("backend", "advices", "systems", "type_"), ( ( - "ArmNNTFLiteDelegate", + "armnn-tflite-delegate", [AdviceCategory.COMPATIBILITY], None, BackendType.BUILTIN, ), ( - "Corstone-300", + "corstone-300", [AdviceCategory.PERFORMANCE, AdviceCategory.OPTIMIZATION], [System.LINUX_AMD64], BackendType.CUSTOM, ), ( - "Corstone-310", + "corstone-310", [AdviceCategory.PERFORMANCE, AdviceCategory.OPTIMIZATION], [System.LINUX_AMD64], BackendType.CUSTOM, @@ -43,7 +43,7 @@ from mlia.core.common import AdviceCategory BackendType.WHEEL, ), ( - "Vela", + "vela", [ AdviceCategory.COMPATIBILITY, AdviceCategory.PERFORMANCE, @@ -86,11 +86,11 @@ def test_backend_registry( def test_get_supported_backends() -> None: """Test function get_supported_backends.""" assert get_supported_backends() == [ - "ArmNNTFLiteDelegate", - "Corstone-300", - "Corstone-310", - "Vela", + "armnn-tflite-delegate", + "corstone-300", + "corstone-310", "tosa-checker", + "vela", ] diff --git a/tests/test_backend_vela_compat.py b/tests/test_backend_vela_compat.py index 4e0f149..0c39dd6 100644 --- a/tests/test_backend_vela_compat.py +++ b/tests/test_backend_vela_compat.py @@ -55,9 +55,11 @@ from mlia.utils.filesystem import working_directory ) def test_operators(test_models_path: Path, model: str, expected_ops: Operators) -> None: """Test operators function.""" - target = EthosUConfiguration.load_profile("ethos-u55-256") + target_config = EthosUConfiguration.load_profile("ethos-u55-256") - operators = supported_operators(test_models_path / model, target.compiler_options) + operators = supported_operators( + test_models_path / model, target_config.compiler_options + ) for expected, actual in zip(expected_ops.ops, operators.ops): # do not compare names as they could be different on each model generation assert expected.op_type == actual.op_type diff --git a/tests/test_backend_vela_compiler.py b/tests/test_backend_vela_compiler.py index 0434ccf..9b69ada 100644 --- a/tests/test_backend_vela_compiler.py +++ b/tests/test_backend_vela_compiler.py @@ -158,8 +158,10 @@ def test_optimize_model(tmp_path: Path, test_tflite_model: Path) -> None: """Test model optimization and saving into file.""" tmp_file = tmp_path / "temp.tflite" - target = EthosUConfiguration.load_profile("ethos-u55-256") - optimize_model(test_tflite_model, target.compiler_options, tmp_file.absolute()) + target_config = EthosUConfiguration.load_profile("ethos-u55-256") + optimize_model( + test_tflite_model, target_config.compiler_options, tmp_file.absolute() + ) assert tmp_file.is_file() assert tmp_file.stat().st_size > 0 diff --git a/tests/test_backend_vela_performance.py b/tests/test_backend_vela_performance.py index 68b96ab..df2ce08 100644 --- a/tests/test_backend_vela_performance.py +++ b/tests/test_backend_vela_performance.py @@ -14,8 +14,10 @@ from mlia.target.ethos_u.config import EthosUConfiguration def test_estimate_performance(test_tflite_model: Path) -> None: """Test getting performance estimations.""" - target = EthosUConfiguration.load_profile("ethos-u55-256") - perf_metrics = estimate_performance(test_tflite_model, target.compiler_options) + target_config = EthosUConfiguration.load_profile("ethos-u55-256") + perf_metrics = estimate_performance( + test_tflite_model, target_config.compiler_options + ) assert isinstance(perf_metrics, PerformanceMetrics) @@ -24,16 +26,18 @@ def test_estimate_performance_already_optimized( tmp_path: Path, test_tflite_model: Path ) -> None: """Test that performance estimation should fail for already optimized model.""" - target = EthosUConfiguration.load_profile("ethos-u55-256") + target_config = EthosUConfiguration.load_profile("ethos-u55-256") optimized_model_path = tmp_path / "optimized_model.tflite" - optimize_model(test_tflite_model, target.compiler_options, optimized_model_path) + optimize_model( + test_tflite_model, target_config.compiler_options, optimized_model_path + ) with pytest.raises( Exception, match="Unable to estimate performance for the given optimized model" ): - estimate_performance(optimized_model_path, target.compiler_options) + estimate_performance(optimized_model_path, target_config.compiler_options) def test_read_invalid_model(test_tflite_invalid_model: Path) -> None: @@ -41,8 +45,8 @@ def test_read_invalid_model(test_tflite_invalid_model: Path) -> None: with pytest.raises( Exception, match=f"Unable to read model {test_tflite_invalid_model}" ): - target = EthosUConfiguration.load_profile("ethos-u55-256") - estimate_performance(test_tflite_invalid_model, target.compiler_options) + target_config = EthosUConfiguration.load_profile("ethos-u55-256") + estimate_performance(test_tflite_invalid_model, target_config.compiler_options) def test_compile_invalid_model( @@ -58,7 +62,7 @@ def test_compile_invalid_model( with pytest.raises( Exception, match="Model could not be optimized with Vela compiler" ): - target = EthosUConfiguration.load_profile("ethos-u55-256") - optimize_model(test_tflite_model, target.compiler_options, model_path) + target_config = EthosUConfiguration.load_profile("ethos-u55-256") + optimize_model(test_tflite_model, target_config.compiler_options, model_path) assert not model_path.exists() diff --git a/tests/test_cli_command_validators.py b/tests/test_cli_command_validators.py index 29813f4..1ac82db 100644 --- a/tests/test_cli_command_validators.py +++ b/tests/test_cli_command_validators.py @@ -109,17 +109,17 @@ def test_validate_check_target_profile( ], [ "tosa", - ["Corstone-310"], + ["corstone-310"], True, - "Backend Corstone-310 not supported with target-profile tosa.", + "Backend corstone-310 not supported with target-profile tosa.", None, ], [ "cortex-a", - ["ArmNNTFLiteDelegate"], + ["armnn-tflite-delegate"], False, None, - ["ArmNNTFLiteDelegate"], + ["armnn-tflite-delegate"], ], [ "cortex-a", @@ -130,14 +130,14 @@ def test_validate_check_target_profile( ], [ "ethos-u55-256", - ["Vela", "Corstone-310"], + ["vela", "corstone-310"], False, None, - ["Vela", "Corstone-310"], + ["vela", "corstone-310"], ], [ "ethos-u65-256", - ["Vela", "Corstone-310", "tosa-checker"], + ["vela", "corstone-310", "tosa-checker"], True, "Backend tosa-checker not supported with target-profile ethos-u65-256.", None, diff --git a/tests/test_cli_main.py b/tests/test_cli_main.py index 673031c..3472e31 100644 --- a/tests/test_cli_main.py +++ b/tests/test_cli_main.py @@ -239,7 +239,7 @@ def test_commands_execution( monkeypatch.setattr( "mlia.cli.options.get_available_backends", - MagicMock(return_value=["Vela", "some_backend"]), + MagicMock(return_value=["vela", "some_backend"]), ) for command in ["check", "optimize"]: diff --git a/tests/test_target_registry.py b/tests/test_target_registry.py index 2cbd97d..ca1ad82 100644 --- a/tests/test_target_registry.py +++ b/tests/test_target_registry.py @@ -63,13 +63,13 @@ def test_supported_advice( @pytest.mark.parametrize( ("backend", "target", "expected_result"), ( - ("ArmNNTFLiteDelegate", None, True), - ("ArmNNTFLiteDelegate", "cortex-a", True), - ("ArmNNTFLiteDelegate", "tosa", False), - ("Corstone-310", None, True), - ("Corstone-310", "ethos-u55", True), - ("Corstone-310", "ethos-u65", True), - ("Corstone-310", "cortex-a", False), + ("armnn-tflite-delegate", None, True), + ("armnn-tflite-delegate", "cortex-a", True), + ("armnn-tflite-delegate", "tosa", False), + ("corstone-310", None, True), + ("corstone-310", "ethos-u55", True), + ("corstone-310", "ethos-u65", True), + ("corstone-310", "cortex-a", False), ), ) def test_is_supported(backend: str, target: str | None, expected_result: bool) -> None: @@ -80,9 +80,9 @@ def test_is_supported(backend: str, target: str | None, expected_result: bool) - @pytest.mark.parametrize( ("target_name", "expected_backends"), ( - ("cortex-a", ["ArmNNTFLiteDelegate"]), - ("ethos-u55", ["Corstone-300", "Corstone-310", "Vela"]), - ("ethos-u65", ["Corstone-300", "Corstone-310", "Vela"]), + ("cortex-a", ["armnn-tflite-delegate"]), + ("ethos-u55", ["corstone-300", "corstone-310", "vela"]), + ("ethos-u65", ["corstone-300", "corstone-310", "vela"]), ("tosa", ["tosa-checker"]), ), ) @@ -107,21 +107,21 @@ def test_supported_targets(advice: AdviceCategory, expected_targets: list[str]) def test_all_supported_backends() -> None: """Test function all_supported_backends.""" assert all_supported_backends() == { - "Vela", + "vela", "tosa-checker", - "ArmNNTFLiteDelegate", - "Corstone-310", - "Corstone-300", + "armnn-tflite-delegate", + "corstone-310", + "corstone-300", } @pytest.mark.parametrize( ("target", "expected_default_backends", "is_subset_only"), [ - ["cortex-a", ["ArmNNTFLiteDelegate"], False], + ["cortex-a", ["armnn-tflite-delegate"], False], ["tosa", ["tosa-checker"], False], - ["ethos-u55", ["Vela"], True], - ["ethos-u65", ["Vela"], True], + ["ethos-u55", ["vela"], True], + ["ethos-u65", ["vela"], True], ], ) def test_default_backends( -- cgit v1.2.1