aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBenjamin Klimczak <benjamin.klimczak@arm.com>2023-02-10 13:12:57 +0000
committerBenjamin Klimczak <benjamin.klimczak@arm.com>2023-02-10 14:56:43 +0000
commit718277eaece76902c4950f18d428907b39a18ef1 (patch)
tree6fda1b8f8c2c0151f3697a2e9f31c2f5551cc2ff
parentfa1fad9332e2912f12a44a1b07716ee434174308 (diff)
downloadmlia-718277eaece76902c4950f18d428907b39a18ef1.tar.gz
MLIA-769 Add "pretty names" for targets / backends
- Provide "pretty names" to print information for targets and backends. - Use 'target_config' instead of 'target' if a target profile is used. - Fix minor issue in output regarding the output directory. Change-Id: Ib38231f30b4d609a0d1e8f9c52b2fb547c69cb6a
-rw-r--r--README.md4
-rw-r--r--src/mlia/backend/armnn_tflite_delegate/__init__.py4
-rw-r--r--src/mlia/backend/corstone/__init__.py8
-rw-r--r--src/mlia/backend/corstone/install.py4
-rw-r--r--src/mlia/backend/corstone/performance.py14
-rw-r--r--src/mlia/backend/tosa_checker/__init__.py1
-rw-r--r--src/mlia/backend/vela/__init__.py3
-rw-r--r--src/mlia/cli/main.py7
-rw-r--r--src/mlia/target/cortex_a/__init__.py5
-rw-r--r--src/mlia/target/cortex_a/events.py2
-rw-r--r--src/mlia/target/cortex_a/handlers.py2
-rw-r--r--src/mlia/target/cortex_a/reporters.py4
-rw-r--r--src/mlia/target/ethos_u/__init__.py10
-rw-r--r--src/mlia/target/ethos_u/advisor.py16
-rw-r--r--src/mlia/target/ethos_u/data_collection.py16
-rw-r--r--src/mlia/target/ethos_u/events.py2
-rw-r--r--src/mlia/target/ethos_u/handlers.py2
-rw-r--r--src/mlia/target/ethos_u/performance.py42
-rw-r--r--src/mlia/target/ethos_u/reporters.py8
-rw-r--r--src/mlia/target/registry.py20
-rw-r--r--src/mlia/target/tosa/__init__.py1
-rw-r--r--src/mlia/target/tosa/reporters.py4
-rw-r--r--src/mlia/utils/registry.py14
-rw-r--r--tests/conftest.py6
-rw-r--r--tests/test_backend_corstone.py4
-rw-r--r--tests/test_backend_corstone_performance.py4
-rw-r--r--tests/test_backend_registry.py16
-rw-r--r--tests/test_backend_vela_compat.py6
-rw-r--r--tests/test_backend_vela_compiler.py6
-rw-r--r--tests/test_backend_vela_performance.py22
-rw-r--r--tests/test_cli_command_validators.py14
-rw-r--r--tests/test_cli_main.py2
-rw-r--r--tests/test_target_registry.py34
33 files changed, 170 insertions, 137 deletions
diff --git a/README.md b/README.md
index 37ba6fc..1abe0d0 100644
--- a/README.md
+++ b/README.md
@@ -149,8 +149,8 @@ mlia check ~/models/mobilenet_v1_1.0_224_quant.tflite \
mlia check ~/models/ds_cnn_large_fully_quantized_int8.tflite \
--target-profile ethos-u65-512 \
--performance \
- --backend "Vela" \
- --backend "Corstone-300"
+ --backend "vela" \
+ --backend "corstone-300"
# Get help and further information
mlia check --help
diff --git a/src/mlia/backend/armnn_tflite_delegate/__init__.py b/src/mlia/backend/armnn_tflite_delegate/__init__.py
index ccb7e38..4fe8639 100644
--- a/src/mlia/backend/armnn_tflite_delegate/__init__.py
+++ b/src/mlia/backend/armnn_tflite_delegate/__init__.py
@@ -1,16 +1,18 @@
# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
"""Arm NN TensorFlow Lite delegate backend module."""
+from mlia.backend.armnn_tflite_delegate.compat import ARMNN_TFLITE_DELEGATE
from mlia.backend.config import BackendConfiguration
from mlia.backend.config import BackendType
from mlia.backend.registry import registry
from mlia.core.common import AdviceCategory
registry.register(
- "ArmNNTFLiteDelegate",
+ "armnn-tflite-delegate",
BackendConfiguration(
supported_advice=[AdviceCategory.COMPATIBILITY],
supported_systems=None,
backend_type=BackendType.BUILTIN,
),
+ pretty_name=ARMNN_TFLITE_DELEGATE["metadata"]["backend"],
)
diff --git a/src/mlia/backend/corstone/__init__.py b/src/mlia/backend/corstone/__init__.py
index b59ab65..5aa688b 100644
--- a/src/mlia/backend/corstone/__init__.py
+++ b/src/mlia/backend/corstone/__init__.py
@@ -12,15 +12,19 @@ CORSTONE_PRIORITY = ("Corstone-310", "Corstone-300")
for corstone_name in CORSTONE_PRIORITY:
registry.register(
- corstone_name,
+ corstone_name.lower(),
BackendConfiguration(
supported_advice=[AdviceCategory.PERFORMANCE, AdviceCategory.OPTIMIZATION],
supported_systems=[System.LINUX_AMD64],
backend_type=BackendType.CUSTOM,
),
+ pretty_name=corstone_name,
)
def is_corstone_backend(backend_name: str) -> bool:
"""Check if backend belongs to Corstone."""
- return backend_name in CORSTONE_PRIORITY
+ return any(
+ name in CORSTONE_PRIORITY
+ for name in (backend_name, registry.pretty_name(backend_name))
+ )
diff --git a/src/mlia/backend/corstone/install.py b/src/mlia/backend/corstone/install.py
index c57a47b..35976cf 100644
--- a/src/mlia/backend/corstone/install.py
+++ b/src/mlia/backend/corstone/install.py
@@ -62,7 +62,7 @@ def get_corstone_300_installation() -> Installation:
"""Get Corstone-300 installation."""
corstone_300 = BackendInstallation(
# pylint: disable=line-too-long
- name="Corstone-300",
+ name="corstone-300",
description="Corstone-300 FVP",
fvp_dir_name="corstone_300",
download_artifact=DownloadArtifact(
@@ -102,7 +102,7 @@ def get_corstone_300_installation() -> Installation:
def get_corstone_310_installation() -> Installation:
"""Get Corstone-310 installation."""
corstone_310 = BackendInstallation(
- name="Corstone-310",
+ name="corstone-310",
description="Corstone-310 FVP",
fvp_dir_name="corstone_310",
download_artifact=None,
diff --git a/src/mlia/backend/corstone/performance.py b/src/mlia/backend/corstone/performance.py
index 8fd3e40..5012821 100644
--- a/src/mlia/backend/corstone/performance.py
+++ b/src/mlia/backend/corstone/performance.py
@@ -95,7 +95,7 @@ def get_generic_inference_app_path(fvp: str, target: str) -> Path:
"""Return path to the generic inference runner binary."""
apps_path = get_mlia_resources() / "backends/applications"
- fvp_mapping = {"Corstone-300": "300", "Corstone-310": "310"}
+ fvp_mapping = {"corstone-300": "300", "corstone-310": "310"}
target_mapping = {"ethos-u55": "U55", "ethos-u65": "U65"}
fvp_version = f"sse-{fvp_mapping[fvp]}"
@@ -108,12 +108,12 @@ def get_generic_inference_app_path(fvp: str, target: str) -> Path:
def get_executable_name(fvp: str, profile: str, target: str) -> str:
"""Return name of the executable for selected FVP and profile."""
executable_name_mapping = {
- ("Corstone-300", "AVH", "ethos-u55"): "VHT_Corstone_SSE-300_Ethos-U55",
- ("Corstone-300", "AVH", "ethos-u65"): "VHT_Corstone_SSE-300_Ethos-U65",
- ("Corstone-300", "default", "ethos-u55"): "FVP_Corstone_SSE-300_Ethos-U55",
- ("Corstone-300", "default", "ethos-u65"): "FVP_Corstone_SSE-300_Ethos-U65",
- ("Corstone-310", "AVH", "ethos-u55"): "VHT_Corstone_SSE-310",
- ("Corstone-310", "AVH", "ethos-u65"): "VHT_Corstone_SSE-310_Ethos-U65",
+ ("corstone-300", "AVH", "ethos-u55"): "VHT_Corstone_SSE-300_Ethos-U55",
+ ("corstone-300", "AVH", "ethos-u65"): "VHT_Corstone_SSE-300_Ethos-U65",
+ ("corstone-300", "default", "ethos-u55"): "FVP_Corstone_SSE-300_Ethos-U55",
+ ("corstone-300", "default", "ethos-u65"): "FVP_Corstone_SSE-300_Ethos-U65",
+ ("corstone-310", "AVH", "ethos-u55"): "VHT_Corstone_SSE-310",
+ ("corstone-310", "AVH", "ethos-u65"): "VHT_Corstone_SSE-310_Ethos-U65",
}
return executable_name_mapping[(fvp, profile, target)]
diff --git a/src/mlia/backend/tosa_checker/__init__.py b/src/mlia/backend/tosa_checker/__init__.py
index e11034f..d2364cb 100644
--- a/src/mlia/backend/tosa_checker/__init__.py
+++ b/src/mlia/backend/tosa_checker/__init__.py
@@ -14,4 +14,5 @@ registry.register(
supported_systems=[System.LINUX_AMD64],
backend_type=BackendType.WHEEL,
),
+ pretty_name="TOSA Checker",
)
diff --git a/src/mlia/backend/vela/__init__.py b/src/mlia/backend/vela/__init__.py
index 68fbcba..7325630 100644
--- a/src/mlia/backend/vela/__init__.py
+++ b/src/mlia/backend/vela/__init__.py
@@ -8,7 +8,7 @@ from mlia.backend.registry import registry
from mlia.core.common import AdviceCategory
registry.register(
- "Vela",
+ "vela",
BackendConfiguration(
supported_advice=[
AdviceCategory.COMPATIBILITY,
@@ -23,4 +23,5 @@ registry.register(
],
backend_type=BackendType.BUILTIN,
),
+ pretty_name="Vela",
)
diff --git a/src/mlia/cli/main.py b/src/mlia/cli/main.py
index b3a9d4c..cc97494 100644
--- a/src/mlia/cli/main.py
+++ b/src/mlia/cli/main.py
@@ -84,7 +84,7 @@ def get_commands() -> list[CommandInfo]:
),
partial(
add_backend_options,
- backends_to_skip=["tosa-checker", "ArmNNTFLiteDelegate"],
+ backends_to_skip=["tosa-checker", "armnn-tflite-delegate"],
),
add_multi_optimization_options,
add_output_options,
@@ -197,12 +197,9 @@ def run_command(args: argparse.Namespace) -> int:
try:
logger.info("ML Inference Advisor %s", __version__)
- logger.info(
- "\nThis execution of MLIA uses output directory: %s", ctx.output_dir
- )
if copy_profile_file(ctx, func_args):
logger.info(
- "Target profile information copied to %s/target_profile.toml",
+ "\nThe target profile (.toml) is copied to the output directory: %s",
ctx.output_dir,
)
args.func(**func_args)
diff --git a/src/mlia/target/cortex_a/__init__.py b/src/mlia/target/cortex_a/__init__.py
index 87f268a..729dd30 100644
--- a/src/mlia/target/cortex_a/__init__.py
+++ b/src/mlia/target/cortex_a/__init__.py
@@ -9,9 +9,10 @@ from mlia.target.registry import TargetInfo
registry.register(
"cortex-a",
TargetInfo(
- supported_backends=["ArmNNTFLiteDelegate"],
- default_backends=["ArmNNTFLiteDelegate"],
+ supported_backends=["armnn-tflite-delegate"],
+ default_backends=["armnn-tflite-delegate"],
advisor_factory_func=configure_and_get_cortexa_advisor,
target_profile_cls=CortexAConfiguration,
),
+ pretty_name="Cortex-A",
)
diff --git a/src/mlia/target/cortex_a/events.py b/src/mlia/target/cortex_a/events.py
index 76f17ba..675335d 100644
--- a/src/mlia/target/cortex_a/events.py
+++ b/src/mlia/target/cortex_a/events.py
@@ -14,7 +14,7 @@ class CortexAAdvisorStartedEvent(Event):
"""Event with Cortex-A advisor parameters."""
model: Path
- target: CortexAConfiguration
+ target_config: CortexAConfiguration
class CortexAAdvisorEventHandler(EventDispatcher):
diff --git a/src/mlia/target/cortex_a/handlers.py b/src/mlia/target/cortex_a/handlers.py
index d46197c..a952c39 100644
--- a/src/mlia/target/cortex_a/handlers.py
+++ b/src/mlia/target/cortex_a/handlers.py
@@ -35,4 +35,4 @@ class CortexAEventHandler(WorkflowEventsHandler, CortexAAdvisorEventHandler):
def on_cortex_a_advisor_started(self, event: CortexAAdvisorStartedEvent) -> None:
"""Handle CortexAAdvisorStarted event."""
- self.reporter.submit(event.target)
+ self.reporter.submit(event.target_config)
diff --git a/src/mlia/target/cortex_a/reporters.py b/src/mlia/target/cortex_a/reporters.py
index e23bf4d..d214b09 100644
--- a/src/mlia/target/cortex_a/reporters.py
+++ b/src/mlia/target/cortex_a/reporters.py
@@ -23,13 +23,13 @@ from mlia.utils.console import style_improvement
from mlia.utils.types import is_list_of
-def report_target(target: CortexAConfiguration) -> Report:
+def report_target(target_config: CortexAConfiguration) -> Report:
"""Generate report for the target."""
return NestedReport(
"Target information",
"target",
[
- ReportItem("Target", alias="target", value=target.target),
+ ReportItem("Target", alias="target", value=target_config.target),
],
)
diff --git a/src/mlia/target/ethos_u/__init__.py b/src/mlia/target/ethos_u/__init__.py
index 6b6777d..7fa8af1 100644
--- a/src/mlia/target/ethos_u/__init__.py
+++ b/src/mlia/target/ethos_u/__init__.py
@@ -8,16 +8,20 @@ from mlia.target.ethos_u.config import get_default_ethos_u_backends
from mlia.target.registry import registry
from mlia.target.registry import TargetInfo
-SUPPORTED_BACKENDS_PRIORITY = ["Vela", *CORSTONE_PRIORITY]
+SUPPORTED_BACKENDS_PRIORITY = [
+ "vela",
+ *(corstone.lower() for corstone in CORSTONE_PRIORITY),
+]
-for ethos_u in ("ethos-u55", "ethos-u65"):
+for name in ("Ethos-U55", "Ethos-U65"):
registry.register(
- ethos_u,
+ name.lower(),
TargetInfo(
supported_backends=SUPPORTED_BACKENDS_PRIORITY,
default_backends=get_default_ethos_u_backends(SUPPORTED_BACKENDS_PRIORITY),
advisor_factory_func=configure_and_get_ethosu_advisor,
target_profile_cls=EthosUConfiguration,
),
+ pretty_name=name,
)
diff --git a/src/mlia/target/ethos_u/advisor.py b/src/mlia/target/ethos_u/advisor.py
index b34d1e0..0eec6aa 100644
--- a/src/mlia/target/ethos_u/advisor.py
+++ b/src/mlia/target/ethos_u/advisor.py
@@ -41,13 +41,13 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor):
def get_collectors(self, context: Context) -> list[DataCollector]:
"""Return list of the data collectors."""
model = self.get_model(context)
- target = self._get_target_cfg(context)
+ target_config = self._get_target_config(context)
backends = self._get_backends(context)
collectors: list[DataCollector] = []
if context.category_enabled(AdviceCategory.COMPATIBILITY):
- collectors.append(EthosUOperatorCompatibility(model, target))
+ collectors.append(EthosUOperatorCompatibility(model, target_config))
# Performance and optimization are mutually exclusive.
# Decide which one to use (taking into account the model format).
@@ -58,18 +58,18 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor):
"Command 'optimization' is not supported for TensorFlow Lite files."
)
if context.category_enabled(AdviceCategory.PERFORMANCE):
- collectors.append(EthosUPerformance(model, target, backends))
+ collectors.append(EthosUPerformance(model, target_config, backends))
else:
# Keras/SavedModel: Prefer optimization
if context.category_enabled(AdviceCategory.OPTIMIZATION):
optimization_settings = self._get_optimization_settings(context)
collectors.append(
EthosUOptimizationPerformance(
- model, target, optimization_settings, backends
+ model, target_config, optimization_settings, backends
)
)
elif context.category_enabled(AdviceCategory.PERFORMANCE):
- collectors.append(EthosUPerformance(model, target, backends))
+ collectors.append(EthosUPerformance(model, target_config, backends))
return collectors
@@ -89,13 +89,13 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor):
def get_events(self, context: Context) -> list[Event]:
"""Return list of the startup events."""
model = self.get_model(context)
- target = self._get_target_cfg(context)
+ target_config = self._get_target_config(context)
return [
- EthosUAdvisorStartedEvent(target=target, model=model),
+ EthosUAdvisorStartedEvent(target_config=target_config, model=model),
]
- def _get_target_cfg(self, context: Context) -> EthosUConfiguration:
+ def _get_target_config(self, context: Context) -> EthosUConfiguration:
"""Get target configuration."""
target_profile = self.get_target_profile(context)
return cast(EthosUConfiguration, profile(target_profile))
diff --git a/src/mlia/target/ethos_u/data_collection.py b/src/mlia/target/ethos_u/data_collection.py
index 96fe240..4fdfe96 100644
--- a/src/mlia/target/ethos_u/data_collection.py
+++ b/src/mlia/target/ethos_u/data_collection.py
@@ -31,10 +31,10 @@ logger = logging.getLogger(__name__)
class EthosUOperatorCompatibility(ContextAwareDataCollector):
"""Collect operator compatibility information."""
- def __init__(self, model: Path, target: EthosUConfiguration) -> None:
+ def __init__(self, model: Path, target_config: EthosUConfiguration) -> None:
"""Init operator compatibility data collector."""
self.model = model
- self.target = target
+ self.target_config = target_config
def collect_data(self) -> Operators:
"""Collect operator compatibility information."""
@@ -42,7 +42,7 @@ class EthosUOperatorCompatibility(ContextAwareDataCollector):
with log_action("Checking operator compatibility ..."):
return supported_operators(
- Path(tflite_model.model_path), self.target.compiler_options
+ Path(tflite_model.model_path), self.target_config.compiler_options
)
@classmethod
@@ -57,12 +57,12 @@ class EthosUPerformance(ContextAwareDataCollector):
def __init__(
self,
model: Path,
- target: EthosUConfiguration,
+ target_config: EthosUConfiguration,
backends: list[str] | None = None,
) -> None:
"""Init performance data collector."""
self.model = model
- self.target = target
+ self.target_config = target_config
self.backends = backends
def collect_data(self) -> PerformanceMetrics:
@@ -70,7 +70,7 @@ class EthosUPerformance(ContextAwareDataCollector):
tflite_model = get_tflite_model(self.model, self.context)
estimator = EthosUPerformanceEstimator(
self.context,
- self.target,
+ self.target_config,
self.backends,
)
@@ -113,13 +113,13 @@ class EthosUOptimizationPerformance(ContextAwareDataCollector):
def __init__(
self,
model: Path,
- target: EthosUConfiguration,
+ target_config: EthosUConfiguration,
optimizations: list[list[dict]],
backends: list[str] | None = None,
) -> None:
"""Init performance optimizations data collector."""
self.model = model
- self.target = target
+ self.target = target_config
self.optimizations = optimizations
self.backends = backends
diff --git a/src/mlia/target/ethos_u/events.py b/src/mlia/target/ethos_u/events.py
index 8060382..17e5e7f 100644
--- a/src/mlia/target/ethos_u/events.py
+++ b/src/mlia/target/ethos_u/events.py
@@ -14,7 +14,7 @@ class EthosUAdvisorStartedEvent(Event):
"""Event with Ethos-U advisor parameters."""
model: Path
- target: EthosUConfiguration
+ target_config: EthosUConfiguration
class EthosUAdvisorEventHandler(EventDispatcher):
diff --git a/src/mlia/target/ethos_u/handlers.py b/src/mlia/target/ethos_u/handlers.py
index c9d0dc7..b9c89e8 100644
--- a/src/mlia/target/ethos_u/handlers.py
+++ b/src/mlia/target/ethos_u/handlers.py
@@ -51,4 +51,4 @@ class EthosUEventHandler(WorkflowEventsHandler, EthosUAdvisorEventHandler):
def on_ethos_u_advisor_started(self, event: EthosUAdvisorStartedEvent) -> None:
"""Handle EthosUAdvisorStarted event."""
- self.reporter.submit(event.target)
+ self.reporter.submit(event.target_config)
diff --git a/src/mlia/target/ethos_u/performance.py b/src/mlia/target/ethos_u/performance.py
index 5bcafab..f7f9a8c 100644
--- a/src/mlia/target/ethos_u/performance.py
+++ b/src/mlia/target/ethos_u/performance.py
@@ -92,17 +92,19 @@ class MemoryUsage:
class PerformanceMetrics:
"""Performance metrics."""
- target: EthosUConfiguration
+ target_config: EthosUConfiguration
npu_cycles: NPUCycles | None
memory_usage: MemoryUsage | None
def in_kilobytes(self) -> PerformanceMetrics:
"""Return metrics with memory usage in KiB."""
if self.memory_usage is None:
- return PerformanceMetrics(self.target, self.npu_cycles, self.memory_usage)
+ return PerformanceMetrics(
+ self.target_config, self.npu_cycles, self.memory_usage
+ )
return PerformanceMetrics(
- self.target, self.npu_cycles, self.memory_usage.in_kilobytes()
+ self.target_config, self.npu_cycles, self.memory_usage.in_kilobytes()
)
@@ -121,10 +123,10 @@ class VelaPerformanceEstimator(
):
"""Vela based performance estimator."""
- def __init__(self, context: Context, target: EthosUConfiguration) -> None:
+ def __init__(self, context: Context, target_config: EthosUConfiguration) -> None:
"""Init Vela based performance estimator."""
self.context = context
- self.target = target
+ self.target = target_config
def estimate(self, model: Path | ModelConfiguration) -> MemoryUsage:
"""Estimate performance."""
@@ -154,11 +156,11 @@ class CorstonePerformanceEstimator(
"""Corstone-based performance estimator."""
def __init__(
- self, context: Context, target: EthosUConfiguration, backend: str
+ self, context: Context, target_config: EthosUConfiguration, backend: str
) -> None:
"""Init Corstone-based performance estimator."""
self.context = context
- self.target = target
+ self.target_config = target_config
self.backend = backend
def estimate(self, model: Path | ModelConfiguration) -> NPUCycles:
@@ -180,12 +182,12 @@ class CorstonePerformanceEstimator(
)
vela_comp.optimize_model(
- model_path, self.target.compiler_options, optimized_model_path
+ model_path, self.target_config.compiler_options, optimized_model_path
)
corstone_perf_metrics = estimate_performance(
- self.target.target,
- self.target.mac,
+ self.target_config.target,
+ self.target_config.mac,
optimized_model_path,
self.backend,
)
@@ -208,17 +210,17 @@ class EthosUPerformanceEstimator(
def __init__(
self,
context: Context,
- target: EthosUConfiguration,
+ target_config: EthosUConfiguration,
backends: list[str] | None = None,
) -> None:
"""Init performance estimator."""
self.context = context
- self.target = target
+ self.target_config = target_config
if backends is None:
- backends = ["Vela"] # Only Vela is always available as default
- ethos_u_backends = supported_backends(target.target)
+ backends = ["vela"] # Only Vela is always available as default
+ ethos_u_backends = supported_backends(target_config.target)
for backend in backends:
- if backend != "Vela" and backend not in ethos_u_backends:
+ if backend != "vela" and backend not in ethos_u_backends:
raise ValueError(
f"Unsupported backend '{backend}'. "
f"Only 'Vela' and {ethos_u_backends} "
@@ -237,12 +239,14 @@ class EthosUPerformanceEstimator(
memory_usage = None
npu_cycles = None
for backend in self.backends:
- if backend == "Vela":
- vela_estimator = VelaPerformanceEstimator(self.context, self.target)
+ if backend == "vela":
+ vela_estimator = VelaPerformanceEstimator(
+ self.context, self.target_config
+ )
memory_usage = vela_estimator.estimate(tflite_model)
elif is_corstone_backend(backend):
corstone_estimator = CorstonePerformanceEstimator(
- self.context, self.target, backend
+ self.context, self.target_config, backend
)
npu_cycles = corstone_estimator.estimate(tflite_model)
else:
@@ -252,4 +256,4 @@ class EthosUPerformanceEstimator(
backend,
)
- return PerformanceMetrics(self.target, npu_cycles, memory_usage)
+ return PerformanceMetrics(self.target_config, npu_cycles, memory_usage)
diff --git a/src/mlia/target/ethos_u/reporters.py b/src/mlia/target/ethos_u/reporters.py
index 4de60bb..2a5b5d3 100644
--- a/src/mlia/target/ethos_u/reporters.py
+++ b/src/mlia/target/ethos_u/reporters.py
@@ -110,9 +110,9 @@ def report_operators(ops: list[Operator]) -> Report:
return Table(columns, rows, name="Operators", alias="operators")
-def report_target_details(target: EthosUConfiguration) -> Report:
+def report_target_details(target_config: EthosUConfiguration) -> Report:
"""Return table representation for the target."""
- compiler_config = target.resolved_compiler_config
+ compiler_config = target_config.resolved_compiler_config
memory_settings = [
ReportItem(
@@ -211,8 +211,8 @@ def report_target_details(target: EthosUConfiguration) -> Report:
"Target information",
"target",
[
- ReportItem("Target", alias="target", value=target.target),
- ReportItem("MAC", alias="mac", value=target.mac),
+ ReportItem("Target", alias="target", value=target_config.target),
+ ReportItem("MAC", alias="mac", value=target_config.mac),
ReportItem(
"Memory mode",
alias="memory_mode",
diff --git a/src/mlia/target/registry.py b/src/mlia/target/registry.py
index 9fccecb..b7b6193 100644
--- a/src/mlia/target/registry.py
+++ b/src/mlia/target/registry.py
@@ -25,12 +25,14 @@ from mlia.utils.registry import Registry
class TargetRegistry(Registry[TargetInfo]):
"""Registry for targets."""
- def register(self, name: str, item: TargetInfo) -> bool:
+ def register(
+ self, name: str, item: TargetInfo, pretty_name: str | None = None
+ ) -> bool:
"""Register an item: returns `False` if already registered."""
assert all(
backend in backend_registry.items for backend in item.supported_backends
)
- return super().register(name, item)
+ return super().register(name, item, pretty_name)
# All supported targets are required to be registered here.
@@ -159,15 +161,12 @@ def table() -> Table:
rows = [
(
- name,
- Table(
- columns=[Column("Backend"), Column("Status")],
- rows=[
- (backend, get_status(backend))
- for backend in info.supported_backends
- ],
- name="Backends",
+ f"{registry.pretty_name(name)}\n<{name}>",
+ "\n".join(
+ f"{backend_registry.pretty_name(backend)}\n<{backend}>"
+ for backend in info.supported_backends
),
+ "\n\n".join(get_status(backend) for backend in info.supported_backends),
"/".join(get_advice(name)),
)
for name, info in registry.items.items()
@@ -177,6 +176,7 @@ def table() -> Table:
columns=[
Column("Target"),
Column("Backend(s)"),
+ Column("Status"),
Column("Advice: comp/perf/opt"),
],
rows=rows,
diff --git a/src/mlia/target/tosa/__init__.py b/src/mlia/target/tosa/__init__.py
index 3830ce5..12077bd 100644
--- a/src/mlia/target/tosa/__init__.py
+++ b/src/mlia/target/tosa/__init__.py
@@ -14,4 +14,5 @@ registry.register(
advisor_factory_func=configure_and_get_tosa_advisor,
target_profile_cls=TOSAConfiguration,
),
+ pretty_name="TOSA",
)
diff --git a/src/mlia/target/tosa/reporters.py b/src/mlia/target/tosa/reporters.py
index 5c015ff..9575978 100644
--- a/src/mlia/target/tosa/reporters.py
+++ b/src/mlia/target/tosa/reporters.py
@@ -42,13 +42,13 @@ class MetadataDisplay: # pylint: disable=too-few-public-methods
self.model_name = model_meta.model_name
-def report_target(target: TOSAConfiguration) -> Report:
+def report_target(target_config: TOSAConfiguration) -> Report:
"""Generate report for the target."""
return NestedReport(
"Target information",
"target",
[
- ReportItem("Target", alias="target", value=target.target),
+ ReportItem("Target", alias="target", value=target_config.target),
],
)
diff --git a/src/mlia/utils/registry.py b/src/mlia/utils/registry.py
index 9b25a81..a886376 100644
--- a/src/mlia/utils/registry.py
+++ b/src/mlia/utils/registry.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
"""Generic registry class."""
from __future__ import annotations
@@ -15,17 +15,25 @@ class Registry(Generic[T]):
def __init__(self) -> None:
"""Create an empty registry."""
self.items: dict[str, T] = {}
+ self.pretty_names: dict[str, str] = {}
def __str__(self) -> str:
"""List all registered items."""
return "\n".join(
- f"- {name}: {item}"
+ f"- {self.pretty_names[name] if name in self.pretty_names else name}: "
+ f"{item}"
for name, item in sorted(self.items.items(), key=lambda v: v[0])
)
- def register(self, name: str, item: T) -> bool:
+ def register(self, name: str, item: T, pretty_name: str | None = None) -> bool:
"""Register an item: returns `False` if already registered."""
if name in self.items:
return False # already registered
self.items[name] = item
+ if pretty_name:
+ self.pretty_names[name] = pretty_name
return True
+
+ def pretty_name(self, name: str) -> str:
+ """Get the pretty name (if available) or return the name as is otherwise."""
+ return self.pretty_names[name] if name in self.pretty_names else name
diff --git a/tests/conftest.py b/tests/conftest.py
index 55b296f..30889ca 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -84,9 +84,11 @@ def fixture_test_models_path(
tflite_vela_model = tmp_path / "test_model_vela.tflite"
- target_profile = EthosUConfiguration.load_profile("ethos-u55-256")
+ target_config = EthosUConfiguration.load_profile("ethos-u55-256")
optimize_model(
- tflite_model_path, target_profile.compiler_options, tflite_vela_model
+ tflite_model_path,
+ target_config.compiler_options,
+ tflite_vela_model,
)
tf.saved_model.save(keras_model, str(tmp_path / "tf_model_test_model"))
diff --git a/tests/test_backend_corstone.py b/tests/test_backend_corstone.py
index 29ef084..190931a 100644
--- a/tests/test_backend_corstone.py
+++ b/tests/test_backend_corstone.py
@@ -6,6 +6,6 @@ from mlia.backend.corstone import is_corstone_backend
def test_is_corstone_backend() -> None:
"""Test function is_corstone_backend."""
- assert is_corstone_backend("Corstone-300") is True
- assert is_corstone_backend("Corstone-310") is True
+ assert is_corstone_backend("corstone-300") is True
+ assert is_corstone_backend("corstone-310") is True
assert is_corstone_backend("New backend") is False
diff --git a/tests/test_backend_corstone_performance.py b/tests/test_backend_corstone_performance.py
index 2d5b196..d3d378e 100644
--- a/tests/test_backend_corstone_performance.py
+++ b/tests/test_backend_corstone_performance.py
@@ -81,7 +81,7 @@ def test_generic_inference_output_parser_failure(wrong_fvp_output: list[str]) ->
[
[
Path("backend_path"),
- "Corstone-300",
+ "corstone-300",
"ethos-u55",
256,
Path("model.tflite"),
@@ -163,7 +163,7 @@ def test_estimate_performance(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr("mlia.utils.proc.command_output", command_output_mock)
result = estimate_performance(
- "ethos-u55", 256, Path("model.tflite"), "Corstone-300"
+ "ethos-u55", 256, Path("model.tflite"), "corstone-300"
)
assert result == PerformanceMetrics(1, 2, 3, 4, 5, 6)
diff --git a/tests/test_backend_registry.py b/tests/test_backend_registry.py
index e3e2da2..cc05632 100644
--- a/tests/test_backend_registry.py
+++ b/tests/test_backend_registry.py
@@ -19,19 +19,19 @@ from mlia.core.common import AdviceCategory
("backend", "advices", "systems", "type_"),
(
(
- "ArmNNTFLiteDelegate",
+ "armnn-tflite-delegate",
[AdviceCategory.COMPATIBILITY],
None,
BackendType.BUILTIN,
),
(
- "Corstone-300",
+ "corstone-300",
[AdviceCategory.PERFORMANCE, AdviceCategory.OPTIMIZATION],
[System.LINUX_AMD64],
BackendType.CUSTOM,
),
(
- "Corstone-310",
+ "corstone-310",
[AdviceCategory.PERFORMANCE, AdviceCategory.OPTIMIZATION],
[System.LINUX_AMD64],
BackendType.CUSTOM,
@@ -43,7 +43,7 @@ from mlia.core.common import AdviceCategory
BackendType.WHEEL,
),
(
- "Vela",
+ "vela",
[
AdviceCategory.COMPATIBILITY,
AdviceCategory.PERFORMANCE,
@@ -86,11 +86,11 @@ def test_backend_registry(
def test_get_supported_backends() -> None:
"""Test function get_supported_backends."""
assert get_supported_backends() == [
- "ArmNNTFLiteDelegate",
- "Corstone-300",
- "Corstone-310",
- "Vela",
+ "armnn-tflite-delegate",
+ "corstone-300",
+ "corstone-310",
"tosa-checker",
+ "vela",
]
diff --git a/tests/test_backend_vela_compat.py b/tests/test_backend_vela_compat.py
index 4e0f149..0c39dd6 100644
--- a/tests/test_backend_vela_compat.py
+++ b/tests/test_backend_vela_compat.py
@@ -55,9 +55,11 @@ from mlia.utils.filesystem import working_directory
)
def test_operators(test_models_path: Path, model: str, expected_ops: Operators) -> None:
"""Test operators function."""
- target = EthosUConfiguration.load_profile("ethos-u55-256")
+ target_config = EthosUConfiguration.load_profile("ethos-u55-256")
- operators = supported_operators(test_models_path / model, target.compiler_options)
+ operators = supported_operators(
+ test_models_path / model, target_config.compiler_options
+ )
for expected, actual in zip(expected_ops.ops, operators.ops):
# do not compare names as they could be different on each model generation
assert expected.op_type == actual.op_type
diff --git a/tests/test_backend_vela_compiler.py b/tests/test_backend_vela_compiler.py
index 0434ccf..9b69ada 100644
--- a/tests/test_backend_vela_compiler.py
+++ b/tests/test_backend_vela_compiler.py
@@ -158,8 +158,10 @@ def test_optimize_model(tmp_path: Path, test_tflite_model: Path) -> None:
"""Test model optimization and saving into file."""
tmp_file = tmp_path / "temp.tflite"
- target = EthosUConfiguration.load_profile("ethos-u55-256")
- optimize_model(test_tflite_model, target.compiler_options, tmp_file.absolute())
+ target_config = EthosUConfiguration.load_profile("ethos-u55-256")
+ optimize_model(
+ test_tflite_model, target_config.compiler_options, tmp_file.absolute()
+ )
assert tmp_file.is_file()
assert tmp_file.stat().st_size > 0
diff --git a/tests/test_backend_vela_performance.py b/tests/test_backend_vela_performance.py
index 68b96ab..df2ce08 100644
--- a/tests/test_backend_vela_performance.py
+++ b/tests/test_backend_vela_performance.py
@@ -14,8 +14,10 @@ from mlia.target.ethos_u.config import EthosUConfiguration
def test_estimate_performance(test_tflite_model: Path) -> None:
"""Test getting performance estimations."""
- target = EthosUConfiguration.load_profile("ethos-u55-256")
- perf_metrics = estimate_performance(test_tflite_model, target.compiler_options)
+ target_config = EthosUConfiguration.load_profile("ethos-u55-256")
+ perf_metrics = estimate_performance(
+ test_tflite_model, target_config.compiler_options
+ )
assert isinstance(perf_metrics, PerformanceMetrics)
@@ -24,16 +26,18 @@ def test_estimate_performance_already_optimized(
tmp_path: Path, test_tflite_model: Path
) -> None:
"""Test that performance estimation should fail for already optimized model."""
- target = EthosUConfiguration.load_profile("ethos-u55-256")
+ target_config = EthosUConfiguration.load_profile("ethos-u55-256")
optimized_model_path = tmp_path / "optimized_model.tflite"
- optimize_model(test_tflite_model, target.compiler_options, optimized_model_path)
+ optimize_model(
+ test_tflite_model, target_config.compiler_options, optimized_model_path
+ )
with pytest.raises(
Exception, match="Unable to estimate performance for the given optimized model"
):
- estimate_performance(optimized_model_path, target.compiler_options)
+ estimate_performance(optimized_model_path, target_config.compiler_options)
def test_read_invalid_model(test_tflite_invalid_model: Path) -> None:
@@ -41,8 +45,8 @@ def test_read_invalid_model(test_tflite_invalid_model: Path) -> None:
with pytest.raises(
Exception, match=f"Unable to read model {test_tflite_invalid_model}"
):
- target = EthosUConfiguration.load_profile("ethos-u55-256")
- estimate_performance(test_tflite_invalid_model, target.compiler_options)
+ target_config = EthosUConfiguration.load_profile("ethos-u55-256")
+ estimate_performance(test_tflite_invalid_model, target_config.compiler_options)
def test_compile_invalid_model(
@@ -58,7 +62,7 @@ def test_compile_invalid_model(
with pytest.raises(
Exception, match="Model could not be optimized with Vela compiler"
):
- target = EthosUConfiguration.load_profile("ethos-u55-256")
- optimize_model(test_tflite_model, target.compiler_options, model_path)
+ target_config = EthosUConfiguration.load_profile("ethos-u55-256")
+ optimize_model(test_tflite_model, target_config.compiler_options, model_path)
assert not model_path.exists()
diff --git a/tests/test_cli_command_validators.py b/tests/test_cli_command_validators.py
index 29813f4..1ac82db 100644
--- a/tests/test_cli_command_validators.py
+++ b/tests/test_cli_command_validators.py
@@ -109,17 +109,17 @@ def test_validate_check_target_profile(
],
[
"tosa",
- ["Corstone-310"],
+ ["corstone-310"],
True,
- "Backend Corstone-310 not supported with target-profile tosa.",
+ "Backend corstone-310 not supported with target-profile tosa.",
None,
],
[
"cortex-a",
- ["ArmNNTFLiteDelegate"],
+ ["armnn-tflite-delegate"],
False,
None,
- ["ArmNNTFLiteDelegate"],
+ ["armnn-tflite-delegate"],
],
[
"cortex-a",
@@ -130,14 +130,14 @@ def test_validate_check_target_profile(
],
[
"ethos-u55-256",
- ["Vela", "Corstone-310"],
+ ["vela", "corstone-310"],
False,
None,
- ["Vela", "Corstone-310"],
+ ["vela", "corstone-310"],
],
[
"ethos-u65-256",
- ["Vela", "Corstone-310", "tosa-checker"],
+ ["vela", "corstone-310", "tosa-checker"],
True,
"Backend tosa-checker not supported with target-profile ethos-u65-256.",
None,
diff --git a/tests/test_cli_main.py b/tests/test_cli_main.py
index 673031c..3472e31 100644
--- a/tests/test_cli_main.py
+++ b/tests/test_cli_main.py
@@ -239,7 +239,7 @@ def test_commands_execution(
monkeypatch.setattr(
"mlia.cli.options.get_available_backends",
- MagicMock(return_value=["Vela", "some_backend"]),
+ MagicMock(return_value=["vela", "some_backend"]),
)
for command in ["check", "optimize"]:
diff --git a/tests/test_target_registry.py b/tests/test_target_registry.py
index 2cbd97d..ca1ad82 100644
--- a/tests/test_target_registry.py
+++ b/tests/test_target_registry.py
@@ -63,13 +63,13 @@ def test_supported_advice(
@pytest.mark.parametrize(
("backend", "target", "expected_result"),
(
- ("ArmNNTFLiteDelegate", None, True),
- ("ArmNNTFLiteDelegate", "cortex-a", True),
- ("ArmNNTFLiteDelegate", "tosa", False),
- ("Corstone-310", None, True),
- ("Corstone-310", "ethos-u55", True),
- ("Corstone-310", "ethos-u65", True),
- ("Corstone-310", "cortex-a", False),
+ ("armnn-tflite-delegate", None, True),
+ ("armnn-tflite-delegate", "cortex-a", True),
+ ("armnn-tflite-delegate", "tosa", False),
+ ("corstone-310", None, True),
+ ("corstone-310", "ethos-u55", True),
+ ("corstone-310", "ethos-u65", True),
+ ("corstone-310", "cortex-a", False),
),
)
def test_is_supported(backend: str, target: str | None, expected_result: bool) -> None:
@@ -80,9 +80,9 @@ def test_is_supported(backend: str, target: str | None, expected_result: bool) -
@pytest.mark.parametrize(
("target_name", "expected_backends"),
(
- ("cortex-a", ["ArmNNTFLiteDelegate"]),
- ("ethos-u55", ["Corstone-300", "Corstone-310", "Vela"]),
- ("ethos-u65", ["Corstone-300", "Corstone-310", "Vela"]),
+ ("cortex-a", ["armnn-tflite-delegate"]),
+ ("ethos-u55", ["corstone-300", "corstone-310", "vela"]),
+ ("ethos-u65", ["corstone-300", "corstone-310", "vela"]),
("tosa", ["tosa-checker"]),
),
)
@@ -107,21 +107,21 @@ def test_supported_targets(advice: AdviceCategory, expected_targets: list[str])
def test_all_supported_backends() -> None:
"""Test function all_supported_backends."""
assert all_supported_backends() == {
- "Vela",
+ "vela",
"tosa-checker",
- "ArmNNTFLiteDelegate",
- "Corstone-310",
- "Corstone-300",
+ "armnn-tflite-delegate",
+ "corstone-310",
+ "corstone-300",
}
@pytest.mark.parametrize(
("target", "expected_default_backends", "is_subset_only"),
[
- ["cortex-a", ["ArmNNTFLiteDelegate"], False],
+ ["cortex-a", ["armnn-tflite-delegate"], False],
["tosa", ["tosa-checker"], False],
- ["ethos-u55", ["Vela"], True],
- ["ethos-u65", ["Vela"], True],
+ ["ethos-u55", ["vela"], True],
+ ["ethos-u65", ["vela"], True],
],
)
def test_default_backends(