From 718277eaece76902c4950f18d428907b39a18ef1 Mon Sep 17 00:00:00 2001 From: Benjamin Klimczak Date: Fri, 10 Feb 2023 13:12:57 +0000 Subject: MLIA-769 Add "pretty names" for targets / backends - Provide "pretty names" to print information for targets and backends. - Use 'target_config' instead of 'target' if a target profile is used. - Fix minor issue in output regarding the output directory. Change-Id: Ib38231f30b4d609a0d1e8f9c52b2fb547c69cb6a --- src/mlia/target/cortex_a/__init__.py | 5 ++-- src/mlia/target/cortex_a/events.py | 2 +- src/mlia/target/cortex_a/handlers.py | 2 +- src/mlia/target/cortex_a/reporters.py | 4 +-- src/mlia/target/ethos_u/__init__.py | 10 ++++--- src/mlia/target/ethos_u/advisor.py | 16 ++++++------ src/mlia/target/ethos_u/data_collection.py | 16 ++++++------ src/mlia/target/ethos_u/events.py | 2 +- src/mlia/target/ethos_u/handlers.py | 2 +- src/mlia/target/ethos_u/performance.py | 42 ++++++++++++++++-------------- src/mlia/target/ethos_u/reporters.py | 8 +++--- src/mlia/target/registry.py | 20 +++++++------- src/mlia/target/tosa/__init__.py | 1 + src/mlia/target/tosa/reporters.py | 4 +-- 14 files changed, 72 insertions(+), 62 deletions(-) (limited to 'src/mlia/target') diff --git a/src/mlia/target/cortex_a/__init__.py b/src/mlia/target/cortex_a/__init__.py index 87f268a..729dd30 100644 --- a/src/mlia/target/cortex_a/__init__.py +++ b/src/mlia/target/cortex_a/__init__.py @@ -9,9 +9,10 @@ from mlia.target.registry import TargetInfo registry.register( "cortex-a", TargetInfo( - supported_backends=["ArmNNTFLiteDelegate"], - default_backends=["ArmNNTFLiteDelegate"], + supported_backends=["armnn-tflite-delegate"], + default_backends=["armnn-tflite-delegate"], advisor_factory_func=configure_and_get_cortexa_advisor, target_profile_cls=CortexAConfiguration, ), + pretty_name="Cortex-A", ) diff --git a/src/mlia/target/cortex_a/events.py b/src/mlia/target/cortex_a/events.py index 76f17ba..675335d 100644 --- a/src/mlia/target/cortex_a/events.py +++ b/src/mlia/target/cortex_a/events.py @@ -14,7 +14,7 @@ class CortexAAdvisorStartedEvent(Event): """Event with Cortex-A advisor parameters.""" model: Path - target: CortexAConfiguration + target_config: CortexAConfiguration class CortexAAdvisorEventHandler(EventDispatcher): diff --git a/src/mlia/target/cortex_a/handlers.py b/src/mlia/target/cortex_a/handlers.py index d46197c..a952c39 100644 --- a/src/mlia/target/cortex_a/handlers.py +++ b/src/mlia/target/cortex_a/handlers.py @@ -35,4 +35,4 @@ class CortexAEventHandler(WorkflowEventsHandler, CortexAAdvisorEventHandler): def on_cortex_a_advisor_started(self, event: CortexAAdvisorStartedEvent) -> None: """Handle CortexAAdvisorStarted event.""" - self.reporter.submit(event.target) + self.reporter.submit(event.target_config) diff --git a/src/mlia/target/cortex_a/reporters.py b/src/mlia/target/cortex_a/reporters.py index e23bf4d..d214b09 100644 --- a/src/mlia/target/cortex_a/reporters.py +++ b/src/mlia/target/cortex_a/reporters.py @@ -23,13 +23,13 @@ from mlia.utils.console import style_improvement from mlia.utils.types import is_list_of -def report_target(target: CortexAConfiguration) -> Report: +def report_target(target_config: CortexAConfiguration) -> Report: """Generate report for the target.""" return NestedReport( "Target information", "target", [ - ReportItem("Target", alias="target", value=target.target), + ReportItem("Target", alias="target", value=target_config.target), ], ) diff --git a/src/mlia/target/ethos_u/__init__.py b/src/mlia/target/ethos_u/__init__.py index 6b6777d..7fa8af1 100644 --- a/src/mlia/target/ethos_u/__init__.py +++ b/src/mlia/target/ethos_u/__init__.py @@ -8,16 +8,20 @@ from mlia.target.ethos_u.config import get_default_ethos_u_backends from mlia.target.registry import registry from mlia.target.registry import TargetInfo -SUPPORTED_BACKENDS_PRIORITY = ["Vela", *CORSTONE_PRIORITY] +SUPPORTED_BACKENDS_PRIORITY = [ + "vela", + *(corstone.lower() for corstone in CORSTONE_PRIORITY), +] -for ethos_u in ("ethos-u55", "ethos-u65"): +for name in ("Ethos-U55", "Ethos-U65"): registry.register( - ethos_u, + name.lower(), TargetInfo( supported_backends=SUPPORTED_BACKENDS_PRIORITY, default_backends=get_default_ethos_u_backends(SUPPORTED_BACKENDS_PRIORITY), advisor_factory_func=configure_and_get_ethosu_advisor, target_profile_cls=EthosUConfiguration, ), + pretty_name=name, ) diff --git a/src/mlia/target/ethos_u/advisor.py b/src/mlia/target/ethos_u/advisor.py index b34d1e0..0eec6aa 100644 --- a/src/mlia/target/ethos_u/advisor.py +++ b/src/mlia/target/ethos_u/advisor.py @@ -41,13 +41,13 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor): def get_collectors(self, context: Context) -> list[DataCollector]: """Return list of the data collectors.""" model = self.get_model(context) - target = self._get_target_cfg(context) + target_config = self._get_target_config(context) backends = self._get_backends(context) collectors: list[DataCollector] = [] if context.category_enabled(AdviceCategory.COMPATIBILITY): - collectors.append(EthosUOperatorCompatibility(model, target)) + collectors.append(EthosUOperatorCompatibility(model, target_config)) # Performance and optimization are mutually exclusive. # Decide which one to use (taking into account the model format). @@ -58,18 +58,18 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor): "Command 'optimization' is not supported for TensorFlow Lite files." ) if context.category_enabled(AdviceCategory.PERFORMANCE): - collectors.append(EthosUPerformance(model, target, backends)) + collectors.append(EthosUPerformance(model, target_config, backends)) else: # Keras/SavedModel: Prefer optimization if context.category_enabled(AdviceCategory.OPTIMIZATION): optimization_settings = self._get_optimization_settings(context) collectors.append( EthosUOptimizationPerformance( - model, target, optimization_settings, backends + model, target_config, optimization_settings, backends ) ) elif context.category_enabled(AdviceCategory.PERFORMANCE): - collectors.append(EthosUPerformance(model, target, backends)) + collectors.append(EthosUPerformance(model, target_config, backends)) return collectors @@ -89,13 +89,13 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor): def get_events(self, context: Context) -> list[Event]: """Return list of the startup events.""" model = self.get_model(context) - target = self._get_target_cfg(context) + target_config = self._get_target_config(context) return [ - EthosUAdvisorStartedEvent(target=target, model=model), + EthosUAdvisorStartedEvent(target_config=target_config, model=model), ] - def _get_target_cfg(self, context: Context) -> EthosUConfiguration: + def _get_target_config(self, context: Context) -> EthosUConfiguration: """Get target configuration.""" target_profile = self.get_target_profile(context) return cast(EthosUConfiguration, profile(target_profile)) diff --git a/src/mlia/target/ethos_u/data_collection.py b/src/mlia/target/ethos_u/data_collection.py index 96fe240..4fdfe96 100644 --- a/src/mlia/target/ethos_u/data_collection.py +++ b/src/mlia/target/ethos_u/data_collection.py @@ -31,10 +31,10 @@ logger = logging.getLogger(__name__) class EthosUOperatorCompatibility(ContextAwareDataCollector): """Collect operator compatibility information.""" - def __init__(self, model: Path, target: EthosUConfiguration) -> None: + def __init__(self, model: Path, target_config: EthosUConfiguration) -> None: """Init operator compatibility data collector.""" self.model = model - self.target = target + self.target_config = target_config def collect_data(self) -> Operators: """Collect operator compatibility information.""" @@ -42,7 +42,7 @@ class EthosUOperatorCompatibility(ContextAwareDataCollector): with log_action("Checking operator compatibility ..."): return supported_operators( - Path(tflite_model.model_path), self.target.compiler_options + Path(tflite_model.model_path), self.target_config.compiler_options ) @classmethod @@ -57,12 +57,12 @@ class EthosUPerformance(ContextAwareDataCollector): def __init__( self, model: Path, - target: EthosUConfiguration, + target_config: EthosUConfiguration, backends: list[str] | None = None, ) -> None: """Init performance data collector.""" self.model = model - self.target = target + self.target_config = target_config self.backends = backends def collect_data(self) -> PerformanceMetrics: @@ -70,7 +70,7 @@ class EthosUPerformance(ContextAwareDataCollector): tflite_model = get_tflite_model(self.model, self.context) estimator = EthosUPerformanceEstimator( self.context, - self.target, + self.target_config, self.backends, ) @@ -113,13 +113,13 @@ class EthosUOptimizationPerformance(ContextAwareDataCollector): def __init__( self, model: Path, - target: EthosUConfiguration, + target_config: EthosUConfiguration, optimizations: list[list[dict]], backends: list[str] | None = None, ) -> None: """Init performance optimizations data collector.""" self.model = model - self.target = target + self.target = target_config self.optimizations = optimizations self.backends = backends diff --git a/src/mlia/target/ethos_u/events.py b/src/mlia/target/ethos_u/events.py index 8060382..17e5e7f 100644 --- a/src/mlia/target/ethos_u/events.py +++ b/src/mlia/target/ethos_u/events.py @@ -14,7 +14,7 @@ class EthosUAdvisorStartedEvent(Event): """Event with Ethos-U advisor parameters.""" model: Path - target: EthosUConfiguration + target_config: EthosUConfiguration class EthosUAdvisorEventHandler(EventDispatcher): diff --git a/src/mlia/target/ethos_u/handlers.py b/src/mlia/target/ethos_u/handlers.py index c9d0dc7..b9c89e8 100644 --- a/src/mlia/target/ethos_u/handlers.py +++ b/src/mlia/target/ethos_u/handlers.py @@ -51,4 +51,4 @@ class EthosUEventHandler(WorkflowEventsHandler, EthosUAdvisorEventHandler): def on_ethos_u_advisor_started(self, event: EthosUAdvisorStartedEvent) -> None: """Handle EthosUAdvisorStarted event.""" - self.reporter.submit(event.target) + self.reporter.submit(event.target_config) diff --git a/src/mlia/target/ethos_u/performance.py b/src/mlia/target/ethos_u/performance.py index 5bcafab..f7f9a8c 100644 --- a/src/mlia/target/ethos_u/performance.py +++ b/src/mlia/target/ethos_u/performance.py @@ -92,17 +92,19 @@ class MemoryUsage: class PerformanceMetrics: """Performance metrics.""" - target: EthosUConfiguration + target_config: EthosUConfiguration npu_cycles: NPUCycles | None memory_usage: MemoryUsage | None def in_kilobytes(self) -> PerformanceMetrics: """Return metrics with memory usage in KiB.""" if self.memory_usage is None: - return PerformanceMetrics(self.target, self.npu_cycles, self.memory_usage) + return PerformanceMetrics( + self.target_config, self.npu_cycles, self.memory_usage + ) return PerformanceMetrics( - self.target, self.npu_cycles, self.memory_usage.in_kilobytes() + self.target_config, self.npu_cycles, self.memory_usage.in_kilobytes() ) @@ -121,10 +123,10 @@ class VelaPerformanceEstimator( ): """Vela based performance estimator.""" - def __init__(self, context: Context, target: EthosUConfiguration) -> None: + def __init__(self, context: Context, target_config: EthosUConfiguration) -> None: """Init Vela based performance estimator.""" self.context = context - self.target = target + self.target = target_config def estimate(self, model: Path | ModelConfiguration) -> MemoryUsage: """Estimate performance.""" @@ -154,11 +156,11 @@ class CorstonePerformanceEstimator( """Corstone-based performance estimator.""" def __init__( - self, context: Context, target: EthosUConfiguration, backend: str + self, context: Context, target_config: EthosUConfiguration, backend: str ) -> None: """Init Corstone-based performance estimator.""" self.context = context - self.target = target + self.target_config = target_config self.backend = backend def estimate(self, model: Path | ModelConfiguration) -> NPUCycles: @@ -180,12 +182,12 @@ class CorstonePerformanceEstimator( ) vela_comp.optimize_model( - model_path, self.target.compiler_options, optimized_model_path + model_path, self.target_config.compiler_options, optimized_model_path ) corstone_perf_metrics = estimate_performance( - self.target.target, - self.target.mac, + self.target_config.target, + self.target_config.mac, optimized_model_path, self.backend, ) @@ -208,17 +210,17 @@ class EthosUPerformanceEstimator( def __init__( self, context: Context, - target: EthosUConfiguration, + target_config: EthosUConfiguration, backends: list[str] | None = None, ) -> None: """Init performance estimator.""" self.context = context - self.target = target + self.target_config = target_config if backends is None: - backends = ["Vela"] # Only Vela is always available as default - ethos_u_backends = supported_backends(target.target) + backends = ["vela"] # Only Vela is always available as default + ethos_u_backends = supported_backends(target_config.target) for backend in backends: - if backend != "Vela" and backend not in ethos_u_backends: + if backend != "vela" and backend not in ethos_u_backends: raise ValueError( f"Unsupported backend '{backend}'. " f"Only 'Vela' and {ethos_u_backends} " @@ -237,12 +239,14 @@ class EthosUPerformanceEstimator( memory_usage = None npu_cycles = None for backend in self.backends: - if backend == "Vela": - vela_estimator = VelaPerformanceEstimator(self.context, self.target) + if backend == "vela": + vela_estimator = VelaPerformanceEstimator( + self.context, self.target_config + ) memory_usage = vela_estimator.estimate(tflite_model) elif is_corstone_backend(backend): corstone_estimator = CorstonePerformanceEstimator( - self.context, self.target, backend + self.context, self.target_config, backend ) npu_cycles = corstone_estimator.estimate(tflite_model) else: @@ -252,4 +256,4 @@ class EthosUPerformanceEstimator( backend, ) - return PerformanceMetrics(self.target, npu_cycles, memory_usage) + return PerformanceMetrics(self.target_config, npu_cycles, memory_usage) diff --git a/src/mlia/target/ethos_u/reporters.py b/src/mlia/target/ethos_u/reporters.py index 4de60bb..2a5b5d3 100644 --- a/src/mlia/target/ethos_u/reporters.py +++ b/src/mlia/target/ethos_u/reporters.py @@ -110,9 +110,9 @@ def report_operators(ops: list[Operator]) -> Report: return Table(columns, rows, name="Operators", alias="operators") -def report_target_details(target: EthosUConfiguration) -> Report: +def report_target_details(target_config: EthosUConfiguration) -> Report: """Return table representation for the target.""" - compiler_config = target.resolved_compiler_config + compiler_config = target_config.resolved_compiler_config memory_settings = [ ReportItem( @@ -211,8 +211,8 @@ def report_target_details(target: EthosUConfiguration) -> Report: "Target information", "target", [ - ReportItem("Target", alias="target", value=target.target), - ReportItem("MAC", alias="mac", value=target.mac), + ReportItem("Target", alias="target", value=target_config.target), + ReportItem("MAC", alias="mac", value=target_config.mac), ReportItem( "Memory mode", alias="memory_mode", diff --git a/src/mlia/target/registry.py b/src/mlia/target/registry.py index 9fccecb..b7b6193 100644 --- a/src/mlia/target/registry.py +++ b/src/mlia/target/registry.py @@ -25,12 +25,14 @@ from mlia.utils.registry import Registry class TargetRegistry(Registry[TargetInfo]): """Registry for targets.""" - def register(self, name: str, item: TargetInfo) -> bool: + def register( + self, name: str, item: TargetInfo, pretty_name: str | None = None + ) -> bool: """Register an item: returns `False` if already registered.""" assert all( backend in backend_registry.items for backend in item.supported_backends ) - return super().register(name, item) + return super().register(name, item, pretty_name) # All supported targets are required to be registered here. @@ -159,15 +161,12 @@ def table() -> Table: rows = [ ( - name, - Table( - columns=[Column("Backend"), Column("Status")], - rows=[ - (backend, get_status(backend)) - for backend in info.supported_backends - ], - name="Backends", + f"{registry.pretty_name(name)}\n<{name}>", + "\n".join( + f"{backend_registry.pretty_name(backend)}\n<{backend}>" + for backend in info.supported_backends ), + "\n\n".join(get_status(backend) for backend in info.supported_backends), "/".join(get_advice(name)), ) for name, info in registry.items.items() @@ -177,6 +176,7 @@ def table() -> Table: columns=[ Column("Target"), Column("Backend(s)"), + Column("Status"), Column("Advice: comp/perf/opt"), ], rows=rows, diff --git a/src/mlia/target/tosa/__init__.py b/src/mlia/target/tosa/__init__.py index 3830ce5..12077bd 100644 --- a/src/mlia/target/tosa/__init__.py +++ b/src/mlia/target/tosa/__init__.py @@ -14,4 +14,5 @@ registry.register( advisor_factory_func=configure_and_get_tosa_advisor, target_profile_cls=TOSAConfiguration, ), + pretty_name="TOSA", ) diff --git a/src/mlia/target/tosa/reporters.py b/src/mlia/target/tosa/reporters.py index 5c015ff..9575978 100644 --- a/src/mlia/target/tosa/reporters.py +++ b/src/mlia/target/tosa/reporters.py @@ -42,13 +42,13 @@ class MetadataDisplay: # pylint: disable=too-few-public-methods self.model_name = model_meta.model_name -def report_target(target: TOSAConfiguration) -> Report: +def report_target(target_config: TOSAConfiguration) -> Report: """Generate report for the target.""" return NestedReport( "Target information", "target", [ - ReportItem("Target", alias="target", value=target.target), + ReportItem("Target", alias="target", value=target_config.target), ], ) -- cgit v1.2.1