From 5800fc990ed1e36ce7d06670f911fbb12a0ec771 Mon Sep 17 00:00:00 2001 From: Raul Farkas Date: Tue, 29 Nov 2022 13:29:04 +0000 Subject: MLIA-650 Implement new CLI changes Breaking change in the CLI and API: Sub-commands "optimization", "operators", and "performance" were replaced by "check", which incorporates compatibility and performance checks, and "optimize" which is used for optimization. "get_advice" API was adapted to these CLI changes. API changes: * Remove previous advice category "all" that would perform all three operations (when possible). Replace them with the ability to pass a set of the advice categories. * Update api.get_advice method docstring to reflect new changes. * Set default advice category to COMPATIBILITY * Update core.common.AdviceCategory by changing the "OPERATORS" advice category to "COMPATIBILITY" and removing "ALL" enum type. Update all subsequent methods that previously used "OPERATORS" to use "COMPATIBILITY". * Update core.context.ExecutionContext to have "COMPATIBILITY" as default advice_category instead of "ALL". * Remove api.generate_supported_operators_report and all related functions from cli.commands, cli.helpers, cli.main, cli.options, core.helpers * Update tests to reflect new API changes. CLI changes: * Update README.md to contain information on the new CLI * Remove the ability to generate supported operators support from MLIA CLI * Replace `mlia ops` and `mlia perf` with the new `mlia check` command that can be used to perform both operations. * Replace `mlia opt` with the new `mlia optimize` command. * Replace `--evaluate-on` flag with `--backend` flag * Replace `--verbose` flag with `--debug` flag (no behaviour change). * Remove the ability for the user to select MLIA working directory. Create and use a temporary directory in /temp instead. * Change behaviour of `--output` flag to not format the content automatically based on file extension anymore. Instead it will simply redirect to a file. * Add the `--json` flag to specfy that the format of the output should be json. * Add command validators that are used to validate inter-dependent flags (e.g. backend validation based on target_profile). * Add support for selecting built-in backends for both `check` and `optimize` commands. * Add new unit tests and update old ones to test the new CLI changes. * Update RELEASES.md * Update copyright notice Change-Id: Ia6340797c7bee3acbbd26601950e5a16ad5602db --- src/mlia/target/ethos_u/advice_generation.py | 34 ++++++++++++++++------------ src/mlia/target/ethos_u/advisor.py | 14 ++++++------ src/mlia/target/ethos_u/handlers.py | 6 ++--- 3 files changed, 29 insertions(+), 25 deletions(-) (limited to 'src/mlia/target/ethos_u') diff --git a/src/mlia/target/ethos_u/advice_generation.py b/src/mlia/target/ethos_u/advice_generation.py index edd78fd..daae4f4 100644 --- a/src/mlia/target/ethos_u/advice_generation.py +++ b/src/mlia/target/ethos_u/advice_generation.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Ethos-U advice generation.""" from __future__ import annotations @@ -26,7 +26,7 @@ class EthosUAdviceProducer(FactBasedAdviceProducer): """Produce advice.""" @produce_advice.register - @advice_category(AdviceCategory.OPERATORS, AdviceCategory.ALL) + @advice_category(AdviceCategory.COMPATIBILITY) def handle_cpu_only_ops(self, data_item: HasCPUOnlyOperators) -> None: """Advice for CPU only operators.""" cpu_only_ops = ",".join(sorted(set(data_item.cpu_only_ops))) @@ -40,11 +40,10 @@ class EthosUAdviceProducer(FactBasedAdviceProducer): "Using operators that are supported by the NPU will " "improve performance.", ] - + self.context.action_resolver.supported_operators_info() ) @produce_advice.register - @advice_category(AdviceCategory.OPERATORS, AdviceCategory.ALL) + @advice_category(AdviceCategory.COMPATIBILITY) def handle_unsupported_operators( self, data_item: HasUnsupportedOnNPUOperators ) -> None: @@ -60,21 +59,25 @@ class EthosUAdviceProducer(FactBasedAdviceProducer): ) @produce_advice.register - @advice_category(AdviceCategory.OPERATORS, AdviceCategory.ALL) + @advice_category(AdviceCategory.COMPATIBILITY) def handle_all_operators_supported( self, _data_item: AllOperatorsSupportedOnNPU ) -> None: """Advice if all operators supported.""" - self.add_advice( - [ - "You don't have any unsupported operators, your model will " - "run completely on NPU." - ] - + self.context.action_resolver.check_performance() - ) + advice = [ + "You don't have any unsupported operators, your model will " + "run completely on NPU." + ] + if self.context.advice_category != ( + AdviceCategory.COMPATIBILITY, + AdviceCategory.PERFORMANCE, + ): + advice += self.context.action_resolver.check_performance() + + self.add_advice(advice) @produce_advice.register - @advice_category(AdviceCategory.OPTIMIZATION, AdviceCategory.ALL) + @advice_category(AdviceCategory.OPTIMIZATION) def handle_optimization_results(self, data_item: OptimizationResults) -> None: """Advice based on optimization results.""" if not data_item.diffs or len(data_item.diffs) != 1: @@ -202,5 +205,6 @@ class EthosUStaticAdviceProducer(ContextAwareAdviceProducer): ) ], } - - return advice_per_category.get(self.context.advice_category, []) + if len(self.context.advice_category) == 1: + return advice_per_category.get(list(self.context.advice_category)[0], []) + return [] diff --git a/src/mlia/target/ethos_u/advisor.py b/src/mlia/target/ethos_u/advisor.py index b9d64ff..640c3e1 100644 --- a/src/mlia/target/ethos_u/advisor.py +++ b/src/mlia/target/ethos_u/advisor.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Ethos-U MLIA module.""" from __future__ import annotations @@ -10,12 +10,12 @@ from mlia.core.advice_generation import AdviceProducer from mlia.core.advisor import DefaultInferenceAdvisor from mlia.core.advisor import InferenceAdvisor from mlia.core.common import AdviceCategory +from mlia.core.common import FormattedFilePath from mlia.core.context import Context from mlia.core.context import ExecutionContext from mlia.core.data_analysis import DataAnalyzer from mlia.core.data_collection import DataCollector from mlia.core.events import Event -from mlia.core.typing import PathOrFileLike from mlia.nn.tensorflow.utils import is_tflite_model from mlia.target.ethos_u.advice_generation import EthosUAdviceProducer from mlia.target.ethos_u.advice_generation import EthosUStaticAdviceProducer @@ -46,7 +46,7 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor): collectors: list[DataCollector] = [] - if AdviceCategory.OPERATORS in context.advice_category: + if context.category_enabled(AdviceCategory.COMPATIBILITY): collectors.append(EthosUOperatorCompatibility(model, device)) # Performance and optimization are mutually exclusive. @@ -57,18 +57,18 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor): raise Exception( "Command 'optimization' is not supported for TensorFlow Lite files." ) - if AdviceCategory.PERFORMANCE in context.advice_category: + if context.category_enabled(AdviceCategory.PERFORMANCE): collectors.append(EthosUPerformance(model, device, backends)) else: # Keras/SavedModel: Prefer optimization - if AdviceCategory.OPTIMIZATION in context.advice_category: + if context.category_enabled(AdviceCategory.OPTIMIZATION): optimization_settings = self._get_optimization_settings(context) collectors.append( EthosUOptimizationPerformance( model, device, optimization_settings, backends ) ) - elif AdviceCategory.PERFORMANCE in context.advice_category: + elif context.category_enabled(AdviceCategory.PERFORMANCE): collectors.append(EthosUPerformance(model, device, backends)) return collectors @@ -126,7 +126,7 @@ def configure_and_get_ethosu_advisor( context: ExecutionContext, target_profile: str, model: str | Path, - output: PathOrFileLike | None = None, + output: FormattedFilePath | None = None, **extra_args: Any, ) -> InferenceAdvisor: """Create and configure Ethos-U advisor.""" diff --git a/src/mlia/target/ethos_u/handlers.py b/src/mlia/target/ethos_u/handlers.py index 84a9554..91f6015 100644 --- a/src/mlia/target/ethos_u/handlers.py +++ b/src/mlia/target/ethos_u/handlers.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Event handler.""" from __future__ import annotations @@ -6,9 +6,9 @@ from __future__ import annotations import logging from mlia.backend.vela.compat import Operators +from mlia.core.common import FormattedFilePath from mlia.core.events import CollectedDataEvent from mlia.core.handlers import WorkflowEventsHandler -from mlia.core.typing import PathOrFileLike from mlia.target.ethos_u.events import EthosUAdvisorEventHandler from mlia.target.ethos_u.events import EthosUAdvisorStartedEvent from mlia.target.ethos_u.performance import OptimizationPerformanceMetrics @@ -21,7 +21,7 @@ logger = logging.getLogger(__name__) class EthosUEventHandler(WorkflowEventsHandler, EthosUAdvisorEventHandler): """CLI event handler.""" - def __init__(self, output: PathOrFileLike | None = None) -> None: + def __init__(self, output: FormattedFilePath | None = None) -> None: """Init event handler.""" super().__init__(ethos_u_formatters, output) -- cgit v1.2.1