diff options
author | Benjamin Klimczak <benjamin.klimczak@arm.com> | 2022-06-27 09:10:49 +0100 |
---|---|---|
committer | Benjamin Klimczak <benjamin.klimczak@arm.com> | 2022-06-27 16:18:55 +0100 |
commit | 0ee201dfa023a9f42e1c8a282e96c231c6769e07 (patch) | |
tree | abd0048d0f1922470e3fe6befcc32f62d38a52f5 | |
parent | 7faf2c4763f299ee53b1ed100025ba50021c8313 (diff) | |
download | mlia-0ee201dfa023a9f42e1c8a282e96c231c6769e07.tar.gz |
MLIA-522 No 'perf' in mode 'all' for TFLite files
Fix the issue that no performance information is shown for TFLite files
when the mode 'all_tests' is used.
Change-Id: I8b4df4ab84ba9783b582ad449a34bf6177037e14
-rw-r--r-- | src/mlia/api.py | 3 | ||||
-rw-r--r-- | src/mlia/core/common.py | 13 | ||||
-rw-r--r-- | src/mlia/core/context.py | 6 | ||||
-rw-r--r-- | src/mlia/devices/ethosu/advice_generation.py | 3 | ||||
-rw-r--r-- | src/mlia/devices/ethosu/advisor.py | 38 | ||||
-rw-r--r-- | tests/mlia/test_api.py | 2 | ||||
-rw-r--r-- | tests/mlia/test_core_context.py | 4 | ||||
-rw-r--r-- | tests/mlia/test_devices_ethosu_advice_generation.py | 5 |
8 files changed, 38 insertions, 36 deletions
diff --git a/src/mlia/api.py b/src/mlia/api.py index 53ea4c8..0f950db 100644 --- a/src/mlia/api.py +++ b/src/mlia/api.py @@ -93,8 +93,7 @@ def get_advice( event_handlers = _get_event_handlers(output) if context is not None: - if context.advice_category is None: - context.advice_category = advice_category + context.advice_category = advice_category if context.config_parameters is None: context.config_parameters = config_parameters diff --git a/src/mlia/core/common.py b/src/mlia/core/common.py index 5fbad42..a11bf9a 100644 --- a/src/mlia/core/common.py +++ b/src/mlia/core/common.py @@ -7,7 +7,8 @@ core module. """ from abc import ABC from abc import abstractmethod -from enum import Enum +from enum import auto +from enum import Flag from typing import Any # This type is used as type alias for the items which are being passed around @@ -17,16 +18,16 @@ from typing import Any DataItem = Any -class AdviceCategory(Enum): +class AdviceCategory(Flag): """Advice category. Enumeration of advice categories supported by ML Inference Advisor. """ - OPERATORS = 1 - PERFORMANCE = 2 - OPTIMIZATION = 3 - ALL = 4 + OPERATORS = auto() + PERFORMANCE = auto() + OPTIMIZATION = auto() + ALL = OPERATORS | PERFORMANCE | OPTIMIZATION @classmethod def from_string(cls, value: str) -> "AdviceCategory": diff --git a/src/mlia/core/context.py b/src/mlia/core/context.py index 8b3dd2c..83d2f7c 100644 --- a/src/mlia/core/context.py +++ b/src/mlia/core/context.py @@ -55,7 +55,7 @@ class Context(ABC): @property @abstractmethod - def advice_category(self) -> Optional[AdviceCategory]: + def advice_category(self) -> AdviceCategory: """Return advice category.""" @property @@ -97,7 +97,7 @@ class ExecutionContext(Context): def __init__( self, *, - advice_category: Optional[AdviceCategory] = None, + advice_category: AdviceCategory = AdviceCategory.ALL, config_parameters: Optional[Mapping[str, Any]] = None, working_dir: Optional[Union[str, Path]] = None, event_handlers: Optional[List[EventHandler]] = None, @@ -141,7 +141,7 @@ class ExecutionContext(Context): self._action_resolver = action_resolver or APIActionResolver() @property - def advice_category(self) -> Optional[AdviceCategory]: + def advice_category(self) -> AdviceCategory: """Return advice category.""" return self._advice_category diff --git a/src/mlia/devices/ethosu/advice_generation.py b/src/mlia/devices/ethosu/advice_generation.py index 7a818c9..0b1352b 100644 --- a/src/mlia/devices/ethosu/advice_generation.py +++ b/src/mlia/devices/ethosu/advice_generation.py @@ -175,9 +175,6 @@ class EthosUStaticAdviceProducer(ContextAwareAdviceProducer): def get_advice(self) -> Union[Advice, List[Advice]]: """Return predefined advice based on category.""" - if self.context.advice_category is None: - return [] - advice_per_category = { AdviceCategory.PERFORMANCE: [ Advice( diff --git a/src/mlia/devices/ethosu/advisor.py b/src/mlia/devices/ethosu/advisor.py index 802826b..e93858f 100644 --- a/src/mlia/devices/ethosu/advisor.py +++ b/src/mlia/devices/ethosu/advisor.py @@ -23,6 +23,7 @@ from mlia.devices.ethosu.data_collection import EthosUOperatorCompatibility from mlia.devices.ethosu.data_collection import EthosUOptimizationPerformance from mlia.devices.ethosu.data_collection import EthosUPerformance from mlia.devices.ethosu.events import EthosUAdvisorStartedEvent +from mlia.nn.tensorflow.utils import is_tflite_model class EthosUInferenceAdvisor(InferenceAdvisor, ParameterResolverMixin): @@ -63,25 +64,30 @@ class EthosUInferenceAdvisor(InferenceAdvisor, ParameterResolverMixin): """Get collectors.""" collectors: List[DataCollector] = [] - if context.any_category_enabled( - AdviceCategory.OPERATORS, - AdviceCategory.ALL, - ): + if AdviceCategory.OPERATORS in context.advice_category: collectors.append(EthosUOperatorCompatibility(model, device)) - if context.category_enabled(AdviceCategory.PERFORMANCE): - collectors.append(EthosUPerformance(model, device, backends)) - - if context.any_category_enabled( - AdviceCategory.OPTIMIZATION, - AdviceCategory.ALL, - ): - optimization_settings = self._get_optimization_settings(context) - collectors.append( - EthosUOptimizationPerformance( - model, device, optimization_settings, backends + # Performance and optimization are mutually exclusive. + # Decide which one to use (taking into account the model format). + if is_tflite_model(model): + # TFLite models do not support optimization (only performance)! + if context.advice_category == AdviceCategory.OPTIMIZATION: + raise Exception( + "Command 'optimization' is not supported for TFLite files." ) - ) + if AdviceCategory.PERFORMANCE in context.advice_category: + collectors.append(EthosUPerformance(model, device, backends)) + else: + # Keras/SavedModel: Prefer optimization + if AdviceCategory.OPTIMIZATION in context.advice_category: + optimization_settings = self._get_optimization_settings(context) + collectors.append( + EthosUOptimizationPerformance( + model, device, optimization_settings, backends + ) + ) + elif AdviceCategory.PERFORMANCE in context.advice_category: + collectors.append(EthosUPerformance(model, device, backends)) return collectors diff --git a/tests/mlia/test_api.py b/tests/mlia/test_api.py index 54d4796..09bc509 100644 --- a/tests/mlia/test_api.py +++ b/tests/mlia/test_api.py @@ -55,7 +55,7 @@ def test_get_advice_wrong_category(test_keras_model: Path) -> None: [ "all", ExecutionContext(advice_category=AdviceCategory.PERFORMANCE), - AdviceCategory.PERFORMANCE, + AdviceCategory.ALL, ], [ "all", diff --git a/tests/mlia/test_core_context.py b/tests/mlia/test_core_context.py index 10015aa..44eb976 100644 --- a/tests/mlia/test_core_context.py +++ b/tests/mlia/test_core_context.py @@ -41,7 +41,7 @@ def test_execution_context(tmpdir: str) -> None: ) context_with_default_params = ExecutionContext(working_dir=tmpdir) - assert context_with_default_params.advice_category is None + assert context_with_default_params.advice_category is AdviceCategory.ALL assert context_with_default_params.config_parameters is None assert context_with_default_params.event_handlers is None assert isinstance( @@ -55,7 +55,7 @@ def test_execution_context(tmpdir: str) -> None: expected_str = ( f"ExecutionContext: working_dir={tmpdir}, " - "advice_category=<not set>, " + "advice_category=ALL, " "config_parameters=None, " "verbose=False" ) diff --git a/tests/mlia/test_devices_ethosu_advice_generation.py b/tests/mlia/test_devices_ethosu_advice_generation.py index 98c8a57..5d37376 100644 --- a/tests/mlia/test_devices_ethosu_advice_generation.py +++ b/tests/mlia/test_devices_ethosu_advice_generation.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 """Tests for Ethos-U advice generation.""" from typing import List -from typing import Optional import pytest @@ -387,7 +386,7 @@ def test_ethosu_advice_producer( "advice_category, action_resolver, expected_advice", [ [ - None, + AdviceCategory.ALL, None, [], ], @@ -467,7 +466,7 @@ def test_ethosu_advice_producer( ) def test_ethosu_static_advice_producer( tmpdir: str, - advice_category: Optional[AdviceCategory], + advice_category: AdviceCategory, action_resolver: ActionResolver, expected_advice: List[Advice], ) -> None: |