From 5800fc990ed1e36ce7d06670f911fbb12a0ec771 Mon Sep 17 00:00:00 2001 From: Raul Farkas Date: Tue, 29 Nov 2022 13:29:04 +0000 Subject: MLIA-650 Implement new CLI changes Breaking change in the CLI and API: Sub-commands "optimization", "operators", and "performance" were replaced by "check", which incorporates compatibility and performance checks, and "optimize" which is used for optimization. "get_advice" API was adapted to these CLI changes. API changes: * Remove previous advice category "all" that would perform all three operations (when possible). Replace them with the ability to pass a set of the advice categories. * Update api.get_advice method docstring to reflect new changes. * Set default advice category to COMPATIBILITY * Update core.common.AdviceCategory by changing the "OPERATORS" advice category to "COMPATIBILITY" and removing "ALL" enum type. Update all subsequent methods that previously used "OPERATORS" to use "COMPATIBILITY". * Update core.context.ExecutionContext to have "COMPATIBILITY" as default advice_category instead of "ALL". * Remove api.generate_supported_operators_report and all related functions from cli.commands, cli.helpers, cli.main, cli.options, core.helpers * Update tests to reflect new API changes. CLI changes: * Update README.md to contain information on the new CLI * Remove the ability to generate supported operators support from MLIA CLI * Replace `mlia ops` and `mlia perf` with the new `mlia check` command that can be used to perform both operations. * Replace `mlia opt` with the new `mlia optimize` command. * Replace `--evaluate-on` flag with `--backend` flag * Replace `--verbose` flag with `--debug` flag (no behaviour change). * Remove the ability for the user to select MLIA working directory. Create and use a temporary directory in /temp instead. * Change behaviour of `--output` flag to not format the content automatically based on file extension anymore. Instead it will simply redirect to a file. * Add the `--json` flag to specfy that the format of the output should be json. * Add command validators that are used to validate inter-dependent flags (e.g. backend validation based on target_profile). * Add support for selecting built-in backends for both `check` and `optimize` commands. * Add new unit tests and update old ones to test the new CLI changes. * Update RELEASES.md * Update copyright notice Change-Id: Ia6340797c7bee3acbbd26601950e5a16ad5602db --- tests/test_cli_commands.py | 97 ++++++++++++---------------------------------- 1 file changed, 24 insertions(+), 73 deletions(-) (limited to 'tests/test_cli_commands.py') diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index aed5c42..03ee9d2 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for cli.commands module.""" from __future__ import annotations @@ -14,9 +14,8 @@ from mlia.backend.manager import DefaultInstallationManager from mlia.cli.commands import backend_install from mlia.cli.commands import backend_list from mlia.cli.commands import backend_uninstall -from mlia.cli.commands import operators -from mlia.cli.commands import optimization -from mlia.cli.commands import performance +from mlia.cli.commands import check +from mlia.cli.commands import optimize from mlia.core.context import ExecutionContext from mlia.target.ethos_u.config import EthosUConfiguration from mlia.target.ethos_u.performance import MemoryUsage @@ -27,7 +26,7 @@ from mlia.target.ethos_u.performance import PerformanceMetrics def test_operators_expected_parameters(sample_context: ExecutionContext) -> None: """Test operators command wrong parameters.""" with pytest.raises(Exception, match="Model is not provided"): - operators(sample_context, "ethos-u55-256") + check(sample_context, "ethos-u55-256") def test_performance_unknown_target( @@ -35,93 +34,45 @@ def test_performance_unknown_target( ) -> None: """Test that command should fail if unknown target passed.""" with pytest.raises(Exception, match="Unable to find target profile unknown"): - performance( - sample_context, model=str(test_tflite_model), target_profile="unknown" + check( + sample_context, + model=str(test_tflite_model), + target_profile="unknown", + performance=True, ) @pytest.mark.parametrize( - "target_profile, optimization_type, optimization_target, expected_error", + "target_profile, pruning, clustering, pruning_target, clustering_target", [ - [ - "ethos-u55-256", - None, - "0.5", - pytest.raises(Exception, match="Optimization type is not provided"), - ], - [ - "ethos-u65-512", - "unknown", - "16", - pytest.raises(Exception, match="Unsupported optimization type: unknown"), - ], - [ - "ethos-u55-256", - "pruning", - None, - pytest.raises(Exception, match="Optimization target is not provided"), - ], - [ - "ethos-u65-512", - "clustering", - None, - pytest.raises(Exception, match="Optimization target is not provided"), - ], - [ - "unknown", - "clustering", - "16", - pytest.raises(Exception, match="Unable to find target profile unknown"), - ], - ], -) -def test_opt_expected_parameters( - sample_context: ExecutionContext, - target_profile: str, - monkeypatch: pytest.MonkeyPatch, - optimization_type: str, - optimization_target: str, - expected_error: Any, - test_keras_model: Path, -) -> None: - """Test that command should fail if no or unknown optimization type provided.""" - mock_performance_estimation(monkeypatch) - - with expected_error: - optimization( - ctx=sample_context, - target_profile=target_profile, - model=str(test_keras_model), - optimization_type=optimization_type, - optimization_target=optimization_target, - ) - - -@pytest.mark.parametrize( - "target_profile, optimization_type, optimization_target", - [ - ["ethos-u55-256", "pruning", "0.5"], - ["ethos-u65-512", "clustering", "32"], - ["ethos-u55-256", "pruning,clustering", "0.5,32"], + ["ethos-u55-256", True, False, 0.5, None], + ["ethos-u65-512", False, True, 0.5, 32], + ["ethos-u55-256", True, True, 0.5, None], + ["ethos-u55-256", False, False, 0.5, None], + ["ethos-u55-256", False, True, "invalid", 32], ], ) def test_opt_valid_optimization_target( target_profile: str, sample_context: ExecutionContext, - optimization_type: str, - optimization_target: str, + pruning: bool, + clustering: bool, + pruning_target: float | None, + clustering_target: int | None, monkeypatch: pytest.MonkeyPatch, test_keras_model: Path, ) -> None: """Test that command should not fail with valid optimization targets.""" mock_performance_estimation(monkeypatch) - optimization( + optimize( ctx=sample_context, target_profile=target_profile, model=str(test_keras_model), - optimization_type=optimization_type, - optimization_target=optimization_target, + pruning=pruning, + clustering=clustering, + pruning_target=pruning_target, + clustering_target=clustering_target, ) -- cgit v1.2.1