diff options
author | Benjamin Klimczak <benjamin.klimczak@arm.com> | 2022-07-11 12:33:42 +0100 |
---|---|---|
committer | Benjamin Klimczak <benjamin.klimczak@arm.com> | 2022-07-26 14:08:21 +0100 |
commit | 5d81f37de09efe10f90512e50252be9c36925fcf (patch) | |
tree | b4d7cdfd051da0a6e882bdfcf280fd7ca7b39e57 /tests/test_cli_commands.py | |
parent | 7899b908c1fe6d86b92a80f3827ddd0ac05b674b (diff) | |
download | mlia-5d81f37de09efe10f90512e50252be9c36925fcf.tar.gz |
MLIA-551 Rework remains of AIET architecture
Re-factoring the code base to further merge the old AIET code into MLIA.
- Remove last traces of the backend type 'tool'
- Controlled systems removed, including SSH protocol, controller,
RunningCommand, locks etc.
- Build command / build dir and deploy functionality removed from
Applications and Systems
- Moving working_dir()
- Replace module 'output_parser' with new module 'output_consumer' and
merge Base64 parsing into it
- Change the output consumption to optionally remove (i.e. actually
consume) lines
- Use Base64 parsing in GenericInferenceOutputParser, replacing the
regex-based parsing and remove the now unused regex parsing
- Remove AIET reporting
- Pre-install applications by moving them to src/mlia/resources/backends
- Rename aiet-config.json to backend-config.json
- Move tests from tests/mlia/ to tests/
- Adapt unit tests to code changes
- Dependencies removed: paramiko, filelock, psutil
- Fix bug in corstone.py: The wrong resource directory was used which
broke the functionality to download backends.
- Use f-string formatting.
- Use logging instead of print.
Change-Id: I768bc3bb6b2eda57d219ad01be4a8e0a74167d76
Diffstat (limited to 'tests/test_cli_commands.py')
-rw-r--r-- | tests/test_cli_commands.py | 204 |
1 files changed, 204 insertions, 0 deletions
diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py new file mode 100644 index 0000000..bf17339 --- /dev/null +++ b/tests/test_cli_commands.py @@ -0,0 +1,204 @@ +# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for cli.commands module.""" +from pathlib import Path +from typing import Any +from typing import Optional +from unittest.mock import call +from unittest.mock import MagicMock + +import pytest + +from mlia.cli.commands import backend +from mlia.cli.commands import operators +from mlia.cli.commands import optimization +from mlia.cli.commands import performance +from mlia.core.context import ExecutionContext +from mlia.devices.ethosu.config import EthosUConfiguration +from mlia.devices.ethosu.performance import MemoryUsage +from mlia.devices.ethosu.performance import NPUCycles +from mlia.devices.ethosu.performance import PerformanceMetrics +from mlia.tools.metadata.common import InstallationManager + + +def test_operators_expected_parameters(dummy_context: ExecutionContext) -> None: + """Test operators command wrong parameters.""" + with pytest.raises(Exception, match="Model is not provided"): + operators(dummy_context, "ethos-u55-256") + + +def test_performance_unknown_target( + dummy_context: ExecutionContext, test_tflite_model: Path +) -> None: + """Test that command should fail if unknown target passed.""" + with pytest.raises(Exception, match="Unable to find target profile unknown"): + performance( + dummy_context, model=str(test_tflite_model), target_profile="unknown" + ) + + +@pytest.mark.parametrize( + "target_profile, optimization_type, optimization_target, expected_error", + [ + [ + "ethos-u55-256", + None, + "0.5", + pytest.raises(Exception, match="Optimization type is not provided"), + ], + [ + "ethos-u65-512", + "unknown", + "16", + pytest.raises(Exception, match="Unsupported optimization type: unknown"), + ], + [ + "ethos-u55-256", + "pruning", + None, + pytest.raises(Exception, match="Optimization target is not provided"), + ], + [ + "ethos-u65-512", + "clustering", + None, + pytest.raises(Exception, match="Optimization target is not provided"), + ], + [ + "unknown", + "clustering", + "16", + pytest.raises(Exception, match="Unable to find target profile unknown"), + ], + ], +) +def test_opt_expected_parameters( + dummy_context: ExecutionContext, + target_profile: str, + monkeypatch: pytest.MonkeyPatch, + optimization_type: str, + optimization_target: str, + expected_error: Any, + test_keras_model: Path, +) -> None: + """Test that command should fail if no or unknown optimization type provided.""" + mock_performance_estimation(monkeypatch) + + with expected_error: + optimization( + ctx=dummy_context, + target_profile=target_profile, + model=str(test_keras_model), + optimization_type=optimization_type, + optimization_target=optimization_target, + ) + + +@pytest.mark.parametrize( + "target_profile, optimization_type, optimization_target", + [ + ["ethos-u55-256", "pruning", "0.5"], + ["ethos-u65-512", "clustering", "32"], + ["ethos-u55-256", "pruning,clustering", "0.5,32"], + ], +) +def test_opt_valid_optimization_target( + target_profile: str, + dummy_context: ExecutionContext, + optimization_type: str, + optimization_target: str, + monkeypatch: pytest.MonkeyPatch, + test_keras_model: Path, +) -> None: + """Test that command should not fail with valid optimization targets.""" + mock_performance_estimation(monkeypatch) + + optimization( + ctx=dummy_context, + target_profile=target_profile, + model=str(test_keras_model), + optimization_type=optimization_type, + optimization_target=optimization_target, + ) + + +def mock_performance_estimation(monkeypatch: pytest.MonkeyPatch) -> None: + """Mock performance estimation.""" + metrics = PerformanceMetrics( + EthosUConfiguration("ethos-u55-256"), + NPUCycles(1, 2, 3, 4, 5, 6), + MemoryUsage(1, 2, 3, 4, 5), + ) + monkeypatch.setattr( + "mlia.devices.ethosu.data_collection.EthosUPerformanceEstimator.estimate", + MagicMock(return_value=metrics), + ) + + +@pytest.fixture(name="installation_manager_mock") +def fixture_mock_installation_manager(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + """Mock installation manager.""" + install_manager_mock = MagicMock(spec=InstallationManager) + monkeypatch.setattr( + "mlia.cli.commands.get_installation_manager", + MagicMock(return_value=install_manager_mock), + ) + return install_manager_mock + + +def test_backend_command_action_status(installation_manager_mock: MagicMock) -> None: + """Test backend command "status".""" + backend(backend_action="status") + + installation_manager_mock.show_env_details.assert_called_once() + + +@pytest.mark.parametrize( + "i_agree_to_the_contained_eula, backend_name, expected_calls", + [ + [False, None, [call(None, True)]], + [True, None, [call(None, False)]], + [False, "backend_name", [call("backend_name", True)]], + [True, "backend_name", [call("backend_name", False)]], + ], +) +def test_backend_command_action_add_downoad( + installation_manager_mock: MagicMock, + i_agree_to_the_contained_eula: bool, + backend_name: Optional[str], + expected_calls: Any, +) -> None: + """Test backend command "install" with download option.""" + backend( + backend_action="install", + download=True, + name=backend_name, + i_agree_to_the_contained_eula=i_agree_to_the_contained_eula, + ) + + assert installation_manager_mock.download_and_install.mock_calls == expected_calls + + +@pytest.mark.parametrize("backend_name", [None, "backend_name"]) +def test_backend_command_action_install_from_path( + installation_manager_mock: MagicMock, + tmp_path: Path, + backend_name: Optional[str], +) -> None: + """Test backend command "install" with backend path.""" + backend(backend_action="install", path=tmp_path, name=backend_name) + + installation_manager_mock.install_from(tmp_path, backend_name) + + +def test_backend_command_action_install_only_one_action( + installation_manager_mock: MagicMock, # pylint: disable=unused-argument + tmp_path: Path, +) -> None: + """Test that only one of action type allowed.""" + with pytest.raises( + Exception, + match="Please select only one action: download or " + "provide path to the backend installation", + ): + backend(backend_action="install", download=True, path=tmp_path) |