diff options
Diffstat (limited to 'tests')
-rw-r--r-- | tests/conftest.py | 7 | ||||
-rw-r--r-- | tests/test_api.py | 8 | ||||
-rw-r--r-- | tests/test_cli_command_validators.py | 57 | ||||
-rw-r--r-- | tests/test_cli_commands.py | 8 | ||||
-rw-r--r-- | tests/test_cli_config.py | 48 | ||||
-rw-r--r-- | tests/test_cli_helpers.py | 11 | ||||
-rw-r--r-- | tests/test_target_config.py | 63 | ||||
-rw-r--r-- | tests/test_target_ethos_u_data_analysis.py | 9 | ||||
-rw-r--r-- | tests/test_target_ethos_u_reporters.py | 5 | ||||
-rw-r--r-- | tests/test_target_registry.py | 57 |
10 files changed, 154 insertions, 119 deletions
diff --git a/tests/conftest.py b/tests/conftest.py index d797869..55b296f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -83,8 +83,11 @@ def fixture_test_models_path( save_tflite_model(tflite_model, tflite_model_path) tflite_vela_model = tmp_path / "test_model_vela.tflite" - device = EthosUConfiguration.load_profile("ethos-u55-256") - optimize_model(tflite_model_path, device.compiler_options, tflite_vela_model) + + target_profile = EthosUConfiguration.load_profile("ethos-u55-256") + optimize_model( + tflite_model_path, target_profile.compiler_options, tflite_vela_model + ) tf.saved_model.save(keras_model, str(tmp_path / "tf_model_test_model")) diff --git a/tests/test_api.py b/tests/test_api.py index 251d5ac..b40c55b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -19,12 +19,8 @@ from mlia.target.tosa.advisor import TOSAInferenceAdvisor def test_get_advice_no_target_provided(test_keras_model: Path) -> None: """Test getting advice when no target provided.""" - with pytest.raises(Exception, match="Target profile is not provided"): - get_advice( - None, # type:ignore - test_keras_model, - {"compatibility"}, - ) + with pytest.raises(Exception, match="No valid target profile was provided."): + get_advice(None, test_keras_model, {"compatibility"}) # type: ignore def test_get_advice_wrong_category(test_keras_model: Path) -> None: diff --git a/tests/test_cli_command_validators.py b/tests/test_cli_command_validators.py index cd048ee..29813f4 100644 --- a/tests/test_cli_command_validators.py +++ b/tests/test_cli_command_validators.py @@ -5,7 +5,6 @@ from __future__ import annotations import argparse from contextlib import ExitStack -from unittest.mock import MagicMock import pytest @@ -93,72 +92,66 @@ def test_validate_check_target_profile( @pytest.mark.parametrize( - "input_target_profile, input_backends, throws_exception," - "exception_message, output_backends", + ( + "input_target_profile", + "input_backends", + "throws_exception", + "exception_message", + "output_backends", + ), [ [ "tosa", - ["Vela"], - True, - "Vela backend not supported with target-profile tosa.", + ["tosa-checker"], + False, None, + ["tosa-checker"], ], [ "tosa", - ["Corstone-300, Vela"], + ["Corstone-310"], True, - "Corstone-300, Vela backend not supported with target-profile tosa.", + "Backend Corstone-310 not supported with target-profile tosa.", None, ], [ "cortex-a", - ["Corstone-310", "tosa-checker"], - True, - "Corstone-310, tosa-checker backend not supported " - "with target-profile cortex-a.", + ["ArmNNTFLiteDelegate"], + False, None, + ["ArmNNTFLiteDelegate"], ], [ - "ethos-u55-256", - ["tosa-checker", "Corstone-310"], + "cortex-a", + ["tosa-checker"], True, - "tosa-checker backend not supported with target-profile ethos-u55-256.", + "Backend tosa-checker not supported with target-profile cortex-a.", None, ], - ["tosa", None, False, None, ["tosa-checker"]], - ["cortex-a", None, False, None, ["ArmNNTFLiteDelegate"]], - ["tosa", ["tosa-checker"], False, None, ["tosa-checker"]], - ["cortex-a", ["ArmNNTFLiteDelegate"], False, None, ["ArmNNTFLiteDelegate"]], [ "ethos-u55-256", - ["Vela", "Corstone-300"], + ["Vela", "Corstone-310"], False, None, - ["Vela", "Corstone-300"], + ["Vela", "Corstone-310"], ], [ - "ethos-u55-256", - None, - False, + "ethos-u65-256", + ["Vela", "Corstone-310", "tosa-checker"], + True, + "Backend tosa-checker not supported with target-profile ethos-u65-256.", None, - ["Vela", "Corstone-300"], ], ], ) def test_validate_backend( - monkeypatch: pytest.MonkeyPatch, input_target_profile: str, - input_backends: list[str] | None, + input_backends: list[str], throws_exception: bool, exception_message: str, output_backends: list[str] | None, ) -> None: """Test backend validation with target-profiles and backends.""" - monkeypatch.setattr( - "mlia.cli.config.get_available_backends", - MagicMock(return_value=["Vela", "Corstone-300"]), - ) - exit_stack = ExitStack() if throws_exception: exit_stack.enter_context( diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 61cc5a6..f3213c4 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -33,7 +33,13 @@ def test_performance_unknown_target( sample_context: ExecutionContext, test_tflite_model: Path ) -> None: """Test that command should fail if unknown target passed.""" - with pytest.raises(Exception, match=r"File not found:*"): + with pytest.raises( + Exception, + match=( + r"Profile 'unknown' is neither a valid built-in target profile " + r"name or a valid file path." + ), + ): check( sample_context, model=str(test_tflite_model), diff --git a/tests/test_cli_config.py b/tests/test_cli_config.py deleted file mode 100644 index 8494d73..0000000 --- a/tests/test_cli_config.py +++ /dev/null @@ -1,48 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Tests for cli.config module.""" -from __future__ import annotations - -from unittest.mock import MagicMock - -import pytest - -from mlia.cli.config import get_default_backends - - -@pytest.mark.parametrize( - "available_backends, expected_default_backends", - [ - [["Vela"], ["Vela"]], - [["Corstone-300"], ["Corstone-300"]], - [["Corstone-310"], ["Corstone-310"]], - [["Corstone-300", "Corstone-310"], ["Corstone-310"]], - [["Vela", "Corstone-300", "Corstone-310"], ["Vela", "Corstone-310"]], - [ - ["Vela", "Corstone-300", "Corstone-310", "New backend"], - ["Vela", "Corstone-310", "New backend"], - ], - [ - ["Vela", "Corstone-300", "New backend"], - ["Vela", "Corstone-300", "New backend"], - ], - [["ArmNNTFLiteDelegate"], ["ArmNNTFLiteDelegate"]], - [["tosa-checker"], ["tosa-checker"]], - [ - ["ArmNNTFLiteDelegate", "Corstone-300"], - ["ArmNNTFLiteDelegate", "Corstone-300"], - ], - ], -) -def test_get_default_backends( - monkeypatch: pytest.MonkeyPatch, - available_backends: list[str], - expected_default_backends: list[str], -) -> None: - """Test function get_default backends.""" - monkeypatch.setattr( - "mlia.cli.config.get_available_backends", - MagicMock(return_value=available_backends), - ) - - assert get_default_backends() == expected_default_backends diff --git a/tests/test_cli_helpers.py b/tests/test_cli_helpers.py index 8f7e4b0..6d19207 100644 --- a/tests/test_cli_helpers.py +++ b/tests/test_cli_helpers.py @@ -3,11 +3,13 @@ """Tests for the helper classes.""" from __future__ import annotations +from pathlib import Path from typing import Any import pytest from mlia.cli.helpers import CLIActionResolver +from mlia.cli.helpers import copy_profile_file_to_output_dir from mlia.nn.tensorflow.optimizations.select import OptimizationSettings @@ -139,3 +141,12 @@ class TestCliActionResolver: """Test checking operator compatibility info.""" resolver = CLIActionResolver(args) assert resolver.check_operator_compatibility() == expected_result + + +def test_copy_profile_file_to_output_dir(tmp_path: Path) -> None: + """Test if the profile file is copied into the output directory.""" + test_target_profile_name = "ethos-u55-128" + test_file_path = Path(f"{tmp_path}/{test_target_profile_name}.toml") + + copy_profile_file_to_output_dir(test_target_profile_name, tmp_path) + assert Path.is_file(test_file_path) diff --git a/tests/test_target_config.py b/tests/test_target_config.py index c6235a5..368d394 100644 --- a/tests/test_target_config.py +++ b/tests/test_target_config.py @@ -3,35 +3,28 @@ """Tests for the backend config module.""" from __future__ import annotations -from pathlib import Path - import pytest from mlia.backend.config import BackendConfiguration from mlia.backend.config import BackendType from mlia.backend.config import System from mlia.core.common import AdviceCategory -from mlia.target.config import copy_profile_file_to_output_dir +from mlia.target.config import BUILTIN_SUPPORTED_PROFILE_NAMES +from mlia.target.config import get_builtin_profile_path from mlia.target.config import get_builtin_supported_profile_names -from mlia.target.config import get_profile_file +from mlia.target.config import is_builtin_profile from mlia.target.config import load_profile from mlia.target.config import TargetInfo from mlia.target.config import TargetProfile +from mlia.target.cortex_a.advisor import CortexAInferenceAdvisor +from mlia.target.cortex_a.config import CortexAConfiguration from mlia.utils.registry import Registry -def test_copy_profile_file_to_output_dir(tmp_path: Path) -> None: - """Test if the profile file is copied into the output directory.""" - test_target_profile_name = "ethos-u55-128" - test_file_path = Path(f"{tmp_path}/{test_target_profile_name}.toml") - - copy_profile_file_to_output_dir(test_target_profile_name, tmp_path) - assert Path.is_file(test_file_path) - - -def test_get_builtin_supported_profile_names() -> None: - """Test profile names getter.""" - assert get_builtin_supported_profile_names() == [ +def test_builtin_supported_profile_names() -> None: + """Test built-in profile names.""" + assert BUILTIN_SUPPORTED_PROFILE_NAMES == get_builtin_supported_profile_names() + assert BUILTIN_SUPPORTED_PROFILE_NAMES == [ "cortex-a", "ethos-u55-128", "ethos-u55-256", @@ -39,23 +32,24 @@ def test_get_builtin_supported_profile_names() -> None: "ethos-u65-512", "tosa", ] + for profile_name in BUILTIN_SUPPORTED_PROFILE_NAMES: + assert is_builtin_profile(profile_name) + profile_file = get_builtin_profile_path(profile_name) + assert profile_file.is_file() -def test_get_profile_file() -> None: - """Test function 'get_profile_file'.""" - profile_file = get_profile_file("cortex-a") +def test_builtin_profile_files() -> None: + """Test function 'get_bulitin_profile_file'.""" + profile_file = get_builtin_profile_path("cortex-a") assert profile_file.is_file() - assert profile_file == get_profile_file(profile_file) - with pytest.raises(Exception): - get_profile_file("UNKNOWN") - with pytest.raises(Exception): - get_profile_file("") + profile_file = get_builtin_profile_path("UNKNOWN_FILE_THAT_DOES_NOT_EXIST") + assert not profile_file.exists() def test_load_profile() -> None: """Test getting profile data.""" - profile_file = get_profile_file("ethos-u55-256") + profile_file = get_builtin_profile_path("ethos-u55-256") assert load_profile(profile_file) == { "target": "ethos-u55", "mac": 256, @@ -80,6 +74,9 @@ def test_target_profile() -> None: profile = MyTargetProfile("AnyTarget") assert profile.target == "AnyTarget" + profile = MyTargetProfile.load_json_data({"target": "MySuperTarget"}) + assert profile.target == "MySuperTarget" + profile = MyTargetProfile("") with pytest.raises(ValueError): profile.verify() @@ -101,7 +98,12 @@ def test_target_info( supported: bool, ) -> None: """Test the class 'TargetInfo'.""" - info = TargetInfo(["backend"]) + info = TargetInfo( + ["backend"], + ["backend"], + CortexAInferenceAdvisor, + CortexAConfiguration, + ) backend_registry = Registry[BackendConfiguration]() backend_registry.register( @@ -116,3 +118,12 @@ def test_target_info( assert info.is_supported(advice, check_system) == supported assert bool(info.filter_supported_backends(advice, check_system)) == supported + + info = TargetInfo( + ["unknown_backend"], + ["unknown_backend"], + CortexAInferenceAdvisor, + CortexAConfiguration, + ) + assert not info.is_supported(advice, check_system) + assert not info.filter_supported_backends(advice, check_system) diff --git a/tests/test_target_ethos_u_data_analysis.py b/tests/test_target_ethos_u_data_analysis.py index e919f5d..8e63946 100644 --- a/tests/test_target_ethos_u_data_analysis.py +++ b/tests/test_target_ethos_u_data_analysis.py @@ -3,6 +3,8 @@ """Tests for Ethos-U data analysis module.""" from __future__ import annotations +from typing import cast + import pytest from mlia.backend.vela.compat import NpuSupported @@ -23,6 +25,7 @@ from mlia.target.ethos_u.performance import MemoryUsage from mlia.target.ethos_u.performance import NPUCycles from mlia.target.ethos_u.performance import OptimizationPerformanceMetrics from mlia.target.ethos_u.performance import PerformanceMetrics +from mlia.target.registry import profile def test_perf_metrics_diff() -> None: @@ -84,7 +87,7 @@ def test_perf_metrics_diff() -> None: [ OptimizationPerformanceMetrics( PerformanceMetrics( - EthosUConfiguration.load_profile("ethos-u55-256"), + cast(EthosUConfiguration, profile("ethos-u55-256")), NPUCycles(1, 2, 3, 4, 5, 6), # memory metrics are in kilobytes MemoryUsage(*[i * 1024 for i in range(1, 6)]), # type: ignore @@ -95,7 +98,7 @@ def test_perf_metrics_diff() -> None: OptimizationSettings("pruning", 0.5, None), ], PerformanceMetrics( - EthosUConfiguration.load_profile("ethos-u55-256"), + cast(EthosUConfiguration, profile("ethos-u55-256")), NPUCycles(1, 2, 3, 4, 5, 6), # memory metrics are in kilobytes MemoryUsage( @@ -127,7 +130,7 @@ def test_perf_metrics_diff() -> None: [ OptimizationPerformanceMetrics( PerformanceMetrics( - EthosUConfiguration.load_profile("ethos-u55-256"), + cast(EthosUConfiguration, profile("ethos-u55-256")), NPUCycles(1, 2, 3, 4, 5, 6), # memory metrics are in kilobytes MemoryUsage(*[i * 1024 for i in range(1, 6)]), # type: ignore diff --git a/tests/test_target_ethos_u_reporters.py b/tests/test_target_ethos_u_reporters.py index 0c5764e..9707dff 100644 --- a/tests/test_target_ethos_u_reporters.py +++ b/tests/test_target_ethos_u_reporters.py @@ -3,6 +3,8 @@ """Tests for reports module.""" from __future__ import annotations +from typing import cast + import pytest from mlia.backend.vela.compat import NpuSupported @@ -12,6 +14,7 @@ from mlia.core.reporting import Table from mlia.target.ethos_u.config import EthosUConfiguration from mlia.target.ethos_u.reporters import report_device_details from mlia.target.ethos_u.reporters import report_operators +from mlia.target.registry import profile from mlia.utils.console import remove_ascii_codes @@ -118,7 +121,7 @@ def test_report_operators( "device, expected_plain_text, expected_json_dict", [ [ - EthosUConfiguration.load_profile("ethos-u55-256"), + cast(EthosUConfiguration, profile("ethos-u55-256")), """Device information: Target ethos-u55 MAC 256 diff --git a/tests/test_target_registry.py b/tests/test_target_registry.py index 5012148..2cbd97d 100644 --- a/tests/test_target_registry.py +++ b/tests/test_target_registry.py @@ -6,7 +6,11 @@ from __future__ import annotations import pytest from mlia.core.common import AdviceCategory +from mlia.target.config import get_builtin_profile_path from mlia.target.registry import all_supported_backends +from mlia.target.registry import default_backends +from mlia.target.registry import is_supported +from mlia.target.registry import profile from mlia.target.registry import registry from mlia.target.registry import supported_advice from mlia.target.registry import supported_backends @@ -57,6 +61,23 @@ def test_supported_advice( @pytest.mark.parametrize( + ("backend", "target", "expected_result"), + ( + ("ArmNNTFLiteDelegate", None, True), + ("ArmNNTFLiteDelegate", "cortex-a", True), + ("ArmNNTFLiteDelegate", "tosa", False), + ("Corstone-310", None, True), + ("Corstone-310", "ethos-u55", True), + ("Corstone-310", "ethos-u65", True), + ("Corstone-310", "cortex-a", False), + ), +) +def test_is_supported(backend: str, target: str | None, expected_result: bool) -> None: + """Test function is_supported().""" + assert is_supported(backend, target) == expected_result + + +@pytest.mark.parametrize( ("target_name", "expected_backends"), ( ("cortex-a", ["ArmNNTFLiteDelegate"]), @@ -92,3 +113,39 @@ def test_all_supported_backends() -> None: "Corstone-310", "Corstone-300", } + + +@pytest.mark.parametrize( + ("target", "expected_default_backends", "is_subset_only"), + [ + ["cortex-a", ["ArmNNTFLiteDelegate"], False], + ["tosa", ["tosa-checker"], False], + ["ethos-u55", ["Vela"], True], + ["ethos-u65", ["Vela"], True], + ], +) +def test_default_backends( + target: str, + expected_default_backends: list[str], + is_subset_only: bool, +) -> None: + """Test function default_backends().""" + if is_subset_only: + assert set(expected_default_backends).issubset(default_backends(target)) + else: + assert default_backends(target) == expected_default_backends + + +@pytest.mark.parametrize( + "target_profile", ("cortex-a", "tosa", "ethos-u55-128", "ethos-u65-256") +) +def test_profile(target_profile: str) -> None: + """Test function profile().""" + # Test loading by built-in profile name + cfg = profile(target_profile) + assert target_profile.startswith(cfg.target) + + # Test loading the file directly + profile_file = get_builtin_profile_path(target_profile) + cfg = profile(profile_file) + assert target_profile.startswith(cfg.target) |