diff options
author | Annie Tallund <annie.tallund@arm.com> | 2023-01-12 07:49:06 +0100 |
---|---|---|
committer | Benjamin Klimczak <benjamin.klimczak@arm.com> | 2023-02-08 15:23:29 +0000 |
commit | 836efd40317a397761ec8b66e3f4398faac43ad0 (patch) | |
tree | 5133ffd51d8d6772551333a4b337d36a501a8a91 /tests | |
parent | a4fb8c72f15146c95df16c25e75f03344e9814fd (diff) | |
download | mlia-836efd40317a397761ec8b66e3f4398faac43ad0.tar.gz |
MLIA-770 List all available backends
- Rely on target and backend registry for support information
- Make above information less Ethos(TM)-U specific
Change-Id: I8dbfb84401016412a3d719a84eb592f21d79c46b
Diffstat (limited to 'tests')
-rw-r--r-- | tests/test_backend_corstone_performance.py | 66 | ||||
-rw-r--r-- | tests/test_backend_install.py | 51 | ||||
-rw-r--r-- | tests/test_cli_config.py | 12 |
3 files changed, 52 insertions, 77 deletions
diff --git a/tests/test_backend_corstone_performance.py b/tests/test_backend_corstone_performance.py index 1734eb9..d41062f 100644 --- a/tests/test_backend_corstone_performance.py +++ b/tests/test_backend_corstone_performance.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for module backend/manager.""" from __future__ import annotations @@ -25,9 +25,8 @@ from mlia.backend.executor.application import get_application from mlia.backend.executor.execution import ExecutionContext from mlia.backend.executor.output_consumer import Base64OutputConsumer from mlia.backend.executor.system import get_system -from mlia.backend.install import get_system_name -from mlia.backend.install import is_supported -from mlia.backend.install import supported_backends +from mlia.backend.registry import get_supported_backends +from mlia.target.registry import is_supported def _mock_encode_b64(data: dict[str, int]) -> str: @@ -150,14 +149,14 @@ def test_generic_inference_output_parser( "device, system, application, backend, expected_error", [ ( - DeviceInfo(device_type="ethos-u55", mac=32), + DeviceInfo(device_type="Ethos-U55", mac=32), ("Corstone-300: Cortex-M55+Ethos-U55", True), ("Generic Inference Runner: Ethos-U55", True), "Corstone-300", does_not_raise(), ), ( - DeviceInfo(device_type="ethos-u55", mac=32), + DeviceInfo(device_type="Ethos-U55", mac=32), ("Corstone-300: Cortex-M55+Ethos-U55", False), ("Generic Inference Runner: Ethos-U55", False), "Corstone-300", @@ -167,7 +166,7 @@ def test_generic_inference_output_parser( ), ), ( - DeviceInfo(device_type="ethos-u55", mac=32), + DeviceInfo(device_type="Ethos-U55", mac=32), ("Corstone-300: Cortex-M55+Ethos-U55", True), ("Generic Inference Runner: Ethos-U55", False), "Corstone-300", @@ -178,14 +177,14 @@ def test_generic_inference_output_parser( ), ), ( - DeviceInfo(device_type="ethos-u55", mac=32), + DeviceInfo(device_type="Ethos-U55", mac=32), ("Corstone-310: Cortex-M85+Ethos-U55", True), ("Generic Inference Runner: Ethos-U55", True), "Corstone-310", does_not_raise(), ), ( - DeviceInfo(device_type="ethos-u55", mac=32), + DeviceInfo(device_type="Ethos-U55", mac=32), ("Corstone-310: Cortex-M85+Ethos-U55", False), ("Generic Inference Runner: Ethos-U55", False), "Corstone-310", @@ -195,7 +194,7 @@ def test_generic_inference_output_parser( ), ), ( - DeviceInfo(device_type="ethos-u55", mac=32), + DeviceInfo(device_type="Ethos-U55", mac=32), ("Corstone-310: Cortex-M85+Ethos-U55", True), ("Generic Inference Runner: Ethos-U55", False), "Corstone-310", @@ -206,14 +205,14 @@ def test_generic_inference_output_parser( ), ), ( - DeviceInfo(device_type="ethos-u65", mac=512), + DeviceInfo(device_type="Ethos-U65", mac=512), ("Corstone-300: Cortex-M55+Ethos-U65", True), ("Generic Inference Runner: Ethos-U65", True), "Corstone-300", does_not_raise(), ), ( - DeviceInfo(device_type="ethos-u65", mac=512), + DeviceInfo(device_type="Ethos-U65", mac=512), ("Corstone-300: Cortex-M55+Ethos-U65", False), ("Generic Inference Runner: Ethos-U65", False), "Corstone-300", @@ -223,7 +222,7 @@ def test_generic_inference_output_parser( ), ), ( - DeviceInfo(device_type="ethos-u65", mac=512), + DeviceInfo(device_type="Ethos-U65", mac=512), ("Corstone-300: Cortex-M55+Ethos-U65", True), ("Generic Inference Runner: Ethos-U65", False), "Corstone-300", @@ -234,14 +233,14 @@ def test_generic_inference_output_parser( ), ), ( - DeviceInfo(device_type="ethos-u65", mac=512), + DeviceInfo(device_type="Ethos-U65", mac=512), ("Corstone-310: Cortex-M85+Ethos-U65", True), ("Generic Inference Runner: Ethos-U65", True), "Corstone-310", does_not_raise(), ), ( - DeviceInfo(device_type="ethos-u65", mac=512), + DeviceInfo(device_type="Ethos-U65", mac=512), ("Corstone-310: Cortex-M85+Ethos-U65", False), ("Generic Inference Runner: Ethos-U65", False), "Corstone-310", @@ -251,7 +250,7 @@ def test_generic_inference_output_parser( ), ), ( - DeviceInfo(device_type="ethos-u65", mac=512), + DeviceInfo(device_type="Ethos-U65", mac=512), ("Corstone-310: Cortex-M85+Ethos-U65", True), ("Generic Inference Runner: Ethos-U65", False), "Corstone-310", @@ -349,7 +348,7 @@ def test_estimate_performance_insufficient_data( with pytest.raises( Exception, match="Unable to get performance metrics, insufficient data" ): - device = DeviceInfo(device_type="ethos-u55", mac=32) + device = DeviceInfo(device_type="Ethos-U55", mac=32) estimate_performance(ModelInfo(test_tflite_model), device, backend) @@ -388,7 +387,7 @@ def test_estimate_performance_invalid_output( with pytest.raises(Exception, match="Unable to get performance metrics"): estimate_performance( ModelInfo(test_tflite_model), - DeviceInfo(device_type="ethos-u55", mac=256), + DeviceInfo(device_type="Ethos-U55", mac=256), backend=backend, ) @@ -396,7 +395,7 @@ def test_estimate_performance_invalid_output( @pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) def test_get_generic_runner(backend: str) -> None: """Test function get_generic_runner().""" - device_info = DeviceInfo("ethos-u55", 256) + device_info = DeviceInfo("Ethos-U55", 256) runner = get_generic_runner(device_info=device_info, backend=backend) assert isinstance(runner, GenericInferenceRunnerEthosU) @@ -408,9 +407,12 @@ def test_get_generic_runner(backend: str) -> None: @pytest.mark.parametrize( ("backend", "device_type"), ( - ("Corstone-300", "ethos-u55"), - ("Corstone-300", "ethos-u65"), - ("Corstone-310", "ethos-u55"), + ("Corstone-300", "Ethos-U55"), + ("Corstone-300", "Ethos-U65"), + ("Corstone-310", "Ethos-U55"), + ("ArmNNTFLiteDelegate", "Cortex-A"), + ("TOSA-Checker", "TOSA"), + ("Corstone-300", None), ), ) def test_backend_support(backend: str, device_type: str) -> None: @@ -418,9 +420,7 @@ def test_backend_support(backend: str, device_type: str) -> None: assert is_supported(backend) assert is_supported(backend, device_type) - assert get_system_name(backend, device_type) - - assert backend in supported_backends() + assert backend in get_supported_backends() class TestGenericInferenceRunnerEthosU: @@ -431,25 +431,25 @@ class TestGenericInferenceRunnerEthosU: "device, backend, expected_system, expected_app", [ [ - DeviceInfo("ethos-u55", 256), + DeviceInfo("Ethos-U55", 256), "Corstone-300", "Corstone-300: Cortex-M55+Ethos-U55", "Generic Inference Runner: Ethos-U55", ], [ - DeviceInfo("ethos-u65", 256), + DeviceInfo("Ethos-U65", 256), "Corstone-300", "Corstone-300: Cortex-M55+Ethos-U65", "Generic Inference Runner: Ethos-U65", ], [ - DeviceInfo("ethos-u55", 256), + DeviceInfo("Ethos-U55", 256), "Corstone-310", "Corstone-310: Cortex-M85+Ethos-U55", "Generic Inference Runner: Ethos-U55", ], [ - DeviceInfo("ethos-u65", 256), + DeviceInfo("Ethos-U65", 256), "Corstone-310", "Corstone-310: Cortex-M85+Ethos-U65", "Generic Inference Runner: Ethos-U65", @@ -470,9 +470,9 @@ class TestGenericInferenceRunnerEthosU: def test_artifact_resolver_unsupported_backend() -> None: """Test that it should be not possible to use unsupported backends.""" with pytest.raises( - RuntimeError, match="Unsupported device ethos-u65 for backend test_backend" + RuntimeError, match="Unsupported device Ethos-U65 for backend test_backend" ): - get_generic_runner(DeviceInfo("ethos-u65", 256), "test_backend") + get_generic_runner(DeviceInfo("Ethos-U65", 256), "test_backend") @staticmethod @pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) @@ -482,7 +482,7 @@ class TestGenericInferenceRunnerEthosU: """Test that inference should fail if system is not installed.""" backend_runner.is_system_installed.return_value = False - generic_runner = get_generic_runner(DeviceInfo("ethos-u55", 256), backend) + generic_runner = get_generic_runner(DeviceInfo("Ethos-U55", 256), backend) with pytest.raises( Exception, match=r"System Corstone-3[01]0: Cortex-M[58]5\+Ethos-U55 is not installed", @@ -498,7 +498,7 @@ class TestGenericInferenceRunnerEthosU: backend_runner.is_system_installed.return_value = True backend_runner.is_application_installed.return_value = False - generic_runner = get_generic_runner(DeviceInfo("ethos-u55", 256), backend) + generic_runner = get_generic_runner(DeviceInfo("Ethos-U55", 256), backend) with pytest.raises( Exception, match="Application Generic Inference Runner: Ethos-U55" diff --git a/tests/test_backend_install.py b/tests/test_backend_install.py index 024a833..c3efe09 100644 --- a/tests/test_backend_install.py +++ b/tests/test_backend_install.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for common management functionality.""" from __future__ import annotations @@ -9,11 +9,9 @@ import pytest from mlia.backend.install import BackendInfo from mlia.backend.install import get_all_application_names -from mlia.backend.install import get_all_system_names -from mlia.backend.install import get_system_name -from mlia.backend.install import is_supported from mlia.backend.install import StaticPathChecker -from mlia.backend.install import supported_backends +from mlia.backend.registry import get_supported_backends +from mlia.target.registry import is_supported @pytest.mark.parametrize( @@ -49,7 +47,13 @@ def test_static_path_checker_invalid_path(tmp_path: Path) -> None: def test_supported_backends() -> None: """Test function supported backends.""" - assert supported_backends() == ["Corstone-300", "Corstone-310"] + assert get_supported_backends() == [ + "ArmNNTFLiteDelegate", + "Corstone-300", + "Corstone-310", + "TOSA-Checker", + "Vela", + ] @pytest.mark.parametrize( @@ -71,30 +75,6 @@ def test_is_supported(backend: str, expected_result: bool) -> None: [ "Corstone-300", [ - "Corstone-300: Cortex-M55+Ethos-U55", - "Corstone-300: Cortex-M55+Ethos-U65", - ], - ], - [ - "Corstone-310", - [ - "Corstone-310: Cortex-M85+Ethos-U55", - "Corstone-310: Cortex-M85+Ethos-U65", - ], - ], - ], -) -def test_get_all_system_names(backend: str, expected_result: list[str]) -> None: - """Test function get_all_system_names.""" - assert sorted(get_all_system_names(backend)) == expected_result - - -@pytest.mark.parametrize( - "backend, expected_result", - [ - [ - "Corstone-300", - [ "Generic Inference Runner: Ethos-U55", "Generic Inference Runner: Ethos-U65", ], @@ -111,14 +91,3 @@ def test_get_all_system_names(backend: str, expected_result: list[str]) -> None: def test_get_all_application_names(backend: str, expected_result: list[str]) -> None: """Test function get_all_application_names.""" assert sorted(get_all_application_names(backend)) == expected_result - - -def test_get_system_name() -> None: - """Test function get_system_name.""" - assert ( - get_system_name("Corstone-300", "ethos-u55") - == "Corstone-300: Cortex-M55+Ethos-U55" - ) - - with pytest.raises(KeyError): - get_system_name("some_backend", "some_type") diff --git a/tests/test_cli_config.py b/tests/test_cli_config.py index b007052..1487f11 100644 --- a/tests/test_cli_config.py +++ b/tests/test_cli_config.py @@ -7,7 +7,7 @@ from unittest.mock import MagicMock import pytest -from mlia.cli.config import get_ethos_u_default_backends +from mlia.cli.config import get_default_backends from mlia.cli.config import is_corstone_backend @@ -27,9 +27,15 @@ from mlia.cli.config import is_corstone_backend ["Vela", "Corstone-300", "New backend"], ["Vela", "Corstone-300", "New backend"], ], + [["ArmNNTFLiteDelegate"], ["ArmNNTFLiteDelegate"]], + [["TOSA-Checker"], ["TOSA-Checker"]], + [ + ["ArmNNTFLiteDelegate", "Corstone-300"], + ["ArmNNTFLiteDelegate", "Corstone-300"], + ], ], ) -def test_get_ethos_u_default_backends( +def test_get_default_backends( monkeypatch: pytest.MonkeyPatch, available_backends: list[str], expected_default_backends: list[str], @@ -40,7 +46,7 @@ def test_get_ethos_u_default_backends( MagicMock(return_value=available_backends), ) - assert get_ethos_u_default_backends() == expected_default_backends + assert get_default_backends() == expected_default_backends def test_is_corstone_backend() -> None: |