diff options
author | Benjamin Klimczak <benjamin.klimczak@arm.com> | 2023-02-10 13:12:57 +0000 |
---|---|---|
committer | Benjamin Klimczak <benjamin.klimczak@arm.com> | 2023-02-10 14:56:43 +0000 |
commit | 718277eaece76902c4950f18d428907b39a18ef1 (patch) | |
tree | 6fda1b8f8c2c0151f3697a2e9f31c2f5551cc2ff /tests | |
parent | fa1fad9332e2912f12a44a1b07716ee434174308 (diff) | |
download | mlia-718277eaece76902c4950f18d428907b39a18ef1.tar.gz |
MLIA-769 Add "pretty names" for targets / backends
- Provide "pretty names" to print information for targets and backends.
- Use 'target_config' instead of 'target' if a target profile is
used.
- Fix minor issue in output regarding the output directory.
Change-Id: Ib38231f30b4d609a0d1e8f9c52b2fb547c69cb6a
Diffstat (limited to 'tests')
-rw-r--r-- | tests/conftest.py | 6 | ||||
-rw-r--r-- | tests/test_backend_corstone.py | 4 | ||||
-rw-r--r-- | tests/test_backend_corstone_performance.py | 4 | ||||
-rw-r--r-- | tests/test_backend_registry.py | 16 | ||||
-rw-r--r-- | tests/test_backend_vela_compat.py | 6 | ||||
-rw-r--r-- | tests/test_backend_vela_compiler.py | 6 | ||||
-rw-r--r-- | tests/test_backend_vela_performance.py | 22 | ||||
-rw-r--r-- | tests/test_cli_command_validators.py | 14 | ||||
-rw-r--r-- | tests/test_cli_main.py | 2 | ||||
-rw-r--r-- | tests/test_target_registry.py | 34 |
10 files changed, 62 insertions, 52 deletions
diff --git a/tests/conftest.py b/tests/conftest.py index 55b296f..30889ca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -84,9 +84,11 @@ def fixture_test_models_path( tflite_vela_model = tmp_path / "test_model_vela.tflite" - target_profile = EthosUConfiguration.load_profile("ethos-u55-256") + target_config = EthosUConfiguration.load_profile("ethos-u55-256") optimize_model( - tflite_model_path, target_profile.compiler_options, tflite_vela_model + tflite_model_path, + target_config.compiler_options, + tflite_vela_model, ) tf.saved_model.save(keras_model, str(tmp_path / "tf_model_test_model")) diff --git a/tests/test_backend_corstone.py b/tests/test_backend_corstone.py index 29ef084..190931a 100644 --- a/tests/test_backend_corstone.py +++ b/tests/test_backend_corstone.py @@ -6,6 +6,6 @@ from mlia.backend.corstone import is_corstone_backend def test_is_corstone_backend() -> None: """Test function is_corstone_backend.""" - assert is_corstone_backend("Corstone-300") is True - assert is_corstone_backend("Corstone-310") is True + assert is_corstone_backend("corstone-300") is True + assert is_corstone_backend("corstone-310") is True assert is_corstone_backend("New backend") is False diff --git a/tests/test_backend_corstone_performance.py b/tests/test_backend_corstone_performance.py index 2d5b196..d3d378e 100644 --- a/tests/test_backend_corstone_performance.py +++ b/tests/test_backend_corstone_performance.py @@ -81,7 +81,7 @@ def test_generic_inference_output_parser_failure(wrong_fvp_output: list[str]) -> [ [ Path("backend_path"), - "Corstone-300", + "corstone-300", "ethos-u55", 256, Path("model.tflite"), @@ -163,7 +163,7 @@ def test_estimate_performance(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr("mlia.utils.proc.command_output", command_output_mock) result = estimate_performance( - "ethos-u55", 256, Path("model.tflite"), "Corstone-300" + "ethos-u55", 256, Path("model.tflite"), "corstone-300" ) assert result == PerformanceMetrics(1, 2, 3, 4, 5, 6) diff --git a/tests/test_backend_registry.py b/tests/test_backend_registry.py index e3e2da2..cc05632 100644 --- a/tests/test_backend_registry.py +++ b/tests/test_backend_registry.py @@ -19,19 +19,19 @@ from mlia.core.common import AdviceCategory ("backend", "advices", "systems", "type_"), ( ( - "ArmNNTFLiteDelegate", + "armnn-tflite-delegate", [AdviceCategory.COMPATIBILITY], None, BackendType.BUILTIN, ), ( - "Corstone-300", + "corstone-300", [AdviceCategory.PERFORMANCE, AdviceCategory.OPTIMIZATION], [System.LINUX_AMD64], BackendType.CUSTOM, ), ( - "Corstone-310", + "corstone-310", [AdviceCategory.PERFORMANCE, AdviceCategory.OPTIMIZATION], [System.LINUX_AMD64], BackendType.CUSTOM, @@ -43,7 +43,7 @@ from mlia.core.common import AdviceCategory BackendType.WHEEL, ), ( - "Vela", + "vela", [ AdviceCategory.COMPATIBILITY, AdviceCategory.PERFORMANCE, @@ -86,11 +86,11 @@ def test_backend_registry( def test_get_supported_backends() -> None: """Test function get_supported_backends.""" assert get_supported_backends() == [ - "ArmNNTFLiteDelegate", - "Corstone-300", - "Corstone-310", - "Vela", + "armnn-tflite-delegate", + "corstone-300", + "corstone-310", "tosa-checker", + "vela", ] diff --git a/tests/test_backend_vela_compat.py b/tests/test_backend_vela_compat.py index 4e0f149..0c39dd6 100644 --- a/tests/test_backend_vela_compat.py +++ b/tests/test_backend_vela_compat.py @@ -55,9 +55,11 @@ from mlia.utils.filesystem import working_directory ) def test_operators(test_models_path: Path, model: str, expected_ops: Operators) -> None: """Test operators function.""" - target = EthosUConfiguration.load_profile("ethos-u55-256") + target_config = EthosUConfiguration.load_profile("ethos-u55-256") - operators = supported_operators(test_models_path / model, target.compiler_options) + operators = supported_operators( + test_models_path / model, target_config.compiler_options + ) for expected, actual in zip(expected_ops.ops, operators.ops): # do not compare names as they could be different on each model generation assert expected.op_type == actual.op_type diff --git a/tests/test_backend_vela_compiler.py b/tests/test_backend_vela_compiler.py index 0434ccf..9b69ada 100644 --- a/tests/test_backend_vela_compiler.py +++ b/tests/test_backend_vela_compiler.py @@ -158,8 +158,10 @@ def test_optimize_model(tmp_path: Path, test_tflite_model: Path) -> None: """Test model optimization and saving into file.""" tmp_file = tmp_path / "temp.tflite" - target = EthosUConfiguration.load_profile("ethos-u55-256") - optimize_model(test_tflite_model, target.compiler_options, tmp_file.absolute()) + target_config = EthosUConfiguration.load_profile("ethos-u55-256") + optimize_model( + test_tflite_model, target_config.compiler_options, tmp_file.absolute() + ) assert tmp_file.is_file() assert tmp_file.stat().st_size > 0 diff --git a/tests/test_backend_vela_performance.py b/tests/test_backend_vela_performance.py index 68b96ab..df2ce08 100644 --- a/tests/test_backend_vela_performance.py +++ b/tests/test_backend_vela_performance.py @@ -14,8 +14,10 @@ from mlia.target.ethos_u.config import EthosUConfiguration def test_estimate_performance(test_tflite_model: Path) -> None: """Test getting performance estimations.""" - target = EthosUConfiguration.load_profile("ethos-u55-256") - perf_metrics = estimate_performance(test_tflite_model, target.compiler_options) + target_config = EthosUConfiguration.load_profile("ethos-u55-256") + perf_metrics = estimate_performance( + test_tflite_model, target_config.compiler_options + ) assert isinstance(perf_metrics, PerformanceMetrics) @@ -24,16 +26,18 @@ def test_estimate_performance_already_optimized( tmp_path: Path, test_tflite_model: Path ) -> None: """Test that performance estimation should fail for already optimized model.""" - target = EthosUConfiguration.load_profile("ethos-u55-256") + target_config = EthosUConfiguration.load_profile("ethos-u55-256") optimized_model_path = tmp_path / "optimized_model.tflite" - optimize_model(test_tflite_model, target.compiler_options, optimized_model_path) + optimize_model( + test_tflite_model, target_config.compiler_options, optimized_model_path + ) with pytest.raises( Exception, match="Unable to estimate performance for the given optimized model" ): - estimate_performance(optimized_model_path, target.compiler_options) + estimate_performance(optimized_model_path, target_config.compiler_options) def test_read_invalid_model(test_tflite_invalid_model: Path) -> None: @@ -41,8 +45,8 @@ def test_read_invalid_model(test_tflite_invalid_model: Path) -> None: with pytest.raises( Exception, match=f"Unable to read model {test_tflite_invalid_model}" ): - target = EthosUConfiguration.load_profile("ethos-u55-256") - estimate_performance(test_tflite_invalid_model, target.compiler_options) + target_config = EthosUConfiguration.load_profile("ethos-u55-256") + estimate_performance(test_tflite_invalid_model, target_config.compiler_options) def test_compile_invalid_model( @@ -58,7 +62,7 @@ def test_compile_invalid_model( with pytest.raises( Exception, match="Model could not be optimized with Vela compiler" ): - target = EthosUConfiguration.load_profile("ethos-u55-256") - optimize_model(test_tflite_model, target.compiler_options, model_path) + target_config = EthosUConfiguration.load_profile("ethos-u55-256") + optimize_model(test_tflite_model, target_config.compiler_options, model_path) assert not model_path.exists() diff --git a/tests/test_cli_command_validators.py b/tests/test_cli_command_validators.py index 29813f4..1ac82db 100644 --- a/tests/test_cli_command_validators.py +++ b/tests/test_cli_command_validators.py @@ -109,17 +109,17 @@ def test_validate_check_target_profile( ], [ "tosa", - ["Corstone-310"], + ["corstone-310"], True, - "Backend Corstone-310 not supported with target-profile tosa.", + "Backend corstone-310 not supported with target-profile tosa.", None, ], [ "cortex-a", - ["ArmNNTFLiteDelegate"], + ["armnn-tflite-delegate"], False, None, - ["ArmNNTFLiteDelegate"], + ["armnn-tflite-delegate"], ], [ "cortex-a", @@ -130,14 +130,14 @@ def test_validate_check_target_profile( ], [ "ethos-u55-256", - ["Vela", "Corstone-310"], + ["vela", "corstone-310"], False, None, - ["Vela", "Corstone-310"], + ["vela", "corstone-310"], ], [ "ethos-u65-256", - ["Vela", "Corstone-310", "tosa-checker"], + ["vela", "corstone-310", "tosa-checker"], True, "Backend tosa-checker not supported with target-profile ethos-u65-256.", None, diff --git a/tests/test_cli_main.py b/tests/test_cli_main.py index 673031c..3472e31 100644 --- a/tests/test_cli_main.py +++ b/tests/test_cli_main.py @@ -239,7 +239,7 @@ def test_commands_execution( monkeypatch.setattr( "mlia.cli.options.get_available_backends", - MagicMock(return_value=["Vela", "some_backend"]), + MagicMock(return_value=["vela", "some_backend"]), ) for command in ["check", "optimize"]: diff --git a/tests/test_target_registry.py b/tests/test_target_registry.py index 2cbd97d..ca1ad82 100644 --- a/tests/test_target_registry.py +++ b/tests/test_target_registry.py @@ -63,13 +63,13 @@ def test_supported_advice( @pytest.mark.parametrize( ("backend", "target", "expected_result"), ( - ("ArmNNTFLiteDelegate", None, True), - ("ArmNNTFLiteDelegate", "cortex-a", True), - ("ArmNNTFLiteDelegate", "tosa", False), - ("Corstone-310", None, True), - ("Corstone-310", "ethos-u55", True), - ("Corstone-310", "ethos-u65", True), - ("Corstone-310", "cortex-a", False), + ("armnn-tflite-delegate", None, True), + ("armnn-tflite-delegate", "cortex-a", True), + ("armnn-tflite-delegate", "tosa", False), + ("corstone-310", None, True), + ("corstone-310", "ethos-u55", True), + ("corstone-310", "ethos-u65", True), + ("corstone-310", "cortex-a", False), ), ) def test_is_supported(backend: str, target: str | None, expected_result: bool) -> None: @@ -80,9 +80,9 @@ def test_is_supported(backend: str, target: str | None, expected_result: bool) - @pytest.mark.parametrize( ("target_name", "expected_backends"), ( - ("cortex-a", ["ArmNNTFLiteDelegate"]), - ("ethos-u55", ["Corstone-300", "Corstone-310", "Vela"]), - ("ethos-u65", ["Corstone-300", "Corstone-310", "Vela"]), + ("cortex-a", ["armnn-tflite-delegate"]), + ("ethos-u55", ["corstone-300", "corstone-310", "vela"]), + ("ethos-u65", ["corstone-300", "corstone-310", "vela"]), ("tosa", ["tosa-checker"]), ), ) @@ -107,21 +107,21 @@ def test_supported_targets(advice: AdviceCategory, expected_targets: list[str]) def test_all_supported_backends() -> None: """Test function all_supported_backends.""" assert all_supported_backends() == { - "Vela", + "vela", "tosa-checker", - "ArmNNTFLiteDelegate", - "Corstone-310", - "Corstone-300", + "armnn-tflite-delegate", + "corstone-310", + "corstone-300", } @pytest.mark.parametrize( ("target", "expected_default_backends", "is_subset_only"), [ - ["cortex-a", ["ArmNNTFLiteDelegate"], False], + ["cortex-a", ["armnn-tflite-delegate"], False], ["tosa", ["tosa-checker"], False], - ["ethos-u55", ["Vela"], True], - ["ethos-u65", ["Vela"], True], + ["ethos-u55", ["vela"], True], + ["ethos-u65", ["vela"], True], ], ) def test_default_backends( |