diff options
Diffstat (limited to 'tests')
-rw-r--r-- | tests/aiet/__init__.py | 3 | ||||
-rw-r--r-- | tests/aiet/conftest.py | 139 | ||||
-rw-r--r-- | tests/aiet/test_backend_tool.py | 60 | ||||
-rw-r--r-- | tests/aiet/test_check_model.py | 162 | ||||
-rw-r--r-- | tests/aiet/test_cli.py | 37 | ||||
-rw-r--r-- | tests/aiet/test_cli_application.py | 1153 | ||||
-rw-r--r-- | tests/aiet/test_cli_common.py | 37 | ||||
-rw-r--r-- | tests/aiet/test_cli_system.py | 240 | ||||
-rw-r--r-- | tests/aiet/test_cli_tool.py | 333 | ||||
-rw-r--r-- | tests/aiet/test_main.py | 16 | ||||
-rw-r--r-- | tests/aiet/test_resources/tools/tool1/aiet-config.json | 30 | ||||
-rw-r--r-- | tests/aiet/test_resources/tools/tool2/aiet-config.json | 26 | ||||
-rw-r--r-- | tests/aiet/test_run_vela_script.py | 152 | ||||
-rw-r--r-- | tests/aiet/test_utils_helpers.py | 27 | ||||
-rw-r--r-- | tests/mlia/conftest.py | 91 | ||||
-rw-r--r-- | tests/mlia/test_backend_application.py (renamed from tests/aiet/test_backend_application.py) | 50 | ||||
-rw-r--r-- | tests/mlia/test_backend_common.py (renamed from tests/aiet/test_backend_common.py) | 36 | ||||
-rw-r--r-- | tests/mlia/test_backend_controller.py (renamed from tests/aiet/test_backend_controller.py) | 8 | ||||
-rw-r--r-- | tests/mlia/test_backend_execution.py (renamed from tests/aiet/test_backend_execution.py) | 110 | ||||
-rw-r--r-- | tests/mlia/test_backend_fs.py (renamed from tests/aiet/test_utils_fs.py) | 28 | ||||
-rw-r--r-- | tests/mlia/test_backend_manager.py (renamed from tests/mlia/test_tools_aiet_wrapper.py) | 326 | ||||
-rw-r--r-- | tests/mlia/test_backend_output_parser.py (renamed from tests/aiet/test_backend_output_parser.py) | 6 | ||||
-rw-r--r-- | tests/mlia/test_backend_proc.py (renamed from tests/aiet/test_utils_proc.py) | 26 | ||||
-rw-r--r-- | tests/mlia/test_backend_protocol.py (renamed from tests/aiet/test_backend_protocol.py) | 24 | ||||
-rw-r--r-- | tests/mlia/test_backend_source.py (renamed from tests/aiet/test_backend_source.py) | 18 | ||||
-rw-r--r-- | tests/mlia/test_backend_system.py (renamed from tests/aiet/test_backend_system.py) | 69 | ||||
-rw-r--r-- | tests/mlia/test_cli_logging.py | 10 | ||||
-rw-r--r-- | tests/mlia/test_devices_ethosu_performance.py | 2 | ||||
-rw-r--r-- | tests/mlia/test_resources/application_config.json (renamed from tests/aiet/test_resources/application_config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/application_config.json.license (renamed from tests/aiet/test_resources/application_config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application1/aiet-config.json (renamed from tests/aiet/test_resources/applications/application1/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application1/aiet-config.json.license (renamed from tests/aiet/test_resources/applications/application1/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application2/aiet-config.json (renamed from tests/aiet/test_resources/applications/application2/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application2/aiet-config.json.license (renamed from tests/aiet/test_resources/applications/application2/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application3/readme.txt (renamed from tests/aiet/test_resources/applications/application3/readme.txt) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application4/aiet-config.json (renamed from tests/aiet/test_resources/applications/application4/aiet-config.json) | 5 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application4/aiet-config.json.license (renamed from tests/aiet/test_resources/applications/application4/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application4/hello_app.txt (renamed from tests/aiet/test_resources/applications/application4/hello_app.txt) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application5/aiet-config.json (renamed from tests/aiet/test_resources/applications/application5/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application5/aiet-config.json.license (renamed from tests/aiet/test_resources/applications/application5/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application6/aiet-config.json | 42 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application6/aiet-config.json.license (renamed from tests/aiet/test_resources/systems/system1/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/readme.txt (renamed from tests/aiet/test_resources/applications/readme.txt) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system1/aiet-config.json (renamed from tests/aiet/test_resources/systems/system1/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system1/aiet-config.json.license (renamed from tests/aiet/test_resources/systems/system2/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system1/system_artifact/dummy.txt (renamed from tests/aiet/test_resources/systems/system1/system_artifact/dummy.txt) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system2/aiet-config.json (renamed from tests/aiet/test_resources/systems/system2/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system2/aiet-config.json.license (renamed from tests/aiet/test_resources/systems/system4/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system3/readme.txt (renamed from tests/aiet/test_resources/systems/system3/readme.txt) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system4/aiet-config.json (renamed from tests/aiet/test_resources/systems/system4/aiet-config.json) | 2 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system4/aiet-config.json.license (renamed from tests/aiet/test_resources/tools/tool1/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system6/aiet-config.json | 34 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system6/aiet-config.json.license (renamed from tests/aiet/test_resources/tools/tool2/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/hello_world.json (renamed from tests/aiet/test_resources/hello_world.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/hello_world.json.license (renamed from tests/aiet/test_resources/hello_world.json.license) | 0 | ||||
-rwxr-xr-x | tests/mlia/test_resources/scripts/test_backend_run (renamed from tests/aiet/test_resources/scripts/test_backend_run) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/scripts/test_backend_run_script.sh (renamed from tests/aiet/test_resources/scripts/test_backend_run_script.sh) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json (renamed from tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json.license (renamed from tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json (renamed from tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json.license (renamed from tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_tools_metadata_corstone.py | 90 |
72 files changed, 591 insertions, 2801 deletions
diff --git a/tests/aiet/__init__.py b/tests/aiet/__init__.py deleted file mode 100644 index 873a7df..0000000 --- a/tests/aiet/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""AIET tests module.""" diff --git a/tests/aiet/conftest.py b/tests/aiet/conftest.py deleted file mode 100644 index cab3dc2..0000000 --- a/tests/aiet/conftest.py +++ /dev/null @@ -1,139 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=redefined-outer-name -"""conftest for pytest.""" -import shutil -import tarfile -from pathlib import Path -from typing import Any - -import pytest -from click.testing import CliRunner - -from aiet.backend.common import get_backend_configs - - -@pytest.fixture(scope="session") -def test_systems_path(test_resources_path: Path) -> Path: - """Return test systems path in a pytest fixture.""" - return test_resources_path / "systems" - - -@pytest.fixture(scope="session") -def test_applications_path(test_resources_path: Path) -> Path: - """Return test applications path in a pytest fixture.""" - return test_resources_path / "applications" - - -@pytest.fixture(scope="session") -def test_tools_path(test_resources_path: Path) -> Path: - """Return test tools path in a pytest fixture.""" - return test_resources_path / "tools" - - -@pytest.fixture(scope="session") -def test_resources_path() -> Path: - """Return test resources path in a pytest fixture.""" - current_path = Path(__file__).parent.absolute() - return current_path / "test_resources" - - -@pytest.fixture(scope="session") -def non_optimised_input_model_file(test_tflite_model: Path) -> Path: - """Provide the path to a quantized dummy model file.""" - return test_tflite_model - - -@pytest.fixture(scope="session") -def optimised_input_model_file(test_tflite_vela_model: Path) -> Path: - """Provide path to Vela-optimised dummy model file.""" - return test_tflite_vela_model - - -@pytest.fixture(scope="session") -def invalid_input_model_file(test_tflite_invalid_model: Path) -> Path: - """Provide the path to an invalid dummy model file.""" - return test_tflite_invalid_model - - -@pytest.fixture(autouse=True) -def test_resources(monkeypatch: pytest.MonkeyPatch, test_resources_path: Path) -> Any: - """Force using test resources as middleware's repository.""" - - def get_test_resources() -> Path: - """Return path to the test resources.""" - return test_resources_path - - monkeypatch.setattr("aiet.utils.fs.get_aiet_resources", get_test_resources) - yield - - -@pytest.fixture(scope="session", autouse=True) -def add_tools(test_resources_path: Path) -> Any: - """Symlink the tools from the original resources path to the test resources path.""" - # tool_dirs = get_available_tool_directory_names() - tool_dirs = [cfg.parent for cfg in get_backend_configs("tools")] - - links = { - src_dir: (test_resources_path / "tools" / src_dir.name) for src_dir in tool_dirs - } - for src_dir, dst_dir in links.items(): - if not dst_dir.exists(): - dst_dir.symlink_to(src_dir, target_is_directory=True) - yield - # Remove symlinks - for dst_dir in links.values(): - if dst_dir.is_symlink(): - dst_dir.unlink() - - -def create_archive( - archive_name: str, source: Path, destination: Path, with_root_folder: bool = False -) -> None: - """Create archive from directory source.""" - with tarfile.open(destination / archive_name, mode="w:gz") as tar: - for item in source.iterdir(): - item_name = item.name - if with_root_folder: - item_name = f"{source.name}/{item_name}" - tar.add(item, item_name) - - -def process_directory(source: Path, destination: Path) -> None: - """Process resource directory.""" - destination.mkdir() - - for item in source.iterdir(): - if item.is_dir(): - create_archive(f"{item.name}.tar.gz", item, destination) - create_archive(f"{item.name}_dir.tar.gz", item, destination, True) - - -@pytest.fixture(scope="session", autouse=True) -def add_archives( - test_resources_path: Path, tmp_path_factory: pytest.TempPathFactory -) -> Any: - """Generate archives of the test resources.""" - tmp_path = tmp_path_factory.mktemp("archives") - - archives_path = tmp_path / "archives" - archives_path.mkdir() - - if (archives_path_link := test_resources_path / "archives").is_symlink(): - archives_path.unlink() - - archives_path_link.symlink_to(archives_path, target_is_directory=True) - - for item in ["applications", "systems"]: - process_directory(test_resources_path / item, archives_path / item) - - yield - - archives_path_link.unlink() - shutil.rmtree(tmp_path) - - -@pytest.fixture(scope="module") -def cli_runner() -> CliRunner: - """Return CliRunner instance in a pytest fixture.""" - return CliRunner() diff --git a/tests/aiet/test_backend_tool.py b/tests/aiet/test_backend_tool.py deleted file mode 100644 index fd5960d..0000000 --- a/tests/aiet/test_backend_tool.py +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=no-self-use -"""Tests for the tool backend.""" -from collections import Counter - -import pytest - -from aiet.backend.common import ConfigurationException -from aiet.backend.config import ToolConfig -from aiet.backend.tool import get_available_tool_directory_names -from aiet.backend.tool import get_available_tools -from aiet.backend.tool import get_tool -from aiet.backend.tool import Tool - - -def test_get_available_tool_directory_names() -> None: - """Test get_available_tools mocking get_resources.""" - directory_names = get_available_tool_directory_names() - assert Counter(directory_names) == Counter(["tool1", "tool2", "vela"]) - - -def test_get_available_tools() -> None: - """Test get_available_tools mocking get_resources.""" - available_tools = get_available_tools() - expected_tool_names = sorted( - [ - "tool_1", - "tool_2", - "vela", - "vela", - "vela", - ] - ) - - assert all(isinstance(s, Tool) for s in available_tools) - assert all(s != 42 for s in available_tools) - assert any(s == available_tools[0] for s in available_tools) - assert len(available_tools) == len(expected_tool_names) - available_tool_names = sorted(str(s) for s in available_tools) - assert available_tool_names == expected_tool_names - - -def test_get_tool() -> None: - """Test get_tool mocking get_resoures.""" - tools = get_tool("tool_1") - assert len(tools) == 1 - tool = tools[0] - assert tool is not None - assert isinstance(tool, Tool) - assert tool.name == "tool_1" - - tools = get_tool("unknown tool") - assert not tools - - -def test_tool_creation() -> None: - """Test edge cases when creating a Tool instance.""" - with pytest.raises(ConfigurationException): - Tool(ToolConfig(name="test", commands={"test": []})) # no 'run' command diff --git a/tests/aiet/test_check_model.py b/tests/aiet/test_check_model.py deleted file mode 100644 index 4eafe59..0000000 --- a/tests/aiet/test_check_model.py +++ /dev/null @@ -1,162 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=redefined-outer-name,no-self-use -"""Module for testing check_model.py script.""" -from pathlib import Path -from typing import Any - -import pytest -from ethosu.vela.tflite.Model import Model -from ethosu.vela.tflite.OperatorCode import OperatorCode - -from aiet.cli.common import InvalidTFLiteFileError -from aiet.cli.common import ModelOptimisedException -from aiet.resources.tools.vela.check_model import check_custom_codes_for_ethosu -from aiet.resources.tools.vela.check_model import check_model -from aiet.resources.tools.vela.check_model import get_custom_codes_from_operators -from aiet.resources.tools.vela.check_model import get_model_from_file -from aiet.resources.tools.vela.check_model import get_operators_from_model -from aiet.resources.tools.vela.check_model import is_vela_optimised - - -@pytest.fixture(scope="session") -def optimised_tflite_model( - optimised_input_model_file: Path, -) -> Model: - """Return Model instance read from a Vela-optimised TFLite file.""" - return get_model_from_file(optimised_input_model_file) - - -@pytest.fixture(scope="session") -def non_optimised_tflite_model( - non_optimised_input_model_file: Path, -) -> Model: - """Return Model instance read from a Vela-optimised TFLite file.""" - return get_model_from_file(non_optimised_input_model_file) - - -class TestIsVelaOptimised: - """Test class for is_vela_optimised() function.""" - - def test_return_true_when_input_is_optimised( - self, - optimised_tflite_model: Model, - ) -> None: - """Verify True returned when input is optimised model.""" - output = is_vela_optimised(optimised_tflite_model) - - assert output is True - - def test_return_false_when_input_is_not_optimised( - self, - non_optimised_tflite_model: Model, - ) -> None: - """Verify False returned when input is non-optimised model.""" - output = is_vela_optimised(non_optimised_tflite_model) - - assert output is False - - -def test_get_operator_list_returns_correct_instances( - optimised_tflite_model: Model, -) -> None: - """Verify list of OperatorCode instances returned by get_operator_list().""" - operator_list = get_operators_from_model(optimised_tflite_model) - - assert all(isinstance(operator, OperatorCode) for operator in operator_list) - - -class TestGetCustomCodesFromOperators: - """Test the get_custom_codes_from_operators() function.""" - - def test_returns_empty_list_when_input_operators_have_no_custom_codes( - self, monkeypatch: Any - ) -> None: - """Verify function returns empty list when operators have no custom codes.""" - # Mock OperatorCode.CustomCode() function to return None - monkeypatch.setattr( - "ethosu.vela.tflite.OperatorCode.OperatorCode.CustomCode", lambda _: None - ) - - operators = [OperatorCode()] * 3 - - custom_codes = get_custom_codes_from_operators(operators) - - assert custom_codes == [] - - def test_returns_custom_codes_when_input_operators_have_custom_codes( - self, monkeypatch: Any - ) -> None: - """Verify list of bytes objects returned representing the CustomCodes.""" - # Mock OperatorCode.CustomCode() function to return a byte string - monkeypatch.setattr( - "ethosu.vela.tflite.OperatorCode.OperatorCode.CustomCode", - lambda _: b"custom-code", - ) - - operators = [OperatorCode()] * 3 - - custom_codes = get_custom_codes_from_operators(operators) - - assert custom_codes == [b"custom-code", b"custom-code", b"custom-code"] - - -@pytest.mark.parametrize( - "custom_codes, expected_output", - [ - ([b"ethos-u", b"something else"], True), - ([b"custom-code-1", b"custom-code-2"], False), - ], -) -def test_check_list_for_ethosu(custom_codes: list, expected_output: bool) -> None: - """Verify function detects 'ethos-u' bytes in the input list.""" - output = check_custom_codes_for_ethosu(custom_codes) - assert output is expected_output - - -class TestGetModelFromFile: - """Test the get_model_from_file() function.""" - - def test_error_raised_when_input_is_invalid_model_file( - self, - invalid_input_model_file: Path, - ) -> None: - """Verify error thrown when an invalid model file is given.""" - with pytest.raises(InvalidTFLiteFileError): - get_model_from_file(invalid_input_model_file) - - def test_model_instance_returned_when_input_is_valid_model_file( - self, - optimised_input_model_file: Path, - ) -> None: - """Verify file is read successfully and returns model instance.""" - tflite_model = get_model_from_file(optimised_input_model_file) - - assert isinstance(tflite_model, Model) - - -class TestCheckModel: - """Test the check_model() function.""" - - def test_check_model_with_non_optimised_input( - self, - non_optimised_input_model_file: Path, - ) -> None: - """Verify no error occurs for a valid input file.""" - check_model(non_optimised_input_model_file) - - def test_check_model_with_optimised_input( - self, - optimised_input_model_file: Path, - ) -> None: - """Verify that the right exception is raised with already optimised input.""" - with pytest.raises(ModelOptimisedException): - check_model(optimised_input_model_file) - - def test_check_model_with_invalid_input( - self, - invalid_input_model_file: Path, - ) -> None: - """Verify that an exception is raised with invalid input.""" - with pytest.raises(Exception): - check_model(invalid_input_model_file) diff --git a/tests/aiet/test_cli.py b/tests/aiet/test_cli.py deleted file mode 100644 index e8589fa..0000000 --- a/tests/aiet/test_cli.py +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Module for testing CLI top command.""" -from typing import Any -from unittest.mock import ANY -from unittest.mock import MagicMock - -from click.testing import CliRunner - -from aiet.cli import cli - - -def test_cli(cli_runner: CliRunner) -> None: - """Test CLI top level command.""" - result = cli_runner.invoke(cli) - assert result.exit_code == 0 - assert "system" in cli.commands - assert "application" in cli.commands - - -def test_cli_version(cli_runner: CliRunner) -> None: - """Test version option.""" - result = cli_runner.invoke(cli, ["--version"]) - assert result.exit_code == 0 - assert "version" in result.output - - -def test_cli_verbose(cli_runner: CliRunner, monkeypatch: Any) -> None: - """Test verbose option.""" - with monkeypatch.context() as mock_context: - mock = MagicMock() - # params[1] is the verbose option and we need to replace the - # callback with a mock object - mock_context.setattr(cli.params[1], "callback", mock) - cli_runner.invoke(cli, ["-vvvv"]) - # 4 is the number -v called earlier - mock.assert_called_once_with(ANY, ANY, 4) diff --git a/tests/aiet/test_cli_application.py b/tests/aiet/test_cli_application.py deleted file mode 100644 index f1ccc44..0000000 --- a/tests/aiet/test_cli_application.py +++ /dev/null @@ -1,1153 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=attribute-defined-outside-init,no-member,line-too-long,too-many-arguments,too-many-locals,redefined-outer-name,too-many-lines -"""Module for testing CLI application subcommand.""" -import base64 -import json -import re -import time -from contextlib import contextmanager -from contextlib import ExitStack -from pathlib import Path -from typing import Any -from typing import Generator -from typing import IO -from typing import List -from typing import Optional -from typing import TypedDict -from unittest.mock import MagicMock - -import click -import pytest -from click.testing import CliRunner -from filelock import FileLock - -from aiet.backend.application import Application -from aiet.backend.config import ApplicationConfig -from aiet.backend.config import LocalProtocolConfig -from aiet.backend.config import SSHConfig -from aiet.backend.config import SystemConfig -from aiet.backend.config import UserParamConfig -from aiet.backend.output_parser import Base64OutputParser -from aiet.backend.protocol import SSHProtocol -from aiet.backend.system import load_system -from aiet.cli.application import application_cmd -from aiet.cli.application import details_cmd -from aiet.cli.application import execute_cmd -from aiet.cli.application import install_cmd -from aiet.cli.application import list_cmd -from aiet.cli.application import parse_payload_run_config -from aiet.cli.application import remove_cmd -from aiet.cli.application import run_cmd -from aiet.cli.common import MiddlewareExitCode - - -def test_application_cmd() -> None: - """Test application commands.""" - commands = ["list", "details", "install", "remove", "execute", "run"] - assert all(command in application_cmd.commands for command in commands) - - -@pytest.mark.parametrize("format_", ["json", "cli"]) -def test_application_cmd_context(cli_runner: CliRunner, format_: str) -> None: - """Test setting command context parameters.""" - result = cli_runner.invoke(application_cmd, ["--format", format_]) - # command should fail if no subcommand provided - assert result.exit_code == 2 - - result = cli_runner.invoke(application_cmd, ["--format", format_, "list"]) - assert result.exit_code == 0 - - -@pytest.mark.parametrize( - "format_, system_name, expected_output", - [ - ( - "json", - None, - '{"type": "application", "available": ["application_1", "application_2"]}\n', - ), - ( - "json", - "system_1", - '{"type": "application", "available": ["application_1"]}\n', - ), - ("cli", None, "Available applications:\n\napplication_1\napplication_2\n"), - ("cli", "system_1", "Available applications:\n\napplication_1\n"), - ], -) -def test_list_cmd( - cli_runner: CliRunner, - monkeypatch: Any, - format_: str, - system_name: str, - expected_output: str, -) -> None: - """Test available applications commands.""" - # Mock some applications - mock_application_1 = MagicMock(spec=Application) - mock_application_1.name = "application_1" - mock_application_1.can_run_on.return_value = system_name == "system_1" - mock_application_2 = MagicMock(spec=Application) - mock_application_2.name = "application_2" - mock_application_2.can_run_on.return_value = system_name == "system_2" - - # Monkey patch the call get_available_applications - mock_available_applications = MagicMock() - mock_available_applications.return_value = [mock_application_1, mock_application_2] - - monkeypatch.setattr( - "aiet.backend.application.get_available_applications", - mock_available_applications, - ) - - obj = {"format": format_} - args = [] - if system_name: - list_cmd.params[0].type = click.Choice([system_name]) - args = ["--system", system_name] - result = cli_runner.invoke(list_cmd, obj=obj, args=args) - assert result.output == expected_output - - -def get_test_application() -> Application: - """Return test system details.""" - config = ApplicationConfig( - name="application", - description="test", - build_dir="", - supported_systems=[], - deploy_data=[], - user_params={}, - commands={ - "clean": ["clean"], - "build": ["build"], - "run": ["run"], - "post_run": ["post_run"], - }, - ) - - return Application(config) - - -def get_details_cmd_json_output() -> str: - """Get JSON output for details command.""" - json_output = """ -[ - { - "type": "application", - "name": "application", - "description": "test", - "supported_systems": [], - "commands": { - "clean": { - "command_strings": [ - "clean" - ], - "user_params": [] - }, - "build": { - "command_strings": [ - "build" - ], - "user_params": [] - }, - "run": { - "command_strings": [ - "run" - ], - "user_params": [] - }, - "post_run": { - "command_strings": [ - "post_run" - ], - "user_params": [] - } - } - } -]""" - return json.dumps(json.loads(json_output)) + "\n" - - -def get_details_cmd_console_output() -> str: - """Get console output for details command.""" - return ( - 'Application "application" details' - + "\nDescription: test" - + "\n\nSupported systems: " - + "\n\nclean commands:" - + "\nCommands: ['clean']" - + "\n\nbuild commands:" - + "\nCommands: ['build']" - + "\n\nrun commands:" - + "\nCommands: ['run']" - + "\n\npost_run commands:" - + "\nCommands: ['post_run']" - + "\n" - ) - - -@pytest.mark.parametrize( - "application_name,format_, expected_output", - [ - ("application", "json", get_details_cmd_json_output()), - ("application", "cli", get_details_cmd_console_output()), - ], -) -def test_details_cmd( - cli_runner: CliRunner, - monkeypatch: Any, - application_name: str, - format_: str, - expected_output: str, -) -> None: - """Test application details command.""" - monkeypatch.setattr( - "aiet.cli.application.get_application", - MagicMock(return_value=[get_test_application()]), - ) - - details_cmd.params[0].type = click.Choice(["application"]) - result = cli_runner.invoke( - details_cmd, obj={"format": format_}, args=["--name", application_name] - ) - assert result.exception is None - assert result.output == expected_output - - -def test_details_cmd_wrong_system(cli_runner: CliRunner, monkeypatch: Any) -> None: - """Test details command fails if application is not supported by the system.""" - monkeypatch.setattr( - "aiet.backend.execution.get_application", MagicMock(return_value=[]) - ) - - details_cmd.params[0].type = click.Choice(["application"]) - details_cmd.params[1].type = click.Choice(["system"]) - result = cli_runner.invoke( - details_cmd, args=["--name", "application", "--system", "system"] - ) - assert result.exit_code == 2 - assert ( - "Application 'application' doesn't support the system 'system'" in result.stdout - ) - - -def test_install_cmd(cli_runner: CliRunner, monkeypatch: Any) -> None: - """Test install application command.""" - mock_install_application = MagicMock() - monkeypatch.setattr( - "aiet.cli.application.install_application", mock_install_application - ) - - args = ["--source", "test"] - cli_runner.invoke(install_cmd, args=args) - mock_install_application.assert_called_once_with(Path("test")) - - -def test_remove_cmd(cli_runner: CliRunner, monkeypatch: Any) -> None: - """Test remove application command.""" - mock_remove_application = MagicMock() - monkeypatch.setattr( - "aiet.cli.application.remove_application", mock_remove_application - ) - remove_cmd.params[0].type = click.Choice(["test"]) - - args = ["--directory_name", "test"] - cli_runner.invoke(remove_cmd, args=args) - mock_remove_application.assert_called_once_with("test") - - -class ExecutionCase(TypedDict, total=False): - """Execution case.""" - - args: List[str] - lock_path: str - can_establish_connection: bool - establish_connection_delay: int - app_exit_code: int - exit_code: int - output: str - - -@pytest.mark.parametrize( - "application_config, system_config, executions", - [ - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - config_location=Path("wrong_location"), - commands={"build": ["echo build {application.name}"]}, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - config_location=Path("wrong_location"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ), - [ - ExecutionCase( - args=["-c", "build"], - exit_code=MiddlewareExitCode.CONFIGURATION_ERROR, - output="Error: Application test_application has wrong config location\n", - ) - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - build_dir="build", - deploy_data=[("sample_file", "/tmp/sample_file")], - commands={"build": ["echo build {application.name}"]}, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ), - [ - ExecutionCase( - args=["-c", "run"], - exit_code=MiddlewareExitCode.CONFIGURATION_ERROR, - output="Error: System test_system does not support data deploy\n", - ) - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - commands={"build": ["echo build {application.name}"]}, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ), - [ - ExecutionCase( - args=["-c", "build"], - exit_code=MiddlewareExitCode.CONFIGURATION_ERROR, - output="Error: No build directory defined for the app test_application\n", - ) - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["new_system"], - build_dir="build", - commands={ - "build": ["echo build {application.name} with {user_params:0}"] - }, - user_params={ - "build": [ - UserParamConfig( - name="param", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - ) - ] - }, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ), - [ - ExecutionCase( - args=["-c", "build"], - exit_code=1, - output="Error: Application 'test_application' doesn't support the system 'test_system'\n", - ) - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - build_dir="build", - commands={"build": ["false"]}, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ), - [ - ExecutionCase( - args=["-c", "build"], - exit_code=MiddlewareExitCode.BACKEND_ERROR, - output="""Running: false -Error: Execution failed. Please check output for the details.\n""", - ) - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - lock=True, - build_dir="build", - commands={ - "build": ["echo build {application.name} with {user_params:0}"] - }, - user_params={ - "build": [ - UserParamConfig( - name="param", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - ) - ] - }, - ), - SystemConfig( - name="test_system", - description="Test system", - lock=True, - data_transfer=LocalProtocolConfig(protocol="local"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ), - [ - ExecutionCase( - args=["-c", "build"], - exit_code=MiddlewareExitCode.SUCCESS, - output="""Running: echo build test_application with param default -build test_application with param default\n""", - ), - ExecutionCase( - args=["-c", "build"], - lock_path="/tmp/middleware_test_application_test_system.lock", - exit_code=MiddlewareExitCode.CONCURRENT_ERROR, - output="Error: Another instance of the system is running\n", - ), - ExecutionCase( - args=["-c", "build", "--param=param=val3"], - exit_code=MiddlewareExitCode.SUCCESS, - output="""Running: echo build test_application with param val3 -build test_application with param val3\n""", - ), - ExecutionCase( - args=["-c", "build", "--param=param=newval"], - exit_code=1, - output="Error: Application parameter 'param=newval' not valid for command 'build'\n", - ), - ExecutionCase( - args=["-c", "some_command"], - exit_code=MiddlewareExitCode.CONFIGURATION_ERROR, - output="Error: Unsupported command some_command\n", - ), - ExecutionCase( - args=["-c", "run"], - exit_code=MiddlewareExitCode.SUCCESS, - output="""Generating commands to execute -Running: echo run test_application on test_system -run test_application on test_system\n""", - ), - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - deploy_data=[("sample_file", "/tmp/sample_file")], - commands={ - "run": [ - "echo run {application.name} with {user_params:param} on {system.name}" - ] - }, - user_params={ - "run": [ - UserParamConfig( - name="param=", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - alias="param", - ) - ] - }, - ), - SystemConfig( - name="test_system", - description="Test system", - lock=True, - data_transfer=SSHConfig( - protocol="ssh", - username="username", - password="password", - hostname="localhost", - port="8022", - ), - commands={"run": ["sleep 100"]}, - ), - [ - ExecutionCase( - args=["-c", "run"], - exit_code=MiddlewareExitCode.SUCCESS, - output="""Generating commands to execute -Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds . -Deploying {application.config_location}/sample_file onto /tmp/sample_file -Running: echo run test_application with param=default on test_system -Shutting down sequence... -Stopping test_system... (It could take few seconds) -test_system stopped successfully.\n""", - ), - ExecutionCase( - args=["-c", "run"], - lock_path="/tmp/middleware_test_system.lock", - exit_code=MiddlewareExitCode.CONCURRENT_ERROR, - output="Error: Another instance of the system is running\n", - ), - ExecutionCase( - args=[ - "-c", - "run", - "--deploy={application.config_location}/sample_file:/tmp/sample_file", - ], - exit_code=0, - output="""Generating commands to execute -Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds . -Deploying {application.config_location}/sample_file onto /tmp/sample_file -Deploying {application.config_location}/sample_file onto /tmp/sample_file -Running: echo run test_application with param=default on test_system -Shutting down sequence... -Stopping test_system... (It could take few seconds) -test_system stopped successfully.\n""", - ), - ExecutionCase( - args=["-c", "run"], - app_exit_code=1, - exit_code=0, - output="""Generating commands to execute -Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds . -Deploying {application.config_location}/sample_file onto /tmp/sample_file -Running: echo run test_application with param=default on test_system -Application exited with exit code 1 -Shutting down sequence... -Stopping test_system... (It could take few seconds) -test_system stopped successfully.\n""", - ), - ExecutionCase( - args=["-c", "run"], - exit_code=MiddlewareExitCode.CONNECTION_ERROR, - can_establish_connection=False, - output="""Generating commands to execute -Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds .......................................................................................... -Shutting down sequence... -Stopping test_system... (It could take few seconds) -test_system stopped successfully. -Error: Couldn't connect to 'localhost:8022'.\n""", - ), - ExecutionCase( - args=["-c", "run", "--deploy=bad_format"], - exit_code=1, - output="Error: Invalid deploy parameter 'bad_format' for command run\n", - ), - ExecutionCase( - args=["-c", "run", "--deploy=:"], - exit_code=1, - output="Error: Invalid deploy parameter ':' for command run\n", - ), - ExecutionCase( - args=["-c", "run", "--deploy= : "], - exit_code=1, - output="Error: Invalid deploy parameter ' : ' for command run\n", - ), - ExecutionCase( - args=["-c", "run", "--deploy=some_src_file:"], - exit_code=1, - output="Error: Invalid deploy parameter 'some_src_file:' for command run\n", - ), - ExecutionCase( - args=["-c", "run", "--deploy=:some_dst_file"], - exit_code=1, - output="Error: Invalid deploy parameter ':some_dst_file' for command run\n", - ), - ExecutionCase( - args=["-c", "run", "--deploy=unknown_file:/tmp/dest"], - exit_code=1, - output="Error: Path unknown_file does not exist\n", - ), - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - commands={ - "run": [ - "echo run {application.name} with {user_params:param} on {system.name}" - ] - }, - user_params={ - "run": [ - UserParamConfig( - name="param=", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - alias="param", - ) - ] - }, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=SSHConfig( - protocol="ssh", - username="username", - password="password", - hostname="localhost", - port="8022", - ), - commands={"run": ["echo Unable to start system"]}, - ), - [ - ExecutionCase( - args=["-c", "run"], - exit_code=4, - can_establish_connection=False, - establish_connection_delay=1, - output="""Generating commands to execute -Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds . - ----------- test_system execution failed ---------- -Unable to start system - - - -Shutting down sequence... -Stopping test_system... (It could take few seconds) -test_system stopped successfully. -Error: Execution failed. Please check output for the details.\n""", - ) - ], - ], - ], -) -def test_application_command_execution( - application_config: ApplicationConfig, - system_config: SystemConfig, - executions: List[ExecutionCase], - tmpdir: Any, - cli_runner: CliRunner, - monkeypatch: Any, -) -> None: - """Test application command execution.""" - - @contextmanager - def lock_execution(lock_path: str) -> Generator[None, None, None]: - lock = FileLock(lock_path) - lock.acquire(timeout=1) - - try: - yield - finally: - lock.release() - - def replace_vars(str_val: str) -> str: - """Replace variables.""" - application_config_location = str( - application_config["config_location"].absolute() - ) - - return str_val.replace( - "{application.config_location}", application_config_location - ) - - for execution in executions: - init_execution_test( - monkeypatch, - tmpdir, - application_config, - system_config, - can_establish_connection=execution.get("can_establish_connection", True), - establish_conection_delay=execution.get("establish_connection_delay", 0), - remote_app_exit_code=execution.get("app_exit_code", 0), - ) - - lock_path = execution.get("lock_path") - - with ExitStack() as stack: - if lock_path: - stack.enter_context(lock_execution(lock_path)) - - args = [replace_vars(arg) for arg in execution["args"]] - - result = cli_runner.invoke( - execute_cmd, - args=["-n", application_config["name"], "-s", system_config["name"]] - + args, - ) - output = replace_vars(execution["output"]) - assert result.exit_code == execution["exit_code"] - assert result.stdout == output - - -@pytest.fixture(params=[False, True], ids=["run-cli", "run-json"]) -def payload_path_or_none(request: Any, tmp_path_factory: Any) -> Optional[Path]: - """Drives tests for run command so that it executes them both to use a json file, and to use CLI.""" - if request.param: - ret: Path = tmp_path_factory.getbasetemp() / "system_config_payload_file.json" - return ret - return None - - -def write_system_payload_config( - payload_file: IO[str], - application_config: ApplicationConfig, - system_config: SystemConfig, -) -> None: - """Write a json payload file for the given test configuration.""" - payload_dict = { - "id": system_config["name"], - "arguments": { - "application": application_config["name"], - }, - } - json.dump(payload_dict, payload_file) - - -@pytest.mark.parametrize( - "application_config, system_config, executions", - [ - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - build_dir="build", - commands={ - "build": ["echo build {application.name} with {user_params:0}"] - }, - user_params={ - "build": [ - UserParamConfig( - name="param", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - ) - ] - }, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ), - [ - ExecutionCase( - args=[], - exit_code=MiddlewareExitCode.SUCCESS, - output="""Running: echo build test_application with param default -build test_application with param default -Generating commands to execute -Running: echo run test_application on test_system -run test_application on test_system\n""", - ) - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - commands={ - "run": [ - "echo run {application.name} with {user_params:param} on {system.name}" - ] - }, - user_params={ - "run": [ - UserParamConfig( - name="param=", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - alias="param", - ) - ] - }, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=SSHConfig( - protocol="ssh", - username="username", - password="password", - hostname="localhost", - port="8022", - ), - commands={"run": ["sleep 100"]}, - ), - [ - ExecutionCase( - args=[], - exit_code=MiddlewareExitCode.SUCCESS, - output="""Generating commands to execute -Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds . -Running: echo run test_application with param=default on test_system -Shutting down sequence... -Stopping test_system... (It could take few seconds) -test_system stopped successfully.\n""", - ) - ], - ], - ], -) -def test_application_run( - application_config: ApplicationConfig, - system_config: SystemConfig, - executions: List[ExecutionCase], - tmpdir: Any, - cli_runner: CliRunner, - monkeypatch: Any, - payload_path_or_none: Path, -) -> None: - """Test application command execution.""" - for execution in executions: - init_execution_test(monkeypatch, tmpdir, application_config, system_config) - - if payload_path_or_none: - with open(payload_path_or_none, "w", encoding="utf-8") as payload_file: - write_system_payload_config( - payload_file, application_config, system_config - ) - - result = cli_runner.invoke( - run_cmd, - args=["--config", str(payload_path_or_none)], - ) - else: - result = cli_runner.invoke( - run_cmd, - args=["-n", application_config["name"], "-s", system_config["name"]] - + execution["args"], - ) - - assert result.stdout == execution["output"] - assert result.exit_code == execution["exit_code"] - - -@pytest.mark.parametrize( - "cmdline,error_pattern", - [ - [ - "--config {payload} -s test_system", - "when --config is set, the following parameters should not be provided", - ], - [ - "--config {payload} -n test_application", - "when --config is set, the following parameters should not be provided", - ], - [ - "--config {payload} -p mypar:3", - "when --config is set, the following parameters should not be provided", - ], - [ - "-p mypar:3", - "when --config is not set, the following parameters are required", - ], - ["-s test_system", "when --config is not set, --name is required"], - ["-n test_application", "when --config is not set, --system is required"], - ], -) -def test_application_run_invalid_param_combinations( - cmdline: str, - error_pattern: str, - cli_runner: CliRunner, - monkeypatch: Any, - tmp_path: Any, - tmpdir: Any, -) -> None: - """Test that invalid combinations arguments result in error as expected.""" - application_config = ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - build_dir="build", - commands={"build": ["echo build {application.name} with {user_params:0}"]}, - user_params={ - "build": [ - UserParamConfig( - name="param", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - ) - ] - }, - ) - system_config = SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ) - - init_execution_test(monkeypatch, tmpdir, application_config, system_config) - - payload_file = tmp_path / "payload.json" - payload_file.write_text("dummy") - result = cli_runner.invoke( - run_cmd, - args=cmdline.format(payload=payload_file).split(), - ) - found = re.search(error_pattern, result.stdout) - assert found, f"Cannot find pattern: [{error_pattern}] in \n[\n{result.stdout}\n]" - - -@pytest.mark.parametrize( - "payload,expected", - [ - pytest.param( - {"arguments": {}}, - None, - marks=pytest.mark.xfail(reason="no system 'id''", strict=True), - ), - pytest.param( - {"id": "testsystem"}, - None, - marks=pytest.mark.xfail(reason="no arguments object", strict=True), - ), - ( - {"id": "testsystem", "arguments": {"application": "testapp"}}, - ("testsystem", "testapp", [], [], [], None), - ), - ( - { - "id": "testsystem", - "arguments": {"application": "testapp", "par1": "val1"}, - }, - ("testsystem", "testapp", ["par1=val1"], [], [], None), - ), - ( - { - "id": "testsystem", - "arguments": {"application": "testapp", "application/par1": "val1"}, - }, - ("testsystem", "testapp", ["par1=val1"], [], [], None), - ), - ( - { - "id": "testsystem", - "arguments": {"application": "testapp", "system/par1": "val1"}, - }, - ("testsystem", "testapp", [], ["par1=val1"], [], None), - ), - ( - { - "id": "testsystem", - "arguments": {"application": "testapp", "deploy/par1": "val1"}, - }, - ("testsystem", "testapp", [], [], ["par1"], None), - ), - ( - { - "id": "testsystem", - "arguments": { - "application": "testapp", - "appar1": "val1", - "application/appar2": "val2", - "system/syspar1": "val3", - "deploy/depploypar1": "val4", - "application/appar3": "val5", - "system/syspar2": "val6", - "deploy/depploypar2": "val7", - }, - }, - ( - "testsystem", - "testapp", - ["appar1=val1", "appar2=val2", "appar3=val5"], - ["syspar1=val3", "syspar2=val6"], - ["depploypar1", "depploypar2"], - None, - ), - ), - ], -) -def test_parse_payload_run_config(payload: dict, expected: tuple) -> None: - """Test parsing of the JSON payload for the run_config command.""" - assert parse_payload_run_config(payload) == expected - - -def test_application_run_report( - tmpdir: Any, - cli_runner: CliRunner, - monkeypatch: Any, -) -> None: - """Test flag '--report' of command 'application run'.""" - app_metrics = {"app_metric": 3.14} - app_metrics_b64 = base64.b64encode(json.dumps(app_metrics).encode("utf-8")) - application_config = ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - build_dir="build", - commands={"build": ["echo build {application.name} with {user_params:0}"]}, - user_params={ - "build": [ - UserParamConfig( - name="param", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - ), - UserParamConfig( - name="p2", - description="another parameter, not overridden", - default_value="the-right-choice", - values=["the-right-choice", "the-bad-choice"], - ), - ] - }, - ) - system_config = SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={ - "run": [ - "echo run {application.name} on {system.name}", - f"echo build <{Base64OutputParser.TAG_NAME}>{app_metrics_b64.decode('utf-8')}</{Base64OutputParser.TAG_NAME}>", - ] - }, - reporting={ - "regex": { - "app_name": { - "pattern": r"run (.\S*) ", - "type": "str", - }, - "sys_name": { - "pattern": r"on (.\S*)", - "type": "str", - }, - } - }, - ) - report_file = Path(tmpdir) / "test_report.json" - param_val = "param=val1" - exit_code = MiddlewareExitCode.SUCCESS - - init_execution_test(monkeypatch, tmpdir, application_config, system_config) - - result = cli_runner.invoke( - run_cmd, - args=[ - "-n", - application_config["name"], - "-s", - system_config["name"], - "--report", - str(report_file), - "--param", - param_val, - ], - ) - assert result.exit_code == exit_code - assert report_file.is_file() - with open(report_file, "r", encoding="utf-8") as file: - report = json.load(file) - - assert report == { - "application": { - "metrics": {"0": {"app_metric": 3.14}}, - "name": "test_application", - "params": {"param": "val1", "p2": "the-right-choice"}, - }, - "system": { - "metrics": {"app_name": "test_application", "sys_name": "test_system"}, - "name": "test_system", - "params": {}, - }, - } - - -def init_execution_test( - monkeypatch: Any, - tmpdir: Any, - application_config: ApplicationConfig, - system_config: SystemConfig, - can_establish_connection: bool = True, - establish_conection_delay: float = 0, - remote_app_exit_code: int = 0, -) -> None: - """Init execution test.""" - application_name = application_config["name"] - system_name = system_config["name"] - - execute_cmd.params[0].type = click.Choice([application_name]) - execute_cmd.params[1].type = click.Choice([system_name]) - execute_cmd.params[2].type = click.Choice(["build", "run", "some_command"]) - - run_cmd.params[0].type = click.Choice([application_name]) - run_cmd.params[1].type = click.Choice([system_name]) - - if "config_location" not in application_config: - application_path = Path(tmpdir) / "application" - application_path.mkdir() - application_config["config_location"] = application_path - - # this file could be used as deploy parameter value or - # as deploy parameter in application configuration - sample_file = application_path / "sample_file" - sample_file.touch() - monkeypatch.setattr( - "aiet.backend.application.get_available_applications", - MagicMock(return_value=[Application(application_config)]), - ) - - ssh_protocol_mock = MagicMock(spec=SSHProtocol) - - def mock_establish_connection() -> bool: - """Mock establish connection function.""" - # give some time for the system to start - time.sleep(establish_conection_delay) - return can_establish_connection - - ssh_protocol_mock.establish_connection.side_effect = mock_establish_connection - ssh_protocol_mock.connection_details.return_value = ("localhost", 8022) - ssh_protocol_mock.run.return_value = ( - remote_app_exit_code, - bytearray(), - bytearray(), - ) - monkeypatch.setattr( - "aiet.backend.protocol.SSHProtocol", MagicMock(return_value=ssh_protocol_mock) - ) - - if "config_location" not in system_config: - system_path = Path(tmpdir) / "system" - system_path.mkdir() - system_config["config_location"] = system_path - monkeypatch.setattr( - "aiet.backend.system.get_available_systems", - MagicMock(return_value=[load_system(system_config)]), - ) - - monkeypatch.setattr("aiet.backend.execution.wait", MagicMock()) diff --git a/tests/aiet/test_cli_common.py b/tests/aiet/test_cli_common.py deleted file mode 100644 index d018e44..0000000 --- a/tests/aiet/test_cli_common.py +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Test for cli common module.""" -from typing import Any - -import pytest - -from aiet.cli.common import print_command_details -from aiet.cli.common import raise_exception_at_signal - - -def test_print_command_details(capsys: Any) -> None: - """Test print_command_details function.""" - command = { - "command_strings": ["echo test"], - "user_params": [ - {"name": "param_name", "description": "param_description"}, - { - "name": "param_name2", - "description": "param_description2", - "alias": "alias2", - }, - ], - } - print_command_details(command) - captured = capsys.readouterr() - assert "echo test" in captured.out - assert "param_name" in captured.out - assert "alias2" in captured.out - - -def test_raise_exception_at_signal() -> None: - """Test raise_exception_at_signal graceful shutdown.""" - with pytest.raises(Exception) as err: - raise_exception_at_signal(1, "") - - assert str(err.value) == "Middleware shutdown requested" diff --git a/tests/aiet/test_cli_system.py b/tests/aiet/test_cli_system.py deleted file mode 100644 index fd39f31..0000000 --- a/tests/aiet/test_cli_system.py +++ /dev/null @@ -1,240 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Module for testing CLI system subcommand.""" -import json -from pathlib import Path -from typing import Any -from typing import Dict -from typing import List -from typing import Optional -from typing import Union -from unittest.mock import MagicMock - -import click -import pytest -from click.testing import CliRunner - -from aiet.backend.config import SystemConfig -from aiet.backend.system import load_system -from aiet.backend.system import System -from aiet.cli.system import details_cmd -from aiet.cli.system import install_cmd -from aiet.cli.system import list_cmd -from aiet.cli.system import remove_cmd -from aiet.cli.system import system_cmd - - -def test_system_cmd() -> None: - """Test system commands.""" - commands = ["list", "details", "install", "remove"] - assert all(command in system_cmd.commands for command in commands) - - -@pytest.mark.parametrize("format_", ["json", "cli"]) -def test_system_cmd_context(cli_runner: CliRunner, format_: str) -> None: - """Test setting command context parameters.""" - result = cli_runner.invoke(system_cmd, ["--format", format_]) - # command should fail if no subcommand provided - assert result.exit_code == 2 - - result = cli_runner.invoke(system_cmd, ["--format", format_, "list"]) - assert result.exit_code == 0 - - -@pytest.mark.parametrize( - "format_,expected_output", - [ - ("json", '{"type": "system", "available": ["system1", "system2"]}\n'), - ("cli", "Available systems:\n\nsystem1\nsystem2\n"), - ], -) -def test_list_cmd_with_format( - cli_runner: CliRunner, monkeypatch: Any, format_: str, expected_output: str -) -> None: - """Test available systems command with different formats output.""" - # Mock some systems - mock_system1 = MagicMock() - mock_system1.name = "system1" - mock_system2 = MagicMock() - mock_system2.name = "system2" - - # Monkey patch the call get_available_systems - mock_available_systems = MagicMock() - mock_available_systems.return_value = [mock_system1, mock_system2] - monkeypatch.setattr("aiet.cli.system.get_available_systems", mock_available_systems) - - obj = {"format": format_} - result = cli_runner.invoke(list_cmd, obj=obj) - assert result.output == expected_output - - -def get_test_system( - annotations: Optional[Dict[str, Union[str, List[str]]]] = None -) -> System: - """Return test system details.""" - config = SystemConfig( - name="system", - description="test", - data_transfer={ - "protocol": "ssh", - "username": "root", - "password": "root", - "hostname": "localhost", - "port": "8022", - }, - commands={ - "clean": ["clean"], - "build": ["build"], - "run": ["run"], - "post_run": ["post_run"], - }, - annotations=annotations or {}, - ) - - return load_system(config) - - -def get_details_cmd_json_output( - annotations: Optional[Dict[str, Union[str, List[str]]]] = None -) -> str: - """Test JSON output for details command.""" - ann_str = "" - if annotations is not None: - ann_str = '"annotations":{},'.format(json.dumps(annotations)) - - json_output = ( - """ -{ - "type": "system", - "name": "system", - "description": "test", - "data_transfer_protocol": "ssh", - "commands": { - "clean": - { - "command_strings": ["clean"], - "user_params": [] - }, - "build": - { - "command_strings": ["build"], - "user_params": [] - }, - "run": - { - "command_strings": ["run"], - "user_params": [] - }, - "post_run": - { - "command_strings": ["post_run"], - "user_params": [] - } - }, -""" - + ann_str - + """ - "available_application" : [] - } -""" - ) - return json.dumps(json.loads(json_output)) + "\n" - - -def get_details_cmd_console_output( - annotations: Optional[Dict[str, Union[str, List[str]]]] = None -) -> str: - """Test console output for details command.""" - ann_str = "" - if annotations: - val_str = "".join( - "\n\t{}: {}".format(ann_name, ann_value) - for ann_name, ann_value in annotations.items() - ) - ann_str = "\nAnnotations:{}".format(val_str) - return ( - 'System "system" details' - + "\nDescription: test" - + "\nData Transfer Protocol: ssh" - + "\nAvailable Applications: " - + ann_str - + "\n\nclean commands:" - + "\nCommands: ['clean']" - + "\n\nbuild commands:" - + "\nCommands: ['build']" - + "\n\nrun commands:" - + "\nCommands: ['run']" - + "\n\npost_run commands:" - + "\nCommands: ['post_run']" - + "\n" - ) - - -@pytest.mark.parametrize( - "format_,system,expected_output", - [ - ( - "json", - get_test_system(annotations={"ann1": "annotation1", "ann2": ["a1", "a2"]}), - get_details_cmd_json_output( - annotations={"ann1": "annotation1", "ann2": ["a1", "a2"]} - ), - ), - ( - "cli", - get_test_system(annotations={"ann1": "annotation1", "ann2": ["a1", "a2"]}), - get_details_cmd_console_output( - annotations={"ann1": "annotation1", "ann2": ["a1", "a2"]} - ), - ), - ( - "json", - get_test_system(annotations={}), - get_details_cmd_json_output(annotations={}), - ), - ( - "cli", - get_test_system(annotations={}), - get_details_cmd_console_output(annotations={}), - ), - ], -) -def test_details_cmd( - cli_runner: CliRunner, - monkeypatch: Any, - format_: str, - system: System, - expected_output: str, -) -> None: - """Test details command with different formats output.""" - mock_get_system = MagicMock() - mock_get_system.return_value = system - monkeypatch.setattr("aiet.cli.system.get_system", mock_get_system) - - args = ["--name", "system"] - obj = {"format": format_} - details_cmd.params[0].type = click.Choice(["system"]) - - result = cli_runner.invoke(details_cmd, args=args, obj=obj) - assert result.output == expected_output - - -def test_install_cmd(cli_runner: CliRunner, monkeypatch: Any) -> None: - """Test install system command.""" - mock_install_system = MagicMock() - monkeypatch.setattr("aiet.cli.system.install_system", mock_install_system) - - args = ["--source", "test"] - cli_runner.invoke(install_cmd, args=args) - mock_install_system.assert_called_once_with(Path("test")) - - -def test_remove_cmd(cli_runner: CliRunner, monkeypatch: Any) -> None: - """Test remove system command.""" - mock_remove_system = MagicMock() - monkeypatch.setattr("aiet.cli.system.remove_system", mock_remove_system) - remove_cmd.params[0].type = click.Choice(["test"]) - - args = ["--directory_name", "test"] - cli_runner.invoke(remove_cmd, args=args) - mock_remove_system.assert_called_once_with("test") diff --git a/tests/aiet/test_cli_tool.py b/tests/aiet/test_cli_tool.py deleted file mode 100644 index 45d45c8..0000000 --- a/tests/aiet/test_cli_tool.py +++ /dev/null @@ -1,333 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=attribute-defined-outside-init,no-member,line-too-long,too-many-arguments,too-many-locals -"""Module for testing CLI tool subcommand.""" -import json -from pathlib import Path -from typing import Any -from typing import List -from typing import Optional -from typing import Sequence -from unittest.mock import MagicMock - -import click -import pytest -from click.testing import CliRunner -from click.testing import Result - -from aiet.backend.tool import get_unique_tool_names -from aiet.backend.tool import Tool -from aiet.cli.tool import details_cmd -from aiet.cli.tool import execute_cmd -from aiet.cli.tool import list_cmd -from aiet.cli.tool import tool_cmd - - -def test_tool_cmd() -> None: - """Test tool commands.""" - commands = ["list", "details", "execute"] - assert all(command in tool_cmd.commands for command in commands) - - -@pytest.mark.parametrize("format_", ["json", "cli"]) -def test_tool_cmd_context(cli_runner: CliRunner, format_: str) -> None: - """Test setting command context parameters.""" - result = cli_runner.invoke(tool_cmd, ["--format", format_]) - # command should fail if no subcommand provided - assert result.exit_code == 2 - - result = cli_runner.invoke(tool_cmd, ["--format", format_, "list"]) - assert result.exit_code == 0 - - -@pytest.mark.parametrize( - "format_, expected_output", - [ - ( - "json", - '{"type": "tool", "available": ["tool_1", "tool_2"]}\n', - ), - ("cli", "Available tools:\n\ntool_1\ntool_2\n"), - ], -) -def test_list_cmd( - cli_runner: CliRunner, - monkeypatch: Any, - format_: str, - expected_output: str, -) -> None: - """Test available tool commands.""" - # Mock some tools - mock_tool_1 = MagicMock(spec=Tool) - mock_tool_1.name = "tool_1" - mock_tool_2 = MagicMock(spec=Tool) - mock_tool_2.name = "tool_2" - - # Monkey patch the call get_available_tools - mock_available_tools = MagicMock() - mock_available_tools.return_value = [mock_tool_1, mock_tool_2] - - monkeypatch.setattr("aiet.backend.tool.get_available_tools", mock_available_tools) - - obj = {"format": format_} - args: Sequence[str] = [] - result = cli_runner.invoke(list_cmd, obj=obj, args=args) - assert result.output == expected_output - - -def get_details_cmd_json_output() -> List[dict]: - """Get JSON output for details command.""" - json_output = [ - { - "type": "tool", - "name": "tool_1", - "description": "This is tool 1", - "supported_systems": ["System 1"], - "commands": { - "clean": {"command_strings": ["echo 'clean'"], "user_params": []}, - "build": {"command_strings": ["echo 'build'"], "user_params": []}, - "run": {"command_strings": ["echo 'run'"], "user_params": []}, - "post_run": {"command_strings": ["echo 'post_run'"], "user_params": []}, - }, - } - ] - - return json_output - - -def get_details_cmd_console_output() -> str: - """Get console output for details command.""" - return ( - 'Tool "tool_1" details' - "\nDescription: This is tool 1" - "\n\nSupported systems: System 1" - "\n\nclean commands:" - "\nCommands: [\"echo 'clean'\"]" - "\n\nbuild commands:" - "\nCommands: [\"echo 'build'\"]" - "\n\nrun commands:\nCommands: [\"echo 'run'\"]" - "\n\npost_run commands:" - "\nCommands: [\"echo 'post_run'\"]" - "\n" - ) - - -@pytest.mark.parametrize( - [ - "tool_name", - "format_", - "expected_success", - "expected_output", - ], - [ - ("tool_1", "json", True, get_details_cmd_json_output()), - ("tool_1", "cli", True, get_details_cmd_console_output()), - ("non-existent tool", "json", False, None), - ("non-existent tool", "cli", False, None), - ], -) -def test_details_cmd( - cli_runner: CliRunner, - tool_name: str, - format_: str, - expected_success: bool, - expected_output: str, -) -> None: - """Test tool details command.""" - details_cmd.params[0].type = click.Choice(["tool_1", "tool_2", "vela"]) - result = cli_runner.invoke( - details_cmd, obj={"format": format_}, args=["--name", tool_name] - ) - success = result.exit_code == 0 - assert success == expected_success, result.output - if expected_success: - assert result.exception is None - output = json.loads(result.output) if format_ == "json" else result.output - assert output == expected_output - - -@pytest.mark.parametrize( - "system_name", - [ - "", - "Corstone-300: Cortex-M55+Ethos-U55", - "Corstone-300: Cortex-M55+Ethos-U65", - "Corstone-310: Cortex-M85+Ethos-U55", - ], -) -def test_details_cmd_vela(cli_runner: CliRunner, system_name: str) -> None: - """Test tool details command for Vela.""" - details_cmd.params[0].type = click.Choice(get_unique_tool_names()) - details_cmd.params[1].type = click.Choice([system_name]) - args = ["--name", "vela"] - if system_name: - args += ["--system", system_name] - result = cli_runner.invoke(details_cmd, obj={"format": "json"}, args=args) - success = result.exit_code == 0 - assert success, result.output - result_json = json.loads(result.output) - assert result_json - if system_name: - assert len(result_json) == 1 - tool = result_json[0] - assert len(tool["supported_systems"]) == 1 - assert system_name == tool["supported_systems"][0] - else: # no system specified => list details for all systems - assert len(result_json) == 3 - assert all(len(tool["supported_systems"]) == 1 for tool in result_json) - - -@pytest.fixture(scope="session") -def input_model_file(non_optimised_input_model_file: Path) -> Path: - """Provide the path to a quantized dummy model file in the test_resources_path.""" - return non_optimised_input_model_file - - -def execute_vela( - cli_runner: CliRunner, - tool_name: str = "vela", - system_name: Optional[str] = None, - input_model: Optional[Path] = None, - output_model: Optional[Path] = None, - mac: Optional[int] = None, - format_: str = "cli", -) -> Result: - """Run Vela with different parameters.""" - execute_cmd.params[0].type = click.Choice(get_unique_tool_names()) - execute_cmd.params[2].type = click.Choice([system_name or "dummy_system"]) - args = ["--name", tool_name] - if system_name is not None: - args += ["--system", system_name] - if input_model is not None: - args += ["--param", "input={}".format(input_model)] - if output_model is not None: - args += ["--param", "output={}".format(output_model)] - if mac is not None: - args += ["--param", "mac={}".format(mac)] - result = cli_runner.invoke( - execute_cmd, - args=args, - obj={"format": format_}, - ) - return result - - -@pytest.mark.parametrize("format_", ["cli, json"]) -@pytest.mark.parametrize( - ["tool_name", "system_name", "mac", "expected_success", "expected_output"], - [ - ("vela", "System 1", 32, False, None), # system not supported - ("vela", "NON-EXISTENT SYSTEM", 128, False, None), # system does not exist - ("vela", "Corstone-300: Cortex-M55+Ethos-U55", 32, True, None), - ("NON-EXISTENT TOOL", "Corstone-300: Cortex-M55+Ethos-U55", 32, False, None), - ("vela", "Corstone-300: Cortex-M55+Ethos-U55", 64, True, None), - ("vela", "Corstone-300: Cortex-M55+Ethos-U55", 128, True, None), - ("vela", "Corstone-300: Cortex-M55+Ethos-U55", 256, True, None), - ( - "vela", - "Corstone-300: Cortex-M55+Ethos-U55", - 512, - False, - None, - ), # mac not supported - ( - "vela", - "Corstone-300: Cortex-M55+Ethos-U65", - 32, - False, - None, - ), # mac not supported - ("vela", "Corstone-300: Cortex-M55+Ethos-U65", 256, True, None), - ("vela", "Corstone-300: Cortex-M55+Ethos-U65", 512, True, None), - ( - "vela", - None, - 512, - False, - "Error: Please specify the system for tool vela.", - ), # no system specified - ( - "NON-EXISTENT TOOL", - "Corstone-300: Cortex-M55+Ethos-U65", - 512, - False, - None, - ), # tool does not exist - ("vela", "Corstone-310: Cortex-M85+Ethos-U55", 128, True, None), - ], -) -def test_vela_run( - cli_runner: CliRunner, - format_: str, - input_model_file: Path, # pylint: disable=redefined-outer-name - tool_name: str, - system_name: Optional[str], - mac: int, - expected_success: bool, - expected_output: Optional[str], - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - """Test the execution of the Vela command.""" - monkeypatch.chdir(tmp_path) - - output_file = Path("vela_output.tflite") - - result = execute_vela( - cli_runner, - tool_name=tool_name, - system_name=system_name, - input_model=input_model_file, - output_model=output_file, - mac=mac, - format_=format_, - ) - - success = result.exit_code == 0 - assert success == expected_success - if success: - # Check output file - output_file = output_file.resolve() - assert output_file.is_file() - if expected_output: - assert result.output.strip() == expected_output - - -@pytest.mark.parametrize("include_input_model", [True, False]) -@pytest.mark.parametrize("include_output_model", [True, False]) -@pytest.mark.parametrize("include_mac", [True, False]) -def test_vela_run_missing_params( - cli_runner: CliRunner, - input_model_file: Path, # pylint: disable=redefined-outer-name - include_input_model: bool, - include_output_model: bool, - include_mac: bool, - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - """Test the execution of the Vela command with missing user parameters.""" - monkeypatch.chdir(tmp_path) - - output_model_file = Path("output_model.tflite") - system_name = "Corstone-300: Cortex-M55+Ethos-U65" - mac = 256 - # input_model is a required parameters, but mac and output_model have default values. - expected_success = include_input_model - - result = execute_vela( - cli_runner, - tool_name="vela", - system_name=system_name, - input_model=input_model_file if include_input_model else None, - output_model=output_model_file if include_output_model else None, - mac=mac if include_mac else None, - ) - - success = result.exit_code == 0 - assert success == expected_success, ( - f"Success is {success}, but expected {expected_success}. " - f"Included params: [" - f"input_model={include_input_model}, " - f"output_model={include_output_model}, " - f"mac={include_mac}]" - ) diff --git a/tests/aiet/test_main.py b/tests/aiet/test_main.py deleted file mode 100644 index f2ebae2..0000000 --- a/tests/aiet/test_main.py +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Module for testing AIET main.py.""" -from typing import Any -from unittest.mock import MagicMock - -from aiet import main - - -def test_main(monkeypatch: Any) -> None: - """Test main entry point function.""" - with monkeypatch.context() as mock_context: - mock = MagicMock() - mock_context.setattr(main, "cli", mock) - main.main() - mock.assert_called_once() diff --git a/tests/aiet/test_resources/tools/tool1/aiet-config.json b/tests/aiet/test_resources/tools/tool1/aiet-config.json deleted file mode 100644 index 067ef7e..0000000 --- a/tests/aiet/test_resources/tools/tool1/aiet-config.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "name": "tool_1", - "description": "This is tool 1", - "build_dir": "build", - "supported_systems": [ - { - "name": "System 1" - } - ], - "commands": { - "clean": [ - "echo 'clean'" - ], - "build": [ - "echo 'build'" - ], - "run": [ - "echo 'run'" - ], - "post_run": [ - "echo 'post_run'" - ] - }, - "user_params": { - "build": [], - "run": [] - } - } -] diff --git a/tests/aiet/test_resources/tools/tool2/aiet-config.json b/tests/aiet/test_resources/tools/tool2/aiet-config.json deleted file mode 100644 index 6eee9a6..0000000 --- a/tests/aiet/test_resources/tools/tool2/aiet-config.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "name": "tool_2", - "description": "This is tool 2 with no supported systems", - "build_dir": "build", - "supported_systems": [], - "commands": { - "clean": [ - "echo 'clean'" - ], - "build": [ - "echo 'build'" - ], - "run": [ - "echo 'run'" - ], - "post_run": [ - "echo 'post_run'" - ] - }, - "user_params": { - "build": [], - "run": [] - } - } -] diff --git a/tests/aiet/test_run_vela_script.py b/tests/aiet/test_run_vela_script.py deleted file mode 100644 index 971856e..0000000 --- a/tests/aiet/test_run_vela_script.py +++ /dev/null @@ -1,152 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=redefined-outer-name,no-self-use -"""Module for testing run_vela.py script.""" -from pathlib import Path -from typing import Any -from typing import List - -import pytest -from click.testing import CliRunner - -from aiet.cli.common import MiddlewareExitCode -from aiet.resources.tools.vela.check_model import get_model_from_file -from aiet.resources.tools.vela.check_model import is_vela_optimised -from aiet.resources.tools.vela.run_vela import run_vela - - -@pytest.fixture(scope="session") -def vela_config_path(test_tools_path: Path) -> Path: - """Return test systems path in a pytest fixture.""" - return test_tools_path / "vela" / "vela.ini" - - -@pytest.fixture( - params=[ - ["ethos-u65-256", "Ethos_U65_High_End", "U65_Shared_Sram"], - ["ethos-u55-32", "Ethos_U55_High_End_Embedded", "U55_Shared_Sram"], - ] -) -def ethos_config(request: Any) -> Any: - """Fixture to provide different configuration for Ethos-U optimization with Vela.""" - return request.param - - -# pylint: disable=too-many-arguments -def generate_args( - input_: Path, - output: Path, - cfg: Path, - acc_config: str, - system_config: str, - memory_mode: str, -) -> List[str]: - """Generate arguments that can be passed to script 'run_vela'.""" - return [ - "-i", - str(input_), - "-o", - str(output), - "--config", - str(cfg), - "--accelerator-config", - acc_config, - "--system-config", - system_config, - "--memory-mode", - memory_mode, - "--optimise", - "Performance", - ] - - -def check_run_vela( - cli_runner: CliRunner, args: List, expected_success: bool, output_file: Path -) -> None: - """Run Vela with the given arguments and check the result.""" - result = cli_runner.invoke(run_vela, args) - success = result.exit_code == MiddlewareExitCode.SUCCESS - assert success == expected_success - if success: - model = get_model_from_file(output_file) - assert is_vela_optimised(model) - - -def run_vela_script( - cli_runner: CliRunner, - input_model_file: Path, - output_model_file: Path, - vela_config: Path, - expected_success: bool, - acc_config: str, - system_config: str, - memory_mode: str, -) -> None: - """Run the command 'run_vela' on the command line.""" - args = generate_args( - input_model_file, - output_model_file, - vela_config, - acc_config, - system_config, - memory_mode, - ) - check_run_vela(cli_runner, args, expected_success, output_model_file) - - -class TestRunVelaCli: - """Test the command-line execution of the run_vela command.""" - - def test_non_optimised_model( - self, - cli_runner: CliRunner, - non_optimised_input_model_file: Path, - tmp_path: Path, - vela_config_path: Path, - ethos_config: List, - ) -> None: - """Verify Vela is run correctly on an unoptimised model.""" - run_vela_script( - cli_runner, - non_optimised_input_model_file, - tmp_path / "test.tflite", - vela_config_path, - True, - *ethos_config, - ) - - def test_optimised_model( - self, - cli_runner: CliRunner, - optimised_input_model_file: Path, - tmp_path: Path, - vela_config_path: Path, - ethos_config: List, - ) -> None: - """Verify Vela is run correctly on an already optimised model.""" - run_vela_script( - cli_runner, - optimised_input_model_file, - tmp_path / "test.tflite", - vela_config_path, - True, - *ethos_config, - ) - - def test_invalid_model( - self, - cli_runner: CliRunner, - invalid_input_model_file: Path, - tmp_path: Path, - vela_config_path: Path, - ethos_config: List, - ) -> None: - """Verify an error is raised when the input model is not valid.""" - run_vela_script( - cli_runner, - invalid_input_model_file, - tmp_path / "test.tflite", - vela_config_path, - False, - *ethos_config, - ) diff --git a/tests/aiet/test_utils_helpers.py b/tests/aiet/test_utils_helpers.py deleted file mode 100644 index bbe03fc..0000000 --- a/tests/aiet/test_utils_helpers.py +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Module for testing helpers.py.""" -import logging -from typing import Any -from typing import List -from unittest.mock import call -from unittest.mock import MagicMock - -import pytest - -from aiet.utils.helpers import set_verbosity - - -@pytest.mark.parametrize( - "verbosity,expected_calls", - [(0, []), (1, [call(logging.INFO)]), (2, [call(logging.DEBUG)])], -) -def test_set_verbosity( - verbosity: int, expected_calls: List[Any], monkeypatch: Any -) -> None: - """Test set_verbosity() with different verbsosity levels.""" - with monkeypatch.context() as mock_context: - logging_mock = MagicMock() - mock_context.setattr(logging.getLogger(), "setLevel", logging_mock) - set_verbosity(None, None, verbosity) - logging_mock.assert_has_calls(expected_calls) diff --git a/tests/mlia/conftest.py b/tests/mlia/conftest.py index f683fca..0b4b2aa 100644 --- a/tests/mlia/conftest.py +++ b/tests/mlia/conftest.py @@ -1,7 +1,10 @@ # SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Pytest conf module.""" +import shutil +import tarfile from pathlib import Path +from typing import Any import pytest @@ -18,3 +21,91 @@ def fixture_test_resources_path() -> Path: def fixture_dummy_context(tmpdir: str) -> ExecutionContext: """Return dummy context fixture.""" return ExecutionContext(working_dir=tmpdir) + + +@pytest.fixture(scope="session") +def test_systems_path(test_resources_path: Path) -> Path: + """Return test systems path in a pytest fixture.""" + return test_resources_path / "backends" / "systems" + + +@pytest.fixture(scope="session") +def test_applications_path(test_resources_path: Path) -> Path: + """Return test applications path in a pytest fixture.""" + return test_resources_path / "backends" / "applications" + + +@pytest.fixture(scope="session") +def non_optimised_input_model_file(test_tflite_model: Path) -> Path: + """Provide the path to a quantized dummy model file.""" + return test_tflite_model + + +@pytest.fixture(scope="session") +def optimised_input_model_file(test_tflite_vela_model: Path) -> Path: + """Provide path to Vela-optimised dummy model file.""" + return test_tflite_vela_model + + +@pytest.fixture(scope="session") +def invalid_input_model_file(test_tflite_invalid_model: Path) -> Path: + """Provide the path to an invalid dummy model file.""" + return test_tflite_invalid_model + + +@pytest.fixture(autouse=True) +def test_resources(monkeypatch: pytest.MonkeyPatch, test_resources_path: Path) -> Any: + """Force using test resources as middleware's repository.""" + + def get_test_resources() -> Path: + """Return path to the test resources.""" + return test_resources_path / "backends" + + monkeypatch.setattr("mlia.backend.fs.get_backend_resources", get_test_resources) + yield + + +def create_archive( + archive_name: str, source: Path, destination: Path, with_root_folder: bool = False +) -> None: + """Create archive from directory source.""" + with tarfile.open(destination / archive_name, mode="w:gz") as tar: + for item in source.iterdir(): + item_name = item.name + if with_root_folder: + item_name = f"{source.name}/{item_name}" + tar.add(item, item_name) + + +def process_directory(source: Path, destination: Path) -> None: + """Process resource directory.""" + destination.mkdir() + + for item in source.iterdir(): + if item.is_dir(): + create_archive(f"{item.name}.tar.gz", item, destination) + create_archive(f"{item.name}_dir.tar.gz", item, destination, True) + + +@pytest.fixture(scope="session", autouse=True) +def add_archives( + test_resources_path: Path, tmp_path_factory: pytest.TempPathFactory +) -> Any: + """Generate archives of the test resources.""" + tmp_path = tmp_path_factory.mktemp("archives") + + archives_path = tmp_path / "archives" + archives_path.mkdir() + + if (archives_path_link := test_resources_path / "archives").is_symlink(): + archives_path_link.unlink() + + archives_path_link.symlink_to(archives_path, target_is_directory=True) + + for item in ["applications", "systems"]: + process_directory(test_resources_path / "backends" / item, archives_path / item) + + yield + + archives_path_link.unlink() + shutil.rmtree(tmp_path) diff --git a/tests/aiet/test_backend_application.py b/tests/mlia/test_backend_application.py index abfab00..2cfb2ef 100644 --- a/tests/aiet/test_backend_application.py +++ b/tests/mlia/test_backend_application.py @@ -11,28 +11,34 @@ from unittest.mock import MagicMock import pytest -from aiet.backend.application import Application -from aiet.backend.application import get_application -from aiet.backend.application import get_available_application_directory_names -from aiet.backend.application import get_available_applications -from aiet.backend.application import get_unique_application_names -from aiet.backend.application import install_application -from aiet.backend.application import load_applications -from aiet.backend.application import remove_application -from aiet.backend.common import Command -from aiet.backend.common import DataPaths -from aiet.backend.common import Param -from aiet.backend.common import UserParamConfig -from aiet.backend.config import ApplicationConfig -from aiet.backend.config import ExtendedApplicationConfig -from aiet.backend.config import NamedExecutionConfig +from mlia.backend.application import Application +from mlia.backend.application import get_application +from mlia.backend.application import get_available_application_directory_names +from mlia.backend.application import get_available_applications +from mlia.backend.application import get_unique_application_names +from mlia.backend.application import install_application +from mlia.backend.application import load_applications +from mlia.backend.application import remove_application +from mlia.backend.common import Command +from mlia.backend.common import DataPaths +from mlia.backend.common import Param +from mlia.backend.common import UserParamConfig +from mlia.backend.config import ApplicationConfig +from mlia.backend.config import ExtendedApplicationConfig +from mlia.backend.config import NamedExecutionConfig def test_get_available_application_directory_names() -> None: """Test get_available_applicationss mocking get_resources.""" directory_names = get_available_application_directory_names() assert Counter(directory_names) == Counter( - ["application1", "application2", "application4", "application5"] + [ + "application1", + "application2", + "application4", + "application5", + "application6", + ] ) @@ -42,7 +48,7 @@ def test_get_available_applications() -> None: assert all(isinstance(s, Application) for s in available_applications) assert all(s != 42 for s in available_applications) - assert len(available_applications) == 9 + assert len(available_applications) == 10 # application_5 has multiply items with multiply supported systems assert [str(s) for s in available_applications] == [ "application_1", @@ -54,6 +60,7 @@ def test_get_available_applications() -> None: "application_5A", "application_5B", "application_5B", + "application_6", ] @@ -70,6 +77,7 @@ def test_get_unique_application_names() -> None: "application_5", "application_5A", "application_5B", + "application_6", ] @@ -121,14 +129,14 @@ def test_get_application() -> None: pytest.raises(Exception, match="Unable to read application definition"), ), ( - "applications/application1", + "backends/applications/application1", 0, pytest.raises( Exception, match=r"Applications \[application_1\] are already installed" ), ), ( - "applications/application3", + "backends/applications/application3", 0, pytest.raises(Exception, match="Unable to read application definition"), ), @@ -144,7 +152,7 @@ def test_install_application( """Test application install from archive.""" mock_create_destination_and_install = MagicMock() monkeypatch.setattr( - "aiet.backend.application.create_destination_and_install", + "mlia.backend.application.create_destination_and_install", mock_create_destination_and_install, ) @@ -156,7 +164,7 @@ def test_install_application( def test_remove_application(monkeypatch: Any) -> None: """Test application removal.""" mock_remove_backend = MagicMock() - monkeypatch.setattr("aiet.backend.application.remove_backend", mock_remove_backend) + monkeypatch.setattr("mlia.backend.application.remove_backend", mock_remove_backend) remove_application("some_application_directory") mock_remove_backend.assert_called_once() diff --git a/tests/aiet/test_backend_common.py b/tests/mlia/test_backend_common.py index 12c30ec..82a985a 100644 --- a/tests/aiet/test_backend_common.py +++ b/tests/mlia/test_backend_common.py @@ -16,20 +16,20 @@ from unittest.mock import MagicMock import pytest -from aiet.backend.application import Application -from aiet.backend.common import Backend -from aiet.backend.common import BaseBackendConfig -from aiet.backend.common import Command -from aiet.backend.common import ConfigurationException -from aiet.backend.common import load_config -from aiet.backend.common import Param -from aiet.backend.common import parse_raw_parameter -from aiet.backend.common import remove_backend -from aiet.backend.config import ApplicationConfig -from aiet.backend.config import UserParamConfig -from aiet.backend.execution import ExecutionContext -from aiet.backend.execution import ParamResolver -from aiet.backend.system import System +from mlia.backend.application import Application +from mlia.backend.common import Backend +from mlia.backend.common import BaseBackendConfig +from mlia.backend.common import Command +from mlia.backend.common import ConfigurationException +from mlia.backend.common import load_config +from mlia.backend.common import Param +from mlia.backend.common import parse_raw_parameter +from mlia.backend.common import remove_backend +from mlia.backend.config import ApplicationConfig +from mlia.backend.config import UserParamConfig +from mlia.backend.execution import ExecutionContext +from mlia.backend.execution import ParamResolver +from mlia.backend.system import System @pytest.mark.parametrize( @@ -44,7 +44,7 @@ def test_remove_backend( ) -> None: """Test remove_backend function.""" mock_remove_resource = MagicMock() - monkeypatch.setattr("aiet.backend.common.remove_resource", mock_remove_resource) + monkeypatch.setattr("mlia.backend.common.remove_resource", mock_remove_resource) with expected_exception: remove_backend(directory_name, "applications") @@ -75,7 +75,7 @@ def test_load_config( ) for config in configs: json_mock = MagicMock() - monkeypatch.setattr("aiet.backend.common.json.load", json_mock) + monkeypatch.setattr("mlia.backend.common.json.load", json_mock) load_config(config) json_mock.assert_called_once() @@ -175,7 +175,7 @@ class TestBackend: "variables": {"var_A": "value for variable A"}, } - monkeypatch.setattr("aiet.backend.system.ProtocolFactory", MagicMock()) + monkeypatch.setattr("mlia.backend.system.ProtocolFactory", MagicMock()) application, system = Application(config), System(config) # type: ignore context = ExecutionContext( app=application, @@ -290,7 +290,7 @@ class TestBackend: expected_output: List[Tuple[Optional[str], Param]], ) -> None: """Test command building.""" - monkeypatch.setattr("aiet.backend.system.ProtocolFactory", MagicMock()) + monkeypatch.setattr("mlia.backend.system.ProtocolFactory", MagicMock()) backend = class_(config) params = backend.resolved_parameters( diff --git a/tests/aiet/test_backend_controller.py b/tests/mlia/test_backend_controller.py index 8836ec5..a047adf 100644 --- a/tests/aiet/test_backend_controller.py +++ b/tests/mlia/test_backend_controller.py @@ -10,10 +10,10 @@ from typing import Any import psutil import pytest -from aiet.backend.common import ConfigurationException -from aiet.backend.controller import SystemController -from aiet.backend.controller import SystemControllerSingleInstance -from aiet.utils.proc import ShellCommand +from mlia.backend.common import ConfigurationException +from mlia.backend.controller import SystemController +from mlia.backend.controller import SystemControllerSingleInstance +from mlia.backend.proc import ShellCommand def get_system_controller(**kwargs: Any) -> SystemController: diff --git a/tests/aiet/test_backend_execution.py b/tests/mlia/test_backend_execution.py index 8aa45f1..9395352 100644 --- a/tests/aiet/test_backend_execution.py +++ b/tests/mlia/test_backend_execution.py @@ -1,41 +1,39 @@ # SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 # pylint: disable=no-self-use -"""Test backend context module.""" +"""Test backend execution module.""" from contextlib import ExitStack as does_not_raise from pathlib import Path from typing import Any from typing import Dict -from typing import Optional from unittest import mock from unittest.mock import MagicMock import pytest from sh import CommandNotFound -from aiet.backend.application import Application -from aiet.backend.application import get_application -from aiet.backend.common import ConfigurationException -from aiet.backend.common import DataPaths -from aiet.backend.common import UserParamConfig -from aiet.backend.config import ApplicationConfig -from aiet.backend.config import LocalProtocolConfig -from aiet.backend.config import SystemConfig -from aiet.backend.execution import deploy_data -from aiet.backend.execution import execute_commands_locally -from aiet.backend.execution import ExecutionContext -from aiet.backend.execution import get_application_and_system -from aiet.backend.execution import get_application_by_name_and_system -from aiet.backend.execution import get_file_lock_path -from aiet.backend.execution import get_tool_by_system -from aiet.backend.execution import ParamResolver -from aiet.backend.execution import Reporter -from aiet.backend.execution import wait -from aiet.backend.output_parser import OutputParser -from aiet.backend.system import get_system -from aiet.backend.system import load_system -from aiet.backend.tool import get_tool -from aiet.utils.proc import CommandFailedException +from mlia.backend.application import Application +from mlia.backend.application import get_application +from mlia.backend.common import DataPaths +from mlia.backend.common import UserParamConfig +from mlia.backend.config import ApplicationConfig +from mlia.backend.config import LocalProtocolConfig +from mlia.backend.config import SystemConfig +from mlia.backend.execution import deploy_data +from mlia.backend.execution import execute_commands_locally +from mlia.backend.execution import ExecutionContext +from mlia.backend.execution import get_application_and_system +from mlia.backend.execution import get_application_by_name_and_system +from mlia.backend.execution import get_file_lock_path +from mlia.backend.execution import ParamResolver +from mlia.backend.execution import Reporter +from mlia.backend.execution import wait +from mlia.backend.output_parser import Base64OutputParser +from mlia.backend.output_parser import OutputParser +from mlia.backend.output_parser import RegexOutputParser +from mlia.backend.proc import CommandFailedException +from mlia.backend.system import get_system +from mlia.backend.system import load_system def test_context_param_resolver(tmpdir: Any) -> None: @@ -319,7 +317,7 @@ def test_get_file_lock_path( def test_get_application_by_name_and_system(monkeypatch: Any) -> None: """Test exceptional case for get_application_by_name_and_system.""" monkeypatch.setattr( - "aiet.backend.execution.get_application", + "mlia.backend.execution.get_application", MagicMock(return_value=[MagicMock(), MagicMock()]), ) @@ -334,7 +332,7 @@ def test_get_application_by_name_and_system(monkeypatch: Any) -> None: def test_get_application_and_system(monkeypatch: Any) -> None: """Test exceptional case for get_application_and_system.""" monkeypatch.setattr( - "aiet.backend.execution.get_system", MagicMock(return_value=None) + "mlia.backend.execution.get_system", MagicMock(return_value=None) ) with pytest.raises(ValueError, match="System test_system is not found"): @@ -379,40 +377,32 @@ def test_deployment_execution_context() -> None: with pytest.raises(AssertionError): deploy_data(ctx) + +def test_reporter_execution_context(tmp_path: Path) -> None: + """Test ExecutionContext creates a reporter when a report file is provided.""" + # Configure regex parser for the system manually + sys = get_system("System 1") + assert sys is not None + sys.reporting = { + "regex": { + "simulated_time": {"pattern": "Simulated time.*: (.*)s", "type": "float"} + } + } + report_file_path = tmp_path / "test_report.json" + ctx = ExecutionContext( - app=get_tool("tool_1")[0], + app=get_application("application_1")[0], app_params=[], - system=None, + system=sys, system_params=[], + report_file=report_file_path, + ) + assert isinstance(ctx.reporter, Reporter) + assert len(ctx.reporter.parsers) == 2 + assert any(isinstance(parser, RegexOutputParser) for parser in ctx.reporter.parsers) + assert any( + isinstance(parser, Base64OutputParser) for parser in ctx.reporter.parsers ) - assert not ctx.is_deploy_needed - deploy_data(ctx) # should be a NOP - - -@pytest.mark.parametrize( - ["tool_name", "system_name", "exception"], - [ - ("vela", "Corstone-300: Cortex-M55+Ethos-U65", None), - ("unknown tool", "Corstone-300: Cortex-M55+Ethos-U65", ConfigurationException), - ("vela", "unknown system", ConfigurationException), - ("vela", None, ConfigurationException), - ], -) -def test_get_tool_by_system( - tool_name: str, system_name: Optional[str], exception: Optional[Any] -) -> None: - """Test exceptions thrown by function get_tool_by_system().""" - - def test() -> None: - """Test call of get_tool_by_system().""" - tool = get_tool_by_system(tool_name, system_name) - assert tool is not None - - if exception is None: - test() - else: - with pytest.raises(exception): - test() class TestExecuteCommandsLocally: @@ -465,7 +455,7 @@ class TestExecuteCommandsLocally: """Ensure commands following an error-exit-code command don't run.""" # Mock execute_command() function execute_command_mock = mock.MagicMock() - monkeypatch.setattr("aiet.utils.proc.execute_command", execute_command_mock) + monkeypatch.setattr("mlia.backend.proc.execute_command", execute_command_mock) # Mock Command object and assign as return value to execute_command() cmd_mock = mock.MagicMock() @@ -473,7 +463,9 @@ class TestExecuteCommandsLocally: # Mock the terminate_command (speed up test) terminate_command_mock = mock.MagicMock() - monkeypatch.setattr("aiet.utils.proc.terminate_command", terminate_command_mock) + monkeypatch.setattr( + "mlia.backend.proc.terminate_command", terminate_command_mock + ) # Mock a thrown Exception and assign to Command().exit_code exit_code_mock = mock.PropertyMock(side_effect=Exception("Exception.")) diff --git a/tests/aiet/test_utils_fs.py b/tests/mlia/test_backend_fs.py index 46d276e..ff9c2ae 100644 --- a/tests/aiet/test_utils_fs.py +++ b/tests/mlia/test_backend_fs.py @@ -10,14 +10,14 @@ from unittest.mock import MagicMock import pytest -from aiet.utils.fs import get_resources -from aiet.utils.fs import read_file_as_bytearray -from aiet.utils.fs import read_file_as_string -from aiet.utils.fs import recreate_directory -from aiet.utils.fs import remove_directory -from aiet.utils.fs import remove_resource -from aiet.utils.fs import ResourceType -from aiet.utils.fs import valid_for_filename +from mlia.backend.fs import get_backends_path +from mlia.backend.fs import read_file_as_bytearray +from mlia.backend.fs import read_file_as_string +from mlia.backend.fs import recreate_directory +from mlia.backend.fs import remove_directory +from mlia.backend.fs import remove_resource +from mlia.backend.fs import ResourceType +from mlia.backend.fs import valid_for_filename @pytest.mark.parametrize( @@ -29,10 +29,10 @@ from aiet.utils.fs import valid_for_filename (None, pytest.raises(ResourceWarning)), ], ) -def test_get_resources(resource_name: ResourceType, expected_path: Any) -> None: +def test_get_backends_path(resource_name: ResourceType, expected_path: Any) -> None: """Test get_resources() with multiple parameters.""" with expected_path: - resource_path = get_resources(resource_name) + resource_path = get_backends_path(resource_name) assert resource_path.exists() @@ -41,10 +41,10 @@ def test_remove_resource_wrong_directory( ) -> None: """Test removing resource with wrong directory.""" mock_get_resources = MagicMock(return_value=test_applications_path) - monkeypatch.setattr("aiet.utils.fs.get_resources", mock_get_resources) + monkeypatch.setattr("mlia.backend.fs.get_backends_path", mock_get_resources) mock_shutil_rmtree = MagicMock() - monkeypatch.setattr("aiet.utils.fs.shutil.rmtree", mock_shutil_rmtree) + monkeypatch.setattr("mlia.backend.fs.shutil.rmtree", mock_shutil_rmtree) with pytest.raises(Exception, match="Resource .* does not exist"): remove_resource("unknown", "applications") @@ -58,10 +58,10 @@ def test_remove_resource_wrong_directory( def test_remove_resource(monkeypatch: Any, test_applications_path: Path) -> None: """Test removing resource data.""" mock_get_resources = MagicMock(return_value=test_applications_path) - monkeypatch.setattr("aiet.utils.fs.get_resources", mock_get_resources) + monkeypatch.setattr("mlia.backend.fs.get_backends_path", mock_get_resources) mock_shutil_rmtree = MagicMock() - monkeypatch.setattr("aiet.utils.fs.shutil.rmtree", mock_shutil_rmtree) + monkeypatch.setattr("mlia.backend.fs.shutil.rmtree", mock_shutil_rmtree) remove_resource("application1", "applications") mock_shutil_rmtree.assert_called_once() diff --git a/tests/mlia/test_tools_aiet_wrapper.py b/tests/mlia/test_backend_manager.py index ab55b71..c81366f 100644 --- a/tests/mlia/test_tools_aiet_wrapper.py +++ b/tests/mlia/test_backend_manager.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 -"""Tests for module tools/aiet_wrapper.""" +"""Tests for module backend/manager.""" +import os from contextlib import ExitStack as does_not_raise from pathlib import Path from typing import Any @@ -13,20 +14,23 @@ from unittest.mock import PropertyMock import pytest -from mlia.tools.aiet_wrapper import AIETRunner -from mlia.tools.aiet_wrapper import DeviceInfo -from mlia.tools.aiet_wrapper import estimate_performance -from mlia.tools.aiet_wrapper import ExecutionParams -from mlia.tools.aiet_wrapper import GenericInferenceOutputParser -from mlia.tools.aiet_wrapper import GenericInferenceRunnerEthosU -from mlia.tools.aiet_wrapper import get_aiet_runner -from mlia.tools.aiet_wrapper import get_generic_runner -from mlia.tools.aiet_wrapper import get_system_name -from mlia.tools.aiet_wrapper import is_supported -from mlia.tools.aiet_wrapper import ModelInfo -from mlia.tools.aiet_wrapper import PerformanceMetrics -from mlia.tools.aiet_wrapper import supported_backends -from mlia.utils.proc import RunningCommand +from mlia.backend.application import get_application +from mlia.backend.common import DataPaths +from mlia.backend.execution import ExecutionContext +from mlia.backend.execution import run_application +from mlia.backend.manager import BackendRunner +from mlia.backend.manager import DeviceInfo +from mlia.backend.manager import estimate_performance +from mlia.backend.manager import ExecutionParams +from mlia.backend.manager import GenericInferenceOutputParser +from mlia.backend.manager import GenericInferenceRunnerEthosU +from mlia.backend.manager import get_generic_runner +from mlia.backend.manager import get_system_name +from mlia.backend.manager import is_supported +from mlia.backend.manager import ModelInfo +from mlia.backend.manager import PerformanceMetrics +from mlia.backend.manager import supported_backends +from mlia.backend.system import get_system @pytest.mark.parametrize( @@ -108,16 +112,16 @@ def test_generic_inference_output_parser( assert parser.missed_keys() == missed_keys -class TestAIETRunner: - """Tests for AIETRunner class.""" +class TestBackendRunner: + """Tests for BackendRunner class.""" @staticmethod - def _setup_aiet( + def _setup_backends( monkeypatch: pytest.MonkeyPatch, available_systems: Optional[List[str]] = None, available_apps: Optional[List[str]] = None, ) -> None: - """Set up AIET metadata.""" + """Set up backend metadata.""" def mock_system(system: str) -> MagicMock: """Mock the System instance.""" @@ -134,13 +138,13 @@ class TestAIETRunner: system_mocks = [mock_system(name) for name in (available_systems or [])] monkeypatch.setattr( - "mlia.tools.aiet_wrapper.get_available_systems", + "mlia.backend.manager.get_available_systems", MagicMock(return_value=system_mocks), ) apps_mock = [mock_app(name) for name in (available_apps or [])] monkeypatch.setattr( - "mlia.tools.aiet_wrapper.get_available_applications", + "mlia.backend.manager.get_available_applications", MagicMock(return_value=apps_mock), ) @@ -159,13 +163,11 @@ class TestAIETRunner: monkeypatch: pytest.MonkeyPatch, ) -> None: """Test method is_system_installed.""" - mock_executor = MagicMock() - aiet_runner = AIETRunner(mock_executor) + backend_runner = BackendRunner() - self._setup_aiet(monkeypatch, available_systems) + self._setup_backends(monkeypatch, available_systems) - assert aiet_runner.is_system_installed(system) == installed - mock_executor.assert_not_called() + assert backend_runner.is_system_installed(system) == installed @pytest.mark.parametrize( "available_systems, systems", @@ -181,28 +183,21 @@ class TestAIETRunner: monkeypatch: pytest.MonkeyPatch, ) -> None: """Test method installed_systems.""" - mock_executor = MagicMock() - aiet_runner = AIETRunner(mock_executor) + backend_runner = BackendRunner() - self._setup_aiet(monkeypatch, available_systems) - assert aiet_runner.get_installed_systems() == systems - - mock_executor.assert_not_called() + self._setup_backends(monkeypatch, available_systems) + assert backend_runner.get_installed_systems() == systems @staticmethod def test_install_system(monkeypatch: pytest.MonkeyPatch) -> None: """Test system installation.""" install_system_mock = MagicMock() - monkeypatch.setattr( - "mlia.tools.aiet_wrapper.install_system", install_system_mock - ) + monkeypatch.setattr("mlia.backend.manager.install_system", install_system_mock) - mock_executor = MagicMock() - aiet_runner = AIETRunner(mock_executor) - aiet_runner.install_system(Path("test_system_path")) + backend_runner = BackendRunner() + backend_runner.install_system(Path("test_system_path")) install_system_mock.assert_called_once_with(Path("test_system_path")) - mock_executor.assert_not_called() @pytest.mark.parametrize( "available_systems, systems, expected_result", @@ -222,14 +217,11 @@ class TestAIETRunner: monkeypatch: pytest.MonkeyPatch, ) -> None: """Test method systems_installed.""" - self._setup_aiet(monkeypatch, available_systems) + self._setup_backends(monkeypatch, available_systems) - mock_executor = MagicMock() - aiet_runner = AIETRunner(mock_executor) + backend_runner = BackendRunner() - assert aiet_runner.systems_installed(systems) is expected_result - - mock_executor.assert_not_called() + assert backend_runner.systems_installed(systems) is expected_result @pytest.mark.parametrize( "available_apps, applications, expected_result", @@ -249,12 +241,10 @@ class TestAIETRunner: monkeypatch: pytest.MonkeyPatch, ) -> None: """Test method applications_installed.""" - self._setup_aiet(monkeypatch, [], available_apps) - mock_executor = MagicMock() - aiet_runner = AIETRunner(mock_executor) + self._setup_backends(monkeypatch, [], available_apps) + backend_runner = BackendRunner() - assert aiet_runner.applications_installed(applications) is expected_result - mock_executor.assert_not_called() + assert backend_runner.applications_installed(applications) is expected_result @pytest.mark.parametrize( "available_apps, applications", @@ -273,30 +263,23 @@ class TestAIETRunner: monkeypatch: pytest.MonkeyPatch, ) -> None: """Test method get_installed_applications.""" - mock_executor = MagicMock() - self._setup_aiet(monkeypatch, [], available_apps) - - aiet_runner = AIETRunner(mock_executor) - assert applications == aiet_runner.get_installed_applications() + self._setup_backends(monkeypatch, [], available_apps) - mock_executor.assert_not_called() + backend_runner = BackendRunner() + assert applications == backend_runner.get_installed_applications() @staticmethod def test_install_application(monkeypatch: pytest.MonkeyPatch) -> None: """Test application installation.""" mock_install_application = MagicMock() monkeypatch.setattr( - "mlia.tools.aiet_wrapper.install_application", mock_install_application + "mlia.backend.manager.install_application", mock_install_application ) - mock_executor = MagicMock() - - aiet_runner = AIETRunner(mock_executor) - aiet_runner.install_application(Path("test_application_path")) + backend_runner = BackendRunner() + backend_runner.install_application(Path("test_application_path")) mock_install_application.assert_called_once_with(Path("test_application_path")) - mock_executor.assert_not_called() - @pytest.mark.parametrize( "available_apps, application, installed", [ @@ -321,66 +304,113 @@ class TestAIETRunner: monkeypatch: pytest.MonkeyPatch, ) -> None: """Test method is_application_installed.""" - self._setup_aiet(monkeypatch, [], available_apps) - - mock_executor = MagicMock() - aiet_runner = AIETRunner(mock_executor) - assert installed == aiet_runner.is_application_installed(application, "system1") + self._setup_backends(monkeypatch, [], available_apps) - mock_executor.assert_not_called() + backend_runner = BackendRunner() + assert installed == backend_runner.is_application_installed( + application, "system1" + ) @staticmethod @pytest.mark.parametrize( "execution_params, expected_command", [ ( - ExecutionParams("application1", "system1", [], [], []), - ["aiet", "application", "run", "-n", "application1", "-s", "system1"], + ExecutionParams("application_4", "System 4", [], [], []), + ["application_4", [], "System 4", [], []], ), ( ExecutionParams( - "application1", - "system1", - ["input_file=123.txt", "size=777"], - ["param1=456", "param2=789"], + "application_6", + "System 6", + ["param1=value2"], + ["sys-param1=value2"], + [], + ), + [ + "application_6", + ["param1=value2"], + "System 6", + ["sys-param1=value2"], + [], + ], + ), + ], + ) + def test_run_application_local( + monkeypatch: pytest.MonkeyPatch, + execution_params: ExecutionParams, + expected_command: List[str], + ) -> None: + """Test method run_application with local systems.""" + run_app = MagicMock(wraps=run_application) + monkeypatch.setattr("mlia.backend.manager.run_application", run_app) + + backend_runner = BackendRunner() + backend_runner.run_application(execution_params) + + run_app.assert_called_once_with(*expected_command) + + @staticmethod + @pytest.mark.parametrize( + "execution_params, expected_command", + [ + ( + ExecutionParams( + "application_1", + "System 1", + [], + [], ["source1.txt:dest1.txt", "source2.txt:dest2.txt"], ), [ - "aiet", - "application", - "run", - "-n", - "application1", - "-s", - "system1", - "-p", - "input_file=123.txt", - "-p", - "size=777", - "--system-param", - "param1=456", - "--system-param", - "param2=789", - "--deploy", - "source1.txt:dest1.txt", - "--deploy", - "source2.txt:dest2.txt", + "application_1", + [], + "System 1", + [], + [ + DataPaths(Path("source1.txt"), "dest1.txt"), + DataPaths(Path("source2.txt"), "dest2.txt"), + ], ], ), ], ) - def test_run_application( - execution_params: ExecutionParams, expected_command: List[str] + def test_run_application_connected( + monkeypatch: pytest.MonkeyPatch, + execution_params: ExecutionParams, + expected_command: List[str], ) -> None: - """Test method run_application.""" - mock_executor = MagicMock() - mock_running_command = MagicMock() - mock_executor.submit.return_value = mock_running_command + """Test method run_application with connectable systems (SSH).""" + app = get_application(execution_params.application, execution_params.system)[0] + sys = get_system(execution_params.system) + + assert sys is not None + + connect_mock = MagicMock(return_value=True, name="connect_mock") + deploy_mock = MagicMock(return_value=True, name="deploy_mock") + run_mock = MagicMock( + return_value=(os.EX_OK, bytearray(), bytearray()), name="run_mock" + ) + sys.establish_connection = connect_mock # type: ignore + sys.deploy = deploy_mock # type: ignore + sys.run = run_mock # type: ignore + + monkeypatch.setattr( + "mlia.backend.execution.get_application_and_system", + MagicMock(return_value=(app, sys)), + ) - aiet_runner = AIETRunner(mock_executor) - aiet_runner.run_application(execution_params) + run_app_mock = MagicMock(wraps=run_application) + monkeypatch.setattr("mlia.backend.manager.run_application", run_app_mock) - mock_executor.submit.assert_called_once_with(expected_command) + backend_runner = BackendRunner() + backend_runner.run_application(execution_params) + + run_app_mock.assert_called_once_with(*expected_command) + + connect_mock.assert_called_once() + assert deploy_mock.call_count == 2 @pytest.mark.parametrize( @@ -490,16 +520,16 @@ def test_estimate_performance( backend: str, expected_error: Any, test_tflite_model: Path, - aiet_runner: MagicMock, + backend_runner: MagicMock, ) -> None: """Test getting performance estimations.""" system_name, system_installed = system application_name, application_installed = application - aiet_runner.is_system_installed.return_value = system_installed - aiet_runner.is_application_installed.return_value = application_installed + backend_runner.is_system_installed.return_value = system_installed + backend_runner.is_application_installed.return_value = application_installed - mock_process = create_mock_process( + mock_context = create_mock_context( [ "NPU AXI0_RD_DATA_BEAT_RECEIVED beats: 1", "NPU AXI0_WR_DATA_BEAT_WRITTEN beats: 2", @@ -507,12 +537,10 @@ def test_estimate_performance( "NPU ACTIVE cycles: 4", "NPU IDLE cycles: 5", "NPU TOTAL cycles: 6", - ], - [], + ] ) - mock_generic_inference_run = RunningCommand(mock_process) - aiet_runner.run_application.return_value = mock_generic_inference_run + backend_runner.run_application.return_value = mock_context with expected_error: perf_metrics = estimate_performance( @@ -529,19 +557,19 @@ def test_estimate_performance( npu_total_cycles=6, ) - assert aiet_runner.is_system_installed.called_once_with(system_name) - assert aiet_runner.is_application_installed.called_once_with( + assert backend_runner.is_system_installed.called_once_with(system_name) + assert backend_runner.is_application_installed.called_once_with( application_name, system_name ) @pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) def test_estimate_performance_insufficient_data( - aiet_runner: MagicMock, test_tflite_model: Path, backend: str + backend_runner: MagicMock, test_tflite_model: Path, backend: str ) -> None: """Test that performance could not be estimated when not all data presented.""" - aiet_runner.is_system_installed.return_value = True - aiet_runner.is_application_installed.return_value = True + backend_runner.is_system_installed.return_value = True + backend_runner.is_application_installed.return_value = True no_total_cycles_output = [ "NPU AXI0_RD_DATA_BEAT_RECEIVED beats: 1", @@ -550,13 +578,9 @@ def test_estimate_performance_insufficient_data( "NPU ACTIVE cycles: 4", "NPU IDLE cycles: 5", ] - mock_process = create_mock_process( - no_total_cycles_output, - [], - ) + mock_context = create_mock_context(no_total_cycles_output) - mock_generic_inference_run = RunningCommand(mock_process) - aiet_runner.run_application.return_value = mock_generic_inference_run + backend_runner.run_application.return_value = mock_context with pytest.raises( Exception, match="Unable to get performance metrics, insufficient data" @@ -567,16 +591,14 @@ def test_estimate_performance_insufficient_data( @pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) def test_estimate_performance_invalid_output( - test_tflite_model: Path, aiet_runner: MagicMock, backend: str + test_tflite_model: Path, backend_runner: MagicMock, backend: str ) -> None: """Test estimation could not be done if inference produces unexpected output.""" - aiet_runner.is_system_installed.return_value = True - aiet_runner.is_application_installed.return_value = True + backend_runner.is_system_installed.return_value = True + backend_runner.is_application_installed.return_value = True - mock_process = create_mock_process( - ["Something", "is", "wrong"], ["What a nice error!"] - ) - aiet_runner.run_application.return_value = RunningCommand(mock_process) + mock_context = create_mock_context(["Something", "is", "wrong"]) + backend_runner.run_application.return_value = mock_context with pytest.raises(Exception, match="Unable to get performance metrics"): estimate_performance( @@ -586,12 +608,6 @@ def test_estimate_performance_invalid_output( ) -def test_get_aiet_runner() -> None: - """Test getting aiet runner.""" - aiet_runner = get_aiet_runner() - assert isinstance(aiet_runner, AIETRunner) - - def create_mock_process(stdout: List[str], stderr: List[str]) -> MagicMock: """Mock underlying process.""" mock_process = MagicMock() @@ -601,6 +617,18 @@ def create_mock_process(stdout: List[str], stderr: List[str]) -> MagicMock: return mock_process +def create_mock_context(stdout: List[str]) -> ExecutionContext: + """Mock ExecutionContext.""" + ctx = ExecutionContext( + app=get_application("application_1")[0], + app_params=[], + system=get_system("System 1"), + system_params=[], + ) + ctx.stdout = bytearray("\n".join(stdout).encode("utf-8")) + return ctx + + @pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) def test_get_generic_runner(backend: str) -> None: """Test function get_generic_runner().""" @@ -621,8 +649,8 @@ def test_get_generic_runner(backend: str) -> None: ("Corstone-310", "ethos-u55"), ), ) -def test_aiet_backend_support(backend: str, device_type: str) -> None: - """Test AIET backend & device support.""" +def test_backend_support(backend: str, device_type: str) -> None: + """Test backend & device support.""" assert is_supported(backend) assert is_supported(backend, device_type) @@ -714,10 +742,10 @@ class TestGenericInferenceRunnerEthosU: @staticmethod @pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) def test_inference_should_fail_if_system_not_installed( - aiet_runner: MagicMock, test_tflite_model: Path, backend: str + backend_runner: MagicMock, test_tflite_model: Path, backend: str ) -> None: """Test that inference should fail if system is not installed.""" - aiet_runner.is_system_installed.return_value = False + backend_runner.is_system_installed.return_value = False generic_runner = get_generic_runner( DeviceInfo("ethos-u55", 256, memory_mode="Shared_Sram"), backend @@ -731,11 +759,11 @@ class TestGenericInferenceRunnerEthosU: @staticmethod @pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) def test_inference_should_fail_is_apps_not_installed( - aiet_runner: MagicMock, test_tflite_model: Path, backend: str + backend_runner: MagicMock, test_tflite_model: Path, backend: str ) -> None: """Test that inference should fail if apps are not installed.""" - aiet_runner.is_system_installed.return_value = True - aiet_runner.is_application_installed.return_value = False + backend_runner.is_system_installed.return_value = True + backend_runner.is_application_installed.return_value = False generic_runner = get_generic_runner( DeviceInfo("ethos-u55", 256, memory_mode="Shared_Sram"), backend @@ -749,12 +777,12 @@ class TestGenericInferenceRunnerEthosU: generic_runner.run(ModelInfo(test_tflite_model), []) -@pytest.fixture(name="aiet_runner") -def fixture_aiet_runner(monkeypatch: pytest.MonkeyPatch) -> MagicMock: - """Mock AIET runner.""" - aiet_runner_mock = MagicMock(spec=AIETRunner) +@pytest.fixture(name="backend_runner") +def fixture_backend_runner(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + """Mock backend runner.""" + backend_runner_mock = MagicMock(spec=BackendRunner) monkeypatch.setattr( - "mlia.tools.aiet_wrapper.get_aiet_runner", - MagicMock(return_value=aiet_runner_mock), + "mlia.backend.manager.get_backend_runner", + MagicMock(return_value=backend_runner_mock), ) - return aiet_runner_mock + return backend_runner_mock diff --git a/tests/aiet/test_backend_output_parser.py b/tests/mlia/test_backend_output_parser.py index d659812..d86aac8 100644 --- a/tests/aiet/test_backend_output_parser.py +++ b/tests/mlia/test_backend_output_parser.py @@ -8,9 +8,9 @@ from typing import Dict import pytest -from aiet.backend.output_parser import Base64OutputParser -from aiet.backend.output_parser import OutputParser -from aiet.backend.output_parser import RegexOutputParser +from mlia.backend.output_parser import Base64OutputParser +from mlia.backend.output_parser import OutputParser +from mlia.backend.output_parser import RegexOutputParser OUTPUT_MATCH_ALL = bytearray( diff --git a/tests/aiet/test_utils_proc.py b/tests/mlia/test_backend_proc.py index 9fb48dd..9ca4788 100644 --- a/tests/aiet/test_utils_proc.py +++ b/tests/mlia/test_backend_proc.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 # pylint: disable=attribute-defined-outside-init,no-self-use,not-callable -"""Pytests for testing aiet/utils/proc.py.""" +"""Pytests for testing mlia/backend/proc.py.""" from pathlib import Path from typing import Any from unittest import mock @@ -10,16 +10,16 @@ import psutil import pytest from sh import ErrorReturnCode -from aiet.utils.proc import Command -from aiet.utils.proc import CommandFailedException -from aiet.utils.proc import CommandNotFound -from aiet.utils.proc import parse_command -from aiet.utils.proc import print_command_stdout -from aiet.utils.proc import run_and_wait -from aiet.utils.proc import save_process_info -from aiet.utils.proc import ShellCommand -from aiet.utils.proc import terminate_command -from aiet.utils.proc import terminate_external_process +from mlia.backend.proc import Command +from mlia.backend.proc import CommandFailedException +from mlia.backend.proc import CommandNotFound +from mlia.backend.proc import parse_command +from mlia.backend.proc import print_command_stdout +from mlia.backend.proc import run_and_wait +from mlia.backend.proc import save_process_info +from mlia.backend.proc import ShellCommand +from mlia.backend.proc import terminate_command +from mlia.backend.proc import terminate_external_process class TestShellCommand: @@ -211,12 +211,12 @@ class TestRunAndWait: """Init test method.""" self.execute_command_mock = mock.MagicMock() monkeypatch.setattr( - "aiet.utils.proc.execute_command", self.execute_command_mock + "mlia.backend.proc.execute_command", self.execute_command_mock ) self.terminate_command_mock = mock.MagicMock() monkeypatch.setattr( - "aiet.utils.proc.terminate_command", self.terminate_command_mock + "mlia.backend.proc.terminate_command", self.terminate_command_mock ) def test_if_execute_command_raises_exception(self) -> None: diff --git a/tests/aiet/test_backend_protocol.py b/tests/mlia/test_backend_protocol.py index 2103238..35e9986 100644 --- a/tests/aiet/test_backend_protocol.py +++ b/tests/mlia/test_backend_protocol.py @@ -10,12 +10,12 @@ from unittest.mock import MagicMock import paramiko import pytest -from aiet.backend.common import ConfigurationException -from aiet.backend.config import LocalProtocolConfig -from aiet.backend.protocol import CustomSFTPClient -from aiet.backend.protocol import LocalProtocol -from aiet.backend.protocol import ProtocolFactory -from aiet.backend.protocol import SSHProtocol +from mlia.backend.common import ConfigurationException +from mlia.backend.config import LocalProtocolConfig +from mlia.backend.protocol import CustomSFTPClient +from mlia.backend.protocol import LocalProtocol +from mlia.backend.protocol import ProtocolFactory +from mlia.backend.protocol import SSHProtocol class TestProtocolFactory: @@ -94,13 +94,13 @@ class TestSSHProtocol: self.mock_ssh_channel.recv_stderr_ready.side_effect = [False, True] monkeypatch.setattr( - "aiet.backend.protocol.paramiko.client.SSHClient", + "mlia.backend.protocol.paramiko.client.SSHClient", MagicMock(return_value=self.mock_ssh_client), ) self.mock_sftp_client = MagicMock(spec=CustomSFTPClient) monkeypatch.setattr( - "aiet.backend.protocol.CustomSFTPClient.from_transport", + "mlia.backend.protocol.CustomSFTPClient.from_transport", MagicMock(return_value=self.mock_sftp_client), ) @@ -116,7 +116,7 @@ class TestSSHProtocol: def test_unable_create_ssh_client(self, monkeypatch: Any) -> None: """Test that command should fail if unable to create ssh client instance.""" monkeypatch.setattr( - "aiet.backend.protocol.paramiko.client.SSHClient", + "mlia.backend.protocol.paramiko.client.SSHClient", MagicMock(side_effect=OSError("Error!")), ) @@ -201,14 +201,14 @@ class TestCustomSFTPClient: self.mock_mkdir = MagicMock() self.mock_put = MagicMock() monkeypatch.setattr( - "aiet.backend.protocol.paramiko.SFTPClient.__init__", + "mlia.backend.protocol.paramiko.SFTPClient.__init__", MagicMock(return_value=None), ) monkeypatch.setattr( - "aiet.backend.protocol.paramiko.SFTPClient.mkdir", self.mock_mkdir + "mlia.backend.protocol.paramiko.SFTPClient.mkdir", self.mock_mkdir ) monkeypatch.setattr( - "aiet.backend.protocol.paramiko.SFTPClient.put", self.mock_put + "mlia.backend.protocol.paramiko.SFTPClient.put", self.mock_put ) self.sftp_client = CustomSFTPClient(MagicMock()) diff --git a/tests/aiet/test_backend_source.py b/tests/mlia/test_backend_source.py index 13b2c6d..84a6a77 100644 --- a/tests/aiet/test_backend_source.py +++ b/tests/mlia/test_backend_source.py @@ -11,11 +11,11 @@ from unittest.mock import patch import pytest -from aiet.backend.common import ConfigurationException -from aiet.backend.source import create_destination_and_install -from aiet.backend.source import DirectorySource -from aiet.backend.source import get_source -from aiet.backend.source import TarArchiveSource +from mlia.backend.common import ConfigurationException +from mlia.backend.source import create_destination_and_install +from mlia.backend.source import DirectorySource +from mlia.backend.source import get_source +from mlia.backend.source import TarArchiveSource def test_create_destination_and_install(test_systems_path: Path, tmpdir: Any) -> None: @@ -28,7 +28,7 @@ def test_create_destination_and_install(test_systems_path: Path, tmpdir: Any) -> assert (resources / "system1").is_dir() -@patch("aiet.backend.source.DirectorySource.create_destination", return_value=False) +@patch("mlia.backend.source.DirectorySource.create_destination", return_value=False) def test_create_destination_and_install_if_dest_creation_not_required( mock_ds_create_destination: Any, tmpdir: Any ) -> None: @@ -66,7 +66,11 @@ def test_create_destination_and_install_if_name_is_empty() -> None: @pytest.mark.parametrize( "source_path, expected_class, expected_error", [ - (Path("applications/application1/"), DirectorySource, does_not_raise()), + ( + Path("backends/applications/application1/"), + DirectorySource, + does_not_raise(), + ), ( Path("archives/applications/application1.tar.gz"), TarArchiveSource, diff --git a/tests/aiet/test_backend_system.py b/tests/mlia/test_backend_system.py index a581547..21187ff 100644 --- a/tests/aiet/test_backend_system.py +++ b/tests/mlia/test_backend_system.py @@ -13,29 +13,29 @@ from unittest.mock import MagicMock import pytest -from aiet.backend.common import Command -from aiet.backend.common import ConfigurationException -from aiet.backend.common import Param -from aiet.backend.common import UserParamConfig -from aiet.backend.config import LocalProtocolConfig -from aiet.backend.config import ProtocolConfig -from aiet.backend.config import SSHConfig -from aiet.backend.config import SystemConfig -from aiet.backend.controller import SystemController -from aiet.backend.controller import SystemControllerSingleInstance -from aiet.backend.protocol import LocalProtocol -from aiet.backend.protocol import SSHProtocol -from aiet.backend.protocol import SupportsClose -from aiet.backend.protocol import SupportsDeploy -from aiet.backend.system import ControlledSystem -from aiet.backend.system import get_available_systems -from aiet.backend.system import get_controller -from aiet.backend.system import get_system -from aiet.backend.system import install_system -from aiet.backend.system import load_system -from aiet.backend.system import remove_system -from aiet.backend.system import StandaloneSystem -from aiet.backend.system import System +from mlia.backend.common import Command +from mlia.backend.common import ConfigurationException +from mlia.backend.common import Param +from mlia.backend.common import UserParamConfig +from mlia.backend.config import LocalProtocolConfig +from mlia.backend.config import ProtocolConfig +from mlia.backend.config import SSHConfig +from mlia.backend.config import SystemConfig +from mlia.backend.controller import SystemController +from mlia.backend.controller import SystemControllerSingleInstance +from mlia.backend.protocol import LocalProtocol +from mlia.backend.protocol import SSHProtocol +from mlia.backend.protocol import SupportsClose +from mlia.backend.protocol import SupportsDeploy +from mlia.backend.system import ControlledSystem +from mlia.backend.system import get_available_systems +from mlia.backend.system import get_controller +from mlia.backend.system import get_system +from mlia.backend.system import install_system +from mlia.backend.system import load_system +from mlia.backend.system import remove_system +from mlia.backend.system import StandaloneSystem +from mlia.backend.system import System def dummy_resolver( @@ -56,8 +56,13 @@ def test_get_available_systems() -> None: """Test get_available_systems mocking get_resources.""" available_systems = get_available_systems() assert all(isinstance(s, System) for s in available_systems) - assert len(available_systems) == 3 - assert [str(s) for s in available_systems] == ["System 1", "System 2", "System 4"] + assert len(available_systems) == 4 + assert [str(s) for s in available_systems] == [ + "System 1", + "System 2", + "System 4", + "System 6", + ] def test_get_system() -> None: @@ -91,12 +96,12 @@ def test_get_system() -> None: pytest.raises(Exception, match="Unable to read system definition"), ), ( - "systems/system1", + "backends/systems/system1", 0, pytest.raises(Exception, match="Systems .* are already installed"), ), ( - "systems/system3", + "backends/systems/system3", 0, pytest.raises(Exception, match="Unable to read system definition"), ), @@ -119,7 +124,7 @@ def test_install_system( """Test system installation from archive.""" mock_create_destination_and_install = MagicMock() monkeypatch.setattr( - "aiet.backend.system.create_destination_and_install", + "mlia.backend.system.create_destination_and_install", mock_create_destination_and_install, ) @@ -132,7 +137,7 @@ def test_install_system( def test_remove_system(monkeypatch: Any) -> None: """Test system removal.""" mock_remove_backend = MagicMock() - monkeypatch.setattr("aiet.backend.system.remove_backend", mock_remove_backend) + monkeypatch.setattr("mlia.backend.system.remove_backend", mock_remove_backend) remove_system("some_system_dir") mock_remove_backend.assert_called_once() @@ -140,7 +145,7 @@ def test_remove_system(monkeypatch: Any) -> None: def test_system(monkeypatch: Any) -> None: """Test the System class.""" config = SystemConfig(name="System 1") - monkeypatch.setattr("aiet.backend.system.ProtocolFactory", MagicMock()) + monkeypatch.setattr("mlia.backend.system.ProtocolFactory", MagicMock()) system = System(config) assert str(system) == "System 1" assert system.name == "System 1" @@ -202,7 +207,7 @@ def test_system_supports_deploy(system_name: str, expected_value: bool) -> None: def test_system_start_and_stop(monkeypatch: Any, mock_protocol: MagicMock) -> None: """Test system start, run commands and stop.""" monkeypatch.setattr( - "aiet.backend.system.ProtocolFactory.get_protocol", + "mlia.backend.system.ProtocolFactory.get_protocol", MagicMock(return_value=mock_protocol), ) @@ -516,7 +521,7 @@ def test_system_deploy_data( ) -> None: """Test deploy data functionality.""" monkeypatch.setattr( - "aiet.backend.system.ProtocolFactory.get_protocol", + "mlia.backend.system.ProtocolFactory.get_protocol", MagicMock(return_value=mock_protocol), ) diff --git a/tests/mlia/test_cli_logging.py b/tests/mlia/test_cli_logging.py index 7c5f299..3f59cb6 100644 --- a/tests/mlia/test_cli_logging.py +++ b/tests/mlia/test_cli_logging.py @@ -32,7 +32,7 @@ def teardown_function() -> None: ( None, True, - """mlia.tools.aiet_wrapper - aiet debug + """mlia.backend.manager - backends debug cli info mlia.cli - cli debug """, @@ -41,11 +41,11 @@ mlia.cli - cli debug ( "logs", True, - """mlia.tools.aiet_wrapper - aiet debug + """mlia.backend.manager - backends debug cli info mlia.cli - cli debug """, - """mlia.tools.aiet_wrapper - DEBUG - aiet debug + """mlia.backend.manager - DEBUG - backends debug mlia.cli - DEBUG - cli debug """, ), @@ -64,8 +64,8 @@ def test_setup_logging( setup_logging(logs_dir_path, verbose) - aiet_logger = logging.getLogger("mlia.tools.aiet_wrapper") - aiet_logger.debug("aiet debug") + backend_logger = logging.getLogger("mlia.backend.manager") + backend_logger.debug("backends debug") cli_logger = logging.getLogger("mlia.cli") cli_logger.info("cli info") diff --git a/tests/mlia/test_devices_ethosu_performance.py b/tests/mlia/test_devices_ethosu_performance.py index e27efa0..b3e5298 100644 --- a/tests/mlia/test_devices_ethosu_performance.py +++ b/tests/mlia/test_devices_ethosu_performance.py @@ -23,6 +23,6 @@ def test_memory_usage_conversion() -> None: def mock_performance_estimation(monkeypatch: pytest.MonkeyPatch) -> None: """Mock performance estimation.""" monkeypatch.setattr( - "mlia.tools.aiet_wrapper.estimate_performance", + "mlia.backend.manager.estimate_performance", MagicMock(return_value=MagicMock()), ) diff --git a/tests/aiet/test_resources/application_config.json b/tests/mlia/test_resources/application_config.json index 2dfcfec..2dfcfec 100644 --- a/tests/aiet/test_resources/application_config.json +++ b/tests/mlia/test_resources/application_config.json diff --git a/tests/aiet/test_resources/application_config.json.license b/tests/mlia/test_resources/application_config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/application_config.json.license +++ b/tests/mlia/test_resources/application_config.json.license diff --git a/tests/aiet/test_resources/applications/application1/aiet-config.json b/tests/mlia/test_resources/backends/applications/application1/aiet-config.json index 97f0401..97f0401 100644 --- a/tests/aiet/test_resources/applications/application1/aiet-config.json +++ b/tests/mlia/test_resources/backends/applications/application1/aiet-config.json diff --git a/tests/aiet/test_resources/applications/application1/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application1/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/applications/application1/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/applications/application1/aiet-config.json.license diff --git a/tests/aiet/test_resources/applications/application2/aiet-config.json b/tests/mlia/test_resources/backends/applications/application2/aiet-config.json index e9122d3..e9122d3 100644 --- a/tests/aiet/test_resources/applications/application2/aiet-config.json +++ b/tests/mlia/test_resources/backends/applications/application2/aiet-config.json diff --git a/tests/aiet/test_resources/applications/application2/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application2/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/applications/application2/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/applications/application2/aiet-config.json.license diff --git a/tests/aiet/test_resources/applications/application3/readme.txt b/tests/mlia/test_resources/backends/applications/application3/readme.txt index 8c72c05..8c72c05 100644 --- a/tests/aiet/test_resources/applications/application3/readme.txt +++ b/tests/mlia/test_resources/backends/applications/application3/readme.txt diff --git a/tests/aiet/test_resources/applications/application4/aiet-config.json b/tests/mlia/test_resources/backends/applications/application4/aiet-config.json index 34dc780..ffb5746 100644 --- a/tests/aiet/test_resources/applications/application4/aiet-config.json +++ b/tests/mlia/test_resources/backends/applications/application4/aiet-config.json @@ -10,10 +10,11 @@ ], "commands": { "build": [ - "cp ../hello_app.txt . # {user_params:0}" + "cp ../hello_app.txt .", + "echo '{user_params:0}' > params.txt" ], "run": [ - "{application.build_dir}/hello_app.txt" + "cat {application.build_dir}/hello_app.txt" ] }, "user_params": { diff --git a/tests/aiet/test_resources/applications/application4/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application4/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/applications/application4/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/applications/application4/aiet-config.json.license diff --git a/tests/aiet/test_resources/applications/application4/hello_app.txt b/tests/mlia/test_resources/backends/applications/application4/hello_app.txt index 2ec0d1d..2ec0d1d 100644 --- a/tests/aiet/test_resources/applications/application4/hello_app.txt +++ b/tests/mlia/test_resources/backends/applications/application4/hello_app.txt diff --git a/tests/aiet/test_resources/applications/application5/aiet-config.json b/tests/mlia/test_resources/backends/applications/application5/aiet-config.json index 5269409..5269409 100644 --- a/tests/aiet/test_resources/applications/application5/aiet-config.json +++ b/tests/mlia/test_resources/backends/applications/application5/aiet-config.json diff --git a/tests/aiet/test_resources/applications/application5/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application5/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/applications/application5/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/applications/application5/aiet-config.json.license diff --git a/tests/mlia/test_resources/backends/applications/application6/aiet-config.json b/tests/mlia/test_resources/backends/applications/application6/aiet-config.json new file mode 100644 index 0000000..56ad807 --- /dev/null +++ b/tests/mlia/test_resources/backends/applications/application6/aiet-config.json @@ -0,0 +1,42 @@ +[ + { + "name": "application_6", + "description": "This is application 6", + "supported_systems": [ + { + "name": "System 6" + } + ], + "build_dir": "build", + "commands": { + "clean": [ + "echo 'clean'" + ], + "build": [ + "echo 'build'" + ], + "run": [ + "echo 'run {user_params:param1}'" + ], + "post_run": [ + "echo 'post_run'" + ] + }, + "user_params": { + "build": [], + "run": [ + { + "name": "--param1", + "description": "Test parameter", + "values": [ + "value1", + "value2", + "value3" + ], + "default_value": "value3", + "alias": "param1" + } + ] + } + } +] diff --git a/tests/aiet/test_resources/systems/system1/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application6/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/systems/system1/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/applications/application6/aiet-config.json.license diff --git a/tests/aiet/test_resources/applications/readme.txt b/tests/mlia/test_resources/backends/applications/readme.txt index a1f8209..a1f8209 100644 --- a/tests/aiet/test_resources/applications/readme.txt +++ b/tests/mlia/test_resources/backends/applications/readme.txt diff --git a/tests/aiet/test_resources/systems/system1/aiet-config.json b/tests/mlia/test_resources/backends/systems/system1/aiet-config.json index 4b5dd19..4b5dd19 100644 --- a/tests/aiet/test_resources/systems/system1/aiet-config.json +++ b/tests/mlia/test_resources/backends/systems/system1/aiet-config.json diff --git a/tests/aiet/test_resources/systems/system2/aiet-config.json.license b/tests/mlia/test_resources/backends/systems/system1/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/systems/system2/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/systems/system1/aiet-config.json.license diff --git a/tests/aiet/test_resources/systems/system1/system_artifact/dummy.txt b/tests/mlia/test_resources/backends/systems/system1/system_artifact/dummy.txt index 487e9d8..487e9d8 100644 --- a/tests/aiet/test_resources/systems/system1/system_artifact/dummy.txt +++ b/tests/mlia/test_resources/backends/systems/system1/system_artifact/dummy.txt diff --git a/tests/aiet/test_resources/systems/system2/aiet-config.json b/tests/mlia/test_resources/backends/systems/system2/aiet-config.json index a9e0eb3..a9e0eb3 100644 --- a/tests/aiet/test_resources/systems/system2/aiet-config.json +++ b/tests/mlia/test_resources/backends/systems/system2/aiet-config.json diff --git a/tests/aiet/test_resources/systems/system4/aiet-config.json.license b/tests/mlia/test_resources/backends/systems/system2/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/systems/system4/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/systems/system2/aiet-config.json.license diff --git a/tests/aiet/test_resources/systems/system3/readme.txt b/tests/mlia/test_resources/backends/systems/system3/readme.txt index aba5a9c..aba5a9c 100644 --- a/tests/aiet/test_resources/systems/system3/readme.txt +++ b/tests/mlia/test_resources/backends/systems/system3/readme.txt diff --git a/tests/aiet/test_resources/systems/system4/aiet-config.json b/tests/mlia/test_resources/backends/systems/system4/aiet-config.json index 295e00f..7b13160 100644 --- a/tests/aiet/test_resources/systems/system4/aiet-config.json +++ b/tests/mlia/test_resources/backends/systems/system4/aiet-config.json @@ -9,7 +9,7 @@ "commands": { "run": [ "echo {application.name}", - "cat {application.commands.run:0}" + "{application.commands.run:0}" ] }, "user_params": { diff --git a/tests/aiet/test_resources/tools/tool1/aiet-config.json.license b/tests/mlia/test_resources/backends/systems/system4/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/tools/tool1/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/systems/system4/aiet-config.json.license diff --git a/tests/mlia/test_resources/backends/systems/system6/aiet-config.json b/tests/mlia/test_resources/backends/systems/system6/aiet-config.json new file mode 100644 index 0000000..4242f64 --- /dev/null +++ b/tests/mlia/test_resources/backends/systems/system6/aiet-config.json @@ -0,0 +1,34 @@ +[ + { + "name": "System 6", + "description": "This is system 6", + "build_dir": "build", + "data_transfer": { + "protocol": "local" + }, + "variables": { + "var1": "{user_params:sys-param1}" + }, + "commands": { + "run": [ + "echo {application.name}", + "{application.commands.run:0}" + ] + }, + "user_params": { + "run": [ + { + "name": "--sys-param1", + "description": "Test parameter", + "values": [ + "value1", + "value2", + "value3" + ], + "default_value": "value1", + "alias": "sys-param1" + } + ] + } + } +] diff --git a/tests/aiet/test_resources/tools/tool2/aiet-config.json.license b/tests/mlia/test_resources/backends/systems/system6/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/tools/tool2/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/systems/system6/aiet-config.json.license diff --git a/tests/aiet/test_resources/hello_world.json b/tests/mlia/test_resources/hello_world.json index 8a9a448..8a9a448 100644 --- a/tests/aiet/test_resources/hello_world.json +++ b/tests/mlia/test_resources/hello_world.json diff --git a/tests/aiet/test_resources/hello_world.json.license b/tests/mlia/test_resources/hello_world.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/hello_world.json.license +++ b/tests/mlia/test_resources/hello_world.json.license diff --git a/tests/aiet/test_resources/scripts/test_backend_run b/tests/mlia/test_resources/scripts/test_backend_run index 548f577..548f577 100755 --- a/tests/aiet/test_resources/scripts/test_backend_run +++ b/tests/mlia/test_resources/scripts/test_backend_run diff --git a/tests/aiet/test_resources/scripts/test_backend_run_script.sh b/tests/mlia/test_resources/scripts/test_backend_run_script.sh index 548f577..548f577 100644 --- a/tests/aiet/test_resources/scripts/test_backend_run_script.sh +++ b/tests/mlia/test_resources/scripts/test_backend_run_script.sh diff --git a/tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json index fe51488..fe51488 100644 --- a/tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json +++ b/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json diff --git a/tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json.license +++ b/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json.license diff --git a/tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json index ff1cf1a..ff1cf1a 100644 --- a/tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json +++ b/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json diff --git a/tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json.license +++ b/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json.license diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json index 724b31b..724b31b 100644 --- a/tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json +++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license +++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json index 1ebb29c..1ebb29c 100644 --- a/tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json +++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license +++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json index 410d12d..410d12d 100644 --- a/tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json +++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license +++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license diff --git a/tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json b/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json index fe51488..fe51488 100644 --- a/tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json +++ b/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json diff --git a/tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json.license b/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json.license +++ b/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json.license diff --git a/tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json b/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json index 20142e9..20142e9 100644 --- a/tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json +++ b/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json diff --git a/tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json.license b/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json.license +++ b/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json.license diff --git a/tests/mlia/test_tools_metadata_corstone.py b/tests/mlia/test_tools_metadata_corstone.py index 2ce3610..017d0c7 100644 --- a/tests/mlia/test_tools_metadata_corstone.py +++ b/tests/mlia/test_tools_metadata_corstone.py @@ -9,13 +9,13 @@ from unittest.mock import MagicMock import pytest -from mlia.tools.aiet_wrapper import AIETRunner +from mlia.backend.manager import BackendRunner from mlia.tools.metadata.common import DownloadAndInstall from mlia.tools.metadata.common import InstallFromPath -from mlia.tools.metadata.corstone import AIETBasedInstallation -from mlia.tools.metadata.corstone import AIETMetadata from mlia.tools.metadata.corstone import BackendInfo +from mlia.tools.metadata.corstone import BackendInstallation from mlia.tools.metadata.corstone import BackendInstaller +from mlia.tools.metadata.corstone import BackendMetadata from mlia.tools.metadata.corstone import CompoundPathChecker from mlia.tools.metadata.corstone import Corstone300Installer from mlia.tools.metadata.corstone import get_corstone_installations @@ -40,8 +40,8 @@ def fixture_test_mlia_resources( return mlia_resources -def get_aiet_based_installation( # pylint: disable=too-many-arguments - aiet_runner_mock: MagicMock = MagicMock(), +def get_backend_installation( # pylint: disable=too-many-arguments + backend_runner_mock: MagicMock = MagicMock(), name: str = "test_name", description: str = "test_description", download_artifact: Optional[MagicMock] = None, @@ -50,11 +50,11 @@ def get_aiet_based_installation( # pylint: disable=too-many-arguments system_config: Optional[str] = None, backend_installer: BackendInstaller = MagicMock(), supported_platforms: Optional[List[str]] = None, -) -> AIETBasedInstallation: - """Get AIET based installation.""" - return AIETBasedInstallation( - aiet_runner=aiet_runner_mock, - metadata=AIETMetadata( +) -> BackendInstallation: + """Get backend installation.""" + return BackendInstallation( + backend_runner=backend_runner_mock, + metadata=BackendMetadata( name=name, description=description, system_config=system_config or "", @@ -90,10 +90,10 @@ def test_could_be_installed_depends_on_platform( monkeypatch.setattr( "mlia.tools.metadata.corstone.all_paths_valid", MagicMock(return_value=True) ) - aiet_runner_mock = MagicMock(spec=AIETRunner) + backend_runner_mock = MagicMock(spec=BackendRunner) - installation = get_aiet_based_installation( - aiet_runner_mock, + installation = get_backend_installation( + backend_runner_mock, supported_platforms=supported_platforms, ) assert installation.could_be_installed == expected_result @@ -103,53 +103,53 @@ def test_get_corstone_installations() -> None: """Test function get_corstone_installation.""" installs = get_corstone_installations() assert len(installs) == 2 - assert all(isinstance(install, AIETBasedInstallation) for install in installs) + assert all(isinstance(install, BackendInstallation) for install in installs) -def test_aiet_based_installation_metadata_resolving() -> None: - """Test AIET based installation metadata resolving.""" - aiet_runner_mock = MagicMock(spec=AIETRunner) - installation = get_aiet_based_installation(aiet_runner_mock) +def test_backend_installation_metadata_resolving() -> None: + """Test backend installation metadata resolving.""" + backend_runner_mock = MagicMock(spec=BackendRunner) + installation = get_backend_installation(backend_runner_mock) assert installation.name == "test_name" assert installation.description == "test_description" - aiet_runner_mock.all_installed.return_value = False + backend_runner_mock.all_installed.return_value = False assert installation.already_installed is False assert installation.could_be_installed is True -def test_aiet_based_installation_supported_install_types(tmp_path: Path) -> None: +def test_backend_installation_supported_install_types(tmp_path: Path) -> None: """Test supported installation types.""" - installation_no_download_artifact = get_aiet_based_installation() + installation_no_download_artifact = get_backend_installation() assert installation_no_download_artifact.supports(DownloadAndInstall()) is False - installation_with_download_artifact = get_aiet_based_installation( + installation_with_download_artifact = get_backend_installation( download_artifact=MagicMock() ) assert installation_with_download_artifact.supports(DownloadAndInstall()) is True path_checker_mock = MagicMock(return_value=BackendInfo(tmp_path)) - installation_can_install_from_dir = get_aiet_based_installation( + installation_can_install_from_dir = get_backend_installation( path_checker=path_checker_mock ) assert installation_can_install_from_dir.supports(InstallFromPath(tmp_path)) is True - any_installation = get_aiet_based_installation() + any_installation = get_backend_installation() assert any_installation.supports("unknown_install_type") is False # type: ignore -def test_aiet_based_installation_install_wrong_type() -> None: +def test_backend_installation_install_wrong_type() -> None: """Test that operation should fail if wrong install type provided.""" with pytest.raises(Exception, match="Unable to install wrong_install_type"): - aiet_runner_mock = MagicMock(spec=AIETRunner) - installation = get_aiet_based_installation(aiet_runner_mock) + backend_runner_mock = MagicMock(spec=BackendRunner) + installation = get_backend_installation(backend_runner_mock) installation.install("wrong_install_type") # type: ignore -def test_aiet_based_installation_install_from_path( +def test_backend_installation_install_from_path( tmp_path: Path, test_mlia_resources: Path ) -> None: """Test installation from the path.""" @@ -164,9 +164,9 @@ def test_aiet_based_installation_install_from_path( path_checker_mock = MagicMock(return_value=BackendInfo(dist_dir)) - aiet_runner_mock = MagicMock(spec=AIETRunner) - installation = get_aiet_based_installation( - aiet_runner_mock=aiet_runner_mock, + backend_runner_mock = MagicMock(spec=BackendRunner) + installation = get_backend_installation( + backend_runner_mock=backend_runner_mock, path_checker=path_checker_mock, apps_resources=[sample_app.name], system_config="example_config.json", @@ -175,12 +175,12 @@ def test_aiet_based_installation_install_from_path( assert installation.supports(InstallFromPath(dist_dir)) is True installation.install(InstallFromPath(dist_dir)) - aiet_runner_mock.install_system.assert_called_once() - aiet_runner_mock.install_application.assert_called_once_with(sample_app) + backend_runner_mock.install_system.assert_called_once() + backend_runner_mock.install_application.assert_called_once_with(sample_app) @pytest.mark.parametrize("copy_source", [True, False]) -def test_aiet_based_installation_install_from_static_path( +def test_backend_installation_install_from_static_path( tmp_path: Path, test_mlia_resources: Path, copy_source: bool ) -> None: """Test installation from the predefined path.""" @@ -204,7 +204,7 @@ def test_aiet_based_installation_install_from_static_path( nested_file = predefined_location_dir / "nested_file.txt" nested_file.touch() - aiet_runner_mock = MagicMock(spec=AIETRunner) + backend_runner_mock = MagicMock(spec=BackendRunner) def check_install_dir(install_dir: Path) -> None: """Check content of the install dir.""" @@ -220,10 +220,10 @@ def test_aiet_based_installation_install_from_static_path( assert install_dir / "custom_config.json" in files - aiet_runner_mock.install_system.side_effect = check_install_dir + backend_runner_mock.install_system.side_effect = check_install_dir - installation = get_aiet_based_installation( - aiet_runner_mock=aiet_runner_mock, + installation = get_backend_installation( + backend_runner_mock=backend_runner_mock, path_checker=StaticPathChecker( predefined_location, ["file.txt"], @@ -237,8 +237,8 @@ def test_aiet_based_installation_install_from_static_path( assert installation.supports(InstallFromPath(predefined_location)) is True installation.install(InstallFromPath(predefined_location)) - aiet_runner_mock.install_system.assert_called_once() - aiet_runner_mock.install_application.assert_called_once_with(sample_app) + backend_runner_mock.install_system.assert_called_once() + backend_runner_mock.install_application.assert_called_once_with(sample_app) def create_sample_fvp_archive(tmp_path: Path) -> Path: @@ -259,7 +259,7 @@ def create_sample_fvp_archive(tmp_path: Path) -> Path: return fvp_archive -def test_aiet_based_installation_download_and_install( +def test_backend_installation_download_and_install( test_mlia_resources: Path, tmp_path: Path ) -> None: """Test downloading and installation process.""" @@ -277,9 +277,9 @@ def test_aiet_based_installation_download_and_install( """Sample installer.""" return dist_dir - aiet_runner_mock = MagicMock(spec=AIETRunner) - installation = get_aiet_based_installation( - aiet_runner_mock, + backend_runner_mock = MagicMock(spec=BackendRunner) + installation = get_backend_installation( + backend_runner_mock, download_artifact=download_artifact_mock, backend_installer=installer, path_checker=path_checker, @@ -288,7 +288,7 @@ def test_aiet_based_installation_download_and_install( installation.install(DownloadAndInstall()) - aiet_runner_mock.install_system.assert_called_once() + backend_runner_mock.install_system.assert_called_once() @pytest.mark.parametrize( |