diff options
Diffstat (limited to 'tests')
-rw-r--r-- | tests/conftest.py | 103 | ||||
-rw-r--r-- | tests/mlia/__init__.py | 3 | ||||
-rw-r--r-- | tests/mlia/conftest.py | 111 | ||||
-rw-r--r-- | tests/mlia/test_backend_controller.py | 160 | ||||
-rw-r--r-- | tests/mlia/test_backend_execution.py | 518 | ||||
-rw-r--r-- | tests/mlia/test_backend_output_parser.py | 152 | ||||
-rw-r--r-- | tests/mlia/test_backend_protocol.py | 231 | ||||
-rw-r--r-- | tests/mlia/test_utils_proc.py | 149 | ||||
-rw-r--r-- | tests/test_api.py (renamed from tests/mlia/test_api.py) | 0 | ||||
-rw-r--r-- | tests/test_backend_application.py (renamed from tests/mlia/test_backend_application.py) | 48 | ||||
-rw-r--r-- | tests/test_backend_common.py (renamed from tests/mlia/test_backend_common.py) | 10 | ||||
-rw-r--r-- | tests/test_backend_execution.py | 203 | ||||
-rw-r--r-- | tests/test_backend_fs.py (renamed from tests/mlia/test_backend_fs.py) | 34 | ||||
-rw-r--r-- | tests/test_backend_manager.py (renamed from tests/mlia/test_backend_manager.py) | 190 | ||||
-rw-r--r-- | tests/test_backend_output_consumer.py | 99 | ||||
-rw-r--r-- | tests/test_backend_proc.py (renamed from tests/mlia/test_backend_proc.py) | 69 | ||||
-rw-r--r-- | tests/test_backend_source.py (renamed from tests/mlia/test_backend_source.py) | 4 | ||||
-rw-r--r-- | tests/test_backend_system.py (renamed from tests/mlia/test_backend_system.py) | 204 | ||||
-rw-r--r-- | tests/test_cli_commands.py (renamed from tests/mlia/test_cli_commands.py) | 0 | ||||
-rw-r--r-- | tests/test_cli_config.py (renamed from tests/mlia/test_cli_config.py) | 0 | ||||
-rw-r--r-- | tests/test_cli_helpers.py (renamed from tests/mlia/test_cli_helpers.py) | 0 | ||||
-rw-r--r-- | tests/test_cli_logging.py (renamed from tests/mlia/test_cli_logging.py) | 2 | ||||
-rw-r--r-- | tests/test_cli_main.py (renamed from tests/mlia/test_cli_main.py) | 2 | ||||
-rw-r--r-- | tests/test_cli_options.py (renamed from tests/mlia/test_cli_options.py) | 0 | ||||
-rw-r--r-- | tests/test_core_advice_generation.py (renamed from tests/mlia/test_core_advice_generation.py) | 0 | ||||
-rw-r--r-- | tests/test_core_advisor.py (renamed from tests/mlia/test_core_advisor.py) | 0 | ||||
-rw-r--r-- | tests/test_core_context.py (renamed from tests/mlia/test_core_context.py) | 0 | ||||
-rw-r--r-- | tests/test_core_data_analysis.py (renamed from tests/mlia/test_core_data_analysis.py) | 0 | ||||
-rw-r--r-- | tests/test_core_events.py (renamed from tests/mlia/test_core_events.py) | 0 | ||||
-rw-r--r-- | tests/test_core_helpers.py (renamed from tests/mlia/test_core_helpers.py) | 0 | ||||
-rw-r--r-- | tests/test_core_mixins.py (renamed from tests/mlia/test_core_mixins.py) | 0 | ||||
-rw-r--r-- | tests/test_core_performance.py (renamed from tests/mlia/test_core_performance.py) | 0 | ||||
-rw-r--r-- | tests/test_core_reporting.py (renamed from tests/mlia/test_core_reporting.py) | 0 | ||||
-rw-r--r-- | tests/test_core_workflow.py (renamed from tests/mlia/test_core_workflow.py) | 0 | ||||
-rw-r--r-- | tests/test_devices_ethosu_advice_generation.py (renamed from tests/mlia/test_devices_ethosu_advice_generation.py) | 0 | ||||
-rw-r--r-- | tests/test_devices_ethosu_advisor.py (renamed from tests/mlia/test_devices_ethosu_advisor.py) | 0 | ||||
-rw-r--r-- | tests/test_devices_ethosu_config.py (renamed from tests/mlia/test_devices_ethosu_config.py) | 0 | ||||
-rw-r--r-- | tests/test_devices_ethosu_data_analysis.py (renamed from tests/mlia/test_devices_ethosu_data_analysis.py) | 0 | ||||
-rw-r--r-- | tests/test_devices_ethosu_data_collection.py (renamed from tests/mlia/test_devices_ethosu_data_collection.py) | 0 | ||||
-rw-r--r-- | tests/test_devices_ethosu_performance.py (renamed from tests/mlia/test_devices_ethosu_performance.py) | 0 | ||||
-rw-r--r-- | tests/test_devices_ethosu_reporters.py (renamed from tests/mlia/test_devices_ethosu_reporters.py) | 0 | ||||
-rw-r--r-- | tests/test_nn_tensorflow_config.py (renamed from tests/mlia/test_nn_tensorflow_config.py) | 0 | ||||
-rw-r--r-- | tests/test_nn_tensorflow_optimizations_clustering.py (renamed from tests/mlia/test_nn_tensorflow_optimizations_clustering.py) | 4 | ||||
-rw-r--r-- | tests/test_nn_tensorflow_optimizations_pruning.py (renamed from tests/mlia/test_nn_tensorflow_optimizations_pruning.py) | 4 | ||||
-rw-r--r-- | tests/test_nn_tensorflow_optimizations_select.py (renamed from tests/mlia/test_nn_tensorflow_optimizations_select.py) | 0 | ||||
-rw-r--r-- | tests/test_nn_tensorflow_tflite_metrics.py (renamed from tests/mlia/test_nn_tensorflow_tflite_metrics.py) | 16 | ||||
-rw-r--r-- | tests/test_nn_tensorflow_utils.py (renamed from tests/mlia/test_nn_tensorflow_utils.py) | 0 | ||||
-rw-r--r-- | tests/test_resources/application_config.json (renamed from tests/mlia/test_resources/application_config.json) | 2 | ||||
-rw-r--r-- | tests/test_resources/application_config.json.license (renamed from tests/mlia/test_resources/application_config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/backends/applications/application1/backend-config.json (renamed from tests/mlia/test_resources/backends/applications/application1/aiet-config.json) | 1 | ||||
-rw-r--r-- | tests/test_resources/backends/applications/application1/backend-config.json.license (renamed from tests/mlia/test_resources/backends/applications/application1/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/backends/applications/application2/backend-config.json (renamed from tests/mlia/test_resources/backends/applications/application2/aiet-config.json) | 1 | ||||
-rw-r--r-- | tests/test_resources/backends/applications/application2/backend-config.json.license (renamed from tests/mlia/test_resources/backends/applications/application2/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/backends/applications/application3/readme.txt (renamed from tests/mlia/test_resources/backends/applications/application3/readme.txt) | 0 | ||||
-rw-r--r-- | tests/test_resources/backends/applications/application4/backend-config.json (renamed from tests/mlia/test_resources/backends/applications/application4/aiet-config.json) | 17 | ||||
-rw-r--r-- | tests/test_resources/backends/applications/application4/backend-config.json.license (renamed from tests/mlia/test_resources/backends/applications/application4/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/backends/applications/application4/hello_app.txt (renamed from tests/mlia/test_resources/backends/applications/application4/hello_app.txt) | 0 | ||||
-rw-r--r-- | tests/test_resources/backends/applications/application5/backend-config.json (renamed from tests/mlia/test_resources/backends/applications/application5/aiet-config.json) | 28 | ||||
-rw-r--r-- | tests/test_resources/backends/applications/application5/backend-config.json.license (renamed from tests/mlia/test_resources/backends/applications/application5/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/backends/applications/application6/backend-config.json (renamed from tests/mlia/test_resources/backends/applications/application6/aiet-config.json) | 1 | ||||
-rw-r--r-- | tests/test_resources/backends/applications/application6/backend-config.json.license (renamed from tests/mlia/test_resources/backends/applications/application6/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/backends/applications/readme.txt (renamed from tests/mlia/test_resources/backends/applications/readme.txt) | 0 | ||||
-rw-r--r-- | tests/test_resources/backends/systems/system1/backend-config.json (renamed from tests/mlia/test_resources/backends/systems/system1/aiet-config.json) | 11 | ||||
-rw-r--r-- | tests/test_resources/backends/systems/system1/backend-config.json.license (renamed from tests/mlia/test_resources/backends/systems/system1/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/backends/systems/system1/system_artifact/dummy.txt (renamed from tests/mlia/test_resources/backends/systems/system1/system_artifact/dummy.txt) | 0 | ||||
-rw-r--r-- | tests/test_resources/backends/systems/system2/backend-config.json (renamed from tests/mlia/test_resources/backends/systems/system2/aiet-config.json) | 8 | ||||
-rw-r--r-- | tests/test_resources/backends/systems/system2/backend-config.json.license (renamed from tests/mlia/test_resources/backends/systems/system2/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/backends/systems/system3/readme.txt (renamed from tests/mlia/test_resources/backends/systems/system3/readme.txt) | 0 | ||||
-rw-r--r-- | tests/test_resources/backends/systems/system4/backend-config.json (renamed from tests/mlia/test_resources/backends/systems/system4/aiet-config.json) | 8 | ||||
-rw-r--r-- | tests/test_resources/backends/systems/system4/backend-config.json.license (renamed from tests/mlia/test_resources/backends/systems/system4/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/backends/systems/system6/backend-config.json (renamed from tests/mlia/test_resources/backends/systems/system6/aiet-config.json) | 4 | ||||
-rw-r--r-- | tests/test_resources/backends/systems/system6/backend-config.json.license (renamed from tests/mlia/test_resources/backends/systems/system6/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/hello_world.json (renamed from tests/mlia/test_resources/hello_world.json) | 1 | ||||
-rw-r--r-- | tests/test_resources/hello_world.json.license (renamed from tests/mlia/test_resources/hello_world.json.license) | 0 | ||||
-rwxr-xr-x | tests/test_resources/scripts/test_backend_run (renamed from tests/mlia/test_resources/scripts/test_backend_run) | 0 | ||||
-rw-r--r-- | tests/test_resources/scripts/test_backend_run_script.sh (renamed from tests/mlia/test_resources/scripts/test_backend_run_script.sh) | 0 | ||||
-rw-r--r-- | tests/test_resources/various/applications/application_with_empty_config/backend-config.json (renamed from tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/test_resources/various/applications/application_with_empty_config/backend-config.json.license (renamed from tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/various/applications/application_with_valid_config/backend-config.json (renamed from tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json) | 6 | ||||
-rw-r--r-- | tests/test_resources/various/applications/application_with_valid_config/backend-config.json.license (renamed from tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/various/applications/application_with_wrong_config1/backend-config.json (renamed from tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/test_resources/various/applications/application_with_wrong_config1/backend-config.json.license (renamed from tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/various/applications/application_with_wrong_config2/backend-config.json (renamed from tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json) | 6 | ||||
-rw-r--r-- | tests/test_resources/various/applications/application_with_wrong_config2/backend-config.json.license (renamed from tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/various/applications/application_with_wrong_config3/backend-config.json (renamed from tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json) | 6 | ||||
-rw-r--r-- | tests/test_resources/various/applications/application_with_wrong_config3/backend-config.json.license (renamed from tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/various/systems/system_with_empty_config/backend-config.json (renamed from tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/test_resources/various/systems/system_with_empty_config/backend-config.json.license (renamed from tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/various/systems/system_with_valid_config/backend-config.json (renamed from tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json) | 4 | ||||
-rw-r--r-- | tests/test_resources/various/systems/system_with_valid_config/backend-config.json.license (renamed from tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/test_resources/vela/sample_vela.ini (renamed from tests/mlia/test_resources/vela/sample_vela.ini) | 0 | ||||
-rw-r--r-- | tests/test_tools_metadata_common.py (renamed from tests/mlia/test_tools_metadata_common.py) | 0 | ||||
-rw-r--r-- | tests/test_tools_metadata_corstone.py (renamed from tests/mlia/test_tools_metadata_corstone.py) | 0 | ||||
-rw-r--r-- | tests/test_tools_vela_wrapper.py (renamed from tests/mlia/test_tools_vela_wrapper.py) | 2 | ||||
-rw-r--r-- | tests/test_utils_console.py (renamed from tests/mlia/test_utils_console.py) | 0 | ||||
-rw-r--r-- | tests/test_utils_download.py (renamed from tests/mlia/test_utils_download.py) | 0 | ||||
-rw-r--r-- | tests/test_utils_filesystem.py (renamed from tests/mlia/test_utils_filesystem.py) | 25 | ||||
-rw-r--r-- | tests/test_utils_logging.py (renamed from tests/mlia/test_utils_logging.py) | 0 | ||||
-rw-r--r-- | tests/test_utils_misc.py (renamed from tests/mlia/test_utils_misc.py) | 0 | ||||
-rw-r--r-- | tests/test_utils_types.py (renamed from tests/mlia/test_utils_types.py) | 0 | ||||
-rw-r--r-- | tests/utils/__init__.py (renamed from tests/mlia/utils/__init__.py) | 0 | ||||
-rw-r--r-- | tests/utils/common.py (renamed from tests/mlia/utils/common.py) | 0 | ||||
-rw-r--r-- | tests/utils/logging.py (renamed from tests/mlia/utils/logging.py) | 0 |
103 files changed, 578 insertions, 1869 deletions
diff --git a/tests/conftest.py b/tests/conftest.py index 5c6156c..4d12033 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,12 +2,15 @@ # SPDX-License-Identifier: Apache-2.0 """Pytest conf module.""" import shutil +import tarfile from pathlib import Path +from typing import Any from typing import Generator import pytest import tensorflow as tf +from mlia.core.context import ExecutionContext from mlia.devices.ethosu.config import EthosUConfiguration from mlia.nn.tensorflow.utils import convert_to_tflite from mlia.nn.tensorflow.utils import save_keras_model @@ -15,6 +18,106 @@ from mlia.nn.tensorflow.utils import save_tflite_model from mlia.tools.vela_wrapper import optimize_model +@pytest.fixture(scope="session", name="test_resources_path") +def fixture_test_resources_path() -> Path: + """Return test resources path.""" + return Path(__file__).parent / "test_resources" + + +@pytest.fixture(name="dummy_context") +def fixture_dummy_context(tmpdir: str) -> ExecutionContext: + """Return dummy context fixture.""" + return ExecutionContext(working_dir=tmpdir) + + +@pytest.fixture(scope="session") +def test_systems_path(test_resources_path: Path) -> Path: + """Return test systems path in a pytest fixture.""" + return test_resources_path / "backends" / "systems" + + +@pytest.fixture(scope="session") +def test_applications_path(test_resources_path: Path) -> Path: + """Return test applications path in a pytest fixture.""" + return test_resources_path / "backends" / "applications" + + +@pytest.fixture(scope="session") +def non_optimised_input_model_file(test_tflite_model: Path) -> Path: + """Provide the path to a quantized dummy model file.""" + return test_tflite_model + + +@pytest.fixture(scope="session") +def optimised_input_model_file(test_tflite_vela_model: Path) -> Path: + """Provide path to Vela-optimised dummy model file.""" + return test_tflite_vela_model + + +@pytest.fixture(scope="session") +def invalid_input_model_file(test_tflite_invalid_model: Path) -> Path: + """Provide the path to an invalid dummy model file.""" + return test_tflite_invalid_model + + +@pytest.fixture(autouse=True) +def test_resources(monkeypatch: pytest.MonkeyPatch, test_resources_path: Path) -> Any: + """Force using test resources as middleware's repository.""" + + def get_test_resources() -> Path: + """Return path to the test resources.""" + return test_resources_path / "backends" + + monkeypatch.setattr("mlia.backend.fs.get_backend_resources", get_test_resources) + yield + + +def create_archive( + archive_name: str, source: Path, destination: Path, with_root_folder: bool = False +) -> None: + """Create archive from directory source.""" + with tarfile.open(destination / archive_name, mode="w:gz") as tar: + for item in source.iterdir(): + item_name = item.name + if with_root_folder: + item_name = f"{source.name}/{item_name}" + tar.add(item, item_name) + + +def process_directory(source: Path, destination: Path) -> None: + """Process resource directory.""" + destination.mkdir() + + for item in source.iterdir(): + if item.is_dir(): + create_archive(f"{item.name}.tar.gz", item, destination) + create_archive(f"{item.name}_dir.tar.gz", item, destination, True) + + +@pytest.fixture(scope="session", autouse=True) +def add_archives( + test_resources_path: Path, tmp_path_factory: pytest.TempPathFactory +) -> Any: + """Generate archives of the test resources.""" + tmp_path = tmp_path_factory.mktemp("archives") + + archives_path = tmp_path / "archives" + archives_path.mkdir() + + if (archives_path_link := test_resources_path / "archives").is_symlink(): + archives_path_link.unlink() + + archives_path_link.symlink_to(archives_path, target_is_directory=True) + + for item in ["applications", "systems"]: + process_directory(test_resources_path / "backends" / item, archives_path / item) + + yield + + archives_path_link.unlink() + shutil.rmtree(tmp_path) + + def get_test_keras_model() -> tf.keras.Model: """Return test Keras model.""" model = tf.keras.Sequential( diff --git a/tests/mlia/__init__.py b/tests/mlia/__init__.py deleted file mode 100644 index 0687f14..0000000 --- a/tests/mlia/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""MLIA tests module.""" diff --git a/tests/mlia/conftest.py b/tests/mlia/conftest.py deleted file mode 100644 index 0b4b2aa..0000000 --- a/tests/mlia/conftest.py +++ /dev/null @@ -1,111 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Pytest conf module.""" -import shutil -import tarfile -from pathlib import Path -from typing import Any - -import pytest - -from mlia.core.context import ExecutionContext - - -@pytest.fixture(scope="session", name="test_resources_path") -def fixture_test_resources_path() -> Path: - """Return test resources path.""" - return Path(__file__).parent / "test_resources" - - -@pytest.fixture(name="dummy_context") -def fixture_dummy_context(tmpdir: str) -> ExecutionContext: - """Return dummy context fixture.""" - return ExecutionContext(working_dir=tmpdir) - - -@pytest.fixture(scope="session") -def test_systems_path(test_resources_path: Path) -> Path: - """Return test systems path in a pytest fixture.""" - return test_resources_path / "backends" / "systems" - - -@pytest.fixture(scope="session") -def test_applications_path(test_resources_path: Path) -> Path: - """Return test applications path in a pytest fixture.""" - return test_resources_path / "backends" / "applications" - - -@pytest.fixture(scope="session") -def non_optimised_input_model_file(test_tflite_model: Path) -> Path: - """Provide the path to a quantized dummy model file.""" - return test_tflite_model - - -@pytest.fixture(scope="session") -def optimised_input_model_file(test_tflite_vela_model: Path) -> Path: - """Provide path to Vela-optimised dummy model file.""" - return test_tflite_vela_model - - -@pytest.fixture(scope="session") -def invalid_input_model_file(test_tflite_invalid_model: Path) -> Path: - """Provide the path to an invalid dummy model file.""" - return test_tflite_invalid_model - - -@pytest.fixture(autouse=True) -def test_resources(monkeypatch: pytest.MonkeyPatch, test_resources_path: Path) -> Any: - """Force using test resources as middleware's repository.""" - - def get_test_resources() -> Path: - """Return path to the test resources.""" - return test_resources_path / "backends" - - monkeypatch.setattr("mlia.backend.fs.get_backend_resources", get_test_resources) - yield - - -def create_archive( - archive_name: str, source: Path, destination: Path, with_root_folder: bool = False -) -> None: - """Create archive from directory source.""" - with tarfile.open(destination / archive_name, mode="w:gz") as tar: - for item in source.iterdir(): - item_name = item.name - if with_root_folder: - item_name = f"{source.name}/{item_name}" - tar.add(item, item_name) - - -def process_directory(source: Path, destination: Path) -> None: - """Process resource directory.""" - destination.mkdir() - - for item in source.iterdir(): - if item.is_dir(): - create_archive(f"{item.name}.tar.gz", item, destination) - create_archive(f"{item.name}_dir.tar.gz", item, destination, True) - - -@pytest.fixture(scope="session", autouse=True) -def add_archives( - test_resources_path: Path, tmp_path_factory: pytest.TempPathFactory -) -> Any: - """Generate archives of the test resources.""" - tmp_path = tmp_path_factory.mktemp("archives") - - archives_path = tmp_path / "archives" - archives_path.mkdir() - - if (archives_path_link := test_resources_path / "archives").is_symlink(): - archives_path_link.unlink() - - archives_path_link.symlink_to(archives_path, target_is_directory=True) - - for item in ["applications", "systems"]: - process_directory(test_resources_path / "backends" / item, archives_path / item) - - yield - - archives_path_link.unlink() - shutil.rmtree(tmp_path) diff --git a/tests/mlia/test_backend_controller.py b/tests/mlia/test_backend_controller.py deleted file mode 100644 index a047adf..0000000 --- a/tests/mlia/test_backend_controller.py +++ /dev/null @@ -1,160 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Tests for system controller.""" -import csv -import os -import time -from pathlib import Path -from typing import Any - -import psutil -import pytest - -from mlia.backend.common import ConfigurationException -from mlia.backend.controller import SystemController -from mlia.backend.controller import SystemControllerSingleInstance -from mlia.backend.proc import ShellCommand - - -def get_system_controller(**kwargs: Any) -> SystemController: - """Get service controller.""" - single_instance = kwargs.get("single_instance", False) - if single_instance: - pid_file_path = kwargs.get("pid_file_path") - return SystemControllerSingleInstance(pid_file_path) - - return SystemController() - - -def test_service_controller() -> None: - """Test service controller functionality.""" - service_controller = get_system_controller() - - assert service_controller.get_output() == ("", "") - with pytest.raises(ConfigurationException, match="Wrong working directory"): - service_controller.start(["sleep 100"], Path("unknown")) - - service_controller.start(["sleep 100"], Path.cwd()) - assert service_controller.is_running() - - service_controller.stop(True) - assert not service_controller.is_running() - assert service_controller.get_output() == ("", "") - - service_controller.stop() - - with pytest.raises( - ConfigurationException, match="System should have only one command to run" - ): - service_controller.start(["sleep 100", "sleep 101"], Path.cwd()) - - with pytest.raises(ConfigurationException, match="No startup command provided"): - service_controller.start([""], Path.cwd()) - - -def test_service_controller_bad_configuration() -> None: - """Test service controller functionality for bad configuration.""" - with pytest.raises(Exception, match="No pid file path presented"): - service_controller = get_system_controller( - single_instance=True, pid_file_path=None - ) - service_controller.start(["sleep 100"], Path.cwd()) - - -def test_service_controller_writes_process_info_correctly(tmpdir: Any) -> None: - """Test that controller writes process info correctly.""" - pid_file = Path(tmpdir) / "test.pid" - - service_controller = get_system_controller( - single_instance=True, pid_file_path=Path(tmpdir) / "test.pid" - ) - - service_controller.start(["sleep 100"], Path.cwd()) - assert service_controller.is_running() - assert pid_file.is_file() - - with open(pid_file, "r", encoding="utf-8") as file: - csv_reader = csv.reader(file) - rows = list(csv_reader) - assert len(rows) == 1 - - name, *_ = rows[0] - assert name == "sleep" - - service_controller.stop() - assert pid_file.exists() - - -def test_service_controller_does_not_write_process_info_if_process_finishes( - tmpdir: Any, -) -> None: - """Test that controller does not write process info if process already finished.""" - pid_file = Path(tmpdir) / "test.pid" - service_controller = get_system_controller( - single_instance=True, pid_file_path=pid_file - ) - service_controller.is_running = lambda: False # type: ignore - service_controller.start(["echo hello"], Path.cwd()) - - assert not pid_file.exists() - - -def test_service_controller_searches_for_previous_instances_correctly( - tmpdir: Any, -) -> None: - """Test that controller searches for previous instances correctly.""" - pid_file = Path(tmpdir) / "test.pid" - command = ShellCommand().run("sleep", "100") - assert command.is_alive() - - pid = command.process.pid - process = psutil.Process(pid) - with open(pid_file, "w", encoding="utf-8") as file: - csv_writer = csv.writer(file) - csv_writer.writerow(("some_process", "some_program", "some_cwd", os.getpid())) - csv_writer.writerow((process.name(), process.exe(), process.cwd(), process.pid)) - csv_writer.writerow(("some_old_process", "not_running", "from_nowhere", 77777)) - - service_controller = get_system_controller( - single_instance=True, pid_file_path=pid_file - ) - service_controller.start(["sleep 100"], Path.cwd()) - # controller should stop this process as it is currently running and - # mentioned in pid file - assert not command.is_alive() - - service_controller.stop() - - -@pytest.mark.parametrize( - "executable", ["test_backend_run_script.sh", "test_backend_run"] -) -def test_service_controller_run_shell_script( - executable: str, test_resources_path: Path -) -> None: - """Test controller's ability to run shell scripts.""" - script_path = test_resources_path / "scripts" - - service_controller = get_system_controller() - - service_controller.start([executable], script_path) - - assert service_controller.is_running() - # give time for the command to produce output - time.sleep(2) - service_controller.stop(wait=True) - assert not service_controller.is_running() - stdout, stderr = service_controller.get_output() - assert stdout == "Hello from script\n" - assert stderr == "Oops!\n" - - -def test_service_controller_does_nothing_if_not_started(tmpdir: Any) -> None: - """Test that nothing happened if controller is not started.""" - service_controller = get_system_controller( - single_instance=True, pid_file_path=Path(tmpdir) / "test.pid" - ) - - assert not service_controller.is_running() - service_controller.stop() - assert not service_controller.is_running() diff --git a/tests/mlia/test_backend_execution.py b/tests/mlia/test_backend_execution.py deleted file mode 100644 index 9395352..0000000 --- a/tests/mlia/test_backend_execution.py +++ /dev/null @@ -1,518 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=no-self-use -"""Test backend execution module.""" -from contextlib import ExitStack as does_not_raise -from pathlib import Path -from typing import Any -from typing import Dict -from unittest import mock -from unittest.mock import MagicMock - -import pytest -from sh import CommandNotFound - -from mlia.backend.application import Application -from mlia.backend.application import get_application -from mlia.backend.common import DataPaths -from mlia.backend.common import UserParamConfig -from mlia.backend.config import ApplicationConfig -from mlia.backend.config import LocalProtocolConfig -from mlia.backend.config import SystemConfig -from mlia.backend.execution import deploy_data -from mlia.backend.execution import execute_commands_locally -from mlia.backend.execution import ExecutionContext -from mlia.backend.execution import get_application_and_system -from mlia.backend.execution import get_application_by_name_and_system -from mlia.backend.execution import get_file_lock_path -from mlia.backend.execution import ParamResolver -from mlia.backend.execution import Reporter -from mlia.backend.execution import wait -from mlia.backend.output_parser import Base64OutputParser -from mlia.backend.output_parser import OutputParser -from mlia.backend.output_parser import RegexOutputParser -from mlia.backend.proc import CommandFailedException -from mlia.backend.system import get_system -from mlia.backend.system import load_system - - -def test_context_param_resolver(tmpdir: Any) -> None: - """Test parameter resolving.""" - system_config_location = Path(tmpdir) / "system" - system_config_location.mkdir() - - application_config_location = Path(tmpdir) / "application" - application_config_location.mkdir() - - ctx = ExecutionContext( - app=Application( - ApplicationConfig( - name="test_application", - description="Test application", - config_location=application_config_location, - build_dir="build-{application.name}-{system.name}", - commands={ - "run": [ - "run_command1 {user_params:0}", - "run_command2 {user_params:1}", - ] - }, - variables={"var_1": "value for var_1"}, - user_params={ - "run": [ - UserParamConfig( - name="--param1", - description="Param 1", - default_value="123", - alias="param_1", - ), - UserParamConfig( - name="--param2", description="Param 2", default_value="456" - ), - UserParamConfig( - name="--param3", description="Param 3", alias="param_3" - ), - UserParamConfig( - name="--param4=", - description="Param 4", - default_value="456", - alias="param_4", - ), - UserParamConfig( - description="Param 5", - default_value="789", - alias="param_5", - ), - ] - }, - ) - ), - app_params=["--param2=789"], - system=load_system( - SystemConfig( - name="test_system", - description="Test system", - config_location=system_config_location, - build_dir="build", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={ - "build": ["build_command1 {user_params:0}"], - "run": ["run_command {application.commands.run:1}"], - }, - variables={"var_1": "value for var_1"}, - user_params={ - "build": [ - UserParamConfig( - name="--param1", description="Param 1", default_value="aaa" - ), - UserParamConfig(name="--param2", description="Param 2"), - ] - }, - ) - ), - system_params=["--param1=bbb"], - custom_deploy_data=[], - ) - - param_resolver = ParamResolver(ctx) - expected_values = { - "application.name": "test_application", - "application.description": "Test application", - "application.config_dir": str(application_config_location), - "application.build_dir": "{}/build-test_application-test_system".format( - application_config_location - ), - "application.commands.run:0": "run_command1 --param1 123", - "application.commands.run.params:0": "123", - "application.commands.run.params:param_1": "123", - "application.commands.run:1": "run_command2 --param2 789", - "application.commands.run.params:1": "789", - "application.variables:var_1": "value for var_1", - "system.name": "test_system", - "system.description": "Test system", - "system.config_dir": str(system_config_location), - "system.commands.build:0": "build_command1 --param1 bbb", - "system.commands.run:0": "run_command run_command2 --param2 789", - "system.commands.build.params:0": "bbb", - "system.variables:var_1": "value for var_1", - } - - for param, value in expected_values.items(): - assert param_resolver(param) == value - - assert ctx.build_dir() == Path( - "{}/build-test_application-test_system".format(application_config_location) - ) - - expected_errors = { - "application.variables:var_2": pytest.raises( - Exception, match="Unknown variable var_2" - ), - "application.commands.clean:0": pytest.raises( - Exception, match="Command clean not found" - ), - "application.commands.run:2": pytest.raises( - Exception, match="Invalid index 2 for command run" - ), - "application.commands.run.params:5": pytest.raises( - Exception, match="Invalid parameter index 5 for command run" - ), - "application.commands.run.params:param_2": pytest.raises( - Exception, - match="No value for parameter with index or alias param_2 of command run", - ), - "UNKNOWN": pytest.raises( - Exception, match="Unable to resolve parameter UNKNOWN" - ), - "system.commands.build.params:1": pytest.raises( - Exception, - match="No value for parameter with index or alias 1 of command build", - ), - "system.commands.build:A": pytest.raises( - Exception, match="Bad command index A" - ), - "system.variables:var_2": pytest.raises( - Exception, match="Unknown variable var_2" - ), - } - for param, error in expected_errors.items(): - with error: - param_resolver(param) - - resolved_params = ctx.app.resolved_parameters("run", []) - expected_user_params = { - "user_params:0": "--param1 123", - "user_params:param_1": "--param1 123", - "user_params:2": "--param3", - "user_params:param_3": "--param3", - "user_params:3": "--param4=456", - "user_params:param_4": "--param4=456", - "user_params:param_5": "789", - } - for param, expected_value in expected_user_params.items(): - assert param_resolver(param, "run", resolved_params) == expected_value - - with pytest.raises( - Exception, match="Invalid index 5 for user params of command run" - ): - param_resolver("user_params:5", "run", resolved_params) - - with pytest.raises( - Exception, match="No user parameter for command 'run' with alias 'param_2'." - ): - param_resolver("user_params:param_2", "run", resolved_params) - - with pytest.raises(Exception, match="Unable to resolve user params"): - param_resolver("user_params:0", "", resolved_params) - - bad_ctx = ExecutionContext( - app=Application( - ApplicationConfig( - name="test_application", - config_location=application_config_location, - build_dir="build-{user_params:0}", - ) - ), - app_params=["--param2=789"], - system=load_system( - SystemConfig( - name="test_system", - description="Test system", - config_location=system_config_location, - build_dir="build-{system.commands.run.params:123}", - data_transfer=LocalProtocolConfig(protocol="local"), - ) - ), - system_params=["--param1=bbb"], - custom_deploy_data=[], - ) - param_resolver = ParamResolver(bad_ctx) - with pytest.raises(Exception, match="Unable to resolve user params"): - bad_ctx.build_dir() - - -# pylint: disable=too-many-arguments -@pytest.mark.parametrize( - "application_name, soft_lock, sys_lock, lock_dir, expected_error, expected_path", - ( - ( - "test_application", - True, - True, - Path("/tmp"), - does_not_raise(), - Path("/tmp/middleware_test_application_test_system.lock"), - ), - ( - "$$test_application$!:", - True, - True, - Path("/tmp"), - does_not_raise(), - Path("/tmp/middleware_test_application_test_system.lock"), - ), - ( - "test_application", - True, - True, - Path("unknown"), - pytest.raises( - Exception, match="Invalid directory unknown for lock files provided" - ), - None, - ), - ( - "test_application", - False, - True, - Path("/tmp"), - does_not_raise(), - Path("/tmp/middleware_test_system.lock"), - ), - ( - "test_application", - True, - False, - Path("/tmp"), - does_not_raise(), - Path("/tmp/middleware_test_application.lock"), - ), - ( - "test_application", - False, - False, - Path("/tmp"), - pytest.raises(Exception, match="No filename for lock provided"), - None, - ), - ), -) -def test_get_file_lock_path( - application_name: str, - soft_lock: bool, - sys_lock: bool, - lock_dir: Path, - expected_error: Any, - expected_path: Path, -) -> None: - """Test get_file_lock_path function.""" - with expected_error: - ctx = ExecutionContext( - app=Application(ApplicationConfig(name=application_name, lock=soft_lock)), - app_params=[], - system=load_system( - SystemConfig( - name="test_system", - lock=sys_lock, - data_transfer=LocalProtocolConfig(protocol="local"), - ) - ), - system_params=[], - custom_deploy_data=[], - ) - path = get_file_lock_path(ctx, lock_dir) - assert path == expected_path - - -def test_get_application_by_name_and_system(monkeypatch: Any) -> None: - """Test exceptional case for get_application_by_name_and_system.""" - monkeypatch.setattr( - "mlia.backend.execution.get_application", - MagicMock(return_value=[MagicMock(), MagicMock()]), - ) - - with pytest.raises( - ValueError, - match="Error during getting application test_application for the " - "system test_system", - ): - get_application_by_name_and_system("test_application", "test_system") - - -def test_get_application_and_system(monkeypatch: Any) -> None: - """Test exceptional case for get_application_and_system.""" - monkeypatch.setattr( - "mlia.backend.execution.get_system", MagicMock(return_value=None) - ) - - with pytest.raises(ValueError, match="System test_system is not found"): - get_application_and_system("test_application", "test_system") - - -def test_wait_function(monkeypatch: Any) -> None: - """Test wait function.""" - sleep_mock = MagicMock() - monkeypatch.setattr("time.sleep", sleep_mock) - wait(0.1) - sleep_mock.assert_called_once() - - -def test_deployment_execution_context() -> None: - """Test property 'is_deploy_needed' of the ExecutionContext.""" - ctx = ExecutionContext( - app=get_application("application_1")[0], - app_params=[], - system=get_system("System 1"), - system_params=[], - ) - assert not ctx.is_deploy_needed - deploy_data(ctx) # should be a NOP - - ctx = ExecutionContext( - app=get_application("application_1")[0], - app_params=[], - system=get_system("System 1"), - system_params=[], - custom_deploy_data=[DataPaths(Path("README.md"), ".")], - ) - assert ctx.is_deploy_needed - - ctx = ExecutionContext( - app=get_application("application_1")[0], - app_params=[], - system=None, - system_params=[], - ) - assert not ctx.is_deploy_needed - with pytest.raises(AssertionError): - deploy_data(ctx) - - -def test_reporter_execution_context(tmp_path: Path) -> None: - """Test ExecutionContext creates a reporter when a report file is provided.""" - # Configure regex parser for the system manually - sys = get_system("System 1") - assert sys is not None - sys.reporting = { - "regex": { - "simulated_time": {"pattern": "Simulated time.*: (.*)s", "type": "float"} - } - } - report_file_path = tmp_path / "test_report.json" - - ctx = ExecutionContext( - app=get_application("application_1")[0], - app_params=[], - system=sys, - system_params=[], - report_file=report_file_path, - ) - assert isinstance(ctx.reporter, Reporter) - assert len(ctx.reporter.parsers) == 2 - assert any(isinstance(parser, RegexOutputParser) for parser in ctx.reporter.parsers) - assert any( - isinstance(parser, Base64OutputParser) for parser in ctx.reporter.parsers - ) - - -class TestExecuteCommandsLocally: - """Test execute_commands_locally() function.""" - - @pytest.mark.parametrize( - "first_command, exception, expected_output", - ( - ( - "echo 'hello'", - None, - "Running: echo 'hello'\nhello\nRunning: echo 'goodbye'\ngoodbye\n", - ), - ( - "non-existent-command", - CommandNotFound, - "Running: non-existent-command\n", - ), - ("false", CommandFailedException, "Running: false\n"), - ), - ids=( - "runs_multiple_commands", - "stops_executing_on_non_existent_command", - "stops_executing_when_command_exits_with_error_code", - ), - ) - def test_execution( - self, - first_command: str, - exception: Any, - expected_output: str, - test_resources_path: Path, - capsys: Any, - ) -> None: - """Test expected behaviour of the function.""" - commands = [first_command, "echo 'goodbye'"] - cwd = test_resources_path - if exception is None: - execute_commands_locally(commands, cwd) - else: - with pytest.raises(exception): - execute_commands_locally(commands, cwd) - - captured = capsys.readouterr() - assert captured.out == expected_output - - def test_stops_executing_on_exception( - self, monkeypatch: Any, test_resources_path: Path - ) -> None: - """Ensure commands following an error-exit-code command don't run.""" - # Mock execute_command() function - execute_command_mock = mock.MagicMock() - monkeypatch.setattr("mlia.backend.proc.execute_command", execute_command_mock) - - # Mock Command object and assign as return value to execute_command() - cmd_mock = mock.MagicMock() - execute_command_mock.return_value = cmd_mock - - # Mock the terminate_command (speed up test) - terminate_command_mock = mock.MagicMock() - monkeypatch.setattr( - "mlia.backend.proc.terminate_command", terminate_command_mock - ) - - # Mock a thrown Exception and assign to Command().exit_code - exit_code_mock = mock.PropertyMock(side_effect=Exception("Exception.")) - type(cmd_mock).exit_code = exit_code_mock - - with pytest.raises(Exception, match="Exception."): - execute_commands_locally( - ["command_1", "command_2"], cwd=test_resources_path - ) - - # Assert only "command_1" was executed - assert execute_command_mock.call_count == 1 - - -def test_reporter(tmpdir: Any) -> None: - """Test class 'Reporter'.""" - ctx = ExecutionContext( - app=get_application("application_4")[0], - app_params=["--app=TestApp"], - system=get_system("System 4"), - system_params=[], - ) - assert ctx.system is not None - - class MockParser(OutputParser): - """Mock implementation of an output parser.""" - - def __init__(self, metrics: Dict[str, Any]) -> None: - """Set up the MockParser.""" - super().__init__(name="test") - self.metrics = metrics - - def __call__(self, output: bytearray) -> Dict[str, Any]: - """Return mock metrics (ignoring the given output).""" - return self.metrics - - metrics = {"Metric": 123, "AnotherMetric": 456} - reporter = Reporter( - parsers=[MockParser(metrics={key: val}) for key, val in metrics.items()], - ) - reporter.parse(bytearray()) - report = reporter.report(ctx) - assert report["system"]["name"] == ctx.system.name - assert report["system"]["params"] == {} - assert report["application"]["name"] == ctx.app.name - assert report["application"]["params"] == {"--app": "TestApp"} - assert report["test"]["metrics"] == metrics - report_file = Path(tmpdir) / "report.json" - reporter.save(report, report_file) - assert report_file.is_file() diff --git a/tests/mlia/test_backend_output_parser.py b/tests/mlia/test_backend_output_parser.py deleted file mode 100644 index d86aac8..0000000 --- a/tests/mlia/test_backend_output_parser.py +++ /dev/null @@ -1,152 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Tests for the output parsing.""" -import base64 -import json -from typing import Any -from typing import Dict - -import pytest - -from mlia.backend.output_parser import Base64OutputParser -from mlia.backend.output_parser import OutputParser -from mlia.backend.output_parser import RegexOutputParser - - -OUTPUT_MATCH_ALL = bytearray( - """ -String1: My awesome string! -String2: STRINGS_ARE_GREAT!!! -Int: 12 -Float: 3.14 -""", - encoding="utf-8", -) - -OUTPUT_NO_MATCH = bytearray( - """ -This contains no matches... -Test1234567890!"£$%^&*()_+@~{}[]/.,<>?| -""", - encoding="utf-8", -) - -OUTPUT_PARTIAL_MATCH = bytearray( - "String1: My awesome string!", - encoding="utf-8", -) - -REGEX_CONFIG = { - "FirstString": {"pattern": r"String1.*: (.*)", "type": "str"}, - "SecondString": {"pattern": r"String2.*: (.*)!!!", "type": "str"}, - "IntegerValue": {"pattern": r"Int.*: (.*)", "type": "int"}, - "FloatValue": {"pattern": r"Float.*: (.*)", "type": "float"}, -} - -EMPTY_REGEX_CONFIG: Dict[str, Dict[str, Any]] = {} - -EXPECTED_METRICS_ALL = { - "FirstString": "My awesome string!", - "SecondString": "STRINGS_ARE_GREAT", - "IntegerValue": 12, - "FloatValue": 3.14, -} - -EXPECTED_METRICS_PARTIAL = { - "FirstString": "My awesome string!", -} - - -class TestRegexOutputParser: - """Collect tests for the RegexOutputParser.""" - - @staticmethod - @pytest.mark.parametrize( - ["output", "config", "expected_metrics"], - [ - (OUTPUT_MATCH_ALL, REGEX_CONFIG, EXPECTED_METRICS_ALL), - (OUTPUT_MATCH_ALL + OUTPUT_NO_MATCH, REGEX_CONFIG, EXPECTED_METRICS_ALL), - (OUTPUT_MATCH_ALL + OUTPUT_NO_MATCH, REGEX_CONFIG, EXPECTED_METRICS_ALL), - ( - OUTPUT_MATCH_ALL + OUTPUT_PARTIAL_MATCH, - REGEX_CONFIG, - EXPECTED_METRICS_ALL, - ), - (OUTPUT_NO_MATCH, REGEX_CONFIG, {}), - (OUTPUT_MATCH_ALL, EMPTY_REGEX_CONFIG, {}), - (bytearray(), EMPTY_REGEX_CONFIG, {}), - (bytearray(), REGEX_CONFIG, {}), - ], - ) - def test_parsing(output: bytearray, config: Dict, expected_metrics: Dict) -> None: - """ - Make sure the RegexOutputParser yields valid results. - - I.e. return an empty dict if either the input or the config is empty and - return the parsed metrics otherwise. - """ - parser = RegexOutputParser(name="Test", regex_config=config) - assert parser.name == "Test" - assert isinstance(parser, OutputParser) - res = parser(output) - assert res == expected_metrics - - @staticmethod - def test_unsupported_type() -> None: - """An unsupported type in the regex_config must raise an exception.""" - config = {"BrokenMetric": {"pattern": "(.*)", "type": "UNSUPPORTED_TYPE"}} - with pytest.raises(TypeError): - RegexOutputParser(name="Test", regex_config=config) - - @staticmethod - @pytest.mark.parametrize( - "config", - ( - {"TooManyGroups": {"pattern": r"(\w)(\d)", "type": "str"}}, - {"NoGroups": {"pattern": r"\W", "type": "str"}}, - ), - ) - def test_invalid_pattern(config: Dict) -> None: - """Exactly one capturing parenthesis is allowed in the regex pattern.""" - with pytest.raises(ValueError): - RegexOutputParser(name="Test", regex_config=config) - - -@pytest.mark.parametrize( - "expected_metrics", - [ - EXPECTED_METRICS_ALL, - EXPECTED_METRICS_PARTIAL, - ], -) -def test_base64_output_parser(expected_metrics: Dict) -> None: - """ - Make sure the Base64OutputParser yields valid results. - - I.e. return an empty dict if either the input or the config is empty and - return the parsed metrics otherwise. - """ - parser = Base64OutputParser(name="Test") - assert parser.name == "Test" - assert isinstance(parser, OutputParser) - - def create_base64_output(expected_metrics: Dict) -> bytearray: - json_str = json.dumps(expected_metrics, indent=4) - json_b64 = base64.b64encode(json_str.encode("utf-8")) - return ( - OUTPUT_MATCH_ALL # Should not be matched by the Base64OutputParser - + f"<{Base64OutputParser.TAG_NAME}>".encode("utf-8") - + bytearray(json_b64) - + f"</{Base64OutputParser.TAG_NAME}>".encode("utf-8") - + OUTPUT_NO_MATCH # Just to add some difficulty... - ) - - output = create_base64_output(expected_metrics) - res = parser(output) - assert len(res) == 1 - assert isinstance(res, dict) - for val in res.values(): - assert val == expected_metrics - - output = parser.filter_out_parsed_content(output) - assert output == (OUTPUT_MATCH_ALL + OUTPUT_NO_MATCH) diff --git a/tests/mlia/test_backend_protocol.py b/tests/mlia/test_backend_protocol.py deleted file mode 100644 index 35e9986..0000000 --- a/tests/mlia/test_backend_protocol.py +++ /dev/null @@ -1,231 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=no-self-use,attribute-defined-outside-init,protected-access -"""Tests for the protocol backend module.""" -from contextlib import ExitStack as does_not_raise -from pathlib import Path -from typing import Any -from unittest.mock import MagicMock - -import paramiko -import pytest - -from mlia.backend.common import ConfigurationException -from mlia.backend.config import LocalProtocolConfig -from mlia.backend.protocol import CustomSFTPClient -from mlia.backend.protocol import LocalProtocol -from mlia.backend.protocol import ProtocolFactory -from mlia.backend.protocol import SSHProtocol - - -class TestProtocolFactory: - """Test ProtocolFactory class.""" - - @pytest.mark.parametrize( - "config, expected_class, exception", - [ - ( - { - "protocol": "ssh", - "username": "user", - "password": "pass", - "hostname": "hostname", - "port": "22", - }, - SSHProtocol, - does_not_raise(), - ), - ({"protocol": "local"}, LocalProtocol, does_not_raise()), - ( - {"protocol": "something"}, - None, - pytest.raises(Exception, match="Protocol not supported"), - ), - (None, None, pytest.raises(Exception, match="No protocol config provided")), - ], - ) - def test_get_protocol( - self, config: Any, expected_class: type, exception: Any - ) -> None: - """Test get_protocol method.""" - factory = ProtocolFactory() - with exception: - protocol = factory.get_protocol(config) - assert isinstance(protocol, expected_class) - - -class TestLocalProtocol: - """Test local protocol.""" - - def test_local_protocol_run_command(self) -> None: - """Test local protocol run command.""" - config = LocalProtocolConfig(protocol="local") - protocol = LocalProtocol(config, cwd=Path("/tmp")) - ret, stdout, stderr = protocol.run("pwd") - assert ret == 0 - assert stdout.decode("utf-8").strip() == "/tmp" - assert stderr.decode("utf-8") == "" - - def test_local_protocol_run_wrong_cwd(self) -> None: - """Execution should fail if wrong working directory provided.""" - config = LocalProtocolConfig(protocol="local") - protocol = LocalProtocol(config, cwd=Path("unknown_directory")) - with pytest.raises( - ConfigurationException, match="Wrong working directory unknown_directory" - ): - protocol.run("pwd") - - -class TestSSHProtocol: - """Test SSH protocol.""" - - @pytest.fixture(autouse=True) - def setup_method(self, monkeypatch: Any) -> None: - """Set up protocol mocks.""" - self.mock_ssh_client = MagicMock(spec=paramiko.client.SSHClient) - - self.mock_ssh_channel = ( - self.mock_ssh_client.get_transport.return_value.open_session.return_value - ) - self.mock_ssh_channel.mock_add_spec(spec=paramiko.channel.Channel) - self.mock_ssh_channel.exit_status_ready.side_effect = [False, True] - self.mock_ssh_channel.recv_exit_status.return_value = True - self.mock_ssh_channel.recv_ready.side_effect = [False, True] - self.mock_ssh_channel.recv_stderr_ready.side_effect = [False, True] - - monkeypatch.setattr( - "mlia.backend.protocol.paramiko.client.SSHClient", - MagicMock(return_value=self.mock_ssh_client), - ) - - self.mock_sftp_client = MagicMock(spec=CustomSFTPClient) - monkeypatch.setattr( - "mlia.backend.protocol.CustomSFTPClient.from_transport", - MagicMock(return_value=self.mock_sftp_client), - ) - - ssh_config = { - "protocol": "ssh", - "username": "user", - "password": "pass", - "hostname": "hostname", - "port": "22", - } - self.protocol = SSHProtocol(ssh_config) - - def test_unable_create_ssh_client(self, monkeypatch: Any) -> None: - """Test that command should fail if unable to create ssh client instance.""" - monkeypatch.setattr( - "mlia.backend.protocol.paramiko.client.SSHClient", - MagicMock(side_effect=OSError("Error!")), - ) - - with pytest.raises(Exception, match="Couldn't connect to 'hostname:22'"): - self.protocol.run("command_example", retry=False) - - def test_ssh_protocol_run_command(self) -> None: - """Test that command run via ssh successfully.""" - self.protocol.run("command_example") - self.mock_ssh_channel.exec_command.assert_called_once() - - def test_ssh_protocol_run_command_connect_failed(self) -> None: - """Test that if connection is not possible then correct exception is raised.""" - self.mock_ssh_client.connect.side_effect = OSError("Unable to connect") - self.mock_ssh_client.close.side_effect = Exception("Error!") - - with pytest.raises(Exception, match="Couldn't connect to 'hostname:22'"): - self.protocol.run("command_example", retry=False) - - def test_ssh_protocol_run_command_bad_transport(self) -> None: - """Test that command should fail if unable to get transport.""" - self.mock_ssh_client.get_transport.return_value = None - - with pytest.raises(Exception, match="Unable to get transport"): - self.protocol.run("command_example", retry=False) - - def test_ssh_protocol_deploy_command_file( - self, test_applications_path: Path - ) -> None: - """Test that files could be deployed over ssh.""" - file_for_deploy = test_applications_path / "readme.txt" - dest = "/tmp/dest" - - self.protocol.deploy(file_for_deploy, dest) - self.mock_sftp_client.put.assert_called_once_with(str(file_for_deploy), dest) - - def test_ssh_protocol_deploy_command_unknown_file(self) -> None: - """Test that deploy will fail if file does not exist.""" - with pytest.raises(Exception, match="Deploy error: file type not supported"): - self.protocol.deploy(Path("unknown_file"), "/tmp/dest") - - def test_ssh_protocol_deploy_command_bad_transport(self) -> None: - """Test that deploy should fail if unable to get transport.""" - self.mock_ssh_client.get_transport.return_value = None - - with pytest.raises(Exception, match="Unable to get transport"): - self.protocol.deploy(Path("some_file"), "/tmp/dest") - - def test_ssh_protocol_deploy_command_directory( - self, test_resources_path: Path - ) -> None: - """Test that directory could be deployed over ssh.""" - directory_for_deploy = test_resources_path / "scripts" - dest = "/tmp/dest" - - self.protocol.deploy(directory_for_deploy, dest) - self.mock_sftp_client.put_dir.assert_called_once_with( - directory_for_deploy, dest - ) - - @pytest.mark.parametrize("establish_connection", (True, False)) - def test_ssh_protocol_close(self, establish_connection: bool) -> None: - """Test protocol close operation.""" - if establish_connection: - self.protocol.establish_connection() - self.protocol.close() - - call_count = 1 if establish_connection else 0 - assert self.mock_ssh_channel.exec_command.call_count == call_count - - def test_connection_details(self) -> None: - """Test getting connection details.""" - assert self.protocol.connection_details() == ("hostname", 22) - - -class TestCustomSFTPClient: - """Test CustomSFTPClient class.""" - - @pytest.fixture(autouse=True) - def setup_method(self, monkeypatch: Any) -> None: - """Set up mocks for CustomSFTPClient instance.""" - self.mock_mkdir = MagicMock() - self.mock_put = MagicMock() - monkeypatch.setattr( - "mlia.backend.protocol.paramiko.SFTPClient.__init__", - MagicMock(return_value=None), - ) - monkeypatch.setattr( - "mlia.backend.protocol.paramiko.SFTPClient.mkdir", self.mock_mkdir - ) - monkeypatch.setattr( - "mlia.backend.protocol.paramiko.SFTPClient.put", self.mock_put - ) - - self.sftp_client = CustomSFTPClient(MagicMock()) - - def test_put_dir(self, test_systems_path: Path) -> None: - """Test deploying directory to remote host.""" - directory_for_deploy = test_systems_path / "system1" - - self.sftp_client.put_dir(directory_for_deploy, "/tmp/dest") - assert self.mock_put.call_count == 3 - assert self.mock_mkdir.call_count == 3 - - def test_mkdir(self) -> None: - """Test creating directory on remote host.""" - self.mock_mkdir.side_effect = IOError("Cannot create directory") - - self.sftp_client._mkdir("new_directory", ignore_existing=True) - - with pytest.raises(IOError, match="Cannot create directory"): - self.sftp_client._mkdir("new_directory", ignore_existing=False) diff --git a/tests/mlia/test_utils_proc.py b/tests/mlia/test_utils_proc.py deleted file mode 100644 index 8316ca5..0000000 --- a/tests/mlia/test_utils_proc.py +++ /dev/null @@ -1,149 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Tests for the module utils/proc.""" -import signal -import subprocess -import time -from pathlib import Path -from unittest.mock import MagicMock - -import pytest - -from mlia.utils.proc import CommandExecutor -from mlia.utils.proc import working_directory - - -class TestCommandExecutor: - """Tests for class CommandExecutor.""" - - @staticmethod - def test_execute() -> None: - """Test command execution.""" - executor = CommandExecutor() - - retcode, stdout, stderr = executor.execute(["echo", "hello world!"]) - assert retcode == 0 - assert stdout.decode().strip() == "hello world!" - assert stderr.decode() == "" - - @staticmethod - def test_submit() -> None: - """Test command submittion.""" - executor = CommandExecutor() - - running_command = executor.submit(["sleep", "10"]) - assert running_command.is_alive() is True - assert running_command.exit_code() is None - - running_command.kill() - for _ in range(3): - time.sleep(0.5) - if not running_command.is_alive(): - break - - assert running_command.is_alive() is False - assert running_command.exit_code() == -9 - - with pytest.raises(subprocess.CalledProcessError): - executor.execute(["sleep", "-1"]) - - @staticmethod - @pytest.mark.parametrize("wait", [True, False]) - def test_stop(wait: bool) -> None: - """Test command termination.""" - executor = CommandExecutor() - - running_command = executor.submit(["sleep", "10"]) - running_command.stop(wait=wait) - - if wait: - assert running_command.is_alive() is False - - @staticmethod - def test_unable_to_stop(monkeypatch: pytest.MonkeyPatch) -> None: - """Test when command could not be stopped.""" - running_command_mock = MagicMock() - running_command_mock.poll.return_value = None - - monkeypatch.setattr( - "mlia.utils.proc.subprocess.Popen", - MagicMock(return_value=running_command_mock), - ) - - with pytest.raises(Exception, match="Unable to stop running command"): - executor = CommandExecutor() - running_command = executor.submit(["sleep", "10"]) - - running_command.stop(num_of_attempts=1, interval=0.1) - - running_command_mock.send_signal.assert_called_once_with(signal.SIGINT) - - @staticmethod - def test_stop_after_several_attempts(monkeypatch: pytest.MonkeyPatch) -> None: - """Test when command could be stopped after several attempts.""" - running_command_mock = MagicMock() - running_command_mock.poll.side_effect = [None, 0] - - monkeypatch.setattr( - "mlia.utils.proc.subprocess.Popen", - MagicMock(return_value=running_command_mock), - ) - - executor = CommandExecutor() - running_command = executor.submit(["sleep", "10"]) - - running_command.stop(num_of_attempts=1, interval=0.1) - running_command_mock.send_signal.assert_called_once_with(signal.SIGINT) - - @staticmethod - def test_send_signal() -> None: - """Test sending signal.""" - executor = CommandExecutor() - running_command = executor.submit(["sleep", "10"]) - running_command.send_signal(signal.SIGINT) - - # wait a bit for a signal processing - time.sleep(1) - - assert running_command.is_alive() is False - assert running_command.exit_code() == -2 - - @staticmethod - @pytest.mark.parametrize( - "redirect_output, expected_output", [[True, "hello\n"], [False, ""]] - ) - def test_wait( - capsys: pytest.CaptureFixture, redirect_output: bool, expected_output: str - ) -> None: - """Test wait completion functionality.""" - executor = CommandExecutor() - - running_command = executor.submit(["echo", "hello"]) - running_command.wait(redirect_output=redirect_output) - - out, _ = capsys.readouterr() - assert out == expected_output - - -@pytest.mark.parametrize( - "should_exist, create_dir", - [ - [True, False], - [False, True], - ], -) -def test_working_directory_context_manager( - tmp_path: Path, should_exist: bool, create_dir: bool -) -> None: - """Test working_directory context manager.""" - prev_wd = Path.cwd() - - working_dir = tmp_path / "work_dir" - if should_exist: - working_dir.mkdir() - - with working_directory(working_dir, create_dir=create_dir) as current_working_dir: - assert current_working_dir.is_dir() - assert Path.cwd() == current_working_dir - - assert Path.cwd() == prev_wd diff --git a/tests/mlia/test_api.py b/tests/test_api.py index 09bc509..09bc509 100644 --- a/tests/mlia/test_api.py +++ b/tests/test_api.py diff --git a/tests/mlia/test_backend_application.py b/tests/test_backend_application.py index 2cfb2ef..6860ecb 100644 --- a/tests/mlia/test_backend_application.py +++ b/tests/test_backend_application.py @@ -20,7 +20,6 @@ from mlia.backend.application import install_application from mlia.backend.application import load_applications from mlia.backend.application import remove_application from mlia.backend.common import Command -from mlia.backend.common import DataPaths from mlia.backend.common import Param from mlia.backend.common import UserParamConfig from mlia.backend.config import ApplicationConfig @@ -186,7 +185,6 @@ class TestApplication: config = ApplicationConfig( # Application supported_systems=["system1", "system2"], - build_dir="build_dir", # inherited from Backend name="name", description="description", @@ -222,24 +220,6 @@ class TestApplication: assert len(applications) == 1 assert applications[0].can_run_on("System 1") - def test_get_deploy_data(self, tmp_path: Path) -> None: - """Test Application can run on.""" - src, dest = "src", "dest" - config = ApplicationConfig( - name="application", deploy_data=[(src, dest)], config_location=tmp_path - ) - src_path = tmp_path / src - src_path.mkdir() - application = Application(config) - assert application.get_deploy_data() == [DataPaths(src_path, dest)] - - def test_get_deploy_data_no_config_location(self) -> None: - """Test that getting deploy data fails if no config location provided.""" - with pytest.raises( - Exception, match="Unable to get application .* config location" - ): - Application(ApplicationConfig(name="application")).get_deploy_data() - def test_unable_to_create_application_without_name(self) -> None: """Test that it is not possible to create application without name.""" with pytest.raises(Exception, match="Name is empty"): @@ -394,14 +374,11 @@ def test_load_application() -> None: default_variables = {"var1": "value1", "var2": "value2"} application_5_0 = application_5[0] - assert application_5_0.build_dir == "default_build_dir" assert application_5_0.supported_systems == ["System 1"] assert application_5_0.commands == default_commands assert application_5_0.variables == default_variables - assert application_5_0.lock is False application_5_1 = application_5[1] - assert application_5_1.build_dir == application_5_0.build_dir assert application_5_1.supported_systems == ["System 2"] assert application_5_1.commands == application_5_1.commands assert application_5_1.variables == default_variables @@ -411,50 +388,31 @@ def test_load_application() -> None: application_5a_0 = application_5a[0] assert application_5a_0.supported_systems == ["System 1"] - assert application_5a_0.build_dir == "build_5A" assert application_5a_0.commands == default_commands assert application_5a_0.variables == {"var1": "new value1", "var2": "value2"} - assert application_5a_0.lock is False application_5a_1 = application_5a[1] assert application_5a_1.supported_systems == ["System 2"] - assert application_5a_1.build_dir == "build" assert application_5a_1.commands == { - "build": Command(["default build command"]), + "build": default_commands["build"], "run": Command(["run command on system 2"]), } assert application_5a_1.variables == {"var1": "value1", "var2": "new value2"} - assert application_5a_1.lock is True application_5b = get_application("application_5B") assert len(application_5b) == 2 application_5b_0 = application_5b[0] - assert application_5b_0.build_dir == "build_5B" assert application_5b_0.supported_systems == ["System 1"] assert application_5b_0.commands == { - "build": Command(["default build command with value for var1 System1"], []), + "build": Command(["default build command with value for var1 System1"]), "run": Command(["default run command with value for var2 System1"]), } assert "non_used_command" not in application_5b_0.commands application_5b_1 = application_5b[1] - assert application_5b_1.build_dir == "build" assert application_5b_1.supported_systems == ["System 2"] assert application_5b_1.commands == { - "build": Command( - [ - "build command on system 2 with value" - " for var1 System2 {user_params:param1}" - ], - [ - Param( - "--param", - "Sample command param", - ["value1", "value2", "value3"], - "value1", - ) - ], - ), + "build": Command(["default build command with value for var1 System2"]), "run": Command(["run command on system 2"], []), } diff --git a/tests/mlia/test_backend_common.py b/tests/test_backend_common.py index 82a985a..0533ef6 100644 --- a/tests/mlia/test_backend_common.py +++ b/tests/test_backend_common.py @@ -149,7 +149,7 @@ class TestBackend: application.validate_parameter("foo", "bar") assert "Unknown command: 'foo'" in str(err.value) - def test_build_command(self, monkeypatch: Any) -> None: + def test_build_command(self) -> None: """Test command building.""" config = { "name": "test", @@ -175,14 +175,12 @@ class TestBackend: "variables": {"var_A": "value for variable A"}, } - monkeypatch.setattr("mlia.backend.system.ProtocolFactory", MagicMock()) application, system = Application(config), System(config) # type: ignore context = ExecutionContext( app=application, app_params=[], system=system, system_params=[], - custom_deploy_data=[], ) param_resolver = ParamResolver(context) @@ -284,13 +282,11 @@ class TestBackend: ) def test_resolved_parameters( self, - monkeypatch: Any, class_: type, config: Dict, expected_output: List[Tuple[Optional[str], Param]], ) -> None: """Test command building.""" - monkeypatch.setattr("mlia.backend.system.ProtocolFactory", MagicMock()) backend = class_(config) params = backend.resolved_parameters( @@ -438,7 +434,7 @@ class TestCommand: Param("param", "param description", [], None, "alias"), Param("param", "param description", [], None, "alias"), ], - pytest.raises(ConfigurationException, match="Non unique aliases alias"), + pytest.raises(ConfigurationException, match="Non-unique aliases alias"), ], [ [ @@ -475,7 +471,7 @@ class TestCommand: Param("param4", "param4 description", [], None, "alias2"), ], pytest.raises( - ConfigurationException, match="Non unique aliases alias1, alias2" + ConfigurationException, match="Non-unique aliases alias1, alias2" ), ], ], diff --git a/tests/test_backend_execution.py b/tests/test_backend_execution.py new file mode 100644 index 0000000..07e7c98 --- /dev/null +++ b/tests/test_backend_execution.py @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +# pylint: disable=no-self-use +"""Test backend execution module.""" +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from mlia.backend.application import Application +from mlia.backend.common import UserParamConfig +from mlia.backend.config import ApplicationConfig +from mlia.backend.config import SystemConfig +from mlia.backend.execution import ExecutionContext +from mlia.backend.execution import get_application_and_system +from mlia.backend.execution import get_application_by_name_and_system +from mlia.backend.execution import ParamResolver +from mlia.backend.system import load_system + + +def test_context_param_resolver(tmpdir: Any) -> None: + """Test parameter resolving.""" + system_config_location = Path(tmpdir) / "system" + system_config_location.mkdir() + + application_config_location = Path(tmpdir) / "application" + application_config_location.mkdir() + + ctx = ExecutionContext( + app=Application( + ApplicationConfig( + name="test_application", + description="Test application", + config_location=application_config_location, + commands={ + "run": [ + "run_command1 {user_params:0}", + "run_command2 {user_params:1}", + ] + }, + variables={"var_1": "value for var_1"}, + user_params={ + "run": [ + UserParamConfig( + name="--param1", + description="Param 1", + default_value="123", + alias="param_1", + ), + UserParamConfig( + name="--param2", description="Param 2", default_value="456" + ), + UserParamConfig( + name="--param3", description="Param 3", alias="param_3" + ), + UserParamConfig( + name="--param4=", + description="Param 4", + default_value="456", + alias="param_4", + ), + UserParamConfig( + description="Param 5", + default_value="789", + alias="param_5", + ), + ] + }, + ) + ), + app_params=["--param2=789"], + system=load_system( + SystemConfig( + name="test_system", + description="Test system", + config_location=system_config_location, + commands={ + "build": ["build_command1 {user_params:0}"], + "run": ["run_command {application.commands.run:1}"], + }, + variables={"var_1": "value for var_1"}, + user_params={ + "build": [ + UserParamConfig( + name="--param1", description="Param 1", default_value="aaa" + ), + UserParamConfig(name="--param2", description="Param 2"), + ] + }, + ) + ), + system_params=["--param1=bbb"], + ) + + param_resolver = ParamResolver(ctx) + expected_values = { + "application.name": "test_application", + "application.description": "Test application", + "application.config_dir": str(application_config_location), + "application.commands.run:0": "run_command1 --param1 123", + "application.commands.run.params:0": "123", + "application.commands.run.params:param_1": "123", + "application.commands.run:1": "run_command2 --param2 789", + "application.commands.run.params:1": "789", + "application.variables:var_1": "value for var_1", + "system.name": "test_system", + "system.description": "Test system", + "system.config_dir": str(system_config_location), + "system.commands.build:0": "build_command1 --param1 bbb", + "system.commands.run:0": "run_command run_command2 --param2 789", + "system.commands.build.params:0": "bbb", + "system.variables:var_1": "value for var_1", + } + + for param, value in expected_values.items(): + assert param_resolver(param) == value + + expected_errors = { + "application.variables:var_2": pytest.raises( + Exception, match="Unknown variable var_2" + ), + "application.commands.clean:0": pytest.raises( + Exception, match="Command clean not found" + ), + "application.commands.run:2": pytest.raises( + Exception, match="Invalid index 2 for command run" + ), + "application.commands.run.params:5": pytest.raises( + Exception, match="Invalid parameter index 5 for command run" + ), + "application.commands.run.params:param_2": pytest.raises( + Exception, + match="No value for parameter with index or alias param_2 of command run", + ), + "UNKNOWN": pytest.raises( + Exception, match="Unable to resolve parameter UNKNOWN" + ), + "system.commands.build.params:1": pytest.raises( + Exception, + match="No value for parameter with index or alias 1 of command build", + ), + "system.commands.build:A": pytest.raises( + Exception, match="Bad command index A" + ), + "system.variables:var_2": pytest.raises( + Exception, match="Unknown variable var_2" + ), + } + for param, error in expected_errors.items(): + with error: + param_resolver(param) + + resolved_params = ctx.app.resolved_parameters("run", []) + expected_user_params = { + "user_params:0": "--param1 123", + "user_params:param_1": "--param1 123", + "user_params:2": "--param3", + "user_params:param_3": "--param3", + "user_params:3": "--param4=456", + "user_params:param_4": "--param4=456", + "user_params:param_5": "789", + } + for param, expected_value in expected_user_params.items(): + assert param_resolver(param, "run", resolved_params) == expected_value + + with pytest.raises( + Exception, match="Invalid index 5 for user params of command run" + ): + param_resolver("user_params:5", "run", resolved_params) + + with pytest.raises( + Exception, match="No user parameter for command 'run' with alias 'param_2'." + ): + param_resolver("user_params:param_2", "run", resolved_params) + + with pytest.raises(Exception, match="Unable to resolve user params"): + param_resolver("user_params:0", "", resolved_params) + + +def test_get_application_by_name_and_system(monkeypatch: Any) -> None: + """Test exceptional case for get_application_by_name_and_system.""" + monkeypatch.setattr( + "mlia.backend.execution.get_application", + MagicMock(return_value=[MagicMock(), MagicMock()]), + ) + + with pytest.raises( + ValueError, + match="Error during getting application test_application for the " + "system test_system", + ): + get_application_by_name_and_system("test_application", "test_system") + + +def test_get_application_and_system(monkeypatch: Any) -> None: + """Test exceptional case for get_application_and_system.""" + monkeypatch.setattr( + "mlia.backend.execution.get_system", MagicMock(return_value=None) + ) + + with pytest.raises(ValueError, match="System test_system is not found"): + get_application_and_system("test_application", "test_system") diff --git a/tests/mlia/test_backend_fs.py b/tests/test_backend_fs.py index ff9c2ae..7423222 100644 --- a/tests/mlia/test_backend_fs.py +++ b/tests/test_backend_fs.py @@ -11,8 +11,6 @@ from unittest.mock import MagicMock import pytest from mlia.backend.fs import get_backends_path -from mlia.backend.fs import read_file_as_bytearray -from mlia.backend.fs import read_file_as_string from mlia.backend.fs import recreate_directory from mlia.backend.fs import remove_directory from mlia.backend.fs import remove_resource @@ -120,38 +118,6 @@ def write_to_file( return tmpfile -class TestReadFileAsString: - """Test read_file_as_string() function.""" - - def test_returns_text_from_valid_file(self, tmpdir: Any) -> None: - """Ensure the string written to a file read correctly.""" - file_path = write_to_file(tmpdir, "w", "hello") - assert read_file_as_string(file_path) == "hello" - - def test_output_is_empty_string_when_input_file_non_existent( - self, tmpdir: Any - ) -> None: - """Ensure empty string returned when reading from non-existent file.""" - file_path = Path(tmpdir / "non-existent.txt") - assert read_file_as_string(file_path) == "" - - -class TestReadFileAsByteArray: - """Test read_file_as_bytearray() function.""" - - def test_returns_bytes_from_valid_file(self, tmpdir: Any) -> None: - """Ensure the bytes written to a file read correctly.""" - file_path = write_to_file(tmpdir, "wb", b"hello bytes") - assert read_file_as_bytearray(file_path) == b"hello bytes" - - def test_output_is_empty_bytearray_when_input_file_non_existent( - self, tmpdir: Any - ) -> None: - """Ensure empty bytearray returned when reading from non-existent file.""" - file_path = Path(tmpdir / "non-existent.txt") - assert read_file_as_bytearray(file_path) == bytearray() - - @pytest.mark.parametrize( "value, replacement, expected_result", [ diff --git a/tests/mlia/test_backend_manager.py b/tests/test_backend_manager.py index c81366f..1b5fea1 100644 --- a/tests/mlia/test_backend_manager.py +++ b/tests/test_backend_manager.py @@ -1,13 +1,15 @@ # SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for module backend/manager.""" -import os +import base64 +import json from contextlib import ExitStack as does_not_raise from pathlib import Path from typing import Any from typing import Dict from typing import List from typing import Optional +from typing import Set from typing import Tuple from unittest.mock import MagicMock from unittest.mock import PropertyMock @@ -15,7 +17,6 @@ from unittest.mock import PropertyMock import pytest from mlia.backend.application import get_application -from mlia.backend.common import DataPaths from mlia.backend.execution import ExecutionContext from mlia.backend.execution import run_application from mlia.backend.manager import BackendRunner @@ -30,9 +31,45 @@ from mlia.backend.manager import is_supported from mlia.backend.manager import ModelInfo from mlia.backend.manager import PerformanceMetrics from mlia.backend.manager import supported_backends +from mlia.backend.output_consumer import Base64OutputConsumer from mlia.backend.system import get_system +def _mock_encode_b64(data: Dict[str, int]) -> str: + """ + Encode the given data into a mock base64-encoded string of JSON. + + This reproduces the base64 encoding done in the Corstone applications. + + JSON example: + + ```json + [{'count': 1, + 'profiling_group': 'Inference', + 'samples': [{'name': 'NPU IDLE', 'value': [612]}, + {'name': 'NPU AXI0_RD_DATA_BEAT_RECEIVED', 'value': [165872]}, + {'name': 'NPU AXI0_WR_DATA_BEAT_WRITTEN', 'value': [88712]}, + {'name': 'NPU AXI1_RD_DATA_BEAT_RECEIVED', 'value': [57540]}, + {'name': 'NPU ACTIVE', 'value': [520489]}, + {'name': 'NPU TOTAL', 'value': [521101]}]}] + ``` + """ + wrapped_data = [ + { + "count": 1, + "profiling_group": "Inference", + "samples": [ + {"name": name, "value": [value]} for name, value in data.items() + ], + } + ] + json_str = json.dumps(wrapped_data) + json_bytes = bytearray(json_str, encoding="utf-8") + json_b64 = base64.b64encode(json_bytes).decode("utf-8") + tag = Base64OutputConsumer.TAG_NAME + return f"<{tag}>{json_b64}</{tag}>" + + @pytest.mark.parametrize( "data, is_ready, result, missed_keys", [ @@ -40,50 +77,52 @@ from mlia.backend.system import get_system [], False, {}, - [ + { "npu_active_cycles", "npu_axi0_rd_data_beat_received", "npu_axi0_wr_data_beat_written", "npu_axi1_rd_data_beat_received", "npu_idle_cycles", "npu_total_cycles", - ], + }, ), ( ["sample text"], False, {}, - [ + { "npu_active_cycles", "npu_axi0_rd_data_beat_received", "npu_axi0_wr_data_beat_written", "npu_axi1_rd_data_beat_received", "npu_idle_cycles", "npu_total_cycles", - ], + }, ), ( - [ - ["NPU AXI0_RD_DATA_BEAT_RECEIVED beats: 123"], - False, - {"npu_axi0_rd_data_beat_received": 123}, - [ - "npu_active_cycles", - "npu_axi0_wr_data_beat_written", - "npu_axi1_rd_data_beat_received", - "npu_idle_cycles", - "npu_total_cycles", - ], - ] + [_mock_encode_b64({"NPU AXI0_RD_DATA_BEAT_RECEIVED": 123})], + False, + {"npu_axi0_rd_data_beat_received": 123}, + { + "npu_active_cycles", + "npu_axi0_wr_data_beat_written", + "npu_axi1_rd_data_beat_received", + "npu_idle_cycles", + "npu_total_cycles", + }, ), ( [ - "NPU AXI0_RD_DATA_BEAT_RECEIVED beats: 1", - "NPU AXI0_WR_DATA_BEAT_WRITTEN beats: 2", - "NPU AXI1_RD_DATA_BEAT_RECEIVED beats: 3", - "NPU ACTIVE cycles: 4", - "NPU IDLE cycles: 5", - "NPU TOTAL cycles: 6", + _mock_encode_b64( + { + "NPU AXI0_RD_DATA_BEAT_RECEIVED": 1, + "NPU AXI0_WR_DATA_BEAT_WRITTEN": 2, + "NPU AXI1_RD_DATA_BEAT_RECEIVED": 3, + "NPU ACTIVE": 4, + "NPU IDLE": 5, + "NPU TOTAL": 6, + } + ) ], True, { @@ -94,12 +133,12 @@ from mlia.backend.system import get_system "npu_idle_cycles": 5, "npu_total_cycles": 6, }, - [], + set(), ), ], ) def test_generic_inference_output_parser( - data: List[str], is_ready: bool, result: Dict, missed_keys: List[str] + data: Dict[str, int], is_ready: bool, result: Dict, missed_keys: Set[str] ) -> None: """Test generic runner output parser.""" parser = GenericInferenceOutputParser() @@ -316,8 +355,8 @@ class TestBackendRunner: "execution_params, expected_command", [ ( - ExecutionParams("application_4", "System 4", [], [], []), - ["application_4", [], "System 4", [], []], + ExecutionParams("application_4", "System 4", [], []), + ["application_4", [], "System 4", []], ), ( ExecutionParams( @@ -325,14 +364,12 @@ class TestBackendRunner: "System 6", ["param1=value2"], ["sys-param1=value2"], - [], ), [ "application_6", ["param1=value2"], "System 6", ["sys-param1=value2"], - [], ], ), ], @@ -351,67 +388,6 @@ class TestBackendRunner: run_app.assert_called_once_with(*expected_command) - @staticmethod - @pytest.mark.parametrize( - "execution_params, expected_command", - [ - ( - ExecutionParams( - "application_1", - "System 1", - [], - [], - ["source1.txt:dest1.txt", "source2.txt:dest2.txt"], - ), - [ - "application_1", - [], - "System 1", - [], - [ - DataPaths(Path("source1.txt"), "dest1.txt"), - DataPaths(Path("source2.txt"), "dest2.txt"), - ], - ], - ), - ], - ) - def test_run_application_connected( - monkeypatch: pytest.MonkeyPatch, - execution_params: ExecutionParams, - expected_command: List[str], - ) -> None: - """Test method run_application with connectable systems (SSH).""" - app = get_application(execution_params.application, execution_params.system)[0] - sys = get_system(execution_params.system) - - assert sys is not None - - connect_mock = MagicMock(return_value=True, name="connect_mock") - deploy_mock = MagicMock(return_value=True, name="deploy_mock") - run_mock = MagicMock( - return_value=(os.EX_OK, bytearray(), bytearray()), name="run_mock" - ) - sys.establish_connection = connect_mock # type: ignore - sys.deploy = deploy_mock # type: ignore - sys.run = run_mock # type: ignore - - monkeypatch.setattr( - "mlia.backend.execution.get_application_and_system", - MagicMock(return_value=(app, sys)), - ) - - run_app_mock = MagicMock(wraps=run_application) - monkeypatch.setattr("mlia.backend.manager.run_application", run_app_mock) - - backend_runner = BackendRunner() - backend_runner.run_application(execution_params) - - run_app_mock.assert_called_once_with(*expected_command) - - connect_mock.assert_called_once() - assert deploy_mock.call_count == 2 - @pytest.mark.parametrize( "device, system, application, backend, expected_error", @@ -531,12 +507,16 @@ def test_estimate_performance( mock_context = create_mock_context( [ - "NPU AXI0_RD_DATA_BEAT_RECEIVED beats: 1", - "NPU AXI0_WR_DATA_BEAT_WRITTEN beats: 2", - "NPU AXI1_RD_DATA_BEAT_RECEIVED beats: 3", - "NPU ACTIVE cycles: 4", - "NPU IDLE cycles: 5", - "NPU TOTAL cycles: 6", + _mock_encode_b64( + { + "NPU AXI0_RD_DATA_BEAT_RECEIVED": 1, + "NPU AXI0_WR_DATA_BEAT_WRITTEN": 2, + "NPU AXI1_RD_DATA_BEAT_RECEIVED": 3, + "NPU ACTIVE": 4, + "NPU IDLE": 5, + "NPU TOTAL": 6, + } + ) ] ) @@ -571,14 +551,14 @@ def test_estimate_performance_insufficient_data( backend_runner.is_system_installed.return_value = True backend_runner.is_application_installed.return_value = True - no_total_cycles_output = [ - "NPU AXI0_RD_DATA_BEAT_RECEIVED beats: 1", - "NPU AXI0_WR_DATA_BEAT_WRITTEN beats: 2", - "NPU AXI1_RD_DATA_BEAT_RECEIVED beats: 3", - "NPU ACTIVE cycles: 4", - "NPU IDLE cycles: 5", - ] - mock_context = create_mock_context(no_total_cycles_output) + no_total_cycles_output = { + "NPU AXI0_RD_DATA_BEAT_RECEIVED": 1, + "NPU AXI0_WR_DATA_BEAT_WRITTEN": 2, + "NPU AXI1_RD_DATA_BEAT_RECEIVED": 3, + "NPU ACTIVE": 4, + "NPU IDLE": 5, + } + mock_context = create_mock_context([_mock_encode_b64(no_total_cycles_output)]) backend_runner.run_application.return_value = mock_context diff --git a/tests/test_backend_output_consumer.py b/tests/test_backend_output_consumer.py new file mode 100644 index 0000000..881112e --- /dev/null +++ b/tests/test_backend_output_consumer.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the output parsing.""" +import base64 +import json +from typing import Any +from typing import Dict + +import pytest + +from mlia.backend.output_consumer import Base64OutputConsumer +from mlia.backend.output_consumer import OutputConsumer + + +OUTPUT_MATCH_ALL = bytearray( + """ +String1: My awesome string! +String2: STRINGS_ARE_GREAT!!! +Int: 12 +Float: 3.14 +""", + encoding="utf-8", +) + +OUTPUT_NO_MATCH = bytearray( + """ +This contains no matches... +Test1234567890!"£$%^&*()_+@~{}[]/.,<>?| +""", + encoding="utf-8", +) + +OUTPUT_PARTIAL_MATCH = bytearray( + "String1: My awesome string!", + encoding="utf-8", +) + +REGEX_CONFIG = { + "FirstString": {"pattern": r"String1.*: (.*)", "type": "str"}, + "SecondString": {"pattern": r"String2.*: (.*)!!!", "type": "str"}, + "IntegerValue": {"pattern": r"Int.*: (.*)", "type": "int"}, + "FloatValue": {"pattern": r"Float.*: (.*)", "type": "float"}, +} + +EMPTY_REGEX_CONFIG: Dict[str, Dict[str, Any]] = {} + +EXPECTED_METRICS_ALL = { + "FirstString": "My awesome string!", + "SecondString": "STRINGS_ARE_GREAT", + "IntegerValue": 12, + "FloatValue": 3.14, +} + +EXPECTED_METRICS_PARTIAL = { + "FirstString": "My awesome string!", +} + + +@pytest.mark.parametrize( + "expected_metrics", + [ + EXPECTED_METRICS_ALL, + EXPECTED_METRICS_PARTIAL, + ], +) +def test_base64_output_consumer(expected_metrics: Dict) -> None: + """ + Make sure the Base64OutputConsumer yields valid results. + + I.e. return an empty dict if either the input or the config is empty and + return the parsed metrics otherwise. + """ + parser = Base64OutputConsumer() + assert isinstance(parser, OutputConsumer) + + def create_base64_output(expected_metrics: Dict) -> bytearray: + json_str = json.dumps(expected_metrics, indent=4) + json_b64 = base64.b64encode(json_str.encode("utf-8")) + return ( + OUTPUT_MATCH_ALL # Should not be matched by the Base64OutputConsumer + + f"<{Base64OutputConsumer.TAG_NAME}>".encode("utf-8") + + bytearray(json_b64) + + f"</{Base64OutputConsumer.TAG_NAME}>".encode("utf-8") + + OUTPUT_NO_MATCH # Just to add some difficulty... + ) + + output = create_base64_output(expected_metrics) + + consumed = False + for line in output.splitlines(): + if parser.feed(line.decode("utf-8")): + consumed = True + assert consumed # we should have consumed at least one line + + res = parser.parsed_output + assert len(res) == 1 + assert isinstance(res, list) + for val in res: + assert val == expected_metrics diff --git a/tests/mlia/test_backend_proc.py b/tests/test_backend_proc.py index 9ca4788..f47c244 100644 --- a/tests/mlia/test_backend_proc.py +++ b/tests/test_backend_proc.py @@ -6,7 +6,6 @@ from pathlib import Path from typing import Any from unittest import mock -import psutil import pytest from sh import ErrorReturnCode @@ -16,10 +15,8 @@ from mlia.backend.proc import CommandNotFound from mlia.backend.proc import parse_command from mlia.backend.proc import print_command_stdout from mlia.backend.proc import run_and_wait -from mlia.backend.proc import save_process_info from mlia.backend.proc import ShellCommand from mlia.backend.proc import terminate_command -from mlia.backend.proc import terminate_external_process class TestShellCommand: @@ -109,61 +106,6 @@ def test_print_command_stdout_not_alive(mock_print: Any) -> None: mock_print.assert_called_once_with("test") -def test_terminate_external_process_no_process(capsys: Any) -> None: - """Test that non existed process could be terminated.""" - mock_command = mock.MagicMock() - mock_command.terminate.side_effect = psutil.Error("Error!") - - terminate_external_process(mock_command) - captured = capsys.readouterr() - assert captured.out == "Unable to terminate process\n" - - -def test_terminate_external_process_case1() -> None: - """Test when process terminated immediately.""" - mock_command = mock.MagicMock() - mock_command.is_running.return_value = False - - terminate_external_process(mock_command) - mock_command.terminate.assert_called_once() - mock_command.is_running.assert_called_once() - - -def test_terminate_external_process_case2() -> None: - """Test when process termination takes time.""" - mock_command = mock.MagicMock() - mock_command.is_running.side_effect = [True, True, False] - - terminate_external_process(mock_command) - mock_command.terminate.assert_called_once() - assert mock_command.is_running.call_count == 3 - - -def test_terminate_external_process_case3() -> None: - """Test when process termination takes more time.""" - mock_command = mock.MagicMock() - mock_command.is_running.side_effect = [True, True, True] - - terminate_external_process( - mock_command, number_of_attempts=2, wait_period=0.1, wait_for_termination=0.1 - ) - assert mock_command.is_running.call_count == 3 - assert mock_command.terminate.call_count == 2 - - -def test_terminate_external_process_case4() -> None: - """Test when process termination takes more time.""" - mock_command = mock.MagicMock() - mock_command.is_running.side_effect = [True, True, False] - - terminate_external_process( - mock_command, number_of_attempts=2, wait_period=0.1, wait_for_termination=0.1 - ) - mock_command.terminate.assert_called_once() - assert mock_command.is_running.call_count == 3 - assert mock_command.terminate.call_count == 1 - - def test_terminate_command_no_process() -> None: """Test command termination when process does not exist.""" mock_command = mock.MagicMock() @@ -253,17 +195,6 @@ class TestRunAndWait: assert self.terminate_command_mock.call_count == call_count -def test_save_process_info_no_process(monkeypatch: Any, tmpdir: Any) -> None: - """Test save_process_info function.""" - mock_process = mock.MagicMock() - monkeypatch.setattr("psutil.Process", mock.MagicMock(return_value=mock_process)) - mock_process.children.side_effect = psutil.NoSuchProcess(555) - - pid_file_path = Path(tmpdir) / "test.pid" - save_process_info(555, pid_file_path) - assert not pid_file_path.exists() - - def test_parse_command() -> None: """Test parse_command function.""" assert parse_command("1.sh") == ["bash", "1.sh"] diff --git a/tests/mlia/test_backend_source.py b/tests/test_backend_source.py index 84a6a77..11f1781 100644 --- a/tests/mlia/test_backend_source.py +++ b/tests/test_backend_source.py @@ -160,8 +160,8 @@ class TestTarArchiveSource: tmpdir_path = Path(tmpdir) tar_source.install_into(tmpdir_path) source_files = [ - "aiet-config.json.license", - "aiet-config.json", + "backend-config.json.license", + "backend-config.json", "system_artifact", ] dest_files = [f.name for f in tmpdir_path.iterdir()] diff --git a/tests/mlia/test_backend_system.py b/tests/test_backend_system.py index 21187ff..13347c6 100644 --- a/tests/mlia/test_backend_system.py +++ b/tests/test_backend_system.py @@ -17,24 +17,12 @@ from mlia.backend.common import Command from mlia.backend.common import ConfigurationException from mlia.backend.common import Param from mlia.backend.common import UserParamConfig -from mlia.backend.config import LocalProtocolConfig -from mlia.backend.config import ProtocolConfig -from mlia.backend.config import SSHConfig from mlia.backend.config import SystemConfig -from mlia.backend.controller import SystemController -from mlia.backend.controller import SystemControllerSingleInstance -from mlia.backend.protocol import LocalProtocol -from mlia.backend.protocol import SSHProtocol -from mlia.backend.protocol import SupportsClose -from mlia.backend.protocol import SupportsDeploy -from mlia.backend.system import ControlledSystem from mlia.backend.system import get_available_systems -from mlia.backend.system import get_controller from mlia.backend.system import get_system from mlia.backend.system import install_system from mlia.backend.system import load_system from mlia.backend.system import remove_system -from mlia.backend.system import StandaloneSystem from mlia.backend.system import System @@ -68,9 +56,7 @@ def test_get_available_systems() -> None: def test_get_system() -> None: """Test get_system.""" system1 = get_system("System 1") - assert isinstance(system1, ControlledSystem) - assert system1.connectable is True - assert system1.connection_details() == ("localhost", 8021) + assert isinstance(system1, System) assert system1.name == "System 1" system2 = get_system("System 2") @@ -78,8 +64,10 @@ def test_get_system() -> None: assert system1 != 42 assert system1 != system2 - system = get_system("Unknown system") - assert system is None + with pytest.raises( + ConfigurationException, match="System 'Unknown system' not found." + ): + get_system("Unknown system") @pytest.mark.parametrize( @@ -142,10 +130,9 @@ def test_remove_system(monkeypatch: Any) -> None: mock_remove_backend.assert_called_once() -def test_system(monkeypatch: Any) -> None: +def test_system() -> None: """Test the System class.""" config = SystemConfig(name="System 1") - monkeypatch.setattr("mlia.backend.system.ProtocolFactory", MagicMock()) system = System(config) assert str(system) == "System 1" assert system.name == "System 1" @@ -162,134 +149,34 @@ def test_system_with_empty_parameter_name() -> None: System(bad_config) -def test_system_standalone_run() -> None: - """Test run operation for standalone system.""" +def test_system_run() -> None: + """Test run operation for system.""" system = get_system("System 4") - assert isinstance(system, StandaloneSystem) - - with pytest.raises( - ConfigurationException, match="System .* does not support connections" - ): - system.connection_details() - - with pytest.raises( - ConfigurationException, match="System .* does not support connections" - ): - system.establish_connection() - - assert system.connectable is False + assert isinstance(system, System) system.run("echo 'application run'") -@pytest.mark.parametrize( - "system_name, expected_value", [("System 1", True), ("System 4", False)] -) -def test_system_supports_deploy(system_name: str, expected_value: bool) -> None: - """Test system property supports_deploy.""" - system = get_system(system_name) - if system is None: - pytest.fail("Unable to get system {}".format(system_name)) - assert system.supports_deploy == expected_value - - -@pytest.mark.parametrize( - "mock_protocol", - [ - MagicMock(spec=SSHProtocol), - MagicMock( - spec=SSHProtocol, - **{"close.side_effect": ValueError("Unable to close protocol")} - ), - MagicMock(spec=LocalProtocol), - ], -) -def test_system_start_and_stop(monkeypatch: Any, mock_protocol: MagicMock) -> None: - """Test system start, run commands and stop.""" - monkeypatch.setattr( - "mlia.backend.system.ProtocolFactory.get_protocol", - MagicMock(return_value=mock_protocol), - ) - - system = get_system("System 1") - if system is None: - pytest.fail("Unable to get system") - assert isinstance(system, ControlledSystem) - - with pytest.raises(Exception, match="System has not been started"): - system.stop() - - assert not system.is_running() - assert system.get_output() == ("", "") - system.start(["sleep 10"], False) - assert system.is_running() - system.stop(wait=True) - assert not system.is_running() - assert system.get_output() == ("", "") - - if isinstance(mock_protocol, SupportsClose): - mock_protocol.close.assert_called_once() - - if isinstance(mock_protocol, SSHProtocol): - system.establish_connection() - - def test_system_start_no_config_location() -> None: """Test that system without config location could not start.""" - system = load_system( - SystemConfig( - name="test", - data_transfer=SSHConfig( - protocol="ssh", - username="user", - password="user", - hostname="localhost", - port="123", - ), - ) - ) + system = load_system(SystemConfig(name="test")) - assert isinstance(system, ControlledSystem) + assert isinstance(system, System) with pytest.raises( - ConfigurationException, match="System test has wrong config location" + ConfigurationException, match="System has invalid config location: None" ): - system.start(["sleep 100"]) + system.run("sleep 100") @pytest.mark.parametrize( "config, expected_class, expected_error", [ ( - SystemConfig( - name="test", - data_transfer=SSHConfig( - protocol="ssh", - username="user", - password="user", - hostname="localhost", - port="123", - ), - ), - ControlledSystem, + SystemConfig(name="test"), + System, does_not_raise(), ), - ( - SystemConfig( - name="test", data_transfer=LocalProtocolConfig(protocol="local") - ), - StandaloneSystem, - does_not_raise(), - ), - ( - SystemConfig( - name="test", - data_transfer=ProtocolConfig(protocol="cool_protocol"), # type: ignore - ), - None, - pytest.raises( - Exception, match="Unsupported execution type for protocol cool_protocol" - ), - ), + (SystemConfig(), None, pytest.raises(ConfigurationException)), ], ) def test_load_system( @@ -310,7 +197,6 @@ def test_load_system_populate_shared_params() -> None: load_system( SystemConfig( name="test_system", - data_transfer=LocalProtocolConfig(protocol="local"), user_params={ "shared": [ UserParamConfig( @@ -330,7 +216,6 @@ def test_load_system_populate_shared_params() -> None: load_system( SystemConfig( name="test_system", - data_transfer=LocalProtocolConfig(protocol="local"), user_params={ "shared": [ UserParamConfig( @@ -355,7 +240,6 @@ def test_load_system_populate_shared_params() -> None: system0 = load_system( SystemConfig( name="test_system", - data_transfer=LocalProtocolConfig(protocol="local"), commands={"run": ["run_command"]}, user_params={ "shared": [], @@ -389,7 +273,6 @@ def test_load_system_populate_shared_params() -> None: system1 = load_system( SystemConfig( name="test_system", - data_transfer=LocalProtocolConfig(protocol="local"), user_params={ "shared": [ UserParamConfig( @@ -412,20 +295,7 @@ def test_load_system_populate_shared_params() -> None: }, ) ) - assert len(system1.commands) == 2 - build_command1 = system1.commands["build"] - assert build_command1 == Command( - [], - [ - Param( - "--shared_param1", - "Shared parameter", - ["1", "2", "3"], - "1", - "shared_param1", - ) - ], - ) + assert len(system1.commands) == 1 run_command1 = system1.commands["run"] assert run_command1 == Command( @@ -451,7 +321,6 @@ def test_load_system_populate_shared_params() -> None: system2 = load_system( SystemConfig( name="test_system", - data_transfer=LocalProtocolConfig(protocol="local"), commands={"build": ["build_command"]}, user_params={ "shared": [ @@ -479,15 +348,7 @@ def test_load_system_populate_shared_params() -> None: build_command2 = system2.commands["build"] assert build_command2 == Command( ["build_command"], - [ - Param( - "--shared_param1", - "Shared parameter", - ["1", "2", "3"], - "1", - "shared_param1", - ) - ], + [], ) run_command2 = system1.commands["run"] @@ -510,32 +371,3 @@ def test_load_system_populate_shared_params() -> None: ), ], ) - - -@pytest.mark.parametrize( - "mock_protocol, expected_call_count", - [(MagicMock(spec=SupportsDeploy), 1), (MagicMock(), 0)], -) -def test_system_deploy_data( - monkeypatch: Any, mock_protocol: MagicMock, expected_call_count: int -) -> None: - """Test deploy data functionality.""" - monkeypatch.setattr( - "mlia.backend.system.ProtocolFactory.get_protocol", - MagicMock(return_value=mock_protocol), - ) - - system = ControlledSystem(SystemConfig(name="test")) - system.deploy(Path("some_file"), "some_dest") - - assert mock_protocol.deploy.call_count == expected_call_count - - -@pytest.mark.parametrize( - "single_instance, controller_class", - ((False, SystemController), (True, SystemControllerSingleInstance)), -) -def test_get_controller(single_instance: bool, controller_class: type) -> None: - """Test function get_controller.""" - controller = get_controller(single_instance) - assert isinstance(controller, controller_class) diff --git a/tests/mlia/test_cli_commands.py b/tests/test_cli_commands.py index bf17339..bf17339 100644 --- a/tests/mlia/test_cli_commands.py +++ b/tests/test_cli_commands.py diff --git a/tests/mlia/test_cli_config.py b/tests/test_cli_config.py index 6d19eec..6d19eec 100644 --- a/tests/mlia/test_cli_config.py +++ b/tests/test_cli_config.py diff --git a/tests/mlia/test_cli_helpers.py b/tests/test_cli_helpers.py index 2c52885..2c52885 100644 --- a/tests/mlia/test_cli_helpers.py +++ b/tests/test_cli_helpers.py diff --git a/tests/mlia/test_cli_logging.py b/tests/test_cli_logging.py index 3f59cb6..5d26551 100644 --- a/tests/mlia/test_cli_logging.py +++ b/tests/test_cli_logging.py @@ -8,7 +8,7 @@ from typing import Optional import pytest from mlia.cli.logging import setup_logging -from tests.mlia.utils.logging import clear_loggers +from tests.utils.logging import clear_loggers def teardown_function() -> None: diff --git a/tests/mlia/test_cli_main.py b/tests/test_cli_main.py index a0937d5..28abc7b 100644 --- a/tests/mlia/test_cli_main.py +++ b/tests/test_cli_main.py @@ -17,7 +17,7 @@ import mlia from mlia.cli.main import CommandInfo from mlia.cli.main import main from mlia.core.context import ExecutionContext -from tests.mlia.utils.logging import clear_loggers +from tests.utils.logging import clear_loggers def teardown_function() -> None: diff --git a/tests/mlia/test_cli_options.py b/tests/test_cli_options.py index a441e58..a441e58 100644 --- a/tests/mlia/test_cli_options.py +++ b/tests/test_cli_options.py diff --git a/tests/mlia/test_core_advice_generation.py b/tests/test_core_advice_generation.py index 05db698..05db698 100644 --- a/tests/mlia/test_core_advice_generation.py +++ b/tests/test_core_advice_generation.py diff --git a/tests/mlia/test_core_advisor.py b/tests/test_core_advisor.py index 375ff62..375ff62 100644 --- a/tests/mlia/test_core_advisor.py +++ b/tests/test_core_advisor.py diff --git a/tests/mlia/test_core_context.py b/tests/test_core_context.py index 44eb976..44eb976 100644 --- a/tests/mlia/test_core_context.py +++ b/tests/test_core_context.py diff --git a/tests/mlia/test_core_data_analysis.py b/tests/test_core_data_analysis.py index a782159..a782159 100644 --- a/tests/mlia/test_core_data_analysis.py +++ b/tests/test_core_data_analysis.py diff --git a/tests/mlia/test_core_events.py b/tests/test_core_events.py index faaab7c..faaab7c 100644 --- a/tests/mlia/test_core_events.py +++ b/tests/test_core_events.py diff --git a/tests/mlia/test_core_helpers.py b/tests/test_core_helpers.py index 8577617..8577617 100644 --- a/tests/mlia/test_core_helpers.py +++ b/tests/test_core_helpers.py diff --git a/tests/mlia/test_core_mixins.py b/tests/test_core_mixins.py index d66213d..d66213d 100644 --- a/tests/mlia/test_core_mixins.py +++ b/tests/test_core_mixins.py diff --git a/tests/mlia/test_core_performance.py b/tests/test_core_performance.py index 0d28fe8..0d28fe8 100644 --- a/tests/mlia/test_core_performance.py +++ b/tests/test_core_performance.py diff --git a/tests/mlia/test_core_reporting.py b/tests/test_core_reporting.py index 2f7ec22..2f7ec22 100644 --- a/tests/mlia/test_core_reporting.py +++ b/tests/test_core_reporting.py diff --git a/tests/mlia/test_core_workflow.py b/tests/test_core_workflow.py index 470e572..470e572 100644 --- a/tests/mlia/test_core_workflow.py +++ b/tests/test_core_workflow.py diff --git a/tests/mlia/test_devices_ethosu_advice_generation.py b/tests/test_devices_ethosu_advice_generation.py index 5d37376..5d37376 100644 --- a/tests/mlia/test_devices_ethosu_advice_generation.py +++ b/tests/test_devices_ethosu_advice_generation.py diff --git a/tests/mlia/test_devices_ethosu_advisor.py b/tests/test_devices_ethosu_advisor.py index 74d2408..74d2408 100644 --- a/tests/mlia/test_devices_ethosu_advisor.py +++ b/tests/test_devices_ethosu_advisor.py diff --git a/tests/mlia/test_devices_ethosu_config.py b/tests/test_devices_ethosu_config.py index 49c999a..49c999a 100644 --- a/tests/mlia/test_devices_ethosu_config.py +++ b/tests/test_devices_ethosu_config.py diff --git a/tests/mlia/test_devices_ethosu_data_analysis.py b/tests/test_devices_ethosu_data_analysis.py index 4b1d38b..4b1d38b 100644 --- a/tests/mlia/test_devices_ethosu_data_analysis.py +++ b/tests/test_devices_ethosu_data_analysis.py diff --git a/tests/mlia/test_devices_ethosu_data_collection.py b/tests/test_devices_ethosu_data_collection.py index 897cf41..897cf41 100644 --- a/tests/mlia/test_devices_ethosu_data_collection.py +++ b/tests/test_devices_ethosu_data_collection.py diff --git a/tests/mlia/test_devices_ethosu_performance.py b/tests/test_devices_ethosu_performance.py index b3e5298..b3e5298 100644 --- a/tests/mlia/test_devices_ethosu_performance.py +++ b/tests/test_devices_ethosu_performance.py diff --git a/tests/mlia/test_devices_ethosu_reporters.py b/tests/test_devices_ethosu_reporters.py index 0da50e0..0da50e0 100644 --- a/tests/mlia/test_devices_ethosu_reporters.py +++ b/tests/test_devices_ethosu_reporters.py diff --git a/tests/mlia/test_nn_tensorflow_config.py b/tests/test_nn_tensorflow_config.py index 1ac9f97..1ac9f97 100644 --- a/tests/mlia/test_nn_tensorflow_config.py +++ b/tests/test_nn_tensorflow_config.py diff --git a/tests/mlia/test_nn_tensorflow_optimizations_clustering.py b/tests/test_nn_tensorflow_optimizations_clustering.py index 9bcf918..c12a1e8 100644 --- a/tests/mlia/test_nn_tensorflow_optimizations_clustering.py +++ b/tests/test_nn_tensorflow_optimizations_clustering.py @@ -16,8 +16,8 @@ from mlia.nn.tensorflow.tflite_metrics import ReportClusterMode from mlia.nn.tensorflow.tflite_metrics import TFLiteMetrics from mlia.nn.tensorflow.utils import convert_to_tflite from mlia.nn.tensorflow.utils import save_tflite_model -from tests.mlia.utils.common import get_dataset -from tests.mlia.utils.common import train_model +from tests.utils.common import get_dataset +from tests.utils.common import train_model def _prune_model( diff --git a/tests/mlia/test_nn_tensorflow_optimizations_pruning.py b/tests/test_nn_tensorflow_optimizations_pruning.py index 64030a6..5d92f5e 100644 --- a/tests/mlia/test_nn_tensorflow_optimizations_pruning.py +++ b/tests/test_nn_tensorflow_optimizations_pruning.py @@ -14,8 +14,8 @@ from mlia.nn.tensorflow.optimizations.pruning import PruningConfiguration from mlia.nn.tensorflow.tflite_metrics import TFLiteMetrics from mlia.nn.tensorflow.utils import convert_to_tflite from mlia.nn.tensorflow.utils import save_tflite_model -from tests.mlia.utils.common import get_dataset -from tests.mlia.utils.common import train_model +from tests.utils.common import get_dataset +from tests.utils.common import train_model def _test_sparsity( diff --git a/tests/mlia/test_nn_tensorflow_optimizations_select.py b/tests/test_nn_tensorflow_optimizations_select.py index 5cac8ba..5cac8ba 100644 --- a/tests/mlia/test_nn_tensorflow_optimizations_select.py +++ b/tests/test_nn_tensorflow_optimizations_select.py diff --git a/tests/mlia/test_nn_tensorflow_tflite_metrics.py b/tests/test_nn_tensorflow_tflite_metrics.py index cf7aaeb..00eacef 100644 --- a/tests/mlia/test_nn_tensorflow_tflite_metrics.py +++ b/tests/test_nn_tensorflow_tflite_metrics.py @@ -77,9 +77,7 @@ class TestTFLiteMetrics: # Check sparsity calculation sparsity_per_layer = metrics.sparsity_per_layer() for name, sparsity in sparsity_per_layer.items(): - assert isclose(sparsity, 0.5), "Layer '{}' has incorrect sparsity.".format( - name - ) + assert isclose(sparsity, 0.5), f"Layer '{name}' has incorrect sparsity." assert isclose(metrics.sparsity_overall(), 0.5) @staticmethod @@ -95,7 +93,7 @@ class TestTFLiteMetrics: for num_unique in num_unique_per_axis: assert ( num_unique == 2 - ), "Layer '{}' has incorrect number of clusters.".format(name) + ), f"Layer '{name}' has incorrect number of clusters." # NUM_CLUSTERS_HISTOGRAM hists = metrics.num_unique_weights(ReportClusterMode.NUM_CLUSTERS_HISTOGRAM) assert hists @@ -105,15 +103,13 @@ class TestTFLiteMetrics: # The histogram starts with the bin for for num_clusters == 1 num_clusters = idx + 1 msg = ( - "Histogram of layer '{}': There are {} axes with {} " - "clusters".format(name, num_axes, num_clusters) + f"Histogram of layer '{name}': There are {num_axes} axes " + f"with {num_clusters} clusters" ) if num_clusters == 2: - assert num_axes > 0, "{}, but there should be at least one.".format( - msg - ) + assert num_axes > 0, f"{msg}, but there should be at least one." else: - assert num_axes == 0, "{}, but there should be none.".format(msg) + assert num_axes == 0, f"{msg}, but there should be none." @staticmethod @pytest.mark.parametrize("report_sparsity", (False, True)) diff --git a/tests/mlia/test_nn_tensorflow_utils.py b/tests/test_nn_tensorflow_utils.py index 6d27299..6d27299 100644 --- a/tests/mlia/test_nn_tensorflow_utils.py +++ b/tests/test_nn_tensorflow_utils.py diff --git a/tests/mlia/test_resources/application_config.json b/tests/test_resources/application_config.json index 2dfcfec..8c5d2b5 100644 --- a/tests/mlia/test_resources/application_config.json +++ b/tests/test_resources/application_config.json @@ -6,7 +6,6 @@ "system_1", "system_2" ], - "build_dir": "build_dir_11", "commands": { "clean": [ "clean_cmd_11" @@ -56,7 +55,6 @@ "supported_systems": [ "system_2" ], - "build_dir": "build_dir_21", "commands": { "clean": [ "clean_cmd_21" diff --git a/tests/mlia/test_resources/application_config.json.license b/tests/test_resources/application_config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/application_config.json.license +++ b/tests/test_resources/application_config.json.license diff --git a/tests/mlia/test_resources/backends/applications/application1/aiet-config.json b/tests/test_resources/backends/applications/application1/backend-config.json index 97f0401..96d5420 100644 --- a/tests/mlia/test_resources/backends/applications/application1/aiet-config.json +++ b/tests/test_resources/backends/applications/application1/backend-config.json @@ -7,7 +7,6 @@ "name": "System 1" } ], - "build_dir": "build", "commands": { "clean": [ "echo 'clean'" diff --git a/tests/mlia/test_resources/backends/applications/application1/aiet-config.json.license b/tests/test_resources/backends/applications/application1/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/backends/applications/application1/aiet-config.json.license +++ b/tests/test_resources/backends/applications/application1/backend-config.json.license diff --git a/tests/mlia/test_resources/backends/applications/application2/aiet-config.json b/tests/test_resources/backends/applications/application2/backend-config.json index e9122d3..3a3969a 100644 --- a/tests/mlia/test_resources/backends/applications/application2/aiet-config.json +++ b/tests/test_resources/backends/applications/application2/backend-config.json @@ -7,7 +7,6 @@ "name": "System 2" } ], - "build_dir": "build", "commands": { "clean": [ "echo 'clean'" diff --git a/tests/mlia/test_resources/backends/applications/application2/aiet-config.json.license b/tests/test_resources/backends/applications/application2/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/backends/applications/application2/aiet-config.json.license +++ b/tests/test_resources/backends/applications/application2/backend-config.json.license diff --git a/tests/mlia/test_resources/backends/applications/application3/readme.txt b/tests/test_resources/backends/applications/application3/readme.txt index 8c72c05..8c72c05 100644 --- a/tests/mlia/test_resources/backends/applications/application3/readme.txt +++ b/tests/test_resources/backends/applications/application3/readme.txt diff --git a/tests/mlia/test_resources/backends/applications/application4/aiet-config.json b/tests/test_resources/backends/applications/application4/backend-config.json index ffb5746..d4362be 100644 --- a/tests/mlia/test_resources/backends/applications/application4/aiet-config.json +++ b/tests/test_resources/backends/applications/application4/backend-config.json @@ -2,23 +2,23 @@ { "name": "application_4", "description": "This is application 4", - "build_dir": "build", + "variables": { + "build_dir": "build" + }, "supported_systems": [ { "name": "System 4" } ], "commands": { - "build": [ - "cp ../hello_app.txt .", - "echo '{user_params:0}' > params.txt" - ], "run": [ - "cat {application.build_dir}/hello_app.txt" + "cp {application.config_dir}/hello_app.txt {system.config_dir}", + "echo '{user_params:0}' > {system.config_dir}/params.txt", + "cat hello_app.txt" ] }, "user_params": { - "build": [ + "run": [ { "name": "--app", "description": "Sample command param", @@ -29,8 +29,7 @@ ], "default_value": "application1" } - ], - "run": [] + ] } } ] diff --git a/tests/mlia/test_resources/backends/applications/application4/aiet-config.json.license b/tests/test_resources/backends/applications/application4/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/backends/applications/application4/aiet-config.json.license +++ b/tests/test_resources/backends/applications/application4/backend-config.json.license diff --git a/tests/mlia/test_resources/backends/applications/application4/hello_app.txt b/tests/test_resources/backends/applications/application4/hello_app.txt index 2ec0d1d..2ec0d1d 100644 --- a/tests/mlia/test_resources/backends/applications/application4/hello_app.txt +++ b/tests/test_resources/backends/applications/application4/hello_app.txt diff --git a/tests/mlia/test_resources/backends/applications/application5/aiet-config.json b/tests/test_resources/backends/applications/application5/backend-config.json index 5269409..219494c 100644 --- a/tests/mlia/test_resources/backends/applications/application5/aiet-config.json +++ b/tests/test_resources/backends/applications/application5/backend-config.json @@ -2,11 +2,9 @@ { "name": "application_5", "description": "This is application 5", - "build_dir": "default_build_dir", "supported_systems": [ { - "name": "System 1", - "lock": false + "name": "System 1" }, { "name": "System 2" @@ -16,7 +14,6 @@ "var1": "value1", "var2": "value2" }, - "lock": true, "commands": { "build": [ "default build command" @@ -36,7 +33,6 @@ "supported_systems": [ { "name": "System 1", - "build_dir": "build_5A", "variables": { "var1": "new value1" } @@ -46,7 +42,6 @@ "variables": { "var2": "new value2" }, - "lock": true, "commands": { "run": [ "run command on system 2" @@ -58,7 +53,6 @@ "var1": "value1", "var2": "value2" }, - "build_dir": "build", "commands": { "build": [ "default build command" @@ -78,25 +72,9 @@ "supported_systems": [ { "name": "System 1", - "build_dir": "build_5B", "variables": { "var1": "value for var1 System1", "var2": "value for var2 System1" - }, - "user_params": { - "build": [ - { - "name": "--param_5B", - "description": "Sample command param", - "values": [ - "value1", - "value2", - "value3" - ], - "default_value": "value1", - "alias": "param1" - } - ] } }, { @@ -106,9 +84,6 @@ "var2": "value for var2 System2" }, "commands": { - "build": [ - "build command on system 2 with {variables:var1} {user_params:param1}" - ], "run": [ "run command on system 2" ] @@ -118,7 +93,6 @@ } } ], - "build_dir": "build", "commands": { "build": [ "default build command with {variables:var1}" diff --git a/tests/mlia/test_resources/backends/applications/application5/aiet-config.json.license b/tests/test_resources/backends/applications/application5/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/backends/applications/application5/aiet-config.json.license +++ b/tests/test_resources/backends/applications/application5/backend-config.json.license diff --git a/tests/mlia/test_resources/backends/applications/application6/aiet-config.json b/tests/test_resources/backends/applications/application6/backend-config.json index 56ad807..81afebd 100644 --- a/tests/mlia/test_resources/backends/applications/application6/aiet-config.json +++ b/tests/test_resources/backends/applications/application6/backend-config.json @@ -7,7 +7,6 @@ "name": "System 6" } ], - "build_dir": "build", "commands": { "clean": [ "echo 'clean'" diff --git a/tests/mlia/test_resources/backends/applications/application6/aiet-config.json.license b/tests/test_resources/backends/applications/application6/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/backends/applications/application6/aiet-config.json.license +++ b/tests/test_resources/backends/applications/application6/backend-config.json.license diff --git a/tests/mlia/test_resources/backends/applications/readme.txt b/tests/test_resources/backends/applications/readme.txt index a1f8209..a1f8209 100644 --- a/tests/mlia/test_resources/backends/applications/readme.txt +++ b/tests/test_resources/backends/applications/readme.txt diff --git a/tests/mlia/test_resources/backends/systems/system1/aiet-config.json b/tests/test_resources/backends/systems/system1/backend-config.json index 4b5dd19..4454695 100644 --- a/tests/mlia/test_resources/backends/systems/system1/aiet-config.json +++ b/tests/test_resources/backends/systems/system1/backend-config.json @@ -2,14 +2,6 @@ { "name": "System 1", "description": "This is system 1", - "build_dir": "build", - "data_transfer": { - "protocol": "ssh", - "username": "root", - "password": "root", - "hostname": "localhost", - "port": "8021" - }, "commands": { "clean": [ "echo 'clean'" @@ -22,9 +14,6 @@ ], "post_run": [ "echo 'post_run'" - ], - "deploy": [ - "echo 'deploy'" ] }, "user_params": { diff --git a/tests/mlia/test_resources/backends/systems/system1/aiet-config.json.license b/tests/test_resources/backends/systems/system1/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/backends/systems/system1/aiet-config.json.license +++ b/tests/test_resources/backends/systems/system1/backend-config.json.license diff --git a/tests/mlia/test_resources/backends/systems/system1/system_artifact/dummy.txt b/tests/test_resources/backends/systems/system1/system_artifact/dummy.txt index 487e9d8..487e9d8 100644 --- a/tests/mlia/test_resources/backends/systems/system1/system_artifact/dummy.txt +++ b/tests/test_resources/backends/systems/system1/system_artifact/dummy.txt diff --git a/tests/mlia/test_resources/backends/systems/system2/aiet-config.json b/tests/test_resources/backends/systems/system2/backend-config.json index a9e0eb3..3359d3d 100644 --- a/tests/mlia/test_resources/backends/systems/system2/aiet-config.json +++ b/tests/test_resources/backends/systems/system2/backend-config.json @@ -2,14 +2,6 @@ { "name": "System 2", "description": "This is system 2", - "build_dir": "build", - "data_transfer": { - "protocol": "ssh", - "username": "root", - "password": "root", - "hostname": "localhost", - "port": "8021" - }, "commands": { "clean": [ "echo 'clean'" diff --git a/tests/mlia/test_resources/backends/systems/system2/aiet-config.json.license b/tests/test_resources/backends/systems/system2/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/backends/systems/system2/aiet-config.json.license +++ b/tests/test_resources/backends/systems/system2/backend-config.json.license diff --git a/tests/mlia/test_resources/backends/systems/system3/readme.txt b/tests/test_resources/backends/systems/system3/readme.txt index aba5a9c..aba5a9c 100644 --- a/tests/mlia/test_resources/backends/systems/system3/readme.txt +++ b/tests/test_resources/backends/systems/system3/readme.txt diff --git a/tests/mlia/test_resources/backends/systems/system4/aiet-config.json b/tests/test_resources/backends/systems/system4/backend-config.json index 7b13160..7701c05 100644 --- a/tests/mlia/test_resources/backends/systems/system4/aiet-config.json +++ b/tests/test_resources/backends/systems/system4/backend-config.json @@ -2,14 +2,12 @@ { "name": "System 4", "description": "This is system 4", - "build_dir": "build", - "data_transfer": { - "protocol": "local" - }, "commands": { "run": [ "echo {application.name}", - "{application.commands.run:0}" + "{application.commands.run:0}", + "{application.commands.run:1}", + "{application.commands.run:2}" ] }, "user_params": { diff --git a/tests/mlia/test_resources/backends/systems/system4/aiet-config.json.license b/tests/test_resources/backends/systems/system4/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/backends/systems/system4/aiet-config.json.license +++ b/tests/test_resources/backends/systems/system4/backend-config.json.license diff --git a/tests/mlia/test_resources/backends/systems/system6/aiet-config.json b/tests/test_resources/backends/systems/system6/backend-config.json index 4242f64..5180799 100644 --- a/tests/mlia/test_resources/backends/systems/system6/aiet-config.json +++ b/tests/test_resources/backends/systems/system6/backend-config.json @@ -2,10 +2,6 @@ { "name": "System 6", "description": "This is system 6", - "build_dir": "build", - "data_transfer": { - "protocol": "local" - }, "variables": { "var1": "{user_params:sys-param1}" }, diff --git a/tests/mlia/test_resources/backends/systems/system6/aiet-config.json.license b/tests/test_resources/backends/systems/system6/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/backends/systems/system6/aiet-config.json.license +++ b/tests/test_resources/backends/systems/system6/backend-config.json.license diff --git a/tests/mlia/test_resources/hello_world.json b/tests/test_resources/hello_world.json index 8a9a448..99e9439 100644 --- a/tests/mlia/test_resources/hello_world.json +++ b/tests/test_resources/hello_world.json @@ -5,7 +5,6 @@ "supported_systems": [ "Dummy System" ], - "build_dir": "build", "deploy_data": [ [ "src", diff --git a/tests/mlia/test_resources/hello_world.json.license b/tests/test_resources/hello_world.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/hello_world.json.license +++ b/tests/test_resources/hello_world.json.license diff --git a/tests/mlia/test_resources/scripts/test_backend_run b/tests/test_resources/scripts/test_backend_run index 548f577..548f577 100755 --- a/tests/mlia/test_resources/scripts/test_backend_run +++ b/tests/test_resources/scripts/test_backend_run diff --git a/tests/mlia/test_resources/scripts/test_backend_run_script.sh b/tests/test_resources/scripts/test_backend_run_script.sh index 548f577..548f577 100644 --- a/tests/mlia/test_resources/scripts/test_backend_run_script.sh +++ b/tests/test_resources/scripts/test_backend_run_script.sh diff --git a/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json b/tests/test_resources/various/applications/application_with_empty_config/backend-config.json index fe51488..fe51488 100644 --- a/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json +++ b/tests/test_resources/various/applications/application_with_empty_config/backend-config.json diff --git a/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json.license b/tests/test_resources/various/applications/application_with_empty_config/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json.license +++ b/tests/test_resources/various/applications/application_with_empty_config/backend-config.json.license diff --git a/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json b/tests/test_resources/various/applications/application_with_valid_config/backend-config.json index ff1cf1a..a457d9b 100644 --- a/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json +++ b/tests/test_resources/various/applications/application_with_valid_config/backend-config.json @@ -2,7 +2,9 @@ { "name": "test_application", "description": "This is test_application", - "build_dir": "build", + "variables": { + "build_dir": "build" + }, "supported_systems": [ { "name": "System 4" @@ -13,7 +15,7 @@ "cp ../hello_app.txt ." ], "run": [ - "{application.build_dir}/hello_app.txt" + "{application.variables:build_dir}/hello_app.txt" ] }, "user_params": { diff --git a/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json.license b/tests/test_resources/various/applications/application_with_valid_config/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json.license +++ b/tests/test_resources/various/applications/application_with_valid_config/backend-config.json.license diff --git a/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json b/tests/test_resources/various/applications/application_with_wrong_config1/backend-config.json index 724b31b..724b31b 100644 --- a/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json +++ b/tests/test_resources/various/applications/application_with_wrong_config1/backend-config.json diff --git a/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license b/tests/test_resources/various/applications/application_with_wrong_config1/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license +++ b/tests/test_resources/various/applications/application_with_wrong_config1/backend-config.json.license diff --git a/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json b/tests/test_resources/various/applications/application_with_wrong_config2/backend-config.json index 1ebb29c..b64e6f8 100644 --- a/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json +++ b/tests/test_resources/various/applications/application_with_wrong_config2/backend-config.json @@ -2,13 +2,15 @@ { "name": "test_application", "description": "This is test_application", - "build_dir": "build", + "variables": { + "build_dir": "build" + }, "commands": { "build": [ "cp ../hello_app.txt ." ], "run": [ - "{application.build_dir}/hello_app.txt" + "{application.variables:build_dir}/hello_app.txt" ] }, "user_params": { diff --git a/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license b/tests/test_resources/various/applications/application_with_wrong_config2/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license +++ b/tests/test_resources/various/applications/application_with_wrong_config2/backend-config.json.license diff --git a/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json b/tests/test_resources/various/applications/application_with_wrong_config3/backend-config.json index 410d12d..4a70cdd 100644 --- a/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json +++ b/tests/test_resources/various/applications/application_with_wrong_config3/backend-config.json @@ -2,7 +2,9 @@ { "name": "test_application", "description": "This is test_application", - "build_dir": "build", + "variables": { + "build_dir": "build" + }, "supported_systems": [ { "anme": "System 4" @@ -13,7 +15,7 @@ "cp ../hello_app.txt ." ], "run": [ - "{application.build_dir}/hello_app.txt" + "{application.variables:build_dir}/hello_app.txt" ] }, "user_params": { diff --git a/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license b/tests/test_resources/various/applications/application_with_wrong_config3/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license +++ b/tests/test_resources/various/applications/application_with_wrong_config3/backend-config.json.license diff --git a/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json b/tests/test_resources/various/systems/system_with_empty_config/backend-config.json index fe51488..fe51488 100644 --- a/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json +++ b/tests/test_resources/various/systems/system_with_empty_config/backend-config.json diff --git a/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json.license b/tests/test_resources/various/systems/system_with_empty_config/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json.license +++ b/tests/test_resources/various/systems/system_with_empty_config/backend-config.json.license diff --git a/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json b/tests/test_resources/various/systems/system_with_valid_config/backend-config.json index 20142e9..83c3025 100644 --- a/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json +++ b/tests/test_resources/various/systems/system_with_valid_config/backend-config.json @@ -2,10 +2,6 @@ { "name": "Test system", "description": "This is a test system", - "build_dir": "build", - "data_transfer": { - "protocol": "local" - }, "commands": { "run": [] }, diff --git a/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json.license b/tests/test_resources/various/systems/system_with_valid_config/backend-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json.license +++ b/tests/test_resources/various/systems/system_with_valid_config/backend-config.json.license diff --git a/tests/mlia/test_resources/vela/sample_vela.ini b/tests/test_resources/vela/sample_vela.ini index c992458..c992458 100644 --- a/tests/mlia/test_resources/vela/sample_vela.ini +++ b/tests/test_resources/vela/sample_vela.ini diff --git a/tests/mlia/test_tools_metadata_common.py b/tests/test_tools_metadata_common.py index 7663b83..7663b83 100644 --- a/tests/mlia/test_tools_metadata_common.py +++ b/tests/test_tools_metadata_common.py diff --git a/tests/mlia/test_tools_metadata_corstone.py b/tests/test_tools_metadata_corstone.py index 017d0c7..017d0c7 100644 --- a/tests/mlia/test_tools_metadata_corstone.py +++ b/tests/test_tools_metadata_corstone.py diff --git a/tests/mlia/test_tools_vela_wrapper.py b/tests/test_tools_vela_wrapper.py index 875d2ff..0efcb0f 100644 --- a/tests/mlia/test_tools_vela_wrapper.py +++ b/tests/test_tools_vela_wrapper.py @@ -20,7 +20,7 @@ from mlia.tools.vela_wrapper import PerformanceMetrics from mlia.tools.vela_wrapper import supported_operators from mlia.tools.vela_wrapper import VelaCompiler from mlia.tools.vela_wrapper import VelaCompilerOptions -from mlia.utils.proc import working_directory +from mlia.utils.filesystem import working_directory def test_default_vela_compiler() -> None: diff --git a/tests/mlia/test_utils_console.py b/tests/test_utils_console.py index 36975f8..36975f8 100644 --- a/tests/mlia/test_utils_console.py +++ b/tests/test_utils_console.py diff --git a/tests/mlia/test_utils_download.py b/tests/test_utils_download.py index 4f8e2dc..4f8e2dc 100644 --- a/tests/mlia/test_utils_download.py +++ b/tests/test_utils_download.py diff --git a/tests/mlia/test_utils_filesystem.py b/tests/test_utils_filesystem.py index 4d8d955..7cf32e7 100644 --- a/tests/mlia/test_utils_filesystem.py +++ b/tests/test_utils_filesystem.py @@ -20,6 +20,7 @@ from mlia.utils.filesystem import get_vela_config from mlia.utils.filesystem import sha256 from mlia.utils.filesystem import temp_directory from mlia.utils.filesystem import temp_file +from mlia.utils.filesystem import working_directory def test_get_mlia_resources() -> None: @@ -164,3 +165,27 @@ def test_copy_all(tmp_path: Path) -> None: assert (dest_dir / sample.name).is_file() assert (dest_dir / sample_nested_file.name).is_file() + + +@pytest.mark.parametrize( + "should_exist, create_dir", + [ + [True, False], + [False, True], + ], +) +def test_working_directory_context_manager( + tmp_path: Path, should_exist: bool, create_dir: bool +) -> None: + """Test working_directory context manager.""" + prev_wd = Path.cwd() + + working_dir = tmp_path / "work_dir" + if should_exist: + working_dir.mkdir() + + with working_directory(working_dir, create_dir=create_dir) as current_working_dir: + assert current_working_dir.is_dir() + assert Path.cwd() == current_working_dir + + assert Path.cwd() == prev_wd diff --git a/tests/mlia/test_utils_logging.py b/tests/test_utils_logging.py index 75ebceb..75ebceb 100644 --- a/tests/mlia/test_utils_logging.py +++ b/tests/test_utils_logging.py diff --git a/tests/mlia/test_utils_misc.py b/tests/test_utils_misc.py index 011d09e..011d09e 100644 --- a/tests/mlia/test_utils_misc.py +++ b/tests/test_utils_misc.py diff --git a/tests/mlia/test_utils_types.py b/tests/test_utils_types.py index 4909efe..4909efe 100644 --- a/tests/mlia/test_utils_types.py +++ b/tests/test_utils_types.py diff --git a/tests/mlia/utils/__init__.py b/tests/utils/__init__.py index 27166ef..27166ef 100644 --- a/tests/mlia/utils/__init__.py +++ b/tests/utils/__init__.py diff --git a/tests/mlia/utils/common.py b/tests/utils/common.py index 932343e..932343e 100644 --- a/tests/mlia/utils/common.py +++ b/tests/utils/common.py diff --git a/tests/mlia/utils/logging.py b/tests/utils/logging.py index d223fb2..d223fb2 100644 --- a/tests/mlia/utils/logging.py +++ b/tests/utils/logging.py |