From 3e3dcb9bd5abb88adcd85b4f89e8a81e7f6fa293 Mon Sep 17 00:00:00 2001 From: Dmitrii Agibov Date: Fri, 27 Jan 2023 09:12:50 +0000 Subject: MLIA-595 Remove old backend configuration mechanism - Remove old backend configuration code - Install backends into directory ~/.mlia - Rename targets/backends in registry to make it consistent across codebase. Change-Id: I9c8b012fe863280f1c692940c0dcad3ef638aaae --- tests/conftest.py | 74 --- tests/test_backend_corstone.py | 11 + tests/test_backend_corstone_install.py | 483 +---------------- tests/test_backend_corstone_performance.py | 589 +++++---------------- tests/test_backend_executor_application.py | 422 --------------- tests/test_backend_executor_common.py | 482 ----------------- tests/test_backend_executor_execution.py | 212 -------- tests/test_backend_executor_fs.py | 138 ----- tests/test_backend_executor_output_consumer.py | 100 ---- tests/test_backend_executor_proc.py | 190 ------- tests/test_backend_executor_runner.py | 254 --------- tests/test_backend_executor_source.py | 205 ------- tests/test_backend_executor_system.py | 358 ------------- tests/test_backend_install.py | 240 ++++++--- tests/test_backend_manager.py | 59 ++- tests/test_backend_registry.py | 20 +- tests/test_backend_repo.py | 140 +++++ tests/test_cli_command_validators.py | 16 +- tests/test_cli_config.py | 10 +- tests/test_resources/application_config.json | 94 ---- .../test_resources/application_config.json.license | 3 - .../applications/application1/backend-config.json | 29 - .../application1/backend-config.json.license | 3 - .../applications/application2/backend-config.json | 29 - .../application2/backend-config.json.license | 3 - .../backends/applications/application3/readme.txt | 4 - .../applications/application4/backend-config.json | 33 -- .../application4/backend-config.json.license | 3 - .../applications/application5/backend-config.json | 134 ----- .../application5/backend-config.json.license | 3 - .../applications/application6/backend-config.json | 41 -- .../application6/backend-config.json.license | 3 - .../backends/applications/readme.txt | 4 - .../backends/systems/system1/backend-config.json | 24 - .../systems/system1/backend-config.json.license | 3 - .../systems/system1/system_artifact/empty.txt | 2 - .../backends/systems/system2/backend-config.json | 24 - .../systems/system2/backend-config.json.license | 3 - .../backends/systems/system3/readme.txt | 4 - .../backends/systems/system4/backend-config.json | 15 - .../systems/system4/backend-config.json.license | 3 - .../backends/systems/system6/backend-config.json | 30 -- .../systems/system6/backend-config.json.license | 3 - tests/test_resources/hello_world.json | 53 -- tests/test_resources/hello_world.json.license | 3 - tests/test_resources/scripts/test_backend_run | 8 - .../scripts/test_backend_run_script.sh | 8 - .../backend-config.json | 1 - .../backend-config.json.license | 3 - .../backend-config.json | 37 -- .../backend-config.json.license | 3 - .../backend-config.json | 2 - .../backend-config.json.license | 3 - .../backend-config.json | 32 -- .../backend-config.json.license | 3 - .../backend-config.json | 37 -- .../backend-config.json.license | 3 - .../system_with_empty_config/backend-config.json | 1 - .../backend-config.json.license | 3 - .../system_with_valid_config/backend-config.json | 12 - .../backend-config.json.license | 3 - tests/test_target_registry.py | 36 +- tests/test_utils_proc.py | 17 + 63 files changed, 603 insertions(+), 4165 deletions(-) create mode 100644 tests/test_backend_corstone.py delete mode 100644 tests/test_backend_executor_application.py delete mode 100644 tests/test_backend_executor_common.py delete mode 100644 tests/test_backend_executor_execution.py delete mode 100644 tests/test_backend_executor_fs.py delete mode 100644 tests/test_backend_executor_output_consumer.py delete mode 100644 tests/test_backend_executor_proc.py delete mode 100644 tests/test_backend_executor_runner.py delete mode 100644 tests/test_backend_executor_source.py delete mode 100644 tests/test_backend_executor_system.py create mode 100644 tests/test_backend_repo.py delete mode 100644 tests/test_resources/application_config.json delete mode 100644 tests/test_resources/application_config.json.license delete mode 100644 tests/test_resources/backends/applications/application1/backend-config.json delete mode 100644 tests/test_resources/backends/applications/application1/backend-config.json.license delete mode 100644 tests/test_resources/backends/applications/application2/backend-config.json delete mode 100644 tests/test_resources/backends/applications/application2/backend-config.json.license delete mode 100644 tests/test_resources/backends/applications/application3/readme.txt delete mode 100644 tests/test_resources/backends/applications/application4/backend-config.json delete mode 100644 tests/test_resources/backends/applications/application4/backend-config.json.license delete mode 100644 tests/test_resources/backends/applications/application5/backend-config.json delete mode 100644 tests/test_resources/backends/applications/application5/backend-config.json.license delete mode 100644 tests/test_resources/backends/applications/application6/backend-config.json delete mode 100644 tests/test_resources/backends/applications/application6/backend-config.json.license delete mode 100644 tests/test_resources/backends/applications/readme.txt delete mode 100644 tests/test_resources/backends/systems/system1/backend-config.json delete mode 100644 tests/test_resources/backends/systems/system1/backend-config.json.license delete mode 100644 tests/test_resources/backends/systems/system1/system_artifact/empty.txt delete mode 100644 tests/test_resources/backends/systems/system2/backend-config.json delete mode 100644 tests/test_resources/backends/systems/system2/backend-config.json.license delete mode 100644 tests/test_resources/backends/systems/system3/readme.txt delete mode 100644 tests/test_resources/backends/systems/system4/backend-config.json delete mode 100644 tests/test_resources/backends/systems/system4/backend-config.json.license delete mode 100644 tests/test_resources/backends/systems/system6/backend-config.json delete mode 100644 tests/test_resources/backends/systems/system6/backend-config.json.license delete mode 100644 tests/test_resources/hello_world.json delete mode 100644 tests/test_resources/hello_world.json.license delete mode 100755 tests/test_resources/scripts/test_backend_run delete mode 100644 tests/test_resources/scripts/test_backend_run_script.sh delete mode 100644 tests/test_resources/various/applications/application_with_empty_config/backend-config.json delete mode 100644 tests/test_resources/various/applications/application_with_empty_config/backend-config.json.license delete mode 100644 tests/test_resources/various/applications/application_with_valid_config/backend-config.json delete mode 100644 tests/test_resources/various/applications/application_with_valid_config/backend-config.json.license delete mode 100644 tests/test_resources/various/applications/application_with_wrong_config1/backend-config.json delete mode 100644 tests/test_resources/various/applications/application_with_wrong_config1/backend-config.json.license delete mode 100644 tests/test_resources/various/applications/application_with_wrong_config2/backend-config.json delete mode 100644 tests/test_resources/various/applications/application_with_wrong_config2/backend-config.json.license delete mode 100644 tests/test_resources/various/applications/application_with_wrong_config3/backend-config.json delete mode 100644 tests/test_resources/various/applications/application_with_wrong_config3/backend-config.json.license delete mode 100644 tests/test_resources/various/systems/system_with_empty_config/backend-config.json delete mode 100644 tests/test_resources/various/systems/system_with_empty_config/backend-config.json.license delete mode 100644 tests/test_resources/various/systems/system_with_valid_config/backend-config.json delete mode 100644 tests/test_resources/various/systems/system_with_valid_config/backend-config.json.license create mode 100644 tests/test_utils_proc.py (limited to 'tests') diff --git a/tests/conftest.py b/tests/conftest.py index 67549e7..d797869 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,9 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Pytest conf module.""" import shutil -import tarfile from pathlib import Path -from typing import Any from typing import Generator import pytest @@ -30,18 +28,6 @@ def fixture_sample_context(tmpdir: str) -> ExecutionContext: return ExecutionContext(output_dir=tmpdir) -@pytest.fixture(scope="session") -def test_systems_path(test_resources_path: Path) -> Path: - """Return test systems path in a pytest fixture.""" - return test_resources_path / "backends" / "systems" - - -@pytest.fixture(scope="session") -def test_applications_path(test_resources_path: Path) -> Path: - """Return test applications path in a pytest fixture.""" - return test_resources_path / "backends" / "applications" - - @pytest.fixture(scope="session") def non_optimised_input_model_file(test_tflite_model: Path) -> Path: """Provide the path to a quantized test model file.""" @@ -60,66 +46,6 @@ def invalid_input_model_file(test_tflite_invalid_model: Path) -> Path: return test_tflite_invalid_model -@pytest.fixture(autouse=True) -def test_resources(monkeypatch: pytest.MonkeyPatch, test_resources_path: Path) -> Any: - """Force using test resources as middleware's repository.""" - - def get_test_resources() -> Path: - """Return path to the test resources.""" - return test_resources_path / "backends" - - monkeypatch.setattr( - "mlia.backend.executor.fs.get_backend_resources", get_test_resources - ) - yield - - -def create_archive( - archive_name: str, source: Path, destination: Path, with_root_folder: bool = False -) -> None: - """Create archive from directory source.""" - with tarfile.open(destination / archive_name, mode="w:gz") as tar: - for item in source.iterdir(): - item_name = item.name - if with_root_folder: - item_name = f"{source.name}/{item_name}" - tar.add(item, item_name) - - -def process_directory(source: Path, destination: Path) -> None: - """Process resource directory.""" - destination.mkdir() - - for item in source.iterdir(): - if item.is_dir(): - create_archive(f"{item.name}.tar.gz", item, destination) - create_archive(f"{item.name}_dir.tar.gz", item, destination, True) - - -@pytest.fixture(scope="session", autouse=True) -def add_archives( - test_resources_path: Path, tmp_path_factory: pytest.TempPathFactory -) -> Any: - """Generate archives of the test resources.""" - tmp_path = tmp_path_factory.mktemp("archives") - - archives_path = tmp_path / "archives" - archives_path.mkdir() - - if (archives_path_link := test_resources_path / "archives").is_symlink(): - archives_path_link.unlink() - - archives_path_link.symlink_to(archives_path, target_is_directory=True) - - for item in ["applications", "systems"]: - process_directory(test_resources_path / "backends" / item, archives_path / item) - - yield - - archives_path_link.unlink() - shutil.rmtree(tmp_path) - - def get_test_keras_model() -> tf.keras.Model: """Return test Keras model.""" model = tf.keras.Sequential( diff --git a/tests/test_backend_corstone.py b/tests/test_backend_corstone.py new file mode 100644 index 0000000..29ef084 --- /dev/null +++ b/tests/test_backend_corstone.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright 2023, Arm Limited and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for Corstone backend.""" +from mlia.backend.corstone import is_corstone_backend + + +def test_is_corstone_backend() -> None: + """Test function is_corstone_backend.""" + assert is_corstone_backend("Corstone-300") is True + assert is_corstone_backend("Corstone-310") is True + assert is_corstone_backend("New backend") is False diff --git a/tests/test_backend_corstone_install.py b/tests/test_backend_corstone_install.py index 3b05a49..b9e6569 100644 --- a/tests/test_backend_corstone_install.py +++ b/tests/test_backend_corstone_install.py @@ -1,490 +1,63 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for Corstone related installation functions..""" from __future__ import annotations -import tarfile from pathlib import Path -from typing import Iterable +from typing import Any +from unittest.mock import call from unittest.mock import MagicMock import pytest from mlia.backend.corstone.install import Corstone300Installer -from mlia.backend.corstone.install import get_corstone_300_installation -from mlia.backend.corstone.install import get_corstone_310_installation from mlia.backend.corstone.install import get_corstone_installations -from mlia.backend.corstone.install import PackagePathChecker -from mlia.backend.corstone.install import StaticPathChecker -from mlia.backend.executor.runner import BackendRunner -from mlia.backend.install import BackendInfo -from mlia.backend.install import BackendInstallation -from mlia.backend.install import BackendInstaller -from mlia.backend.install import BackendMetadata -from mlia.backend.install import CompoundPathChecker -from mlia.backend.install import DownloadAndInstall -from mlia.backend.install import InstallFromPath -from mlia.backend.install import PathChecker - - -@pytest.fixture(name="test_mlia_resources") -def fixture_test_mlia_resources( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> Path: - """Redirect MLIA resources resolution to the temp directory.""" - mlia_resources = tmp_path / "resources" - mlia_resources.mkdir() - - monkeypatch.setattr( - "mlia.backend.install.get_mlia_resources", - MagicMock(return_value=mlia_resources), - ) - - return mlia_resources - - -def get_backend_installation( # pylint: disable=too-many-arguments - backend_runner_mock: MagicMock = MagicMock(), - name: str = "test_name", - description: str = "test_description", - download_artifact: MagicMock | None = None, - path_checker: PathChecker = MagicMock(), - apps_resources: list[str] | None = None, - system_config: str | None = None, - backend_installer: BackendInstaller = MagicMock(), - supported_platforms: list[str] | None = None, -) -> BackendInstallation: - """Get backend installation.""" - return BackendInstallation( - backend_runner=backend_runner_mock, - metadata=BackendMetadata( - name=name, - description=description, - system_config=system_config or "", - apps_resources=apps_resources or [], - fvp_dir_name="sample_dir", - download_artifact=download_artifact, - supported_platforms=supported_platforms, - ), - path_checker=path_checker, - backend_installer=backend_installer, - ) - - -@pytest.mark.parametrize( - "platform, supported_platforms, expected_result", - [ - ["Linux", ["Linux"], True], - ["Linux", [], True], - ["Linux", None, True], - ["Windows", ["Linux"], False], - ], -) -def test_could_be_installed_depends_on_platform( - platform: str, - supported_platforms: list[str] | None, - expected_result: bool, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Test that installation could not be installed on unsupported platform.""" - monkeypatch.setattr( - "mlia.backend.install.platform.system", - MagicMock(return_value=platform), - ) - monkeypatch.setattr( - "mlia.backend.install.all_paths_valid", - MagicMock(return_value=True), - ) - backend_runner_mock = MagicMock(spec=BackendRunner) - - installation = get_backend_installation( - backend_runner_mock, - supported_platforms=supported_platforms, - ) - assert installation.could_be_installed == expected_result +from mlia.backend.install import Installation def test_get_corstone_installations() -> None: - """Test function get_corstone_installation.""" - installs = get_corstone_installations() - assert len(installs) == 2 - assert all(isinstance(install, BackendInstallation) for install in installs) - - -def test_backend_installation_metadata_resolving() -> None: - """Test backend installation metadata resolving.""" - backend_runner_mock = MagicMock(spec=BackendRunner) - installation = get_backend_installation(backend_runner_mock) - - assert installation.name == "test_name" - assert installation.description == "test_description" - - backend_runner_mock.all_installed.return_value = False - assert installation.already_installed is False - - assert installation.could_be_installed is True - - -def test_backend_installation_supported_install_types(tmp_path: Path) -> None: - """Test supported installation types.""" - installation_no_download_artifact = get_backend_installation() - assert installation_no_download_artifact.supports(DownloadAndInstall()) is False - - installation_with_download_artifact = get_backend_installation( - download_artifact=MagicMock() - ) - assert installation_with_download_artifact.supports(DownloadAndInstall()) is True - - path_checker_mock = MagicMock(return_value=BackendInfo(tmp_path)) - installation_can_install_from_dir = get_backend_installation( - path_checker=path_checker_mock - ) - assert installation_can_install_from_dir.supports(InstallFromPath(tmp_path)) is True - - any_installation = get_backend_installation() - assert any_installation.supports("unknown_install_type") is False # type: ignore - - -def test_backend_installation_install_wrong_type() -> None: - """Test that operation should fail if wrong install type provided.""" - with pytest.raises(Exception, match="Unable to install wrong_install_type"): - backend_runner_mock = MagicMock(spec=BackendRunner) - installation = get_backend_installation(backend_runner_mock) - - installation.install("wrong_install_type") # type: ignore - - -def test_backend_installation_install_from_path( - tmp_path: Path, test_mlia_resources: Path -) -> None: - """Test installation from the path.""" - system_config = test_mlia_resources / "example_config.json" - system_config.touch() - - sample_app = test_mlia_resources / "sample_app" - sample_app.mkdir() - - dist_dir = tmp_path / "dist" - dist_dir.mkdir() - - path_checker_mock = MagicMock(return_value=BackendInfo(dist_dir)) - - backend_runner_mock = MagicMock(spec=BackendRunner) - installation = get_backend_installation( - backend_runner_mock=backend_runner_mock, - path_checker=path_checker_mock, - apps_resources=[sample_app.name], - system_config="example_config.json", - ) - - assert installation.supports(InstallFromPath(dist_dir)) is True - installation.install(InstallFromPath(dist_dir)) - - backend_runner_mock.install_system.assert_called_once() - backend_runner_mock.install_application.assert_called_once_with(sample_app) - - -@pytest.mark.parametrize("copy_source", [True, False]) -def test_backend_installation_install_from_static_path( - tmp_path: Path, test_mlia_resources: Path, copy_source: bool -) -> None: - """Test installation from the predefined path.""" - system_config = test_mlia_resources / "example_config.json" - system_config.touch() - - custom_system_config = test_mlia_resources / "custom_config.json" - custom_system_config.touch() - - sample_app = test_mlia_resources / "sample_app" - sample_app.mkdir() - - predefined_location = tmp_path / "backend" - predefined_location.mkdir() - - predefined_location_file = predefined_location / "file.txt" - predefined_location_file.touch() - - predefined_location_dir = predefined_location / "folder" - predefined_location_dir.mkdir() - nested_file = predefined_location_dir / "nested_file.txt" - nested_file.touch() - - backend_runner_mock = MagicMock(spec=BackendRunner) - - def check_install_dir(install_dir: Path) -> None: - """Check content of the install dir.""" - assert install_dir.is_dir() - files = list(install_dir.iterdir()) - - if copy_source: - assert len(files) == 3 - assert all(install_dir / item in files for item in ["file.txt", "folder"]) - assert (install_dir / "folder/nested_file.txt").is_file() - else: - assert len(files) == 1 - - assert install_dir / "custom_config.json" in files - - backend_runner_mock.install_system.side_effect = check_install_dir - - installation = get_backend_installation( - backend_runner_mock=backend_runner_mock, - path_checker=StaticPathChecker( - predefined_location, - ["file.txt"], - copy_source=copy_source, - system_config=str(custom_system_config), - ), - apps_resources=[sample_app.name], - system_config="example_config.json", - ) - - assert installation.supports(InstallFromPath(predefined_location)) is True - installation.install(InstallFromPath(predefined_location)) - - backend_runner_mock.install_system.assert_called_once() - backend_runner_mock.install_application.assert_called_once_with(sample_app) - - -def create_sample_fvp_archive(tmp_path: Path) -> Path: - """Create sample FVP tar archive.""" - fvp_archive_dir = tmp_path / "archive" - fvp_archive_dir.mkdir() - - sample_file = fvp_archive_dir / "sample.txt" - sample_file.write_text("Sample file") - - sample_dir = fvp_archive_dir / "sample_dir" - sample_dir.mkdir() - - fvp_archive = tmp_path / "archive.tgz" - with tarfile.open(fvp_archive, "w:gz") as fvp_archive_tar: - fvp_archive_tar.add(fvp_archive_dir, arcname=fvp_archive_dir.name) - - return fvp_archive - - -def test_backend_installation_download_and_install( - test_mlia_resources: Path, tmp_path: Path -) -> None: - """Test downloading and installation process.""" - fvp_archive = create_sample_fvp_archive(tmp_path) - - system_config = test_mlia_resources / "example_config.json" - system_config.touch() - - download_artifact_mock = MagicMock() - download_artifact_mock.download_to.return_value = fvp_archive + """Test function get_corstone_installations.""" + installations = get_corstone_installations() + assert len(installations) == 2 - path_checker = PackagePathChecker(["archive/sample.txt"], "archive/sample_dir") - - def installer(_eula_agreement: bool, dist_dir: Path) -> Path: - """Sample installer.""" - return dist_dir - - backend_runner_mock = MagicMock(spec=BackendRunner) - installation = get_backend_installation( - backend_runner_mock, - download_artifact=download_artifact_mock, - backend_installer=installer, - path_checker=path_checker, - system_config="example_config.json", - ) - - installation.install(DownloadAndInstall()) - - backend_runner_mock.install_system.assert_called_once() - - -@pytest.mark.parametrize( - "dir_content, expected_result", - [ - [ - ["models/", "file1.txt", "file2.txt"], - "models", - ], - [ - ["file1.txt", "file2.txt"], - None, - ], - [ - ["models/", "file2.txt"], - None, - ], - ], -) -def test_corstone_path_checker_valid_path( - tmp_path: Path, dir_content: list[str], expected_result: str | None -) -> None: - """Test Corstone path checker valid scenario.""" - path_checker = PackagePathChecker(["file1.txt", "file2.txt"], "models") - - for item in dir_content: - if item.endswith("/"): - item_dir = tmp_path / item - item_dir.mkdir() - else: - item_file = tmp_path / item - item_file.touch() - - result = path_checker(tmp_path) - expected = ( - None if expected_result is None else BackendInfo(tmp_path / expected_result) - ) - - assert result == expected - - -@pytest.mark.parametrize("system_config", [None, "system_config"]) -@pytest.mark.parametrize("copy_source", [True, False]) -def test_static_path_checker( - tmp_path: Path, copy_source: bool, system_config: str | None -) -> None: - """Test static path checker.""" - static_checker = StaticPathChecker( - tmp_path, [], copy_source=copy_source, system_config=system_config - ) - assert static_checker(tmp_path) == BackendInfo( - tmp_path, copy_source=copy_source, system_config=system_config - ) - - -def test_static_path_checker_not_valid_path(tmp_path: Path) -> None: - """Test static path checker should return None if path is not valid.""" - static_checker = StaticPathChecker(tmp_path, ["file.txt"]) - assert static_checker(tmp_path / "backend") is None - - -def test_static_path_checker_not_valid_structure(tmp_path: Path) -> None: - """Test static path checker should return None if files are missing.""" - static_checker = StaticPathChecker(tmp_path, ["file.txt"]) - assert static_checker(tmp_path) is None - - missing_file = tmp_path / "file.txt" - missing_file.touch() - - assert static_checker(tmp_path) == BackendInfo(tmp_path, copy_source=False) - - -def test_compound_path_checker(tmp_path: Path) -> None: - """Test compound path checker.""" - path_checker_path_valid_path = MagicMock(return_value=BackendInfo(tmp_path)) - path_checker_path_not_valid_path = MagicMock(return_value=None) - - checker = CompoundPathChecker( - path_checker_path_valid_path, path_checker_path_not_valid_path - ) - assert checker(tmp_path) == BackendInfo(tmp_path) - - checker = CompoundPathChecker(path_checker_path_not_valid_path) - assert checker(tmp_path) is None + assert all(isinstance(item, Installation) for item in installations) @pytest.mark.parametrize( - "eula_agreement, expected_command", + "eula_agreement, expected_calls", [ - [ - True, - [ - "./FVP_Corstone_SSE-300.sh", - "-q", - "-d", - "corstone-300", - ], - ], + [True, [call(["./FVP_Corstone_SSE-300.sh", "-q", "-d", "corstone-300"])]], [ False, [ - "./FVP_Corstone_SSE-300.sh", - "-q", - "-d", - "corstone-300", - "--nointeractive", - "--i-agree-to-the-contained-eula", + call( + [ + "./FVP_Corstone_SSE-300.sh", + "-q", + "-d", + "corstone-300", + "--nointeractive", + "--i-agree-to-the-contained-eula", + ] + ) ], ], ], ) -def test_corstone_300_installer( +def test_corstone_installer( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, eula_agreement: bool, - expected_command: list[str], + expected_calls: Any, ) -> None: - """Test Corstone-300 installer.""" - command_mock = MagicMock() + """Test Corstone 300 installer.""" + mock_check_call = MagicMock() monkeypatch.setattr( - "mlia.backend.corstone.install.subprocess.check_call", command_mock + "mlia.backend.corstone.install.subprocess.check_call", mock_check_call ) - installer = Corstone300Installer() - result = installer(eula_agreement, tmp_path) - - command_mock.assert_called_once_with(expected_command) - assert result == tmp_path / "corstone-300" - - -@pytest.mark.parametrize( - "corstone_installation, expected_paths", - [ - [ - get_corstone_300_installation(), - { - "/opt/VHT/VHT_Corstone_SSE-300_Ethos-U55", - "/opt/VHT/VHT_Corstone_SSE-300_Ethos-U65", - }, - ], - [ - get_corstone_310_installation(), - { - "/opt/VHT/VHT_Corstone_SSE-310", - "/opt/VHT/VHT_Corstone_SSE-310_Ethos-U65", - }, - ], - ], -) -def test_corstone_vht_install( - corstone_installation: BackendInstallation, - expected_paths: set, - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Test if Corstone 300/310 could be installed from /opt/VHT.""" - - def _all_files_exist(paths: Iterable[Path]) -> bool: - """Check if all files exist.""" - pathset = {item.as_posix() for item in paths} - return pathset == expected_paths - create_destination_and_install_mock = MagicMock() - - monkeypatch.setattr("mlia.backend.install.all_files_exist", _all_files_exist) - - monkeypatch.setattr( - "mlia.backend.executor.system.get_available_systems", lambda: [] - ) - - monkeypatch.setattr( - "mlia.backend.executor.system.create_destination_and_install", - create_destination_and_install_mock, - ) - - corstone_installation.install(InstallFromPath(Path("/opt/VHT"))) - - create_destination_and_install_mock.assert_called_once() - - -def test_corstone_uninstall( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Test the uninstall function in Corstone.""" - remove_system_mock = MagicMock() - - monkeypatch.setattr( - "mlia.backend.install.remove_system", - remove_system_mock, - ) - - installation = get_corstone_300_installation() + installer = Corstone300Installer() + installer(eula_agreement, tmp_path) - installation.uninstall() - remove_system_mock.assert_called_once_with("corstone_300") + assert mock_check_call.mock_calls == expected_calls diff --git a/tests/test_backend_corstone_performance.py b/tests/test_backend_corstone_performance.py index d41062f..2d5b196 100644 --- a/tests/test_backend_corstone_performance.py +++ b/tests/test_backend_corstone_performance.py @@ -4,516 +4,167 @@ from __future__ import annotations import base64 -import json -from contextlib import ExitStack as does_not_raise from pathlib import Path -from typing import Any +from typing import Generator from unittest.mock import MagicMock -from unittest.mock import PropertyMock import pytest -from mlia.backend.corstone.performance import BackendRunner -from mlia.backend.corstone.performance import DeviceInfo +from mlia.backend.corstone.performance import build_corstone_command from mlia.backend.corstone.performance import estimate_performance from mlia.backend.corstone.performance import GenericInferenceOutputParser -from mlia.backend.corstone.performance import GenericInferenceRunnerEthosU -from mlia.backend.corstone.performance import get_generic_runner -from mlia.backend.corstone.performance import ModelInfo +from mlia.backend.corstone.performance import get_metrics from mlia.backend.corstone.performance import PerformanceMetrics -from mlia.backend.executor.application import get_application -from mlia.backend.executor.execution import ExecutionContext -from mlia.backend.executor.output_consumer import Base64OutputConsumer -from mlia.backend.executor.system import get_system -from mlia.backend.registry import get_supported_backends -from mlia.target.registry import is_supported - +from mlia.backend.errors import BackendExecutionFailed +from mlia.utils.proc import Command + + +def encode_b64(data: str) -> str: + """Encode data in base64 format.""" + return base64.b64encode(data.encode()).decode() + + +def valid_fvp_output() -> list[str]: + """Return valid FVP output that could be succesfully parsed.""" + json_data = """[ + { + "profiling_group": "Inference", + "count": 1, + "samples": [ + {"name": "NPU IDLE", "value": [2]}, + {"name": "NPU AXI0_RD_DATA_BEAT_RECEIVED", "value": [4]}, + {"name": "NPU AXI0_WR_DATA_BEAT_WRITTEN", "value": [5]}, + {"name": "NPU AXI1_RD_DATA_BEAT_RECEIVED", "value": [6]}, + {"name": "NPU ACTIVE", "value": [1]}, + {"name": "NPU TOTAL", "value": [3]} + ] + } +]""" -def _mock_encode_b64(data: dict[str, int]) -> str: - """ - Encode the given data into a mock base64-encoded string of JSON. + return [ + "some output", + f"{encode_b64(json_data)}", + "some_output", + ] - This reproduces the base64 encoding done in the Corstone applications. - JSON example: +def test_generic_inference_output_parser_success() -> None: + """Test successful generic inference output parsing.""" + output_parser = GenericInferenceOutputParser() + for line in valid_fvp_output(): + output_parser(line) - ```json - [{'count': 1, - 'profiling_group': 'Inference', - 'samples': [{'name': 'NPU IDLE', 'value': [612]}, - {'name': 'NPU AXI0_RD_DATA_BEAT_RECEIVED', 'value': [165872]}, - {'name': 'NPU AXI0_WR_DATA_BEAT_WRITTEN', 'value': [88712]}, - {'name': 'NPU AXI1_RD_DATA_BEAT_RECEIVED', 'value': [57540]}, - {'name': 'NPU ACTIVE', 'value': [520489]}, - {'name': 'NPU TOTAL', 'value': [521101]}]}] - ``` - """ - wrapped_data = [ - { - "count": 1, - "profiling_group": "Inference", - "samples": [ - {"name": name, "value": [value]} for name, value in data.items() - ], - } - ] - json_str = json.dumps(wrapped_data) - json_bytes = bytearray(json_str, encoding="utf-8") - json_b64 = base64.b64encode(json_bytes).decode("utf-8") - tag = Base64OutputConsumer.TAG_NAME - return f"<{tag}>{json_b64}" + assert output_parser.get_metrics() == PerformanceMetrics(1, 2, 3, 4, 5, 6) @pytest.mark.parametrize( - "data, is_ready, result, missed_keys", + "wrong_fvp_output", [ - ( - [], - False, - {}, - { - "npu_active_cycles", - "npu_axi0_rd_data_beat_received", - "npu_axi0_wr_data_beat_written", - "npu_axi1_rd_data_beat_received", - "npu_idle_cycles", - "npu_total_cycles", - }, - ), - ( - ["sample text"], - False, - {}, - { - "npu_active_cycles", - "npu_axi0_rd_data_beat_received", - "npu_axi0_wr_data_beat_written", - "npu_axi1_rd_data_beat_received", - "npu_idle_cycles", - "npu_total_cycles", - }, - ), - ( - [_mock_encode_b64({"NPU AXI0_RD_DATA_BEAT_RECEIVED": 123})], - False, - {"npu_axi0_rd_data_beat_received": 123}, - { - "npu_active_cycles", - "npu_axi0_wr_data_beat_written", - "npu_axi1_rd_data_beat_received", - "npu_idle_cycles", - "npu_total_cycles", - }, - ), - ( - [ - _mock_encode_b64( - { - "NPU AXI0_RD_DATA_BEAT_RECEIVED": 1, - "NPU AXI0_WR_DATA_BEAT_WRITTEN": 2, - "NPU AXI1_RD_DATA_BEAT_RECEIVED": 3, - "NPU ACTIVE": 4, - "NPU IDLE": 5, - "NPU TOTAL": 6, - } - ) - ], - True, - { - "npu_axi0_rd_data_beat_received": 1, - "npu_axi0_wr_data_beat_written": 2, - "npu_axi1_rd_data_beat_received": 3, - "npu_active_cycles": 4, - "npu_idle_cycles": 5, - "npu_total_cycles": 6, - }, - set(), - ), + [], + ["NPU IDLE: 123"], + ["123"], ], ) -def test_generic_inference_output_parser( - data: dict[str, int], is_ready: bool, result: dict, missed_keys: set[str] -) -> None: - """Test generic runner output parser.""" - parser = GenericInferenceOutputParser() +def test_generic_inference_output_parser_failure(wrong_fvp_output: list[str]) -> None: + """Test unsuccessful generic inference output parsing.""" + output_parser = GenericInferenceOutputParser() - for line in data: - parser.feed(line) + for line in wrong_fvp_output: + output_parser(line) - assert parser.is_ready() == is_ready - assert parser.result == result - assert parser.missed_keys() == missed_keys + with pytest.raises(ValueError, match="Unable to parse output and get metrics"): + output_parser.get_metrics() @pytest.mark.parametrize( - "device, system, application, backend, expected_error", + "backend_path, fvp, target, mac, model, profile, expected_command", [ - ( - DeviceInfo(device_type="Ethos-U55", mac=32), - ("Corstone-300: Cortex-M55+Ethos-U55", True), - ("Generic Inference Runner: Ethos-U55", True), - "Corstone-300", - does_not_raise(), - ), - ( - DeviceInfo(device_type="Ethos-U55", mac=32), - ("Corstone-300: Cortex-M55+Ethos-U55", False), - ("Generic Inference Runner: Ethos-U55", False), - "Corstone-300", - pytest.raises( - Exception, - match=r"System Corstone-300: Cortex-M55\+Ethos-U55 is not installed", - ), - ), - ( - DeviceInfo(device_type="Ethos-U55", mac=32), - ("Corstone-300: Cortex-M55+Ethos-U55", True), - ("Generic Inference Runner: Ethos-U55", False), - "Corstone-300", - pytest.raises( - Exception, - match=r"Application Generic Inference Runner: Ethos-U55 " - r"for the system Corstone-300: Cortex-M55\+Ethos-U55 is not installed", - ), - ), - ( - DeviceInfo(device_type="Ethos-U55", mac=32), - ("Corstone-310: Cortex-M85+Ethos-U55", True), - ("Generic Inference Runner: Ethos-U55", True), - "Corstone-310", - does_not_raise(), - ), - ( - DeviceInfo(device_type="Ethos-U55", mac=32), - ("Corstone-310: Cortex-M85+Ethos-U55", False), - ("Generic Inference Runner: Ethos-U55", False), - "Corstone-310", - pytest.raises( - Exception, - match=r"System Corstone-310: Cortex-M85\+Ethos-U55 is not installed", - ), - ), - ( - DeviceInfo(device_type="Ethos-U55", mac=32), - ("Corstone-310: Cortex-M85+Ethos-U55", True), - ("Generic Inference Runner: Ethos-U55", False), - "Corstone-310", - pytest.raises( - Exception, - match=r"Application Generic Inference Runner: Ethos-U55 " - r"for the system Corstone-310: Cortex-M85\+Ethos-U55 is not installed", - ), - ), - ( - DeviceInfo(device_type="Ethos-U65", mac=512), - ("Corstone-300: Cortex-M55+Ethos-U65", True), - ("Generic Inference Runner: Ethos-U65", True), - "Corstone-300", - does_not_raise(), - ), - ( - DeviceInfo(device_type="Ethos-U65", mac=512), - ("Corstone-300: Cortex-M55+Ethos-U65", False), - ("Generic Inference Runner: Ethos-U65", False), - "Corstone-300", - pytest.raises( - Exception, - match=r"System Corstone-300: Cortex-M55\+Ethos-U65 is not installed", - ), - ), - ( - DeviceInfo(device_type="Ethos-U65", mac=512), - ("Corstone-300: Cortex-M55+Ethos-U65", True), - ("Generic Inference Runner: Ethos-U65", False), + [ + Path("backend_path"), "Corstone-300", - pytest.raises( - Exception, - match=r"Application Generic Inference Runner: Ethos-U65 " - r"for the system Corstone-300: Cortex-M55\+Ethos-U65 is not installed", + "ethos-u55", + 256, + Path("model.tflite"), + "default", + Command( + [ + "backend_path/FVP_Corstone_SSE-300_Ethos-U55", + "-a", + "apps/backends/applications/" + "inference_runner-sse-300-22.08.02-ethos-U55-Default-noTA/" + "ethos-u-inference_runner.axf", + "--data", + "model.tflite@0x90000000", + "-C", + "ethosu.num_macs=256", + "-C", + "mps3_board.telnetterminal0.start_telnet=0", + "-C", + "mps3_board.uart0.out_file='-'", + "-C", + "mps3_board.uart0.shutdown_on_eot=1", + "-C", + "mps3_board.visualisation.disable-visualisation=1", + "--stat", + ] ), - ), - ( - DeviceInfo(device_type="Ethos-U65", mac=512), - ("Corstone-310: Cortex-M85+Ethos-U65", True), - ("Generic Inference Runner: Ethos-U65", True), - "Corstone-310", - does_not_raise(), - ), - ( - DeviceInfo(device_type="Ethos-U65", mac=512), - ("Corstone-310: Cortex-M85+Ethos-U65", False), - ("Generic Inference Runner: Ethos-U65", False), - "Corstone-310", - pytest.raises( - Exception, - match=r"System Corstone-310: Cortex-M85\+Ethos-U65 is not installed", - ), - ), - ( - DeviceInfo(device_type="Ethos-U65", mac=512), - ("Corstone-310: Cortex-M85+Ethos-U65", True), - ("Generic Inference Runner: Ethos-U65", False), - "Corstone-310", - pytest.raises( - Exception, - match=r"Application Generic Inference Runner: Ethos-U65 " - r"for the system Corstone-310: Cortex-M85\+Ethos-U65 is not installed", - ), - ), - ( - DeviceInfo( - device_type="unknown_device", # type: ignore - mac=None, # type: ignore - ), - ("some_system", False), - ("some_application", False), - "some backend", - pytest.raises(Exception, match="Unsupported device unknown_device"), - ), + ], ], ) -def test_estimate_performance( - device: DeviceInfo, - system: tuple[str, bool], - application: tuple[str, bool], - backend: str, - expected_error: Any, - test_tflite_model: Path, - backend_runner: MagicMock, +def test_build_corsone_command( + monkeypatch: pytest.MonkeyPatch, + backend_path: Path, + fvp: str, + target: str, + mac: int, + model: Path, + profile: str, + expected_command: Command, ) -> None: - """Test getting performance estimations.""" - system_name, system_installed = system - application_name, application_installed = application - - backend_runner.is_system_installed.return_value = system_installed - backend_runner.is_application_installed.return_value = application_installed - - mock_context = create_mock_context( - [ - _mock_encode_b64( - { - "NPU AXI0_RD_DATA_BEAT_RECEIVED": 1, - "NPU AXI0_WR_DATA_BEAT_WRITTEN": 2, - "NPU AXI1_RD_DATA_BEAT_RECEIVED": 3, - "NPU ACTIVE": 4, - "NPU IDLE": 5, - "NPU TOTAL": 6, - } - ) - ] + """Test function build_corstone_command.""" + monkeypatch.setattr( + "mlia.backend.corstone.performance.get_mlia_resources", lambda: Path("apps") ) - backend_runner.run_application.return_value = mock_context + command = build_corstone_command(backend_path, fvp, target, mac, model, profile) + assert command == expected_command - with expected_error: - perf_metrics = estimate_performance( - ModelInfo(test_tflite_model), device, backend - ) - - assert isinstance(perf_metrics, PerformanceMetrics) - assert perf_metrics == PerformanceMetrics( - npu_axi0_rd_data_beat_received=1, - npu_axi0_wr_data_beat_written=2, - npu_axi1_rd_data_beat_received=3, - npu_active_cycles=4, - npu_idle_cycles=5, - npu_total_cycles=6, - ) - - assert backend_runner.is_system_installed.called_once_with(system_name) - assert backend_runner.is_application_installed.called_once_with( - application_name, system_name - ) - - -@pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) -def test_estimate_performance_insufficient_data( - backend_runner: MagicMock, test_tflite_model: Path, backend: str -) -> None: - """Test that performance could not be estimated when not all data presented.""" - backend_runner.is_system_installed.return_value = True - backend_runner.is_application_installed.return_value = True - - no_total_cycles_output = { - "NPU AXI0_RD_DATA_BEAT_RECEIVED": 1, - "NPU AXI0_WR_DATA_BEAT_WRITTEN": 2, - "NPU AXI1_RD_DATA_BEAT_RECEIVED": 3, - "NPU ACTIVE": 4, - "NPU IDLE": 5, - } - mock_context = create_mock_context([_mock_encode_b64(no_total_cycles_output)]) - - backend_runner.run_application.return_value = mock_context +def test_get_metrics_wrong_fvp() -> None: + """Test that command construction should fail for wrong FVP.""" with pytest.raises( - Exception, match="Unable to get performance metrics, insufficient data" + BackendExecutionFailed, match=r"Unable to construct a command line for some_fvp" ): - device = DeviceInfo(device_type="Ethos-U55", mac=32) - estimate_performance(ModelInfo(test_tflite_model), device, backend) - - -def create_mock_process(stdout: list[str], stderr: list[str]) -> MagicMock: - """Mock underlying process.""" - mock_process = MagicMock() - mock_process.poll.return_value = 0 - type(mock_process).stdout = PropertyMock(return_value=iter(stdout)) - type(mock_process).stderr = PropertyMock(return_value=iter(stderr)) - return mock_process - - -def create_mock_context(stdout: list[str]) -> ExecutionContext: - """Mock ExecutionContext.""" - ctx = ExecutionContext( - app=get_application("application_1")[0], - app_params=[], - system=get_system("System 1"), - system_params=[], - ) - ctx.stdout = bytearray("\n".join(stdout).encode("utf-8")) - return ctx - - -@pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) -def test_estimate_performance_invalid_output( - test_tflite_model: Path, backend_runner: MagicMock, backend: str -) -> None: - """Test estimation could not be done if inference produces unexpected output.""" - backend_runner.is_system_installed.return_value = True - backend_runner.is_application_installed.return_value = True - - mock_context = create_mock_context(["Something", "is", "wrong"]) - backend_runner.run_application.return_value = mock_context - - with pytest.raises(Exception, match="Unable to get performance metrics"): - estimate_performance( - ModelInfo(test_tflite_model), - DeviceInfo(device_type="Ethos-U55", mac=256), - backend=backend, + get_metrics( + Path("backend_path"), + "some_fvp", + "ethos-u55", + 256, + Path("model.tflite"), ) -@pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) -def test_get_generic_runner(backend: str) -> None: - """Test function get_generic_runner().""" - device_info = DeviceInfo("Ethos-U55", 256) - - runner = get_generic_runner(device_info=device_info, backend=backend) - assert isinstance(runner, GenericInferenceRunnerEthosU) - - with pytest.raises(RuntimeError): - get_generic_runner(device_info=device_info, backend="UNKNOWN_BACKEND") - - -@pytest.mark.parametrize( - ("backend", "device_type"), - ( - ("Corstone-300", "Ethos-U55"), - ("Corstone-300", "Ethos-U65"), - ("Corstone-310", "Ethos-U55"), - ("ArmNNTFLiteDelegate", "Cortex-A"), - ("TOSA-Checker", "TOSA"), - ("Corstone-300", None), - ), -) -def test_backend_support(backend: str, device_type: str) -> None: - """Test backend & device support.""" - assert is_supported(backend) - assert is_supported(backend, device_type) - - assert backend in get_supported_backends() - - -class TestGenericInferenceRunnerEthosU: - """Test for the class GenericInferenceRunnerEthosU.""" +def test_estimate_performance(monkeypatch: pytest.MonkeyPatch) -> None: + """Test function estimate_performance.""" + mock_repository = MagicMock() + mock_repository.get_backend_settings.return_value = Path("backend_path"), { + "profile": "default" + } - @staticmethod - @pytest.mark.parametrize( - "device, backend, expected_system, expected_app", - [ - [ - DeviceInfo("Ethos-U55", 256), - "Corstone-300", - "Corstone-300: Cortex-M55+Ethos-U55", - "Generic Inference Runner: Ethos-U55", - ], - [ - DeviceInfo("Ethos-U65", 256), - "Corstone-300", - "Corstone-300: Cortex-M55+Ethos-U65", - "Generic Inference Runner: Ethos-U65", - ], - [ - DeviceInfo("Ethos-U55", 256), - "Corstone-310", - "Corstone-310: Cortex-M85+Ethos-U55", - "Generic Inference Runner: Ethos-U55", - ], - [ - DeviceInfo("Ethos-U65", 256), - "Corstone-310", - "Corstone-310: Cortex-M85+Ethos-U65", - "Generic Inference Runner: Ethos-U65", - ], - ], + monkeypatch.setattr( + "mlia.backend.corstone.performance.get_backend_repository", + lambda: mock_repository, ) - def test_artifact_resolver( - device: DeviceInfo, backend: str, expected_system: str, expected_app: str - ) -> None: - """Test artifact resolving based on the provided parameters.""" - generic_runner = get_generic_runner(device, backend) - assert isinstance(generic_runner, GenericInferenceRunnerEthosU) - - assert generic_runner.system_name == expected_system - assert generic_runner.app_name == expected_app - - @staticmethod - def test_artifact_resolver_unsupported_backend() -> None: - """Test that it should be not possible to use unsupported backends.""" - with pytest.raises( - RuntimeError, match="Unsupported device Ethos-U65 for backend test_backend" - ): - get_generic_runner(DeviceInfo("Ethos-U65", 256), "test_backend") - - @staticmethod - @pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) - def test_inference_should_fail_if_system_not_installed( - backend_runner: MagicMock, test_tflite_model: Path, backend: str - ) -> None: - """Test that inference should fail if system is not installed.""" - backend_runner.is_system_installed.return_value = False - generic_runner = get_generic_runner(DeviceInfo("Ethos-U55", 256), backend) - with pytest.raises( - Exception, - match=r"System Corstone-3[01]0: Cortex-M[58]5\+Ethos-U55 is not installed", - ): - generic_runner.run(ModelInfo(test_tflite_model), []) + def command_output_mock(_command: Command) -> Generator[str, None, None]: + """Mock FVP output.""" + yield from valid_fvp_output() - @staticmethod - @pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) - def test_inference_should_fail_is_apps_not_installed( - backend_runner: MagicMock, test_tflite_model: Path, backend: str - ) -> None: - """Test that inference should fail if apps are not installed.""" - backend_runner.is_system_installed.return_value = True - backend_runner.is_application_installed.return_value = False + monkeypatch.setattr("mlia.utils.proc.command_output", command_output_mock) - generic_runner = get_generic_runner(DeviceInfo("Ethos-U55", 256), backend) - with pytest.raises( - Exception, - match="Application Generic Inference Runner: Ethos-U55" - r" for the system Corstone-3[01]0: Cortex-M[58]5\+Ethos-U55 is not " - r"installed", - ): - generic_runner.run(ModelInfo(test_tflite_model), []) - - -@pytest.fixture(name="backend_runner") -def fixture_backend_runner(monkeypatch: pytest.MonkeyPatch) -> MagicMock: - """Mock backend runner.""" - backend_runner_mock = MagicMock(spec=BackendRunner) - monkeypatch.setattr( - "mlia.backend.corstone.performance.get_backend_runner", - MagicMock(return_value=backend_runner_mock), + result = estimate_performance( + "ethos-u55", 256, Path("model.tflite"), "Corstone-300" ) - return backend_runner_mock + assert result == PerformanceMetrics(1, 2, 3, 4, 5, 6) + + mock_repository.get_backend_settings.assert_called_once() diff --git a/tests/test_backend_executor_application.py b/tests/test_backend_executor_application.py deleted file mode 100644 index 8962a0a..0000000 --- a/tests/test_backend_executor_application.py +++ /dev/null @@ -1,422 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Tests for the application backend.""" -from __future__ import annotations - -from collections import Counter -from contextlib import ExitStack as does_not_raise -from pathlib import Path -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from mlia.backend.executor.application import Application -from mlia.backend.executor.application import get_application -from mlia.backend.executor.application import ( - get_available_application_directory_names, -) -from mlia.backend.executor.application import get_available_applications -from mlia.backend.executor.application import get_unique_application_names -from mlia.backend.executor.application import install_application -from mlia.backend.executor.application import load_applications -from mlia.backend.executor.application import remove_application -from mlia.backend.executor.common import Command -from mlia.backend.executor.common import Param -from mlia.backend.executor.common import UserParamConfig -from mlia.backend.executor.config import ApplicationConfig -from mlia.backend.executor.config import ExtendedApplicationConfig -from mlia.backend.executor.config import NamedExecutionConfig - - -def test_get_available_application_directory_names() -> None: - """Test get_available_applicationss mocking get_resources.""" - directory_names = get_available_application_directory_names() - assert Counter(directory_names) == Counter( - [ - "application1", - "application2", - "application4", - "application5", - "application6", - ] - ) - - -def test_get_available_applications() -> None: - """Test get_available_applicationss mocking get_resources.""" - available_applications = get_available_applications() - - assert all(isinstance(s, Application) for s in available_applications) - assert all(s != 42 for s in available_applications) - assert len(available_applications) == 10 - # application_5 has multiply items with multiply supported systems - assert [str(s) for s in available_applications] == [ - "application_1", - "application_2", - "application_4", - "application_5", - "application_5", - "application_5A", - "application_5A", - "application_5B", - "application_5B", - "application_6", - ] - - -def test_get_unique_application_names() -> None: - """Test get_unique_application_names.""" - unique_names = get_unique_application_names() - - assert all(isinstance(s, str) for s in unique_names) - assert all(s for s in unique_names) - assert sorted(unique_names) == [ - "application_1", - "application_2", - "application_4", - "application_5", - "application_5A", - "application_5B", - "application_6", - ] - - -def test_get_application() -> None: - """Test get_application mocking get_resoures.""" - application = get_application("application_1") - if len(application) != 1: - pytest.fail("Unable to get application") - assert application[0].name == "application_1" - - application = get_application("unknown application") - assert len(application) == 0 - - -@pytest.mark.parametrize( - "source, call_count, expected_exception", - ( - ( - "archives/applications/application1.tar.gz", - 0, - pytest.raises( - Exception, match=r"Applications \[application_1\] are already installed" - ), - ), - ( - "various/applications/application_with_empty_config", - 0, - pytest.raises(Exception, match="No application definition found"), - ), - ( - "various/applications/application_with_wrong_config1", - 0, - pytest.raises(Exception, match="Unable to read application definition"), - ), - ( - "various/applications/application_with_wrong_config2", - 0, - pytest.raises(Exception, match="Unable to read application definition"), - ), - ( - "various/applications/application_with_wrong_config3", - 0, - pytest.raises(Exception, match="Unable to read application definition"), - ), - ("various/applications/application_with_valid_config", 1, does_not_raise()), - ( - "archives/applications/application3.tar.gz", - 0, - pytest.raises(Exception, match="Unable to read application definition"), - ), - ( - "backends/applications/application1", - 0, - pytest.raises( - Exception, match=r"Applications \[application_1\] are already installed" - ), - ), - ( - "backends/applications/application3", - 0, - pytest.raises(Exception, match="Unable to read application definition"), - ), - ), -) -def test_install_application( - monkeypatch: Any, - test_resources_path: Path, - source: str, - call_count: int, - expected_exception: Any, -) -> None: - """Test application install from archive.""" - mock_create_destination_and_install = MagicMock() - monkeypatch.setattr( - "mlia.backend.executor.application.create_destination_and_install", - mock_create_destination_and_install, - ) - - with expected_exception: - install_application(test_resources_path / source) - assert mock_create_destination_and_install.call_count == call_count - - -def test_remove_application(monkeypatch: Any) -> None: - """Test application removal.""" - mock_remove_backend = MagicMock() - monkeypatch.setattr( - "mlia.backend.executor.application.remove_backend", mock_remove_backend - ) - - remove_application("some_application_directory") - mock_remove_backend.assert_called_once() - - -def test_application_config_without_commands() -> None: - """Test application config without commands.""" - config = ApplicationConfig(name="application") - application = Application(config) - # pylint: disable=use-implicit-booleaness-not-comparison - assert application.commands == {} - - -class TestApplication: - """Test for application class methods.""" - - def test___eq__(self) -> None: - """Test overloaded __eq__ method.""" - config = ApplicationConfig( - # Application - supported_systems=["system1", "system2"], - # inherited from Backend - name="name", - description="description", - commands={}, - ) - application1 = Application(config) - application2 = Application(config) # Identical - assert application1 == application2 - - application3 = Application(config) # changed - # Change one single attribute so not equal, but same Type - setattr(application3, "supported_systems", ["somewhere/else"]) - assert application1 != application3 - - # different Type - application4 = "Not the Application you are looking for" - assert application1 != application4 - - application5 = Application(config) - # supported systems could be in any order - setattr(application5, "supported_systems", ["system2", "system1"]) - assert application1 == application5 - - def test_can_run_on(self) -> None: - """Test Application can run on.""" - config = ApplicationConfig(name="application", supported_systems=["System-A"]) - - application = Application(config) - assert application.can_run_on("System-A") - assert not application.can_run_on("System-B") - - applications = get_application("application_1", "System 1") - assert len(applications) == 1 - assert applications[0].can_run_on("System 1") - - def test_unable_to_create_application_without_name(self) -> None: - """Test that it is not possible to create application without name.""" - with pytest.raises(Exception, match="Name is empty"): - Application(ApplicationConfig()) - - def test_application_config_without_commands(self) -> None: - """Test application config without commands.""" - config = ApplicationConfig(name="application") - application = Application(config) - # pylint: disable=use-implicit-booleaness-not-comparison - assert application.commands == {} - - @pytest.mark.parametrize( - "config, expected_params", - ( - ( - ApplicationConfig( - name="application", - commands={"command": ["cmd {user_params:0} {user_params:1}"]}, - user_params={ - "command": [ - UserParamConfig( - name="--param1", description="param1", alias="param1" - ), - UserParamConfig( - name="--param2", description="param2", alias="param2" - ), - ] - }, - ), - [Param("--param1", "param1"), Param("--param2", "param2")], - ), - ( - ApplicationConfig( - name="application", - commands={"command": ["cmd {user_params:param1} {user_params:1}"]}, - user_params={ - "command": [ - UserParamConfig( - name="--param1", description="param1", alias="param1" - ), - UserParamConfig( - name="--param2", description="param2", alias="param2" - ), - ] - }, - ), - [Param("--param1", "param1"), Param("--param2", "param2")], - ), - ( - ApplicationConfig( - name="application", - commands={"command": ["cmd {user_params:param1}"]}, - user_params={ - "command": [ - UserParamConfig( - name="--param1", description="param1", alias="param1" - ), - UserParamConfig( - name="--param2", description="param2", alias="param2" - ), - ] - }, - ), - [Param("--param1", "param1")], - ), - ), - ) - def test_remove_unused_params( - self, config: ApplicationConfig, expected_params: list[Param] - ) -> None: - """Test mod remove_unused_parameter.""" - application = Application(config) - application.remove_unused_params() - assert application.commands["command"].params == expected_params - - -@pytest.mark.parametrize( - "config, expected_error", - ( - ( - ExtendedApplicationConfig(name="application"), - pytest.raises(Exception, match="No supported systems definition provided"), - ), - ( - ExtendedApplicationConfig( - name="application", supported_systems=[NamedExecutionConfig(name="")] - ), - pytest.raises( - Exception, - match="Unable to read supported system definition, name is missed", - ), - ), - ( - ExtendedApplicationConfig( - name="application", - supported_systems=[ - NamedExecutionConfig( - name="system", - commands={"command": ["cmd"]}, - user_params={"command": [UserParamConfig(name="param")]}, - ) - ], - commands={"command": ["cmd {user_params:0}"]}, - user_params={"command": [UserParamConfig(name="param")]}, - ), - pytest.raises( - Exception, match="Default parameters for command .* should have aliases" - ), - ), - ( - ExtendedApplicationConfig( - name="application", - supported_systems=[ - NamedExecutionConfig( - name="system", - commands={"command": ["cmd"]}, - user_params={"command": [UserParamConfig(name="param")]}, - ) - ], - commands={"command": ["cmd {user_params:0}"]}, - user_params={"command": [UserParamConfig(name="param", alias="param")]}, - ), - pytest.raises( - Exception, match="system parameters for command .* should have aliases" - ), - ), - ), -) -def test_load_application_exceptional_cases( - config: ExtendedApplicationConfig, expected_error: Any -) -> None: - """Test exceptional cases for application load function.""" - with expected_error: - load_applications(config) - - -def test_load_application() -> None: - """Test application load function. - - The main purpose of this test is to test configuration for application - for different systems. All configuration should be correctly - overridden if needed. - """ - application_5 = get_application("application_5") - assert len(application_5) == 2 - - default_commands = { - "build": Command(["default build command"]), - "run": Command(["default run command"]), - } - default_variables = {"var1": "value1", "var2": "value2"} - - application_5_0 = application_5[0] - assert application_5_0.supported_systems == ["System 1"] - assert application_5_0.commands == default_commands - assert application_5_0.variables == default_variables - - application_5_1 = application_5[1] - assert application_5_1.supported_systems == ["System 2"] - assert application_5_1.commands == application_5_1.commands - assert application_5_1.variables == default_variables - - application_5a = get_application("application_5A") - assert len(application_5a) == 2 - - application_5a_0 = application_5a[0] - assert application_5a_0.supported_systems == ["System 1"] - assert application_5a_0.commands == default_commands - assert application_5a_0.variables == {"var1": "new value1", "var2": "value2"} - - application_5a_1 = application_5a[1] - assert application_5a_1.supported_systems == ["System 2"] - assert application_5a_1.commands == { - "build": default_commands["build"], - "run": Command(["run command on system 2"]), - } - assert application_5a_1.variables == {"var1": "value1", "var2": "new value2"} - - application_5b = get_application("application_5B") - assert len(application_5b) == 2 - - application_5b_0 = application_5b[0] - assert application_5b_0.supported_systems == ["System 1"] - assert application_5b_0.commands == { - "build": Command(["default build command with value for var1 System1"]), - "run": Command(["default run command with value for var2 System1"]), - } - assert "non_used_command" not in application_5b_0.commands - - application_5b_1 = application_5b[1] - assert application_5b_1.supported_systems == ["System 2"] - assert application_5b_1.commands == { - "build": Command(["default build command with value for var1 System2"]), - "run": Command(["run command on system 2"], []), - } diff --git a/tests/test_backend_executor_common.py b/tests/test_backend_executor_common.py deleted file mode 100644 index e881462..0000000 --- a/tests/test_backend_executor_common.py +++ /dev/null @@ -1,482 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=protected-access -"""Tests for the common backend module.""" -from __future__ import annotations - -from contextlib import ExitStack as does_not_raise -from pathlib import Path -from typing import Any -from typing import cast -from typing import IO -from typing import List -from unittest.mock import MagicMock - -import pytest - -from mlia.backend.executor.application import Application -from mlia.backend.executor.common import Backend -from mlia.backend.executor.common import BaseBackendConfig -from mlia.backend.executor.common import Command -from mlia.backend.executor.common import ConfigurationException -from mlia.backend.executor.common import load_config -from mlia.backend.executor.common import Param -from mlia.backend.executor.common import parse_raw_parameter -from mlia.backend.executor.common import remove_backend -from mlia.backend.executor.config import ApplicationConfig -from mlia.backend.executor.config import UserParamConfig -from mlia.backend.executor.execution import ExecutionContext -from mlia.backend.executor.execution import ParamResolver -from mlia.backend.executor.system import System - - -@pytest.mark.parametrize( - "directory_name, expected_exception", - ( - ("some_dir", does_not_raise()), - (None, pytest.raises(Exception, match="No directory name provided")), - ), -) -def test_remove_backend( - monkeypatch: Any, directory_name: str, expected_exception: Any -) -> None: - """Test remove_backend function.""" - mock_remove_resource = MagicMock() - monkeypatch.setattr( - "mlia.backend.executor.common.remove_resource", mock_remove_resource - ) - - with expected_exception: - remove_backend(directory_name, "applications") - - -@pytest.mark.parametrize( - "filename, expected_exception", - ( - ("application_config.json", does_not_raise()), - (None, pytest.raises(Exception, match="Unable to read config")), - ), -) -def test_load_config( - filename: str, expected_exception: Any, test_resources_path: Path, monkeypatch: Any -) -> None: - """Test load_config.""" - with expected_exception: - configs: list[Path | IO[bytes] | None] = ( - [None] - if not filename - else [ - # Ignore pylint warning as 'with' can't be used inside of a - # generator expression. - # pylint: disable=consider-using-with - open(test_resources_path / filename, "rb"), - test_resources_path / filename, - ] - ) - for config in configs: - json_mock = MagicMock() - monkeypatch.setattr("mlia.backend.executor.common.json.load", json_mock) - load_config(config) - json_mock.assert_called_once() - - -class TestBackend: - """Test Backend class.""" - - def test___repr__(self) -> None: - """Test the representation of Backend instance.""" - backend = Backend( - BaseBackendConfig(name="Testing name", description="Testing description") - ) - assert str(backend) == "Testing name" - - def test__eq__(self) -> None: - """Test equality method with different cases.""" - backend1 = Backend(BaseBackendConfig(name="name", description="description")) - backend1.commands = {"command": Command(["command"])} - - backend2 = Backend(BaseBackendConfig(name="name", description="description")) - backend2.commands = {"command": Command(["command"])} - - backend3 = Backend( - BaseBackendConfig( - name="Ben", description="This is not the Backend you are looking for" - ) - ) - backend3.commands = {"wave": Command(["wave hand"])} - - backend4 = "Foo" # checking not isinstance(backend4, Backend) - - assert backend1 == backend2 - assert backend1 != backend3 - assert backend1 != backend4 - - @pytest.mark.parametrize( - "parameter, valid", - [ - ("--choice-param value_1", True), - ("--choice-param wrong_value", False), - ("--open-param something", True), - ("--wrong-param value", False), - ], - ) - def test_validate_parameter( - self, parameter: str, valid: bool, test_resources_path: Path - ) -> None: - """Test validate_parameter.""" - config = cast( - List[ApplicationConfig], - load_config(test_resources_path / "hello_world.json"), - ) - # The application configuration is a list of configurations so we need - # only the first one - # Exercise the validate_parameter test using the Application classe which - # inherits from Backend. - application = Application(config[0]) - assert application.validate_parameter("run", parameter) == valid - - def test_validate_parameter_with_invalid_command( - self, test_resources_path: Path - ) -> None: - """Test validate_parameter with an invalid command_name.""" - config = cast( - List[ApplicationConfig], - load_config(test_resources_path / "hello_world.json"), - ) - application = Application(config[0]) - with pytest.raises(AttributeError) as err: - # command foo does not exist, so raise an error - application.validate_parameter("foo", "bar") - assert "Unknown command: 'foo'" in str(err.value) - - def test_build_command(self) -> None: - """Test command building.""" - config = { - "name": "test", - "commands": { - "build": ["build {user_params:0} {user_params:1}"], - "run": ["run {user_params:0}"], - "post_run": ["post_run {application_params:0} on {system_params:0}"], - "some_command": ["Command with {variables:var_A}"], - "empty_command": [""], - }, - "user_params": { - "build": [ - { - "name": "choice_param_0=", - "values": [1, 2, 3], - "default_value": 1, - }, - {"name": "choice_param_1", "values": [3, 4, 5], "default_value": 3}, - {"name": "choice_param_3", "values": [6, 7, 8]}, - ], - "run": [{"name": "flag_param_0"}], - }, - "variables": {"var_A": "value for variable A"}, - } - - application, system = Application(config), System(config) # type: ignore - context = ExecutionContext( - app=application, - app_params=[], - system=system, - system_params=[], - ) - - param_resolver = ParamResolver(context) - - cmd = application.build_command( - "build", ["choice_param_0=2", "choice_param_1=4"], param_resolver - ) - assert cmd == ["build choice_param_0=2 choice_param_1 4"] - - cmd = application.build_command("build", ["choice_param_0=2"], param_resolver) - assert cmd == ["build choice_param_0=2 choice_param_1 3"] - - cmd = application.build_command( - "build", ["choice_param_0=2", "choice_param_3=7"], param_resolver - ) - assert cmd == ["build choice_param_0=2 choice_param_1 3"] - - with pytest.raises( - ConfigurationException, match="Command 'foo' could not be found." - ): - application.build_command("foo", [""], param_resolver) - - cmd = application.build_command("some_command", [], param_resolver) - assert cmd == ["Command with value for variable A"] - - cmd = application.build_command("empty_command", [], param_resolver) - assert cmd == [""] - - @pytest.mark.parametrize("class_", [Application, System]) - def test_build_command_unknown_variable(self, class_: type) -> None: - """Test that unable to construct backend with unknown variable.""" - with pytest.raises(Exception, match="Unknown variable var1"): - config = {"name": "test", "commands": {"run": ["run {variables:var1}"]}} - class_(config) - - @pytest.mark.parametrize( - "class_, config, expected_output", - [ - ( - Application, - { - "name": "test", - "commands": { - "build": ["build {user_params:0} {user_params:1}"], - "run": ["run {user_params:0}"], - }, - "user_params": { - "build": [ - { - "name": "choice_param_0=", - "values": ["a", "b", "c"], - "default_value": "a", - "alias": "param_1", - }, - { - "name": "choice_param_1", - "values": ["a", "b", "c"], - "default_value": "a", - "alias": "param_2", - }, - {"name": "choice_param_3", "values": ["a", "b", "c"]}, - ], - "run": [{"name": "flag_param_0"}], - }, - }, - [ - ( - "b", - Param( - name="choice_param_0=", - description="", - values=["a", "b", "c"], - default_value="a", - alias="param_1", - ), - ), - ( - "a", - Param( - name="choice_param_1", - description="", - values=["a", "b", "c"], - default_value="a", - alias="param_2", - ), - ), - ( - "c", - Param( - name="choice_param_3", - description="", - values=["a", "b", "c"], - ), - ), - ], - ), - (System, {"name": "test"}, []), - ], - ) - def test_resolved_parameters( - self, - class_: type, - config: dict, - expected_output: list[tuple[str | None, Param]], - ) -> None: - """Test command building.""" - backend = class_(config) - - params = backend.resolved_parameters( - "build", ["choice_param_0=b", "choice_param_3=c"] - ) - assert params == expected_output - - @pytest.mark.parametrize( - ["param_name", "user_param", "expected_value"], - [ - ( - "test_name", - "test_name=1234", - "1234", - ), # optional parameter using '=' - ( - "test_name", - "test_name 1234", - "1234", - ), # optional parameter using ' ' - ("test_name", "test_name", None), # flag - (None, "test_name=1234", "1234"), # positional parameter - ], - ) - def test_resolved_user_parameters( - self, param_name: str, user_param: str, expected_value: str - ) -> None: - """Test different variants to provide user parameters.""" - # A sample config providing one backend config - config = { - "name": "test_backend", - "commands": { - "test": ["user_param:test_param"], - }, - "user_params": { - "test": [UserParamConfig(name=param_name, alias="test_name")], - }, - } - backend = Backend(cast(BaseBackendConfig, config)) - params = backend.resolved_parameters( - command_name="test", user_params=[user_param] - ) - assert len(params) == 1 - value, param = params[0] - assert param_name == param.name - assert expected_value == value - - @pytest.mark.parametrize( - "input_param,expected", - [ - ("--param=1", ("--param", "1")), - ("--param 1", ("--param", "1")), - ("--flag", ("--flag", None)), - ], - ) - def test__parse_raw_parameter( - self, input_param: str, expected: tuple[str, str | None] - ) -> None: - """Test internal method of parsing a single raw parameter.""" - assert parse_raw_parameter(input_param) == expected - - -class TestParam: - """Test Param class.""" - - def test__eq__(self) -> None: - """Test equality method with different cases.""" - param1 = Param(name="test", description="desc", values=["values"]) - param2 = Param(name="test", description="desc", values=["values"]) - param3 = Param(name="test1", description="desc", values=["values"]) - param4 = object() - - assert param1 == param2 - assert param1 != param3 - assert param1 != param4 - - def test_get_details(self) -> None: - """Test get_details() method.""" - param1 = Param(name="test", description="desc", values=["values"]) - assert param1.get_details() == { - "name": "test", - "values": ["values"], - "description": "desc", - } - - def test_invalid(self) -> None: - """Test invalid use cases for the Param class.""" - with pytest.raises( - ConfigurationException, - match="Either name, alias or both must be set to identify a parameter.", - ): - Param(name=None, description="desc", values=["values"]) - - -class TestCommand: - """Test Command class.""" - - def test_get_details(self) -> None: - """Test get_details() method.""" - param1 = Param(name="test", description="desc", values=["values"]) - command1 = Command(command_strings=["echo test"], params=[param1]) - assert command1.get_details() == { - "command_strings": ["echo test"], - "user_params": [ - {"name": "test", "values": ["values"], "description": "desc"} - ], - } - - def test__eq__(self) -> None: - """Test equality method with different cases.""" - param1 = Param("test", "desc", ["values"]) - param2 = Param("test1", "desc1", ["values1"]) - command1 = Command(command_strings=["echo test"], params=[param1]) - command2 = Command(command_strings=["echo test"], params=[param1]) - command3 = Command(command_strings=["echo test"]) - command4 = Command(command_strings=["echo test"], params=[param2]) - command5 = object() - - assert command1 == command2 - assert command1 != command3 - assert command1 != command4 - assert command1 != command5 - - @pytest.mark.parametrize( - "params, expected_error", - [ - [[], does_not_raise()], - [[Param("param", "param description", [])], does_not_raise()], - [ - [ - Param("param", "param description", [], None, "alias"), - Param("param", "param description", [], None), - ], - does_not_raise(), - ], - [ - [ - Param("param1", "param1 description", [], None, "alias1"), - Param("param2", "param2 description", [], None, "alias2"), - ], - does_not_raise(), - ], - [ - [ - Param("param", "param description", [], None, "alias"), - Param("param", "param description", [], None, "alias"), - ], - pytest.raises(ConfigurationException, match="Non-unique aliases alias"), - ], - [ - [ - Param("alias", "param description", [], None, "alias1"), - Param("param", "param description", [], None, "alias"), - ], - pytest.raises( - ConfigurationException, - match="Aliases .* could not be used as parameter name", - ), - ], - [ - [ - Param("alias", "param description", [], None, "alias"), - Param("param1", "param1 description", [], None, "alias1"), - ], - does_not_raise(), - ], - [ - [ - Param("alias", "param description", [], None, "alias"), - Param("alias", "param1 description", [], None, "alias1"), - ], - pytest.raises( - ConfigurationException, - match="Aliases .* could not be used as parameter name", - ), - ], - [ - [ - Param("param1", "param1 description", [], None, "alias1"), - Param("param2", "param2 description", [], None, "alias1"), - Param("param3", "param3 description", [], None, "alias2"), - Param("param4", "param4 description", [], None, "alias2"), - ], - pytest.raises( - ConfigurationException, match="Non-unique aliases alias1, alias2" - ), - ], - ], - ) - def test_validate_params(self, params: list[Param], expected_error: Any) -> None: - """Test command validation function.""" - with expected_error: - Command([], params) diff --git a/tests/test_backend_executor_execution.py b/tests/test_backend_executor_execution.py deleted file mode 100644 index 6a6ea08..0000000 --- a/tests/test_backend_executor_execution.py +++ /dev/null @@ -1,212 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Test backend execution module.""" -from pathlib import Path -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from mlia.backend.executor.application import Application -from mlia.backend.executor.common import UserParamConfig -from mlia.backend.executor.config import ApplicationConfig -from mlia.backend.executor.config import SystemConfig -from mlia.backend.executor.execution import ExecutionContext -from mlia.backend.executor.execution import get_application_and_system -from mlia.backend.executor.execution import get_application_by_name_and_system -from mlia.backend.executor.execution import ParamResolver -from mlia.backend.executor.execution import run_application -from mlia.backend.executor.system import load_system - - -def test_context_param_resolver(tmpdir: Any) -> None: - """Test parameter resolving.""" - system_config_location = Path(tmpdir) / "system" - system_config_location.mkdir() - - application_config_location = Path(tmpdir) / "application" - application_config_location.mkdir() - - ctx = ExecutionContext( - app=Application( - ApplicationConfig( - name="test_application", - description="Test application", - config_location=application_config_location, - commands={ - "run": [ - "run_command1 {user_params:0}", - "run_command2 {user_params:1}", - ] - }, - variables={"var_1": "value for var_1"}, - user_params={ - "run": [ - UserParamConfig( - name="--param1", - description="Param 1", - default_value="123", - alias="param_1", - ), - UserParamConfig( - name="--param2", description="Param 2", default_value="456" - ), - UserParamConfig( - name="--param3", description="Param 3", alias="param_3" - ), - UserParamConfig( - name="--param4=", - description="Param 4", - default_value="456", - alias="param_4", - ), - UserParamConfig( - description="Param 5", - default_value="789", - alias="param_5", - ), - ] - }, - ) - ), - app_params=["--param2=789"], - system=load_system( - SystemConfig( - name="test_system", - description="Test system", - config_location=system_config_location, - commands={ - "build": ["build_command1 {user_params:0}"], - "run": ["run_command {application.commands.run:1}"], - }, - variables={"var_1": "value for var_1"}, - user_params={ - "build": [ - UserParamConfig( - name="--param1", description="Param 1", default_value="aaa" - ), - UserParamConfig(name="--param2", description="Param 2"), - ] - }, - ) - ), - system_params=["--param1=bbb"], - ) - - param_resolver = ParamResolver(ctx) - expected_values = { - "application.name": "test_application", - "application.description": "Test application", - "application.config_dir": str(application_config_location), - "application.commands.run:0": "run_command1 --param1 123", - "application.commands.run.params:0": "123", - "application.commands.run.params:param_1": "123", - "application.commands.run:1": "run_command2 --param2 789", - "application.commands.run.params:1": "789", - "application.variables:var_1": "value for var_1", - "system.name": "test_system", - "system.description": "Test system", - "system.config_dir": str(system_config_location), - "system.commands.build:0": "build_command1 --param1 bbb", - "system.commands.run:0": "run_command run_command2 --param2 789", - "system.commands.build.params:0": "bbb", - "system.variables:var_1": "value for var_1", - } - - for param, value in expected_values.items(): - assert param_resolver(param) == value - - expected_errors = { - "application.variables:var_2": pytest.raises( - Exception, match="Unknown variable var_2" - ), - "application.commands.clean:0": pytest.raises( - Exception, match="Command clean not found" - ), - "application.commands.run:2": pytest.raises( - Exception, match="Invalid index 2 for command run" - ), - "application.commands.run.params:5": pytest.raises( - Exception, match="Invalid parameter index 5 for command run" - ), - "application.commands.run.params:param_2": pytest.raises( - Exception, - match="No value for parameter with index or alias param_2 of command run", - ), - "UNKNOWN": pytest.raises( - Exception, match="Unable to resolve parameter UNKNOWN" - ), - "system.commands.build.params:1": pytest.raises( - Exception, - match="No value for parameter with index or alias 1 of command build", - ), - "system.commands.build:A": pytest.raises( - Exception, match="Bad command index A" - ), - "system.variables:var_2": pytest.raises( - Exception, match="Unknown variable var_2" - ), - } - for param, error in expected_errors.items(): - with error: - param_resolver(param) - - resolved_params = ctx.app.resolved_parameters("run", []) - expected_user_params = { - "user_params:0": "--param1 123", - "user_params:param_1": "--param1 123", - "user_params:2": "--param3", - "user_params:param_3": "--param3", - "user_params:3": "--param4=456", - "user_params:param_4": "--param4=456", - "user_params:param_5": "789", - } - for param, expected_value in expected_user_params.items(): - assert param_resolver(param, "run", resolved_params) == expected_value - - with pytest.raises( - Exception, match="Invalid index 5 for user params of command run" - ): - param_resolver("user_params:5", "run", resolved_params) - - with pytest.raises( - Exception, match="No user parameter for command 'run' with alias 'param_2'." - ): - param_resolver("user_params:param_2", "run", resolved_params) - - with pytest.raises(Exception, match="Unable to resolve user params"): - param_resolver("user_params:0", "", resolved_params) - - -def test_get_application_by_name_and_system(monkeypatch: Any) -> None: - """Test exceptional case for get_application_by_name_and_system.""" - monkeypatch.setattr( - "mlia.backend.executor.execution.get_application", - MagicMock(return_value=[MagicMock(), MagicMock()]), - ) - - with pytest.raises( - ValueError, - match="Error during getting application test_application for the " - "system test_system", - ): - get_application_by_name_and_system("test_application", "test_system") - - -def test_get_application_and_system(monkeypatch: Any) -> None: - """Test exceptional case for get_application_and_system.""" - monkeypatch.setattr( - "mlia.backend.executor.execution.get_system", MagicMock(return_value=None) - ) - - with pytest.raises(ValueError, match="System test_system is not found"): - get_application_and_system("test_application", "test_system") - - -def test_run_application() -> None: - """Test function run_application.""" - ctx = run_application("application_4", [], "System 4", []) - - assert isinstance(ctx, ExecutionContext) - assert ctx.stderr is not None and not ctx.stderr.decode() - assert ctx.stdout is not None and ctx.stdout.decode().strip() == "application_4" diff --git a/tests/test_backend_executor_fs.py b/tests/test_backend_executor_fs.py deleted file mode 100644 index 298b8db..0000000 --- a/tests/test_backend_executor_fs.py +++ /dev/null @@ -1,138 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Module for testing fs.py.""" -from __future__ import annotations - -from contextlib import ExitStack as does_not_raise -from pathlib import Path -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from mlia.backend.executor.fs import get_backends_path -from mlia.backend.executor.fs import recreate_directory -from mlia.backend.executor.fs import remove_directory -from mlia.backend.executor.fs import remove_resource -from mlia.backend.executor.fs import ResourceType -from mlia.backend.executor.fs import valid_for_filename - - -@pytest.mark.parametrize( - "resource_name,expected_path", - [ - ("systems", does_not_raise()), - ("applications", does_not_raise()), - ("whaaat", pytest.raises(ResourceWarning)), - (None, pytest.raises(ResourceWarning)), - ], -) -def test_get_backends_path(resource_name: ResourceType, expected_path: Any) -> None: - """Test get_resources() with multiple parameters.""" - with expected_path: - resource_path = get_backends_path(resource_name) - assert resource_path.exists() - - -def test_remove_resource_wrong_directory( - monkeypatch: Any, test_applications_path: Path -) -> None: - """Test removing resource with wrong directory.""" - mock_get_resources = MagicMock(return_value=test_applications_path) - monkeypatch.setattr( - "mlia.backend.executor.fs.get_backends_path", mock_get_resources - ) - - mock_shutil_rmtree = MagicMock() - monkeypatch.setattr("mlia.backend.executor.fs.shutil.rmtree", mock_shutil_rmtree) - - with pytest.raises(Exception, match="Resource .* does not exist"): - remove_resource("unknown", "applications") - mock_shutil_rmtree.assert_not_called() - - with pytest.raises(Exception, match="Wrong resource .*"): - remove_resource("readme.txt", "applications") - mock_shutil_rmtree.assert_not_called() - - -def test_remove_resource(monkeypatch: Any, test_applications_path: Path) -> None: - """Test removing resource data.""" - mock_get_resources = MagicMock(return_value=test_applications_path) - monkeypatch.setattr( - "mlia.backend.executor.fs.get_backends_path", mock_get_resources - ) - - mock_shutil_rmtree = MagicMock() - monkeypatch.setattr("mlia.backend.executor.fs.shutil.rmtree", mock_shutil_rmtree) - - remove_resource("application1", "applications") - mock_shutil_rmtree.assert_called_once() - - -def test_remove_directory(tmpdir: Any) -> None: - """Test directory removal.""" - tmpdir_path = Path(tmpdir) - tmpfile = tmpdir_path / "temp.txt" - - for item in [None, tmpfile]: - with pytest.raises(Exception, match="No directory path provided"): - remove_directory(item) - - newdir = tmpdir_path / "newdir" - newdir.mkdir() - - assert newdir.is_dir() - remove_directory(newdir) - assert not newdir.exists() - - -def test_recreate_directory(tmpdir: Any) -> None: - """Test directory recreation.""" - with pytest.raises(Exception, match="No directory path provided"): - recreate_directory(None) - - tmpdir_path = Path(tmpdir) - tmpfile = tmpdir_path / "temp.txt" - tmpfile.touch() - with pytest.raises(Exception, match="Path .* does exist and it is not a directory"): - recreate_directory(tmpfile) - - newdir = tmpdir_path / "newdir" - newdir.mkdir() - newfile = newdir / "newfile" - newfile.touch() - assert list(newdir.iterdir()) == [newfile] - recreate_directory(newdir) - assert not list(newdir.iterdir()) - - newdir2 = tmpdir_path / "newdir2" - assert not newdir2.exists() - recreate_directory(newdir2) - assert newdir2.is_dir() - - -def write_to_file( - write_directory: Any, write_mode: str, write_text: str | bytes -) -> Path: - """Write some text to a temporary test file.""" - tmpdir_path = Path(write_directory) - tmpfile = tmpdir_path / "file_name.txt" - with open(tmpfile, write_mode) as file: # pylint: disable=unspecified-encoding - file.write(write_text) - return tmpfile - - -@pytest.mark.parametrize( - "value, replacement, expected_result", - [ - ["", "", ""], - ["123", "", "123"], - ["123", "_", "123"], - ["/some_folder/some_script.sh", "", "some_foldersome_script.sh"], - ["/some_folder/some_script.sh", "_", "_some_folder_some_script.sh"], - ["!;'some_name$%^!", "_", "___some_name____"], - ], -) -def test_valid_for_filename(value: str, replacement: str, expected_result: str) -> None: - """Test function valid_for_filename.""" - assert valid_for_filename(value, replacement) == expected_result diff --git a/tests/test_backend_executor_output_consumer.py b/tests/test_backend_executor_output_consumer.py deleted file mode 100644 index 537084f..0000000 --- a/tests/test_backend_executor_output_consumer.py +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Tests for the output parsing.""" -from __future__ import annotations - -import base64 -import json -from typing import Any - -import pytest - -from mlia.backend.executor.output_consumer import Base64OutputConsumer -from mlia.backend.executor.output_consumer import OutputConsumer - - -OUTPUT_MATCH_ALL = bytearray( - """ -String1: My awesome string! -String2: STRINGS_ARE_GREAT!!! -Int: 12 -Float: 3.14 -""", - encoding="utf-8", -) - -OUTPUT_NO_MATCH = bytearray( - """ -This contains no matches... -Test1234567890!"£$%^&*()_+@~{}[]/.,<>?| -""", - encoding="utf-8", -) - -OUTPUT_PARTIAL_MATCH = bytearray( - "String1: My awesome string!", - encoding="utf-8", -) - -REGEX_CONFIG = { - "FirstString": {"pattern": r"String1.*: (.*)", "type": "str"}, - "SecondString": {"pattern": r"String2.*: (.*)!!!", "type": "str"}, - "IntegerValue": {"pattern": r"Int.*: (.*)", "type": "int"}, - "FloatValue": {"pattern": r"Float.*: (.*)", "type": "float"}, -} - -EMPTY_REGEX_CONFIG: dict[str, dict[str, Any]] = {} - -EXPECTED_METRICS_ALL = { - "FirstString": "My awesome string!", - "SecondString": "STRINGS_ARE_GREAT", - "IntegerValue": 12, - "FloatValue": 3.14, -} - -EXPECTED_METRICS_PARTIAL = { - "FirstString": "My awesome string!", -} - - -@pytest.mark.parametrize( - "expected_metrics", - [ - EXPECTED_METRICS_ALL, - EXPECTED_METRICS_PARTIAL, - ], -) -def test_base64_output_consumer(expected_metrics: dict) -> None: - """ - Make sure the Base64OutputConsumer yields valid results. - - I.e. return an empty dict if either the input or the config is empty and - return the parsed metrics otherwise. - """ - parser = Base64OutputConsumer() - assert isinstance(parser, OutputConsumer) - - def create_base64_output(expected_metrics: dict) -> bytearray: - json_str = json.dumps(expected_metrics, indent=4) - json_b64 = base64.b64encode(json_str.encode("utf-8")) - return ( - OUTPUT_MATCH_ALL # Should not be matched by the Base64OutputConsumer - + f"<{Base64OutputConsumer.TAG_NAME}>".encode() - + bytearray(json_b64) - + f"".encode() - + OUTPUT_NO_MATCH # Just to add some difficulty... - ) - - output = create_base64_output(expected_metrics) - - consumed = False - for line in output.splitlines(): - if parser.feed(line.decode("utf-8")): - consumed = True - assert consumed # we should have consumed at least one line - - res = parser.parsed_output - assert len(res) == 1 - assert isinstance(res, list) - for val in res: - assert val == expected_metrics diff --git a/tests/test_backend_executor_proc.py b/tests/test_backend_executor_proc.py deleted file mode 100644 index e8caf8a..0000000 --- a/tests/test_backend_executor_proc.py +++ /dev/null @@ -1,190 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=attribute-defined-outside-init,not-callable -"""Pytests for testing mlia/backend/proc.py.""" -from pathlib import Path -from typing import Any -from unittest import mock - -import pytest -from sh import ErrorReturnCode - -from mlia.backend.executor.proc import Command -from mlia.backend.executor.proc import CommandFailedException -from mlia.backend.executor.proc import CommandNotFound -from mlia.backend.executor.proc import parse_command -from mlia.backend.executor.proc import print_command_stdout -from mlia.backend.executor.proc import run_and_wait -from mlia.backend.executor.proc import ShellCommand -from mlia.backend.executor.proc import terminate_command - - -class TestShellCommand: - """Sample class for collecting tests.""" - - def test_run_ls(self, monkeypatch: Any) -> None: - """Test a simple ls command.""" - mock_command = mock.MagicMock() - monkeypatch.setattr(Command, "bake", mock_command) - - mock_get_stdout_stderr_paths = mock.MagicMock() - mock_get_stdout_stderr_paths.return_value = ("/path/std.out", "/path/std.err") - monkeypatch.setattr( - ShellCommand, "get_stdout_stderr_paths", mock_get_stdout_stderr_paths - ) - - shell_command = ShellCommand() - shell_command.run("ls", "-l") - assert mock_command.mock_calls[0] == mock.call(("-l",)) - assert mock_command.mock_calls[1] == mock.call()( - _bg=True, - _err="/path/std.err", - _out="/path/std.out", - _tee=True, - _bg_exc=False, - ) - - def test_run_command_not_found(self) -> None: - """Test whe the command doesn't exist.""" - shell_command = ShellCommand() - with pytest.raises(CommandNotFound): - shell_command.run("lsl", "-l") - - def test_get_stdout_stderr_paths(self) -> None: - """Test the method to get files to store stdout and stderr.""" - shell_command = ShellCommand() - out, err = shell_command.get_stdout_stderr_paths("cmd") - assert out.exists() and out.is_file() - assert err.exists() and err.is_file() - assert "cmd" in out.name - assert "cmd" in err.name - - -@mock.patch("builtins.print") -def test_print_command_stdout_alive(mock_print: Any) -> None: - """Test the print command stdout with an alive (running) process.""" - mock_command = mock.MagicMock() - mock_command.is_alive.return_value = True - mock_command.next.side_effect = ["test1", "test2", StopIteration] - - print_command_stdout(mock_command) - - mock_command.assert_has_calls( - [mock.call.is_alive(), mock.call.next(), mock.call.next()] - ) - mock_print.assert_has_calls( - [mock.call("test1", end=""), mock.call("test2", end="")] - ) - - -@mock.patch("builtins.print") -def test_print_command_stdout_not_alive(mock_print: Any) -> None: - """Test the print command stdout with a not alive (exited) process.""" - mock_command = mock.MagicMock() - mock_command.is_alive.return_value = False - mock_command.stdout = "test" - - print_command_stdout(mock_command) - mock_command.assert_has_calls([mock.call.is_alive()]) - mock_print.assert_called_once_with("test") - - -def test_terminate_command_no_process() -> None: - """Test command termination when process does not exist.""" - mock_command = mock.MagicMock() - mock_command.process.signal_group.side_effect = ProcessLookupError() - - terminate_command(mock_command) - mock_command.process.signal_group.assert_called_once() - mock_command.is_alive.assert_not_called() - - -def test_terminate_command() -> None: - """Test command termination.""" - mock_command = mock.MagicMock() - mock_command.is_alive.return_value = False - - terminate_command(mock_command) - mock_command.process.signal_group.assert_called_once() - - -def test_terminate_command_case1() -> None: - """Test command termination when it takes time..""" - mock_command = mock.MagicMock() - mock_command.is_alive.side_effect = [True, True, False] - - terminate_command(mock_command, wait_period=0.1) - mock_command.process.signal_group.assert_called_once() - assert mock_command.is_alive.call_count == 3 - - -def test_terminate_command_case2() -> None: - """Test command termination when it takes much time..""" - mock_command = mock.MagicMock() - mock_command.is_alive.side_effect = [True, True, True] - - terminate_command(mock_command, number_of_attempts=3, wait_period=0.1) - assert mock_command.is_alive.call_count == 3 - assert mock_command.process.signal_group.call_count == 2 - - -class TestRunAndWait: - """Test run_and_wait function.""" - - @pytest.fixture(autouse=True) - def setup_method(self, monkeypatch: Any) -> None: - """Init test method.""" - self.execute_command_mock = mock.MagicMock() - monkeypatch.setattr( - "mlia.backend.executor.proc.execute_command", self.execute_command_mock - ) - - self.terminate_command_mock = mock.MagicMock() - monkeypatch.setattr( - "mlia.backend.executor.proc.terminate_command", - self.terminate_command_mock, - ) - - def test_if_execute_command_raises_exception(self) -> None: - """Test if execute_command fails.""" - self.execute_command_mock.side_effect = Exception("Error!") - with pytest.raises(Exception, match="Error!"): - run_and_wait("command", Path.cwd()) - - def test_if_command_finishes_with_error(self) -> None: - """Test if command finishes with error.""" - cmd_mock = mock.MagicMock() - self.execute_command_mock.return_value = cmd_mock - exit_code_mock = mock.PropertyMock( - side_effect=ErrorReturnCode("cmd", bytearray(), bytearray()) - ) - type(cmd_mock).exit_code = exit_code_mock - - with pytest.raises(CommandFailedException): - run_and_wait("command", Path.cwd()) - - @pytest.mark.parametrize("terminate_on_error, call_count", ((False, 0), (True, 1))) - def test_if_command_finishes_with_exception( - self, terminate_on_error: bool, call_count: int - ) -> None: - """Test if command finishes with error.""" - cmd_mock = mock.MagicMock() - self.execute_command_mock.return_value = cmd_mock - exit_code_mock = mock.PropertyMock(side_effect=Exception("Error!")) - type(cmd_mock).exit_code = exit_code_mock - - with pytest.raises(Exception, match="Error!"): - run_and_wait("command", Path.cwd(), terminate_on_error=terminate_on_error) - - assert self.terminate_command_mock.call_count == call_count - - -def test_parse_command() -> None: - """Test parse_command function.""" - assert parse_command("1.sh") == ["bash", "1.sh"] - # The following line raises a B604 bandit error. In our case we specify - # what shell to use instead of using the default one. It is a safe use - # we are ignoring this instance. - assert parse_command("1.sh", shell="sh") == ["sh", "1.sh"] # nosec - assert parse_command("command") == ["command"] - assert parse_command("command 123 --param=1") == ["command", "123", "--param=1"] diff --git a/tests/test_backend_executor_runner.py b/tests/test_backend_executor_runner.py deleted file mode 100644 index 36c6e5e..0000000 --- a/tests/test_backend_executor_runner.py +++ /dev/null @@ -1,254 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Tests for module backend/manager.""" -from __future__ import annotations - -from pathlib import Path -from unittest.mock import MagicMock -from unittest.mock import PropertyMock - -import pytest - -from mlia.backend.corstone.performance import BackendRunner -from mlia.backend.corstone.performance import ExecutionParams - - -class TestBackendRunner: - """Tests for BackendRunner class.""" - - @staticmethod - def _setup_backends( - monkeypatch: pytest.MonkeyPatch, - available_systems: list[str] | None = None, - available_apps: list[str] | None = None, - ) -> None: - """Set up backend metadata.""" - - def mock_system(system: str) -> MagicMock: - """Mock the System instance.""" - mock = MagicMock() - type(mock).name = PropertyMock(return_value=system) - return mock - - def mock_app(app: str) -> MagicMock: - """Mock the Application instance.""" - mock = MagicMock() - type(mock).name = PropertyMock(return_value=app) - mock.can_run_on.return_value = True - return mock - - system_mocks = [mock_system(name) for name in (available_systems or [])] - monkeypatch.setattr( - "mlia.backend.executor.runner.get_available_systems", - MagicMock(return_value=system_mocks), - ) - - apps_mock = [mock_app(name) for name in (available_apps or [])] - monkeypatch.setattr( - "mlia.backend.executor.runner.get_available_applications", - MagicMock(return_value=apps_mock), - ) - - @pytest.mark.parametrize( - "available_systems, system, installed", - [ - ([], "system1", False), - (["system1", "system2"], "system1", True), - ], - ) - def test_is_system_installed( - self, - available_systems: list, - system: str, - installed: bool, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """Test method is_system_installed.""" - backend_runner = BackendRunner() - - self._setup_backends(monkeypatch, available_systems) - - assert backend_runner.is_system_installed(system) == installed - - @pytest.mark.parametrize( - "available_systems, systems", - [ - ([], []), - (["system1"], ["system1"]), - ], - ) - def test_installed_systems( - self, - available_systems: list[str], - systems: list[str], - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """Test method installed_systems.""" - backend_runner = BackendRunner() - - self._setup_backends(monkeypatch, available_systems) - assert backend_runner.get_installed_systems() == systems - - @staticmethod - def test_install_system(monkeypatch: pytest.MonkeyPatch) -> None: - """Test system installation.""" - install_system_mock = MagicMock() - monkeypatch.setattr( - "mlia.backend.executor.runner.install_system", install_system_mock - ) - - backend_runner = BackendRunner() - backend_runner.install_system(Path("test_system_path")) - - install_system_mock.assert_called_once_with(Path("test_system_path")) - - @pytest.mark.parametrize( - "available_systems, systems, expected_result", - [ - ([], [], False), - (["system1"], [], False), - (["system1"], ["system1"], True), - (["system1", "system2"], ["system1", "system3"], False), - (["system1", "system2"], ["system1", "system2"], True), - ], - ) - def test_systems_installed( - self, - available_systems: list[str], - systems: list[str], - expected_result: bool, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """Test method systems_installed.""" - self._setup_backends(monkeypatch, available_systems) - - backend_runner = BackendRunner() - - assert backend_runner.systems_installed(systems) is expected_result - - @pytest.mark.parametrize( - "available_apps, applications, expected_result", - [ - ([], [], False), - (["app1"], [], False), - (["app1"], ["app1"], True), - (["app1", "app2"], ["app1", "app3"], False), - (["app1", "app2"], ["app1", "app2"], True), - ], - ) - def test_applications_installed( - self, - available_apps: list[str], - applications: list[str], - expected_result: bool, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """Test method applications_installed.""" - self._setup_backends(monkeypatch, [], available_apps) - backend_runner = BackendRunner() - - assert backend_runner.applications_installed(applications) is expected_result - - @pytest.mark.parametrize( - "available_apps, applications", - [ - ([], []), - ( - ["application1", "application2"], - ["application1", "application2"], - ), - ], - ) - def test_get_installed_applications( - self, - available_apps: list[str], - applications: list[str], - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """Test method get_installed_applications.""" - self._setup_backends(monkeypatch, [], available_apps) - - backend_runner = BackendRunner() - assert applications == backend_runner.get_installed_applications() - - @staticmethod - def test_install_application(monkeypatch: pytest.MonkeyPatch) -> None: - """Test application installation.""" - mock_install_application = MagicMock() - monkeypatch.setattr( - "mlia.backend.executor.runner.install_application", - mock_install_application, - ) - - backend_runner = BackendRunner() - backend_runner.install_application(Path("test_application_path")) - mock_install_application.assert_called_once_with(Path("test_application_path")) - - @pytest.mark.parametrize( - "available_apps, application, installed", - [ - ([], "system1", False), - ( - ["application1", "application2"], - "application1", - True, - ), - ( - [], - "application1", - False, - ), - ], - ) - def test_is_application_installed( - self, - available_apps: list[str], - application: str, - installed: bool, - monkeypatch: pytest.MonkeyPatch, - ) -> None: - """Test method is_application_installed.""" - self._setup_backends(monkeypatch, [], available_apps) - - backend_runner = BackendRunner() - assert installed == backend_runner.is_application_installed( - application, "system1" - ) - - @staticmethod - @pytest.mark.parametrize( - "execution_params, expected_command", - [ - ( - ExecutionParams("application_4", "System 4", [], []), - ["application_4", [], "System 4", []], - ), - ( - ExecutionParams( - "application_6", - "System 6", - ["param1=value2"], - ["sys-param1=value2"], - ), - [ - "application_6", - ["param1=value2"], - "System 6", - ["sys-param1=value2"], - ], - ), - ], - ) - def test_run_application_local( - monkeypatch: pytest.MonkeyPatch, - execution_params: ExecutionParams, - expected_command: list[str], - ) -> None: - """Test method run_application with local systems.""" - run_app = MagicMock() - monkeypatch.setattr("mlia.backend.executor.runner.run_application", run_app) - - backend_runner = BackendRunner() - backend_runner.run_application(execution_params) - - run_app.assert_called_once_with(*expected_command) diff --git a/tests/test_backend_executor_source.py b/tests/test_backend_executor_source.py deleted file mode 100644 index 3aa336e..0000000 --- a/tests/test_backend_executor_source.py +++ /dev/null @@ -1,205 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Tests for the source backend module.""" -from collections import Counter -from contextlib import ExitStack as does_not_raise -from pathlib import Path -from typing import Any -from unittest.mock import MagicMock -from unittest.mock import patch - -import pytest - -from mlia.backend.executor.common import ConfigurationException -from mlia.backend.executor.source import create_destination_and_install -from mlia.backend.executor.source import DirectorySource -from mlia.backend.executor.source import get_source -from mlia.backend.executor.source import TarArchiveSource - - -def test_create_destination_and_install(test_systems_path: Path, tmpdir: Any) -> None: - """Test create_destination_and_install function.""" - system_directory = test_systems_path / "system1" - - dir_source = DirectorySource(system_directory) - resources = Path(tmpdir) - create_destination_and_install(dir_source, resources) - assert (resources / "system1").is_dir() - - -@patch( - "mlia.backend.executor.source.DirectorySource.create_destination", - return_value=False, -) -def test_create_destination_and_install_if_dest_creation_not_required( - mock_ds_create_destination: Any, tmpdir: Any -) -> None: - """Test create_destination_and_install function.""" - dir_source = DirectorySource(Path("unknown")) - resources = Path(tmpdir) - with pytest.raises(Exception): - create_destination_and_install(dir_source, resources) - - mock_ds_create_destination.assert_called_once() - - -def test_create_destination_and_install_if_installation_fails(tmpdir: Any) -> None: - """Test create_destination_and_install function if installation fails.""" - dir_source = DirectorySource(Path("unknown")) - resources = Path(tmpdir) - with pytest.raises(Exception, match="Directory .* does not exist"): - create_destination_and_install(dir_source, resources) - assert not (resources / "unknown").exists() - assert resources.exists() - - -def test_create_destination_and_install_if_name_is_empty() -> None: - """Test create_destination_and_install function fails if source name is empty.""" - source = MagicMock() - source.create_destination.return_value = True - source.name.return_value = None - - with pytest.raises(Exception, match="Unable to get source name"): - create_destination_and_install(source, Path("some_path")) - - source.install_into.assert_not_called() - - -@pytest.mark.parametrize( - "source_path, expected_class, expected_error", - [ - ( - Path("backends/applications/application1/"), - DirectorySource, - does_not_raise(), - ), - ( - Path("archives/applications/application1.tar.gz"), - TarArchiveSource, - does_not_raise(), - ), - ( - Path("doesnt/exist"), - None, - pytest.raises( - ConfigurationException, match="Unable to read .*doesnt/exist" - ), - ), - ], -) -def test_get_source( - source_path: Path, - expected_class: Any, - expected_error: Any, - test_resources_path: Path, -) -> None: - """Test get_source function.""" - with expected_error: - full_source_path = test_resources_path / source_path - source = get_source(full_source_path) - assert isinstance(source, expected_class) - - -class TestDirectorySource: - """Test DirectorySource class.""" - - @pytest.mark.parametrize( - "directory, name", - [ - (Path("/some/path/some_system"), "some_system"), - (Path("some_system"), "some_system"), - ], - ) - def test_name(self, directory: Path, name: str) -> None: - """Test getting source name.""" - assert DirectorySource(directory).name() == name - - def test_install_into(self, test_systems_path: Path, tmpdir: Any) -> None: - """Test install directory into destination.""" - system_directory = test_systems_path / "system1" - - dir_source = DirectorySource(system_directory) - with pytest.raises(Exception, match="Wrong destination .*"): - dir_source.install_into(Path("unknown_destination")) - - tmpdir_path = Path(tmpdir) - dir_source.install_into(tmpdir_path) - source_files = [f.name for f in system_directory.iterdir()] - dest_files = [f.name for f in tmpdir_path.iterdir()] - assert Counter(source_files) == Counter(dest_files) - - def test_install_into_unknown_source_directory(self, tmpdir: Any) -> None: - """Test install system from unknown directory.""" - with pytest.raises(Exception, match="Directory .* does not exist"): - DirectorySource(Path("unknown_directory")).install_into(Path(tmpdir)) - - -class TestTarArchiveSource: - """Test TarArchiveSource class.""" - - @pytest.mark.parametrize( - "archive, name", - [ - (Path("some_archive.tgz"), "some_archive"), - (Path("some_archive.tar.gz"), "some_archive"), - (Path("some_archive"), "some_archive"), - ("archives/systems/system1.tar.gz", "system1"), - ("archives/systems/system1_dir.tar.gz", "system1"), - ], - ) - def test_name(self, test_resources_path: Path, archive: Path, name: str) -> None: - """Test getting source name.""" - assert TarArchiveSource(test_resources_path / archive).name() == name - - def test_install_into(self, test_resources_path: Path, tmpdir: Any) -> None: - """Test install archive into destination.""" - system_archive = test_resources_path / "archives/systems/system1.tar.gz" - - tar_source = TarArchiveSource(system_archive) - with pytest.raises(Exception, match="Wrong destination .*"): - tar_source.install_into(Path("unknown_destination")) - - tmpdir_path = Path(tmpdir) - tar_source.install_into(tmpdir_path) - source_files = [ - "backend-config.json.license", - "backend-config.json", - "system_artifact", - ] - dest_files = [f.name for f in tmpdir_path.iterdir()] - assert Counter(source_files) == Counter(dest_files) - - def test_install_into_unknown_source_archive(self, tmpdir: Any) -> None: - """Test install unknown source archive.""" - with pytest.raises(Exception, match="File .* does not exist"): - TarArchiveSource(Path("unknown.tar.gz")).install_into(Path(tmpdir)) - - def test_install_into_unsupported_source_archive(self, tmpdir: Any) -> None: - """Test install unsupported file type.""" - plain_text_file = Path(tmpdir) / "test_file" - plain_text_file.write_text("Not a system config") - - with pytest.raises(Exception, match="Unsupported archive type .*"): - TarArchiveSource(plain_text_file).install_into(Path(tmpdir)) - - def test_lazy_property_init(self, test_resources_path: Path) -> None: - """Test that class properties initialized correctly.""" - system_archive = test_resources_path / "archives/systems/system1.tar.gz" - - tar_source = TarArchiveSource(system_archive) - assert tar_source.name() == "system1" - assert tar_source.config() is not None - assert tar_source.create_destination() - - tar_source = TarArchiveSource(system_archive) - assert tar_source.config() is not None - assert tar_source.create_destination() - assert tar_source.name() == "system1" - - def test_create_destination_property(self, test_resources_path: Path) -> None: - """Test create_destination property filled correctly for different archives.""" - system_archive1 = test_resources_path / "archives/systems/system1.tar.gz" - system_archive2 = test_resources_path / "archives/systems/system1_dir.tar.gz" - - assert TarArchiveSource(system_archive1).create_destination() - assert not TarArchiveSource(system_archive2).create_destination() diff --git a/tests/test_backend_executor_system.py b/tests/test_backend_executor_system.py deleted file mode 100644 index c94ef30..0000000 --- a/tests/test_backend_executor_system.py +++ /dev/null @@ -1,358 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Tests for system backend.""" -from __future__ import annotations - -from contextlib import ExitStack as does_not_raise -from pathlib import Path -from typing import Any -from unittest.mock import MagicMock - -import pytest - -from mlia.backend.executor.common import Command -from mlia.backend.executor.common import ConfigurationException -from mlia.backend.executor.common import Param -from mlia.backend.executor.common import UserParamConfig -from mlia.backend.executor.config import SystemConfig -from mlia.backend.executor.system import get_available_systems -from mlia.backend.executor.system import get_system -from mlia.backend.executor.system import install_system -from mlia.backend.executor.system import load_system -from mlia.backend.executor.system import remove_system -from mlia.backend.executor.system import System - - -def test_get_available_systems() -> None: - """Test get_available_systems mocking get_resources.""" - available_systems = get_available_systems() - assert all(isinstance(s, System) for s in available_systems) - assert len(available_systems) == 4 - assert [str(s) for s in available_systems] == [ - "System 1", - "System 2", - "System 4", - "System 6", - ] - - -def test_get_system() -> None: - """Test get_system.""" - system1 = get_system("System 1") - assert isinstance(system1, System) - assert system1.name == "System 1" - - system2 = get_system("System 2") - # check that comparison with object of another type returns false - assert system1 != 42 - assert system1 != system2 - - with pytest.raises( - ConfigurationException, match="System 'Unknown system' not found." - ): - get_system("Unknown system") - - -@pytest.mark.parametrize( - "source, call_count, exception_type", - ( - ( - "archives/systems/system1.tar.gz", - 0, - pytest.raises(Exception, match="Systems .* are already installed"), - ), - ( - "archives/systems/system3.tar.gz", - 0, - pytest.raises(Exception, match="Unable to read system definition"), - ), - ( - "backends/systems/system1", - 0, - pytest.raises(Exception, match="Systems .* are already installed"), - ), - ( - "backends/systems/system3", - 0, - pytest.raises(Exception, match="Unable to read system definition"), - ), - ("unknown_path", 0, pytest.raises(Exception, match="Unable to read")), - ( - "various/systems/system_with_empty_config", - 0, - pytest.raises(Exception, match="No system definition found"), - ), - ("various/systems/system_with_valid_config", 1, does_not_raise()), - ), -) -def test_install_system( - monkeypatch: Any, - test_resources_path: Path, - source: str, - call_count: int, - exception_type: Any, -) -> None: - """Test system installation from archive.""" - mock_create_destination_and_install = MagicMock() - monkeypatch.setattr( - "mlia.backend.executor.system.create_destination_and_install", - mock_create_destination_and_install, - ) - - with exception_type: - install_system(test_resources_path / source) - - assert mock_create_destination_and_install.call_count == call_count - - -def test_remove_system(monkeypatch: Any) -> None: - """Test system removal.""" - mock_remove_backend = MagicMock() - monkeypatch.setattr( - "mlia.backend.executor.system.remove_backend", mock_remove_backend - ) - remove_system("some_system_dir") - mock_remove_backend.assert_called_once() - - -def test_system() -> None: - """Test the System class.""" - config = SystemConfig(name="System 1") - system = System(config) - assert str(system) == "System 1" - assert system.name == "System 1" - - -def test_system_with_empty_parameter_name() -> None: - """Test that configuration fails if parameter name is empty.""" - bad_config = SystemConfig( - name="System 1", - commands={"run": ["run"]}, - user_params={"run": [{"name": "", "values": ["1", "2", "3"]}]}, - ) - with pytest.raises(Exception, match="Parameter has an empty 'name' attribute."): - System(bad_config) - - -def test_system_run() -> None: - """Test run operation for system.""" - system = get_system("System 4") - assert isinstance(system, System) - - system.run("echo 'application run'") - - -def test_system_start_no_config_location() -> None: - """Test that system without config location could not start.""" - system = load_system(SystemConfig(name="test")) - - assert isinstance(system, System) - with pytest.raises( - ConfigurationException, match="System has invalid config location: None" - ): - system.run("sleep 100") - - -@pytest.mark.parametrize( - "config, expected_class, expected_error", - [ - ( - SystemConfig(name="test"), - System, - does_not_raise(), - ), - (SystemConfig(), None, pytest.raises(ConfigurationException)), - ], -) -def test_load_system( - config: SystemConfig, expected_class: type, expected_error: Any -) -> None: - """Test load_system function.""" - if not expected_class: - with expected_error: - load_system(config) - else: - system = load_system(config) - assert isinstance(system, expected_class) - - -def test_load_system_populate_shared_params() -> None: - """Test shared parameters population.""" - with pytest.raises(Exception, match="All shared parameters should have aliases"): - load_system( - SystemConfig( - name="test_system", - user_params={ - "shared": [ - UserParamConfig( - name="--shared_param1", - description="Shared parameter", - values=["1", "2", "3"], - default_value="1", - ) - ] - }, - ) - ) - - with pytest.raises( - Exception, match="All parameters for command run should have aliases" - ): - load_system( - SystemConfig( - name="test_system", - user_params={ - "shared": [ - UserParamConfig( - name="--shared_param1", - description="Shared parameter", - values=["1", "2", "3"], - default_value="1", - alias="shared_param1", - ) - ], - "run": [ - UserParamConfig( - name="--run_param1", - description="Run specific parameter", - values=["1", "2", "3"], - default_value="2", - ) - ], - }, - ) - ) - system0 = load_system( - SystemConfig( - name="test_system", - commands={"run": ["run_command"]}, - user_params={ - "shared": [], - "run": [ - UserParamConfig( - name="--run_param1", - description="Run specific parameter", - values=["1", "2", "3"], - default_value="2", - alias="run_param1", - ) - ], - }, - ) - ) - assert len(system0.commands) == 1 - run_command1 = system0.commands["run"] - assert run_command1 == Command( - ["run_command"], - [ - Param( - "--run_param1", - "Run specific parameter", - ["1", "2", "3"], - "2", - "run_param1", - ) - ], - ) - - system1 = load_system( - SystemConfig( - name="test_system", - user_params={ - "shared": [ - UserParamConfig( - name="--shared_param1", - description="Shared parameter", - values=["1", "2", "3"], - default_value="1", - alias="shared_param1", - ) - ], - "run": [ - UserParamConfig( - name="--run_param1", - description="Run specific parameter", - values=["1", "2", "3"], - default_value="2", - alias="run_param1", - ) - ], - }, - ) - ) - assert len(system1.commands) == 1 - - run_command1 = system1.commands["run"] - assert run_command1 == Command( - [], - [ - Param( - "--shared_param1", - "Shared parameter", - ["1", "2", "3"], - "1", - "shared_param1", - ), - Param( - "--run_param1", - "Run specific parameter", - ["1", "2", "3"], - "2", - "run_param1", - ), - ], - ) - - system2 = load_system( - SystemConfig( - name="test_system", - commands={"build": ["build_command"]}, - user_params={ - "shared": [ - UserParamConfig( - name="--shared_param1", - description="Shared parameter", - values=["1", "2", "3"], - default_value="1", - alias="shared_param1", - ) - ], - "run": [ - UserParamConfig( - name="--run_param1", - description="Run specific parameter", - values=["1", "2", "3"], - default_value="2", - alias="run_param1", - ) - ], - }, - ) - ) - assert len(system2.commands) == 2 - build_command2 = system2.commands["build"] - assert build_command2 == Command( - ["build_command"], - [], - ) - - run_command2 = system1.commands["run"] - assert run_command2 == Command( - [], - [ - Param( - "--shared_param1", - "Shared parameter", - ["1", "2", "3"], - "1", - "shared_param1", - ), - Param( - "--run_param1", - "Run specific parameter", - ["1", "2", "3"], - "2", - "run_param1", - ), - ], - ) diff --git a/tests/test_backend_install.py b/tests/test_backend_install.py index c3efe09..dacb1aa 100644 --- a/tests/test_backend_install.py +++ b/tests/test_backend_install.py @@ -3,91 +3,203 @@ """Tests for common management functionality.""" from __future__ import annotations +import tarfile from pathlib import Path +from unittest.mock import ANY +from unittest.mock import MagicMock import pytest from mlia.backend.install import BackendInfo -from mlia.backend.install import get_all_application_names +from mlia.backend.install import BackendInstallation +from mlia.backend.install import CompoundPathChecker +from mlia.backend.install import DownloadAndInstall +from mlia.backend.install import InstallFromPath +from mlia.backend.install import PackagePathChecker from mlia.backend.install import StaticPathChecker -from mlia.backend.registry import get_supported_backends -from mlia.target.registry import is_supported +from mlia.backend.repo import BackendRepository + + +@pytest.fixture(name="backend_repo") +def mock_backend_repo(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + """Mock backend repository.""" + mock = MagicMock(spec=BackendRepository) + monkeypatch.setattr("mlia.backend.install.get_backend_repository", lambda: mock) + + return mock + + +def test_wrong_install_type() -> None: + """Test that installation should fail for wrong install type.""" + installation = BackendInstallation( + "sample_backend", + "Sample backend", + "sample_backend", + None, + None, + lambda path: None, + None, + ) + + assert not installation.supports("some_path") # type: ignore + + with pytest.raises(Exception): + installation.install("some_path") # type: ignore @pytest.mark.parametrize( - "copy_source, system_config", + "supported_platforms, expected_result", [ - (True, "system_config.json"), - (True, None), - (False, "system_config.json"), - (False, None), + [None, True], + [["UNKNOWN"], False], ], ) -def test_static_path_checker( - tmp_path: Path, copy_source: bool, system_config: str +def test_backend_could_be_installed( + supported_platforms: list[str] | None, expected_result: bool +) -> None: + """Test method could_be_installed.""" + installation = BackendInstallation( + "sample_backend", + "Sample backend", + "sample_backend", + None, + supported_platforms, + lambda path: None, + None, + ) + + assert installation.could_be_installed == expected_result + + +@pytest.mark.parametrize("copy_source", [True, False]) +def test_backend_installation_from_path( + tmp_path: Path, backend_repo: MagicMock, copy_source: bool +) -> None: + """Test methods of backend installation.""" + installation = BackendInstallation( + "sample_backend", + "Sample backend", + "sample_backend", + None, + None, + lambda path: BackendInfo(path, copy_source=copy_source), + None, + ) + + assert installation.supports(InstallFromPath(tmp_path)) + assert not installation.supports(DownloadAndInstall()) + + installation.install(InstallFromPath(tmp_path)) + + if copy_source: + backend_repo.copy_backend.assert_called_with( + "sample_backend", tmp_path, "sample_backend", None + ) + backend_repo.add_backend.assert_not_called() + else: + backend_repo.copy_backend.assert_not_called() + backend_repo.add_backend.assert_called_with("sample_backend", tmp_path, None) + + +def test_backend_installation_download_and_install( + tmp_path: Path, backend_repo: MagicMock ) -> None: - """Test static path checker.""" - checker = StaticPathChecker(tmp_path, ["file1.txt"], copy_source, system_config) - tmp_path.joinpath("file1.txt").touch() + """Test methods of backend installation.""" + download_artifact_mock = MagicMock() - result = checker(tmp_path) - assert result == BackendInfo(tmp_path, copy_source, system_config) + tmp_archive = tmp_path.joinpath("sample.tgz") + sample_file = tmp_path.joinpath("sample.txt") + sample_file.touch() + with tarfile.open(tmp_archive, "w:gz") as archive: + archive.add(sample_file) -def test_static_path_checker_invalid_path(tmp_path: Path) -> None: - """Test static path checker with invalid path.""" - checker = StaticPathChecker(tmp_path, ["file1.txt"]) + download_artifact_mock.download_to.return_value = tmp_archive - result = checker(tmp_path) - assert result is None + installation = BackendInstallation( + "sample_backend", + "Sample backend", + "sample_backend", + download_artifact_mock, + None, + lambda path: BackendInfo(path, copy_source=False), + lambda eula_agreement, path: path, + ) - result = checker(tmp_path / "unknown_directory") - assert result is None + assert installation.supports(DownloadAndInstall()) + installation.install(DownloadAndInstall()) + backend_repo.add_backend.assert_called_with("sample_backend", ANY, None) -def test_supported_backends() -> None: - """Test function supported backends.""" - assert get_supported_backends() == [ - "ArmNNTFLiteDelegate", - "Corstone-300", - "Corstone-310", - "TOSA-Checker", - "Vela", - ] +def test_backend_installation_unable_to_download() -> None: + """Test that installation should fail when downloading fails.""" + download_artifact_mock = MagicMock() + download_artifact_mock.download_to.side_effect = Exception("Download error") -@pytest.mark.parametrize( - "backend, expected_result", - [ - ["unknown_backend", False], - ["Corstone-300", True], - ["Corstone-310", True], - ], -) -def test_is_supported(backend: str, expected_result: bool) -> None: - """Test function is_supported.""" - assert is_supported(backend) == expected_result + installation = BackendInstallation( + "sample_backend", + "Sample backend", + "sample_backend", + download_artifact_mock, + None, + lambda path: BackendInfo(path, copy_source=False), + lambda eula_agreement, path: path, + ) + with pytest.raises(Exception, match="Unable to download backend artifact"): + installation.install(DownloadAndInstall()) -@pytest.mark.parametrize( - "backend, expected_result", - [ - [ - "Corstone-300", - [ - "Generic Inference Runner: Ethos-U55", - "Generic Inference Runner: Ethos-U65", - ], - ], - [ - "Corstone-310", - [ - "Generic Inference Runner: Ethos-U55", - "Generic Inference Runner: Ethos-U65", - ], - ], - ], -) -def test_get_all_application_names(backend: str, expected_result: list[str]) -> None: - """Test function get_all_application_names.""" - assert sorted(get_all_application_names(backend)) == expected_result + +def test_static_path_checker(tmp_path: Path) -> None: + """Test for StaticPathChecker.""" + checker1 = StaticPathChecker(tmp_path, []) + assert checker1(tmp_path) == BackendInfo(tmp_path, copy_source=False) + + checker2 = StaticPathChecker(tmp_path / "dist", []) + assert checker2(tmp_path) is None + + checker3 = StaticPathChecker(tmp_path, ["sample.txt"]) + + assert checker3(tmp_path) is None + + sample_file = tmp_path.joinpath("sample.txt") + sample_file.touch() + + assert checker3(tmp_path) == BackendInfo(tmp_path, copy_source=False) + + +def test_compound_path_checker(tmp_path: Path) -> None: + """Test for CompoundPathChecker.""" + static_checker = StaticPathChecker(tmp_path, []) + compound_checker = CompoundPathChecker(static_checker) + + assert compound_checker(tmp_path) == BackendInfo(tmp_path, copy_source=False) + + +def test_package_path_checker(tmp_path: Path) -> None: + """Test PackagePathChecker.""" + sample_dir = tmp_path.joinpath("sample") + sample_dir.mkdir() + + checker1 = PackagePathChecker([], "sample") + assert checker1(tmp_path) == BackendInfo(tmp_path / "sample") + + checker2 = PackagePathChecker(["sample.txt"], "sample") + assert checker2(tmp_path) is None + + +def test_backend_installation_uninstall(backend_repo: MagicMock) -> None: + """Test backend removing process.""" + installation = BackendInstallation( + "sample_backend", + "Sample backend", + "sample_backend", + None, + None, + lambda path: None, + None, + ) + + installation.uninstall() + backend_repo.remove_backend.assert_called_with("sample_backend") diff --git a/tests/test_backend_manager.py b/tests/test_backend_manager.py index 19cb357..879353e 100644 --- a/tests/test_backend_manager.py +++ b/tests/test_backend_manager.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for installation manager.""" from __future__ import annotations @@ -16,6 +16,8 @@ from mlia.backend.install import Installation from mlia.backend.install import InstallationType from mlia.backend.install import InstallFromPath from mlia.backend.manager import DefaultInstallationManager +from mlia.core.errors import ConfigurationError +from mlia.core.errors import InternalError def get_default_installation_manager_mock( @@ -255,6 +257,25 @@ def test_installation_manager_install_from( install_mock.uninstall.assert_not_called() +def test_installation_manager_unsupported_install_type( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """Test that installation could not be installed via unsupported type.""" + download_install_mock = _could_be_downloaded_and_installed_mock() + install_from_mock = _could_be_installed_from_mock() + install_mocks = [download_install_mock, install_from_mock] + + manager = get_installation_manager(False, install_mocks, monkeypatch) + manager.install_from(tmp_path, "could_be_downloaded_and_installed") + + manager.download_and_install("could_be_installed_from") + + for mock in install_mocks: + mock.install.assert_not_called() + mock.uninstall.assert_not_called() + + @pytest.mark.parametrize("noninteractive", [True, False]) @pytest.mark.parametrize( "install_mock, backend_name, expected_call", @@ -280,3 +301,39 @@ def test_installation_manager_uninstall( manager.uninstall(backend_name) assert install_mock.uninstall.mock_calls == expected_call + + +def test_installation_internal_error(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that manager should be able to detect wrong state.""" + install_mock = _ready_for_uninstall_mock() + manager = get_installation_manager(False, [install_mock, install_mock], monkeypatch) + + with pytest.raises( + InternalError, + match="More than one installed backend with name already_installed found", + ): + manager.uninstall("already_installed") + + +def test_uninstall_unknown_backend(monkeypatch: pytest.MonkeyPatch) -> None: + """Test that uninstall should fail for uknown backend.""" + install_mock = _ready_for_uninstall_mock() + manager = get_installation_manager(False, [install_mock, install_mock], monkeypatch) + + with pytest.raises( + ConfigurationError, match="Backend 'some_backend' is not installed" + ): + manager.uninstall("some_backend") + + +def test_show_env_details(monkeypatch: pytest.MonkeyPatch) -> None: + """Test method show_env_details.""" + ready_to_install_mock = _ready_for_installation_mock() + could_be_installed_mock = _could_be_installed_from_mock() + + manager = get_installation_manager( + False, + [ready_to_install_mock, could_be_installed_mock], + monkeypatch, + ) + manager.show_env_details() diff --git a/tests/test_backend_registry.py b/tests/test_backend_registry.py index 703e699..e3e2da2 100644 --- a/tests/test_backend_registry.py +++ b/tests/test_backend_registry.py @@ -9,6 +9,8 @@ import pytest from mlia.backend.config import BackendType from mlia.backend.config import System +from mlia.backend.registry import get_supported_backends +from mlia.backend.registry import get_supported_systems from mlia.backend.registry import registry from mlia.core.common import AdviceCategory @@ -35,7 +37,7 @@ from mlia.core.common import AdviceCategory BackendType.CUSTOM, ), ( - "TOSA-Checker", + "tosa-checker", [AdviceCategory.COMPATIBILITY], [System.LINUX_AMD64], BackendType.WHEEL, @@ -79,3 +81,19 @@ def test_backend_registry( cfg.supported_systems ), f"Supported systems differs: {advices} != {cfg.supported_advice}" assert cfg.type == type_ + + +def test_get_supported_backends() -> None: + """Test function get_supported_backends.""" + assert get_supported_backends() == [ + "ArmNNTFLiteDelegate", + "Corstone-300", + "Corstone-310", + "Vela", + "tosa-checker", + ] + + +def test_get_supported_systems() -> None: + """Test function get_supported_systems.""" + assert get_supported_systems() diff --git a/tests/test_backend_repo.py b/tests/test_backend_repo.py new file mode 100644 index 0000000..5071989 --- /dev/null +++ b/tests/test_backend_repo.py @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright 2023, Arm Limited and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for backend repository.""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from mlia.backend.repo import BackendRepository +from mlia.backend.repo import get_backend_repository + + +def test_get_backend_repository(tmp_path: Path) -> None: + """Test function get_backend_repository.""" + repo_path = tmp_path / "repo" + repo = get_backend_repository(repo_path) + + assert isinstance(repo, BackendRepository) + + backends_dir = repo_path / "backends" + assert backends_dir.is_dir() + assert not list(backends_dir.iterdir()) + + config_file = repo_path / "mlia_config.json" + assert config_file.is_file() + assert json.loads(config_file.read_text()) == {"backends": []} + + +def test_backend_repository_wrong_directory(tmp_path: Path) -> None: + """Test that repository instance should throw error for the wrong directory.""" + with pytest.raises( + Exception, match=f"Directory {tmp_path} could not be used as MLIA repository." + ): + BackendRepository(tmp_path) + + +def test_empty_backend_repository(tmp_path: Path) -> None: + """Test empty backend repository.""" + repo_path = tmp_path / "repo" + repo = get_backend_repository(repo_path) + + assert not repo.is_backend_installed("sample_backend") + + with pytest.raises(Exception, match="Backend sample_backend is not installed."): + repo.remove_backend("sample_backend") + + with pytest.raises(Exception, match="Backend sample_backend is not installed."): + repo.get_backend_settings("sample_backend") + + +def test_adding_backend(tmp_path: Path) -> None: + """Test adding backend to the repository.""" + repo_path = tmp_path / "repo" + repo = get_backend_repository(repo_path) + + backend_path = tmp_path.joinpath("backend") + backend_path.mkdir() + + settings = {"param": "value"} + repo.add_backend("sample_backend", backend_path, settings) + + backends_dir = repo_path / "backends" + assert backends_dir.is_dir() + assert not list(backends_dir.iterdir()) + + assert repo.is_backend_installed("sample_backend") + expected_settings = { + "param": "value", + "backend_path": backend_path.as_posix(), + } + assert repo.get_backend_settings("sample_backend") == ( + backend_path, + expected_settings, + ) + + config_file = repo_path / "mlia_config.json" + expected_content = { + "backends": [ + { + "name": "sample_backend", + "settings": { + "backend_path": backend_path.as_posix(), + "param": "value", + }, + } + ] + } + assert json.loads(config_file.read_text()) == expected_content + + with pytest.raises(Exception, match="Backend sample_backend already installed"): + repo.add_backend("sample_backend", backend_path, settings) + + repo.remove_backend("sample_backend") + assert not repo.is_backend_installed("sample_backend") + + +def test_copy_backend(tmp_path: Path) -> None: + """Test copying backend to the repository.""" + repo_path = tmp_path / "repo" + repo = get_backend_repository(repo_path) + + backend_path = tmp_path.joinpath("backend") + backend_path.mkdir() + + backend_path.joinpath("sample.txt").touch() + + settings = {"param": "value"} + repo.copy_backend("sample_backend", backend_path, "sample_backend_dir", settings) + + repo_backend_path = repo_path.joinpath("backends", "sample_backend_dir") + assert repo_backend_path.is_dir() + assert repo_backend_path.joinpath("sample.txt").is_file() + + config_file = repo_path / "mlia_config.json" + expected_content = { + "backends": [ + { + "name": "sample_backend", + "settings": { + "backend_dir": "sample_backend_dir", + "param": "value", + }, + } + ] + } + assert json.loads(config_file.read_text()) == expected_content + + expected_settings = { + "param": "value", + "backend_dir": "sample_backend_dir", + } + assert repo.get_backend_settings("sample_backend") == ( + repo_backend_path, + expected_settings, + ) + + repo.remove_backend("sample_backend") + assert not repo_backend_path.exists() diff --git a/tests/test_cli_command_validators.py b/tests/test_cli_command_validators.py index 13514a5..cd048ee 100644 --- a/tests/test_cli_command_validators.py +++ b/tests/test_cli_command_validators.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +from contextlib import ExitStack from unittest.mock import MagicMock import pytest @@ -125,9 +126,9 @@ def test_validate_check_target_profile( None, ], ["tosa", None, False, None, ["tosa-checker"]], - ["cortex-a", None, False, None, ["armnn-tflitedelegate"]], + ["cortex-a", None, False, None, ["ArmNNTFLiteDelegate"]], ["tosa", ["tosa-checker"], False, None, ["tosa-checker"]], - ["cortex-a", ["armnn-tflitedelegate"], False, None, ["armnn-tflitedelegate"]], + ["cortex-a", ["ArmNNTFLiteDelegate"], False, None, ["ArmNNTFLiteDelegate"]], [ "ethos-u55-256", ["Vela", "Corstone-300"], @@ -158,10 +159,11 @@ def test_validate_backend( MagicMock(return_value=["Vela", "Corstone-300"]), ) + exit_stack = ExitStack() if throws_exception: - with pytest.raises(argparse.ArgumentError) as err: - validate_backend(input_target_profile, input_backends) - assert str(err.value.message) == exception_message - return + exit_stack.enter_context( + pytest.raises(argparse.ArgumentError, match=exception_message) + ) - assert validate_backend(input_target_profile, input_backends) == output_backends + with exit_stack: + assert validate_backend(input_target_profile, input_backends) == output_backends diff --git a/tests/test_cli_config.py b/tests/test_cli_config.py index 1487f11..8494d73 100644 --- a/tests/test_cli_config.py +++ b/tests/test_cli_config.py @@ -8,7 +8,6 @@ from unittest.mock import MagicMock import pytest from mlia.cli.config import get_default_backends -from mlia.cli.config import is_corstone_backend @pytest.mark.parametrize( @@ -28,7 +27,7 @@ from mlia.cli.config import is_corstone_backend ["Vela", "Corstone-300", "New backend"], ], [["ArmNNTFLiteDelegate"], ["ArmNNTFLiteDelegate"]], - [["TOSA-Checker"], ["TOSA-Checker"]], + [["tosa-checker"], ["tosa-checker"]], [ ["ArmNNTFLiteDelegate", "Corstone-300"], ["ArmNNTFLiteDelegate", "Corstone-300"], @@ -47,10 +46,3 @@ def test_get_default_backends( ) assert get_default_backends() == expected_default_backends - - -def test_is_corstone_backend() -> None: - """Test function is_corstone_backend.""" - assert is_corstone_backend("Corstone-300") is True - assert is_corstone_backend("Corstone-310") is True - assert is_corstone_backend("New backend") is False diff --git a/tests/test_resources/application_config.json b/tests/test_resources/application_config.json deleted file mode 100644 index 8c5d2b5..0000000 --- a/tests/test_resources/application_config.json +++ /dev/null @@ -1,94 +0,0 @@ -[ - { - "name": "application_1", - "description": "application number one", - "supported_systems": [ - "system_1", - "system_2" - ], - "commands": { - "clean": [ - "clean_cmd_11" - ], - "build": [ - "build_cmd_11" - ], - "run": [ - "run_cmd_11" - ], - "post_run": [ - "post_run_cmd_11" - ] - }, - "user_params": { - "run": [ - { - "name": "run_param_11", - "values": [], - "description": "run param number one" - } - ], - "build": [ - { - "name": "build_param_11", - "values": [], - "description": "build param number one" - }, - { - "name": "build_param_12", - "values": [], - "description": "build param number two" - }, - { - "name": "build_param_13", - "values": [ - "value_1" - ], - "description": "build param number three with some value" - } - ] - } - }, - { - "name": "application_2", - "description": "application number two", - "supported_systems": [ - "system_2" - ], - "commands": { - "clean": [ - "clean_cmd_21" - ], - "build": [ - "build_cmd_21", - "build_cmd_22" - ], - "run": [ - "run_cmd_21" - ], - "post_run": [ - "post_run_cmd_21" - ] - }, - "user_params": { - "build": [ - { - "name": "build_param_21", - "values": [], - "description": "build param number one" - }, - { - "name": "build_param_22", - "values": [], - "description": "build param number two" - }, - { - "name": "build_param_23", - "values": [], - "description": "build param number three" - } - ], - "run": [] - } - } -] diff --git a/tests/test_resources/application_config.json.license b/tests/test_resources/application_config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/application_config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/backends/applications/application1/backend-config.json b/tests/test_resources/backends/applications/application1/backend-config.json deleted file mode 100644 index 96d5420..0000000 --- a/tests/test_resources/backends/applications/application1/backend-config.json +++ /dev/null @@ -1,29 +0,0 @@ -[ - { - "name": "application_1", - "description": "This is application 1", - "supported_systems": [ - { - "name": "System 1" - } - ], - "commands": { - "clean": [ - "echo 'clean'" - ], - "build": [ - "echo 'build'" - ], - "run": [ - "echo 'run'" - ], - "post_run": [ - "echo 'post_run'" - ] - }, - "user_params": { - "build": [], - "run": [] - } - } -] diff --git a/tests/test_resources/backends/applications/application1/backend-config.json.license b/tests/test_resources/backends/applications/application1/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/backends/applications/application1/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/backends/applications/application2/backend-config.json b/tests/test_resources/backends/applications/application2/backend-config.json deleted file mode 100644 index 3a3969a..0000000 --- a/tests/test_resources/backends/applications/application2/backend-config.json +++ /dev/null @@ -1,29 +0,0 @@ -[ - { - "name": "application_2", - "description": "This is application 2", - "supported_systems": [ - { - "name": "System 2" - } - ], - "commands": { - "clean": [ - "echo 'clean'" - ], - "build": [ - "echo 'build'" - ], - "run": [ - "echo 'run'" - ], - "post_run": [ - "echo 'post_run'" - ] - }, - "user_params": { - "build": [], - "run": [] - } - } -] diff --git a/tests/test_resources/backends/applications/application2/backend-config.json.license b/tests/test_resources/backends/applications/application2/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/backends/applications/application2/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/backends/applications/application3/readme.txt b/tests/test_resources/backends/applications/application3/readme.txt deleted file mode 100644 index 8c72c05..0000000 --- a/tests/test_resources/backends/applications/application3/readme.txt +++ /dev/null @@ -1,4 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -SPDX-License-Identifier: Apache-2.0 - -This application does not have json configuration file diff --git a/tests/test_resources/backends/applications/application4/backend-config.json b/tests/test_resources/backends/applications/application4/backend-config.json deleted file mode 100644 index 11017e4..0000000 --- a/tests/test_resources/backends/applications/application4/backend-config.json +++ /dev/null @@ -1,33 +0,0 @@ -[ - { - "name": "application_4", - "description": "This is application 4", - "variables": { - "build_dir": "build" - }, - "supported_systems": [ - { - "name": "System 4" - } - ], - "commands": { - "run": [ - "echo {application.name}" - ] - }, - "user_params": { - "run": [ - { - "name": "--app", - "description": "Sample command param", - "values": [ - "application1", - "application2", - "application3" - ], - "default_value": "application1" - } - ] - } - } -] diff --git a/tests/test_resources/backends/applications/application4/backend-config.json.license b/tests/test_resources/backends/applications/application4/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/backends/applications/application4/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/backends/applications/application5/backend-config.json b/tests/test_resources/backends/applications/application5/backend-config.json deleted file mode 100644 index 219494c..0000000 --- a/tests/test_resources/backends/applications/application5/backend-config.json +++ /dev/null @@ -1,134 +0,0 @@ -[ - { - "name": "application_5", - "description": "This is application 5", - "supported_systems": [ - { - "name": "System 1" - }, - { - "name": "System 2" - } - ], - "variables": { - "var1": "value1", - "var2": "value2" - }, - "commands": { - "build": [ - "default build command" - ], - "run": [ - "default run command" - ] - }, - "user_params": { - "build": [], - "run": [] - } - }, - { - "name": "application_5A", - "description": "This is application 5A", - "supported_systems": [ - { - "name": "System 1", - "variables": { - "var1": "new value1" - } - }, - { - "name": "System 2", - "variables": { - "var2": "new value2" - }, - "commands": { - "run": [ - "run command on system 2" - ] - } - } - ], - "variables": { - "var1": "value1", - "var2": "value2" - }, - "commands": { - "build": [ - "default build command" - ], - "run": [ - "default run command" - ] - }, - "user_params": { - "build": [], - "run": [] - } - }, - { - "name": "application_5B", - "description": "This is application 5B", - "supported_systems": [ - { - "name": "System 1", - "variables": { - "var1": "value for var1 System1", - "var2": "value for var2 System1" - } - }, - { - "name": "System 2", - "variables": { - "var1": "value for var1 System2", - "var2": "value for var2 System2" - }, - "commands": { - "run": [ - "run command on system 2" - ] - }, - "user_params": { - "run": [] - } - } - ], - "commands": { - "build": [ - "default build command with {variables:var1}" - ], - "run": [ - "default run command with {variables:var2}" - ] - }, - "user_params": { - "build": [ - { - "name": "--param", - "description": "Sample command param", - "values": [ - "value1", - "value2", - "value3" - ], - "default_value": "value1", - "alias": "param1" - } - ], - "run": [], - "non_used_command": [ - { - "name": "--not-used", - "description": "Not used param anywhere", - "values": [ - "value1", - "value2", - "value3" - ], - "default_value": "value1", - "alias": "param1" - } - ] - } - } -] diff --git a/tests/test_resources/backends/applications/application5/backend-config.json.license b/tests/test_resources/backends/applications/application5/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/backends/applications/application5/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/backends/applications/application6/backend-config.json b/tests/test_resources/backends/applications/application6/backend-config.json deleted file mode 100644 index 81afebd..0000000 --- a/tests/test_resources/backends/applications/application6/backend-config.json +++ /dev/null @@ -1,41 +0,0 @@ -[ - { - "name": "application_6", - "description": "This is application 6", - "supported_systems": [ - { - "name": "System 6" - } - ], - "commands": { - "clean": [ - "echo 'clean'" - ], - "build": [ - "echo 'build'" - ], - "run": [ - "echo 'run {user_params:param1}'" - ], - "post_run": [ - "echo 'post_run'" - ] - }, - "user_params": { - "build": [], - "run": [ - { - "name": "--param1", - "description": "Test parameter", - "values": [ - "value1", - "value2", - "value3" - ], - "default_value": "value3", - "alias": "param1" - } - ] - } - } -] diff --git a/tests/test_resources/backends/applications/application6/backend-config.json.license b/tests/test_resources/backends/applications/application6/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/backends/applications/application6/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/backends/applications/readme.txt b/tests/test_resources/backends/applications/readme.txt deleted file mode 100644 index d3e6fe2..0000000 --- a/tests/test_resources/backends/applications/readme.txt +++ /dev/null @@ -1,4 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -SPDX-License-Identifier: Apache-2.0 - -File for test purposes diff --git a/tests/test_resources/backends/systems/system1/backend-config.json b/tests/test_resources/backends/systems/system1/backend-config.json deleted file mode 100644 index 4454695..0000000 --- a/tests/test_resources/backends/systems/system1/backend-config.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "name": "System 1", - "description": "This is system 1", - "commands": { - "clean": [ - "echo 'clean'" - ], - "build": [ - "echo 'build'" - ], - "run": [ - "echo 'run'" - ], - "post_run": [ - "echo 'post_run'" - ] - }, - "user_params": { - "build": [], - "run": [] - } - } -] diff --git a/tests/test_resources/backends/systems/system1/backend-config.json.license b/tests/test_resources/backends/systems/system1/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/backends/systems/system1/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/backends/systems/system1/system_artifact/empty.txt b/tests/test_resources/backends/systems/system1/system_artifact/empty.txt deleted file mode 100644 index 487e9d8..0000000 --- a/tests/test_resources/backends/systems/system1/system_artifact/empty.txt +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/backends/systems/system2/backend-config.json b/tests/test_resources/backends/systems/system2/backend-config.json deleted file mode 100644 index 3359d3d..0000000 --- a/tests/test_resources/backends/systems/system2/backend-config.json +++ /dev/null @@ -1,24 +0,0 @@ -[ - { - "name": "System 2", - "description": "This is system 2", - "commands": { - "clean": [ - "echo 'clean'" - ], - "build": [ - "echo 'build'" - ], - "run": [ - "echo 'run'" - ], - "post_run": [ - "echo 'post_run'" - ] - }, - "user_params": { - "build": [], - "run": [] - } - } -] diff --git a/tests/test_resources/backends/systems/system2/backend-config.json.license b/tests/test_resources/backends/systems/system2/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/backends/systems/system2/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/backends/systems/system3/readme.txt b/tests/test_resources/backends/systems/system3/readme.txt deleted file mode 100644 index aba5a9c..0000000 --- a/tests/test_resources/backends/systems/system3/readme.txt +++ /dev/null @@ -1,4 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -SPDX-License-Identifier: Apache-2.0 - -This system does not have the json configuration file diff --git a/tests/test_resources/backends/systems/system4/backend-config.json b/tests/test_resources/backends/systems/system4/backend-config.json deleted file mode 100644 index daa4025..0000000 --- a/tests/test_resources/backends/systems/system4/backend-config.json +++ /dev/null @@ -1,15 +0,0 @@ -[ - { - "name": "System 4", - "description": "This is system 4", - "commands": { - "run": [ - "echo {system.name}", - "{application.commands.run:0}" - ] - }, - "user_params": { - "run": [] - } - } -] diff --git a/tests/test_resources/backends/systems/system4/backend-config.json.license b/tests/test_resources/backends/systems/system4/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/backends/systems/system4/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/backends/systems/system6/backend-config.json b/tests/test_resources/backends/systems/system6/backend-config.json deleted file mode 100644 index 5180799..0000000 --- a/tests/test_resources/backends/systems/system6/backend-config.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "name": "System 6", - "description": "This is system 6", - "variables": { - "var1": "{user_params:sys-param1}" - }, - "commands": { - "run": [ - "echo {application.name}", - "{application.commands.run:0}" - ] - }, - "user_params": { - "run": [ - { - "name": "--sys-param1", - "description": "Test parameter", - "values": [ - "value1", - "value2", - "value3" - ], - "default_value": "value1", - "alias": "sys-param1" - } - ] - } - } -] diff --git a/tests/test_resources/backends/systems/system6/backend-config.json.license b/tests/test_resources/backends/systems/system6/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/backends/systems/system6/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/hello_world.json b/tests/test_resources/hello_world.json deleted file mode 100644 index 28d7bd9..0000000 --- a/tests/test_resources/hello_world.json +++ /dev/null @@ -1,53 +0,0 @@ -[ - { - "name": "Hello world", - "description": "Sample application that displays 'Hello world!'", - "supported_systems": [ - "Sample System" - ], - "deploy_data": [ - [ - "src", - "/tmp/" - ], - [ - "README", - "/tmp/README.md" - ] - ], - "commands": { - "clean": [], - "build": [], - "run": [ - "echo 'Hello world!'", - "ls -l /tmp" - ], - "post_run": [] - }, - "user_params": { - "run": [ - { - "name": "--choice-param", - "values": [ - "value_1", - "value_2" - ], - "default_value": "value_1", - "description": "Choice param" - }, - { - "name": "--open-param", - "values": [], - "default_value": "value_4", - "description": "Open param" - }, - { - "name": "--enable-flag", - "default_value": "value_4", - "description": "Flag param" - } - ], - "build": [] - } - } -] diff --git a/tests/test_resources/hello_world.json.license b/tests/test_resources/hello_world.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/hello_world.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/scripts/test_backend_run b/tests/test_resources/scripts/test_backend_run deleted file mode 100755 index 548f577..0000000 --- a/tests/test_resources/scripts/test_backend_run +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 - -echo "Hello from script" ->&2 echo "Oops!" -sleep 100 diff --git a/tests/test_resources/scripts/test_backend_run_script.sh b/tests/test_resources/scripts/test_backend_run_script.sh deleted file mode 100644 index 548f577..0000000 --- a/tests/test_resources/scripts/test_backend_run_script.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 - -echo "Hello from script" ->&2 echo "Oops!" -sleep 100 diff --git a/tests/test_resources/various/applications/application_with_empty_config/backend-config.json b/tests/test_resources/various/applications/application_with_empty_config/backend-config.json deleted file mode 100644 index fe51488..0000000 --- a/tests/test_resources/various/applications/application_with_empty_config/backend-config.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_resources/various/applications/application_with_empty_config/backend-config.json.license b/tests/test_resources/various/applications/application_with_empty_config/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/various/applications/application_with_empty_config/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/various/applications/application_with_valid_config/backend-config.json b/tests/test_resources/various/applications/application_with_valid_config/backend-config.json deleted file mode 100644 index a457d9b..0000000 --- a/tests/test_resources/various/applications/application_with_valid_config/backend-config.json +++ /dev/null @@ -1,37 +0,0 @@ -[ - { - "name": "test_application", - "description": "This is test_application", - "variables": { - "build_dir": "build" - }, - "supported_systems": [ - { - "name": "System 4" - } - ], - "commands": { - "build": [ - "cp ../hello_app.txt ." - ], - "run": [ - "{application.variables:build_dir}/hello_app.txt" - ] - }, - "user_params": { - "build": [ - { - "name": "--app", - "description": "Sample command param", - "values": [ - "application1", - "application2", - "application3" - ], - "default_value": "application1" - } - ], - "run": [] - } - } -] diff --git a/tests/test_resources/various/applications/application_with_valid_config/backend-config.json.license b/tests/test_resources/various/applications/application_with_valid_config/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/various/applications/application_with_valid_config/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/various/applications/application_with_wrong_config1/backend-config.json b/tests/test_resources/various/applications/application_with_wrong_config1/backend-config.json deleted file mode 100644 index 724b31b..0000000 --- a/tests/test_resources/various/applications/application_with_wrong_config1/backend-config.json +++ /dev/null @@ -1,2 +0,0 @@ -This is not valid json file -{ diff --git a/tests/test_resources/various/applications/application_with_wrong_config1/backend-config.json.license b/tests/test_resources/various/applications/application_with_wrong_config1/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/various/applications/application_with_wrong_config1/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/various/applications/application_with_wrong_config2/backend-config.json b/tests/test_resources/various/applications/application_with_wrong_config2/backend-config.json deleted file mode 100644 index b64e6f8..0000000 --- a/tests/test_resources/various/applications/application_with_wrong_config2/backend-config.json +++ /dev/null @@ -1,32 +0,0 @@ -[ - { - "name": "test_application", - "description": "This is test_application", - "variables": { - "build_dir": "build" - }, - "commands": { - "build": [ - "cp ../hello_app.txt ." - ], - "run": [ - "{application.variables:build_dir}/hello_app.txt" - ] - }, - "user_params": { - "build": [ - { - "name": "--app", - "description": "Sample command param", - "values": [ - "application1", - "application2", - "application3" - ], - "default_value": "application1" - } - ], - "run": [] - } - } -] diff --git a/tests/test_resources/various/applications/application_with_wrong_config2/backend-config.json.license b/tests/test_resources/various/applications/application_with_wrong_config2/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/various/applications/application_with_wrong_config2/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/various/applications/application_with_wrong_config3/backend-config.json b/tests/test_resources/various/applications/application_with_wrong_config3/backend-config.json deleted file mode 100644 index 4a70cdd..0000000 --- a/tests/test_resources/various/applications/application_with_wrong_config3/backend-config.json +++ /dev/null @@ -1,37 +0,0 @@ -[ - { - "name": "test_application", - "description": "This is test_application", - "variables": { - "build_dir": "build" - }, - "supported_systems": [ - { - "anme": "System 4" - } - ], - "commands": { - "build": [ - "cp ../hello_app.txt ." - ], - "run": [ - "{application.variables:build_dir}/hello_app.txt" - ] - }, - "user_params": { - "build": [ - { - "name": "--app", - "description": "Sample command param", - "values": [ - "application1", - "application2", - "application3" - ], - "default_value": "application1" - } - ], - "run": [] - } - } -] diff --git a/tests/test_resources/various/applications/application_with_wrong_config3/backend-config.json.license b/tests/test_resources/various/applications/application_with_wrong_config3/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/various/applications/application_with_wrong_config3/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/various/systems/system_with_empty_config/backend-config.json b/tests/test_resources/various/systems/system_with_empty_config/backend-config.json deleted file mode 100644 index fe51488..0000000 --- a/tests/test_resources/various/systems/system_with_empty_config/backend-config.json +++ /dev/null @@ -1 +0,0 @@ -[] diff --git a/tests/test_resources/various/systems/system_with_empty_config/backend-config.json.license b/tests/test_resources/various/systems/system_with_empty_config/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/various/systems/system_with_empty_config/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_resources/various/systems/system_with_valid_config/backend-config.json b/tests/test_resources/various/systems/system_with_valid_config/backend-config.json deleted file mode 100644 index 83c3025..0000000 --- a/tests/test_resources/various/systems/system_with_valid_config/backend-config.json +++ /dev/null @@ -1,12 +0,0 @@ -[ - { - "name": "Test system", - "description": "This is a test system", - "commands": { - "run": [] - }, - "user_params": { - "run": [] - } - } -] diff --git a/tests/test_resources/various/systems/system_with_valid_config/backend-config.json.license b/tests/test_resources/various/systems/system_with_valid_config/backend-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/test_resources/various/systems/system_with_valid_config/backend-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/test_target_registry.py b/tests/test_target_registry.py index e6028a9..5012148 100644 --- a/tests/test_target_registry.py +++ b/tests/test_target_registry.py @@ -6,6 +6,7 @@ from __future__ import annotations import pytest from mlia.core.common import AdviceCategory +from mlia.target.registry import all_supported_backends from mlia.target.registry import registry from mlia.target.registry import supported_advice from mlia.target.registry import supported_backends @@ -13,7 +14,7 @@ from mlia.target.registry import supported_targets @pytest.mark.parametrize( - "expected_target", ("Cortex-A", "Ethos-U55", "Ethos-U65", "TOSA") + "expected_target", ("cortex-a", "ethos-u55", "ethos-u65", "tosa") ) def test_target_registry(expected_target: str) -> None: """Test the target registry.""" @@ -26,9 +27,9 @@ def test_target_registry(expected_target: str) -> None: @pytest.mark.parametrize( ("target_name", "expected_advices"), ( - ("Cortex-A", [AdviceCategory.COMPATIBILITY]), + ("cortex-a", [AdviceCategory.COMPATIBILITY]), ( - "Ethos-U55", + "ethos-u55", [ AdviceCategory.COMPATIBILITY, AdviceCategory.OPTIMIZATION, @@ -36,14 +37,14 @@ def test_target_registry(expected_target: str) -> None: ], ), ( - "Ethos-U65", + "ethos-u65", [ AdviceCategory.COMPATIBILITY, AdviceCategory.OPTIMIZATION, AdviceCategory.PERFORMANCE, ], ), - ("TOSA", [AdviceCategory.COMPATIBILITY]), + ("tosa", [AdviceCategory.COMPATIBILITY]), ), ) def test_supported_advice( @@ -58,10 +59,10 @@ def test_supported_advice( @pytest.mark.parametrize( ("target_name", "expected_backends"), ( - ("Cortex-A", ["ArmNNTFLiteDelegate"]), - ("Ethos-U55", ["Corstone-300", "Corstone-310", "Vela"]), - ("Ethos-U65", ["Corstone-300", "Corstone-310", "Vela"]), - ("TOSA", ["TOSA-Checker"]), + ("cortex-a", ["ArmNNTFLiteDelegate"]), + ("ethos-u55", ["Corstone-300", "Corstone-310", "Vela"]), + ("ethos-u65", ["Corstone-300", "Corstone-310", "Vela"]), + ("tosa", ["tosa-checker"]), ), ) def test_supported_backends(target_name: str, expected_backends: list[str]) -> None: @@ -72,11 +73,22 @@ def test_supported_backends(target_name: str, expected_backends: list[str]) -> N @pytest.mark.parametrize( ("advice", "expected_targets"), ( - (AdviceCategory.COMPATIBILITY, ["Cortex-A", "Ethos-U55", "Ethos-U65", "TOSA"]), - (AdviceCategory.OPTIMIZATION, ["Ethos-U55", "Ethos-U65"]), - (AdviceCategory.PERFORMANCE, ["Ethos-U55", "Ethos-U65"]), + (AdviceCategory.COMPATIBILITY, ["cortex-a", "ethos-u55", "ethos-u65", "tosa"]), + (AdviceCategory.OPTIMIZATION, ["ethos-u55", "ethos-u65"]), + (AdviceCategory.PERFORMANCE, ["ethos-u55", "ethos-u65"]), ), ) def test_supported_targets(advice: AdviceCategory, expected_targets: list[str]) -> None: """Test function supported_targets().""" assert sorted(expected_targets) == sorted(supported_targets(advice)) + + +def test_all_supported_backends() -> None: + """Test function all_supported_backends.""" + assert all_supported_backends() == { + "Vela", + "tosa-checker", + "ArmNNTFLiteDelegate", + "Corstone-310", + "Corstone-300", + } diff --git a/tests/test_utils_proc.py b/tests/test_utils_proc.py new file mode 100644 index 0000000..bea431f --- /dev/null +++ b/tests/test_utils_proc.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright 2023, Arm Limited and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for process management functions.""" +from unittest.mock import MagicMock + +from mlia.utils.proc import Command +from mlia.utils.proc import process_command_output + + +def test_process_command_output() -> None: + """Test function process_command_output.""" + command = Command(["echo", "-n", "sample message"]) + + output_consumer = MagicMock() + process_command_output(command, [output_consumer]) + + output_consumer.assert_called_once_with("sample message") -- cgit v1.2.1