diff options
author | Benjamin Klimczak <benjamin.klimczak@arm.com> | 2022-06-28 10:29:35 +0100 |
---|---|---|
committer | Benjamin Klimczak <benjamin.klimczak@arm.com> | 2022-07-08 10:57:19 +0100 |
commit | c9b4089b3037b5943565d76242d3016b8776f8d2 (patch) | |
tree | 3de24f79dedf0f26f492a7fa1562bf684e13a055 | |
parent | ba2c7fcccf37e8c81946f0776714c64f73191787 (diff) | |
download | mlia-c9b4089b3037b5943565d76242d3016b8776f8d2.tar.gz |
MLIA-546 Merge AIET into MLIA
Merge the deprecated AIET interface for backend execution into MLIA:
- Execute backends directly (without subprocess and the aiet CLI)
- Fix issues with the unit tests
- Remove src/aiet and tests/aiet
- Re-factor code to replace 'aiet' with 'backend'
- Adapt and improve unit tests after re-factoring
- Remove dependencies that are not needed anymore (click and cloup)
Change-Id: I450734c6a3f705ba9afde41862b29e797e511f7c
-rw-r--r-- | setup.cfg | 4 | ||||
-rw-r--r-- | src/aiet/__init__.py | 7 | ||||
-rw-r--r-- | src/aiet/backend/tool.py | 109 | ||||
-rw-r--r-- | src/aiet/cli/__init__.py | 28 | ||||
-rw-r--r-- | src/aiet/cli/application.py | 362 | ||||
-rw-r--r-- | src/aiet/cli/common.py | 173 | ||||
-rw-r--r-- | src/aiet/cli/completion.py | 72 | ||||
-rw-r--r-- | src/aiet/cli/system.py | 122 | ||||
-rw-r--r-- | src/aiet/cli/tool.py | 143 | ||||
-rw-r--r-- | src/aiet/main.py | 13 | ||||
-rw-r--r-- | src/aiet/resources/tools/vela/aiet-config.json | 73 | ||||
-rw-r--r-- | src/aiet/resources/tools/vela/check_model.py | 75 | ||||
-rw-r--r-- | src/aiet/resources/tools/vela/run_vela.py | 65 | ||||
-rw-r--r-- | src/aiet/resources/tools/vela/vela.ini | 53 | ||||
-rw-r--r-- | src/aiet/utils/__init__.py | 3 | ||||
-rw-r--r-- | src/aiet/utils/helpers.py | 17 | ||||
-rw-r--r-- | src/mlia/backend/__init__.py (renamed from src/aiet/backend/__init__.py) | 0 | ||||
-rw-r--r-- | src/mlia/backend/application.py (renamed from src/aiet/backend/application.py) | 28 | ||||
-rw-r--r-- | src/mlia/backend/common.py (renamed from src/aiet/backend/common.py) | 22 | ||||
-rw-r--r-- | src/mlia/backend/config.py (renamed from src/aiet/backend/config.py) | 18 | ||||
-rw-r--r-- | src/mlia/backend/controller.py (renamed from src/aiet/backend/controller.py) | 16 | ||||
-rw-r--r-- | src/mlia/backend/execution.py (renamed from src/aiet/backend/execution.py) | 148 | ||||
-rw-r--r-- | src/mlia/backend/fs.py (renamed from src/aiet/utils/fs.py) | 19 | ||||
-rw-r--r-- | src/mlia/backend/manager.py (renamed from src/mlia/tools/aiet_wrapper.py) | 108 | ||||
-rw-r--r-- | src/mlia/backend/output_parser.py (renamed from src/aiet/backend/output_parser.py) | 0 | ||||
-rw-r--r-- | src/mlia/backend/proc.py (renamed from src/aiet/utils/proc.py) | 2 | ||||
-rw-r--r-- | src/mlia/backend/protocol.py (renamed from src/aiet/backend/protocol.py) | 8 | ||||
-rw-r--r-- | src/mlia/backend/source.py (renamed from src/aiet/backend/source.py) | 18 | ||||
-rw-r--r-- | src/mlia/backend/system.py (renamed from src/aiet/backend/system.py) | 34 | ||||
-rw-r--r-- | src/mlia/cli/config.py | 6 | ||||
-rw-r--r-- | src/mlia/devices/ethosu/performance.py | 37 | ||||
-rw-r--r-- | src/mlia/resources/aiet/applications/APPLICATIONS.txt | 5 | ||||
-rw-r--r-- | src/mlia/resources/aiet/systems/SYSTEMS.txt | 3 | ||||
-rw-r--r-- | src/mlia/resources/backends/applications/.gitignore (renamed from src/aiet/resources/applications/.gitignore) | 0 | ||||
-rw-r--r-- | src/mlia/resources/backends/systems/.gitignore (renamed from src/aiet/resources/systems/.gitignore) | 0 | ||||
-rw-r--r-- | src/mlia/tools/metadata/corstone.py | 61 | ||||
-rw-r--r-- | src/mlia/utils/proc.py | 20 | ||||
-rw-r--r-- | tests/aiet/__init__.py | 3 | ||||
-rw-r--r-- | tests/aiet/conftest.py | 139 | ||||
-rw-r--r-- | tests/aiet/test_backend_tool.py | 60 | ||||
-rw-r--r-- | tests/aiet/test_check_model.py | 162 | ||||
-rw-r--r-- | tests/aiet/test_cli.py | 37 | ||||
-rw-r--r-- | tests/aiet/test_cli_application.py | 1153 | ||||
-rw-r--r-- | tests/aiet/test_cli_common.py | 37 | ||||
-rw-r--r-- | tests/aiet/test_cli_system.py | 240 | ||||
-rw-r--r-- | tests/aiet/test_cli_tool.py | 333 | ||||
-rw-r--r-- | tests/aiet/test_main.py | 16 | ||||
-rw-r--r-- | tests/aiet/test_resources/tools/tool1/aiet-config.json | 30 | ||||
-rw-r--r-- | tests/aiet/test_resources/tools/tool2/aiet-config.json | 26 | ||||
-rw-r--r-- | tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json.license | 3 | ||||
-rw-r--r-- | tests/aiet/test_run_vela_script.py | 152 | ||||
-rw-r--r-- | tests/aiet/test_utils_helpers.py | 27 | ||||
-rw-r--r-- | tests/mlia/conftest.py | 91 | ||||
-rw-r--r-- | tests/mlia/test_backend_application.py (renamed from tests/aiet/test_backend_application.py) | 50 | ||||
-rw-r--r-- | tests/mlia/test_backend_common.py (renamed from tests/aiet/test_backend_common.py) | 36 | ||||
-rw-r--r-- | tests/mlia/test_backend_controller.py (renamed from tests/aiet/test_backend_controller.py) | 8 | ||||
-rw-r--r-- | tests/mlia/test_backend_execution.py (renamed from tests/aiet/test_backend_execution.py) | 110 | ||||
-rw-r--r-- | tests/mlia/test_backend_fs.py (renamed from tests/aiet/test_utils_fs.py) | 28 | ||||
-rw-r--r-- | tests/mlia/test_backend_manager.py (renamed from tests/mlia/test_tools_aiet_wrapper.py) | 326 | ||||
-rw-r--r-- | tests/mlia/test_backend_output_parser.py (renamed from tests/aiet/test_backend_output_parser.py) | 6 | ||||
-rw-r--r-- | tests/mlia/test_backend_proc.py (renamed from tests/aiet/test_utils_proc.py) | 26 | ||||
-rw-r--r-- | tests/mlia/test_backend_protocol.py (renamed from tests/aiet/test_backend_protocol.py) | 24 | ||||
-rw-r--r-- | tests/mlia/test_backend_source.py (renamed from tests/aiet/test_backend_source.py) | 18 | ||||
-rw-r--r-- | tests/mlia/test_backend_system.py (renamed from tests/aiet/test_backend_system.py) | 69 | ||||
-rw-r--r-- | tests/mlia/test_cli_logging.py | 10 | ||||
-rw-r--r-- | tests/mlia/test_devices_ethosu_performance.py | 2 | ||||
-rw-r--r-- | tests/mlia/test_resources/application_config.json (renamed from tests/aiet/test_resources/application_config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/application_config.json.license (renamed from tests/aiet/test_resources/application_config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application1/aiet-config.json (renamed from tests/aiet/test_resources/applications/application1/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application1/aiet-config.json.license (renamed from src/aiet/resources/tools/vela/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application2/aiet-config.json (renamed from tests/aiet/test_resources/applications/application2/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application2/aiet-config.json.license (renamed from tests/aiet/test_resources/applications/application1/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application3/readme.txt (renamed from tests/aiet/test_resources/applications/application3/readme.txt) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application4/aiet-config.json (renamed from tests/aiet/test_resources/applications/application4/aiet-config.json) | 5 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application4/aiet-config.json.license (renamed from tests/aiet/test_resources/applications/application2/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application4/hello_app.txt (renamed from tests/aiet/test_resources/applications/application4/hello_app.txt) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application5/aiet-config.json (renamed from tests/aiet/test_resources/applications/application5/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application5/aiet-config.json.license (renamed from tests/aiet/test_resources/applications/application4/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application6/aiet-config.json | 42 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/application6/aiet-config.json.license (renamed from tests/aiet/test_resources/applications/application5/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/applications/readme.txt (renamed from tests/aiet/test_resources/applications/readme.txt) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system1/aiet-config.json (renamed from tests/aiet/test_resources/systems/system1/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system1/aiet-config.json.license (renamed from tests/aiet/test_resources/systems/system1/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system1/system_artifact/dummy.txt (renamed from tests/aiet/test_resources/systems/system1/system_artifact/dummy.txt) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system2/aiet-config.json (renamed from tests/aiet/test_resources/systems/system2/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system2/aiet-config.json.license (renamed from tests/aiet/test_resources/systems/system2/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system3/readme.txt (renamed from tests/aiet/test_resources/systems/system3/readme.txt) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system4/aiet-config.json (renamed from tests/aiet/test_resources/systems/system4/aiet-config.json) | 2 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system4/aiet-config.json.license (renamed from tests/aiet/test_resources/systems/system4/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system6/aiet-config.json | 34 | ||||
-rw-r--r-- | tests/mlia/test_resources/backends/systems/system6/aiet-config.json.license (renamed from tests/aiet/test_resources/tools/tool1/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/hello_world.json (renamed from tests/aiet/test_resources/hello_world.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/hello_world.json.license (renamed from tests/aiet/test_resources/hello_world.json.license) | 0 | ||||
-rwxr-xr-x | tests/mlia/test_resources/scripts/test_backend_run (renamed from tests/aiet/test_resources/scripts/test_backend_run) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/scripts/test_backend_run_script.sh (renamed from tests/aiet/test_resources/scripts/test_backend_run_script.sh) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json.license (renamed from tests/aiet/test_resources/tools/tool2/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json (renamed from tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json (renamed from tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json) | 0 | ||||
-rw-r--r-- | tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json.license (renamed from tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json.license) | 0 | ||||
-rw-r--r-- | tests/mlia/test_tools_metadata_corstone.py | 90 |
110 files changed, 821 insertions, 4446 deletions
@@ -34,12 +34,10 @@ install_requires = ethos-u-vela~=3.3.0 requests rich - click sh paramiko filelock psutil - cloup>=0.12.0 [options.packages.find] where = src @@ -47,8 +45,6 @@ where = src [options.entry_points] console_scripts = mlia=mlia.cli.main:main - aiet=aiet.main:main - run_vela=aiet.resources.tools.vela.run_vela:main [options.extras_require] dev = diff --git a/src/aiet/__init__.py b/src/aiet/__init__.py deleted file mode 100644 index 49304b1..0000000 --- a/src/aiet/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Init of aiet.""" -import pkg_resources - - -__version__ = pkg_resources.get_distribution("mlia").version diff --git a/src/aiet/backend/tool.py b/src/aiet/backend/tool.py deleted file mode 100644 index d643665..0000000 --- a/src/aiet/backend/tool.py +++ /dev/null @@ -1,109 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Tool backend module.""" -from typing import Any -from typing import cast -from typing import Dict -from typing import List -from typing import Optional - -from aiet.backend.common import Backend -from aiet.backend.common import ConfigurationException -from aiet.backend.common import get_backend_configs -from aiet.backend.common import get_backend_directories -from aiet.backend.common import load_application_or_tool_configs -from aiet.backend.common import load_config -from aiet.backend.config import ExtendedToolConfig -from aiet.backend.config import ToolConfig - - -def get_available_tool_directory_names() -> List[str]: - """Return a list of directory names for all available tools.""" - return [entry.name for entry in get_backend_directories("tools")] - - -def get_available_tools() -> List["Tool"]: - """Return a list with all available tools.""" - available_tools = [] - for config_json in get_backend_configs("tools"): - config_entries = cast(List[ExtendedToolConfig], load_config(config_json)) - for config_entry in config_entries: - config_entry["config_location"] = config_json.parent.absolute() - tools = load_tools(config_entry) - available_tools += tools - - return sorted(available_tools, key=lambda tool: tool.name) - - -def get_tool(tool_name: str, system_name: Optional[str] = None) -> List["Tool"]: - """Return a tool instance with the same name passed as argument.""" - return [ - tool - for tool in get_available_tools() - if tool.name == tool_name and (not system_name or tool.can_run_on(system_name)) - ] - - -def get_unique_tool_names(system_name: Optional[str] = None) -> List[str]: - """Extract a list of unique tool names of all tools available.""" - return list( - set( - tool.name - for tool in get_available_tools() - if not system_name or tool.can_run_on(system_name) - ) - ) - - -class Tool(Backend): - """Class for representing a single tool component.""" - - def __init__(self, config: ToolConfig) -> None: - """Construct a Tool instance from a dict.""" - super().__init__(config) - - self.supported_systems = config.get("supported_systems", []) - - if "run" not in self.commands: - raise ConfigurationException("A Tool must have a 'run' command.") - - def __eq__(self, other: object) -> bool: - """Overload operator ==.""" - if not isinstance(other, Tool): - return False - - return ( - super().__eq__(other) - and self.name == other.name - and set(self.supported_systems) == set(other.supported_systems) - ) - - def can_run_on(self, system_name: str) -> bool: - """Check if the tool can run on the system passed as argument.""" - return system_name in self.supported_systems - - def get_details(self) -> Dict[str, Any]: - """Return dictionary with all relevant information of the Tool instance.""" - output = { - "type": "tool", - "name": self.name, - "description": self.description, - "supported_systems": self.supported_systems, - "commands": self._get_command_details(), - } - - return output - - -def load_tools(config: ExtendedToolConfig) -> List[Tool]: - """Load tool. - - Tool configuration could contain different parameters/commands for different - supported systems. For each supported system this function will return separate - Tool instance with appropriate configuration. - """ - configs = load_application_or_tool_configs( - config, ToolConfig, is_system_required=False - ) - tools = [Tool(cfg) for cfg in configs] - return tools diff --git a/src/aiet/cli/__init__.py b/src/aiet/cli/__init__.py deleted file mode 100644 index bcd17c3..0000000 --- a/src/aiet/cli/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Module to mange the CLI interface.""" -import click - -from aiet import __version__ -from aiet.cli.application import application_cmd -from aiet.cli.completion import completion_cmd -from aiet.cli.system import system_cmd -from aiet.cli.tool import tool_cmd -from aiet.utils.helpers import set_verbosity - - -@click.group() -@click.version_option(__version__) -@click.option( - "-v", "--verbose", default=0, count=True, callback=set_verbosity, expose_value=False -) -@click.pass_context -def cli(ctx: click.Context) -> None: # pylint: disable=unused-argument - """AIET: AI Evaluation Toolkit.""" - # Unused arguments must be present here in definition to pass click context. - - -cli.add_command(application_cmd) -cli.add_command(system_cmd) -cli.add_command(tool_cmd) -cli.add_command(completion_cmd) diff --git a/src/aiet/cli/application.py b/src/aiet/cli/application.py deleted file mode 100644 index 59b652d..0000000 --- a/src/aiet/cli/application.py +++ /dev/null @@ -1,362 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-FileCopyrightText: Copyright (c) 2021, Gianluca Gippetto. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause -"""Module to manage the CLI interface of applications.""" -import json -import logging -import re -from pathlib import Path -from typing import Any -from typing import IO -from typing import List -from typing import Optional -from typing import Tuple - -import click -import cloup - -from aiet.backend.application import get_application -from aiet.backend.application import get_available_application_directory_names -from aiet.backend.application import get_unique_application_names -from aiet.backend.application import install_application -from aiet.backend.application import remove_application -from aiet.backend.common import DataPaths -from aiet.backend.execution import execute_application_command -from aiet.backend.execution import run_application -from aiet.backend.system import get_available_systems -from aiet.cli.common import get_format -from aiet.cli.common import middleware_exception_handler -from aiet.cli.common import middleware_signal_handler -from aiet.cli.common import print_command_details -from aiet.cli.common import set_format - - -@click.group(name="application") -@click.option( - "-f", - "--format", - "format_", - type=click.Choice(["cli", "json"]), - default="cli", - show_default=True, -) -@click.pass_context -def application_cmd(ctx: click.Context, format_: str) -> None: - """Sub command to manage applications.""" - set_format(ctx, format_) - - -@application_cmd.command(name="list") -@click.pass_context -@click.option( - "-s", - "--system", - "system_name", - type=click.Choice([s.name for s in get_available_systems()]), - required=False, -) -def list_cmd(ctx: click.Context, system_name: str) -> None: - """List all available applications.""" - unique_application_names = get_unique_application_names(system_name) - unique_application_names.sort() - if get_format(ctx) == "json": - data = {"type": "application", "available": unique_application_names} - print(json.dumps(data)) - else: - print("Available applications:\n") - print(*unique_application_names, sep="\n") - - -@application_cmd.command(name="details") -@click.option( - "-n", - "--name", - "application_name", - type=click.Choice(get_unique_application_names()), - required=True, -) -@click.option( - "-s", - "--system", - "system_name", - type=click.Choice([s.name for s in get_available_systems()]), - required=False, -) -@click.pass_context -def details_cmd(ctx: click.Context, application_name: str, system_name: str) -> None: - """Details of a specific application.""" - applications = get_application(application_name, system_name) - if not applications: - raise click.UsageError( - "Application '{}' doesn't support the system '{}'".format( - application_name, system_name - ) - ) - - if get_format(ctx) == "json": - applications_details = [s.get_details() for s in applications] - print(json.dumps(applications_details)) - else: - for application in applications: - application_details = application.get_details() - application_details_template = ( - 'Application "{name}" details\nDescription: {description}' - ) - - print( - application_details_template.format( - name=application_details["name"], - description=application_details["description"], - ) - ) - - print( - "\nSupported systems: {}".format( - ", ".join(application_details["supported_systems"]) - ) - ) - - command_details = application_details["commands"] - - for command, details in command_details.items(): - print("\n{} commands:".format(command)) - print_command_details(details) - - -# pylint: disable=too-many-arguments -@application_cmd.command(name="execute") -@click.option( - "-n", - "--name", - "application_name", - type=click.Choice(get_unique_application_names()), - required=True, -) -@click.option( - "-s", - "--system", - "system_name", - type=click.Choice([s.name for s in get_available_systems()]), - required=True, -) -@click.option( - "-c", - "--command", - "command_name", - type=click.Choice(["build", "run"]), - required=True, -) -@click.option("-p", "--param", "application_params", multiple=True) -@click.option("--system-param", "system_params", multiple=True) -@click.option("-d", "--deploy", "deploy_params", multiple=True) -@middleware_signal_handler -@middleware_exception_handler -def execute_cmd( - application_name: str, - system_name: str, - command_name: str, - application_params: List[str], - system_params: List[str], - deploy_params: List[str], -) -> None: - """Execute application commands. DEPRECATED! Use 'aiet application run' instead.""" - logging.warning( - "Please use 'aiet application run' instead. Use of 'aiet application " - "execute' is deprecated and might be removed in a future release." - ) - - custom_deploy_data = get_custom_deploy_data(command_name, deploy_params) - - execute_application_command( - command_name, - application_name, - application_params, - system_name, - system_params, - custom_deploy_data, - ) - - -@cloup.command(name="run") -@cloup.option( - "-n", - "--name", - "application_name", - type=click.Choice(get_unique_application_names()), -) -@cloup.option( - "-s", - "--system", - "system_name", - type=click.Choice([s.name for s in get_available_systems()]), -) -@cloup.option("-p", "--param", "application_params", multiple=True) -@cloup.option("--system-param", "system_params", multiple=True) -@cloup.option("-d", "--deploy", "deploy_params", multiple=True) -@click.option( - "-r", - "--report", - "report_file", - type=Path, - help="Create a report file in JSON format containing metrics parsed from " - "the simulation output as specified in the aiet-config.json.", -) -@cloup.option( - "--config", - "config_file", - type=click.File("r"), - help="Read options from a config file rather than from the command line. " - "The config file is a json file.", -) -@cloup.constraint( - cloup.constraints.If( - cloup.constraints.conditions.Not( - cloup.constraints.conditions.IsSet("config_file") - ), - then=cloup.constraints.require_all, - ), - ["system_name", "application_name"], -) -@cloup.constraint( - cloup.constraints.If("config_file", then=cloup.constraints.accept_none), - [ - "system_name", - "application_name", - "application_params", - "system_params", - "deploy_params", - ], -) -@middleware_signal_handler -@middleware_exception_handler -def run_cmd( - application_name: str, - system_name: str, - application_params: List[str], - system_params: List[str], - deploy_params: List[str], - report_file: Optional[Path], - config_file: Optional[IO[str]], -) -> None: - """Execute application commands.""" - if config_file: - payload_data = json.load(config_file) - ( - system_name, - application_name, - application_params, - system_params, - deploy_params, - report_file, - ) = parse_payload_run_config(payload_data) - - custom_deploy_data = get_custom_deploy_data("run", deploy_params) - - run_application( - application_name, - application_params, - system_name, - system_params, - custom_deploy_data, - report_file, - ) - - -application_cmd.add_command(run_cmd) - - -def parse_payload_run_config( - payload_data: dict, -) -> Tuple[str, str, List[str], List[str], List[str], Optional[Path]]: - """Parse the payload into a tuple.""" - system_id = payload_data.get("id") - arguments: Optional[Any] = payload_data.get("arguments") - - if not isinstance(system_id, str): - raise click.ClickException("invalid payload json: no system 'id'") - if not isinstance(arguments, dict): - raise click.ClickException("invalid payload json: no arguments object") - - application_name = arguments.pop("application", None) - if not isinstance(application_name, str): - raise click.ClickException("invalid payload json: no application_id") - - report_path = arguments.pop("report_path", None) - - application_params = [] - system_params = [] - deploy_params = [] - - for (param_key, value) in arguments.items(): - (par, _) = re.subn("^application/", "", param_key) - (par, found_sys_param) = re.subn("^system/", "", par) - (par, found_deploy_param) = re.subn("^deploy/", "", par) - - param_expr = par + "=" + value - if found_sys_param: - system_params.append(param_expr) - elif found_deploy_param: - deploy_params.append(par) - else: - application_params.append(param_expr) - - return ( - system_id, - application_name, - application_params, - system_params, - deploy_params, - report_path, - ) - - -def get_custom_deploy_data( - command_name: str, deploy_params: List[str] -) -> List[DataPaths]: - """Get custom deploy data information.""" - custom_deploy_data: List[DataPaths] = [] - if not deploy_params: - return custom_deploy_data - - for param in deploy_params: - parts = param.split(":") - if not len(parts) == 2 or any(not part.strip() for part in parts): - raise click.ClickException( - "Invalid deploy parameter '{}' for command {}".format( - param, command_name - ) - ) - data_path = DataPaths(Path(parts[0]), parts[1]) - if not data_path.src.exists(): - raise click.ClickException("Path {} does not exist".format(data_path.src)) - custom_deploy_data.append(data_path) - - return custom_deploy_data - - -@application_cmd.command(name="install") -@click.option( - "-s", - "--source", - "source", - required=True, - help="Path to the directory or archive with application definition", -) -def install_cmd(source: str) -> None: - """Install new application.""" - source_path = Path(source) - install_application(source_path) - - -@application_cmd.command(name="remove") -@click.option( - "-d", - "--directory_name", - "directory_name", - type=click.Choice(get_available_application_directory_names()), - required=True, - help="Name of the directory with application", -) -def remove_cmd(directory_name: str) -> None: - """Remove application.""" - remove_application(directory_name) diff --git a/src/aiet/cli/common.py b/src/aiet/cli/common.py deleted file mode 100644 index 1d157b6..0000000 --- a/src/aiet/cli/common.py +++ /dev/null @@ -1,173 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Common functions for cli module.""" -import enum -import logging -from functools import wraps -from signal import SIG_IGN -from signal import SIGINT -from signal import signal as signal_handler -from signal import SIGTERM -from typing import Any -from typing import Callable -from typing import cast -from typing import Dict - -from click import ClickException -from click import Context -from click import UsageError - -from aiet.backend.common import ConfigurationException -from aiet.backend.execution import AnotherInstanceIsRunningException -from aiet.backend.execution import ConnectionException -from aiet.backend.protocol import SSHConnectionException -from aiet.utils.proc import CommandFailedException - - -class MiddlewareExitCode(enum.IntEnum): - """Middleware exit codes.""" - - SUCCESS = 0 - # exit codes 1 and 2 are used by click - SHUTDOWN_REQUESTED = 3 - BACKEND_ERROR = 4 - CONCURRENT_ERROR = 5 - CONNECTION_ERROR = 6 - CONFIGURATION_ERROR = 7 - MODEL_OPTIMISED_ERROR = 8 - INVALID_TFLITE_FILE_ERROR = 9 - - -class CustomClickException(ClickException): - """Custom click exception.""" - - def show(self, file: Any = None) -> None: - """Override show method.""" - super().show(file) - - logging.debug("Execution failed with following exception: ", exc_info=self) - - -class MiddlewareShutdownException(CustomClickException): - """Exception indicates that user requested middleware shutdown.""" - - exit_code = int(MiddlewareExitCode.SHUTDOWN_REQUESTED) - - -class BackendException(CustomClickException): - """Exception indicates that command failed.""" - - exit_code = int(MiddlewareExitCode.BACKEND_ERROR) - - -class ConcurrentErrorException(CustomClickException): - """Exception indicates concurrent execution error.""" - - exit_code = int(MiddlewareExitCode.CONCURRENT_ERROR) - - -class BackendConnectionException(CustomClickException): - """Exception indicates that connection could not be established.""" - - exit_code = int(MiddlewareExitCode.CONNECTION_ERROR) - - -class BackendConfigurationException(CustomClickException): - """Exception indicates some configuration issue.""" - - exit_code = int(MiddlewareExitCode.CONFIGURATION_ERROR) - - -class ModelOptimisedException(CustomClickException): - """Exception indicates input file has previously been Vela optimised.""" - - exit_code = int(MiddlewareExitCode.MODEL_OPTIMISED_ERROR) - - -class InvalidTFLiteFileError(CustomClickException): - """Exception indicates input TFLite file is misformatted.""" - - exit_code = int(MiddlewareExitCode.INVALID_TFLITE_FILE_ERROR) - - -def print_command_details(command: Dict) -> None: - """Print command details including parameters.""" - command_strings = command["command_strings"] - print("Commands: {}".format(command_strings)) - user_params = command["user_params"] - for i, param in enumerate(user_params, 1): - print("User parameter #{}".format(i)) - print("\tName: {}".format(param.get("name", "-"))) - print("\tDescription: {}".format(param["description"])) - print("\tPossible values: {}".format(param.get("values", "-"))) - print("\tDefault value: {}".format(param.get("default_value", "-"))) - print("\tAlias: {}".format(param.get("alias", "-"))) - - -def raise_exception_at_signal( - signum: int, frame: Any # pylint: disable=unused-argument -) -> None: - """Handle signals.""" - # Disable both SIGINT and SIGTERM signals. Further SIGINT and SIGTERM - # signals will be ignored as we allow a graceful shutdown. - # Unused arguments must be present here in definition as used in signal handler - # callback - - signal_handler(SIGINT, SIG_IGN) - signal_handler(SIGTERM, SIG_IGN) - raise MiddlewareShutdownException("Middleware shutdown requested") - - -def middleware_exception_handler(func: Callable) -> Callable: - """Handle backend exceptions decorator.""" - - @wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> Any: - try: - return func(*args, **kwargs) - except (MiddlewareShutdownException, UsageError, ClickException) as error: - # click should take care of these exceptions - raise error - except ValueError as error: - raise ClickException(str(error)) from error - except AnotherInstanceIsRunningException as error: - raise ConcurrentErrorException( - "Another instance of the system is running" - ) from error - except (SSHConnectionException, ConnectionException) as error: - raise BackendConnectionException(str(error)) from error - except ConfigurationException as error: - raise BackendConfigurationException(str(error)) from error - except (CommandFailedException, Exception) as error: - raise BackendException( - "Execution failed. Please check output for the details." - ) from error - - return wrapper - - -def middleware_signal_handler(func: Callable) -> Callable: - """Handle signals decorator.""" - - @wraps(func) - def wrapper(*args: Any, **kwargs: Any) -> Any: - # Set up signal handlers for SIGINT (ctrl-c) and SIGTERM (kill command) - # The handler ignores further signals and it raises an exception - signal_handler(SIGINT, raise_exception_at_signal) - signal_handler(SIGTERM, raise_exception_at_signal) - - return func(*args, **kwargs) - - return wrapper - - -def set_format(ctx: Context, format_: str) -> None: - """Save format in click context.""" - ctx_obj = ctx.ensure_object(dict) - ctx_obj["format"] = format_ - - -def get_format(ctx: Context) -> str: - """Get format from click context.""" - ctx_obj = cast(Dict[str, str], ctx.ensure_object(dict)) - return ctx_obj["format"] diff --git a/src/aiet/cli/completion.py b/src/aiet/cli/completion.py deleted file mode 100644 index 71f054f..0000000 --- a/src/aiet/cli/completion.py +++ /dev/null @@ -1,72 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -""" -Add auto completion to different shells with these helpers. - -See: https://click.palletsprojects.com/en/8.0.x/shell-completion/ -""" -import click - - -def _get_package_name() -> str: - return __name__.split(".", maxsplit=1)[0] - - -# aiet completion bash -@click.group(name="completion") -def completion_cmd() -> None: - """Enable auto completion for your shell.""" - - -@completion_cmd.command(name="bash") -def bash_cmd() -> None: - """ - Enable auto completion for bash. - - Use this command to activate completion in the current bash: - - eval "`aiet completion bash`" - - Use this command to add auto completion to bash globally, if you have aiet - installed globally (requires starting a new shell afterwards): - - aiet completion bash >> ~/.bashrc - """ - package_name = _get_package_name() - print(f'eval "$(_{package_name.upper()}_COMPLETE=bash_source {package_name})"') - - -@completion_cmd.command(name="zsh") -def zsh_cmd() -> None: - """ - Enable auto completion for zsh. - - Use this command to activate completion in the current zsh: - - eval "`aiet completion zsh`" - - Use this command to add auto completion to zsh globally, if you have aiet - installed globally (requires starting a new shell afterwards): - - aiet completion zsh >> ~/.zshrc - """ - package_name = _get_package_name() - print(f'eval "$(_{package_name.upper()}_COMPLETE=zsh_source {package_name})"') - - -@completion_cmd.command(name="fish") -def fish_cmd() -> None: - """ - Enable auto completion for fish. - - Use this command to activate completion in the current fish: - - eval "`aiet completion fish`" - - Use this command to add auto completion to fish globally, if you have aiet - installed globally (requires starting a new shell afterwards): - - aiet completion fish >> ~/.config/fish/completions/aiet.fish - """ - package_name = _get_package_name() - print(f'eval "(env _{package_name.upper()}_COMPLETE=fish_source {package_name})"') diff --git a/src/aiet/cli/system.py b/src/aiet/cli/system.py deleted file mode 100644 index f1f7637..0000000 --- a/src/aiet/cli/system.py +++ /dev/null @@ -1,122 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Module to manage the CLI interface of systems.""" -import json -from pathlib import Path -from typing import cast - -import click - -from aiet.backend.application import get_available_applications -from aiet.backend.system import get_available_systems -from aiet.backend.system import get_available_systems_directory_names -from aiet.backend.system import get_system -from aiet.backend.system import install_system -from aiet.backend.system import remove_system -from aiet.backend.system import System -from aiet.cli.common import get_format -from aiet.cli.common import print_command_details -from aiet.cli.common import set_format - - -@click.group(name="system") -@click.option( - "-f", - "--format", - "format_", - type=click.Choice(["cli", "json"]), - default="cli", - show_default=True, -) -@click.pass_context -def system_cmd(ctx: click.Context, format_: str) -> None: - """Sub command to manage systems.""" - set_format(ctx, format_) - - -@system_cmd.command(name="list") -@click.pass_context -def list_cmd(ctx: click.Context) -> None: - """List all available systems.""" - available_systems = get_available_systems() - system_names = [system.name for system in available_systems] - if get_format(ctx) == "json": - data = {"type": "system", "available": system_names} - print(json.dumps(data)) - else: - print("Available systems:\n") - print(*system_names, sep="\n") - - -@system_cmd.command(name="details") -@click.option( - "-n", - "--name", - "system_name", - type=click.Choice([s.name for s in get_available_systems()]), - required=True, -) -@click.pass_context -def details_cmd(ctx: click.Context, system_name: str) -> None: - """Details of a specific system.""" - system = cast(System, get_system(system_name)) - applications = [ - s.name for s in get_available_applications() if s.can_run_on(system.name) - ] - system_details = system.get_details() - if get_format(ctx) == "json": - system_details["available_application"] = applications - print(json.dumps(system_details)) - else: - system_details_template = ( - 'System "{name}" details\n' - "Description: {description}\n" - "Data Transfer Protocol: {protocol}\n" - "Available Applications: {available_application}" - ) - print( - system_details_template.format( - name=system_details["name"], - description=system_details["description"], - protocol=system_details["data_transfer_protocol"], - available_application=", ".join(applications), - ) - ) - - if system_details["annotations"]: - print("Annotations:") - for ann_name, ann_value in system_details["annotations"].items(): - print("\t{}: {}".format(ann_name, ann_value)) - - command_details = system_details["commands"] - for command, details in command_details.items(): - print("\n{} commands:".format(command)) - print_command_details(details) - - -@system_cmd.command(name="install") -@click.option( - "-s", - "--source", - "source", - required=True, - help="Path to the directory or archive with system definition", -) -def install_cmd(source: str) -> None: - """Install new system.""" - source_path = Path(source) - install_system(source_path) - - -@system_cmd.command(name="remove") -@click.option( - "-d", - "--directory_name", - "directory_name", - type=click.Choice(get_available_systems_directory_names()), - required=True, - help="Name of the directory with system", -) -def remove_cmd(directory_name: str) -> None: - """Remove system by given name.""" - remove_system(directory_name) diff --git a/src/aiet/cli/tool.py b/src/aiet/cli/tool.py deleted file mode 100644 index 2c80821..0000000 --- a/src/aiet/cli/tool.py +++ /dev/null @@ -1,143 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Module to manage the CLI interface of tools.""" -import json -from typing import Any -from typing import List -from typing import Optional - -import click - -from aiet.backend.execution import execute_tool_command -from aiet.backend.tool import get_tool -from aiet.backend.tool import get_unique_tool_names -from aiet.cli.common import get_format -from aiet.cli.common import middleware_exception_handler -from aiet.cli.common import middleware_signal_handler -from aiet.cli.common import print_command_details -from aiet.cli.common import set_format - - -@click.group(name="tool") -@click.option( - "-f", - "--format", - "format_", - type=click.Choice(["cli", "json"]), - default="cli", - show_default=True, -) -@click.pass_context -def tool_cmd(ctx: click.Context, format_: str) -> None: - """Sub command to manage tools.""" - set_format(ctx, format_) - - -@tool_cmd.command(name="list") -@click.pass_context -def list_cmd(ctx: click.Context) -> None: - """List all available tools.""" - # raise NotImplementedError("TODO") - tool_names = get_unique_tool_names() - tool_names.sort() - if get_format(ctx) == "json": - data = {"type": "tool", "available": tool_names} - print(json.dumps(data)) - else: - print("Available tools:\n") - print(*tool_names, sep="\n") - - -def validate_system( - ctx: click.Context, - _: click.Parameter, # param is not used - value: Any, -) -> Any: - """Validate provided system name depending on the the tool name.""" - tool_name = ctx.params["tool_name"] - tools = get_tool(tool_name, value) - if not tools: - supported_systems = [tool.supported_systems[0] for tool in get_tool(tool_name)] - raise click.BadParameter( - message="'{}' is not one of {}.".format( - value, - ", ".join("'{}'".format(system) for system in supported_systems), - ), - ctx=ctx, - ) - return value - - -@tool_cmd.command(name="details") -@click.option( - "-n", - "--name", - "tool_name", - type=click.Choice(get_unique_tool_names()), - required=True, -) -@click.option( - "-s", - "--system", - "system_name", - callback=validate_system, - required=False, -) -@click.pass_context -@middleware_signal_handler -@middleware_exception_handler -def details_cmd(ctx: click.Context, tool_name: str, system_name: Optional[str]) -> None: - """Details of a specific tool.""" - tools = get_tool(tool_name, system_name) - if get_format(ctx) == "json": - tools_details = [s.get_details() for s in tools] - print(json.dumps(tools_details)) - else: - for tool in tools: - tool_details = tool.get_details() - tool_details_template = 'Tool "{name}" details\nDescription: {description}' - - print( - tool_details_template.format( - name=tool_details["name"], - description=tool_details["description"], - ) - ) - - print( - "\nSupported systems: {}".format( - ", ".join(tool_details["supported_systems"]) - ) - ) - - command_details = tool_details["commands"] - - for command, details in command_details.items(): - print("\n{} commands:".format(command)) - print_command_details(details) - - -# pylint: disable=too-many-arguments -@tool_cmd.command(name="execute") -@click.option( - "-n", - "--name", - "tool_name", - type=click.Choice(get_unique_tool_names()), - required=True, -) -@click.option("-p", "--param", "tool_params", multiple=True) -@click.option( - "-s", - "--system", - "system_name", - callback=validate_system, - required=False, -) -@middleware_signal_handler -@middleware_exception_handler -def execute_cmd( - tool_name: str, tool_params: List[str], system_name: Optional[str] -) -> None: - """Execute tool commands.""" - execute_tool_command(tool_name, tool_params, system_name) diff --git a/src/aiet/main.py b/src/aiet/main.py deleted file mode 100644 index 6898ad9..0000000 --- a/src/aiet/main.py +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Entry point module of AIET.""" -from aiet.cli import cli - - -def main() -> None: - """Entry point of aiet application.""" - cli() # pylint: disable=no-value-for-parameter - - -if __name__ == "__main__": - main() diff --git a/src/aiet/resources/tools/vela/aiet-config.json b/src/aiet/resources/tools/vela/aiet-config.json deleted file mode 100644 index c12f291..0000000 --- a/src/aiet/resources/tools/vela/aiet-config.json +++ /dev/null @@ -1,73 +0,0 @@ -[ - { - "name": "vela", - "description": "Neural network model compiler for Arm Ethos-U NPUs", - "supported_systems": [ - { - "name": "Corstone-300: Cortex-M55+Ethos-U55" - }, - { - "name": "Corstone-310: Cortex-M85+Ethos-U55" - }, - { - "name": "Corstone-300: Cortex-M55+Ethos-U65", - "variables": { - "accelerator_config_prefix": "ethos-u65", - "system_config": "Ethos_U65_High_End", - "shared_sram": "U65_Shared_Sram" - }, - "user_params": { - "run": [ - { - "description": "MACs per cycle", - "values": [ - "256", - "512" - ], - "default_value": "512", - "alias": "mac" - } - ] - } - } - ], - "variables": { - "accelerator_config_prefix": "ethos-u55", - "system_config": "Ethos_U55_High_End_Embedded", - "shared_sram": "U55_Shared_Sram" - }, - "commands": { - "run": [ - "run_vela {user_params:input} {user_params:output} --config {tool.config_dir}/vela.ini --accelerator-config {variables:accelerator_config_prefix}-{user_params:mac} --system-config {variables:system_config} --memory-mode {variables:shared_sram} --optimise Performance" - ] - }, - "user_params": { - "run": [ - { - "description": "MACs per cycle", - "values": [ - "32", - "64", - "128", - "256" - ], - "default_value": "128", - "alias": "mac" - }, - { - "name": "--input-model", - "description": "Path to the TFLite model", - "values": [], - "alias": "input" - }, - { - "name": "--output-model", - "description": "Path to the output model file of the vela-optimisation step. The vela output is saved in the parent directory.", - "values": [], - "default_value": "output_model.tflite", - "alias": "output" - } - ] - } - } -] diff --git a/src/aiet/resources/tools/vela/check_model.py b/src/aiet/resources/tools/vela/check_model.py deleted file mode 100644 index 7c700b1..0000000 --- a/src/aiet/resources/tools/vela/check_model.py +++ /dev/null @@ -1,75 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2020, 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Check if a TFLite model file is Vela-optimised.""" -import struct -from pathlib import Path - -from ethosu.vela.tflite.Model import Model - -from aiet.cli.common import InvalidTFLiteFileError -from aiet.cli.common import ModelOptimisedException -from aiet.utils.fs import read_file_as_bytearray - - -def get_model_from_file(input_model_file: Path) -> Model: - """Generate Model instance from TFLite file using flatc generated code.""" - buffer = read_file_as_bytearray(input_model_file) - try: - model = Model.GetRootAsModel(buffer, 0) - except (TypeError, RuntimeError, struct.error) as tflite_error: - raise InvalidTFLiteFileError( - f"Error reading in model from {input_model_file}." - ) from tflite_error - return model - - -def is_vela_optimised(tflite_model: Model) -> bool: - """Return True if 'ethos-u' custom operator found in the Model.""" - operators = get_operators_from_model(tflite_model) - - custom_codes = get_custom_codes_from_operators(operators) - - return check_custom_codes_for_ethosu(custom_codes) - - -def get_operators_from_model(tflite_model: Model) -> list: - """Return list of the unique operator codes used in the Model.""" - return [ - tflite_model.OperatorCodes(index) - for index in range(tflite_model.OperatorCodesLength()) - ] - - -def get_custom_codes_from_operators(operators: list) -> list: - """Return list of each operator's CustomCode() strings, if they exist.""" - return [ - operator.CustomCode() - for operator in operators - if operator.CustomCode() is not None - ] - - -def check_custom_codes_for_ethosu(custom_codes: list) -> bool: - """Check for existence of ethos-u string in the custom codes.""" - return any( - custom_code_name.decode("utf-8") == "ethos-u" - for custom_code_name in custom_codes - ) - - -def check_model(tflite_file_name: str) -> None: - """Raise an exception if model in given file is Vela optimised.""" - tflite_path = Path(tflite_file_name) - - tflite_model = get_model_from_file(tflite_path) - - if is_vela_optimised(tflite_model): - raise ModelOptimisedException( - f"TFLite model in {tflite_file_name} is already " - f"vela optimised ('ethos-u' custom op detected)." - ) - - print( - f"TFLite model in {tflite_file_name} is not vela optimised " - f"('ethos-u' custom op not detected)." - ) diff --git a/src/aiet/resources/tools/vela/run_vela.py b/src/aiet/resources/tools/vela/run_vela.py deleted file mode 100644 index 2c1b0be..0000000 --- a/src/aiet/resources/tools/vela/run_vela.py +++ /dev/null @@ -1,65 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Wrapper to only run Vela when the input is not already optimised.""" -import shutil -import subprocess -from pathlib import Path -from typing import Tuple - -import click - -from aiet.cli.common import ModelOptimisedException -from aiet.resources.tools.vela.check_model import check_model - - -def vela_output_model_path(input_model: str, output_dir: str) -> Path: - """Construct the path to the Vela output file.""" - in_path = Path(input_model) - tflite_vela = Path(output_dir) / f"{in_path.stem}_vela{in_path.suffix}" - return tflite_vela - - -def execute_vela(vela_args: Tuple, output_dir: Path, input_model: str) -> None: - """Execute vela as external call.""" - cmd = ["vela"] + list(vela_args) - cmd += ["--output-dir", str(output_dir)] # Re-add parsed out_dir to arguments - cmd += [input_model] - subprocess.run(cmd, check=True) - - -@click.command(context_settings=dict(ignore_unknown_options=True)) -@click.option( - "--input-model", - "-i", - type=click.Path(exists=True, file_okay=True, readable=True), - required=True, -) -@click.option("--output-model", "-o", type=click.Path(), required=True) -# Collect the remaining arguments to be directly forwarded to Vela -@click.argument("vela-args", nargs=-1, type=click.UNPROCESSED) -def run_vela(input_model: str, output_model: str, vela_args: Tuple) -> None: - """Check input, run Vela (if needed) and copy optimised file to destination.""" - output_dir = Path(output_model).parent - try: - check_model(input_model) # raises an exception if already Vela-optimised - execute_vela(vela_args, output_dir, input_model) - print("Vela optimisation complete.") - src_model = vela_output_model_path(input_model, str(output_dir)) - except ModelOptimisedException as ex: - # Input already optimized: copy input file to destination path and return - print(f"Input already vela-optimised.\n{ex}") - src_model = Path(input_model) - except subprocess.CalledProcessError as ex: - print(ex) - raise SystemExit(ex.returncode) from ex - - try: - shutil.copyfile(src_model, output_model) - except (shutil.SameFileError, OSError) as ex: - print(ex) - raise SystemExit(ex.errno) from ex - - -def main() -> None: - """Entry point of check_model application.""" - run_vela() # pylint: disable=no-value-for-parameter diff --git a/src/aiet/resources/tools/vela/vela.ini b/src/aiet/resources/tools/vela/vela.ini deleted file mode 100644 index 5996553..0000000 --- a/src/aiet/resources/tools/vela/vela.ini +++ /dev/null @@ -1,53 +0,0 @@ -; SPDX-FileCopyrightText: Copyright 2021-2022, Arm Limited and/or its affiliates. -; SPDX-License-Identifier: Apache-2.0 - -; ----------------------------------------------------------------------------- -; Vela configuration file - -; ----------------------------------------------------------------------------- -; System Configuration - -; Ethos-U55 High-End Embedded: SRAM (4 GB/s) and Flash (0.5 GB/s) -[System_Config.Ethos_U55_High_End_Embedded] -core_clock=500e6 -axi0_port=Sram -axi1_port=OffChipFlash -Sram_clock_scale=1.0 -Sram_burst_length=32 -Sram_read_latency=32 -Sram_write_latency=32 -OffChipFlash_clock_scale=0.125 -OffChipFlash_burst_length=128 -OffChipFlash_read_latency=64 -OffChipFlash_write_latency=64 - -; Ethos-U65 High-End: SRAM (16 GB/s) and DRAM (3.75 GB/s) -[System_Config.Ethos_U65_High_End] -core_clock=1e9 -axi0_port=Sram -axi1_port=Dram -Sram_clock_scale=1.0 -Sram_burst_length=32 -Sram_read_latency=32 -Sram_write_latency=32 -Dram_clock_scale=0.234375 -Dram_burst_length=128 -Dram_read_latency=500 -Dram_write_latency=250 - -; ----------------------------------------------------------------------------- -; Memory Mode - -; Shared SRAM: the SRAM is shared between the Ethos-U and the Cortex-M software -; The non-SRAM memory is assumed to be read-only -[Memory_Mode.U55_Shared_Sram] -const_mem_area=Axi1 -arena_mem_area=Axi0 -cache_mem_area=Axi0 -arena_cache_size=4194304 - -[Memory_Mode.U65_Shared_Sram] -const_mem_area=Axi1 -arena_mem_area=Axi0 -cache_mem_area=Axi0 -arena_cache_size=2097152 diff --git a/src/aiet/utils/__init__.py b/src/aiet/utils/__init__.py deleted file mode 100644 index fc7ef7c..0000000 --- a/src/aiet/utils/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""This module contains all utils shared across aiet project.""" diff --git a/src/aiet/utils/helpers.py b/src/aiet/utils/helpers.py deleted file mode 100644 index 6d3cd22..0000000 --- a/src/aiet/utils/helpers.py +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Helpers functions.""" -import logging -from typing import Any - - -def set_verbosity( - ctx: Any, option: Any, verbosity: Any # pylint: disable=unused-argument -) -> None: - """Set the logging level according to the verbosity.""" - # Unused arguments must be present here in definition as these are required in - # function definition when set as a callback - if verbosity == 1: - logging.getLogger().setLevel(logging.INFO) - elif verbosity > 1: - logging.getLogger().setLevel(logging.DEBUG) diff --git a/src/aiet/backend/__init__.py b/src/mlia/backend/__init__.py index 3d60372..3d60372 100644 --- a/src/aiet/backend/__init__.py +++ b/src/mlia/backend/__init__.py diff --git a/src/aiet/backend/application.py b/src/mlia/backend/application.py index f6ef815..eb85212 100644 --- a/src/aiet/backend/application.py +++ b/src/mlia/backend/application.py @@ -9,19 +9,19 @@ from typing import Dict from typing import List from typing import Optional -from aiet.backend.common import Backend -from aiet.backend.common import ConfigurationException -from aiet.backend.common import DataPaths -from aiet.backend.common import get_backend_configs -from aiet.backend.common import get_backend_directories -from aiet.backend.common import load_application_or_tool_configs -from aiet.backend.common import load_config -from aiet.backend.common import remove_backend -from aiet.backend.config import ApplicationConfig -from aiet.backend.config import ExtendedApplicationConfig -from aiet.backend.source import create_destination_and_install -from aiet.backend.source import get_source -from aiet.utils.fs import get_resources +from mlia.backend.common import Backend +from mlia.backend.common import ConfigurationException +from mlia.backend.common import DataPaths +from mlia.backend.common import get_backend_configs +from mlia.backend.common import get_backend_directories +from mlia.backend.common import load_application_or_tool_configs +from mlia.backend.common import load_config +from mlia.backend.common import remove_backend +from mlia.backend.config import ApplicationConfig +from mlia.backend.config import ExtendedApplicationConfig +from mlia.backend.fs import get_backends_path +from mlia.backend.source import create_destination_and_install +from mlia.backend.source import get_source def get_available_application_directory_names() -> List[str]: @@ -78,7 +78,7 @@ def install_application(source_path: Path) -> None: "Applications [{}] are already installed".format(",".join(names)) ) - create_destination_and_install(source, get_resources("applications")) + create_destination_and_install(source, get_backends_path("applications")) def remove_application(directory_name: str) -> None: diff --git a/src/aiet/backend/common.py b/src/mlia/backend/common.py index b887ee7..2bbb9d3 100644 --- a/src/aiet/backend/common.py +++ b/src/mlia/backend/common.py @@ -23,17 +23,17 @@ from typing import Tuple from typing import Type from typing import Union -from aiet.backend.config import BackendConfig -from aiet.backend.config import BaseBackendConfig -from aiet.backend.config import NamedExecutionConfig -from aiet.backend.config import UserParamConfig -from aiet.backend.config import UserParamsConfig -from aiet.utils.fs import get_resources -from aiet.utils.fs import remove_resource -from aiet.utils.fs import ResourceType +from mlia.backend.config import BackendConfig +from mlia.backend.config import BaseBackendConfig +from mlia.backend.config import NamedExecutionConfig +from mlia.backend.config import UserParamConfig +from mlia.backend.config import UserParamsConfig +from mlia.backend.fs import get_backends_path +from mlia.backend.fs import remove_resource +from mlia.backend.fs import ResourceType -AIET_CONFIG_FILE: Final[str] = "aiet-config.json" +BACKEND_CONFIG_FILE: Final[str] = "aiet-config.json" class ConfigurationException(Exception): @@ -42,7 +42,7 @@ class ConfigurationException(Exception): def get_backend_config(dir_path: Path) -> Path: """Get path to backendir configuration file.""" - return dir_path / AIET_CONFIG_FILE + return dir_path / BACKEND_CONFIG_FILE def get_backend_configs(resource_type: ResourceType) -> Iterable[Path]: @@ -56,7 +56,7 @@ def get_backend_directories(resource_type: ResourceType) -> Iterable[Path]: """Get path to the backend directories for provided resource_type.""" return ( entry - for entry in get_resources(resource_type).iterdir() + for entry in get_backends_path(resource_type).iterdir() if is_backend_directory(entry) ) diff --git a/src/aiet/backend/config.py b/src/mlia/backend/config.py index dd42012..657adef 100644 --- a/src/aiet/backend/config.py +++ b/src/mlia/backend/config.py @@ -89,19 +89,5 @@ class SystemConfig(BaseBackendConfig, total=False): reporting: Dict[str, Dict] -class ToolConfig(BaseBackendConfig, total=False): - """Tool configuration.""" - - supported_systems: List[str] - - -class ExtendedToolConfig(BaseBackendConfig, total=False): - """Extended tool configuration.""" - - supported_systems: List[NamedExecutionConfig] - - -BackendItemConfig = Union[ApplicationConfig, SystemConfig, ToolConfig] -BackendConfig = Union[ - List[ExtendedApplicationConfig], List[SystemConfig], List[ToolConfig] -] +BackendItemConfig = Union[ApplicationConfig, SystemConfig] +BackendConfig = Union[List[ExtendedApplicationConfig], List[SystemConfig]] diff --git a/src/aiet/backend/controller.py b/src/mlia/backend/controller.py index 2650902..f1b68a9 100644 --- a/src/aiet/backend/controller.py +++ b/src/mlia/backend/controller.py @@ -10,14 +10,14 @@ from typing import Tuple import psutil import sh -from aiet.backend.common import ConfigurationException -from aiet.utils.fs import read_file_as_string -from aiet.utils.proc import execute_command -from aiet.utils.proc import get_stdout_stderr_paths -from aiet.utils.proc import read_process_info -from aiet.utils.proc import save_process_info -from aiet.utils.proc import terminate_command -from aiet.utils.proc import terminate_external_process +from mlia.backend.common import ConfigurationException +from mlia.backend.fs import read_file_as_string +from mlia.backend.proc import execute_command +from mlia.backend.proc import get_stdout_stderr_paths +from mlia.backend.proc import read_process_info +from mlia.backend.proc import save_process_info +from mlia.backend.proc import terminate_command +from mlia.backend.proc import terminate_external_process class SystemController: diff --git a/src/aiet/backend/execution.py b/src/mlia/backend/execution.py index 1653ee2..749ccdb 100644 --- a/src/aiet/backend/execution.py +++ b/src/mlia/backend/execution.py @@ -8,7 +8,6 @@ import re import string import sys import time -import warnings from collections import defaultdict from contextlib import contextmanager from contextlib import ExitStack @@ -24,32 +23,29 @@ from typing import Optional from typing import Sequence from typing import Tuple from typing import TypedDict -from typing import Union from filelock import FileLock from filelock import Timeout -from aiet.backend.application import Application -from aiet.backend.application import get_application -from aiet.backend.common import Backend -from aiet.backend.common import ConfigurationException -from aiet.backend.common import DataPaths -from aiet.backend.common import Param -from aiet.backend.common import parse_raw_parameter -from aiet.backend.common import resolve_all_parameters -from aiet.backend.output_parser import Base64OutputParser -from aiet.backend.output_parser import OutputParser -from aiet.backend.output_parser import RegexOutputParser -from aiet.backend.system import ControlledSystem -from aiet.backend.system import get_system -from aiet.backend.system import StandaloneSystem -from aiet.backend.system import System -from aiet.backend.tool import get_tool -from aiet.backend.tool import Tool -from aiet.utils.fs import recreate_directory -from aiet.utils.fs import remove_directory -from aiet.utils.fs import valid_for_filename -from aiet.utils.proc import run_and_wait +from mlia.backend.application import Application +from mlia.backend.application import get_application +from mlia.backend.common import Backend +from mlia.backend.common import ConfigurationException +from mlia.backend.common import DataPaths +from mlia.backend.common import Param +from mlia.backend.common import parse_raw_parameter +from mlia.backend.common import resolve_all_parameters +from mlia.backend.fs import recreate_directory +from mlia.backend.fs import remove_directory +from mlia.backend.fs import valid_for_filename +from mlia.backend.output_parser import Base64OutputParser +from mlia.backend.output_parser import OutputParser +from mlia.backend.output_parser import RegexOutputParser +from mlia.backend.proc import run_and_wait +from mlia.backend.system import ControlledSystem +from mlia.backend.system import get_system +from mlia.backend.system import StandaloneSystem +from mlia.backend.system import System class AnotherInstanceIsRunningException(Exception): @@ -73,7 +69,7 @@ class ExecutionContext: # pylint: disable=too-many-arguments,too-many-instance-attributes def __init__( self, - app: Union[Application, Tool], + app: Application, app_params: List[str], system: Optional[System], system_params: List[str], @@ -106,14 +102,13 @@ class ExecutionContext: self.param_resolver = ParamResolver(self) self._resolved_build_dir: Optional[Path] = None + self.stdout: Optional[bytearray] = None + self.stderr: Optional[bytearray] = None + @property def is_deploy_needed(self) -> bool: """Check if application requires data deployment.""" - if isinstance(self.app, Application): - return ( - len(self.app.get_deploy_data()) > 0 or len(self.custom_deploy_data) > 0 - ) - return False + return len(self.app.get_deploy_data()) > 0 or len(self.custom_deploy_data) > 0 @property def is_locking_required(self) -> bool: @@ -533,45 +528,6 @@ def get_application_and_system( return application, system -def execute_application_command( # pylint: disable=too-many-arguments - command_name: str, - application_name: str, - application_params: List[str], - system_name: str, - system_params: List[str], - custom_deploy_data: List[DataPaths], -) -> None: - """Execute application command. - - .. deprecated:: 21.12 - """ - warnings.warn( - "Use 'run_application()' instead. Use of 'execute_application_command()' is " - "deprecated and might be removed in a future release.", - DeprecationWarning, - ) - - if command_name not in ["build", "run"]: - raise ConfigurationException("Unsupported command {}".format(command_name)) - - application, system = get_application_and_system(application_name, system_name) - validate_parameters(application, [command_name], application_params) - validate_parameters(system, [command_name], system_params) - - ctx = ExecutionContext( - app=application, - app_params=application_params, - system=system, - system_params=system_params, - custom_deploy_data=custom_deploy_data, - ) - - if command_name == "run": - execute_application_command_run(ctx) - else: - execute_application_command_build(ctx) - - # pylint: disable=too-many-arguments def run_application( application_name: str, @@ -580,7 +536,7 @@ def run_application( system_params: List[str], custom_deploy_data: List[DataPaths], report_file: Optional[Path] = None, -) -> None: +) -> ExecutionContext: """Run application on the provided system.""" application, system = get_application_and_system(application_name, system_name) validate_parameters(application, ["build", "run"], application_params) @@ -607,6 +563,8 @@ def run_application( execute_application_command_run(ctx) + return ctx + def execute_application_command_build(ctx: ExecutionContext) -> None: """Execute application command 'build'.""" @@ -655,17 +613,14 @@ def execute_application_command_run(ctx: ExecutionContext) -> None: for command in commands_to_run: print("Running: {}".format(command)) - exit_code, std_output, std_err = ctx.system.run(command) + exit_code, ctx.stdout, ctx.stderr = ctx.system.run(command) if exit_code != 0: print("Application exited with exit code {}".format(exit_code)) if ctx.reporter: - ctx.reporter.parse(std_output) - std_output = ctx.reporter.get_filtered_output(std_output) - - print(std_output.decode("utf8"), end="") - print(std_err.decode("utf8"), end="") + ctx.reporter.parse(ctx.stdout) + ctx.stdout = ctx.reporter.get_filtered_output(ctx.stdout) if ctx.reporter: report = ctx.reporter.report(ctx) @@ -717,12 +672,10 @@ def wait(interval: float) -> None: def deploy_data(ctx: ExecutionContext) -> None: """Deploy data to the system.""" - if isinstance(ctx.app, Application): - # Only application can deploy data (tools can not) - assert ctx.system is not None, "System is required." - for item in itertools.chain(ctx.app.get_deploy_data(), ctx.custom_deploy_data): - print("Deploying {} onto {}".format(item.src, item.dst)) - ctx.system.deploy(item.src, item.dst) + assert ctx.system is not None, "System is required." + for item in itertools.chain(ctx.app.get_deploy_data(), ctx.custom_deploy_data): + print("Deploying {} onto {}".format(item.src, item.dst)) + ctx.system.deploy(item.src, item.dst) def build_run_commands(ctx: ExecutionContext) -> List[str]: @@ -824,36 +777,3 @@ def get_context_managers( managers.append(controlled_system_manager) return managers - - -def get_tool_by_system(tool_name: str, system_name: Optional[str]) -> Tool: - """Return tool (optionally by provided system name.""" - tools = get_tool(tool_name, system_name) - if not tools: - raise ConfigurationException( - "Tool '{}' not found or doesn't support the system '{}'".format( - tool_name, system_name - ) - ) - if len(tools) != 1: - raise ConfigurationException( - "Please specify the system for tool {}.".format(tool_name) - ) - tool = tools[0] - - return tool - - -def execute_tool_command( - tool_name: str, - tool_params: List[str], - system_name: Optional[str] = None, -) -> None: - """Execute the tool command locally calling the 'run' command.""" - tool = get_tool_by_system(tool_name, system_name) - ctx = ExecutionContext( - app=tool, app_params=tool_params, system=None, system_params=[] - ) - commands = tool.build_command("run", tool_params, ctx.param_resolver) - - execute_commands_locally(commands, Path.cwd()) diff --git a/src/aiet/utils/fs.py b/src/mlia/backend/fs.py index ea99a69..9979fcb 100644 --- a/src/aiet/utils/fs.py +++ b/src/mlia/backend/fs.py @@ -1,7 +1,6 @@ # SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Module to host all file system related functions.""" -import importlib.resources as pkg_resources import re import shutil from pathlib import Path @@ -9,17 +8,17 @@ from typing import Any from typing import Literal from typing import Optional -ResourceType = Literal["applications", "systems", "tools"] +from mlia.utils.filesystem import get_mlia_resources +ResourceType = Literal["applications", "systems"] -def get_aiet_resources() -> Path: - """Get resources folder path.""" - with pkg_resources.path("aiet", "__init__.py") as init_path: - project_root = init_path.parent - return project_root / "resources" +def get_backend_resources() -> Path: + """Get backend resources folder path.""" + return get_mlia_resources() / "backends" -def get_resources(name: ResourceType) -> Path: + +def get_backends_path(name: ResourceType) -> Path: """Return the absolute path of the specified resource. It uses importlib to return resources packaged with MANIFEST.in. @@ -27,7 +26,7 @@ def get_resources(name: ResourceType) -> Path: if not name: raise ResourceWarning("Resource name is not provided") - resource_path = get_aiet_resources() / name + resource_path = get_backend_resources() / name if resource_path.is_dir(): return resource_path @@ -48,7 +47,7 @@ def copy_directory_content(source: Path, destination: Path) -> None: def remove_resource(resource_directory: str, resource_type: ResourceType) -> None: """Remove resource data.""" - resources = get_resources(resource_type) + resources = get_backends_path(resource_type) resource_location = resources / resource_directory if not resource_location.exists(): diff --git a/src/mlia/tools/aiet_wrapper.py b/src/mlia/backend/manager.py index 73e82ee..3a1016c 100644 --- a/src/mlia/tools/aiet_wrapper.py +++ b/src/mlia/backend/manager.py @@ -1,6 +1,6 @@ # SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 -"""Module for AIET integration.""" +"""Module for backend integration.""" import logging import re from abc import ABC @@ -14,11 +14,13 @@ from typing import Literal from typing import Optional from typing import Tuple -from aiet.backend.application import get_available_applications -from aiet.backend.application import install_application -from aiet.backend.system import get_available_systems -from aiet.backend.system import install_system -from mlia.utils.proc import CommandExecutor +from mlia.backend.application import get_available_applications +from mlia.backend.application import install_application +from mlia.backend.common import DataPaths +from mlia.backend.execution import ExecutionContext +from mlia.backend.execution import run_application +from mlia.backend.system import get_available_systems +from mlia.backend.system import install_system from mlia.utils.proc import OutputConsumer from mlia.utils.proc import RunningCommand @@ -54,7 +56,7 @@ _SYSTEM_TO_APP_MAP = { def get_system_name(backend: str, device_type: str) -> str: - """Get the AIET system name for the given backend and device type.""" + """Get the system name for the given backend and device type.""" return _SUPPORTED_SYSTEMS[backend][device_type] @@ -71,7 +73,7 @@ def is_supported(backend: str, device_type: Optional[str] = None) -> bool: def supported_backends() -> List[str]: - """Get a list of all backends supported by the AIET wrapper.""" + """Get a list of all backends supported by the backend manager.""" return list(_SUPPORTED_SYSTEMS.keys()) @@ -129,8 +131,8 @@ class ExecutionParams: deploy_params: List[str] -class AIETLogWriter(OutputConsumer): - """Redirect AIET command output to the logger.""" +class LogWriter(OutputConsumer): + """Redirect output to the logger.""" def feed(self, line: str) -> None: """Process line from the output.""" @@ -211,12 +213,11 @@ class GenericInferenceOutputParser(OutputConsumer): return sorted(self.PATTERNS.keys() - self.result.keys()) -class AIETRunner: - """AIET runner.""" +class BackendRunner: + """Backend runner.""" - def __init__(self, executor: CommandExecutor) -> None: - """Init AIET runner instance.""" - self.executor = executor + def __init__(self) -> None: + """Init BackendRunner instance.""" @staticmethod def get_installed_systems() -> List[str]: @@ -270,39 +271,40 @@ class AIETRunner: """Install application.""" install_application(app_path) - def run_application(self, execution_params: ExecutionParams) -> RunningCommand: + @staticmethod + def run_application(execution_params: ExecutionParams) -> ExecutionContext: """Run requested application.""" - command = [ - "aiet", - "application", - "run", - "-n", + + def to_data_paths(paths: str) -> DataPaths: + """Split input into two and create new DataPaths object.""" + src, dst = paths.split(sep=":", maxsplit=1) + return DataPaths(Path(src), dst) + + deploy_data_paths = [ + to_data_paths(paths) for paths in execution_params.deploy_params + ] + + ctx = run_application( execution_params.application, - "-s", + execution_params.application_params, execution_params.system, - *self._params("-p", execution_params.application_params), - *self._params("--system-param", execution_params.system_params), - *self._params("--deploy", execution_params.deploy_params), - ] + execution_params.system_params, + deploy_data_paths, + ) - return self._submit(command) + return ctx @staticmethod def _params(name: str, params: List[str]) -> List[str]: return [p for item in [(name, param) for param in params] for p in item] - def _submit(self, command: List[str]) -> RunningCommand: - """Submit command for the execution.""" - logger.debug("Submit command %s", " ".join(command)) - return self.executor.submit(command) - class GenericInferenceRunner(ABC): """Abstract class for generic inference runner.""" - def __init__(self, aiet_runner: AIETRunner): + def __init__(self, backend_runner: BackendRunner): """Init generic inference runner instance.""" - self.aiet_runner = aiet_runner + self.backend_runner = backend_runner self.running_inference: Optional[RunningCommand] = None def run( @@ -311,9 +313,9 @@ class GenericInferenceRunner(ABC): """Run generic inference for the provided device/model.""" execution_params = self.get_execution_params(model_info) - self.running_inference = self.aiet_runner.run_application(execution_params) - self.running_inference.output_consumers = output_consumers - self.running_inference.consume_output() + ctx = self.backend_runner.run_application(execution_params) + if ctx.stdout is not None: + self.consume_output(ctx.stdout, output_consumers) def stop(self) -> None: """Stop running inference.""" @@ -336,24 +338,31 @@ class GenericInferenceRunner(ABC): def check_system_and_application(self, system_name: str, app_name: str) -> None: """Check if requested system and application installed.""" - if not self.aiet_runner.is_system_installed(system_name): + if not self.backend_runner.is_system_installed(system_name): raise Exception(f"System {system_name} is not installed") - if not self.aiet_runner.is_application_installed(app_name, system_name): + if not self.backend_runner.is_application_installed(app_name, system_name): raise Exception( f"Application {app_name} for the system {system_name} " "is not installed" ) + @staticmethod + def consume_output(output: bytearray, consumers: List[OutputConsumer]) -> None: + """Pass program's output to the consumers.""" + for line in output.decode("utf8").splitlines(): + for consumer in consumers: + consumer.feed(line) + class GenericInferenceRunnerEthosU(GenericInferenceRunner): """Generic inference runner on U55/65.""" def __init__( - self, aiet_runner: AIETRunner, device_info: DeviceInfo, backend: str + self, backend_runner: BackendRunner, device_info: DeviceInfo, backend: str ) -> None: """Init generic inference runner instance.""" - super().__init__(aiet_runner) + super().__init__(backend_runner) system_name, app_name = self.resolve_system_and_app(device_info, backend) self.system_name = system_name @@ -405,8 +414,8 @@ class GenericInferenceRunnerEthosU(GenericInferenceRunner): def get_generic_runner(device_info: DeviceInfo, backend: str) -> GenericInferenceRunner: """Get generic runner for provided device and backend.""" - aiet_runner = get_aiet_runner() - return GenericInferenceRunnerEthosU(aiet_runner, device_info, backend) + backend_runner = get_backend_runner() + return GenericInferenceRunnerEthosU(backend_runner, device_info, backend) def estimate_performance( @@ -415,7 +424,7 @@ def estimate_performance( """Get performance estimations.""" with get_generic_runner(device_info, backend) as generic_runner: output_parser = GenericInferenceOutputParser() - output_consumers = [output_parser, AIETLogWriter()] + output_consumers = [output_parser, LogWriter()] generic_runner.run(model_info, output_consumers) @@ -429,7 +438,10 @@ def estimate_performance( return PerformanceMetrics(**output_parser.result) -def get_aiet_runner() -> AIETRunner: - """Return AIET runner.""" - executor = CommandExecutor() - return AIETRunner(executor) +def get_backend_runner() -> BackendRunner: + """ + Return BackendRunner instance. + + Note: This is needed for the unit tests. + """ + return BackendRunner() diff --git a/src/aiet/backend/output_parser.py b/src/mlia/backend/output_parser.py index 111772a..111772a 100644 --- a/src/aiet/backend/output_parser.py +++ b/src/mlia/backend/output_parser.py diff --git a/src/aiet/utils/proc.py b/src/mlia/backend/proc.py index b6f4357..90ff414 100644 --- a/src/aiet/utils/proc.py +++ b/src/mlia/backend/proc.py @@ -24,7 +24,7 @@ from sh import CommandNotFound from sh import ErrorReturnCode from sh import RunningCommand -from aiet.utils.fs import valid_for_filename +from mlia.backend.fs import valid_for_filename class CommandFailedException(Exception): diff --git a/src/aiet/backend/protocol.py b/src/mlia/backend/protocol.py index c621436..ebfe69a 100644 --- a/src/aiet/backend/protocol.py +++ b/src/mlia/backend/protocol.py @@ -14,10 +14,10 @@ from typing import Union import paramiko -from aiet.backend.common import ConfigurationException -from aiet.backend.config import LocalProtocolConfig -from aiet.backend.config import SSHConfig -from aiet.utils.proc import run_and_wait +from mlia.backend.common import ConfigurationException +from mlia.backend.config import LocalProtocolConfig +from mlia.backend.config import SSHConfig +from mlia.backend.proc import run_and_wait # Redirect all paramiko thread exceptions to a file otherwise these will be diff --git a/src/aiet/backend/source.py b/src/mlia/backend/source.py index dec175a..dcf6835 100644 --- a/src/aiet/backend/source.py +++ b/src/mlia/backend/source.py @@ -11,13 +11,13 @@ from tarfile import TarFile from typing import Optional from typing import Union -from aiet.backend.common import AIET_CONFIG_FILE -from aiet.backend.common import ConfigurationException -from aiet.backend.common import get_backend_config -from aiet.backend.common import is_backend_directory -from aiet.backend.common import load_config -from aiet.backend.config import BackendConfig -from aiet.utils.fs import copy_directory_content +from mlia.backend.common import BACKEND_CONFIG_FILE +from mlia.backend.common import ConfigurationException +from mlia.backend.common import get_backend_config +from mlia.backend.common import is_backend_directory +from mlia.backend.common import load_config +from mlia.backend.config import BackendConfig +from mlia.backend.fs import copy_directory_content class Source(ABC): @@ -99,7 +99,7 @@ class TarArchiveSource(Source): with self._open(self.archive_path) as archive: try: - config_entry = archive.getmember(AIET_CONFIG_FILE) + config_entry = archive.getmember(BACKEND_CONFIG_FILE) self._has_top_level_folder = False except KeyError as error_no_config: try: @@ -112,7 +112,7 @@ class TarArchiveSource(Source): "Archive has no top level directory" ) from error_no_config - config_path = "{}/{}".format(top_level_dir, AIET_CONFIG_FILE) + config_path = "{}/{}".format(top_level_dir, BACKEND_CONFIG_FILE) config_entry = archive.getmember(config_path) self._has_top_level_folder = True diff --git a/src/aiet/backend/system.py b/src/mlia/backend/system.py index 48f1bb1..469083e 100644 --- a/src/aiet/backend/system.py +++ b/src/mlia/backend/system.py @@ -10,22 +10,22 @@ from typing import Optional from typing import Tuple from typing import Union -from aiet.backend.common import Backend -from aiet.backend.common import ConfigurationException -from aiet.backend.common import get_backend_configs -from aiet.backend.common import get_backend_directories -from aiet.backend.common import load_config -from aiet.backend.common import remove_backend -from aiet.backend.config import SystemConfig -from aiet.backend.controller import SystemController -from aiet.backend.controller import SystemControllerSingleInstance -from aiet.backend.protocol import ProtocolFactory -from aiet.backend.protocol import SupportsClose -from aiet.backend.protocol import SupportsConnection -from aiet.backend.protocol import SupportsDeploy -from aiet.backend.source import create_destination_and_install -from aiet.backend.source import get_source -from aiet.utils.fs import get_resources +from mlia.backend.common import Backend +from mlia.backend.common import ConfigurationException +from mlia.backend.common import get_backend_configs +from mlia.backend.common import get_backend_directories +from mlia.backend.common import load_config +from mlia.backend.common import remove_backend +from mlia.backend.config import SystemConfig +from mlia.backend.controller import SystemController +from mlia.backend.controller import SystemControllerSingleInstance +from mlia.backend.fs import get_backends_path +from mlia.backend.protocol import ProtocolFactory +from mlia.backend.protocol import SupportsClose +from mlia.backend.protocol import SupportsConnection +from mlia.backend.protocol import SupportsDeploy +from mlia.backend.source import create_destination_and_install +from mlia.backend.source import get_source def get_available_systems_directory_names() -> List[str]: @@ -75,7 +75,7 @@ def install_system(source_path: Path) -> None: "Systems [{}] are already installed".format(",".join(names)) ) - create_destination_and_install(source, get_resources("systems")) + create_destination_and_install(source, get_backends_path("systems")) def remove_system(directory_name: str) -> None: diff --git a/src/mlia/cli/config.py b/src/mlia/cli/config.py index 838b051..a673230 100644 --- a/src/mlia/cli/config.py +++ b/src/mlia/cli/config.py @@ -5,7 +5,7 @@ import logging from functools import lru_cache from typing import List -import mlia.tools.aiet_wrapper as aiet +import mlia.backend.manager as backend_manager from mlia.tools.metadata.common import DefaultInstallationManager from mlia.tools.metadata.common import InstallationManager from mlia.tools.metadata.corstone import get_corstone_installations @@ -25,12 +25,12 @@ def get_available_backends() -> List[str]: """Return list of the available backends.""" available_backends = ["Vela"] - # Add backends using AIET + # Add backends using backend manager manager = get_installation_manager() available_backends.extend( ( backend - for backend in aiet.supported_backends() + for backend in backend_manager.supported_backends() if manager.backend_installed(backend) ) ) diff --git a/src/mlia/devices/ethosu/performance.py b/src/mlia/devices/ethosu/performance.py index b0718a5..a73045a 100644 --- a/src/mlia/devices/ethosu/performance.py +++ b/src/mlia/devices/ethosu/performance.py @@ -10,7 +10,7 @@ from typing import Optional from typing import Tuple from typing import Union -import mlia.tools.aiet_wrapper as aiet +import mlia.backend.manager as backend_manager import mlia.tools.vela_wrapper as vela from mlia.core.context import Context from mlia.core.performance import PerformanceEstimator @@ -147,15 +147,15 @@ class VelaPerformanceEstimator( return memory_usage -class AIETPerformanceEstimator( +class CorstonePerformanceEstimator( PerformanceEstimator[Union[Path, ModelConfiguration], NPUCycles] ): - """AIET based performance estimator.""" + """Corstone-based performance estimator.""" def __init__( self, context: Context, device: EthosUConfiguration, backend: str ) -> None: - """Init AIET based performance estimator.""" + """Init Corstone-based performance estimator.""" self.context = context self.device = device self.backend = backend @@ -179,24 +179,24 @@ class AIETPerformanceEstimator( model_path, self.device.compiler_options, optimized_model_path ) - model_info = aiet.ModelInfo(model_path=optimized_model_path) - device_info = aiet.DeviceInfo( + model_info = backend_manager.ModelInfo(model_path=optimized_model_path) + device_info = backend_manager.DeviceInfo( device_type=self.device.target, # type: ignore mac=self.device.mac, memory_mode=self.device.compiler_options.memory_mode, # type: ignore ) - aiet_perf_metrics = aiet.estimate_performance( + corstone_perf_metrics = backend_manager.estimate_performance( model_info, device_info, self.backend ) npu_cycles = NPUCycles( - aiet_perf_metrics.npu_active_cycles, - aiet_perf_metrics.npu_idle_cycles, - aiet_perf_metrics.npu_total_cycles, - aiet_perf_metrics.npu_axi0_rd_data_beat_received, - aiet_perf_metrics.npu_axi0_wr_data_beat_written, - aiet_perf_metrics.npu_axi1_rd_data_beat_received, + corstone_perf_metrics.npu_active_cycles, + corstone_perf_metrics.npu_idle_cycles, + corstone_perf_metrics.npu_total_cycles, + corstone_perf_metrics.npu_axi0_rd_data_beat_received, + corstone_perf_metrics.npu_axi0_wr_data_beat_written, + corstone_perf_metrics.npu_axi1_rd_data_beat_received, ) logger.info("Done\n") @@ -220,10 +220,11 @@ class EthosUPerformanceEstimator( if backends is None: backends = ["Vela"] # Only Vela is always available as default for backend in backends: - if backend != "Vela" and not aiet.is_supported(backend): + if backend != "Vela" and not backend_manager.is_supported(backend): raise ValueError( f"Unsupported backend '{backend}'. " - f"Only 'Vela' and {aiet.supported_backends()} are supported." + f"Only 'Vela' and {backend_manager.supported_backends()} " + "are supported." ) self.backends = set(backends) @@ -242,11 +243,11 @@ class EthosUPerformanceEstimator( if backend == "Vela": vela_estimator = VelaPerformanceEstimator(self.context, self.device) memory_usage = vela_estimator.estimate(tflite_model) - elif backend in aiet.supported_backends(): - aiet_estimator = AIETPerformanceEstimator( + elif backend in backend_manager.supported_backends(): + corstone_estimator = CorstonePerformanceEstimator( self.context, self.device, backend ) - npu_cycles = aiet_estimator.estimate(tflite_model) + npu_cycles = corstone_estimator.estimate(tflite_model) else: logger.warning( "Backend '%s' is not supported for Ethos-U performance " diff --git a/src/mlia/resources/aiet/applications/APPLICATIONS.txt b/src/mlia/resources/aiet/applications/APPLICATIONS.txt index 09127f8..a702e19 100644 --- a/src/mlia/resources/aiet/applications/APPLICATIONS.txt +++ b/src/mlia/resources/aiet/applications/APPLICATIONS.txt @@ -1,6 +1,7 @@ SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. SPDX-License-Identifier: Apache-2.0 -This directory contains the Generic Inference Runner application packages for AIET +This directory contains the application packages for the Generic Inference +Runner. -Each package should contain its own aiet-config.json file +Each package should contain its own aiet-config.json file. diff --git a/src/mlia/resources/aiet/systems/SYSTEMS.txt b/src/mlia/resources/aiet/systems/SYSTEMS.txt index bc27e73..3861769 100644 --- a/src/mlia/resources/aiet/systems/SYSTEMS.txt +++ b/src/mlia/resources/aiet/systems/SYSTEMS.txt @@ -1,8 +1,7 @@ SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. SPDX-License-Identifier: Apache-2.0 -This directory contains the configuration files of the systems for the AIET -middleware. +This directory contains the configuration files of the system backends. Supported systems: diff --git a/src/aiet/resources/applications/.gitignore b/src/mlia/resources/backends/applications/.gitignore index 0226166..0226166 100644 --- a/src/aiet/resources/applications/.gitignore +++ b/src/mlia/resources/backends/applications/.gitignore diff --git a/src/aiet/resources/systems/.gitignore b/src/mlia/resources/backends/systems/.gitignore index 0226166..0226166 100644 --- a/src/aiet/resources/systems/.gitignore +++ b/src/mlia/resources/backends/systems/.gitignore diff --git a/src/mlia/tools/metadata/corstone.py b/src/mlia/tools/metadata/corstone.py index 7a9d113..a92f81c 100644 --- a/src/mlia/tools/metadata/corstone.py +++ b/src/mlia/tools/metadata/corstone.py @@ -12,7 +12,8 @@ from typing import Iterable from typing import List from typing import Optional -import mlia.tools.aiet_wrapper as aiet +import mlia.backend.manager as backend_manager +from mlia.backend.fs import get_backend_resources from mlia.tools.metadata.common import DownloadAndInstall from mlia.tools.metadata.common import Installation from mlia.tools.metadata.common import InstallationType @@ -41,8 +42,8 @@ PathChecker = Callable[[Path], Optional[BackendInfo]] BackendInstaller = Callable[[bool, Path], Path] -class AIETMetadata: - """AIET installation metadata.""" +class BackendMetadata: + """Backend installation metadata.""" def __init__( self, @@ -55,7 +56,7 @@ class AIETMetadata: supported_platforms: Optional[List[str]] = None, ) -> None: """ - Initialize AIETMetaData. + Initialize BackendMetadata. Members expected_systems and expected_apps are filled automatically. """ @@ -67,15 +68,15 @@ class AIETMetadata: self.download_artifact = download_artifact self.supported_platforms = supported_platforms - self.expected_systems = aiet.get_all_system_names(name) - self.expected_apps = aiet.get_all_application_names(name) + self.expected_systems = backend_manager.get_all_system_names(name) + self.expected_apps = backend_manager.get_all_application_names(name) @property def expected_resources(self) -> Iterable[Path]: """Return list of expected resources.""" resources = [self.system_config, *self.apps_resources] - return (get_mlia_resources() / resource for resource in resources) + return (get_backend_resources() / resource for resource in resources) @property def supported_platform(self) -> bool: @@ -86,49 +87,49 @@ class AIETMetadata: return platform.system() in self.supported_platforms -class AIETBasedInstallation(Installation): - """Backend installation based on AIET functionality.""" +class BackendInstallation(Installation): + """Backend installation.""" def __init__( self, - aiet_runner: aiet.AIETRunner, - metadata: AIETMetadata, + backend_runner: backend_manager.BackendRunner, + metadata: BackendMetadata, path_checker: PathChecker, backend_installer: Optional[BackendInstaller], ) -> None: - """Init the tool installation.""" - self.aiet_runner = aiet_runner + """Init the backend installation.""" + self.backend_runner = backend_runner self.metadata = metadata self.path_checker = path_checker self.backend_installer = backend_installer @property def name(self) -> str: - """Return name of the tool.""" + """Return name of the backend.""" return self.metadata.name @property def description(self) -> str: - """Return description of the tool.""" + """Return description of the backend.""" return self.metadata.description @property def already_installed(self) -> bool: - """Return true if tool already installed.""" - return self.aiet_runner.all_installed( + """Return true if backend already installed.""" + return self.backend_runner.all_installed( self.metadata.expected_systems, self.metadata.expected_apps ) @property def could_be_installed(self) -> bool: - """Return true if tool could be installed.""" + """Return true if backend could be installed.""" if not self.metadata.supported_platform: return False return all_paths_valid(self.metadata.expected_resources) def supports(self, install_type: InstallationType) -> bool: - """Return true if tools supported type of the installation.""" + """Return true if backends supported type of the installation.""" if isinstance(install_type, DownloadAndInstall): return self.metadata.download_artifact is not None @@ -138,7 +139,7 @@ class AIETBasedInstallation(Installation): return False # type: ignore def install(self, install_type: InstallationType) -> None: - """Install the tool.""" + """Install the backend.""" if isinstance(install_type, DownloadAndInstall): download_artifact = self.metadata.download_artifact assert download_artifact is not None, "No artifact provided" @@ -153,7 +154,7 @@ class AIETBasedInstallation(Installation): raise Exception(f"Unable to install {install_type}") def install_from(self, backend_info: BackendInfo) -> None: - """Install tool from the directory.""" + """Install backend from the directory.""" mlia_resources = get_mlia_resources() with temp_directory() as tmpdir: @@ -169,15 +170,15 @@ class AIETBasedInstallation(Installation): copy_all(*resources_to_copy, dest=fvp_dist_dir) - self.aiet_runner.install_system(fvp_dist_dir) + self.backend_runner.install_system(fvp_dist_dir) for app in self.metadata.apps_resources: - self.aiet_runner.install_application(mlia_resources / app) + self.backend_runner.install_application(mlia_resources / app) def download_and_install( self, download_artifact: DownloadArtifact, eula_agrement: bool ) -> None: - """Download and install the tool.""" + """Download and install the backend.""" with temp_directory() as tmpdir: try: downloaded_to = download_artifact.download_to(tmpdir) @@ -307,10 +308,10 @@ class Corstone300Installer: def get_corstone_300_installation() -> Installation: """Get Corstone-300 installation.""" - corstone_300 = AIETBasedInstallation( - aiet_runner=aiet.get_aiet_runner(), + corstone_300 = BackendInstallation( + backend_runner=backend_manager.BackendRunner(), # pylint: disable=line-too-long - metadata=AIETMetadata( + metadata=BackendMetadata( name="Corstone-300", description="Corstone-300 FVP", system_config="aiet/systems/corstone-300/aiet-config.json", @@ -356,10 +357,10 @@ def get_corstone_300_installation() -> Installation: def get_corstone_310_installation() -> Installation: """Get Corstone-310 installation.""" - corstone_310 = AIETBasedInstallation( - aiet_runner=aiet.get_aiet_runner(), + corstone_310 = BackendInstallation( + backend_runner=backend_manager.BackendRunner(), # pylint: disable=line-too-long - metadata=AIETMetadata( + metadata=BackendMetadata( name="Corstone-310", description="Corstone-310 FVP", system_config="aiet/systems/corstone-310/aiet-config.json", diff --git a/src/mlia/utils/proc.py b/src/mlia/utils/proc.py index 39aca43..18a4305 100644 --- a/src/mlia/utils/proc.py +++ b/src/mlia/utils/proc.py @@ -8,7 +8,6 @@ import time from abc import ABC from abc import abstractmethod from contextlib import contextmanager -from contextlib import suppress from pathlib import Path from typing import Any from typing import Generator @@ -23,7 +22,7 @@ class OutputConsumer(ABC): @abstractmethod def feed(self, line: str) -> None: - """Feed new line to the consumerr.""" + """Feed new line to the consumer.""" class RunningCommand: @@ -32,7 +31,7 @@ class RunningCommand: def __init__(self, process: subprocess.Popen) -> None: """Init running command instance.""" self.process = process - self._output_consumers: Optional[List[OutputConsumer]] = None + self.output_consumers: List[OutputConsumer] = [] def is_alive(self) -> bool: """Return true if process is still alive.""" @@ -57,25 +56,14 @@ class RunningCommand: """Send signal to the process.""" self.process.send_signal(signal_num) - @property - def output_consumers(self) -> Optional[List[OutputConsumer]]: - """Property output_consumers.""" - return self._output_consumers - - @output_consumers.setter - def output_consumers(self, output_consumers: List[OutputConsumer]) -> None: - """Set output consumers.""" - self._output_consumers = output_consumers - def consume_output(self) -> None: """Pass program's output to the consumers.""" - if self.process is None or self.output_consumers is None: + if self.process is None or not self.output_consumers: return for line in self.stdout(): for consumer in self.output_consumers: - with suppress(): - consumer.feed(line) + consumer.feed(line) def stop( self, wait: bool = True, num_of_attempts: int = 5, interval: float = 0.5 diff --git a/tests/aiet/__init__.py b/tests/aiet/__init__.py deleted file mode 100644 index 873a7df..0000000 --- a/tests/aiet/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""AIET tests module.""" diff --git a/tests/aiet/conftest.py b/tests/aiet/conftest.py deleted file mode 100644 index cab3dc2..0000000 --- a/tests/aiet/conftest.py +++ /dev/null @@ -1,139 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=redefined-outer-name -"""conftest for pytest.""" -import shutil -import tarfile -from pathlib import Path -from typing import Any - -import pytest -from click.testing import CliRunner - -from aiet.backend.common import get_backend_configs - - -@pytest.fixture(scope="session") -def test_systems_path(test_resources_path: Path) -> Path: - """Return test systems path in a pytest fixture.""" - return test_resources_path / "systems" - - -@pytest.fixture(scope="session") -def test_applications_path(test_resources_path: Path) -> Path: - """Return test applications path in a pytest fixture.""" - return test_resources_path / "applications" - - -@pytest.fixture(scope="session") -def test_tools_path(test_resources_path: Path) -> Path: - """Return test tools path in a pytest fixture.""" - return test_resources_path / "tools" - - -@pytest.fixture(scope="session") -def test_resources_path() -> Path: - """Return test resources path in a pytest fixture.""" - current_path = Path(__file__).parent.absolute() - return current_path / "test_resources" - - -@pytest.fixture(scope="session") -def non_optimised_input_model_file(test_tflite_model: Path) -> Path: - """Provide the path to a quantized dummy model file.""" - return test_tflite_model - - -@pytest.fixture(scope="session") -def optimised_input_model_file(test_tflite_vela_model: Path) -> Path: - """Provide path to Vela-optimised dummy model file.""" - return test_tflite_vela_model - - -@pytest.fixture(scope="session") -def invalid_input_model_file(test_tflite_invalid_model: Path) -> Path: - """Provide the path to an invalid dummy model file.""" - return test_tflite_invalid_model - - -@pytest.fixture(autouse=True) -def test_resources(monkeypatch: pytest.MonkeyPatch, test_resources_path: Path) -> Any: - """Force using test resources as middleware's repository.""" - - def get_test_resources() -> Path: - """Return path to the test resources.""" - return test_resources_path - - monkeypatch.setattr("aiet.utils.fs.get_aiet_resources", get_test_resources) - yield - - -@pytest.fixture(scope="session", autouse=True) -def add_tools(test_resources_path: Path) -> Any: - """Symlink the tools from the original resources path to the test resources path.""" - # tool_dirs = get_available_tool_directory_names() - tool_dirs = [cfg.parent for cfg in get_backend_configs("tools")] - - links = { - src_dir: (test_resources_path / "tools" / src_dir.name) for src_dir in tool_dirs - } - for src_dir, dst_dir in links.items(): - if not dst_dir.exists(): - dst_dir.symlink_to(src_dir, target_is_directory=True) - yield - # Remove symlinks - for dst_dir in links.values(): - if dst_dir.is_symlink(): - dst_dir.unlink() - - -def create_archive( - archive_name: str, source: Path, destination: Path, with_root_folder: bool = False -) -> None: - """Create archive from directory source.""" - with tarfile.open(destination / archive_name, mode="w:gz") as tar: - for item in source.iterdir(): - item_name = item.name - if with_root_folder: - item_name = f"{source.name}/{item_name}" - tar.add(item, item_name) - - -def process_directory(source: Path, destination: Path) -> None: - """Process resource directory.""" - destination.mkdir() - - for item in source.iterdir(): - if item.is_dir(): - create_archive(f"{item.name}.tar.gz", item, destination) - create_archive(f"{item.name}_dir.tar.gz", item, destination, True) - - -@pytest.fixture(scope="session", autouse=True) -def add_archives( - test_resources_path: Path, tmp_path_factory: pytest.TempPathFactory -) -> Any: - """Generate archives of the test resources.""" - tmp_path = tmp_path_factory.mktemp("archives") - - archives_path = tmp_path / "archives" - archives_path.mkdir() - - if (archives_path_link := test_resources_path / "archives").is_symlink(): - archives_path.unlink() - - archives_path_link.symlink_to(archives_path, target_is_directory=True) - - for item in ["applications", "systems"]: - process_directory(test_resources_path / item, archives_path / item) - - yield - - archives_path_link.unlink() - shutil.rmtree(tmp_path) - - -@pytest.fixture(scope="module") -def cli_runner() -> CliRunner: - """Return CliRunner instance in a pytest fixture.""" - return CliRunner() diff --git a/tests/aiet/test_backend_tool.py b/tests/aiet/test_backend_tool.py deleted file mode 100644 index fd5960d..0000000 --- a/tests/aiet/test_backend_tool.py +++ /dev/null @@ -1,60 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=no-self-use -"""Tests for the tool backend.""" -from collections import Counter - -import pytest - -from aiet.backend.common import ConfigurationException -from aiet.backend.config import ToolConfig -from aiet.backend.tool import get_available_tool_directory_names -from aiet.backend.tool import get_available_tools -from aiet.backend.tool import get_tool -from aiet.backend.tool import Tool - - -def test_get_available_tool_directory_names() -> None: - """Test get_available_tools mocking get_resources.""" - directory_names = get_available_tool_directory_names() - assert Counter(directory_names) == Counter(["tool1", "tool2", "vela"]) - - -def test_get_available_tools() -> None: - """Test get_available_tools mocking get_resources.""" - available_tools = get_available_tools() - expected_tool_names = sorted( - [ - "tool_1", - "tool_2", - "vela", - "vela", - "vela", - ] - ) - - assert all(isinstance(s, Tool) for s in available_tools) - assert all(s != 42 for s in available_tools) - assert any(s == available_tools[0] for s in available_tools) - assert len(available_tools) == len(expected_tool_names) - available_tool_names = sorted(str(s) for s in available_tools) - assert available_tool_names == expected_tool_names - - -def test_get_tool() -> None: - """Test get_tool mocking get_resoures.""" - tools = get_tool("tool_1") - assert len(tools) == 1 - tool = tools[0] - assert tool is not None - assert isinstance(tool, Tool) - assert tool.name == "tool_1" - - tools = get_tool("unknown tool") - assert not tools - - -def test_tool_creation() -> None: - """Test edge cases when creating a Tool instance.""" - with pytest.raises(ConfigurationException): - Tool(ToolConfig(name="test", commands={"test": []})) # no 'run' command diff --git a/tests/aiet/test_check_model.py b/tests/aiet/test_check_model.py deleted file mode 100644 index 4eafe59..0000000 --- a/tests/aiet/test_check_model.py +++ /dev/null @@ -1,162 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=redefined-outer-name,no-self-use -"""Module for testing check_model.py script.""" -from pathlib import Path -from typing import Any - -import pytest -from ethosu.vela.tflite.Model import Model -from ethosu.vela.tflite.OperatorCode import OperatorCode - -from aiet.cli.common import InvalidTFLiteFileError -from aiet.cli.common import ModelOptimisedException -from aiet.resources.tools.vela.check_model import check_custom_codes_for_ethosu -from aiet.resources.tools.vela.check_model import check_model -from aiet.resources.tools.vela.check_model import get_custom_codes_from_operators -from aiet.resources.tools.vela.check_model import get_model_from_file -from aiet.resources.tools.vela.check_model import get_operators_from_model -from aiet.resources.tools.vela.check_model import is_vela_optimised - - -@pytest.fixture(scope="session") -def optimised_tflite_model( - optimised_input_model_file: Path, -) -> Model: - """Return Model instance read from a Vela-optimised TFLite file.""" - return get_model_from_file(optimised_input_model_file) - - -@pytest.fixture(scope="session") -def non_optimised_tflite_model( - non_optimised_input_model_file: Path, -) -> Model: - """Return Model instance read from a Vela-optimised TFLite file.""" - return get_model_from_file(non_optimised_input_model_file) - - -class TestIsVelaOptimised: - """Test class for is_vela_optimised() function.""" - - def test_return_true_when_input_is_optimised( - self, - optimised_tflite_model: Model, - ) -> None: - """Verify True returned when input is optimised model.""" - output = is_vela_optimised(optimised_tflite_model) - - assert output is True - - def test_return_false_when_input_is_not_optimised( - self, - non_optimised_tflite_model: Model, - ) -> None: - """Verify False returned when input is non-optimised model.""" - output = is_vela_optimised(non_optimised_tflite_model) - - assert output is False - - -def test_get_operator_list_returns_correct_instances( - optimised_tflite_model: Model, -) -> None: - """Verify list of OperatorCode instances returned by get_operator_list().""" - operator_list = get_operators_from_model(optimised_tflite_model) - - assert all(isinstance(operator, OperatorCode) for operator in operator_list) - - -class TestGetCustomCodesFromOperators: - """Test the get_custom_codes_from_operators() function.""" - - def test_returns_empty_list_when_input_operators_have_no_custom_codes( - self, monkeypatch: Any - ) -> None: - """Verify function returns empty list when operators have no custom codes.""" - # Mock OperatorCode.CustomCode() function to return None - monkeypatch.setattr( - "ethosu.vela.tflite.OperatorCode.OperatorCode.CustomCode", lambda _: None - ) - - operators = [OperatorCode()] * 3 - - custom_codes = get_custom_codes_from_operators(operators) - - assert custom_codes == [] - - def test_returns_custom_codes_when_input_operators_have_custom_codes( - self, monkeypatch: Any - ) -> None: - """Verify list of bytes objects returned representing the CustomCodes.""" - # Mock OperatorCode.CustomCode() function to return a byte string - monkeypatch.setattr( - "ethosu.vela.tflite.OperatorCode.OperatorCode.CustomCode", - lambda _: b"custom-code", - ) - - operators = [OperatorCode()] * 3 - - custom_codes = get_custom_codes_from_operators(operators) - - assert custom_codes == [b"custom-code", b"custom-code", b"custom-code"] - - -@pytest.mark.parametrize( - "custom_codes, expected_output", - [ - ([b"ethos-u", b"something else"], True), - ([b"custom-code-1", b"custom-code-2"], False), - ], -) -def test_check_list_for_ethosu(custom_codes: list, expected_output: bool) -> None: - """Verify function detects 'ethos-u' bytes in the input list.""" - output = check_custom_codes_for_ethosu(custom_codes) - assert output is expected_output - - -class TestGetModelFromFile: - """Test the get_model_from_file() function.""" - - def test_error_raised_when_input_is_invalid_model_file( - self, - invalid_input_model_file: Path, - ) -> None: - """Verify error thrown when an invalid model file is given.""" - with pytest.raises(InvalidTFLiteFileError): - get_model_from_file(invalid_input_model_file) - - def test_model_instance_returned_when_input_is_valid_model_file( - self, - optimised_input_model_file: Path, - ) -> None: - """Verify file is read successfully and returns model instance.""" - tflite_model = get_model_from_file(optimised_input_model_file) - - assert isinstance(tflite_model, Model) - - -class TestCheckModel: - """Test the check_model() function.""" - - def test_check_model_with_non_optimised_input( - self, - non_optimised_input_model_file: Path, - ) -> None: - """Verify no error occurs for a valid input file.""" - check_model(non_optimised_input_model_file) - - def test_check_model_with_optimised_input( - self, - optimised_input_model_file: Path, - ) -> None: - """Verify that the right exception is raised with already optimised input.""" - with pytest.raises(ModelOptimisedException): - check_model(optimised_input_model_file) - - def test_check_model_with_invalid_input( - self, - invalid_input_model_file: Path, - ) -> None: - """Verify that an exception is raised with invalid input.""" - with pytest.raises(Exception): - check_model(invalid_input_model_file) diff --git a/tests/aiet/test_cli.py b/tests/aiet/test_cli.py deleted file mode 100644 index e8589fa..0000000 --- a/tests/aiet/test_cli.py +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Module for testing CLI top command.""" -from typing import Any -from unittest.mock import ANY -from unittest.mock import MagicMock - -from click.testing import CliRunner - -from aiet.cli import cli - - -def test_cli(cli_runner: CliRunner) -> None: - """Test CLI top level command.""" - result = cli_runner.invoke(cli) - assert result.exit_code == 0 - assert "system" in cli.commands - assert "application" in cli.commands - - -def test_cli_version(cli_runner: CliRunner) -> None: - """Test version option.""" - result = cli_runner.invoke(cli, ["--version"]) - assert result.exit_code == 0 - assert "version" in result.output - - -def test_cli_verbose(cli_runner: CliRunner, monkeypatch: Any) -> None: - """Test verbose option.""" - with monkeypatch.context() as mock_context: - mock = MagicMock() - # params[1] is the verbose option and we need to replace the - # callback with a mock object - mock_context.setattr(cli.params[1], "callback", mock) - cli_runner.invoke(cli, ["-vvvv"]) - # 4 is the number -v called earlier - mock.assert_called_once_with(ANY, ANY, 4) diff --git a/tests/aiet/test_cli_application.py b/tests/aiet/test_cli_application.py deleted file mode 100644 index f1ccc44..0000000 --- a/tests/aiet/test_cli_application.py +++ /dev/null @@ -1,1153 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=attribute-defined-outside-init,no-member,line-too-long,too-many-arguments,too-many-locals,redefined-outer-name,too-many-lines -"""Module for testing CLI application subcommand.""" -import base64 -import json -import re -import time -from contextlib import contextmanager -from contextlib import ExitStack -from pathlib import Path -from typing import Any -from typing import Generator -from typing import IO -from typing import List -from typing import Optional -from typing import TypedDict -from unittest.mock import MagicMock - -import click -import pytest -from click.testing import CliRunner -from filelock import FileLock - -from aiet.backend.application import Application -from aiet.backend.config import ApplicationConfig -from aiet.backend.config import LocalProtocolConfig -from aiet.backend.config import SSHConfig -from aiet.backend.config import SystemConfig -from aiet.backend.config import UserParamConfig -from aiet.backend.output_parser import Base64OutputParser -from aiet.backend.protocol import SSHProtocol -from aiet.backend.system import load_system -from aiet.cli.application import application_cmd -from aiet.cli.application import details_cmd -from aiet.cli.application import execute_cmd -from aiet.cli.application import install_cmd -from aiet.cli.application import list_cmd -from aiet.cli.application import parse_payload_run_config -from aiet.cli.application import remove_cmd -from aiet.cli.application import run_cmd -from aiet.cli.common import MiddlewareExitCode - - -def test_application_cmd() -> None: - """Test application commands.""" - commands = ["list", "details", "install", "remove", "execute", "run"] - assert all(command in application_cmd.commands for command in commands) - - -@pytest.mark.parametrize("format_", ["json", "cli"]) -def test_application_cmd_context(cli_runner: CliRunner, format_: str) -> None: - """Test setting command context parameters.""" - result = cli_runner.invoke(application_cmd, ["--format", format_]) - # command should fail if no subcommand provided - assert result.exit_code == 2 - - result = cli_runner.invoke(application_cmd, ["--format", format_, "list"]) - assert result.exit_code == 0 - - -@pytest.mark.parametrize( - "format_, system_name, expected_output", - [ - ( - "json", - None, - '{"type": "application", "available": ["application_1", "application_2"]}\n', - ), - ( - "json", - "system_1", - '{"type": "application", "available": ["application_1"]}\n', - ), - ("cli", None, "Available applications:\n\napplication_1\napplication_2\n"), - ("cli", "system_1", "Available applications:\n\napplication_1\n"), - ], -) -def test_list_cmd( - cli_runner: CliRunner, - monkeypatch: Any, - format_: str, - system_name: str, - expected_output: str, -) -> None: - """Test available applications commands.""" - # Mock some applications - mock_application_1 = MagicMock(spec=Application) - mock_application_1.name = "application_1" - mock_application_1.can_run_on.return_value = system_name == "system_1" - mock_application_2 = MagicMock(spec=Application) - mock_application_2.name = "application_2" - mock_application_2.can_run_on.return_value = system_name == "system_2" - - # Monkey patch the call get_available_applications - mock_available_applications = MagicMock() - mock_available_applications.return_value = [mock_application_1, mock_application_2] - - monkeypatch.setattr( - "aiet.backend.application.get_available_applications", - mock_available_applications, - ) - - obj = {"format": format_} - args = [] - if system_name: - list_cmd.params[0].type = click.Choice([system_name]) - args = ["--system", system_name] - result = cli_runner.invoke(list_cmd, obj=obj, args=args) - assert result.output == expected_output - - -def get_test_application() -> Application: - """Return test system details.""" - config = ApplicationConfig( - name="application", - description="test", - build_dir="", - supported_systems=[], - deploy_data=[], - user_params={}, - commands={ - "clean": ["clean"], - "build": ["build"], - "run": ["run"], - "post_run": ["post_run"], - }, - ) - - return Application(config) - - -def get_details_cmd_json_output() -> str: - """Get JSON output for details command.""" - json_output = """ -[ - { - "type": "application", - "name": "application", - "description": "test", - "supported_systems": [], - "commands": { - "clean": { - "command_strings": [ - "clean" - ], - "user_params": [] - }, - "build": { - "command_strings": [ - "build" - ], - "user_params": [] - }, - "run": { - "command_strings": [ - "run" - ], - "user_params": [] - }, - "post_run": { - "command_strings": [ - "post_run" - ], - "user_params": [] - } - } - } -]""" - return json.dumps(json.loads(json_output)) + "\n" - - -def get_details_cmd_console_output() -> str: - """Get console output for details command.""" - return ( - 'Application "application" details' - + "\nDescription: test" - + "\n\nSupported systems: " - + "\n\nclean commands:" - + "\nCommands: ['clean']" - + "\n\nbuild commands:" - + "\nCommands: ['build']" - + "\n\nrun commands:" - + "\nCommands: ['run']" - + "\n\npost_run commands:" - + "\nCommands: ['post_run']" - + "\n" - ) - - -@pytest.mark.parametrize( - "application_name,format_, expected_output", - [ - ("application", "json", get_details_cmd_json_output()), - ("application", "cli", get_details_cmd_console_output()), - ], -) -def test_details_cmd( - cli_runner: CliRunner, - monkeypatch: Any, - application_name: str, - format_: str, - expected_output: str, -) -> None: - """Test application details command.""" - monkeypatch.setattr( - "aiet.cli.application.get_application", - MagicMock(return_value=[get_test_application()]), - ) - - details_cmd.params[0].type = click.Choice(["application"]) - result = cli_runner.invoke( - details_cmd, obj={"format": format_}, args=["--name", application_name] - ) - assert result.exception is None - assert result.output == expected_output - - -def test_details_cmd_wrong_system(cli_runner: CliRunner, monkeypatch: Any) -> None: - """Test details command fails if application is not supported by the system.""" - monkeypatch.setattr( - "aiet.backend.execution.get_application", MagicMock(return_value=[]) - ) - - details_cmd.params[0].type = click.Choice(["application"]) - details_cmd.params[1].type = click.Choice(["system"]) - result = cli_runner.invoke( - details_cmd, args=["--name", "application", "--system", "system"] - ) - assert result.exit_code == 2 - assert ( - "Application 'application' doesn't support the system 'system'" in result.stdout - ) - - -def test_install_cmd(cli_runner: CliRunner, monkeypatch: Any) -> None: - """Test install application command.""" - mock_install_application = MagicMock() - monkeypatch.setattr( - "aiet.cli.application.install_application", mock_install_application - ) - - args = ["--source", "test"] - cli_runner.invoke(install_cmd, args=args) - mock_install_application.assert_called_once_with(Path("test")) - - -def test_remove_cmd(cli_runner: CliRunner, monkeypatch: Any) -> None: - """Test remove application command.""" - mock_remove_application = MagicMock() - monkeypatch.setattr( - "aiet.cli.application.remove_application", mock_remove_application - ) - remove_cmd.params[0].type = click.Choice(["test"]) - - args = ["--directory_name", "test"] - cli_runner.invoke(remove_cmd, args=args) - mock_remove_application.assert_called_once_with("test") - - -class ExecutionCase(TypedDict, total=False): - """Execution case.""" - - args: List[str] - lock_path: str - can_establish_connection: bool - establish_connection_delay: int - app_exit_code: int - exit_code: int - output: str - - -@pytest.mark.parametrize( - "application_config, system_config, executions", - [ - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - config_location=Path("wrong_location"), - commands={"build": ["echo build {application.name}"]}, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - config_location=Path("wrong_location"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ), - [ - ExecutionCase( - args=["-c", "build"], - exit_code=MiddlewareExitCode.CONFIGURATION_ERROR, - output="Error: Application test_application has wrong config location\n", - ) - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - build_dir="build", - deploy_data=[("sample_file", "/tmp/sample_file")], - commands={"build": ["echo build {application.name}"]}, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ), - [ - ExecutionCase( - args=["-c", "run"], - exit_code=MiddlewareExitCode.CONFIGURATION_ERROR, - output="Error: System test_system does not support data deploy\n", - ) - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - commands={"build": ["echo build {application.name}"]}, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ), - [ - ExecutionCase( - args=["-c", "build"], - exit_code=MiddlewareExitCode.CONFIGURATION_ERROR, - output="Error: No build directory defined for the app test_application\n", - ) - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["new_system"], - build_dir="build", - commands={ - "build": ["echo build {application.name} with {user_params:0}"] - }, - user_params={ - "build": [ - UserParamConfig( - name="param", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - ) - ] - }, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ), - [ - ExecutionCase( - args=["-c", "build"], - exit_code=1, - output="Error: Application 'test_application' doesn't support the system 'test_system'\n", - ) - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - build_dir="build", - commands={"build": ["false"]}, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ), - [ - ExecutionCase( - args=["-c", "build"], - exit_code=MiddlewareExitCode.BACKEND_ERROR, - output="""Running: false -Error: Execution failed. Please check output for the details.\n""", - ) - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - lock=True, - build_dir="build", - commands={ - "build": ["echo build {application.name} with {user_params:0}"] - }, - user_params={ - "build": [ - UserParamConfig( - name="param", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - ) - ] - }, - ), - SystemConfig( - name="test_system", - description="Test system", - lock=True, - data_transfer=LocalProtocolConfig(protocol="local"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ), - [ - ExecutionCase( - args=["-c", "build"], - exit_code=MiddlewareExitCode.SUCCESS, - output="""Running: echo build test_application with param default -build test_application with param default\n""", - ), - ExecutionCase( - args=["-c", "build"], - lock_path="/tmp/middleware_test_application_test_system.lock", - exit_code=MiddlewareExitCode.CONCURRENT_ERROR, - output="Error: Another instance of the system is running\n", - ), - ExecutionCase( - args=["-c", "build", "--param=param=val3"], - exit_code=MiddlewareExitCode.SUCCESS, - output="""Running: echo build test_application with param val3 -build test_application with param val3\n""", - ), - ExecutionCase( - args=["-c", "build", "--param=param=newval"], - exit_code=1, - output="Error: Application parameter 'param=newval' not valid for command 'build'\n", - ), - ExecutionCase( - args=["-c", "some_command"], - exit_code=MiddlewareExitCode.CONFIGURATION_ERROR, - output="Error: Unsupported command some_command\n", - ), - ExecutionCase( - args=["-c", "run"], - exit_code=MiddlewareExitCode.SUCCESS, - output="""Generating commands to execute -Running: echo run test_application on test_system -run test_application on test_system\n""", - ), - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - deploy_data=[("sample_file", "/tmp/sample_file")], - commands={ - "run": [ - "echo run {application.name} with {user_params:param} on {system.name}" - ] - }, - user_params={ - "run": [ - UserParamConfig( - name="param=", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - alias="param", - ) - ] - }, - ), - SystemConfig( - name="test_system", - description="Test system", - lock=True, - data_transfer=SSHConfig( - protocol="ssh", - username="username", - password="password", - hostname="localhost", - port="8022", - ), - commands={"run": ["sleep 100"]}, - ), - [ - ExecutionCase( - args=["-c", "run"], - exit_code=MiddlewareExitCode.SUCCESS, - output="""Generating commands to execute -Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds . -Deploying {application.config_location}/sample_file onto /tmp/sample_file -Running: echo run test_application with param=default on test_system -Shutting down sequence... -Stopping test_system... (It could take few seconds) -test_system stopped successfully.\n""", - ), - ExecutionCase( - args=["-c", "run"], - lock_path="/tmp/middleware_test_system.lock", - exit_code=MiddlewareExitCode.CONCURRENT_ERROR, - output="Error: Another instance of the system is running\n", - ), - ExecutionCase( - args=[ - "-c", - "run", - "--deploy={application.config_location}/sample_file:/tmp/sample_file", - ], - exit_code=0, - output="""Generating commands to execute -Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds . -Deploying {application.config_location}/sample_file onto /tmp/sample_file -Deploying {application.config_location}/sample_file onto /tmp/sample_file -Running: echo run test_application with param=default on test_system -Shutting down sequence... -Stopping test_system... (It could take few seconds) -test_system stopped successfully.\n""", - ), - ExecutionCase( - args=["-c", "run"], - app_exit_code=1, - exit_code=0, - output="""Generating commands to execute -Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds . -Deploying {application.config_location}/sample_file onto /tmp/sample_file -Running: echo run test_application with param=default on test_system -Application exited with exit code 1 -Shutting down sequence... -Stopping test_system... (It could take few seconds) -test_system stopped successfully.\n""", - ), - ExecutionCase( - args=["-c", "run"], - exit_code=MiddlewareExitCode.CONNECTION_ERROR, - can_establish_connection=False, - output="""Generating commands to execute -Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds .......................................................................................... -Shutting down sequence... -Stopping test_system... (It could take few seconds) -test_system stopped successfully. -Error: Couldn't connect to 'localhost:8022'.\n""", - ), - ExecutionCase( - args=["-c", "run", "--deploy=bad_format"], - exit_code=1, - output="Error: Invalid deploy parameter 'bad_format' for command run\n", - ), - ExecutionCase( - args=["-c", "run", "--deploy=:"], - exit_code=1, - output="Error: Invalid deploy parameter ':' for command run\n", - ), - ExecutionCase( - args=["-c", "run", "--deploy= : "], - exit_code=1, - output="Error: Invalid deploy parameter ' : ' for command run\n", - ), - ExecutionCase( - args=["-c", "run", "--deploy=some_src_file:"], - exit_code=1, - output="Error: Invalid deploy parameter 'some_src_file:' for command run\n", - ), - ExecutionCase( - args=["-c", "run", "--deploy=:some_dst_file"], - exit_code=1, - output="Error: Invalid deploy parameter ':some_dst_file' for command run\n", - ), - ExecutionCase( - args=["-c", "run", "--deploy=unknown_file:/tmp/dest"], - exit_code=1, - output="Error: Path unknown_file does not exist\n", - ), - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - commands={ - "run": [ - "echo run {application.name} with {user_params:param} on {system.name}" - ] - }, - user_params={ - "run": [ - UserParamConfig( - name="param=", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - alias="param", - ) - ] - }, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=SSHConfig( - protocol="ssh", - username="username", - password="password", - hostname="localhost", - port="8022", - ), - commands={"run": ["echo Unable to start system"]}, - ), - [ - ExecutionCase( - args=["-c", "run"], - exit_code=4, - can_establish_connection=False, - establish_connection_delay=1, - output="""Generating commands to execute -Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds . - ----------- test_system execution failed ---------- -Unable to start system - - - -Shutting down sequence... -Stopping test_system... (It could take few seconds) -test_system stopped successfully. -Error: Execution failed. Please check output for the details.\n""", - ) - ], - ], - ], -) -def test_application_command_execution( - application_config: ApplicationConfig, - system_config: SystemConfig, - executions: List[ExecutionCase], - tmpdir: Any, - cli_runner: CliRunner, - monkeypatch: Any, -) -> None: - """Test application command execution.""" - - @contextmanager - def lock_execution(lock_path: str) -> Generator[None, None, None]: - lock = FileLock(lock_path) - lock.acquire(timeout=1) - - try: - yield - finally: - lock.release() - - def replace_vars(str_val: str) -> str: - """Replace variables.""" - application_config_location = str( - application_config["config_location"].absolute() - ) - - return str_val.replace( - "{application.config_location}", application_config_location - ) - - for execution in executions: - init_execution_test( - monkeypatch, - tmpdir, - application_config, - system_config, - can_establish_connection=execution.get("can_establish_connection", True), - establish_conection_delay=execution.get("establish_connection_delay", 0), - remote_app_exit_code=execution.get("app_exit_code", 0), - ) - - lock_path = execution.get("lock_path") - - with ExitStack() as stack: - if lock_path: - stack.enter_context(lock_execution(lock_path)) - - args = [replace_vars(arg) for arg in execution["args"]] - - result = cli_runner.invoke( - execute_cmd, - args=["-n", application_config["name"], "-s", system_config["name"]] - + args, - ) - output = replace_vars(execution["output"]) - assert result.exit_code == execution["exit_code"] - assert result.stdout == output - - -@pytest.fixture(params=[False, True], ids=["run-cli", "run-json"]) -def payload_path_or_none(request: Any, tmp_path_factory: Any) -> Optional[Path]: - """Drives tests for run command so that it executes them both to use a json file, and to use CLI.""" - if request.param: - ret: Path = tmp_path_factory.getbasetemp() / "system_config_payload_file.json" - return ret - return None - - -def write_system_payload_config( - payload_file: IO[str], - application_config: ApplicationConfig, - system_config: SystemConfig, -) -> None: - """Write a json payload file for the given test configuration.""" - payload_dict = { - "id": system_config["name"], - "arguments": { - "application": application_config["name"], - }, - } - json.dump(payload_dict, payload_file) - - -@pytest.mark.parametrize( - "application_config, system_config, executions", - [ - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - build_dir="build", - commands={ - "build": ["echo build {application.name} with {user_params:0}"] - }, - user_params={ - "build": [ - UserParamConfig( - name="param", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - ) - ] - }, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ), - [ - ExecutionCase( - args=[], - exit_code=MiddlewareExitCode.SUCCESS, - output="""Running: echo build test_application with param default -build test_application with param default -Generating commands to execute -Running: echo run test_application on test_system -run test_application on test_system\n""", - ) - ], - ], - [ - ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - commands={ - "run": [ - "echo run {application.name} with {user_params:param} on {system.name}" - ] - }, - user_params={ - "run": [ - UserParamConfig( - name="param=", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - alias="param", - ) - ] - }, - ), - SystemConfig( - name="test_system", - description="Test system", - data_transfer=SSHConfig( - protocol="ssh", - username="username", - password="password", - hostname="localhost", - port="8022", - ), - commands={"run": ["sleep 100"]}, - ), - [ - ExecutionCase( - args=[], - exit_code=MiddlewareExitCode.SUCCESS, - output="""Generating commands to execute -Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds . -Running: echo run test_application with param=default on test_system -Shutting down sequence... -Stopping test_system... (It could take few seconds) -test_system stopped successfully.\n""", - ) - ], - ], - ], -) -def test_application_run( - application_config: ApplicationConfig, - system_config: SystemConfig, - executions: List[ExecutionCase], - tmpdir: Any, - cli_runner: CliRunner, - monkeypatch: Any, - payload_path_or_none: Path, -) -> None: - """Test application command execution.""" - for execution in executions: - init_execution_test(monkeypatch, tmpdir, application_config, system_config) - - if payload_path_or_none: - with open(payload_path_or_none, "w", encoding="utf-8") as payload_file: - write_system_payload_config( - payload_file, application_config, system_config - ) - - result = cli_runner.invoke( - run_cmd, - args=["--config", str(payload_path_or_none)], - ) - else: - result = cli_runner.invoke( - run_cmd, - args=["-n", application_config["name"], "-s", system_config["name"]] - + execution["args"], - ) - - assert result.stdout == execution["output"] - assert result.exit_code == execution["exit_code"] - - -@pytest.mark.parametrize( - "cmdline,error_pattern", - [ - [ - "--config {payload} -s test_system", - "when --config is set, the following parameters should not be provided", - ], - [ - "--config {payload} -n test_application", - "when --config is set, the following parameters should not be provided", - ], - [ - "--config {payload} -p mypar:3", - "when --config is set, the following parameters should not be provided", - ], - [ - "-p mypar:3", - "when --config is not set, the following parameters are required", - ], - ["-s test_system", "when --config is not set, --name is required"], - ["-n test_application", "when --config is not set, --system is required"], - ], -) -def test_application_run_invalid_param_combinations( - cmdline: str, - error_pattern: str, - cli_runner: CliRunner, - monkeypatch: Any, - tmp_path: Any, - tmpdir: Any, -) -> None: - """Test that invalid combinations arguments result in error as expected.""" - application_config = ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - build_dir="build", - commands={"build": ["echo build {application.name} with {user_params:0}"]}, - user_params={ - "build": [ - UserParamConfig( - name="param", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - ) - ] - }, - ) - system_config = SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={"run": ["echo run {application.name} on {system.name}"]}, - ) - - init_execution_test(monkeypatch, tmpdir, application_config, system_config) - - payload_file = tmp_path / "payload.json" - payload_file.write_text("dummy") - result = cli_runner.invoke( - run_cmd, - args=cmdline.format(payload=payload_file).split(), - ) - found = re.search(error_pattern, result.stdout) - assert found, f"Cannot find pattern: [{error_pattern}] in \n[\n{result.stdout}\n]" - - -@pytest.mark.parametrize( - "payload,expected", - [ - pytest.param( - {"arguments": {}}, - None, - marks=pytest.mark.xfail(reason="no system 'id''", strict=True), - ), - pytest.param( - {"id": "testsystem"}, - None, - marks=pytest.mark.xfail(reason="no arguments object", strict=True), - ), - ( - {"id": "testsystem", "arguments": {"application": "testapp"}}, - ("testsystem", "testapp", [], [], [], None), - ), - ( - { - "id": "testsystem", - "arguments": {"application": "testapp", "par1": "val1"}, - }, - ("testsystem", "testapp", ["par1=val1"], [], [], None), - ), - ( - { - "id": "testsystem", - "arguments": {"application": "testapp", "application/par1": "val1"}, - }, - ("testsystem", "testapp", ["par1=val1"], [], [], None), - ), - ( - { - "id": "testsystem", - "arguments": {"application": "testapp", "system/par1": "val1"}, - }, - ("testsystem", "testapp", [], ["par1=val1"], [], None), - ), - ( - { - "id": "testsystem", - "arguments": {"application": "testapp", "deploy/par1": "val1"}, - }, - ("testsystem", "testapp", [], [], ["par1"], None), - ), - ( - { - "id": "testsystem", - "arguments": { - "application": "testapp", - "appar1": "val1", - "application/appar2": "val2", - "system/syspar1": "val3", - "deploy/depploypar1": "val4", - "application/appar3": "val5", - "system/syspar2": "val6", - "deploy/depploypar2": "val7", - }, - }, - ( - "testsystem", - "testapp", - ["appar1=val1", "appar2=val2", "appar3=val5"], - ["syspar1=val3", "syspar2=val6"], - ["depploypar1", "depploypar2"], - None, - ), - ), - ], -) -def test_parse_payload_run_config(payload: dict, expected: tuple) -> None: - """Test parsing of the JSON payload for the run_config command.""" - assert parse_payload_run_config(payload) == expected - - -def test_application_run_report( - tmpdir: Any, - cli_runner: CliRunner, - monkeypatch: Any, -) -> None: - """Test flag '--report' of command 'application run'.""" - app_metrics = {"app_metric": 3.14} - app_metrics_b64 = base64.b64encode(json.dumps(app_metrics).encode("utf-8")) - application_config = ApplicationConfig( - name="test_application", - description="Test application", - supported_systems=["test_system"], - build_dir="build", - commands={"build": ["echo build {application.name} with {user_params:0}"]}, - user_params={ - "build": [ - UserParamConfig( - name="param", - description="sample parameter", - default_value="default", - values=["val1", "val2", "val3"], - ), - UserParamConfig( - name="p2", - description="another parameter, not overridden", - default_value="the-right-choice", - values=["the-right-choice", "the-bad-choice"], - ), - ] - }, - ) - system_config = SystemConfig( - name="test_system", - description="Test system", - data_transfer=LocalProtocolConfig(protocol="local"), - commands={ - "run": [ - "echo run {application.name} on {system.name}", - f"echo build <{Base64OutputParser.TAG_NAME}>{app_metrics_b64.decode('utf-8')}</{Base64OutputParser.TAG_NAME}>", - ] - }, - reporting={ - "regex": { - "app_name": { - "pattern": r"run (.\S*) ", - "type": "str", - }, - "sys_name": { - "pattern": r"on (.\S*)", - "type": "str", - }, - } - }, - ) - report_file = Path(tmpdir) / "test_report.json" - param_val = "param=val1" - exit_code = MiddlewareExitCode.SUCCESS - - init_execution_test(monkeypatch, tmpdir, application_config, system_config) - - result = cli_runner.invoke( - run_cmd, - args=[ - "-n", - application_config["name"], - "-s", - system_config["name"], - "--report", - str(report_file), - "--param", - param_val, - ], - ) - assert result.exit_code == exit_code - assert report_file.is_file() - with open(report_file, "r", encoding="utf-8") as file: - report = json.load(file) - - assert report == { - "application": { - "metrics": {"0": {"app_metric": 3.14}}, - "name": "test_application", - "params": {"param": "val1", "p2": "the-right-choice"}, - }, - "system": { - "metrics": {"app_name": "test_application", "sys_name": "test_system"}, - "name": "test_system", - "params": {}, - }, - } - - -def init_execution_test( - monkeypatch: Any, - tmpdir: Any, - application_config: ApplicationConfig, - system_config: SystemConfig, - can_establish_connection: bool = True, - establish_conection_delay: float = 0, - remote_app_exit_code: int = 0, -) -> None: - """Init execution test.""" - application_name = application_config["name"] - system_name = system_config["name"] - - execute_cmd.params[0].type = click.Choice([application_name]) - execute_cmd.params[1].type = click.Choice([system_name]) - execute_cmd.params[2].type = click.Choice(["build", "run", "some_command"]) - - run_cmd.params[0].type = click.Choice([application_name]) - run_cmd.params[1].type = click.Choice([system_name]) - - if "config_location" not in application_config: - application_path = Path(tmpdir) / "application" - application_path.mkdir() - application_config["config_location"] = application_path - - # this file could be used as deploy parameter value or - # as deploy parameter in application configuration - sample_file = application_path / "sample_file" - sample_file.touch() - monkeypatch.setattr( - "aiet.backend.application.get_available_applications", - MagicMock(return_value=[Application(application_config)]), - ) - - ssh_protocol_mock = MagicMock(spec=SSHProtocol) - - def mock_establish_connection() -> bool: - """Mock establish connection function.""" - # give some time for the system to start - time.sleep(establish_conection_delay) - return can_establish_connection - - ssh_protocol_mock.establish_connection.side_effect = mock_establish_connection - ssh_protocol_mock.connection_details.return_value = ("localhost", 8022) - ssh_protocol_mock.run.return_value = ( - remote_app_exit_code, - bytearray(), - bytearray(), - ) - monkeypatch.setattr( - "aiet.backend.protocol.SSHProtocol", MagicMock(return_value=ssh_protocol_mock) - ) - - if "config_location" not in system_config: - system_path = Path(tmpdir) / "system" - system_path.mkdir() - system_config["config_location"] = system_path - monkeypatch.setattr( - "aiet.backend.system.get_available_systems", - MagicMock(return_value=[load_system(system_config)]), - ) - - monkeypatch.setattr("aiet.backend.execution.wait", MagicMock()) diff --git a/tests/aiet/test_cli_common.py b/tests/aiet/test_cli_common.py deleted file mode 100644 index d018e44..0000000 --- a/tests/aiet/test_cli_common.py +++ /dev/null @@ -1,37 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Test for cli common module.""" -from typing import Any - -import pytest - -from aiet.cli.common import print_command_details -from aiet.cli.common import raise_exception_at_signal - - -def test_print_command_details(capsys: Any) -> None: - """Test print_command_details function.""" - command = { - "command_strings": ["echo test"], - "user_params": [ - {"name": "param_name", "description": "param_description"}, - { - "name": "param_name2", - "description": "param_description2", - "alias": "alias2", - }, - ], - } - print_command_details(command) - captured = capsys.readouterr() - assert "echo test" in captured.out - assert "param_name" in captured.out - assert "alias2" in captured.out - - -def test_raise_exception_at_signal() -> None: - """Test raise_exception_at_signal graceful shutdown.""" - with pytest.raises(Exception) as err: - raise_exception_at_signal(1, "") - - assert str(err.value) == "Middleware shutdown requested" diff --git a/tests/aiet/test_cli_system.py b/tests/aiet/test_cli_system.py deleted file mode 100644 index fd39f31..0000000 --- a/tests/aiet/test_cli_system.py +++ /dev/null @@ -1,240 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Module for testing CLI system subcommand.""" -import json -from pathlib import Path -from typing import Any -from typing import Dict -from typing import List -from typing import Optional -from typing import Union -from unittest.mock import MagicMock - -import click -import pytest -from click.testing import CliRunner - -from aiet.backend.config import SystemConfig -from aiet.backend.system import load_system -from aiet.backend.system import System -from aiet.cli.system import details_cmd -from aiet.cli.system import install_cmd -from aiet.cli.system import list_cmd -from aiet.cli.system import remove_cmd -from aiet.cli.system import system_cmd - - -def test_system_cmd() -> None: - """Test system commands.""" - commands = ["list", "details", "install", "remove"] - assert all(command in system_cmd.commands for command in commands) - - -@pytest.mark.parametrize("format_", ["json", "cli"]) -def test_system_cmd_context(cli_runner: CliRunner, format_: str) -> None: - """Test setting command context parameters.""" - result = cli_runner.invoke(system_cmd, ["--format", format_]) - # command should fail if no subcommand provided - assert result.exit_code == 2 - - result = cli_runner.invoke(system_cmd, ["--format", format_, "list"]) - assert result.exit_code == 0 - - -@pytest.mark.parametrize( - "format_,expected_output", - [ - ("json", '{"type": "system", "available": ["system1", "system2"]}\n'), - ("cli", "Available systems:\n\nsystem1\nsystem2\n"), - ], -) -def test_list_cmd_with_format( - cli_runner: CliRunner, monkeypatch: Any, format_: str, expected_output: str -) -> None: - """Test available systems command with different formats output.""" - # Mock some systems - mock_system1 = MagicMock() - mock_system1.name = "system1" - mock_system2 = MagicMock() - mock_system2.name = "system2" - - # Monkey patch the call get_available_systems - mock_available_systems = MagicMock() - mock_available_systems.return_value = [mock_system1, mock_system2] - monkeypatch.setattr("aiet.cli.system.get_available_systems", mock_available_systems) - - obj = {"format": format_} - result = cli_runner.invoke(list_cmd, obj=obj) - assert result.output == expected_output - - -def get_test_system( - annotations: Optional[Dict[str, Union[str, List[str]]]] = None -) -> System: - """Return test system details.""" - config = SystemConfig( - name="system", - description="test", - data_transfer={ - "protocol": "ssh", - "username": "root", - "password": "root", - "hostname": "localhost", - "port": "8022", - }, - commands={ - "clean": ["clean"], - "build": ["build"], - "run": ["run"], - "post_run": ["post_run"], - }, - annotations=annotations or {}, - ) - - return load_system(config) - - -def get_details_cmd_json_output( - annotations: Optional[Dict[str, Union[str, List[str]]]] = None -) -> str: - """Test JSON output for details command.""" - ann_str = "" - if annotations is not None: - ann_str = '"annotations":{},'.format(json.dumps(annotations)) - - json_output = ( - """ -{ - "type": "system", - "name": "system", - "description": "test", - "data_transfer_protocol": "ssh", - "commands": { - "clean": - { - "command_strings": ["clean"], - "user_params": [] - }, - "build": - { - "command_strings": ["build"], - "user_params": [] - }, - "run": - { - "command_strings": ["run"], - "user_params": [] - }, - "post_run": - { - "command_strings": ["post_run"], - "user_params": [] - } - }, -""" - + ann_str - + """ - "available_application" : [] - } -""" - ) - return json.dumps(json.loads(json_output)) + "\n" - - -def get_details_cmd_console_output( - annotations: Optional[Dict[str, Union[str, List[str]]]] = None -) -> str: - """Test console output for details command.""" - ann_str = "" - if annotations: - val_str = "".join( - "\n\t{}: {}".format(ann_name, ann_value) - for ann_name, ann_value in annotations.items() - ) - ann_str = "\nAnnotations:{}".format(val_str) - return ( - 'System "system" details' - + "\nDescription: test" - + "\nData Transfer Protocol: ssh" - + "\nAvailable Applications: " - + ann_str - + "\n\nclean commands:" - + "\nCommands: ['clean']" - + "\n\nbuild commands:" - + "\nCommands: ['build']" - + "\n\nrun commands:" - + "\nCommands: ['run']" - + "\n\npost_run commands:" - + "\nCommands: ['post_run']" - + "\n" - ) - - -@pytest.mark.parametrize( - "format_,system,expected_output", - [ - ( - "json", - get_test_system(annotations={"ann1": "annotation1", "ann2": ["a1", "a2"]}), - get_details_cmd_json_output( - annotations={"ann1": "annotation1", "ann2": ["a1", "a2"]} - ), - ), - ( - "cli", - get_test_system(annotations={"ann1": "annotation1", "ann2": ["a1", "a2"]}), - get_details_cmd_console_output( - annotations={"ann1": "annotation1", "ann2": ["a1", "a2"]} - ), - ), - ( - "json", - get_test_system(annotations={}), - get_details_cmd_json_output(annotations={}), - ), - ( - "cli", - get_test_system(annotations={}), - get_details_cmd_console_output(annotations={}), - ), - ], -) -def test_details_cmd( - cli_runner: CliRunner, - monkeypatch: Any, - format_: str, - system: System, - expected_output: str, -) -> None: - """Test details command with different formats output.""" - mock_get_system = MagicMock() - mock_get_system.return_value = system - monkeypatch.setattr("aiet.cli.system.get_system", mock_get_system) - - args = ["--name", "system"] - obj = {"format": format_} - details_cmd.params[0].type = click.Choice(["system"]) - - result = cli_runner.invoke(details_cmd, args=args, obj=obj) - assert result.output == expected_output - - -def test_install_cmd(cli_runner: CliRunner, monkeypatch: Any) -> None: - """Test install system command.""" - mock_install_system = MagicMock() - monkeypatch.setattr("aiet.cli.system.install_system", mock_install_system) - - args = ["--source", "test"] - cli_runner.invoke(install_cmd, args=args) - mock_install_system.assert_called_once_with(Path("test")) - - -def test_remove_cmd(cli_runner: CliRunner, monkeypatch: Any) -> None: - """Test remove system command.""" - mock_remove_system = MagicMock() - monkeypatch.setattr("aiet.cli.system.remove_system", mock_remove_system) - remove_cmd.params[0].type = click.Choice(["test"]) - - args = ["--directory_name", "test"] - cli_runner.invoke(remove_cmd, args=args) - mock_remove_system.assert_called_once_with("test") diff --git a/tests/aiet/test_cli_tool.py b/tests/aiet/test_cli_tool.py deleted file mode 100644 index 45d45c8..0000000 --- a/tests/aiet/test_cli_tool.py +++ /dev/null @@ -1,333 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=attribute-defined-outside-init,no-member,line-too-long,too-many-arguments,too-many-locals -"""Module for testing CLI tool subcommand.""" -import json -from pathlib import Path -from typing import Any -from typing import List -from typing import Optional -from typing import Sequence -from unittest.mock import MagicMock - -import click -import pytest -from click.testing import CliRunner -from click.testing import Result - -from aiet.backend.tool import get_unique_tool_names -from aiet.backend.tool import Tool -from aiet.cli.tool import details_cmd -from aiet.cli.tool import execute_cmd -from aiet.cli.tool import list_cmd -from aiet.cli.tool import tool_cmd - - -def test_tool_cmd() -> None: - """Test tool commands.""" - commands = ["list", "details", "execute"] - assert all(command in tool_cmd.commands for command in commands) - - -@pytest.mark.parametrize("format_", ["json", "cli"]) -def test_tool_cmd_context(cli_runner: CliRunner, format_: str) -> None: - """Test setting command context parameters.""" - result = cli_runner.invoke(tool_cmd, ["--format", format_]) - # command should fail if no subcommand provided - assert result.exit_code == 2 - - result = cli_runner.invoke(tool_cmd, ["--format", format_, "list"]) - assert result.exit_code == 0 - - -@pytest.mark.parametrize( - "format_, expected_output", - [ - ( - "json", - '{"type": "tool", "available": ["tool_1", "tool_2"]}\n', - ), - ("cli", "Available tools:\n\ntool_1\ntool_2\n"), - ], -) -def test_list_cmd( - cli_runner: CliRunner, - monkeypatch: Any, - format_: str, - expected_output: str, -) -> None: - """Test available tool commands.""" - # Mock some tools - mock_tool_1 = MagicMock(spec=Tool) - mock_tool_1.name = "tool_1" - mock_tool_2 = MagicMock(spec=Tool) - mock_tool_2.name = "tool_2" - - # Monkey patch the call get_available_tools - mock_available_tools = MagicMock() - mock_available_tools.return_value = [mock_tool_1, mock_tool_2] - - monkeypatch.setattr("aiet.backend.tool.get_available_tools", mock_available_tools) - - obj = {"format": format_} - args: Sequence[str] = [] - result = cli_runner.invoke(list_cmd, obj=obj, args=args) - assert result.output == expected_output - - -def get_details_cmd_json_output() -> List[dict]: - """Get JSON output for details command.""" - json_output = [ - { - "type": "tool", - "name": "tool_1", - "description": "This is tool 1", - "supported_systems": ["System 1"], - "commands": { - "clean": {"command_strings": ["echo 'clean'"], "user_params": []}, - "build": {"command_strings": ["echo 'build'"], "user_params": []}, - "run": {"command_strings": ["echo 'run'"], "user_params": []}, - "post_run": {"command_strings": ["echo 'post_run'"], "user_params": []}, - }, - } - ] - - return json_output - - -def get_details_cmd_console_output() -> str: - """Get console output for details command.""" - return ( - 'Tool "tool_1" details' - "\nDescription: This is tool 1" - "\n\nSupported systems: System 1" - "\n\nclean commands:" - "\nCommands: [\"echo 'clean'\"]" - "\n\nbuild commands:" - "\nCommands: [\"echo 'build'\"]" - "\n\nrun commands:\nCommands: [\"echo 'run'\"]" - "\n\npost_run commands:" - "\nCommands: [\"echo 'post_run'\"]" - "\n" - ) - - -@pytest.mark.parametrize( - [ - "tool_name", - "format_", - "expected_success", - "expected_output", - ], - [ - ("tool_1", "json", True, get_details_cmd_json_output()), - ("tool_1", "cli", True, get_details_cmd_console_output()), - ("non-existent tool", "json", False, None), - ("non-existent tool", "cli", False, None), - ], -) -def test_details_cmd( - cli_runner: CliRunner, - tool_name: str, - format_: str, - expected_success: bool, - expected_output: str, -) -> None: - """Test tool details command.""" - details_cmd.params[0].type = click.Choice(["tool_1", "tool_2", "vela"]) - result = cli_runner.invoke( - details_cmd, obj={"format": format_}, args=["--name", tool_name] - ) - success = result.exit_code == 0 - assert success == expected_success, result.output - if expected_success: - assert result.exception is None - output = json.loads(result.output) if format_ == "json" else result.output - assert output == expected_output - - -@pytest.mark.parametrize( - "system_name", - [ - "", - "Corstone-300: Cortex-M55+Ethos-U55", - "Corstone-300: Cortex-M55+Ethos-U65", - "Corstone-310: Cortex-M85+Ethos-U55", - ], -) -def test_details_cmd_vela(cli_runner: CliRunner, system_name: str) -> None: - """Test tool details command for Vela.""" - details_cmd.params[0].type = click.Choice(get_unique_tool_names()) - details_cmd.params[1].type = click.Choice([system_name]) - args = ["--name", "vela"] - if system_name: - args += ["--system", system_name] - result = cli_runner.invoke(details_cmd, obj={"format": "json"}, args=args) - success = result.exit_code == 0 - assert success, result.output - result_json = json.loads(result.output) - assert result_json - if system_name: - assert len(result_json) == 1 - tool = result_json[0] - assert len(tool["supported_systems"]) == 1 - assert system_name == tool["supported_systems"][0] - else: # no system specified => list details for all systems - assert len(result_json) == 3 - assert all(len(tool["supported_systems"]) == 1 for tool in result_json) - - -@pytest.fixture(scope="session") -def input_model_file(non_optimised_input_model_file: Path) -> Path: - """Provide the path to a quantized dummy model file in the test_resources_path.""" - return non_optimised_input_model_file - - -def execute_vela( - cli_runner: CliRunner, - tool_name: str = "vela", - system_name: Optional[str] = None, - input_model: Optional[Path] = None, - output_model: Optional[Path] = None, - mac: Optional[int] = None, - format_: str = "cli", -) -> Result: - """Run Vela with different parameters.""" - execute_cmd.params[0].type = click.Choice(get_unique_tool_names()) - execute_cmd.params[2].type = click.Choice([system_name or "dummy_system"]) - args = ["--name", tool_name] - if system_name is not None: - args += ["--system", system_name] - if input_model is not None: - args += ["--param", "input={}".format(input_model)] - if output_model is not None: - args += ["--param", "output={}".format(output_model)] - if mac is not None: - args += ["--param", "mac={}".format(mac)] - result = cli_runner.invoke( - execute_cmd, - args=args, - obj={"format": format_}, - ) - return result - - -@pytest.mark.parametrize("format_", ["cli, json"]) -@pytest.mark.parametrize( - ["tool_name", "system_name", "mac", "expected_success", "expected_output"], - [ - ("vela", "System 1", 32, False, None), # system not supported - ("vela", "NON-EXISTENT SYSTEM", 128, False, None), # system does not exist - ("vela", "Corstone-300: Cortex-M55+Ethos-U55", 32, True, None), - ("NON-EXISTENT TOOL", "Corstone-300: Cortex-M55+Ethos-U55", 32, False, None), - ("vela", "Corstone-300: Cortex-M55+Ethos-U55", 64, True, None), - ("vela", "Corstone-300: Cortex-M55+Ethos-U55", 128, True, None), - ("vela", "Corstone-300: Cortex-M55+Ethos-U55", 256, True, None), - ( - "vela", - "Corstone-300: Cortex-M55+Ethos-U55", - 512, - False, - None, - ), # mac not supported - ( - "vela", - "Corstone-300: Cortex-M55+Ethos-U65", - 32, - False, - None, - ), # mac not supported - ("vela", "Corstone-300: Cortex-M55+Ethos-U65", 256, True, None), - ("vela", "Corstone-300: Cortex-M55+Ethos-U65", 512, True, None), - ( - "vela", - None, - 512, - False, - "Error: Please specify the system for tool vela.", - ), # no system specified - ( - "NON-EXISTENT TOOL", - "Corstone-300: Cortex-M55+Ethos-U65", - 512, - False, - None, - ), # tool does not exist - ("vela", "Corstone-310: Cortex-M85+Ethos-U55", 128, True, None), - ], -) -def test_vela_run( - cli_runner: CliRunner, - format_: str, - input_model_file: Path, # pylint: disable=redefined-outer-name - tool_name: str, - system_name: Optional[str], - mac: int, - expected_success: bool, - expected_output: Optional[str], - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - """Test the execution of the Vela command.""" - monkeypatch.chdir(tmp_path) - - output_file = Path("vela_output.tflite") - - result = execute_vela( - cli_runner, - tool_name=tool_name, - system_name=system_name, - input_model=input_model_file, - output_model=output_file, - mac=mac, - format_=format_, - ) - - success = result.exit_code == 0 - assert success == expected_success - if success: - # Check output file - output_file = output_file.resolve() - assert output_file.is_file() - if expected_output: - assert result.output.strip() == expected_output - - -@pytest.mark.parametrize("include_input_model", [True, False]) -@pytest.mark.parametrize("include_output_model", [True, False]) -@pytest.mark.parametrize("include_mac", [True, False]) -def test_vela_run_missing_params( - cli_runner: CliRunner, - input_model_file: Path, # pylint: disable=redefined-outer-name - include_input_model: bool, - include_output_model: bool, - include_mac: bool, - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - """Test the execution of the Vela command with missing user parameters.""" - monkeypatch.chdir(tmp_path) - - output_model_file = Path("output_model.tflite") - system_name = "Corstone-300: Cortex-M55+Ethos-U65" - mac = 256 - # input_model is a required parameters, but mac and output_model have default values. - expected_success = include_input_model - - result = execute_vela( - cli_runner, - tool_name="vela", - system_name=system_name, - input_model=input_model_file if include_input_model else None, - output_model=output_model_file if include_output_model else None, - mac=mac if include_mac else None, - ) - - success = result.exit_code == 0 - assert success == expected_success, ( - f"Success is {success}, but expected {expected_success}. " - f"Included params: [" - f"input_model={include_input_model}, " - f"output_model={include_output_model}, " - f"mac={include_mac}]" - ) diff --git a/tests/aiet/test_main.py b/tests/aiet/test_main.py deleted file mode 100644 index f2ebae2..0000000 --- a/tests/aiet/test_main.py +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Module for testing AIET main.py.""" -from typing import Any -from unittest.mock import MagicMock - -from aiet import main - - -def test_main(monkeypatch: Any) -> None: - """Test main entry point function.""" - with monkeypatch.context() as mock_context: - mock = MagicMock() - mock_context.setattr(main, "cli", mock) - main.main() - mock.assert_called_once() diff --git a/tests/aiet/test_resources/tools/tool1/aiet-config.json b/tests/aiet/test_resources/tools/tool1/aiet-config.json deleted file mode 100644 index 067ef7e..0000000 --- a/tests/aiet/test_resources/tools/tool1/aiet-config.json +++ /dev/null @@ -1,30 +0,0 @@ -[ - { - "name": "tool_1", - "description": "This is tool 1", - "build_dir": "build", - "supported_systems": [ - { - "name": "System 1" - } - ], - "commands": { - "clean": [ - "echo 'clean'" - ], - "build": [ - "echo 'build'" - ], - "run": [ - "echo 'run'" - ], - "post_run": [ - "echo 'post_run'" - ] - }, - "user_params": { - "build": [], - "run": [] - } - } -] diff --git a/tests/aiet/test_resources/tools/tool2/aiet-config.json b/tests/aiet/test_resources/tools/tool2/aiet-config.json deleted file mode 100644 index 6eee9a6..0000000 --- a/tests/aiet/test_resources/tools/tool2/aiet-config.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "name": "tool_2", - "description": "This is tool 2 with no supported systems", - "build_dir": "build", - "supported_systems": [], - "commands": { - "clean": [ - "echo 'clean'" - ], - "build": [ - "echo 'build'" - ], - "run": [ - "echo 'run'" - ], - "post_run": [ - "echo 'post_run'" - ] - }, - "user_params": { - "build": [], - "run": [] - } - } -] diff --git a/tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json.license b/tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json.license deleted file mode 100644 index 9b83bfc..0000000 --- a/tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. - -SPDX-License-Identifier: Apache-2.0 diff --git a/tests/aiet/test_run_vela_script.py b/tests/aiet/test_run_vela_script.py deleted file mode 100644 index 971856e..0000000 --- a/tests/aiet/test_run_vela_script.py +++ /dev/null @@ -1,152 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -# pylint: disable=redefined-outer-name,no-self-use -"""Module for testing run_vela.py script.""" -from pathlib import Path -from typing import Any -from typing import List - -import pytest -from click.testing import CliRunner - -from aiet.cli.common import MiddlewareExitCode -from aiet.resources.tools.vela.check_model import get_model_from_file -from aiet.resources.tools.vela.check_model import is_vela_optimised -from aiet.resources.tools.vela.run_vela import run_vela - - -@pytest.fixture(scope="session") -def vela_config_path(test_tools_path: Path) -> Path: - """Return test systems path in a pytest fixture.""" - return test_tools_path / "vela" / "vela.ini" - - -@pytest.fixture( - params=[ - ["ethos-u65-256", "Ethos_U65_High_End", "U65_Shared_Sram"], - ["ethos-u55-32", "Ethos_U55_High_End_Embedded", "U55_Shared_Sram"], - ] -) -def ethos_config(request: Any) -> Any: - """Fixture to provide different configuration for Ethos-U optimization with Vela.""" - return request.param - - -# pylint: disable=too-many-arguments -def generate_args( - input_: Path, - output: Path, - cfg: Path, - acc_config: str, - system_config: str, - memory_mode: str, -) -> List[str]: - """Generate arguments that can be passed to script 'run_vela'.""" - return [ - "-i", - str(input_), - "-o", - str(output), - "--config", - str(cfg), - "--accelerator-config", - acc_config, - "--system-config", - system_config, - "--memory-mode", - memory_mode, - "--optimise", - "Performance", - ] - - -def check_run_vela( - cli_runner: CliRunner, args: List, expected_success: bool, output_file: Path -) -> None: - """Run Vela with the given arguments and check the result.""" - result = cli_runner.invoke(run_vela, args) - success = result.exit_code == MiddlewareExitCode.SUCCESS - assert success == expected_success - if success: - model = get_model_from_file(output_file) - assert is_vela_optimised(model) - - -def run_vela_script( - cli_runner: CliRunner, - input_model_file: Path, - output_model_file: Path, - vela_config: Path, - expected_success: bool, - acc_config: str, - system_config: str, - memory_mode: str, -) -> None: - """Run the command 'run_vela' on the command line.""" - args = generate_args( - input_model_file, - output_model_file, - vela_config, - acc_config, - system_config, - memory_mode, - ) - check_run_vela(cli_runner, args, expected_success, output_model_file) - - -class TestRunVelaCli: - """Test the command-line execution of the run_vela command.""" - - def test_non_optimised_model( - self, - cli_runner: CliRunner, - non_optimised_input_model_file: Path, - tmp_path: Path, - vela_config_path: Path, - ethos_config: List, - ) -> None: - """Verify Vela is run correctly on an unoptimised model.""" - run_vela_script( - cli_runner, - non_optimised_input_model_file, - tmp_path / "test.tflite", - vela_config_path, - True, - *ethos_config, - ) - - def test_optimised_model( - self, - cli_runner: CliRunner, - optimised_input_model_file: Path, - tmp_path: Path, - vela_config_path: Path, - ethos_config: List, - ) -> None: - """Verify Vela is run correctly on an already optimised model.""" - run_vela_script( - cli_runner, - optimised_input_model_file, - tmp_path / "test.tflite", - vela_config_path, - True, - *ethos_config, - ) - - def test_invalid_model( - self, - cli_runner: CliRunner, - invalid_input_model_file: Path, - tmp_path: Path, - vela_config_path: Path, - ethos_config: List, - ) -> None: - """Verify an error is raised when the input model is not valid.""" - run_vela_script( - cli_runner, - invalid_input_model_file, - tmp_path / "test.tflite", - vela_config_path, - False, - *ethos_config, - ) diff --git a/tests/aiet/test_utils_helpers.py b/tests/aiet/test_utils_helpers.py deleted file mode 100644 index bbe03fc..0000000 --- a/tests/aiet/test_utils_helpers.py +++ /dev/null @@ -1,27 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Module for testing helpers.py.""" -import logging -from typing import Any -from typing import List -from unittest.mock import call -from unittest.mock import MagicMock - -import pytest - -from aiet.utils.helpers import set_verbosity - - -@pytest.mark.parametrize( - "verbosity,expected_calls", - [(0, []), (1, [call(logging.INFO)]), (2, [call(logging.DEBUG)])], -) -def test_set_verbosity( - verbosity: int, expected_calls: List[Any], monkeypatch: Any -) -> None: - """Test set_verbosity() with different verbsosity levels.""" - with monkeypatch.context() as mock_context: - logging_mock = MagicMock() - mock_context.setattr(logging.getLogger(), "setLevel", logging_mock) - set_verbosity(None, None, verbosity) - logging_mock.assert_has_calls(expected_calls) diff --git a/tests/mlia/conftest.py b/tests/mlia/conftest.py index f683fca..0b4b2aa 100644 --- a/tests/mlia/conftest.py +++ b/tests/mlia/conftest.py @@ -1,7 +1,10 @@ # SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Pytest conf module.""" +import shutil +import tarfile from pathlib import Path +from typing import Any import pytest @@ -18,3 +21,91 @@ def fixture_test_resources_path() -> Path: def fixture_dummy_context(tmpdir: str) -> ExecutionContext: """Return dummy context fixture.""" return ExecutionContext(working_dir=tmpdir) + + +@pytest.fixture(scope="session") +def test_systems_path(test_resources_path: Path) -> Path: + """Return test systems path in a pytest fixture.""" + return test_resources_path / "backends" / "systems" + + +@pytest.fixture(scope="session") +def test_applications_path(test_resources_path: Path) -> Path: + """Return test applications path in a pytest fixture.""" + return test_resources_path / "backends" / "applications" + + +@pytest.fixture(scope="session") +def non_optimised_input_model_file(test_tflite_model: Path) -> Path: + """Provide the path to a quantized dummy model file.""" + return test_tflite_model + + +@pytest.fixture(scope="session") +def optimised_input_model_file(test_tflite_vela_model: Path) -> Path: + """Provide path to Vela-optimised dummy model file.""" + return test_tflite_vela_model + + +@pytest.fixture(scope="session") +def invalid_input_model_file(test_tflite_invalid_model: Path) -> Path: + """Provide the path to an invalid dummy model file.""" + return test_tflite_invalid_model + + +@pytest.fixture(autouse=True) +def test_resources(monkeypatch: pytest.MonkeyPatch, test_resources_path: Path) -> Any: + """Force using test resources as middleware's repository.""" + + def get_test_resources() -> Path: + """Return path to the test resources.""" + return test_resources_path / "backends" + + monkeypatch.setattr("mlia.backend.fs.get_backend_resources", get_test_resources) + yield + + +def create_archive( + archive_name: str, source: Path, destination: Path, with_root_folder: bool = False +) -> None: + """Create archive from directory source.""" + with tarfile.open(destination / archive_name, mode="w:gz") as tar: + for item in source.iterdir(): + item_name = item.name + if with_root_folder: + item_name = f"{source.name}/{item_name}" + tar.add(item, item_name) + + +def process_directory(source: Path, destination: Path) -> None: + """Process resource directory.""" + destination.mkdir() + + for item in source.iterdir(): + if item.is_dir(): + create_archive(f"{item.name}.tar.gz", item, destination) + create_archive(f"{item.name}_dir.tar.gz", item, destination, True) + + +@pytest.fixture(scope="session", autouse=True) +def add_archives( + test_resources_path: Path, tmp_path_factory: pytest.TempPathFactory +) -> Any: + """Generate archives of the test resources.""" + tmp_path = tmp_path_factory.mktemp("archives") + + archives_path = tmp_path / "archives" + archives_path.mkdir() + + if (archives_path_link := test_resources_path / "archives").is_symlink(): + archives_path_link.unlink() + + archives_path_link.symlink_to(archives_path, target_is_directory=True) + + for item in ["applications", "systems"]: + process_directory(test_resources_path / "backends" / item, archives_path / item) + + yield + + archives_path_link.unlink() + shutil.rmtree(tmp_path) diff --git a/tests/aiet/test_backend_application.py b/tests/mlia/test_backend_application.py index abfab00..2cfb2ef 100644 --- a/tests/aiet/test_backend_application.py +++ b/tests/mlia/test_backend_application.py @@ -11,28 +11,34 @@ from unittest.mock import MagicMock import pytest -from aiet.backend.application import Application -from aiet.backend.application import get_application -from aiet.backend.application import get_available_application_directory_names -from aiet.backend.application import get_available_applications -from aiet.backend.application import get_unique_application_names -from aiet.backend.application import install_application -from aiet.backend.application import load_applications -from aiet.backend.application import remove_application -from aiet.backend.common import Command -from aiet.backend.common import DataPaths -from aiet.backend.common import Param -from aiet.backend.common import UserParamConfig -from aiet.backend.config import ApplicationConfig -from aiet.backend.config import ExtendedApplicationConfig -from aiet.backend.config import NamedExecutionConfig +from mlia.backend.application import Application +from mlia.backend.application import get_application +from mlia.backend.application import get_available_application_directory_names +from mlia.backend.application import get_available_applications +from mlia.backend.application import get_unique_application_names +from mlia.backend.application import install_application +from mlia.backend.application import load_applications +from mlia.backend.application import remove_application +from mlia.backend.common import Command +from mlia.backend.common import DataPaths +from mlia.backend.common import Param +from mlia.backend.common import UserParamConfig +from mlia.backend.config import ApplicationConfig +from mlia.backend.config import ExtendedApplicationConfig +from mlia.backend.config import NamedExecutionConfig def test_get_available_application_directory_names() -> None: """Test get_available_applicationss mocking get_resources.""" directory_names = get_available_application_directory_names() assert Counter(directory_names) == Counter( - ["application1", "application2", "application4", "application5"] + [ + "application1", + "application2", + "application4", + "application5", + "application6", + ] ) @@ -42,7 +48,7 @@ def test_get_available_applications() -> None: assert all(isinstance(s, Application) for s in available_applications) assert all(s != 42 for s in available_applications) - assert len(available_applications) == 9 + assert len(available_applications) == 10 # application_5 has multiply items with multiply supported systems assert [str(s) for s in available_applications] == [ "application_1", @@ -54,6 +60,7 @@ def test_get_available_applications() -> None: "application_5A", "application_5B", "application_5B", + "application_6", ] @@ -70,6 +77,7 @@ def test_get_unique_application_names() -> None: "application_5", "application_5A", "application_5B", + "application_6", ] @@ -121,14 +129,14 @@ def test_get_application() -> None: pytest.raises(Exception, match="Unable to read application definition"), ), ( - "applications/application1", + "backends/applications/application1", 0, pytest.raises( Exception, match=r"Applications \[application_1\] are already installed" ), ), ( - "applications/application3", + "backends/applications/application3", 0, pytest.raises(Exception, match="Unable to read application definition"), ), @@ -144,7 +152,7 @@ def test_install_application( """Test application install from archive.""" mock_create_destination_and_install = MagicMock() monkeypatch.setattr( - "aiet.backend.application.create_destination_and_install", + "mlia.backend.application.create_destination_and_install", mock_create_destination_and_install, ) @@ -156,7 +164,7 @@ def test_install_application( def test_remove_application(monkeypatch: Any) -> None: """Test application removal.""" mock_remove_backend = MagicMock() - monkeypatch.setattr("aiet.backend.application.remove_backend", mock_remove_backend) + monkeypatch.setattr("mlia.backend.application.remove_backend", mock_remove_backend) remove_application("some_application_directory") mock_remove_backend.assert_called_once() diff --git a/tests/aiet/test_backend_common.py b/tests/mlia/test_backend_common.py index 12c30ec..82a985a 100644 --- a/tests/aiet/test_backend_common.py +++ b/tests/mlia/test_backend_common.py @@ -16,20 +16,20 @@ from unittest.mock import MagicMock import pytest -from aiet.backend.application import Application -from aiet.backend.common import Backend -from aiet.backend.common import BaseBackendConfig -from aiet.backend.common import Command -from aiet.backend.common import ConfigurationException -from aiet.backend.common import load_config -from aiet.backend.common import Param -from aiet.backend.common import parse_raw_parameter -from aiet.backend.common import remove_backend -from aiet.backend.config import ApplicationConfig -from aiet.backend.config import UserParamConfig -from aiet.backend.execution import ExecutionContext -from aiet.backend.execution import ParamResolver -from aiet.backend.system import System +from mlia.backend.application import Application +from mlia.backend.common import Backend +from mlia.backend.common import BaseBackendConfig +from mlia.backend.common import Command +from mlia.backend.common import ConfigurationException +from mlia.backend.common import load_config +from mlia.backend.common import Param +from mlia.backend.common import parse_raw_parameter +from mlia.backend.common import remove_backend +from mlia.backend.config import ApplicationConfig +from mlia.backend.config import UserParamConfig +from mlia.backend.execution import ExecutionContext +from mlia.backend.execution import ParamResolver +from mlia.backend.system import System @pytest.mark.parametrize( @@ -44,7 +44,7 @@ def test_remove_backend( ) -> None: """Test remove_backend function.""" mock_remove_resource = MagicMock() - monkeypatch.setattr("aiet.backend.common.remove_resource", mock_remove_resource) + monkeypatch.setattr("mlia.backend.common.remove_resource", mock_remove_resource) with expected_exception: remove_backend(directory_name, "applications") @@ -75,7 +75,7 @@ def test_load_config( ) for config in configs: json_mock = MagicMock() - monkeypatch.setattr("aiet.backend.common.json.load", json_mock) + monkeypatch.setattr("mlia.backend.common.json.load", json_mock) load_config(config) json_mock.assert_called_once() @@ -175,7 +175,7 @@ class TestBackend: "variables": {"var_A": "value for variable A"}, } - monkeypatch.setattr("aiet.backend.system.ProtocolFactory", MagicMock()) + monkeypatch.setattr("mlia.backend.system.ProtocolFactory", MagicMock()) application, system = Application(config), System(config) # type: ignore context = ExecutionContext( app=application, @@ -290,7 +290,7 @@ class TestBackend: expected_output: List[Tuple[Optional[str], Param]], ) -> None: """Test command building.""" - monkeypatch.setattr("aiet.backend.system.ProtocolFactory", MagicMock()) + monkeypatch.setattr("mlia.backend.system.ProtocolFactory", MagicMock()) backend = class_(config) params = backend.resolved_parameters( diff --git a/tests/aiet/test_backend_controller.py b/tests/mlia/test_backend_controller.py index 8836ec5..a047adf 100644 --- a/tests/aiet/test_backend_controller.py +++ b/tests/mlia/test_backend_controller.py @@ -10,10 +10,10 @@ from typing import Any import psutil import pytest -from aiet.backend.common import ConfigurationException -from aiet.backend.controller import SystemController -from aiet.backend.controller import SystemControllerSingleInstance -from aiet.utils.proc import ShellCommand +from mlia.backend.common import ConfigurationException +from mlia.backend.controller import SystemController +from mlia.backend.controller import SystemControllerSingleInstance +from mlia.backend.proc import ShellCommand def get_system_controller(**kwargs: Any) -> SystemController: diff --git a/tests/aiet/test_backend_execution.py b/tests/mlia/test_backend_execution.py index 8aa45f1..9395352 100644 --- a/tests/aiet/test_backend_execution.py +++ b/tests/mlia/test_backend_execution.py @@ -1,41 +1,39 @@ # SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 # pylint: disable=no-self-use -"""Test backend context module.""" +"""Test backend execution module.""" from contextlib import ExitStack as does_not_raise from pathlib import Path from typing import Any from typing import Dict -from typing import Optional from unittest import mock from unittest.mock import MagicMock import pytest from sh import CommandNotFound -from aiet.backend.application import Application -from aiet.backend.application import get_application -from aiet.backend.common import ConfigurationException -from aiet.backend.common import DataPaths -from aiet.backend.common import UserParamConfig -from aiet.backend.config import ApplicationConfig -from aiet.backend.config import LocalProtocolConfig -from aiet.backend.config import SystemConfig -from aiet.backend.execution import deploy_data -from aiet.backend.execution import execute_commands_locally -from aiet.backend.execution import ExecutionContext -from aiet.backend.execution import get_application_and_system -from aiet.backend.execution import get_application_by_name_and_system -from aiet.backend.execution import get_file_lock_path -from aiet.backend.execution import get_tool_by_system -from aiet.backend.execution import ParamResolver -from aiet.backend.execution import Reporter -from aiet.backend.execution import wait -from aiet.backend.output_parser import OutputParser -from aiet.backend.system import get_system -from aiet.backend.system import load_system -from aiet.backend.tool import get_tool -from aiet.utils.proc import CommandFailedException +from mlia.backend.application import Application +from mlia.backend.application import get_application +from mlia.backend.common import DataPaths +from mlia.backend.common import UserParamConfig +from mlia.backend.config import ApplicationConfig +from mlia.backend.config import LocalProtocolConfig +from mlia.backend.config import SystemConfig +from mlia.backend.execution import deploy_data +from mlia.backend.execution import execute_commands_locally +from mlia.backend.execution import ExecutionContext +from mlia.backend.execution import get_application_and_system +from mlia.backend.execution import get_application_by_name_and_system +from mlia.backend.execution import get_file_lock_path +from mlia.backend.execution import ParamResolver +from mlia.backend.execution import Reporter +from mlia.backend.execution import wait +from mlia.backend.output_parser import Base64OutputParser +from mlia.backend.output_parser import OutputParser +from mlia.backend.output_parser import RegexOutputParser +from mlia.backend.proc import CommandFailedException +from mlia.backend.system import get_system +from mlia.backend.system import load_system def test_context_param_resolver(tmpdir: Any) -> None: @@ -319,7 +317,7 @@ def test_get_file_lock_path( def test_get_application_by_name_and_system(monkeypatch: Any) -> None: """Test exceptional case for get_application_by_name_and_system.""" monkeypatch.setattr( - "aiet.backend.execution.get_application", + "mlia.backend.execution.get_application", MagicMock(return_value=[MagicMock(), MagicMock()]), ) @@ -334,7 +332,7 @@ def test_get_application_by_name_and_system(monkeypatch: Any) -> None: def test_get_application_and_system(monkeypatch: Any) -> None: """Test exceptional case for get_application_and_system.""" monkeypatch.setattr( - "aiet.backend.execution.get_system", MagicMock(return_value=None) + "mlia.backend.execution.get_system", MagicMock(return_value=None) ) with pytest.raises(ValueError, match="System test_system is not found"): @@ -379,40 +377,32 @@ def test_deployment_execution_context() -> None: with pytest.raises(AssertionError): deploy_data(ctx) + +def test_reporter_execution_context(tmp_path: Path) -> None: + """Test ExecutionContext creates a reporter when a report file is provided.""" + # Configure regex parser for the system manually + sys = get_system("System 1") + assert sys is not None + sys.reporting = { + "regex": { + "simulated_time": {"pattern": "Simulated time.*: (.*)s", "type": "float"} + } + } + report_file_path = tmp_path / "test_report.json" + ctx = ExecutionContext( - app=get_tool("tool_1")[0], + app=get_application("application_1")[0], app_params=[], - system=None, + system=sys, system_params=[], + report_file=report_file_path, + ) + assert isinstance(ctx.reporter, Reporter) + assert len(ctx.reporter.parsers) == 2 + assert any(isinstance(parser, RegexOutputParser) for parser in ctx.reporter.parsers) + assert any( + isinstance(parser, Base64OutputParser) for parser in ctx.reporter.parsers ) - assert not ctx.is_deploy_needed - deploy_data(ctx) # should be a NOP - - -@pytest.mark.parametrize( - ["tool_name", "system_name", "exception"], - [ - ("vela", "Corstone-300: Cortex-M55+Ethos-U65", None), - ("unknown tool", "Corstone-300: Cortex-M55+Ethos-U65", ConfigurationException), - ("vela", "unknown system", ConfigurationException), - ("vela", None, ConfigurationException), - ], -) -def test_get_tool_by_system( - tool_name: str, system_name: Optional[str], exception: Optional[Any] -) -> None: - """Test exceptions thrown by function get_tool_by_system().""" - - def test() -> None: - """Test call of get_tool_by_system().""" - tool = get_tool_by_system(tool_name, system_name) - assert tool is not None - - if exception is None: - test() - else: - with pytest.raises(exception): - test() class TestExecuteCommandsLocally: @@ -465,7 +455,7 @@ class TestExecuteCommandsLocally: """Ensure commands following an error-exit-code command don't run.""" # Mock execute_command() function execute_command_mock = mock.MagicMock() - monkeypatch.setattr("aiet.utils.proc.execute_command", execute_command_mock) + monkeypatch.setattr("mlia.backend.proc.execute_command", execute_command_mock) # Mock Command object and assign as return value to execute_command() cmd_mock = mock.MagicMock() @@ -473,7 +463,9 @@ class TestExecuteCommandsLocally: # Mock the terminate_command (speed up test) terminate_command_mock = mock.MagicMock() - monkeypatch.setattr("aiet.utils.proc.terminate_command", terminate_command_mock) + monkeypatch.setattr( + "mlia.backend.proc.terminate_command", terminate_command_mock + ) # Mock a thrown Exception and assign to Command().exit_code exit_code_mock = mock.PropertyMock(side_effect=Exception("Exception.")) diff --git a/tests/aiet/test_utils_fs.py b/tests/mlia/test_backend_fs.py index 46d276e..ff9c2ae 100644 --- a/tests/aiet/test_utils_fs.py +++ b/tests/mlia/test_backend_fs.py @@ -10,14 +10,14 @@ from unittest.mock import MagicMock import pytest -from aiet.utils.fs import get_resources -from aiet.utils.fs import read_file_as_bytearray -from aiet.utils.fs import read_file_as_string -from aiet.utils.fs import recreate_directory -from aiet.utils.fs import remove_directory -from aiet.utils.fs import remove_resource -from aiet.utils.fs import ResourceType -from aiet.utils.fs import valid_for_filename +from mlia.backend.fs import get_backends_path +from mlia.backend.fs import read_file_as_bytearray +from mlia.backend.fs import read_file_as_string +from mlia.backend.fs import recreate_directory +from mlia.backend.fs import remove_directory +from mlia.backend.fs import remove_resource +from mlia.backend.fs import ResourceType +from mlia.backend.fs import valid_for_filename @pytest.mark.parametrize( @@ -29,10 +29,10 @@ from aiet.utils.fs import valid_for_filename (None, pytest.raises(ResourceWarning)), ], ) -def test_get_resources(resource_name: ResourceType, expected_path: Any) -> None: +def test_get_backends_path(resource_name: ResourceType, expected_path: Any) -> None: """Test get_resources() with multiple parameters.""" with expected_path: - resource_path = get_resources(resource_name) + resource_path = get_backends_path(resource_name) assert resource_path.exists() @@ -41,10 +41,10 @@ def test_remove_resource_wrong_directory( ) -> None: """Test removing resource with wrong directory.""" mock_get_resources = MagicMock(return_value=test_applications_path) - monkeypatch.setattr("aiet.utils.fs.get_resources", mock_get_resources) + monkeypatch.setattr("mlia.backend.fs.get_backends_path", mock_get_resources) mock_shutil_rmtree = MagicMock() - monkeypatch.setattr("aiet.utils.fs.shutil.rmtree", mock_shutil_rmtree) + monkeypatch.setattr("mlia.backend.fs.shutil.rmtree", mock_shutil_rmtree) with pytest.raises(Exception, match="Resource .* does not exist"): remove_resource("unknown", "applications") @@ -58,10 +58,10 @@ def test_remove_resource_wrong_directory( def test_remove_resource(monkeypatch: Any, test_applications_path: Path) -> None: """Test removing resource data.""" mock_get_resources = MagicMock(return_value=test_applications_path) - monkeypatch.setattr("aiet.utils.fs.get_resources", mock_get_resources) + monkeypatch.setattr("mlia.backend.fs.get_backends_path", mock_get_resources) mock_shutil_rmtree = MagicMock() - monkeypatch.setattr("aiet.utils.fs.shutil.rmtree", mock_shutil_rmtree) + monkeypatch.setattr("mlia.backend.fs.shutil.rmtree", mock_shutil_rmtree) remove_resource("application1", "applications") mock_shutil_rmtree.assert_called_once() diff --git a/tests/mlia/test_tools_aiet_wrapper.py b/tests/mlia/test_backend_manager.py index ab55b71..c81366f 100644 --- a/tests/mlia/test_tools_aiet_wrapper.py +++ b/tests/mlia/test_backend_manager.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 -"""Tests for module tools/aiet_wrapper.""" +"""Tests for module backend/manager.""" +import os from contextlib import ExitStack as does_not_raise from pathlib import Path from typing import Any @@ -13,20 +14,23 @@ from unittest.mock import PropertyMock import pytest -from mlia.tools.aiet_wrapper import AIETRunner -from mlia.tools.aiet_wrapper import DeviceInfo -from mlia.tools.aiet_wrapper import estimate_performance -from mlia.tools.aiet_wrapper import ExecutionParams -from mlia.tools.aiet_wrapper import GenericInferenceOutputParser -from mlia.tools.aiet_wrapper import GenericInferenceRunnerEthosU -from mlia.tools.aiet_wrapper import get_aiet_runner -from mlia.tools.aiet_wrapper import get_generic_runner -from mlia.tools.aiet_wrapper import get_system_name -from mlia.tools.aiet_wrapper import is_supported -from mlia.tools.aiet_wrapper import ModelInfo -from mlia.tools.aiet_wrapper import PerformanceMetrics -from mlia.tools.aiet_wrapper import supported_backends -from mlia.utils.proc import RunningCommand +from mlia.backend.application import get_application +from mlia.backend.common import DataPaths +from mlia.backend.execution import ExecutionContext +from mlia.backend.execution import run_application +from mlia.backend.manager import BackendRunner +from mlia.backend.manager import DeviceInfo +from mlia.backend.manager import estimate_performance +from mlia.backend.manager import ExecutionParams +from mlia.backend.manager import GenericInferenceOutputParser +from mlia.backend.manager import GenericInferenceRunnerEthosU +from mlia.backend.manager import get_generic_runner +from mlia.backend.manager import get_system_name +from mlia.backend.manager import is_supported +from mlia.backend.manager import ModelInfo +from mlia.backend.manager import PerformanceMetrics +from mlia.backend.manager import supported_backends +from mlia.backend.system import get_system @pytest.mark.parametrize( @@ -108,16 +112,16 @@ def test_generic_inference_output_parser( assert parser.missed_keys() == missed_keys -class TestAIETRunner: - """Tests for AIETRunner class.""" +class TestBackendRunner: + """Tests for BackendRunner class.""" @staticmethod - def _setup_aiet( + def _setup_backends( monkeypatch: pytest.MonkeyPatch, available_systems: Optional[List[str]] = None, available_apps: Optional[List[str]] = None, ) -> None: - """Set up AIET metadata.""" + """Set up backend metadata.""" def mock_system(system: str) -> MagicMock: """Mock the System instance.""" @@ -134,13 +138,13 @@ class TestAIETRunner: system_mocks = [mock_system(name) for name in (available_systems or [])] monkeypatch.setattr( - "mlia.tools.aiet_wrapper.get_available_systems", + "mlia.backend.manager.get_available_systems", MagicMock(return_value=system_mocks), ) apps_mock = [mock_app(name) for name in (available_apps or [])] monkeypatch.setattr( - "mlia.tools.aiet_wrapper.get_available_applications", + "mlia.backend.manager.get_available_applications", MagicMock(return_value=apps_mock), ) @@ -159,13 +163,11 @@ class TestAIETRunner: monkeypatch: pytest.MonkeyPatch, ) -> None: """Test method is_system_installed.""" - mock_executor = MagicMock() - aiet_runner = AIETRunner(mock_executor) + backend_runner = BackendRunner() - self._setup_aiet(monkeypatch, available_systems) + self._setup_backends(monkeypatch, available_systems) - assert aiet_runner.is_system_installed(system) == installed - mock_executor.assert_not_called() + assert backend_runner.is_system_installed(system) == installed @pytest.mark.parametrize( "available_systems, systems", @@ -181,28 +183,21 @@ class TestAIETRunner: monkeypatch: pytest.MonkeyPatch, ) -> None: """Test method installed_systems.""" - mock_executor = MagicMock() - aiet_runner = AIETRunner(mock_executor) + backend_runner = BackendRunner() - self._setup_aiet(monkeypatch, available_systems) - assert aiet_runner.get_installed_systems() == systems - - mock_executor.assert_not_called() + self._setup_backends(monkeypatch, available_systems) + assert backend_runner.get_installed_systems() == systems @staticmethod def test_install_system(monkeypatch: pytest.MonkeyPatch) -> None: """Test system installation.""" install_system_mock = MagicMock() - monkeypatch.setattr( - "mlia.tools.aiet_wrapper.install_system", install_system_mock - ) + monkeypatch.setattr("mlia.backend.manager.install_system", install_system_mock) - mock_executor = MagicMock() - aiet_runner = AIETRunner(mock_executor) - aiet_runner.install_system(Path("test_system_path")) + backend_runner = BackendRunner() + backend_runner.install_system(Path("test_system_path")) install_system_mock.assert_called_once_with(Path("test_system_path")) - mock_executor.assert_not_called() @pytest.mark.parametrize( "available_systems, systems, expected_result", @@ -222,14 +217,11 @@ class TestAIETRunner: monkeypatch: pytest.MonkeyPatch, ) -> None: """Test method systems_installed.""" - self._setup_aiet(monkeypatch, available_systems) + self._setup_backends(monkeypatch, available_systems) - mock_executor = MagicMock() - aiet_runner = AIETRunner(mock_executor) + backend_runner = BackendRunner() - assert aiet_runner.systems_installed(systems) is expected_result - - mock_executor.assert_not_called() + assert backend_runner.systems_installed(systems) is expected_result @pytest.mark.parametrize( "available_apps, applications, expected_result", @@ -249,12 +241,10 @@ class TestAIETRunner: monkeypatch: pytest.MonkeyPatch, ) -> None: """Test method applications_installed.""" - self._setup_aiet(monkeypatch, [], available_apps) - mock_executor = MagicMock() - aiet_runner = AIETRunner(mock_executor) + self._setup_backends(monkeypatch, [], available_apps) + backend_runner = BackendRunner() - assert aiet_runner.applications_installed(applications) is expected_result - mock_executor.assert_not_called() + assert backend_runner.applications_installed(applications) is expected_result @pytest.mark.parametrize( "available_apps, applications", @@ -273,30 +263,23 @@ class TestAIETRunner: monkeypatch: pytest.MonkeyPatch, ) -> None: """Test method get_installed_applications.""" - mock_executor = MagicMock() - self._setup_aiet(monkeypatch, [], available_apps) - - aiet_runner = AIETRunner(mock_executor) - assert applications == aiet_runner.get_installed_applications() + self._setup_backends(monkeypatch, [], available_apps) - mock_executor.assert_not_called() + backend_runner = BackendRunner() + assert applications == backend_runner.get_installed_applications() @staticmethod def test_install_application(monkeypatch: pytest.MonkeyPatch) -> None: """Test application installation.""" mock_install_application = MagicMock() monkeypatch.setattr( - "mlia.tools.aiet_wrapper.install_application", mock_install_application + "mlia.backend.manager.install_application", mock_install_application ) - mock_executor = MagicMock() - - aiet_runner = AIETRunner(mock_executor) - aiet_runner.install_application(Path("test_application_path")) + backend_runner = BackendRunner() + backend_runner.install_application(Path("test_application_path")) mock_install_application.assert_called_once_with(Path("test_application_path")) - mock_executor.assert_not_called() - @pytest.mark.parametrize( "available_apps, application, installed", [ @@ -321,66 +304,113 @@ class TestAIETRunner: monkeypatch: pytest.MonkeyPatch, ) -> None: """Test method is_application_installed.""" - self._setup_aiet(monkeypatch, [], available_apps) - - mock_executor = MagicMock() - aiet_runner = AIETRunner(mock_executor) - assert installed == aiet_runner.is_application_installed(application, "system1") + self._setup_backends(monkeypatch, [], available_apps) - mock_executor.assert_not_called() + backend_runner = BackendRunner() + assert installed == backend_runner.is_application_installed( + application, "system1" + ) @staticmethod @pytest.mark.parametrize( "execution_params, expected_command", [ ( - ExecutionParams("application1", "system1", [], [], []), - ["aiet", "application", "run", "-n", "application1", "-s", "system1"], + ExecutionParams("application_4", "System 4", [], [], []), + ["application_4", [], "System 4", [], []], ), ( ExecutionParams( - "application1", - "system1", - ["input_file=123.txt", "size=777"], - ["param1=456", "param2=789"], + "application_6", + "System 6", + ["param1=value2"], + ["sys-param1=value2"], + [], + ), + [ + "application_6", + ["param1=value2"], + "System 6", + ["sys-param1=value2"], + [], + ], + ), + ], + ) + def test_run_application_local( + monkeypatch: pytest.MonkeyPatch, + execution_params: ExecutionParams, + expected_command: List[str], + ) -> None: + """Test method run_application with local systems.""" + run_app = MagicMock(wraps=run_application) + monkeypatch.setattr("mlia.backend.manager.run_application", run_app) + + backend_runner = BackendRunner() + backend_runner.run_application(execution_params) + + run_app.assert_called_once_with(*expected_command) + + @staticmethod + @pytest.mark.parametrize( + "execution_params, expected_command", + [ + ( + ExecutionParams( + "application_1", + "System 1", + [], + [], ["source1.txt:dest1.txt", "source2.txt:dest2.txt"], ), [ - "aiet", - "application", - "run", - "-n", - "application1", - "-s", - "system1", - "-p", - "input_file=123.txt", - "-p", - "size=777", - "--system-param", - "param1=456", - "--system-param", - "param2=789", - "--deploy", - "source1.txt:dest1.txt", - "--deploy", - "source2.txt:dest2.txt", + "application_1", + [], + "System 1", + [], + [ + DataPaths(Path("source1.txt"), "dest1.txt"), + DataPaths(Path("source2.txt"), "dest2.txt"), + ], ], ), ], ) - def test_run_application( - execution_params: ExecutionParams, expected_command: List[str] + def test_run_application_connected( + monkeypatch: pytest.MonkeyPatch, + execution_params: ExecutionParams, + expected_command: List[str], ) -> None: - """Test method run_application.""" - mock_executor = MagicMock() - mock_running_command = MagicMock() - mock_executor.submit.return_value = mock_running_command + """Test method run_application with connectable systems (SSH).""" + app = get_application(execution_params.application, execution_params.system)[0] + sys = get_system(execution_params.system) + + assert sys is not None + + connect_mock = MagicMock(return_value=True, name="connect_mock") + deploy_mock = MagicMock(return_value=True, name="deploy_mock") + run_mock = MagicMock( + return_value=(os.EX_OK, bytearray(), bytearray()), name="run_mock" + ) + sys.establish_connection = connect_mock # type: ignore + sys.deploy = deploy_mock # type: ignore + sys.run = run_mock # type: ignore + + monkeypatch.setattr( + "mlia.backend.execution.get_application_and_system", + MagicMock(return_value=(app, sys)), + ) - aiet_runner = AIETRunner(mock_executor) - aiet_runner.run_application(execution_params) + run_app_mock = MagicMock(wraps=run_application) + monkeypatch.setattr("mlia.backend.manager.run_application", run_app_mock) - mock_executor.submit.assert_called_once_with(expected_command) + backend_runner = BackendRunner() + backend_runner.run_application(execution_params) + + run_app_mock.assert_called_once_with(*expected_command) + + connect_mock.assert_called_once() + assert deploy_mock.call_count == 2 @pytest.mark.parametrize( @@ -490,16 +520,16 @@ def test_estimate_performance( backend: str, expected_error: Any, test_tflite_model: Path, - aiet_runner: MagicMock, + backend_runner: MagicMock, ) -> None: """Test getting performance estimations.""" system_name, system_installed = system application_name, application_installed = application - aiet_runner.is_system_installed.return_value = system_installed - aiet_runner.is_application_installed.return_value = application_installed + backend_runner.is_system_installed.return_value = system_installed + backend_runner.is_application_installed.return_value = application_installed - mock_process = create_mock_process( + mock_context = create_mock_context( [ "NPU AXI0_RD_DATA_BEAT_RECEIVED beats: 1", "NPU AXI0_WR_DATA_BEAT_WRITTEN beats: 2", @@ -507,12 +537,10 @@ def test_estimate_performance( "NPU ACTIVE cycles: 4", "NPU IDLE cycles: 5", "NPU TOTAL cycles: 6", - ], - [], + ] ) - mock_generic_inference_run = RunningCommand(mock_process) - aiet_runner.run_application.return_value = mock_generic_inference_run + backend_runner.run_application.return_value = mock_context with expected_error: perf_metrics = estimate_performance( @@ -529,19 +557,19 @@ def test_estimate_performance( npu_total_cycles=6, ) - assert aiet_runner.is_system_installed.called_once_with(system_name) - assert aiet_runner.is_application_installed.called_once_with( + assert backend_runner.is_system_installed.called_once_with(system_name) + assert backend_runner.is_application_installed.called_once_with( application_name, system_name ) @pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) def test_estimate_performance_insufficient_data( - aiet_runner: MagicMock, test_tflite_model: Path, backend: str + backend_runner: MagicMock, test_tflite_model: Path, backend: str ) -> None: """Test that performance could not be estimated when not all data presented.""" - aiet_runner.is_system_installed.return_value = True - aiet_runner.is_application_installed.return_value = True + backend_runner.is_system_installed.return_value = True + backend_runner.is_application_installed.return_value = True no_total_cycles_output = [ "NPU AXI0_RD_DATA_BEAT_RECEIVED beats: 1", @@ -550,13 +578,9 @@ def test_estimate_performance_insufficient_data( "NPU ACTIVE cycles: 4", "NPU IDLE cycles: 5", ] - mock_process = create_mock_process( - no_total_cycles_output, - [], - ) + mock_context = create_mock_context(no_total_cycles_output) - mock_generic_inference_run = RunningCommand(mock_process) - aiet_runner.run_application.return_value = mock_generic_inference_run + backend_runner.run_application.return_value = mock_context with pytest.raises( Exception, match="Unable to get performance metrics, insufficient data" @@ -567,16 +591,14 @@ def test_estimate_performance_insufficient_data( @pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) def test_estimate_performance_invalid_output( - test_tflite_model: Path, aiet_runner: MagicMock, backend: str + test_tflite_model: Path, backend_runner: MagicMock, backend: str ) -> None: """Test estimation could not be done if inference produces unexpected output.""" - aiet_runner.is_system_installed.return_value = True - aiet_runner.is_application_installed.return_value = True + backend_runner.is_system_installed.return_value = True + backend_runner.is_application_installed.return_value = True - mock_process = create_mock_process( - ["Something", "is", "wrong"], ["What a nice error!"] - ) - aiet_runner.run_application.return_value = RunningCommand(mock_process) + mock_context = create_mock_context(["Something", "is", "wrong"]) + backend_runner.run_application.return_value = mock_context with pytest.raises(Exception, match="Unable to get performance metrics"): estimate_performance( @@ -586,12 +608,6 @@ def test_estimate_performance_invalid_output( ) -def test_get_aiet_runner() -> None: - """Test getting aiet runner.""" - aiet_runner = get_aiet_runner() - assert isinstance(aiet_runner, AIETRunner) - - def create_mock_process(stdout: List[str], stderr: List[str]) -> MagicMock: """Mock underlying process.""" mock_process = MagicMock() @@ -601,6 +617,18 @@ def create_mock_process(stdout: List[str], stderr: List[str]) -> MagicMock: return mock_process +def create_mock_context(stdout: List[str]) -> ExecutionContext: + """Mock ExecutionContext.""" + ctx = ExecutionContext( + app=get_application("application_1")[0], + app_params=[], + system=get_system("System 1"), + system_params=[], + ) + ctx.stdout = bytearray("\n".join(stdout).encode("utf-8")) + return ctx + + @pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) def test_get_generic_runner(backend: str) -> None: """Test function get_generic_runner().""" @@ -621,8 +649,8 @@ def test_get_generic_runner(backend: str) -> None: ("Corstone-310", "ethos-u55"), ), ) -def test_aiet_backend_support(backend: str, device_type: str) -> None: - """Test AIET backend & device support.""" +def test_backend_support(backend: str, device_type: str) -> None: + """Test backend & device support.""" assert is_supported(backend) assert is_supported(backend, device_type) @@ -714,10 +742,10 @@ class TestGenericInferenceRunnerEthosU: @staticmethod @pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) def test_inference_should_fail_if_system_not_installed( - aiet_runner: MagicMock, test_tflite_model: Path, backend: str + backend_runner: MagicMock, test_tflite_model: Path, backend: str ) -> None: """Test that inference should fail if system is not installed.""" - aiet_runner.is_system_installed.return_value = False + backend_runner.is_system_installed.return_value = False generic_runner = get_generic_runner( DeviceInfo("ethos-u55", 256, memory_mode="Shared_Sram"), backend @@ -731,11 +759,11 @@ class TestGenericInferenceRunnerEthosU: @staticmethod @pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310")) def test_inference_should_fail_is_apps_not_installed( - aiet_runner: MagicMock, test_tflite_model: Path, backend: str + backend_runner: MagicMock, test_tflite_model: Path, backend: str ) -> None: """Test that inference should fail if apps are not installed.""" - aiet_runner.is_system_installed.return_value = True - aiet_runner.is_application_installed.return_value = False + backend_runner.is_system_installed.return_value = True + backend_runner.is_application_installed.return_value = False generic_runner = get_generic_runner( DeviceInfo("ethos-u55", 256, memory_mode="Shared_Sram"), backend @@ -749,12 +777,12 @@ class TestGenericInferenceRunnerEthosU: generic_runner.run(ModelInfo(test_tflite_model), []) -@pytest.fixture(name="aiet_runner") -def fixture_aiet_runner(monkeypatch: pytest.MonkeyPatch) -> MagicMock: - """Mock AIET runner.""" - aiet_runner_mock = MagicMock(spec=AIETRunner) +@pytest.fixture(name="backend_runner") +def fixture_backend_runner(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + """Mock backend runner.""" + backend_runner_mock = MagicMock(spec=BackendRunner) monkeypatch.setattr( - "mlia.tools.aiet_wrapper.get_aiet_runner", - MagicMock(return_value=aiet_runner_mock), + "mlia.backend.manager.get_backend_runner", + MagicMock(return_value=backend_runner_mock), ) - return aiet_runner_mock + return backend_runner_mock diff --git a/tests/aiet/test_backend_output_parser.py b/tests/mlia/test_backend_output_parser.py index d659812..d86aac8 100644 --- a/tests/aiet/test_backend_output_parser.py +++ b/tests/mlia/test_backend_output_parser.py @@ -8,9 +8,9 @@ from typing import Dict import pytest -from aiet.backend.output_parser import Base64OutputParser -from aiet.backend.output_parser import OutputParser -from aiet.backend.output_parser import RegexOutputParser +from mlia.backend.output_parser import Base64OutputParser +from mlia.backend.output_parser import OutputParser +from mlia.backend.output_parser import RegexOutputParser OUTPUT_MATCH_ALL = bytearray( diff --git a/tests/aiet/test_utils_proc.py b/tests/mlia/test_backend_proc.py index 9fb48dd..9ca4788 100644 --- a/tests/aiet/test_utils_proc.py +++ b/tests/mlia/test_backend_proc.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 # pylint: disable=attribute-defined-outside-init,no-self-use,not-callable -"""Pytests for testing aiet/utils/proc.py.""" +"""Pytests for testing mlia/backend/proc.py.""" from pathlib import Path from typing import Any from unittest import mock @@ -10,16 +10,16 @@ import psutil import pytest from sh import ErrorReturnCode -from aiet.utils.proc import Command -from aiet.utils.proc import CommandFailedException -from aiet.utils.proc import CommandNotFound -from aiet.utils.proc import parse_command -from aiet.utils.proc import print_command_stdout -from aiet.utils.proc import run_and_wait -from aiet.utils.proc import save_process_info -from aiet.utils.proc import ShellCommand -from aiet.utils.proc import terminate_command -from aiet.utils.proc import terminate_external_process +from mlia.backend.proc import Command +from mlia.backend.proc import CommandFailedException +from mlia.backend.proc import CommandNotFound +from mlia.backend.proc import parse_command +from mlia.backend.proc import print_command_stdout +from mlia.backend.proc import run_and_wait +from mlia.backend.proc import save_process_info +from mlia.backend.proc import ShellCommand +from mlia.backend.proc import terminate_command +from mlia.backend.proc import terminate_external_process class TestShellCommand: @@ -211,12 +211,12 @@ class TestRunAndWait: """Init test method.""" self.execute_command_mock = mock.MagicMock() monkeypatch.setattr( - "aiet.utils.proc.execute_command", self.execute_command_mock + "mlia.backend.proc.execute_command", self.execute_command_mock ) self.terminate_command_mock = mock.MagicMock() monkeypatch.setattr( - "aiet.utils.proc.terminate_command", self.terminate_command_mock + "mlia.backend.proc.terminate_command", self.terminate_command_mock ) def test_if_execute_command_raises_exception(self) -> None: diff --git a/tests/aiet/test_backend_protocol.py b/tests/mlia/test_backend_protocol.py index 2103238..35e9986 100644 --- a/tests/aiet/test_backend_protocol.py +++ b/tests/mlia/test_backend_protocol.py @@ -10,12 +10,12 @@ from unittest.mock import MagicMock import paramiko import pytest -from aiet.backend.common import ConfigurationException -from aiet.backend.config import LocalProtocolConfig -from aiet.backend.protocol import CustomSFTPClient -from aiet.backend.protocol import LocalProtocol -from aiet.backend.protocol import ProtocolFactory -from aiet.backend.protocol import SSHProtocol +from mlia.backend.common import ConfigurationException +from mlia.backend.config import LocalProtocolConfig +from mlia.backend.protocol import CustomSFTPClient +from mlia.backend.protocol import LocalProtocol +from mlia.backend.protocol import ProtocolFactory +from mlia.backend.protocol import SSHProtocol class TestProtocolFactory: @@ -94,13 +94,13 @@ class TestSSHProtocol: self.mock_ssh_channel.recv_stderr_ready.side_effect = [False, True] monkeypatch.setattr( - "aiet.backend.protocol.paramiko.client.SSHClient", + "mlia.backend.protocol.paramiko.client.SSHClient", MagicMock(return_value=self.mock_ssh_client), ) self.mock_sftp_client = MagicMock(spec=CustomSFTPClient) monkeypatch.setattr( - "aiet.backend.protocol.CustomSFTPClient.from_transport", + "mlia.backend.protocol.CustomSFTPClient.from_transport", MagicMock(return_value=self.mock_sftp_client), ) @@ -116,7 +116,7 @@ class TestSSHProtocol: def test_unable_create_ssh_client(self, monkeypatch: Any) -> None: """Test that command should fail if unable to create ssh client instance.""" monkeypatch.setattr( - "aiet.backend.protocol.paramiko.client.SSHClient", + "mlia.backend.protocol.paramiko.client.SSHClient", MagicMock(side_effect=OSError("Error!")), ) @@ -201,14 +201,14 @@ class TestCustomSFTPClient: self.mock_mkdir = MagicMock() self.mock_put = MagicMock() monkeypatch.setattr( - "aiet.backend.protocol.paramiko.SFTPClient.__init__", + "mlia.backend.protocol.paramiko.SFTPClient.__init__", MagicMock(return_value=None), ) monkeypatch.setattr( - "aiet.backend.protocol.paramiko.SFTPClient.mkdir", self.mock_mkdir + "mlia.backend.protocol.paramiko.SFTPClient.mkdir", self.mock_mkdir ) monkeypatch.setattr( - "aiet.backend.protocol.paramiko.SFTPClient.put", self.mock_put + "mlia.backend.protocol.paramiko.SFTPClient.put", self.mock_put ) self.sftp_client = CustomSFTPClient(MagicMock()) diff --git a/tests/aiet/test_backend_source.py b/tests/mlia/test_backend_source.py index 13b2c6d..84a6a77 100644 --- a/tests/aiet/test_backend_source.py +++ b/tests/mlia/test_backend_source.py @@ -11,11 +11,11 @@ from unittest.mock import patch import pytest -from aiet.backend.common import ConfigurationException -from aiet.backend.source import create_destination_and_install -from aiet.backend.source import DirectorySource -from aiet.backend.source import get_source -from aiet.backend.source import TarArchiveSource +from mlia.backend.common import ConfigurationException +from mlia.backend.source import create_destination_and_install +from mlia.backend.source import DirectorySource +from mlia.backend.source import get_source +from mlia.backend.source import TarArchiveSource def test_create_destination_and_install(test_systems_path: Path, tmpdir: Any) -> None: @@ -28,7 +28,7 @@ def test_create_destination_and_install(test_systems_path: Path, tmpdir: Any) -> assert (resources / "system1").is_dir() -@patch("aiet.backend.source.DirectorySource.create_destination", return_value=False) +@patch("mlia.backend.source.DirectorySource.create_destination", return_value=False) def test_create_destination_and_install_if_dest_creation_not_required( mock_ds_create_destination: Any, tmpdir: Any ) -> None: @@ -66,7 +66,11 @@ def test_create_destination_and_install_if_name_is_empty() -> None: @pytest.mark.parametrize( "source_path, expected_class, expected_error", [ - (Path("applications/application1/"), DirectorySource, does_not_raise()), + ( + Path("backends/applications/application1/"), + DirectorySource, + does_not_raise(), + ), ( Path("archives/applications/application1.tar.gz"), TarArchiveSource, diff --git a/tests/aiet/test_backend_system.py b/tests/mlia/test_backend_system.py index a581547..21187ff 100644 --- a/tests/aiet/test_backend_system.py +++ b/tests/mlia/test_backend_system.py @@ -13,29 +13,29 @@ from unittest.mock import MagicMock import pytest -from aiet.backend.common import Command -from aiet.backend.common import ConfigurationException -from aiet.backend.common import Param -from aiet.backend.common import UserParamConfig -from aiet.backend.config import LocalProtocolConfig -from aiet.backend.config import ProtocolConfig -from aiet.backend.config import SSHConfig -from aiet.backend.config import SystemConfig -from aiet.backend.controller import SystemController -from aiet.backend.controller import SystemControllerSingleInstance -from aiet.backend.protocol import LocalProtocol -from aiet.backend.protocol import SSHProtocol -from aiet.backend.protocol import SupportsClose -from aiet.backend.protocol import SupportsDeploy -from aiet.backend.system import ControlledSystem -from aiet.backend.system import get_available_systems -from aiet.backend.system import get_controller -from aiet.backend.system import get_system -from aiet.backend.system import install_system -from aiet.backend.system import load_system -from aiet.backend.system import remove_system -from aiet.backend.system import StandaloneSystem -from aiet.backend.system import System +from mlia.backend.common import Command +from mlia.backend.common import ConfigurationException +from mlia.backend.common import Param +from mlia.backend.common import UserParamConfig +from mlia.backend.config import LocalProtocolConfig +from mlia.backend.config import ProtocolConfig +from mlia.backend.config import SSHConfig +from mlia.backend.config import SystemConfig +from mlia.backend.controller import SystemController +from mlia.backend.controller import SystemControllerSingleInstance +from mlia.backend.protocol import LocalProtocol +from mlia.backend.protocol import SSHProtocol +from mlia.backend.protocol import SupportsClose +from mlia.backend.protocol import SupportsDeploy +from mlia.backend.system import ControlledSystem +from mlia.backend.system import get_available_systems +from mlia.backend.system import get_controller +from mlia.backend.system import get_system +from mlia.backend.system import install_system +from mlia.backend.system import load_system +from mlia.backend.system import remove_system +from mlia.backend.system import StandaloneSystem +from mlia.backend.system import System def dummy_resolver( @@ -56,8 +56,13 @@ def test_get_available_systems() -> None: """Test get_available_systems mocking get_resources.""" available_systems = get_available_systems() assert all(isinstance(s, System) for s in available_systems) - assert len(available_systems) == 3 - assert [str(s) for s in available_systems] == ["System 1", "System 2", "System 4"] + assert len(available_systems) == 4 + assert [str(s) for s in available_systems] == [ + "System 1", + "System 2", + "System 4", + "System 6", + ] def test_get_system() -> None: @@ -91,12 +96,12 @@ def test_get_system() -> None: pytest.raises(Exception, match="Unable to read system definition"), ), ( - "systems/system1", + "backends/systems/system1", 0, pytest.raises(Exception, match="Systems .* are already installed"), ), ( - "systems/system3", + "backends/systems/system3", 0, pytest.raises(Exception, match="Unable to read system definition"), ), @@ -119,7 +124,7 @@ def test_install_system( """Test system installation from archive.""" mock_create_destination_and_install = MagicMock() monkeypatch.setattr( - "aiet.backend.system.create_destination_and_install", + "mlia.backend.system.create_destination_and_install", mock_create_destination_and_install, ) @@ -132,7 +137,7 @@ def test_install_system( def test_remove_system(monkeypatch: Any) -> None: """Test system removal.""" mock_remove_backend = MagicMock() - monkeypatch.setattr("aiet.backend.system.remove_backend", mock_remove_backend) + monkeypatch.setattr("mlia.backend.system.remove_backend", mock_remove_backend) remove_system("some_system_dir") mock_remove_backend.assert_called_once() @@ -140,7 +145,7 @@ def test_remove_system(monkeypatch: Any) -> None: def test_system(monkeypatch: Any) -> None: """Test the System class.""" config = SystemConfig(name="System 1") - monkeypatch.setattr("aiet.backend.system.ProtocolFactory", MagicMock()) + monkeypatch.setattr("mlia.backend.system.ProtocolFactory", MagicMock()) system = System(config) assert str(system) == "System 1" assert system.name == "System 1" @@ -202,7 +207,7 @@ def test_system_supports_deploy(system_name: str, expected_value: bool) -> None: def test_system_start_and_stop(monkeypatch: Any, mock_protocol: MagicMock) -> None: """Test system start, run commands and stop.""" monkeypatch.setattr( - "aiet.backend.system.ProtocolFactory.get_protocol", + "mlia.backend.system.ProtocolFactory.get_protocol", MagicMock(return_value=mock_protocol), ) @@ -516,7 +521,7 @@ def test_system_deploy_data( ) -> None: """Test deploy data functionality.""" monkeypatch.setattr( - "aiet.backend.system.ProtocolFactory.get_protocol", + "mlia.backend.system.ProtocolFactory.get_protocol", MagicMock(return_value=mock_protocol), ) diff --git a/tests/mlia/test_cli_logging.py b/tests/mlia/test_cli_logging.py index 7c5f299..3f59cb6 100644 --- a/tests/mlia/test_cli_logging.py +++ b/tests/mlia/test_cli_logging.py @@ -32,7 +32,7 @@ def teardown_function() -> None: ( None, True, - """mlia.tools.aiet_wrapper - aiet debug + """mlia.backend.manager - backends debug cli info mlia.cli - cli debug """, @@ -41,11 +41,11 @@ mlia.cli - cli debug ( "logs", True, - """mlia.tools.aiet_wrapper - aiet debug + """mlia.backend.manager - backends debug cli info mlia.cli - cli debug """, - """mlia.tools.aiet_wrapper - DEBUG - aiet debug + """mlia.backend.manager - DEBUG - backends debug mlia.cli - DEBUG - cli debug """, ), @@ -64,8 +64,8 @@ def test_setup_logging( setup_logging(logs_dir_path, verbose) - aiet_logger = logging.getLogger("mlia.tools.aiet_wrapper") - aiet_logger.debug("aiet debug") + backend_logger = logging.getLogger("mlia.backend.manager") + backend_logger.debug("backends debug") cli_logger = logging.getLogger("mlia.cli") cli_logger.info("cli info") diff --git a/tests/mlia/test_devices_ethosu_performance.py b/tests/mlia/test_devices_ethosu_performance.py index e27efa0..b3e5298 100644 --- a/tests/mlia/test_devices_ethosu_performance.py +++ b/tests/mlia/test_devices_ethosu_performance.py @@ -23,6 +23,6 @@ def test_memory_usage_conversion() -> None: def mock_performance_estimation(monkeypatch: pytest.MonkeyPatch) -> None: """Mock performance estimation.""" monkeypatch.setattr( - "mlia.tools.aiet_wrapper.estimate_performance", + "mlia.backend.manager.estimate_performance", MagicMock(return_value=MagicMock()), ) diff --git a/tests/aiet/test_resources/application_config.json b/tests/mlia/test_resources/application_config.json index 2dfcfec..2dfcfec 100644 --- a/tests/aiet/test_resources/application_config.json +++ b/tests/mlia/test_resources/application_config.json diff --git a/tests/aiet/test_resources/application_config.json.license b/tests/mlia/test_resources/application_config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/application_config.json.license +++ b/tests/mlia/test_resources/application_config.json.license diff --git a/tests/aiet/test_resources/applications/application1/aiet-config.json b/tests/mlia/test_resources/backends/applications/application1/aiet-config.json index 97f0401..97f0401 100644 --- a/tests/aiet/test_resources/applications/application1/aiet-config.json +++ b/tests/mlia/test_resources/backends/applications/application1/aiet-config.json diff --git a/src/aiet/resources/tools/vela/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application1/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/src/aiet/resources/tools/vela/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/applications/application1/aiet-config.json.license diff --git a/tests/aiet/test_resources/applications/application2/aiet-config.json b/tests/mlia/test_resources/backends/applications/application2/aiet-config.json index e9122d3..e9122d3 100644 --- a/tests/aiet/test_resources/applications/application2/aiet-config.json +++ b/tests/mlia/test_resources/backends/applications/application2/aiet-config.json diff --git a/tests/aiet/test_resources/applications/application1/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application2/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/applications/application1/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/applications/application2/aiet-config.json.license diff --git a/tests/aiet/test_resources/applications/application3/readme.txt b/tests/mlia/test_resources/backends/applications/application3/readme.txt index 8c72c05..8c72c05 100644 --- a/tests/aiet/test_resources/applications/application3/readme.txt +++ b/tests/mlia/test_resources/backends/applications/application3/readme.txt diff --git a/tests/aiet/test_resources/applications/application4/aiet-config.json b/tests/mlia/test_resources/backends/applications/application4/aiet-config.json index 34dc780..ffb5746 100644 --- a/tests/aiet/test_resources/applications/application4/aiet-config.json +++ b/tests/mlia/test_resources/backends/applications/application4/aiet-config.json @@ -10,10 +10,11 @@ ], "commands": { "build": [ - "cp ../hello_app.txt . # {user_params:0}" + "cp ../hello_app.txt .", + "echo '{user_params:0}' > params.txt" ], "run": [ - "{application.build_dir}/hello_app.txt" + "cat {application.build_dir}/hello_app.txt" ] }, "user_params": { diff --git a/tests/aiet/test_resources/applications/application2/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application4/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/applications/application2/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/applications/application4/aiet-config.json.license diff --git a/tests/aiet/test_resources/applications/application4/hello_app.txt b/tests/mlia/test_resources/backends/applications/application4/hello_app.txt index 2ec0d1d..2ec0d1d 100644 --- a/tests/aiet/test_resources/applications/application4/hello_app.txt +++ b/tests/mlia/test_resources/backends/applications/application4/hello_app.txt diff --git a/tests/aiet/test_resources/applications/application5/aiet-config.json b/tests/mlia/test_resources/backends/applications/application5/aiet-config.json index 5269409..5269409 100644 --- a/tests/aiet/test_resources/applications/application5/aiet-config.json +++ b/tests/mlia/test_resources/backends/applications/application5/aiet-config.json diff --git a/tests/aiet/test_resources/applications/application4/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application5/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/applications/application4/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/applications/application5/aiet-config.json.license diff --git a/tests/mlia/test_resources/backends/applications/application6/aiet-config.json b/tests/mlia/test_resources/backends/applications/application6/aiet-config.json new file mode 100644 index 0000000..56ad807 --- /dev/null +++ b/tests/mlia/test_resources/backends/applications/application6/aiet-config.json @@ -0,0 +1,42 @@ +[ + { + "name": "application_6", + "description": "This is application 6", + "supported_systems": [ + { + "name": "System 6" + } + ], + "build_dir": "build", + "commands": { + "clean": [ + "echo 'clean'" + ], + "build": [ + "echo 'build'" + ], + "run": [ + "echo 'run {user_params:param1}'" + ], + "post_run": [ + "echo 'post_run'" + ] + }, + "user_params": { + "build": [], + "run": [ + { + "name": "--param1", + "description": "Test parameter", + "values": [ + "value1", + "value2", + "value3" + ], + "default_value": "value3", + "alias": "param1" + } + ] + } + } +] diff --git a/tests/aiet/test_resources/applications/application5/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application6/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/applications/application5/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/applications/application6/aiet-config.json.license diff --git a/tests/aiet/test_resources/applications/readme.txt b/tests/mlia/test_resources/backends/applications/readme.txt index a1f8209..a1f8209 100644 --- a/tests/aiet/test_resources/applications/readme.txt +++ b/tests/mlia/test_resources/backends/applications/readme.txt diff --git a/tests/aiet/test_resources/systems/system1/aiet-config.json b/tests/mlia/test_resources/backends/systems/system1/aiet-config.json index 4b5dd19..4b5dd19 100644 --- a/tests/aiet/test_resources/systems/system1/aiet-config.json +++ b/tests/mlia/test_resources/backends/systems/system1/aiet-config.json diff --git a/tests/aiet/test_resources/systems/system1/aiet-config.json.license b/tests/mlia/test_resources/backends/systems/system1/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/systems/system1/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/systems/system1/aiet-config.json.license diff --git a/tests/aiet/test_resources/systems/system1/system_artifact/dummy.txt b/tests/mlia/test_resources/backends/systems/system1/system_artifact/dummy.txt index 487e9d8..487e9d8 100644 --- a/tests/aiet/test_resources/systems/system1/system_artifact/dummy.txt +++ b/tests/mlia/test_resources/backends/systems/system1/system_artifact/dummy.txt diff --git a/tests/aiet/test_resources/systems/system2/aiet-config.json b/tests/mlia/test_resources/backends/systems/system2/aiet-config.json index a9e0eb3..a9e0eb3 100644 --- a/tests/aiet/test_resources/systems/system2/aiet-config.json +++ b/tests/mlia/test_resources/backends/systems/system2/aiet-config.json diff --git a/tests/aiet/test_resources/systems/system2/aiet-config.json.license b/tests/mlia/test_resources/backends/systems/system2/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/systems/system2/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/systems/system2/aiet-config.json.license diff --git a/tests/aiet/test_resources/systems/system3/readme.txt b/tests/mlia/test_resources/backends/systems/system3/readme.txt index aba5a9c..aba5a9c 100644 --- a/tests/aiet/test_resources/systems/system3/readme.txt +++ b/tests/mlia/test_resources/backends/systems/system3/readme.txt diff --git a/tests/aiet/test_resources/systems/system4/aiet-config.json b/tests/mlia/test_resources/backends/systems/system4/aiet-config.json index 295e00f..7b13160 100644 --- a/tests/aiet/test_resources/systems/system4/aiet-config.json +++ b/tests/mlia/test_resources/backends/systems/system4/aiet-config.json @@ -9,7 +9,7 @@ "commands": { "run": [ "echo {application.name}", - "cat {application.commands.run:0}" + "{application.commands.run:0}" ] }, "user_params": { diff --git a/tests/aiet/test_resources/systems/system4/aiet-config.json.license b/tests/mlia/test_resources/backends/systems/system4/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/systems/system4/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/systems/system4/aiet-config.json.license diff --git a/tests/mlia/test_resources/backends/systems/system6/aiet-config.json b/tests/mlia/test_resources/backends/systems/system6/aiet-config.json new file mode 100644 index 0000000..4242f64 --- /dev/null +++ b/tests/mlia/test_resources/backends/systems/system6/aiet-config.json @@ -0,0 +1,34 @@ +[ + { + "name": "System 6", + "description": "This is system 6", + "build_dir": "build", + "data_transfer": { + "protocol": "local" + }, + "variables": { + "var1": "{user_params:sys-param1}" + }, + "commands": { + "run": [ + "echo {application.name}", + "{application.commands.run:0}" + ] + }, + "user_params": { + "run": [ + { + "name": "--sys-param1", + "description": "Test parameter", + "values": [ + "value1", + "value2", + "value3" + ], + "default_value": "value1", + "alias": "sys-param1" + } + ] + } + } +] diff --git a/tests/aiet/test_resources/tools/tool1/aiet-config.json.license b/tests/mlia/test_resources/backends/systems/system6/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/tools/tool1/aiet-config.json.license +++ b/tests/mlia/test_resources/backends/systems/system6/aiet-config.json.license diff --git a/tests/aiet/test_resources/hello_world.json b/tests/mlia/test_resources/hello_world.json index 8a9a448..8a9a448 100644 --- a/tests/aiet/test_resources/hello_world.json +++ b/tests/mlia/test_resources/hello_world.json diff --git a/tests/aiet/test_resources/hello_world.json.license b/tests/mlia/test_resources/hello_world.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/hello_world.json.license +++ b/tests/mlia/test_resources/hello_world.json.license diff --git a/tests/aiet/test_resources/scripts/test_backend_run b/tests/mlia/test_resources/scripts/test_backend_run index 548f577..548f577 100755 --- a/tests/aiet/test_resources/scripts/test_backend_run +++ b/tests/mlia/test_resources/scripts/test_backend_run diff --git a/tests/aiet/test_resources/scripts/test_backend_run_script.sh b/tests/mlia/test_resources/scripts/test_backend_run_script.sh index 548f577..548f577 100644 --- a/tests/aiet/test_resources/scripts/test_backend_run_script.sh +++ b/tests/mlia/test_resources/scripts/test_backend_run_script.sh diff --git a/tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json index fe51488..fe51488 100644 --- a/tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json +++ b/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json diff --git a/tests/aiet/test_resources/tools/tool2/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/tools/tool2/aiet-config.json.license +++ b/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json.license diff --git a/tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json index ff1cf1a..ff1cf1a 100644 --- a/tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json +++ b/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json diff --git a/tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json.license +++ b/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json.license diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json index 724b31b..724b31b 100644 --- a/tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json +++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json diff --git a/tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json.license +++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json index 1ebb29c..1ebb29c 100644 --- a/tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json +++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license +++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json index 410d12d..410d12d 100644 --- a/tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json +++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license +++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license diff --git a/tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json b/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json index fe51488..fe51488 100644 --- a/tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json +++ b/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license b/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license +++ b/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json.license diff --git a/tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json b/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json index 20142e9..20142e9 100644 --- a/tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json +++ b/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json diff --git a/tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json.license b/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json.license index 9b83bfc..9b83bfc 100644 --- a/tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json.license +++ b/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json.license diff --git a/tests/mlia/test_tools_metadata_corstone.py b/tests/mlia/test_tools_metadata_corstone.py index 2ce3610..017d0c7 100644 --- a/tests/mlia/test_tools_metadata_corstone.py +++ b/tests/mlia/test_tools_metadata_corstone.py @@ -9,13 +9,13 @@ from unittest.mock import MagicMock import pytest -from mlia.tools.aiet_wrapper import AIETRunner +from mlia.backend.manager import BackendRunner from mlia.tools.metadata.common import DownloadAndInstall from mlia.tools.metadata.common import InstallFromPath -from mlia.tools.metadata.corstone import AIETBasedInstallation -from mlia.tools.metadata.corstone import AIETMetadata from mlia.tools.metadata.corstone import BackendInfo +from mlia.tools.metadata.corstone import BackendInstallation from mlia.tools.metadata.corstone import BackendInstaller +from mlia.tools.metadata.corstone import BackendMetadata from mlia.tools.metadata.corstone import CompoundPathChecker from mlia.tools.metadata.corstone import Corstone300Installer from mlia.tools.metadata.corstone import get_corstone_installations @@ -40,8 +40,8 @@ def fixture_test_mlia_resources( return mlia_resources -def get_aiet_based_installation( # pylint: disable=too-many-arguments - aiet_runner_mock: MagicMock = MagicMock(), +def get_backend_installation( # pylint: disable=too-many-arguments + backend_runner_mock: MagicMock = MagicMock(), name: str = "test_name", description: str = "test_description", download_artifact: Optional[MagicMock] = None, @@ -50,11 +50,11 @@ def get_aiet_based_installation( # pylint: disable=too-many-arguments system_config: Optional[str] = None, backend_installer: BackendInstaller = MagicMock(), supported_platforms: Optional[List[str]] = None, -) -> AIETBasedInstallation: - """Get AIET based installation.""" - return AIETBasedInstallation( - aiet_runner=aiet_runner_mock, - metadata=AIETMetadata( +) -> BackendInstallation: + """Get backend installation.""" + return BackendInstallation( + backend_runner=backend_runner_mock, + metadata=BackendMetadata( name=name, description=description, system_config=system_config or "", @@ -90,10 +90,10 @@ def test_could_be_installed_depends_on_platform( monkeypatch.setattr( "mlia.tools.metadata.corstone.all_paths_valid", MagicMock(return_value=True) ) - aiet_runner_mock = MagicMock(spec=AIETRunner) + backend_runner_mock = MagicMock(spec=BackendRunner) - installation = get_aiet_based_installation( - aiet_runner_mock, + installation = get_backend_installation( + backend_runner_mock, supported_platforms=supported_platforms, ) assert installation.could_be_installed == expected_result @@ -103,53 +103,53 @@ def test_get_corstone_installations() -> None: """Test function get_corstone_installation.""" installs = get_corstone_installations() assert len(installs) == 2 - assert all(isinstance(install, AIETBasedInstallation) for install in installs) + assert all(isinstance(install, BackendInstallation) for install in installs) -def test_aiet_based_installation_metadata_resolving() -> None: - """Test AIET based installation metadata resolving.""" - aiet_runner_mock = MagicMock(spec=AIETRunner) - installation = get_aiet_based_installation(aiet_runner_mock) +def test_backend_installation_metadata_resolving() -> None: + """Test backend installation metadata resolving.""" + backend_runner_mock = MagicMock(spec=BackendRunner) + installation = get_backend_installation(backend_runner_mock) assert installation.name == "test_name" assert installation.description == "test_description" - aiet_runner_mock.all_installed.return_value = False + backend_runner_mock.all_installed.return_value = False assert installation.already_installed is False assert installation.could_be_installed is True -def test_aiet_based_installation_supported_install_types(tmp_path: Path) -> None: +def test_backend_installation_supported_install_types(tmp_path: Path) -> None: """Test supported installation types.""" - installation_no_download_artifact = get_aiet_based_installation() + installation_no_download_artifact = get_backend_installation() assert installation_no_download_artifact.supports(DownloadAndInstall()) is False - installation_with_download_artifact = get_aiet_based_installation( + installation_with_download_artifact = get_backend_installation( download_artifact=MagicMock() ) assert installation_with_download_artifact.supports(DownloadAndInstall()) is True path_checker_mock = MagicMock(return_value=BackendInfo(tmp_path)) - installation_can_install_from_dir = get_aiet_based_installation( + installation_can_install_from_dir = get_backend_installation( path_checker=path_checker_mock ) assert installation_can_install_from_dir.supports(InstallFromPath(tmp_path)) is True - any_installation = get_aiet_based_installation() + any_installation = get_backend_installation() assert any_installation.supports("unknown_install_type") is False # type: ignore -def test_aiet_based_installation_install_wrong_type() -> None: +def test_backend_installation_install_wrong_type() -> None: """Test that operation should fail if wrong install type provided.""" with pytest.raises(Exception, match="Unable to install wrong_install_type"): - aiet_runner_mock = MagicMock(spec=AIETRunner) - installation = get_aiet_based_installation(aiet_runner_mock) + backend_runner_mock = MagicMock(spec=BackendRunner) + installation = get_backend_installation(backend_runner_mock) installation.install("wrong_install_type") # type: ignore -def test_aiet_based_installation_install_from_path( +def test_backend_installation_install_from_path( tmp_path: Path, test_mlia_resources: Path ) -> None: """Test installation from the path.""" @@ -164,9 +164,9 @@ def test_aiet_based_installation_install_from_path( path_checker_mock = MagicMock(return_value=BackendInfo(dist_dir)) - aiet_runner_mock = MagicMock(spec=AIETRunner) - installation = get_aiet_based_installation( - aiet_runner_mock=aiet_runner_mock, + backend_runner_mock = MagicMock(spec=BackendRunner) + installation = get_backend_installation( + backend_runner_mock=backend_runner_mock, path_checker=path_checker_mock, apps_resources=[sample_app.name], system_config="example_config.json", @@ -175,12 +175,12 @@ def test_aiet_based_installation_install_from_path( assert installation.supports(InstallFromPath(dist_dir)) is True installation.install(InstallFromPath(dist_dir)) - aiet_runner_mock.install_system.assert_called_once() - aiet_runner_mock.install_application.assert_called_once_with(sample_app) + backend_runner_mock.install_system.assert_called_once() + backend_runner_mock.install_application.assert_called_once_with(sample_app) @pytest.mark.parametrize("copy_source", [True, False]) -def test_aiet_based_installation_install_from_static_path( +def test_backend_installation_install_from_static_path( tmp_path: Path, test_mlia_resources: Path, copy_source: bool ) -> None: """Test installation from the predefined path.""" @@ -204,7 +204,7 @@ def test_aiet_based_installation_install_from_static_path( nested_file = predefined_location_dir / "nested_file.txt" nested_file.touch() - aiet_runner_mock = MagicMock(spec=AIETRunner) + backend_runner_mock = MagicMock(spec=BackendRunner) def check_install_dir(install_dir: Path) -> None: """Check content of the install dir.""" @@ -220,10 +220,10 @@ def test_aiet_based_installation_install_from_static_path( assert install_dir / "custom_config.json" in files - aiet_runner_mock.install_system.side_effect = check_install_dir + backend_runner_mock.install_system.side_effect = check_install_dir - installation = get_aiet_based_installation( - aiet_runner_mock=aiet_runner_mock, + installation = get_backend_installation( + backend_runner_mock=backend_runner_mock, path_checker=StaticPathChecker( predefined_location, ["file.txt"], @@ -237,8 +237,8 @@ def test_aiet_based_installation_install_from_static_path( assert installation.supports(InstallFromPath(predefined_location)) is True installation.install(InstallFromPath(predefined_location)) - aiet_runner_mock.install_system.assert_called_once() - aiet_runner_mock.install_application.assert_called_once_with(sample_app) + backend_runner_mock.install_system.assert_called_once() + backend_runner_mock.install_application.assert_called_once_with(sample_app) def create_sample_fvp_archive(tmp_path: Path) -> Path: @@ -259,7 +259,7 @@ def create_sample_fvp_archive(tmp_path: Path) -> Path: return fvp_archive -def test_aiet_based_installation_download_and_install( +def test_backend_installation_download_and_install( test_mlia_resources: Path, tmp_path: Path ) -> None: """Test downloading and installation process.""" @@ -277,9 +277,9 @@ def test_aiet_based_installation_download_and_install( """Sample installer.""" return dist_dir - aiet_runner_mock = MagicMock(spec=AIETRunner) - installation = get_aiet_based_installation( - aiet_runner_mock, + backend_runner_mock = MagicMock(spec=BackendRunner) + installation = get_backend_installation( + backend_runner_mock, download_artifact=download_artifact_mock, backend_installer=installer, path_checker=path_checker, @@ -288,7 +288,7 @@ def test_aiet_based_installation_download_and_install( installation.install(DownloadAndInstall()) - aiet_runner_mock.install_system.assert_called_once() + backend_runner_mock.install_system.assert_called_once() @pytest.mark.parametrize( |