aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorBenjamin Klimczak <benjamin.klimczak@arm.com>2022-06-28 10:29:35 +0100
committerBenjamin Klimczak <benjamin.klimczak@arm.com>2022-07-08 10:57:19 +0100
commitc9b4089b3037b5943565d76242d3016b8776f8d2 (patch)
tree3de24f79dedf0f26f492a7fa1562bf684e13a055
parentba2c7fcccf37e8c81946f0776714c64f73191787 (diff)
downloadmlia-c9b4089b3037b5943565d76242d3016b8776f8d2.tar.gz
MLIA-546 Merge AIET into MLIA
Merge the deprecated AIET interface for backend execution into MLIA: - Execute backends directly (without subprocess and the aiet CLI) - Fix issues with the unit tests - Remove src/aiet and tests/aiet - Re-factor code to replace 'aiet' with 'backend' - Adapt and improve unit tests after re-factoring - Remove dependencies that are not needed anymore (click and cloup) Change-Id: I450734c6a3f705ba9afde41862b29e797e511f7c
-rw-r--r--setup.cfg4
-rw-r--r--src/aiet/__init__.py7
-rw-r--r--src/aiet/backend/tool.py109
-rw-r--r--src/aiet/cli/__init__.py28
-rw-r--r--src/aiet/cli/application.py362
-rw-r--r--src/aiet/cli/common.py173
-rw-r--r--src/aiet/cli/completion.py72
-rw-r--r--src/aiet/cli/system.py122
-rw-r--r--src/aiet/cli/tool.py143
-rw-r--r--src/aiet/main.py13
-rw-r--r--src/aiet/resources/tools/vela/aiet-config.json73
-rw-r--r--src/aiet/resources/tools/vela/check_model.py75
-rw-r--r--src/aiet/resources/tools/vela/run_vela.py65
-rw-r--r--src/aiet/resources/tools/vela/vela.ini53
-rw-r--r--src/aiet/utils/__init__.py3
-rw-r--r--src/aiet/utils/helpers.py17
-rw-r--r--src/mlia/backend/__init__.py (renamed from src/aiet/backend/__init__.py)0
-rw-r--r--src/mlia/backend/application.py (renamed from src/aiet/backend/application.py)28
-rw-r--r--src/mlia/backend/common.py (renamed from src/aiet/backend/common.py)22
-rw-r--r--src/mlia/backend/config.py (renamed from src/aiet/backend/config.py)18
-rw-r--r--src/mlia/backend/controller.py (renamed from src/aiet/backend/controller.py)16
-rw-r--r--src/mlia/backend/execution.py (renamed from src/aiet/backend/execution.py)148
-rw-r--r--src/mlia/backend/fs.py (renamed from src/aiet/utils/fs.py)19
-rw-r--r--src/mlia/backend/manager.py (renamed from src/mlia/tools/aiet_wrapper.py)108
-rw-r--r--src/mlia/backend/output_parser.py (renamed from src/aiet/backend/output_parser.py)0
-rw-r--r--src/mlia/backend/proc.py (renamed from src/aiet/utils/proc.py)2
-rw-r--r--src/mlia/backend/protocol.py (renamed from src/aiet/backend/protocol.py)8
-rw-r--r--src/mlia/backend/source.py (renamed from src/aiet/backend/source.py)18
-rw-r--r--src/mlia/backend/system.py (renamed from src/aiet/backend/system.py)34
-rw-r--r--src/mlia/cli/config.py6
-rw-r--r--src/mlia/devices/ethosu/performance.py37
-rw-r--r--src/mlia/resources/aiet/applications/APPLICATIONS.txt5
-rw-r--r--src/mlia/resources/aiet/systems/SYSTEMS.txt3
-rw-r--r--src/mlia/resources/backends/applications/.gitignore (renamed from src/aiet/resources/applications/.gitignore)0
-rw-r--r--src/mlia/resources/backends/systems/.gitignore (renamed from src/aiet/resources/systems/.gitignore)0
-rw-r--r--src/mlia/tools/metadata/corstone.py61
-rw-r--r--src/mlia/utils/proc.py20
-rw-r--r--tests/aiet/__init__.py3
-rw-r--r--tests/aiet/conftest.py139
-rw-r--r--tests/aiet/test_backend_tool.py60
-rw-r--r--tests/aiet/test_check_model.py162
-rw-r--r--tests/aiet/test_cli.py37
-rw-r--r--tests/aiet/test_cli_application.py1153
-rw-r--r--tests/aiet/test_cli_common.py37
-rw-r--r--tests/aiet/test_cli_system.py240
-rw-r--r--tests/aiet/test_cli_tool.py333
-rw-r--r--tests/aiet/test_main.py16
-rw-r--r--tests/aiet/test_resources/tools/tool1/aiet-config.json30
-rw-r--r--tests/aiet/test_resources/tools/tool2/aiet-config.json26
-rw-r--r--tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json.license3
-rw-r--r--tests/aiet/test_run_vela_script.py152
-rw-r--r--tests/aiet/test_utils_helpers.py27
-rw-r--r--tests/mlia/conftest.py91
-rw-r--r--tests/mlia/test_backend_application.py (renamed from tests/aiet/test_backend_application.py)50
-rw-r--r--tests/mlia/test_backend_common.py (renamed from tests/aiet/test_backend_common.py)36
-rw-r--r--tests/mlia/test_backend_controller.py (renamed from tests/aiet/test_backend_controller.py)8
-rw-r--r--tests/mlia/test_backend_execution.py (renamed from tests/aiet/test_backend_execution.py)110
-rw-r--r--tests/mlia/test_backend_fs.py (renamed from tests/aiet/test_utils_fs.py)28
-rw-r--r--tests/mlia/test_backend_manager.py (renamed from tests/mlia/test_tools_aiet_wrapper.py)326
-rw-r--r--tests/mlia/test_backend_output_parser.py (renamed from tests/aiet/test_backend_output_parser.py)6
-rw-r--r--tests/mlia/test_backend_proc.py (renamed from tests/aiet/test_utils_proc.py)26
-rw-r--r--tests/mlia/test_backend_protocol.py (renamed from tests/aiet/test_backend_protocol.py)24
-rw-r--r--tests/mlia/test_backend_source.py (renamed from tests/aiet/test_backend_source.py)18
-rw-r--r--tests/mlia/test_backend_system.py (renamed from tests/aiet/test_backend_system.py)69
-rw-r--r--tests/mlia/test_cli_logging.py10
-rw-r--r--tests/mlia/test_devices_ethosu_performance.py2
-rw-r--r--tests/mlia/test_resources/application_config.json (renamed from tests/aiet/test_resources/application_config.json)0
-rw-r--r--tests/mlia/test_resources/application_config.json.license (renamed from tests/aiet/test_resources/application_config.json.license)0
-rw-r--r--tests/mlia/test_resources/backends/applications/application1/aiet-config.json (renamed from tests/aiet/test_resources/applications/application1/aiet-config.json)0
-rw-r--r--tests/mlia/test_resources/backends/applications/application1/aiet-config.json.license (renamed from src/aiet/resources/tools/vela/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/backends/applications/application2/aiet-config.json (renamed from tests/aiet/test_resources/applications/application2/aiet-config.json)0
-rw-r--r--tests/mlia/test_resources/backends/applications/application2/aiet-config.json.license (renamed from tests/aiet/test_resources/applications/application1/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/backends/applications/application3/readme.txt (renamed from tests/aiet/test_resources/applications/application3/readme.txt)0
-rw-r--r--tests/mlia/test_resources/backends/applications/application4/aiet-config.json (renamed from tests/aiet/test_resources/applications/application4/aiet-config.json)5
-rw-r--r--tests/mlia/test_resources/backends/applications/application4/aiet-config.json.license (renamed from tests/aiet/test_resources/applications/application2/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/backends/applications/application4/hello_app.txt (renamed from tests/aiet/test_resources/applications/application4/hello_app.txt)0
-rw-r--r--tests/mlia/test_resources/backends/applications/application5/aiet-config.json (renamed from tests/aiet/test_resources/applications/application5/aiet-config.json)0
-rw-r--r--tests/mlia/test_resources/backends/applications/application5/aiet-config.json.license (renamed from tests/aiet/test_resources/applications/application4/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/backends/applications/application6/aiet-config.json42
-rw-r--r--tests/mlia/test_resources/backends/applications/application6/aiet-config.json.license (renamed from tests/aiet/test_resources/applications/application5/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/backends/applications/readme.txt (renamed from tests/aiet/test_resources/applications/readme.txt)0
-rw-r--r--tests/mlia/test_resources/backends/systems/system1/aiet-config.json (renamed from tests/aiet/test_resources/systems/system1/aiet-config.json)0
-rw-r--r--tests/mlia/test_resources/backends/systems/system1/aiet-config.json.license (renamed from tests/aiet/test_resources/systems/system1/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/backends/systems/system1/system_artifact/dummy.txt (renamed from tests/aiet/test_resources/systems/system1/system_artifact/dummy.txt)0
-rw-r--r--tests/mlia/test_resources/backends/systems/system2/aiet-config.json (renamed from tests/aiet/test_resources/systems/system2/aiet-config.json)0
-rw-r--r--tests/mlia/test_resources/backends/systems/system2/aiet-config.json.license (renamed from tests/aiet/test_resources/systems/system2/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/backends/systems/system3/readme.txt (renamed from tests/aiet/test_resources/systems/system3/readme.txt)0
-rw-r--r--tests/mlia/test_resources/backends/systems/system4/aiet-config.json (renamed from tests/aiet/test_resources/systems/system4/aiet-config.json)2
-rw-r--r--tests/mlia/test_resources/backends/systems/system4/aiet-config.json.license (renamed from tests/aiet/test_resources/systems/system4/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/backends/systems/system6/aiet-config.json34
-rw-r--r--tests/mlia/test_resources/backends/systems/system6/aiet-config.json.license (renamed from tests/aiet/test_resources/tools/tool1/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/hello_world.json (renamed from tests/aiet/test_resources/hello_world.json)0
-rw-r--r--tests/mlia/test_resources/hello_world.json.license (renamed from tests/aiet/test_resources/hello_world.json.license)0
-rwxr-xr-xtests/mlia/test_resources/scripts/test_backend_run (renamed from tests/aiet/test_resources/scripts/test_backend_run)0
-rw-r--r--tests/mlia/test_resources/scripts/test_backend_run_script.sh (renamed from tests/aiet/test_resources/scripts/test_backend_run_script.sh)0
-rw-r--r--tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json)0
-rw-r--r--tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json.license (renamed from tests/aiet/test_resources/tools/tool2/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json)0
-rw-r--r--tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json)0
-rw-r--r--tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json)0
-rw-r--r--tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json)0
-rw-r--r--tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json (renamed from tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json)0
-rw-r--r--tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json.license (renamed from tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json (renamed from tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json)0
-rw-r--r--tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json.license (renamed from tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json.license)0
-rw-r--r--tests/mlia/test_tools_metadata_corstone.py90
110 files changed, 821 insertions, 4446 deletions
diff --git a/setup.cfg b/setup.cfg
index 9391fa2..3021043 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -34,12 +34,10 @@ install_requires =
ethos-u-vela~=3.3.0
requests
rich
- click
sh
paramiko
filelock
psutil
- cloup>=0.12.0
[options.packages.find]
where = src
@@ -47,8 +45,6 @@ where = src
[options.entry_points]
console_scripts =
mlia=mlia.cli.main:main
- aiet=aiet.main:main
- run_vela=aiet.resources.tools.vela.run_vela:main
[options.extras_require]
dev =
diff --git a/src/aiet/__init__.py b/src/aiet/__init__.py
deleted file mode 100644
index 49304b1..0000000
--- a/src/aiet/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Init of aiet."""
-import pkg_resources
-
-
-__version__ = pkg_resources.get_distribution("mlia").version
diff --git a/src/aiet/backend/tool.py b/src/aiet/backend/tool.py
deleted file mode 100644
index d643665..0000000
--- a/src/aiet/backend/tool.py
+++ /dev/null
@@ -1,109 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Tool backend module."""
-from typing import Any
-from typing import cast
-from typing import Dict
-from typing import List
-from typing import Optional
-
-from aiet.backend.common import Backend
-from aiet.backend.common import ConfigurationException
-from aiet.backend.common import get_backend_configs
-from aiet.backend.common import get_backend_directories
-from aiet.backend.common import load_application_or_tool_configs
-from aiet.backend.common import load_config
-from aiet.backend.config import ExtendedToolConfig
-from aiet.backend.config import ToolConfig
-
-
-def get_available_tool_directory_names() -> List[str]:
- """Return a list of directory names for all available tools."""
- return [entry.name for entry in get_backend_directories("tools")]
-
-
-def get_available_tools() -> List["Tool"]:
- """Return a list with all available tools."""
- available_tools = []
- for config_json in get_backend_configs("tools"):
- config_entries = cast(List[ExtendedToolConfig], load_config(config_json))
- for config_entry in config_entries:
- config_entry["config_location"] = config_json.parent.absolute()
- tools = load_tools(config_entry)
- available_tools += tools
-
- return sorted(available_tools, key=lambda tool: tool.name)
-
-
-def get_tool(tool_name: str, system_name: Optional[str] = None) -> List["Tool"]:
- """Return a tool instance with the same name passed as argument."""
- return [
- tool
- for tool in get_available_tools()
- if tool.name == tool_name and (not system_name or tool.can_run_on(system_name))
- ]
-
-
-def get_unique_tool_names(system_name: Optional[str] = None) -> List[str]:
- """Extract a list of unique tool names of all tools available."""
- return list(
- set(
- tool.name
- for tool in get_available_tools()
- if not system_name or tool.can_run_on(system_name)
- )
- )
-
-
-class Tool(Backend):
- """Class for representing a single tool component."""
-
- def __init__(self, config: ToolConfig) -> None:
- """Construct a Tool instance from a dict."""
- super().__init__(config)
-
- self.supported_systems = config.get("supported_systems", [])
-
- if "run" not in self.commands:
- raise ConfigurationException("A Tool must have a 'run' command.")
-
- def __eq__(self, other: object) -> bool:
- """Overload operator ==."""
- if not isinstance(other, Tool):
- return False
-
- return (
- super().__eq__(other)
- and self.name == other.name
- and set(self.supported_systems) == set(other.supported_systems)
- )
-
- def can_run_on(self, system_name: str) -> bool:
- """Check if the tool can run on the system passed as argument."""
- return system_name in self.supported_systems
-
- def get_details(self) -> Dict[str, Any]:
- """Return dictionary with all relevant information of the Tool instance."""
- output = {
- "type": "tool",
- "name": self.name,
- "description": self.description,
- "supported_systems": self.supported_systems,
- "commands": self._get_command_details(),
- }
-
- return output
-
-
-def load_tools(config: ExtendedToolConfig) -> List[Tool]:
- """Load tool.
-
- Tool configuration could contain different parameters/commands for different
- supported systems. For each supported system this function will return separate
- Tool instance with appropriate configuration.
- """
- configs = load_application_or_tool_configs(
- config, ToolConfig, is_system_required=False
- )
- tools = [Tool(cfg) for cfg in configs]
- return tools
diff --git a/src/aiet/cli/__init__.py b/src/aiet/cli/__init__.py
deleted file mode 100644
index bcd17c3..0000000
--- a/src/aiet/cli/__init__.py
+++ /dev/null
@@ -1,28 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Module to mange the CLI interface."""
-import click
-
-from aiet import __version__
-from aiet.cli.application import application_cmd
-from aiet.cli.completion import completion_cmd
-from aiet.cli.system import system_cmd
-from aiet.cli.tool import tool_cmd
-from aiet.utils.helpers import set_verbosity
-
-
-@click.group()
-@click.version_option(__version__)
-@click.option(
- "-v", "--verbose", default=0, count=True, callback=set_verbosity, expose_value=False
-)
-@click.pass_context
-def cli(ctx: click.Context) -> None: # pylint: disable=unused-argument
- """AIET: AI Evaluation Toolkit."""
- # Unused arguments must be present here in definition to pass click context.
-
-
-cli.add_command(application_cmd)
-cli.add_command(system_cmd)
-cli.add_command(tool_cmd)
-cli.add_command(completion_cmd)
diff --git a/src/aiet/cli/application.py b/src/aiet/cli/application.py
deleted file mode 100644
index 59b652d..0000000
--- a/src/aiet/cli/application.py
+++ /dev/null
@@ -1,362 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-FileCopyrightText: Copyright (c) 2021, Gianluca Gippetto. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0 AND BSD-3-Clause
-"""Module to manage the CLI interface of applications."""
-import json
-import logging
-import re
-from pathlib import Path
-from typing import Any
-from typing import IO
-from typing import List
-from typing import Optional
-from typing import Tuple
-
-import click
-import cloup
-
-from aiet.backend.application import get_application
-from aiet.backend.application import get_available_application_directory_names
-from aiet.backend.application import get_unique_application_names
-from aiet.backend.application import install_application
-from aiet.backend.application import remove_application
-from aiet.backend.common import DataPaths
-from aiet.backend.execution import execute_application_command
-from aiet.backend.execution import run_application
-from aiet.backend.system import get_available_systems
-from aiet.cli.common import get_format
-from aiet.cli.common import middleware_exception_handler
-from aiet.cli.common import middleware_signal_handler
-from aiet.cli.common import print_command_details
-from aiet.cli.common import set_format
-
-
-@click.group(name="application")
-@click.option(
- "-f",
- "--format",
- "format_",
- type=click.Choice(["cli", "json"]),
- default="cli",
- show_default=True,
-)
-@click.pass_context
-def application_cmd(ctx: click.Context, format_: str) -> None:
- """Sub command to manage applications."""
- set_format(ctx, format_)
-
-
-@application_cmd.command(name="list")
-@click.pass_context
-@click.option(
- "-s",
- "--system",
- "system_name",
- type=click.Choice([s.name for s in get_available_systems()]),
- required=False,
-)
-def list_cmd(ctx: click.Context, system_name: str) -> None:
- """List all available applications."""
- unique_application_names = get_unique_application_names(system_name)
- unique_application_names.sort()
- if get_format(ctx) == "json":
- data = {"type": "application", "available": unique_application_names}
- print(json.dumps(data))
- else:
- print("Available applications:\n")
- print(*unique_application_names, sep="\n")
-
-
-@application_cmd.command(name="details")
-@click.option(
- "-n",
- "--name",
- "application_name",
- type=click.Choice(get_unique_application_names()),
- required=True,
-)
-@click.option(
- "-s",
- "--system",
- "system_name",
- type=click.Choice([s.name for s in get_available_systems()]),
- required=False,
-)
-@click.pass_context
-def details_cmd(ctx: click.Context, application_name: str, system_name: str) -> None:
- """Details of a specific application."""
- applications = get_application(application_name, system_name)
- if not applications:
- raise click.UsageError(
- "Application '{}' doesn't support the system '{}'".format(
- application_name, system_name
- )
- )
-
- if get_format(ctx) == "json":
- applications_details = [s.get_details() for s in applications]
- print(json.dumps(applications_details))
- else:
- for application in applications:
- application_details = application.get_details()
- application_details_template = (
- 'Application "{name}" details\nDescription: {description}'
- )
-
- print(
- application_details_template.format(
- name=application_details["name"],
- description=application_details["description"],
- )
- )
-
- print(
- "\nSupported systems: {}".format(
- ", ".join(application_details["supported_systems"])
- )
- )
-
- command_details = application_details["commands"]
-
- for command, details in command_details.items():
- print("\n{} commands:".format(command))
- print_command_details(details)
-
-
-# pylint: disable=too-many-arguments
-@application_cmd.command(name="execute")
-@click.option(
- "-n",
- "--name",
- "application_name",
- type=click.Choice(get_unique_application_names()),
- required=True,
-)
-@click.option(
- "-s",
- "--system",
- "system_name",
- type=click.Choice([s.name for s in get_available_systems()]),
- required=True,
-)
-@click.option(
- "-c",
- "--command",
- "command_name",
- type=click.Choice(["build", "run"]),
- required=True,
-)
-@click.option("-p", "--param", "application_params", multiple=True)
-@click.option("--system-param", "system_params", multiple=True)
-@click.option("-d", "--deploy", "deploy_params", multiple=True)
-@middleware_signal_handler
-@middleware_exception_handler
-def execute_cmd(
- application_name: str,
- system_name: str,
- command_name: str,
- application_params: List[str],
- system_params: List[str],
- deploy_params: List[str],
-) -> None:
- """Execute application commands. DEPRECATED! Use 'aiet application run' instead."""
- logging.warning(
- "Please use 'aiet application run' instead. Use of 'aiet application "
- "execute' is deprecated and might be removed in a future release."
- )
-
- custom_deploy_data = get_custom_deploy_data(command_name, deploy_params)
-
- execute_application_command(
- command_name,
- application_name,
- application_params,
- system_name,
- system_params,
- custom_deploy_data,
- )
-
-
-@cloup.command(name="run")
-@cloup.option(
- "-n",
- "--name",
- "application_name",
- type=click.Choice(get_unique_application_names()),
-)
-@cloup.option(
- "-s",
- "--system",
- "system_name",
- type=click.Choice([s.name for s in get_available_systems()]),
-)
-@cloup.option("-p", "--param", "application_params", multiple=True)
-@cloup.option("--system-param", "system_params", multiple=True)
-@cloup.option("-d", "--deploy", "deploy_params", multiple=True)
-@click.option(
- "-r",
- "--report",
- "report_file",
- type=Path,
- help="Create a report file in JSON format containing metrics parsed from "
- "the simulation output as specified in the aiet-config.json.",
-)
-@cloup.option(
- "--config",
- "config_file",
- type=click.File("r"),
- help="Read options from a config file rather than from the command line. "
- "The config file is a json file.",
-)
-@cloup.constraint(
- cloup.constraints.If(
- cloup.constraints.conditions.Not(
- cloup.constraints.conditions.IsSet("config_file")
- ),
- then=cloup.constraints.require_all,
- ),
- ["system_name", "application_name"],
-)
-@cloup.constraint(
- cloup.constraints.If("config_file", then=cloup.constraints.accept_none),
- [
- "system_name",
- "application_name",
- "application_params",
- "system_params",
- "deploy_params",
- ],
-)
-@middleware_signal_handler
-@middleware_exception_handler
-def run_cmd(
- application_name: str,
- system_name: str,
- application_params: List[str],
- system_params: List[str],
- deploy_params: List[str],
- report_file: Optional[Path],
- config_file: Optional[IO[str]],
-) -> None:
- """Execute application commands."""
- if config_file:
- payload_data = json.load(config_file)
- (
- system_name,
- application_name,
- application_params,
- system_params,
- deploy_params,
- report_file,
- ) = parse_payload_run_config(payload_data)
-
- custom_deploy_data = get_custom_deploy_data("run", deploy_params)
-
- run_application(
- application_name,
- application_params,
- system_name,
- system_params,
- custom_deploy_data,
- report_file,
- )
-
-
-application_cmd.add_command(run_cmd)
-
-
-def parse_payload_run_config(
- payload_data: dict,
-) -> Tuple[str, str, List[str], List[str], List[str], Optional[Path]]:
- """Parse the payload into a tuple."""
- system_id = payload_data.get("id")
- arguments: Optional[Any] = payload_data.get("arguments")
-
- if not isinstance(system_id, str):
- raise click.ClickException("invalid payload json: no system 'id'")
- if not isinstance(arguments, dict):
- raise click.ClickException("invalid payload json: no arguments object")
-
- application_name = arguments.pop("application", None)
- if not isinstance(application_name, str):
- raise click.ClickException("invalid payload json: no application_id")
-
- report_path = arguments.pop("report_path", None)
-
- application_params = []
- system_params = []
- deploy_params = []
-
- for (param_key, value) in arguments.items():
- (par, _) = re.subn("^application/", "", param_key)
- (par, found_sys_param) = re.subn("^system/", "", par)
- (par, found_deploy_param) = re.subn("^deploy/", "", par)
-
- param_expr = par + "=" + value
- if found_sys_param:
- system_params.append(param_expr)
- elif found_deploy_param:
- deploy_params.append(par)
- else:
- application_params.append(param_expr)
-
- return (
- system_id,
- application_name,
- application_params,
- system_params,
- deploy_params,
- report_path,
- )
-
-
-def get_custom_deploy_data(
- command_name: str, deploy_params: List[str]
-) -> List[DataPaths]:
- """Get custom deploy data information."""
- custom_deploy_data: List[DataPaths] = []
- if not deploy_params:
- return custom_deploy_data
-
- for param in deploy_params:
- parts = param.split(":")
- if not len(parts) == 2 or any(not part.strip() for part in parts):
- raise click.ClickException(
- "Invalid deploy parameter '{}' for command {}".format(
- param, command_name
- )
- )
- data_path = DataPaths(Path(parts[0]), parts[1])
- if not data_path.src.exists():
- raise click.ClickException("Path {} does not exist".format(data_path.src))
- custom_deploy_data.append(data_path)
-
- return custom_deploy_data
-
-
-@application_cmd.command(name="install")
-@click.option(
- "-s",
- "--source",
- "source",
- required=True,
- help="Path to the directory or archive with application definition",
-)
-def install_cmd(source: str) -> None:
- """Install new application."""
- source_path = Path(source)
- install_application(source_path)
-
-
-@application_cmd.command(name="remove")
-@click.option(
- "-d",
- "--directory_name",
- "directory_name",
- type=click.Choice(get_available_application_directory_names()),
- required=True,
- help="Name of the directory with application",
-)
-def remove_cmd(directory_name: str) -> None:
- """Remove application."""
- remove_application(directory_name)
diff --git a/src/aiet/cli/common.py b/src/aiet/cli/common.py
deleted file mode 100644
index 1d157b6..0000000
--- a/src/aiet/cli/common.py
+++ /dev/null
@@ -1,173 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Common functions for cli module."""
-import enum
-import logging
-from functools import wraps
-from signal import SIG_IGN
-from signal import SIGINT
-from signal import signal as signal_handler
-from signal import SIGTERM
-from typing import Any
-from typing import Callable
-from typing import cast
-from typing import Dict
-
-from click import ClickException
-from click import Context
-from click import UsageError
-
-from aiet.backend.common import ConfigurationException
-from aiet.backend.execution import AnotherInstanceIsRunningException
-from aiet.backend.execution import ConnectionException
-from aiet.backend.protocol import SSHConnectionException
-from aiet.utils.proc import CommandFailedException
-
-
-class MiddlewareExitCode(enum.IntEnum):
- """Middleware exit codes."""
-
- SUCCESS = 0
- # exit codes 1 and 2 are used by click
- SHUTDOWN_REQUESTED = 3
- BACKEND_ERROR = 4
- CONCURRENT_ERROR = 5
- CONNECTION_ERROR = 6
- CONFIGURATION_ERROR = 7
- MODEL_OPTIMISED_ERROR = 8
- INVALID_TFLITE_FILE_ERROR = 9
-
-
-class CustomClickException(ClickException):
- """Custom click exception."""
-
- def show(self, file: Any = None) -> None:
- """Override show method."""
- super().show(file)
-
- logging.debug("Execution failed with following exception: ", exc_info=self)
-
-
-class MiddlewareShutdownException(CustomClickException):
- """Exception indicates that user requested middleware shutdown."""
-
- exit_code = int(MiddlewareExitCode.SHUTDOWN_REQUESTED)
-
-
-class BackendException(CustomClickException):
- """Exception indicates that command failed."""
-
- exit_code = int(MiddlewareExitCode.BACKEND_ERROR)
-
-
-class ConcurrentErrorException(CustomClickException):
- """Exception indicates concurrent execution error."""
-
- exit_code = int(MiddlewareExitCode.CONCURRENT_ERROR)
-
-
-class BackendConnectionException(CustomClickException):
- """Exception indicates that connection could not be established."""
-
- exit_code = int(MiddlewareExitCode.CONNECTION_ERROR)
-
-
-class BackendConfigurationException(CustomClickException):
- """Exception indicates some configuration issue."""
-
- exit_code = int(MiddlewareExitCode.CONFIGURATION_ERROR)
-
-
-class ModelOptimisedException(CustomClickException):
- """Exception indicates input file has previously been Vela optimised."""
-
- exit_code = int(MiddlewareExitCode.MODEL_OPTIMISED_ERROR)
-
-
-class InvalidTFLiteFileError(CustomClickException):
- """Exception indicates input TFLite file is misformatted."""
-
- exit_code = int(MiddlewareExitCode.INVALID_TFLITE_FILE_ERROR)
-
-
-def print_command_details(command: Dict) -> None:
- """Print command details including parameters."""
- command_strings = command["command_strings"]
- print("Commands: {}".format(command_strings))
- user_params = command["user_params"]
- for i, param in enumerate(user_params, 1):
- print("User parameter #{}".format(i))
- print("\tName: {}".format(param.get("name", "-")))
- print("\tDescription: {}".format(param["description"]))
- print("\tPossible values: {}".format(param.get("values", "-")))
- print("\tDefault value: {}".format(param.get("default_value", "-")))
- print("\tAlias: {}".format(param.get("alias", "-")))
-
-
-def raise_exception_at_signal(
- signum: int, frame: Any # pylint: disable=unused-argument
-) -> None:
- """Handle signals."""
- # Disable both SIGINT and SIGTERM signals. Further SIGINT and SIGTERM
- # signals will be ignored as we allow a graceful shutdown.
- # Unused arguments must be present here in definition as used in signal handler
- # callback
-
- signal_handler(SIGINT, SIG_IGN)
- signal_handler(SIGTERM, SIG_IGN)
- raise MiddlewareShutdownException("Middleware shutdown requested")
-
-
-def middleware_exception_handler(func: Callable) -> Callable:
- """Handle backend exceptions decorator."""
-
- @wraps(func)
- def wrapper(*args: Any, **kwargs: Any) -> Any:
- try:
- return func(*args, **kwargs)
- except (MiddlewareShutdownException, UsageError, ClickException) as error:
- # click should take care of these exceptions
- raise error
- except ValueError as error:
- raise ClickException(str(error)) from error
- except AnotherInstanceIsRunningException as error:
- raise ConcurrentErrorException(
- "Another instance of the system is running"
- ) from error
- except (SSHConnectionException, ConnectionException) as error:
- raise BackendConnectionException(str(error)) from error
- except ConfigurationException as error:
- raise BackendConfigurationException(str(error)) from error
- except (CommandFailedException, Exception) as error:
- raise BackendException(
- "Execution failed. Please check output for the details."
- ) from error
-
- return wrapper
-
-
-def middleware_signal_handler(func: Callable) -> Callable:
- """Handle signals decorator."""
-
- @wraps(func)
- def wrapper(*args: Any, **kwargs: Any) -> Any:
- # Set up signal handlers for SIGINT (ctrl-c) and SIGTERM (kill command)
- # The handler ignores further signals and it raises an exception
- signal_handler(SIGINT, raise_exception_at_signal)
- signal_handler(SIGTERM, raise_exception_at_signal)
-
- return func(*args, **kwargs)
-
- return wrapper
-
-
-def set_format(ctx: Context, format_: str) -> None:
- """Save format in click context."""
- ctx_obj = ctx.ensure_object(dict)
- ctx_obj["format"] = format_
-
-
-def get_format(ctx: Context) -> str:
- """Get format from click context."""
- ctx_obj = cast(Dict[str, str], ctx.ensure_object(dict))
- return ctx_obj["format"]
diff --git a/src/aiet/cli/completion.py b/src/aiet/cli/completion.py
deleted file mode 100644
index 71f054f..0000000
--- a/src/aiet/cli/completion.py
+++ /dev/null
@@ -1,72 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""
-Add auto completion to different shells with these helpers.
-
-See: https://click.palletsprojects.com/en/8.0.x/shell-completion/
-"""
-import click
-
-
-def _get_package_name() -> str:
- return __name__.split(".", maxsplit=1)[0]
-
-
-# aiet completion bash
-@click.group(name="completion")
-def completion_cmd() -> None:
- """Enable auto completion for your shell."""
-
-
-@completion_cmd.command(name="bash")
-def bash_cmd() -> None:
- """
- Enable auto completion for bash.
-
- Use this command to activate completion in the current bash:
-
- eval "`aiet completion bash`"
-
- Use this command to add auto completion to bash globally, if you have aiet
- installed globally (requires starting a new shell afterwards):
-
- aiet completion bash >> ~/.bashrc
- """
- package_name = _get_package_name()
- print(f'eval "$(_{package_name.upper()}_COMPLETE=bash_source {package_name})"')
-
-
-@completion_cmd.command(name="zsh")
-def zsh_cmd() -> None:
- """
- Enable auto completion for zsh.
-
- Use this command to activate completion in the current zsh:
-
- eval "`aiet completion zsh`"
-
- Use this command to add auto completion to zsh globally, if you have aiet
- installed globally (requires starting a new shell afterwards):
-
- aiet completion zsh >> ~/.zshrc
- """
- package_name = _get_package_name()
- print(f'eval "$(_{package_name.upper()}_COMPLETE=zsh_source {package_name})"')
-
-
-@completion_cmd.command(name="fish")
-def fish_cmd() -> None:
- """
- Enable auto completion for fish.
-
- Use this command to activate completion in the current fish:
-
- eval "`aiet completion fish`"
-
- Use this command to add auto completion to fish globally, if you have aiet
- installed globally (requires starting a new shell afterwards):
-
- aiet completion fish >> ~/.config/fish/completions/aiet.fish
- """
- package_name = _get_package_name()
- print(f'eval "(env _{package_name.upper()}_COMPLETE=fish_source {package_name})"')
diff --git a/src/aiet/cli/system.py b/src/aiet/cli/system.py
deleted file mode 100644
index f1f7637..0000000
--- a/src/aiet/cli/system.py
+++ /dev/null
@@ -1,122 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Module to manage the CLI interface of systems."""
-import json
-from pathlib import Path
-from typing import cast
-
-import click
-
-from aiet.backend.application import get_available_applications
-from aiet.backend.system import get_available_systems
-from aiet.backend.system import get_available_systems_directory_names
-from aiet.backend.system import get_system
-from aiet.backend.system import install_system
-from aiet.backend.system import remove_system
-from aiet.backend.system import System
-from aiet.cli.common import get_format
-from aiet.cli.common import print_command_details
-from aiet.cli.common import set_format
-
-
-@click.group(name="system")
-@click.option(
- "-f",
- "--format",
- "format_",
- type=click.Choice(["cli", "json"]),
- default="cli",
- show_default=True,
-)
-@click.pass_context
-def system_cmd(ctx: click.Context, format_: str) -> None:
- """Sub command to manage systems."""
- set_format(ctx, format_)
-
-
-@system_cmd.command(name="list")
-@click.pass_context
-def list_cmd(ctx: click.Context) -> None:
- """List all available systems."""
- available_systems = get_available_systems()
- system_names = [system.name for system in available_systems]
- if get_format(ctx) == "json":
- data = {"type": "system", "available": system_names}
- print(json.dumps(data))
- else:
- print("Available systems:\n")
- print(*system_names, sep="\n")
-
-
-@system_cmd.command(name="details")
-@click.option(
- "-n",
- "--name",
- "system_name",
- type=click.Choice([s.name for s in get_available_systems()]),
- required=True,
-)
-@click.pass_context
-def details_cmd(ctx: click.Context, system_name: str) -> None:
- """Details of a specific system."""
- system = cast(System, get_system(system_name))
- applications = [
- s.name for s in get_available_applications() if s.can_run_on(system.name)
- ]
- system_details = system.get_details()
- if get_format(ctx) == "json":
- system_details["available_application"] = applications
- print(json.dumps(system_details))
- else:
- system_details_template = (
- 'System "{name}" details\n'
- "Description: {description}\n"
- "Data Transfer Protocol: {protocol}\n"
- "Available Applications: {available_application}"
- )
- print(
- system_details_template.format(
- name=system_details["name"],
- description=system_details["description"],
- protocol=system_details["data_transfer_protocol"],
- available_application=", ".join(applications),
- )
- )
-
- if system_details["annotations"]:
- print("Annotations:")
- for ann_name, ann_value in system_details["annotations"].items():
- print("\t{}: {}".format(ann_name, ann_value))
-
- command_details = system_details["commands"]
- for command, details in command_details.items():
- print("\n{} commands:".format(command))
- print_command_details(details)
-
-
-@system_cmd.command(name="install")
-@click.option(
- "-s",
- "--source",
- "source",
- required=True,
- help="Path to the directory or archive with system definition",
-)
-def install_cmd(source: str) -> None:
- """Install new system."""
- source_path = Path(source)
- install_system(source_path)
-
-
-@system_cmd.command(name="remove")
-@click.option(
- "-d",
- "--directory_name",
- "directory_name",
- type=click.Choice(get_available_systems_directory_names()),
- required=True,
- help="Name of the directory with system",
-)
-def remove_cmd(directory_name: str) -> None:
- """Remove system by given name."""
- remove_system(directory_name)
diff --git a/src/aiet/cli/tool.py b/src/aiet/cli/tool.py
deleted file mode 100644
index 2c80821..0000000
--- a/src/aiet/cli/tool.py
+++ /dev/null
@@ -1,143 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Module to manage the CLI interface of tools."""
-import json
-from typing import Any
-from typing import List
-from typing import Optional
-
-import click
-
-from aiet.backend.execution import execute_tool_command
-from aiet.backend.tool import get_tool
-from aiet.backend.tool import get_unique_tool_names
-from aiet.cli.common import get_format
-from aiet.cli.common import middleware_exception_handler
-from aiet.cli.common import middleware_signal_handler
-from aiet.cli.common import print_command_details
-from aiet.cli.common import set_format
-
-
-@click.group(name="tool")
-@click.option(
- "-f",
- "--format",
- "format_",
- type=click.Choice(["cli", "json"]),
- default="cli",
- show_default=True,
-)
-@click.pass_context
-def tool_cmd(ctx: click.Context, format_: str) -> None:
- """Sub command to manage tools."""
- set_format(ctx, format_)
-
-
-@tool_cmd.command(name="list")
-@click.pass_context
-def list_cmd(ctx: click.Context) -> None:
- """List all available tools."""
- # raise NotImplementedError("TODO")
- tool_names = get_unique_tool_names()
- tool_names.sort()
- if get_format(ctx) == "json":
- data = {"type": "tool", "available": tool_names}
- print(json.dumps(data))
- else:
- print("Available tools:\n")
- print(*tool_names, sep="\n")
-
-
-def validate_system(
- ctx: click.Context,
- _: click.Parameter, # param is not used
- value: Any,
-) -> Any:
- """Validate provided system name depending on the the tool name."""
- tool_name = ctx.params["tool_name"]
- tools = get_tool(tool_name, value)
- if not tools:
- supported_systems = [tool.supported_systems[0] for tool in get_tool(tool_name)]
- raise click.BadParameter(
- message="'{}' is not one of {}.".format(
- value,
- ", ".join("'{}'".format(system) for system in supported_systems),
- ),
- ctx=ctx,
- )
- return value
-
-
-@tool_cmd.command(name="details")
-@click.option(
- "-n",
- "--name",
- "tool_name",
- type=click.Choice(get_unique_tool_names()),
- required=True,
-)
-@click.option(
- "-s",
- "--system",
- "system_name",
- callback=validate_system,
- required=False,
-)
-@click.pass_context
-@middleware_signal_handler
-@middleware_exception_handler
-def details_cmd(ctx: click.Context, tool_name: str, system_name: Optional[str]) -> None:
- """Details of a specific tool."""
- tools = get_tool(tool_name, system_name)
- if get_format(ctx) == "json":
- tools_details = [s.get_details() for s in tools]
- print(json.dumps(tools_details))
- else:
- for tool in tools:
- tool_details = tool.get_details()
- tool_details_template = 'Tool "{name}" details\nDescription: {description}'
-
- print(
- tool_details_template.format(
- name=tool_details["name"],
- description=tool_details["description"],
- )
- )
-
- print(
- "\nSupported systems: {}".format(
- ", ".join(tool_details["supported_systems"])
- )
- )
-
- command_details = tool_details["commands"]
-
- for command, details in command_details.items():
- print("\n{} commands:".format(command))
- print_command_details(details)
-
-
-# pylint: disable=too-many-arguments
-@tool_cmd.command(name="execute")
-@click.option(
- "-n",
- "--name",
- "tool_name",
- type=click.Choice(get_unique_tool_names()),
- required=True,
-)
-@click.option("-p", "--param", "tool_params", multiple=True)
-@click.option(
- "-s",
- "--system",
- "system_name",
- callback=validate_system,
- required=False,
-)
-@middleware_signal_handler
-@middleware_exception_handler
-def execute_cmd(
- tool_name: str, tool_params: List[str], system_name: Optional[str]
-) -> None:
- """Execute tool commands."""
- execute_tool_command(tool_name, tool_params, system_name)
diff --git a/src/aiet/main.py b/src/aiet/main.py
deleted file mode 100644
index 6898ad9..0000000
--- a/src/aiet/main.py
+++ /dev/null
@@ -1,13 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Entry point module of AIET."""
-from aiet.cli import cli
-
-
-def main() -> None:
- """Entry point of aiet application."""
- cli() # pylint: disable=no-value-for-parameter
-
-
-if __name__ == "__main__":
- main()
diff --git a/src/aiet/resources/tools/vela/aiet-config.json b/src/aiet/resources/tools/vela/aiet-config.json
deleted file mode 100644
index c12f291..0000000
--- a/src/aiet/resources/tools/vela/aiet-config.json
+++ /dev/null
@@ -1,73 +0,0 @@
-[
- {
- "name": "vela",
- "description": "Neural network model compiler for Arm Ethos-U NPUs",
- "supported_systems": [
- {
- "name": "Corstone-300: Cortex-M55+Ethos-U55"
- },
- {
- "name": "Corstone-310: Cortex-M85+Ethos-U55"
- },
- {
- "name": "Corstone-300: Cortex-M55+Ethos-U65",
- "variables": {
- "accelerator_config_prefix": "ethos-u65",
- "system_config": "Ethos_U65_High_End",
- "shared_sram": "U65_Shared_Sram"
- },
- "user_params": {
- "run": [
- {
- "description": "MACs per cycle",
- "values": [
- "256",
- "512"
- ],
- "default_value": "512",
- "alias": "mac"
- }
- ]
- }
- }
- ],
- "variables": {
- "accelerator_config_prefix": "ethos-u55",
- "system_config": "Ethos_U55_High_End_Embedded",
- "shared_sram": "U55_Shared_Sram"
- },
- "commands": {
- "run": [
- "run_vela {user_params:input} {user_params:output} --config {tool.config_dir}/vela.ini --accelerator-config {variables:accelerator_config_prefix}-{user_params:mac} --system-config {variables:system_config} --memory-mode {variables:shared_sram} --optimise Performance"
- ]
- },
- "user_params": {
- "run": [
- {
- "description": "MACs per cycle",
- "values": [
- "32",
- "64",
- "128",
- "256"
- ],
- "default_value": "128",
- "alias": "mac"
- },
- {
- "name": "--input-model",
- "description": "Path to the TFLite model",
- "values": [],
- "alias": "input"
- },
- {
- "name": "--output-model",
- "description": "Path to the output model file of the vela-optimisation step. The vela output is saved in the parent directory.",
- "values": [],
- "default_value": "output_model.tflite",
- "alias": "output"
- }
- ]
- }
- }
-]
diff --git a/src/aiet/resources/tools/vela/check_model.py b/src/aiet/resources/tools/vela/check_model.py
deleted file mode 100644
index 7c700b1..0000000
--- a/src/aiet/resources/tools/vela/check_model.py
+++ /dev/null
@@ -1,75 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2020, 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Check if a TFLite model file is Vela-optimised."""
-import struct
-from pathlib import Path
-
-from ethosu.vela.tflite.Model import Model
-
-from aiet.cli.common import InvalidTFLiteFileError
-from aiet.cli.common import ModelOptimisedException
-from aiet.utils.fs import read_file_as_bytearray
-
-
-def get_model_from_file(input_model_file: Path) -> Model:
- """Generate Model instance from TFLite file using flatc generated code."""
- buffer = read_file_as_bytearray(input_model_file)
- try:
- model = Model.GetRootAsModel(buffer, 0)
- except (TypeError, RuntimeError, struct.error) as tflite_error:
- raise InvalidTFLiteFileError(
- f"Error reading in model from {input_model_file}."
- ) from tflite_error
- return model
-
-
-def is_vela_optimised(tflite_model: Model) -> bool:
- """Return True if 'ethos-u' custom operator found in the Model."""
- operators = get_operators_from_model(tflite_model)
-
- custom_codes = get_custom_codes_from_operators(operators)
-
- return check_custom_codes_for_ethosu(custom_codes)
-
-
-def get_operators_from_model(tflite_model: Model) -> list:
- """Return list of the unique operator codes used in the Model."""
- return [
- tflite_model.OperatorCodes(index)
- for index in range(tflite_model.OperatorCodesLength())
- ]
-
-
-def get_custom_codes_from_operators(operators: list) -> list:
- """Return list of each operator's CustomCode() strings, if they exist."""
- return [
- operator.CustomCode()
- for operator in operators
- if operator.CustomCode() is not None
- ]
-
-
-def check_custom_codes_for_ethosu(custom_codes: list) -> bool:
- """Check for existence of ethos-u string in the custom codes."""
- return any(
- custom_code_name.decode("utf-8") == "ethos-u"
- for custom_code_name in custom_codes
- )
-
-
-def check_model(tflite_file_name: str) -> None:
- """Raise an exception if model in given file is Vela optimised."""
- tflite_path = Path(tflite_file_name)
-
- tflite_model = get_model_from_file(tflite_path)
-
- if is_vela_optimised(tflite_model):
- raise ModelOptimisedException(
- f"TFLite model in {tflite_file_name} is already "
- f"vela optimised ('ethos-u' custom op detected)."
- )
-
- print(
- f"TFLite model in {tflite_file_name} is not vela optimised "
- f"('ethos-u' custom op not detected)."
- )
diff --git a/src/aiet/resources/tools/vela/run_vela.py b/src/aiet/resources/tools/vela/run_vela.py
deleted file mode 100644
index 2c1b0be..0000000
--- a/src/aiet/resources/tools/vela/run_vela.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Wrapper to only run Vela when the input is not already optimised."""
-import shutil
-import subprocess
-from pathlib import Path
-from typing import Tuple
-
-import click
-
-from aiet.cli.common import ModelOptimisedException
-from aiet.resources.tools.vela.check_model import check_model
-
-
-def vela_output_model_path(input_model: str, output_dir: str) -> Path:
- """Construct the path to the Vela output file."""
- in_path = Path(input_model)
- tflite_vela = Path(output_dir) / f"{in_path.stem}_vela{in_path.suffix}"
- return tflite_vela
-
-
-def execute_vela(vela_args: Tuple, output_dir: Path, input_model: str) -> None:
- """Execute vela as external call."""
- cmd = ["vela"] + list(vela_args)
- cmd += ["--output-dir", str(output_dir)] # Re-add parsed out_dir to arguments
- cmd += [input_model]
- subprocess.run(cmd, check=True)
-
-
-@click.command(context_settings=dict(ignore_unknown_options=True))
-@click.option(
- "--input-model",
- "-i",
- type=click.Path(exists=True, file_okay=True, readable=True),
- required=True,
-)
-@click.option("--output-model", "-o", type=click.Path(), required=True)
-# Collect the remaining arguments to be directly forwarded to Vela
-@click.argument("vela-args", nargs=-1, type=click.UNPROCESSED)
-def run_vela(input_model: str, output_model: str, vela_args: Tuple) -> None:
- """Check input, run Vela (if needed) and copy optimised file to destination."""
- output_dir = Path(output_model).parent
- try:
- check_model(input_model) # raises an exception if already Vela-optimised
- execute_vela(vela_args, output_dir, input_model)
- print("Vela optimisation complete.")
- src_model = vela_output_model_path(input_model, str(output_dir))
- except ModelOptimisedException as ex:
- # Input already optimized: copy input file to destination path and return
- print(f"Input already vela-optimised.\n{ex}")
- src_model = Path(input_model)
- except subprocess.CalledProcessError as ex:
- print(ex)
- raise SystemExit(ex.returncode) from ex
-
- try:
- shutil.copyfile(src_model, output_model)
- except (shutil.SameFileError, OSError) as ex:
- print(ex)
- raise SystemExit(ex.errno) from ex
-
-
-def main() -> None:
- """Entry point of check_model application."""
- run_vela() # pylint: disable=no-value-for-parameter
diff --git a/src/aiet/resources/tools/vela/vela.ini b/src/aiet/resources/tools/vela/vela.ini
deleted file mode 100644
index 5996553..0000000
--- a/src/aiet/resources/tools/vela/vela.ini
+++ /dev/null
@@ -1,53 +0,0 @@
-; SPDX-FileCopyrightText: Copyright 2021-2022, Arm Limited and/or its affiliates.
-; SPDX-License-Identifier: Apache-2.0
-
-; -----------------------------------------------------------------------------
-; Vela configuration file
-
-; -----------------------------------------------------------------------------
-; System Configuration
-
-; Ethos-U55 High-End Embedded: SRAM (4 GB/s) and Flash (0.5 GB/s)
-[System_Config.Ethos_U55_High_End_Embedded]
-core_clock=500e6
-axi0_port=Sram
-axi1_port=OffChipFlash
-Sram_clock_scale=1.0
-Sram_burst_length=32
-Sram_read_latency=32
-Sram_write_latency=32
-OffChipFlash_clock_scale=0.125
-OffChipFlash_burst_length=128
-OffChipFlash_read_latency=64
-OffChipFlash_write_latency=64
-
-; Ethos-U65 High-End: SRAM (16 GB/s) and DRAM (3.75 GB/s)
-[System_Config.Ethos_U65_High_End]
-core_clock=1e9
-axi0_port=Sram
-axi1_port=Dram
-Sram_clock_scale=1.0
-Sram_burst_length=32
-Sram_read_latency=32
-Sram_write_latency=32
-Dram_clock_scale=0.234375
-Dram_burst_length=128
-Dram_read_latency=500
-Dram_write_latency=250
-
-; -----------------------------------------------------------------------------
-; Memory Mode
-
-; Shared SRAM: the SRAM is shared between the Ethos-U and the Cortex-M software
-; The non-SRAM memory is assumed to be read-only
-[Memory_Mode.U55_Shared_Sram]
-const_mem_area=Axi1
-arena_mem_area=Axi0
-cache_mem_area=Axi0
-arena_cache_size=4194304
-
-[Memory_Mode.U65_Shared_Sram]
-const_mem_area=Axi1
-arena_mem_area=Axi0
-cache_mem_area=Axi0
-arena_cache_size=2097152
diff --git a/src/aiet/utils/__init__.py b/src/aiet/utils/__init__.py
deleted file mode 100644
index fc7ef7c..0000000
--- a/src/aiet/utils/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""This module contains all utils shared across aiet project."""
diff --git a/src/aiet/utils/helpers.py b/src/aiet/utils/helpers.py
deleted file mode 100644
index 6d3cd22..0000000
--- a/src/aiet/utils/helpers.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Helpers functions."""
-import logging
-from typing import Any
-
-
-def set_verbosity(
- ctx: Any, option: Any, verbosity: Any # pylint: disable=unused-argument
-) -> None:
- """Set the logging level according to the verbosity."""
- # Unused arguments must be present here in definition as these are required in
- # function definition when set as a callback
- if verbosity == 1:
- logging.getLogger().setLevel(logging.INFO)
- elif verbosity > 1:
- logging.getLogger().setLevel(logging.DEBUG)
diff --git a/src/aiet/backend/__init__.py b/src/mlia/backend/__init__.py
index 3d60372..3d60372 100644
--- a/src/aiet/backend/__init__.py
+++ b/src/mlia/backend/__init__.py
diff --git a/src/aiet/backend/application.py b/src/mlia/backend/application.py
index f6ef815..eb85212 100644
--- a/src/aiet/backend/application.py
+++ b/src/mlia/backend/application.py
@@ -9,19 +9,19 @@ from typing import Dict
from typing import List
from typing import Optional
-from aiet.backend.common import Backend
-from aiet.backend.common import ConfigurationException
-from aiet.backend.common import DataPaths
-from aiet.backend.common import get_backend_configs
-from aiet.backend.common import get_backend_directories
-from aiet.backend.common import load_application_or_tool_configs
-from aiet.backend.common import load_config
-from aiet.backend.common import remove_backend
-from aiet.backend.config import ApplicationConfig
-from aiet.backend.config import ExtendedApplicationConfig
-from aiet.backend.source import create_destination_and_install
-from aiet.backend.source import get_source
-from aiet.utils.fs import get_resources
+from mlia.backend.common import Backend
+from mlia.backend.common import ConfigurationException
+from mlia.backend.common import DataPaths
+from mlia.backend.common import get_backend_configs
+from mlia.backend.common import get_backend_directories
+from mlia.backend.common import load_application_or_tool_configs
+from mlia.backend.common import load_config
+from mlia.backend.common import remove_backend
+from mlia.backend.config import ApplicationConfig
+from mlia.backend.config import ExtendedApplicationConfig
+from mlia.backend.fs import get_backends_path
+from mlia.backend.source import create_destination_and_install
+from mlia.backend.source import get_source
def get_available_application_directory_names() -> List[str]:
@@ -78,7 +78,7 @@ def install_application(source_path: Path) -> None:
"Applications [{}] are already installed".format(",".join(names))
)
- create_destination_and_install(source, get_resources("applications"))
+ create_destination_and_install(source, get_backends_path("applications"))
def remove_application(directory_name: str) -> None:
diff --git a/src/aiet/backend/common.py b/src/mlia/backend/common.py
index b887ee7..2bbb9d3 100644
--- a/src/aiet/backend/common.py
+++ b/src/mlia/backend/common.py
@@ -23,17 +23,17 @@ from typing import Tuple
from typing import Type
from typing import Union
-from aiet.backend.config import BackendConfig
-from aiet.backend.config import BaseBackendConfig
-from aiet.backend.config import NamedExecutionConfig
-from aiet.backend.config import UserParamConfig
-from aiet.backend.config import UserParamsConfig
-from aiet.utils.fs import get_resources
-from aiet.utils.fs import remove_resource
-from aiet.utils.fs import ResourceType
+from mlia.backend.config import BackendConfig
+from mlia.backend.config import BaseBackendConfig
+from mlia.backend.config import NamedExecutionConfig
+from mlia.backend.config import UserParamConfig
+from mlia.backend.config import UserParamsConfig
+from mlia.backend.fs import get_backends_path
+from mlia.backend.fs import remove_resource
+from mlia.backend.fs import ResourceType
-AIET_CONFIG_FILE: Final[str] = "aiet-config.json"
+BACKEND_CONFIG_FILE: Final[str] = "aiet-config.json"
class ConfigurationException(Exception):
@@ -42,7 +42,7 @@ class ConfigurationException(Exception):
def get_backend_config(dir_path: Path) -> Path:
"""Get path to backendir configuration file."""
- return dir_path / AIET_CONFIG_FILE
+ return dir_path / BACKEND_CONFIG_FILE
def get_backend_configs(resource_type: ResourceType) -> Iterable[Path]:
@@ -56,7 +56,7 @@ def get_backend_directories(resource_type: ResourceType) -> Iterable[Path]:
"""Get path to the backend directories for provided resource_type."""
return (
entry
- for entry in get_resources(resource_type).iterdir()
+ for entry in get_backends_path(resource_type).iterdir()
if is_backend_directory(entry)
)
diff --git a/src/aiet/backend/config.py b/src/mlia/backend/config.py
index dd42012..657adef 100644
--- a/src/aiet/backend/config.py
+++ b/src/mlia/backend/config.py
@@ -89,19 +89,5 @@ class SystemConfig(BaseBackendConfig, total=False):
reporting: Dict[str, Dict]
-class ToolConfig(BaseBackendConfig, total=False):
- """Tool configuration."""
-
- supported_systems: List[str]
-
-
-class ExtendedToolConfig(BaseBackendConfig, total=False):
- """Extended tool configuration."""
-
- supported_systems: List[NamedExecutionConfig]
-
-
-BackendItemConfig = Union[ApplicationConfig, SystemConfig, ToolConfig]
-BackendConfig = Union[
- List[ExtendedApplicationConfig], List[SystemConfig], List[ToolConfig]
-]
+BackendItemConfig = Union[ApplicationConfig, SystemConfig]
+BackendConfig = Union[List[ExtendedApplicationConfig], List[SystemConfig]]
diff --git a/src/aiet/backend/controller.py b/src/mlia/backend/controller.py
index 2650902..f1b68a9 100644
--- a/src/aiet/backend/controller.py
+++ b/src/mlia/backend/controller.py
@@ -10,14 +10,14 @@ from typing import Tuple
import psutil
import sh
-from aiet.backend.common import ConfigurationException
-from aiet.utils.fs import read_file_as_string
-from aiet.utils.proc import execute_command
-from aiet.utils.proc import get_stdout_stderr_paths
-from aiet.utils.proc import read_process_info
-from aiet.utils.proc import save_process_info
-from aiet.utils.proc import terminate_command
-from aiet.utils.proc import terminate_external_process
+from mlia.backend.common import ConfigurationException
+from mlia.backend.fs import read_file_as_string
+from mlia.backend.proc import execute_command
+from mlia.backend.proc import get_stdout_stderr_paths
+from mlia.backend.proc import read_process_info
+from mlia.backend.proc import save_process_info
+from mlia.backend.proc import terminate_command
+from mlia.backend.proc import terminate_external_process
class SystemController:
diff --git a/src/aiet/backend/execution.py b/src/mlia/backend/execution.py
index 1653ee2..749ccdb 100644
--- a/src/aiet/backend/execution.py
+++ b/src/mlia/backend/execution.py
@@ -8,7 +8,6 @@ import re
import string
import sys
import time
-import warnings
from collections import defaultdict
from contextlib import contextmanager
from contextlib import ExitStack
@@ -24,32 +23,29 @@ from typing import Optional
from typing import Sequence
from typing import Tuple
from typing import TypedDict
-from typing import Union
from filelock import FileLock
from filelock import Timeout
-from aiet.backend.application import Application
-from aiet.backend.application import get_application
-from aiet.backend.common import Backend
-from aiet.backend.common import ConfigurationException
-from aiet.backend.common import DataPaths
-from aiet.backend.common import Param
-from aiet.backend.common import parse_raw_parameter
-from aiet.backend.common import resolve_all_parameters
-from aiet.backend.output_parser import Base64OutputParser
-from aiet.backend.output_parser import OutputParser
-from aiet.backend.output_parser import RegexOutputParser
-from aiet.backend.system import ControlledSystem
-from aiet.backend.system import get_system
-from aiet.backend.system import StandaloneSystem
-from aiet.backend.system import System
-from aiet.backend.tool import get_tool
-from aiet.backend.tool import Tool
-from aiet.utils.fs import recreate_directory
-from aiet.utils.fs import remove_directory
-from aiet.utils.fs import valid_for_filename
-from aiet.utils.proc import run_and_wait
+from mlia.backend.application import Application
+from mlia.backend.application import get_application
+from mlia.backend.common import Backend
+from mlia.backend.common import ConfigurationException
+from mlia.backend.common import DataPaths
+from mlia.backend.common import Param
+from mlia.backend.common import parse_raw_parameter
+from mlia.backend.common import resolve_all_parameters
+from mlia.backend.fs import recreate_directory
+from mlia.backend.fs import remove_directory
+from mlia.backend.fs import valid_for_filename
+from mlia.backend.output_parser import Base64OutputParser
+from mlia.backend.output_parser import OutputParser
+from mlia.backend.output_parser import RegexOutputParser
+from mlia.backend.proc import run_and_wait
+from mlia.backend.system import ControlledSystem
+from mlia.backend.system import get_system
+from mlia.backend.system import StandaloneSystem
+from mlia.backend.system import System
class AnotherInstanceIsRunningException(Exception):
@@ -73,7 +69,7 @@ class ExecutionContext:
# pylint: disable=too-many-arguments,too-many-instance-attributes
def __init__(
self,
- app: Union[Application, Tool],
+ app: Application,
app_params: List[str],
system: Optional[System],
system_params: List[str],
@@ -106,14 +102,13 @@ class ExecutionContext:
self.param_resolver = ParamResolver(self)
self._resolved_build_dir: Optional[Path] = None
+ self.stdout: Optional[bytearray] = None
+ self.stderr: Optional[bytearray] = None
+
@property
def is_deploy_needed(self) -> bool:
"""Check if application requires data deployment."""
- if isinstance(self.app, Application):
- return (
- len(self.app.get_deploy_data()) > 0 or len(self.custom_deploy_data) > 0
- )
- return False
+ return len(self.app.get_deploy_data()) > 0 or len(self.custom_deploy_data) > 0
@property
def is_locking_required(self) -> bool:
@@ -533,45 +528,6 @@ def get_application_and_system(
return application, system
-def execute_application_command( # pylint: disable=too-many-arguments
- command_name: str,
- application_name: str,
- application_params: List[str],
- system_name: str,
- system_params: List[str],
- custom_deploy_data: List[DataPaths],
-) -> None:
- """Execute application command.
-
- .. deprecated:: 21.12
- """
- warnings.warn(
- "Use 'run_application()' instead. Use of 'execute_application_command()' is "
- "deprecated and might be removed in a future release.",
- DeprecationWarning,
- )
-
- if command_name not in ["build", "run"]:
- raise ConfigurationException("Unsupported command {}".format(command_name))
-
- application, system = get_application_and_system(application_name, system_name)
- validate_parameters(application, [command_name], application_params)
- validate_parameters(system, [command_name], system_params)
-
- ctx = ExecutionContext(
- app=application,
- app_params=application_params,
- system=system,
- system_params=system_params,
- custom_deploy_data=custom_deploy_data,
- )
-
- if command_name == "run":
- execute_application_command_run(ctx)
- else:
- execute_application_command_build(ctx)
-
-
# pylint: disable=too-many-arguments
def run_application(
application_name: str,
@@ -580,7 +536,7 @@ def run_application(
system_params: List[str],
custom_deploy_data: List[DataPaths],
report_file: Optional[Path] = None,
-) -> None:
+) -> ExecutionContext:
"""Run application on the provided system."""
application, system = get_application_and_system(application_name, system_name)
validate_parameters(application, ["build", "run"], application_params)
@@ -607,6 +563,8 @@ def run_application(
execute_application_command_run(ctx)
+ return ctx
+
def execute_application_command_build(ctx: ExecutionContext) -> None:
"""Execute application command 'build'."""
@@ -655,17 +613,14 @@ def execute_application_command_run(ctx: ExecutionContext) -> None:
for command in commands_to_run:
print("Running: {}".format(command))
- exit_code, std_output, std_err = ctx.system.run(command)
+ exit_code, ctx.stdout, ctx.stderr = ctx.system.run(command)
if exit_code != 0:
print("Application exited with exit code {}".format(exit_code))
if ctx.reporter:
- ctx.reporter.parse(std_output)
- std_output = ctx.reporter.get_filtered_output(std_output)
-
- print(std_output.decode("utf8"), end="")
- print(std_err.decode("utf8"), end="")
+ ctx.reporter.parse(ctx.stdout)
+ ctx.stdout = ctx.reporter.get_filtered_output(ctx.stdout)
if ctx.reporter:
report = ctx.reporter.report(ctx)
@@ -717,12 +672,10 @@ def wait(interval: float) -> None:
def deploy_data(ctx: ExecutionContext) -> None:
"""Deploy data to the system."""
- if isinstance(ctx.app, Application):
- # Only application can deploy data (tools can not)
- assert ctx.system is not None, "System is required."
- for item in itertools.chain(ctx.app.get_deploy_data(), ctx.custom_deploy_data):
- print("Deploying {} onto {}".format(item.src, item.dst))
- ctx.system.deploy(item.src, item.dst)
+ assert ctx.system is not None, "System is required."
+ for item in itertools.chain(ctx.app.get_deploy_data(), ctx.custom_deploy_data):
+ print("Deploying {} onto {}".format(item.src, item.dst))
+ ctx.system.deploy(item.src, item.dst)
def build_run_commands(ctx: ExecutionContext) -> List[str]:
@@ -824,36 +777,3 @@ def get_context_managers(
managers.append(controlled_system_manager)
return managers
-
-
-def get_tool_by_system(tool_name: str, system_name: Optional[str]) -> Tool:
- """Return tool (optionally by provided system name."""
- tools = get_tool(tool_name, system_name)
- if not tools:
- raise ConfigurationException(
- "Tool '{}' not found or doesn't support the system '{}'".format(
- tool_name, system_name
- )
- )
- if len(tools) != 1:
- raise ConfigurationException(
- "Please specify the system for tool {}.".format(tool_name)
- )
- tool = tools[0]
-
- return tool
-
-
-def execute_tool_command(
- tool_name: str,
- tool_params: List[str],
- system_name: Optional[str] = None,
-) -> None:
- """Execute the tool command locally calling the 'run' command."""
- tool = get_tool_by_system(tool_name, system_name)
- ctx = ExecutionContext(
- app=tool, app_params=tool_params, system=None, system_params=[]
- )
- commands = tool.build_command("run", tool_params, ctx.param_resolver)
-
- execute_commands_locally(commands, Path.cwd())
diff --git a/src/aiet/utils/fs.py b/src/mlia/backend/fs.py
index ea99a69..9979fcb 100644
--- a/src/aiet/utils/fs.py
+++ b/src/mlia/backend/fs.py
@@ -1,7 +1,6 @@
# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
"""Module to host all file system related functions."""
-import importlib.resources as pkg_resources
import re
import shutil
from pathlib import Path
@@ -9,17 +8,17 @@ from typing import Any
from typing import Literal
from typing import Optional
-ResourceType = Literal["applications", "systems", "tools"]
+from mlia.utils.filesystem import get_mlia_resources
+ResourceType = Literal["applications", "systems"]
-def get_aiet_resources() -> Path:
- """Get resources folder path."""
- with pkg_resources.path("aiet", "__init__.py") as init_path:
- project_root = init_path.parent
- return project_root / "resources"
+def get_backend_resources() -> Path:
+ """Get backend resources folder path."""
+ return get_mlia_resources() / "backends"
-def get_resources(name: ResourceType) -> Path:
+
+def get_backends_path(name: ResourceType) -> Path:
"""Return the absolute path of the specified resource.
It uses importlib to return resources packaged with MANIFEST.in.
@@ -27,7 +26,7 @@ def get_resources(name: ResourceType) -> Path:
if not name:
raise ResourceWarning("Resource name is not provided")
- resource_path = get_aiet_resources() / name
+ resource_path = get_backend_resources() / name
if resource_path.is_dir():
return resource_path
@@ -48,7 +47,7 @@ def copy_directory_content(source: Path, destination: Path) -> None:
def remove_resource(resource_directory: str, resource_type: ResourceType) -> None:
"""Remove resource data."""
- resources = get_resources(resource_type)
+ resources = get_backends_path(resource_type)
resource_location = resources / resource_directory
if not resource_location.exists():
diff --git a/src/mlia/tools/aiet_wrapper.py b/src/mlia/backend/manager.py
index 73e82ee..3a1016c 100644
--- a/src/mlia/tools/aiet_wrapper.py
+++ b/src/mlia/backend/manager.py
@@ -1,6 +1,6 @@
# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
-"""Module for AIET integration."""
+"""Module for backend integration."""
import logging
import re
from abc import ABC
@@ -14,11 +14,13 @@ from typing import Literal
from typing import Optional
from typing import Tuple
-from aiet.backend.application import get_available_applications
-from aiet.backend.application import install_application
-from aiet.backend.system import get_available_systems
-from aiet.backend.system import install_system
-from mlia.utils.proc import CommandExecutor
+from mlia.backend.application import get_available_applications
+from mlia.backend.application import install_application
+from mlia.backend.common import DataPaths
+from mlia.backend.execution import ExecutionContext
+from mlia.backend.execution import run_application
+from mlia.backend.system import get_available_systems
+from mlia.backend.system import install_system
from mlia.utils.proc import OutputConsumer
from mlia.utils.proc import RunningCommand
@@ -54,7 +56,7 @@ _SYSTEM_TO_APP_MAP = {
def get_system_name(backend: str, device_type: str) -> str:
- """Get the AIET system name for the given backend and device type."""
+ """Get the system name for the given backend and device type."""
return _SUPPORTED_SYSTEMS[backend][device_type]
@@ -71,7 +73,7 @@ def is_supported(backend: str, device_type: Optional[str] = None) -> bool:
def supported_backends() -> List[str]:
- """Get a list of all backends supported by the AIET wrapper."""
+ """Get a list of all backends supported by the backend manager."""
return list(_SUPPORTED_SYSTEMS.keys())
@@ -129,8 +131,8 @@ class ExecutionParams:
deploy_params: List[str]
-class AIETLogWriter(OutputConsumer):
- """Redirect AIET command output to the logger."""
+class LogWriter(OutputConsumer):
+ """Redirect output to the logger."""
def feed(self, line: str) -> None:
"""Process line from the output."""
@@ -211,12 +213,11 @@ class GenericInferenceOutputParser(OutputConsumer):
return sorted(self.PATTERNS.keys() - self.result.keys())
-class AIETRunner:
- """AIET runner."""
+class BackendRunner:
+ """Backend runner."""
- def __init__(self, executor: CommandExecutor) -> None:
- """Init AIET runner instance."""
- self.executor = executor
+ def __init__(self) -> None:
+ """Init BackendRunner instance."""
@staticmethod
def get_installed_systems() -> List[str]:
@@ -270,39 +271,40 @@ class AIETRunner:
"""Install application."""
install_application(app_path)
- def run_application(self, execution_params: ExecutionParams) -> RunningCommand:
+ @staticmethod
+ def run_application(execution_params: ExecutionParams) -> ExecutionContext:
"""Run requested application."""
- command = [
- "aiet",
- "application",
- "run",
- "-n",
+
+ def to_data_paths(paths: str) -> DataPaths:
+ """Split input into two and create new DataPaths object."""
+ src, dst = paths.split(sep=":", maxsplit=1)
+ return DataPaths(Path(src), dst)
+
+ deploy_data_paths = [
+ to_data_paths(paths) for paths in execution_params.deploy_params
+ ]
+
+ ctx = run_application(
execution_params.application,
- "-s",
+ execution_params.application_params,
execution_params.system,
- *self._params("-p", execution_params.application_params),
- *self._params("--system-param", execution_params.system_params),
- *self._params("--deploy", execution_params.deploy_params),
- ]
+ execution_params.system_params,
+ deploy_data_paths,
+ )
- return self._submit(command)
+ return ctx
@staticmethod
def _params(name: str, params: List[str]) -> List[str]:
return [p for item in [(name, param) for param in params] for p in item]
- def _submit(self, command: List[str]) -> RunningCommand:
- """Submit command for the execution."""
- logger.debug("Submit command %s", " ".join(command))
- return self.executor.submit(command)
-
class GenericInferenceRunner(ABC):
"""Abstract class for generic inference runner."""
- def __init__(self, aiet_runner: AIETRunner):
+ def __init__(self, backend_runner: BackendRunner):
"""Init generic inference runner instance."""
- self.aiet_runner = aiet_runner
+ self.backend_runner = backend_runner
self.running_inference: Optional[RunningCommand] = None
def run(
@@ -311,9 +313,9 @@ class GenericInferenceRunner(ABC):
"""Run generic inference for the provided device/model."""
execution_params = self.get_execution_params(model_info)
- self.running_inference = self.aiet_runner.run_application(execution_params)
- self.running_inference.output_consumers = output_consumers
- self.running_inference.consume_output()
+ ctx = self.backend_runner.run_application(execution_params)
+ if ctx.stdout is not None:
+ self.consume_output(ctx.stdout, output_consumers)
def stop(self) -> None:
"""Stop running inference."""
@@ -336,24 +338,31 @@ class GenericInferenceRunner(ABC):
def check_system_and_application(self, system_name: str, app_name: str) -> None:
"""Check if requested system and application installed."""
- if not self.aiet_runner.is_system_installed(system_name):
+ if not self.backend_runner.is_system_installed(system_name):
raise Exception(f"System {system_name} is not installed")
- if not self.aiet_runner.is_application_installed(app_name, system_name):
+ if not self.backend_runner.is_application_installed(app_name, system_name):
raise Exception(
f"Application {app_name} for the system {system_name} "
"is not installed"
)
+ @staticmethod
+ def consume_output(output: bytearray, consumers: List[OutputConsumer]) -> None:
+ """Pass program's output to the consumers."""
+ for line in output.decode("utf8").splitlines():
+ for consumer in consumers:
+ consumer.feed(line)
+
class GenericInferenceRunnerEthosU(GenericInferenceRunner):
"""Generic inference runner on U55/65."""
def __init__(
- self, aiet_runner: AIETRunner, device_info: DeviceInfo, backend: str
+ self, backend_runner: BackendRunner, device_info: DeviceInfo, backend: str
) -> None:
"""Init generic inference runner instance."""
- super().__init__(aiet_runner)
+ super().__init__(backend_runner)
system_name, app_name = self.resolve_system_and_app(device_info, backend)
self.system_name = system_name
@@ -405,8 +414,8 @@ class GenericInferenceRunnerEthosU(GenericInferenceRunner):
def get_generic_runner(device_info: DeviceInfo, backend: str) -> GenericInferenceRunner:
"""Get generic runner for provided device and backend."""
- aiet_runner = get_aiet_runner()
- return GenericInferenceRunnerEthosU(aiet_runner, device_info, backend)
+ backend_runner = get_backend_runner()
+ return GenericInferenceRunnerEthosU(backend_runner, device_info, backend)
def estimate_performance(
@@ -415,7 +424,7 @@ def estimate_performance(
"""Get performance estimations."""
with get_generic_runner(device_info, backend) as generic_runner:
output_parser = GenericInferenceOutputParser()
- output_consumers = [output_parser, AIETLogWriter()]
+ output_consumers = [output_parser, LogWriter()]
generic_runner.run(model_info, output_consumers)
@@ -429,7 +438,10 @@ def estimate_performance(
return PerformanceMetrics(**output_parser.result)
-def get_aiet_runner() -> AIETRunner:
- """Return AIET runner."""
- executor = CommandExecutor()
- return AIETRunner(executor)
+def get_backend_runner() -> BackendRunner:
+ """
+ Return BackendRunner instance.
+
+ Note: This is needed for the unit tests.
+ """
+ return BackendRunner()
diff --git a/src/aiet/backend/output_parser.py b/src/mlia/backend/output_parser.py
index 111772a..111772a 100644
--- a/src/aiet/backend/output_parser.py
+++ b/src/mlia/backend/output_parser.py
diff --git a/src/aiet/utils/proc.py b/src/mlia/backend/proc.py
index b6f4357..90ff414 100644
--- a/src/aiet/utils/proc.py
+++ b/src/mlia/backend/proc.py
@@ -24,7 +24,7 @@ from sh import CommandNotFound
from sh import ErrorReturnCode
from sh import RunningCommand
-from aiet.utils.fs import valid_for_filename
+from mlia.backend.fs import valid_for_filename
class CommandFailedException(Exception):
diff --git a/src/aiet/backend/protocol.py b/src/mlia/backend/protocol.py
index c621436..ebfe69a 100644
--- a/src/aiet/backend/protocol.py
+++ b/src/mlia/backend/protocol.py
@@ -14,10 +14,10 @@ from typing import Union
import paramiko
-from aiet.backend.common import ConfigurationException
-from aiet.backend.config import LocalProtocolConfig
-from aiet.backend.config import SSHConfig
-from aiet.utils.proc import run_and_wait
+from mlia.backend.common import ConfigurationException
+from mlia.backend.config import LocalProtocolConfig
+from mlia.backend.config import SSHConfig
+from mlia.backend.proc import run_and_wait
# Redirect all paramiko thread exceptions to a file otherwise these will be
diff --git a/src/aiet/backend/source.py b/src/mlia/backend/source.py
index dec175a..dcf6835 100644
--- a/src/aiet/backend/source.py
+++ b/src/mlia/backend/source.py
@@ -11,13 +11,13 @@ from tarfile import TarFile
from typing import Optional
from typing import Union
-from aiet.backend.common import AIET_CONFIG_FILE
-from aiet.backend.common import ConfigurationException
-from aiet.backend.common import get_backend_config
-from aiet.backend.common import is_backend_directory
-from aiet.backend.common import load_config
-from aiet.backend.config import BackendConfig
-from aiet.utils.fs import copy_directory_content
+from mlia.backend.common import BACKEND_CONFIG_FILE
+from mlia.backend.common import ConfigurationException
+from mlia.backend.common import get_backend_config
+from mlia.backend.common import is_backend_directory
+from mlia.backend.common import load_config
+from mlia.backend.config import BackendConfig
+from mlia.backend.fs import copy_directory_content
class Source(ABC):
@@ -99,7 +99,7 @@ class TarArchiveSource(Source):
with self._open(self.archive_path) as archive:
try:
- config_entry = archive.getmember(AIET_CONFIG_FILE)
+ config_entry = archive.getmember(BACKEND_CONFIG_FILE)
self._has_top_level_folder = False
except KeyError as error_no_config:
try:
@@ -112,7 +112,7 @@ class TarArchiveSource(Source):
"Archive has no top level directory"
) from error_no_config
- config_path = "{}/{}".format(top_level_dir, AIET_CONFIG_FILE)
+ config_path = "{}/{}".format(top_level_dir, BACKEND_CONFIG_FILE)
config_entry = archive.getmember(config_path)
self._has_top_level_folder = True
diff --git a/src/aiet/backend/system.py b/src/mlia/backend/system.py
index 48f1bb1..469083e 100644
--- a/src/aiet/backend/system.py
+++ b/src/mlia/backend/system.py
@@ -10,22 +10,22 @@ from typing import Optional
from typing import Tuple
from typing import Union
-from aiet.backend.common import Backend
-from aiet.backend.common import ConfigurationException
-from aiet.backend.common import get_backend_configs
-from aiet.backend.common import get_backend_directories
-from aiet.backend.common import load_config
-from aiet.backend.common import remove_backend
-from aiet.backend.config import SystemConfig
-from aiet.backend.controller import SystemController
-from aiet.backend.controller import SystemControllerSingleInstance
-from aiet.backend.protocol import ProtocolFactory
-from aiet.backend.protocol import SupportsClose
-from aiet.backend.protocol import SupportsConnection
-from aiet.backend.protocol import SupportsDeploy
-from aiet.backend.source import create_destination_and_install
-from aiet.backend.source import get_source
-from aiet.utils.fs import get_resources
+from mlia.backend.common import Backend
+from mlia.backend.common import ConfigurationException
+from mlia.backend.common import get_backend_configs
+from mlia.backend.common import get_backend_directories
+from mlia.backend.common import load_config
+from mlia.backend.common import remove_backend
+from mlia.backend.config import SystemConfig
+from mlia.backend.controller import SystemController
+from mlia.backend.controller import SystemControllerSingleInstance
+from mlia.backend.fs import get_backends_path
+from mlia.backend.protocol import ProtocolFactory
+from mlia.backend.protocol import SupportsClose
+from mlia.backend.protocol import SupportsConnection
+from mlia.backend.protocol import SupportsDeploy
+from mlia.backend.source import create_destination_and_install
+from mlia.backend.source import get_source
def get_available_systems_directory_names() -> List[str]:
@@ -75,7 +75,7 @@ def install_system(source_path: Path) -> None:
"Systems [{}] are already installed".format(",".join(names))
)
- create_destination_and_install(source, get_resources("systems"))
+ create_destination_and_install(source, get_backends_path("systems"))
def remove_system(directory_name: str) -> None:
diff --git a/src/mlia/cli/config.py b/src/mlia/cli/config.py
index 838b051..a673230 100644
--- a/src/mlia/cli/config.py
+++ b/src/mlia/cli/config.py
@@ -5,7 +5,7 @@ import logging
from functools import lru_cache
from typing import List
-import mlia.tools.aiet_wrapper as aiet
+import mlia.backend.manager as backend_manager
from mlia.tools.metadata.common import DefaultInstallationManager
from mlia.tools.metadata.common import InstallationManager
from mlia.tools.metadata.corstone import get_corstone_installations
@@ -25,12 +25,12 @@ def get_available_backends() -> List[str]:
"""Return list of the available backends."""
available_backends = ["Vela"]
- # Add backends using AIET
+ # Add backends using backend manager
manager = get_installation_manager()
available_backends.extend(
(
backend
- for backend in aiet.supported_backends()
+ for backend in backend_manager.supported_backends()
if manager.backend_installed(backend)
)
)
diff --git a/src/mlia/devices/ethosu/performance.py b/src/mlia/devices/ethosu/performance.py
index b0718a5..a73045a 100644
--- a/src/mlia/devices/ethosu/performance.py
+++ b/src/mlia/devices/ethosu/performance.py
@@ -10,7 +10,7 @@ from typing import Optional
from typing import Tuple
from typing import Union
-import mlia.tools.aiet_wrapper as aiet
+import mlia.backend.manager as backend_manager
import mlia.tools.vela_wrapper as vela
from mlia.core.context import Context
from mlia.core.performance import PerformanceEstimator
@@ -147,15 +147,15 @@ class VelaPerformanceEstimator(
return memory_usage
-class AIETPerformanceEstimator(
+class CorstonePerformanceEstimator(
PerformanceEstimator[Union[Path, ModelConfiguration], NPUCycles]
):
- """AIET based performance estimator."""
+ """Corstone-based performance estimator."""
def __init__(
self, context: Context, device: EthosUConfiguration, backend: str
) -> None:
- """Init AIET based performance estimator."""
+ """Init Corstone-based performance estimator."""
self.context = context
self.device = device
self.backend = backend
@@ -179,24 +179,24 @@ class AIETPerformanceEstimator(
model_path, self.device.compiler_options, optimized_model_path
)
- model_info = aiet.ModelInfo(model_path=optimized_model_path)
- device_info = aiet.DeviceInfo(
+ model_info = backend_manager.ModelInfo(model_path=optimized_model_path)
+ device_info = backend_manager.DeviceInfo(
device_type=self.device.target, # type: ignore
mac=self.device.mac,
memory_mode=self.device.compiler_options.memory_mode, # type: ignore
)
- aiet_perf_metrics = aiet.estimate_performance(
+ corstone_perf_metrics = backend_manager.estimate_performance(
model_info, device_info, self.backend
)
npu_cycles = NPUCycles(
- aiet_perf_metrics.npu_active_cycles,
- aiet_perf_metrics.npu_idle_cycles,
- aiet_perf_metrics.npu_total_cycles,
- aiet_perf_metrics.npu_axi0_rd_data_beat_received,
- aiet_perf_metrics.npu_axi0_wr_data_beat_written,
- aiet_perf_metrics.npu_axi1_rd_data_beat_received,
+ corstone_perf_metrics.npu_active_cycles,
+ corstone_perf_metrics.npu_idle_cycles,
+ corstone_perf_metrics.npu_total_cycles,
+ corstone_perf_metrics.npu_axi0_rd_data_beat_received,
+ corstone_perf_metrics.npu_axi0_wr_data_beat_written,
+ corstone_perf_metrics.npu_axi1_rd_data_beat_received,
)
logger.info("Done\n")
@@ -220,10 +220,11 @@ class EthosUPerformanceEstimator(
if backends is None:
backends = ["Vela"] # Only Vela is always available as default
for backend in backends:
- if backend != "Vela" and not aiet.is_supported(backend):
+ if backend != "Vela" and not backend_manager.is_supported(backend):
raise ValueError(
f"Unsupported backend '{backend}'. "
- f"Only 'Vela' and {aiet.supported_backends()} are supported."
+ f"Only 'Vela' and {backend_manager.supported_backends()} "
+ "are supported."
)
self.backends = set(backends)
@@ -242,11 +243,11 @@ class EthosUPerformanceEstimator(
if backend == "Vela":
vela_estimator = VelaPerformanceEstimator(self.context, self.device)
memory_usage = vela_estimator.estimate(tflite_model)
- elif backend in aiet.supported_backends():
- aiet_estimator = AIETPerformanceEstimator(
+ elif backend in backend_manager.supported_backends():
+ corstone_estimator = CorstonePerformanceEstimator(
self.context, self.device, backend
)
- npu_cycles = aiet_estimator.estimate(tflite_model)
+ npu_cycles = corstone_estimator.estimate(tflite_model)
else:
logger.warning(
"Backend '%s' is not supported for Ethos-U performance "
diff --git a/src/mlia/resources/aiet/applications/APPLICATIONS.txt b/src/mlia/resources/aiet/applications/APPLICATIONS.txt
index 09127f8..a702e19 100644
--- a/src/mlia/resources/aiet/applications/APPLICATIONS.txt
+++ b/src/mlia/resources/aiet/applications/APPLICATIONS.txt
@@ -1,6 +1,7 @@
SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
SPDX-License-Identifier: Apache-2.0
-This directory contains the Generic Inference Runner application packages for AIET
+This directory contains the application packages for the Generic Inference
+Runner.
-Each package should contain its own aiet-config.json file
+Each package should contain its own aiet-config.json file.
diff --git a/src/mlia/resources/aiet/systems/SYSTEMS.txt b/src/mlia/resources/aiet/systems/SYSTEMS.txt
index bc27e73..3861769 100644
--- a/src/mlia/resources/aiet/systems/SYSTEMS.txt
+++ b/src/mlia/resources/aiet/systems/SYSTEMS.txt
@@ -1,8 +1,7 @@
SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
SPDX-License-Identifier: Apache-2.0
-This directory contains the configuration files of the systems for the AIET
-middleware.
+This directory contains the configuration files of the system backends.
Supported systems:
diff --git a/src/aiet/resources/applications/.gitignore b/src/mlia/resources/backends/applications/.gitignore
index 0226166..0226166 100644
--- a/src/aiet/resources/applications/.gitignore
+++ b/src/mlia/resources/backends/applications/.gitignore
diff --git a/src/aiet/resources/systems/.gitignore b/src/mlia/resources/backends/systems/.gitignore
index 0226166..0226166 100644
--- a/src/aiet/resources/systems/.gitignore
+++ b/src/mlia/resources/backends/systems/.gitignore
diff --git a/src/mlia/tools/metadata/corstone.py b/src/mlia/tools/metadata/corstone.py
index 7a9d113..a92f81c 100644
--- a/src/mlia/tools/metadata/corstone.py
+++ b/src/mlia/tools/metadata/corstone.py
@@ -12,7 +12,8 @@ from typing import Iterable
from typing import List
from typing import Optional
-import mlia.tools.aiet_wrapper as aiet
+import mlia.backend.manager as backend_manager
+from mlia.backend.fs import get_backend_resources
from mlia.tools.metadata.common import DownloadAndInstall
from mlia.tools.metadata.common import Installation
from mlia.tools.metadata.common import InstallationType
@@ -41,8 +42,8 @@ PathChecker = Callable[[Path], Optional[BackendInfo]]
BackendInstaller = Callable[[bool, Path], Path]
-class AIETMetadata:
- """AIET installation metadata."""
+class BackendMetadata:
+ """Backend installation metadata."""
def __init__(
self,
@@ -55,7 +56,7 @@ class AIETMetadata:
supported_platforms: Optional[List[str]] = None,
) -> None:
"""
- Initialize AIETMetaData.
+ Initialize BackendMetadata.
Members expected_systems and expected_apps are filled automatically.
"""
@@ -67,15 +68,15 @@ class AIETMetadata:
self.download_artifact = download_artifact
self.supported_platforms = supported_platforms
- self.expected_systems = aiet.get_all_system_names(name)
- self.expected_apps = aiet.get_all_application_names(name)
+ self.expected_systems = backend_manager.get_all_system_names(name)
+ self.expected_apps = backend_manager.get_all_application_names(name)
@property
def expected_resources(self) -> Iterable[Path]:
"""Return list of expected resources."""
resources = [self.system_config, *self.apps_resources]
- return (get_mlia_resources() / resource for resource in resources)
+ return (get_backend_resources() / resource for resource in resources)
@property
def supported_platform(self) -> bool:
@@ -86,49 +87,49 @@ class AIETMetadata:
return platform.system() in self.supported_platforms
-class AIETBasedInstallation(Installation):
- """Backend installation based on AIET functionality."""
+class BackendInstallation(Installation):
+ """Backend installation."""
def __init__(
self,
- aiet_runner: aiet.AIETRunner,
- metadata: AIETMetadata,
+ backend_runner: backend_manager.BackendRunner,
+ metadata: BackendMetadata,
path_checker: PathChecker,
backend_installer: Optional[BackendInstaller],
) -> None:
- """Init the tool installation."""
- self.aiet_runner = aiet_runner
+ """Init the backend installation."""
+ self.backend_runner = backend_runner
self.metadata = metadata
self.path_checker = path_checker
self.backend_installer = backend_installer
@property
def name(self) -> str:
- """Return name of the tool."""
+ """Return name of the backend."""
return self.metadata.name
@property
def description(self) -> str:
- """Return description of the tool."""
+ """Return description of the backend."""
return self.metadata.description
@property
def already_installed(self) -> bool:
- """Return true if tool already installed."""
- return self.aiet_runner.all_installed(
+ """Return true if backend already installed."""
+ return self.backend_runner.all_installed(
self.metadata.expected_systems, self.metadata.expected_apps
)
@property
def could_be_installed(self) -> bool:
- """Return true if tool could be installed."""
+ """Return true if backend could be installed."""
if not self.metadata.supported_platform:
return False
return all_paths_valid(self.metadata.expected_resources)
def supports(self, install_type: InstallationType) -> bool:
- """Return true if tools supported type of the installation."""
+ """Return true if backends supported type of the installation."""
if isinstance(install_type, DownloadAndInstall):
return self.metadata.download_artifact is not None
@@ -138,7 +139,7 @@ class AIETBasedInstallation(Installation):
return False # type: ignore
def install(self, install_type: InstallationType) -> None:
- """Install the tool."""
+ """Install the backend."""
if isinstance(install_type, DownloadAndInstall):
download_artifact = self.metadata.download_artifact
assert download_artifact is not None, "No artifact provided"
@@ -153,7 +154,7 @@ class AIETBasedInstallation(Installation):
raise Exception(f"Unable to install {install_type}")
def install_from(self, backend_info: BackendInfo) -> None:
- """Install tool from the directory."""
+ """Install backend from the directory."""
mlia_resources = get_mlia_resources()
with temp_directory() as tmpdir:
@@ -169,15 +170,15 @@ class AIETBasedInstallation(Installation):
copy_all(*resources_to_copy, dest=fvp_dist_dir)
- self.aiet_runner.install_system(fvp_dist_dir)
+ self.backend_runner.install_system(fvp_dist_dir)
for app in self.metadata.apps_resources:
- self.aiet_runner.install_application(mlia_resources / app)
+ self.backend_runner.install_application(mlia_resources / app)
def download_and_install(
self, download_artifact: DownloadArtifact, eula_agrement: bool
) -> None:
- """Download and install the tool."""
+ """Download and install the backend."""
with temp_directory() as tmpdir:
try:
downloaded_to = download_artifact.download_to(tmpdir)
@@ -307,10 +308,10 @@ class Corstone300Installer:
def get_corstone_300_installation() -> Installation:
"""Get Corstone-300 installation."""
- corstone_300 = AIETBasedInstallation(
- aiet_runner=aiet.get_aiet_runner(),
+ corstone_300 = BackendInstallation(
+ backend_runner=backend_manager.BackendRunner(),
# pylint: disable=line-too-long
- metadata=AIETMetadata(
+ metadata=BackendMetadata(
name="Corstone-300",
description="Corstone-300 FVP",
system_config="aiet/systems/corstone-300/aiet-config.json",
@@ -356,10 +357,10 @@ def get_corstone_300_installation() -> Installation:
def get_corstone_310_installation() -> Installation:
"""Get Corstone-310 installation."""
- corstone_310 = AIETBasedInstallation(
- aiet_runner=aiet.get_aiet_runner(),
+ corstone_310 = BackendInstallation(
+ backend_runner=backend_manager.BackendRunner(),
# pylint: disable=line-too-long
- metadata=AIETMetadata(
+ metadata=BackendMetadata(
name="Corstone-310",
description="Corstone-310 FVP",
system_config="aiet/systems/corstone-310/aiet-config.json",
diff --git a/src/mlia/utils/proc.py b/src/mlia/utils/proc.py
index 39aca43..18a4305 100644
--- a/src/mlia/utils/proc.py
+++ b/src/mlia/utils/proc.py
@@ -8,7 +8,6 @@ import time
from abc import ABC
from abc import abstractmethod
from contextlib import contextmanager
-from contextlib import suppress
from pathlib import Path
from typing import Any
from typing import Generator
@@ -23,7 +22,7 @@ class OutputConsumer(ABC):
@abstractmethod
def feed(self, line: str) -> None:
- """Feed new line to the consumerr."""
+ """Feed new line to the consumer."""
class RunningCommand:
@@ -32,7 +31,7 @@ class RunningCommand:
def __init__(self, process: subprocess.Popen) -> None:
"""Init running command instance."""
self.process = process
- self._output_consumers: Optional[List[OutputConsumer]] = None
+ self.output_consumers: List[OutputConsumer] = []
def is_alive(self) -> bool:
"""Return true if process is still alive."""
@@ -57,25 +56,14 @@ class RunningCommand:
"""Send signal to the process."""
self.process.send_signal(signal_num)
- @property
- def output_consumers(self) -> Optional[List[OutputConsumer]]:
- """Property output_consumers."""
- return self._output_consumers
-
- @output_consumers.setter
- def output_consumers(self, output_consumers: List[OutputConsumer]) -> None:
- """Set output consumers."""
- self._output_consumers = output_consumers
-
def consume_output(self) -> None:
"""Pass program's output to the consumers."""
- if self.process is None or self.output_consumers is None:
+ if self.process is None or not self.output_consumers:
return
for line in self.stdout():
for consumer in self.output_consumers:
- with suppress():
- consumer.feed(line)
+ consumer.feed(line)
def stop(
self, wait: bool = True, num_of_attempts: int = 5, interval: float = 0.5
diff --git a/tests/aiet/__init__.py b/tests/aiet/__init__.py
deleted file mode 100644
index 873a7df..0000000
--- a/tests/aiet/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""AIET tests module."""
diff --git a/tests/aiet/conftest.py b/tests/aiet/conftest.py
deleted file mode 100644
index cab3dc2..0000000
--- a/tests/aiet/conftest.py
+++ /dev/null
@@ -1,139 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-# pylint: disable=redefined-outer-name
-"""conftest for pytest."""
-import shutil
-import tarfile
-from pathlib import Path
-from typing import Any
-
-import pytest
-from click.testing import CliRunner
-
-from aiet.backend.common import get_backend_configs
-
-
-@pytest.fixture(scope="session")
-def test_systems_path(test_resources_path: Path) -> Path:
- """Return test systems path in a pytest fixture."""
- return test_resources_path / "systems"
-
-
-@pytest.fixture(scope="session")
-def test_applications_path(test_resources_path: Path) -> Path:
- """Return test applications path in a pytest fixture."""
- return test_resources_path / "applications"
-
-
-@pytest.fixture(scope="session")
-def test_tools_path(test_resources_path: Path) -> Path:
- """Return test tools path in a pytest fixture."""
- return test_resources_path / "tools"
-
-
-@pytest.fixture(scope="session")
-def test_resources_path() -> Path:
- """Return test resources path in a pytest fixture."""
- current_path = Path(__file__).parent.absolute()
- return current_path / "test_resources"
-
-
-@pytest.fixture(scope="session")
-def non_optimised_input_model_file(test_tflite_model: Path) -> Path:
- """Provide the path to a quantized dummy model file."""
- return test_tflite_model
-
-
-@pytest.fixture(scope="session")
-def optimised_input_model_file(test_tflite_vela_model: Path) -> Path:
- """Provide path to Vela-optimised dummy model file."""
- return test_tflite_vela_model
-
-
-@pytest.fixture(scope="session")
-def invalid_input_model_file(test_tflite_invalid_model: Path) -> Path:
- """Provide the path to an invalid dummy model file."""
- return test_tflite_invalid_model
-
-
-@pytest.fixture(autouse=True)
-def test_resources(monkeypatch: pytest.MonkeyPatch, test_resources_path: Path) -> Any:
- """Force using test resources as middleware's repository."""
-
- def get_test_resources() -> Path:
- """Return path to the test resources."""
- return test_resources_path
-
- monkeypatch.setattr("aiet.utils.fs.get_aiet_resources", get_test_resources)
- yield
-
-
-@pytest.fixture(scope="session", autouse=True)
-def add_tools(test_resources_path: Path) -> Any:
- """Symlink the tools from the original resources path to the test resources path."""
- # tool_dirs = get_available_tool_directory_names()
- tool_dirs = [cfg.parent for cfg in get_backend_configs("tools")]
-
- links = {
- src_dir: (test_resources_path / "tools" / src_dir.name) for src_dir in tool_dirs
- }
- for src_dir, dst_dir in links.items():
- if not dst_dir.exists():
- dst_dir.symlink_to(src_dir, target_is_directory=True)
- yield
- # Remove symlinks
- for dst_dir in links.values():
- if dst_dir.is_symlink():
- dst_dir.unlink()
-
-
-def create_archive(
- archive_name: str, source: Path, destination: Path, with_root_folder: bool = False
-) -> None:
- """Create archive from directory source."""
- with tarfile.open(destination / archive_name, mode="w:gz") as tar:
- for item in source.iterdir():
- item_name = item.name
- if with_root_folder:
- item_name = f"{source.name}/{item_name}"
- tar.add(item, item_name)
-
-
-def process_directory(source: Path, destination: Path) -> None:
- """Process resource directory."""
- destination.mkdir()
-
- for item in source.iterdir():
- if item.is_dir():
- create_archive(f"{item.name}.tar.gz", item, destination)
- create_archive(f"{item.name}_dir.tar.gz", item, destination, True)
-
-
-@pytest.fixture(scope="session", autouse=True)
-def add_archives(
- test_resources_path: Path, tmp_path_factory: pytest.TempPathFactory
-) -> Any:
- """Generate archives of the test resources."""
- tmp_path = tmp_path_factory.mktemp("archives")
-
- archives_path = tmp_path / "archives"
- archives_path.mkdir()
-
- if (archives_path_link := test_resources_path / "archives").is_symlink():
- archives_path.unlink()
-
- archives_path_link.symlink_to(archives_path, target_is_directory=True)
-
- for item in ["applications", "systems"]:
- process_directory(test_resources_path / item, archives_path / item)
-
- yield
-
- archives_path_link.unlink()
- shutil.rmtree(tmp_path)
-
-
-@pytest.fixture(scope="module")
-def cli_runner() -> CliRunner:
- """Return CliRunner instance in a pytest fixture."""
- return CliRunner()
diff --git a/tests/aiet/test_backend_tool.py b/tests/aiet/test_backend_tool.py
deleted file mode 100644
index fd5960d..0000000
--- a/tests/aiet/test_backend_tool.py
+++ /dev/null
@@ -1,60 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-# pylint: disable=no-self-use
-"""Tests for the tool backend."""
-from collections import Counter
-
-import pytest
-
-from aiet.backend.common import ConfigurationException
-from aiet.backend.config import ToolConfig
-from aiet.backend.tool import get_available_tool_directory_names
-from aiet.backend.tool import get_available_tools
-from aiet.backend.tool import get_tool
-from aiet.backend.tool import Tool
-
-
-def test_get_available_tool_directory_names() -> None:
- """Test get_available_tools mocking get_resources."""
- directory_names = get_available_tool_directory_names()
- assert Counter(directory_names) == Counter(["tool1", "tool2", "vela"])
-
-
-def test_get_available_tools() -> None:
- """Test get_available_tools mocking get_resources."""
- available_tools = get_available_tools()
- expected_tool_names = sorted(
- [
- "tool_1",
- "tool_2",
- "vela",
- "vela",
- "vela",
- ]
- )
-
- assert all(isinstance(s, Tool) for s in available_tools)
- assert all(s != 42 for s in available_tools)
- assert any(s == available_tools[0] for s in available_tools)
- assert len(available_tools) == len(expected_tool_names)
- available_tool_names = sorted(str(s) for s in available_tools)
- assert available_tool_names == expected_tool_names
-
-
-def test_get_tool() -> None:
- """Test get_tool mocking get_resoures."""
- tools = get_tool("tool_1")
- assert len(tools) == 1
- tool = tools[0]
- assert tool is not None
- assert isinstance(tool, Tool)
- assert tool.name == "tool_1"
-
- tools = get_tool("unknown tool")
- assert not tools
-
-
-def test_tool_creation() -> None:
- """Test edge cases when creating a Tool instance."""
- with pytest.raises(ConfigurationException):
- Tool(ToolConfig(name="test", commands={"test": []})) # no 'run' command
diff --git a/tests/aiet/test_check_model.py b/tests/aiet/test_check_model.py
deleted file mode 100644
index 4eafe59..0000000
--- a/tests/aiet/test_check_model.py
+++ /dev/null
@@ -1,162 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-# pylint: disable=redefined-outer-name,no-self-use
-"""Module for testing check_model.py script."""
-from pathlib import Path
-from typing import Any
-
-import pytest
-from ethosu.vela.tflite.Model import Model
-from ethosu.vela.tflite.OperatorCode import OperatorCode
-
-from aiet.cli.common import InvalidTFLiteFileError
-from aiet.cli.common import ModelOptimisedException
-from aiet.resources.tools.vela.check_model import check_custom_codes_for_ethosu
-from aiet.resources.tools.vela.check_model import check_model
-from aiet.resources.tools.vela.check_model import get_custom_codes_from_operators
-from aiet.resources.tools.vela.check_model import get_model_from_file
-from aiet.resources.tools.vela.check_model import get_operators_from_model
-from aiet.resources.tools.vela.check_model import is_vela_optimised
-
-
-@pytest.fixture(scope="session")
-def optimised_tflite_model(
- optimised_input_model_file: Path,
-) -> Model:
- """Return Model instance read from a Vela-optimised TFLite file."""
- return get_model_from_file(optimised_input_model_file)
-
-
-@pytest.fixture(scope="session")
-def non_optimised_tflite_model(
- non_optimised_input_model_file: Path,
-) -> Model:
- """Return Model instance read from a Vela-optimised TFLite file."""
- return get_model_from_file(non_optimised_input_model_file)
-
-
-class TestIsVelaOptimised:
- """Test class for is_vela_optimised() function."""
-
- def test_return_true_when_input_is_optimised(
- self,
- optimised_tflite_model: Model,
- ) -> None:
- """Verify True returned when input is optimised model."""
- output = is_vela_optimised(optimised_tflite_model)
-
- assert output is True
-
- def test_return_false_when_input_is_not_optimised(
- self,
- non_optimised_tflite_model: Model,
- ) -> None:
- """Verify False returned when input is non-optimised model."""
- output = is_vela_optimised(non_optimised_tflite_model)
-
- assert output is False
-
-
-def test_get_operator_list_returns_correct_instances(
- optimised_tflite_model: Model,
-) -> None:
- """Verify list of OperatorCode instances returned by get_operator_list()."""
- operator_list = get_operators_from_model(optimised_tflite_model)
-
- assert all(isinstance(operator, OperatorCode) for operator in operator_list)
-
-
-class TestGetCustomCodesFromOperators:
- """Test the get_custom_codes_from_operators() function."""
-
- def test_returns_empty_list_when_input_operators_have_no_custom_codes(
- self, monkeypatch: Any
- ) -> None:
- """Verify function returns empty list when operators have no custom codes."""
- # Mock OperatorCode.CustomCode() function to return None
- monkeypatch.setattr(
- "ethosu.vela.tflite.OperatorCode.OperatorCode.CustomCode", lambda _: None
- )
-
- operators = [OperatorCode()] * 3
-
- custom_codes = get_custom_codes_from_operators(operators)
-
- assert custom_codes == []
-
- def test_returns_custom_codes_when_input_operators_have_custom_codes(
- self, monkeypatch: Any
- ) -> None:
- """Verify list of bytes objects returned representing the CustomCodes."""
- # Mock OperatorCode.CustomCode() function to return a byte string
- monkeypatch.setattr(
- "ethosu.vela.tflite.OperatorCode.OperatorCode.CustomCode",
- lambda _: b"custom-code",
- )
-
- operators = [OperatorCode()] * 3
-
- custom_codes = get_custom_codes_from_operators(operators)
-
- assert custom_codes == [b"custom-code", b"custom-code", b"custom-code"]
-
-
-@pytest.mark.parametrize(
- "custom_codes, expected_output",
- [
- ([b"ethos-u", b"something else"], True),
- ([b"custom-code-1", b"custom-code-2"], False),
- ],
-)
-def test_check_list_for_ethosu(custom_codes: list, expected_output: bool) -> None:
- """Verify function detects 'ethos-u' bytes in the input list."""
- output = check_custom_codes_for_ethosu(custom_codes)
- assert output is expected_output
-
-
-class TestGetModelFromFile:
- """Test the get_model_from_file() function."""
-
- def test_error_raised_when_input_is_invalid_model_file(
- self,
- invalid_input_model_file: Path,
- ) -> None:
- """Verify error thrown when an invalid model file is given."""
- with pytest.raises(InvalidTFLiteFileError):
- get_model_from_file(invalid_input_model_file)
-
- def test_model_instance_returned_when_input_is_valid_model_file(
- self,
- optimised_input_model_file: Path,
- ) -> None:
- """Verify file is read successfully and returns model instance."""
- tflite_model = get_model_from_file(optimised_input_model_file)
-
- assert isinstance(tflite_model, Model)
-
-
-class TestCheckModel:
- """Test the check_model() function."""
-
- def test_check_model_with_non_optimised_input(
- self,
- non_optimised_input_model_file: Path,
- ) -> None:
- """Verify no error occurs for a valid input file."""
- check_model(non_optimised_input_model_file)
-
- def test_check_model_with_optimised_input(
- self,
- optimised_input_model_file: Path,
- ) -> None:
- """Verify that the right exception is raised with already optimised input."""
- with pytest.raises(ModelOptimisedException):
- check_model(optimised_input_model_file)
-
- def test_check_model_with_invalid_input(
- self,
- invalid_input_model_file: Path,
- ) -> None:
- """Verify that an exception is raised with invalid input."""
- with pytest.raises(Exception):
- check_model(invalid_input_model_file)
diff --git a/tests/aiet/test_cli.py b/tests/aiet/test_cli.py
deleted file mode 100644
index e8589fa..0000000
--- a/tests/aiet/test_cli.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Module for testing CLI top command."""
-from typing import Any
-from unittest.mock import ANY
-from unittest.mock import MagicMock
-
-from click.testing import CliRunner
-
-from aiet.cli import cli
-
-
-def test_cli(cli_runner: CliRunner) -> None:
- """Test CLI top level command."""
- result = cli_runner.invoke(cli)
- assert result.exit_code == 0
- assert "system" in cli.commands
- assert "application" in cli.commands
-
-
-def test_cli_version(cli_runner: CliRunner) -> None:
- """Test version option."""
- result = cli_runner.invoke(cli, ["--version"])
- assert result.exit_code == 0
- assert "version" in result.output
-
-
-def test_cli_verbose(cli_runner: CliRunner, monkeypatch: Any) -> None:
- """Test verbose option."""
- with monkeypatch.context() as mock_context:
- mock = MagicMock()
- # params[1] is the verbose option and we need to replace the
- # callback with a mock object
- mock_context.setattr(cli.params[1], "callback", mock)
- cli_runner.invoke(cli, ["-vvvv"])
- # 4 is the number -v called earlier
- mock.assert_called_once_with(ANY, ANY, 4)
diff --git a/tests/aiet/test_cli_application.py b/tests/aiet/test_cli_application.py
deleted file mode 100644
index f1ccc44..0000000
--- a/tests/aiet/test_cli_application.py
+++ /dev/null
@@ -1,1153 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-# pylint: disable=attribute-defined-outside-init,no-member,line-too-long,too-many-arguments,too-many-locals,redefined-outer-name,too-many-lines
-"""Module for testing CLI application subcommand."""
-import base64
-import json
-import re
-import time
-from contextlib import contextmanager
-from contextlib import ExitStack
-from pathlib import Path
-from typing import Any
-from typing import Generator
-from typing import IO
-from typing import List
-from typing import Optional
-from typing import TypedDict
-from unittest.mock import MagicMock
-
-import click
-import pytest
-from click.testing import CliRunner
-from filelock import FileLock
-
-from aiet.backend.application import Application
-from aiet.backend.config import ApplicationConfig
-from aiet.backend.config import LocalProtocolConfig
-from aiet.backend.config import SSHConfig
-from aiet.backend.config import SystemConfig
-from aiet.backend.config import UserParamConfig
-from aiet.backend.output_parser import Base64OutputParser
-from aiet.backend.protocol import SSHProtocol
-from aiet.backend.system import load_system
-from aiet.cli.application import application_cmd
-from aiet.cli.application import details_cmd
-from aiet.cli.application import execute_cmd
-from aiet.cli.application import install_cmd
-from aiet.cli.application import list_cmd
-from aiet.cli.application import parse_payload_run_config
-from aiet.cli.application import remove_cmd
-from aiet.cli.application import run_cmd
-from aiet.cli.common import MiddlewareExitCode
-
-
-def test_application_cmd() -> None:
- """Test application commands."""
- commands = ["list", "details", "install", "remove", "execute", "run"]
- assert all(command in application_cmd.commands for command in commands)
-
-
-@pytest.mark.parametrize("format_", ["json", "cli"])
-def test_application_cmd_context(cli_runner: CliRunner, format_: str) -> None:
- """Test setting command context parameters."""
- result = cli_runner.invoke(application_cmd, ["--format", format_])
- # command should fail if no subcommand provided
- assert result.exit_code == 2
-
- result = cli_runner.invoke(application_cmd, ["--format", format_, "list"])
- assert result.exit_code == 0
-
-
-@pytest.mark.parametrize(
- "format_, system_name, expected_output",
- [
- (
- "json",
- None,
- '{"type": "application", "available": ["application_1", "application_2"]}\n',
- ),
- (
- "json",
- "system_1",
- '{"type": "application", "available": ["application_1"]}\n',
- ),
- ("cli", None, "Available applications:\n\napplication_1\napplication_2\n"),
- ("cli", "system_1", "Available applications:\n\napplication_1\n"),
- ],
-)
-def test_list_cmd(
- cli_runner: CliRunner,
- monkeypatch: Any,
- format_: str,
- system_name: str,
- expected_output: str,
-) -> None:
- """Test available applications commands."""
- # Mock some applications
- mock_application_1 = MagicMock(spec=Application)
- mock_application_1.name = "application_1"
- mock_application_1.can_run_on.return_value = system_name == "system_1"
- mock_application_2 = MagicMock(spec=Application)
- mock_application_2.name = "application_2"
- mock_application_2.can_run_on.return_value = system_name == "system_2"
-
- # Monkey patch the call get_available_applications
- mock_available_applications = MagicMock()
- mock_available_applications.return_value = [mock_application_1, mock_application_2]
-
- monkeypatch.setattr(
- "aiet.backend.application.get_available_applications",
- mock_available_applications,
- )
-
- obj = {"format": format_}
- args = []
- if system_name:
- list_cmd.params[0].type = click.Choice([system_name])
- args = ["--system", system_name]
- result = cli_runner.invoke(list_cmd, obj=obj, args=args)
- assert result.output == expected_output
-
-
-def get_test_application() -> Application:
- """Return test system details."""
- config = ApplicationConfig(
- name="application",
- description="test",
- build_dir="",
- supported_systems=[],
- deploy_data=[],
- user_params={},
- commands={
- "clean": ["clean"],
- "build": ["build"],
- "run": ["run"],
- "post_run": ["post_run"],
- },
- )
-
- return Application(config)
-
-
-def get_details_cmd_json_output() -> str:
- """Get JSON output for details command."""
- json_output = """
-[
- {
- "type": "application",
- "name": "application",
- "description": "test",
- "supported_systems": [],
- "commands": {
- "clean": {
- "command_strings": [
- "clean"
- ],
- "user_params": []
- },
- "build": {
- "command_strings": [
- "build"
- ],
- "user_params": []
- },
- "run": {
- "command_strings": [
- "run"
- ],
- "user_params": []
- },
- "post_run": {
- "command_strings": [
- "post_run"
- ],
- "user_params": []
- }
- }
- }
-]"""
- return json.dumps(json.loads(json_output)) + "\n"
-
-
-def get_details_cmd_console_output() -> str:
- """Get console output for details command."""
- return (
- 'Application "application" details'
- + "\nDescription: test"
- + "\n\nSupported systems: "
- + "\n\nclean commands:"
- + "\nCommands: ['clean']"
- + "\n\nbuild commands:"
- + "\nCommands: ['build']"
- + "\n\nrun commands:"
- + "\nCommands: ['run']"
- + "\n\npost_run commands:"
- + "\nCommands: ['post_run']"
- + "\n"
- )
-
-
-@pytest.mark.parametrize(
- "application_name,format_, expected_output",
- [
- ("application", "json", get_details_cmd_json_output()),
- ("application", "cli", get_details_cmd_console_output()),
- ],
-)
-def test_details_cmd(
- cli_runner: CliRunner,
- monkeypatch: Any,
- application_name: str,
- format_: str,
- expected_output: str,
-) -> None:
- """Test application details command."""
- monkeypatch.setattr(
- "aiet.cli.application.get_application",
- MagicMock(return_value=[get_test_application()]),
- )
-
- details_cmd.params[0].type = click.Choice(["application"])
- result = cli_runner.invoke(
- details_cmd, obj={"format": format_}, args=["--name", application_name]
- )
- assert result.exception is None
- assert result.output == expected_output
-
-
-def test_details_cmd_wrong_system(cli_runner: CliRunner, monkeypatch: Any) -> None:
- """Test details command fails if application is not supported by the system."""
- monkeypatch.setattr(
- "aiet.backend.execution.get_application", MagicMock(return_value=[])
- )
-
- details_cmd.params[0].type = click.Choice(["application"])
- details_cmd.params[1].type = click.Choice(["system"])
- result = cli_runner.invoke(
- details_cmd, args=["--name", "application", "--system", "system"]
- )
- assert result.exit_code == 2
- assert (
- "Application 'application' doesn't support the system 'system'" in result.stdout
- )
-
-
-def test_install_cmd(cli_runner: CliRunner, monkeypatch: Any) -> None:
- """Test install application command."""
- mock_install_application = MagicMock()
- monkeypatch.setattr(
- "aiet.cli.application.install_application", mock_install_application
- )
-
- args = ["--source", "test"]
- cli_runner.invoke(install_cmd, args=args)
- mock_install_application.assert_called_once_with(Path("test"))
-
-
-def test_remove_cmd(cli_runner: CliRunner, monkeypatch: Any) -> None:
- """Test remove application command."""
- mock_remove_application = MagicMock()
- monkeypatch.setattr(
- "aiet.cli.application.remove_application", mock_remove_application
- )
- remove_cmd.params[0].type = click.Choice(["test"])
-
- args = ["--directory_name", "test"]
- cli_runner.invoke(remove_cmd, args=args)
- mock_remove_application.assert_called_once_with("test")
-
-
-class ExecutionCase(TypedDict, total=False):
- """Execution case."""
-
- args: List[str]
- lock_path: str
- can_establish_connection: bool
- establish_connection_delay: int
- app_exit_code: int
- exit_code: int
- output: str
-
-
-@pytest.mark.parametrize(
- "application_config, system_config, executions",
- [
- [
- ApplicationConfig(
- name="test_application",
- description="Test application",
- supported_systems=["test_system"],
- config_location=Path("wrong_location"),
- commands={"build": ["echo build {application.name}"]},
- ),
- SystemConfig(
- name="test_system",
- description="Test system",
- data_transfer=LocalProtocolConfig(protocol="local"),
- config_location=Path("wrong_location"),
- commands={"run": ["echo run {application.name} on {system.name}"]},
- ),
- [
- ExecutionCase(
- args=["-c", "build"],
- exit_code=MiddlewareExitCode.CONFIGURATION_ERROR,
- output="Error: Application test_application has wrong config location\n",
- )
- ],
- ],
- [
- ApplicationConfig(
- name="test_application",
- description="Test application",
- supported_systems=["test_system"],
- build_dir="build",
- deploy_data=[("sample_file", "/tmp/sample_file")],
- commands={"build": ["echo build {application.name}"]},
- ),
- SystemConfig(
- name="test_system",
- description="Test system",
- data_transfer=LocalProtocolConfig(protocol="local"),
- commands={"run": ["echo run {application.name} on {system.name}"]},
- ),
- [
- ExecutionCase(
- args=["-c", "run"],
- exit_code=MiddlewareExitCode.CONFIGURATION_ERROR,
- output="Error: System test_system does not support data deploy\n",
- )
- ],
- ],
- [
- ApplicationConfig(
- name="test_application",
- description="Test application",
- supported_systems=["test_system"],
- commands={"build": ["echo build {application.name}"]},
- ),
- SystemConfig(
- name="test_system",
- description="Test system",
- data_transfer=LocalProtocolConfig(protocol="local"),
- commands={"run": ["echo run {application.name} on {system.name}"]},
- ),
- [
- ExecutionCase(
- args=["-c", "build"],
- exit_code=MiddlewareExitCode.CONFIGURATION_ERROR,
- output="Error: No build directory defined for the app test_application\n",
- )
- ],
- ],
- [
- ApplicationConfig(
- name="test_application",
- description="Test application",
- supported_systems=["new_system"],
- build_dir="build",
- commands={
- "build": ["echo build {application.name} with {user_params:0}"]
- },
- user_params={
- "build": [
- UserParamConfig(
- name="param",
- description="sample parameter",
- default_value="default",
- values=["val1", "val2", "val3"],
- )
- ]
- },
- ),
- SystemConfig(
- name="test_system",
- description="Test system",
- data_transfer=LocalProtocolConfig(protocol="local"),
- commands={"run": ["echo run {application.name} on {system.name}"]},
- ),
- [
- ExecutionCase(
- args=["-c", "build"],
- exit_code=1,
- output="Error: Application 'test_application' doesn't support the system 'test_system'\n",
- )
- ],
- ],
- [
- ApplicationConfig(
- name="test_application",
- description="Test application",
- supported_systems=["test_system"],
- build_dir="build",
- commands={"build": ["false"]},
- ),
- SystemConfig(
- name="test_system",
- description="Test system",
- data_transfer=LocalProtocolConfig(protocol="local"),
- commands={"run": ["echo run {application.name} on {system.name}"]},
- ),
- [
- ExecutionCase(
- args=["-c", "build"],
- exit_code=MiddlewareExitCode.BACKEND_ERROR,
- output="""Running: false
-Error: Execution failed. Please check output for the details.\n""",
- )
- ],
- ],
- [
- ApplicationConfig(
- name="test_application",
- description="Test application",
- supported_systems=["test_system"],
- lock=True,
- build_dir="build",
- commands={
- "build": ["echo build {application.name} with {user_params:0}"]
- },
- user_params={
- "build": [
- UserParamConfig(
- name="param",
- description="sample parameter",
- default_value="default",
- values=["val1", "val2", "val3"],
- )
- ]
- },
- ),
- SystemConfig(
- name="test_system",
- description="Test system",
- lock=True,
- data_transfer=LocalProtocolConfig(protocol="local"),
- commands={"run": ["echo run {application.name} on {system.name}"]},
- ),
- [
- ExecutionCase(
- args=["-c", "build"],
- exit_code=MiddlewareExitCode.SUCCESS,
- output="""Running: echo build test_application with param default
-build test_application with param default\n""",
- ),
- ExecutionCase(
- args=["-c", "build"],
- lock_path="/tmp/middleware_test_application_test_system.lock",
- exit_code=MiddlewareExitCode.CONCURRENT_ERROR,
- output="Error: Another instance of the system is running\n",
- ),
- ExecutionCase(
- args=["-c", "build", "--param=param=val3"],
- exit_code=MiddlewareExitCode.SUCCESS,
- output="""Running: echo build test_application with param val3
-build test_application with param val3\n""",
- ),
- ExecutionCase(
- args=["-c", "build", "--param=param=newval"],
- exit_code=1,
- output="Error: Application parameter 'param=newval' not valid for command 'build'\n",
- ),
- ExecutionCase(
- args=["-c", "some_command"],
- exit_code=MiddlewareExitCode.CONFIGURATION_ERROR,
- output="Error: Unsupported command some_command\n",
- ),
- ExecutionCase(
- args=["-c", "run"],
- exit_code=MiddlewareExitCode.SUCCESS,
- output="""Generating commands to execute
-Running: echo run test_application on test_system
-run test_application on test_system\n""",
- ),
- ],
- ],
- [
- ApplicationConfig(
- name="test_application",
- description="Test application",
- supported_systems=["test_system"],
- deploy_data=[("sample_file", "/tmp/sample_file")],
- commands={
- "run": [
- "echo run {application.name} with {user_params:param} on {system.name}"
- ]
- },
- user_params={
- "run": [
- UserParamConfig(
- name="param=",
- description="sample parameter",
- default_value="default",
- values=["val1", "val2", "val3"],
- alias="param",
- )
- ]
- },
- ),
- SystemConfig(
- name="test_system",
- description="Test system",
- lock=True,
- data_transfer=SSHConfig(
- protocol="ssh",
- username="username",
- password="password",
- hostname="localhost",
- port="8022",
- ),
- commands={"run": ["sleep 100"]},
- ),
- [
- ExecutionCase(
- args=["-c", "run"],
- exit_code=MiddlewareExitCode.SUCCESS,
- output="""Generating commands to execute
-Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds .
-Deploying {application.config_location}/sample_file onto /tmp/sample_file
-Running: echo run test_application with param=default on test_system
-Shutting down sequence...
-Stopping test_system... (It could take few seconds)
-test_system stopped successfully.\n""",
- ),
- ExecutionCase(
- args=["-c", "run"],
- lock_path="/tmp/middleware_test_system.lock",
- exit_code=MiddlewareExitCode.CONCURRENT_ERROR,
- output="Error: Another instance of the system is running\n",
- ),
- ExecutionCase(
- args=[
- "-c",
- "run",
- "--deploy={application.config_location}/sample_file:/tmp/sample_file",
- ],
- exit_code=0,
- output="""Generating commands to execute
-Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds .
-Deploying {application.config_location}/sample_file onto /tmp/sample_file
-Deploying {application.config_location}/sample_file onto /tmp/sample_file
-Running: echo run test_application with param=default on test_system
-Shutting down sequence...
-Stopping test_system... (It could take few seconds)
-test_system stopped successfully.\n""",
- ),
- ExecutionCase(
- args=["-c", "run"],
- app_exit_code=1,
- exit_code=0,
- output="""Generating commands to execute
-Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds .
-Deploying {application.config_location}/sample_file onto /tmp/sample_file
-Running: echo run test_application with param=default on test_system
-Application exited with exit code 1
-Shutting down sequence...
-Stopping test_system... (It could take few seconds)
-test_system stopped successfully.\n""",
- ),
- ExecutionCase(
- args=["-c", "run"],
- exit_code=MiddlewareExitCode.CONNECTION_ERROR,
- can_establish_connection=False,
- output="""Generating commands to execute
-Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds ..........................................................................................
-Shutting down sequence...
-Stopping test_system... (It could take few seconds)
-test_system stopped successfully.
-Error: Couldn't connect to 'localhost:8022'.\n""",
- ),
- ExecutionCase(
- args=["-c", "run", "--deploy=bad_format"],
- exit_code=1,
- output="Error: Invalid deploy parameter 'bad_format' for command run\n",
- ),
- ExecutionCase(
- args=["-c", "run", "--deploy=:"],
- exit_code=1,
- output="Error: Invalid deploy parameter ':' for command run\n",
- ),
- ExecutionCase(
- args=["-c", "run", "--deploy= : "],
- exit_code=1,
- output="Error: Invalid deploy parameter ' : ' for command run\n",
- ),
- ExecutionCase(
- args=["-c", "run", "--deploy=some_src_file:"],
- exit_code=1,
- output="Error: Invalid deploy parameter 'some_src_file:' for command run\n",
- ),
- ExecutionCase(
- args=["-c", "run", "--deploy=:some_dst_file"],
- exit_code=1,
- output="Error: Invalid deploy parameter ':some_dst_file' for command run\n",
- ),
- ExecutionCase(
- args=["-c", "run", "--deploy=unknown_file:/tmp/dest"],
- exit_code=1,
- output="Error: Path unknown_file does not exist\n",
- ),
- ],
- ],
- [
- ApplicationConfig(
- name="test_application",
- description="Test application",
- supported_systems=["test_system"],
- commands={
- "run": [
- "echo run {application.name} with {user_params:param} on {system.name}"
- ]
- },
- user_params={
- "run": [
- UserParamConfig(
- name="param=",
- description="sample parameter",
- default_value="default",
- values=["val1", "val2", "val3"],
- alias="param",
- )
- ]
- },
- ),
- SystemConfig(
- name="test_system",
- description="Test system",
- data_transfer=SSHConfig(
- protocol="ssh",
- username="username",
- password="password",
- hostname="localhost",
- port="8022",
- ),
- commands={"run": ["echo Unable to start system"]},
- ),
- [
- ExecutionCase(
- args=["-c", "run"],
- exit_code=4,
- can_establish_connection=False,
- establish_connection_delay=1,
- output="""Generating commands to execute
-Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds .
-
----------- test_system execution failed ----------
-Unable to start system
-
-
-
-Shutting down sequence...
-Stopping test_system... (It could take few seconds)
-test_system stopped successfully.
-Error: Execution failed. Please check output for the details.\n""",
- )
- ],
- ],
- ],
-)
-def test_application_command_execution(
- application_config: ApplicationConfig,
- system_config: SystemConfig,
- executions: List[ExecutionCase],
- tmpdir: Any,
- cli_runner: CliRunner,
- monkeypatch: Any,
-) -> None:
- """Test application command execution."""
-
- @contextmanager
- def lock_execution(lock_path: str) -> Generator[None, None, None]:
- lock = FileLock(lock_path)
- lock.acquire(timeout=1)
-
- try:
- yield
- finally:
- lock.release()
-
- def replace_vars(str_val: str) -> str:
- """Replace variables."""
- application_config_location = str(
- application_config["config_location"].absolute()
- )
-
- return str_val.replace(
- "{application.config_location}", application_config_location
- )
-
- for execution in executions:
- init_execution_test(
- monkeypatch,
- tmpdir,
- application_config,
- system_config,
- can_establish_connection=execution.get("can_establish_connection", True),
- establish_conection_delay=execution.get("establish_connection_delay", 0),
- remote_app_exit_code=execution.get("app_exit_code", 0),
- )
-
- lock_path = execution.get("lock_path")
-
- with ExitStack() as stack:
- if lock_path:
- stack.enter_context(lock_execution(lock_path))
-
- args = [replace_vars(arg) for arg in execution["args"]]
-
- result = cli_runner.invoke(
- execute_cmd,
- args=["-n", application_config["name"], "-s", system_config["name"]]
- + args,
- )
- output = replace_vars(execution["output"])
- assert result.exit_code == execution["exit_code"]
- assert result.stdout == output
-
-
-@pytest.fixture(params=[False, True], ids=["run-cli", "run-json"])
-def payload_path_or_none(request: Any, tmp_path_factory: Any) -> Optional[Path]:
- """Drives tests for run command so that it executes them both to use a json file, and to use CLI."""
- if request.param:
- ret: Path = tmp_path_factory.getbasetemp() / "system_config_payload_file.json"
- return ret
- return None
-
-
-def write_system_payload_config(
- payload_file: IO[str],
- application_config: ApplicationConfig,
- system_config: SystemConfig,
-) -> None:
- """Write a json payload file for the given test configuration."""
- payload_dict = {
- "id": system_config["name"],
- "arguments": {
- "application": application_config["name"],
- },
- }
- json.dump(payload_dict, payload_file)
-
-
-@pytest.mark.parametrize(
- "application_config, system_config, executions",
- [
- [
- ApplicationConfig(
- name="test_application",
- description="Test application",
- supported_systems=["test_system"],
- build_dir="build",
- commands={
- "build": ["echo build {application.name} with {user_params:0}"]
- },
- user_params={
- "build": [
- UserParamConfig(
- name="param",
- description="sample parameter",
- default_value="default",
- values=["val1", "val2", "val3"],
- )
- ]
- },
- ),
- SystemConfig(
- name="test_system",
- description="Test system",
- data_transfer=LocalProtocolConfig(protocol="local"),
- commands={"run": ["echo run {application.name} on {system.name}"]},
- ),
- [
- ExecutionCase(
- args=[],
- exit_code=MiddlewareExitCode.SUCCESS,
- output="""Running: echo build test_application with param default
-build test_application with param default
-Generating commands to execute
-Running: echo run test_application on test_system
-run test_application on test_system\n""",
- )
- ],
- ],
- [
- ApplicationConfig(
- name="test_application",
- description="Test application",
- supported_systems=["test_system"],
- commands={
- "run": [
- "echo run {application.name} with {user_params:param} on {system.name}"
- ]
- },
- user_params={
- "run": [
- UserParamConfig(
- name="param=",
- description="sample parameter",
- default_value="default",
- values=["val1", "val2", "val3"],
- alias="param",
- )
- ]
- },
- ),
- SystemConfig(
- name="test_system",
- description="Test system",
- data_transfer=SSHConfig(
- protocol="ssh",
- username="username",
- password="password",
- hostname="localhost",
- port="8022",
- ),
- commands={"run": ["sleep 100"]},
- ),
- [
- ExecutionCase(
- args=[],
- exit_code=MiddlewareExitCode.SUCCESS,
- output="""Generating commands to execute
-Trying to establish connection with 'localhost:8022' - 90 retries every 15.0 seconds .
-Running: echo run test_application with param=default on test_system
-Shutting down sequence...
-Stopping test_system... (It could take few seconds)
-test_system stopped successfully.\n""",
- )
- ],
- ],
- ],
-)
-def test_application_run(
- application_config: ApplicationConfig,
- system_config: SystemConfig,
- executions: List[ExecutionCase],
- tmpdir: Any,
- cli_runner: CliRunner,
- monkeypatch: Any,
- payload_path_or_none: Path,
-) -> None:
- """Test application command execution."""
- for execution in executions:
- init_execution_test(monkeypatch, tmpdir, application_config, system_config)
-
- if payload_path_or_none:
- with open(payload_path_or_none, "w", encoding="utf-8") as payload_file:
- write_system_payload_config(
- payload_file, application_config, system_config
- )
-
- result = cli_runner.invoke(
- run_cmd,
- args=["--config", str(payload_path_or_none)],
- )
- else:
- result = cli_runner.invoke(
- run_cmd,
- args=["-n", application_config["name"], "-s", system_config["name"]]
- + execution["args"],
- )
-
- assert result.stdout == execution["output"]
- assert result.exit_code == execution["exit_code"]
-
-
-@pytest.mark.parametrize(
- "cmdline,error_pattern",
- [
- [
- "--config {payload} -s test_system",
- "when --config is set, the following parameters should not be provided",
- ],
- [
- "--config {payload} -n test_application",
- "when --config is set, the following parameters should not be provided",
- ],
- [
- "--config {payload} -p mypar:3",
- "when --config is set, the following parameters should not be provided",
- ],
- [
- "-p mypar:3",
- "when --config is not set, the following parameters are required",
- ],
- ["-s test_system", "when --config is not set, --name is required"],
- ["-n test_application", "when --config is not set, --system is required"],
- ],
-)
-def test_application_run_invalid_param_combinations(
- cmdline: str,
- error_pattern: str,
- cli_runner: CliRunner,
- monkeypatch: Any,
- tmp_path: Any,
- tmpdir: Any,
-) -> None:
- """Test that invalid combinations arguments result in error as expected."""
- application_config = ApplicationConfig(
- name="test_application",
- description="Test application",
- supported_systems=["test_system"],
- build_dir="build",
- commands={"build": ["echo build {application.name} with {user_params:0}"]},
- user_params={
- "build": [
- UserParamConfig(
- name="param",
- description="sample parameter",
- default_value="default",
- values=["val1", "val2", "val3"],
- )
- ]
- },
- )
- system_config = SystemConfig(
- name="test_system",
- description="Test system",
- data_transfer=LocalProtocolConfig(protocol="local"),
- commands={"run": ["echo run {application.name} on {system.name}"]},
- )
-
- init_execution_test(monkeypatch, tmpdir, application_config, system_config)
-
- payload_file = tmp_path / "payload.json"
- payload_file.write_text("dummy")
- result = cli_runner.invoke(
- run_cmd,
- args=cmdline.format(payload=payload_file).split(),
- )
- found = re.search(error_pattern, result.stdout)
- assert found, f"Cannot find pattern: [{error_pattern}] in \n[\n{result.stdout}\n]"
-
-
-@pytest.mark.parametrize(
- "payload,expected",
- [
- pytest.param(
- {"arguments": {}},
- None,
- marks=pytest.mark.xfail(reason="no system 'id''", strict=True),
- ),
- pytest.param(
- {"id": "testsystem"},
- None,
- marks=pytest.mark.xfail(reason="no arguments object", strict=True),
- ),
- (
- {"id": "testsystem", "arguments": {"application": "testapp"}},
- ("testsystem", "testapp", [], [], [], None),
- ),
- (
- {
- "id": "testsystem",
- "arguments": {"application": "testapp", "par1": "val1"},
- },
- ("testsystem", "testapp", ["par1=val1"], [], [], None),
- ),
- (
- {
- "id": "testsystem",
- "arguments": {"application": "testapp", "application/par1": "val1"},
- },
- ("testsystem", "testapp", ["par1=val1"], [], [], None),
- ),
- (
- {
- "id": "testsystem",
- "arguments": {"application": "testapp", "system/par1": "val1"},
- },
- ("testsystem", "testapp", [], ["par1=val1"], [], None),
- ),
- (
- {
- "id": "testsystem",
- "arguments": {"application": "testapp", "deploy/par1": "val1"},
- },
- ("testsystem", "testapp", [], [], ["par1"], None),
- ),
- (
- {
- "id": "testsystem",
- "arguments": {
- "application": "testapp",
- "appar1": "val1",
- "application/appar2": "val2",
- "system/syspar1": "val3",
- "deploy/depploypar1": "val4",
- "application/appar3": "val5",
- "system/syspar2": "val6",
- "deploy/depploypar2": "val7",
- },
- },
- (
- "testsystem",
- "testapp",
- ["appar1=val1", "appar2=val2", "appar3=val5"],
- ["syspar1=val3", "syspar2=val6"],
- ["depploypar1", "depploypar2"],
- None,
- ),
- ),
- ],
-)
-def test_parse_payload_run_config(payload: dict, expected: tuple) -> None:
- """Test parsing of the JSON payload for the run_config command."""
- assert parse_payload_run_config(payload) == expected
-
-
-def test_application_run_report(
- tmpdir: Any,
- cli_runner: CliRunner,
- monkeypatch: Any,
-) -> None:
- """Test flag '--report' of command 'application run'."""
- app_metrics = {"app_metric": 3.14}
- app_metrics_b64 = base64.b64encode(json.dumps(app_metrics).encode("utf-8"))
- application_config = ApplicationConfig(
- name="test_application",
- description="Test application",
- supported_systems=["test_system"],
- build_dir="build",
- commands={"build": ["echo build {application.name} with {user_params:0}"]},
- user_params={
- "build": [
- UserParamConfig(
- name="param",
- description="sample parameter",
- default_value="default",
- values=["val1", "val2", "val3"],
- ),
- UserParamConfig(
- name="p2",
- description="another parameter, not overridden",
- default_value="the-right-choice",
- values=["the-right-choice", "the-bad-choice"],
- ),
- ]
- },
- )
- system_config = SystemConfig(
- name="test_system",
- description="Test system",
- data_transfer=LocalProtocolConfig(protocol="local"),
- commands={
- "run": [
- "echo run {application.name} on {system.name}",
- f"echo build <{Base64OutputParser.TAG_NAME}>{app_metrics_b64.decode('utf-8')}</{Base64OutputParser.TAG_NAME}>",
- ]
- },
- reporting={
- "regex": {
- "app_name": {
- "pattern": r"run (.\S*) ",
- "type": "str",
- },
- "sys_name": {
- "pattern": r"on (.\S*)",
- "type": "str",
- },
- }
- },
- )
- report_file = Path(tmpdir) / "test_report.json"
- param_val = "param=val1"
- exit_code = MiddlewareExitCode.SUCCESS
-
- init_execution_test(monkeypatch, tmpdir, application_config, system_config)
-
- result = cli_runner.invoke(
- run_cmd,
- args=[
- "-n",
- application_config["name"],
- "-s",
- system_config["name"],
- "--report",
- str(report_file),
- "--param",
- param_val,
- ],
- )
- assert result.exit_code == exit_code
- assert report_file.is_file()
- with open(report_file, "r", encoding="utf-8") as file:
- report = json.load(file)
-
- assert report == {
- "application": {
- "metrics": {"0": {"app_metric": 3.14}},
- "name": "test_application",
- "params": {"param": "val1", "p2": "the-right-choice"},
- },
- "system": {
- "metrics": {"app_name": "test_application", "sys_name": "test_system"},
- "name": "test_system",
- "params": {},
- },
- }
-
-
-def init_execution_test(
- monkeypatch: Any,
- tmpdir: Any,
- application_config: ApplicationConfig,
- system_config: SystemConfig,
- can_establish_connection: bool = True,
- establish_conection_delay: float = 0,
- remote_app_exit_code: int = 0,
-) -> None:
- """Init execution test."""
- application_name = application_config["name"]
- system_name = system_config["name"]
-
- execute_cmd.params[0].type = click.Choice([application_name])
- execute_cmd.params[1].type = click.Choice([system_name])
- execute_cmd.params[2].type = click.Choice(["build", "run", "some_command"])
-
- run_cmd.params[0].type = click.Choice([application_name])
- run_cmd.params[1].type = click.Choice([system_name])
-
- if "config_location" not in application_config:
- application_path = Path(tmpdir) / "application"
- application_path.mkdir()
- application_config["config_location"] = application_path
-
- # this file could be used as deploy parameter value or
- # as deploy parameter in application configuration
- sample_file = application_path / "sample_file"
- sample_file.touch()
- monkeypatch.setattr(
- "aiet.backend.application.get_available_applications",
- MagicMock(return_value=[Application(application_config)]),
- )
-
- ssh_protocol_mock = MagicMock(spec=SSHProtocol)
-
- def mock_establish_connection() -> bool:
- """Mock establish connection function."""
- # give some time for the system to start
- time.sleep(establish_conection_delay)
- return can_establish_connection
-
- ssh_protocol_mock.establish_connection.side_effect = mock_establish_connection
- ssh_protocol_mock.connection_details.return_value = ("localhost", 8022)
- ssh_protocol_mock.run.return_value = (
- remote_app_exit_code,
- bytearray(),
- bytearray(),
- )
- monkeypatch.setattr(
- "aiet.backend.protocol.SSHProtocol", MagicMock(return_value=ssh_protocol_mock)
- )
-
- if "config_location" not in system_config:
- system_path = Path(tmpdir) / "system"
- system_path.mkdir()
- system_config["config_location"] = system_path
- monkeypatch.setattr(
- "aiet.backend.system.get_available_systems",
- MagicMock(return_value=[load_system(system_config)]),
- )
-
- monkeypatch.setattr("aiet.backend.execution.wait", MagicMock())
diff --git a/tests/aiet/test_cli_common.py b/tests/aiet/test_cli_common.py
deleted file mode 100644
index d018e44..0000000
--- a/tests/aiet/test_cli_common.py
+++ /dev/null
@@ -1,37 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Test for cli common module."""
-from typing import Any
-
-import pytest
-
-from aiet.cli.common import print_command_details
-from aiet.cli.common import raise_exception_at_signal
-
-
-def test_print_command_details(capsys: Any) -> None:
- """Test print_command_details function."""
- command = {
- "command_strings": ["echo test"],
- "user_params": [
- {"name": "param_name", "description": "param_description"},
- {
- "name": "param_name2",
- "description": "param_description2",
- "alias": "alias2",
- },
- ],
- }
- print_command_details(command)
- captured = capsys.readouterr()
- assert "echo test" in captured.out
- assert "param_name" in captured.out
- assert "alias2" in captured.out
-
-
-def test_raise_exception_at_signal() -> None:
- """Test raise_exception_at_signal graceful shutdown."""
- with pytest.raises(Exception) as err:
- raise_exception_at_signal(1, "")
-
- assert str(err.value) == "Middleware shutdown requested"
diff --git a/tests/aiet/test_cli_system.py b/tests/aiet/test_cli_system.py
deleted file mode 100644
index fd39f31..0000000
--- a/tests/aiet/test_cli_system.py
+++ /dev/null
@@ -1,240 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Module for testing CLI system subcommand."""
-import json
-from pathlib import Path
-from typing import Any
-from typing import Dict
-from typing import List
-from typing import Optional
-from typing import Union
-from unittest.mock import MagicMock
-
-import click
-import pytest
-from click.testing import CliRunner
-
-from aiet.backend.config import SystemConfig
-from aiet.backend.system import load_system
-from aiet.backend.system import System
-from aiet.cli.system import details_cmd
-from aiet.cli.system import install_cmd
-from aiet.cli.system import list_cmd
-from aiet.cli.system import remove_cmd
-from aiet.cli.system import system_cmd
-
-
-def test_system_cmd() -> None:
- """Test system commands."""
- commands = ["list", "details", "install", "remove"]
- assert all(command in system_cmd.commands for command in commands)
-
-
-@pytest.mark.parametrize("format_", ["json", "cli"])
-def test_system_cmd_context(cli_runner: CliRunner, format_: str) -> None:
- """Test setting command context parameters."""
- result = cli_runner.invoke(system_cmd, ["--format", format_])
- # command should fail if no subcommand provided
- assert result.exit_code == 2
-
- result = cli_runner.invoke(system_cmd, ["--format", format_, "list"])
- assert result.exit_code == 0
-
-
-@pytest.mark.parametrize(
- "format_,expected_output",
- [
- ("json", '{"type": "system", "available": ["system1", "system2"]}\n'),
- ("cli", "Available systems:\n\nsystem1\nsystem2\n"),
- ],
-)
-def test_list_cmd_with_format(
- cli_runner: CliRunner, monkeypatch: Any, format_: str, expected_output: str
-) -> None:
- """Test available systems command with different formats output."""
- # Mock some systems
- mock_system1 = MagicMock()
- mock_system1.name = "system1"
- mock_system2 = MagicMock()
- mock_system2.name = "system2"
-
- # Monkey patch the call get_available_systems
- mock_available_systems = MagicMock()
- mock_available_systems.return_value = [mock_system1, mock_system2]
- monkeypatch.setattr("aiet.cli.system.get_available_systems", mock_available_systems)
-
- obj = {"format": format_}
- result = cli_runner.invoke(list_cmd, obj=obj)
- assert result.output == expected_output
-
-
-def get_test_system(
- annotations: Optional[Dict[str, Union[str, List[str]]]] = None
-) -> System:
- """Return test system details."""
- config = SystemConfig(
- name="system",
- description="test",
- data_transfer={
- "protocol": "ssh",
- "username": "root",
- "password": "root",
- "hostname": "localhost",
- "port": "8022",
- },
- commands={
- "clean": ["clean"],
- "build": ["build"],
- "run": ["run"],
- "post_run": ["post_run"],
- },
- annotations=annotations or {},
- )
-
- return load_system(config)
-
-
-def get_details_cmd_json_output(
- annotations: Optional[Dict[str, Union[str, List[str]]]] = None
-) -> str:
- """Test JSON output for details command."""
- ann_str = ""
- if annotations is not None:
- ann_str = '"annotations":{},'.format(json.dumps(annotations))
-
- json_output = (
- """
-{
- "type": "system",
- "name": "system",
- "description": "test",
- "data_transfer_protocol": "ssh",
- "commands": {
- "clean":
- {
- "command_strings": ["clean"],
- "user_params": []
- },
- "build":
- {
- "command_strings": ["build"],
- "user_params": []
- },
- "run":
- {
- "command_strings": ["run"],
- "user_params": []
- },
- "post_run":
- {
- "command_strings": ["post_run"],
- "user_params": []
- }
- },
-"""
- + ann_str
- + """
- "available_application" : []
- }
-"""
- )
- return json.dumps(json.loads(json_output)) + "\n"
-
-
-def get_details_cmd_console_output(
- annotations: Optional[Dict[str, Union[str, List[str]]]] = None
-) -> str:
- """Test console output for details command."""
- ann_str = ""
- if annotations:
- val_str = "".join(
- "\n\t{}: {}".format(ann_name, ann_value)
- for ann_name, ann_value in annotations.items()
- )
- ann_str = "\nAnnotations:{}".format(val_str)
- return (
- 'System "system" details'
- + "\nDescription: test"
- + "\nData Transfer Protocol: ssh"
- + "\nAvailable Applications: "
- + ann_str
- + "\n\nclean commands:"
- + "\nCommands: ['clean']"
- + "\n\nbuild commands:"
- + "\nCommands: ['build']"
- + "\n\nrun commands:"
- + "\nCommands: ['run']"
- + "\n\npost_run commands:"
- + "\nCommands: ['post_run']"
- + "\n"
- )
-
-
-@pytest.mark.parametrize(
- "format_,system,expected_output",
- [
- (
- "json",
- get_test_system(annotations={"ann1": "annotation1", "ann2": ["a1", "a2"]}),
- get_details_cmd_json_output(
- annotations={"ann1": "annotation1", "ann2": ["a1", "a2"]}
- ),
- ),
- (
- "cli",
- get_test_system(annotations={"ann1": "annotation1", "ann2": ["a1", "a2"]}),
- get_details_cmd_console_output(
- annotations={"ann1": "annotation1", "ann2": ["a1", "a2"]}
- ),
- ),
- (
- "json",
- get_test_system(annotations={}),
- get_details_cmd_json_output(annotations={}),
- ),
- (
- "cli",
- get_test_system(annotations={}),
- get_details_cmd_console_output(annotations={}),
- ),
- ],
-)
-def test_details_cmd(
- cli_runner: CliRunner,
- monkeypatch: Any,
- format_: str,
- system: System,
- expected_output: str,
-) -> None:
- """Test details command with different formats output."""
- mock_get_system = MagicMock()
- mock_get_system.return_value = system
- monkeypatch.setattr("aiet.cli.system.get_system", mock_get_system)
-
- args = ["--name", "system"]
- obj = {"format": format_}
- details_cmd.params[0].type = click.Choice(["system"])
-
- result = cli_runner.invoke(details_cmd, args=args, obj=obj)
- assert result.output == expected_output
-
-
-def test_install_cmd(cli_runner: CliRunner, monkeypatch: Any) -> None:
- """Test install system command."""
- mock_install_system = MagicMock()
- monkeypatch.setattr("aiet.cli.system.install_system", mock_install_system)
-
- args = ["--source", "test"]
- cli_runner.invoke(install_cmd, args=args)
- mock_install_system.assert_called_once_with(Path("test"))
-
-
-def test_remove_cmd(cli_runner: CliRunner, monkeypatch: Any) -> None:
- """Test remove system command."""
- mock_remove_system = MagicMock()
- monkeypatch.setattr("aiet.cli.system.remove_system", mock_remove_system)
- remove_cmd.params[0].type = click.Choice(["test"])
-
- args = ["--directory_name", "test"]
- cli_runner.invoke(remove_cmd, args=args)
- mock_remove_system.assert_called_once_with("test")
diff --git a/tests/aiet/test_cli_tool.py b/tests/aiet/test_cli_tool.py
deleted file mode 100644
index 45d45c8..0000000
--- a/tests/aiet/test_cli_tool.py
+++ /dev/null
@@ -1,333 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-# pylint: disable=attribute-defined-outside-init,no-member,line-too-long,too-many-arguments,too-many-locals
-"""Module for testing CLI tool subcommand."""
-import json
-from pathlib import Path
-from typing import Any
-from typing import List
-from typing import Optional
-from typing import Sequence
-from unittest.mock import MagicMock
-
-import click
-import pytest
-from click.testing import CliRunner
-from click.testing import Result
-
-from aiet.backend.tool import get_unique_tool_names
-from aiet.backend.tool import Tool
-from aiet.cli.tool import details_cmd
-from aiet.cli.tool import execute_cmd
-from aiet.cli.tool import list_cmd
-from aiet.cli.tool import tool_cmd
-
-
-def test_tool_cmd() -> None:
- """Test tool commands."""
- commands = ["list", "details", "execute"]
- assert all(command in tool_cmd.commands for command in commands)
-
-
-@pytest.mark.parametrize("format_", ["json", "cli"])
-def test_tool_cmd_context(cli_runner: CliRunner, format_: str) -> None:
- """Test setting command context parameters."""
- result = cli_runner.invoke(tool_cmd, ["--format", format_])
- # command should fail if no subcommand provided
- assert result.exit_code == 2
-
- result = cli_runner.invoke(tool_cmd, ["--format", format_, "list"])
- assert result.exit_code == 0
-
-
-@pytest.mark.parametrize(
- "format_, expected_output",
- [
- (
- "json",
- '{"type": "tool", "available": ["tool_1", "tool_2"]}\n',
- ),
- ("cli", "Available tools:\n\ntool_1\ntool_2\n"),
- ],
-)
-def test_list_cmd(
- cli_runner: CliRunner,
- monkeypatch: Any,
- format_: str,
- expected_output: str,
-) -> None:
- """Test available tool commands."""
- # Mock some tools
- mock_tool_1 = MagicMock(spec=Tool)
- mock_tool_1.name = "tool_1"
- mock_tool_2 = MagicMock(spec=Tool)
- mock_tool_2.name = "tool_2"
-
- # Monkey patch the call get_available_tools
- mock_available_tools = MagicMock()
- mock_available_tools.return_value = [mock_tool_1, mock_tool_2]
-
- monkeypatch.setattr("aiet.backend.tool.get_available_tools", mock_available_tools)
-
- obj = {"format": format_}
- args: Sequence[str] = []
- result = cli_runner.invoke(list_cmd, obj=obj, args=args)
- assert result.output == expected_output
-
-
-def get_details_cmd_json_output() -> List[dict]:
- """Get JSON output for details command."""
- json_output = [
- {
- "type": "tool",
- "name": "tool_1",
- "description": "This is tool 1",
- "supported_systems": ["System 1"],
- "commands": {
- "clean": {"command_strings": ["echo 'clean'"], "user_params": []},
- "build": {"command_strings": ["echo 'build'"], "user_params": []},
- "run": {"command_strings": ["echo 'run'"], "user_params": []},
- "post_run": {"command_strings": ["echo 'post_run'"], "user_params": []},
- },
- }
- ]
-
- return json_output
-
-
-def get_details_cmd_console_output() -> str:
- """Get console output for details command."""
- return (
- 'Tool "tool_1" details'
- "\nDescription: This is tool 1"
- "\n\nSupported systems: System 1"
- "\n\nclean commands:"
- "\nCommands: [\"echo 'clean'\"]"
- "\n\nbuild commands:"
- "\nCommands: [\"echo 'build'\"]"
- "\n\nrun commands:\nCommands: [\"echo 'run'\"]"
- "\n\npost_run commands:"
- "\nCommands: [\"echo 'post_run'\"]"
- "\n"
- )
-
-
-@pytest.mark.parametrize(
- [
- "tool_name",
- "format_",
- "expected_success",
- "expected_output",
- ],
- [
- ("tool_1", "json", True, get_details_cmd_json_output()),
- ("tool_1", "cli", True, get_details_cmd_console_output()),
- ("non-existent tool", "json", False, None),
- ("non-existent tool", "cli", False, None),
- ],
-)
-def test_details_cmd(
- cli_runner: CliRunner,
- tool_name: str,
- format_: str,
- expected_success: bool,
- expected_output: str,
-) -> None:
- """Test tool details command."""
- details_cmd.params[0].type = click.Choice(["tool_1", "tool_2", "vela"])
- result = cli_runner.invoke(
- details_cmd, obj={"format": format_}, args=["--name", tool_name]
- )
- success = result.exit_code == 0
- assert success == expected_success, result.output
- if expected_success:
- assert result.exception is None
- output = json.loads(result.output) if format_ == "json" else result.output
- assert output == expected_output
-
-
-@pytest.mark.parametrize(
- "system_name",
- [
- "",
- "Corstone-300: Cortex-M55+Ethos-U55",
- "Corstone-300: Cortex-M55+Ethos-U65",
- "Corstone-310: Cortex-M85+Ethos-U55",
- ],
-)
-def test_details_cmd_vela(cli_runner: CliRunner, system_name: str) -> None:
- """Test tool details command for Vela."""
- details_cmd.params[0].type = click.Choice(get_unique_tool_names())
- details_cmd.params[1].type = click.Choice([system_name])
- args = ["--name", "vela"]
- if system_name:
- args += ["--system", system_name]
- result = cli_runner.invoke(details_cmd, obj={"format": "json"}, args=args)
- success = result.exit_code == 0
- assert success, result.output
- result_json = json.loads(result.output)
- assert result_json
- if system_name:
- assert len(result_json) == 1
- tool = result_json[0]
- assert len(tool["supported_systems"]) == 1
- assert system_name == tool["supported_systems"][0]
- else: # no system specified => list details for all systems
- assert len(result_json) == 3
- assert all(len(tool["supported_systems"]) == 1 for tool in result_json)
-
-
-@pytest.fixture(scope="session")
-def input_model_file(non_optimised_input_model_file: Path) -> Path:
- """Provide the path to a quantized dummy model file in the test_resources_path."""
- return non_optimised_input_model_file
-
-
-def execute_vela(
- cli_runner: CliRunner,
- tool_name: str = "vela",
- system_name: Optional[str] = None,
- input_model: Optional[Path] = None,
- output_model: Optional[Path] = None,
- mac: Optional[int] = None,
- format_: str = "cli",
-) -> Result:
- """Run Vela with different parameters."""
- execute_cmd.params[0].type = click.Choice(get_unique_tool_names())
- execute_cmd.params[2].type = click.Choice([system_name or "dummy_system"])
- args = ["--name", tool_name]
- if system_name is not None:
- args += ["--system", system_name]
- if input_model is not None:
- args += ["--param", "input={}".format(input_model)]
- if output_model is not None:
- args += ["--param", "output={}".format(output_model)]
- if mac is not None:
- args += ["--param", "mac={}".format(mac)]
- result = cli_runner.invoke(
- execute_cmd,
- args=args,
- obj={"format": format_},
- )
- return result
-
-
-@pytest.mark.parametrize("format_", ["cli, json"])
-@pytest.mark.parametrize(
- ["tool_name", "system_name", "mac", "expected_success", "expected_output"],
- [
- ("vela", "System 1", 32, False, None), # system not supported
- ("vela", "NON-EXISTENT SYSTEM", 128, False, None), # system does not exist
- ("vela", "Corstone-300: Cortex-M55+Ethos-U55", 32, True, None),
- ("NON-EXISTENT TOOL", "Corstone-300: Cortex-M55+Ethos-U55", 32, False, None),
- ("vela", "Corstone-300: Cortex-M55+Ethos-U55", 64, True, None),
- ("vela", "Corstone-300: Cortex-M55+Ethos-U55", 128, True, None),
- ("vela", "Corstone-300: Cortex-M55+Ethos-U55", 256, True, None),
- (
- "vela",
- "Corstone-300: Cortex-M55+Ethos-U55",
- 512,
- False,
- None,
- ), # mac not supported
- (
- "vela",
- "Corstone-300: Cortex-M55+Ethos-U65",
- 32,
- False,
- None,
- ), # mac not supported
- ("vela", "Corstone-300: Cortex-M55+Ethos-U65", 256, True, None),
- ("vela", "Corstone-300: Cortex-M55+Ethos-U65", 512, True, None),
- (
- "vela",
- None,
- 512,
- False,
- "Error: Please specify the system for tool vela.",
- ), # no system specified
- (
- "NON-EXISTENT TOOL",
- "Corstone-300: Cortex-M55+Ethos-U65",
- 512,
- False,
- None,
- ), # tool does not exist
- ("vela", "Corstone-310: Cortex-M85+Ethos-U55", 128, True, None),
- ],
-)
-def test_vela_run(
- cli_runner: CliRunner,
- format_: str,
- input_model_file: Path, # pylint: disable=redefined-outer-name
- tool_name: str,
- system_name: Optional[str],
- mac: int,
- expected_success: bool,
- expected_output: Optional[str],
- monkeypatch: pytest.MonkeyPatch,
- tmp_path: Path,
-) -> None:
- """Test the execution of the Vela command."""
- monkeypatch.chdir(tmp_path)
-
- output_file = Path("vela_output.tflite")
-
- result = execute_vela(
- cli_runner,
- tool_name=tool_name,
- system_name=system_name,
- input_model=input_model_file,
- output_model=output_file,
- mac=mac,
- format_=format_,
- )
-
- success = result.exit_code == 0
- assert success == expected_success
- if success:
- # Check output file
- output_file = output_file.resolve()
- assert output_file.is_file()
- if expected_output:
- assert result.output.strip() == expected_output
-
-
-@pytest.mark.parametrize("include_input_model", [True, False])
-@pytest.mark.parametrize("include_output_model", [True, False])
-@pytest.mark.parametrize("include_mac", [True, False])
-def test_vela_run_missing_params(
- cli_runner: CliRunner,
- input_model_file: Path, # pylint: disable=redefined-outer-name
- include_input_model: bool,
- include_output_model: bool,
- include_mac: bool,
- monkeypatch: pytest.MonkeyPatch,
- tmp_path: Path,
-) -> None:
- """Test the execution of the Vela command with missing user parameters."""
- monkeypatch.chdir(tmp_path)
-
- output_model_file = Path("output_model.tflite")
- system_name = "Corstone-300: Cortex-M55+Ethos-U65"
- mac = 256
- # input_model is a required parameters, but mac and output_model have default values.
- expected_success = include_input_model
-
- result = execute_vela(
- cli_runner,
- tool_name="vela",
- system_name=system_name,
- input_model=input_model_file if include_input_model else None,
- output_model=output_model_file if include_output_model else None,
- mac=mac if include_mac else None,
- )
-
- success = result.exit_code == 0
- assert success == expected_success, (
- f"Success is {success}, but expected {expected_success}. "
- f"Included params: ["
- f"input_model={include_input_model}, "
- f"output_model={include_output_model}, "
- f"mac={include_mac}]"
- )
diff --git a/tests/aiet/test_main.py b/tests/aiet/test_main.py
deleted file mode 100644
index f2ebae2..0000000
--- a/tests/aiet/test_main.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Module for testing AIET main.py."""
-from typing import Any
-from unittest.mock import MagicMock
-
-from aiet import main
-
-
-def test_main(monkeypatch: Any) -> None:
- """Test main entry point function."""
- with monkeypatch.context() as mock_context:
- mock = MagicMock()
- mock_context.setattr(main, "cli", mock)
- main.main()
- mock.assert_called_once()
diff --git a/tests/aiet/test_resources/tools/tool1/aiet-config.json b/tests/aiet/test_resources/tools/tool1/aiet-config.json
deleted file mode 100644
index 067ef7e..0000000
--- a/tests/aiet/test_resources/tools/tool1/aiet-config.json
+++ /dev/null
@@ -1,30 +0,0 @@
-[
- {
- "name": "tool_1",
- "description": "This is tool 1",
- "build_dir": "build",
- "supported_systems": [
- {
- "name": "System 1"
- }
- ],
- "commands": {
- "clean": [
- "echo 'clean'"
- ],
- "build": [
- "echo 'build'"
- ],
- "run": [
- "echo 'run'"
- ],
- "post_run": [
- "echo 'post_run'"
- ]
- },
- "user_params": {
- "build": [],
- "run": []
- }
- }
-]
diff --git a/tests/aiet/test_resources/tools/tool2/aiet-config.json b/tests/aiet/test_resources/tools/tool2/aiet-config.json
deleted file mode 100644
index 6eee9a6..0000000
--- a/tests/aiet/test_resources/tools/tool2/aiet-config.json
+++ /dev/null
@@ -1,26 +0,0 @@
-[
- {
- "name": "tool_2",
- "description": "This is tool 2 with no supported systems",
- "build_dir": "build",
- "supported_systems": [],
- "commands": {
- "clean": [
- "echo 'clean'"
- ],
- "build": [
- "echo 'build'"
- ],
- "run": [
- "echo 'run'"
- ],
- "post_run": [
- "echo 'post_run'"
- ]
- },
- "user_params": {
- "build": [],
- "run": []
- }
- }
-]
diff --git a/tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json.license b/tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json.license
deleted file mode 100644
index 9b83bfc..0000000
--- a/tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json.license
+++ /dev/null
@@ -1,3 +0,0 @@
-SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-
-SPDX-License-Identifier: Apache-2.0
diff --git a/tests/aiet/test_run_vela_script.py b/tests/aiet/test_run_vela_script.py
deleted file mode 100644
index 971856e..0000000
--- a/tests/aiet/test_run_vela_script.py
+++ /dev/null
@@ -1,152 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-# pylint: disable=redefined-outer-name,no-self-use
-"""Module for testing run_vela.py script."""
-from pathlib import Path
-from typing import Any
-from typing import List
-
-import pytest
-from click.testing import CliRunner
-
-from aiet.cli.common import MiddlewareExitCode
-from aiet.resources.tools.vela.check_model import get_model_from_file
-from aiet.resources.tools.vela.check_model import is_vela_optimised
-from aiet.resources.tools.vela.run_vela import run_vela
-
-
-@pytest.fixture(scope="session")
-def vela_config_path(test_tools_path: Path) -> Path:
- """Return test systems path in a pytest fixture."""
- return test_tools_path / "vela" / "vela.ini"
-
-
-@pytest.fixture(
- params=[
- ["ethos-u65-256", "Ethos_U65_High_End", "U65_Shared_Sram"],
- ["ethos-u55-32", "Ethos_U55_High_End_Embedded", "U55_Shared_Sram"],
- ]
-)
-def ethos_config(request: Any) -> Any:
- """Fixture to provide different configuration for Ethos-U optimization with Vela."""
- return request.param
-
-
-# pylint: disable=too-many-arguments
-def generate_args(
- input_: Path,
- output: Path,
- cfg: Path,
- acc_config: str,
- system_config: str,
- memory_mode: str,
-) -> List[str]:
- """Generate arguments that can be passed to script 'run_vela'."""
- return [
- "-i",
- str(input_),
- "-o",
- str(output),
- "--config",
- str(cfg),
- "--accelerator-config",
- acc_config,
- "--system-config",
- system_config,
- "--memory-mode",
- memory_mode,
- "--optimise",
- "Performance",
- ]
-
-
-def check_run_vela(
- cli_runner: CliRunner, args: List, expected_success: bool, output_file: Path
-) -> None:
- """Run Vela with the given arguments and check the result."""
- result = cli_runner.invoke(run_vela, args)
- success = result.exit_code == MiddlewareExitCode.SUCCESS
- assert success == expected_success
- if success:
- model = get_model_from_file(output_file)
- assert is_vela_optimised(model)
-
-
-def run_vela_script(
- cli_runner: CliRunner,
- input_model_file: Path,
- output_model_file: Path,
- vela_config: Path,
- expected_success: bool,
- acc_config: str,
- system_config: str,
- memory_mode: str,
-) -> None:
- """Run the command 'run_vela' on the command line."""
- args = generate_args(
- input_model_file,
- output_model_file,
- vela_config,
- acc_config,
- system_config,
- memory_mode,
- )
- check_run_vela(cli_runner, args, expected_success, output_model_file)
-
-
-class TestRunVelaCli:
- """Test the command-line execution of the run_vela command."""
-
- def test_non_optimised_model(
- self,
- cli_runner: CliRunner,
- non_optimised_input_model_file: Path,
- tmp_path: Path,
- vela_config_path: Path,
- ethos_config: List,
- ) -> None:
- """Verify Vela is run correctly on an unoptimised model."""
- run_vela_script(
- cli_runner,
- non_optimised_input_model_file,
- tmp_path / "test.tflite",
- vela_config_path,
- True,
- *ethos_config,
- )
-
- def test_optimised_model(
- self,
- cli_runner: CliRunner,
- optimised_input_model_file: Path,
- tmp_path: Path,
- vela_config_path: Path,
- ethos_config: List,
- ) -> None:
- """Verify Vela is run correctly on an already optimised model."""
- run_vela_script(
- cli_runner,
- optimised_input_model_file,
- tmp_path / "test.tflite",
- vela_config_path,
- True,
- *ethos_config,
- )
-
- def test_invalid_model(
- self,
- cli_runner: CliRunner,
- invalid_input_model_file: Path,
- tmp_path: Path,
- vela_config_path: Path,
- ethos_config: List,
- ) -> None:
- """Verify an error is raised when the input model is not valid."""
- run_vela_script(
- cli_runner,
- invalid_input_model_file,
- tmp_path / "test.tflite",
- vela_config_path,
- False,
- *ethos_config,
- )
diff --git a/tests/aiet/test_utils_helpers.py b/tests/aiet/test_utils_helpers.py
deleted file mode 100644
index bbe03fc..0000000
--- a/tests/aiet/test_utils_helpers.py
+++ /dev/null
@@ -1,27 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Module for testing helpers.py."""
-import logging
-from typing import Any
-from typing import List
-from unittest.mock import call
-from unittest.mock import MagicMock
-
-import pytest
-
-from aiet.utils.helpers import set_verbosity
-
-
-@pytest.mark.parametrize(
- "verbosity,expected_calls",
- [(0, []), (1, [call(logging.INFO)]), (2, [call(logging.DEBUG)])],
-)
-def test_set_verbosity(
- verbosity: int, expected_calls: List[Any], monkeypatch: Any
-) -> None:
- """Test set_verbosity() with different verbsosity levels."""
- with monkeypatch.context() as mock_context:
- logging_mock = MagicMock()
- mock_context.setattr(logging.getLogger(), "setLevel", logging_mock)
- set_verbosity(None, None, verbosity)
- logging_mock.assert_has_calls(expected_calls)
diff --git a/tests/mlia/conftest.py b/tests/mlia/conftest.py
index f683fca..0b4b2aa 100644
--- a/tests/mlia/conftest.py
+++ b/tests/mlia/conftest.py
@@ -1,7 +1,10 @@
# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
"""Pytest conf module."""
+import shutil
+import tarfile
from pathlib import Path
+from typing import Any
import pytest
@@ -18,3 +21,91 @@ def fixture_test_resources_path() -> Path:
def fixture_dummy_context(tmpdir: str) -> ExecutionContext:
"""Return dummy context fixture."""
return ExecutionContext(working_dir=tmpdir)
+
+
+@pytest.fixture(scope="session")
+def test_systems_path(test_resources_path: Path) -> Path:
+ """Return test systems path in a pytest fixture."""
+ return test_resources_path / "backends" / "systems"
+
+
+@pytest.fixture(scope="session")
+def test_applications_path(test_resources_path: Path) -> Path:
+ """Return test applications path in a pytest fixture."""
+ return test_resources_path / "backends" / "applications"
+
+
+@pytest.fixture(scope="session")
+def non_optimised_input_model_file(test_tflite_model: Path) -> Path:
+ """Provide the path to a quantized dummy model file."""
+ return test_tflite_model
+
+
+@pytest.fixture(scope="session")
+def optimised_input_model_file(test_tflite_vela_model: Path) -> Path:
+ """Provide path to Vela-optimised dummy model file."""
+ return test_tflite_vela_model
+
+
+@pytest.fixture(scope="session")
+def invalid_input_model_file(test_tflite_invalid_model: Path) -> Path:
+ """Provide the path to an invalid dummy model file."""
+ return test_tflite_invalid_model
+
+
+@pytest.fixture(autouse=True)
+def test_resources(monkeypatch: pytest.MonkeyPatch, test_resources_path: Path) -> Any:
+ """Force using test resources as middleware's repository."""
+
+ def get_test_resources() -> Path:
+ """Return path to the test resources."""
+ return test_resources_path / "backends"
+
+ monkeypatch.setattr("mlia.backend.fs.get_backend_resources", get_test_resources)
+ yield
+
+
+def create_archive(
+ archive_name: str, source: Path, destination: Path, with_root_folder: bool = False
+) -> None:
+ """Create archive from directory source."""
+ with tarfile.open(destination / archive_name, mode="w:gz") as tar:
+ for item in source.iterdir():
+ item_name = item.name
+ if with_root_folder:
+ item_name = f"{source.name}/{item_name}"
+ tar.add(item, item_name)
+
+
+def process_directory(source: Path, destination: Path) -> None:
+ """Process resource directory."""
+ destination.mkdir()
+
+ for item in source.iterdir():
+ if item.is_dir():
+ create_archive(f"{item.name}.tar.gz", item, destination)
+ create_archive(f"{item.name}_dir.tar.gz", item, destination, True)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def add_archives(
+ test_resources_path: Path, tmp_path_factory: pytest.TempPathFactory
+) -> Any:
+ """Generate archives of the test resources."""
+ tmp_path = tmp_path_factory.mktemp("archives")
+
+ archives_path = tmp_path / "archives"
+ archives_path.mkdir()
+
+ if (archives_path_link := test_resources_path / "archives").is_symlink():
+ archives_path_link.unlink()
+
+ archives_path_link.symlink_to(archives_path, target_is_directory=True)
+
+ for item in ["applications", "systems"]:
+ process_directory(test_resources_path / "backends" / item, archives_path / item)
+
+ yield
+
+ archives_path_link.unlink()
+ shutil.rmtree(tmp_path)
diff --git a/tests/aiet/test_backend_application.py b/tests/mlia/test_backend_application.py
index abfab00..2cfb2ef 100644
--- a/tests/aiet/test_backend_application.py
+++ b/tests/mlia/test_backend_application.py
@@ -11,28 +11,34 @@ from unittest.mock import MagicMock
import pytest
-from aiet.backend.application import Application
-from aiet.backend.application import get_application
-from aiet.backend.application import get_available_application_directory_names
-from aiet.backend.application import get_available_applications
-from aiet.backend.application import get_unique_application_names
-from aiet.backend.application import install_application
-from aiet.backend.application import load_applications
-from aiet.backend.application import remove_application
-from aiet.backend.common import Command
-from aiet.backend.common import DataPaths
-from aiet.backend.common import Param
-from aiet.backend.common import UserParamConfig
-from aiet.backend.config import ApplicationConfig
-from aiet.backend.config import ExtendedApplicationConfig
-from aiet.backend.config import NamedExecutionConfig
+from mlia.backend.application import Application
+from mlia.backend.application import get_application
+from mlia.backend.application import get_available_application_directory_names
+from mlia.backend.application import get_available_applications
+from mlia.backend.application import get_unique_application_names
+from mlia.backend.application import install_application
+from mlia.backend.application import load_applications
+from mlia.backend.application import remove_application
+from mlia.backend.common import Command
+from mlia.backend.common import DataPaths
+from mlia.backend.common import Param
+from mlia.backend.common import UserParamConfig
+from mlia.backend.config import ApplicationConfig
+from mlia.backend.config import ExtendedApplicationConfig
+from mlia.backend.config import NamedExecutionConfig
def test_get_available_application_directory_names() -> None:
"""Test get_available_applicationss mocking get_resources."""
directory_names = get_available_application_directory_names()
assert Counter(directory_names) == Counter(
- ["application1", "application2", "application4", "application5"]
+ [
+ "application1",
+ "application2",
+ "application4",
+ "application5",
+ "application6",
+ ]
)
@@ -42,7 +48,7 @@ def test_get_available_applications() -> None:
assert all(isinstance(s, Application) for s in available_applications)
assert all(s != 42 for s in available_applications)
- assert len(available_applications) == 9
+ assert len(available_applications) == 10
# application_5 has multiply items with multiply supported systems
assert [str(s) for s in available_applications] == [
"application_1",
@@ -54,6 +60,7 @@ def test_get_available_applications() -> None:
"application_5A",
"application_5B",
"application_5B",
+ "application_6",
]
@@ -70,6 +77,7 @@ def test_get_unique_application_names() -> None:
"application_5",
"application_5A",
"application_5B",
+ "application_6",
]
@@ -121,14 +129,14 @@ def test_get_application() -> None:
pytest.raises(Exception, match="Unable to read application definition"),
),
(
- "applications/application1",
+ "backends/applications/application1",
0,
pytest.raises(
Exception, match=r"Applications \[application_1\] are already installed"
),
),
(
- "applications/application3",
+ "backends/applications/application3",
0,
pytest.raises(Exception, match="Unable to read application definition"),
),
@@ -144,7 +152,7 @@ def test_install_application(
"""Test application install from archive."""
mock_create_destination_and_install = MagicMock()
monkeypatch.setattr(
- "aiet.backend.application.create_destination_and_install",
+ "mlia.backend.application.create_destination_and_install",
mock_create_destination_and_install,
)
@@ -156,7 +164,7 @@ def test_install_application(
def test_remove_application(monkeypatch: Any) -> None:
"""Test application removal."""
mock_remove_backend = MagicMock()
- monkeypatch.setattr("aiet.backend.application.remove_backend", mock_remove_backend)
+ monkeypatch.setattr("mlia.backend.application.remove_backend", mock_remove_backend)
remove_application("some_application_directory")
mock_remove_backend.assert_called_once()
diff --git a/tests/aiet/test_backend_common.py b/tests/mlia/test_backend_common.py
index 12c30ec..82a985a 100644
--- a/tests/aiet/test_backend_common.py
+++ b/tests/mlia/test_backend_common.py
@@ -16,20 +16,20 @@ from unittest.mock import MagicMock
import pytest
-from aiet.backend.application import Application
-from aiet.backend.common import Backend
-from aiet.backend.common import BaseBackendConfig
-from aiet.backend.common import Command
-from aiet.backend.common import ConfigurationException
-from aiet.backend.common import load_config
-from aiet.backend.common import Param
-from aiet.backend.common import parse_raw_parameter
-from aiet.backend.common import remove_backend
-from aiet.backend.config import ApplicationConfig
-from aiet.backend.config import UserParamConfig
-from aiet.backend.execution import ExecutionContext
-from aiet.backend.execution import ParamResolver
-from aiet.backend.system import System
+from mlia.backend.application import Application
+from mlia.backend.common import Backend
+from mlia.backend.common import BaseBackendConfig
+from mlia.backend.common import Command
+from mlia.backend.common import ConfigurationException
+from mlia.backend.common import load_config
+from mlia.backend.common import Param
+from mlia.backend.common import parse_raw_parameter
+from mlia.backend.common import remove_backend
+from mlia.backend.config import ApplicationConfig
+from mlia.backend.config import UserParamConfig
+from mlia.backend.execution import ExecutionContext
+from mlia.backend.execution import ParamResolver
+from mlia.backend.system import System
@pytest.mark.parametrize(
@@ -44,7 +44,7 @@ def test_remove_backend(
) -> None:
"""Test remove_backend function."""
mock_remove_resource = MagicMock()
- monkeypatch.setattr("aiet.backend.common.remove_resource", mock_remove_resource)
+ monkeypatch.setattr("mlia.backend.common.remove_resource", mock_remove_resource)
with expected_exception:
remove_backend(directory_name, "applications")
@@ -75,7 +75,7 @@ def test_load_config(
)
for config in configs:
json_mock = MagicMock()
- monkeypatch.setattr("aiet.backend.common.json.load", json_mock)
+ monkeypatch.setattr("mlia.backend.common.json.load", json_mock)
load_config(config)
json_mock.assert_called_once()
@@ -175,7 +175,7 @@ class TestBackend:
"variables": {"var_A": "value for variable A"},
}
- monkeypatch.setattr("aiet.backend.system.ProtocolFactory", MagicMock())
+ monkeypatch.setattr("mlia.backend.system.ProtocolFactory", MagicMock())
application, system = Application(config), System(config) # type: ignore
context = ExecutionContext(
app=application,
@@ -290,7 +290,7 @@ class TestBackend:
expected_output: List[Tuple[Optional[str], Param]],
) -> None:
"""Test command building."""
- monkeypatch.setattr("aiet.backend.system.ProtocolFactory", MagicMock())
+ monkeypatch.setattr("mlia.backend.system.ProtocolFactory", MagicMock())
backend = class_(config)
params = backend.resolved_parameters(
diff --git a/tests/aiet/test_backend_controller.py b/tests/mlia/test_backend_controller.py
index 8836ec5..a047adf 100644
--- a/tests/aiet/test_backend_controller.py
+++ b/tests/mlia/test_backend_controller.py
@@ -10,10 +10,10 @@ from typing import Any
import psutil
import pytest
-from aiet.backend.common import ConfigurationException
-from aiet.backend.controller import SystemController
-from aiet.backend.controller import SystemControllerSingleInstance
-from aiet.utils.proc import ShellCommand
+from mlia.backend.common import ConfigurationException
+from mlia.backend.controller import SystemController
+from mlia.backend.controller import SystemControllerSingleInstance
+from mlia.backend.proc import ShellCommand
def get_system_controller(**kwargs: Any) -> SystemController:
diff --git a/tests/aiet/test_backend_execution.py b/tests/mlia/test_backend_execution.py
index 8aa45f1..9395352 100644
--- a/tests/aiet/test_backend_execution.py
+++ b/tests/mlia/test_backend_execution.py
@@ -1,41 +1,39 @@
# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
# pylint: disable=no-self-use
-"""Test backend context module."""
+"""Test backend execution module."""
from contextlib import ExitStack as does_not_raise
from pathlib import Path
from typing import Any
from typing import Dict
-from typing import Optional
from unittest import mock
from unittest.mock import MagicMock
import pytest
from sh import CommandNotFound
-from aiet.backend.application import Application
-from aiet.backend.application import get_application
-from aiet.backend.common import ConfigurationException
-from aiet.backend.common import DataPaths
-from aiet.backend.common import UserParamConfig
-from aiet.backend.config import ApplicationConfig
-from aiet.backend.config import LocalProtocolConfig
-from aiet.backend.config import SystemConfig
-from aiet.backend.execution import deploy_data
-from aiet.backend.execution import execute_commands_locally
-from aiet.backend.execution import ExecutionContext
-from aiet.backend.execution import get_application_and_system
-from aiet.backend.execution import get_application_by_name_and_system
-from aiet.backend.execution import get_file_lock_path
-from aiet.backend.execution import get_tool_by_system
-from aiet.backend.execution import ParamResolver
-from aiet.backend.execution import Reporter
-from aiet.backend.execution import wait
-from aiet.backend.output_parser import OutputParser
-from aiet.backend.system import get_system
-from aiet.backend.system import load_system
-from aiet.backend.tool import get_tool
-from aiet.utils.proc import CommandFailedException
+from mlia.backend.application import Application
+from mlia.backend.application import get_application
+from mlia.backend.common import DataPaths
+from mlia.backend.common import UserParamConfig
+from mlia.backend.config import ApplicationConfig
+from mlia.backend.config import LocalProtocolConfig
+from mlia.backend.config import SystemConfig
+from mlia.backend.execution import deploy_data
+from mlia.backend.execution import execute_commands_locally
+from mlia.backend.execution import ExecutionContext
+from mlia.backend.execution import get_application_and_system
+from mlia.backend.execution import get_application_by_name_and_system
+from mlia.backend.execution import get_file_lock_path
+from mlia.backend.execution import ParamResolver
+from mlia.backend.execution import Reporter
+from mlia.backend.execution import wait
+from mlia.backend.output_parser import Base64OutputParser
+from mlia.backend.output_parser import OutputParser
+from mlia.backend.output_parser import RegexOutputParser
+from mlia.backend.proc import CommandFailedException
+from mlia.backend.system import get_system
+from mlia.backend.system import load_system
def test_context_param_resolver(tmpdir: Any) -> None:
@@ -319,7 +317,7 @@ def test_get_file_lock_path(
def test_get_application_by_name_and_system(monkeypatch: Any) -> None:
"""Test exceptional case for get_application_by_name_and_system."""
monkeypatch.setattr(
- "aiet.backend.execution.get_application",
+ "mlia.backend.execution.get_application",
MagicMock(return_value=[MagicMock(), MagicMock()]),
)
@@ -334,7 +332,7 @@ def test_get_application_by_name_and_system(monkeypatch: Any) -> None:
def test_get_application_and_system(monkeypatch: Any) -> None:
"""Test exceptional case for get_application_and_system."""
monkeypatch.setattr(
- "aiet.backend.execution.get_system", MagicMock(return_value=None)
+ "mlia.backend.execution.get_system", MagicMock(return_value=None)
)
with pytest.raises(ValueError, match="System test_system is not found"):
@@ -379,40 +377,32 @@ def test_deployment_execution_context() -> None:
with pytest.raises(AssertionError):
deploy_data(ctx)
+
+def test_reporter_execution_context(tmp_path: Path) -> None:
+ """Test ExecutionContext creates a reporter when a report file is provided."""
+ # Configure regex parser for the system manually
+ sys = get_system("System 1")
+ assert sys is not None
+ sys.reporting = {
+ "regex": {
+ "simulated_time": {"pattern": "Simulated time.*: (.*)s", "type": "float"}
+ }
+ }
+ report_file_path = tmp_path / "test_report.json"
+
ctx = ExecutionContext(
- app=get_tool("tool_1")[0],
+ app=get_application("application_1")[0],
app_params=[],
- system=None,
+ system=sys,
system_params=[],
+ report_file=report_file_path,
+ )
+ assert isinstance(ctx.reporter, Reporter)
+ assert len(ctx.reporter.parsers) == 2
+ assert any(isinstance(parser, RegexOutputParser) for parser in ctx.reporter.parsers)
+ assert any(
+ isinstance(parser, Base64OutputParser) for parser in ctx.reporter.parsers
)
- assert not ctx.is_deploy_needed
- deploy_data(ctx) # should be a NOP
-
-
-@pytest.mark.parametrize(
- ["tool_name", "system_name", "exception"],
- [
- ("vela", "Corstone-300: Cortex-M55+Ethos-U65", None),
- ("unknown tool", "Corstone-300: Cortex-M55+Ethos-U65", ConfigurationException),
- ("vela", "unknown system", ConfigurationException),
- ("vela", None, ConfigurationException),
- ],
-)
-def test_get_tool_by_system(
- tool_name: str, system_name: Optional[str], exception: Optional[Any]
-) -> None:
- """Test exceptions thrown by function get_tool_by_system()."""
-
- def test() -> None:
- """Test call of get_tool_by_system()."""
- tool = get_tool_by_system(tool_name, system_name)
- assert tool is not None
-
- if exception is None:
- test()
- else:
- with pytest.raises(exception):
- test()
class TestExecuteCommandsLocally:
@@ -465,7 +455,7 @@ class TestExecuteCommandsLocally:
"""Ensure commands following an error-exit-code command don't run."""
# Mock execute_command() function
execute_command_mock = mock.MagicMock()
- monkeypatch.setattr("aiet.utils.proc.execute_command", execute_command_mock)
+ monkeypatch.setattr("mlia.backend.proc.execute_command", execute_command_mock)
# Mock Command object and assign as return value to execute_command()
cmd_mock = mock.MagicMock()
@@ -473,7 +463,9 @@ class TestExecuteCommandsLocally:
# Mock the terminate_command (speed up test)
terminate_command_mock = mock.MagicMock()
- monkeypatch.setattr("aiet.utils.proc.terminate_command", terminate_command_mock)
+ monkeypatch.setattr(
+ "mlia.backend.proc.terminate_command", terminate_command_mock
+ )
# Mock a thrown Exception and assign to Command().exit_code
exit_code_mock = mock.PropertyMock(side_effect=Exception("Exception."))
diff --git a/tests/aiet/test_utils_fs.py b/tests/mlia/test_backend_fs.py
index 46d276e..ff9c2ae 100644
--- a/tests/aiet/test_utils_fs.py
+++ b/tests/mlia/test_backend_fs.py
@@ -10,14 +10,14 @@ from unittest.mock import MagicMock
import pytest
-from aiet.utils.fs import get_resources
-from aiet.utils.fs import read_file_as_bytearray
-from aiet.utils.fs import read_file_as_string
-from aiet.utils.fs import recreate_directory
-from aiet.utils.fs import remove_directory
-from aiet.utils.fs import remove_resource
-from aiet.utils.fs import ResourceType
-from aiet.utils.fs import valid_for_filename
+from mlia.backend.fs import get_backends_path
+from mlia.backend.fs import read_file_as_bytearray
+from mlia.backend.fs import read_file_as_string
+from mlia.backend.fs import recreate_directory
+from mlia.backend.fs import remove_directory
+from mlia.backend.fs import remove_resource
+from mlia.backend.fs import ResourceType
+from mlia.backend.fs import valid_for_filename
@pytest.mark.parametrize(
@@ -29,10 +29,10 @@ from aiet.utils.fs import valid_for_filename
(None, pytest.raises(ResourceWarning)),
],
)
-def test_get_resources(resource_name: ResourceType, expected_path: Any) -> None:
+def test_get_backends_path(resource_name: ResourceType, expected_path: Any) -> None:
"""Test get_resources() with multiple parameters."""
with expected_path:
- resource_path = get_resources(resource_name)
+ resource_path = get_backends_path(resource_name)
assert resource_path.exists()
@@ -41,10 +41,10 @@ def test_remove_resource_wrong_directory(
) -> None:
"""Test removing resource with wrong directory."""
mock_get_resources = MagicMock(return_value=test_applications_path)
- monkeypatch.setattr("aiet.utils.fs.get_resources", mock_get_resources)
+ monkeypatch.setattr("mlia.backend.fs.get_backends_path", mock_get_resources)
mock_shutil_rmtree = MagicMock()
- monkeypatch.setattr("aiet.utils.fs.shutil.rmtree", mock_shutil_rmtree)
+ monkeypatch.setattr("mlia.backend.fs.shutil.rmtree", mock_shutil_rmtree)
with pytest.raises(Exception, match="Resource .* does not exist"):
remove_resource("unknown", "applications")
@@ -58,10 +58,10 @@ def test_remove_resource_wrong_directory(
def test_remove_resource(monkeypatch: Any, test_applications_path: Path) -> None:
"""Test removing resource data."""
mock_get_resources = MagicMock(return_value=test_applications_path)
- monkeypatch.setattr("aiet.utils.fs.get_resources", mock_get_resources)
+ monkeypatch.setattr("mlia.backend.fs.get_backends_path", mock_get_resources)
mock_shutil_rmtree = MagicMock()
- monkeypatch.setattr("aiet.utils.fs.shutil.rmtree", mock_shutil_rmtree)
+ monkeypatch.setattr("mlia.backend.fs.shutil.rmtree", mock_shutil_rmtree)
remove_resource("application1", "applications")
mock_shutil_rmtree.assert_called_once()
diff --git a/tests/mlia/test_tools_aiet_wrapper.py b/tests/mlia/test_backend_manager.py
index ab55b71..c81366f 100644
--- a/tests/mlia/test_tools_aiet_wrapper.py
+++ b/tests/mlia/test_backend_manager.py
@@ -1,6 +1,7 @@
# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
-"""Tests for module tools/aiet_wrapper."""
+"""Tests for module backend/manager."""
+import os
from contextlib import ExitStack as does_not_raise
from pathlib import Path
from typing import Any
@@ -13,20 +14,23 @@ from unittest.mock import PropertyMock
import pytest
-from mlia.tools.aiet_wrapper import AIETRunner
-from mlia.tools.aiet_wrapper import DeviceInfo
-from mlia.tools.aiet_wrapper import estimate_performance
-from mlia.tools.aiet_wrapper import ExecutionParams
-from mlia.tools.aiet_wrapper import GenericInferenceOutputParser
-from mlia.tools.aiet_wrapper import GenericInferenceRunnerEthosU
-from mlia.tools.aiet_wrapper import get_aiet_runner
-from mlia.tools.aiet_wrapper import get_generic_runner
-from mlia.tools.aiet_wrapper import get_system_name
-from mlia.tools.aiet_wrapper import is_supported
-from mlia.tools.aiet_wrapper import ModelInfo
-from mlia.tools.aiet_wrapper import PerformanceMetrics
-from mlia.tools.aiet_wrapper import supported_backends
-from mlia.utils.proc import RunningCommand
+from mlia.backend.application import get_application
+from mlia.backend.common import DataPaths
+from mlia.backend.execution import ExecutionContext
+from mlia.backend.execution import run_application
+from mlia.backend.manager import BackendRunner
+from mlia.backend.manager import DeviceInfo
+from mlia.backend.manager import estimate_performance
+from mlia.backend.manager import ExecutionParams
+from mlia.backend.manager import GenericInferenceOutputParser
+from mlia.backend.manager import GenericInferenceRunnerEthosU
+from mlia.backend.manager import get_generic_runner
+from mlia.backend.manager import get_system_name
+from mlia.backend.manager import is_supported
+from mlia.backend.manager import ModelInfo
+from mlia.backend.manager import PerformanceMetrics
+from mlia.backend.manager import supported_backends
+from mlia.backend.system import get_system
@pytest.mark.parametrize(
@@ -108,16 +112,16 @@ def test_generic_inference_output_parser(
assert parser.missed_keys() == missed_keys
-class TestAIETRunner:
- """Tests for AIETRunner class."""
+class TestBackendRunner:
+ """Tests for BackendRunner class."""
@staticmethod
- def _setup_aiet(
+ def _setup_backends(
monkeypatch: pytest.MonkeyPatch,
available_systems: Optional[List[str]] = None,
available_apps: Optional[List[str]] = None,
) -> None:
- """Set up AIET metadata."""
+ """Set up backend metadata."""
def mock_system(system: str) -> MagicMock:
"""Mock the System instance."""
@@ -134,13 +138,13 @@ class TestAIETRunner:
system_mocks = [mock_system(name) for name in (available_systems or [])]
monkeypatch.setattr(
- "mlia.tools.aiet_wrapper.get_available_systems",
+ "mlia.backend.manager.get_available_systems",
MagicMock(return_value=system_mocks),
)
apps_mock = [mock_app(name) for name in (available_apps or [])]
monkeypatch.setattr(
- "mlia.tools.aiet_wrapper.get_available_applications",
+ "mlia.backend.manager.get_available_applications",
MagicMock(return_value=apps_mock),
)
@@ -159,13 +163,11 @@ class TestAIETRunner:
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Test method is_system_installed."""
- mock_executor = MagicMock()
- aiet_runner = AIETRunner(mock_executor)
+ backend_runner = BackendRunner()
- self._setup_aiet(monkeypatch, available_systems)
+ self._setup_backends(monkeypatch, available_systems)
- assert aiet_runner.is_system_installed(system) == installed
- mock_executor.assert_not_called()
+ assert backend_runner.is_system_installed(system) == installed
@pytest.mark.parametrize(
"available_systems, systems",
@@ -181,28 +183,21 @@ class TestAIETRunner:
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Test method installed_systems."""
- mock_executor = MagicMock()
- aiet_runner = AIETRunner(mock_executor)
+ backend_runner = BackendRunner()
- self._setup_aiet(monkeypatch, available_systems)
- assert aiet_runner.get_installed_systems() == systems
-
- mock_executor.assert_not_called()
+ self._setup_backends(monkeypatch, available_systems)
+ assert backend_runner.get_installed_systems() == systems
@staticmethod
def test_install_system(monkeypatch: pytest.MonkeyPatch) -> None:
"""Test system installation."""
install_system_mock = MagicMock()
- monkeypatch.setattr(
- "mlia.tools.aiet_wrapper.install_system", install_system_mock
- )
+ monkeypatch.setattr("mlia.backend.manager.install_system", install_system_mock)
- mock_executor = MagicMock()
- aiet_runner = AIETRunner(mock_executor)
- aiet_runner.install_system(Path("test_system_path"))
+ backend_runner = BackendRunner()
+ backend_runner.install_system(Path("test_system_path"))
install_system_mock.assert_called_once_with(Path("test_system_path"))
- mock_executor.assert_not_called()
@pytest.mark.parametrize(
"available_systems, systems, expected_result",
@@ -222,14 +217,11 @@ class TestAIETRunner:
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Test method systems_installed."""
- self._setup_aiet(monkeypatch, available_systems)
+ self._setup_backends(monkeypatch, available_systems)
- mock_executor = MagicMock()
- aiet_runner = AIETRunner(mock_executor)
+ backend_runner = BackendRunner()
- assert aiet_runner.systems_installed(systems) is expected_result
-
- mock_executor.assert_not_called()
+ assert backend_runner.systems_installed(systems) is expected_result
@pytest.mark.parametrize(
"available_apps, applications, expected_result",
@@ -249,12 +241,10 @@ class TestAIETRunner:
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Test method applications_installed."""
- self._setup_aiet(monkeypatch, [], available_apps)
- mock_executor = MagicMock()
- aiet_runner = AIETRunner(mock_executor)
+ self._setup_backends(monkeypatch, [], available_apps)
+ backend_runner = BackendRunner()
- assert aiet_runner.applications_installed(applications) is expected_result
- mock_executor.assert_not_called()
+ assert backend_runner.applications_installed(applications) is expected_result
@pytest.mark.parametrize(
"available_apps, applications",
@@ -273,30 +263,23 @@ class TestAIETRunner:
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Test method get_installed_applications."""
- mock_executor = MagicMock()
- self._setup_aiet(monkeypatch, [], available_apps)
-
- aiet_runner = AIETRunner(mock_executor)
- assert applications == aiet_runner.get_installed_applications()
+ self._setup_backends(monkeypatch, [], available_apps)
- mock_executor.assert_not_called()
+ backend_runner = BackendRunner()
+ assert applications == backend_runner.get_installed_applications()
@staticmethod
def test_install_application(monkeypatch: pytest.MonkeyPatch) -> None:
"""Test application installation."""
mock_install_application = MagicMock()
monkeypatch.setattr(
- "mlia.tools.aiet_wrapper.install_application", mock_install_application
+ "mlia.backend.manager.install_application", mock_install_application
)
- mock_executor = MagicMock()
-
- aiet_runner = AIETRunner(mock_executor)
- aiet_runner.install_application(Path("test_application_path"))
+ backend_runner = BackendRunner()
+ backend_runner.install_application(Path("test_application_path"))
mock_install_application.assert_called_once_with(Path("test_application_path"))
- mock_executor.assert_not_called()
-
@pytest.mark.parametrize(
"available_apps, application, installed",
[
@@ -321,66 +304,113 @@ class TestAIETRunner:
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Test method is_application_installed."""
- self._setup_aiet(monkeypatch, [], available_apps)
-
- mock_executor = MagicMock()
- aiet_runner = AIETRunner(mock_executor)
- assert installed == aiet_runner.is_application_installed(application, "system1")
+ self._setup_backends(monkeypatch, [], available_apps)
- mock_executor.assert_not_called()
+ backend_runner = BackendRunner()
+ assert installed == backend_runner.is_application_installed(
+ application, "system1"
+ )
@staticmethod
@pytest.mark.parametrize(
"execution_params, expected_command",
[
(
- ExecutionParams("application1", "system1", [], [], []),
- ["aiet", "application", "run", "-n", "application1", "-s", "system1"],
+ ExecutionParams("application_4", "System 4", [], [], []),
+ ["application_4", [], "System 4", [], []],
),
(
ExecutionParams(
- "application1",
- "system1",
- ["input_file=123.txt", "size=777"],
- ["param1=456", "param2=789"],
+ "application_6",
+ "System 6",
+ ["param1=value2"],
+ ["sys-param1=value2"],
+ [],
+ ),
+ [
+ "application_6",
+ ["param1=value2"],
+ "System 6",
+ ["sys-param1=value2"],
+ [],
+ ],
+ ),
+ ],
+ )
+ def test_run_application_local(
+ monkeypatch: pytest.MonkeyPatch,
+ execution_params: ExecutionParams,
+ expected_command: List[str],
+ ) -> None:
+ """Test method run_application with local systems."""
+ run_app = MagicMock(wraps=run_application)
+ monkeypatch.setattr("mlia.backend.manager.run_application", run_app)
+
+ backend_runner = BackendRunner()
+ backend_runner.run_application(execution_params)
+
+ run_app.assert_called_once_with(*expected_command)
+
+ @staticmethod
+ @pytest.mark.parametrize(
+ "execution_params, expected_command",
+ [
+ (
+ ExecutionParams(
+ "application_1",
+ "System 1",
+ [],
+ [],
["source1.txt:dest1.txt", "source2.txt:dest2.txt"],
),
[
- "aiet",
- "application",
- "run",
- "-n",
- "application1",
- "-s",
- "system1",
- "-p",
- "input_file=123.txt",
- "-p",
- "size=777",
- "--system-param",
- "param1=456",
- "--system-param",
- "param2=789",
- "--deploy",
- "source1.txt:dest1.txt",
- "--deploy",
- "source2.txt:dest2.txt",
+ "application_1",
+ [],
+ "System 1",
+ [],
+ [
+ DataPaths(Path("source1.txt"), "dest1.txt"),
+ DataPaths(Path("source2.txt"), "dest2.txt"),
+ ],
],
),
],
)
- def test_run_application(
- execution_params: ExecutionParams, expected_command: List[str]
+ def test_run_application_connected(
+ monkeypatch: pytest.MonkeyPatch,
+ execution_params: ExecutionParams,
+ expected_command: List[str],
) -> None:
- """Test method run_application."""
- mock_executor = MagicMock()
- mock_running_command = MagicMock()
- mock_executor.submit.return_value = mock_running_command
+ """Test method run_application with connectable systems (SSH)."""
+ app = get_application(execution_params.application, execution_params.system)[0]
+ sys = get_system(execution_params.system)
+
+ assert sys is not None
+
+ connect_mock = MagicMock(return_value=True, name="connect_mock")
+ deploy_mock = MagicMock(return_value=True, name="deploy_mock")
+ run_mock = MagicMock(
+ return_value=(os.EX_OK, bytearray(), bytearray()), name="run_mock"
+ )
+ sys.establish_connection = connect_mock # type: ignore
+ sys.deploy = deploy_mock # type: ignore
+ sys.run = run_mock # type: ignore
+
+ monkeypatch.setattr(
+ "mlia.backend.execution.get_application_and_system",
+ MagicMock(return_value=(app, sys)),
+ )
- aiet_runner = AIETRunner(mock_executor)
- aiet_runner.run_application(execution_params)
+ run_app_mock = MagicMock(wraps=run_application)
+ monkeypatch.setattr("mlia.backend.manager.run_application", run_app_mock)
- mock_executor.submit.assert_called_once_with(expected_command)
+ backend_runner = BackendRunner()
+ backend_runner.run_application(execution_params)
+
+ run_app_mock.assert_called_once_with(*expected_command)
+
+ connect_mock.assert_called_once()
+ assert deploy_mock.call_count == 2
@pytest.mark.parametrize(
@@ -490,16 +520,16 @@ def test_estimate_performance(
backend: str,
expected_error: Any,
test_tflite_model: Path,
- aiet_runner: MagicMock,
+ backend_runner: MagicMock,
) -> None:
"""Test getting performance estimations."""
system_name, system_installed = system
application_name, application_installed = application
- aiet_runner.is_system_installed.return_value = system_installed
- aiet_runner.is_application_installed.return_value = application_installed
+ backend_runner.is_system_installed.return_value = system_installed
+ backend_runner.is_application_installed.return_value = application_installed
- mock_process = create_mock_process(
+ mock_context = create_mock_context(
[
"NPU AXI0_RD_DATA_BEAT_RECEIVED beats: 1",
"NPU AXI0_WR_DATA_BEAT_WRITTEN beats: 2",
@@ -507,12 +537,10 @@ def test_estimate_performance(
"NPU ACTIVE cycles: 4",
"NPU IDLE cycles: 5",
"NPU TOTAL cycles: 6",
- ],
- [],
+ ]
)
- mock_generic_inference_run = RunningCommand(mock_process)
- aiet_runner.run_application.return_value = mock_generic_inference_run
+ backend_runner.run_application.return_value = mock_context
with expected_error:
perf_metrics = estimate_performance(
@@ -529,19 +557,19 @@ def test_estimate_performance(
npu_total_cycles=6,
)
- assert aiet_runner.is_system_installed.called_once_with(system_name)
- assert aiet_runner.is_application_installed.called_once_with(
+ assert backend_runner.is_system_installed.called_once_with(system_name)
+ assert backend_runner.is_application_installed.called_once_with(
application_name, system_name
)
@pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310"))
def test_estimate_performance_insufficient_data(
- aiet_runner: MagicMock, test_tflite_model: Path, backend: str
+ backend_runner: MagicMock, test_tflite_model: Path, backend: str
) -> None:
"""Test that performance could not be estimated when not all data presented."""
- aiet_runner.is_system_installed.return_value = True
- aiet_runner.is_application_installed.return_value = True
+ backend_runner.is_system_installed.return_value = True
+ backend_runner.is_application_installed.return_value = True
no_total_cycles_output = [
"NPU AXI0_RD_DATA_BEAT_RECEIVED beats: 1",
@@ -550,13 +578,9 @@ def test_estimate_performance_insufficient_data(
"NPU ACTIVE cycles: 4",
"NPU IDLE cycles: 5",
]
- mock_process = create_mock_process(
- no_total_cycles_output,
- [],
- )
+ mock_context = create_mock_context(no_total_cycles_output)
- mock_generic_inference_run = RunningCommand(mock_process)
- aiet_runner.run_application.return_value = mock_generic_inference_run
+ backend_runner.run_application.return_value = mock_context
with pytest.raises(
Exception, match="Unable to get performance metrics, insufficient data"
@@ -567,16 +591,14 @@ def test_estimate_performance_insufficient_data(
@pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310"))
def test_estimate_performance_invalid_output(
- test_tflite_model: Path, aiet_runner: MagicMock, backend: str
+ test_tflite_model: Path, backend_runner: MagicMock, backend: str
) -> None:
"""Test estimation could not be done if inference produces unexpected output."""
- aiet_runner.is_system_installed.return_value = True
- aiet_runner.is_application_installed.return_value = True
+ backend_runner.is_system_installed.return_value = True
+ backend_runner.is_application_installed.return_value = True
- mock_process = create_mock_process(
- ["Something", "is", "wrong"], ["What a nice error!"]
- )
- aiet_runner.run_application.return_value = RunningCommand(mock_process)
+ mock_context = create_mock_context(["Something", "is", "wrong"])
+ backend_runner.run_application.return_value = mock_context
with pytest.raises(Exception, match="Unable to get performance metrics"):
estimate_performance(
@@ -586,12 +608,6 @@ def test_estimate_performance_invalid_output(
)
-def test_get_aiet_runner() -> None:
- """Test getting aiet runner."""
- aiet_runner = get_aiet_runner()
- assert isinstance(aiet_runner, AIETRunner)
-
-
def create_mock_process(stdout: List[str], stderr: List[str]) -> MagicMock:
"""Mock underlying process."""
mock_process = MagicMock()
@@ -601,6 +617,18 @@ def create_mock_process(stdout: List[str], stderr: List[str]) -> MagicMock:
return mock_process
+def create_mock_context(stdout: List[str]) -> ExecutionContext:
+ """Mock ExecutionContext."""
+ ctx = ExecutionContext(
+ app=get_application("application_1")[0],
+ app_params=[],
+ system=get_system("System 1"),
+ system_params=[],
+ )
+ ctx.stdout = bytearray("\n".join(stdout).encode("utf-8"))
+ return ctx
+
+
@pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310"))
def test_get_generic_runner(backend: str) -> None:
"""Test function get_generic_runner()."""
@@ -621,8 +649,8 @@ def test_get_generic_runner(backend: str) -> None:
("Corstone-310", "ethos-u55"),
),
)
-def test_aiet_backend_support(backend: str, device_type: str) -> None:
- """Test AIET backend & device support."""
+def test_backend_support(backend: str, device_type: str) -> None:
+ """Test backend & device support."""
assert is_supported(backend)
assert is_supported(backend, device_type)
@@ -714,10 +742,10 @@ class TestGenericInferenceRunnerEthosU:
@staticmethod
@pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310"))
def test_inference_should_fail_if_system_not_installed(
- aiet_runner: MagicMock, test_tflite_model: Path, backend: str
+ backend_runner: MagicMock, test_tflite_model: Path, backend: str
) -> None:
"""Test that inference should fail if system is not installed."""
- aiet_runner.is_system_installed.return_value = False
+ backend_runner.is_system_installed.return_value = False
generic_runner = get_generic_runner(
DeviceInfo("ethos-u55", 256, memory_mode="Shared_Sram"), backend
@@ -731,11 +759,11 @@ class TestGenericInferenceRunnerEthosU:
@staticmethod
@pytest.mark.parametrize("backend", ("Corstone-300", "Corstone-310"))
def test_inference_should_fail_is_apps_not_installed(
- aiet_runner: MagicMock, test_tflite_model: Path, backend: str
+ backend_runner: MagicMock, test_tflite_model: Path, backend: str
) -> None:
"""Test that inference should fail if apps are not installed."""
- aiet_runner.is_system_installed.return_value = True
- aiet_runner.is_application_installed.return_value = False
+ backend_runner.is_system_installed.return_value = True
+ backend_runner.is_application_installed.return_value = False
generic_runner = get_generic_runner(
DeviceInfo("ethos-u55", 256, memory_mode="Shared_Sram"), backend
@@ -749,12 +777,12 @@ class TestGenericInferenceRunnerEthosU:
generic_runner.run(ModelInfo(test_tflite_model), [])
-@pytest.fixture(name="aiet_runner")
-def fixture_aiet_runner(monkeypatch: pytest.MonkeyPatch) -> MagicMock:
- """Mock AIET runner."""
- aiet_runner_mock = MagicMock(spec=AIETRunner)
+@pytest.fixture(name="backend_runner")
+def fixture_backend_runner(monkeypatch: pytest.MonkeyPatch) -> MagicMock:
+ """Mock backend runner."""
+ backend_runner_mock = MagicMock(spec=BackendRunner)
monkeypatch.setattr(
- "mlia.tools.aiet_wrapper.get_aiet_runner",
- MagicMock(return_value=aiet_runner_mock),
+ "mlia.backend.manager.get_backend_runner",
+ MagicMock(return_value=backend_runner_mock),
)
- return aiet_runner_mock
+ return backend_runner_mock
diff --git a/tests/aiet/test_backend_output_parser.py b/tests/mlia/test_backend_output_parser.py
index d659812..d86aac8 100644
--- a/tests/aiet/test_backend_output_parser.py
+++ b/tests/mlia/test_backend_output_parser.py
@@ -8,9 +8,9 @@ from typing import Dict
import pytest
-from aiet.backend.output_parser import Base64OutputParser
-from aiet.backend.output_parser import OutputParser
-from aiet.backend.output_parser import RegexOutputParser
+from mlia.backend.output_parser import Base64OutputParser
+from mlia.backend.output_parser import OutputParser
+from mlia.backend.output_parser import RegexOutputParser
OUTPUT_MATCH_ALL = bytearray(
diff --git a/tests/aiet/test_utils_proc.py b/tests/mlia/test_backend_proc.py
index 9fb48dd..9ca4788 100644
--- a/tests/aiet/test_utils_proc.py
+++ b/tests/mlia/test_backend_proc.py
@@ -1,7 +1,7 @@
# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
# SPDX-License-Identifier: Apache-2.0
# pylint: disable=attribute-defined-outside-init,no-self-use,not-callable
-"""Pytests for testing aiet/utils/proc.py."""
+"""Pytests for testing mlia/backend/proc.py."""
from pathlib import Path
from typing import Any
from unittest import mock
@@ -10,16 +10,16 @@ import psutil
import pytest
from sh import ErrorReturnCode
-from aiet.utils.proc import Command
-from aiet.utils.proc import CommandFailedException
-from aiet.utils.proc import CommandNotFound
-from aiet.utils.proc import parse_command
-from aiet.utils.proc import print_command_stdout
-from aiet.utils.proc import run_and_wait
-from aiet.utils.proc import save_process_info
-from aiet.utils.proc import ShellCommand
-from aiet.utils.proc import terminate_command
-from aiet.utils.proc import terminate_external_process
+from mlia.backend.proc import Command
+from mlia.backend.proc import CommandFailedException
+from mlia.backend.proc import CommandNotFound
+from mlia.backend.proc import parse_command
+from mlia.backend.proc import print_command_stdout
+from mlia.backend.proc import run_and_wait
+from mlia.backend.proc import save_process_info
+from mlia.backend.proc import ShellCommand
+from mlia.backend.proc import terminate_command
+from mlia.backend.proc import terminate_external_process
class TestShellCommand:
@@ -211,12 +211,12 @@ class TestRunAndWait:
"""Init test method."""
self.execute_command_mock = mock.MagicMock()
monkeypatch.setattr(
- "aiet.utils.proc.execute_command", self.execute_command_mock
+ "mlia.backend.proc.execute_command", self.execute_command_mock
)
self.terminate_command_mock = mock.MagicMock()
monkeypatch.setattr(
- "aiet.utils.proc.terminate_command", self.terminate_command_mock
+ "mlia.backend.proc.terminate_command", self.terminate_command_mock
)
def test_if_execute_command_raises_exception(self) -> None:
diff --git a/tests/aiet/test_backend_protocol.py b/tests/mlia/test_backend_protocol.py
index 2103238..35e9986 100644
--- a/tests/aiet/test_backend_protocol.py
+++ b/tests/mlia/test_backend_protocol.py
@@ -10,12 +10,12 @@ from unittest.mock import MagicMock
import paramiko
import pytest
-from aiet.backend.common import ConfigurationException
-from aiet.backend.config import LocalProtocolConfig
-from aiet.backend.protocol import CustomSFTPClient
-from aiet.backend.protocol import LocalProtocol
-from aiet.backend.protocol import ProtocolFactory
-from aiet.backend.protocol import SSHProtocol
+from mlia.backend.common import ConfigurationException
+from mlia.backend.config import LocalProtocolConfig
+from mlia.backend.protocol import CustomSFTPClient
+from mlia.backend.protocol import LocalProtocol
+from mlia.backend.protocol import ProtocolFactory
+from mlia.backend.protocol import SSHProtocol
class TestProtocolFactory:
@@ -94,13 +94,13 @@ class TestSSHProtocol:
self.mock_ssh_channel.recv_stderr_ready.side_effect = [False, True]
monkeypatch.setattr(
- "aiet.backend.protocol.paramiko.client.SSHClient",
+ "mlia.backend.protocol.paramiko.client.SSHClient",
MagicMock(return_value=self.mock_ssh_client),
)
self.mock_sftp_client = MagicMock(spec=CustomSFTPClient)
monkeypatch.setattr(
- "aiet.backend.protocol.CustomSFTPClient.from_transport",
+ "mlia.backend.protocol.CustomSFTPClient.from_transport",
MagicMock(return_value=self.mock_sftp_client),
)
@@ -116,7 +116,7 @@ class TestSSHProtocol:
def test_unable_create_ssh_client(self, monkeypatch: Any) -> None:
"""Test that command should fail if unable to create ssh client instance."""
monkeypatch.setattr(
- "aiet.backend.protocol.paramiko.client.SSHClient",
+ "mlia.backend.protocol.paramiko.client.SSHClient",
MagicMock(side_effect=OSError("Error!")),
)
@@ -201,14 +201,14 @@ class TestCustomSFTPClient:
self.mock_mkdir = MagicMock()
self.mock_put = MagicMock()
monkeypatch.setattr(
- "aiet.backend.protocol.paramiko.SFTPClient.__init__",
+ "mlia.backend.protocol.paramiko.SFTPClient.__init__",
MagicMock(return_value=None),
)
monkeypatch.setattr(
- "aiet.backend.protocol.paramiko.SFTPClient.mkdir", self.mock_mkdir
+ "mlia.backend.protocol.paramiko.SFTPClient.mkdir", self.mock_mkdir
)
monkeypatch.setattr(
- "aiet.backend.protocol.paramiko.SFTPClient.put", self.mock_put
+ "mlia.backend.protocol.paramiko.SFTPClient.put", self.mock_put
)
self.sftp_client = CustomSFTPClient(MagicMock())
diff --git a/tests/aiet/test_backend_source.py b/tests/mlia/test_backend_source.py
index 13b2c6d..84a6a77 100644
--- a/tests/aiet/test_backend_source.py
+++ b/tests/mlia/test_backend_source.py
@@ -11,11 +11,11 @@ from unittest.mock import patch
import pytest
-from aiet.backend.common import ConfigurationException
-from aiet.backend.source import create_destination_and_install
-from aiet.backend.source import DirectorySource
-from aiet.backend.source import get_source
-from aiet.backend.source import TarArchiveSource
+from mlia.backend.common import ConfigurationException
+from mlia.backend.source import create_destination_and_install
+from mlia.backend.source import DirectorySource
+from mlia.backend.source import get_source
+from mlia.backend.source import TarArchiveSource
def test_create_destination_and_install(test_systems_path: Path, tmpdir: Any) -> None:
@@ -28,7 +28,7 @@ def test_create_destination_and_install(test_systems_path: Path, tmpdir: Any) ->
assert (resources / "system1").is_dir()
-@patch("aiet.backend.source.DirectorySource.create_destination", return_value=False)
+@patch("mlia.backend.source.DirectorySource.create_destination", return_value=False)
def test_create_destination_and_install_if_dest_creation_not_required(
mock_ds_create_destination: Any, tmpdir: Any
) -> None:
@@ -66,7 +66,11 @@ def test_create_destination_and_install_if_name_is_empty() -> None:
@pytest.mark.parametrize(
"source_path, expected_class, expected_error",
[
- (Path("applications/application1/"), DirectorySource, does_not_raise()),
+ (
+ Path("backends/applications/application1/"),
+ DirectorySource,
+ does_not_raise(),
+ ),
(
Path("archives/applications/application1.tar.gz"),
TarArchiveSource,
diff --git a/tests/aiet/test_backend_system.py b/tests/mlia/test_backend_system.py
index a581547..21187ff 100644
--- a/tests/aiet/test_backend_system.py
+++ b/tests/mlia/test_backend_system.py
@@ -13,29 +13,29 @@ from unittest.mock import MagicMock
import pytest
-from aiet.backend.common import Command
-from aiet.backend.common import ConfigurationException
-from aiet.backend.common import Param
-from aiet.backend.common import UserParamConfig
-from aiet.backend.config import LocalProtocolConfig
-from aiet.backend.config import ProtocolConfig
-from aiet.backend.config import SSHConfig
-from aiet.backend.config import SystemConfig
-from aiet.backend.controller import SystemController
-from aiet.backend.controller import SystemControllerSingleInstance
-from aiet.backend.protocol import LocalProtocol
-from aiet.backend.protocol import SSHProtocol
-from aiet.backend.protocol import SupportsClose
-from aiet.backend.protocol import SupportsDeploy
-from aiet.backend.system import ControlledSystem
-from aiet.backend.system import get_available_systems
-from aiet.backend.system import get_controller
-from aiet.backend.system import get_system
-from aiet.backend.system import install_system
-from aiet.backend.system import load_system
-from aiet.backend.system import remove_system
-from aiet.backend.system import StandaloneSystem
-from aiet.backend.system import System
+from mlia.backend.common import Command
+from mlia.backend.common import ConfigurationException
+from mlia.backend.common import Param
+from mlia.backend.common import UserParamConfig
+from mlia.backend.config import LocalProtocolConfig
+from mlia.backend.config import ProtocolConfig
+from mlia.backend.config import SSHConfig
+from mlia.backend.config import SystemConfig
+from mlia.backend.controller import SystemController
+from mlia.backend.controller import SystemControllerSingleInstance
+from mlia.backend.protocol import LocalProtocol
+from mlia.backend.protocol import SSHProtocol
+from mlia.backend.protocol import SupportsClose
+from mlia.backend.protocol import SupportsDeploy
+from mlia.backend.system import ControlledSystem
+from mlia.backend.system import get_available_systems
+from mlia.backend.system import get_controller
+from mlia.backend.system import get_system
+from mlia.backend.system import install_system
+from mlia.backend.system import load_system
+from mlia.backend.system import remove_system
+from mlia.backend.system import StandaloneSystem
+from mlia.backend.system import System
def dummy_resolver(
@@ -56,8 +56,13 @@ def test_get_available_systems() -> None:
"""Test get_available_systems mocking get_resources."""
available_systems = get_available_systems()
assert all(isinstance(s, System) for s in available_systems)
- assert len(available_systems) == 3
- assert [str(s) for s in available_systems] == ["System 1", "System 2", "System 4"]
+ assert len(available_systems) == 4
+ assert [str(s) for s in available_systems] == [
+ "System 1",
+ "System 2",
+ "System 4",
+ "System 6",
+ ]
def test_get_system() -> None:
@@ -91,12 +96,12 @@ def test_get_system() -> None:
pytest.raises(Exception, match="Unable to read system definition"),
),
(
- "systems/system1",
+ "backends/systems/system1",
0,
pytest.raises(Exception, match="Systems .* are already installed"),
),
(
- "systems/system3",
+ "backends/systems/system3",
0,
pytest.raises(Exception, match="Unable to read system definition"),
),
@@ -119,7 +124,7 @@ def test_install_system(
"""Test system installation from archive."""
mock_create_destination_and_install = MagicMock()
monkeypatch.setattr(
- "aiet.backend.system.create_destination_and_install",
+ "mlia.backend.system.create_destination_and_install",
mock_create_destination_and_install,
)
@@ -132,7 +137,7 @@ def test_install_system(
def test_remove_system(monkeypatch: Any) -> None:
"""Test system removal."""
mock_remove_backend = MagicMock()
- monkeypatch.setattr("aiet.backend.system.remove_backend", mock_remove_backend)
+ monkeypatch.setattr("mlia.backend.system.remove_backend", mock_remove_backend)
remove_system("some_system_dir")
mock_remove_backend.assert_called_once()
@@ -140,7 +145,7 @@ def test_remove_system(monkeypatch: Any) -> None:
def test_system(monkeypatch: Any) -> None:
"""Test the System class."""
config = SystemConfig(name="System 1")
- monkeypatch.setattr("aiet.backend.system.ProtocolFactory", MagicMock())
+ monkeypatch.setattr("mlia.backend.system.ProtocolFactory", MagicMock())
system = System(config)
assert str(system) == "System 1"
assert system.name == "System 1"
@@ -202,7 +207,7 @@ def test_system_supports_deploy(system_name: str, expected_value: bool) -> None:
def test_system_start_and_stop(monkeypatch: Any, mock_protocol: MagicMock) -> None:
"""Test system start, run commands and stop."""
monkeypatch.setattr(
- "aiet.backend.system.ProtocolFactory.get_protocol",
+ "mlia.backend.system.ProtocolFactory.get_protocol",
MagicMock(return_value=mock_protocol),
)
@@ -516,7 +521,7 @@ def test_system_deploy_data(
) -> None:
"""Test deploy data functionality."""
monkeypatch.setattr(
- "aiet.backend.system.ProtocolFactory.get_protocol",
+ "mlia.backend.system.ProtocolFactory.get_protocol",
MagicMock(return_value=mock_protocol),
)
diff --git a/tests/mlia/test_cli_logging.py b/tests/mlia/test_cli_logging.py
index 7c5f299..3f59cb6 100644
--- a/tests/mlia/test_cli_logging.py
+++ b/tests/mlia/test_cli_logging.py
@@ -32,7 +32,7 @@ def teardown_function() -> None:
(
None,
True,
- """mlia.tools.aiet_wrapper - aiet debug
+ """mlia.backend.manager - backends debug
cli info
mlia.cli - cli debug
""",
@@ -41,11 +41,11 @@ mlia.cli - cli debug
(
"logs",
True,
- """mlia.tools.aiet_wrapper - aiet debug
+ """mlia.backend.manager - backends debug
cli info
mlia.cli - cli debug
""",
- """mlia.tools.aiet_wrapper - DEBUG - aiet debug
+ """mlia.backend.manager - DEBUG - backends debug
mlia.cli - DEBUG - cli debug
""",
),
@@ -64,8 +64,8 @@ def test_setup_logging(
setup_logging(logs_dir_path, verbose)
- aiet_logger = logging.getLogger("mlia.tools.aiet_wrapper")
- aiet_logger.debug("aiet debug")
+ backend_logger = logging.getLogger("mlia.backend.manager")
+ backend_logger.debug("backends debug")
cli_logger = logging.getLogger("mlia.cli")
cli_logger.info("cli info")
diff --git a/tests/mlia/test_devices_ethosu_performance.py b/tests/mlia/test_devices_ethosu_performance.py
index e27efa0..b3e5298 100644
--- a/tests/mlia/test_devices_ethosu_performance.py
+++ b/tests/mlia/test_devices_ethosu_performance.py
@@ -23,6 +23,6 @@ def test_memory_usage_conversion() -> None:
def mock_performance_estimation(monkeypatch: pytest.MonkeyPatch) -> None:
"""Mock performance estimation."""
monkeypatch.setattr(
- "mlia.tools.aiet_wrapper.estimate_performance",
+ "mlia.backend.manager.estimate_performance",
MagicMock(return_value=MagicMock()),
)
diff --git a/tests/aiet/test_resources/application_config.json b/tests/mlia/test_resources/application_config.json
index 2dfcfec..2dfcfec 100644
--- a/tests/aiet/test_resources/application_config.json
+++ b/tests/mlia/test_resources/application_config.json
diff --git a/tests/aiet/test_resources/application_config.json.license b/tests/mlia/test_resources/application_config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/application_config.json.license
+++ b/tests/mlia/test_resources/application_config.json.license
diff --git a/tests/aiet/test_resources/applications/application1/aiet-config.json b/tests/mlia/test_resources/backends/applications/application1/aiet-config.json
index 97f0401..97f0401 100644
--- a/tests/aiet/test_resources/applications/application1/aiet-config.json
+++ b/tests/mlia/test_resources/backends/applications/application1/aiet-config.json
diff --git a/src/aiet/resources/tools/vela/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application1/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/src/aiet/resources/tools/vela/aiet-config.json.license
+++ b/tests/mlia/test_resources/backends/applications/application1/aiet-config.json.license
diff --git a/tests/aiet/test_resources/applications/application2/aiet-config.json b/tests/mlia/test_resources/backends/applications/application2/aiet-config.json
index e9122d3..e9122d3 100644
--- a/tests/aiet/test_resources/applications/application2/aiet-config.json
+++ b/tests/mlia/test_resources/backends/applications/application2/aiet-config.json
diff --git a/tests/aiet/test_resources/applications/application1/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application2/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/applications/application1/aiet-config.json.license
+++ b/tests/mlia/test_resources/backends/applications/application2/aiet-config.json.license
diff --git a/tests/aiet/test_resources/applications/application3/readme.txt b/tests/mlia/test_resources/backends/applications/application3/readme.txt
index 8c72c05..8c72c05 100644
--- a/tests/aiet/test_resources/applications/application3/readme.txt
+++ b/tests/mlia/test_resources/backends/applications/application3/readme.txt
diff --git a/tests/aiet/test_resources/applications/application4/aiet-config.json b/tests/mlia/test_resources/backends/applications/application4/aiet-config.json
index 34dc780..ffb5746 100644
--- a/tests/aiet/test_resources/applications/application4/aiet-config.json
+++ b/tests/mlia/test_resources/backends/applications/application4/aiet-config.json
@@ -10,10 +10,11 @@
],
"commands": {
"build": [
- "cp ../hello_app.txt . # {user_params:0}"
+ "cp ../hello_app.txt .",
+ "echo '{user_params:0}' > params.txt"
],
"run": [
- "{application.build_dir}/hello_app.txt"
+ "cat {application.build_dir}/hello_app.txt"
]
},
"user_params": {
diff --git a/tests/aiet/test_resources/applications/application2/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application4/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/applications/application2/aiet-config.json.license
+++ b/tests/mlia/test_resources/backends/applications/application4/aiet-config.json.license
diff --git a/tests/aiet/test_resources/applications/application4/hello_app.txt b/tests/mlia/test_resources/backends/applications/application4/hello_app.txt
index 2ec0d1d..2ec0d1d 100644
--- a/tests/aiet/test_resources/applications/application4/hello_app.txt
+++ b/tests/mlia/test_resources/backends/applications/application4/hello_app.txt
diff --git a/tests/aiet/test_resources/applications/application5/aiet-config.json b/tests/mlia/test_resources/backends/applications/application5/aiet-config.json
index 5269409..5269409 100644
--- a/tests/aiet/test_resources/applications/application5/aiet-config.json
+++ b/tests/mlia/test_resources/backends/applications/application5/aiet-config.json
diff --git a/tests/aiet/test_resources/applications/application4/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application5/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/applications/application4/aiet-config.json.license
+++ b/tests/mlia/test_resources/backends/applications/application5/aiet-config.json.license
diff --git a/tests/mlia/test_resources/backends/applications/application6/aiet-config.json b/tests/mlia/test_resources/backends/applications/application6/aiet-config.json
new file mode 100644
index 0000000..56ad807
--- /dev/null
+++ b/tests/mlia/test_resources/backends/applications/application6/aiet-config.json
@@ -0,0 +1,42 @@
+[
+ {
+ "name": "application_6",
+ "description": "This is application 6",
+ "supported_systems": [
+ {
+ "name": "System 6"
+ }
+ ],
+ "build_dir": "build",
+ "commands": {
+ "clean": [
+ "echo 'clean'"
+ ],
+ "build": [
+ "echo 'build'"
+ ],
+ "run": [
+ "echo 'run {user_params:param1}'"
+ ],
+ "post_run": [
+ "echo 'post_run'"
+ ]
+ },
+ "user_params": {
+ "build": [],
+ "run": [
+ {
+ "name": "--param1",
+ "description": "Test parameter",
+ "values": [
+ "value1",
+ "value2",
+ "value3"
+ ],
+ "default_value": "value3",
+ "alias": "param1"
+ }
+ ]
+ }
+ }
+]
diff --git a/tests/aiet/test_resources/applications/application5/aiet-config.json.license b/tests/mlia/test_resources/backends/applications/application6/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/applications/application5/aiet-config.json.license
+++ b/tests/mlia/test_resources/backends/applications/application6/aiet-config.json.license
diff --git a/tests/aiet/test_resources/applications/readme.txt b/tests/mlia/test_resources/backends/applications/readme.txt
index a1f8209..a1f8209 100644
--- a/tests/aiet/test_resources/applications/readme.txt
+++ b/tests/mlia/test_resources/backends/applications/readme.txt
diff --git a/tests/aiet/test_resources/systems/system1/aiet-config.json b/tests/mlia/test_resources/backends/systems/system1/aiet-config.json
index 4b5dd19..4b5dd19 100644
--- a/tests/aiet/test_resources/systems/system1/aiet-config.json
+++ b/tests/mlia/test_resources/backends/systems/system1/aiet-config.json
diff --git a/tests/aiet/test_resources/systems/system1/aiet-config.json.license b/tests/mlia/test_resources/backends/systems/system1/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/systems/system1/aiet-config.json.license
+++ b/tests/mlia/test_resources/backends/systems/system1/aiet-config.json.license
diff --git a/tests/aiet/test_resources/systems/system1/system_artifact/dummy.txt b/tests/mlia/test_resources/backends/systems/system1/system_artifact/dummy.txt
index 487e9d8..487e9d8 100644
--- a/tests/aiet/test_resources/systems/system1/system_artifact/dummy.txt
+++ b/tests/mlia/test_resources/backends/systems/system1/system_artifact/dummy.txt
diff --git a/tests/aiet/test_resources/systems/system2/aiet-config.json b/tests/mlia/test_resources/backends/systems/system2/aiet-config.json
index a9e0eb3..a9e0eb3 100644
--- a/tests/aiet/test_resources/systems/system2/aiet-config.json
+++ b/tests/mlia/test_resources/backends/systems/system2/aiet-config.json
diff --git a/tests/aiet/test_resources/systems/system2/aiet-config.json.license b/tests/mlia/test_resources/backends/systems/system2/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/systems/system2/aiet-config.json.license
+++ b/tests/mlia/test_resources/backends/systems/system2/aiet-config.json.license
diff --git a/tests/aiet/test_resources/systems/system3/readme.txt b/tests/mlia/test_resources/backends/systems/system3/readme.txt
index aba5a9c..aba5a9c 100644
--- a/tests/aiet/test_resources/systems/system3/readme.txt
+++ b/tests/mlia/test_resources/backends/systems/system3/readme.txt
diff --git a/tests/aiet/test_resources/systems/system4/aiet-config.json b/tests/mlia/test_resources/backends/systems/system4/aiet-config.json
index 295e00f..7b13160 100644
--- a/tests/aiet/test_resources/systems/system4/aiet-config.json
+++ b/tests/mlia/test_resources/backends/systems/system4/aiet-config.json
@@ -9,7 +9,7 @@
"commands": {
"run": [
"echo {application.name}",
- "cat {application.commands.run:0}"
+ "{application.commands.run:0}"
]
},
"user_params": {
diff --git a/tests/aiet/test_resources/systems/system4/aiet-config.json.license b/tests/mlia/test_resources/backends/systems/system4/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/systems/system4/aiet-config.json.license
+++ b/tests/mlia/test_resources/backends/systems/system4/aiet-config.json.license
diff --git a/tests/mlia/test_resources/backends/systems/system6/aiet-config.json b/tests/mlia/test_resources/backends/systems/system6/aiet-config.json
new file mode 100644
index 0000000..4242f64
--- /dev/null
+++ b/tests/mlia/test_resources/backends/systems/system6/aiet-config.json
@@ -0,0 +1,34 @@
+[
+ {
+ "name": "System 6",
+ "description": "This is system 6",
+ "build_dir": "build",
+ "data_transfer": {
+ "protocol": "local"
+ },
+ "variables": {
+ "var1": "{user_params:sys-param1}"
+ },
+ "commands": {
+ "run": [
+ "echo {application.name}",
+ "{application.commands.run:0}"
+ ]
+ },
+ "user_params": {
+ "run": [
+ {
+ "name": "--sys-param1",
+ "description": "Test parameter",
+ "values": [
+ "value1",
+ "value2",
+ "value3"
+ ],
+ "default_value": "value1",
+ "alias": "sys-param1"
+ }
+ ]
+ }
+ }
+]
diff --git a/tests/aiet/test_resources/tools/tool1/aiet-config.json.license b/tests/mlia/test_resources/backends/systems/system6/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/tools/tool1/aiet-config.json.license
+++ b/tests/mlia/test_resources/backends/systems/system6/aiet-config.json.license
diff --git a/tests/aiet/test_resources/hello_world.json b/tests/mlia/test_resources/hello_world.json
index 8a9a448..8a9a448 100644
--- a/tests/aiet/test_resources/hello_world.json
+++ b/tests/mlia/test_resources/hello_world.json
diff --git a/tests/aiet/test_resources/hello_world.json.license b/tests/mlia/test_resources/hello_world.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/hello_world.json.license
+++ b/tests/mlia/test_resources/hello_world.json.license
diff --git a/tests/aiet/test_resources/scripts/test_backend_run b/tests/mlia/test_resources/scripts/test_backend_run
index 548f577..548f577 100755
--- a/tests/aiet/test_resources/scripts/test_backend_run
+++ b/tests/mlia/test_resources/scripts/test_backend_run
diff --git a/tests/aiet/test_resources/scripts/test_backend_run_script.sh b/tests/mlia/test_resources/scripts/test_backend_run_script.sh
index 548f577..548f577 100644
--- a/tests/aiet/test_resources/scripts/test_backend_run_script.sh
+++ b/tests/mlia/test_resources/scripts/test_backend_run_script.sh
diff --git a/tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json
index fe51488..fe51488 100644
--- a/tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json
+++ b/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json
diff --git a/tests/aiet/test_resources/tools/tool2/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/tools/tool2/aiet-config.json.license
+++ b/tests/mlia/test_resources/various/applications/application_with_empty_config/aiet-config.json.license
diff --git a/tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json
index ff1cf1a..ff1cf1a 100644
--- a/tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json
+++ b/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json
diff --git a/tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/various/applications/application_with_empty_config/aiet-config.json.license
+++ b/tests/mlia/test_resources/various/applications/application_with_valid_config/aiet-config.json.license
diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json
index 724b31b..724b31b 100644
--- a/tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json
+++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json
diff --git a/tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/various/applications/application_with_valid_config/aiet-config.json.license
+++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license
diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json
index 1ebb29c..1ebb29c 100644
--- a/tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json
+++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json
diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/various/applications/application_with_wrong_config1/aiet-config.json.license
+++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license
diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json b/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json
index 410d12d..410d12d 100644
--- a/tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json
+++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json
diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license b/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/various/applications/application_with_wrong_config2/aiet-config.json.license
+++ b/tests/mlia/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license
diff --git a/tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json b/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json
index fe51488..fe51488 100644
--- a/tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json
+++ b/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json
diff --git a/tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license b/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/various/applications/application_with_wrong_config3/aiet-config.json.license
+++ b/tests/mlia/test_resources/various/systems/system_with_empty_config/aiet-config.json.license
diff --git a/tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json b/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json
index 20142e9..20142e9 100644
--- a/tests/aiet/test_resources/various/systems/system_with_valid_config/aiet-config.json
+++ b/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json
diff --git a/tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json.license b/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json.license
index 9b83bfc..9b83bfc 100644
--- a/tests/aiet/test_resources/various/systems/system_with_empty_config/aiet-config.json.license
+++ b/tests/mlia/test_resources/various/systems/system_with_valid_config/aiet-config.json.license
diff --git a/tests/mlia/test_tools_metadata_corstone.py b/tests/mlia/test_tools_metadata_corstone.py
index 2ce3610..017d0c7 100644
--- a/tests/mlia/test_tools_metadata_corstone.py
+++ b/tests/mlia/test_tools_metadata_corstone.py
@@ -9,13 +9,13 @@ from unittest.mock import MagicMock
import pytest
-from mlia.tools.aiet_wrapper import AIETRunner
+from mlia.backend.manager import BackendRunner
from mlia.tools.metadata.common import DownloadAndInstall
from mlia.tools.metadata.common import InstallFromPath
-from mlia.tools.metadata.corstone import AIETBasedInstallation
-from mlia.tools.metadata.corstone import AIETMetadata
from mlia.tools.metadata.corstone import BackendInfo
+from mlia.tools.metadata.corstone import BackendInstallation
from mlia.tools.metadata.corstone import BackendInstaller
+from mlia.tools.metadata.corstone import BackendMetadata
from mlia.tools.metadata.corstone import CompoundPathChecker
from mlia.tools.metadata.corstone import Corstone300Installer
from mlia.tools.metadata.corstone import get_corstone_installations
@@ -40,8 +40,8 @@ def fixture_test_mlia_resources(
return mlia_resources
-def get_aiet_based_installation( # pylint: disable=too-many-arguments
- aiet_runner_mock: MagicMock = MagicMock(),
+def get_backend_installation( # pylint: disable=too-many-arguments
+ backend_runner_mock: MagicMock = MagicMock(),
name: str = "test_name",
description: str = "test_description",
download_artifact: Optional[MagicMock] = None,
@@ -50,11 +50,11 @@ def get_aiet_based_installation( # pylint: disable=too-many-arguments
system_config: Optional[str] = None,
backend_installer: BackendInstaller = MagicMock(),
supported_platforms: Optional[List[str]] = None,
-) -> AIETBasedInstallation:
- """Get AIET based installation."""
- return AIETBasedInstallation(
- aiet_runner=aiet_runner_mock,
- metadata=AIETMetadata(
+) -> BackendInstallation:
+ """Get backend installation."""
+ return BackendInstallation(
+ backend_runner=backend_runner_mock,
+ metadata=BackendMetadata(
name=name,
description=description,
system_config=system_config or "",
@@ -90,10 +90,10 @@ def test_could_be_installed_depends_on_platform(
monkeypatch.setattr(
"mlia.tools.metadata.corstone.all_paths_valid", MagicMock(return_value=True)
)
- aiet_runner_mock = MagicMock(spec=AIETRunner)
+ backend_runner_mock = MagicMock(spec=BackendRunner)
- installation = get_aiet_based_installation(
- aiet_runner_mock,
+ installation = get_backend_installation(
+ backend_runner_mock,
supported_platforms=supported_platforms,
)
assert installation.could_be_installed == expected_result
@@ -103,53 +103,53 @@ def test_get_corstone_installations() -> None:
"""Test function get_corstone_installation."""
installs = get_corstone_installations()
assert len(installs) == 2
- assert all(isinstance(install, AIETBasedInstallation) for install in installs)
+ assert all(isinstance(install, BackendInstallation) for install in installs)
-def test_aiet_based_installation_metadata_resolving() -> None:
- """Test AIET based installation metadata resolving."""
- aiet_runner_mock = MagicMock(spec=AIETRunner)
- installation = get_aiet_based_installation(aiet_runner_mock)
+def test_backend_installation_metadata_resolving() -> None:
+ """Test backend installation metadata resolving."""
+ backend_runner_mock = MagicMock(spec=BackendRunner)
+ installation = get_backend_installation(backend_runner_mock)
assert installation.name == "test_name"
assert installation.description == "test_description"
- aiet_runner_mock.all_installed.return_value = False
+ backend_runner_mock.all_installed.return_value = False
assert installation.already_installed is False
assert installation.could_be_installed is True
-def test_aiet_based_installation_supported_install_types(tmp_path: Path) -> None:
+def test_backend_installation_supported_install_types(tmp_path: Path) -> None:
"""Test supported installation types."""
- installation_no_download_artifact = get_aiet_based_installation()
+ installation_no_download_artifact = get_backend_installation()
assert installation_no_download_artifact.supports(DownloadAndInstall()) is False
- installation_with_download_artifact = get_aiet_based_installation(
+ installation_with_download_artifact = get_backend_installation(
download_artifact=MagicMock()
)
assert installation_with_download_artifact.supports(DownloadAndInstall()) is True
path_checker_mock = MagicMock(return_value=BackendInfo(tmp_path))
- installation_can_install_from_dir = get_aiet_based_installation(
+ installation_can_install_from_dir = get_backend_installation(
path_checker=path_checker_mock
)
assert installation_can_install_from_dir.supports(InstallFromPath(tmp_path)) is True
- any_installation = get_aiet_based_installation()
+ any_installation = get_backend_installation()
assert any_installation.supports("unknown_install_type") is False # type: ignore
-def test_aiet_based_installation_install_wrong_type() -> None:
+def test_backend_installation_install_wrong_type() -> None:
"""Test that operation should fail if wrong install type provided."""
with pytest.raises(Exception, match="Unable to install wrong_install_type"):
- aiet_runner_mock = MagicMock(spec=AIETRunner)
- installation = get_aiet_based_installation(aiet_runner_mock)
+ backend_runner_mock = MagicMock(spec=BackendRunner)
+ installation = get_backend_installation(backend_runner_mock)
installation.install("wrong_install_type") # type: ignore
-def test_aiet_based_installation_install_from_path(
+def test_backend_installation_install_from_path(
tmp_path: Path, test_mlia_resources: Path
) -> None:
"""Test installation from the path."""
@@ -164,9 +164,9 @@ def test_aiet_based_installation_install_from_path(
path_checker_mock = MagicMock(return_value=BackendInfo(dist_dir))
- aiet_runner_mock = MagicMock(spec=AIETRunner)
- installation = get_aiet_based_installation(
- aiet_runner_mock=aiet_runner_mock,
+ backend_runner_mock = MagicMock(spec=BackendRunner)
+ installation = get_backend_installation(
+ backend_runner_mock=backend_runner_mock,
path_checker=path_checker_mock,
apps_resources=[sample_app.name],
system_config="example_config.json",
@@ -175,12 +175,12 @@ def test_aiet_based_installation_install_from_path(
assert installation.supports(InstallFromPath(dist_dir)) is True
installation.install(InstallFromPath(dist_dir))
- aiet_runner_mock.install_system.assert_called_once()
- aiet_runner_mock.install_application.assert_called_once_with(sample_app)
+ backend_runner_mock.install_system.assert_called_once()
+ backend_runner_mock.install_application.assert_called_once_with(sample_app)
@pytest.mark.parametrize("copy_source", [True, False])
-def test_aiet_based_installation_install_from_static_path(
+def test_backend_installation_install_from_static_path(
tmp_path: Path, test_mlia_resources: Path, copy_source: bool
) -> None:
"""Test installation from the predefined path."""
@@ -204,7 +204,7 @@ def test_aiet_based_installation_install_from_static_path(
nested_file = predefined_location_dir / "nested_file.txt"
nested_file.touch()
- aiet_runner_mock = MagicMock(spec=AIETRunner)
+ backend_runner_mock = MagicMock(spec=BackendRunner)
def check_install_dir(install_dir: Path) -> None:
"""Check content of the install dir."""
@@ -220,10 +220,10 @@ def test_aiet_based_installation_install_from_static_path(
assert install_dir / "custom_config.json" in files
- aiet_runner_mock.install_system.side_effect = check_install_dir
+ backend_runner_mock.install_system.side_effect = check_install_dir
- installation = get_aiet_based_installation(
- aiet_runner_mock=aiet_runner_mock,
+ installation = get_backend_installation(
+ backend_runner_mock=backend_runner_mock,
path_checker=StaticPathChecker(
predefined_location,
["file.txt"],
@@ -237,8 +237,8 @@ def test_aiet_based_installation_install_from_static_path(
assert installation.supports(InstallFromPath(predefined_location)) is True
installation.install(InstallFromPath(predefined_location))
- aiet_runner_mock.install_system.assert_called_once()
- aiet_runner_mock.install_application.assert_called_once_with(sample_app)
+ backend_runner_mock.install_system.assert_called_once()
+ backend_runner_mock.install_application.assert_called_once_with(sample_app)
def create_sample_fvp_archive(tmp_path: Path) -> Path:
@@ -259,7 +259,7 @@ def create_sample_fvp_archive(tmp_path: Path) -> Path:
return fvp_archive
-def test_aiet_based_installation_download_and_install(
+def test_backend_installation_download_and_install(
test_mlia_resources: Path, tmp_path: Path
) -> None:
"""Test downloading and installation process."""
@@ -277,9 +277,9 @@ def test_aiet_based_installation_download_and_install(
"""Sample installer."""
return dist_dir
- aiet_runner_mock = MagicMock(spec=AIETRunner)
- installation = get_aiet_based_installation(
- aiet_runner_mock,
+ backend_runner_mock = MagicMock(spec=BackendRunner)
+ installation = get_backend_installation(
+ backend_runner_mock,
download_artifact=download_artifact_mock,
backend_installer=installer,
path_checker=path_checker,
@@ -288,7 +288,7 @@ def test_aiet_based_installation_download_and_install(
installation.install(DownloadAndInstall())
- aiet_runner_mock.install_system.assert_called_once()
+ backend_runner_mock.install_system.assert_called_once()
@pytest.mark.parametrize(