From ba9aeace4c3d3f4f31830d68b5cc7dd3f4bf1dde Mon Sep 17 00:00:00 2001 From: Benjamin Klimczak Date: Tue, 6 Dec 2022 16:55:51 +0000 Subject: MLIA-468 Skip e2e tests if backend is unavailable The behavior can be modified using the flag '--no-skip' for the e2e tests, i.e. providing the flag will let the tests fail when the backend is unavailable and otherwise these tests are skipped per default. Also use commas instead of semi-colons to separate Python versions. Change-Id: Ib2b9f6c66ce4d500b0d50080d127c06e43616c3d --- Dockerfile | 2 +- docker/install_python_versions.sh | 2 +- tests_e2e/conftest.py | 23 ++++++++++ tests_e2e/test_e2e.py | 93 +++++++++++---------------------------- 4 files changed, 50 insertions(+), 70 deletions(-) create mode 100644 tests_e2e/conftest.py diff --git a/Dockerfile b/Dockerfile index c00482b..28a2707 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,7 +79,7 @@ RUN curl https://pyenv.run | bash ENV PYENV_ROOT /home/foo/.pyenv ENV PATH $PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH -# Python versions separated by semicolons. E.g. "3.8;3.9" +# Python versions separated by commas, e.g. "3.8,3.9" ARG PYTHON_VERSIONS # Install Python versions and set them to be available globally COPY docker/install_python_versions.sh /home/foo diff --git a/docker/install_python_versions.sh b/docker/install_python_versions.sh index c3cd976..6343e63 100755 --- a/docker/install_python_versions.sh +++ b/docker/install_python_versions.sh @@ -27,7 +27,7 @@ function set_python_global_versions() { # available globally. py_versions=$1 parameters=$2 -for i in $(echo "$py_versions" | tr ";" "\n") +for i in $(echo "$py_versions" | tr "," "\n") do pyenv install "$i":latest done diff --git a/tests_e2e/conftest.py b/tests_e2e/conftest.py new file mode 100644 index 0000000..8cb85f7 --- /dev/null +++ b/tests_e2e/conftest.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +"""Test configuration for the end-to-end tests.""" +from typing import cast + +import pytest + + +def pytest_addoption(parser: pytest.Parser) -> None: + """Add command line options to control the e2e test behavior.""" + parser.addoption( + "--no-skip", + action="store_true", + help="If set, forces tests to run regardless of the availability of " + "MLIA backends required for the test. If not set, tests will be " + "skipped if the required backend is not available.", + ) + + +@pytest.fixture(scope="session", name="no_skip") +def fixture_no_skip(request: pytest.FixtureRequest) -> bool: + """Fixture for easy access to the '--no-skip' parameter.""" + return cast(bool, request.config.getoption("--no-skip", default=True)) diff --git a/tests_e2e/test_e2e.py b/tests_e2e/test_e2e.py index 439723b..fb40735 100644 --- a/tests_e2e/test_e2e.py +++ b/tests_e2e/test_e2e.py @@ -14,12 +14,13 @@ from contextlib import ExitStack from dataclasses import dataclass from pathlib import Path from typing import Any -from typing import cast from typing import Generator from typing import Iterable import pytest +from mlia.cli.config import get_available_backends +from mlia.cli.config import get_default_backends from mlia.cli.main import get_commands from mlia.cli.main import get_possible_command_names from mlia.cli.main import init_commands @@ -34,52 +35,6 @@ pytestmark = pytest.mark.e2e VALID_COMMANDS = get_possible_command_names(get_commands()) -@dataclass -class CommandExecution: - """Command execution.""" - - parsed_args: argparse.Namespace - parameters: list[str] - - def __str__(self) -> str: - """Return string representation.""" - command = self._get_param("command") - target_profile = self._get_param("target_profile") - - model_path = Path(self._get_param("model")) - model = model_path.name - - evaluate_on = self._get_param("evaluate_on", None) - evalute_on_opts = f" evaluate_on={','.join(evaluate_on)}" if evaluate_on else "" - - opt_type = self._get_param("optimization_type", None) - opt_target = self._get_param("optimization_target", None) - - opts = ( - f" optimization={opts}" - if (opts := self._merge(opt_type, opt_target)) - else "" - ) - - return f"command {command}: {target_profile=} {model=}{evalute_on_opts}{opts}" - - def _get_param(self, param: str, default: str | None = "unknown") -> Any: - return getattr(self.parsed_args, param, default) - - @staticmethod - def _merge(value1: str, value2: str, sep: str = ",") -> str: - """Split and merge values into a string.""" - if not value1 or not value2: - return "" - - values = [ - f"{v1} {v2}" - for v1, v2 in zip(str(value1).split(sep), str(value2).split(sep)) - ] - - return ",".join(values) - - @dataclass class ExecutionConfiguration: """Execution configuration.""" @@ -271,38 +226,40 @@ def get_all_commands_combinations(executions: Any) -> Generator[list[str], None, ) -def try_to_parse_args(combination: list[str]) -> argparse.Namespace: - """Try to parse command.""" - try: - # parser contains some static data and could not be reused - # this is why it is being created for each combination - args_parser = get_args_parser() - return cast(argparse.Namespace, args_parser.parse_args(combination)) - except SystemExit as err: - raise Exception( - f"Configuration contains invalid parameters: {combination}" - ) from err +def check_args(args: list[str], no_skip: bool) -> None: + """Check the arguments and skip/fail test cases based on that.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--evaluate-on", + help="Backends to use for evaluation (default: %(default)s)", + nargs="*", + default=get_default_backends(), + ) + + parsed_args, _ = parser.parse_known_args(args) + required_backends = set(parsed_args.evaluate_on) + available_backends = set(get_available_backends()) + missing_backends = required_backends.difference(available_backends) + + if missing_backends and not no_skip: + pytest.skip(f"Missing backend(s): {','.join(missing_backends)}") -def get_execution_definitions() -> Generator[CommandExecution, None, None]: +def get_execution_definitions() -> Generator[list[str], None, None]: """Collect all execution definitions from configuration file.""" config_file = get_config_file() executions = get_config_content(config_file) executions = resolve_parameters(executions) - for combination in get_all_commands_combinations(executions): - # parse parameters to generate meaningful test description - args = try_to_parse_args(combination) - - yield CommandExecution(args, combination) + return get_all_commands_combinations(executions) class TestEndToEnd: """End to end command tests.""" - @pytest.mark.parametrize("command_execution", get_execution_definitions(), ids=str) - def test_command(self, command_execution: CommandExecution) -> None: + @pytest.mark.parametrize("command", get_execution_definitions(), ids=str) + def test_e2e(self, command: list[str], no_skip: bool) -> None: """Test MLIA command with the provided parameters.""" - mlia_command = ["mlia", *command_execution.parameters] - + check_args(command, no_skip) + mlia_command = ["mlia", *command] run_command(mlia_command) -- cgit v1.2.1