diff options
author | Raul Farkas <raul.farkas@arm.com> | 2022-11-29 13:29:04 +0000 |
---|---|---|
committer | Raul Farkas <raul.farkas@arm.com> | 2023-01-10 10:46:07 +0000 |
commit | 5800fc990ed1e36ce7d06670f911fbb12a0ec771 (patch) | |
tree | 294605295cd2624ba63e6ad3df335a2a4b2700ab /tests/test_cli_command_validators.py | |
parent | dcd0bd31985c27e1d07333351b26cf8ad12ad1fd (diff) | |
download | mlia-5800fc990ed1e36ce7d06670f911fbb12a0ec771.tar.gz |
MLIA-650 Implement new CLI changes
Breaking change in the CLI and API: Sub-commands "optimization",
"operators", and "performance" were replaced by "check", which
incorporates compatibility and performance checks, and "optimize" which
is used for optimization. "get_advice" API was adapted to these CLI
changes.
API changes:
* Remove previous advice category "all" that would perform all three
operations (when possible). Replace them with the ability to pass a
set of the advice categories.
* Update api.get_advice method docstring to reflect new changes.
* Set default advice category to COMPATIBILITY
* Update core.common.AdviceCategory by changing the "OPERATORS" advice
category to "COMPATIBILITY" and removing "ALL" enum type.
Update all subsequent methods that previously used "OPERATORS" to use
"COMPATIBILITY".
* Update core.context.ExecutionContext to have "COMPATIBILITY" as
default advice_category instead of "ALL".
* Remove api.generate_supported_operators_report and all related
functions from cli.commands, cli.helpers, cli.main, cli.options,
core.helpers
* Update tests to reflect new API changes.
CLI changes:
* Update README.md to contain information on the new CLI
* Remove the ability to generate supported operators support from MLIA
CLI
* Replace `mlia ops` and `mlia perf` with the new `mlia check` command
that can be used to perform both operations.
* Replace `mlia opt` with the new `mlia optimize` command.
* Replace `--evaluate-on` flag with `--backend` flag
* Replace `--verbose` flag with `--debug` flag (no behaviour change).
* Remove the ability for the user to select MLIA working directory.
Create and use a temporary directory in /temp instead.
* Change behaviour of `--output` flag to not format the content
automatically based on file extension anymore. Instead it will simply
redirect to a file.
* Add the `--json` flag to specfy that the format of the output should
be json.
* Add command validators that are used to validate inter-dependent
flags (e.g. backend validation based on target_profile).
* Add support for selecting built-in backends for both `check` and
`optimize` commands.
* Add new unit tests and update old ones to test the new CLI changes.
* Update RELEASES.md
* Update copyright notice
Change-Id: Ia6340797c7bee3acbbd26601950e5a16ad5602db
Diffstat (limited to 'tests/test_cli_command_validators.py')
-rw-r--r-- | tests/test_cli_command_validators.py | 167 |
1 files changed, 167 insertions, 0 deletions
diff --git a/tests/test_cli_command_validators.py b/tests/test_cli_command_validators.py new file mode 100644 index 0000000..13514a5 --- /dev/null +++ b/tests/test_cli_command_validators.py @@ -0,0 +1,167 @@ +# SPDX-FileCopyrightText: Copyright 2023, Arm Limited and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +"""Tests for cli.command_validators module.""" +from __future__ import annotations + +import argparse +from unittest.mock import MagicMock + +import pytest + +from mlia.cli.command_validators import validate_backend +from mlia.cli.command_validators import validate_check_target_profile + + +@pytest.mark.parametrize( + "target_profile, category, expected_warnings, sys_exits", + [ + ["ethos-u55-256", {"compatibility", "performance"}, [], False], + ["ethos-u55-256", {"compatibility"}, [], False], + ["ethos-u55-256", {"performance"}, [], False], + [ + "tosa", + {"compatibility", "performance"}, + [ + ( + "\nWARNING: Performance checks skipped as they cannot be " + "performed with target profile tosa." + ) + ], + False, + ], + [ + "tosa", + {"performance"}, + [ + ( + "\nWARNING: Performance checks skipped as they cannot be " + "performed with target profile tosa. No operation was performed." + ) + ], + True, + ], + ["tosa", "compatibility", [], False], + [ + "cortex-a", + {"performance"}, + [ + ( + "\nWARNING: Performance checks skipped as they cannot be " + "performed with target profile cortex-a. " + "No operation was performed." + ) + ], + True, + ], + [ + "cortex-a", + {"compatibility", "performance"}, + [ + ( + "\nWARNING: Performance checks skipped as they cannot be " + "performed with target profile cortex-a." + ) + ], + False, + ], + ["cortex-a", "compatibility", [], False], + ], +) +def test_validate_check_target_profile( + caplog: pytest.LogCaptureFixture, + target_profile: str, + category: set[str], + expected_warnings: list[str], + sys_exits: bool, +) -> None: + """Test outcomes of category dependent target profile validation.""" + # Capture if program terminates + if sys_exits: + with pytest.raises(SystemExit) as sys_ex: + validate_check_target_profile(target_profile, category) + assert sys_ex.value.code == 0 + return + + validate_check_target_profile(target_profile, category) + + log_records = caplog.records + # Get all log records with level 30 (warning level) + warning_messages = {x.message for x in log_records if x.levelno == 30} + # Ensure the warnings coincide with the expected ones + assert warning_messages == set(expected_warnings) + + +@pytest.mark.parametrize( + "input_target_profile, input_backends, throws_exception," + "exception_message, output_backends", + [ + [ + "tosa", + ["Vela"], + True, + "Vela backend not supported with target-profile tosa.", + None, + ], + [ + "tosa", + ["Corstone-300, Vela"], + True, + "Corstone-300, Vela backend not supported with target-profile tosa.", + None, + ], + [ + "cortex-a", + ["Corstone-310", "tosa-checker"], + True, + "Corstone-310, tosa-checker backend not supported " + "with target-profile cortex-a.", + None, + ], + [ + "ethos-u55-256", + ["tosa-checker", "Corstone-310"], + True, + "tosa-checker backend not supported with target-profile ethos-u55-256.", + None, + ], + ["tosa", None, False, None, ["tosa-checker"]], + ["cortex-a", None, False, None, ["armnn-tflitedelegate"]], + ["tosa", ["tosa-checker"], False, None, ["tosa-checker"]], + ["cortex-a", ["armnn-tflitedelegate"], False, None, ["armnn-tflitedelegate"]], + [ + "ethos-u55-256", + ["Vela", "Corstone-300"], + False, + None, + ["Vela", "Corstone-300"], + ], + [ + "ethos-u55-256", + None, + False, + None, + ["Vela", "Corstone-300"], + ], + ], +) +def test_validate_backend( + monkeypatch: pytest.MonkeyPatch, + input_target_profile: str, + input_backends: list[str] | None, + throws_exception: bool, + exception_message: str, + output_backends: list[str] | None, +) -> None: + """Test backend validation with target-profiles and backends.""" + monkeypatch.setattr( + "mlia.cli.config.get_available_backends", + MagicMock(return_value=["Vela", "Corstone-300"]), + ) + + if throws_exception: + with pytest.raises(argparse.ArgumentError) as err: + validate_backend(input_target_profile, input_backends) + assert str(err.value.message) == exception_message + return + + assert validate_backend(input_target_profile, input_backends) == output_backends |