MLIA-650 Implement new CLI changes

Breaking change in the CLI and API: Sub-commands "optimization",
"operators", and "performance" were replaced by "check", which
incorporates compatibility and performance checks, and "optimize" which
is used for optimization. "get_advice" API was adapted to these CLI
changes.

API changes:

* Remove previous advice category "all" that would perform all three
  operations (when possible). Replace them with the ability to pass a
  set of the advice categories.
* Update api.get_advice method docstring to reflect new changes.
* Set default advice category to COMPATIBILITY
* Update core.common.AdviceCategory by changing the "OPERATORS" advice
  category to "COMPATIBILITY" and removing "ALL" enum type.
  Update all subsequent methods that previously used "OPERATORS" to use
  "COMPATIBILITY".
* Update core.context.ExecutionContext to have "COMPATIBILITY" as
  default advice_category instead of "ALL".
* Remove api.generate_supported_operators_report and all related
  functions from cli.commands, cli.helpers, cli.main, cli.options,
  core.helpers
* Update tests to reflect new API changes.

CLI changes:

* Update README.md to contain information on the new CLI
* Remove the ability to generate supported operators support from MLIA
  CLI
* Replace `mlia ops` and `mlia perf` with the new `mlia check` command
  that can be used to perform both operations.
* Replace `mlia opt` with the new `mlia optimize` command.
* Replace `--evaluate-on` flag with `--backend` flag
* Replace `--verbose` flag with `--debug` flag (no behaviour change).
* Remove the ability for the user to select MLIA working directory.
  Create and use a temporary directory in /temp instead.
* Change behaviour of `--output` flag to not format the content
  automatically based on file extension anymore. Instead it will simply
  redirect to a file.
* Add the `--json` flag to specfy that the format of the output should
  be json.
* Add command validators that are used to validate inter-dependent
  flags (e.g. backend validation based on target_profile).
* Add support for selecting built-in backends for both `check` and
  `optimize` commands.
* Add new unit tests and update old ones to test the new CLI changes.
* Update RELEASES.md
* Update copyright notice

Change-Id: Ia6340797c7bee3acbbd26601950e5a16ad5602db

5 files changed, 75 insertions, 49 deletions

diff --git a/src/mlia/core/common.py b/src/mlia/core/common.py
index 6c9dde1..53df001 100644
--- a/src/mlia/core/common.py
+++ b/src/mlia/core/common.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Common module.
 
@@ -13,6 +13,9 @@ from enum import auto
 from enum import Flag
 from typing import Any
 
+from mlia.core.typing import OutputFormat
+from mlia.core.typing import PathOrFileLike
+
 # This type is used as type alias for the items which are being passed around
 # in advisor workflow. There are no restrictions on the type of the
 # object. This alias used only to emphasize the nature of the input/output
@@ -20,31 +23,55 @@ from typing import Any
 DataItem = Any
 
 
+class FormattedFilePath:
+    """Class used to keep track of the format that a path points to."""
+
+    def __init__(self, path: PathOrFileLike, fmt: OutputFormat = "plain_text") -> None:
+        """Init FormattedFilePath."""
+        self._path = path
+        self._fmt = fmt
+
+    @property
+    def fmt(self) -> OutputFormat:
+        """Return file format."""
+        return self._fmt
+
+    @property
+    def path(self) -> PathOrFileLike:
+        """Return file path."""
+        return self._path
+
+    def __eq__(self, other: object) -> bool:
+        """Check for equality with other objects."""
+        if isinstance(other, FormattedFilePath):
+            return other.fmt == self.fmt and other.path == self.path
+
+        return False
+
+    def __repr__(self) -> str:
+        """Represent object."""
+        return f"FormattedFilePath {self.path=}, {self.fmt=}"
+
+
 class AdviceCategory(Flag):
     """Advice category.
 
     Enumeration of advice categories supported by ML Inference Advisor.
     """
 
-    OPERATORS = auto()
+    COMPATIBILITY = auto()
     PERFORMANCE = auto()
     OPTIMIZATION = auto()
-    ALL = (
-        # pylint: disable=unsupported-binary-operation
-        OPERATORS
-        | PERFORMANCE
-        | OPTIMIZATION
-        # pylint: enable=unsupported-binary-operation
-    )
 
     @classmethod
-    def from_string(cls, value: str) -> AdviceCategory:
+    def from_string(cls, values: set[str]) -> set[AdviceCategory]:
         """Resolve enum value from string value."""
         category_names = [item.name for item in AdviceCategory]
-        if not value or value.upper() not in category_names:
-            raise Exception(f"Invalid advice category {value}")
+        for advice_value in values:
+            if advice_value.upper() not in category_names:
+                raise Exception(f"Invalid advice category {advice_value}")
 
-        return AdviceCategory[value.upper()]
+        return {AdviceCategory[value.upper()] for value in values}
 
 
 class NamedEntity(ABC):
diff --git a/src/mlia/core/context.py b/src/mlia/core/context.py
index a4737bb..94aa885 100644
--- a/src/mlia/core/context.py
+++ b/src/mlia/core/context.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Context module.
 
@@ -10,6 +10,7 @@ parameters).
 from __future__ import annotations
 
 import logging
+import tempfile
 from abc import ABC
 from abc import abstractmethod
 from pathlib import Path
@@ -54,7 +55,7 @@ class Context(ABC):
 
     @property
     @abstractmethod
-    def advice_category(self) -> AdviceCategory:
+    def advice_category(self) -> set[AdviceCategory]:
         """Return advice category."""
 
     @property
@@ -71,7 +72,7 @@ class Context(ABC):
     def update(
         self,
         *,
-        advice_category: AdviceCategory,
+        advice_category: set[AdviceCategory],
         event_handlers: list[EventHandler],
         config_parameters: Mapping[str, Any],
     ) -> None:
@@ -79,11 +80,11 @@ class Context(ABC):
 
     def category_enabled(self, category: AdviceCategory) -> bool:
         """Check if category enabled."""
-        return category == self.advice_category
+        return category in self.advice_category
 
     def any_category_enabled(self, *categories: AdviceCategory) -> bool:
         """Return true if any category is enabled."""
-        return self.advice_category in categories
+        return all(category in self.advice_category for category in categories)
 
     def register_event_handlers(self) -> None:
         """Register event handlers."""
@@ -96,7 +97,7 @@ class ExecutionContext(Context):
     def __init__(
         self,
         *,
-        advice_category: AdviceCategory = AdviceCategory.ALL,
+        advice_category: set[AdviceCategory] = None,
         config_parameters: Mapping[str, Any] | None = None,
         working_dir: str | Path | None = None,
         event_handlers: list[EventHandler] | None = None,
@@ -108,7 +109,7 @@ class ExecutionContext(Context):
     ) -> None:
         """Init execution context.
 
-        :param advice_category: requested advice category
+        :param advice_category: requested advice categories
         :param config_parameters: dictionary like object with input parameters
         :param working_dir: path to the directory that will be used as a place
                to store temporary files, logs, models. If not provided then
@@ -124,13 +125,13 @@ class ExecutionContext(Context):
         :param action_resolver: instance of the action resolver that could make
                advice actionable
         """
-        self._advice_category = advice_category
+        self._advice_category = advice_category or {AdviceCategory.COMPATIBILITY}
         self._config_parameters = config_parameters
 
-        self._working_dir_path = Path.cwd()
         if working_dir:
             self._working_dir_path = Path(working_dir)
-            self._working_dir_path.mkdir(exist_ok=True)
+        else:
+            self._working_dir_path = generate_temp_workdir()
 
         self._event_handlers = event_handlers
         self._event_publisher = event_publisher or DefaultEventPublisher()
@@ -140,12 +141,17 @@ class ExecutionContext(Context):
         self._action_resolver = action_resolver or APIActionResolver()
 
     @property
-    def advice_category(self) -> AdviceCategory:
+    def working_dir(self) -> Path:
+        """Return working dir path."""
+        return self._working_dir_path
+
+    @property
+    def advice_category(self) -> set[AdviceCategory]:
         """Return advice category."""
         return self._advice_category
 
     @advice_category.setter
-    def advice_category(self, advice_category: AdviceCategory) -> None:
+    def advice_category(self, advice_category: set[AdviceCategory]) -> None:
         """Setter for the advice category."""
         self._advice_category = advice_category
 
@@ -194,7 +200,7 @@ class ExecutionContext(Context):
     def update(
         self,
         *,
-        advice_category: AdviceCategory,
+        advice_category: set[AdviceCategory],
         event_handlers: list[EventHandler],
         config_parameters: Mapping[str, Any],
     ) -> None:
@@ -206,7 +212,9 @@ class ExecutionContext(Context):
     def __str__(self) -> str:
         """Return string representation."""
         category = (
-            "<not set>" if self.advice_category is None else self.advice_category.name
+            "<not set>"
+            if self.advice_category is None
+            else {x.name for x in self.advice_category}
         )
 
         return (
@@ -215,3 +223,9 @@ class ExecutionContext(Context):
             f"config_parameters={self.config_parameters}, "
             f"verbose={self.verbose}"
         )
+
+
+def generate_temp_workdir() -> Path:
+    """Generate a temporary working dir and returns the path."""
+    working_dir = tempfile.mkdtemp(suffix=None, prefix="mlia-", dir=None)
+    return Path(working_dir)
diff --git a/src/mlia/core/handlers.py b/src/mlia/core/handlers.py
index a3255ae..6e50934 100644
--- a/src/mlia/core/handlers.py
+++ b/src/mlia/core/handlers.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Event handlers module."""
 from __future__ import annotations
@@ -9,6 +9,7 @@ from typing import Callable
 
 from mlia.core.advice_generation import Advice
 from mlia.core.advice_generation import AdviceEvent
+from mlia.core.common import FormattedFilePath
 from mlia.core.events import ActionFinishedEvent
 from mlia.core.events import ActionStartedEvent
 from mlia.core.events import AdviceStageFinishedEvent
@@ -26,7 +27,6 @@ from mlia.core.events import ExecutionFinishedEvent
 from mlia.core.events import ExecutionStartedEvent
 from mlia.core.reporting import Report
 from mlia.core.reporting import Reporter
-from mlia.core.reporting import resolve_output_format
 from mlia.core.typing import PathOrFileLike
 from mlia.utils.console import create_section_header
 
@@ -101,12 +101,12 @@ class WorkflowEventsHandler(SystemEventsHandler):
     def __init__(
         self,
         formatter_resolver: Callable[[Any], Callable[[Any], Report]],
-        output: PathOrFileLike | None = None,
+        output: FormattedFilePath | None = None,
     ) -> None:
         """Init event handler."""
-        output_format = resolve_output_format(output)
+        output_format = output.fmt if output else "plain_text"
         self.reporter = Reporter(formatter_resolver, output_format)
-        self.output = output
+        self.output = output.path if output else None
 
         self.advice: list[Advice] = []
 
diff --git a/src/mlia/core/helpers.py b/src/mlia/core/helpers.py
index f4a9df6..ed43d04 100644
--- a/src/mlia/core/helpers.py
+++ b/src/mlia/core/helpers.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Module for various helper classes."""
 # pylint: disable=unused-argument
@@ -14,10 +14,6 @@ class ActionResolver:
         """Return action details for applying optimizations."""
         return []
 
-    def supported_operators_info(self) -> list[str]:
-        """Return action details for generating supported ops report."""
-        return []
-
     def check_performance(self) -> list[str]:
         """Return action details for checking performance."""
         return []
diff --git a/src/mlia/core/reporting.py b/src/mlia/core/reporting.py
index b96a6b5..19644b2 100644
--- a/src/mlia/core/reporting.py
+++ b/src/mlia/core/reporting.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Reporting module."""
 from __future__ import annotations
@@ -639,14 +639,3 @@ def _apply_format_parameters(
         return report
 
     return wrapper
-
-
-def resolve_output_format(output: PathOrFileLike | None) -> OutputFormat:
-    """Resolve output format based on the output name."""
-    if isinstance(output, (str, Path)):
-        format_from_filename = Path(output).suffix.lstrip(".")
-
-        if format_from_filename in OUTPUT_FORMATS:
-            return cast(OutputFormat, format_from_filename)
-
-    return "plain_text"