MLIA-650 Implement new CLI changes

Breaking change in the CLI and API: Sub-commands "optimization",
"operators", and "performance" were replaced by "check", which
incorporates compatibility and performance checks, and "optimize" which
is used for optimization. "get_advice" API was adapted to these CLI
changes.

API changes:

* Remove previous advice category "all" that would perform all three
  operations (when possible). Replace them with the ability to pass a
  set of the advice categories.
* Update api.get_advice method docstring to reflect new changes.
* Set default advice category to COMPATIBILITY
* Update core.common.AdviceCategory by changing the "OPERATORS" advice
  category to "COMPATIBILITY" and removing "ALL" enum type.
  Update all subsequent methods that previously used "OPERATORS" to use
  "COMPATIBILITY".
* Update core.context.ExecutionContext to have "COMPATIBILITY" as
  default advice_category instead of "ALL".
* Remove api.generate_supported_operators_report and all related
  functions from cli.commands, cli.helpers, cli.main, cli.options,
  core.helpers
* Update tests to reflect new API changes.

CLI changes:

* Update README.md to contain information on the new CLI
* Remove the ability to generate supported operators support from MLIA
  CLI
* Replace `mlia ops` and `mlia perf` with the new `mlia check` command
  that can be used to perform both operations.
* Replace `mlia opt` with the new `mlia optimize` command.
* Replace `--evaluate-on` flag with `--backend` flag
* Replace `--verbose` flag with `--debug` flag (no behaviour change).
* Remove the ability for the user to select MLIA working directory.
  Create and use a temporary directory in /temp instead.
* Change behaviour of `--output` flag to not format the content
  automatically based on file extension anymore. Instead it will simply
  redirect to a file.
* Add the `--json` flag to specfy that the format of the output should
  be json.
* Add command validators that are used to validate inter-dependent
  flags (e.g. backend validation based on target_profile).
* Add support for selecting built-in backends for both `check` and
  `optimize` commands.
* Add new unit tests and update old ones to test the new CLI changes.
* Update RELEASES.md
* Update copyright notice

Change-Id: Ia6340797c7bee3acbbd26601950e5a16ad5602db

9 files changed, 52 insertions, 48 deletions

diff --git a/src/mlia/target/cortex_a/advice_generation.py b/src/mlia/target/cortex_a/advice_generation.py
index b68106e..98e8c06 100644
--- a/src/mlia/target/cortex_a/advice_generation.py
+++ b/src/mlia/target/cortex_a/advice_generation.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Cortex-A advice generation."""
 from functools import singledispatchmethod
@@ -29,7 +29,7 @@ class CortexAAdviceProducer(FactBasedAdviceProducer):
         """Produce advice."""
 
     @produce_advice.register
-    @advice_category(AdviceCategory.ALL, AdviceCategory.OPERATORS)
+    @advice_category(AdviceCategory.COMPATIBILITY)
     def handle_model_is_cortex_a_compatible(
         self, data_item: ModelIsCortexACompatible
     ) -> None:
@@ -43,7 +43,7 @@ class CortexAAdviceProducer(FactBasedAdviceProducer):
         )
 
     @produce_advice.register
-    @advice_category(AdviceCategory.ALL, AdviceCategory.OPERATORS)
+    @advice_category(AdviceCategory.COMPATIBILITY)
     def handle_model_is_not_cortex_a_compatible(
         self, data_item: ModelIsNotCortexACompatible
     ) -> None:
@@ -83,7 +83,7 @@ class CortexAAdviceProducer(FactBasedAdviceProducer):
         )
 
     @produce_advice.register
-    @advice_category(AdviceCategory.ALL, AdviceCategory.OPERATORS)
+    @advice_category(AdviceCategory.COMPATIBILITY)
     def handle_model_is_not_tflite_compatible(
         self, data_item: ModelIsNotTFLiteCompatible
     ) -> None:
@@ -127,7 +127,7 @@ class CortexAAdviceProducer(FactBasedAdviceProducer):
             )
 
     @produce_advice.register
-    @advice_category(AdviceCategory.ALL, AdviceCategory.OPERATORS)
+    @advice_category(AdviceCategory.COMPATIBILITY)
     def handle_tflite_check_failed(
         self, _data_item: TFLiteCompatibilityCheckFailed
     ) -> None:
@@ -140,7 +140,7 @@ class CortexAAdviceProducer(FactBasedAdviceProducer):
         )
 
     @produce_advice.register
-    @advice_category(AdviceCategory.ALL, AdviceCategory.OPERATORS)
+    @advice_category(AdviceCategory.COMPATIBILITY)
     def handle_model_has_custom_operators(
         self, _data_item: ModelHasCustomOperators
     ) -> None:
diff --git a/src/mlia/target/cortex_a/advisor.py b/src/mlia/target/cortex_a/advisor.py
index 5912e38..b649f0d 100644
--- a/src/mlia/target/cortex_a/advisor.py
+++ b/src/mlia/target/cortex_a/advisor.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Cortex-A MLIA module."""
 from __future__ import annotations
@@ -10,12 +10,12 @@ from mlia.core.advice_generation import AdviceProducer
 from mlia.core.advisor import DefaultInferenceAdvisor
 from mlia.core.advisor import InferenceAdvisor
 from mlia.core.common import AdviceCategory
+from mlia.core.common import FormattedFilePath
 from mlia.core.context import Context
 from mlia.core.context import ExecutionContext
 from mlia.core.data_analysis import DataAnalyzer
 from mlia.core.data_collection import DataCollector
 from mlia.core.events import Event
-from mlia.core.typing import PathOrFileLike
 from mlia.target.cortex_a.advice_generation import CortexAAdviceProducer
 from mlia.target.cortex_a.config import CortexAConfiguration
 from mlia.target.cortex_a.data_analysis import CortexADataAnalyzer
@@ -38,7 +38,7 @@ class CortexAInferenceAdvisor(DefaultInferenceAdvisor):
 
         collectors: list[DataCollector] = []
 
-        if AdviceCategory.OPERATORS in context.advice_category:
+        if context.category_enabled(AdviceCategory.COMPATIBILITY):
             collectors.append(CortexAOperatorCompatibility(model))
 
         return collectors
@@ -67,7 +67,7 @@ def configure_and_get_cortexa_advisor(
     context: ExecutionContext,
     target_profile: str,
     model: str | Path,
-    output: PathOrFileLike | None = None,
+    output: FormattedFilePath | None = None,
     **_extra_args: Any,
 ) -> InferenceAdvisor:
     """Create and configure Cortex-A advisor."""
diff --git a/src/mlia/target/cortex_a/handlers.py b/src/mlia/target/cortex_a/handlers.py
index b2d5faa..d6acde5 100644
--- a/src/mlia/target/cortex_a/handlers.py
+++ b/src/mlia/target/cortex_a/handlers.py
@@ -1,13 +1,13 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Event handler."""
 from __future__ import annotations
 
 import logging
 
+from mlia.core.common import FormattedFilePath
 from mlia.core.events import CollectedDataEvent
 from mlia.core.handlers import WorkflowEventsHandler
-from mlia.core.typing import PathOrFileLike
 from mlia.nn.tensorflow.tflite_compat import TFLiteCompatibilityInfo
 from mlia.target.cortex_a.events import CortexAAdvisorEventHandler
 from mlia.target.cortex_a.events import CortexAAdvisorStartedEvent
@@ -20,7 +20,7 @@ logger = logging.getLogger(__name__)
 class CortexAEventHandler(WorkflowEventsHandler, CortexAAdvisorEventHandler):
     """CLI event handler."""
 
-    def __init__(self, output: PathOrFileLike | None = None) -> None:
+    def __init__(self, output: FormattedFilePath | None = None) -> None:
         """Init event handler."""
         super().__init__(cortex_a_formatters, output)
 
diff --git a/src/mlia/target/ethos_u/advice_generation.py b/src/mlia/target/ethos_u/advice_generation.py
index edd78fd..daae4f4 100644
--- a/src/mlia/target/ethos_u/advice_generation.py
+++ b/src/mlia/target/ethos_u/advice_generation.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Ethos-U advice generation."""
 from __future__ import annotations
@@ -26,7 +26,7 @@ class EthosUAdviceProducer(FactBasedAdviceProducer):
         """Produce advice."""
 
     @produce_advice.register
-    @advice_category(AdviceCategory.OPERATORS, AdviceCategory.ALL)
+    @advice_category(AdviceCategory.COMPATIBILITY)
     def handle_cpu_only_ops(self, data_item: HasCPUOnlyOperators) -> None:
         """Advice for CPU only operators."""
         cpu_only_ops = ",".join(sorted(set(data_item.cpu_only_ops)))
@@ -40,11 +40,10 @@ class EthosUAdviceProducer(FactBasedAdviceProducer):
                 "Using operators that are supported by the NPU will "
                 "improve performance.",
             ]
-            + self.context.action_resolver.supported_operators_info()
         )
 
     @produce_advice.register
-    @advice_category(AdviceCategory.OPERATORS, AdviceCategory.ALL)
+    @advice_category(AdviceCategory.COMPATIBILITY)
     def handle_unsupported_operators(
         self, data_item: HasUnsupportedOnNPUOperators
     ) -> None:
@@ -60,21 +59,25 @@ class EthosUAdviceProducer(FactBasedAdviceProducer):
         )
 
     @produce_advice.register
-    @advice_category(AdviceCategory.OPERATORS, AdviceCategory.ALL)
+    @advice_category(AdviceCategory.COMPATIBILITY)
     def handle_all_operators_supported(
         self, _data_item: AllOperatorsSupportedOnNPU
     ) -> None:
         """Advice if all operators supported."""
-        self.add_advice(
-            [
-                "You don't have any unsupported operators, your model will "
-                "run completely on NPU."
-            ]
-            + self.context.action_resolver.check_performance()
-        )
+        advice = [
+            "You don't have any unsupported operators, your model will "
+            "run completely on NPU."
+        ]
+        if self.context.advice_category != (
+            AdviceCategory.COMPATIBILITY,
+            AdviceCategory.PERFORMANCE,
+        ):
+            advice += self.context.action_resolver.check_performance()
+
+        self.add_advice(advice)
 
     @produce_advice.register
-    @advice_category(AdviceCategory.OPTIMIZATION, AdviceCategory.ALL)
+    @advice_category(AdviceCategory.OPTIMIZATION)
     def handle_optimization_results(self, data_item: OptimizationResults) -> None:
         """Advice based on optimization results."""
         if not data_item.diffs or len(data_item.diffs) != 1:
@@ -202,5 +205,6 @@ class EthosUStaticAdviceProducer(ContextAwareAdviceProducer):
                 )
             ],
         }
-
-        return advice_per_category.get(self.context.advice_category, [])
+        if len(self.context.advice_category) == 1:
+            return advice_per_category.get(list(self.context.advice_category)[0], [])
+        return []
diff --git a/src/mlia/target/ethos_u/advisor.py b/src/mlia/target/ethos_u/advisor.py
index b9d64ff..640c3e1 100644
--- a/src/mlia/target/ethos_u/advisor.py
+++ b/src/mlia/target/ethos_u/advisor.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Ethos-U MLIA module."""
 from __future__ import annotations
@@ -10,12 +10,12 @@ from mlia.core.advice_generation import AdviceProducer
 from mlia.core.advisor import DefaultInferenceAdvisor
 from mlia.core.advisor import InferenceAdvisor
 from mlia.core.common import AdviceCategory
+from mlia.core.common import FormattedFilePath
 from mlia.core.context import Context
 from mlia.core.context import ExecutionContext
 from mlia.core.data_analysis import DataAnalyzer
 from mlia.core.data_collection import DataCollector
 from mlia.core.events import Event
-from mlia.core.typing import PathOrFileLike
 from mlia.nn.tensorflow.utils import is_tflite_model
 from mlia.target.ethos_u.advice_generation import EthosUAdviceProducer
 from mlia.target.ethos_u.advice_generation import EthosUStaticAdviceProducer
@@ -46,7 +46,7 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor):
 
         collectors: list[DataCollector] = []
 
-        if AdviceCategory.OPERATORS in context.advice_category:
+        if context.category_enabled(AdviceCategory.COMPATIBILITY):
             collectors.append(EthosUOperatorCompatibility(model, device))
 
         # Performance and optimization are mutually exclusive.
@@ -57,18 +57,18 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor):
                 raise Exception(
                     "Command 'optimization' is not supported for TensorFlow Lite files."
                 )
-            if AdviceCategory.PERFORMANCE in context.advice_category:
+            if context.category_enabled(AdviceCategory.PERFORMANCE):
                 collectors.append(EthosUPerformance(model, device, backends))
         else:
             # Keras/SavedModel: Prefer optimization
-            if AdviceCategory.OPTIMIZATION in context.advice_category:
+            if context.category_enabled(AdviceCategory.OPTIMIZATION):
                 optimization_settings = self._get_optimization_settings(context)
                 collectors.append(
                     EthosUOptimizationPerformance(
                         model, device, optimization_settings, backends
                     )
                 )
-            elif AdviceCategory.PERFORMANCE in context.advice_category:
+            elif context.category_enabled(AdviceCategory.PERFORMANCE):
                 collectors.append(EthosUPerformance(model, device, backends))
 
         return collectors
@@ -126,7 +126,7 @@ def configure_and_get_ethosu_advisor(
     context: ExecutionContext,
     target_profile: str,
     model: str | Path,
-    output: PathOrFileLike | None = None,
+    output: FormattedFilePath | None = None,
     **extra_args: Any,
 ) -> InferenceAdvisor:
     """Create and configure Ethos-U advisor."""
diff --git a/src/mlia/target/ethos_u/handlers.py b/src/mlia/target/ethos_u/handlers.py
index 84a9554..91f6015 100644
--- a/src/mlia/target/ethos_u/handlers.py
+++ b/src/mlia/target/ethos_u/handlers.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Event handler."""
 from __future__ import annotations
@@ -6,9 +6,9 @@ from __future__ import annotations
 import logging
 
 from mlia.backend.vela.compat import Operators
+from mlia.core.common import FormattedFilePath
 from mlia.core.events import CollectedDataEvent
 from mlia.core.handlers import WorkflowEventsHandler
-from mlia.core.typing import PathOrFileLike
 from mlia.target.ethos_u.events import EthosUAdvisorEventHandler
 from mlia.target.ethos_u.events import EthosUAdvisorStartedEvent
 from mlia.target.ethos_u.performance import OptimizationPerformanceMetrics
@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 class EthosUEventHandler(WorkflowEventsHandler, EthosUAdvisorEventHandler):
     """CLI event handler."""
 
-    def __init__(self, output: PathOrFileLike | None = None) -> None:
+    def __init__(self, output: FormattedFilePath | None = None) -> None:
         """Init event handler."""
         super().__init__(ethos_u_formatters, output)
 
diff --git a/src/mlia/target/tosa/advice_generation.py b/src/mlia/target/tosa/advice_generation.py
index f531b84..b8b9abf 100644
--- a/src/mlia/target/tosa/advice_generation.py
+++ b/src/mlia/target/tosa/advice_generation.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """TOSA advice generation."""
 from functools import singledispatchmethod
@@ -19,7 +19,7 @@ class TOSAAdviceProducer(FactBasedAdviceProducer):
         """Produce advice."""
 
     @produce_advice.register
-    @advice_category(AdviceCategory.ALL, AdviceCategory.OPERATORS)
+    @advice_category(AdviceCategory.COMPATIBILITY)
     def handle_model_is_tosa_compatible(
         self, _data_item: ModelIsTOSACompatible
     ) -> None:
@@ -27,7 +27,7 @@ class TOSAAdviceProducer(FactBasedAdviceProducer):
         self.add_advice(["Model is fully TOSA compatible."])
 
     @produce_advice.register
-    @advice_category(AdviceCategory.ALL, AdviceCategory.OPERATORS)
+    @advice_category(AdviceCategory.COMPATIBILITY)
     def handle_model_is_not_tosa_compatible(
         self, _data_item: ModelIsNotTOSACompatible
     ) -> None:
diff --git a/src/mlia/target/tosa/advisor.py b/src/mlia/target/tosa/advisor.py
index 2739dfd..4851113 100644
--- a/src/mlia/target/tosa/advisor.py
+++ b/src/mlia/target/tosa/advisor.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """TOSA advisor."""
 from __future__ import annotations
@@ -10,12 +10,12 @@ from mlia.core.advice_generation import AdviceCategory
 from mlia.core.advice_generation import AdviceProducer
 from mlia.core.advisor import DefaultInferenceAdvisor
 from mlia.core.advisor import InferenceAdvisor
+from mlia.core.common import FormattedFilePath
 from mlia.core.context import Context
 from mlia.core.context import ExecutionContext
 from mlia.core.data_analysis import DataAnalyzer
 from mlia.core.data_collection import DataCollector
 from mlia.core.events import Event
-from mlia.core.typing import PathOrFileLike
 from mlia.target.tosa.advice_generation import TOSAAdviceProducer
 from mlia.target.tosa.config import TOSAConfiguration
 from mlia.target.tosa.data_analysis import TOSADataAnalyzer
@@ -38,7 +38,7 @@ class TOSAInferenceAdvisor(DefaultInferenceAdvisor):
 
         collectors: list[DataCollector] = []
 
-        if AdviceCategory.OPERATORS in context.advice_category:
+        if context.category_enabled(AdviceCategory.COMPATIBILITY):
             collectors.append(TOSAOperatorCompatibility(model))
 
         return collectors
@@ -69,7 +69,7 @@ def configure_and_get_tosa_advisor(
     context: ExecutionContext,
     target_profile: str,
     model: str | Path,
-    output: PathOrFileLike | None = None,
+    output: FormattedFilePath | None = None,
     **_extra_args: Any,
 ) -> InferenceAdvisor:
     """Create and configure TOSA advisor."""
diff --git a/src/mlia/target/tosa/handlers.py b/src/mlia/target/tosa/handlers.py
index 863558c..1037ba1 100644
--- a/src/mlia/target/tosa/handlers.py
+++ b/src/mlia/target/tosa/handlers.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """TOSA Advisor event handlers."""
 # pylint: disable=R0801
@@ -7,9 +7,9 @@ from __future__ import annotations
 import logging
 
 from mlia.backend.tosa_checker.compat import TOSACompatibilityInfo
+from mlia.core.common import FormattedFilePath
 from mlia.core.events import CollectedDataEvent
 from mlia.core.handlers import WorkflowEventsHandler
-from mlia.core.typing import PathOrFileLike
 from mlia.target.tosa.events import TOSAAdvisorEventHandler
 from mlia.target.tosa.events import TOSAAdvisorStartedEvent
 from mlia.target.tosa.reporters import tosa_formatters
@@ -20,7 +20,7 @@ logger = logging.getLogger(__name__)
 class TOSAEventHandler(WorkflowEventsHandler, TOSAAdvisorEventHandler):
     """Event handler for TOSA advisor."""
 
-    def __init__(self, output: PathOrFileLike | None = None) -> None:
+    def __init__(self, output: FormattedFilePath | None = None) -> None:
         """Init event handler."""
         super().__init__(tosa_formatters, output)