MLIA-769 Expand use of target/backend registries

- Use the target/backend registries to avoid hard-coded names.
- Cache target profiles to avoid re-loading them

Change-Id: I474b7c9ef23894e1d8a3ea06d13a37652054c62e

10 files changed, 154 insertions, 119 deletions

diff --git a/tests/conftest.py b/tests/conftest.py
index d797869..55b296f 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -83,8 +83,11 @@ def fixture_test_models_path(
     save_tflite_model(tflite_model, tflite_model_path)
 
     tflite_vela_model = tmp_path / "test_model_vela.tflite"
-    device = EthosUConfiguration.load_profile("ethos-u55-256")
-    optimize_model(tflite_model_path, device.compiler_options, tflite_vela_model)
+
+    target_profile = EthosUConfiguration.load_profile("ethos-u55-256")
+    optimize_model(
+        tflite_model_path, target_profile.compiler_options, tflite_vela_model
+    )
 
     tf.saved_model.save(keras_model, str(tmp_path / "tf_model_test_model"))
 
diff --git a/tests/test_api.py b/tests/test_api.py
index 251d5ac..b40c55b 100644
--- a/tests/test_api.py
+++ b/tests/test_api.py
@@ -19,12 +19,8 @@ from mlia.target.tosa.advisor import TOSAInferenceAdvisor
 
 def test_get_advice_no_target_provided(test_keras_model: Path) -> None:
     """Test getting advice when no target provided."""
-    with pytest.raises(Exception, match="Target profile is not provided"):
-        get_advice(
-            None,  # type:ignore
-            test_keras_model,
-            {"compatibility"},
-        )
+    with pytest.raises(Exception, match="No valid target profile was provided."):
+        get_advice(None, test_keras_model, {"compatibility"})  # type: ignore
 
 
 def test_get_advice_wrong_category(test_keras_model: Path) -> None:
diff --git a/tests/test_cli_command_validators.py b/tests/test_cli_command_validators.py
index cd048ee..29813f4 100644
--- a/tests/test_cli_command_validators.py
+++ b/tests/test_cli_command_validators.py
@@ -5,7 +5,6 @@ from __future__ import annotations
 
 import argparse
 from contextlib import ExitStack
-from unittest.mock import MagicMock
 
 import pytest
 
@@ -93,72 +92,66 @@ def test_validate_check_target_profile(
 
 
 @pytest.mark.parametrize(
-    "input_target_profile, input_backends, throws_exception,"
-    "exception_message, output_backends",
+    (
+        "input_target_profile",
+        "input_backends",
+        "throws_exception",
+        "exception_message",
+        "output_backends",
+    ),
     [
         [
             "tosa",
-            ["Vela"],
-            True,
-            "Vela backend not supported with target-profile tosa.",
+            ["tosa-checker"],
+            False,
             None,
+            ["tosa-checker"],
         ],
         [
             "tosa",
-            ["Corstone-300, Vela"],
+            ["Corstone-310"],
             True,
-            "Corstone-300, Vela backend not supported with target-profile tosa.",
+            "Backend Corstone-310 not supported with target-profile tosa.",
             None,
         ],
         [
             "cortex-a",
-            ["Corstone-310", "tosa-checker"],
-            True,
-            "Corstone-310, tosa-checker backend not supported "
-            "with target-profile cortex-a.",
+            ["ArmNNTFLiteDelegate"],
+            False,
             None,
+            ["ArmNNTFLiteDelegate"],
         ],
         [
-            "ethos-u55-256",
-            ["tosa-checker", "Corstone-310"],
+            "cortex-a",
+            ["tosa-checker"],
             True,
-            "tosa-checker backend not supported with target-profile ethos-u55-256.",
+            "Backend tosa-checker not supported with target-profile cortex-a.",
             None,
         ],
-        ["tosa", None, False, None, ["tosa-checker"]],
-        ["cortex-a", None, False, None, ["ArmNNTFLiteDelegate"]],
-        ["tosa", ["tosa-checker"], False, None, ["tosa-checker"]],
-        ["cortex-a", ["ArmNNTFLiteDelegate"], False, None, ["ArmNNTFLiteDelegate"]],
         [
             "ethos-u55-256",
-            ["Vela", "Corstone-300"],
+            ["Vela", "Corstone-310"],
             False,
             None,
-            ["Vela", "Corstone-300"],
+            ["Vela", "Corstone-310"],
         ],
         [
-            "ethos-u55-256",
-            None,
-            False,
+            "ethos-u65-256",
+            ["Vela", "Corstone-310", "tosa-checker"],
+            True,
+            "Backend tosa-checker not supported with target-profile ethos-u65-256.",
             None,
-            ["Vela", "Corstone-300"],
         ],
     ],
 )
 def test_validate_backend(
-    monkeypatch: pytest.MonkeyPatch,
     input_target_profile: str,
-    input_backends: list[str] | None,
+    input_backends: list[str],
     throws_exception: bool,
     exception_message: str,
     output_backends: list[str] | None,
 ) -> None:
     """Test backend validation with target-profiles and backends."""
-    monkeypatch.setattr(
-        "mlia.cli.config.get_available_backends",
-        MagicMock(return_value=["Vela", "Corstone-300"]),
-    )
-
     exit_stack = ExitStack()
     if throws_exception:
         exit_stack.enter_context(
diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py
index 61cc5a6..f3213c4 100644
--- a/tests/test_cli_commands.py
+++ b/tests/test_cli_commands.py
@@ -33,7 +33,13 @@ def test_performance_unknown_target(
     sample_context: ExecutionContext, test_tflite_model: Path
 ) -> None:
     """Test that command should fail if unknown target passed."""
-    with pytest.raises(Exception, match=r"File not found:*"):
+    with pytest.raises(
+        Exception,
+        match=(
+            r"Profile 'unknown' is neither a valid built-in target profile "
+            r"name or a valid file path."
+        ),
+    ):
         check(
             sample_context,
             model=str(test_tflite_model),
diff --git a/tests/test_cli_config.py b/tests/test_cli_config.py
deleted file mode 100644
index 8494d73..0000000
--- a/tests/test_cli_config.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
-# SPDX-License-Identifier: Apache-2.0
-"""Tests for cli.config module."""
-from __future__ import annotations
-
-from unittest.mock import MagicMock
-
-import pytest
-
-from mlia.cli.config import get_default_backends
-
-
-@pytest.mark.parametrize(
-    "available_backends, expected_default_backends",
-    [
-        [["Vela"], ["Vela"]],
-        [["Corstone-300"], ["Corstone-300"]],
-        [["Corstone-310"], ["Corstone-310"]],
-        [["Corstone-300", "Corstone-310"], ["Corstone-310"]],
-        [["Vela", "Corstone-300", "Corstone-310"], ["Vela", "Corstone-310"]],
-        [
-            ["Vela", "Corstone-300", "Corstone-310", "New backend"],
-            ["Vela", "Corstone-310", "New backend"],
-        ],
-        [
-            ["Vela", "Corstone-300", "New backend"],
-            ["Vela", "Corstone-300", "New backend"],
-        ],
-        [["ArmNNTFLiteDelegate"], ["ArmNNTFLiteDelegate"]],
-        [["tosa-checker"], ["tosa-checker"]],
-        [
-            ["ArmNNTFLiteDelegate", "Corstone-300"],
-            ["ArmNNTFLiteDelegate", "Corstone-300"],
-        ],
-    ],
-)
-def test_get_default_backends(
-    monkeypatch: pytest.MonkeyPatch,
-    available_backends: list[str],
-    expected_default_backends: list[str],
-) -> None:
-    """Test function get_default backends."""
-    monkeypatch.setattr(
-        "mlia.cli.config.get_available_backends",
-        MagicMock(return_value=available_backends),
-    )
-
-    assert get_default_backends() == expected_default_backends
diff --git a/tests/test_cli_helpers.py b/tests/test_cli_helpers.py
index 8f7e4b0..6d19207 100644
--- a/tests/test_cli_helpers.py
+++ b/tests/test_cli_helpers.py
@@ -3,11 +3,13 @@
 """Tests for the helper classes."""
 from __future__ import annotations
 
+from pathlib import Path
 from typing import Any
 
 import pytest
 
 from mlia.cli.helpers import CLIActionResolver
+from mlia.cli.helpers import copy_profile_file_to_output_dir
 from mlia.nn.tensorflow.optimizations.select import OptimizationSettings
 
 
@@ -139,3 +141,12 @@ class TestCliActionResolver:
         """Test checking operator compatibility info."""
         resolver = CLIActionResolver(args)
         assert resolver.check_operator_compatibility() == expected_result
+
+
+def test_copy_profile_file_to_output_dir(tmp_path: Path) -> None:
+    """Test if the profile file is copied into the output directory."""
+    test_target_profile_name = "ethos-u55-128"
+    test_file_path = Path(f"{tmp_path}/{test_target_profile_name}.toml")
+
+    copy_profile_file_to_output_dir(test_target_profile_name, tmp_path)
+    assert Path.is_file(test_file_path)
diff --git a/tests/test_target_config.py b/tests/test_target_config.py
index c6235a5..368d394 100644
--- a/tests/test_target_config.py
+++ b/tests/test_target_config.py
@@ -3,35 +3,28 @@
 """Tests for the backend config module."""
 from __future__ import annotations
 
-from pathlib import Path
-
 import pytest
 
 from mlia.backend.config import BackendConfiguration
 from mlia.backend.config import BackendType
 from mlia.backend.config import System
 from mlia.core.common import AdviceCategory
-from mlia.target.config import copy_profile_file_to_output_dir
+from mlia.target.config import BUILTIN_SUPPORTED_PROFILE_NAMES
+from mlia.target.config import get_builtin_profile_path
 from mlia.target.config import get_builtin_supported_profile_names
-from mlia.target.config import get_profile_file
+from mlia.target.config import is_builtin_profile
 from mlia.target.config import load_profile
 from mlia.target.config import TargetInfo
 from mlia.target.config import TargetProfile
+from mlia.target.cortex_a.advisor import CortexAInferenceAdvisor
+from mlia.target.cortex_a.config import CortexAConfiguration
 from mlia.utils.registry import Registry
 
 
-def test_copy_profile_file_to_output_dir(tmp_path: Path) -> None:
-    """Test if the profile file is copied into the output directory."""
-    test_target_profile_name = "ethos-u55-128"
-    test_file_path = Path(f"{tmp_path}/{test_target_profile_name}.toml")
-
-    copy_profile_file_to_output_dir(test_target_profile_name, tmp_path)
-    assert Path.is_file(test_file_path)
-
-
-def test_get_builtin_supported_profile_names() -> None:
-    """Test profile names getter."""
-    assert get_builtin_supported_profile_names() == [
+def test_builtin_supported_profile_names() -> None:
+    """Test built-in profile names."""
+    assert BUILTIN_SUPPORTED_PROFILE_NAMES == get_builtin_supported_profile_names()
+    assert BUILTIN_SUPPORTED_PROFILE_NAMES == [
         "cortex-a",
         "ethos-u55-128",
         "ethos-u55-256",
@@ -39,23 +32,24 @@ def test_get_builtin_supported_profile_names() -> None:
         "ethos-u65-512",
         "tosa",
     ]
+    for profile_name in BUILTIN_SUPPORTED_PROFILE_NAMES:
+        assert is_builtin_profile(profile_name)
+        profile_file = get_builtin_profile_path(profile_name)
+        assert profile_file.is_file()
 
 
-def test_get_profile_file() -> None:
-    """Test function 'get_profile_file'."""
-    profile_file = get_profile_file("cortex-a")
+def test_builtin_profile_files() -> None:
+    """Test function 'get_bulitin_profile_file'."""
+    profile_file = get_builtin_profile_path("cortex-a")
     assert profile_file.is_file()
-    assert profile_file == get_profile_file(profile_file)
 
-    with pytest.raises(Exception):
-        get_profile_file("UNKNOWN")
-    with pytest.raises(Exception):
-        get_profile_file("")
+    profile_file = get_builtin_profile_path("UNKNOWN_FILE_THAT_DOES_NOT_EXIST")
+    assert not profile_file.exists()
 
 
 def test_load_profile() -> None:
     """Test getting profile data."""
-    profile_file = get_profile_file("ethos-u55-256")
+    profile_file = get_builtin_profile_path("ethos-u55-256")
     assert load_profile(profile_file) == {
         "target": "ethos-u55",
         "mac": 256,
@@ -80,6 +74,9 @@ def test_target_profile() -> None:
     profile = MyTargetProfile("AnyTarget")
     assert profile.target == "AnyTarget"
 
+    profile = MyTargetProfile.load_json_data({"target": "MySuperTarget"})
+    assert profile.target == "MySuperTarget"
+
     profile = MyTargetProfile("")
     with pytest.raises(ValueError):
         profile.verify()
@@ -101,7 +98,12 @@ def test_target_info(
     supported: bool,
 ) -> None:
     """Test the class 'TargetInfo'."""
-    info = TargetInfo(["backend"])
+    info = TargetInfo(
+        ["backend"],
+        ["backend"],
+        CortexAInferenceAdvisor,
+        CortexAConfiguration,
+    )
 
     backend_registry = Registry[BackendConfiguration]()
     backend_registry.register(
@@ -116,3 +118,12 @@ def test_target_info(
 
     assert info.is_supported(advice, check_system) == supported
     assert bool(info.filter_supported_backends(advice, check_system)) == supported
+
+    info = TargetInfo(
+        ["unknown_backend"],
+        ["unknown_backend"],
+        CortexAInferenceAdvisor,
+        CortexAConfiguration,
+    )
+    assert not info.is_supported(advice, check_system)
+    assert not info.filter_supported_backends(advice, check_system)
diff --git a/tests/test_target_ethos_u_data_analysis.py b/tests/test_target_ethos_u_data_analysis.py
index e919f5d..8e63946 100644
--- a/tests/test_target_ethos_u_data_analysis.py
+++ b/tests/test_target_ethos_u_data_analysis.py
@@ -3,6 +3,8 @@
 """Tests for Ethos-U data analysis module."""
 from __future__ import annotations
 
+from typing import cast
+
 import pytest
 
 from mlia.backend.vela.compat import NpuSupported
@@ -23,6 +25,7 @@ from mlia.target.ethos_u.performance import MemoryUsage
 from mlia.target.ethos_u.performance import NPUCycles
 from mlia.target.ethos_u.performance import OptimizationPerformanceMetrics
 from mlia.target.ethos_u.performance import PerformanceMetrics
+from mlia.target.registry import profile
 
 
 def test_perf_metrics_diff() -> None:
@@ -84,7 +87,7 @@ def test_perf_metrics_diff() -> None:
         [
             OptimizationPerformanceMetrics(
                 PerformanceMetrics(
-                    EthosUConfiguration.load_profile("ethos-u55-256"),
+                    cast(EthosUConfiguration, profile("ethos-u55-256")),
                     NPUCycles(1, 2, 3, 4, 5, 6),
                     # memory metrics are in kilobytes
                     MemoryUsage(*[i * 1024 for i in range(1, 6)]),  # type: ignore
@@ -95,7 +98,7 @@ def test_perf_metrics_diff() -> None:
                             OptimizationSettings("pruning", 0.5, None),
                         ],
                         PerformanceMetrics(
-                            EthosUConfiguration.load_profile("ethos-u55-256"),
+                            cast(EthosUConfiguration, profile("ethos-u55-256")),
                             NPUCycles(1, 2, 3, 4, 5, 6),
                             # memory metrics are in kilobytes
                             MemoryUsage(
@@ -127,7 +130,7 @@ def test_perf_metrics_diff() -> None:
         [
             OptimizationPerformanceMetrics(
                 PerformanceMetrics(
-                    EthosUConfiguration.load_profile("ethos-u55-256"),
+                    cast(EthosUConfiguration, profile("ethos-u55-256")),
                     NPUCycles(1, 2, 3, 4, 5, 6),
                     # memory metrics are in kilobytes
                     MemoryUsage(*[i * 1024 for i in range(1, 6)]),  # type: ignore
diff --git a/tests/test_target_ethos_u_reporters.py b/tests/test_target_ethos_u_reporters.py
index 0c5764e..9707dff 100644
--- a/tests/test_target_ethos_u_reporters.py
+++ b/tests/test_target_ethos_u_reporters.py
@@ -3,6 +3,8 @@
 """Tests for reports module."""
 from __future__ import annotations
 
+from typing import cast
+
 import pytest
 
 from mlia.backend.vela.compat import NpuSupported
@@ -12,6 +14,7 @@ from mlia.core.reporting import Table
 from mlia.target.ethos_u.config import EthosUConfiguration
 from mlia.target.ethos_u.reporters import report_device_details
 from mlia.target.ethos_u.reporters import report_operators
+from mlia.target.registry import profile
 from mlia.utils.console import remove_ascii_codes
 
 
@@ -118,7 +121,7 @@ def test_report_operators(
     "device, expected_plain_text, expected_json_dict",
     [
         [
-            EthosUConfiguration.load_profile("ethos-u55-256"),
+            cast(EthosUConfiguration, profile("ethos-u55-256")),
             """Device information:
   Target                                                     ethos-u55
   MAC                                                              256
diff --git a/tests/test_target_registry.py b/tests/test_target_registry.py
index 5012148..2cbd97d 100644
--- a/tests/test_target_registry.py
+++ b/tests/test_target_registry.py
@@ -6,7 +6,11 @@ from __future__ import annotations
 import pytest
 
 from mlia.core.common import AdviceCategory
+from mlia.target.config import get_builtin_profile_path
 from mlia.target.registry import all_supported_backends
+from mlia.target.registry import default_backends
+from mlia.target.registry import is_supported
+from mlia.target.registry import profile
 from mlia.target.registry import registry
 from mlia.target.registry import supported_advice
 from mlia.target.registry import supported_backends
@@ -57,6 +61,23 @@ def test_supported_advice(
 
 
 @pytest.mark.parametrize(
+    ("backend", "target", "expected_result"),
+    (
+        ("ArmNNTFLiteDelegate", None, True),
+        ("ArmNNTFLiteDelegate", "cortex-a", True),
+        ("ArmNNTFLiteDelegate", "tosa", False),
+        ("Corstone-310", None, True),
+        ("Corstone-310", "ethos-u55", True),
+        ("Corstone-310", "ethos-u65", True),
+        ("Corstone-310", "cortex-a", False),
+    ),
+)
+def test_is_supported(backend: str, target: str | None, expected_result: bool) -> None:
+    """Test function is_supported()."""
+    assert is_supported(backend, target) == expected_result
+
+
+@pytest.mark.parametrize(
     ("target_name", "expected_backends"),
     (
         ("cortex-a", ["ArmNNTFLiteDelegate"]),
@@ -92,3 +113,39 @@ def test_all_supported_backends() -> None:
         "Corstone-310",
         "Corstone-300",
     }
+
+
+@pytest.mark.parametrize(
+    ("target", "expected_default_backends", "is_subset_only"),
+    [
+        ["cortex-a", ["ArmNNTFLiteDelegate"], False],
+        ["tosa", ["tosa-checker"], False],
+        ["ethos-u55", ["Vela"], True],
+        ["ethos-u65", ["Vela"], True],
+    ],
+)
+def test_default_backends(
+    target: str,
+    expected_default_backends: list[str],
+    is_subset_only: bool,
+) -> None:
+    """Test function default_backends()."""
+    if is_subset_only:
+        assert set(expected_default_backends).issubset(default_backends(target))
+    else:
+        assert default_backends(target) == expected_default_backends
+
+
+@pytest.mark.parametrize(
+    "target_profile", ("cortex-a", "tosa", "ethos-u55-128", "ethos-u65-256")
+)
+def test_profile(target_profile: str) -> None:
+    """Test function profile()."""
+    # Test loading by built-in profile name
+    cfg = profile(target_profile)
+    assert target_profile.startswith(cfg.target)
+
+    # Test loading the file directly
+    profile_file = get_builtin_profile_path(target_profile)
+    cfg = profile(profile_file)
+    assert target_profile.startswith(cfg.target)