MLIA-590 Support path to custom target profiles

- Start using TOML format for target profile
- Add support for loading custom target profile files

Change-Id: I6be019d4341e93115440ccdbdb6dafdc1c85b966

19 files changed, 135 insertions, 122 deletions

diff --git a/setup.cfg b/setup.cfg
index 966eac5..b70c955 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-FileCopyrightText: Copyright (c) 2020 Troy Comi
 # SPDX-License-Identifier: Apache-2.0 AND MIT
 
@@ -36,6 +36,7 @@ install_requires =
     requests~=2.28.1
     rich~=12.6.0
     sh~=1.14.3
+    tomli~=2.0.1 ; python_version<"3.11"
 
 [options.packages.find]
 where = src
diff --git a/src/mlia/api.py b/src/mlia/api.py
index 8105276..437c457 100644
--- a/src/mlia/api.py
+++ b/src/mlia/api.py
@@ -79,7 +79,7 @@ def get_advice(
 
 def get_advisor(
     context: ExecutionContext,
-    target_profile: str,
+    target_profile: str | Path,
     model: str | Path,
     **extra_args: Any,
 ) -> InferenceAdvisor:
diff --git a/src/mlia/cli/options.py b/src/mlia/cli/options.py
index d154646..1b92958 100644
--- a/src/mlia/cli/options.py
+++ b/src/mlia/cli/options.py
@@ -13,7 +13,7 @@ from mlia.cli.config import DEFAULT_PRUNING_TARGET
 from mlia.cli.config import get_available_backends
 from mlia.cli.config import is_corstone_backend
 from mlia.core.typing import OutputFormat
-from mlia.utils.filesystem import get_supported_profile_names
+from mlia.utils.filesystem import get_builtin_supported_profile_names
 
 
 def add_check_category_options(parser: argparse.ArgumentParser) -> None:
@@ -35,18 +35,21 @@ def add_target_options(
     required: bool = True,
 ) -> None:
     """Add target specific options."""
-    target_profiles = get_supported_profile_names()
+    target_profiles = get_builtin_supported_profile_names()
     if profiles_to_skip:
         target_profiles = [tp for tp in target_profiles if tp not in profiles_to_skip]
 
+    default_target_profile = "ethos-u55-256"
+
     target_group = parser.add_argument_group("target options")
     target_group.add_argument(
         "-t",
         "--target-profile",
-        choices=target_profiles,
         required=required,
-        default="",
-        help="Target profile that will set the target options "
+        default=default_target_profile,
+        help="Builtin target profile: {target_profiles}"
+        "or path to custom target profile"
+        "Target profile that will set the target options "
         "such as target, mac value, memory mode, etc. "
         "For the values associated with each target profile "
         "please refer to the documentation.",
diff --git a/src/mlia/resources/profiles.json b/src/mlia/resources/profiles.json
deleted file mode 100644
index 990cdb3..0000000
--- a/src/mlia/resources/profiles.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
-    "ethos-u55-256": {
-        "target": "ethos-u55",
-        "mac": 256,
-        "system_config": "Ethos_U55_High_End_Embedded",
-        "memory_mode": "Shared_Sram"
-    },
-    "ethos-u55-128": {
-        "target": "ethos-u55",
-        "mac": 128,
-        "system_config": "Ethos_U55_High_End_Embedded",
-        "memory_mode": "Shared_Sram"
-    },
-    "ethos-u65-512": {
-        "target": "ethos-u65",
-        "mac": 512,
-        "system_config": "Ethos_U65_High_End",
-        "memory_mode": "Dedicated_Sram"
-    },
-    "ethos-u65-256": {
-        "target": "ethos-u65",
-        "mac": 256,
-        "system_config": "Ethos_U65_High_End",
-        "memory_mode": "Dedicated_Sram"
-    },
-    "tosa": {
-        "target": "tosa"
-    },
-    "cortex-a": {
-        "target": "cortex-a"
-    }
-}
diff --git a/src/mlia/resources/profiles.json.license b/src/mlia/resources/profiles.json.license
deleted file mode 100644
index 9b83bfc..0000000
--- a/src/mlia/resources/profiles.json.license
+++ /dev/null
@@ -1,3 +0,0 @@
-SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
-
-SPDX-License-Identifier: Apache-2.0
diff --git a/src/mlia/resources/target_profiles/cortex-a.toml b/src/mlia/resources/target_profiles/cortex-a.toml
new file mode 100644
index 0000000..9de9cee
--- /dev/null
+++ b/src/mlia/resources/target_profiles/cortex-a.toml
@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: Copyright 2023, Arm Limited and/or its affiliates.
+# SPDX-License-Identifier: Apache-2.0
+
+target="cortex-a"
diff --git a/src/mlia/resources/target_profiles/ethos-u55-128.toml b/src/mlia/resources/target_profiles/ethos-u55-128.toml
new file mode 100644
index 0000000..71c6ec6
--- /dev/null
+++ b/src/mlia/resources/target_profiles/ethos-u55-128.toml
@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: Copyright 2023, Arm Limited and/or its affiliates.
+# SPDX-License-Identifier: Apache-2.0
+
+target="ethos-u55"
+
+# Number of MACs [32, 64, 128, 256]
+mac=128
+
+# Memory mode: [SRAM Only, Shared SRAM, Dedicated SRAM]
+memory_mode="Shared_Sram"
+
+# System configuration
+system_config="Ethos_U55_High_End_Embedded"
diff --git a/src/mlia/resources/target_profiles/ethos-u55-256.toml b/src/mlia/resources/target_profiles/ethos-u55-256.toml
new file mode 100644
index 0000000..f44cdfe
--- /dev/null
+++ b/src/mlia/resources/target_profiles/ethos-u55-256.toml
@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: Copyright 2023, Arm Limited and/or its affiliates.
+# SPDX-License-Identifier: Apache-2.0
+
+target="ethos-u55"
+
+# Number of MACs [32, 64, 128, 256]
+mac=256
+
+# Memory mode: [SRAM Only, Shared SRAM, Dedicated SRAM]
+memory_mode="Shared_Sram"
+
+# System configuration
+system_config="Ethos_U55_High_End_Embedded"
diff --git a/src/mlia/resources/target_profiles/ethos-u65-256.toml b/src/mlia/resources/target_profiles/ethos-u65-256.toml
new file mode 100644
index 0000000..078f60a
--- /dev/null
+++ b/src/mlia/resources/target_profiles/ethos-u65-256.toml
@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: Copyright 2023, Arm Limited and/or its affiliates.
+# SPDX-License-Identifier: Apache-2.0
+
+target="ethos-u65"
+
+# Number of MACs [256, 512]
+mac=256
+
+# Memory mode: [SRAM Only, Shared SRAM, Dedicated SRAM]
+memory_mode="Dedicated_Sram"
+
+# System configuration
+system_config="Ethos_U65_High_End"
diff --git a/src/mlia/resources/target_profiles/ethos-u65-512.toml b/src/mlia/resources/target_profiles/ethos-u65-512.toml
new file mode 100644
index 0000000..6d32e63
--- /dev/null
+++ b/src/mlia/resources/target_profiles/ethos-u65-512.toml
@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: Copyright 2023, Arm Limited and/or its affiliates.
+# SPDX-License-Identifier: Apache-2.0
+
+target="ethos-u65"
+
+# Number of MACs [256, 512]
+mac=512
+
+# Memory mode: [SRAM Only, Shared SRAM, Dedicated SRAM]
+memory_mode="Dedicated_Sram"
+
+# System configuration
+system_config="Ethos_U65_High_End"
diff --git a/src/mlia/resources/target_profiles/tosa.toml b/src/mlia/resources/target_profiles/tosa.toml
new file mode 100644
index 0000000..fb179ab
--- /dev/null
+++ b/src/mlia/resources/target_profiles/tosa.toml
@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: Copyright 2023, Arm Limited and/or its affiliates.
+# SPDX-License-Identifier: Apache-2.0
+
+target="tosa"
diff --git a/src/mlia/target/cortex_a/advisor.py b/src/mlia/target/cortex_a/advisor.py
index 1249d93..52af592 100644
--- a/src/mlia/target/cortex_a/advisor.py
+++ b/src/mlia/target/cortex_a/advisor.py
@@ -64,7 +64,7 @@ class CortexAInferenceAdvisor(DefaultInferenceAdvisor):
 
 def configure_and_get_cortexa_advisor(
     context: ExecutionContext,
-    target_profile: str,
+    target_profile: str | Path,
     model: str | Path,
     **_extra_args: Any,
 ) -> InferenceAdvisor:
@@ -78,7 +78,9 @@ def configure_and_get_cortexa_advisor(
     return CortexAInferenceAdvisor()
 
 
-def _get_config_parameters(model: str | Path, target_profile: str) -> dict[str, Any]:
+def _get_config_parameters(
+    model: str | Path, target_profile: str | Path
+) -> dict[str, Any]:
     """Get configuration parameters for the advisor."""
     advisor_parameters: dict[str, Any] = {
         "cortex_a_inference_advisor": {
diff --git a/src/mlia/target/ethos_u/advisor.py b/src/mlia/target/ethos_u/advisor.py
index ce4e0fc..937e91c 100644
--- a/src/mlia/target/ethos_u/advisor.py
+++ b/src/mlia/target/ethos_u/advisor.py
@@ -123,7 +123,7 @@ class EthosUInferenceAdvisor(DefaultInferenceAdvisor):
 
 def configure_and_get_ethosu_advisor(
     context: ExecutionContext,
-    target_profile: str,
+    target_profile: str | Path,
     model: str | Path,
     **extra_args: Any,
 ) -> InferenceAdvisor:
@@ -155,7 +155,7 @@ _DEFAULT_OPTIMIZATION_TARGETS = [
 
 def _get_config_parameters(
     model: str | Path,
-    target_profile: str,
+    target_profile: str | Path,
     **extra_args: Any,
 ) -> dict[str, Any]:
     """Get configuration parameters for the advisor."""
diff --git a/src/mlia/target/tosa/advisor.py b/src/mlia/target/tosa/advisor.py
index b60e824..e8aad53 100644
--- a/src/mlia/target/tosa/advisor.py
+++ b/src/mlia/target/tosa/advisor.py
@@ -78,7 +78,7 @@ class TOSAInferenceAdvisor(DefaultInferenceAdvisor):
 
 def configure_and_get_tosa_advisor(
     context: ExecutionContext,
-    target_profile: str,
+    target_profile: str | Path,
     model: str | Path,
     **_extra_args: Any,
 ) -> InferenceAdvisor:
@@ -92,7 +92,9 @@ def configure_and_get_tosa_advisor(
     return TOSAInferenceAdvisor()
 
 
-def _get_config_parameters(model: str | Path, target_profile: str) -> dict[str, Any]:
+def _get_config_parameters(
+    model: str | Path, target_profile: str | Path
+) -> dict[str, Any]:
     """Get configuration parameters for the advisor."""
     advisor_parameters: dict[str, Any] = {
         "tosa_inference_advisor": {
diff --git a/src/mlia/utils/filesystem.py b/src/mlia/utils/filesystem.py
index fcd09b5..4734a84 100644
--- a/src/mlia/utils/filesystem.py
+++ b/src/mlia/utils/filesystem.py
@@ -1,11 +1,10 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Utils related to file management."""
 from __future__ import annotations
 
 import hashlib
 import importlib.resources as pkg_resources
-import json
 import os
 import shutil
 from contextlib import contextmanager
@@ -17,6 +16,11 @@ from typing import cast
 from typing import Generator
 from typing import Iterable
 
+try:
+    import tomllib
+except ModuleNotFoundError:
+    import tomli as tomllib  # type: ignore
+
 
 def get_mlia_resources() -> Path:
     """Get the path to the resources directory."""
@@ -30,44 +34,53 @@ def get_vela_config() -> Path:
     return get_mlia_resources() / "vela/vela.ini"
 
 
-def get_profiles_file() -> Path:
+def get_mlia_target_profiles_dir() -> Path:
     """Get the profiles file."""
-    return get_mlia_resources() / "profiles.json"
+    return get_mlia_resources() / "target_profiles"
 
 
-def get_profiles_data() -> dict[str, dict[str, Any]]:
-    """Get the profile values as a dictionary."""
-    with open(get_profiles_file(), encoding="utf-8") as json_file:
-        profiles = json.load(json_file)
+def get_profile_toml_file(target_profile: str | Path) -> str | Path:
+    """Get the target profile toml file."""
+    if not target_profile:
+        raise Exception("Target profile is not provided")
 
-        if not isinstance(profiles, dict):
-            raise Exception("Profiles data format is not valid")
+    profile_toml_file = Path(get_mlia_target_profiles_dir() / f"{target_profile}.toml")
+    if not profile_toml_file.is_file():
+        profile_toml_file = Path(target_profile)
 
-        return profiles
+    if not profile_toml_file.exists():
+        raise Exception(f"File not found: {profile_toml_file}.")
+    return profile_toml_file
 
 
-def get_profile(target_profile: str) -> dict[str, Any]:
+def get_profile(target_profile: str | Path) -> dict[str, Any]:
     """Get settings for the provided target profile."""
     if not target_profile:
         raise Exception("Target profile is not provided")
 
-    profiles = get_profiles_data()
+    toml_file = get_profile_toml_file(target_profile)
 
-    try:
-        return profiles[target_profile]
-    except KeyError as err:
-        raise Exception(f"Unable to find target profile {target_profile}") from err
+    with open(toml_file, "rb") as file:
+        profile = tomllib.load(file)
+
+    return cast(dict, profile)
 
 
-def get_supported_profile_names() -> list[str]:
-    """Get the supported Ethos-U profile names."""
-    return list(get_profiles_data().keys())
+def get_builtin_supported_profile_names() -> list[str]:
+    """Return list of default profiles in the target profiles directory."""
+    return sorted(
+        [
+            item.stem
+            for item in get_mlia_target_profiles_dir().iterdir()
+            if item.is_file() and item.suffix == ".toml"
+        ]
+    )
 
 
-def get_target(target_profile: str) -> str:
+def get_target(target_profile: str | Path) -> str:
     """Return target for the provided target_profile."""
-    profile_data = get_profile(target_profile)
-    return cast(str, profile_data["target"])
+    profile = get_profile(target_profile)
+    return cast(str, profile["target"])
 
 
 @contextmanager
diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py
index 03ee9d2..b65d90e 100644
--- a/tests/test_cli_commands.py
+++ b/tests/test_cli_commands.py
@@ -33,7 +33,7 @@ def test_performance_unknown_target(
     sample_context: ExecutionContext, test_tflite_model: Path
 ) -> None:
     """Test that command should fail if unknown target passed."""
-    with pytest.raises(Exception, match="Unable to find target profile unknown"):
+    with pytest.raises(Exception, match=r"File not found:*"):
         check(
             sample_context,
             model=str(test_tflite_model),
diff --git a/tests/test_target_ethos_u_config.py b/tests/test_target_ethos_u_config.py
index 3160190..08a20ff 100644
--- a/tests/test_target_ethos_u_config.py
+++ b/tests/test_target_ethos_u_config.py
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Tests for config module."""
 from __future__ import annotations
@@ -45,7 +45,7 @@ def test_get_target() -> None:
     with pytest.raises(Exception, match="No target profile given"):
         get_target(None)  # type: ignore
 
-    with pytest.raises(Exception, match="Unable to find target profile unknown"):
+    with pytest.raises(Exception, match=r"File not found:*"):
         get_target("unknown")
 
     u65_device = get_target("ethos-u65-512")
diff --git a/tests/test_utils_filesystem.py b/tests/test_utils_filesystem.py
index 9dd51e1..954f9e3 100644
--- a/tests/test_utils_filesystem.py
+++ b/tests/test_utils_filesystem.py
@@ -1,21 +1,18 @@
-# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates.
 # SPDX-License-Identifier: Apache-2.0
 """Tests for the filesystem module."""
 import contextlib
-import json
 from pathlib import Path
-from unittest.mock import MagicMock
 
 import pytest
 
 from mlia.utils.filesystem import all_files_exist
 from mlia.utils.filesystem import all_paths_valid
 from mlia.utils.filesystem import copy_all
+from mlia.utils.filesystem import get_builtin_supported_profile_names
 from mlia.utils.filesystem import get_mlia_resources
+from mlia.utils.filesystem import get_mlia_target_profiles_dir
 from mlia.utils.filesystem import get_profile
-from mlia.utils.filesystem import get_profiles_data
-from mlia.utils.filesystem import get_profiles_file
-from mlia.utils.filesystem import get_supported_profile_names
 from mlia.utils.filesystem import get_vela_config
 from mlia.utils.filesystem import sha256
 from mlia.utils.filesystem import temp_directory
@@ -34,50 +31,20 @@ def test_get_vela_config() -> None:
     assert get_vela_config().name == "vela.ini"
 
 
-def test_profiles_file() -> None:
-    """Test profiles file getter."""
-    assert get_profiles_file().is_file()
-    assert get_profiles_file().name == "profiles.json"
+def test_get_mlia_target_profiles() -> None:
+    """Test target profiles getter."""
+    assert get_mlia_target_profiles_dir().is_dir()
 
 
-def test_profiles_data() -> None:
-    """Test profiles data getter."""
-    assert list(get_profiles_data().keys()) == [
-        "ethos-u55-256",
-        "ethos-u55-128",
-        "ethos-u65-512",
-        "ethos-u65-256",
-        "tosa",
-        "cortex-a",
-    ]
-
-
-def test_profiles_data_wrong_format(
-    monkeypatch: pytest.MonkeyPatch, tmp_path: Path
-) -> None:
-    """Test if profile data has wrong format."""
-    wrong_profile_data = tmp_path / "bad.json"
-    with open(wrong_profile_data, "w", encoding="utf-8") as file:
-        json.dump([], file)
-
-    monkeypatch.setattr(
-        "mlia.utils.filesystem.get_profiles_file",
-        MagicMock(return_value=wrong_profile_data),
-    )
-
-    with pytest.raises(Exception, match="Profiles data format is not valid"):
-        get_profiles_data()
-
-
-def test_get_supported_profile_names() -> None:
+def test_get_builtin_supported_profile_names() -> None:
     """Test profile names getter."""
-    assert list(get_supported_profile_names()) == [
-        "ethos-u55-256",
+    assert get_builtin_supported_profile_names() == [
+        "cortex-a",
         "ethos-u55-128",
-        "ethos-u65-512",
+        "ethos-u55-256",
         "ethos-u65-256",
+        "ethos-u65-512",
         "tosa",
-        "cortex-a",
     ]
 
 
@@ -86,11 +53,11 @@ def test_get_profile() -> None:
     assert get_profile("ethos-u55-256") == {
         "target": "ethos-u55",
         "mac": 256,
-        "system_config": "Ethos_U55_High_End_Embedded",
         "memory_mode": "Shared_Sram",
+        "system_config": "Ethos_U55_High_End_Embedded",
     }
 
-    with pytest.raises(Exception, match="Unable to find target profile unknown"):
+    with pytest.raises(Exception, match=r"File not found:*"):
         get_profile("unknown")
 
 
diff --git a/tests_e2e/test_e2e.py b/tests_e2e/test_e2e.py
index 4de640b..546bd7e 100644
--- a/tests_e2e/test_e2e.py
+++ b/tests_e2e/test_e2e.py
@@ -24,7 +24,7 @@ from mlia.cli.config import get_available_backends
 from mlia.cli.main import get_commands
 from mlia.cli.main import get_possible_command_names
 from mlia.cli.main import init_parser
-from mlia.utils.filesystem import get_supported_profile_names
+from mlia.utils.filesystem import get_builtin_supported_profile_names
 from mlia.utils.types import is_list_of
 
 
@@ -175,7 +175,7 @@ def resolve(params: list[str]) -> Generator[list[str], None, None]:
         if prev == "--target-profile" and param == "*":
             resolved = (
                 replace_element(params, idx, profile)
-                for profile in get_supported_profile_names()
+                for profile in get_builtin_supported_profile_names()
             )
         elif param.startswith("e2e_config") and (
             filenames := glob.glob(f"{Path.cwd()}/{param}", recursive=True)