From 2ba39623502551ec073fbc67b59e0458af084c7e Mon Sep 17 00:00:00 2001 From: Benedetta Delfino Date: Wed, 28 Feb 2024 17:38:42 +0000 Subject: feat: Add support for Arm Corstone-300 and Corstone-310 on AArch64 - Add support for Corstone-300 download on AArch64 - Add support for Corstone-310 download on AArch64 - Add support for Corstone-310 download on x86 - Add e2e tests and unit tests - Edited README.md to reflect updates Resolves: MLIA-1017 Signed-off-by: Benedetta Delfino Change-Id: I8d54a721f91d67123f65c076313cef12b7df92bd --- README.md | 18 ++-- src/mlia/backend/corstone/__init__.py | 10 ++- src/mlia/backend/corstone/install.py | 140 ++++++++++++++++++++++++------- src/mlia/backend/corstone/performance.py | 5 +- src/mlia/utils/filesystem.py | 4 +- tests/test_backend_corstone_install.py | 39 +++++++-- tests/test_backend_registry.py | 18 ++-- tests_e2e/test_e2e.py | 12 ++- 8 files changed, 187 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 9d6b951..ee53247 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # ML Inference Advisor - Introduction @@ -262,7 +262,8 @@ Please, find more details in the section for the ## TOSA The target profile *tosa* can be used for TOSA compatibility checks of your -model. It requires the [TOSA Checker](#tosa-checker) backend. +model. It requires the [TOSA Checker](#tosa-checker) backend. Please note that +TOSA is currently only available for x86 architecture. For more information, see TOSA Checker's: @@ -324,16 +325,16 @@ the following table shows some compatibility information: | Backend | Linux | Windows | Python | +============================================================================= | Arm NN | | | | -| TensorFlow | x86_64 | Windows 10 | Python>=3.8 | +| TensorFlow | x86_64 and AArch64 | Windows 10 | Python>=3.8 | | Lite Delegate | | | | +----------------------------------------------------------------------------- -| Corstone-300 | x86_64 | Not compatible | Python>=3.8 | +| Corstone-300 | x86_64 and AArch64 | Not compatible | Python>=3.8 | +----------------------------------------------------------------------------- -| Corstone-310 | x86_64 | Not compatible | Python>=3.8 | +| Corstone-310 | x86_64 and AArch64 | Not compatible | Python>=3.8 | +----------------------------------------------------------------------------- | TOSA checker | x86_64 (manylinux2014) | Not compatible | 3.7<=Python<=3.9 | +----------------------------------------------------------------------------- -| Vela | x86_64 | Windows 10 | Python~=3.7 | +| Vela | x86_64 and AArch64 | Windows 10 | Python~=3.7 | +----------------------------------------------------------------------------+ ``` @@ -368,8 +369,7 @@ For further information about Corstone-300 please refer to: ### Corstone-310 Corstone-310 is a backend that provides performance metrics for systems based -on Cortex-M85 and Ethos-U. It is available as Arm Virtual Hardware (AVH) only, -i.e. it can not be downloaded automatically. +on Cortex-M85 and Ethos-U. * For access to AVH for Corstone-310 please refer to: @@ -379,7 +379,7 @@ i.e. it can not be downloaded automatically. ### TOSA Checker The TOSA Checker backend provides operator compatibility checks against the -TOSA specification. +TOSA specification. Please note that TOSA is currently only available for x86 architecture. Please, install it into the same environment as MLIA using this command: diff --git a/src/mlia/backend/corstone/__init__.py b/src/mlia/backend/corstone/__init__.py index 7575ceb..c6c2227 100644 --- a/src/mlia/backend/corstone/__init__.py +++ b/src/mlia/backend/corstone/__init__.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Corstone backend module.""" from mlia.backend.config import BackendConfiguration @@ -20,8 +20,12 @@ for corstone_name, installation in CORSTONE_PRIORITY.items(): registry.register( corstone_name.lower(), BackendConfiguration( - supported_advice=[AdviceCategory.PERFORMANCE, AdviceCategory.OPTIMIZATION], - supported_systems=[System.LINUX_AMD64], + supported_advice=[ + AdviceCategory.COMPATIBILITY, + AdviceCategory.PERFORMANCE, + AdviceCategory.OPTIMIZATION, + ], + supported_systems=[System.LINUX_AMD64, System.LINUX_AARCH64], backend_type=BackendType.CUSTOM, installation=installation, ), diff --git a/src/mlia/backend/corstone/install.py b/src/mlia/backend/corstone/install.py index d6101cf..40b5530 100644 --- a/src/mlia/backend/corstone/install.py +++ b/src/mlia/backend/corstone/install.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Module for Corstone based FVPs. @@ -12,6 +12,7 @@ import logging import subprocess # nosec from pathlib import Path +from mlia.backend.config import System from mlia.backend.install import BackendInstallation from mlia.backend.install import CompoundPathChecker from mlia.backend.install import Installation @@ -24,21 +25,35 @@ from mlia.utils.filesystem import working_directory logger = logging.getLogger(__name__) -class Corstone300Installer: - """Helper class that wraps Corstone 300 installation logic.""" +class CorstoneInstaller: + """Helper class that wraps Corstone installation logic.""" + + def __init__(self, name: str): + """Define name of the Corstone installer.""" + self.name = name def __call__(self, eula_agreement: bool, dist_dir: Path) -> Path: - """Install Corstone-300 and return path to the models.""" + """Install Corstone and return path to the models.""" with working_directory(dist_dir): - install_dir = "corstone-300" + install_dir = self.name + + if self.name == "corstone-300": + fvp = "./FVP_Corstone_SSE-300.sh" + elif self.name == "corstone-310": + fvp = "./FVP_Corstone_SSE-310.sh" + else: + raise RuntimeError( + f"Couldn't find fvp file during '{self.name}' installation" + ) try: fvp_install_cmd = [ - "./FVP_Corstone_SSE-300.sh", + fvp, "-q", "-d", install_dir, ] + if not eula_agreement: fvp_install_cmd += [ "--nointeractive", @@ -47,12 +62,12 @@ class Corstone300Installer: # The following line raises a B603 error for bandit. In this # specific case, the input is pretty much static and cannot be - # changed byt the user hence disabling the security check for + # changed by the user hence disabling the security check for # this instance subprocess.check_call(fvp_install_cmd) # nosec except subprocess.CalledProcessError as err: raise RuntimeError( - "Error occurred during Corstone-300 installation" + f"Error occurred during '{self.name}' installation" ) from err return dist_dir / install_dir @@ -60,27 +75,54 @@ class Corstone300Installer: def get_corstone_300_installation() -> Installation: """Get Corstone-300 installation.""" + corstone_name = "corstone-300" + if System.CURRENT == System.LINUX_AARCH64: + url = ( + "https://developer.arm.com/-/media/Arm%20Developer%20Community/" + "Downloads/OSS/FVP/Corstone-300/" + "FVP_Corstone_SSE-300_11.22_35_Linux64_armv8l.tgz" + ) + + filename = "FVP_Corstone_SSE-300_11.22_35_Linux64_armv8l.tgz" + version = "11.22_35" + sha256_hash = "0414d3dccbf7037ad24df7002ff1b48975c213f3c1d44544d95033080d0f9ce3" + expected_files = [ + "models/Linux64_armv8l_GCC-9.3/FVP_Corstone_SSE-300_Ethos-U55", + "models/Linux64_armv8l_GCC-9.3/FVP_Corstone_SSE-300_Ethos-U65", + ] + backend_subfolder = "models/Linux64_armv8l_GCC-9.3" + + else: + url = ( + "https://developer.arm.com/-/media/Arm%20Developer%20Community" + "/Downloads/OSS/FVP/Corstone-300/" + "FVP_Corstone_SSE-300_11.16_26.tgz" + ) + filename = "FVP_Corstone_SSE-300_11.16_26.tgz" + version = "11.16_26" + sha256_hash = "e26139be756b5003a30d978c629de638aed1934d597dc24a17043d4708e934d7" + expected_files = [ + "models/Linux64_GCC-6.4/FVP_Corstone_SSE-300_Ethos-U55", + "models/Linux64_GCC-6.4/FVP_Corstone_SSE-300_Ethos-U65", + ] + backend_subfolder = "models/Linux64_GCC-6.4" + corstone_300 = BackendInstallation( - # pylint: disable=line-too-long - name="corstone-300", + name=corstone_name, description="Corstone-300 FVP", - fvp_dir_name="corstone_300", + fvp_dir_name=corstone_name.replace("-", "_"), download_artifact=DownloadArtifact( name="Corstone-300 FVP", - url="https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-300/FVP_Corstone_SSE-300_11.16_26.tgz", - filename="FVP_Corstone_SSE-300_11.16_26.tgz", - version="11.16_26", - sha256_hash="e26139be756b5003a30d978c629de638aed1934d597dc24a17043d4708e934d7", + url=url, + filename=filename, + version=version, + sha256_hash=sha256_hash, ), supported_platforms=["Linux"], - # pylint: enable=line-too-long path_checker=CompoundPathChecker( PackagePathChecker( - expected_files=[ - "models/Linux64_GCC-6.4/FVP_Corstone_SSE-300_Ethos-U55", - "models/Linux64_GCC-6.4/FVP_Corstone_SSE-300_Ethos-U65", - ], - backend_subfolder="models/Linux64_GCC-6.4", + expected_files=expected_files, + backend_subfolder=backend_subfolder, settings={"profile": "default"}, ), StaticPathChecker( @@ -93,7 +135,7 @@ def get_corstone_300_installation() -> Installation: settings={"profile": "AVH"}, ), ), - backend_installer=Corstone300Installer(), + backend_installer=CorstoneInstaller(name=corstone_name), ) return corstone_300 @@ -101,32 +143,66 @@ def get_corstone_300_installation() -> Installation: def get_corstone_310_installation() -> Installation: """Get Corstone-310 installation.""" + corstone_name = "corstone-310" + if System.CURRENT == System.LINUX_AARCH64: + url = ( + "https://developer.arm.com/-/media/Arm%20Developer%20Community" + "/Downloads/OSS/FVP/Corstone-310/" + "FVP_Corstone_SSE-310_11.24_13_Linux64_armv8l.tgz" + ) + filename = "FVP_Corstone_SSE-310_11.24_13_Linux64_armv8l.tgz" + version = "11.24_13" + sha256_hash = "61be18564a7d70c8eb73736e433a65cc16ae4b59f9b028ae86d258e2c28af526" + expected_files = [ + "models/Linux64_armv8l_GCC-9.3/FVP_Corstone_SSE-310", + "models/Linux64_armv8l_GCC-9.3/FVP_Corstone_SSE-310_Ethos-U65", + ] + backend_subfolder = "models/Linux64_armv8l_GCC-9.3" + + else: + url = ( + "https://developer.arm.com/-/media/Arm%20Developer%20Community" + "/Downloads/OSS/FVP/Corstone-310/" + "FVP_Corstone_SSE-310_11.24_13_Linux64.tgz" + ) + filename = "FVP_Corstone_SSE-310_11.24_13_Linux64.tgz" + version = "11.24_13" + sha256_hash = "616ecc0e82067fe0684790cf99638b3496f9ead11051a58d766e8258e766c556" + expected_files = [ + "models/Linux64_GCC-9.3/FVP_Corstone_SSE-310", + "models/Linux64_GCC-9.3/FVP_Corstone_SSE-310_Ethos-U65", + ] + backend_subfolder = "models/Linux64_GCC-9.3" + corstone_310 = BackendInstallation( - name="corstone-310", + name=corstone_name, description="Corstone-310 FVP", - fvp_dir_name="corstone_310", - download_artifact=None, + fvp_dir_name=corstone_name.replace("-", "_"), + download_artifact=DownloadArtifact( + name="Corstone-310 FVP", + url=url, + filename=filename, + version=version, + sha256_hash=sha256_hash, + ), supported_platforms=["Linux"], path_checker=CompoundPathChecker( PackagePathChecker( - expected_files=[ - "models/Linux64_GCC-9.3/FVP_Corstone_SSE-310", - "models/Linux64_GCC-9.3/FVP_Corstone_SSE-310_Ethos-U65", - ], - backend_subfolder="models/Linux64_GCC-9.3", + expected_files=expected_files, + backend_subfolder=backend_subfolder, settings={"profile": "default"}, ), StaticPathChecker( static_backend_path=Path("/opt/VHT"), expected_files=[ - "VHT_Corstone_SSE-310", + "VHT_Corstone_SSE-310_Ethos-U55", "VHT_Corstone_SSE-310_Ethos-U65", ], copy_source=False, settings={"profile": "AVH"}, ), ), - backend_installer=None, + backend_installer=CorstoneInstaller(name=corstone_name), ) return corstone_310 diff --git a/src/mlia/backend/corstone/performance.py b/src/mlia/backend/corstone/performance.py index 5012821..fc50109 100644 --- a/src/mlia/backend/corstone/performance.py +++ b/src/mlia/backend/corstone/performance.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Module for backend integration.""" from __future__ import annotations @@ -114,6 +114,8 @@ def get_executable_name(fvp: str, profile: str, target: str) -> str: ("corstone-300", "default", "ethos-u65"): "FVP_Corstone_SSE-300_Ethos-U65", ("corstone-310", "AVH", "ethos-u55"): "VHT_Corstone_SSE-310", ("corstone-310", "AVH", "ethos-u65"): "VHT_Corstone_SSE-310_Ethos-U65", + ("corstone-310", "default", "ethos-u55"): "FVP_Corstone_SSE-310", + ("corstone-310", "default", "ethos-u65"): "FVP_Corstone_SSE-310_Ethos-U65", } return executable_name_mapping[(fvp, profile, target)] @@ -122,6 +124,7 @@ def get_executable_name(fvp: str, profile: str, target: str) -> str: def get_fvp_metadata(fvp: str, profile: str, target: str) -> FVPMetadata: """Return metadata for selected Corstone backend.""" executable_name = get_executable_name(fvp, profile, target) + app = get_generic_inference_app_path(fvp, target) return FVPMetadata(executable_name, app) diff --git a/src/mlia/utils/filesystem.py b/src/mlia/utils/filesystem.py index f8e8962..e3ef7db 100644 --- a/src/mlia/utils/filesystem.py +++ b/src/mlia/utils/filesystem.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Utils related to file management.""" from __future__ import annotations @@ -77,7 +77,7 @@ def sha256(filepath: Path) -> str: def all_files_exist(paths: Iterable[Path]) -> bool: - """Check if all files are exist.""" + """Check if all files exist.""" return all(item.is_file() for item in paths) diff --git a/tests/test_backend_corstone_install.py b/tests/test_backend_corstone_install.py index b9e6569..496e378 100644 --- a/tests/test_backend_corstone_install.py +++ b/tests/test_backend_corstone_install.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for Corstone related installation functions..""" from __future__ import annotations @@ -10,7 +10,7 @@ from unittest.mock import MagicMock import pytest -from mlia.backend.corstone.install import Corstone300Installer +from mlia.backend.corstone.install import CorstoneInstaller from mlia.backend.corstone.install import get_corstone_installations from mlia.backend.install import Installation @@ -24,10 +24,15 @@ def test_get_corstone_installations() -> None: @pytest.mark.parametrize( - "eula_agreement, expected_calls", + "corstone_name, eula_agreement, expected_calls", [ - [True, [call(["./FVP_Corstone_SSE-300.sh", "-q", "-d", "corstone-300"])]], [ + "corstone-300", + True, + [call(["./FVP_Corstone_SSE-300.sh", "-q", "-d", "corstone-300"])], + ], + [ + "corstone-300", False, [ call( @@ -42,22 +47,44 @@ def test_get_corstone_installations() -> None: ) ], ], + [ + "corstone-310", + True, + [call(["./FVP_Corstone_SSE-310.sh", "-q", "-d", "corstone-310"])], + ], + [ + "corstone-310", + False, + [ + call( + [ + "./FVP_Corstone_SSE-310.sh", + "-q", + "-d", + "corstone-310", + "--nointeractive", + "--i-agree-to-the-contained-eula", + ] + ) + ], + ], ], ) def test_corstone_installer( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + corstone_name: str, eula_agreement: bool, expected_calls: Any, ) -> None: - """Test Corstone 300 installer.""" + """Test Corstone installer.""" mock_check_call = MagicMock() monkeypatch.setattr( "mlia.backend.corstone.install.subprocess.check_call", mock_check_call ) - installer = Corstone300Installer() + installer = CorstoneInstaller(name=corstone_name) installer(eula_agreement, tmp_path) assert mock_check_call.mock_calls == expected_calls diff --git a/tests/test_backend_registry.py b/tests/test_backend_registry.py index cc05632..1f729b6 100644 --- a/tests/test_backend_registry.py +++ b/tests/test_backend_registry.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """Tests for the backend registry module.""" from __future__ import annotations @@ -26,14 +26,22 @@ from mlia.core.common import AdviceCategory ), ( "corstone-300", - [AdviceCategory.PERFORMANCE, AdviceCategory.OPTIMIZATION], - [System.LINUX_AMD64], + [ + AdviceCategory.COMPATIBILITY, + AdviceCategory.PERFORMANCE, + AdviceCategory.OPTIMIZATION, + ], + [System.LINUX_AMD64, System.LINUX_AARCH64], BackendType.CUSTOM, ), ( "corstone-310", - [AdviceCategory.PERFORMANCE, AdviceCategory.OPTIMIZATION], - [System.LINUX_AMD64], + [ + AdviceCategory.COMPATIBILITY, + AdviceCategory.PERFORMANCE, + AdviceCategory.OPTIMIZATION, + ], + [System.LINUX_AMD64, System.LINUX_AARCH64], BackendType.CUSTOM, ), ( diff --git a/tests_e2e/test_e2e.py b/tests_e2e/test_e2e.py index 602a653..74ff51c 100644 --- a/tests_e2e/test_e2e.py +++ b/tests_e2e/test_e2e.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2024, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 """End to end tests for MLIA CLI.""" from __future__ import annotations @@ -20,6 +20,7 @@ from typing import Sequence import pytest +from mlia.backend.config import System from mlia.backend.manager import get_available_backends from mlia.cli.main import get_commands from mlia.cli.main import get_possible_command_names @@ -251,11 +252,16 @@ def get_all_commands_combinations( def check_args(args: list[str], no_skip: bool) -> None: """Check the arguments and skip/fail test cases based on that.""" parser = argparse.ArgumentParser() + parser.add_argument( "--backend", help="Backends to use for evaluation.", action="append", ) + parser.add_argument( + "--target-profile", + help="Target profiles to use for evaluation.", + ) parsed_args, _ = parser.parse_known_args(args) if parsed_args.backend: @@ -266,6 +272,10 @@ def check_args(args: list[str], no_skip: bool) -> None: if missing_backends and not no_skip: pytest.skip(f"Missing backend(s): {','.join(missing_backends)}") + if parsed_args.target_profile == "tosa": + if System.CURRENT == System.LINUX_AARCH64: + pytest.skip("TOSA is not yet available for AArch64, skipping this test.") + def get_execution_definitions( executions: dict, -- cgit v1.2.1