From 37959522a805a5e23c930ed79aac84920c3cb208 Mon Sep 17 00:00:00 2001 From: Dmitrii Agibov Date: Fri, 18 Nov 2022 16:34:03 +0000 Subject: Move backends functionality into separate modules - Move backend management/executor code into module backend_core - Create separate module for each backend in "backend" module - Move each backend into corresponding module - Split Vela wrapper into several submodules Change-Id: If01b6774aab6501951212541cc5d7f5aa7c97e95 --- src/mlia/tools/metadata/corstone.py | 417 ------------------------------------ 1 file changed, 417 deletions(-) delete mode 100644 src/mlia/tools/metadata/corstone.py (limited to 'src/mlia/tools/metadata/corstone.py') diff --git a/src/mlia/tools/metadata/corstone.py b/src/mlia/tools/metadata/corstone.py deleted file mode 100644 index df2dcdb..0000000 --- a/src/mlia/tools/metadata/corstone.py +++ /dev/null @@ -1,417 +0,0 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. -# SPDX-License-Identifier: Apache-2.0 -"""Module for Corstone based FVPs. - -The import of subprocess module raises a B404 bandit error. MLIA usage of -subprocess is needed and can be considered safe hence disabling the security -check. -""" -from __future__ import annotations - -import logging -import platform -import subprocess # nosec -import tarfile -from dataclasses import dataclass -from pathlib import Path -from typing import Callable -from typing import Iterable -from typing import Optional - -import mlia.backend.manager as backend_manager -from mlia.backend.system import remove_system -from mlia.tools.metadata.common import DownloadAndInstall -from mlia.tools.metadata.common import Installation -from mlia.tools.metadata.common import InstallationType -from mlia.tools.metadata.common import InstallFromPath -from mlia.utils.download import DownloadArtifact -from mlia.utils.filesystem import all_files_exist -from mlia.utils.filesystem import all_paths_valid -from mlia.utils.filesystem import copy_all -from mlia.utils.filesystem import get_mlia_resources -from mlia.utils.filesystem import temp_directory -from mlia.utils.filesystem import working_directory - - -logger = logging.getLogger(__name__) - - -@dataclass -class BackendInfo: - """Backend information.""" - - backend_path: Path - copy_source: bool = True - system_config: str | None = None - - -PathChecker = Callable[[Path], Optional[BackendInfo]] -BackendInstaller = Callable[[bool, Path], Path] - - -class BackendMetadata: - """Backend installation metadata.""" - - def __init__( - self, - name: str, - description: str, - system_config: str, - apps_resources: list[str], - fvp_dir_name: str, - download_artifact: DownloadArtifact | None, - supported_platforms: list[str] | None = None, - ) -> None: - """ - Initialize BackendMetadata. - - Members expected_systems and expected_apps are filled automatically. - """ - self.name = name - self.description = description - self.system_config = system_config - self.apps_resources = apps_resources - self.fvp_dir_name = fvp_dir_name - self.download_artifact = download_artifact - self.supported_platforms = supported_platforms - - self.expected_systems = backend_manager.get_all_system_names(name) - self.expected_apps = backend_manager.get_all_application_names(name) - - @property - def expected_resources(self) -> Iterable[Path]: - """Return list of expected resources.""" - resources = [self.system_config, *self.apps_resources] - - return (get_mlia_resources() / resource for resource in resources) - - @property - def supported_platform(self) -> bool: - """Return true if current platform supported.""" - if not self.supported_platforms: - return True - - return platform.system() in self.supported_platforms - - -class BackendInstallation(Installation): - """Backend installation.""" - - def __init__( - self, - backend_runner: backend_manager.BackendRunner, - metadata: BackendMetadata, - path_checker: PathChecker, - backend_installer: BackendInstaller | None, - ) -> None: - """Init the backend installation.""" - self.backend_runner = backend_runner - self.metadata = metadata - self.path_checker = path_checker - self.backend_installer = backend_installer - - @property - def name(self) -> str: - """Return name of the backend.""" - return self.metadata.name - - @property - def description(self) -> str: - """Return description of the backend.""" - return self.metadata.description - - @property - def already_installed(self) -> bool: - """Return true if backend already installed.""" - return self.backend_runner.all_installed( - self.metadata.expected_systems, self.metadata.expected_apps - ) - - @property - def could_be_installed(self) -> bool: - """Return true if backend could be installed.""" - if not self.metadata.supported_platform: - return False - - return all_paths_valid(self.metadata.expected_resources) - - def supports(self, install_type: InstallationType) -> bool: - """Return true if backends supported type of the installation.""" - if isinstance(install_type, DownloadAndInstall): - return self.metadata.download_artifact is not None - - if isinstance(install_type, InstallFromPath): - return self.path_checker(install_type.backend_path) is not None - - return False # type: ignore - - def install(self, install_type: InstallationType) -> None: - """Install the backend.""" - if isinstance(install_type, DownloadAndInstall): - download_artifact = self.metadata.download_artifact - assert download_artifact is not None, "No artifact provided" - - self.download_and_install(download_artifact, install_type.eula_agreement) - elif isinstance(install_type, InstallFromPath): - backend_path = self.path_checker(install_type.backend_path) - assert backend_path is not None, "Unable to resolve backend path" - - self.install_from(backend_path) - else: - raise Exception(f"Unable to install {install_type}") - - def install_from(self, backend_info: BackendInfo) -> None: - """Install backend from the directory.""" - mlia_resources = get_mlia_resources() - - with temp_directory() as tmpdir: - fvp_dist_dir = tmpdir / self.metadata.fvp_dir_name - - system_config = self.metadata.system_config - if backend_info.system_config: - system_config = backend_info.system_config - - resources_to_copy = [mlia_resources / system_config] - if backend_info.copy_source: - resources_to_copy.append(backend_info.backend_path) - - copy_all(*resources_to_copy, dest=fvp_dist_dir) - - self.backend_runner.install_system(fvp_dist_dir) - - for app in self.metadata.apps_resources: - self.backend_runner.install_application(mlia_resources / app) - - def download_and_install( - self, download_artifact: DownloadArtifact, eula_agrement: bool - ) -> None: - """Download and install the backend.""" - with temp_directory() as tmpdir: - try: - downloaded_to = download_artifact.download_to(tmpdir) - except Exception as err: - raise Exception("Unable to download backend artifact") from err - - with working_directory(tmpdir / "dist", create_dir=True) as dist_dir: - with tarfile.open(downloaded_to) as archive: - archive.extractall(dist_dir) - - assert self.backend_installer, ( - f"Backend '{self.metadata.name}' does not support " - "download and installation." - ) - backend_path = self.backend_installer(eula_agrement, dist_dir) - if self.path_checker(backend_path) is None: - raise Exception("Downloaded artifact has invalid structure") - - self.install(InstallFromPath(backend_path)) - - def uninstall(self) -> None: - """Uninstall the backend.""" - remove_system(self.metadata.fvp_dir_name) - - -class PackagePathChecker: - """Package path checker.""" - - def __init__( - self, expected_files: list[str], backend_subfolder: str | None = None - ) -> None: - """Init the path checker.""" - self.expected_files = expected_files - self.backend_subfolder = backend_subfolder - - def __call__(self, backend_path: Path) -> BackendInfo | None: - """Check if directory contains all expected files.""" - resolved_paths = (backend_path / file for file in self.expected_files) - if not all_files_exist(resolved_paths): - return None - - if self.backend_subfolder: - subfolder = backend_path / self.backend_subfolder - - if not subfolder.is_dir(): - return None - - return BackendInfo(subfolder) - - return BackendInfo(backend_path) - - -class StaticPathChecker: - """Static path checker.""" - - def __init__( - self, - static_backend_path: Path, - expected_files: list[str], - copy_source: bool = False, - system_config: str | None = None, - ) -> None: - """Init static path checker.""" - self.static_backend_path = static_backend_path - self.expected_files = expected_files - self.copy_source = copy_source - self.system_config = system_config - - def __call__(self, backend_path: Path) -> BackendInfo | None: - """Check if directory equals static backend path with all expected files.""" - if backend_path != self.static_backend_path: - return None - - resolved_paths = (backend_path / file for file in self.expected_files) - if not all_files_exist(resolved_paths): - return None - - return BackendInfo( - backend_path, - copy_source=self.copy_source, - system_config=self.system_config, - ) - - -class CompoundPathChecker: - """Compound path checker.""" - - def __init__(self, *path_checkers: PathChecker) -> None: - """Init compound path checker.""" - self.path_checkers = path_checkers - - def __call__(self, backend_path: Path) -> BackendInfo | None: - """Iterate over checkers and return first non empty backend info.""" - first_resolved_backend_info = ( - backend_info - for path_checker in self.path_checkers - if (backend_info := path_checker(backend_path)) is not None - ) - - return next(first_resolved_backend_info, None) - - -class Corstone300Installer: - """Helper class that wraps Corstone 300 installation logic.""" - - def __call__(self, eula_agreement: bool, dist_dir: Path) -> Path: - """Install Corstone-300 and return path to the models.""" - with working_directory(dist_dir): - install_dir = "corstone-300" - try: - fvp_install_cmd = [ - "./FVP_Corstone_SSE-300.sh", - "-q", - "-d", - install_dir, - ] - if not eula_agreement: - fvp_install_cmd += [ - "--nointeractive", - "--i-agree-to-the-contained-eula", - ] - - # The following line raises a B603 error for bandit. In this - # specific case, the input is pretty much static and cannot be - # changed byt the user hence disabling the security check for - # this instance - subprocess.check_call(fvp_install_cmd) # nosec - except subprocess.CalledProcessError as err: - raise Exception( - "Error occurred during Corstone-300 installation" - ) from err - - return dist_dir / install_dir - - -def get_corstone_300_installation() -> Installation: - """Get Corstone-300 installation.""" - corstone_300 = BackendInstallation( - backend_runner=backend_manager.BackendRunner(), - # pylint: disable=line-too-long - metadata=BackendMetadata( - name="Corstone-300", - description="Corstone-300 FVP", - system_config="backend_configs/systems/corstone-300/backend-config.json", - apps_resources=[], - fvp_dir_name="corstone_300", - download_artifact=DownloadArtifact( - name="Corstone-300 FVP", - url="https://developer.arm.com/-/media/Arm%20Developer%20Community/Downloads/OSS/FVP/Corstone-300/FVP_Corstone_SSE-300_11.16_26.tgz", - filename="FVP_Corstone_SSE-300_11.16_26.tgz", - version="11.16_26", - sha256_hash="e26139be756b5003a30d978c629de638aed1934d597dc24a17043d4708e934d7", - ), - supported_platforms=["Linux"], - ), - # pylint: enable=line-too-long - path_checker=CompoundPathChecker( - PackagePathChecker( - expected_files=[ - "models/Linux64_GCC-6.4/FVP_Corstone_SSE-300_Ethos-U55", - "models/Linux64_GCC-6.4/FVP_Corstone_SSE-300_Ethos-U65", - ], - backend_subfolder="models/Linux64_GCC-6.4", - ), - StaticPathChecker( - static_backend_path=Path("/opt/VHT"), - expected_files=[ - "VHT_Corstone_SSE-300_Ethos-U55", - "VHT_Corstone_SSE-300_Ethos-U65", - ], - copy_source=False, - system_config=( - "backend_configs/systems/corstone-300-vht/backend-config.json" - ), - ), - ), - backend_installer=Corstone300Installer(), - ) - - return corstone_300 - - -def get_corstone_310_installation() -> Installation: - """Get Corstone-310 installation.""" - corstone_310 = BackendInstallation( - backend_runner=backend_manager.BackendRunner(), - # pylint: disable=line-too-long - metadata=BackendMetadata( - name="Corstone-310", - description="Corstone-310 FVP", - system_config="backend_configs/systems/corstone-310/backend-config.json", - apps_resources=[], - fvp_dir_name="corstone_310", - download_artifact=None, - supported_platforms=["Linux"], - ), - # pylint: enable=line-too-long - path_checker=CompoundPathChecker( - PackagePathChecker( - expected_files=[ - "models/Linux64_GCC-9.3/FVP_Corstone_SSE-310", - "models/Linux64_GCC-9.3/FVP_Corstone_SSE-310_Ethos-U65", - ], - backend_subfolder="models/Linux64_GCC-9.3", - ), - StaticPathChecker( - static_backend_path=Path("/opt/VHT"), - expected_files=[ - "VHT_Corstone_SSE-310", - "VHT_Corstone_SSE-310_Ethos-U65", - ], - copy_source=False, - system_config=( - "backend_configs/systems/corstone-310-vht/backend-config.json" - ), - ), - ), - backend_installer=None, - ) - - return corstone_310 - - -def get_corstone_installations() -> list[Installation]: - """Get Corstone installations.""" - return [ - get_corstone_300_installation(), - get_corstone_310_installation(), - ] -- cgit v1.2.1