From ba251631768f25b840e93ece6a4af3db119e6dd1 Mon Sep 17 00:00:00 2001 From: Benjamin Klimczak Date: Mon, 28 Aug 2023 16:57:24 +0100 Subject: MLIA-961 Update pre-commit hook versions - Update dependencies in .pre-commit.yaml - Fix code issues with new linters Change-Id: I36964ecf1a405dd8faac01b4470b56122a7cad17 Signed-off-by: Benjamin Klimczak --- .pre-commit-config.yaml | 24 ++++++++++++------------ README.md | 4 ++-- RELEASES.md | 4 ++-- src/mlia/backend/install.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 58 insertions(+), 17 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2de4fac..57ef192 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-FileCopyrightText: Copyright 2022-2023, Arm Limited and/or its affiliates. # SPDX-License-Identifier: Apache-2.0 # Pre-commit checks # @@ -11,7 +11,7 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 + rev: v4.4.0 hooks: - id: check-yaml @@ -38,57 +38,57 @@ repos: args: [--whitespaces-count, '8'] - repo: https://github.com/asottile/reorder_python_imports - rev: v3.9.0 + rev: v3.10.0 hooks: - id: reorder-python-imports args: ["--application-directories", ".:src"] - repo: https://github.com/asottile/pyupgrade - rev: v3.2.0 + rev: v3.10.1 hooks: - id: pyupgrade args: ["--py38-plus"] - repo: https://github.com/psf/black - rev: 22.10.0 + rev: 23.7.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: '5.0.4' + rev: 6.1.0 hooks: - id: flake8 - repo: https://github.com/PyCQA/pydocstyle - rev: 6.1.1 + rev: 6.3.0 hooks: - id: pydocstyle - repo: https://github.com/markdownlint/markdownlint - rev: v0.11.0 + rev: v0.12.0 hooks: - id: markdownlint args: ["-r", "~MD024,~MD002"] - repo: https://github.com/ryanrhee/shellcheck-py - rev: v0.8.0.4 + rev: v0.9.0.5 hooks: - id: shellcheck args: [-f, gcc, -x] - repo: https://github.com/fsfe/reuse-tool - rev: v1.0.0 + rev: v2.1.0 hooks: - id: reuse - repo: https://github.com/PrincetonUniversity/blocklint - rev: v0.2.3 + rev: v0.2.4 hooks: - id: blocklint exclude: setup.cfg - repo: https://github.com/PyCQA/bandit - rev: '1.7.4' + rev: 1.7.5 hooks: - id: bandit args: ["--skip", "B101"] diff --git a/README.md b/README.md index 7c9209a..725af53 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,8 @@ typical setup for MLIA requires: tested on this one specifically) * Python® >= 3.8.1 * Ethos™-U Vela dependencies (Linux® only) - * For more details, please refer to the - [prerequisites of Vela](https://pypi.org/project/ethos-u-vela/) + * For more details, please refer to the + [prerequisites of Vela](https://pypi.org/project/ethos-u-vela/) ## Installation diff --git a/RELEASES.md b/RELEASES.md index 992b146..4ac1b9b 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -135,7 +135,7 @@ scheme. * Ethos-U operator compatibility, performance estimation and optimization advice * Arm IP support: - * Ethos-U55 via Corstone-300 and Corstone-310 - * Ethos-U65 via Corstone-300 + * Ethos-U55 via Corstone-300 and Corstone-310 + * Ethos-U65 via Corstone-300 Note: Corstone-310 is available on AVH only. diff --git a/src/mlia/backend/install.py b/src/mlia/backend/install.py index 721b660..f405511 100644 --- a/src/mlia/backend/install.py +++ b/src/mlia/backend/install.py @@ -11,6 +11,7 @@ from abc import abstractmethod from dataclasses import dataclass from pathlib import Path from typing import Callable +from typing import Iterable from typing import Optional from typing import Union @@ -177,7 +178,47 @@ class BackendInstallation(Installation): with working_directory(tmpdir / "dist", create_dir=True) as dist_dir: with tarfile.open(downloaded_to) as archive: - archive.extractall(dist_dir) + + def get_filtered_members( + members: Iterable[tarfile.TarInfo], + ) -> Iterable[tarfile.TarInfo]: + """ + Make sure we only handle safe files from the tar file. + + To avoid traversal attacks we only allow files that are + - regular files, i.e. not a symlink etc. + - relative paths, i.e. no absolute file paths + - not including directory traversal sequences '..' + """ + for member in members: + try: + if not (member.isfile() or member.isdir()): + raise ValueError("Path is not a regular file.") + path = Path(member.path) + if path.is_absolute(): + raise ValueError( + "Path is absolute, but must be relative." + ) + abs_path = (dist_dir / path).resolve() + abs_path.relative_to(dist_dir) + yield member + except ValueError as ex: + logger.warning( + "File '%s' ignored while extracting from %s: %s", + member.path, + downloaded_to, + ex, + ) + + # Filter files from the tarfile to avoid traversal attacks. + # Note: bandit is still putting out a low severity / + # low confidence warning despite the check + # From Python 3.8.17 on there is a built-in feature to fix + # this using the new argument filter="data", see + # https://docs.python.org/3.8/library/tarfile.html#tarfile.TarFile.extractall + archive.extractall( # nosec + dist_dir, members=get_filtered_members(archive.getmembers()) + ) backend_path = dist_dir if self.backend_installer: -- cgit v1.2.1