From b4240d3aa133b8eefd253498e3f2cc321e24ab84 Mon Sep 17 00:00:00 2001 From: Tom Allsop Date: Fri, 4 Nov 2022 10:40:10 +0000 Subject: Added ASAN & UBSAN build options and Dockerfile for sanitized builds * Added SanitizerBuild.Dockerfile for running sanitized builds. * Added dependencies for bandit into SanitizerBuild.Dockerfile. * Added --sanitizer option to setup.py. * Added .bandit.yaml. Change-Id: I4dd41bc52790a1b7f17ffca556362e37860ab572 --- .bandit.yaml | 5 ++++ .bazelrc | 28 ++++++++++++++++++++ docker/CI.Dockerfile | 31 ++++++++++++++++++++++ docker/README.md | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++- setup.py | 43 ++++++++++++++++++++---------- 5 files changed, 167 insertions(+), 15 deletions(-) create mode 100644 .bandit.yaml create mode 100644 docker/CI.Dockerfile diff --git a/.bandit.yaml b/.bandit.yaml new file mode 100644 index 0000000..a77306d --- /dev/null +++ b/.bandit.yaml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +assert_used: + skips: ['*/test_*.py'] +exclude_dirs: ['build', 'dist', 'bazel-bin', 'bazel-out', 'bazel-tosa_checker','bazel-testlogs','.pytest_cache'] \ No newline at end of file diff --git a/.bazelrc b/.bazelrc index 97fd61c..717b2dc 100644 --- a/.bazelrc +++ b/.bazelrc @@ -6,3 +6,31 @@ common --experimental_repo_remote_exec common --cxxopt=-std=c++17 common --host_cxxopt=-std=c++17 common --copt=-w + +# Address sanitizer +# bazel build --config asan +build:asan --action_env=CC=clang +build:asan --action_env=CXX=clang++ +build:asan --strip=never +build:asan --copt -fsanitize=address +build:asan --copt -DADDRESS_SANITIZER +build:asan --copt -g +build:asan --copt -O1 +build:asan --copt -fno-omit-frame-pointer +build:asan --copt -fno-sanitize-recover=all +build:asan --linkopt -fsanitize=address +build:asan --linkopt -shared-libasan + +# Undefined Behavior Sanitizer +# bazel build --config ubsan +build:ubsan --action_env=CC=clang +build:ubsan --action_env=CXX=clang++ +build:ubsan --strip=never +build:ubsan --copt -fsanitize=undefined +build:ubsan --copt -DUNDEFINED_BEHAVIOR_SANITIZER +build:ubsan --copt -g +build:ubsan --copt -O1 +build:ubsan --copt -fno-omit-frame-pointer +build:ubsan --copt -fno-sanitize-recover=all +build:ubsan --linkopt -fsanitize=undefined +build:ubsan --linkopt -lubsan diff --git a/docker/CI.Dockerfile b/docker/CI.Dockerfile new file mode 100644 index 0000000..d5ebfce --- /dev/null +++ b/docker/CI.Dockerfile @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates. +# SPDX-License-Identifier: Apache-2.0 +FROM ubuntu:22.04 + +ARG PYTHON_VERSION=3.9 +ARG BAZEL_VERSION=5.1.1 +ARG TENSORFLOW_VERSION=2.9.0 + +RUN apt-get update +RUN apt-get install -y build-essential software-properties-common clang curl unzip git libc++-dev libc++abi-dev + +RUN add-apt-repository -y ppa:deadsnakes/ppa +RUN apt-get update + +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Europe/London + +RUN apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-venv python${PYTHON_VERSION}-dev +RUN python${PYTHON_VERSION} -m ensurepip + +RUN python${PYTHON_VERSION} -m pip install --no-cache-dir setuptools pybind11 numpy twine keyrings.alt wheel bandit==1.7.4 + +COPY install/install_bazel.sh /install/ +RUN /install/install_bazel.sh ${BAZEL_VERSION} + +ENV PYTHON_BIN_PATH=/usr/bin/python${PYTHON_VERSION} +ENV CI_BUILD_PYTHON=/usr/bin/python${PYTHON_VERSION} +ENV CROSSTOOL_PYTHON_INCLUDE_PATH=/usr/bin/python${PYTHON_VERSION} + +ARG CACHE_STOP=1 +RUN git clone --depth=1 https://github.com/tensorflow/tensorflow.git --branch v${TENSORFLOW_VERSION} /tensorflow_src \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index 7558954..8475813 100644 --- a/docker/README.md +++ b/docker/README.md @@ -34,13 +34,86 @@ Generate the new manylinux wheel from the `tosa_checker` wheel: ```console auditwheel repair dist/.whl -w dist/ ``` -The `tosa_checker` manylinux wheel can now be found in the `/dist` directory. +The `tosa_checker` manylinux wheel can now be found in the `dist/` directory. Install the `tosa_checker` manylinux wheel: ```console pip install dist/.whl ``` +## How to use the TOSA Checker Docker™ image with security countermeasures + +A Docker™ image is provided for builds of the TOSA Checker with security countermeasures +that are used in the project's continuous integration system. The following countermeasures are provided +in this image: + +* [Address Sanitizer (ASAN)](https://clang.llvm.org/docs/AddressSanitizer.html) +* [Undefined Behavior Sanitizer (UBSAN)](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html) +* [Bandit](https://pypi.org/project/bandit/) + +This section will explain how to use this Docker™ image to build the TOSA checker with +sanitizers and lint the Python source code with Bandit. + +### Building the TOSA Checker with sanitizers +To build the Docker™ image run the command below: + +```console +docker build . -t tc-cp39-countermeasures --build-arg PYTHON_VERSION=3.9 -f CI.Dockerfile +``` + +*Note: In this example, the image is built with Python 3.9. This can be changed using the PYTHON_VERSION argument.* + +After this, run the container as follows, mounting the source code to the container: + +```console +docker run -it -v :/tosa_checker tc-cp39-countermeasures +``` + +Following this, build the TOSA checker using the following command: + +```console +cd tosa_checker +python3.9 setup.py --tensorflow_src_dir /tensorflow_src --sanitizer bdist_wheel +``` +Choose between `asan` or `ubsan` as the sanitizer option. The `tosa_checker` wheel can be found in the `dist/` directory. + +The TOSA Checker wheel can then be installed as follows: + +```console +python3.9 -m pip install dist/.whl +``` + +To then run the unit test of the TOSA Checker, the requirements for this must be installed: + +```console +cd tests +python3.9 -m pip install -r requirements.txt +``` + +Then, if you're using the ASAN option: +```console +export LD_PRELOAD=$(clang -print-file-name=libclang_rt.asan-x86_64.so) +ASAN_OPTIONS=detect_leaks=0 python3.9 -m pytest --capture=no . +``` + +For the UBSAN option, run the following command: +```console +UBSAN_OPTIONS=print_stacktrace=1 python3.9 -m pytest --capture=no . +``` + +### Running the Bandit linter +Firstly build and run the Docker™ image: +```console +docker build . -t tc-cp39-countermeasures --build-arg PYTHON_VERSION=3.9 -f CI.Dockerfile +docker run -it -v :/tosa_checker tc-cp39-countermeasures +``` + +After this, a HTML report can be generated with Bandit as follows: +```console +cd tosa_checker +python3.9 -m bandit --configfile .bandit.yaml -r . -f html -o report.html +``` + ## Trademarks and Copyrights * Python® is a registered trademark of the PSF. diff --git a/setup.py b/setup.py index a62cd22..92b3fb4 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,12 @@ argparser = argparse.ArgumentParser() argparser.add_argument( "--tensorflow_src_dir", help="TensorFlow source directory path", required=False ) +argparser.add_argument( + "--sanitizer", + help="Build using a sanitizer (choose from asan or ubsan)", + choices=["asan", "ubsan"], + required=False +) args, unknown = argparser.parse_known_args() sys.argv = [sys.argv[0]] + unknown @@ -51,21 +57,30 @@ class BazelBuildExtension(setuptools.command.build_ext.build_ext): ext.tensorflow_version, ) - self.spawn( - [ - "bazel", - "build", - "-c", - "opt", - # FIXME Some of the Bazel targets dependencies we use have - # a 'friends' visibility, check if our Bazel target can be added - # to the 'friends' list. - "--check_visibility=false", - "--override_repository=org_tensorflow=" - + os.path.abspath(tensorflow_src_dir), - ext.bazel_target, + commands = [ + "bazel", + "build" + ] + + if args.sanitizer: + commands += [ + "--config={}".format(args.sanitizer) ] - ) + + commands += [ + # FIXME Some of the Bazel targets dependencies we use have + # a 'friends' visibility, check if our Bazel target can be added + # to the 'friends' list. + "-c", + "opt", + "--check_visibility=false", + "--override_repository=org_tensorflow={}".format( + os.path.abspath(tensorflow_src_dir) + ), + ext.bazel_target + ] + + self.spawn(commands) shared_lib_dest_path = self.get_ext_fullpath(ext.name) shared_lib_dest_dir = os.path.dirname(shared_lib_dest_path) -- cgit v1.2.1