aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTom Allsop <tom.allsop@arm.com>2022-11-04 10:40:10 +0000
committerTom Allsop <tom.allsop@arm.com>2022-11-14 11:35:09 +0000
commitb4240d3aa133b8eefd253498e3f2cc321e24ab84 (patch)
tree0fc478cdfc2ed902eda7d0c9afe2734987047c89
parent04a2328f9683b7fe97579a3310d2f473f38717dd (diff)
downloadtosa_checker-b4240d3aa133b8eefd253498e3f2cc321e24ab84.tar.gz
Added ASAN & UBSAN build options and Dockerfile for sanitized builds
* Added SanitizerBuild.Dockerfile for running sanitized builds. * Added dependencies for bandit into SanitizerBuild.Dockerfile. * Added --sanitizer option to setup.py. * Added .bandit.yaml. Change-Id: I4dd41bc52790a1b7f17ffca556362e37860ab572
-rw-r--r--.bandit.yaml5
-rw-r--r--.bazelrc28
-rw-r--r--docker/CI.Dockerfile31
-rw-r--r--docker/README.md75
-rw-r--r--setup.py43
5 files changed, 167 insertions, 15 deletions
diff --git a/.bandit.yaml b/.bandit.yaml
new file mode 100644
index 0000000..a77306d
--- /dev/null
+++ b/.bandit.yaml
@@ -0,0 +1,5 @@
+# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-License-Identifier: Apache-2.0
+assert_used:
+ skips: ['*/test_*.py']
+exclude_dirs: ['build', 'dist', 'bazel-bin', 'bazel-out', 'bazel-tosa_checker','bazel-testlogs','.pytest_cache'] \ No newline at end of file
diff --git a/.bazelrc b/.bazelrc
index 97fd61c..717b2dc 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -6,3 +6,31 @@ common --experimental_repo_remote_exec
common --cxxopt=-std=c++17
common --host_cxxopt=-std=c++17
common --copt=-w
+
+# Address sanitizer
+# bazel build --config asan
+build:asan --action_env=CC=clang
+build:asan --action_env=CXX=clang++
+build:asan --strip=never
+build:asan --copt -fsanitize=address
+build:asan --copt -DADDRESS_SANITIZER
+build:asan --copt -g
+build:asan --copt -O1
+build:asan --copt -fno-omit-frame-pointer
+build:asan --copt -fno-sanitize-recover=all
+build:asan --linkopt -fsanitize=address
+build:asan --linkopt -shared-libasan
+
+# Undefined Behavior Sanitizer
+# bazel build --config ubsan
+build:ubsan --action_env=CC=clang
+build:ubsan --action_env=CXX=clang++
+build:ubsan --strip=never
+build:ubsan --copt -fsanitize=undefined
+build:ubsan --copt -DUNDEFINED_BEHAVIOR_SANITIZER
+build:ubsan --copt -g
+build:ubsan --copt -O1
+build:ubsan --copt -fno-omit-frame-pointer
+build:ubsan --copt -fno-sanitize-recover=all
+build:ubsan --linkopt -fsanitize=undefined
+build:ubsan --linkopt -lubsan
diff --git a/docker/CI.Dockerfile b/docker/CI.Dockerfile
new file mode 100644
index 0000000..d5ebfce
--- /dev/null
+++ b/docker/CI.Dockerfile
@@ -0,0 +1,31 @@
+# SPDX-FileCopyrightText: Copyright 2022, Arm Limited and/or its affiliates.
+# SPDX-License-Identifier: Apache-2.0
+FROM ubuntu:22.04
+
+ARG PYTHON_VERSION=3.9
+ARG BAZEL_VERSION=5.1.1
+ARG TENSORFLOW_VERSION=2.9.0
+
+RUN apt-get update
+RUN apt-get install -y build-essential software-properties-common clang curl unzip git libc++-dev libc++abi-dev
+
+RUN add-apt-repository -y ppa:deadsnakes/ppa
+RUN apt-get update
+
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/London
+
+RUN apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-venv python${PYTHON_VERSION}-dev
+RUN python${PYTHON_VERSION} -m ensurepip
+
+RUN python${PYTHON_VERSION} -m pip install --no-cache-dir setuptools pybind11 numpy twine keyrings.alt wheel bandit==1.7.4
+
+COPY install/install_bazel.sh /install/
+RUN /install/install_bazel.sh ${BAZEL_VERSION}
+
+ENV PYTHON_BIN_PATH=/usr/bin/python${PYTHON_VERSION}
+ENV CI_BUILD_PYTHON=/usr/bin/python${PYTHON_VERSION}
+ENV CROSSTOOL_PYTHON_INCLUDE_PATH=/usr/bin/python${PYTHON_VERSION}
+
+ARG CACHE_STOP=1
+RUN git clone --depth=1 https://github.com/tensorflow/tensorflow.git --branch v${TENSORFLOW_VERSION} /tensorflow_src \ No newline at end of file
diff --git a/docker/README.md b/docker/README.md
index 7558954..8475813 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -34,13 +34,86 @@ Generate the new manylinux wheel from the `tosa_checker` wheel:
```console
auditwheel repair dist/<tosa_checker>.whl -w dist/
```
-The `tosa_checker` manylinux wheel can now be found in the `/dist` directory.
+The `tosa_checker` manylinux wheel can now be found in the `dist/` directory.
Install the `tosa_checker` manylinux wheel:
```console
pip install dist/<tosa_checker-manyliux>.whl
```
+## How to use the TOSA Checker Docker™ image with security countermeasures
+
+A Docker™ image is provided for builds of the TOSA Checker with security countermeasures
+that are used in the project's continuous integration system. The following countermeasures are provided
+in this image:
+
+* [Address Sanitizer (ASAN)](https://clang.llvm.org/docs/AddressSanitizer.html)
+* [Undefined Behavior Sanitizer (UBSAN)](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html)
+* [Bandit](https://pypi.org/project/bandit/)
+
+This section will explain how to use this Docker™ image to build the TOSA checker with
+sanitizers and lint the Python source code with Bandit.
+
+### Building the TOSA Checker with sanitizers
+To build the Docker™ image run the command below:
+
+```console
+docker build . -t tc-cp39-countermeasures --build-arg PYTHON_VERSION=3.9 -f CI.Dockerfile
+```
+
+*Note: In this example, the image is built with Python 3.9. This can be changed using the PYTHON_VERSION argument.*
+
+After this, run the container as follows, mounting the source code to the container:
+
+```console
+docker run -it -v <tosa_checker source code on your machine>:/tosa_checker tc-cp39-countermeasures
+```
+
+Following this, build the TOSA checker using the following command:
+
+```console
+cd tosa_checker
+python3.9 setup.py --tensorflow_src_dir /tensorflow_src --sanitizer <sanitizer option> bdist_wheel
+```
+Choose between `asan` or `ubsan` as the sanitizer option. The `tosa_checker` wheel can be found in the `dist/` directory.
+
+The TOSA Checker wheel can then be installed as follows:
+
+```console
+python3.9 -m pip install dist/<tosa_checker>.whl
+```
+
+To then run the unit test of the TOSA Checker, the requirements for this must be installed:
+
+```console
+cd tests
+python3.9 -m pip install -r requirements.txt
+```
+
+Then, if you're using the ASAN option:
+```console
+export LD_PRELOAD=$(clang -print-file-name=libclang_rt.asan-x86_64.so)
+ASAN_OPTIONS=detect_leaks=0 python3.9 -m pytest --capture=no .
+```
+
+For the UBSAN option, run the following command:
+```console
+UBSAN_OPTIONS=print_stacktrace=1 python3.9 -m pytest --capture=no .
+```
+
+### Running the Bandit linter
+Firstly build and run the Docker™ image:
+```console
+docker build . -t tc-cp39-countermeasures --build-arg PYTHON_VERSION=3.9 -f CI.Dockerfile
+docker run -it -v <tosa_checker source code on your machine>:/tosa_checker tc-cp39-countermeasures
+```
+
+After this, a HTML report can be generated with Bandit as follows:
+```console
+cd tosa_checker
+python3.9 -m bandit --configfile .bandit.yaml -r . -f html -o report.html
+```
+
## Trademarks and Copyrights
* Python® is a registered trademark of the PSF.
diff --git a/setup.py b/setup.py
index a62cd22..92b3fb4 100644
--- a/setup.py
+++ b/setup.py
@@ -18,6 +18,12 @@ argparser = argparse.ArgumentParser()
argparser.add_argument(
"--tensorflow_src_dir", help="TensorFlow source directory path", required=False
)
+argparser.add_argument(
+ "--sanitizer",
+ help="Build using a sanitizer (choose from asan or ubsan)",
+ choices=["asan", "ubsan"],
+ required=False
+)
args, unknown = argparser.parse_known_args()
sys.argv = [sys.argv[0]] + unknown
@@ -51,21 +57,30 @@ class BazelBuildExtension(setuptools.command.build_ext.build_ext):
ext.tensorflow_version,
)
- self.spawn(
- [
- "bazel",
- "build",
- "-c",
- "opt",
- # FIXME Some of the Bazel targets dependencies we use have
- # a 'friends' visibility, check if our Bazel target can be added
- # to the 'friends' list.
- "--check_visibility=false",
- "--override_repository=org_tensorflow="
- + os.path.abspath(tensorflow_src_dir),
- ext.bazel_target,
+ commands = [
+ "bazel",
+ "build"
+ ]
+
+ if args.sanitizer:
+ commands += [
+ "--config={}".format(args.sanitizer)
]
- )
+
+ commands += [
+ # FIXME Some of the Bazel targets dependencies we use have
+ # a 'friends' visibility, check if our Bazel target can be added
+ # to the 'friends' list.
+ "-c",
+ "opt",
+ "--check_visibility=false",
+ "--override_repository=org_tensorflow={}".format(
+ os.path.abspath(tensorflow_src_dir)
+ ),
+ ext.bazel_target
+ ]
+
+ self.spawn(commands)
shared_lib_dest_path = self.get_ext_fullpath(ext.name)
shared_lib_dest_dir = os.path.dirname(shared_lib_dest_path)