From 8c7c569fca0988776f108ba4a56c1f60e176b62c Mon Sep 17 00:00:00 2001 From: James Conroy Date: Thu, 4 Aug 2022 16:55:05 +0100 Subject: IVGCVSW-6777 Add Arm NN build-tool Dockerfile * Adds Dockerfile associated with Arm NN build-tool scripts. * The Dockerfile encapsulates the installation of system-wide packages (install-packages.sh), download/install of Arm NN dependencies (setup-armnn.sh) and the building of Arm NN and ACL (build-armnn.sh). * A helper script for copying build contents from the built Docker image is provided for by docker-copy-to-host.sh. * Modified existing scripts: moved the cloning of Arm NN and ACL from setup-armnn.sh to build-armnn.sh and decoupled setup-armnn.sh from scripts outside of build-tool directory e.g. armnn/scripts/get_tensorflow.sh. * The build-armnn.sh script clones the latest release branches of Arm NN and ACL by default. Custom repos can be placed in the build-tool directory prior to 'docker build' and they will be used instead (advanced usage). * Support added for Linux targets only, Android to be added in future work. Signed-off-by: James Conroy Change-Id: I336013cf93821d2cd3e5d9fe2ca4e955ffdd2386 --- build-tool/docker/Dockerfile | 79 ++++++++++++++++++++ build-tool/scripts/build-armnn.sh | 61 ++++++++++++--- build-tool/scripts/common.sh | 17 ++++- build-tool/scripts/docker-copy-to-host.sh | 49 ++++++++++++ build-tool/scripts/install-packages.sh | 119 ++++++++++++++++++++++++++++++ build-tool/scripts/setup-armnn.sh | 55 +------------- 6 files changed, 318 insertions(+), 62 deletions(-) create mode 100644 build-tool/docker/Dockerfile create mode 100755 build-tool/scripts/docker-copy-to-host.sh create mode 100755 build-tool/scripts/install-packages.sh diff --git a/build-tool/docker/Dockerfile b/build-tool/docker/Dockerfile new file mode 100644 index 0000000000..d1f212c684 --- /dev/null +++ b/build-tool/docker/Dockerfile @@ -0,0 +1,79 @@ +# +# Copyright © 2022 Arm Ltd and Contributors. All rights reserved. +# SPDX-License-Identifier: MIT +# + +# Default build type is 'production'. Use 'dev' if supplying custom Arm NN / ACL repos from host +ARG BUILD_TYPE=production + +ARG UBUNTU_VERSION=18.04 +FROM ubuntu:${UBUNTU_VERSION} AS build-production + +ENV DEBIAN_FRONTEND noninteractive + +# Install basic packages for Docker container (not specific to Arm NN) +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + locales \ + vim \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Set locale for Docker container +RUN locale-gen en_GB.UTF-8 && \ + update-locale LC_ALL=en_GB.UTF-8 LANG=en_GB.UTF-8 +ENV LANG en_GB.UTF-8 +ENV LC_ALL en_GB.UTF-8 + +WORKDIR /root + +# Install system-wide packages specific to Arm NN +COPY ./scripts/install-packages.sh . +RUN ./install-packages.sh + +# Define user for non-root processes (overwriteable during 'docker build' with --build-arg) +ARG USER_ID=1000 +ARG GROUP_ID=1000 + +# Create non-root user 'arm-user' based on $USER_ID and $GROUP_ID above +RUN addgroup --gid $GROUP_ID arm-user +RUN useradd --create-home --shell /bin/bash --uid $USER_ID --gid $GROUP_ID arm-user + +# Switch to non-root user +USER arm-user +WORKDIR /home/arm-user + +# Copy scripts required by Setup into WORKDIR +COPY --chown=arm-user:arm-user ./scripts/validation.sh . +COPY --chown=arm-user:arm-user ./scripts/common.sh . +COPY --chown=arm-user:arm-user ./scripts/setup-armnn.sh . + +# Run setup-armnn.sh: download and install Arm NN dependencies +ARG SETUP_ARGS="" +RUN echo "SETUP_ARGS: $SETUP_ARGS" +RUN ./setup-armnn.sh $SETUP_ARGS + +# This build-dev stage (inherits 'build-production' stage) is only used in final image if $BUILD_TYPE is 'dev' +FROM build-production as build-dev + +# Create directory for source repos in WORKDIR +RUN mkdir -p source/armnn source/acl + +# Copy custom armnn/acl source repos from the build-tool directory on the host, if they exist (optional) +# If repos not provided, the build-armnn.sh script will automatically download the latest release branches of Arm NN and ACL +# Copies Dockerfile to ensure COPY works - at least one file must exist for COPY to work +COPY --chown=arm-user:arm-user ./docker/Dockerfile ./armnn* ./source/armnn/ +COPY --chown=arm-user:arm-user ./docker/Dockerfile ./acl* ./source/acl/ + +# Final stage which inherits either 'build-production' or 'build-dev' stage +FROM build-${BUILD_TYPE} as final + +# Copy build script into WORKDIR +COPY --chown=arm-user:arm-user ./scripts/build-armnn.sh . + +# Run build-armnn.sh: build Arm NN and ACL +ARG BUILD_ARGS="" +RUN echo "BUILD_ARGS: $BUILD_ARGS" +RUN ./build-armnn.sh $BUILD_ARGS \ No newline at end of file diff --git a/build-tool/scripts/build-armnn.sh b/build-tool/scripts/build-armnn.sh index 2b8217b313..98b3b3f6fc 100755 --- a/build-tool/scripts/build-armnn.sh +++ b/build-tool/scripts/build-armnn.sh @@ -149,6 +149,43 @@ build_armnn() return 0 } +download_armnn() +{ + cd "$SOURCE_DIR" + + echo -e "\n***** Downloading Arm NN *****" + + rm -rf "$ARMNN_SRC" + + # Latest release branch of Arm NN is checked out by default + git clone https://github.com/ARM-software/armnn.git armnn + + cd "$ARMNN_SRC" + armnn_branch="$(git rev-parse --abbrev-ref HEAD)" + + echo -e "\n***** Arm NN Downloaded: $armnn_branch *****" +} + +download_acl() +{ + cd "$SOURCE_DIR" + + echo -e "\n***** Downloading ACL *****" + + rm -rf "$ACL_SRC" + + git clone https://github.com/ARM-software/ComputeLibrary.git acl + + # Get corresponding release tag for ACL by parsing release branch number for Arm NN + local acl_tag="" + acl_tag="$(echo "$armnn_branch" | tr '\n' ' ' | sed -e 's/[^0-9]/ /g' -e 's/^ *//g' -e 's/ *$//g' | tr -s ' ' | sed 's/ /./g')" + + cd "$ACL_SRC" + git checkout v"$acl_tag" + + echo -e "\n***** ACL Downloaded: $acl_tag *****" +} + usage() { cat </source, downloaded using setup-armnn.sh. +The first execution of this script will download the latest release branches of Arm NN and ACL, by default. Alternatively, place custom/modified repositories named "armnn" and "acl" in /source. +If providing custom repos, both Arm NN and ACL must be provided. The ACL repo will not be used if flags --neon-backend or --cl-backend are not selected. By default, a tarball tar.gz archive of the Arm NN build will be created in the directory from which this script is called from. @@ -356,17 +394,22 @@ if [ ! -d "$BUILD_DIR" ]; then exit 1 fi -# Verify that Arm NN and ACL exist in correct paths prior to script execution -if [ ! -d "$ARMNN_SRC" ]; then - echo -e "\nERROR: Arm NN repo does not exist as expected at $ARMNN_SRC" - exit 1 -fi +# Download Arm NN and ACL if not done already in a previous execution of this script +# Check if Arm NN source directory exists AND that it is a repository (not empty) +if [ -d "$ARMNN_SRC" ] && check_if_repository "$ARMNN_SRC"; then + echo -e "\n***** Arm NN source repository already located at $ARMNN_SRC. Skipping cloning of Arm NN. *****" -if [ "$flag_neon_backend" -eq 1 ] || [ "$flag_cl_backend" -eq 1 ]; then - if [ ! -d "$ACL_SRC" ]; then - echo -e "\nERROR: ACL repo does not exist as expected at $ACL_SRC" + # ACL repo must also be present if Arm NN repo is present + if [ -d "$ACL_SRC" ] && check_if_repository "$ACL_SRC"; then + echo -e "\n***** ACL source repository already located at $ACL_SRC. Skipping cloning of ACL. *****" + else + echo -e "\nERROR: ACL source repository must be provided at $ACL_SRC if Arm NN source is provided. *****" exit 1 fi +else + # Download latest release branches of Arm NN and ACL + download_armnn + download_acl fi # Adjust output build directory names for Arm NN and ACL if debug is enabled diff --git a/build-tool/scripts/common.sh b/build-tool/scripts/common.sh index 522bfb68cd..f9f8f974aa 100755 --- a/build-tool/scripts/common.sh +++ b/build-tool/scripts/common.sh @@ -47,6 +47,7 @@ FLATBUFFERS_BUILD_TARGET="$FLATBUFFERS_BUILD_ROOT"/"$TARGET_ARCH"_build FLATBUFFERS_BUILD_HOST="$FLATBUFFERS_BUILD_ROOT"/"$HOST_ARCH"_build # Location of flatc compiler # Tensorflow +TENSORFLOW_VERSION="tags/v2.5.0" TENSORFLOW_SRC="$SOURCE_DIR"/tensorflow TFLITE_SRC="$TENSORFLOW_SRC"/tensorflow/lite SCHEMA_SRC="$TFLITE_SRC"/schema/schema.fbs @@ -72,4 +73,18 @@ ONNX_BUILD_TARGET="$BUILD_DIR"/onnx/"$TARGET_ARCH"_build # Arm NN / ACL ARMNN_SRC="$SOURCE_DIR"/armnn -ACL_SRC="$SOURCE_DIR"/acl \ No newline at end of file +ACL_SRC="$SOURCE_DIR"/acl + +# Check if directory at $1 is a repository or not +check_if_repository() +{ + pushd "$1" > /dev/null + + if [ "$(git rev-parse --is-inside-work-tree 2> /dev/null)" ]; then + popd > /dev/null + return 0 + else + popd > /dev/null + return 1 + fi +} \ No newline at end of file diff --git a/build-tool/scripts/docker-copy-to-host.sh b/build-tool/scripts/docker-copy-to-host.sh new file mode 100755 index 0000000000..bb120462e5 --- /dev/null +++ b/build-tool/scripts/docker-copy-to-host.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# +# Copyright © 2022 Arm Ltd and Contributors. All rights reserved. +# SPDX-License-Identifier: MIT +# + +# Script which copies a file or directory from the /home/arm-user/ directory in Docker to the host machine +# This script creates a directory called 'docker_output' in the current directory and places the copied contents there +# Takes two arguments: +# 1. Name of created Docker image i.e. "--tag " provided at 'docker build' stage (tag is optional in image naming) +# 2. Relative path to file or directory to copy from the Docker /home/arm-user/ directory +# +# Examples: +# 1. Copy the tarball of the aarch64 build from the /home/arm-user/ directory +# ./scripts/docker-copy-to-host.sh armnn_image armnn_aarch64_build.tar.gz +# 2. Copy the unarchived Arm NN build +# ./scripts/docker-copy-to-host.sh armnn_image build/armnn +# 3. Copy the unarchived ACL build +# ./scripts/docker-copy-to-host.sh armnn_image build/acl + +set -o nounset # Catch references to undefined variables. +set -o pipefail # Catch non zero exit codes within pipelines. +set -o errexit # Catch and propagate non zero exit codes. + +image_name="$1" +file_path="$2" + +name=$(basename "$0") + +echo "***** $name: Copying file(s) from path /home/arm-user/$file_path inside Docker image '$image_name' to host *****" + +echo -e "\n***** Creating directory docker_output on host *****" +mkdir -p docker_output + +# Cleanup old 'armnn_temp' container in case a previous run of this script was not successful +docker rm --force armnn_temp 2> /dev/null + +echo -e "\n***** Creating temporary Docker container named armnn_temp using Docker image '$image_name' *****" +docker create --interactive --tty --name armnn_temp "$image_name" bash > /dev/null + +echo -e "\n***** Running Docker command: docker cp armnn_temp:/home/arm-user/$file_path ./docker_output *****" +docker cp armnn_temp:/home/arm-user/"$file_path" ./docker_output > /dev/null + +echo -e "\n***** Successfully copied file(s) to host in directory docker_output *****" + +# Remove temporary docker container 'armnn_temp' +docker rm --force armnn_temp > /dev/null + +echo -e "\n***** Deleted temporary Docker container armnn_temp *****" \ No newline at end of file diff --git a/build-tool/scripts/install-packages.sh b/build-tool/scripts/install-packages.sh new file mode 100755 index 0000000000..4f62fece2c --- /dev/null +++ b/build-tool/scripts/install-packages.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# +# Copyright © 2022 Arm Ltd and Contributors. All rights reserved. +# SPDX-License-Identifier: MIT +# + +# Script which installs system-wide packages required by setup-armnn.sh and build-armnn.sh +# Downloads and builds CMake from source in the current directory from which this script is called +# CMake will be installed system-wide once this script has completed execution +# Requires sudo privileges + +set -o nounset # Catch references to undefined variables. +set -o pipefail # Catch non zero exit codes within pipelines. +set -o errexit # Catch and propagate non zero exit codes. + +# Host architecture e.g. x86_64, aarch64 +HOST_ARCH=$(uname -m) + +# Number of online cores on host +NUM_THREADS=$(getconf _NPROCESSORS_ONLN) + +# CMake is downloaded and built in the current directory from which this script is called +ROOT_DIR=$(pwd) + +# CMake +CMAKE_VERSION=3.19 +CMAKE_VERSION_FULL=3.19.0 +CMAKE_SRC="$ROOT_DIR"/cmake-"$CMAKE_VERSION_FULL" +CMAKE_BUILD="$ROOT_DIR"/cmake_build + +download_cmake() +{ + cd "$ROOT_DIR" + + echo -e "\n***** Downloading CMake $CMAKE_VERSION *****" + wget -O cmake-"$CMAKE_VERSION_FULL".tar.gz https://cmake.org/files/v"$CMAKE_VERSION"/cmake-"$CMAKE_VERSION_FULL".tar.gz + + echo -e "\n***** Extracting archive *****" + tar -xzf cmake-"$CMAKE_VERSION_FULL".tar.gz + + echo -e "\n***** Removing archive *****" + rm cmake-"$CMAKE_VERSION_FULL".tar.gz + + echo -e "\n***** CMake $CMAKE_VERSION Downloaded *****" +} + +install_cmake() +{ + mkdir -p "$CMAKE_BUILD" + cd "$CMAKE_BUILD" + + apt-get purge -y cmake + + echo -e "\n***** Building CMake $CMAKE_VERSION ***** " + "$CMAKE_SRC"/bootstrap + make + make install -j "$NUM_THREADS" + + if [[ "$(cmake --version 2> /dev/null | grep "$CMAKE_VERSION" )" == *"$CMAKE_VERSION"* ]]; then + echo -e "\n***** Built and Installed CMake $CMAKE_VERSION *****" + else + echo -e "\nERROR: CMake $CMAKE_VERSION not installed correctly after building from source" + exit 1 + fi +} + +install_apt_packages() +{ + apt-get update && apt-get install -y --no-install-recommends \ + autoconf \ + automake \ + build-essential \ + curl \ + git \ + libssl-dev \ + libtool \ + make \ + scons \ + unzip \ + wget + + # Install cross compile toolchains if host is x86_64 + if [ "$HOST_ARCH" == "x86_64" ]; then + apt-get update && apt-get install -y --no-install-recommends \ + crossbuild-essential-arm64 \ + crossbuild-essential-armhf + fi + + apt-get clean + rm -rf /var/lib/apt/lists/* +} + +name=$(basename "$0") + +if [ ! "$(id -u)" -eq 0 ]; then + echo -e "\nERROR: $name must be ran as root (i.e. sudo ./$name)" + exit 1 +fi + +echo -e "\n***** $name: Installing system-wide packages required by setup-armnn.sh and build-armnn.sh *****" +echo -e "\nINFO: This script downloads and builds CMake from source in the current directory from which this script is called" +echo -e "\nINFO: CMake and other apt packages will be installed system-wide once this script has completed execution" +echo -e "\nScript execution will begin in 10 seconds..." + +sleep 10 + +install_apt_packages + +# Download, Build and Install CMake if not already present +if [[ "$(cmake --version 2> /dev/null | grep "$CMAKE_VERSION" )" == *"$CMAKE_VERSION"* ]]; then + echo -e "\n***** CMake $CMAKE_VERSION already installed, skipping CMake install *****" +else + download_cmake + install_cmake +fi + +echo -e "\n***** $name: Successfully installed system-wide packages required by setup-armnn.sh and build-armnn.sh *****\n" + +exit 0 \ No newline at end of file diff --git a/build-tool/scripts/setup-armnn.sh b/build-tool/scripts/setup-armnn.sh index 6a9c22ebef..d6f6fce9db 100755 --- a/build-tool/scripts/setup-armnn.sh +++ b/build-tool/scripts/setup-armnn.sh @@ -22,7 +22,7 @@ download_and_extract() { cd "$SOURCE_DIR" - echo -e "\n***** Downloading $1 *****" + echo -e "\n***** Downloading $1 *****\n" wget -O "$3" "$2" echo -e "\n***** Extracting archive *****" @@ -135,7 +135,7 @@ download_tensorflow() git clone https://github.com/tensorflow/tensorflow.git cd "$TENSORFLOW_SRC" - git checkout "$("$ARMNN_SRC"/scripts/get_tensorflow.sh -p)" + git checkout "$TENSORFLOW_VERSION" echo -e "\n***** TensorFlow downloaded *****" } @@ -220,39 +220,6 @@ generate_onnx_sources() echo -e "\n***** Generated ONNX sources for $TARGET_ARCH *****" } -download_armnn() -{ - cd "$SOURCE_DIR" - - echo -e "\n***** Downloading Arm NN *****" - - # Latest release branch of Arm NN is checked out by default - git clone https://github.com/ARM-software/armnn.git armnn - - cd "$ARMNN_SRC" - armnn_branch="$(git rev-parse --abbrev-ref HEAD)" - - echo -e "\n***** Arm NN Downloaded: $armnn_branch *****" -} - -download_acl() -{ - cd "$SOURCE_DIR" - - echo -e "\n***** Downloading ACL *****" - - git clone https://github.com/ARM-software/ComputeLibrary.git acl - - # Get corresponding release tag for ACL by parsing release branch number for Arm NN - local acl_tag="" - acl_tag="$(echo "$armnn_branch" | tr '\n' ' ' | sed -e 's/[^0-9]/ /g' -e 's/^ *//g' -e 's/ *$//g' | tr -s ' ' | sed 's/ /./g')" - - cd "$ACL_SRC" - git checkout v"$acl_tag" - - echo -e "\n***** ACL Downloaded: $acl_tag *****" -} - usage() { cat </setup-armnn.sh --target-arch=aarch64 --all @@ -387,7 +350,7 @@ echo " root directory: $ROOT_DIR" echo "source directory: $SOURCE_DIR" echo " build directory: $BUILD_DIR" -if [ "$(git rev-parse --is-inside-work-tree 2> /dev/null)" ]; then +if check_if_repository .; then echo -e "\n***** WARNING: Running script inside a git repository. To avoid nested repos, call this script from outside of this repo. *****" fi @@ -398,18 +361,6 @@ sleep 10 mkdir -p "$SOURCE_DIR" mkdir -p "$BUILD_DIR" -if [ -d "$ARMNN_SRC" ]; then - echo -e "\n***** Arm NN source repository already located at $ARMNN_SRC. Skipping cloning of Arm NN. *****" -else - download_armnn -fi - -if [ -d "$ACL_SRC" ]; then - echo -e "\n***** ACL source repository already located at $ACL_SRC. Skipping cloning of ACL. *****" -else - download_acl -fi - if [ "$flag_tflite_delegate" -eq 1 ] || [ "$flag_tflite_parser" -eq 1 ]; then download_flatbuffers -- cgit v1.2.1