From 6ef76bd2eac8c59c51a766a051bfbda681af181a Mon Sep 17 00:00:00 2001 From: Cisco Cervellera Date: Thu, 25 Nov 2021 18:32:27 +0000 Subject: MLECO 2597: Minimal system requirement * Documentation added with missing prerequisite * build_default.py can now be tuned for constraint build systems Change-Id: I74c061359ff663335e664528c4f0616f55cff0f7 --- build_default.py | 54 +++++++++++++++++++++++++++++++++++++++-------- docs/sections/building.md | 26 +++++++++++++++++++++++ 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/build_default.py b/build_default.py index b642b80..3971a05 100755 --- a/build_default.py +++ b/build_default.py @@ -20,19 +20,43 @@ import subprocess import shutil import multiprocessing import logging +import threading import sys -from argparse import ArgumentParser +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter from set_up_default_resources import set_up_resources, \ get_default_npu_config_from_name, \ valid_npu_config_names, \ default_npu_config_names +class PipeLogging(threading.Thread): + + def __init__(self, log_level): + threading.Thread.__init__(self) + self.logLevel = log_level + self.fileRead, self.fileWrite = os.pipe() + self.pipeIn = os.fdopen(self.fileRead) + self.daemon = False + self.start() + + def fileno(self): + return self.fileWrite + + def run(self): + for line in iter(self.pipeIn.readline, ''): + logging.log(self.logLevel, line.strip('\n')) + + self.pipeIn.close() + + def close(self): + os.close(self.fileWrite) def run(toolchain: str, download_resources: bool, run_vela_on_models: bool, - npu_config_name: str): + npu_config_name: str, + make_jobs: int, + make_verbose: bool): """ Run the helpers scripts. @@ -82,6 +106,8 @@ def run(toolchain: str, except Exception as e: logging.error('Failed to delete %s. Reason: %s' % (filepath, e)) + logpipe = PipeLogging(logging.INFO) + os.chdir(build_dir) cmake_toolchain_file = os.path.join(current_file_dir, "scripts", "cmake", "toolchains", toolchain_file_name) @@ -92,19 +118,21 @@ def run(toolchain: str, f" -DETHOS_U_NPU_CONFIG_ID={ethos_u_cfg.ethos_u_config_id}") logging.info(cmake_command) - state = subprocess.run(cmake_command, shell=True, stdout=subprocess.PIPE, + state = subprocess.run(cmake_command, shell=True, stdout=logpipe, stderr=subprocess.STDOUT) - logging.info(state.stdout.decode('utf-8')) - make_command = f"make -j{multiprocessing.cpu_count()}" + make_command = f"make -j{make_jobs}" + if make_verbose : + make_command += " VERBOSE=1" logging.info(make_command) - state = subprocess.run(make_command, shell=True, stdout=subprocess.PIPE, + state = subprocess.run(make_command, shell=True, stdout=logpipe, stderr=subprocess.STDOUT) - logging.info(state.stdout.decode('utf-8')) + + logpipe.close() if __name__ == '__main__': - parser = ArgumentParser() + parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("--toolchain", default="gnu", help=""" Specify the toolchain to use (Arm or GNU). @@ -120,6 +148,12 @@ if __name__ == '__main__': help=f"""Arm Ethos-U configuration to build for. Choose from: {valid_npu_config_names}""", default=default_npu_config_names[0]) + parser.add_argument("--make-jobs", + help="Number of jobs to run with make", + default=multiprocessing.cpu_count()) + parser.add_argument("--make-verbose", + help="Make runs with VERBOSE=1", + action='store_true') args = parser.parse_args() logging.basicConfig(filename='log_build_default.log', level=logging.DEBUG, @@ -129,4 +163,6 @@ if __name__ == '__main__': run(args.toolchain.lower(), not args.skip_download, not args.skip_vela, - args.npu_config_name) + args.npu_config_name, + args.make_jobs, + args.make_verbose) diff --git a/docs/sections/building.md b/docs/sections/building.md index 7ca98c5..b289e52 100644 --- a/docs/sections/building.md +++ b/docs/sections/building.md @@ -2,6 +2,7 @@ - [Building the ML embedded code sample applications from sources](./building.md#building-the-ml-embedded-code-sample-applications-from-sources) - [Build prerequisites](./building.md#build-prerequisites) + - [Third-party build prerequisites](./building.md#third-party-build-prerequisites) - [Build options](./building.md#build-options) - [Build process](./building.md#build-process) - [Preparing build environment](./building.md#preparing-build-environment) @@ -93,6 +94,16 @@ Before proceeding, it is *essential* to ensure that the following prerequisites python3 -m venv ``` +- The build system uses external Python libraries during the building process. Please make sure that the latest pip and libsndfile versions are installed. + + ```commandline + pip3 --version + ``` + + ```log + pip 9.0.1 from /usr/lib/python3/dist-packages (python 3.6) + ``` + - Make ```commandline @@ -111,6 +122,17 @@ Before proceeding, it is *essential* to ensure that the following prerequisites *Ethos™-U55* NPU driver, and CMSIS. Instructions for downloading these are listed under: [preparing build environment](./building.md#preparing-build-environment). +### Third-party build prerequisites + +- The following software is needed by TensorFlow Lite Micro. + + - xxd + - unzip + - Python Pillow + +> **Note:** Due to the fast paced nature of development, this list might not be exhaustive. +Please refer to Tensorflow Lite Micro documentation for more info. + ## Build options The project build system allows you to specify custom neural network models (in the `.tflite` format) for each use-case @@ -337,6 +359,10 @@ Additional command line arguments supported by this script are: - `ethos-u55-256` - `ethos-u65-256` - `ethos-u65-512` +- `--make-jobs`: Specifies the number of concurrent jobs to use for compilation. +The default value is equal to the number of cores in the system. +Lowering this value can be useful in case of limited resources. +- `--make-verbose`: Make the compile process verbose. This is equal to run ```make VERBOSE=1```. To build for *Ethos™-U55* 32 MAC configuration, using `Arm Compiler`, run: -- cgit v1.2.1