From a5e7920ade44d5cd018b404df276587e3cc4d8b0 Mon Sep 17 00:00:00 2001 From: Kristofer Jonsson Date: Tue, 1 Sep 2020 16:32:43 +0200 Subject: Externals configuration file Store configuration of external repositories in a JSON file. Add command to dump configuration to stdout. Change-Id: I84d5388d9bd31e4d1c6626260c9418e07446634a --- .gitignore | 1 + README.md | 16 +++--- externals.json | 39 +++++++++++++++ fetch_externals.py | 142 ++++++++++++++++++++++++++++++++++++++++++----------- 4 files changed, 161 insertions(+), 37 deletions(-) create mode 100644 externals.json diff --git a/.gitignore b/.gitignore index a2a4412..5febf4e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /core_software +/linux_driver_stack /vela diff --git a/README.md b/README.md index 269c7ec..1a90c1e 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # Ethos-u -This is the root repository for all Ethos-u software. It is provided to help +This is the root repository for all Arm Ethos-U software. It is provided to help users download required repositories and place them in a tree structure. ``` -$ ./fetch_externals.py +$ ./fetch_externals.py fetch ``` The script will build following tree structure. @@ -22,10 +22,10 @@ Directory | Directory | Description | --- | --- -| [.](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u) | This is the root directory for all Ethos-u software. | -| [core_software](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-core-software) | The software executing on Arm Cortex-M is referred to as _Core Software_. This folder provides a small build system that illustrates how to build the key components for the Ethos-u core software. | -| [core_driver](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-core-driver) | The Ethos-u NPU driver. | +| [.](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u) | This is the root directory for all Arm Ethos-U software. | +| [core_software](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-core-software) | The software executing on Arm Cortex-M is referred to as _Core Software_. This folder provides a small build system that illustrates how to build the key components for the Arm Ethos-U core software. | +| [core_driver](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-core-driver) | The Arm Ethos-U NPU driver. | | [cmsis](https://github.com/ARM-software/CMSIS_5) | CMSIS provides optimized kernels and generic interfaces to the Arm Cortex-M CPUs. | -| [tensorflow](https://github.com/tensorflow/tensorflow) | The TensorFlowLite micro framework is used to run inferences. | -| [linux_driver_stack](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-linux-driver-stack) | Example driver stack showing how Linux can dispatch inferences to an Ethos-U subsystem. | -| [vela](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-vela) | The Vela optimizer takes a TFLu file as input and replaces operators that are supported by the Ethos-u NPU with custom operators designed to run on the NPU. Operators not supported by the NPU are executed in software. | +| [tensorflow](https://github.com/tensorflow/tensorflow) | The TensorFlow Lite microcontroller framework is used to run inferences. | +| [linux_driver_stack](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-linux-driver-stack) | Example driver stack showing how Linux can dispatch inferences to an Arm Ethos-U subsystem. | +| [vela](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-vela) | The Vela optimizer takes a TFLu file as input and replaces operators that are supported by the Arm Ethos-U NPU with custom operators designed to run on the NPU. Operators not supported by the NPU are executed in software. | diff --git a/externals.json b/externals.json new file mode 100644 index 0000000..6ee5456 --- /dev/null +++ b/externals.json @@ -0,0 +1,39 @@ +{ + "externals": + [ + { + "path": "core_software", + "fetchurl": "https://review.mlplatform.org/ml/ethos-u/ethos-u-core-software", + "pushurl": "ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-core-software", + "revision": "master" + }, + { + "path": "core_software/core_driver", + "fetchurl": "https://review.mlplatform.org/ml/ethos-u/ethos-u-core-driver", + "pushurl": "ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-core-driver", + "revision": "master" + }, + { + "path": "core_software/cmsis", + "fetchurl": "https://github.com/ARM-software/CMSIS_5.git", + "revision": "master" + }, + { + "path": "core_software/tensorflow", + "fetchurl": "https://github.com/tensorflow/tensorflow", + "revision": "master" + }, + { + "path": "linux_driver_stack", + "fetchurl": "https://review.mlplatform.org/ml/ethos-u/ethos-u-linux-driver-stack", + "pushurl": "ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-linux-driver-stack", + "revision": "master" + }, + { + "path": "vela", + "fetchurl": "https://review.mlplatform.org/ml/ethos-u/ethos-u-vela", + "pushurl": "ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-vela", + "revision": "master" + } + ] +} diff --git a/fetch_externals.py b/fetch_externals.py index 920ea4b..3cd6620 100755 --- a/fetch_externals.py +++ b/fetch_externals.py @@ -18,8 +18,11 @@ # limitations under the License. # +import argparse +import json import os import subprocess +import sys def print_args(args, **kwargs): cwd = kwargs['cwd'] @@ -37,37 +40,43 @@ def check_output(args, **kwargs): print_args(args, **kwargs) return subprocess.check_output(args, **kwargs) +############################################################################### +# Git class +############################################################################### + class Git(object): - def __init__(self, path, fetchurl, pushurl=None, revision='origin/master'): + def __init__(self, pwd, path, fetchurl, pushurl=None, revision='origin/master'): + self.pwd = pwd self.path = path + self.absolutepath = os.path.join(pwd, path) self.fetchurl = fetchurl self.pushurl = pushurl self.revision = revision def init(self): - if not os.path.exists(self.path): - os.makedirs(self.path) + if not os.path.exists(self.absolutepath): + os.makedirs(self.absolutepath) - if not os.path.exists(os.path.join(self.path, '.git')): - check_output(['git', 'init'], cwd=self.path) + if not os.path.exists(os.path.join(self.absolutepath, '.git')): + check_output(['git', 'init'], cwd=self.absolutepath) def remote_add(self, name, fetchurl): - remotes = check_output(['git', 'remote'], cwd=self.path).decode('utf-8').split('\n') + remotes = check_output(['git', 'remote'], cwd=self.absolutepath).decode('utf-8').split('\n') if not name in remotes: - check_output(['git', 'remote', 'add', '-m', self.revision, name, self.fetchurl], cwd=self.path) + check_output(['git', 'remote', 'add', '-m', self.revision, name, self.fetchurl], cwd=self.absolutepath) if self.pushurl: - check_output(['git', 'remote', 'set-url', '--add', '--push', name, self.pushurl], cwd=self.path) + check_output(['git', 'remote', 'set-url', '--add', '--push', name, self.pushurl], cwd=self.absolutepath) def fetch(self): - check_output(['git', 'fetch'], cwd=self.path) + check_output(['git', 'fetch'], cwd=self.absolutepath) def checkout(self, revision): rev = self.__get_rev(revision) - check_output(['git', 'checkout', rev], stderr=subprocess.STDOUT, cwd=self.path) + check_output(['git', 'checkout', rev], stderr=subprocess.STDOUT, cwd=self.absolutepath) def clone(self): - if not os.path.exists(os.path.join(self.path, '.git')): + if not os.path.exists(os.path.join(self.absolutepath, '.git')): self.init() self.remote_add('origin', self.fetchurl) self.fetch() @@ -75,28 +84,103 @@ class Git(object): def rebase(self): rev = self.__get_rev(self.revision) - check_output(['git', 'rebase', rev], cwd=self.path) + check_output(['git', 'rebase', rev], cwd=self.absolutepath) + + def set_sha1(self): + self.revision = self.__get_rev(self.revision) + + def get_dict(self): + data = {} + data['path'] = self.path + data['fetchurl'] = self.fetchurl + + if self.pushurl: + data['pushurl'] = self.pushurl + + data['revision'] = self.revision + + return data def __get_rev(self, revision): try: - rev = check_output(['git', 'rev-parse', 'origin/' + self.revision], cwd=self.path) + rev = check_output(['git', 'rev-parse', 'origin/' + self.revision], stderr=subprocess.STDOUT, cwd=self.absolutepath) except: - rev = check_output(['git', 'rev-parse', self.revision], cwd=self.path) + rev = check_output(['git', 'rev-parse', self.revision], cwd=self.absolutepath) return rev.decode('utf-8').strip() -basedir = os.path.dirname(os.path.realpath(__file__)) - -externals = [ - Git(os.path.join(basedir, 'core_software'), "https://review.mlplatform.org/ml/ethos-u/ethos-u-core-software", pushurl='ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-core-software', revision='master'), - Git(os.path.join(basedir, 'core_software/core_driver'), "https://review.mlplatform.org/ml/ethos-u/ethos-u-core-driver", pushurl='ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-core-driver', revision='master'), - Git(os.path.join(basedir, 'core_software/cmsis'), 'https://github.com/ARM-software/CMSIS_5.git', revision='master'), - Git(os.path.join(basedir, 'core_software/tensorflow'), 'https://github.com/tensorflow/tensorflow', revision='master'), - Git(os.path.join(basedir, 'linux_driver_stack'), "https://review.mlplatform.org/ml/ethos-u/ethos-u-linux-driver-stack", pushurl='ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-linux-driver-stack', revision='master'), - Git(os.path.join(basedir, 'vela'), "https://review.mlplatform.org/ml/ethos-u/ethos-u-vela", pushurl='ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-vela', revision='master') -] - -for external in externals: - external.clone() - external.fetch() - external.rebase() +############################################################################### +# Externals class +############################################################################### + +class Externals: + def __init__(self, config): + self.externals = [] + self.load_config(config) + + def load_config(self, config): + self.pwd = os.path.dirname(os.path.realpath(config)) + + with open(config, 'r') as fp: + data = json.load(fp) + + for ext in data['externals']: + git = Git(self.pwd, ext['path'], ext['fetchurl'], ext.get('pushurl', None), ext['revision']) + self.externals.append(git) + + def fetch(self): + for ext in self.externals: + ext.clone() + ext.fetch() + ext.checkout(ext.revision) + + def set_sha1(self): + for ext in self.externals: + ext.set_sha1() + + def get_dict(self): + data = { 'externals': [] } + for ext in self.externals: + data['externals'].append(ext.get_dict()) + + return data + + def dump(self, fhandle): + fhandle.write(json.dumps(self.get_dict(), sort_keys=False, separators=(',', ': '), indent=4)) + +############################################################################### +# Handle functions +############################################################################### + +def handle_fetch(args): + externals = Externals(args.configuration) + externals.fetch() + +def handle_dump(args): + externals = Externals(args.configuration) + + if args.sha1: + externals.set_sha1() + + externals.dump(sys.stdout) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Fetch external repositories.') + parser.add_argument('-c', '--configuration', default='externals.json', help='Externals configuration file') + subparsers = parser.add_subparsers() + + subparser = subparsers.add_parser('fetch', description='Fetch external repositories.') + subparser.set_defaults(func=handle_fetch) + + subparser = subparsers.add_parser('dump', description='Dump configuration.') + subparser.set_defaults(func=handle_dump) + subparser.add_argument('-s', '--sha1', action='store_true', help='Replace revision with current SHA-1') + + args = parser.parse_args() + + if 'func' not in args: + parser.print_help() + sys.exit(2) + + sys.exit(args.func(args)) + -- cgit v1.2.1