Externals configuration file

Store configuration of external repositories in a JSON file.

Add command to dump configuration to stdout.

Change-Id: I84d5388d9bd31e4d1c6626260c9418e07446634a

4 files changed, 161 insertions, 37 deletions

diff --git a/.gitignore b/.gitignore
index a2a4412..5febf4e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 /core_software
+/linux_driver_stack
 /vela
diff --git a/README.md b/README.md
index 269c7ec..1a90c1e 100644
--- a/README.md
+++ b/README.md
@@ -1,10 +1,10 @@
 # Ethos-u
 
-This is the root repository for all Ethos-u software. It is provided to help
+This is the root repository for all Arm Ethos-U software. It is provided to help
 users download required repositories and place them in a tree structure.
 
 ```
-$ ./fetch_externals.py
+$ ./fetch_externals.py fetch
 ```
 
 The script will build following tree structure.
@@ -22,10 +22,10 @@ Directory
 
 | Directory | Description |
 --- | ---
-| [.](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u) | This is the root directory for all Ethos-u software. |
-| [core_software](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-core-software) | The software executing on Arm Cortex-M is referred to as _Core Software_. This folder provides a small build system that illustrates how to build the key components for the Ethos-u core software. |
-| [core_driver](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-core-driver) | The Ethos-u NPU driver. |
+| [.](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u) | This is the root directory for all Arm Ethos-U software. |
+| [core_software](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-core-software) | The software executing on Arm Cortex-M is referred to as _Core Software_. This folder provides a small build system that illustrates how to build the key components for the Arm Ethos-U core software. |
+| [core_driver](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-core-driver) | The Arm Ethos-U NPU driver. |
 | [cmsis](https://github.com/ARM-software/CMSIS_5) | CMSIS provides optimized kernels and generic interfaces to the Arm Cortex-M CPUs. |
-| [tensorflow](https://github.com/tensorflow/tensorflow) | The TensorFlowLite micro framework is used to run inferences. |
-| [linux_driver_stack](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-linux-driver-stack) | Example driver stack showing how Linux can dispatch inferences to an Ethos-U subsystem. |
-| [vela](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-vela) | The Vela optimizer takes a TFLu file as input and replaces operators that are supported by the Ethos-u NPU with custom operators designed to run on the NPU. Operators not supported by the NPU are executed in software. |
+| [tensorflow](https://github.com/tensorflow/tensorflow) | The TensorFlow Lite microcontroller framework is used to run inferences. |
+| [linux_driver_stack](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-linux-driver-stack) | Example driver stack showing how Linux can dispatch inferences to an Arm Ethos-U subsystem. |
+| [vela](https://review.mlplatform.org/plugins/gitiles/ml/ethos-u/ethos-u-vela) | The Vela optimizer takes a TFLu file as input and replaces operators that are supported by the Arm Ethos-U NPU with custom operators designed to run on the NPU. Operators not supported by the NPU are executed in software. |
diff --git a/externals.json b/externals.json
new file mode 100644
index 0000000..6ee5456
--- /dev/null
+++ b/externals.json
@@ -0,0 +1,39 @@
+{
+    "externals":
+    [
+        {
+            "path": "core_software",
+            "fetchurl": "https://review.mlplatform.org/ml/ethos-u/ethos-u-core-software",
+            "pushurl": "ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-core-software",
+            "revision": "master"
+        },
+        {
+            "path": "core_software/core_driver",
+            "fetchurl": "https://review.mlplatform.org/ml/ethos-u/ethos-u-core-driver",
+            "pushurl": "ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-core-driver",
+            "revision": "master"
+        },
+        {
+            "path": "core_software/cmsis",
+            "fetchurl": "https://github.com/ARM-software/CMSIS_5.git",
+            "revision": "master"
+        },
+        {
+            "path": "core_software/tensorflow",
+            "fetchurl": "https://github.com/tensorflow/tensorflow",
+            "revision": "master"
+        },
+        {
+            "path": "linux_driver_stack",
+            "fetchurl": "https://review.mlplatform.org/ml/ethos-u/ethos-u-linux-driver-stack",
+            "pushurl": "ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-linux-driver-stack",
+            "revision": "master"
+        },
+        {
+            "path": "vela",
+            "fetchurl": "https://review.mlplatform.org/ml/ethos-u/ethos-u-vela",
+            "pushurl": "ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-vela",
+            "revision": "master"
+        }
+    ]
+}
diff --git a/fetch_externals.py b/fetch_externals.py
index 920ea4b..3cd6620 100755
--- a/fetch_externals.py
+++ b/fetch_externals.py
@@ -18,8 +18,11 @@
 # limitations under the License.
 #
 
+import argparse
+import json
 import os
 import subprocess
+import sys
 
 def print_args(args, **kwargs):
     cwd = kwargs['cwd']
@@ -37,37 +40,43 @@ def check_output(args, **kwargs):
     print_args(args, **kwargs)
     return subprocess.check_output(args, **kwargs)
 
+###############################################################################
+# Git class
+###############################################################################
+
 class Git(object):
-    def __init__(self, path, fetchurl, pushurl=None, revision='origin/master'):
+    def __init__(self, pwd, path, fetchurl, pushurl=None, revision='origin/master'):
+        self.pwd = pwd
         self.path = path
+        self.absolutepath = os.path.join(pwd, path)
         self.fetchurl = fetchurl
         self.pushurl = pushurl
         self.revision = revision
 
     def init(self):
-        if not os.path.exists(self.path):
-            os.makedirs(self.path)
+        if not os.path.exists(self.absolutepath):
+            os.makedirs(self.absolutepath)
 
-        if not os.path.exists(os.path.join(self.path, '.git')):
-            check_output(['git', 'init'], cwd=self.path)
+        if not os.path.exists(os.path.join(self.absolutepath, '.git')):
+            check_output(['git', 'init'], cwd=self.absolutepath)
 
     def remote_add(self, name, fetchurl):
-        remotes = check_output(['git', 'remote'], cwd=self.path).decode('utf-8').split('\n')
+        remotes = check_output(['git', 'remote'], cwd=self.absolutepath).decode('utf-8').split('\n')
         if not name in remotes:
-            check_output(['git', 'remote', 'add', '-m', self.revision, name, self.fetchurl], cwd=self.path)
+            check_output(['git', 'remote', 'add', '-m', self.revision, name, self.fetchurl], cwd=self.absolutepath)
 
             if self.pushurl:
-                check_output(['git', 'remote', 'set-url', '--add', '--push', name, self.pushurl], cwd=self.path)
+                check_output(['git', 'remote', 'set-url', '--add', '--push', name, self.pushurl], cwd=self.absolutepath)
 
     def fetch(self):
-        check_output(['git', 'fetch'], cwd=self.path)
+        check_output(['git', 'fetch'], cwd=self.absolutepath)
 
     def checkout(self, revision):
         rev = self.__get_rev(revision)
-        check_output(['git', 'checkout', rev], stderr=subprocess.STDOUT, cwd=self.path)
+        check_output(['git', 'checkout', rev], stderr=subprocess.STDOUT, cwd=self.absolutepath)
 
     def clone(self):
-        if not os.path.exists(os.path.join(self.path, '.git')):
+        if not os.path.exists(os.path.join(self.absolutepath, '.git')):
             self.init()
             self.remote_add('origin', self.fetchurl)
             self.fetch()
@@ -75,28 +84,103 @@ class Git(object):
 
     def rebase(self):
         rev = self.__get_rev(self.revision)
-        check_output(['git', 'rebase', rev], cwd=self.path)
+        check_output(['git', 'rebase', rev], cwd=self.absolutepath)
+
+    def set_sha1(self):
+        self.revision = self.__get_rev(self.revision)
+
+    def get_dict(self):
+        data = {}
+        data['path'] = self.path
+        data['fetchurl'] = self.fetchurl
+
+        if self.pushurl:
+            data['pushurl'] = self.pushurl
+
+        data['revision'] = self.revision
+
+        return data
 
     def __get_rev(self, revision):
         try:
-            rev = check_output(['git', 'rev-parse', 'origin/' + self.revision], cwd=self.path)
+            rev = check_output(['git', 'rev-parse', 'origin/' + self.revision], stderr=subprocess.STDOUT, cwd=self.absolutepath)
         except:
-            rev = check_output(['git', 'rev-parse', self.revision], cwd=self.path)
+            rev = check_output(['git', 'rev-parse', self.revision], cwd=self.absolutepath)
 
         return rev.decode('utf-8').strip()
 
-basedir = os.path.dirname(os.path.realpath(__file__))
-
-externals = [
-    Git(os.path.join(basedir, 'core_software'), "https://review.mlplatform.org/ml/ethos-u/ethos-u-core-software", pushurl='ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-core-software', revision='master'),
-    Git(os.path.join(basedir, 'core_software/core_driver'), "https://review.mlplatform.org/ml/ethos-u/ethos-u-core-driver", pushurl='ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-core-driver', revision='master'),
-    Git(os.path.join(basedir, 'core_software/cmsis'), 'https://github.com/ARM-software/CMSIS_5.git', revision='master'),
-    Git(os.path.join(basedir, 'core_software/tensorflow'), 'https://github.com/tensorflow/tensorflow', revision='master'),
-    Git(os.path.join(basedir, 'linux_driver_stack'), "https://review.mlplatform.org/ml/ethos-u/ethos-u-linux-driver-stack", pushurl='ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-linux-driver-stack', revision='master'),
-    Git(os.path.join(basedir, 'vela'), "https://review.mlplatform.org/ml/ethos-u/ethos-u-vela", pushurl='ssh://review.mlplatform.org:29418/ml/ethos-u/ethos-u-vela', revision='master')
-]
-
-for external in externals:
-    external.clone()
-    external.fetch()
-    external.rebase()
+###############################################################################
+# Externals class
+###############################################################################
+
+class Externals:
+    def __init__(self, config):
+        self.externals = []
+        self.load_config(config)
+
+    def load_config(self, config):
+        self.pwd = os.path.dirname(os.path.realpath(config))
+
+        with open(config, 'r') as fp:
+            data = json.load(fp)
+
+        for ext in data['externals']:
+            git = Git(self.pwd, ext['path'], ext['fetchurl'], ext.get('pushurl', None), ext['revision'])
+            self.externals.append(git)
+
+    def fetch(self):
+        for ext in self.externals:
+            ext.clone()
+            ext.fetch()
+            ext.checkout(ext.revision)
+
+    def set_sha1(self):
+        for ext in self.externals:
+            ext.set_sha1()
+
+    def get_dict(self):
+        data = { 'externals': [] }
+        for ext in self.externals:
+            data['externals'].append(ext.get_dict())
+
+        return data
+
+    def dump(self, fhandle):
+        fhandle.write(json.dumps(self.get_dict(), sort_keys=False, separators=(',', ': '), indent=4))
+
+###############################################################################
+# Handle functions
+###############################################################################
+
+def handle_fetch(args):
+    externals = Externals(args.configuration)
+    externals.fetch()
+
+def handle_dump(args):
+    externals = Externals(args.configuration)
+
+    if args.sha1:
+        externals.set_sha1()
+
+    externals.dump(sys.stdout)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Fetch external repositories.')
+    parser.add_argument('-c', '--configuration', default='externals.json', help='Externals configuration file')
+    subparsers = parser.add_subparsers()
+
+    subparser = subparsers.add_parser('fetch', description='Fetch external repositories.')
+    subparser.set_defaults(func=handle_fetch)
+
+    subparser = subparsers.add_parser('dump', description='Dump configuration.')
+    subparser.set_defaults(func=handle_dump)
+    subparser.add_argument('-s', '--sha1', action='store_true', help='Replace revision with current SHA-1')
+
+    args = parser.parse_args()
+
+    if 'func' not in args:
+        parser.print_help()
+        sys.exit(2)
+
+    sys.exit(args.func(args))
+