Firmware resident model

Support referencing a network model by index that has been built into
the firmware binary.

Change-Id: Idd5294376ea82503dfeafe1203dcc0694d296dfe

15 files changed, 179 insertions, 58 deletions

diff --git a/driver_library/include/ethosu.hpp b/driver_library/include/ethosu.hpp
index 98e6969..0738aa2 100644
--- a/driver_library/include/ethosu.hpp
+++ b/driver_library/include/ethosu.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020-2021 Arm Limited. All rights reserved.
+ * Copyright (c) 2020-2022 Arm Limited. All rights reserved.
  *
  * SPDX-License-Identifier: Apache-2.0
  *
@@ -161,6 +161,7 @@ private:
 class Network {
 public:
     Network(const Device &device, std::shared_ptr<Buffer> &buffer);
+    Network(const Device &device, const std::string &model, const unsigned index);
     virtual ~Network();
 
     int ioctl(unsigned long cmd, void *data = nullptr);
@@ -171,6 +172,8 @@ public:
     size_t getOfmSize() const;
 
 private:
+    void parseModel(const char *data);
+
     int fd;
     std::shared_ptr<Buffer> buffer;
     std::vector<size_t> ifmDims;
diff --git a/driver_library/src/ethosu.cpp b/driver_library/src/ethosu.cpp
index 32d179a..16d2db0 100644
--- a/driver_library/src/ethosu.cpp
+++ b/driver_library/src/ethosu.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020-2021 Arm Limited. All rights reserved.
+ * Copyright (c) 2020-2022 Arm Limited. All rights reserved.
  *
  * SPDX-License-Identifier: Apache-2.0
  *
@@ -23,6 +23,7 @@
 
 #include <algorithm>
 #include <exception>
+#include <fstream>
 #include <iostream>
 
 #include <fcntl.h>
@@ -292,25 +293,46 @@ int Buffer::getFd() const {
 Network::Network(const Device &device, shared_ptr<Buffer> &buffer) : fd(-1), buffer(buffer) {
     // Create buffer handle
     ethosu_uapi_network_create uapi;
-    uapi.fd = buffer->getFd();
-    fd      = device.ioctl(ETHOSU_IOCTL_NETWORK_CREATE, static_cast<void *>(&uapi));
+    uapi.type = ETHOSU_UAPI_NETWORK_BUFFER;
+    uapi.fd   = buffer->getFd();
+    fd        = device.ioctl(ETHOSU_IOCTL_NETWORK_CREATE, static_cast<void *>(&uapi));
 
-    // Create model handle
-    const tflite::Model *model = tflite::GetModel(reinterpret_cast<void *>(buffer->data()));
-
-    if (model->subgraphs() == nullptr) {
-        try {
-            eclose(fd);
-        } catch (...) { std::throw_with_nested(EthosU::Exception("Failed to get subgraphs: nullptr")); }
+    try {
+        parseModel(buffer->data());
+    } catch (...) {
+        eclose(fd);
+        throw;
     }
+}
 
-    // Get input dimensions for first subgraph
-    auto *subgraph = *model->subgraphs()->begin();
-    ifmDims        = getSubGraphDims(subgraph, subgraph->inputs());
+Network::Network(const Device &device, const string &model, const unsigned index) : fd(-1) {
+    // Create buffer handle
+    ethosu_uapi_network_create uapi;
+    uapi.type  = ETHOSU_UAPI_NETWORK_INDEX;
+    uapi.index = index;
+    fd         = device.ioctl(ETHOSU_IOCTL_NETWORK_CREATE, static_cast<void *>(&uapi));
 
-    // Get output dimensions for last subgraph
-    subgraph = *model->subgraphs()->rbegin();
-    ofmDims  = getSubGraphDims(subgraph, subgraph->outputs());
+    try {
+        // Open file
+        ifstream ifs(model, std::ios::binary);
+        if (!ifs.is_open()) {
+            throw Exception("Failed to open model file.");
+        }
+
+        // Get file size
+        ifs.seekg(0, ios_base::end);
+        size_t size = ifs.tellg();
+        ifs.seekg(0, ios_base::beg);
+
+        // Read data into buffer
+        vector<char> buffer(size);
+        ifs.read(buffer.data(), size);
+
+        parseModel(buffer.data());
+    } catch (...) {
+        eclose(fd);
+        throw;
+    }
 }
 
 Network::~Network() {
@@ -353,6 +375,23 @@ size_t Network::getOfmSize() const {
     return size;
 }
 
+void Network::parseModel(const char *data) {
+    // Create model handle
+    const tflite::Model *model = tflite::GetModel(reinterpret_cast<const void *>(data));
+
+    if (model->subgraphs() == nullptr) {
+        EthosU::Exception("Failed to get subgraphs: nullptr");
+    }
+
+    // Get input dimensions for first subgraph
+    auto *subgraph = *model->subgraphs()->begin();
+    ifmDims        = getSubGraphDims(subgraph, subgraph->inputs());
+
+    // Get output dimensions for last subgraph
+    subgraph = *model->subgraphs()->rbegin();
+    ofmDims  = getSubGraphDims(subgraph, subgraph->outputs());
+}
+
 /****************************************************************************
  * Inference
  ****************************************************************************/
diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt
index 5aa47bb..6d2beb5 100644
--- a/kernel/CMakeLists.txt
+++ b/kernel/CMakeLists.txt
@@ -1,5 +1,5 @@
 #
-# (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+# Copyright (c) 2020-2022 Arm Limited.
 #
 # This program is free software and is provided to you under the terms of the
 # GNU General Public License version 2 as published by the Free Software
diff --git a/kernel/ethosu_core_interface.h b/kernel/ethosu_core_interface.h
index ef63c3b..cc4fca4 100644
--- a/kernel/ethosu_core_interface.h
+++ b/kernel/ethosu_core_interface.h
@@ -1,5 +1,5 @@
 /*
- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ * Copyright (c) 2020-2022 Arm Limited.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -88,27 +88,61 @@ struct ethosu_core_queue {
 	uint8_t                         data[];
 };
 
+/**
+ * enum ethosu_core_status - Status
+ */
 enum ethosu_core_status {
 	ETHOSU_CORE_STATUS_OK,
 	ETHOSU_CORE_STATUS_ERROR
 };
 
+/**
+ * struct ethosu_core_buffer - Buffer descriptor
+ *
+ * Pointer and size to a buffer withing the Ethos-U
+ * address space.
+ */
 struct ethosu_core_buffer {
 	uint32_t ptr;
 	uint32_t size;
 };
 
+/**
+ * enum ethosu_core_network_type - Network buffer type
+ */
+enum ethosu_core_network_type {
+	ETHOSU_CORE_NETWORK_BUFFER = 1,
+	ETHOSU_CORE_NETWORK_INDEX
+};
+
+/**
+ * struct ethosu_core_network_buffer - Network buffer
+ */
+struct ethosu_core_network_buffer {
+	uint32_t type;
+	union {
+		struct ethosu_core_buffer buffer;
+		uint32_t                  index;
+	};
+};
+
+/**
+ * struct ethosu_core_inference_req - Inference request
+ */
 struct ethosu_core_inference_req {
-	uint64_t                  user_arg;
-	uint32_t                  ifm_count;
-	struct ethosu_core_buffer ifm[ETHOSU_CORE_BUFFER_MAX];
-	uint32_t                  ofm_count;
-	struct ethosu_core_buffer ofm[ETHOSU_CORE_BUFFER_MAX];
-	struct ethosu_core_buffer network;
-	uint8_t                   pmu_event_config[ETHOSU_CORE_PMU_MAX];
-	uint32_t                  pmu_cycle_counter_enable;
+	uint64_t                          user_arg;
+	uint32_t                          ifm_count;
+	struct ethosu_core_buffer         ifm[ETHOSU_CORE_BUFFER_MAX];
+	uint32_t                          ofm_count;
+	struct ethosu_core_buffer         ofm[ETHOSU_CORE_BUFFER_MAX];
+	struct ethosu_core_network_buffer network;
+	uint8_t                           pmu_event_config[ETHOSU_CORE_PMU_MAX];
+	uint32_t                          pmu_cycle_counter_enable;
 };
 
+/**
+ * struct ethosu_core_inference_rsp - Inference response
+ */
 struct ethosu_core_inference_rsp {
 	uint64_t user_arg;
 	uint32_t ofm_count;
@@ -121,7 +155,7 @@ struct ethosu_core_inference_rsp {
 };
 
 /**
- * struct ethosu_core_msg_verson - Message protocol version
+ * struct ethosu_core_msg_version - Message protocol version
  */
 struct ethosu_core_msg_version {
 	uint8_t major;
diff --git a/kernel/ethosu_device.c b/kernel/ethosu_device.c
index e6f1e80..54d70f8 100644
--- a/kernel/ethosu_device.c
+++ b/kernel/ethosu_device.c
@@ -1,5 +1,5 @@
 /*
- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ * Copyright (c) 2020-2022 Arm Limited.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
diff --git a/kernel/ethosu_device.h b/kernel/ethosu_device.h
index 3afdda8..e1d7034 100644
--- a/kernel/ethosu_device.h
+++ b/kernel/ethosu_device.h
@@ -1,5 +1,5 @@
 /*
- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ * Copyright (c) 2020-2022 Arm Limited.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
diff --git a/kernel/ethosu_driver.c b/kernel/ethosu_driver.c
index 9d02431..a530f95 100644
--- a/kernel/ethosu_driver.c
+++ b/kernel/ethosu_driver.c
@@ -1,5 +1,5 @@
 /*
- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ * Copyright (c) 2020-2022 Arm Limited.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
diff --git a/kernel/ethosu_inference.c b/kernel/ethosu_inference.c
index 6fde92c..17beef4 100644
--- a/kernel/ethosu_inference.c
+++ b/kernel/ethosu_inference.c
@@ -1,5 +1,5 @@
 /*
- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ * Copyright (c) 2020, 2022 Arm Limited.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -90,6 +90,7 @@ static int ethosu_inference_send(struct ethosu_inference *inf)
 				       inf->ifm_count, inf->ifm,
 				       inf->ofm_count, inf->ofm,
 				       inf->net->buf,
+				       inf->net->index,
 				       inf->pmu_event_config,
 				       ETHOSU_PMU_EVENT_MAX,
 				       inf->pmu_cycle_counter_enable);
diff --git a/kernel/ethosu_mailbox.c b/kernel/ethosu_mailbox.c
index 7f159f3..ef9a07d 100644
--- a/kernel/ethosu_mailbox.c
+++ b/kernel/ethosu_mailbox.c
@@ -1,5 +1,5 @@
 /*
- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ * Copyright (c) 2020-2022 Arm Limited.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -234,6 +234,7 @@ int ethosu_mailbox_inference(struct ethosu_mailbox *mbox,
 			     uint32_t ofm_count,
 			     struct ethosu_buffer **ofm,
 			     struct ethosu_buffer *network,
+			     uint32_t network_index,
 			     uint8_t *pmu_event_config,
 			     uint8_t pmu_event_config_count,
 			     uint8_t pmu_cycle_counter_enable)
@@ -262,7 +263,13 @@ int ethosu_mailbox_inference(struct ethosu_mailbox *mbox,
 	for (i = 0; i < ETHOSU_CORE_PMU_MAX; i++)
 		inf.pmu_event_config[i] = pmu_event_config[i];
 
-	ethosu_core_set_size(network, &inf.network);
+	if (network != NULL) {
+		inf.network.type = ETHOSU_CORE_NETWORK_BUFFER;
+		ethosu_core_set_size(network, &inf.network.buffer);
+	} else {
+		inf.network.type = ETHOSU_CORE_NETWORK_INDEX;
+		inf.network.index = network_index;
+	}
 
 	return ethosu_queue_write_msg(mbox, ETHOSU_CORE_MSG_INFERENCE_REQ,
 				      &inf, sizeof(inf));
diff --git a/kernel/ethosu_mailbox.h b/kernel/ethosu_mailbox.h
index 5cd5e62..af3d986 100644
--- a/kernel/ethosu_mailbox.h
+++ b/kernel/ethosu_mailbox.h
@@ -1,5 +1,5 @@
 /*
- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ * Copyright (c) 2020-2022 Arm Limited.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -133,6 +133,7 @@ int ethosu_mailbox_inference(struct ethosu_mailbox *mbox,
 			     uint32_t ofm_count,
 			     struct ethosu_buffer **ofm,
 			     struct ethosu_buffer *network,
+			     uint32_t network_index,
 			     uint8_t *pmu_event_config,
 			     uint8_t pmu_event_config_count,
 			     uint8_t pmu_cycle_counter_enable);
diff --git a/kernel/ethosu_network.c b/kernel/ethosu_network.c
index 4d68f05..57ccb62 100644
--- a/kernel/ethosu_network.c
+++ b/kernel/ethosu_network.c
@@ -1,5 +1,5 @@
 /*
- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ * Copyright (c) 2020, 2022 Arm Limited.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -69,7 +69,9 @@ static void ethosu_network_destroy(struct kref *kref)
 
 	dev_info(net->edev->dev, "Network destroy. handle=0x%pK\n", net);
 
-	ethosu_buffer_put(net->buf);
+	if (net->buf != NULL)
+		ethosu_buffer_put(net->buf);
+
 	devm_kfree(net->edev->dev, net);
 }
 
@@ -128,28 +130,31 @@ static long ethosu_network_ioctl(struct file *file,
 int ethosu_network_create(struct ethosu_device *edev,
 			  struct ethosu_uapi_network_create *uapi)
 {
-	struct ethosu_buffer *buf;
 	struct ethosu_network *net;
 	int ret = -ENOMEM;
 
-	buf = ethosu_buffer_get_from_fd(uapi->fd);
-	if (IS_ERR(buf))
-		return PTR_ERR(buf);
-
 	net = devm_kzalloc(edev->dev, sizeof(*net), GFP_KERNEL);
-	if (!net) {
-		ret = -ENOMEM;
-		goto put_buf;
-	}
+	if (!net)
+		return -ENOMEM;
 
 	net->edev = edev;
-	net->buf = buf;
+	net->buf = NULL;
 	kref_init(&net->kref);
 
+	if (uapi->type == ETHOSU_UAPI_NETWORK_BUFFER) {
+		net->buf = ethosu_buffer_get_from_fd(uapi->fd);
+		if (IS_ERR(net->buf)) {
+			ret = PTR_ERR(net->buf);
+			goto free_net;
+		}
+	} else {
+		net->index = uapi->index;
+	}
+
 	ret = anon_inode_getfd("ethosu-network", &ethosu_network_fops, net,
 			       O_RDWR | O_CLOEXEC);
 	if (ret < 0)
-		goto free_net;
+		goto put_buf;
 
 	net->file = fget(ret);
 	fput(net->file);
@@ -159,12 +164,13 @@ int ethosu_network_create(struct ethosu_device *edev,
 
 	return ret;
 
+put_buf:
+	if (net->buf != NULL)
+		ethosu_buffer_put(net->buf);
+
 free_net:
 	devm_kfree(edev->dev, net);
 
-put_buf:
-	ethosu_buffer_put(buf);
-
 	return ret;
 }
 
diff --git a/kernel/ethosu_network.h b/kernel/ethosu_network.h
index bb70afc..e70b46f 100644
--- a/kernel/ethosu_network.h
+++ b/kernel/ethosu_network.h
@@ -1,5 +1,5 @@
 /*
- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ * Copyright (c) 2020, 2022 Arm Limited.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -43,6 +43,7 @@ struct ethosu_network {
 	struct file          *file;
 	struct kref          kref;
 	struct ethosu_buffer *buf;
+	uint32_t             index;
 };
 
 /****************************************************************************
diff --git a/kernel/uapi/ethosu.h b/kernel/uapi/ethosu.h
index 903316d..335c769 100644
--- a/kernel/uapi/ethosu.h
+++ b/kernel/uapi/ethosu.h
@@ -1,5 +1,5 @@
 /*
- * (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+ * Copyright (c) 2020-2022 Arm Limited.
  *
  * This program is free software and is provided to you under the terms of the
  * GNU General Public License version 2 as published by the Free Software
@@ -97,11 +97,28 @@ struct ethosu_uapi_buffer {
 };
 
 /**
+ * enum ethosu_uapi_network_create - Network buffer type.
+ * @ETHOSU_UAPI_NETWORK_BUFFER:	Network is stored in a buffer handle.
+ * @ETHOSU_UAPI_NETWORK_INDEX:	Network is built into firmware and referenced by
+ *                              index.
+ */
+enum ethosu_uapi_network_type {
+	ETHOSU_UAPI_NETWORK_BUFFER = 1,
+	ETHOSU_UAPI_NETWORK_INDEX
+};
+
+/**
  * struct ethosu_uapi_network_create - Create network request
+ * @type:	Buffer type. See @ethosu_uapi_network_type.
  * @fd:		Buffer file descriptor
+ * @index:	Buffer index compiled into firmware binary.
  */
 struct ethosu_uapi_network_create {
-	__u32 fd;
+	uint32_t type;
+	union {
+		__u32 fd;
+		__u32 index;
+	};
 };
 
 /**
diff --git a/mailbox/CMakeLists.txt b/mailbox/CMakeLists.txt
index 1728b4e..44d8c82 100644
--- a/mailbox/CMakeLists.txt
+++ b/mailbox/CMakeLists.txt
@@ -1,5 +1,5 @@
 #
-# (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
+# Copyright (c) 2020-2022 Arm Limited.
 #
 # This program is free software and is provided to you under the terms of the
 # GNU General Public License version 2 as published by the Free Software
diff --git a/utils/inference_runner/inference_runner.cpp b/utils/inference_runner/inference_runner.cpp
index a72a954..08a47b7 100644
--- a/utils/inference_runner/inference_runner.cpp
+++ b/utils/inference_runner/inference_runner.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020-2021 Arm Limited. All rights reserved.
+ * Copyright (c) 2020-2022 Arm Limited. All rights reserved.
  *
  * SPDX-License-Identifier: Apache-2.0
  *
@@ -39,6 +39,7 @@ void help(const string exe) {
     cerr << "Arguments:\n";
     cerr << "    -h --help       Print this help message.\n";
     cerr << "    -n --network    File to read network from.\n";
+    cerr << "       --index      Network model index, stored in firmware binary.\n";
     cerr << "    -i --ifm        File to read IFM from.\n";
     cerr << "    -o --ofm        File to write IFM to.\n";
     cerr << "    -P --pmu [0.." << Inference::getMaxPmuEventCounters() << "] eventid.\n";
@@ -138,6 +139,7 @@ ostream &operator<<(ostream &os, Buffer &buf) {
 int main(int argc, char *argv[]) {
     const string exe = argv[0];
     string networkArg;
+    int networkIndex = -1;
     list<string> ifmArg;
     vector<uint8_t> enabledCounters(Inference::getMaxPmuEventCounters());
     string ofmArg;
@@ -154,6 +156,9 @@ int main(int argc, char *argv[]) {
         } else if (arg == "--network" || arg == "-n") {
             rangeCheck(++i, argc, arg);
             networkArg = argv[i];
+        } else if (arg == "--index") {
+            rangeCheck(++i, argc, arg);
+            networkIndex = stoi(argv[i]);
         } else if (arg == "--ifm" || arg == "-i") {
             rangeCheck(++i, argc, arg);
             ifmArg.push_back(argv[i]);
@@ -228,8 +233,15 @@ int main(int argc, char *argv[]) {
 
         /* Create network */
         cout << "Create network" << endl;
-        shared_ptr<Buffer> networkBuffer = allocAndFill(device, networkArg);
-        shared_ptr<Network> network      = make_shared<Network>(device, networkBuffer);
+
+        shared_ptr<Network> network;
+
+        if (networkIndex < 0) {
+            shared_ptr<Buffer> networkBuffer = allocAndFill(device, networkArg);
+            network                          = make_shared<Network>(device, networkBuffer);
+        } else {
+            network = make_shared<Network>(device, networkArg, networkIndex);
+        }
 
         /* Create one inference per IFM */
         list<shared_ptr<Inference>> inferences;