Reset firmware

Reset the firmware if it becomes unresponsive. Use ping to send
keep alive requests.

Only monitor ping and inference request messages. The other messages
pass no resources to the firmware and can be cancelled without
resetting the firmware.

Change-Id: Ifbcc370f02d79a64f25598f11376a1dc84a7a066

10 files changed, 440 insertions, 166 deletions

diff --git a/kernel/ethosu_device.c b/kernel/ethosu_device.c
index f440331..316df6f 100644
--- a/kernel/ethosu_device.c
+++ b/kernel/ethosu_device.c
@@ -38,8 +38,9 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/of_reserved_mem.h>
-#include <linux/uaccess.h>
+#include <linux/reset.h>
 #include <linux/slab.h>
+#include <linux/uaccess.h>
 
 /****************************************************************************
  * Defines
@@ -58,42 +59,71 @@ static void ethosu_capabilities_destroy(struct kref *kref)
 	struct ethosu_capabilities *cap =
 		container_of(kref, struct ethosu_capabilities, refcount);
 
-	list_del(&cap->list);
+	dev_info(cap->edev->dev, "Capabilities destroy. handle=0x%pK\n", cap);
+
+	list_del(&cap->msg.list);
 
 	devm_kfree(cap->edev->dev, cap);
 }
 
-static int ethosu_capabilities_find(struct ethosu_capabilities *cap,
-				    struct list_head *capabilties_list)
+static int ethosu_capabilities_send(struct ethosu_capabilities *cap)
 {
-	struct ethosu_capabilities *cur;
+	int ret;
 
-	list_for_each_entry(cur, capabilties_list, list) {
-		if (cur == cap)
-			return 0;
-	}
+	ret = ethosu_mailbox_capabilities_request(&cap->edev->mailbox, cap);
+	if (ret)
+		return ret;
 
-	return -EINVAL;
+	return 0;
 }
 
-static int ethosu_capability_rsp(struct ethosu_device *edev,
-				 struct ethosu_core_msg_capabilities_rsp *msg)
+static void ethosu_capabilities_fail(struct ethosu_mailbox_msg *msg)
 {
-	struct ethosu_capabilities *cap;
+	struct ethosu_capabilities *cap =
+		container_of(msg, typeof(*cap), msg);
+
+	cap->errno = -EFAULT;
+	complete(&cap->done);
+}
+
+static int ethosu_capabilities_resend(struct ethosu_mailbox_msg *msg)
+{
+	struct ethosu_capabilities *cap =
+		container_of(msg, typeof(*cap), msg);
+	int ret;
+
+	/* Don't resend request if response has already been received */
+	if (completion_done(&cap->done))
+		return 0;
+
+	/* Resend request */
+	ret = ethosu_capabilities_send(cap);
+	if (ret)
+		return ret;
+
+	return 0;
+}
+
+static void ethosu_capability_rsp(struct ethosu_device *edev,
+				  struct ethosu_core_msg_capabilities_rsp *msg)
+{
+	struct ethosu_capabilities *cap =
+		(struct ethosu_capabilities *)msg->user_arg;
 	struct ethosu_uapi_device_capabilities *capabilities;
 	int ret;
 
-	cap = (struct ethosu_capabilities *)msg->user_arg;
-	ret = ethosu_capabilities_find(cap, &edev->capabilities_list);
-	if (0 != ret) {
+	ret = ethosu_mailbox_find(&edev->mailbox, &cap->msg);
+	if (ret) {
 		dev_warn(edev->dev,
-			 "Handle not found in capabilities list. handle=0x%p\n",
+			 "Capabilities not found in pending list. handle=0x%p\n",
 			 cap);
 
-		/* NOTE: do not call complete or kref_put on invalid data! */
-		return ret;
+		return;
 	}
 
+	if (completion_done(&cap->done))
+		return;
+
 	capabilities = cap->capabilities;
 
 	capabilities->hw_id.version_status = msg->version_status;
@@ -110,11 +140,61 @@ static int ethosu_capability_rsp(struct ethosu_device *edev,
 	capabilities->hw_cfg.cmd_stream_version = msg->cmd_stream_version;
 	capabilities->hw_cfg.custom_dma = msg->custom_dma;
 
+	cap->errno = 0;
 	complete(&cap->done);
+}
 
+static int ethosu_capabilities_request(struct ethosu_device *edev,
+				       void __user *udata)
+{
+	struct ethosu_uapi_device_capabilities uapi;
+	struct ethosu_capabilities *cap;
+	int ret;
+	int timeout;
+
+	cap = devm_kzalloc(edev->dev, sizeof(struct ethosu_capabilities),
+			   GFP_KERNEL);
+	if (!cap)
+		return -ENOMEM;
+
+	cap->edev = edev;
+	cap->capabilities = &uapi;
+	kref_init(&cap->refcount);
+	init_completion(&cap->done);
+	list_add(&cap->msg.list, &edev->mailbox.pending_list);
+	cap->msg.fail = ethosu_capabilities_fail;
+	cap->msg.resend = ethosu_capabilities_resend;
+
+	ret = ethosu_capabilities_send(cap);
+	if (0 != ret)
+		goto put_kref;
+
+	/* Unlock the mutex before going to block on the condition */
+	mutex_unlock(&edev->mutex);
+
+	/* wait for response to arrive back */
+	timeout = wait_for_completion_timeout(&cap->done,
+					      msecs_to_jiffies(
+						      CAPABILITIES_RESP_TIMEOUT_MS));
+
+	/* take back the mutex before resuming to do anything */
+	mutex_lock(&edev->mutex);
+
+	if (0 == timeout) {
+		dev_warn(edev->dev, "Capabilities response timeout");
+		ret = -EIO;
+		goto put_kref;
+	}
+
+	if (cap->errno)
+		goto put_kref;
+
+	ret = copy_to_user(udata, &uapi, sizeof(uapi)) ? -EFAULT : 0;
+
+put_kref:
 	kref_put(&cap->refcount, ethosu_capabilities_destroy);
 
-	return 0;
+	return ret;
 }
 
 /* Incoming messages */
@@ -171,6 +251,7 @@ static int ethosu_handle_msg(struct ethosu_device *edev)
 			 "Msg: Inference response. user_arg=0x%llx, ofm_count=%u, status=%u\n",
 			 data.inf.user_arg, data.inf.ofm_count,
 			 data.inf.status);
+
 		ethosu_inference_rsp(edev, &data.inf);
 		break;
 	case ETHOSU_CORE_MSG_VERSION_RSP:
@@ -223,7 +304,7 @@ static int ethosu_handle_msg(struct ethosu_device *edev)
 			 data.capabilities.cmd_stream_version,
 			 data.capabilities.custom_dma);
 
-		ret = ethosu_capability_rsp(edev, &data.capabilities);
+		ethosu_capability_rsp(edev, &data.capabilities);
 		break;
 	case ETHOSU_CORE_MSG_NETWORK_INFO_RSP:
 		if (header.length != sizeof(data.network_info)) {
@@ -252,6 +333,63 @@ static int ethosu_handle_msg(struct ethosu_device *edev)
 	return ret;
 }
 
+static int ethosu_firmware_reset(struct ethosu_device *edev)
+{
+	int ret;
+
+	/* No reset control for this device */
+	if (IS_ERR(edev->reset))
+		return PTR_ERR(edev->reset);
+
+	dev_info(edev->dev, "Resetting firmware.");
+
+	ret = reset_control_assert(edev->reset);
+	if (ret) {
+		dev_warn(edev->dev, "Failed to reset assert firmware. ret=%d",
+			 ret);
+
+		return ret;
+	}
+
+	/* Initialize mailbox header with illegal values */
+	ethosu_mailbox_wait_prepare(&edev->mailbox);
+
+	/* If this call fails we have a problem. We managed halt the firmware,
+	 * but not to release the reset.
+	 */
+	ret = reset_control_deassert(edev->reset);
+	if (ret) {
+		dev_warn(edev->dev, "Failed to reset deassert firmware. ret=%d",
+			 ret);
+		goto fail;
+	}
+
+	/* Wait for firmware to boot up and initialize mailbox */
+	ret = ethosu_mailbox_wait_firmware(&edev->mailbox);
+	if (ret) {
+		dev_warn(edev->dev, "Wait on firmware boot timed out. ret=%d",
+			 ret);
+		goto fail;
+	}
+
+	edev->mailbox.ping_count = 0;
+	ethosu_watchdog_reset(&edev->watchdog);
+
+	ret = ethosu_mailbox_ping(&edev->mailbox);
+	if (ret)
+		goto fail;
+
+	/* Resend messages */
+	ethosu_mailbox_resend(&edev->mailbox);
+
+	return ret;
+
+fail:
+	ethosu_mailbox_fail(&edev->mailbox);
+
+	return ret;
+}
+
 static void ethosu_watchdog_callback(struct ethosu_watchdog *wdog)
 {
 	struct ethosu_device *edev =
@@ -259,7 +397,13 @@ static void ethosu_watchdog_callback(struct ethosu_watchdog *wdog)
 
 	mutex_lock(&edev->mutex);
 
-	dev_warn(edev->dev, "Device watchdog timeout");
+	dev_warn(edev->dev, "Device watchdog timeout. ping_count=%u",
+		 edev->mailbox.ping_count);
+
+	if (edev->mailbox.ping_count < 1)
+		ethosu_mailbox_ping(&edev->mailbox);
+	else
+		ethosu_firmware_reset(edev);
 
 	mutex_unlock(&edev->mutex);
 }
@@ -277,67 +421,6 @@ static int ethosu_open(struct inode *inode,
 	return nonseekable_open(inode, file);
 }
 
-static int ethosu_send_capabilities_request(struct ethosu_device *edev,
-					    void __user *udata)
-{
-	struct ethosu_uapi_device_capabilities uapi;
-	struct ethosu_capabilities *cap;
-	int ret;
-	int timeout;
-
-	cap = devm_kzalloc(edev->dev, sizeof(struct ethosu_capabilities),
-			   GFP_KERNEL);
-	if (!cap)
-		return -ENOMEM;
-
-	cap->edev = edev;
-	cap->capabilities = &uapi;
-	kref_init(&cap->refcount);
-	init_completion(&cap->done);
-	list_add(&cap->list, &edev->capabilities_list);
-
-	ret = ethosu_mailbox_capabilities_request(&edev->mailbox, cap);
-	if (0 != ret)
-		goto put_kref;
-
-	/*
-	 * Increase ref counter since we sent the pointer out to
-	 * response handler thread. That thread is responsible to
-	 * decrease the ref counter before exiting. So the memory
-	 * can be freed.
-	 *
-	 * NOTE: if no response is received back, the memory is leaked.
-	 */
-	kref_get(&cap->refcount);
-
-	/* Unlock the mutex before going to block on the condition */
-	mutex_unlock(&edev->mutex);
-
-	/* wait for response to arrive back */
-	timeout = wait_for_completion_timeout(&cap->done,
-					      msecs_to_jiffies(
-						      CAPABILITIES_RESP_TIMEOUT_MS));
-
-	/* take back the mutex before resuming to do anything */
-	ret = mutex_lock_interruptible(&edev->mutex);
-	if (0 != ret)
-		goto put_kref;
-
-	if (0 == timeout) {
-		dev_warn(edev->dev,
-			 "Msg: Capabilities response lost - timeout\n");
-		ret = -EIO;
-		goto put_kref;
-	}
-
-	ret = copy_to_user(udata, &uapi, sizeof(uapi)) ? -EFAULT : 0;
-
-put_kref:
-	kref_put(&cap->refcount, ethosu_capabilities_destroy);
-
-	return ret;
-}
-
 static long ethosu_ioctl(struct file *file,
 			 unsigned int cmd,
 			 unsigned long arg)
@@ -359,7 +442,7 @@ static long ethosu_ioctl(struct file *file,
 		break;
 	case ETHOSU_IOCTL_CAPABILITIES_REQ:
 		dev_info(edev->dev, "Ioctl: Send capabilities request\n");
-		ret = ethosu_send_capabilities_request(edev, udata);
+		ret = ethosu_capabilities_request(edev, udata);
 		break;
 	case ETHOSU_IOCTL_PING: {
 		dev_info(edev->dev, "Ioctl: Send ping\n");
@@ -444,9 +527,10 @@ int ethosu_dev_init(struct ethosu_device *edev,
 	edev->class = class;
 	edev->devt = devt;
 	mutex_init(&edev->mutex);
-	INIT_LIST_HEAD(&edev->capabilities_list);
-	INIT_LIST_HEAD(&edev->inference_list);
-	INIT_LIST_HEAD(&edev->network_info_list);
+
+	edev->reset = devm_reset_control_get_by_index(edev->dev, 0);
+	if (IS_ERR(edev->reset))
+		dev_warn(edev->dev, "No reset control found for this device.");
 
 	ret = of_reserved_mem_device_init(edev->dev);
 	if (ret)
@@ -481,6 +565,8 @@ int ethosu_dev_init(struct ethosu_device *edev,
 		goto del_cdev;
 	}
 
+	ethosu_firmware_reset(edev);
+
 	dev_info(edev->dev,
 		 "Created Arm Ethos-U device. name=%s, major=%d, minor=%d\n",
 		 dev_name(sysdev), MAJOR(edev->devt), MINOR(edev->devt));
diff --git a/kernel/ethosu_device.h b/kernel/ethosu_device.h
index 7d8791a..7c6c99d 100644
--- a/kernel/ethosu_device.h
+++ b/kernel/ethosu_device.h
@@ -40,6 +40,8 @@
  * Types
  ****************************************************************************/
 
+struct reset_control;
+
 /**
  * struct ethosu_device - Device structure
  */
@@ -51,9 +53,7 @@ struct ethosu_device {
 	struct mutex           mutex;
 	struct ethosu_mailbox  mailbox;
 	struct ethosu_watchdog watchdog;
-	struct list_head       capabilities_list;
-	struct list_head       inference_list;
-	struct list_head       network_info_list;
+	struct reset_control   *reset;
 };
 
 /**
@@ -64,7 +64,8 @@ struct ethosu_capabilities {
 	struct completion                      done;
 	struct kref                            refcount;
 	struct ethosu_uapi_device_capabilities *capabilities;
-	struct list_head                       list;
+	struct ethosu_mailbox_msg              msg;
+	int                                    errno;
 };
 
 /****************************************************************************
diff --git a/kernel/ethosu_inference.c b/kernel/ethosu_inference.c
index ad31f06..3c18bbd 100644
--- a/kernel/ethosu_inference.c
+++ b/kernel/ethosu_inference.c
@@ -28,7 +28,6 @@
 #include "ethosu_core_interface.h"
 #include "ethosu_device.h"
 #include "ethosu_network.h"
-#include "uapi/ethosu.h"
 
 #include <linux/anon_inodes.h>
 #include <linux/file.h>
@@ -81,9 +80,6 @@ static int ethosu_inference_send(struct ethosu_inference *inf)
 {
 	int ret;
 
-	if (inf->pending)
-		return -EINVAL;
-
 	inf->status = ETHOSU_UAPI_STATUS_ERROR;
 
 	ret = ethosu_mailbox_inference(&inf->edev->mailbox, inf,
@@ -97,24 +93,51 @@ static int ethosu_inference_send(struct ethosu_inference *inf)
 	if (ret)
 		return ret;
 
-	inf->pending = true;
-
 	ethosu_inference_get(inf);
 
 	return 0;
 }
 
-static int ethosu_inference_find(struct ethosu_inference *inf,
-				 struct list_head *inference_list)
+static void ethosu_inference_fail(struct ethosu_mailbox_msg *msg)
 {
-	struct ethosu_inference *cur;
+	struct ethosu_inference *inf =
+		container_of(msg, typeof(*inf), msg);
+	int ret;
 
-	list_for_each_entry(cur, inference_list, list) {
-		if (cur == inf)
-			return 0;
+	/* Decrement reference count if inference was pending reponse */
+	if (!inf->done) {
+		ret = ethosu_inference_put(inf);
+		if (ret)
+			return;
 	}
 
-	return -EINVAL;
+	/* Fail inference and wake up any waiting process */
+	inf->status = ETHOSU_UAPI_STATUS_ERROR;
+	inf->done = true;
+	wake_up_interruptible(&inf->waitq);
+}
+
+static int ethosu_inference_resend(struct ethosu_mailbox_msg *msg)
+{
+	struct ethosu_inference *inf =
+		container_of(msg, typeof(*inf), msg);
+	int ret;
+
+	/* Don't resend request if response has already been received */
+	if (inf->done)
+		return 0;
+
+	/* Decrement reference count for pending request */
+	ret = ethosu_inference_put(inf);
+	if (ret)
+		return 0;
+
+	/* Resend request */
+	ret = ethosu_inference_send(inf);
+	if (ret)
+		return ret;
+
+	return 0;
 }
 
 static bool ethosu_inference_verify(struct file *file)
@@ -131,7 +154,7 @@ static void ethosu_inference_kref_destroy(struct kref *kref)
 		 "Inference destroy. handle=0x%pK, status=%d\n",
 		 inf, inf->status);
 
-	list_del(&inf->list);
+	list_del(&inf->msg.list);
 
 	while (inf->ifm_count-- > 0)
 		ethosu_buffer_put(inf->ifm[inf->ifm_count]);
@@ -165,7 +188,7 @@ static unsigned int ethosu_inference_poll(struct file *file,
 
 	poll_wait(file, &inf->waitq, wait);
 
-	if (!inf->pending)
+	if (inf->done)
 		ret |= POLLIN;
 
 	return ret;
@@ -237,10 +260,12 @@ int ethosu_inference_create(struct ethosu_device *edev,
 
 	inf->edev = edev;
 	inf->net = net;
-	inf->pending = false;
+	inf->done = false;
 	inf->status = ETHOSU_UAPI_STATUS_ERROR;
 	kref_init(&inf->kref);
 	init_waitqueue_head(&inf->waitq);
+	inf->msg.fail = ethosu_inference_fail;
+	inf->msg.resend = ethosu_inference_resend;
 
 	/* Get pointer to IFM buffers */
 	for (i = 0; i < uapi->ifm_count; i++) {
@@ -296,8 +321,8 @@ int ethosu_inference_create(struct ethosu_device *edev,
 	inf->file = fget(ret);
 	fput(inf->file);
 
-	/* Add inference to inference list */
-	list_add(&inf->list, &edev->inference_list);
+	/* Add inference to pending list */
+	list_add(&inf->msg.list, &edev->mailbox.pending_list);
 
 	/* Send inference request to Arm Ethos-U subsystem */
 	(void)ethosu_inference_send(inf);
@@ -350,9 +375,9 @@ void ethosu_inference_get(struct ethosu_inference *inf)
 	kref_get(&inf->kref);
 }
 
-void ethosu_inference_put(struct ethosu_inference *inf)
+int ethosu_inference_put(struct ethosu_inference *inf)
 {
-	kref_put(&inf->kref, &ethosu_inference_kref_destroy);
+	return kref_put(&inf->kref, &ethosu_inference_kref_destroy);
 }
 
 void ethosu_inference_rsp(struct ethosu_device *edev,
@@ -363,7 +388,7 @@ void ethosu_inference_rsp(struct ethosu_device *edev,
 	int ret;
 	int i;
 
-	ret = ethosu_inference_find(inf, &edev->inference_list);
+	ret = ethosu_mailbox_find(&edev->mailbox, &inf->msg);
 	if (ret) {
 		dev_warn(edev->dev,
 			 "Handle not found in inference list. handle=0x%p\n",
@@ -372,8 +397,6 @@ void ethosu_inference_rsp(struct ethosu_device *edev,
 		return;
 	}
 
-	inf->pending = false;
-
 	if (rsp->status == ETHOSU_CORE_STATUS_OK &&
 	    inf->ofm_count <= ETHOSU_CORE_BUFFER_MAX) {
 		uint32_t i;
@@ -412,6 +435,8 @@ void ethosu_inference_rsp(struct ethosu_device *edev,
 		 "PMU cycle counter. enable=%u, count=%llu\n",
 		 inf->pmu_cycle_counter_enable,
 		 inf->pmu_cycle_counter_count);
+
+	inf->done = true;
 	wake_up_interruptible(&inf->waitq);
 
 	ethosu_inference_put(inf);
diff --git a/kernel/ethosu_inference.h b/kernel/ethosu_inference.h
index a41043a..66d4ff9 100644
--- a/kernel/ethosu_inference.h
+++ b/kernel/ethosu_inference.h
@@ -25,6 +25,7 @@
  * Includes
  ****************************************************************************/
 
+#include "ethosu_mailbox.h"
 #include "uapi/ethosu.h"
 
 #include <linux/kref.h>
@@ -48,10 +49,10 @@ struct file;
  * @file:			File handle
  * @kref:			Reference counter
  * @waitq:			Wait queue
+ * @done:			Wait condition is done
  * @ifm:			Pointer to IFM buffer
  * @ofm:			Pointer to OFM buffer
  * @net:			Pointer to network
- * @pending:			Pending response from the firmware
  * @status:			Inference status
  * @pmu_event_config:		PMU event configuration
  * @pmu_event_count:		PMU event count after inference
@@ -59,22 +60,22 @@ struct file;
  * @pmu_cycle_counter_count:	PMU cycle counter count after inference
  */
 struct ethosu_inference {
-	struct ethosu_device    *edev;
-	struct file             *file;
-	struct kref             kref;
-	wait_queue_head_t       waitq;
-	uint32_t                ifm_count;
-	struct ethosu_buffer    *ifm[ETHOSU_FD_MAX];
-	uint32_t                ofm_count;
-	struct ethosu_buffer    *ofm[ETHOSU_FD_MAX];
-	struct ethosu_network   *net;
-	bool                    pending;
-	enum ethosu_uapi_status status;
-	uint8_t                 pmu_event_config[ETHOSU_PMU_EVENT_MAX];
-	uint32_t                pmu_event_count[ETHOSU_PMU_EVENT_MAX];
-	uint32_t                pmu_cycle_counter_enable;
-	uint64_t                pmu_cycle_counter_count;
-	struct list_head        list;
+	struct ethosu_device      *edev;
+	struct file               *file;
+	struct kref               kref;
+	wait_queue_head_t         waitq;
+	bool                      done;
+	uint32_t                  ifm_count;
+	struct ethosu_buffer      *ifm[ETHOSU_FD_MAX];
+	uint32_t                  ofm_count;
+	struct ethosu_buffer      *ofm[ETHOSU_FD_MAX];
+	struct ethosu_network     *net;
+	enum ethosu_uapi_status   status;
+	uint8_t                   pmu_event_config[ETHOSU_PMU_EVENT_MAX];
+	uint32_t                  pmu_event_count[ETHOSU_PMU_EVENT_MAX];
+	uint32_t                  pmu_cycle_counter_enable;
+	uint64_t                  pmu_cycle_counter_count;
+	struct ethosu_mailbox_msg msg;
 };
 
 /****************************************************************************
@@ -108,8 +109,10 @@ void ethosu_inference_get(struct ethosu_inference *inf);
 
 /**
  * ethosu_inference_put() - Put inference
+ *
+ * Return: 1 if object was removed, else 0.
  */
-void ethosu_inference_put(struct ethosu_inference *inf);
+int ethosu_inference_put(struct ethosu_inference *inf);
 
 /**
  * ethosu_inference_rsp() - Handle inference response
diff --git a/kernel/ethosu_mailbox.c b/kernel/ethosu_mailbox.c
index 7753baa..7355361 100644
--- a/kernel/ethosu_mailbox.c
+++ b/kernel/ethosu_mailbox.c
@@ -23,16 +23,29 @@
  ****************************************************************************/
 
 #include "ethosu_mailbox.h"
-#include "ethosu_watchdog.h"
 
 #include "ethosu_buffer.h"
 #include "ethosu_core_interface.h"
 #include "ethosu_device.h"
+#include "ethosu_watchdog.h"
 
+#include <linux/jiffies.h>
 #include <linux/resource.h>
 #include <linux/uio.h>
 
 /****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#ifndef fallthrough
+#if __has_attribute(__fallthrough__)
+#define fallthrough __attribute__((__fallthrough__))
+#else
+#define fallthrough do {} while (0)  /* fallthrough */
+#endif
+#endif
+
+/****************************************************************************
  * Functions
  ****************************************************************************/
 
@@ -41,10 +54,9 @@ static void ethosu_wd_inc(struct ethosu_mailbox *mbox,
 {
 	switch (type) {
 	case ETHOSU_CORE_MSG_PING:
+		mbox->ping_count++;
+		fallthrough;
 	case ETHOSU_CORE_MSG_INFERENCE_REQ:
-	case ETHOSU_CORE_MSG_VERSION_REQ:
-	case ETHOSU_CORE_MSG_CAPABILITIES_REQ:
-	case ETHOSU_CORE_MSG_NETWORK_INFO_REQ:
 		ethosu_watchdog_inc(mbox->wdog);
 		break;
 	default:
@@ -57,10 +69,9 @@ static void ethosu_wd_dec(struct ethosu_mailbox *mbox,
 {
 	switch (type) {
 	case ETHOSU_CORE_MSG_PONG:
+		mbox->ping_count--;
+		fallthrough;
 	case ETHOSU_CORE_MSG_INFERENCE_RSP:
-	case ETHOSU_CORE_MSG_VERSION_RSP:
-	case ETHOSU_CORE_MSG_CAPABILITIES_RSP:
-	case ETHOSU_CORE_MSG_NETWORK_INFO_RSP:
 		ethosu_watchdog_dec(mbox->wdog);
 		break;
 	default:
@@ -190,6 +201,36 @@ void ethosu_mailbox_reset(struct ethosu_mailbox *mbox)
 	mbox->out_queue->header.read = mbox->out_queue->header.write;
 }
 
+void ethosu_mailbox_wait_prepare(struct ethosu_mailbox *mbox)
+{
+	mbox->in_queue->header.size = 0;
+	mbox->in_queue->header.read = 0xffffff;
+	mbox->in_queue->header.write = 0xffffff;
+}
+
+int ethosu_mailbox_wait_firmware(struct ethosu_mailbox *mbox)
+{
+	const unsigned long timeout = 1000;
+	const unsigned long end = jiffies + msecs_to_jiffies(timeout);
+	volatile struct ethosu_core_queue_header *hdr =
+		&mbox->in_queue->header;
+	int ret = -ETIMEDOUT;
+
+	/* Spin wait on mailbox initialization */
+	while ((end - jiffies) < timeout)
+		if (hdr->size != 0 &&
+		    hdr->read != 0xffffff &&
+		    hdr->write != 0xffffff) {
+			ret = 0;
+			break;
+		}
+
+	dev_info(mbox->dev, "mbox: Wait. ret=%d, size=%u, read=%u, write=%u",
+		 ret, hdr->size, hdr->read, hdr->write);
+
+	return ret;
+}
+
 int ethosu_mailbox_read(struct ethosu_mailbox *mbox,
 			struct ethosu_core_msg *header,
 			void *data,
@@ -241,6 +282,45 @@ int ethosu_mailbox_read(struct ethosu_mailbox *mbox,
 	return 0;
 }
 
+int ethosu_mailbox_find(struct ethosu_mailbox *mbox,
+			struct ethosu_mailbox_msg *msg)
+{
+	struct ethosu_mailbox_msg *cur;
+
+	list_for_each_entry(cur, &mbox->pending_list, list) {
+		if (cur == msg)
+			return 0;
+	}
+
+	return -EINVAL;
+}
+
+void ethosu_mailbox_fail(struct ethosu_mailbox *mbox)
+{
+	struct ethosu_mailbox_msg *cur, *cur_tmp;
+
+	list_for_each_entry_safe(cur, cur_tmp, &mbox->pending_list, list) {
+		cur->fail(cur);
+	}
+}
+
+int ethosu_mailbox_resend(struct ethosu_mailbox *mbox)
+{
+	struct ethosu_mailbox_msg *cur, *cur_tmp;
+	int ret;
+
+	list_for_each_entry_safe(cur, cur_tmp, &mbox->pending_list, list) {
+		ret = cur->resend(cur);
+		if (ret) {
+			cur->fail(cur);
+
+			return ret;
+		}
+	}
+
+	return 0;
+}
+
 int ethosu_mailbox_ping(struct ethosu_mailbox *mbox)
 {
 	return ethosu_queue_write_msg(mbox, ETHOSU_CORE_MSG_PING, NULL, 0);
@@ -380,6 +460,8 @@ int ethosu_mailbox_init(struct ethosu_mailbox *mbox,
 	mbox->callback = callback;
 	mbox->user_arg = user_arg;
 	mbox->wdog = wdog;
+	mbox->ping_count = 0;
+	INIT_LIST_HEAD(&mbox->pending_list);
 
 	mbox->client.dev = dev;
 	mbox->client.rx_callback = ethosu_mailbox_rx_callback;
diff --git a/kernel/ethosu_mailbox.h b/kernel/ethosu_mailbox.h
index 956685b..55d4436 100644
--- a/kernel/ethosu_mailbox.h
+++ b/kernel/ethosu_mailbox.h
@@ -55,7 +55,15 @@ struct ethosu_mailbox {
 	struct mbox_chan         *tx;
 	ethosu_mailbox_cb        callback;
 	void                     *user_arg;
+	struct list_head         pending_list;
 	struct ethosu_watchdog   *wdog;
+	unsigned                 ping_count;
+};
+
+struct ethosu_mailbox_msg {
+	struct list_head list;
+	void             (*fail)(struct ethosu_mailbox_msg *msg);
+	int              (*resend)(struct ethosu_mailbox_msg *msg);
 };
 
 /****************************************************************************
@@ -81,6 +89,26 @@ int ethosu_mailbox_init(struct ethosu_mailbox *mbox,
 void ethosu_mailbox_deinit(struct ethosu_mailbox *mbox);
 
 /**
+ * ethosu_mailbox_wait_prepare() - Prepare to wait on firmware
+ *
+ * This function must only be called when the firmware is in a
+ * stopped state. It invalidates the firmware queues setting
+ * size, read and write positions to illegal values.
+ */
+void ethosu_mailbox_wait_prepare(struct ethosu_mailbox *mbox);
+
+/**
+ * ethosu_mailbox_wait_firmware() - Waiting for firmware to initialize
+ *                                  message queues
+ *
+ * Following a call to ethosu_mailbox_wait_prepare() this function waits for
+ * the firmware to boot up and initialize the firmware queues.
+ *
+ * Return: 0 on success, else error code.
+ */
+int ethosu_mailbox_wait_firmware(struct ethosu_mailbox *mbox);
+
+/**
  * ethosu_mailbox_read() - Read message from mailbox
  *
  * Return: 0 message read, else error code.
@@ -91,6 +119,30 @@ int ethosu_mailbox_read(struct ethosu_mailbox *mbox,
 			size_t length);
 
 /**
+ * ethosu_mailbox_find() - Find mailbox message
+ *
+ * Return: 0 on success, else error code.
+ */
+int ethosu_mailbox_find(struct ethosu_mailbox *mbox,
+			struct ethosu_mailbox_msg *msg);
+
+/**
+ * ethosu_mailbox_fail() - Fail mailbox messages
+ *
+ * Call fail() callback on all messages in pending list.
+ */
+void ethosu_mailbox_fail(struct ethosu_mailbox *mbox);
+
+/**
+ * ethosu_mailbox_resend() - Resend mailbox messages
+ *
+ * Call resend() callback on all messages in pending list.
+ *
+ * Return: 0 on success, else error code.
+ */
+int ethosu_mailbox_resend(struct ethosu_mailbox *mbox);
+
+/**
  * ethosu_mailbox_reset() - Reset to end of queue
  */
 void ethosu_mailbox_reset(struct ethosu_mailbox *mbox);
diff --git a/kernel/ethosu_network.c b/kernel/ethosu_network.c
index 0654a79..86ae410 100644
--- a/kernel/ethosu_network.c
+++ b/kernel/ethosu_network.c
@@ -104,6 +104,9 @@ static int ethosu_network_info_request(struct ethosu_network *net,
 	ret = ethosu_network_info_wait(info, 3000);
 	mutex_lock(&net->edev->mutex);
 
+	if (ret)
+		info->msg.fail(&info->msg);
+
 	ethosu_network_info_put(info);
 
 	return ret;
@@ -240,7 +243,7 @@ void ethosu_network_get(struct ethosu_network *net)
 	kref_get(&net->kref);
 }
 
-void ethosu_network_put(struct ethosu_network *net)
+int ethosu_network_put(struct ethosu_network *net)
 {
-	kref_put(&net->kref, ethosu_network_destroy);
+	return kref_put(&net->kref, ethosu_network_destroy);
 }
diff --git a/kernel/ethosu_network.h b/kernel/ethosu_network.h
index 7ddffb5..8ee6818 100644
--- a/kernel/ethosu_network.h
+++ b/kernel/ethosu_network.h
@@ -76,7 +76,9 @@ void ethosu_network_get(struct ethosu_network *net);
 
 /**
  * ethosu_network_put() - Put network
+ *
+ * Return: 1 if object was removed, else 0.
  */
-void ethosu_network_put(struct ethosu_network *net);
+int ethosu_network_put(struct ethosu_network *net);
 
 #endif /* ETHOSU_NETWORK_H */
diff --git a/kernel/ethosu_network_info.c b/kernel/ethosu_network_info.c
index 24be677..c2b6caa 100644
--- a/kernel/ethosu_network_info.c
+++ b/kernel/ethosu_network_info.c
@@ -40,7 +40,7 @@ static void ethosu_network_info_destroy(struct kref *kref)
 
 	dev_info(info->edev->dev, "Network info destroy. handle=0x%pK\n", info);
 
-	list_del(&info->list);
+	list_del(&info->msg.list);
 
 	ethosu_network_put(info->net);
 
@@ -59,8 +59,35 @@ static int ethosu_network_info_send(struct ethosu_network_info *info)
 	if (ret)
 		return ret;
 
-	/* Increase reference count for the pending network info response */
-	ethosu_network_info_get(info);
+	return 0;
+}
+
+static void ethosu_network_info_fail(struct ethosu_mailbox_msg *msg)
+{
+	struct ethosu_network_info *info =
+		container_of(msg, typeof(*info), msg);
+
+	if (completion_done(&info->done))
+		return;
+
+	info->errno = -EFAULT;
+	complete(&info->done);
+}
+
+static int ethosu_network_info_resend(struct ethosu_mailbox_msg *msg)
+{
+	struct ethosu_network_info *info =
+		container_of(msg, typeof(*info), msg);
+	int ret;
+
+	/* Don't resend request if response has already been received */
+	if (completion_done(&info->done))
+		return 0;
+
+	/* Resend request */
+	ret = ethosu_network_info_send(info);
+	if (ret)
+		return ret;
 
 	return 0;
 }
@@ -82,9 +109,11 @@ struct ethosu_network_info *ethosu_network_info_create(
 	info->uapi = uapi;
 	kref_init(&info->kref);
 	init_completion(&info->done);
+	info->msg.fail = ethosu_network_info_fail;
+	info->msg.resend = ethosu_network_info_resend;
 
 	/* Insert network info to network info list */
-	list_add(&info->list, &edev->network_info_list);
+	list_add(&info->msg.list, &edev->mailbox.pending_list);
 
 	/* Get reference to network */
 	ethosu_network_get(net);
@@ -103,27 +132,14 @@ put_info:
 	return ERR_PTR(ret);
 }
 
-static int ethosu_network_info_find(struct ethosu_network_info *info,
-				    struct list_head *network_info_list)
-{
-	struct ethosu_network_info *cur;
-
-	list_for_each_entry(cur, network_info_list, list) {
-		if (cur == info)
-			return 0;
-	}
-
-	return -EINVAL;
-}
-
 void ethosu_network_info_get(struct ethosu_network_info *info)
 {
 	kref_get(&info->kref);
 }
 
-void ethosu_network_info_put(struct ethosu_network_info *info)
+int ethosu_network_info_put(struct ethosu_network_info *info)
 {
-	kref_put(&info->kref, ethosu_network_info_destroy);
+	return kref_put(&info->kref, ethosu_network_info_destroy);
 }
 
 int ethosu_network_info_wait(struct ethosu_network_info *info,
@@ -152,7 +168,7 @@ void ethosu_network_info_rsp(struct ethosu_device *edev,
 	uint32_t i;
 	int ret;
 
-	ret = ethosu_network_info_find(info, &edev->network_info_list);
+	ret = ethosu_mailbox_find(&edev->mailbox, &info->msg);
 	if (0 != ret) {
 		dev_warn(edev->dev,
 			 "Handle not found in network info list. handle=0x%p\n",
@@ -161,6 +177,9 @@ void ethosu_network_info_rsp(struct ethosu_device *edev,
 		return;
 	}
 
+	if (completion_done(&info->done))
+		return;
+
 	info->errno = 0;
 
 	if (rsp->status != ETHOSU_CORE_STATUS_OK) {
@@ -185,6 +204,4 @@ void ethosu_network_info_rsp(struct ethosu_device *edev,
 
 signal_complete:
 	complete(&info->done);
-
-	ethosu_network_info_put(info);
 }
diff --git a/kernel/ethosu_network_info.h b/kernel/ethosu_network_info.h
index 2a333a6..680be80 100644
--- a/kernel/ethosu_network_info.h
+++ b/kernel/ethosu_network_info.h
@@ -26,6 +26,7 @@
  ****************************************************************************/
 
 #include "ethosu_core_interface.h"
+#include "ethosu_mailbox.h"
 
 #include <linux/kref.h>
 #include <linux/types.h>
@@ -44,9 +45,9 @@ struct ethosu_network_info {
 	struct ethosu_network           *net;
 	struct ethosu_uapi_network_info *uapi;
 	struct kref                     kref;
-	struct list_head                list;
 	struct completion               done;
 	int                             errno;
+	struct ethosu_mailbox_msg       msg;
 };
 
 /****************************************************************************
@@ -72,8 +73,10 @@ void ethosu_network_info_get(struct ethosu_network_info *info);
 
 /**
  * ethosu_network_info_put() - Put network info
+ *
+ * Return: 1 if object was removed, else 0.
  */
-void ethosu_network_info_put(struct ethosu_network_info *info);
+int ethosu_network_info_put(struct ethosu_network_info *info);
 
 /**
  * ethosu_network_info_wait() - Get network info