Kernel watchdog timeout

Implement kernel watchdog that detects when firmware becomes
unresponsive.

Change-Id: I5c5b58a56a2ce629e1fd7cabae83b61823239ea6

8 files changed, 317 insertions, 23 deletions

diff --git a/kernel/Kbuild b/kernel/Kbuild
index a6664ac..0b92c12 100644
--- a/kernel/Kbuild
+++ b/kernel/Kbuild
@@ -26,4 +26,5 @@ ethosu-objs := ethosu_driver.o \
                ethosu_inference.o \
                ethosu_mailbox.o \
                ethosu_network.o \
-               ethosu_network_info.o
+               ethosu_network_info.o \
+               ethosu_watchdog.o
diff --git a/kernel/ethosu_device.c b/kernel/ethosu_device.c
index dfdaa58..f440331 100644
--- a/kernel/ethosu_device.c
+++ b/kernel/ethosu_device.c
@@ -252,6 +252,18 @@ static int ethosu_handle_msg(struct ethosu_device *edev)
 	return ret;
 }
 
+static void ethosu_watchdog_callback(struct ethosu_watchdog *wdog)
+{
+	struct ethosu_device *edev =
+		container_of(wdog, struct ethosu_device, watchdog);
+
+	mutex_lock(&edev->mutex);
+
+	dev_warn(edev->dev, "Device watchdog timeout");
+
+	mutex_unlock(&edev->mutex);
+}
+
 static int ethosu_open(struct inode *inode,
 		       struct file *file)
 {
@@ -442,11 +454,16 @@ int ethosu_dev_init(struct ethosu_device *edev,
 
 	dma_set_mask_and_coherent(edev->dev, DMA_BIT_MASK(DMA_ADDR_BITS));
 
-	ret = ethosu_mailbox_init(&edev->mailbox, dev, in_queue, out_queue,
-				  ethosu_mbox_rx, edev);
+	ret = ethosu_watchdog_init(&edev->watchdog, dev,
+				   ethosu_watchdog_callback);
 	if (ret)
 		goto release_reserved_mem;
 
+	ret = ethosu_mailbox_init(&edev->mailbox, dev, in_queue, out_queue,
+				  ethosu_mbox_rx, edev, &edev->watchdog);
+	if (ret)
+		goto deinit_watchdog;
+
 	cdev_init(&edev->cdev, &fops);
 	edev->cdev.owner = THIS_MODULE;
 
@@ -476,6 +493,9 @@ del_cdev:
 deinit_mailbox:
 	ethosu_mailbox_deinit(&edev->mailbox);
 
+deinit_watchdog:
+	ethosu_watchdog_deinit(&edev->watchdog);
+
 release_reserved_mem:
 	of_reserved_mem_device_release(edev->dev);
 
@@ -485,6 +505,7 @@ release_reserved_mem:
 void ethosu_dev_deinit(struct ethosu_device *edev)
 {
 	ethosu_mailbox_deinit(&edev->mailbox);
+	ethosu_watchdog_deinit(&edev->watchdog);
 	device_destroy(edev->class, edev->cdev.dev);
 	cdev_del(&edev->cdev);
 	of_reserved_mem_device_release(edev->dev);
diff --git a/kernel/ethosu_device.h b/kernel/ethosu_device.h
index d096e23..7d8791a 100644
--- a/kernel/ethosu_device.h
+++ b/kernel/ethosu_device.h
@@ -27,6 +27,7 @@
 
 #include "uapi/ethosu.h"
 #include "ethosu_mailbox.h"
+#include "ethosu_watchdog.h"
 
 #include <linux/device.h>
 #include <linux/cdev.h>
@@ -43,15 +44,16 @@
  * struct ethosu_device - Device structure
  */
 struct ethosu_device {
-	struct device         *dev;
-	struct cdev           cdev;
-	struct                class *class;
-	dev_t                 devt;
-	struct mutex          mutex;
-	struct ethosu_mailbox mailbox;
-	struct list_head      capabilities_list;
-	struct list_head      inference_list;
-	struct list_head      network_info_list;
+	struct device          *dev;
+	struct cdev            cdev;
+	struct                 class *class;
+	dev_t                  devt;
+	struct mutex           mutex;
+	struct ethosu_mailbox  mailbox;
+	struct ethosu_watchdog watchdog;
+	struct list_head       capabilities_list;
+	struct list_head       inference_list;
+	struct list_head       network_info_list;
 };
 
 /**
diff --git a/kernel/ethosu_mailbox.c b/kernel/ethosu_mailbox.c
index e3d31fe..7753baa 100644
--- a/kernel/ethosu_mailbox.c
+++ b/kernel/ethosu_mailbox.c
@@ -23,6 +23,7 @@
  ****************************************************************************/
 
 #include "ethosu_mailbox.h"
+#include "ethosu_watchdog.h"
 
 #include "ethosu_buffer.h"
 #include "ethosu_core_interface.h"
@@ -35,6 +36,38 @@
  * Functions
  ****************************************************************************/
 
+static void ethosu_wd_inc(struct ethosu_mailbox *mbox,
+			  enum ethosu_core_msg_type type)
+{
+	switch (type) {
+	case ETHOSU_CORE_MSG_PING:
+	case ETHOSU_CORE_MSG_INFERENCE_REQ:
+	case ETHOSU_CORE_MSG_VERSION_REQ:
+	case ETHOSU_CORE_MSG_CAPABILITIES_REQ:
+	case ETHOSU_CORE_MSG_NETWORK_INFO_REQ:
+		ethosu_watchdog_inc(mbox->wdog);
+		break;
+	default:
+		break;
+	}
+}
+
+static void ethosu_wd_dec(struct ethosu_mailbox *mbox,
+			  enum ethosu_core_msg_type type)
+{
+	switch (type) {
+	case ETHOSU_CORE_MSG_PONG:
+	case ETHOSU_CORE_MSG_INFERENCE_RSP:
+	case ETHOSU_CORE_MSG_VERSION_RSP:
+	case ETHOSU_CORE_MSG_CAPABILITIES_RSP:
+	case ETHOSU_CORE_MSG_NETWORK_INFO_RSP:
+		ethosu_watchdog_dec(mbox->wdog);
+		break;
+	default:
+		break;
+	}
+}
+
 static void ethosu_core_set_size(struct ethosu_buffer *buf,
 				 struct ethosu_core_buffer *cbuf)
 {
@@ -113,8 +146,15 @@ static int ethosu_queue_write_msg(struct ethosu_mailbox *mbox,
 		{ &msg, sizeof(msg) },
 		{ data, length      }
 	};
+	int ret;
+
+	ret = ethosu_queue_write(mbox, vec, 2);
+	if (ret)
+		return ret;
 
-	return ethosu_queue_write(mbox, vec, 2);
+	ethosu_wd_inc(mbox, type);
+
+	return 0;
 }
 
 static int ethosu_queue_read(struct ethosu_mailbox *mbox,
@@ -196,6 +236,8 @@ int ethosu_mailbox_read(struct ethosu_mailbox *mbox,
 		return -EBADMSG;
 	}
 
+	ethosu_wd_dec(mbox, header->type);
+
 	return 0;
 }
 
@@ -329,13 +371,15 @@ int ethosu_mailbox_init(struct ethosu_mailbox *mbox,
 			struct resource *in_queue,
 			struct resource *out_queue,
 			ethosu_mailbox_cb callback,
-			void *user_arg)
+			void *user_arg,
+			struct ethosu_watchdog *wdog)
 {
 	int ret;
 
 	mbox->dev = dev;
 	mbox->callback = callback;
 	mbox->user_arg = user_arg;
+	mbox->wdog = wdog;
 
 	mbox->client.dev = dev;
 	mbox->client.rx_callback = ethosu_mailbox_rx_callback;
diff --git a/kernel/ethosu_mailbox.h b/kernel/ethosu_mailbox.h
index f004f92..956685b 100644
--- a/kernel/ethosu_mailbox.h
+++ b/kernel/ethosu_mailbox.h
@@ -39,6 +39,7 @@ struct ethosu_buffer;
 struct ethosu_device;
 struct ethosu_core_msg;
 struct ethosu_core_queue;
+struct ethosu_watchdog;
 struct resource;
 
 typedef void (*ethosu_mailbox_cb)(void *user_arg);
@@ -54,6 +55,7 @@ struct ethosu_mailbox {
 	struct mbox_chan         *tx;
 	ethosu_mailbox_cb        callback;
 	void                     *user_arg;
+	struct ethosu_watchdog   *wdog;
 };
 
 /****************************************************************************
@@ -70,7 +72,8 @@ int ethosu_mailbox_init(struct ethosu_mailbox *mbox,
 			struct resource *in_queue,
 			struct resource *out_queue,
 			ethosu_mailbox_cb callback,
-			void *user_arg);
+			void *user_arg,
+			struct ethosu_watchdog *wdog);
 
 /**
  * ethosu_mailbox_deinit() - Deinitialize mailbox
diff --git a/kernel/ethosu_network_info.c b/kernel/ethosu_network_info.c
index 2b1f841..24be677 100644
--- a/kernel/ethosu_network_info.c
+++ b/kernel/ethosu_network_info.c
@@ -47,6 +47,24 @@ static void ethosu_network_info_destroy(struct kref *kref)
 	devm_kfree(info->edev->dev, info);
 }
 
+static int ethosu_network_info_send(struct ethosu_network_info *info)
+{
+	int ret;
+
+	/* Send network info request to firmware */
+	ret = ethosu_mailbox_network_info_request(&info->edev->mailbox,
+						  info,
+						  info->net->buf,
+						  info->net->index);
+	if (ret)
+		return ret;
+
+	/* Increase reference count for the pending network info response */
+	ethosu_network_info_get(info);
+
+	return 0;
+}
+
 struct ethosu_network_info *ethosu_network_info_create(
 	struct ethosu_device *edev,
 	struct ethosu_network *net,
@@ -71,17 +89,10 @@ struct ethosu_network_info *ethosu_network_info_create(
 	/* Get reference to network */
 	ethosu_network_get(net);
 
-	/* Send network info request to firmware */
-	ret = ethosu_mailbox_network_info_request(&info->edev->mailbox,
-						  info,
-						  info->net->buf,
-						  info->net->index);
+	ret = ethosu_network_info_send(info);
 	if (ret)
 		goto put_info;
 
-	/* Increase reference count for the pending network info response */
-	ethosu_network_info_get(info);
-
 	dev_info(edev->dev, "Network info create. handle=%p\n", info);
 
 	return info;
diff --git a/kernel/ethosu_watchdog.c b/kernel/ethosu_watchdog.c
new file mode 100644
index 0000000..bde0803
--- /dev/null
+++ b/kernel/ethosu_watchdog.c
@@ -0,0 +1,130 @@
+/*
+ * Copyright (c) 2022 Arm Limited.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include "ethosu_watchdog.h"
+
+#include <linux/device.h>
+#include <linux/jiffies.h>
+#include <linux/moduleparam.h>
+#include <linux/version.h>
+
+/****************************************************************************
+ * Variables
+ ****************************************************************************/
+
+static unsigned long watchdog_timeout_ms = 3000;
+module_param(watchdog_timeout_ms, ulong, 0664);
+MODULE_PARM_DESC(watchdog_timeout_ms,
+		 "Watchdog timeout in milliseconds for unresponsive firmware.");
+
+/****************************************************************************
+ * Functions
+ ****************************************************************************/
+
+static void ethosu_watchdog_update(struct ethosu_watchdog *wdog)
+{
+	int ret;
+
+	ret = mod_timer(&wdog->timer,
+			jiffies + msecs_to_jiffies(watchdog_timeout_ms));
+
+	dev_info(wdog->dev,
+		 "Wdog: Update watchdog timeout. ret=%d, timeout_ms=%lu, refcount=%u", ret,
+		 watchdog_timeout_ms, atomic_read(&wdog->refcount));
+}
+
+static void ethosu_watchdog_work(struct work_struct *work)
+{
+	struct ethosu_watchdog *wdog =
+		container_of(work, struct ethosu_watchdog, work);
+
+	dev_info(wdog->dev, "Wdog: Watchdog timeout. refcount=%u",
+		 atomic_read(&wdog->refcount));
+
+	wdog->callback(wdog);
+}
+
+static void ethosu_watchdog_timeout(struct timer_list *timer)
+{
+	struct ethosu_watchdog *wdog =
+		container_of(timer, struct ethosu_watchdog, timer);
+
+	queue_work(system_unbound_wq, &wdog->work);
+}
+
+#if KERNEL_VERSION(4, 14, 0) > LINUX_VERSION_CODE
+static void ethosu_watchdog_timeout_legacy(unsigned long data)
+{
+	ethosu_watchdog_timeout((struct timer_list *)data);
+}
+
+#endif
+
+int ethosu_watchdog_init(struct ethosu_watchdog *wdog,
+			 struct device *dev,
+			 ethosu_watchdog_cb callback)
+{
+	wdog->dev = dev;
+	wdog->callback = callback;
+	atomic_set(&wdog->refcount, 0);
+	INIT_WORK(&wdog->work, ethosu_watchdog_work);
+
+#if KERNEL_VERSION(4, 14, 0) <= LINUX_VERSION_CODE
+	timer_setup(&wdog->timer, ethosu_watchdog_timeout, 0);
+#else
+	setup_timer(&wdog->timer, ethosu_watchdog_timeout_legacy,
+		    (unsigned long)&wdog->timer);
+#endif
+
+	return 0;
+}
+
+void ethosu_watchdog_deinit(struct ethosu_watchdog *wdog)
+{
+	del_timer(&wdog->timer);
+}
+
+int ethosu_watchdog_reset(struct ethosu_watchdog *wdog)
+{
+	del_timer(&wdog->timer);
+	atomic_set(&wdog->refcount, 0);
+
+	return 0;
+}
+
+void ethosu_watchdog_inc(struct ethosu_watchdog *wdog)
+{
+	atomic_inc(&wdog->refcount);
+	ethosu_watchdog_update(wdog);
+}
+
+void ethosu_watchdog_dec(struct ethosu_watchdog *wdog)
+{
+	if (atomic_dec_and_test(&wdog->refcount)) {
+		dev_info(wdog->dev, "Wdog: Cancel watchdog timeout");
+		del_timer(&wdog->timer);
+	} else {
+		ethosu_watchdog_update(wdog);
+	}
+}
diff --git a/kernel/ethosu_watchdog.h b/kernel/ethosu_watchdog.h
new file mode 100644
index 0000000..d288af4
--- /dev/null
+++ b/kernel/ethosu_watchdog.h
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2022 Arm Limited.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ *
+ * SPDX-License-Identifier: GPL-2.0-only
+ */
+
+#ifndef ETHOSU_WATCHDOG_H
+#define ETHOSU_WATCHDOG_H
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/types.h>
+#include <linux/timer.h>
+#include <linux/workqueue.h>
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+struct ethosu_watchdog;
+
+typedef void (*ethosu_watchdog_cb)(struct ethosu_watchdog *wdog);
+
+struct ethosu_watchdog {
+	struct device      *dev;
+	ethosu_watchdog_cb callback;
+	struct timer_list  timer;
+	struct work_struct work;
+	atomic_t           refcount;
+};
+
+/****************************************************************************
+ * Functions
+ ****************************************************************************/
+
+/**
+ * ethosu_watchdog_init() - Initialize watchdog
+ *
+ * Return: 0 on success, else error code.
+ */
+int ethosu_watchdog_init(struct ethosu_watchdog *wdog,
+			 struct device *dev,
+			 ethosu_watchdog_cb callback);
+
+/**
+ * ethosu_watchdog_deinit() - Deinitialize watchdog
+ */
+void ethosu_watchdog_deinit(struct ethosu_watchdog *wdog);
+
+/**
+ * ethosu_watchdog_reset() - Reset watchdog
+ */
+int ethosu_watchdog_reset(struct ethosu_watchdog *wdog);
+
+/**
+ * ethosu_watchdog_inc() - Increment reference count
+ */
+void ethosu_watchdog_inc(struct ethosu_watchdog *wdog);
+
+/**
+ * ethosu_watchdog_dec() - Decrement reference count
+ */
+void ethosu_watchdog_dec(struct ethosu_watchdog *wdog);
+
+#endif /* ETHOSU_WATCHDOG_H */