From f5b98c965c51def4f63d7fb198f70180e195b2e8 Mon Sep 17 00:00:00 2001 From: Kristofer Jonsson Date: Mon, 14 Mar 2022 16:09:12 +0100 Subject: Kernel watchdog timeout Implement kernel watchdog that detects when firmware becomes unresponsive. Change-Id: I5c5b58a56a2ce629e1fd7cabae83b61823239ea6 --- kernel/Kbuild | 3 +- kernel/ethosu_device.c | 25 ++++++++- kernel/ethosu_device.h | 20 ++++--- kernel/ethosu_mailbox.c | 48 +++++++++++++++- kernel/ethosu_mailbox.h | 5 +- kernel/ethosu_network_info.c | 27 ++++++--- kernel/ethosu_watchdog.c | 130 +++++++++++++++++++++++++++++++++++++++++++ kernel/ethosu_watchdog.h | 82 +++++++++++++++++++++++++++ 8 files changed, 317 insertions(+), 23 deletions(-) create mode 100644 kernel/ethosu_watchdog.c create mode 100644 kernel/ethosu_watchdog.h diff --git a/kernel/Kbuild b/kernel/Kbuild index a6664ac..0b92c12 100644 --- a/kernel/Kbuild +++ b/kernel/Kbuild @@ -26,4 +26,5 @@ ethosu-objs := ethosu_driver.o \ ethosu_inference.o \ ethosu_mailbox.o \ ethosu_network.o \ - ethosu_network_info.o + ethosu_network_info.o \ + ethosu_watchdog.o diff --git a/kernel/ethosu_device.c b/kernel/ethosu_device.c index dfdaa58..f440331 100644 --- a/kernel/ethosu_device.c +++ b/kernel/ethosu_device.c @@ -252,6 +252,18 @@ static int ethosu_handle_msg(struct ethosu_device *edev) return ret; } +static void ethosu_watchdog_callback(struct ethosu_watchdog *wdog) +{ + struct ethosu_device *edev = + container_of(wdog, struct ethosu_device, watchdog); + + mutex_lock(&edev->mutex); + + dev_warn(edev->dev, "Device watchdog timeout"); + + mutex_unlock(&edev->mutex); +} + static int ethosu_open(struct inode *inode, struct file *file) { @@ -442,11 +454,16 @@ int ethosu_dev_init(struct ethosu_device *edev, dma_set_mask_and_coherent(edev->dev, DMA_BIT_MASK(DMA_ADDR_BITS)); - ret = ethosu_mailbox_init(&edev->mailbox, dev, in_queue, out_queue, - ethosu_mbox_rx, edev); + ret = ethosu_watchdog_init(&edev->watchdog, dev, + ethosu_watchdog_callback); if (ret) goto release_reserved_mem; + ret = ethosu_mailbox_init(&edev->mailbox, dev, in_queue, out_queue, + ethosu_mbox_rx, edev, &edev->watchdog); + if (ret) + goto deinit_watchdog; + cdev_init(&edev->cdev, &fops); edev->cdev.owner = THIS_MODULE; @@ -476,6 +493,9 @@ del_cdev: deinit_mailbox: ethosu_mailbox_deinit(&edev->mailbox); +deinit_watchdog: + ethosu_watchdog_deinit(&edev->watchdog); + release_reserved_mem: of_reserved_mem_device_release(edev->dev); @@ -485,6 +505,7 @@ release_reserved_mem: void ethosu_dev_deinit(struct ethosu_device *edev) { ethosu_mailbox_deinit(&edev->mailbox); + ethosu_watchdog_deinit(&edev->watchdog); device_destroy(edev->class, edev->cdev.dev); cdev_del(&edev->cdev); of_reserved_mem_device_release(edev->dev); diff --git a/kernel/ethosu_device.h b/kernel/ethosu_device.h index d096e23..7d8791a 100644 --- a/kernel/ethosu_device.h +++ b/kernel/ethosu_device.h @@ -27,6 +27,7 @@ #include "uapi/ethosu.h" #include "ethosu_mailbox.h" +#include "ethosu_watchdog.h" #include #include @@ -43,15 +44,16 @@ * struct ethosu_device - Device structure */ struct ethosu_device { - struct device *dev; - struct cdev cdev; - struct class *class; - dev_t devt; - struct mutex mutex; - struct ethosu_mailbox mailbox; - struct list_head capabilities_list; - struct list_head inference_list; - struct list_head network_info_list; + struct device *dev; + struct cdev cdev; + struct class *class; + dev_t devt; + struct mutex mutex; + struct ethosu_mailbox mailbox; + struct ethosu_watchdog watchdog; + struct list_head capabilities_list; + struct list_head inference_list; + struct list_head network_info_list; }; /** diff --git a/kernel/ethosu_mailbox.c b/kernel/ethosu_mailbox.c index e3d31fe..7753baa 100644 --- a/kernel/ethosu_mailbox.c +++ b/kernel/ethosu_mailbox.c @@ -23,6 +23,7 @@ ****************************************************************************/ #include "ethosu_mailbox.h" +#include "ethosu_watchdog.h" #include "ethosu_buffer.h" #include "ethosu_core_interface.h" @@ -35,6 +36,38 @@ * Functions ****************************************************************************/ +static void ethosu_wd_inc(struct ethosu_mailbox *mbox, + enum ethosu_core_msg_type type) +{ + switch (type) { + case ETHOSU_CORE_MSG_PING: + case ETHOSU_CORE_MSG_INFERENCE_REQ: + case ETHOSU_CORE_MSG_VERSION_REQ: + case ETHOSU_CORE_MSG_CAPABILITIES_REQ: + case ETHOSU_CORE_MSG_NETWORK_INFO_REQ: + ethosu_watchdog_inc(mbox->wdog); + break; + default: + break; + } +} + +static void ethosu_wd_dec(struct ethosu_mailbox *mbox, + enum ethosu_core_msg_type type) +{ + switch (type) { + case ETHOSU_CORE_MSG_PONG: + case ETHOSU_CORE_MSG_INFERENCE_RSP: + case ETHOSU_CORE_MSG_VERSION_RSP: + case ETHOSU_CORE_MSG_CAPABILITIES_RSP: + case ETHOSU_CORE_MSG_NETWORK_INFO_RSP: + ethosu_watchdog_dec(mbox->wdog); + break; + default: + break; + } +} + static void ethosu_core_set_size(struct ethosu_buffer *buf, struct ethosu_core_buffer *cbuf) { @@ -113,8 +146,15 @@ static int ethosu_queue_write_msg(struct ethosu_mailbox *mbox, { &msg, sizeof(msg) }, { data, length } }; + int ret; + + ret = ethosu_queue_write(mbox, vec, 2); + if (ret) + return ret; - return ethosu_queue_write(mbox, vec, 2); + ethosu_wd_inc(mbox, type); + + return 0; } static int ethosu_queue_read(struct ethosu_mailbox *mbox, @@ -196,6 +236,8 @@ int ethosu_mailbox_read(struct ethosu_mailbox *mbox, return -EBADMSG; } + ethosu_wd_dec(mbox, header->type); + return 0; } @@ -329,13 +371,15 @@ int ethosu_mailbox_init(struct ethosu_mailbox *mbox, struct resource *in_queue, struct resource *out_queue, ethosu_mailbox_cb callback, - void *user_arg) + void *user_arg, + struct ethosu_watchdog *wdog) { int ret; mbox->dev = dev; mbox->callback = callback; mbox->user_arg = user_arg; + mbox->wdog = wdog; mbox->client.dev = dev; mbox->client.rx_callback = ethosu_mailbox_rx_callback; diff --git a/kernel/ethosu_mailbox.h b/kernel/ethosu_mailbox.h index f004f92..956685b 100644 --- a/kernel/ethosu_mailbox.h +++ b/kernel/ethosu_mailbox.h @@ -39,6 +39,7 @@ struct ethosu_buffer; struct ethosu_device; struct ethosu_core_msg; struct ethosu_core_queue; +struct ethosu_watchdog; struct resource; typedef void (*ethosu_mailbox_cb)(void *user_arg); @@ -54,6 +55,7 @@ struct ethosu_mailbox { struct mbox_chan *tx; ethosu_mailbox_cb callback; void *user_arg; + struct ethosu_watchdog *wdog; }; /**************************************************************************** @@ -70,7 +72,8 @@ int ethosu_mailbox_init(struct ethosu_mailbox *mbox, struct resource *in_queue, struct resource *out_queue, ethosu_mailbox_cb callback, - void *user_arg); + void *user_arg, + struct ethosu_watchdog *wdog); /** * ethosu_mailbox_deinit() - Deinitialize mailbox diff --git a/kernel/ethosu_network_info.c b/kernel/ethosu_network_info.c index 2b1f841..24be677 100644 --- a/kernel/ethosu_network_info.c +++ b/kernel/ethosu_network_info.c @@ -47,6 +47,24 @@ static void ethosu_network_info_destroy(struct kref *kref) devm_kfree(info->edev->dev, info); } +static int ethosu_network_info_send(struct ethosu_network_info *info) +{ + int ret; + + /* Send network info request to firmware */ + ret = ethosu_mailbox_network_info_request(&info->edev->mailbox, + info, + info->net->buf, + info->net->index); + if (ret) + return ret; + + /* Increase reference count for the pending network info response */ + ethosu_network_info_get(info); + + return 0; +} + struct ethosu_network_info *ethosu_network_info_create( struct ethosu_device *edev, struct ethosu_network *net, @@ -71,17 +89,10 @@ struct ethosu_network_info *ethosu_network_info_create( /* Get reference to network */ ethosu_network_get(net); - /* Send network info request to firmware */ - ret = ethosu_mailbox_network_info_request(&info->edev->mailbox, - info, - info->net->buf, - info->net->index); + ret = ethosu_network_info_send(info); if (ret) goto put_info; - /* Increase reference count for the pending network info response */ - ethosu_network_info_get(info); - dev_info(edev->dev, "Network info create. handle=%p\n", info); return info; diff --git a/kernel/ethosu_watchdog.c b/kernel/ethosu_watchdog.c new file mode 100644 index 0000000..bde0803 --- /dev/null +++ b/kernel/ethosu_watchdog.c @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0-only + */ + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include "ethosu_watchdog.h" + +#include +#include +#include +#include + +/**************************************************************************** + * Variables + ****************************************************************************/ + +static unsigned long watchdog_timeout_ms = 3000; +module_param(watchdog_timeout_ms, ulong, 0664); +MODULE_PARM_DESC(watchdog_timeout_ms, + "Watchdog timeout in milliseconds for unresponsive firmware."); + +/**************************************************************************** + * Functions + ****************************************************************************/ + +static void ethosu_watchdog_update(struct ethosu_watchdog *wdog) +{ + int ret; + + ret = mod_timer(&wdog->timer, + jiffies + msecs_to_jiffies(watchdog_timeout_ms)); + + dev_info(wdog->dev, + "Wdog: Update watchdog timeout. ret=%d, timeout_ms=%lu, refcount=%u", ret, + watchdog_timeout_ms, atomic_read(&wdog->refcount)); +} + +static void ethosu_watchdog_work(struct work_struct *work) +{ + struct ethosu_watchdog *wdog = + container_of(work, struct ethosu_watchdog, work); + + dev_info(wdog->dev, "Wdog: Watchdog timeout. refcount=%u", + atomic_read(&wdog->refcount)); + + wdog->callback(wdog); +} + +static void ethosu_watchdog_timeout(struct timer_list *timer) +{ + struct ethosu_watchdog *wdog = + container_of(timer, struct ethosu_watchdog, timer); + + queue_work(system_unbound_wq, &wdog->work); +} + +#if KERNEL_VERSION(4, 14, 0) > LINUX_VERSION_CODE +static void ethosu_watchdog_timeout_legacy(unsigned long data) +{ + ethosu_watchdog_timeout((struct timer_list *)data); +} + +#endif + +int ethosu_watchdog_init(struct ethosu_watchdog *wdog, + struct device *dev, + ethosu_watchdog_cb callback) +{ + wdog->dev = dev; + wdog->callback = callback; + atomic_set(&wdog->refcount, 0); + INIT_WORK(&wdog->work, ethosu_watchdog_work); + +#if KERNEL_VERSION(4, 14, 0) <= LINUX_VERSION_CODE + timer_setup(&wdog->timer, ethosu_watchdog_timeout, 0); +#else + setup_timer(&wdog->timer, ethosu_watchdog_timeout_legacy, + (unsigned long)&wdog->timer); +#endif + + return 0; +} + +void ethosu_watchdog_deinit(struct ethosu_watchdog *wdog) +{ + del_timer(&wdog->timer); +} + +int ethosu_watchdog_reset(struct ethosu_watchdog *wdog) +{ + del_timer(&wdog->timer); + atomic_set(&wdog->refcount, 0); + + return 0; +} + +void ethosu_watchdog_inc(struct ethosu_watchdog *wdog) +{ + atomic_inc(&wdog->refcount); + ethosu_watchdog_update(wdog); +} + +void ethosu_watchdog_dec(struct ethosu_watchdog *wdog) +{ + if (atomic_dec_and_test(&wdog->refcount)) { + dev_info(wdog->dev, "Wdog: Cancel watchdog timeout"); + del_timer(&wdog->timer); + } else { + ethosu_watchdog_update(wdog); + } +} diff --git a/kernel/ethosu_watchdog.h b/kernel/ethosu_watchdog.h new file mode 100644 index 0000000..d288af4 --- /dev/null +++ b/kernel/ethosu_watchdog.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2022 Arm Limited. + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + * SPDX-License-Identifier: GPL-2.0-only + */ + +#ifndef ETHOSU_WATCHDOG_H +#define ETHOSU_WATCHDOG_H + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include +#include +#include + +/**************************************************************************** + * Types + ****************************************************************************/ + +struct device; +struct ethosu_watchdog; + +typedef void (*ethosu_watchdog_cb)(struct ethosu_watchdog *wdog); + +struct ethosu_watchdog { + struct device *dev; + ethosu_watchdog_cb callback; + struct timer_list timer; + struct work_struct work; + atomic_t refcount; +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +/** + * ethosu_watchdog_init() - Initialize watchdog + * + * Return: 0 on success, else error code. + */ +int ethosu_watchdog_init(struct ethosu_watchdog *wdog, + struct device *dev, + ethosu_watchdog_cb callback); + +/** + * ethosu_watchdog_deinit() - Deinitialize watchdog + */ +void ethosu_watchdog_deinit(struct ethosu_watchdog *wdog); + +/** + * ethosu_watchdog_reset() - Reset watchdog + */ +int ethosu_watchdog_reset(struct ethosu_watchdog *wdog); + +/** + * ethosu_watchdog_inc() - Increment reference count + */ +void ethosu_watchdog_inc(struct ethosu_watchdog *wdog); + +/** + * ethosu_watchdog_dec() - Decrement reference count + */ +void ethosu_watchdog_dec(struct ethosu_watchdog *wdog); + +#endif /* ETHOSU_WATCHDOG_H */ -- cgit v1.2.1