Reset firmware when inference cancellation fails22.05-rc1

If an inference fails to be cancelled and it is still in a pending
state, reset the whole firmware.  Indeed the assumption is that
something is at fault on the firmware side and it is not progressing.

Change-Id: I2f568b2167d86cda3cb96a5e83551b018f5fc55d

5 files changed, 33 insertions, 15 deletions

diff --git a/kernel/ethosu_cancel_inference.c b/kernel/ethosu_cancel_inference.c
index 09778ee..befdd2f 100644
--- a/kernel/ethosu_cancel_inference.c
+++ b/kernel/ethosu_cancel_inference.c
@@ -159,6 +159,15 @@ int ethosu_cancel_inference_request(struct ethosu_inference *inf,
 		goto put_kref;
 	}
 
+	/* if cancellation failed and the inference did not complete then reset
+	 * the firmware */
+	if (cancellation->uapi->status == ETHOSU_UAPI_STATUS_ERROR &&
+	    !cancellation->inf->done) {
+		ret = ethosu_firmware_reset(cancellation->edev);
+		if (ret)
+			goto put_kref;
+	}
+
 put_kref:
 	kref_put(&cancellation->kref, &ethosu_cancel_inference_destroy);
 
diff --git a/kernel/ethosu_device.c b/kernel/ethosu_device.c
index f66c2ac..6b911ca 100644
--- a/kernel/ethosu_device.c
+++ b/kernel/ethosu_device.c
@@ -83,6 +83,9 @@ static void ethosu_capabilities_fail(struct ethosu_mailbox_msg *msg)
 	struct ethosu_capabilities *cap =
 		container_of(msg, typeof(*cap), msg);
 
+	if (completion_done(&cap->done))
+		return;
+
 	cap->errno = -EFAULT;
 	complete(&cap->done);
 }
@@ -349,7 +352,7 @@ static int ethosu_handle_msg(struct ethosu_device *edev)
 	return ret;
 }
 
-static int ethosu_firmware_reset(struct ethosu_device *edev)
+int ethosu_firmware_reset(struct ethosu_device *edev)
 {
 	int ret;
 
@@ -361,8 +364,8 @@ static int ethosu_firmware_reset(struct ethosu_device *edev)
 
 	ret = reset_control_assert(edev->reset);
 	if (ret) {
-		dev_warn(edev->dev, "Failed to reset assert firmware. ret=%d",
-			 ret);
+		dev_err(edev->dev, "Failed to reset assert firmware. ret=%d",
+			ret);
 
 		return ret;
 	}
@@ -375,16 +378,16 @@ static int ethosu_firmware_reset(struct ethosu_device *edev)
 	 */
 	ret = reset_control_deassert(edev->reset);
 	if (ret) {
-		dev_warn(edev->dev, "Failed to reset deassert firmware. ret=%d",
-			 ret);
+		dev_err(edev->dev, "Failed to reset deassert firmware. ret=%d",
+			ret);
 		goto fail;
 	}
 
 	/* Wait for firmware to boot up and initialize mailbox */
 	ret = ethosu_mailbox_wait_firmware(&edev->mailbox);
 	if (ret) {
-		dev_warn(edev->dev, "Wait on firmware boot timed out. ret=%d",
-			 ret);
+		dev_err(edev->dev, "Wait on firmware boot timed out. ret=%d",
+			ret);
 		goto fail;
 	}
 
@@ -392,8 +395,12 @@ static int ethosu_firmware_reset(struct ethosu_device *edev)
 	ethosu_watchdog_reset(&edev->watchdog);
 
 	ret = ethosu_mailbox_ping(&edev->mailbox);
-	if (ret)
+	if (ret) {
+		dev_warn(edev->dev,
+			 "Failed to send ping after firmware reset. ret=%d",
+			 ret);
 		goto fail;
+	}
 
 	/* Resend messages */
 	ethosu_mailbox_resend(&edev->mailbox);
diff --git a/kernel/ethosu_device.h b/kernel/ethosu_device.h
index 7c6c99d..132dff2 100644
--- a/kernel/ethosu_device.h
+++ b/kernel/ethosu_device.h
@@ -89,4 +89,9 @@ int ethosu_dev_init(struct ethosu_device *edev,
  */
 void ethosu_dev_deinit(struct ethosu_device *edev);
 
+/**
+ * ethosu_firmware_reset() - Reset the device running firmware
+ */
+int ethosu_firmware_reset(struct ethosu_device *edev);
+
 #endif /* ETHOSU_DEVICE_H */
diff --git a/kernel/ethosu_mailbox.c b/kernel/ethosu_mailbox.c
index 5343e56..843cb58 100644
--- a/kernel/ethosu_mailbox.c
+++ b/kernel/ethosu_mailbox.c
@@ -304,7 +304,7 @@ void ethosu_mailbox_fail(struct ethosu_mailbox *mbox)
 	}
 }
 
-int ethosu_mailbox_resend(struct ethosu_mailbox *mbox)
+void ethosu_mailbox_resend(struct ethosu_mailbox *mbox)
 {
 	struct ethosu_mailbox_msg *cur, *cur_tmp;
 	int ret;
@@ -312,13 +312,11 @@ int ethosu_mailbox_resend(struct ethosu_mailbox *mbox)
 	list_for_each_entry_safe(cur, cur_tmp, &mbox->pending_list, list) {
 		ret = cur->resend(cur);
 		if (ret) {
+			dev_warn(mbox->dev, "Failed to resend msg. ret=%d",
+				 ret);
 			cur->fail(cur);
-
-			return ret;
 		}
 	}
-
-	return 0;
 }
 
 int ethosu_mailbox_ping(struct ethosu_mailbox *mbox)
diff --git a/kernel/ethosu_mailbox.h b/kernel/ethosu_mailbox.h
index 7af0c47..07276f6 100644
--- a/kernel/ethosu_mailbox.h
+++ b/kernel/ethosu_mailbox.h
@@ -138,9 +138,8 @@ void ethosu_mailbox_fail(struct ethosu_mailbox *mbox);
  *
  * Call resend() callback on all messages in pending list.
  *
- * Return: 0 on success, else error code.
  */
-int ethosu_mailbox_resend(struct ethosu_mailbox *mbox);
+void ethosu_mailbox_resend(struct ethosu_mailbox *mbox);
 
 /**
  * ethosu_mailbox_reset() - Reset to end of queue