From d779a08a0f7ca3cdde16941720ddc7af96e74520 Mon Sep 17 00:00:00 2001 From: Kristofer Jonsson Date: Wed, 4 Jan 2023 17:09:47 +0100 Subject: Converting Ethos-U driver to rpmsg The Ethos-U kernel driver has been converted from a platform driver with a custom firmware interface into a rpmsg driver. Change-Id: I9ae449f5e79eb02924e6630611d0893e5fec86be --- kernel/ethosu_buffer.c | 90 +++------ kernel/ethosu_buffer.h | 4 +- kernel/ethosu_cancel_inference.c | 12 +- kernel/ethosu_cancel_inference.h | 9 +- kernel/ethosu_capabilities.c | 7 +- kernel/ethosu_capabilities.h | 5 +- kernel/ethosu_device.c | 259 ++++++++++++------------- kernel/ethosu_device.h | 9 +- kernel/ethosu_driver.c | 115 +++++------ kernel/ethosu_inference.c | 10 +- kernel/ethosu_inference.h | 7 +- kernel/ethosu_mailbox.c | 408 ++++++++------------------------------- kernel/ethosu_mailbox.h | 51 +---- kernel/ethosu_network_info.c | 8 +- kernel/ethosu_network_info.h | 7 +- 15 files changed, 336 insertions(+), 665 deletions(-) (limited to 'kernel') diff --git a/kernel/ethosu_buffer.c b/kernel/ethosu_buffer.c index ac32aca..a83a95a 100644 --- a/kernel/ethosu_buffer.c +++ b/kernel/ethosu_buffer.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. + * Copyright 2020-2023 Arm Limited and/or its affiliates * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -32,6 +32,7 @@ #include #include #include +#include #include /**************************************************************************** @@ -61,59 +62,22 @@ static const struct file_operations ethosu_buffer_fops = { * Functions ****************************************************************************/ -/* - * The 'dma-ranges' device tree property for shared dma memory does not seem - * to be fully supported for coherent memory. Therefor we apply the DMA range - * offset ourselves. - */ -static dma_addr_t ethosu_buffer_dma_ranges(struct device *dev, - dma_addr_t dma_addr, - size_t dma_buf_size) +__attribute__((used)) +static dma_addr_t ethosu_pa_to_da(struct device *dev, + phys_addr_t pa, + size_t len) { - struct device_node *node = dev->of_node; - const __be32 *ranges; - int len; - int naddr; - int nsize; - int inc; - int i; - - if (!node) - return dma_addr; - - /* Get the #address-cells and #size-cells properties */ - naddr = of_n_addr_cells(node); - nsize = of_n_size_cells(node); - - /* Read the 'dma-ranges' property */ - ranges = of_get_property(node, "dma-ranges", &len); - if (!ranges || len <= 0) - return dma_addr; - - dev_dbg(dev, "ranges=%p, len=%d, naddr=%d, nsize=%d\n", - ranges, len, naddr, nsize); - - len /= sizeof(*ranges); - inc = naddr + naddr + nsize; - - for (i = 0; (i + inc) <= len; i += inc) { - dma_addr_t daddr; - dma_addr_t paddr; - dma_addr_t size; - - daddr = of_read_number(&ranges[i], naddr); - paddr = of_read_number(&ranges[i + naddr], naddr); - size = of_read_number(&ranges[i + naddr + naddr], nsize); - - dev_dbg(dev, "daddr=0x%llx, paddr=0x%llx, size=0x%llx\n", - daddr, paddr, size); - - if (dma_addr >= paddr && - (dma_addr + dma_buf_size) < (paddr + size)) - return dma_addr + daddr - paddr; + struct rproc *rproc = rproc_get_by_child(dev); + struct rproc_mem_entry *mem; + + list_for_each_entry(mem, &rproc->carveouts, node) { + ssize_t offset = pa - mem->dma; + + if (offset >= 0 && offset + len <= mem->len) + return mem->da + offset; } - return dma_addr; + return (dma_addr_t)-1; } static bool ethosu_buffer_verify(struct file *file) @@ -128,8 +92,8 @@ static void ethosu_buffer_destroy(struct kref *kref) dev_info(buf->edev->dev, "Buffer destroy. buf=0x%pK\n", buf); - dma_free_coherent(buf->edev->dev, buf->capacity, buf->cpu_addr, - buf->dma_addr_orig); + dma_free_coherent(buf->dev, buf->capacity, buf->cpu_addr, + buf->dma_addr); devm_kfree(buf->edev->dev, buf); } @@ -157,8 +121,7 @@ static int ethosu_buffer_mmap(struct file *file, file, buf); ret = dma_mmap_coherent(buf->edev->dev, vma, buf->cpu_addr, - buf->dma_addr_orig, - buf->capacity); + buf->dma_addr, buf->capacity); return ret; } @@ -224,6 +187,8 @@ static long ethosu_buffer_ioctl(struct file *file, int ethosu_buffer_create(struct ethosu_device *edev, size_t capacity) { + struct rproc *rproc = rproc_get_by_child(edev->dev); + struct device *dev = rproc->dev.parent; struct ethosu_buffer *buf; int ret = -ENOMEM; @@ -235,20 +200,17 @@ int ethosu_buffer_create(struct ethosu_device *edev, return -ENOMEM; buf->edev = edev; + buf->dev = dev; buf->capacity = capacity; buf->offset = 0; buf->size = 0; kref_init(&buf->kref); - buf->cpu_addr = dma_alloc_coherent(buf->edev->dev, capacity, - &buf->dma_addr_orig, GFP_KERNEL); + buf->cpu_addr = dma_alloc_coherent(dev, capacity, + &buf->dma_addr, GFP_KERNEL); if (!buf->cpu_addr) goto free_buf; - buf->dma_addr = ethosu_buffer_dma_ranges(buf->edev->dev, - buf->dma_addr_orig, - buf->capacity); - ret = anon_inode_getfd("ethosu-buffer", ðosu_buffer_fops, buf, O_RDWR | O_CLOEXEC); if (ret < 0) @@ -258,15 +220,15 @@ int ethosu_buffer_create(struct ethosu_device *edev, fput(buf->file); dev_info(buf->edev->dev, - "Buffer create. file=0x%pK, fd=%d, buf=0x%pK, capacity=%zu, cpu_addr=0x%pK, dma_addr=0x%llx, dma_addr_orig=0x%llx, phys_addr=0x%llx\n", + "Buffer create. file=0x%pK, fd=%d, buf=0x%pK, capacity=%zu, cpu_addr=0x%pK, dma_addr=0x%llx, phys_addr=0x%llx\n", buf->file, ret, buf, capacity, buf->cpu_addr, buf->dma_addr, - buf->dma_addr_orig, virt_to_phys(buf->cpu_addr)); + virt_to_phys(buf->cpu_addr)); return ret; free_dma: dma_free_coherent(buf->edev->dev, buf->capacity, buf->cpu_addr, - buf->dma_addr_orig); + buf->dma_addr); free_buf: devm_kfree(buf->edev->dev, buf); diff --git a/kernel/ethosu_buffer.h b/kernel/ethosu_buffer.h index 7a21fcb..ada8c4c 100644 --- a/kernel/ethosu_buffer.h +++ b/kernel/ethosu_buffer.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020,2022 ARM Limited. + * Copyright 2020,2022-2023 Arm Limited and/or its affiliates * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -51,6 +51,7 @@ struct device; */ struct ethosu_buffer { struct ethosu_device *edev; + struct device *dev; struct file *file; struct kref kref; size_t capacity; @@ -58,7 +59,6 @@ struct ethosu_buffer { size_t size; void *cpu_addr; dma_addr_t dma_addr; - dma_addr_t dma_addr_orig; }; /**************************************************************************** diff --git a/kernel/ethosu_cancel_inference.c b/kernel/ethosu_cancel_inference.c index c1d31b3..61f7a9c 100644 --- a/kernel/ethosu_cancel_inference.c +++ b/kernel/ethosu_cancel_inference.c @@ -24,7 +24,7 @@ #include "ethosu_cancel_inference.h" -#include "ethosu_core_interface.h" +#include "ethosu_core_rpmsg.h" #include "ethosu_device.h" #include "ethosu_inference.h" @@ -143,17 +143,17 @@ kfree: } void ethosu_cancel_inference_rsp(struct ethosu_device *edev, - struct ethosu_core_cancel_inference_rsp *rsp) + int msg_id, + struct ethosu_core_msg_cancel_inference_rsp *rsp) { - int id = (int)rsp->user_arg; struct ethosu_mailbox_msg *msg; struct ethosu_cancel_inference *cancellation; - msg = ethosu_mailbox_find(&edev->mailbox, id); + msg = ethosu_mailbox_find(&edev->mailbox, msg_id); if (IS_ERR(msg)) { dev_warn(edev->dev, - "Handle not found in cancel inference list. handle=0x%p\n", - rsp); + "Id for cancel inference msg not found. id=%d\n", + msg_id); return; } diff --git a/kernel/ethosu_cancel_inference.h b/kernel/ethosu_cancel_inference.h index 044692c..a21b9c0 100644 --- a/kernel/ethosu_cancel_inference.h +++ b/kernel/ethosu_cancel_inference.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 ARM Limited. + * Copyright 2022-2023 Arm Limited and/or its affiliates * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -35,10 +35,10 @@ * Types ****************************************************************************/ -struct ethosu_core_cancel_inference_rsp; +struct ethosu_core_msg_cancel_inference_rsp; struct ethosu_device; -struct ethosu_uapi_cancel_inference_status; struct ethosu_inference; +struct ethosu_uapi_cancel_inference_status; struct ethosu_cancel_inference { struct ethosu_device *edev; @@ -65,6 +65,7 @@ int ethosu_cancel_inference_request(struct ethosu_inference *inf, * ethosu_cancel_inference_rsp() - Handle cancel inference response */ void ethosu_cancel_inference_rsp(struct ethosu_device *edev, - struct ethosu_core_cancel_inference_rsp *rsp); + int msg_id, + struct ethosu_core_msg_cancel_inference_rsp *rsp); #endif /* ETHOSU_CANCEL_INFERENCE_H */ diff --git a/kernel/ethosu_capabilities.c b/kernel/ethosu_capabilities.c index 9735ee2..57db452 100644 --- a/kernel/ethosu_capabilities.c +++ b/kernel/ethosu_capabilities.c @@ -24,6 +24,7 @@ #include "ethosu_capabilities.h" +#include "ethosu_core_rpmsg.h" #include "ethosu_device.h" #include @@ -59,17 +60,17 @@ static void ethosu_capabilities_fail(struct ethosu_mailbox_msg *msg) } void ethosu_capability_rsp(struct ethosu_device *edev, + int msg_id, struct ethosu_core_msg_capabilities_rsp *rsp) { - int id = (int)rsp->user_arg; struct ethosu_mailbox_msg *msg; struct ethosu_capabilities *cap; - msg = ethosu_mailbox_find(&edev->mailbox, id); + msg = ethosu_mailbox_find(&edev->mailbox, msg_id); if (IS_ERR(msg)) { dev_warn(edev->dev, "Id for capabilities msg not found. id=%d\n", - id); + msg_id); return; } diff --git a/kernel/ethosu_capabilities.h b/kernel/ethosu_capabilities.h index 87cd168..3a79dc5 100644 --- a/kernel/ethosu_capabilities.h +++ b/kernel/ethosu_capabilities.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Arm Limited. + * Copyright 2022-2023 Arm Limited and/or its affiliates * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -25,7 +25,6 @@ * Includes ****************************************************************************/ -#include "ethosu_core_interface.h" #include "ethosu_mailbox.h" #include @@ -35,6 +34,7 @@ * Types ****************************************************************************/ +struct ethosu_core_msg_capabilities_rsp; struct ethosu_device; struct ethosu_uapi_device_capabilities; @@ -57,6 +57,7 @@ int ethosu_capabilities_request(struct ethosu_device *edev, struct ethosu_uapi_device_capabilities *uapi); void ethosu_capability_rsp(struct ethosu_device *edev, + int msg_id, struct ethosu_core_msg_capabilities_rsp *rsp); #endif diff --git a/kernel/ethosu_device.c b/kernel/ethosu_device.c index 36b776f..004b26f 100644 --- a/kernel/ethosu_device.c +++ b/kernel/ethosu_device.c @@ -25,7 +25,7 @@ #include "ethosu_device.h" #include "ethosu_buffer.h" -#include "ethosu_core_interface.h" +#include "ethosu_core_rpmsg.h" #include "ethosu_capabilities.h" #include "ethosu_inference.h" #include "ethosu_cancel_inference.h" @@ -46,102 +46,98 @@ * Defines ****************************************************************************/ -#define DMA_ADDR_BITS 32 /* Number of address bits */ - /**************************************************************************** * Functions ****************************************************************************/ /* Incoming messages */ -static int ethosu_handle_msg(struct ethosu_device *edev) +static int ethosu_handle_msg(struct rpmsg_device *rpdev, + void *data, + int len, + void *priv, + u32 src) { + struct ethosu_device *edev = dev_get_drvdata(&rpdev->dev); + struct device *dev = &rpdev->dev; + struct ethosu_core_rpmsg *rpmsg = data; + int length = len - sizeof(rpmsg->header); int ret; - struct ethosu_core_msg header; - - union { - struct ethosu_core_msg_err error; - struct ethosu_core_inference_rsp inf; - struct ethosu_core_msg_version version; - struct ethosu_core_msg_capabilities_rsp capabilities; - struct ethosu_core_network_info_rsp network_info; - struct ethosu_core_cancel_inference_rsp cancellation; - } data; - - /* Read message */ - ret = ethosu_mailbox_read(&edev->mailbox, &header, &data, sizeof(data)); - if (ret) - return ret; - switch (header.type) { + dev_info(dev, + "Msg: magic=0x%08x, type=%u, msg_id=%llu", + rpmsg->header.magic, rpmsg->header.type, rpmsg->header.msg_id); + + switch (rpmsg->header.type) { case ETHOSU_CORE_MSG_ERR: - if (header.length != sizeof(data.error)) { - dev_warn(edev->dev, - "Msg: Error message of incorrect size. size=%u, expected=%zu\n", header.length, - sizeof(data.error)); + if (length != sizeof(rpmsg->error)) { + dev_warn(dev, + "Msg: Error message of incorrect size. size=%u, expected=%zu", length, + sizeof(rpmsg->error)); ret = -EBADMSG; break; } - data.error.msg[sizeof(data.error.msg) - 1] = '\0'; - dev_warn(edev->dev, "Msg: Error. type=%u, msg=\"%s\"\n", - data.error.type, data.error.msg); + rpmsg->error.msg[sizeof(rpmsg->error.msg) - 1] = '\0'; + dev_warn(dev, "Msg: Error. type=%u, msg=\"%s\"", + rpmsg->error.type, rpmsg->error.msg); ret = -EBADMSG; break; case ETHOSU_CORE_MSG_PING: - dev_info(edev->dev, "Msg: Ping\n"); + dev_info(dev, "Msg: Ping"); ret = ethosu_mailbox_pong(&edev->mailbox); break; case ETHOSU_CORE_MSG_PONG: - dev_info(edev->dev, "Msg: Pong\n"); + dev_info(dev, "Msg: Pong"); break; case ETHOSU_CORE_MSG_INFERENCE_RSP: - if (header.length != sizeof(data.inf)) { - dev_warn(edev->dev, - "Msg: Inference response of incorrect size. size=%u, expected=%zu\n", header.length, - sizeof(data.inf)); + if (length != sizeof(rpmsg->inf_rsp)) { + dev_warn(dev, + "Msg: Inference response of incorrect size. size=%u, expected=%zu", length, + sizeof(rpmsg->inf_rsp)); ret = -EBADMSG; break; } - dev_info(edev->dev, - "Msg: Inference response. user_arg=0x%llx, ofm_count=%u, status=%u\n", - data.inf.user_arg, data.inf.ofm_count, - data.inf.status); + dev_info(dev, + "Msg: Inference response. ofm_count=%u, status=%u", + rpmsg->inf_rsp.ofm_count, rpmsg->inf_rsp.status); - ethosu_inference_rsp(edev, &data.inf); + ethosu_inference_rsp(edev, rpmsg->header.msg_id, + &rpmsg->inf_rsp); break; case ETHOSU_CORE_MSG_CANCEL_INFERENCE_RSP: - if (header.length != sizeof(data.cancellation)) { - dev_warn(edev->dev, - "Msg: Cancel Inference response of incorrect size. size=%u, expected=%zu\n", header.length, - sizeof(data.cancellation)); + if (length != sizeof(rpmsg->cancel_rsp)) { + dev_warn(dev, + "Msg: Cancel Inference response of incorrect size. size=%u, expected=%zu", length, + sizeof(rpmsg->cancel_rsp)); ret = -EBADMSG; break; } - dev_info(edev->dev, - "Msg: Cancel Inference response. user_arg=0x%llx, status=%u\n", - data.cancellation.user_arg, data.cancellation.status); - ethosu_cancel_inference_rsp(edev, &data.cancellation); + dev_info(dev, + "Msg: Cancel Inference response. status=%u", + rpmsg->cancel_rsp.status); + ethosu_cancel_inference_rsp(edev, rpmsg->header.msg_id, + &rpmsg->cancel_rsp); break; case ETHOSU_CORE_MSG_VERSION_RSP: - if (header.length != sizeof(data.version)) { - dev_warn(edev->dev, - "Msg: Version response of incorrect size. size=%u, expected=%zu\n", header.length, - sizeof(data.version)); + if (length != sizeof(rpmsg->version_rsp)) { + dev_warn(dev, + "Msg: Version response of incorrect size. size=%u, expected=%zu", length, + sizeof(rpmsg->version_rsp)); ret = -EBADMSG; break; } - dev_info(edev->dev, "Msg: Version response v%u.%u.%u\n", - data.version.major, data.version.minor, - data.version.patch); + dev_info(dev, "Msg: Version response v%u.%u.%u", + rpmsg->version_rsp.major, rpmsg->version_rsp.minor, + rpmsg->version_rsp.patch); /* Check major and minor version match, else return error */ - if (data.version.major != ETHOSU_CORE_MSG_VERSION_MAJOR || - data.version.minor != ETHOSU_CORE_MSG_VERSION_MINOR) { - dev_warn(edev->dev, "Msg: Version mismatch detected! "); - dev_warn(edev->dev, "Local version: v%u.%u.%u\n", + if (rpmsg->version_rsp.major != ETHOSU_CORE_MSG_VERSION_MAJOR || + rpmsg->version_rsp.minor != ETHOSU_CORE_MSG_VERSION_MINOR) { + dev_warn(dev, "Msg: Version mismatch detected! "); + dev_warn(dev, "Local version: v%u.%u.%u", ETHOSU_CORE_MSG_VERSION_MAJOR, ETHOSU_CORE_MSG_VERSION_MINOR, ETHOSU_CORE_MSG_VERSION_PATCH); @@ -149,53 +145,54 @@ static int ethosu_handle_msg(struct ethosu_device *edev) break; case ETHOSU_CORE_MSG_CAPABILITIES_RSP: - if (header.length != sizeof(data.capabilities)) { - dev_warn(edev->dev, - "Msg: Capabilities response of incorrect size. size=%u, expected=%zu\n", header.length, - sizeof(data.capabilities)); + if (length != sizeof(rpmsg->cap_rsp)) { + dev_warn(dev, + "Msg: Capabilities response of incorrect size. size=%u, expected=%zu", length, + sizeof(rpmsg->cap_rsp)); ret = -EBADMSG; break; } - dev_info(edev->dev, - "Msg: Capabilities response ua%llx vs%hhu v%hhu.%hhu p%hhu av%hhu.%hhu.%hhu dv%hhu.%hhu.%hhu mcc%hhu csv%hhu cd%hhu\n", - data.capabilities.user_arg, - data.capabilities.version_status, - data.capabilities.version_major, - data.capabilities.version_minor, - data.capabilities.product_major, - data.capabilities.arch_major_rev, - data.capabilities.arch_minor_rev, - data.capabilities.arch_patch_rev, - data.capabilities.driver_major_rev, - data.capabilities.driver_minor_rev, - data.capabilities.driver_patch_rev, - data.capabilities.macs_per_cc, - data.capabilities.cmd_stream_version, - data.capabilities.custom_dma); - - ethosu_capability_rsp(edev, &data.capabilities); + dev_info(dev, + "Msg: Capabilities response vs%hhu v%hhu.%hhu p%hhu av%hhu.%hhu.%hhu dv%hhu.%hhu.%hhu mcc%hhu csv%hhu cd%hhu", + rpmsg->cap_rsp.version_status, + rpmsg->cap_rsp.version_major, + rpmsg->cap_rsp.version_minor, + rpmsg->cap_rsp.product_major, + rpmsg->cap_rsp.arch_major_rev, + rpmsg->cap_rsp.arch_minor_rev, + rpmsg->cap_rsp.arch_patch_rev, + rpmsg->cap_rsp.driver_major_rev, + rpmsg->cap_rsp.driver_minor_rev, + rpmsg->cap_rsp.driver_patch_rev, + rpmsg->cap_rsp.macs_per_cc, + rpmsg->cap_rsp.cmd_stream_version, + rpmsg->cap_rsp.custom_dma); + + ethosu_capability_rsp(edev, rpmsg->header.msg_id, + &rpmsg->cap_rsp); break; case ETHOSU_CORE_MSG_NETWORK_INFO_RSP: - if (header.length != sizeof(data.network_info)) { - dev_warn(edev->dev, - "Msg: Network info response of incorrect size. size=%u, expected=%zu\n", header.length, - sizeof(data.network_info)); + if (length != sizeof(rpmsg->net_info_rsp)) { + dev_warn(dev, + "Msg: Network info response of incorrect size. size=%u, expected=%zu", length, + sizeof(rpmsg->net_info_rsp)); ret = -EBADMSG; break; } - dev_info(edev->dev, - "Msg: Network info response. user_arg=0x%llx, status=%u", - data.network_info.user_arg, - data.network_info.status); + dev_info(dev, + "Msg: Network info response. status=%u", + rpmsg->net_info_rsp.status); - ethosu_network_info_rsp(edev, &data.network_info); + ethosu_network_info_rsp(edev, rpmsg->header.msg_id, + &rpmsg->net_info_rsp); break; default: /* This should not happen due to version checks */ - dev_warn(edev->dev, "Msg: Protocol error\n"); + dev_warn(dev, "Msg: Protocol error. type=%u", + rpmsg->header.type); ret = -EPROTO; break; } @@ -211,7 +208,7 @@ static int ethosu_open(struct inode *inode, file->private_data = edev; - dev_info(edev->dev, "Device open. file=0x%pK\n", file); + dev_info(edev->dev, "Device open. file=0x%pK", file); return nonseekable_open(inode, file); } @@ -228,19 +225,19 @@ static long ethosu_ioctl(struct file *file, if (ret) return ret; - dev_info(edev->dev, "Device ioctl. file=0x%pK, cmd=0x%x, arg=0x%lx\n", + dev_info(edev->dev, "Device ioctl. file=0x%pK, cmd=0x%x, arg=0x%lx", file, cmd, arg); switch (cmd) { case ETHOSU_IOCTL_VERSION_REQ: - dev_info(edev->dev, "Device ioctl: Send version request\n"); + dev_info(edev->dev, "Device ioctl: Send version request"); ret = ethosu_mailbox_version_request(&edev->mailbox); break; case ETHOSU_IOCTL_CAPABILITIES_REQ: { struct ethosu_uapi_device_capabilities uapi; dev_info(edev->dev, - "Device ioctl: Send capabilities request\n"); + "Device ioctl: Send capabilities request"); ret = ethosu_capabilities_request(edev, &uapi); if (ret) @@ -250,7 +247,7 @@ static long ethosu_ioctl(struct file *file, break; } case ETHOSU_IOCTL_PING: { - dev_info(edev->dev, "Device ioctl: Send ping\n"); + dev_info(edev->dev, "Device ioctl: Send ping"); ret = ethosu_mailbox_ping(&edev->mailbox); break; } @@ -261,7 +258,7 @@ static long ethosu_ioctl(struct file *file, break; dev_info(edev->dev, - "Device ioctl: Buffer create. capacity=%u\n", + "Device ioctl: Buffer create. capacity=%u", uapi.capacity); ret = ethosu_buffer_create(edev, uapi.capacity); @@ -274,7 +271,7 @@ static long ethosu_ioctl(struct file *file, break; dev_info(edev->dev, - "Device ioctl: Network create. type=%u, fd/index=%u\n", + "Device ioctl: Network create. type=%u, fd/index=%u", uapi.type, uapi.fd); ret = ethosu_network_create(edev, &uapi); @@ -292,31 +289,34 @@ static long ethosu_ioctl(struct file *file, return ret; } -static void ethosu_mbox_rx(void *user_arg) +static struct rpmsg_endpoint *ethosu_create_ept(struct rpmsg_device *rpdev) { - struct ethosu_device *edev = user_arg; - int ret; + struct device *dev = &rpdev->dev; + struct rpmsg_channel_info info; + struct rpmsg_endpoint *ept; - mutex_lock(&edev->mutex); + /* Create rpmsg endpoint */ + strncpy(info.name, rpdev->id.name, sizeof(info.name)); + info.src = 0; + info.dst = rpdev->dst; - do { - ret = ethosu_handle_msg(edev); - if (ret && ret != -ENOMSG) - /* Need to start over in case of error, empty the queue - * by fast-forwarding read position to write position. - * */ - ethosu_mailbox_reset(&edev->mailbox); - } while (ret == 0); + dev_info(dev, "Creating rpmsg endpoint. name=%s, src=%u, dst=%u", + info.name, info.src, info.dst); - mutex_unlock(&edev->mutex); + ept = rpmsg_create_ept(rpdev, ethosu_handle_msg, NULL, info); + if (!ept) { + dev_err(&rpdev->dev, "Failed to create endpoint"); + + return ERR_PTR(-EINVAL); + } + + return ept; } int ethosu_dev_init(struct ethosu_device *edev, - struct device *dev, + struct rpmsg_device *rpdev, struct class *class, - dev_t devt, - struct resource *in_queue, - struct resource *out_queue) + dev_t devt) { static const struct file_operations fops = { .owner = THIS_MODULE, @@ -329,43 +329,43 @@ int ethosu_dev_init(struct ethosu_device *edev, struct device *sysdev; int ret; - edev->dev = dev; + edev->dev = &rpdev->dev; + edev->rpdev = rpdev; edev->class = class; edev->devt = devt; mutex_init(&edev->mutex); - ret = of_reserved_mem_device_init(edev->dev); - if (ret) - return ret; - - dma_set_mask_and_coherent(edev->dev, DMA_BIT_MASK(DMA_ADDR_BITS)); + edev->ept = ethosu_create_ept(rpdev); + if (IS_ERR(edev->ept)) + return PTR_ERR(edev->ept); - ret = ethosu_mailbox_init(&edev->mailbox, dev, in_queue, out_queue, - ethosu_mbox_rx, edev); + ret = ethosu_mailbox_init(&edev->mailbox, edev->dev, edev->ept); if (ret) - goto release_reserved_mem; + return ret; cdev_init(&edev->cdev, &fops); edev->cdev.owner = THIS_MODULE; ret = cdev_add(&edev->cdev, edev->devt, 1); if (ret) { - dev_err(edev->dev, "Failed to add character device.\n"); + dev_err(edev->dev, "Failed to add character device."); goto deinit_mailbox; } sysdev = device_create(edev->class, NULL, edev->devt, edev, "ethosu%d", MINOR(edev->devt)); if (IS_ERR(sysdev)) { - dev_err(edev->dev, "Failed to create device.\n"); + dev_err(edev->dev, "Failed to create device."); ret = PTR_ERR(sysdev); goto del_cdev; } dev_info(edev->dev, - "Created Arm Ethos-U device. name=%s, major=%d, minor=%d\n", + "Created Arm Ethos-U device. name=%s, major=%d, minor=%d", dev_name(sysdev), MAJOR(edev->devt), MINOR(edev->devt)); + ethosu_mailbox_ping(&edev->mailbox); + return 0; del_cdev: @@ -374,18 +374,15 @@ del_cdev: deinit_mailbox: ethosu_mailbox_deinit(&edev->mailbox); -release_reserved_mem: - of_reserved_mem_device_release(edev->dev); - return ret; } void ethosu_dev_deinit(struct ethosu_device *edev) { + dev_info(edev->dev, "%s\n", __FUNCTION__); + ethosu_mailbox_deinit(&edev->mailbox); + rpmsg_destroy_ept(edev->ept); device_destroy(edev->class, edev->cdev.dev); cdev_del(&edev->cdev); - of_reserved_mem_device_release(edev->dev); - - dev_info(edev->dev, "%s\n", __FUNCTION__); } diff --git a/kernel/ethosu_device.h b/kernel/ethosu_device.h index 8e81f91..cc4271d 100644 --- a/kernel/ethosu_device.h +++ b/kernel/ethosu_device.h @@ -32,6 +32,7 @@ #include #include #include +#include /**************************************************************************** * Types @@ -42,6 +43,8 @@ */ struct ethosu_device { struct device *dev; + struct rpmsg_device *rpdev; + struct rpmsg_endpoint *ept; struct cdev cdev; struct class *class; dev_t devt; @@ -59,11 +62,9 @@ struct ethosu_device { * Return: 0 on success, else error code. */ int ethosu_dev_init(struct ethosu_device *edev, - struct device *dev, + struct rpmsg_device *rpdev, struct class *class, - dev_t devt, - struct resource *in_queue, - struct resource *out_queue); + dev_t devt); /** * ethosu_dev_deinit() - Initialize the device diff --git a/kernel/ethosu_driver.c b/kernel/ethosu_driver.c index a530f95..c6fc8cd 100644 --- a/kernel/ethosu_driver.c +++ b/kernel/ethosu_driver.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020-2022 Arm Limited. + * Copyright 2020-2023 Arm Limited and/or its affiliates * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include "ethosu_device.h" @@ -38,6 +38,8 @@ #define MINOR_BASE 0 /* Minor version starts at 0 */ #define MINOR_COUNT 64 /* Allocate minor versions */ +#define DMA_ADDR_BITS 32 /* Number of address bits */ + /**************************************************************************** * Variables ****************************************************************************/ @@ -49,105 +51,98 @@ static dev_t devt; static DECLARE_BITMAP(minors, MINOR_COUNT); /**************************************************************************** - * Arm Ethos-U + * Rpmsg driver ****************************************************************************/ -static int ethosu_pdev_probe(struct platform_device *pdev) +static int ethosu_rpmsg_probe(struct rpmsg_device *rpdev) { + struct device *dev = &rpdev->dev; struct ethosu_device *edev; - struct resource *in_queue_res; - struct resource *out_queue_res; int minor; int ret; - dev_info(&pdev->dev, "Probe\n"); - + /* Reserve minor number for device node */ minor = find_first_zero_bit(minors, MINOR_COUNT); if (minor >= MINOR_COUNT) { - dev_err(&pdev->dev, "No more minor numbers.\n"); + dev_err(dev, "No more minor numbers."); return -ENOMEM; } - /* Get path to TCM memory */ - in_queue_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, - "in_queue"); - if (IS_ERR(in_queue_res)) { - dev_err(&pdev->dev, "Failed to get in_queue resource.\n"); - - return PTR_ERR(in_queue_res); - } - - out_queue_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, - "out_queue"); - if (IS_ERR(out_queue_res)) { - dev_err(&pdev->dev, "Failed to get out_queue resource.\n"); - - return PTR_ERR(out_queue_res); - } - - /* Allocate memory for Arm Ethos-U device */ - edev = devm_kzalloc(&pdev->dev, sizeof(*edev), GFP_KERNEL); + edev = devm_kzalloc(dev, sizeof(*edev), GFP_KERNEL); if (!edev) return -ENOMEM; - platform_set_drvdata(pdev, edev); + dev_set_drvdata(dev, edev); /* Initialize device */ - ret = ethosu_dev_init(edev, &pdev->dev, ethosu_class, - MKDEV(MAJOR(devt), minor), in_queue_res, - out_queue_res); + ret = ethosu_dev_init(edev, rpdev, ethosu_class, + MKDEV(MAJOR(devt), minor)); if (ret) - goto free_dev; + return ret; set_bit(minor, minors); return 0; - -free_dev: - devm_kfree(&pdev->dev, edev); - - return ret; } -static int ethosu_pdev_remove(struct platform_device *pdev) +static void ethosu_rpmsg_remove(struct rpmsg_device *rpdev) { - struct ethosu_device *edev = platform_get_drvdata(pdev); + struct ethosu_device *edev = dev_get_drvdata(&rpdev->dev); + + dev_info(&rpdev->dev, "%s", __FUNCTION__); clear_bit(MINOR(edev->devt), minors); ethosu_dev_deinit(edev); +} - return 0; +static int ethosu_rpmsg_cb(struct rpmsg_device *rpdev, + void *data, + int len, + void *priv, + u32 src) +{ + dev_err(&rpdev->dev, "%s", __FUNCTION__); + + return -EINVAL; } -static const struct of_device_id ethosu_pdev_match[] = { - { .compatible = "arm,ethosu" }, - { /* Sentinel */ }, +static struct rpmsg_device_id ethosu_rpmsg_driver_id_table[] = { + { .name = "ethos-u-0.0" }, + {}, }; -MODULE_DEVICE_TABLE(of, ethosu_pdev_match); +MODULE_DEVICE_TABLE(rpmsg, ethosu_rpmsg_driver_id_table); -static struct platform_driver ethosu_pdev_driver = { - .probe = ðosu_pdev_probe, - .remove = ðosu_pdev_remove, - .driver = { - .name = ETHOSU_DRIVER_NAME, - .owner = THIS_MODULE, - .of_match_table = of_match_ptr(ethosu_pdev_match), +static struct rpmsg_driver ethosu_rpmsg_driver = { + .drv = { + .name = ETHOSU_DRIVER_NAME, + .owner = THIS_MODULE, }, + .id_table = ethosu_rpmsg_driver_id_table, + .probe = ethosu_rpmsg_probe, + .callback = ethosu_rpmsg_cb, + .remove = ethosu_rpmsg_remove, }; /**************************************************************************** * Module init and exit ****************************************************************************/ +static void __exit ethosu_exit(void) +{ + unregister_rpmsg_driver(ðosu_rpmsg_driver); + unregister_chrdev_region(devt, MINOR_COUNT); + class_destroy(ethosu_class); +} + static int __init ethosu_init(void) { int ret; ethosu_class = class_create(THIS_MODULE, ETHOSU_DRIVER_NAME); if (IS_ERR(ethosu_class)) { - printk("Failed to create class '%s'.\n", ETHOSU_DRIVER_NAME); + pr_err("Failed to create class '%s'.\n", ETHOSU_DRIVER_NAME); return PTR_ERR(ethosu_class); } @@ -155,13 +150,13 @@ static int __init ethosu_init(void) ret = alloc_chrdev_region(&devt, MINOR_BASE, MINOR_COUNT, ETHOSU_DRIVER_NAME); if (ret) { - printk("Failed to allocate chrdev region.\n"); + pr_err("Failed to allocate chrdev region.\n"); goto destroy_class; } - ret = platform_driver_register(ðosu_pdev_driver); + ret = register_rpmsg_driver(ðosu_rpmsg_driver); if (ret) { - printk("Failed to register Arm Ethos-U platform driver.\n"); + pr_err("Failed to register Arm Ethos-U rpmsg driver.\n"); goto region_unregister; } @@ -176,15 +171,9 @@ destroy_class: return ret; } -static void __exit ethosu_exit(void) -{ - platform_driver_unregister(ðosu_pdev_driver); - unregister_chrdev_region(devt, MINOR_COUNT); - class_destroy(ethosu_class); -} - module_init(ethosu_init) module_exit(ethosu_exit) + MODULE_LICENSE("GPL v2"); MODULE_AUTHOR("Arm Ltd"); MODULE_DESCRIPTION("Arm Ethos-U NPU Driver"); diff --git a/kernel/ethosu_inference.c b/kernel/ethosu_inference.c index 58501f7..4f56126 100644 --- a/kernel/ethosu_inference.c +++ b/kernel/ethosu_inference.c @@ -25,7 +25,7 @@ #include "ethosu_inference.h" #include "ethosu_buffer.h" -#include "ethosu_core_interface.h" +#include "ethosu_core_rpmsg.h" #include "ethosu_device.h" #include "ethosu_network.h" #include "ethosu_cancel_inference.h" @@ -413,19 +413,19 @@ int ethosu_inference_put(struct ethosu_inference *inf) } void ethosu_inference_rsp(struct ethosu_device *edev, - struct ethosu_core_inference_rsp *rsp) + int msg_id, + struct ethosu_core_msg_inference_rsp *rsp) { - int id = (int)rsp->user_arg; struct ethosu_mailbox_msg *msg; struct ethosu_inference *inf; int ret; int i; - msg = ethosu_mailbox_find(&edev->mailbox, id); + msg = ethosu_mailbox_find(&edev->mailbox, msg_id); if (IS_ERR(msg)) { dev_warn(edev->dev, "Id for inference msg not found. Id=%d\n", - id); + msg_id); return; } diff --git a/kernel/ethosu_inference.h b/kernel/ethosu_inference.h index 2f188b6..5a3b860 100644 --- a/kernel/ethosu_inference.h +++ b/kernel/ethosu_inference.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020,2022 ARM Limited. + * Copyright 2020,2022-2023 Arm Limited and/or its affiliates * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -36,7 +36,7 @@ ****************************************************************************/ struct ethosu_buffer; -struct ethosu_core_inference_rsp; +struct ethosu_core_msg_inference_rsp; struct ethosu_device; struct ethosu_network; struct ethosu_uapi_inference_create; @@ -118,6 +118,7 @@ int ethosu_inference_put(struct ethosu_inference *inf); * ethosu_inference_rsp() - Handle inference response */ void ethosu_inference_rsp(struct ethosu_device *edev, - struct ethosu_core_inference_rsp *rsp); + int msg_id, + struct ethosu_core_msg_inference_rsp *rsp); #endif /* ETHOSU_INFERENCE_H */ diff --git a/kernel/ethosu_mailbox.c b/kernel/ethosu_mailbox.c index 870a4ef..4c64f17 100644 --- a/kernel/ethosu_mailbox.c +++ b/kernel/ethosu_mailbox.c @@ -25,7 +25,7 @@ #include "ethosu_mailbox.h" #include "ethosu_buffer.h" -#include "ethosu_core_interface.h" +#include "ethosu_core_rpmsg.h" #include "ethosu_device.h" #include @@ -62,192 +62,6 @@ static void ethosu_core_set_capacity(struct ethosu_buffer *buf, cbuf->size = (uint32_t)buf->capacity - buf->offset - buf->size; } -static size_t ethosu_queue_available(struct ethosu_core_queue *queue) -{ - size_t size = queue->header.write - queue->header.read; - - if (queue->header.read > queue->header.write) - size += queue->header.size; - - return size; -} - -static size_t ethosu_queue_capacity(struct ethosu_core_queue *queue) -{ - return max_t(ssize_t, - queue->header.size - ethosu_queue_available(queue) - 1, 0); -} - -static int ethosu_queue_write(struct ethosu_mailbox *mbox, - const struct kvec *vec, - size_t length) -{ - struct ethosu_core_queue *queue = mbox->in_queue; - uint8_t *dst = &queue->data[0]; - uint32_t wpos = queue->header.write; - size_t total_size; - size_t i; - int ret; - - for (i = 0, total_size = 0; i < length; i++) - total_size += vec[i].iov_len; - - if (total_size > ethosu_queue_capacity(queue)) - return -EINVAL; - - for (i = 0; i < length; i++) { - const uint8_t *src = vec[i].iov_base; - const uint8_t *end = src + vec[i].iov_len; - - while (src < end) { - dst[wpos] = *src++; - wpos = (wpos + 1) % queue->header.size; - } - } - - queue->header.write = wpos; - - ret = mbox_send_message(mbox->tx, queue); - if (ret < 0) - return ret; - - return 0; -} - -static int ethosu_queue_write_msg(struct ethosu_mailbox *mbox, - uint32_t type, - void *data, - size_t length) -{ - struct ethosu_core_msg msg = { - .magic = ETHOSU_CORE_MSG_MAGIC, - .type = type, .length= length - }; - const struct kvec vec[2] = { - { &msg, sizeof(msg) }, - { data, length } - }; - int ret; - - ret = ethosu_queue_write(mbox, vec, 2); - if (ret) - return ret; - - return 0; -} - -static int ethosu_queue_read(struct ethosu_mailbox *mbox, - void *data, - size_t length) -{ - struct ethosu_core_queue *queue = mbox->out_queue; - uint8_t *src = &queue->data[0]; - uint8_t *dst = (uint8_t *)data; - const uint8_t *end = dst + length; - uint32_t rpos = queue->header.read; - size_t queue_avail = ethosu_queue_available(queue); - - if (length == 0) - return 0; - else if (queue_avail == 0) - return -ENOMSG; - else if (length > queue_avail) - return -EBADMSG; - - while (dst < end) { - *dst++ = src[rpos]; - rpos = (rpos + 1) % queue->header.size; - } - - queue->header.read = rpos; - - return 0; -} - -void ethosu_mailbox_reset(struct ethosu_mailbox *mbox) -{ - mbox->out_queue->header.read = mbox->out_queue->header.write; -} - -void ethosu_mailbox_wait_prepare(struct ethosu_mailbox *mbox) -{ - mbox->in_queue->header.size = 0; - mbox->in_queue->header.read = 0xffffff; - mbox->in_queue->header.write = 0xffffff; -} - -int ethosu_mailbox_wait_firmware(struct ethosu_mailbox *mbox) -{ - const unsigned long timeout = 1000; - const unsigned long end = jiffies + msecs_to_jiffies(timeout); - volatile struct ethosu_core_queue_header *hdr = - &mbox->in_queue->header; - int ret = -ETIMEDOUT; - - /* Spin wait on mailbox initialization */ - while ((end - jiffies) < timeout) - if (hdr->size != 0 && - hdr->read != 0xffffff && - hdr->write != 0xffffff) { - ret = 0; - break; - } - - dev_info(mbox->dev, "mbox: Wait. ret=%d, size=%u, read=%u, write=%u", - ret, hdr->size, hdr->read, hdr->write); - - return ret; -} - -int ethosu_mailbox_read(struct ethosu_mailbox *mbox, - struct ethosu_core_msg *header, - void *data, - size_t length) -{ - int ret; - - /* Read message header magic */ - ret = ethosu_queue_read(mbox, header, sizeof(*header)); - if (ret) { - if (ret != -ENOMSG) - dev_warn(mbox->dev, - "Msg: Failed to read message header\n"); - - return ret; - } - - if (header->magic != ETHOSU_CORE_MSG_MAGIC) { - dev_warn(mbox->dev, - "Msg: Invalid magic. Got: %08X but expected %08X\n", - header->magic, ETHOSU_CORE_MSG_MAGIC); - - return -EINVAL; - } - - dev_info(mbox->dev, - "mbox: Read msg header. magic=%08X, type=%u, length=%u", - header->magic, header->type, header->length); - - /* Check that payload is not larger than allocated buffer */ - if (header->length > length) { - dev_warn(mbox->dev, - "Msg: Buffer size (%zu) too small for message (%u)\n", - sizeof(data), header->length); - - return -ENOMEM; - } - - /* Read payload data */ - ret = ethosu_queue_read(mbox, data, header->length); - if (ret) { - dev_warn(mbox->dev, "Msg: Failed to read payload data\n"); - - return -EBADMSG; - } - - return 0; -} - int ethosu_mailbox_register(struct ethosu_mailbox *mbox, struct ethosu_mailbox_msg *msg) { @@ -288,30 +102,52 @@ void ethosu_mailbox_fail(struct ethosu_mailbox *mbox) int ethosu_mailbox_ping(struct ethosu_mailbox *mbox) { - return ethosu_queue_write_msg(mbox, ETHOSU_CORE_MSG_PING, NULL, 0); + struct ethosu_core_rpmsg rpmsg = { + .header = { + .magic = ETHOSU_CORE_MSG_MAGIC, + .type = ETHOSU_CORE_MSG_PING, + } + }; + + return rpmsg_send(mbox->ept, &rpmsg, sizeof(rpmsg.header)); } int ethosu_mailbox_pong(struct ethosu_mailbox *mbox) { - return ethosu_queue_write_msg(mbox, ETHOSU_CORE_MSG_PONG, NULL, 0); + struct ethosu_core_rpmsg rpmsg = { + .header = { + .magic = ETHOSU_CORE_MSG_MAGIC, + .type = ETHOSU_CORE_MSG_PONG, + } + }; + + return rpmsg_send(mbox->ept, &rpmsg, sizeof(rpmsg.header)); } int ethosu_mailbox_version_request(struct ethosu_mailbox *mbox) { - return ethosu_queue_write_msg(mbox, ETHOSU_CORE_MSG_VERSION_REQ, NULL, - 0); + struct ethosu_core_rpmsg rpmsg = { + .header = { + .magic = ETHOSU_CORE_MSG_MAGIC, + .type = ETHOSU_CORE_MSG_VERSION_REQ, + } + }; + + return rpmsg_send(mbox->ept, &rpmsg, sizeof(rpmsg.header)); } int ethosu_mailbox_capabilities_request(struct ethosu_mailbox *mbox, struct ethosu_mailbox_msg *msg) { - struct ethosu_core_capabilities_req req = { - .user_arg = msg->id + struct ethosu_core_rpmsg rpmsg = { + .header = { + .magic = ETHOSU_CORE_MSG_MAGIC, + .type = ETHOSU_CORE_MSG_CAPABILITIES_REQ, + .msg_id = msg->id + } }; - return ethosu_queue_write_msg(mbox, ETHOSU_CORE_MSG_CAPABILITIES_REQ, - &req, - sizeof(req)); + return rpmsg_send(mbox->ept, &rpmsg, sizeof(rpmsg.header)); } int ethosu_mailbox_inference(struct ethosu_mailbox *mbox, @@ -326,40 +162,46 @@ int ethosu_mailbox_inference(struct ethosu_mailbox *mbox, uint8_t pmu_event_config_count, uint8_t pmu_cycle_counter_enable) { - struct ethosu_core_inference_req inf; + struct ethosu_core_rpmsg rpmsg = { + .header = { + .magic = ETHOSU_CORE_MSG_MAGIC, + .type = ETHOSU_CORE_MSG_INFERENCE_REQ, + .msg_id = msg->id + } + }; + struct ethosu_core_msg_inference_req *inf_req = &rpmsg.inf_req; uint32_t i; /* Verify that the uapi and core has the same number of pmus */ if (pmu_event_config_count != ETHOSU_CORE_PMU_MAX) { - dev_err(mbox->dev, "PMU count misconfigured.\n"); + dev_err(mbox->dev, "PMU count misconfigured."); return -EINVAL; } - inf.user_arg = msg->id; - inf.ifm_count = ifm_count; - inf.ofm_count = ofm_count; - inf.pmu_cycle_counter_enable = pmu_cycle_counter_enable; + inf_req->ifm_count = ifm_count; + inf_req->ofm_count = ofm_count; + inf_req->pmu_cycle_counter_enable = pmu_cycle_counter_enable; for (i = 0; i < ifm_count; i++) - ethosu_core_set_size(ifm[i], &inf.ifm[i]); + ethosu_core_set_size(ifm[i], &inf_req->ifm[i]); for (i = 0; i < ofm_count; i++) - ethosu_core_set_capacity(ofm[i], &inf.ofm[i]); + ethosu_core_set_capacity(ofm[i], &inf_req->ofm[i]); for (i = 0; i < ETHOSU_CORE_PMU_MAX; i++) - inf.pmu_event_config[i] = pmu_event_config[i]; + inf_req->pmu_event_config[i] = pmu_event_config[i]; if (network != NULL) { - inf.network.type = ETHOSU_CORE_NETWORK_BUFFER; - ethosu_core_set_size(network, &inf.network.buffer); + inf_req->network.type = ETHOSU_CORE_NETWORK_BUFFER; + ethosu_core_set_size(network, &inf_req->network.buffer); } else { - inf.network.type = ETHOSU_CORE_NETWORK_INDEX; - inf.network.index = network_index; + inf_req->network.type = ETHOSU_CORE_NETWORK_INDEX; + inf_req->network.index = network_index; } - return ethosu_queue_write_msg(mbox, ETHOSU_CORE_MSG_INFERENCE_REQ, - &inf, sizeof(inf)); + return rpmsg_send(mbox->ept, &rpmsg, + sizeof(rpmsg.header) + sizeof(rpmsg.inf_req)); } int ethosu_mailbox_network_info_request(struct ethosu_mailbox *mbox, @@ -367,141 +209,57 @@ int ethosu_mailbox_network_info_request(struct ethosu_mailbox *mbox, struct ethosu_buffer *network, uint32_t network_index) { - struct ethosu_core_network_info_req info; - - info.user_arg = msg->id; + struct ethosu_core_rpmsg rpmsg = { + .header = { + .magic = ETHOSU_CORE_MSG_MAGIC, + .type = ETHOSU_CORE_MSG_NETWORK_INFO_REQ, + .msg_id = msg->id + } + }; + struct ethosu_core_msg_network_info_req *info_req = &rpmsg.net_info_req; if (network != NULL) { - info.network.type = ETHOSU_CORE_NETWORK_BUFFER; - ethosu_core_set_size(network, &info.network.buffer); + info_req->network.type = ETHOSU_CORE_NETWORK_BUFFER; + ethosu_core_set_size(network, &info_req->network.buffer); } else { - info.network.type = ETHOSU_CORE_NETWORK_INDEX; - info.network.index = network_index; + info_req->network.type = ETHOSU_CORE_NETWORK_INDEX; + info_req->network.index = network_index; } - return ethosu_queue_write_msg(mbox, ETHOSU_CORE_MSG_NETWORK_INFO_REQ, - &info, sizeof(info)); + return rpmsg_send(mbox->ept, &rpmsg, + sizeof(rpmsg.header) + sizeof(rpmsg.net_info_req)); } int ethosu_mailbox_cancel_inference(struct ethosu_mailbox *mbox, struct ethosu_mailbox_msg *msg, int inference_handle) { - struct ethosu_core_cancel_inference_req req; - - req.user_arg = msg->id; - req.inference_handle = inference_handle; - - return ethosu_queue_write_msg(mbox, - ETHOSU_CORE_MSG_CANCEL_INFERENCE_REQ, - &req, sizeof(req)); -} - -static void ethosu_mailbox_rx_work(struct work_struct *work) -{ - struct ethosu_mailbox *mbox = container_of(work, typeof(*mbox), work); - - mbox->callback(mbox->user_arg); -} - -static void ethosu_mailbox_rx_callback(struct mbox_client *client, - void *message) -{ - struct ethosu_mailbox *mbox = - container_of(client, typeof(*mbox), client); - - dev_info(mbox->dev, "mbox: Received message.\n"); - - queue_work(mbox->wq, &mbox->work); -} + struct ethosu_core_rpmsg rpmsg = { + .header = { + .magic = ETHOSU_CORE_MSG_MAGIC, + .type = + ETHOSU_CORE_MSG_CANCEL_INFERENCE_REQ, + .msg_id = msg->id + }, + .cancel_req = { + .inference_handle = inference_handle + } + }; -static void ethosu_mailbox_tx_done(struct mbox_client *client, - void *message, - int r) -{ - if (r) - dev_warn(client->dev, "mbox: Failed sending message (%d)\n", r); - else - dev_info(client->dev, "mbox: Message sent\n"); + return rpmsg_send(mbox->ept, &rpmsg, + sizeof(rpmsg.header) + sizeof(rpmsg.cancel_req)); } int ethosu_mailbox_init(struct ethosu_mailbox *mbox, struct device *dev, - struct resource *in_queue, - struct resource *out_queue, - ethosu_mailbox_cb callback, - void *user_arg) + struct rpmsg_endpoint *ept) { - int ret; - mbox->dev = dev; - mbox->callback = callback; - mbox->user_arg = user_arg; + mbox->ept = ept; idr_init(&mbox->msg_idr); - mbox->client.dev = dev; - mbox->client.rx_callback = ethosu_mailbox_rx_callback; - mbox->client.tx_prepare = NULL; /* preparation of data is handled - * through the - * queue functions */ - mbox->client.tx_done = ethosu_mailbox_tx_done; - mbox->client.tx_block = true; - mbox->client.knows_txdone = false; - mbox->client.tx_tout = 500; - - mbox->in_queue = devm_ioremap_resource(mbox->dev, in_queue); - if (IS_ERR(mbox->in_queue)) - return PTR_ERR(mbox->in_queue); - - mbox->out_queue = devm_ioremap_resource(mbox->dev, out_queue); - if (IS_ERR(mbox->out_queue)) { - ret = PTR_ERR(mbox->out_queue); - goto unmap_in_queue; - } - - mbox->wq = create_singlethread_workqueue("ethosu_workqueue"); - if (!mbox->wq) { - dev_err(mbox->dev, "Failed to create work queue\n"); - ret = -EINVAL; - goto unmap_out_queue; - } - - INIT_WORK(&mbox->work, ethosu_mailbox_rx_work); - - mbox->tx = mbox_request_channel_byname(&mbox->client, "tx"); - if (IS_ERR(mbox->tx)) { - dev_warn(mbox->dev, "mbox: Failed to request tx channel\n"); - ret = PTR_ERR(mbox->tx); - goto workqueue_destroy; - } - - mbox->rx = mbox_request_channel_byname(&mbox->client, "rx"); - if (IS_ERR(mbox->rx)) { - dev_info(dev, "mbox: Using same channel for RX and TX\n"); - mbox->rx = mbox->tx; - } - return 0; - -workqueue_destroy: - destroy_workqueue(mbox->wq); - -unmap_out_queue: - devm_iounmap(mbox->dev, mbox->out_queue); - -unmap_in_queue: - devm_iounmap(mbox->dev, mbox->in_queue); - - return ret; } void ethosu_mailbox_deinit(struct ethosu_mailbox *mbox) -{ - if (mbox->rx != mbox->tx) - mbox_free_channel(mbox->rx); - - mbox_free_channel(mbox->tx); - destroy_workqueue(mbox->wq); - devm_iounmap(mbox->dev, mbox->out_queue); - devm_iounmap(mbox->dev, mbox->in_queue); -} +{} diff --git a/kernel/ethosu_mailbox.h b/kernel/ethosu_mailbox.h index c3f5579..edf922b 100644 --- a/kernel/ethosu_mailbox.h +++ b/kernel/ethosu_mailbox.h @@ -24,7 +24,7 @@ /**************************************************************************** * Includes ****************************************************************************/ -#include "ethosu_core_interface.h" +#include "ethosu_core_rpmsg.h" #include #include @@ -45,17 +45,9 @@ struct resource; typedef void (*ethosu_mailbox_cb)(void *user_arg); struct ethosu_mailbox { - struct device *dev; - struct workqueue_struct *wq; - struct work_struct work; - struct ethosu_core_queue __iomem *in_queue; - struct ethosu_core_queue __iomem *out_queue; - struct mbox_client client; - struct mbox_chan *rx; - struct mbox_chan *tx; - ethosu_mailbox_cb callback; - void *user_arg; - struct idr msg_idr; + struct device *dev; + struct rpmsg_endpoint *ept; + struct idr msg_idr; }; struct ethosu_mailbox_msg { @@ -74,46 +66,13 @@ struct ethosu_mailbox_msg { */ int ethosu_mailbox_init(struct ethosu_mailbox *mbox, struct device *dev, - struct resource *in_queue, - struct resource *out_queue, - ethosu_mailbox_cb callback, - void *user_arg); + struct rpmsg_endpoint *ept); /** * ethosu_mailbox_deinit() - Deinitialize mailbox */ void ethosu_mailbox_deinit(struct ethosu_mailbox *mbox); -/** - * ethosu_mailbox_wait_prepare() - Prepare to wait on firmware - * - * This function must only be called when the firmware is in a - * stopped state. It invalidates the firmware queues setting - * size, read and write positions to illegal values. - */ -void ethosu_mailbox_wait_prepare(struct ethosu_mailbox *mbox); - -/** - * ethosu_mailbox_wait_firmware() - Waiting for firmware to initialize - * message queues - * - * Following a call to ethosu_mailbox_wait_prepare() this function waits for - * the firmware to boot up and initialize the firmware queues. - * - * Return: 0 on success, else error code. - */ -int ethosu_mailbox_wait_firmware(struct ethosu_mailbox *mbox); - -/** - * ethosu_mailbox_read() - Read message from mailbox - * - * Return: 0 message read, else error code. - */ -int ethosu_mailbox_read(struct ethosu_mailbox *mbox, - struct ethosu_core_msg *header, - void *data, - size_t length); - /** * ethosu_mailbox_register() - Register the ethosu_mailbox_msg in ethosu_mailbox * diff --git a/kernel/ethosu_network_info.c b/kernel/ethosu_network_info.c index 5469b6c..5e7a1b9 100644 --- a/kernel/ethosu_network_info.c +++ b/kernel/ethosu_network_info.c @@ -115,19 +115,19 @@ kfree: } void ethosu_network_info_rsp(struct ethosu_device *edev, - struct ethosu_core_network_info_rsp *rsp) + int msg_id, + struct ethosu_core_msg_network_info_rsp *rsp) { int ret; - int id = (int)rsp->user_arg; struct ethosu_mailbox_msg *msg; struct ethosu_network_info *info; uint32_t i; - msg = ethosu_mailbox_find(&edev->mailbox, id); + msg = ethosu_mailbox_find(&edev->mailbox, msg_id); if (IS_ERR(msg)) { dev_warn(edev->dev, "Id for network info msg not found. msg.id=0x%x\n", - id); + msg_id); return; } diff --git a/kernel/ethosu_network_info.h b/kernel/ethosu_network_info.h index facca54..a809d35 100644 --- a/kernel/ethosu_network_info.h +++ b/kernel/ethosu_network_info.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 ARM Limited. + * Copyright 2022-2023 Arm Limited and/or its affiliates * * This program is free software and is provided to you under the terms of the * GNU General Public License version 2 as published by the Free Software @@ -25,7 +25,6 @@ * Includes ****************************************************************************/ -#include "ethosu_core_interface.h" #include "ethosu_mailbox.h" #include @@ -35,6 +34,7 @@ * Types ****************************************************************************/ +struct ethosu_core_msg_network_info_rsp; struct ethosu_device; struct ethosu_network; struct ethosu_uapi_network_info; @@ -66,6 +66,7 @@ int ethosu_network_info_request(struct ethosu_network *net, * ethosu_network_info_rsp() - Handle network info response. */ void ethosu_network_info_rsp(struct ethosu_device *edev, - struct ethosu_core_network_info_rsp *rsp); + int msg_id, + struct ethosu_core_msg_network_info_rsp *rsp); #endif /* ETHOSU_NETWORK_INFO_H */ -- cgit v1.2.1