From d4ad9e55cb8e17a4a42b3a94c64e6bc48529b26e Mon Sep 17 00:00:00 2001 From: Mikael Olsson Date: Wed, 7 Feb 2024 11:22:26 +0100 Subject: Restructure kernel driver source tree As a first step to have a clearer separation of the different parts of the kernel driver, the source files have been placed into separate directories according to their purpose and the different parts are only allowed to use headers from another part in the include folder. Files have been renamed accordingly to namespace them by their purpose. Change-Id: I75e09ebf0002c99a22b6d4b09d34504d186c32b3 Signed-off-by: Mikael Olsson --- CMakeLists.txt | 5 +- README.md | 31 +- kernel/CMakeLists.txt | 4 +- kernel/Kbuild | 26 +- kernel/common/ethosu_buffer.c | 215 ++++++++ kernel/common/ethosu_device.c | 560 +++++++++++++++++++++ kernel/common/ethosu_dma_mem.c | 77 +++ kernel/common/ethosu_driver.c | 161 ++++++ kernel/ethosu_buffer.c | 215 -------- kernel/ethosu_buffer.h | 85 ---- kernel/ethosu_cancel_inference.c | 191 ------- kernel/ethosu_cancel_inference.h | 72 --- kernel/ethosu_capabilities.c | 164 ------ kernel/ethosu_capabilities.h | 64 --- kernel/ethosu_core_rpmsg.h | 249 --------- kernel/ethosu_device.c | 560 --------------------- kernel/ethosu_device.h | 72 --- kernel/ethosu_dma_mem.c | 77 --- kernel/ethosu_dma_mem.h | 58 --- kernel/ethosu_driver.c | 162 ------ kernel/ethosu_inference.c | 486 ------------------ kernel/ethosu_inference.h | 125 ----- kernel/ethosu_mailbox.c | 362 ------------- kernel/ethosu_mailbox.h | 194 ------- kernel/ethosu_network.c | 255 ---------- kernel/ethosu_network.h | 84 ---- kernel/ethosu_network_info.c | 188 ------- kernel/ethosu_network_info.h | 73 --- kernel/ethosu_version.c | 149 ------ kernel/ethosu_version.h | 60 --- kernel/include/common/ethosu_buffer.h | 85 ++++ kernel/include/common/ethosu_device.h | 71 +++ kernel/include/common/ethosu_dma_mem.h | 58 +++ kernel/include/rpmsg/ethosu_rpmsg.h | 248 +++++++++ .../include/rpmsg/ethosu_rpmsg_cancel_inference.h | 71 +++ kernel/include/rpmsg/ethosu_rpmsg_capabilities.h | 63 +++ kernel/include/rpmsg/ethosu_rpmsg_inference.h | 124 +++++ kernel/include/rpmsg/ethosu_rpmsg_mailbox.h | 194 +++++++ kernel/include/rpmsg/ethosu_rpmsg_network.h | 84 ++++ kernel/include/rpmsg/ethosu_rpmsg_network_info.h | 72 +++ kernel/include/rpmsg/ethosu_rpmsg_version.h | 59 +++ kernel/include/uapi/ethosu.h | 264 ++++++++++ kernel/rpmsg/ethosu_rpmsg_cancel_inference.c | 191 +++++++ kernel/rpmsg/ethosu_rpmsg_capabilities.c | 164 ++++++ kernel/rpmsg/ethosu_rpmsg_inference.c | 486 ++++++++++++++++++ kernel/rpmsg/ethosu_rpmsg_mailbox.c | 363 +++++++++++++ kernel/rpmsg/ethosu_rpmsg_network.c | 255 ++++++++++ kernel/rpmsg/ethosu_rpmsg_network_info.c | 187 +++++++ kernel/rpmsg/ethosu_rpmsg_version.c | 147 ++++++ kernel/uapi/ethosu.h | 264 ---------- 50 files changed, 4238 insertions(+), 4236 deletions(-) create mode 100644 kernel/common/ethosu_buffer.c create mode 100644 kernel/common/ethosu_device.c create mode 100644 kernel/common/ethosu_dma_mem.c create mode 100644 kernel/common/ethosu_driver.c delete mode 100644 kernel/ethosu_buffer.c delete mode 100644 kernel/ethosu_buffer.h delete mode 100644 kernel/ethosu_cancel_inference.c delete mode 100644 kernel/ethosu_cancel_inference.h delete mode 100644 kernel/ethosu_capabilities.c delete mode 100644 kernel/ethosu_capabilities.h delete mode 100644 kernel/ethosu_core_rpmsg.h delete mode 100644 kernel/ethosu_device.c delete mode 100644 kernel/ethosu_device.h delete mode 100644 kernel/ethosu_dma_mem.c delete mode 100644 kernel/ethosu_dma_mem.h delete mode 100644 kernel/ethosu_driver.c delete mode 100644 kernel/ethosu_inference.c delete mode 100644 kernel/ethosu_inference.h delete mode 100644 kernel/ethosu_mailbox.c delete mode 100644 kernel/ethosu_mailbox.h delete mode 100644 kernel/ethosu_network.c delete mode 100644 kernel/ethosu_network.h delete mode 100644 kernel/ethosu_network_info.c delete mode 100644 kernel/ethosu_network_info.h delete mode 100644 kernel/ethosu_version.c delete mode 100644 kernel/ethosu_version.h create mode 100644 kernel/include/common/ethosu_buffer.h create mode 100644 kernel/include/common/ethosu_device.h create mode 100644 kernel/include/common/ethosu_dma_mem.h create mode 100644 kernel/include/rpmsg/ethosu_rpmsg.h create mode 100644 kernel/include/rpmsg/ethosu_rpmsg_cancel_inference.h create mode 100644 kernel/include/rpmsg/ethosu_rpmsg_capabilities.h create mode 100644 kernel/include/rpmsg/ethosu_rpmsg_inference.h create mode 100644 kernel/include/rpmsg/ethosu_rpmsg_mailbox.h create mode 100644 kernel/include/rpmsg/ethosu_rpmsg_network.h create mode 100644 kernel/include/rpmsg/ethosu_rpmsg_network_info.h create mode 100644 kernel/include/rpmsg/ethosu_rpmsg_version.h create mode 100644 kernel/include/uapi/ethosu.h create mode 100644 kernel/rpmsg/ethosu_rpmsg_cancel_inference.c create mode 100644 kernel/rpmsg/ethosu_rpmsg_capabilities.c create mode 100644 kernel/rpmsg/ethosu_rpmsg_inference.c create mode 100644 kernel/rpmsg/ethosu_rpmsg_mailbox.c create mode 100644 kernel/rpmsg/ethosu_rpmsg_network.c create mode 100644 kernel/rpmsg/ethosu_rpmsg_network_info.c create mode 100644 kernel/rpmsg/ethosu_rpmsg_version.c delete mode 100644 kernel/uapi/ethosu.h diff --git a/CMakeLists.txt b/CMakeLists.txt index f2a121d..e7ebc1b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,5 @@ # -# Copyright (c) 2020-2022 Arm Limited. All rights reserved. -# +# SPDX-FileCopyrightText: Copyright 2020-2022, 2024 Arm Limited and/or its affiliates # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the License); you may @@ -39,7 +38,7 @@ string(REGEX REPLACE "(.*)gcc$" "\\1" CROSS_COMPILE "${CMAKE_C_COMPILER}") set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib") # Add include directory -include_directories("kernel") +include_directories("kernel/include") # Add sub directories if(BUILD_KERNEL) diff --git a/README.md b/README.md index 82be485..f28414a 100644 --- a/README.md +++ b/README.md @@ -63,9 +63,19 @@ include the rpmsg virtio message queues, memory resources and trace buffers. The [ethosu_remoteproc.c](remoteproc/ethosu_remoteproc.c) is provided for reference, but could be replaced by any remoteproc driver. -## Ethos-U rpmsg driver +## Ethos-U driver -The [Ethos-U rpmsg driver](kernel/ethosu_driver.c) uses dynamic name resolution, +The purpose of the Ethos-U driver is to enable user space applications to +dispatch inferences to the NPU. The driver presents a +[Userspace API (UAPI)](kernel/include/uapi/ethosu.h), which applications use +IOCTL on to allocate buffers, create networks and dispatch inferences. + +To enable the kernel driver to be extended to support other setups, the source +files for the driver are placed in separate common and rpmsg directories. + +### Rpmsg driver + +The [Ethos-U rpmsg driver](kernel/common/ethosu_driver.c) uses dynamic name resolution, which requires a Linux kernel version of 5.12 or later. This means that when the firmware boots up it calls `rpmsg_create_ept()` to create a dynamic endpoint with a name of maximum 32 characters. The service name must be unique and is @@ -81,16 +91,19 @@ driver will consequently only allocate resources while the firmware is running. The [message handler openamp](https://git.mlplatform.org/ml/ethos-u/ethos-u-core-platform.git/tree/applications/message_handler_openamp) creates a new endpoint named `ethos-u-0.0`, which is serviced by the -[Ethos-U rpmsg driver](kernel/ethosu_driver.c). It is important that firmware +[Ethos-U rpmsg driver](kernel/common/ethosu_driver.c). It is important that firmware and rpmsg driver use the same endpoint name, or else the endpoint will not be successfully created. -The purpose of the Ethos-U rpmsg driver is to enable user space applications to -dispatch inferences to the the Ethos-U subsystem. The driver presents a -[Userspace API (UAPI)](kernel/uapi/ethosu.h), which applications use IOCTL on to -allocate buffers, create networks and dispatch inferences. The types for the -rpmsg communication between Linux and the firmware are defined in -[ethosu_core_rpmsg.h](kernel/ethosu_core_rpmsg.h). +The messages used for the rpmsg communication between Linux and the firmware +are defined in [ethosu_core_rpmsg.h](kernel/include/rpmsg/ethosu_rpmsg.h). + +### Folder structure + +To make a clear separation between the different parts of the driver, the +source files are placed into separate directories according to their purpose +and the different parts are only allowed to use headers from another part +in the include folder. # DTB diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 0192bdc..b5ce944 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright 2020-2023 Arm Limited and/or its affiliates +# SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates # SPDX-License-Identifier: GPL-2.0-only # # This program is free software and is provided to you under the terms of the @@ -41,4 +41,4 @@ add_custom_target(kernel ALL # Install the kernel object and headers install(FILES ethosu.ko DESTINATION "modules") -install(FILES "uapi/ethosu.h" DESTINATION "include/uapi") +install(FILES "include/uapi/ethosu.h" DESTINATION "include/uapi") diff --git a/kernel/Kbuild b/kernel/Kbuild index 98bece3..88ce756 100644 --- a/kernel/Kbuild +++ b/kernel/Kbuild @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright 2020,2022-2023 Arm Limited and/or its affiliates +# SPDX-FileCopyrightText: Copyright 2020, 2022-2024 Arm Limited and/or its affiliates # SPDX-License-Identifier: GPL-2.0-only # # This program is free software and is provided to you under the terms of the @@ -17,18 +17,18 @@ # http://www.gnu.org/licenses/gpl-2.0.html. # -ccflags-y += -Werror +ccflags-y += -Werror -I$(src)/include obj-$(CONFIG_ETHOSU) = ethosu.o -ethosu-objs := ethosu_driver.o \ - ethosu_buffer.o \ - ethosu_dma_mem.o \ - ethosu_device.o \ - ethosu_inference.o \ - ethosu_mailbox.o \ - ethosu_network.o \ - ethosu_network_info.o \ - ethosu_capabilities.o \ - ethosu_cancel_inference.o \ - ethosu_version.o +ethosu-objs := common/ethosu_driver.o \ + common/ethosu_buffer.o \ + common/ethosu_dma_mem.o \ + common/ethosu_device.o \ + rpmsg/ethosu_rpmsg_cancel_inference.o \ + rpmsg/ethosu_rpmsg_network.o \ + rpmsg/ethosu_rpmsg_network_info.o \ + rpmsg/ethosu_rpmsg_capabilities.o \ + rpmsg/ethosu_rpmsg_inference.o \ + rpmsg/ethosu_rpmsg_mailbox.o \ + rpmsg/ethosu_rpmsg_version.o diff --git a/kernel/common/ethosu_buffer.c b/kernel/common/ethosu_buffer.c new file mode 100644 index 0000000..cf41b8d --- /dev/null +++ b/kernel/common/ethosu_buffer.c @@ -0,0 +1,215 @@ +/* + * SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +/**************************************************************************** + * Variables + ****************************************************************************/ + +static int ethosu_buffer_release(struct inode *inode, + struct file *file); + +static int ethosu_buffer_mmap(struct file *file, + struct vm_area_struct *vma); + +static loff_t ethosu_buffer_llseek(struct file *file, + loff_t offset, + int whence); + +static const struct file_operations ethosu_buffer_fops = { + .release = ðosu_buffer_release, + .mmap = ðosu_buffer_mmap, + .llseek = ðosu_buffer_llseek, +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +static bool ethosu_buffer_verify(struct file *file) +{ + return file->f_op == ðosu_buffer_fops; +} + +static void ethosu_buffer_destroy(struct kref *kref) +{ + struct ethosu_buffer *buf = + container_of(kref, struct ethosu_buffer, kref); + struct device *dev = buf->dev; + + dev_dbg(dev, "Buffer destroy. buf=0x%pK", buf); + + ethosu_dma_mem_free(&buf->dma_mem); + + memset(buf, 0, sizeof(*buf)); + devm_kfree(dev, buf); +} + +static int ethosu_buffer_release(struct inode *inode, + struct file *file) +{ + struct ethosu_buffer *buf = file->private_data; + struct device *dev = buf->dev; + + dev_dbg(dev, "Buffer release. file=0x%pK, buf=0x%pK\n", + file, buf); + + ethosu_buffer_put(buf); + + return 0; +} + +static int ethosu_buffer_mmap(struct file *file, + struct vm_area_struct *vma) +{ + struct ethosu_buffer *buf = file->private_data; + struct device *dev = buf->dev; + int ret; + + dev_dbg(dev, "Buffer mmap. file=0x%pK, buf=0x%pK\n", + file, buf); + + ret = dma_mmap_coherent(dev, vma, buf->dma_mem->cpu_addr, + buf->dma_mem->dma_addr, buf->dma_mem->size); + + return ret; +} + +static loff_t ethosu_buffer_llseek(struct file *file, + loff_t offset, + int whence) +{ + struct ethosu_buffer *buf = file->private_data; + + if (offset != 0) + return -EINVAL; + + /* + * SEEK_END and SEEK_SET is supported with a zero offset to allow buffer + * size discovery using seek functions e.g. + * size = lseek(buf_fd, 0, SEEK_END); + * lseek(buf_fd, 0, SEEK_SET); + */ + switch (whence) { + case SEEK_END: + return buf->dma_mem->size; + case SEEK_SET: + return 0; + default: + return -EINVAL; + } +} + +int ethosu_buffer_create(struct device *dev, + size_t size) +{ + struct ethosu_buffer *buf; + int ret = -ENOMEM; + + if (!size) + return -EINVAL; + + buf = devm_kzalloc(dev, sizeof(*buf), GFP_KERNEL); + if (!buf) + return -ENOMEM; + + buf->dev = dev; + kref_init(&buf->kref); + + buf->dma_mem = ethosu_dma_mem_alloc(dev, size); + if (IS_ERR(buf->dma_mem)) { + ret = PTR_ERR(buf->dma_mem); + goto free_buf; + } + + ret = anon_inode_getfd("ethosu-buffer", ðosu_buffer_fops, buf, + O_RDWR | O_CLOEXEC); + if (ret < 0) + goto free_dma; + + buf->file = fget(ret); + buf->file->f_mode |= FMODE_LSEEK; + + fput(buf->file); + + dev_dbg(dev, + "Buffer create. file=0x%pK, fd=%d, buf=0x%pK, size=%zu, cpu_addr=0x%pK, dma_addr=0x%llx, phys_addr=0x%llx\n", + buf->file, ret, buf, size, buf->dma_mem->cpu_addr, + buf->dma_mem->dma_addr, virt_to_phys(buf->dma_mem->cpu_addr)); + + return ret; + +free_dma: + ethosu_dma_mem_free(&buf->dma_mem); + +free_buf: + memset(buf, 0, sizeof(*buf)); + devm_kfree(dev, buf); + + return ret; +} + +struct ethosu_buffer *ethosu_buffer_get_from_fd(int fd) +{ + struct ethosu_buffer *buf; + struct file *file; + + file = fget(fd); + if (!file) + return ERR_PTR(-EINVAL); + + if (!ethosu_buffer_verify(file)) { + fput(file); + + return ERR_PTR(-EINVAL); + } + + buf = file->private_data; + ethosu_buffer_get(buf); + fput(file); + + return buf; +} + +void ethosu_buffer_get(struct ethosu_buffer *buf) +{ + kref_get(&buf->kref); +} + +void ethosu_buffer_put(struct ethosu_buffer *buf) +{ + kref_put(&buf->kref, ethosu_buffer_destroy); +} diff --git a/kernel/common/ethosu_device.c b/kernel/common/ethosu_device.c new file mode 100644 index 0000000..f1e00ab --- /dev/null +++ b/kernel/common/ethosu_device.c @@ -0,0 +1,560 @@ +/* + * SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/**************************************************************************** + * Defines + ****************************************************************************/ + +#define MINOR_BASE 0 /* Minor version starts at 0 */ +#define MINOR_COUNT 64 /* Allocate minor versions */ + +/**************************************************************************** + * Variables + ****************************************************************************/ + +static DECLARE_BITMAP(minors, MINOR_COUNT); + +/**************************************************************************** + * Functions + ****************************************************************************/ + +/* Incoming messages */ +static int ethosu_handle_rpmsg(struct rpmsg_device *rpdev, + void *data, + int len, + void *priv, + u32 src) +{ + struct ethosu_device *edev = dev_get_drvdata(&rpdev->dev); + struct device *dev = &edev->dev; + struct ethosu_mailbox *mbox = &edev->mailbox; + struct ethosu_core_rpmsg *rpmsg = data; + int length = len - sizeof(rpmsg->header); + int ret = 0; + + if (unlikely(rpmsg->header.magic != ETHOSU_CORE_MSG_MAGIC)) { + dev_warn(dev, "Msg: Error invalid message magic. magic=0x%08x", + rpmsg->header.magic); + + return -EBADMSG; + } + + device_lock(dev); + + dev_dbg(dev, + "Msg: magic=0x%08x, type=%u, msg_id=%llu", + rpmsg->header.magic, rpmsg->header.type, rpmsg->header.msg_id); + + switch (rpmsg->header.type) { + case ETHOSU_CORE_MSG_ERR: + if (length != sizeof(rpmsg->error)) { + dev_warn(dev, + "Msg: Error message of incorrect size. size=%u, expected=%zu", length, + sizeof(rpmsg->error)); + ret = -EBADMSG; + break; + } + + rpmsg->error.msg[sizeof(rpmsg->error.msg) - 1] = '\0'; + dev_warn(dev, "Msg: Error. type=%u, msg=\"%s\"", + rpmsg->error.type, rpmsg->error.msg); + + rproc_report_crash(rproc_get_by_child(dev), RPROC_FATAL_ERROR); + break; + case ETHOSU_CORE_MSG_PING: + dev_dbg(dev, "Msg: Ping"); + ret = ethosu_mailbox_pong(mbox); + break; + case ETHOSU_CORE_MSG_PONG: + dev_dbg(dev, "Msg: Pong"); + break; + case ETHOSU_CORE_MSG_INFERENCE_RSP: + if (length != sizeof(rpmsg->inf_rsp)) { + dev_warn(dev, + "Msg: Inference response of incorrect size. size=%u, expected=%zu", length, + sizeof(rpmsg->inf_rsp)); + ret = -EBADMSG; + break; + } + + dev_dbg(dev, + "Msg: Inference response. ofm_count=%u, status=%u", + rpmsg->inf_rsp.ofm_count, rpmsg->inf_rsp.status); + + ethosu_inference_rsp(mbox, rpmsg->header.msg_id, + &rpmsg->inf_rsp); + break; + case ETHOSU_CORE_MSG_CANCEL_INFERENCE_RSP: + if (length != sizeof(rpmsg->cancel_rsp)) { + dev_warn(dev, + "Msg: Cancel Inference response of incorrect size. size=%u, expected=%zu", length, + sizeof(rpmsg->cancel_rsp)); + ret = -EBADMSG; + break; + } + + dev_dbg(dev, + "Msg: Cancel Inference response. status=%u", + rpmsg->cancel_rsp.status); + ethosu_cancel_inference_rsp(mbox, + rpmsg->header.msg_id, + &rpmsg->cancel_rsp); + break; + case ETHOSU_CORE_MSG_VERSION_RSP: + if (length != sizeof(rpmsg->version_rsp)) { + dev_warn(dev, + "Msg: Protocol version response of incorrect size. size=%u, expected=%zu", length, + sizeof(rpmsg->version_rsp)); + ret = -EBADMSG; + break; + } + + dev_dbg(dev, "Msg: Protocol version response %u.%u.%u", + rpmsg->version_rsp.major, rpmsg->version_rsp.minor, + rpmsg->version_rsp.patch); + + ethosu_version_rsp(mbox, rpmsg->header.msg_id, + &rpmsg->version_rsp); + break; + case ETHOSU_CORE_MSG_CAPABILITIES_RSP: + if (length != sizeof(rpmsg->cap_rsp)) { + dev_warn(dev, + "Msg: Capabilities response of incorrect size. size=%u, expected=%zu", length, + sizeof(rpmsg->cap_rsp)); + ret = -EBADMSG; + break; + } + + dev_dbg(dev, + "Msg: Capabilities response vs%hhu v%hhu.%hhu p%hhu av%hhu.%hhu.%hhu dv%hhu.%hhu.%hhu mcc%hhu csv%hhu cd%hhu", + rpmsg->cap_rsp.version_status, + rpmsg->cap_rsp.version_major, + rpmsg->cap_rsp.version_minor, + rpmsg->cap_rsp.product_major, + rpmsg->cap_rsp.arch_major_rev, + rpmsg->cap_rsp.arch_minor_rev, + rpmsg->cap_rsp.arch_patch_rev, + rpmsg->cap_rsp.driver_major_rev, + rpmsg->cap_rsp.driver_minor_rev, + rpmsg->cap_rsp.driver_patch_rev, + rpmsg->cap_rsp.macs_per_cc, + rpmsg->cap_rsp.cmd_stream_version, + rpmsg->cap_rsp.custom_dma); + + ethosu_capability_rsp(mbox, rpmsg->header.msg_id, + &rpmsg->cap_rsp); + break; + case ETHOSU_CORE_MSG_NETWORK_INFO_RSP: + if (length != sizeof(rpmsg->net_info_rsp)) { + dev_warn(dev, + "Msg: Network info response of incorrect size. size=%u, expected=%zu", length, + sizeof(rpmsg->net_info_rsp)); + ret = -EBADMSG; + break; + } + + dev_dbg(dev, + "Msg: Network info response. status=%u", + rpmsg->net_info_rsp.status); + + ethosu_network_info_rsp(mbox, + rpmsg->header.msg_id, + &rpmsg->net_info_rsp); + + break; + default: + /* This should not happen due to version checks */ + dev_warn(dev, "Msg: Protocol error. type=%u", + rpmsg->header.type); + ret = -EPROTO; + break; + } + + device_unlock(dev); + + wake_up(&mbox->send_queue); + + return ret; +} + +static int ethosu_open(struct inode *inode, + struct file *file) +{ + struct cdev *cdev = inode->i_cdev; + struct ethosu_device *edev = container_of(cdev, struct ethosu_device, + cdev); + struct rpmsg_device *rpdev = edev->rpdev; + struct device *dev = &edev->dev; + + dev_dbg(dev, "Device open. file=0x%pK", file); + + file->private_data = rpdev; + + return nonseekable_open(inode, file); +} + +static long ethosu_ioctl(struct file *file, + unsigned int cmd, + unsigned long arg) +{ + struct rpmsg_device *rpdev = file->private_data; + struct ethosu_device *edev = dev_get_drvdata(&rpdev->dev); + struct device *dev = &edev->dev; + void __user *udata = (void __user *)arg; + int ret; + + switch (cmd) { + case ETHOSU_IOCTL_DRIVER_VERSION_GET: { + const struct ethosu_uapi_kernel_driver_version version = { + .major = ETHOSU_KERNEL_DRIVER_VERSION_MAJOR, + .minor = ETHOSU_KERNEL_DRIVER_VERSION_MINOR, + .patch = ETHOSU_KERNEL_DRIVER_VERSION_PATCH, + }; + + ret = copy_to_user(udata, &version, + sizeof(version)) ? -EFAULT : 0; + break; + } + case ETHOSU_IOCTL_CAPABILITIES_REQ: { + dev_dbg(dev, "Device ioctl: Capabilities request"); + + ret = copy_to_user(udata, &edev->capabilities, + sizeof(edev->capabilities)) ? -EFAULT : 0; + break; + } + case ETHOSU_IOCTL_PING: { + ret = device_lock_interruptible(dev); + if (ret) + return ret; + + dev_dbg(dev, "Device ioctl: Send ping"); + + ret = ethosu_mailbox_ping(&edev->mailbox); + + device_unlock(dev); + + break; + } + case ETHOSU_IOCTL_BUFFER_CREATE: { + struct ethosu_uapi_buffer_create uapi; + + if (copy_from_user(&uapi, udata, sizeof(uapi))) { + ret = -EFAULT; + break; + } + + ret = device_lock_interruptible(dev); + if (ret) + return ret; + + dev_dbg(dev, + "Device ioctl: Buffer create. size=%u", + uapi.size); + + ret = ethosu_buffer_create(dev, uapi.size); + + device_unlock(dev); + + break; + } + case ETHOSU_IOCTL_NETWORK_CREATE: { + struct ethosu_uapi_network_create uapi; + + if (copy_from_user(&uapi, udata, sizeof(uapi))) { + ret = -EFAULT; + break; + } + + ret = device_lock_interruptible(dev); + if (ret) + return ret; + + dev_dbg(dev, + "Device ioctl: Network create. type=%u\n", uapi.type); + + ret = ethosu_network_create(dev, &edev->mailbox, &uapi); + + device_unlock(dev); + + break; + } + default: { + dev_err(dev, "Invalid ioctl. cmd=%u, arg=%lu", + cmd, arg); + ret = -ENOIOCTLCMD; + break; + } + } + + return ret; +} + +static struct rpmsg_endpoint *ethosu_create_ept(struct rpmsg_device *rpdev) +{ + struct device *dev = &rpdev->dev; + struct rpmsg_channel_info info = { 0 }; + struct rpmsg_endpoint *ept; + + /* Create rpmsg endpoint */ + strncpy(info.name, rpdev->id.name, sizeof(info.name) - 1); + info.src = 0; + info.dst = rpdev->dst; + + dev_dbg(dev, "Creating rpmsg endpoint. name=%s, src=%u, dst=%u", + info.name, info.src, info.dst); + + ept = rpmsg_create_ept(rpdev, ethosu_handle_rpmsg, NULL, info); + if (!ept) { + dev_err(&rpdev->dev, "Failed to create endpoint"); + + return ERR_PTR(-EINVAL); + } + + return ept; +} + +static const struct file_operations fops = { + .owner = THIS_MODULE, + .open = ðosu_open, + .unlocked_ioctl = ðosu_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ðosu_ioctl, +#endif +}; + +static void ethosu_dev_release(struct device *dev) +{ + struct ethosu_device *edev = dev_get_drvdata(dev); + + clear_bit(MINOR(edev->cdev.dev), minors); + + ethosu_mailbox_deinit(&edev->mailbox); + device_destroy(edev->class, edev->cdev.dev); + kfree(edev); +} + +static int ethosu_device_register(struct device *dev, + struct device *parent, + void *drvdata, + dev_t devt) +{ + struct rproc *rproc = rproc_get_by_child(parent); + int ret; + + dev->parent = parent; + dev->release = ethosu_dev_release; + dev_set_drvdata(dev, drvdata); + + ret = dev_set_name(dev, "ethosu%d", MINOR(devt)); + if (ret) { + dev_err(parent, "Failed to set device name. ret=%d", ret); + + return ret; + } + + /* Inherit DMA mask from rproc device */ + ret = dma_coerce_mask_and_coherent(dev, + dma_get_mask(rproc->dev.parent)); + if (ret) { + dev_err(parent, "Failed to set DMA mask. ret=%d", ret); + + return ret; + } + + /* Inherit DMA configuration from rproc device */ + ret = of_dma_configure(dev, rproc->dev.parent->of_node, false); + if (ret) { + dev_err(parent, "Failed to configure DMA. ret=%d", + ret); + + return ret; + } + + /* Inherit reserved memory from rproc device */ + ret = of_reserved_mem_device_init_by_idx(dev, + rproc->dev.parent->of_node, 0); + if (ret) { + dev_err(parent, "Failed to initialize reserved memory. ret=%d", + ret); + + return ret; + } + + ret = device_register(dev); + if (ret) { + dev_err(parent, "Failed to register device. ret=%d", ret); + + return ret; + } + + return 0; +} + +int ethosu_dev_init(struct rpmsg_device *rpdev, + struct class *class, + dev_t devt) +{ + struct device *dev = &rpdev->dev; + struct ethosu_device *edev; + struct device *sysdev; + int minor; + int ret; + + /* Reserve minor number for device node */ + minor = find_first_zero_bit(minors, MINOR_COUNT); + if (minor >= MINOR_COUNT) { + dev_err(dev, "No more minor numbers."); + + return -ENOMEM; + } + + devt = MKDEV(MAJOR(devt), minor); + + /* Allocate and create Ethos-U device */ + edev = kzalloc(sizeof(*edev), GFP_KERNEL); + if (!edev) + return -ENOMEM; + + dev_set_drvdata(&rpdev->dev, edev); + + edev->rpdev = rpdev; + edev->class = class; + + /* Create device object */ + ret = ethosu_device_register(&edev->dev, &rpdev->dev, edev, + devt); + if (ret) { + kfree(edev); + + return ret; + } + + /* Continue with new device */ + dev = &edev->dev; + + /* Create RPMsg endpoint */ + edev->ept = ethosu_create_ept(rpdev); + if (IS_ERR(edev->ept)) { + ret = PTR_ERR(edev->ept); + goto device_unregister; + } + + ret = ethosu_mailbox_init(&edev->mailbox, dev, edev->ept); + if (ret) + goto free_rpmsg_ept; + + device_lock(dev); + ret = ethosu_version_check_request(dev, &edev->mailbox); + device_unlock(dev); + if (ret) { + dev_err(dev, "Protocol version check failed: %d", ret); + goto deinit_mailbox; + } + + device_lock(dev); + ret = ethosu_capabilities_request(dev, &edev->mailbox, + &edev->capabilities); + device_unlock(dev); + if (ret) { + dev_err(dev, "Failed to get device capabilities: %d", ret); + goto deinit_mailbox; + } + + /* Create device node */ + cdev_init(&edev->cdev, &fops); + edev->cdev.owner = THIS_MODULE; + + cdev_set_parent(&edev->cdev, &dev->kobj); + + ret = cdev_add(&edev->cdev, devt, 1); + if (ret) { + dev_err(dev, "Failed to add character device."); + goto deinit_mailbox; + } + + sysdev = device_create(edev->class, NULL, devt, rpdev, + "ethosu%d", MINOR(devt)); + if (IS_ERR(sysdev)) { + dev_err(dev, "Failed to create device."); + ret = PTR_ERR(sysdev); + goto del_cdev; + } + + set_bit(minor, minors); + + dev_info(dev, + "Created Arm Ethos-U device. name=%s, major=%d, minor=%d", + dev_name(sysdev), MAJOR(devt), MINOR(devt)); + + return 0; + +del_cdev: + cdev_del(&edev->cdev); + +deinit_mailbox: + ethosu_mailbox_deinit(&edev->mailbox); + +free_rpmsg_ept: + rpmsg_destroy_ept(edev->ept); + +device_unregister: + device_unregister(dev); + + return ret; +} + +void ethosu_dev_deinit(struct rpmsg_device *rpdev) +{ + struct device *dev = &rpdev->dev; + struct ethosu_device *edev = dev_get_drvdata(dev); + + device_lock(&edev->dev); + ethosu_mailbox_fail(&edev->mailbox); + device_unlock(&edev->dev); + + rpmsg_destroy_ept(edev->ept); + cdev_del(&edev->cdev); + device_unregister(&edev->dev); +} diff --git a/kernel/common/ethosu_dma_mem.c b/kernel/common/ethosu_dma_mem.c new file mode 100644 index 0000000..0e05978 --- /dev/null +++ b/kernel/common/ethosu_dma_mem.c @@ -0,0 +1,77 @@ +/* + * SPDX-FileCopyrightText: Copyright 2023-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include + +#include +#include + +/**************************************************************************** + * Functions + ****************************************************************************/ + +struct ethosu_dma_mem *ethosu_dma_mem_alloc(struct device *dev, + size_t size) +{ + struct ethosu_dma_mem *dma_mem; + + if (!size) + return ERR_PTR(-EINVAL); + + dma_mem = devm_kzalloc(dev, sizeof(*dma_mem), GFP_KERNEL); + if (!dma_mem) + return ERR_PTR(-ENOMEM); + + dma_mem->dev = dev; + dma_mem->size = size; + dma_mem->cpu_addr = dma_alloc_coherent(dev, size, &dma_mem->dma_addr, + GFP_KERNEL); + if (!dma_mem->cpu_addr) { + memset(dma_mem, 0, sizeof(*dma_mem)); + devm_kfree(dev, dma_mem); + + return ERR_PTR(-ENOMEM); + } + + return dma_mem; +} + +void ethosu_dma_mem_free(struct ethosu_dma_mem **dma_mem) +{ + struct device *dev; + struct ethosu_dma_mem *mem; + + if (!dma_mem || !*dma_mem) + return; + + mem = *dma_mem; + dev = mem->dev; + + memset(mem->cpu_addr, 0, mem->size); + dma_free_coherent(dev, mem->size, mem->cpu_addr, mem->dma_addr); + + memset(mem, 0, sizeof(*mem)); + devm_kfree(dev, mem); + + *dma_mem = NULL; +} diff --git a/kernel/common/ethosu_driver.c b/kernel/common/ethosu_driver.c new file mode 100644 index 0000000..15f13b5 --- /dev/null +++ b/kernel/common/ethosu_driver.c @@ -0,0 +1,161 @@ +/* + * SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +/**************************************************************************** + * Defines + ****************************************************************************/ + +#define ETHOSU_DRIVER_STR(s) #s +#define ETHOSU_DRIVER_VERSION_STR(major, minor, patch) \ + ETHOSU_DRIVER_STR(major) "." \ + ETHOSU_DRIVER_STR(minor) "." \ + ETHOSU_DRIVER_STR(patch) +#define ETHOSU_DRIVER_VERSION ETHOSU_DRIVER_VERSION_STR( \ + ETHOSU_KERNEL_DRIVER_VERSION_MAJOR, \ + ETHOSU_KERNEL_DRIVER_VERSION_MINOR, \ + ETHOSU_KERNEL_DRIVER_VERSION_PATCH) + +#define ETHOSU_DRIVER_NAME "ethosu" + +#define MINOR_BASE 0 /* Minor version starts at 0 */ +#define MINOR_COUNT 64 /* Allocate minor versions */ + +/**************************************************************************** + * Variables + ****************************************************************************/ + +static struct class *ethosu_class; + +static dev_t devt; + +/**************************************************************************** + * Rpmsg driver + ****************************************************************************/ + +static int ethosu_rpmsg_probe(struct rpmsg_device *rpdev) +{ + int ret; + + /* Initialize device */ + ret = ethosu_dev_init(rpdev, ethosu_class, devt); + if (ret) + return ret; + + return 0; +} + +static void ethosu_rpmsg_remove(struct rpmsg_device *rpdev) +{ + ethosu_dev_deinit(rpdev); +} + +static int ethosu_rpmsg_cb(struct rpmsg_device *rpdev, + void *data, + int len, + void *priv, + u32 src) +{ + dev_err(&rpdev->dev, "%s", __FUNCTION__); + + return -EINVAL; +} + +static struct rpmsg_device_id ethosu_rpmsg_driver_id_table[] = { + { .name = "ethos-u-0.0" }, + {}, +}; + +MODULE_DEVICE_TABLE(rpmsg, ethosu_rpmsg_driver_id_table); + +static struct rpmsg_driver ethosu_rpmsg_driver = { + .drv = { + .name = ETHOSU_DRIVER_NAME, + .owner = THIS_MODULE, + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, + .id_table = ethosu_rpmsg_driver_id_table, + .probe = ethosu_rpmsg_probe, + .callback = ethosu_rpmsg_cb, + .remove = ethosu_rpmsg_remove, +}; + +/**************************************************************************** + * Module init and exit + ****************************************************************************/ + +static void __exit ethosu_exit(void) +{ + unregister_rpmsg_driver(ðosu_rpmsg_driver); + unregister_chrdev_region(devt, MINOR_COUNT); + class_destroy(ethosu_class); +} + +static int __init ethosu_init(void) +{ + int ret; + + ethosu_class = class_create(THIS_MODULE, ETHOSU_DRIVER_NAME); + if (IS_ERR(ethosu_class)) { + pr_err("Failed to create class '%s'.\n", ETHOSU_DRIVER_NAME); + + return PTR_ERR(ethosu_class); + } + + ret = alloc_chrdev_region(&devt, MINOR_BASE, MINOR_COUNT, + ETHOSU_DRIVER_NAME); + if (ret) { + pr_err("Failed to allocate chrdev region.\n"); + goto destroy_class; + } + + ret = register_rpmsg_driver(ðosu_rpmsg_driver); + if (ret) { + pr_err("Failed to register Arm Ethos-U rpmsg driver.\n"); + goto region_unregister; + } + + return 0; + +region_unregister: + unregister_chrdev_region(devt, MINOR_COUNT); + +destroy_class: + class_destroy(ethosu_class); + + return ret; +} + +module_init(ethosu_init) +module_exit(ethosu_exit) + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Arm Ltd"); +MODULE_DESCRIPTION("Arm Ethos-U NPU Driver"); +MODULE_VERSION(ETHOSU_DRIVER_VERSION); diff --git a/kernel/ethosu_buffer.c b/kernel/ethosu_buffer.c deleted file mode 100644 index 69b5007..0000000 --- a/kernel/ethosu_buffer.c +++ /dev/null @@ -1,215 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright 2020-2023 Arm Limited and/or its affiliates - * SPDX-License-Identifier: GPL-2.0-only - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - */ - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_buffer.h" - -#include "ethosu_device.h" -#include "ethosu_dma_mem.h" -#include "uapi/ethosu.h" - -#include -#include -#include -#include -#include -#include -#include - -/**************************************************************************** - * Variables - ****************************************************************************/ - -static int ethosu_buffer_release(struct inode *inode, - struct file *file); - -static int ethosu_buffer_mmap(struct file *file, - struct vm_area_struct *vma); - -static loff_t ethosu_buffer_llseek(struct file *file, - loff_t offset, - int whence); - -static const struct file_operations ethosu_buffer_fops = { - .release = ðosu_buffer_release, - .mmap = ðosu_buffer_mmap, - .llseek = ðosu_buffer_llseek, -}; - -/**************************************************************************** - * Functions - ****************************************************************************/ - -static bool ethosu_buffer_verify(struct file *file) -{ - return file->f_op == ðosu_buffer_fops; -} - -static void ethosu_buffer_destroy(struct kref *kref) -{ - struct ethosu_buffer *buf = - container_of(kref, struct ethosu_buffer, kref); - struct device *dev = buf->dev; - - dev_dbg(dev, "Buffer destroy. buf=0x%pK", buf); - - ethosu_dma_mem_free(&buf->dma_mem); - - memset(buf, 0, sizeof(*buf)); - devm_kfree(dev, buf); -} - -static int ethosu_buffer_release(struct inode *inode, - struct file *file) -{ - struct ethosu_buffer *buf = file->private_data; - struct device *dev = buf->dev; - - dev_dbg(dev, "Buffer release. file=0x%pK, buf=0x%pK\n", - file, buf); - - ethosu_buffer_put(buf); - - return 0; -} - -static int ethosu_buffer_mmap(struct file *file, - struct vm_area_struct *vma) -{ - struct ethosu_buffer *buf = file->private_data; - struct device *dev = buf->dev; - int ret; - - dev_dbg(dev, "Buffer mmap. file=0x%pK, buf=0x%pK\n", - file, buf); - - ret = dma_mmap_coherent(dev, vma, buf->dma_mem->cpu_addr, - buf->dma_mem->dma_addr, buf->dma_mem->size); - - return ret; -} - -static loff_t ethosu_buffer_llseek(struct file *file, - loff_t offset, - int whence) -{ - struct ethosu_buffer *buf = file->private_data; - - if (offset != 0) - return -EINVAL; - - /* - * SEEK_END and SEEK_SET is supported with a zero offset to allow buffer - * size discovery using seek functions e.g. - * size = lseek(buf_fd, 0, SEEK_END); - * lseek(buf_fd, 0, SEEK_SET); - */ - switch (whence) { - case SEEK_END: - return buf->dma_mem->size; - case SEEK_SET: - return 0; - default: - return -EINVAL; - } -} - -int ethosu_buffer_create(struct device *dev, - size_t size) -{ - struct ethosu_buffer *buf; - int ret = -ENOMEM; - - if (!size) - return -EINVAL; - - buf = devm_kzalloc(dev, sizeof(*buf), GFP_KERNEL); - if (!buf) - return -ENOMEM; - - buf->dev = dev; - kref_init(&buf->kref); - - buf->dma_mem = ethosu_dma_mem_alloc(dev, size); - if (IS_ERR(buf->dma_mem)) { - ret = PTR_ERR(buf->dma_mem); - goto free_buf; - } - - ret = anon_inode_getfd("ethosu-buffer", ðosu_buffer_fops, buf, - O_RDWR | O_CLOEXEC); - if (ret < 0) - goto free_dma; - - buf->file = fget(ret); - buf->file->f_mode |= FMODE_LSEEK; - - fput(buf->file); - - dev_dbg(dev, - "Buffer create. file=0x%pK, fd=%d, buf=0x%pK, size=%zu, cpu_addr=0x%pK, dma_addr=0x%llx, phys_addr=0x%llx\n", - buf->file, ret, buf, size, buf->dma_mem->cpu_addr, - buf->dma_mem->dma_addr, virt_to_phys(buf->dma_mem->cpu_addr)); - - return ret; - -free_dma: - ethosu_dma_mem_free(&buf->dma_mem); - -free_buf: - memset(buf, 0, sizeof(*buf)); - devm_kfree(dev, buf); - - return ret; -} - -struct ethosu_buffer *ethosu_buffer_get_from_fd(int fd) -{ - struct ethosu_buffer *buf; - struct file *file; - - file = fget(fd); - if (!file) - return ERR_PTR(-EINVAL); - - if (!ethosu_buffer_verify(file)) { - fput(file); - - return ERR_PTR(-EINVAL); - } - - buf = file->private_data; - ethosu_buffer_get(buf); - fput(file); - - return buf; -} - -void ethosu_buffer_get(struct ethosu_buffer *buf) -{ - kref_get(&buf->kref); -} - -void ethosu_buffer_put(struct ethosu_buffer *buf) -{ - kref_put(&buf->kref, ethosu_buffer_destroy); -} diff --git a/kernel/ethosu_buffer.h b/kernel/ethosu_buffer.h deleted file mode 100644 index 8bef2d2..0000000 --- a/kernel/ethosu_buffer.h +++ /dev/null @@ -1,85 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright 2020,2022-2023 Arm Limited and/or its affiliates - * SPDX-License-Identifier: GPL-2.0-only - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - */ - -#ifndef ETHOSU_BUFFER_H -#define ETHOSU_BUFFER_H - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include -#include - -/**************************************************************************** - * Types - ****************************************************************************/ - -struct ethosu_dma_mem; -struct ethosu_device; -struct device; - -/** - * struct ethosu_buffer - User data buffer - * @dev: Device - * @file: File - * @kref: Reference counting - * @dma_mem: DMA memory allocated for the buffer - */ -struct ethosu_buffer { - struct device *dev; - struct file *file; - struct kref kref; - struct ethosu_dma_mem *dma_mem; -}; - -/**************************************************************************** - * Functions - ****************************************************************************/ - -/** - * ethosu_buffer_create() - Create buffer - * - * This function must be called in the context of a user space process. - * - * Return: fd on success, else error code. - */ -int ethosu_buffer_create(struct device *dev, - size_t size); - -/** - * ethosu_buffer_get_from_fd() - Get buffer handle from fd - * - * This function must be called from a user space context. - * - * Return: Pointer on success, else ERR_PTR. - */ -struct ethosu_buffer *ethosu_buffer_get_from_fd(int fd); - -/** - * ethosu_buffer_get() - Put buffer - */ -void ethosu_buffer_get(struct ethosu_buffer *buf); - -/** - * ethosu_buffer_put() - Put buffer - */ -void ethosu_buffer_put(struct ethosu_buffer *buf); - -#endif /* ETHOSU_BUFFER_H */ diff --git a/kernel/ethosu_cancel_inference.c b/kernel/ethosu_cancel_inference.c deleted file mode 100644 index 47687b8..0000000 --- a/kernel/ethosu_cancel_inference.c +++ /dev/null @@ -1,191 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright 2022-2024 Arm Limited and/or its affiliates - * SPDX-License-Identifier: GPL-2.0-only - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_cancel_inference.h" - -#include "ethosu_core_rpmsg.h" -#include "ethosu_device.h" -#include "ethosu_inference.h" - -#include -#include - -/**************************************************************************** - * Defines - ****************************************************************************/ - -#define CANCEL_INFERENCE_RESP_TIMEOUT_MS 2000 - -/**************************************************************************** - * Functions - ****************************************************************************/ - -static int ethosu_cancel_inference_send( - struct ethosu_cancel_inference *cancellation, - struct ethosu_mailbox *mailbox) -{ - return ethosu_mailbox_cancel_inference(mailbox, - &cancellation->msg, - cancellation->inf->msg.id); -} - -static void ethosu_cancel_inference_fail(struct ethosu_mailbox_msg *msg) -{ - struct ethosu_cancel_inference *cancellation = - container_of(msg, typeof(*cancellation), msg); - - if (completion_done(&cancellation->done)) - return; - - cancellation->errno = -EFAULT; - cancellation->uapi->status = ETHOSU_UAPI_STATUS_ERROR; - complete(&cancellation->done); -} - -int ethosu_cancel_inference_request(struct device *dev, - struct ethosu_mailbox *mailbox, - struct ethosu_inference *inf, - struct ethosu_uapi_cancel_inference_status *uapi) -{ - struct ethosu_cancel_inference *cancellation; - int ret; - int timeout; - - if (inf->done) { - uapi->status = ETHOSU_UAPI_STATUS_ERROR; - - return 0; - } - - cancellation = - devm_kzalloc(dev, - sizeof(struct ethosu_cancel_inference), - GFP_KERNEL); - if (!cancellation) - return -ENOMEM; - - /* increase ref count on the inference we are refering to */ - ethosu_inference_get(inf); - /* mark inference ABORTING to avoid resending the inference message */ - inf->status = ETHOSU_UAPI_STATUS_ABORTING; - - cancellation->dev = dev; - cancellation->inf = inf; - cancellation->uapi = uapi; - init_completion(&cancellation->done); - cancellation->msg.fail = ethosu_cancel_inference_fail; - - ret = ethosu_mailbox_register(mailbox, - &cancellation->msg); - if (ret < 0) - goto kfree; - - dev_dbg(dev, - "Inference cancellation create. cancel=0x%pK, msg.id=%ddev", - cancellation, cancellation->msg.id); - - ret = ethosu_cancel_inference_send(cancellation, mailbox); - if (0 != ret) - goto deregister; - - /* Unlock the mutex before going to block on the condition */ - device_unlock(dev); - - /* wait for response to arrive back */ - timeout = wait_for_completion_timeout(&cancellation->done, - msecs_to_jiffies( - CANCEL_INFERENCE_RESP_TIMEOUT_MS)); - /* take back the mutex before resuming to do anything */ - ret = device_lock_interruptible(dev); - if (0 != ret) - goto deregister; - - if (0 == timeout /* timed out*/) { - dev_warn(dev, - "Msg: Cancel Inference response lost - timeoutdev"); - ret = -EIO; - - rproc_report_crash(rproc_get_by_child(dev), RPROC_FATAL_ERROR); - goto deregister; - } - - if (cancellation->errno) { - ret = cancellation->errno; - rproc_report_crash(rproc_get_by_child(dev), RPROC_FATAL_ERROR); - goto deregister; - } - - if (inf->status != ETHOSU_UAPI_STATUS_ABORTED) - inf->status = ETHOSU_UAPI_STATUS_ABORTED; - -deregister: - ethosu_mailbox_deregister(mailbox, - &cancellation->msg); - -kfree: - dev_dbg(dev, - "Cancel inference destroy. cancel=0x%pK", cancellation); - - /* decrease the reference on the inference we are refering to */ - ethosu_inference_put(cancellation->inf); - devm_kfree(dev, cancellation); - - return ret; -} - -void ethosu_cancel_inference_rsp(struct ethosu_mailbox *mailbox, - int msg_id, - struct ethosu_core_msg_cancel_inference_rsp *rsp) -{ - struct device *dev = mailbox->dev; - struct ethosu_mailbox_msg *msg; - struct ethosu_cancel_inference *cancellation; - - msg = ethosu_mailbox_find(mailbox, msg_id, - ETHOSU_CORE_MSG_CANCEL_INFERENCE_REQ); - if (IS_ERR(msg)) { - dev_warn(dev, - "Id for cancel inference msg not found. Id=0x%x: %ld", - msg_id, PTR_ERR(msg)); - - return; - } - - cancellation = container_of(msg, typeof(*cancellation), msg); - - if (completion_done(&cancellation->done)) - return; - - cancellation->errno = 0; - switch (rsp->status) { - case ETHOSU_CORE_STATUS_OK: - cancellation->uapi->status = ETHOSU_UAPI_STATUS_OK; - break; - case ETHOSU_CORE_STATUS_ERROR: - cancellation->uapi->status = ETHOSU_UAPI_STATUS_ERROR; - break; - } - - complete(&cancellation->done); -} diff --git a/kernel/ethosu_cancel_inference.h b/kernel/ethosu_cancel_inference.h deleted file mode 100644 index a3982dd..0000000 --- a/kernel/ethosu_cancel_inference.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2022-2023 Arm Limited and/or its affiliates - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - * SPDX-License-Identifier: GPL-2.0-only - */ - -#ifndef ETHOSU_CANCEL_INFERENCE_H -#define ETHOSU_CANCEL_INFERENCE_H - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_mailbox.h" -#include "uapi/ethosu.h" - -#include -#include - -/**************************************************************************** - * Types - ****************************************************************************/ - -struct ethosu_core_msg_cancel_inference_rsp; -struct ethosu_inference; -struct ethosu_uapi_cancel_inference_status; - -struct ethosu_cancel_inference { - struct device *dev; - struct ethosu_inference *inf; - struct ethosu_uapi_cancel_inference_status *uapi; - struct completion done; - struct ethosu_mailbox_msg msg; - int errno; -}; - -/**************************************************************************** - * Functions - ****************************************************************************/ - -/** - * ethosu_cancel_inference_request() - Send cancel inference request - * - * Return: 0 on success, error code otherwise. - */ -int ethosu_cancel_inference_request(struct device *dev, - struct ethosu_mailbox *mailbox, - struct ethosu_inference *inf, - struct ethosu_uapi_cancel_inference_status *uapi); - -/** - * ethosu_cancel_inference_rsp() - Handle cancel inference response - */ -void ethosu_cancel_inference_rsp(struct ethosu_mailbox *mailbox, - int msg_id, - struct ethosu_core_msg_cancel_inference_rsp *rsp); - -#endif /* ETHOSU_CANCEL_INFERENCE_H */ diff --git a/kernel/ethosu_capabilities.c b/kernel/ethosu_capabilities.c deleted file mode 100644 index 2c2d23a..0000000 --- a/kernel/ethosu_capabilities.c +++ /dev/null @@ -1,164 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright 2022-2023 Arm Limited and/or its affiliates - * SPDX-License-Identifier: GPL-2.0-only - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_capabilities.h" - -#include "ethosu_core_rpmsg.h" -#include "ethosu_device.h" - -#include -#include -#include - -/**************************************************************************** - * Defines - ****************************************************************************/ - -#define CAPABILITIES_RESP_TIMEOUT_MS 2000 - -/**************************************************************************** - * Functions - ****************************************************************************/ - -static inline int ethosu_capabilities_send(struct ethosu_capabilities *cap, - struct ethosu_mailbox *mailbox) -{ - return ethosu_mailbox_capabilities_request(mailbox, - &cap->msg); -} - -static void ethosu_capabilities_fail(struct ethosu_mailbox_msg *msg) -{ - struct ethosu_capabilities *cap = - container_of(msg, typeof(*cap), msg); - - if (completion_done(&cap->done)) - return; - - cap->errno = -EFAULT; - complete(&cap->done); -} - -void ethosu_capability_rsp(struct ethosu_mailbox *mailbox, - int msg_id, - struct ethosu_core_msg_capabilities_rsp *rsp) -{ - struct device *dev = mailbox->dev; - struct ethosu_mailbox_msg *msg; - struct ethosu_capabilities *cap; - - msg = ethosu_mailbox_find(mailbox, msg_id, - ETHOSU_CORE_MSG_CAPABILITIES_REQ); - if (IS_ERR(msg)) { - dev_warn(dev, - "Id for capabilities msg not found. Id=0x%0x: %ld\n", - msg_id, PTR_ERR(msg)); - - return; - } - - cap = container_of(msg, typeof(*cap), msg); - - if (completion_done(&cap->done)) - return; - - cap->uapi->hw_id.version_status = rsp->version_status; - cap->uapi->hw_id.version_minor = rsp->version_minor; - cap->uapi->hw_id.version_major = rsp->version_major; - cap->uapi->hw_id.product_major = rsp->product_major; - cap->uapi->hw_id.arch_patch_rev = rsp->arch_patch_rev; - cap->uapi->hw_id.arch_minor_rev = rsp->arch_minor_rev; - cap->uapi->hw_id.arch_major_rev = rsp->arch_major_rev; - cap->uapi->driver_patch_rev = rsp->driver_patch_rev; - cap->uapi->driver_minor_rev = rsp->driver_minor_rev; - cap->uapi->driver_major_rev = rsp->driver_major_rev; - cap->uapi->hw_cfg.macs_per_cc = rsp->macs_per_cc; - cap->uapi->hw_cfg.cmd_stream_version = rsp->cmd_stream_version; - cap->uapi->hw_cfg.custom_dma = rsp->custom_dma; - - cap->errno = 0; - complete(&cap->done); -} - -int ethosu_capabilities_request(struct device *dev, - struct ethosu_mailbox *mailbox, - struct ethosu_uapi_device_capabilities *uapi) -{ - struct ethosu_capabilities *cap; - int ret; - int timeout; - - cap = devm_kzalloc(dev, sizeof(struct ethosu_capabilities), - GFP_KERNEL); - if (!cap) - return -ENOMEM; - - cap->dev = dev; - cap->uapi = uapi; - init_completion(&cap->done); - cap->msg.fail = ethosu_capabilities_fail; - - ret = ethosu_mailbox_register(mailbox, &cap->msg); - if (ret < 0) - goto kfree; - - dev_dbg(dev, "Capabilities create. Id=%d, handle=0x%p\n", - cap->msg.id, cap); - - ret = ethosu_capabilities_send(cap, mailbox); - if (0 != ret) - goto deregister; - - /* Unlock the mutex before going to block on the condition */ - device_unlock(dev); - - /* wait for response to arrive back */ - timeout = wait_for_completion_timeout(&cap->done, - msecs_to_jiffies( - CAPABILITIES_RESP_TIMEOUT_MS)); - - /* take back the mutex before resuming to do anything */ - device_lock(dev); - - if (0 == timeout) { - dev_warn(dev, "Capabilities response timeout"); - ret = -ETIME; - goto deregister; - } - - if (cap->errno) { - ret = cap->errno; - goto deregister; - } - -deregister: - ethosu_mailbox_deregister(mailbox, &cap->msg); - -kfree: - dev_dbg(dev, "Capabilities destroy. Id=%d, handle=0x%p\n", - cap->msg.id, cap); - devm_kfree(dev, cap); - - return ret; -} diff --git a/kernel/ethosu_capabilities.h b/kernel/ethosu_capabilities.h deleted file mode 100644 index 30bf475..0000000 --- a/kernel/ethosu_capabilities.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2022-2023 Arm Limited and/or its affiliates - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - * SPDX-License-Identifier: GPL-2.0-only - */ - -#ifndef ETHOSU_CAPABILITIES_H -#define ETHOSU_CAPABILITIES_H - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_mailbox.h" - -#include -#include - -/**************************************************************************** - * Types - ****************************************************************************/ - -struct ethosu_core_msg_capabilities_rsp; -struct ethosu_device; -struct ethosu_uapi_device_capabilities; - -/** - * struct ethosu_capabilities - Capabilities internal struct - */ -struct ethosu_capabilities { - struct device *dev; - struct completion done; - struct ethosu_uapi_device_capabilities *uapi; - struct ethosu_mailbox_msg msg; - int errno; -}; - -/**************************************************************************** - * Functions - ****************************************************************************/ - -int ethosu_capabilities_request(struct device *dev, - struct ethosu_mailbox *mailbox, - struct ethosu_uapi_device_capabilities *uapi); - -void ethosu_capability_rsp(struct ethosu_mailbox *mailbox, - int msg_id, - struct ethosu_core_msg_capabilities_rsp *rsp); - -#endif diff --git a/kernel/ethosu_core_rpmsg.h b/kernel/ethosu_core_rpmsg.h deleted file mode 100644 index 0b7feab..0000000 --- a/kernel/ethosu_core_rpmsg.h +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Copyright (c) 2020-2023 Arm Limited. - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - * SPDX-License-Identifier: GPL-2.0-only - */ - -#ifndef ETHOSU_CORE_RPMSG_H -#define ETHOSU_CORE_RPMSG_H - -#ifdef __KERNEL__ -#include -#else -#include -#endif - -#ifdef __cplusplus -namespace EthosU { -#endif - -/** Maximum number of IFM/OFM buffers per inference */ -#define ETHOSU_CORE_BUFFER_MAX 16 - -/** Maximum number of PMU counters to be returned for inference */ -#define ETHOSU_CORE_PMU_MAX 8 - -#define ETHOSU_CORE_MSG_MAGIC 0x41457631 -#define ETHOSU_CORE_MSG_VERSION_MAJOR 0 -#define ETHOSU_CORE_MSG_VERSION_MINOR 2 -#define ETHOSU_CORE_MSG_VERSION_PATCH 0 - -/** - * enum ethosu_core_msg_type - Message types - * - * Types for the messages sent between the host and the core subsystem. - */ -enum ethosu_core_msg_type { - ETHOSU_CORE_MSG_ERR = 1, - ETHOSU_CORE_MSG_PING, - ETHOSU_CORE_MSG_PONG, - ETHOSU_CORE_MSG_INFERENCE_REQ, - ETHOSU_CORE_MSG_INFERENCE_RSP, - ETHOSU_CORE_MSG_VERSION_REQ, - ETHOSU_CORE_MSG_VERSION_RSP, - ETHOSU_CORE_MSG_CAPABILITIES_REQ, - ETHOSU_CORE_MSG_CAPABILITIES_RSP, - ETHOSU_CORE_MSG_NETWORK_INFO_REQ, - ETHOSU_CORE_MSG_NETWORK_INFO_RSP, - ETHOSU_CORE_MSG_CANCEL_INFERENCE_REQ, - ETHOSU_CORE_MSG_CANCEL_INFERENCE_RSP, - ETHOSU_CORE_MSG_MAX -}; - -/** - * struct ethosu_core_msg_header - Message header - */ -struct ethosu_core_msg_header { - uint32_t magic; - uint32_t type; - uint64_t msg_id; -}; - -/** - * enum ethosu_core_status - Status - */ -enum ethosu_core_status { - ETHOSU_CORE_STATUS_OK, - ETHOSU_CORE_STATUS_ERROR, - ETHOSU_CORE_STATUS_RUNNING, - ETHOSU_CORE_STATUS_REJECTED, - ETHOSU_CORE_STATUS_ABORTED, - ETHOSU_CORE_STATUS_ABORTING, -}; - -/** - * struct ethosu_core_buffer - Buffer descriptor - * - * Pointer and size to a buffer within the Ethos-U address space. - */ -struct ethosu_core_buffer { - uint32_t ptr; - uint32_t size; -}; - -/** - * enum ethosu_core_network_type - Network buffer type - */ -enum ethosu_core_network_type { - ETHOSU_CORE_NETWORK_BUFFER = 1, - ETHOSU_CORE_NETWORK_INDEX -}; - -/** - * struct ethosu_core_network_buffer - Network buffer - */ -struct ethosu_core_network_buffer { - uint32_t type; - union { - struct ethosu_core_buffer buffer; - uint32_t index; - }; -}; - -/** - * struct ethosu_core_msg_inference_req - Inference request - */ -struct ethosu_core_msg_inference_req { - uint32_t ifm_count; - struct ethosu_core_buffer ifm[ETHOSU_CORE_BUFFER_MAX]; - uint32_t ofm_count; - struct ethosu_core_buffer ofm[ETHOSU_CORE_BUFFER_MAX]; - struct ethosu_core_network_buffer network; - uint8_t pmu_event_config[ETHOSU_CORE_PMU_MAX]; - uint32_t pmu_cycle_counter_enable; -}; - -/** - * struct ethosu_core_msg_inference_rsp - Inference response - */ -struct ethosu_core_msg_inference_rsp { - uint32_t ofm_count; - uint32_t ofm_size[ETHOSU_CORE_BUFFER_MAX]; - uint32_t status; - uint8_t pmu_event_config[ETHOSU_CORE_PMU_MAX]; - uint64_t pmu_event_count[ETHOSU_CORE_PMU_MAX]; - uint32_t pmu_cycle_counter_enable; - uint64_t pmu_cycle_counter_count; -}; - -/** - * struct ethosu_core_msg_network_info_req - Network information request - */ -struct ethosu_core_msg_network_info_req { - struct ethosu_core_network_buffer network; -}; - -/** - * struct ethosu_core_msg_network_info_rsp - Network information response - */ -struct ethosu_core_msg_network_info_rsp { - char desc[32]; - uint32_t ifm_count; - uint32_t ifm_size[ETHOSU_CORE_BUFFER_MAX]; - uint32_t ofm_count; - uint32_t ofm_size[ETHOSU_CORE_BUFFER_MAX]; - uint32_t status; -}; - -/** - * struct ethosu_core_msg_version_rsp - Message protocol version - */ -struct ethosu_core_msg_version_rsp { - uint8_t major; - uint8_t minor; - uint8_t patch; - uint8_t _reserved; -}; - -/** - * struct ethosu_core_msg_capabilities_rsp - Message capabilities response - */ -struct ethosu_core_msg_capabilities_rsp { - uint32_t version_status; - uint32_t version_minor; - uint32_t version_major; - uint32_t product_major; - uint32_t arch_patch_rev; - uint32_t arch_minor_rev; - uint32_t arch_major_rev; - uint32_t driver_patch_rev; - uint32_t driver_minor_rev; - uint32_t driver_major_rev; - uint32_t macs_per_cc; - uint32_t cmd_stream_version; - uint32_t custom_dma; -}; - -/** - * struct ethosu_core_msg_cancel_inference_req - Message cancel inference - * request - */ -struct ethosu_core_msg_cancel_inference_req { - uint64_t inference_handle; -}; - -/** - * struct ethosu_core_msg_cancel_inference_rsp - Message cancel inference - * response - */ -struct ethosu_core_msg_cancel_inference_rsp { - uint32_t status; -}; - -/** - * enum ethosu_core_err_type - Error types - */ -enum ethosu_core_err_type { - ETHOSU_CORE_MSG_ERR_GENERIC = 0, - ETHOSU_CORE_MSG_ERR_UNSUPPORTED_TYPE, - ETHOSU_CORE_MSG_ERR_INVALID_PAYLOAD, - ETHOSU_CORE_MSG_ERR_INVALID_SIZE, - ETHOSU_CORE_MSG_ERR_INVALID_MAGIC, - ETHOSU_CORE_MSG_ERR_MAX -}; - -/** - * struct ethosu_core_msg_err - Error message struct - */ -struct ethosu_core_msg_err { - uint32_t type; /* optional use of extra error code */ - char msg[128]; -}; - -/** - * struct ethosu_core_rpmsg - Rpmsg message - */ -struct ethosu_core_rpmsg { - struct ethosu_core_msg_header header; - union { - struct ethosu_core_msg_inference_req inf_req; - struct ethosu_core_msg_inference_rsp inf_rsp; - struct ethosu_core_msg_network_info_req net_info_req; - struct ethosu_core_msg_network_info_rsp net_info_rsp; - struct ethosu_core_msg_capabilities_rsp cap_rsp; - struct ethosu_core_msg_cancel_inference_req cancel_req; - struct ethosu_core_msg_cancel_inference_rsp cancel_rsp; - struct ethosu_core_msg_version_rsp version_rsp; - struct ethosu_core_msg_err error; - }; -}; - -#ifdef __cplusplus -} /*namespace EthosU */ -#endif - -#endif /* ETHOSU_CORE_RPMSG_H */ diff --git a/kernel/ethosu_device.c b/kernel/ethosu_device.c deleted file mode 100644 index 32fb012..0000000 --- a/kernel/ethosu_device.c +++ /dev/null @@ -1,560 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright 2020-2023 Arm Limited and/or its affiliates - * SPDX-License-Identifier: GPL-2.0-only - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - */ - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_device.h" - -#include "ethosu_buffer.h" -#include "ethosu_core_rpmsg.h" -#include "ethosu_capabilities.h" -#include "ethosu_inference.h" -#include "ethosu_cancel_inference.h" -#include "ethosu_network.h" -#include "ethosu_network_info.h" -#include "ethosu_version.h" -#include "uapi/ethosu.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/**************************************************************************** - * Defines - ****************************************************************************/ - -#define MINOR_BASE 0 /* Minor version starts at 0 */ -#define MINOR_COUNT 64 /* Allocate minor versions */ - -/**************************************************************************** - * Variables - ****************************************************************************/ - -static DECLARE_BITMAP(minors, MINOR_COUNT); - -/**************************************************************************** - * Functions - ****************************************************************************/ - -/* Incoming messages */ -static int ethosu_handle_rpmsg(struct rpmsg_device *rpdev, - void *data, - int len, - void *priv, - u32 src) -{ - struct ethosu_device *edev = dev_get_drvdata(&rpdev->dev); - struct device *dev = &edev->dev; - struct ethosu_mailbox *mbox = &edev->mailbox; - struct ethosu_core_rpmsg *rpmsg = data; - int length = len - sizeof(rpmsg->header); - int ret = 0; - - if (unlikely(rpmsg->header.magic != ETHOSU_CORE_MSG_MAGIC)) { - dev_warn(dev, "Msg: Error invalid message magic. magic=0x%08x", - rpmsg->header.magic); - - return -EBADMSG; - } - - device_lock(dev); - - dev_dbg(dev, - "Msg: magic=0x%08x, type=%u, msg_id=%llu", - rpmsg->header.magic, rpmsg->header.type, rpmsg->header.msg_id); - - switch (rpmsg->header.type) { - case ETHOSU_CORE_MSG_ERR: - if (length != sizeof(rpmsg->error)) { - dev_warn(dev, - "Msg: Error message of incorrect size. size=%u, expected=%zu", length, - sizeof(rpmsg->error)); - ret = -EBADMSG; - break; - } - - rpmsg->error.msg[sizeof(rpmsg->error.msg) - 1] = '\0'; - dev_warn(dev, "Msg: Error. type=%u, msg=\"%s\"", - rpmsg->error.type, rpmsg->error.msg); - - rproc_report_crash(rproc_get_by_child(dev), RPROC_FATAL_ERROR); - break; - case ETHOSU_CORE_MSG_PING: - dev_dbg(dev, "Msg: Ping"); - ret = ethosu_mailbox_pong(mbox); - break; - case ETHOSU_CORE_MSG_PONG: - dev_dbg(dev, "Msg: Pong"); - break; - case ETHOSU_CORE_MSG_INFERENCE_RSP: - if (length != sizeof(rpmsg->inf_rsp)) { - dev_warn(dev, - "Msg: Inference response of incorrect size. size=%u, expected=%zu", length, - sizeof(rpmsg->inf_rsp)); - ret = -EBADMSG; - break; - } - - dev_dbg(dev, - "Msg: Inference response. ofm_count=%u, status=%u", - rpmsg->inf_rsp.ofm_count, rpmsg->inf_rsp.status); - - ethosu_inference_rsp(mbox, rpmsg->header.msg_id, - &rpmsg->inf_rsp); - break; - case ETHOSU_CORE_MSG_CANCEL_INFERENCE_RSP: - if (length != sizeof(rpmsg->cancel_rsp)) { - dev_warn(dev, - "Msg: Cancel Inference response of incorrect size. size=%u, expected=%zu", length, - sizeof(rpmsg->cancel_rsp)); - ret = -EBADMSG; - break; - } - - dev_dbg(dev, - "Msg: Cancel Inference response. status=%u", - rpmsg->cancel_rsp.status); - ethosu_cancel_inference_rsp(mbox, - rpmsg->header.msg_id, - &rpmsg->cancel_rsp); - break; - case ETHOSU_CORE_MSG_VERSION_RSP: - if (length != sizeof(rpmsg->version_rsp)) { - dev_warn(dev, - "Msg: Protocol version response of incorrect size. size=%u, expected=%zu", length, - sizeof(rpmsg->version_rsp)); - ret = -EBADMSG; - break; - } - - dev_dbg(dev, "Msg: Protocol version response %u.%u.%u", - rpmsg->version_rsp.major, rpmsg->version_rsp.minor, - rpmsg->version_rsp.patch); - - ethosu_version_rsp(mbox, rpmsg->header.msg_id, - &rpmsg->version_rsp); - break; - case ETHOSU_CORE_MSG_CAPABILITIES_RSP: - if (length != sizeof(rpmsg->cap_rsp)) { - dev_warn(dev, - "Msg: Capabilities response of incorrect size. size=%u, expected=%zu", length, - sizeof(rpmsg->cap_rsp)); - ret = -EBADMSG; - break; - } - - dev_dbg(dev, - "Msg: Capabilities response vs%hhu v%hhu.%hhu p%hhu av%hhu.%hhu.%hhu dv%hhu.%hhu.%hhu mcc%hhu csv%hhu cd%hhu", - rpmsg->cap_rsp.version_status, - rpmsg->cap_rsp.version_major, - rpmsg->cap_rsp.version_minor, - rpmsg->cap_rsp.product_major, - rpmsg->cap_rsp.arch_major_rev, - rpmsg->cap_rsp.arch_minor_rev, - rpmsg->cap_rsp.arch_patch_rev, - rpmsg->cap_rsp.driver_major_rev, - rpmsg->cap_rsp.driver_minor_rev, - rpmsg->cap_rsp.driver_patch_rev, - rpmsg->cap_rsp.macs_per_cc, - rpmsg->cap_rsp.cmd_stream_version, - rpmsg->cap_rsp.custom_dma); - - ethosu_capability_rsp(mbox, rpmsg->header.msg_id, - &rpmsg->cap_rsp); - break; - case ETHOSU_CORE_MSG_NETWORK_INFO_RSP: - if (length != sizeof(rpmsg->net_info_rsp)) { - dev_warn(dev, - "Msg: Network info response of incorrect size. size=%u, expected=%zu", length, - sizeof(rpmsg->net_info_rsp)); - ret = -EBADMSG; - break; - } - - dev_dbg(dev, - "Msg: Network info response. status=%u", - rpmsg->net_info_rsp.status); - - ethosu_network_info_rsp(mbox, - rpmsg->header.msg_id, - &rpmsg->net_info_rsp); - - break; - default: - /* This should not happen due to version checks */ - dev_warn(dev, "Msg: Protocol error. type=%u", - rpmsg->header.type); - ret = -EPROTO; - break; - } - - device_unlock(dev); - - wake_up(&mbox->send_queue); - - return ret; -} - -static int ethosu_open(struct inode *inode, - struct file *file) -{ - struct cdev *cdev = inode->i_cdev; - struct ethosu_device *edev = container_of(cdev, struct ethosu_device, - cdev); - struct rpmsg_device *rpdev = edev->rpdev; - struct device *dev = &edev->dev; - - dev_dbg(dev, "Device open. file=0x%pK", file); - - file->private_data = rpdev; - - return nonseekable_open(inode, file); -} - -static long ethosu_ioctl(struct file *file, - unsigned int cmd, - unsigned long arg) -{ - struct rpmsg_device *rpdev = file->private_data; - struct ethosu_device *edev = dev_get_drvdata(&rpdev->dev); - struct device *dev = &edev->dev; - void __user *udata = (void __user *)arg; - int ret; - - switch (cmd) { - case ETHOSU_IOCTL_DRIVER_VERSION_GET: { - const struct ethosu_uapi_kernel_driver_version version = { - .major = ETHOSU_KERNEL_DRIVER_VERSION_MAJOR, - .minor = ETHOSU_KERNEL_DRIVER_VERSION_MINOR, - .patch = ETHOSU_KERNEL_DRIVER_VERSION_PATCH, - }; - - ret = copy_to_user(udata, &version, - sizeof(version)) ? -EFAULT : 0; - break; - } - case ETHOSU_IOCTL_CAPABILITIES_REQ: { - dev_dbg(dev, "Device ioctl: Capabilities request"); - - ret = copy_to_user(udata, &edev->capabilities, - sizeof(edev->capabilities)) ? -EFAULT : 0; - break; - } - case ETHOSU_IOCTL_PING: { - ret = device_lock_interruptible(dev); - if (ret) - return ret; - - dev_dbg(dev, "Device ioctl: Send ping"); - - ret = ethosu_mailbox_ping(&edev->mailbox); - - device_unlock(dev); - - break; - } - case ETHOSU_IOCTL_BUFFER_CREATE: { - struct ethosu_uapi_buffer_create uapi; - - if (copy_from_user(&uapi, udata, sizeof(uapi))) { - ret = -EFAULT; - break; - } - - ret = device_lock_interruptible(dev); - if (ret) - return ret; - - dev_dbg(dev, - "Device ioctl: Buffer create. size=%u", - uapi.size); - - ret = ethosu_buffer_create(dev, uapi.size); - - device_unlock(dev); - - break; - } - case ETHOSU_IOCTL_NETWORK_CREATE: { - struct ethosu_uapi_network_create uapi; - - if (copy_from_user(&uapi, udata, sizeof(uapi))) { - ret = -EFAULT; - break; - } - - ret = device_lock_interruptible(dev); - if (ret) - return ret; - - dev_dbg(dev, - "Device ioctl: Network create. type=%u\n", uapi.type); - - ret = ethosu_network_create(dev, &edev->mailbox, &uapi); - - device_unlock(dev); - - break; - } - default: { - dev_err(dev, "Invalid ioctl. cmd=%u, arg=%lu", - cmd, arg); - ret = -ENOIOCTLCMD; - break; - } - } - - return ret; -} - -static struct rpmsg_endpoint *ethosu_create_ept(struct rpmsg_device *rpdev) -{ - struct device *dev = &rpdev->dev; - struct rpmsg_channel_info info = { 0 }; - struct rpmsg_endpoint *ept; - - /* Create rpmsg endpoint */ - strncpy(info.name, rpdev->id.name, sizeof(info.name) - 1); - info.src = 0; - info.dst = rpdev->dst; - - dev_dbg(dev, "Creating rpmsg endpoint. name=%s, src=%u, dst=%u", - info.name, info.src, info.dst); - - ept = rpmsg_create_ept(rpdev, ethosu_handle_rpmsg, NULL, info); - if (!ept) { - dev_err(&rpdev->dev, "Failed to create endpoint"); - - return ERR_PTR(-EINVAL); - } - - return ept; -} - -static const struct file_operations fops = { - .owner = THIS_MODULE, - .open = ðosu_open, - .unlocked_ioctl = ðosu_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = ðosu_ioctl, -#endif -}; - -static void ethosu_dev_release(struct device *dev) -{ - struct ethosu_device *edev = dev_get_drvdata(dev); - - clear_bit(MINOR(edev->cdev.dev), minors); - - ethosu_mailbox_deinit(&edev->mailbox); - device_destroy(edev->class, edev->cdev.dev); - kfree(edev); -} - -static int ethosu_device_register(struct device *dev, - struct device *parent, - void *drvdata, - dev_t devt) -{ - struct rproc *rproc = rproc_get_by_child(parent); - int ret; - - dev->parent = parent; - dev->release = ethosu_dev_release; - dev_set_drvdata(dev, drvdata); - - ret = dev_set_name(dev, "ethosu%d", MINOR(devt)); - if (ret) { - dev_err(parent, "Failed to set device name. ret=%d", ret); - - return ret; - } - - /* Inherit DMA mask from rproc device */ - ret = dma_coerce_mask_and_coherent(dev, - dma_get_mask(rproc->dev.parent)); - if (ret) { - dev_err(parent, "Failed to set DMA mask. ret=%d", ret); - - return ret; - } - - /* Inherit DMA configuration from rproc device */ - ret = of_dma_configure(dev, rproc->dev.parent->of_node, false); - if (ret) { - dev_err(parent, "Failed to configure DMA. ret=%d", - ret); - - return ret; - } - - /* Inherit reserved memory from rproc device */ - ret = of_reserved_mem_device_init_by_idx(dev, - rproc->dev.parent->of_node, 0); - if (ret) { - dev_err(parent, "Failed to initialize reserved memory. ret=%d", - ret); - - return ret; - } - - ret = device_register(dev); - if (ret) { - dev_err(parent, "Failed to register device. ret=%d", ret); - - return ret; - } - - return 0; -} - -int ethosu_dev_init(struct rpmsg_device *rpdev, - struct class *class, - dev_t devt) -{ - struct device *dev = &rpdev->dev; - struct ethosu_device *edev; - struct device *sysdev; - int minor; - int ret; - - /* Reserve minor number for device node */ - minor = find_first_zero_bit(minors, MINOR_COUNT); - if (minor >= MINOR_COUNT) { - dev_err(dev, "No more minor numbers."); - - return -ENOMEM; - } - - devt = MKDEV(MAJOR(devt), minor); - - /* Allocate and create Ethos-U device */ - edev = kzalloc(sizeof(*edev), GFP_KERNEL); - if (!edev) - return -ENOMEM; - - dev_set_drvdata(&rpdev->dev, edev); - - edev->rpdev = rpdev; - edev->class = class; - - /* Create device object */ - ret = ethosu_device_register(&edev->dev, &rpdev->dev, edev, - devt); - if (ret) { - kfree(edev); - - return ret; - } - - /* Continue with new device */ - dev = &edev->dev; - - /* Create RPMsg endpoint */ - edev->ept = ethosu_create_ept(rpdev); - if (IS_ERR(edev->ept)) { - ret = PTR_ERR(edev->ept); - goto device_unregister; - } - - ret = ethosu_mailbox_init(&edev->mailbox, dev, edev->ept); - if (ret) - goto free_rpmsg_ept; - - device_lock(dev); - ret = ethosu_version_check_request(dev, &edev->mailbox); - device_unlock(dev); - if (ret) { - dev_err(dev, "Protocol version check failed: %d", ret); - goto deinit_mailbox; - } - - device_lock(dev); - ret = ethosu_capabilities_request(dev, &edev->mailbox, - &edev->capabilities); - device_unlock(dev); - if (ret) { - dev_err(dev, "Failed to get device capabilities: %d", ret); - goto deinit_mailbox; - } - - /* Create device node */ - cdev_init(&edev->cdev, &fops); - edev->cdev.owner = THIS_MODULE; - - cdev_set_parent(&edev->cdev, &dev->kobj); - - ret = cdev_add(&edev->cdev, devt, 1); - if (ret) { - dev_err(dev, "Failed to add character device."); - goto deinit_mailbox; - } - - sysdev = device_create(edev->class, NULL, devt, rpdev, - "ethosu%d", MINOR(devt)); - if (IS_ERR(sysdev)) { - dev_err(dev, "Failed to create device."); - ret = PTR_ERR(sysdev); - goto del_cdev; - } - - set_bit(minor, minors); - - dev_info(dev, - "Created Arm Ethos-U device. name=%s, major=%d, minor=%d", - dev_name(sysdev), MAJOR(devt), MINOR(devt)); - - return 0; - -del_cdev: - cdev_del(&edev->cdev); - -deinit_mailbox: - ethosu_mailbox_deinit(&edev->mailbox); - -free_rpmsg_ept: - rpmsg_destroy_ept(edev->ept); - -device_unregister: - device_unregister(dev); - - return ret; -} - -void ethosu_dev_deinit(struct rpmsg_device *rpdev) -{ - struct device *dev = &rpdev->dev; - struct ethosu_device *edev = dev_get_drvdata(dev); - - device_lock(&edev->dev); - ethosu_mailbox_fail(&edev->mailbox); - device_unlock(&edev->dev); - - rpmsg_destroy_ept(edev->ept); - cdev_del(&edev->cdev); - device_unregister(&edev->dev); -} diff --git a/kernel/ethosu_device.h b/kernel/ethosu_device.h deleted file mode 100644 index af14c85..0000000 --- a/kernel/ethosu_device.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2020-2023 Arm Limited and/or its affiliates - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - * SPDX-License-Identifier: GPL-2.0-only - */ - -#ifndef ETHOSU_DEVICE_H -#define ETHOSU_DEVICE_H - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "uapi/ethosu.h" -#include "ethosu_mailbox.h" - -#include -#include -#include -#include -#include - -/**************************************************************************** - * Types - ****************************************************************************/ - -/** - * struct ethosu_device - Device structure - */ -struct ethosu_device { - struct device dev; - struct rpmsg_device *rpdev; - struct rpmsg_endpoint *ept; - struct cdev cdev; - struct class *class; - struct ethosu_mailbox mailbox; - struct ethosu_uapi_device_capabilities capabilities; -}; - -/**************************************************************************** - * Functions - ****************************************************************************/ - -/** - * ethosu_dev_init() - Initialize the device - * - * Return: 0 on success, else error code. - */ -int ethosu_dev_init(struct rpmsg_device *rpdev, - struct class *class, - dev_t devt); - -/** - * ethosu_dev_deinit() - Initialize the device - */ -void ethosu_dev_deinit(struct rpmsg_device *rpdev); - -#endif /* ETHOSU_DEVICE_H */ diff --git a/kernel/ethosu_dma_mem.c b/kernel/ethosu_dma_mem.c deleted file mode 100644 index cb63081..0000000 --- a/kernel/ethosu_dma_mem.c +++ /dev/null @@ -1,77 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright 2023 Arm Limited and/or its affiliates - * SPDX-License-Identifier: GPL-2.0-only - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - */ - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_dma_mem.h" - -#include -#include - -/**************************************************************************** - * Functions - ****************************************************************************/ - -struct ethosu_dma_mem *ethosu_dma_mem_alloc(struct device *dev, - size_t size) -{ - struct ethosu_dma_mem *dma_mem; - - if (!size) - return ERR_PTR(-EINVAL); - - dma_mem = devm_kzalloc(dev, sizeof(*dma_mem), GFP_KERNEL); - if (!dma_mem) - return ERR_PTR(-ENOMEM); - - dma_mem->dev = dev; - dma_mem->size = size; - dma_mem->cpu_addr = dma_alloc_coherent(dev, size, &dma_mem->dma_addr, - GFP_KERNEL); - if (!dma_mem->cpu_addr) { - memset(dma_mem, 0, sizeof(*dma_mem)); - devm_kfree(dev, dma_mem); - - return ERR_PTR(-ENOMEM); - } - - return dma_mem; -} - -void ethosu_dma_mem_free(struct ethosu_dma_mem **dma_mem) -{ - struct device *dev; - struct ethosu_dma_mem *mem; - - if (!dma_mem || !*dma_mem) - return; - - mem = *dma_mem; - dev = mem->dev; - - memset(mem->cpu_addr, 0, mem->size); - dma_free_coherent(dev, mem->size, mem->cpu_addr, mem->dma_addr); - - memset(mem, 0, sizeof(*mem)); - devm_kfree(dev, mem); - - *dma_mem = NULL; -} diff --git a/kernel/ethosu_dma_mem.h b/kernel/ethosu_dma_mem.h deleted file mode 100644 index 11cdd39..0000000 --- a/kernel/ethosu_dma_mem.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright 2023 Arm Limited and/or its affiliates - * SPDX-License-Identifier: GPL-2.0-only - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - */ - -#ifndef ETHOSU_DMA_MEM_H -#define ETHOSU_DMA_MEM_H - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include - -/**************************************************************************** - * Types - ****************************************************************************/ - -struct device; - -/** - * struct ethosu_dma_mem - DMA memory allocation - * @dev: Device - * @size: Size of the allocation - * @cpu_addr: Kernel mapped address - * @dma_addr: DMA address - */ -struct ethosu_dma_mem { - struct device *dev; - size_t size; - void *cpu_addr; - dma_addr_t dma_addr; -}; - -/**************************************************************************** - * Functions - ****************************************************************************/ - -struct ethosu_dma_mem *ethosu_dma_mem_alloc(struct device *dev, - size_t size); - -void ethosu_dma_mem_free(struct ethosu_dma_mem **dma_mem); - -#endif /* ETHOSU_DMA_MEM_H */ diff --git a/kernel/ethosu_driver.c b/kernel/ethosu_driver.c deleted file mode 100644 index 48d2155..0000000 --- a/kernel/ethosu_driver.c +++ /dev/null @@ -1,162 +0,0 @@ -/* - * Copyright 2020-2023 Arm Limited and/or its affiliates - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - * SPDX-License-Identifier: GPL-2.0-only - */ - -#include -#include -#include -#include -#include -#include -#include - -#include "ethosu_device.h" -#include "uapi/ethosu.h" - -/**************************************************************************** - * Defines - ****************************************************************************/ - -#define ETHOSU_DRIVER_STR(s) #s -#define ETHOSU_DRIVER_VERSION_STR(major, minor, patch) \ - ETHOSU_DRIVER_STR(major) "." \ - ETHOSU_DRIVER_STR(minor) "." \ - ETHOSU_DRIVER_STR(patch) -#define ETHOSU_DRIVER_VERSION ETHOSU_DRIVER_VERSION_STR( \ - ETHOSU_KERNEL_DRIVER_VERSION_MAJOR, \ - ETHOSU_KERNEL_DRIVER_VERSION_MINOR, \ - ETHOSU_KERNEL_DRIVER_VERSION_PATCH) - -#define ETHOSU_DRIVER_NAME "ethosu" - -#define MINOR_BASE 0 /* Minor version starts at 0 */ -#define MINOR_COUNT 64 /* Allocate minor versions */ - -/**************************************************************************** - * Variables - ****************************************************************************/ - -static struct class *ethosu_class; - -static dev_t devt; - -/**************************************************************************** - * Rpmsg driver - ****************************************************************************/ - -static int ethosu_rpmsg_probe(struct rpmsg_device *rpdev) -{ - int ret; - - /* Initialize device */ - ret = ethosu_dev_init(rpdev, ethosu_class, devt); - if (ret) - return ret; - - return 0; -} - -static void ethosu_rpmsg_remove(struct rpmsg_device *rpdev) -{ - ethosu_dev_deinit(rpdev); -} - -static int ethosu_rpmsg_cb(struct rpmsg_device *rpdev, - void *data, - int len, - void *priv, - u32 src) -{ - dev_err(&rpdev->dev, "%s", __FUNCTION__); - - return -EINVAL; -} - -static struct rpmsg_device_id ethosu_rpmsg_driver_id_table[] = { - { .name = "ethos-u-0.0" }, - {}, -}; - -MODULE_DEVICE_TABLE(rpmsg, ethosu_rpmsg_driver_id_table); - -static struct rpmsg_driver ethosu_rpmsg_driver = { - .drv = { - .name = ETHOSU_DRIVER_NAME, - .owner = THIS_MODULE, - .probe_type = PROBE_PREFER_ASYNCHRONOUS, - }, - .id_table = ethosu_rpmsg_driver_id_table, - .probe = ethosu_rpmsg_probe, - .callback = ethosu_rpmsg_cb, - .remove = ethosu_rpmsg_remove, -}; - -/**************************************************************************** - * Module init and exit - ****************************************************************************/ - -static void __exit ethosu_exit(void) -{ - unregister_rpmsg_driver(ðosu_rpmsg_driver); - unregister_chrdev_region(devt, MINOR_COUNT); - class_destroy(ethosu_class); -} - -static int __init ethosu_init(void) -{ - int ret; - - ethosu_class = class_create(THIS_MODULE, ETHOSU_DRIVER_NAME); - if (IS_ERR(ethosu_class)) { - pr_err("Failed to create class '%s'.\n", ETHOSU_DRIVER_NAME); - - return PTR_ERR(ethosu_class); - } - - ret = alloc_chrdev_region(&devt, MINOR_BASE, MINOR_COUNT, - ETHOSU_DRIVER_NAME); - if (ret) { - pr_err("Failed to allocate chrdev region.\n"); - goto destroy_class; - } - - ret = register_rpmsg_driver(ðosu_rpmsg_driver); - if (ret) { - pr_err("Failed to register Arm Ethos-U rpmsg driver.\n"); - goto region_unregister; - } - - return 0; - -region_unregister: - unregister_chrdev_region(devt, MINOR_COUNT); - -destroy_class: - class_destroy(ethosu_class); - - return ret; -} - -module_init(ethosu_init) -module_exit(ethosu_exit) - -MODULE_LICENSE("GPL v2"); -MODULE_AUTHOR("Arm Ltd"); -MODULE_DESCRIPTION("Arm Ethos-U NPU Driver"); -MODULE_VERSION(ETHOSU_DRIVER_VERSION); diff --git a/kernel/ethosu_inference.c b/kernel/ethosu_inference.c deleted file mode 100644 index 3238d7e..0000000 --- a/kernel/ethosu_inference.c +++ /dev/null @@ -1,486 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright 2020,2022-2024 Arm Limited and/or its affiliates - * SPDX-License-Identifier: GPL-2.0-only - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - */ - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_inference.h" - -#include "ethosu_buffer.h" -#include "ethosu_core_rpmsg.h" -#include "ethosu_device.h" -#include "ethosu_network.h" -#include "ethosu_cancel_inference.h" - -#include -#include -#include -#include - -/**************************************************************************** - * Variables - ****************************************************************************/ - -static int ethosu_inference_release(struct inode *inode, - struct file *file); - -static __poll_t ethosu_inference_poll(struct file *file, - poll_table *wait); - -static long ethosu_inference_ioctl(struct file *file, - unsigned int cmd, - unsigned long arg); - -static const struct file_operations ethosu_inference_fops = { - .release = ðosu_inference_release, - .poll = ðosu_inference_poll, - .unlocked_ioctl = ðosu_inference_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = ðosu_inference_ioctl, -#endif -}; - -/**************************************************************************** - * Functions - ****************************************************************************/ - -static const char *status_to_string(const enum ethosu_uapi_status status) -{ - switch (status) { - case ETHOSU_UAPI_STATUS_OK: { - return "Ok"; - } - case ETHOSU_UAPI_STATUS_ERROR: { - return "Error"; - } - case ETHOSU_UAPI_STATUS_RUNNING: { - return "Running"; - } - case ETHOSU_UAPI_STATUS_REJECTED: { - return "Rejected"; - } - case ETHOSU_UAPI_STATUS_ABORTED: { - return "Aborted"; - } - case ETHOSU_UAPI_STATUS_ABORTING: { - return "Aborting"; - } - default: { - return "Unknown"; - } - } -} - -static int ethosu_inference_send(struct ethosu_inference *inf) -{ - struct device *dev = inf->dev; - int ret; - - inf->status = ETHOSU_UAPI_STATUS_ERROR; - - ret = ethosu_mailbox_inference(inf->mailbox, &inf->msg, - inf->ifm_count, inf->ifm, - inf->ofm_count, inf->ofm, - inf->net, inf->pmu_event_config, - ETHOSU_PMU_EVENT_MAX, - inf->pmu_cycle_counter_enable); - if (ret) { - dev_warn(dev, - "Failed to send inference request. inf=0x%pK, ret=%d", - inf, ret); - - return ret; - } - - inf->status = ETHOSU_UAPI_STATUS_RUNNING; - - ethosu_inference_get(inf); - - return 0; -} - -static void ethosu_inference_fail(struct ethosu_mailbox_msg *msg) -{ - struct ethosu_inference *inf = - container_of(msg, typeof(*inf), msg); - int ret; - - if (inf->done) - return; - - /* Decrement reference count if inference was pending reponse */ - ret = ethosu_inference_put(inf); - if (ret) - return; - - /* Set status accordingly to the inference state */ - inf->status = inf->status == ETHOSU_UAPI_STATUS_ABORTING ? - ETHOSU_UAPI_STATUS_ABORTED : - ETHOSU_UAPI_STATUS_ERROR; - /* Mark it done and wake up the waiting process */ - inf->done = true; - wake_up_interruptible(&inf->waitq); -} - -static bool ethosu_inference_verify(struct file *file) -{ - return file->f_op == ðosu_inference_fops; -} - -static void ethosu_inference_kref_destroy(struct kref *kref) -{ - struct ethosu_inference *inf = - container_of(kref, struct ethosu_inference, kref); - struct device *dev = inf->dev; - - dev_dbg(dev, - "Inference destroy. inf=0x%pK, status=%d, ifm_count=%u, ofm_count=%u", - inf, inf->status, inf->ifm_count, inf->ofm_count); - - ethosu_mailbox_deregister(inf->mailbox, &inf->msg); - - while (inf->ifm_count-- > 0) - ethosu_buffer_put(inf->ifm[inf->ifm_count]); - - while (inf->ofm_count-- > 0) - ethosu_buffer_put(inf->ofm[inf->ofm_count]); - - ethosu_network_put(inf->net); - memset(inf, 0, sizeof(*inf)); - devm_kfree(dev, inf); -} - -static int ethosu_inference_release(struct inode *inode, - struct file *file) -{ - struct ethosu_inference *inf = file->private_data; - struct device *dev = inf->dev; - - dev_dbg(dev, - "Inference release. file=0x%pK, inf=0x%pK", - file, inf); - - device_lock(dev); - ethosu_inference_put(inf); - device_unlock(dev); - - return 0; -} - -static __poll_t ethosu_inference_poll(struct file *file, - poll_table *wait) -{ - struct ethosu_inference *inf = file->private_data; - __poll_t ret = 0; - - poll_wait(file, &inf->waitq, wait); - - if (inf->done) - ret |= EPOLLIN; - - return ret; -} - -static long ethosu_inference_ioctl(struct file *file, - unsigned int cmd, - unsigned long arg) -{ - struct ethosu_inference *inf = file->private_data; - struct device *dev = inf->dev; - void __user *udata = (void __user *)arg; - int ret; - - ret = device_lock_interruptible(dev); - if (ret) - return ret; - - switch (cmd) { - case ETHOSU_IOCTL_INFERENCE_STATUS: { - struct ethosu_uapi_result_status uapi = { 0 }; - int i; - - uapi.status = inf->status; - - for (i = 0; i < ETHOSU_PMU_EVENT_MAX; i++) { - uapi.pmu_config.events[i] = - inf->pmu_event_config[i]; - uapi.pmu_count.events[i] = - inf->pmu_event_count[i]; - } - - uapi.pmu_config.cycle_count = inf->pmu_cycle_counter_enable; - uapi.pmu_count.cycle_count = inf->pmu_cycle_counter_count; - - dev_dbg(dev, - "Inference ioctl: Inference status. status=%s (%d)\n", - status_to_string(uapi.status), uapi.status); - - ret = copy_to_user(udata, &uapi, sizeof(uapi)) ? -EFAULT : 0; - - break; - } - case ETHOSU_IOCTL_INFERENCE_CANCEL: { - struct ethosu_uapi_cancel_inference_status uapi = { 0 }; - - dev_dbg(dev, - "Inference ioctl: Cancel Inference. Handle=%p\n", - inf); - - ret = ethosu_cancel_inference_request(dev, inf->mailbox, inf, - &uapi); - if (ret) - break; - - ret = copy_to_user(udata, &uapi, sizeof(uapi)) ? -EFAULT : 0; - - break; - } - default: { - dev_err(dev, "Invalid ioctl. cmd=%u, arg=%lu\n", - cmd, arg); - ret = -ENOIOCTLCMD; - break; - } - } - - device_unlock(dev); - - return ret; -} - -int ethosu_inference_create(struct device *dev, - struct ethosu_mailbox *mailbox, - struct ethosu_network *net, - struct ethosu_uapi_inference_create *uapi) -{ - struct ethosu_inference *inf; - uint32_t i; - int fd; - int ret = -ENOMEM; - - if (uapi->ifm_count > ETHOSU_FD_MAX || - uapi->ofm_count > ETHOSU_FD_MAX) { - dev_warn(dev, - "Too many IFM and/or OFM buffers for inference. ifm_count=%u, ofm_count=%u", - uapi->ifm_count, uapi->ofm_count); - - return -EFAULT; - } - - inf = devm_kzalloc(dev, sizeof(*inf), GFP_KERNEL); - if (!inf) - return -ENOMEM; - - inf->dev = dev; - inf->mailbox = mailbox; - inf->net = net; - inf->done = false; - inf->status = ETHOSU_UAPI_STATUS_ERROR; - kref_init(&inf->kref); - init_waitqueue_head(&inf->waitq); - inf->msg.fail = ethosu_inference_fail; - - /* Add inference to pending list */ - ret = ethosu_mailbox_register(mailbox, &inf->msg); - if (ret < 0) - goto kfree; - - /* Get pointer to IFM buffers */ - for (i = 0; i < uapi->ifm_count; i++) { - inf->ifm[i] = ethosu_buffer_get_from_fd(uapi->ifm_fd[i]); - if (IS_ERR(inf->ifm[i])) { - ret = PTR_ERR(inf->ifm[i]); - goto put_ifm; - } - - inf->ifm_count++; - } - - /* Get pointer to OFM buffer */ - for (i = 0; i < uapi->ofm_count; i++) { - inf->ofm[i] = ethosu_buffer_get_from_fd(uapi->ofm_fd[i]); - if (IS_ERR(inf->ofm[i])) { - ret = PTR_ERR(inf->ofm[i]); - goto put_ofm; - } - - inf->ofm_count++; - } - - /* Configure PMU and cycle counter */ - dev_dbg(dev, - "Configuring events for PMU. events=[%u, %u, %u, %u]\n", - uapi->pmu_config.events[0], uapi->pmu_config.events[1], - uapi->pmu_config.events[2], uapi->pmu_config.events[3]); - - /* Configure events and reset count for all events */ - for (i = 0; i < ETHOSU_PMU_EVENT_MAX; i++) { - inf->pmu_event_config[i] = uapi->pmu_config.events[i]; - inf->pmu_event_count[i] = 0; - } - - /* Configure cycle counter and reset any previous count */ - inf->pmu_cycle_counter_enable = uapi->pmu_config.cycle_count; - inf->pmu_cycle_counter_count = 0; - - /* Increment network reference count */ - ethosu_network_get(net); - - /* Send inference request to Arm Ethos-U subsystem */ - ret = ethosu_inference_send(inf); - if (ret) - goto put_net; - - /* Create file descriptor */ - ret = fd = anon_inode_getfd("ethosu-inference", ðosu_inference_fops, - inf, O_RDWR | O_CLOEXEC); - if (ret < 0) - goto put_net; - - /* Store pointer to file structure */ - inf->file = fget(ret); - fput(inf->file); - - dev_dbg(dev, - "Inference create. file=0x%pK, fd=%d, inf=0x%p, net=0x%pK, msg.id=0x%x", - inf->file, fd, inf, inf->net, inf->msg.id); - - return fd; - -put_net: - ethosu_network_put(inf->net); - -put_ofm: - while (inf->ofm_count-- > 0) - ethosu_buffer_put(inf->ofm[inf->ofm_count]); - -put_ifm: - while (inf->ifm_count-- > 0) - ethosu_buffer_put(inf->ifm[inf->ifm_count]); - - ethosu_mailbox_deregister(mailbox, &inf->msg); - -kfree: - memset(inf, 0, sizeof(*inf)); - devm_kfree(dev, inf); - - return ret; -} - -struct ethosu_inference *ethosu_inference_get_from_fd(int fd) -{ - struct ethosu_inference *inf; - struct file *file; - - file = fget(fd); - if (!file) - return ERR_PTR(-EINVAL); - - if (!ethosu_inference_verify(file)) { - fput(file); - - return ERR_PTR(-EINVAL); - } - - inf = file->private_data; - ethosu_inference_get(inf); - fput(file); - - return inf; -} - -void ethosu_inference_get(struct ethosu_inference *inf) -{ - kref_get(&inf->kref); -} - -int ethosu_inference_put(struct ethosu_inference *inf) -{ - return kref_put(&inf->kref, ðosu_inference_kref_destroy); -} - -void ethosu_inference_rsp(struct ethosu_mailbox *mailbox, - int msg_id, - struct ethosu_core_msg_inference_rsp *rsp) -{ - struct device *dev = mailbox->dev; - struct ethosu_mailbox_msg *msg; - struct ethosu_inference *inf; - int i; - - msg = ethosu_mailbox_find(mailbox, msg_id, - ETHOSU_CORE_MSG_INFERENCE_REQ); - if (IS_ERR(msg)) { - dev_warn(dev, - "Id for inference msg not found. Id=0x%x: %ld\n", - msg_id, PTR_ERR(msg)); - - return; - } - - inf = container_of(msg, typeof(*inf), msg); - - /* - * Don't handle the response if the inference is aborted or - * in the process of being aborted - */ - if (inf->status == ETHOSU_UAPI_STATUS_ABORTED || - inf->status == ETHOSU_UAPI_STATUS_ABORTING) { - inf->status = ETHOSU_UAPI_STATUS_ABORTED; - goto done; - } - - if (rsp->status == ETHOSU_CORE_STATUS_OK && - inf->ofm_count <= ETHOSU_CORE_BUFFER_MAX) - inf->status = ETHOSU_UAPI_STATUS_OK; - else if (rsp->status == ETHOSU_CORE_STATUS_REJECTED) - inf->status = ETHOSU_UAPI_STATUS_REJECTED; - else if (rsp->status == ETHOSU_CORE_STATUS_ABORTED) - inf->status = ETHOSU_UAPI_STATUS_ABORTED; - else - inf->status = ETHOSU_UAPI_STATUS_ERROR; - - if (inf->status == ETHOSU_UAPI_STATUS_OK) { - for (i = 0; i < ETHOSU_CORE_PMU_MAX; i++) { - inf->pmu_event_config[i] = rsp->pmu_event_config[i]; - inf->pmu_event_count[i] = rsp->pmu_event_count[i]; - } - - inf->pmu_cycle_counter_enable = rsp->pmu_cycle_counter_enable; - inf->pmu_cycle_counter_count = rsp->pmu_cycle_counter_count; - - dev_dbg(dev, - "PMU events. config=[%u, %u, %u, %u], count=[%llu, %llu, %llu, %llu]\n", - inf->pmu_event_config[0], inf->pmu_event_config[1], - inf->pmu_event_config[2], inf->pmu_event_config[3], - inf->pmu_event_count[0], inf->pmu_event_count[1], - inf->pmu_event_count[2], inf->pmu_event_count[3]); - - if (inf->pmu_cycle_counter_enable) - dev_dbg(dev, - "PMU cycle counter: count=%llu\n", - inf->pmu_cycle_counter_count); - } - -done: - inf->done = true; - wake_up_interruptible(&inf->waitq); - ethosu_inference_put(inf); -} diff --git a/kernel/ethosu_inference.h b/kernel/ethosu_inference.h deleted file mode 100644 index 5040ef1..0000000 --- a/kernel/ethosu_inference.h +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2020,2022-2023 Arm Limited and/or its affiliates - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - * SPDX-License-Identifier: GPL-2.0-only - */ - -#ifndef ETHOSU_INFERENCE_H -#define ETHOSU_INFERENCE_H - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_mailbox.h" -#include "uapi/ethosu.h" - -#include -#include - -/**************************************************************************** - * Types - ****************************************************************************/ - -struct ethosu_buffer; -struct ethosu_core_msg_inference_rsp; -struct ethosu_network; -struct ethosu_uapi_inference_create; -struct file; - -/** - * struct ethosu_inference - Inference struct - * @edev: Arm Ethos-U device - * @file: File handle - * @kref: Reference counter - * @waitq: Wait queue - * @done: Wait condition is done - * @ifm: Pointer to IFM buffer - * @ofm: Pointer to OFM buffer - * @net: Pointer to network - * @status: Inference status - * @pmu_event_config: PMU event configuration - * @pmu_event_count: PMU event count after inference - * @pmu_cycle_counter_enable: PMU cycle counter config - * @pmu_cycle_counter_count: PMU cycle counter count after inference - * @msg: Mailbox message - */ -struct ethosu_inference { - struct device *dev; - struct ethosu_mailbox *mailbox; - struct file *file; - struct kref kref; - wait_queue_head_t waitq; - bool done; - uint32_t ifm_count; - struct ethosu_buffer *ifm[ETHOSU_FD_MAX]; - uint32_t ofm_count; - struct ethosu_buffer *ofm[ETHOSU_FD_MAX]; - struct ethosu_network *net; - enum ethosu_uapi_status status; - uint8_t pmu_event_config[ETHOSU_PMU_EVENT_MAX]; - uint64_t pmu_event_count[ETHOSU_PMU_EVENT_MAX]; - uint32_t pmu_cycle_counter_enable; - uint64_t pmu_cycle_counter_count; - struct ethosu_mailbox_msg msg; -}; - -/**************************************************************************** - * Functions - ****************************************************************************/ - -/** - * ethosu_inference_create() - Create inference - * - * This function must be called in the context of a user space process. - * - * Return: fd on success, else error code. - */ -int ethosu_inference_create(struct device *dev, - struct ethosu_mailbox *mailbox, - struct ethosu_network *net, - struct ethosu_uapi_inference_create *uapi); - -/** - * ethosu_inference_get_from_fd() - Get inference handle from fd - * - * This function must be called from a user space context. - * - * Return: Pointer on success, else ERR_PTR. - */ -struct ethosu_inference *ethosu_inference_get_from_fd(int fd); - -/** - * ethosu_inference_get() - Get inference - */ -void ethosu_inference_get(struct ethosu_inference *inf); - -/** - * ethosu_inference_put() - Put inference - * - * Return: 1 if object was removed, else 0. - */ -int ethosu_inference_put(struct ethosu_inference *inf); - -/** - * ethosu_inference_rsp() - Handle inference response - */ -void ethosu_inference_rsp(struct ethosu_mailbox *mailbox, - int msg_id, - struct ethosu_core_msg_inference_rsp *rsp); - -#endif /* ETHOSU_INFERENCE_H */ diff --git a/kernel/ethosu_mailbox.c b/kernel/ethosu_mailbox.c deleted file mode 100644 index 005e83e..0000000 --- a/kernel/ethosu_mailbox.c +++ /dev/null @@ -1,362 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright 2020-2023 Arm Limited and/or its affiliates - * SPDX-License-Identifier: GPL-2.0-only - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - */ - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_mailbox.h" - -#include "ethosu_buffer.h" -#include "ethosu_core_rpmsg.h" -#include "ethosu_device.h" -#include "ethosu_dma_mem.h" -#include "ethosu_network.h" - -#include -#include -#include -#include -#include - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#ifndef fallthrough -#if __has_attribute(__fallthrough__) -#define fallthrough __attribute__((__fallthrough__)) -#else -#define fallthrough do {} while (0) /* fallthrough */ -#endif -#endif - -/**************************************************************************** - * Defines - ****************************************************************************/ - -#define MAILBOX_SEND_TIMEOUT_MS 15000 - -/**************************************************************************** - * Functions - ****************************************************************************/ - -/** - * ethosu_send_locked() - Blocking mailbox message sender - * - * Context: Can sleep and must be called with the device mutex locked. - * - * Return: 0 on success, else error code. - */ -static int ethosu_send_locked(struct ethosu_mailbox *mbox, - void *data, - size_t length) -{ - DEFINE_WAIT_FUNC(wait, woken_wake_function); - struct device *dev = mbox->dev; - long timeout = msecs_to_jiffies(MAILBOX_SEND_TIMEOUT_MS); - bool try_send = !wq_has_sleeper(&mbox->send_queue); - int ret; - - might_sleep(); - - /* Exclusive wait to only wake up one task at a time */ - add_wait_queue_exclusive(&mbox->send_queue, &wait); - for (;;) { - /* Stop if the mailbox is closing down */ - if (atomic_read(&mbox->done)) { - ret = -ENODEV; - break; - } - - /* Attempt to send if queue is empty or task was woken up */ - if (try_send) { - ret = rpmsg_trysend(mbox->ept, data, length); - if (ret != -ENOMEM) - break; - } else { - try_send = true; - } - - /* Unlock device mutex while waiting to not block other tasks */ - device_unlock(dev); - timeout = wait_woken(&wait, TASK_INTERRUPTIBLE, timeout); - device_lock(dev); - - /* Stop if the wait was interrupted */ - if (signal_pending(current)) { - ret = -EINTR; - break; - } - - if (!timeout) { - ret = -ETIME; - break; - } - } - - remove_wait_queue(&mbox->send_queue, &wait); - - /* - * If the message was sent successfully, there may be more TX buffers - * available so wake up the next waiting task. - */ - if (!ret && wq_has_sleeper(&mbox->send_queue)) - wake_up(&mbox->send_queue); - - return ret; -} - -static void ethosu_core_buffer_dma_mem_set(struct ethosu_dma_mem *dma_mem, - struct ethosu_core_buffer *cbuf) -{ - cbuf->ptr = (uint32_t)dma_mem->dma_addr; - cbuf->size = (uint32_t)dma_mem->size; -} - -int ethosu_mailbox_register(struct ethosu_mailbox *mbox, - struct ethosu_mailbox_msg *msg) -{ - WARN_ON_ONCE(!mutex_is_locked(&mbox->dev->mutex)); - msg->id = idr_alloc_cyclic(&mbox->msg_idr, msg, 0, INT_MAX, GFP_KERNEL); - if (msg->id < 0) - return msg->id; - - return 0; -} - -void ethosu_mailbox_deregister(struct ethosu_mailbox *mbox, - struct ethosu_mailbox_msg *msg) -{ - WARN_ON_ONCE(!mutex_is_locked(&mbox->dev->mutex)); - idr_remove(&mbox->msg_idr, msg->id); -} - -struct ethosu_mailbox_msg *ethosu_mailbox_find(struct ethosu_mailbox *mbox, - int msg_id, - uint32_t msg_type) -{ - struct ethosu_mailbox_msg *ptr; - - WARN_ON_ONCE(!mutex_is_locked(&mbox->dev->mutex)); - ptr = (struct ethosu_mailbox_msg *)idr_find(&mbox->msg_idr, msg_id); - - if (ptr == NULL) - return ERR_PTR(-ENOENT); - - if (ptr->type != msg_type) - return ERR_PTR(-EINVAL); - - return ptr; -} - -void ethosu_mailbox_fail(struct ethosu_mailbox *mbox) -{ - struct ethosu_mailbox_msg *cur; - int id; - - WARN_ON_ONCE(!mutex_is_locked(&mbox->dev->mutex)); - idr_for_each_entry(&mbox->msg_idr, cur, id) { - cur->fail(cur); - } -} - -int ethosu_mailbox_ping(struct ethosu_mailbox *mbox) -{ - struct ethosu_core_rpmsg rpmsg = { - .header = { - .magic = ETHOSU_CORE_MSG_MAGIC, - .type = ETHOSU_CORE_MSG_PING, - } - }; - - return ethosu_send_locked(mbox, &rpmsg, sizeof(rpmsg.header)); -} - -int ethosu_mailbox_pong(struct ethosu_mailbox *mbox) -{ - struct ethosu_core_rpmsg rpmsg = { - .header = { - .magic = ETHOSU_CORE_MSG_MAGIC, - .type = ETHOSU_CORE_MSG_PONG, - } - }; - - return ethosu_send_locked(mbox, &rpmsg, sizeof(rpmsg.header)); -} - -int ethosu_mailbox_version_request(struct ethosu_mailbox *mbox, - struct ethosu_mailbox_msg *msg) -{ - struct ethosu_core_rpmsg rpmsg = { - .header = { - .magic = ETHOSU_CORE_MSG_MAGIC, - .type = ETHOSU_CORE_MSG_VERSION_REQ, - .msg_id = msg->id - } - }; - - msg->type = rpmsg.header.type; - - return ethosu_send_locked(mbox, &rpmsg, sizeof(rpmsg.header)); -} - -int ethosu_mailbox_capabilities_request(struct ethosu_mailbox *mbox, - struct ethosu_mailbox_msg *msg) -{ - struct ethosu_core_rpmsg rpmsg = { - .header = { - .magic = ETHOSU_CORE_MSG_MAGIC, - .type = ETHOSU_CORE_MSG_CAPABILITIES_REQ, - .msg_id = msg->id - } - }; - - msg->type = rpmsg.header.type; - - return ethosu_send_locked(mbox, &rpmsg, sizeof(rpmsg.header)); -} - -int ethosu_mailbox_inference(struct ethosu_mailbox *mbox, - struct ethosu_mailbox_msg *msg, - uint32_t ifm_count, - struct ethosu_buffer **ifm, - uint32_t ofm_count, - struct ethosu_buffer **ofm, - struct ethosu_network *network, - uint8_t *pmu_event_config, - uint8_t pmu_event_config_count, - uint8_t pmu_cycle_counter_enable) -{ - struct ethosu_core_rpmsg rpmsg = { - .header = { - .magic = ETHOSU_CORE_MSG_MAGIC, - .type = ETHOSU_CORE_MSG_INFERENCE_REQ, - .msg_id = msg->id - } - }; - struct ethosu_core_msg_inference_req *inf_req = &rpmsg.inf_req; - uint32_t i; - - msg->type = rpmsg.header.type; - - /* Verify that the uapi and core has the same number of pmus */ - if (pmu_event_config_count != ETHOSU_CORE_PMU_MAX) { - dev_err(mbox->dev, "PMU count misconfigured."); - - return -EINVAL; - } - - inf_req->ifm_count = ifm_count; - inf_req->ofm_count = ofm_count; - inf_req->pmu_cycle_counter_enable = pmu_cycle_counter_enable; - - for (i = 0; i < ifm_count; i++) - ethosu_core_buffer_dma_mem_set(ifm[i]->dma_mem, - &inf_req->ifm[i]); - - for (i = 0; i < ofm_count; i++) - ethosu_core_buffer_dma_mem_set(ofm[i]->dma_mem, - &inf_req->ofm[i]); - - for (i = 0; i < ETHOSU_CORE_PMU_MAX; i++) - inf_req->pmu_event_config[i] = pmu_event_config[i]; - - if (network->dma_mem != NULL) { - inf_req->network.type = ETHOSU_CORE_NETWORK_BUFFER; - ethosu_core_buffer_dma_mem_set(network->dma_mem, - &inf_req->network.buffer); - } else { - inf_req->network.type = ETHOSU_CORE_NETWORK_INDEX; - inf_req->network.index = network->index; - } - - return ethosu_send_locked(mbox, &rpmsg, - sizeof(rpmsg.header) + sizeof(rpmsg.inf_req)); -} - -int ethosu_mailbox_network_info_request(struct ethosu_mailbox *mbox, - struct ethosu_mailbox_msg *msg, - struct ethosu_network *network) -{ - struct ethosu_core_rpmsg rpmsg = { - .header = { - .magic = ETHOSU_CORE_MSG_MAGIC, - .type = ETHOSU_CORE_MSG_NETWORK_INFO_REQ, - .msg_id = msg->id - } - }; - struct ethosu_core_msg_network_info_req *info_req = &rpmsg.net_info_req; - - msg->type = rpmsg.header.type; - - if (network->dma_mem != NULL) { - info_req->network.type = ETHOSU_CORE_NETWORK_BUFFER; - ethosu_core_buffer_dma_mem_set(network->dma_mem, - &info_req->network.buffer); - } else { - info_req->network.type = ETHOSU_CORE_NETWORK_INDEX; - info_req->network.index = network->index; - } - - return ethosu_send_locked(mbox, &rpmsg, - sizeof(rpmsg.header) + - sizeof(rpmsg.net_info_req)); -} - -int ethosu_mailbox_cancel_inference(struct ethosu_mailbox *mbox, - struct ethosu_mailbox_msg *msg, - int inference_handle) -{ - struct ethosu_core_rpmsg rpmsg = { - .header = { - .magic = ETHOSU_CORE_MSG_MAGIC, - .type = - ETHOSU_CORE_MSG_CANCEL_INFERENCE_REQ, - .msg_id = msg->id - }, - .cancel_req = { - .inference_handle = inference_handle - } - }; - - msg->type = rpmsg.header.type; - - return ethosu_send_locked(mbox, &rpmsg, - sizeof(rpmsg.header) + - sizeof(rpmsg.cancel_req)); -} - -int ethosu_mailbox_init(struct ethosu_mailbox *mbox, - struct device *dev, - struct rpmsg_endpoint *ept) -{ - mbox->dev = dev; - mbox->ept = ept; - idr_init(&mbox->msg_idr); - init_waitqueue_head(&mbox->send_queue); - - return 0; -} - -void ethosu_mailbox_deinit(struct ethosu_mailbox *mbox) -{ - atomic_set(&mbox->done, 1); - wake_up_all(&mbox->send_queue); -} diff --git a/kernel/ethosu_mailbox.h b/kernel/ethosu_mailbox.h deleted file mode 100644 index ab19613..0000000 --- a/kernel/ethosu_mailbox.h +++ /dev/null @@ -1,194 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright 2020-2023 Arm Limited and/or its affiliates - * SPDX-License-Identifier: GPL-2.0-only - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - */ - -#ifndef ETHOSU_MAILBOX_H -#define ETHOSU_MAILBOX_H - -/**************************************************************************** - * Includes - ****************************************************************************/ -#include "ethosu_core_rpmsg.h" - -#include -#include -#include -#include - -/**************************************************************************** - * Types - ****************************************************************************/ - -struct device; -struct ethosu_buffer; -struct ethosu_core_msg; -struct ethosu_core_queue; -struct ethosu_device; -struct ethosu_network; -struct resource; - -typedef void (*ethosu_mailbox_cb)(void *user_arg); - -struct ethosu_mailbox { - struct device *dev; - struct rpmsg_endpoint *ept; - struct idr msg_idr; - atomic_t done; - wait_queue_head_t send_queue; -}; - -/** - * struct ethosu_mailbox_msg - Mailbox message - * @id: Message id - * @type: Message request type - * @fail: Message failure callback - * - * The fail callback will be called with the device mutex locked - */ -struct ethosu_mailbox_msg { - int id; - uint32_t type; - void (*fail)(struct ethosu_mailbox_msg *msg); -}; - -/**************************************************************************** - * Functions - ****************************************************************************/ - -/** - * ethosu_mailbox_init() - Initialize mailbox - * - * Return: 0 on success, else error code. - */ -int ethosu_mailbox_init(struct ethosu_mailbox *mbox, - struct device *dev, - struct rpmsg_endpoint *ept); - -/** - * ethosu_mailbox_deinit() - Deinitialize mailbox - */ -void ethosu_mailbox_deinit(struct ethosu_mailbox *mbox); - -/** - * ethosu_mailbox_register() - Register the ethosu_mailbox_msg in ethosu_mailbox - * - * Context: Must be called with the device mutex locked - * - * Return: 0 on success, else error code. - */ -int ethosu_mailbox_register(struct ethosu_mailbox *mbox, - struct ethosu_mailbox_msg *msg); - -/** - * ethosu_mailbox_free_id() - Free the id of the ethosu_mailbox_msg - * - * Context: Must be called with the device mutex locked - */ -void ethosu_mailbox_deregister(struct ethosu_mailbox *mbox, - struct ethosu_mailbox_msg *msg); - -/** - * ethosu_mailbox_find() - Find mailbox message - * - * Context: Must be called with the device mutex locked - * - * Return: a valid pointer on success, otherwise an error ptr. - */ -struct ethosu_mailbox_msg *ethosu_mailbox_find(struct ethosu_mailbox *mbox, - int msg_id, - uint32_t msg_type); - -/** - * ethosu_mailbox_fail() - Fail mailbox messages - * - * Call fail() callback on all messages in pending list. - * - * Context: Must be called with the device mutex locked - */ -void ethosu_mailbox_fail(struct ethosu_mailbox *mbox); - -/** - * ethosu_mailbox_reset() - Reset to end of queue - */ -void ethosu_mailbox_reset(struct ethosu_mailbox *mbox); - -/** - * ethosu_mailbox_ping() - Send ping message - * - * Return: 0 on success, else error code. - */ -int ethosu_mailbox_ping(struct ethosu_mailbox *mbox); - -/** - * ethosu_mailbox_pong() - Send pong response - * - * Return: 0 on success, else error code. - */ -int ethosu_mailbox_pong(struct ethosu_mailbox *mbox); - -/** - * ethosu_mailbox_version_request() - Send protocol version request - * - * Return: 0 on succes, else error code - */ -int ethosu_mailbox_version_request(struct ethosu_mailbox *mbox, - struct ethosu_mailbox_msg *msg); - -/** - * ethosu_mailbox_capabilities_request() - Send capabilities request - * - * Return: 0 on success, else error code. - */ -int ethosu_mailbox_capabilities_request(struct ethosu_mailbox *mbox, - struct ethosu_mailbox_msg *msg); - -/** - * ethosu_mailbox_inference() - Send inference - * - * Return: 0 on success, else error code. - */ -int ethosu_mailbox_inference(struct ethosu_mailbox *mbox, - struct ethosu_mailbox_msg *msg, - uint32_t ifm_count, - struct ethosu_buffer **ifm, - uint32_t ofm_count, - struct ethosu_buffer **ofm, - struct ethosu_network *network, - uint8_t *pmu_event_config, - uint8_t pmu_event_config_count, - uint8_t pmu_cycle_counter_enable); - -/** - * ethosu_mailbox_network_info_request() - Send network info request - * - * Return: 0 on success, else error code. - */ -int ethosu_mailbox_network_info_request(struct ethosu_mailbox *mbox, - struct ethosu_mailbox_msg *msg, - struct ethosu_network *network); - -/** - * ethosu_mailbox_cancel_inference() - Send inference cancellation - * - * Return: 0 on success, else error code. - */ -int ethosu_mailbox_cancel_inference(struct ethosu_mailbox *mbox, - struct ethosu_mailbox_msg *msg, - int inference_handle); - -#endif /* ETHOSU_MAILBOX_H */ diff --git a/kernel/ethosu_network.c b/kernel/ethosu_network.c deleted file mode 100644 index 58d5c77..0000000 --- a/kernel/ethosu_network.c +++ /dev/null @@ -1,255 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright 2020,2022-2023 Arm Limited and/or its affiliates - * SPDX-License-Identifier: GPL-2.0-only - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - */ - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_network.h" - -#include "ethosu_device.h" -#include "ethosu_dma_mem.h" -#include "ethosu_inference.h" -#include "ethosu_network_info.h" -#include "uapi/ethosu.h" - -#include -#include -#include -#include - -/**************************************************************************** - * Variables - ****************************************************************************/ - -static int ethosu_network_release(struct inode *inode, - struct file *file); - -static long ethosu_network_ioctl(struct file *file, - unsigned int cmd, - unsigned long arg); - -static const struct file_operations ethosu_network_fops = { - .release = ðosu_network_release, - .unlocked_ioctl = ðosu_network_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = ðosu_network_ioctl, -#endif -}; - -/**************************************************************************** - * Functions - ****************************************************************************/ - -static bool ethosu_network_verify(struct file *file) -{ - return file->f_op == ðosu_network_fops; -} - -static void ethosu_network_destroy(struct kref *kref) -{ - struct ethosu_network *net = - container_of(kref, struct ethosu_network, kref); - struct device *dev = net->dev; - - dev_dbg(dev, "Network destroy. net=0x%pK\n", net); - - if (net->dma_mem != NULL) - ethosu_dma_mem_free(&net->dma_mem); - - memset(net, 0, sizeof(*net)); - devm_kfree(dev, net); -} - -static int ethosu_network_release(struct inode *inode, - struct file *file) -{ - struct ethosu_network *net = file->private_data; - struct device *dev = net->dev; - - dev_dbg(dev, "Network release. file=0x%pK, net=0x%pK\n", - file, net); - - ethosu_network_put(net); - - return 0; -} - -static long ethosu_network_ioctl(struct file *file, - unsigned int cmd, - unsigned long arg) -{ - struct ethosu_network *net = file->private_data; - struct device *dev = net->dev; - void __user *udata = (void __user *)arg; - int ret; - - ret = device_lock_interruptible(net->dev); - if (ret) - return ret; - - switch (cmd) { - case ETHOSU_IOCTL_NETWORK_INFO: { - struct ethosu_uapi_network_info uapi = { 0 }; - - dev_dbg(dev, "Network ioctl: Network info. net=0x%pK\n", net); - - ret = ethosu_network_info_request(dev, net->mailbox, net, - &uapi); - if (ret) - break; - - ret = copy_to_user(udata, &uapi, sizeof(uapi)) ? -EFAULT : 0; - break; - } - case ETHOSU_IOCTL_INFERENCE_CREATE: { - struct ethosu_uapi_inference_create uapi; - - if (copy_from_user(&uapi, udata, sizeof(uapi))) { - ret = -EFAULT; - break; - } - - dev_dbg(dev, - "Network ioctl: Inference. ifm_fd=%u, ofm_fd=%u\n", - uapi.ifm_fd[0], uapi.ofm_fd[0]); - - ret = ethosu_inference_create(dev, net->mailbox, net, &uapi); - break; - } - default: { - dev_err(dev, "Invalid ioctl. cmd=%u, arg=%lu", - cmd, arg); - ret = -ENOIOCTLCMD; - break; - } - } - - device_unlock(net->dev); - - return ret; -} - -int ethosu_network_create(struct device *dev, - struct ethosu_mailbox *mailbox, - struct ethosu_uapi_network_create *uapi) -{ - struct ethosu_network *net; - const void __user *data; - int ret; - - net = devm_kzalloc(dev, sizeof(*net), GFP_KERNEL); - if (!net) - return -ENOMEM; - - net->dev = dev; - net->mailbox = mailbox; - kref_init(&net->kref); - - switch (uapi->type) { - case ETHOSU_UAPI_NETWORK_USER_BUFFER: - if (!uapi->network.data_ptr) { - dev_err(dev, "Invalid network data ptr\n"); - ret = -EINVAL; - goto free_net; - } - - if (!uapi->network.size) { - dev_err(dev, "Invalid network data size\n"); - ret = -EINVAL; - goto free_net; - } - - net->dma_mem = ethosu_dma_mem_alloc(dev, uapi->network.size); - if (IS_ERR(net->dma_mem)) { - ret = PTR_ERR(net->dma_mem); - goto free_net; - } - - data = u64_to_user_ptr(uapi->network.data_ptr); - ret = copy_from_user(net->dma_mem->cpu_addr, data, - uapi->network.size); - if (ret) - goto free_dma_mem; - - break; - case ETHOSU_UAPI_NETWORK_INDEX: - net->index = uapi->index; - break; - default: - ret = -EINVAL; - goto free_net; - } - - ret = anon_inode_getfd("ethosu-network", ðosu_network_fops, net, - O_RDWR | O_CLOEXEC); - if (ret < 0) - goto free_dma_mem; - - net->file = fget(ret); - fput(net->file); - - dev_dbg(dev, - "Network create. file=0x%pK, fd=%d, net=0x%pK, buf=0x%pK, index=%u", - net->file, ret, net, net->dma_mem, net->index); - - return ret; - -free_dma_mem: - if (net->dma_mem != NULL) - ethosu_dma_mem_free(&net->dma_mem); - -free_net: - memset(net, 0, sizeof(*net)); - devm_kfree(dev, net); - - return ret; -} - -struct ethosu_network *ethosu_network_get_from_fd(int fd) -{ - struct ethosu_network *net; - struct file *file; - - file = fget(fd); - if (!file) - return ERR_PTR(-EINVAL); - - if (!ethosu_network_verify(file)) { - fput(file); - - return ERR_PTR(-EINVAL); - } - - net = file->private_data; - ethosu_network_get(net); - fput(file); - - return net; -} - -void ethosu_network_get(struct ethosu_network *net) -{ - kref_get(&net->kref); -} - -int ethosu_network_put(struct ethosu_network *net) -{ - return kref_put(&net->kref, ethosu_network_destroy); -} diff --git a/kernel/ethosu_network.h b/kernel/ethosu_network.h deleted file mode 100644 index 5484bac..0000000 --- a/kernel/ethosu_network.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright 2020,2022-2023 Arm Limited and/or its affiliates - * SPDX-License-Identifier: GPL-2.0-only - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - */ - -#ifndef ETHOSU_NETWORK_H -#define ETHOSU_NETWORK_H - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include -#include - -/**************************************************************************** - * Types - ****************************************************************************/ - -struct ethosu_buffer; -struct ethosu_uapi_network_create; -struct device; -struct file; - -struct ethosu_network { - struct device *dev; - struct ethosu_mailbox *mailbox; - struct file *file; - struct kref kref; - struct ethosu_dma_mem *dma_mem; - uint32_t index; -}; - -/**************************************************************************** - * Functions - ****************************************************************************/ - -/** - * ethosu_network_create() - Create network - * - * This function must be called in the context of a user space process. - * - * Return: fd on success, else error code. - */ -int ethosu_network_create(struct device *dev, - struct ethosu_mailbox *mailbox, - struct ethosu_uapi_network_create *uapi); - -/** - * ethosu_network_get_from_fd() - Get network handle from fd - * - * This function must be called from a user space context. - * - * Return: Pointer on success, else ERR_PTR. - */ -struct ethosu_network *ethosu_network_get_from_fd(int fd); - -/** - * ethosu_network_get() - Get network - */ -void ethosu_network_get(struct ethosu_network *net); - -/** - * ethosu_network_put() - Put network - * - * Return: 1 if object was removed, else 0. - */ -int ethosu_network_put(struct ethosu_network *net); - -#endif /* ETHOSU_NETWORK_H */ diff --git a/kernel/ethosu_network_info.c b/kernel/ethosu_network_info.c deleted file mode 100644 index 011a0b4..0000000 --- a/kernel/ethosu_network_info.c +++ /dev/null @@ -1,188 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright 2022-2023 Arm Limited and/or its affiliates - * SPDX-License-Identifier: GPL-2.0-only - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - */ - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_network_info.h" - -#include "ethosu_device.h" -#include "ethosu_network.h" -#include "ethosu_mailbox.h" -#include "uapi/ethosu.h" - -#include - -#define NETWORK_INFO_RESP_TIMEOUT_MS 3000 - -static inline int ethosu_network_info_send(struct ethosu_network_info *info, - struct ethosu_mailbox *mailbox) -{ - /* Send network info request to firmware */ - return ethosu_mailbox_network_info_request(mailbox, - &info->msg, - info->net); -} - -static void ethosu_network_info_fail(struct ethosu_mailbox_msg *msg) -{ - struct ethosu_network_info *info = - container_of(msg, typeof(*info), msg); - - if (completion_done(&info->done)) - return; - - info->errno = -EFAULT; - complete(&info->done); -} - -int ethosu_network_info_request(struct device *dev, - struct ethosu_mailbox *mailbox, - struct ethosu_network *net, - struct ethosu_uapi_network_info *uapi) -{ - struct ethosu_network_info *info; - int ret; - int timeout; - - info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL); - if (!info) - return -ENOMEM; - - info->dev = dev; - info->net = net; - info->uapi = uapi; - init_completion(&info->done); - info->msg.fail = ethosu_network_info_fail; - - ret = ethosu_mailbox_register(mailbox, &info->msg); - if (ret < 0) - goto kfree; - - /* Get reference to network */ - ethosu_network_get(info->net); - - ret = ethosu_network_info_send(info, mailbox); - if (ret) - goto deregister; - - dev_dbg(dev, - "Network info create. info=0x%pK, net=0x%pK, msg.id=0x%x\n", - info, info->net, info->msg.id); - - /* Unlock the device mutex and wait for completion */ - device_unlock(dev); - timeout = wait_for_completion_timeout(&info->done, - msecs_to_jiffies( - NETWORK_INFO_RESP_TIMEOUT_MS)); - device_lock(dev); - - if (0 == timeout) { - dev_warn(dev, "Network info timed out. info=0x%pK", - info); - - ret = -ETIME; - goto deregister; - } - - ret = info->errno; - -deregister: - ethosu_mailbox_deregister(mailbox, &info->msg); - ethosu_network_put(info->net); - -kfree: - dev_dbg(dev, - "Network info destroy. info=0x%pK, msg.id=0x%x\n", - info, info->msg.id); - devm_kfree(dev, info); - - return ret; -} - -void ethosu_network_info_rsp(struct ethosu_mailbox *mailbox, - int msg_id, - struct ethosu_core_msg_network_info_rsp *rsp) -{ - int ret; - struct device *dev = mailbox->dev; - struct ethosu_mailbox_msg *msg; - struct ethosu_network_info *info; - uint32_t i; - const size_t rsp_desc_size = sizeof(rsp->desc); - - BUILD_BUG_ON(rsp_desc_size != sizeof(info->uapi->desc)); - - msg = ethosu_mailbox_find(mailbox, msg_id, - ETHOSU_CORE_MSG_NETWORK_INFO_REQ); - if (IS_ERR(msg)) { - dev_warn(dev, - "Id for network info msg not found. Id=0x%x: %ld\n", - msg_id, PTR_ERR(msg)); - - return; - } - - info = container_of(msg, typeof(*info), msg); - - if (completion_done(&info->done)) - return; - - info->errno = 0; - - if (rsp->status != ETHOSU_CORE_STATUS_OK) { - dev_err(dev, "Failed to get information about the network\n"); - info->errno = -EBADF; - goto signal_complete; - } - - if (rsp->ifm_count > ETHOSU_FD_MAX || rsp->ofm_count > ETHOSU_FD_MAX) { - dev_err(dev, - "Invalid number of IFMs/OFMs in network info: IFMs=%u OFMs=%u\n", - rsp->ifm_count, rsp->ofm_count); - info->errno = -ENFILE; - goto signal_complete; - } - - if (strnlen(rsp->desc, rsp_desc_size) == rsp_desc_size) { - dev_err(dev, - "Description in network info is not null-terminated\n"); - info->errno = -EMSGSIZE; - goto signal_complete; - } - - ret = strscpy(info->uapi->desc, rsp->desc, sizeof(info->uapi->desc)); - if (ret < 0) { - dev_err(dev, "Failed to copy network info description\n"); - info->errno = ret; - goto signal_complete; - } - - info->uapi->ifm_count = rsp->ifm_count; - for (i = 0; i < rsp->ifm_count; i++) - info->uapi->ifm_size[i] = rsp->ifm_size[i]; - - info->uapi->ofm_count = rsp->ofm_count; - for (i = 0; i < rsp->ofm_count; i++) - info->uapi->ofm_size[i] = rsp->ofm_size[i]; - -signal_complete: - complete(&info->done); -} diff --git a/kernel/ethosu_network_info.h b/kernel/ethosu_network_info.h deleted file mode 100644 index 8c2e659..0000000 --- a/kernel/ethosu_network_info.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2022-2023 Arm Limited and/or its affiliates - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - * SPDX-License-Identifier: GPL-2.0-only - */ - -#ifndef ETHOSU_NETWORK_INFO_H -#define ETHOSU_NETWORK_INFO_H - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_mailbox.h" - -#include -#include - -/**************************************************************************** - * Types - ****************************************************************************/ - -struct ethosu_core_msg_network_info_rsp; -struct ethosu_network; -struct ethosu_uapi_network_info; - -struct ethosu_network_info { - struct device *dev; - struct ethosu_network *net; - struct ethosu_uapi_network_info *uapi; - struct completion done; - int errno; - struct ethosu_mailbox_msg msg; -}; - -/**************************************************************************** - * Functions - ****************************************************************************/ - -/** - * ethosu_network_info_request() - Send a network info request - * - * This function must be called in the context of a user space process. - * - * Return: 0 on success, . - */ -int ethosu_network_info_request(struct device *dev, - struct ethosu_mailbox *mailbox, - struct ethosu_network *net, - struct ethosu_uapi_network_info *uapi); - -/** - * ethosu_network_info_rsp() - Handle network info response. - */ -void ethosu_network_info_rsp(struct ethosu_mailbox *mailbox, - int msg_id, - struct ethosu_core_msg_network_info_rsp *rsp); - -#endif /* ETHOSU_NETWORK_INFO_H */ diff --git a/kernel/ethosu_version.c b/kernel/ethosu_version.c deleted file mode 100644 index 1a92dc2..0000000 --- a/kernel/ethosu_version.c +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright 2023 Arm Limited and/or its affiliates - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - * SPDX-License-Identifier: GPL-2.0-only - */ - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_version.h" - -#include "ethosu_core_rpmsg.h" - -#include - -/**************************************************************************** - * Defines - ****************************************************************************/ - -#define VERSION_RESP_TIMEOUT_MS 2000 - -/**************************************************************************** - * Functions - ****************************************************************************/ - -static void ethosu_version_fail(struct ethosu_mailbox_msg *msg) -{ - struct ethosu_version *version = - container_of(msg, typeof(*version), msg); - - if (completion_done(&version->done)) - return; - - version->errno = -EFAULT; - complete(&version->done); -} - -void ethosu_version_rsp(struct ethosu_mailbox *mailbox, - int msg_id, - struct ethosu_core_msg_version_rsp *rsp) -{ - struct device *dev = mailbox->dev; - struct ethosu_mailbox_msg *msg; - struct ethosu_version *version; - - msg = ethosu_mailbox_find(mailbox, msg_id, - ETHOSU_CORE_MSG_VERSION_REQ); - if (IS_ERR(msg)) { - dev_warn(dev, - "Id for version msg not found. Id=0x%0x: %ld\n", - msg_id, PTR_ERR(msg)); - - return; - } - - version = container_of(msg, typeof(*version), msg); - - if (completion_done(&version->done)) - return; - - if (rsp->major != ETHOSU_CORE_MSG_VERSION_MAJOR || - rsp->minor != ETHOSU_CORE_MSG_VERSION_MINOR) { - dev_warn(dev, - "Msg: Protocol version mismatch. Expected %u.%u.X but got %u.%u.%u", - ETHOSU_CORE_MSG_VERSION_MAJOR, - ETHOSU_CORE_MSG_VERSION_MINOR, - rsp->major, rsp->minor, rsp->patch); - version->errno = -EPROTO; - } else { - version->errno = 0; - } - - complete(&version->done); -} - -int ethosu_version_check_request(struct device *dev, - struct ethosu_mailbox *mailbox) -{ - struct ethosu_version *version; - int ret; - int timeout; - - version = devm_kzalloc(dev, sizeof(*version), GFP_KERNEL); - if (!version) - return -ENOMEM; - - version->dev = dev; - init_completion(&version->done); - version->msg.fail = ethosu_version_fail; - - ret = ethosu_mailbox_register(mailbox, &version->msg); - if (ret < 0) - goto free_version; - - dev_dbg(dev, "Protocol version request created. Id=0x%x, handle=%pK\n", - version->msg.id, version); - - ret = ethosu_mailbox_version_request(mailbox, &version->msg); - if (ret) - goto deregister; - - /* Unlock the mutex to not block other messages while waiting */ - device_unlock(dev); - - /* Wait for version response */ - timeout = wait_for_completion_timeout(&version->done, - msecs_to_jiffies( - VERSION_RESP_TIMEOUT_MS)); - - /* Take back the mutex before resuming to do anything */ - device_lock(dev); - - if (0 == timeout) { - dev_warn(dev, "Protocol version response timeout"); - ret = -ETIME; - goto deregister; - } - - if (version->errno) { - ret = version->errno; - goto deregister; - } - -deregister: - ethosu_mailbox_deregister(mailbox, &version->msg); - -free_version: - dev_dbg(dev, "Protocol version destroy. Id=0x%x, handle=%pK\n", - version->msg.id, - version); - devm_kfree(dev, version); - - return ret; -} diff --git a/kernel/ethosu_version.h b/kernel/ethosu_version.h deleted file mode 100644 index c5c02ad..0000000 --- a/kernel/ethosu_version.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2023 Arm Limited and/or its affiliates - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - * SPDX-License-Identifier: GPL-2.0-only - */ - -#ifndef ETHOSU_VERSION_H -#define ETHOSU_VERSION_H - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include "ethosu_mailbox.h" - -#include -#include - -/**************************************************************************** - * Types - ****************************************************************************/ - -struct ethosu_core_msg_version_rsp; - -/** - * struct ethosu_version - Protocol version internal struct - */ -struct ethosu_version { - struct device *dev; - struct completion done; - struct ethosu_mailbox_msg msg; - int errno; -}; - -/**************************************************************************** - * Functions - ****************************************************************************/ - -void ethosu_version_rsp(struct ethosu_mailbox *mailbox, - int msg_id, - struct ethosu_core_msg_version_rsp *rsp); - -int ethosu_version_check_request(struct device *dev, - struct ethosu_mailbox *mailbox); - -#endif /* ETHOSU_VERSION_H */ diff --git a/kernel/include/common/ethosu_buffer.h b/kernel/include/common/ethosu_buffer.h new file mode 100644 index 0000000..868e783 --- /dev/null +++ b/kernel/include/common/ethosu_buffer.h @@ -0,0 +1,85 @@ +/* + * SPDX-FileCopyrightText: Copyright 2020, 2022-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +#ifndef ETHOSU_BUFFER_H +#define ETHOSU_BUFFER_H + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include +#include + +/**************************************************************************** + * Types + ****************************************************************************/ + +struct ethosu_dma_mem; +struct ethosu_device; +struct device; + +/** + * struct ethosu_buffer - User data buffer + * @dev: Device + * @file: File + * @kref: Reference counting + * @dma_mem: DMA memory allocated for the buffer + */ +struct ethosu_buffer { + struct device *dev; + struct file *file; + struct kref kref; + struct ethosu_dma_mem *dma_mem; +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +/** + * ethosu_buffer_create() - Create buffer + * + * This function must be called in the context of a user space process. + * + * Return: fd on success, else error code. + */ +int ethosu_buffer_create(struct device *dev, + size_t size); + +/** + * ethosu_buffer_get_from_fd() - Get buffer handle from fd + * + * This function must be called from a user space context. + * + * Return: Pointer on success, else ERR_PTR. + */ +struct ethosu_buffer *ethosu_buffer_get_from_fd(int fd); + +/** + * ethosu_buffer_get() - Put buffer + */ +void ethosu_buffer_get(struct ethosu_buffer *buf); + +/** + * ethosu_buffer_put() - Put buffer + */ +void ethosu_buffer_put(struct ethosu_buffer *buf); + +#endif /* ETHOSU_BUFFER_H */ diff --git a/kernel/include/common/ethosu_device.h b/kernel/include/common/ethosu_device.h new file mode 100644 index 0000000..e96543e --- /dev/null +++ b/kernel/include/common/ethosu_device.h @@ -0,0 +1,71 @@ +/* + * SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +#ifndef ETHOSU_DEVICE_H +#define ETHOSU_DEVICE_H + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include +#include + +#include +#include +#include +#include +#include + +/**************************************************************************** + * Types + ****************************************************************************/ + +/** + * struct ethosu_device - Device structure + */ +struct ethosu_device { + struct device dev; + struct rpmsg_device *rpdev; + struct rpmsg_endpoint *ept; + struct cdev cdev; + struct class *class; + struct ethosu_mailbox mailbox; + struct ethosu_uapi_device_capabilities capabilities; +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +/** + * ethosu_dev_init() - Initialize the device + * + * Return: 0 on success, else error code. + */ +int ethosu_dev_init(struct rpmsg_device *rpdev, + struct class *class, + dev_t devt); + +/** + * ethosu_dev_deinit() - Initialize the device + */ +void ethosu_dev_deinit(struct rpmsg_device *rpdev); + +#endif /* ETHOSU_DEVICE_H */ diff --git a/kernel/include/common/ethosu_dma_mem.h b/kernel/include/common/ethosu_dma_mem.h new file mode 100644 index 0000000..eac4b42 --- /dev/null +++ b/kernel/include/common/ethosu_dma_mem.h @@ -0,0 +1,58 @@ +/* + * SPDX-FileCopyrightText: Copyright 2023-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +#ifndef ETHOSU_DMA_MEM_H +#define ETHOSU_DMA_MEM_H + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include + +/**************************************************************************** + * Types + ****************************************************************************/ + +struct device; + +/** + * struct ethosu_dma_mem - DMA memory allocation + * @dev: Device + * @size: Size of the allocation + * @cpu_addr: Kernel mapped address + * @dma_addr: DMA address + */ +struct ethosu_dma_mem { + struct device *dev; + size_t size; + void *cpu_addr; + dma_addr_t dma_addr; +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +struct ethosu_dma_mem *ethosu_dma_mem_alloc(struct device *dev, + size_t size); + +void ethosu_dma_mem_free(struct ethosu_dma_mem **dma_mem); + +#endif /* ETHOSU_DMA_MEM_H */ diff --git a/kernel/include/rpmsg/ethosu_rpmsg.h b/kernel/include/rpmsg/ethosu_rpmsg.h new file mode 100644 index 0000000..c1923c2 --- /dev/null +++ b/kernel/include/rpmsg/ethosu_rpmsg.h @@ -0,0 +1,248 @@ +/* + * SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +#ifndef ETHOSU_CORE_RPMSG_H +#define ETHOSU_CORE_RPMSG_H + +#ifdef __KERNEL__ +#include +#else +#include +#endif + +#ifdef __cplusplus +namespace EthosU { +#endif + +/** Maximum number of IFM/OFM buffers per inference */ +#define ETHOSU_CORE_BUFFER_MAX 16 + +/** Maximum number of PMU counters to be returned for inference */ +#define ETHOSU_CORE_PMU_MAX 8 + +#define ETHOSU_CORE_MSG_MAGIC 0x41457631 +#define ETHOSU_CORE_MSG_VERSION_MAJOR 0 +#define ETHOSU_CORE_MSG_VERSION_MINOR 2 +#define ETHOSU_CORE_MSG_VERSION_PATCH 0 + +/** + * enum ethosu_core_msg_type - Message types + * + * Types for the messages sent between the host and the core subsystem. + */ +enum ethosu_core_msg_type { + ETHOSU_CORE_MSG_ERR = 1, + ETHOSU_CORE_MSG_PING, + ETHOSU_CORE_MSG_PONG, + ETHOSU_CORE_MSG_INFERENCE_REQ, + ETHOSU_CORE_MSG_INFERENCE_RSP, + ETHOSU_CORE_MSG_VERSION_REQ, + ETHOSU_CORE_MSG_VERSION_RSP, + ETHOSU_CORE_MSG_CAPABILITIES_REQ, + ETHOSU_CORE_MSG_CAPABILITIES_RSP, + ETHOSU_CORE_MSG_NETWORK_INFO_REQ, + ETHOSU_CORE_MSG_NETWORK_INFO_RSP, + ETHOSU_CORE_MSG_CANCEL_INFERENCE_REQ, + ETHOSU_CORE_MSG_CANCEL_INFERENCE_RSP, + ETHOSU_CORE_MSG_MAX +}; + +/** + * struct ethosu_core_msg_header - Message header + */ +struct ethosu_core_msg_header { + uint32_t magic; + uint32_t type; + uint64_t msg_id; +}; + +/** + * enum ethosu_core_status - Status + */ +enum ethosu_core_status { + ETHOSU_CORE_STATUS_OK, + ETHOSU_CORE_STATUS_ERROR, + ETHOSU_CORE_STATUS_RUNNING, + ETHOSU_CORE_STATUS_REJECTED, + ETHOSU_CORE_STATUS_ABORTED, + ETHOSU_CORE_STATUS_ABORTING, +}; + +/** + * struct ethosu_core_buffer - Buffer descriptor + * + * Pointer and size to a buffer within the Ethos-U address space. + */ +struct ethosu_core_buffer { + uint32_t ptr; + uint32_t size; +}; + +/** + * enum ethosu_core_network_type - Network buffer type + */ +enum ethosu_core_network_type { + ETHOSU_CORE_NETWORK_BUFFER = 1, + ETHOSU_CORE_NETWORK_INDEX +}; + +/** + * struct ethosu_core_network_buffer - Network buffer + */ +struct ethosu_core_network_buffer { + uint32_t type; + union { + struct ethosu_core_buffer buffer; + uint32_t index; + }; +}; + +/** + * struct ethosu_core_msg_inference_req - Inference request + */ +struct ethosu_core_msg_inference_req { + uint32_t ifm_count; + struct ethosu_core_buffer ifm[ETHOSU_CORE_BUFFER_MAX]; + uint32_t ofm_count; + struct ethosu_core_buffer ofm[ETHOSU_CORE_BUFFER_MAX]; + struct ethosu_core_network_buffer network; + uint8_t pmu_event_config[ETHOSU_CORE_PMU_MAX]; + uint32_t pmu_cycle_counter_enable; +}; + +/** + * struct ethosu_core_msg_inference_rsp - Inference response + */ +struct ethosu_core_msg_inference_rsp { + uint32_t ofm_count; + uint32_t ofm_size[ETHOSU_CORE_BUFFER_MAX]; + uint32_t status; + uint8_t pmu_event_config[ETHOSU_CORE_PMU_MAX]; + uint64_t pmu_event_count[ETHOSU_CORE_PMU_MAX]; + uint32_t pmu_cycle_counter_enable; + uint64_t pmu_cycle_counter_count; +}; + +/** + * struct ethosu_core_msg_network_info_req - Network information request + */ +struct ethosu_core_msg_network_info_req { + struct ethosu_core_network_buffer network; +}; + +/** + * struct ethosu_core_msg_network_info_rsp - Network information response + */ +struct ethosu_core_msg_network_info_rsp { + char desc[32]; + uint32_t ifm_count; + uint32_t ifm_size[ETHOSU_CORE_BUFFER_MAX]; + uint32_t ofm_count; + uint32_t ofm_size[ETHOSU_CORE_BUFFER_MAX]; + uint32_t status; +}; + +/** + * struct ethosu_core_msg_version_rsp - Message protocol version + */ +struct ethosu_core_msg_version_rsp { + uint8_t major; + uint8_t minor; + uint8_t patch; + uint8_t _reserved; +}; + +/** + * struct ethosu_core_msg_capabilities_rsp - Message capabilities response + */ +struct ethosu_core_msg_capabilities_rsp { + uint32_t version_status; + uint32_t version_minor; + uint32_t version_major; + uint32_t product_major; + uint32_t arch_patch_rev; + uint32_t arch_minor_rev; + uint32_t arch_major_rev; + uint32_t driver_patch_rev; + uint32_t driver_minor_rev; + uint32_t driver_major_rev; + uint32_t macs_per_cc; + uint32_t cmd_stream_version; + uint32_t custom_dma; +}; + +/** + * struct ethosu_core_msg_cancel_inference_req - Message cancel inference + * request + */ +struct ethosu_core_msg_cancel_inference_req { + uint64_t inference_handle; +}; + +/** + * struct ethosu_core_msg_cancel_inference_rsp - Message cancel inference + * response + */ +struct ethosu_core_msg_cancel_inference_rsp { + uint32_t status; +}; + +/** + * enum ethosu_core_err_type - Error types + */ +enum ethosu_core_err_type { + ETHOSU_CORE_MSG_ERR_GENERIC = 0, + ETHOSU_CORE_MSG_ERR_UNSUPPORTED_TYPE, + ETHOSU_CORE_MSG_ERR_INVALID_PAYLOAD, + ETHOSU_CORE_MSG_ERR_INVALID_SIZE, + ETHOSU_CORE_MSG_ERR_INVALID_MAGIC, + ETHOSU_CORE_MSG_ERR_MAX +}; + +/** + * struct ethosu_core_msg_err - Error message struct + */ +struct ethosu_core_msg_err { + uint32_t type; /* optional use of extra error code */ + char msg[128]; +}; + +/** + * struct ethosu_core_rpmsg - Rpmsg message + */ +struct ethosu_core_rpmsg { + struct ethosu_core_msg_header header; + union { + struct ethosu_core_msg_inference_req inf_req; + struct ethosu_core_msg_inference_rsp inf_rsp; + struct ethosu_core_msg_network_info_req net_info_req; + struct ethosu_core_msg_network_info_rsp net_info_rsp; + struct ethosu_core_msg_capabilities_rsp cap_rsp; + struct ethosu_core_msg_cancel_inference_req cancel_req; + struct ethosu_core_msg_cancel_inference_rsp cancel_rsp; + struct ethosu_core_msg_version_rsp version_rsp; + struct ethosu_core_msg_err error; + }; +}; + +#ifdef __cplusplus +} /*namespace EthosU */ +#endif + +#endif /* ETHOSU_CORE_RPMSG_H */ diff --git a/kernel/include/rpmsg/ethosu_rpmsg_cancel_inference.h b/kernel/include/rpmsg/ethosu_rpmsg_cancel_inference.h new file mode 100644 index 0000000..1d6cc03 --- /dev/null +++ b/kernel/include/rpmsg/ethosu_rpmsg_cancel_inference.h @@ -0,0 +1,71 @@ +/* + * SPDX-FileCopyrightText: Copyright 2022-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +#ifndef ETHOSU_CANCEL_INFERENCE_H +#define ETHOSU_CANCEL_INFERENCE_H + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include +#include + +#include +#include + +/**************************************************************************** + * Types + ****************************************************************************/ + +struct ethosu_core_msg_cancel_inference_rsp; +struct ethosu_inference; +struct ethosu_uapi_cancel_inference_status; + +struct ethosu_cancel_inference { + struct device *dev; + struct ethosu_inference *inf; + struct ethosu_uapi_cancel_inference_status *uapi; + struct completion done; + struct ethosu_mailbox_msg msg; + int errno; +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +/** + * ethosu_cancel_inference_request() - Send cancel inference request + * + * Return: 0 on success, error code otherwise. + */ +int ethosu_cancel_inference_request(struct device *dev, + struct ethosu_mailbox *mailbox, + struct ethosu_inference *inf, + struct ethosu_uapi_cancel_inference_status *uapi); + +/** + * ethosu_cancel_inference_rsp() - Handle cancel inference response + */ +void ethosu_cancel_inference_rsp(struct ethosu_mailbox *mailbox, + int msg_id, + struct ethosu_core_msg_cancel_inference_rsp *rsp); + +#endif /* ETHOSU_CANCEL_INFERENCE_H */ diff --git a/kernel/include/rpmsg/ethosu_rpmsg_capabilities.h b/kernel/include/rpmsg/ethosu_rpmsg_capabilities.h new file mode 100644 index 0000000..aefdbaa --- /dev/null +++ b/kernel/include/rpmsg/ethosu_rpmsg_capabilities.h @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: Copyright 2022-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +#ifndef ETHOSU_CAPABILITIES_H +#define ETHOSU_CAPABILITIES_H + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include + +#include +#include + +/**************************************************************************** + * Types + ****************************************************************************/ + +struct ethosu_core_msg_capabilities_rsp; +struct ethosu_device; +struct ethosu_uapi_device_capabilities; + +/** + * struct ethosu_capabilities - Capabilities internal struct + */ +struct ethosu_capabilities { + struct device *dev; + struct completion done; + struct ethosu_uapi_device_capabilities *uapi; + struct ethosu_mailbox_msg msg; + int errno; +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +int ethosu_capabilities_request(struct device *dev, + struct ethosu_mailbox *mailbox, + struct ethosu_uapi_device_capabilities *uapi); + +void ethosu_capability_rsp(struct ethosu_mailbox *mailbox, + int msg_id, + struct ethosu_core_msg_capabilities_rsp *rsp); + +#endif diff --git a/kernel/include/rpmsg/ethosu_rpmsg_inference.h b/kernel/include/rpmsg/ethosu_rpmsg_inference.h new file mode 100644 index 0000000..8bdfb8a --- /dev/null +++ b/kernel/include/rpmsg/ethosu_rpmsg_inference.h @@ -0,0 +1,124 @@ +/* + * SPDX-FileCopyrightText: Copyright 2020, 2022-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +#ifndef ETHOSU_INFERENCE_H +#define ETHOSU_INFERENCE_H + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include +#include + +#include +#include + +/**************************************************************************** + * Types + ****************************************************************************/ + +struct ethosu_buffer; +struct ethosu_core_msg_inference_rsp; +struct ethosu_network; +struct ethosu_uapi_inference_create; +struct file; + +/** + * struct ethosu_inference - Inference struct + * @edev: Arm Ethos-U device + * @file: File handle + * @kref: Reference counter + * @waitq: Wait queue + * @done: Wait condition is done + * @ifm: Pointer to IFM buffer + * @ofm: Pointer to OFM buffer + * @net: Pointer to network + * @status: Inference status + * @pmu_event_config: PMU event configuration + * @pmu_event_count: PMU event count after inference + * @pmu_cycle_counter_enable: PMU cycle counter config + * @pmu_cycle_counter_count: PMU cycle counter count after inference + * @msg: Mailbox message + */ +struct ethosu_inference { + struct device *dev; + struct ethosu_mailbox *mailbox; + struct file *file; + struct kref kref; + wait_queue_head_t waitq; + bool done; + uint32_t ifm_count; + struct ethosu_buffer *ifm[ETHOSU_FD_MAX]; + uint32_t ofm_count; + struct ethosu_buffer *ofm[ETHOSU_FD_MAX]; + struct ethosu_network *net; + enum ethosu_uapi_status status; + uint8_t pmu_event_config[ETHOSU_PMU_EVENT_MAX]; + uint64_t pmu_event_count[ETHOSU_PMU_EVENT_MAX]; + uint32_t pmu_cycle_counter_enable; + uint64_t pmu_cycle_counter_count; + struct ethosu_mailbox_msg msg; +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +/** + * ethosu_inference_create() - Create inference + * + * This function must be called in the context of a user space process. + * + * Return: fd on success, else error code. + */ +int ethosu_inference_create(struct device *dev, + struct ethosu_mailbox *mailbox, + struct ethosu_network *net, + struct ethosu_uapi_inference_create *uapi); + +/** + * ethosu_inference_get_from_fd() - Get inference handle from fd + * + * This function must be called from a user space context. + * + * Return: Pointer on success, else ERR_PTR. + */ +struct ethosu_inference *ethosu_inference_get_from_fd(int fd); + +/** + * ethosu_inference_get() - Get inference + */ +void ethosu_inference_get(struct ethosu_inference *inf); + +/** + * ethosu_inference_put() - Put inference + * + * Return: 1 if object was removed, else 0. + */ +int ethosu_inference_put(struct ethosu_inference *inf); + +/** + * ethosu_inference_rsp() - Handle inference response + */ +void ethosu_inference_rsp(struct ethosu_mailbox *mailbox, + int msg_id, + struct ethosu_core_msg_inference_rsp *rsp); + +#endif /* ETHOSU_INFERENCE_H */ diff --git a/kernel/include/rpmsg/ethosu_rpmsg_mailbox.h b/kernel/include/rpmsg/ethosu_rpmsg_mailbox.h new file mode 100644 index 0000000..66eb172 --- /dev/null +++ b/kernel/include/rpmsg/ethosu_rpmsg_mailbox.h @@ -0,0 +1,194 @@ +/* + * SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +#ifndef ETHOSU_MAILBOX_H +#define ETHOSU_MAILBOX_H + +/**************************************************************************** + * Includes + ****************************************************************************/ +#include + +#include +#include +#include +#include + +/**************************************************************************** + * Types + ****************************************************************************/ + +struct device; +struct ethosu_buffer; +struct ethosu_core_msg; +struct ethosu_core_queue; +struct ethosu_device; +struct ethosu_network; +struct resource; + +typedef void (*ethosu_mailbox_cb)(void *user_arg); + +struct ethosu_mailbox { + struct device *dev; + struct rpmsg_endpoint *ept; + struct idr msg_idr; + atomic_t done; + wait_queue_head_t send_queue; +}; + +/** + * struct ethosu_mailbox_msg - Mailbox message + * @id: Message id + * @type: Message request type + * @fail: Message failure callback + * + * The fail callback will be called with the device mutex locked + */ +struct ethosu_mailbox_msg { + int id; + uint32_t type; + void (*fail)(struct ethosu_mailbox_msg *msg); +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +/** + * ethosu_mailbox_init() - Initialize mailbox + * + * Return: 0 on success, else error code. + */ +int ethosu_mailbox_init(struct ethosu_mailbox *mbox, + struct device *dev, + struct rpmsg_endpoint *ept); + +/** + * ethosu_mailbox_deinit() - Deinitialize mailbox + */ +void ethosu_mailbox_deinit(struct ethosu_mailbox *mbox); + +/** + * ethosu_mailbox_register() - Register the ethosu_mailbox_msg in ethosu_mailbox + * + * Context: Must be called with the device mutex locked + * + * Return: 0 on success, else error code. + */ +int ethosu_mailbox_register(struct ethosu_mailbox *mbox, + struct ethosu_mailbox_msg *msg); + +/** + * ethosu_mailbox_free_id() - Free the id of the ethosu_mailbox_msg + * + * Context: Must be called with the device mutex locked + */ +void ethosu_mailbox_deregister(struct ethosu_mailbox *mbox, + struct ethosu_mailbox_msg *msg); + +/** + * ethosu_mailbox_find() - Find mailbox message + * + * Context: Must be called with the device mutex locked + * + * Return: a valid pointer on success, otherwise an error ptr. + */ +struct ethosu_mailbox_msg *ethosu_mailbox_find(struct ethosu_mailbox *mbox, + int msg_id, + uint32_t msg_type); + +/** + * ethosu_mailbox_fail() - Fail mailbox messages + * + * Call fail() callback on all messages in pending list. + * + * Context: Must be called with the device mutex locked + */ +void ethosu_mailbox_fail(struct ethosu_mailbox *mbox); + +/** + * ethosu_mailbox_reset() - Reset to end of queue + */ +void ethosu_mailbox_reset(struct ethosu_mailbox *mbox); + +/** + * ethosu_mailbox_ping() - Send ping message + * + * Return: 0 on success, else error code. + */ +int ethosu_mailbox_ping(struct ethosu_mailbox *mbox); + +/** + * ethosu_mailbox_pong() - Send pong response + * + * Return: 0 on success, else error code. + */ +int ethosu_mailbox_pong(struct ethosu_mailbox *mbox); + +/** + * ethosu_mailbox_version_request() - Send protocol version request + * + * Return: 0 on succes, else error code + */ +int ethosu_mailbox_version_request(struct ethosu_mailbox *mbox, + struct ethosu_mailbox_msg *msg); + +/** + * ethosu_mailbox_capabilities_request() - Send capabilities request + * + * Return: 0 on success, else error code. + */ +int ethosu_mailbox_capabilities_request(struct ethosu_mailbox *mbox, + struct ethosu_mailbox_msg *msg); + +/** + * ethosu_mailbox_inference() - Send inference + * + * Return: 0 on success, else error code. + */ +int ethosu_mailbox_inference(struct ethosu_mailbox *mbox, + struct ethosu_mailbox_msg *msg, + uint32_t ifm_count, + struct ethosu_buffer **ifm, + uint32_t ofm_count, + struct ethosu_buffer **ofm, + struct ethosu_network *network, + uint8_t *pmu_event_config, + uint8_t pmu_event_config_count, + uint8_t pmu_cycle_counter_enable); + +/** + * ethosu_mailbox_network_info_request() - Send network info request + * + * Return: 0 on success, else error code. + */ +int ethosu_mailbox_network_info_request(struct ethosu_mailbox *mbox, + struct ethosu_mailbox_msg *msg, + struct ethosu_network *network); + +/** + * ethosu_mailbox_cancel_inference() - Send inference cancellation + * + * Return: 0 on success, else error code. + */ +int ethosu_mailbox_cancel_inference(struct ethosu_mailbox *mbox, + struct ethosu_mailbox_msg *msg, + int inference_handle); + +#endif /* ETHOSU_MAILBOX_H */ diff --git a/kernel/include/rpmsg/ethosu_rpmsg_network.h b/kernel/include/rpmsg/ethosu_rpmsg_network.h new file mode 100644 index 0000000..269018a --- /dev/null +++ b/kernel/include/rpmsg/ethosu_rpmsg_network.h @@ -0,0 +1,84 @@ +/* + * SPDX-FileCopyrightText: Copyright 2020, 2022-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +#ifndef ETHOSU_NETWORK_H +#define ETHOSU_NETWORK_H + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include +#include + +/**************************************************************************** + * Types + ****************************************************************************/ + +struct ethosu_buffer; +struct ethosu_uapi_network_create; +struct device; +struct file; + +struct ethosu_network { + struct device *dev; + struct ethosu_mailbox *mailbox; + struct file *file; + struct kref kref; + struct ethosu_dma_mem *dma_mem; + uint32_t index; +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +/** + * ethosu_network_create() - Create network + * + * This function must be called in the context of a user space process. + * + * Return: fd on success, else error code. + */ +int ethosu_network_create(struct device *dev, + struct ethosu_mailbox *mailbox, + struct ethosu_uapi_network_create *uapi); + +/** + * ethosu_network_get_from_fd() - Get network handle from fd + * + * This function must be called from a user space context. + * + * Return: Pointer on success, else ERR_PTR. + */ +struct ethosu_network *ethosu_network_get_from_fd(int fd); + +/** + * ethosu_network_get() - Get network + */ +void ethosu_network_get(struct ethosu_network *net); + +/** + * ethosu_network_put() - Put network + * + * Return: 1 if object was removed, else 0. + */ +int ethosu_network_put(struct ethosu_network *net); + +#endif /* ETHOSU_NETWORK_H */ diff --git a/kernel/include/rpmsg/ethosu_rpmsg_network_info.h b/kernel/include/rpmsg/ethosu_rpmsg_network_info.h new file mode 100644 index 0000000..aaa3733 --- /dev/null +++ b/kernel/include/rpmsg/ethosu_rpmsg_network_info.h @@ -0,0 +1,72 @@ +/* + * SPDX-FileCopyrightText: Copyright 2022-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +#ifndef ETHOSU_NETWORK_INFO_H +#define ETHOSU_NETWORK_INFO_H + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include + +#include +#include + +/**************************************************************************** + * Types + ****************************************************************************/ + +struct ethosu_core_msg_network_info_rsp; +struct ethosu_network; +struct ethosu_uapi_network_info; + +struct ethosu_network_info { + struct device *dev; + struct ethosu_network *net; + struct ethosu_uapi_network_info *uapi; + struct completion done; + int errno; + struct ethosu_mailbox_msg msg; +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +/** + * ethosu_network_info_request() - Send a network info request + * + * This function must be called in the context of a user space process. + * + * Return: 0 on success, . + */ +int ethosu_network_info_request(struct device *dev, + struct ethosu_mailbox *mailbox, + struct ethosu_network *net, + struct ethosu_uapi_network_info *uapi); + +/** + * ethosu_network_info_rsp() - Handle network info response. + */ +void ethosu_network_info_rsp(struct ethosu_mailbox *mailbox, + int msg_id, + struct ethosu_core_msg_network_info_rsp *rsp); + +#endif /* ETHOSU_NETWORK_INFO_H */ diff --git a/kernel/include/rpmsg/ethosu_rpmsg_version.h b/kernel/include/rpmsg/ethosu_rpmsg_version.h new file mode 100644 index 0000000..755a3d5 --- /dev/null +++ b/kernel/include/rpmsg/ethosu_rpmsg_version.h @@ -0,0 +1,59 @@ +/* + * SPDX-FileCopyrightText: Copyright 2023-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +#ifndef ETHOSU_VERSION_H +#define ETHOSU_VERSION_H + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include + +#include +#include + +/**************************************************************************** + * Types + ****************************************************************************/ + +struct ethosu_core_msg_version_rsp; + +/** + * struct ethosu_version - Protocol version internal struct + */ +struct ethosu_version { + struct device *dev; + struct completion done; + struct ethosu_mailbox_msg msg; + int errno; +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +void ethosu_version_rsp(struct ethosu_mailbox *mailbox, + int msg_id, + struct ethosu_core_msg_version_rsp *rsp); + +int ethosu_version_check_request(struct device *dev, + struct ethosu_mailbox *mailbox); + +#endif /* ETHOSU_VERSION_H */ diff --git a/kernel/include/uapi/ethosu.h b/kernel/include/uapi/ethosu.h new file mode 100644 index 0000000..58b9c34 --- /dev/null +++ b/kernel/include/uapi/ethosu.h @@ -0,0 +1,264 @@ +/* + * SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +#ifndef ETHOSU_H +#define ETHOSU_H + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include +#include + +#ifdef __cplusplus +namespace EthosU { +#endif + +/**************************************************************************** + * Defines + ****************************************************************************/ + +#define ETHOSU_IOCTL_BASE 0x01 +#define ETHOSU_IO(nr) _IO(ETHOSU_IOCTL_BASE, nr) +#define ETHOSU_IOR(nr, type) _IOR(ETHOSU_IOCTL_BASE, nr, type) +#define ETHOSU_IOW(nr, type) _IOW(ETHOSU_IOCTL_BASE, nr, type) +#define ETHOSU_IOWR(nr, type) _IOWR(ETHOSU_IOCTL_BASE, nr, type) + +#define ETHOSU_IOCTL_PING ETHOSU_IO(0x00) +#define ETHOSU_IOCTL_CAPABILITIES_REQ ETHOSU_IOR(0x02, \ + struct ethosu_uapi_device_capabilities) +#define ETHOSU_IOCTL_DRIVER_VERSION_GET ETHOSU_IOR(0x03, \ + struct ethosu_uapi_kernel_driver_version) +#define ETHOSU_IOCTL_BUFFER_CREATE ETHOSU_IOR(0x10, \ + struct ethosu_uapi_buffer_create) +#define ETHOSU_IOCTL_NETWORK_CREATE ETHOSU_IOR(0x20, \ + struct ethosu_uapi_network_create) +#define ETHOSU_IOCTL_NETWORK_INFO ETHOSU_IOR(0x21, \ + struct ethosu_uapi_network_info) +#define ETHOSU_IOCTL_INFERENCE_CREATE ETHOSU_IOR(0x30, \ + struct ethosu_uapi_inference_create) +#define ETHOSU_IOCTL_INFERENCE_STATUS ETHOSU_IOR(0x31, \ + struct ethosu_uapi_result_status) +#define ETHOSU_IOCTL_INFERENCE_CANCEL ETHOSU_IOR(0x32, \ + struct ethosu_uapi_cancel_inference_status) + +/* Maximum number of IFM/OFM file descriptors per network */ +#define ETHOSU_FD_MAX 16 + +/* Maximum number of PMUs available */ +#define ETHOSU_PMU_EVENT_MAX 8 + +/* Kernel driver version */ +#define ETHOSU_KERNEL_DRIVER_VERSION_MAJOR 3 +#define ETHOSU_KERNEL_DRIVER_VERSION_MINOR 0 +#define ETHOSU_KERNEL_DRIVER_VERSION_PATCH 0 + +/**************************************************************************** + * Types + ****************************************************************************/ + +/** + * enum ethosu_uapi_status - Status + */ +enum ethosu_uapi_status { + ETHOSU_UAPI_STATUS_OK, + ETHOSU_UAPI_STATUS_ERROR, + ETHOSU_UAPI_STATUS_RUNNING, + ETHOSU_UAPI_STATUS_REJECTED, + ETHOSU_UAPI_STATUS_ABORTED, + ETHOSU_UAPI_STATUS_ABORTING, +}; + +/** + * struct ethosu_uapi_kernel_driver_version - Kernel driver version + * @major: Major version + * @minor: Minor version + * @patch: Patch version + */ +struct ethosu_uapi_kernel_driver_version { + __u32 major; + __u32 minor; + __u32 patch; +}; + +/** + * struct ethosu_uapi_buffer_create - Create buffer request + * @size: Size of the requested buffer + */ +struct ethosu_uapi_buffer_create { + __u32 size; +}; + +/** + * enum ethosu_uapi_network_type - Network buffer type. + * @ETHOSU_UAPI_NETWORK_USER_BUFFER: Network data is provided in a user + * buffer. + * @ETHOSU_UAPI_NETWORK_INDEX: Network is built into firmware and + * referenced by index. + */ +enum ethosu_uapi_network_type { + ETHOSU_UAPI_NETWORK_USER_BUFFER = 1, + ETHOSU_UAPI_NETWORK_INDEX, +}; + +/** + * struct ethosu_uapi_network_create - Create network request + * @type: Buffer type. See @ethosu_uapi_network_type. + * @fd: Buffer file descriptor + * @index: Buffer index compiled into firmware binary. + */ +struct ethosu_uapi_network_create { + uint32_t type; + union { + struct { + __u64 data_ptr; + __u32 size; + } network; + __u32 index; + }; +}; + +/** + * struct ethosu_uapi_network_info - Network info + * @desc: Network description + * @ifm_count: Number of IFM buffers + * @ifm_size: IFM buffer sizes + * @ofm_count: Number of OFM buffers + * @ofm_size: OFM buffer sizes + */ +struct ethosu_uapi_network_info { + char desc[32]; + __u32 ifm_count; + __u32 ifm_size[ETHOSU_FD_MAX]; + __u32 ofm_count; + __u32 ofm_size[ETHOSU_FD_MAX]; +}; + +/** + * struct ethosu_uapi_pmu_config - Configure performance counters + * @events: Array of counters to configure, set to non-zero for + * each counter to enable corresponding event. + * @cycle_count: Set to enable the cycle counter. + */ +struct ethosu_uapi_pmu_config { + __u32 events[ETHOSU_PMU_EVENT_MAX]; + __u32 cycle_count; +}; + +/** + * struct ethosu_uapi_pmu_counts - Status of performance counters + * @events: Count for respective configured events. + * @cycle_count: Count for cycle counter. + */ +struct ethosu_uapi_pmu_counts { + __u64 events[ETHOSU_PMU_EVENT_MAX]; + __u64 cycle_count; +}; + +/** + * struct ethosu_uapi_device_hw_id - Device hardware identification + * @version_status: Version status + * @version_minor: Version minor + * @version_major: Version major + * @product_major: Product major + * @arch_patch_rev: Architecture version patch + * @arch_minor_rev: Architecture version minor + * @arch_major_rev: Architecture version major + */ +struct ethosu_uapi_device_hw_id { + __u32 version_status; + __u32 version_minor; + __u32 version_major; + __u32 product_major; + __u32 arch_patch_rev; + __u32 arch_minor_rev; + __u32 arch_major_rev; +}; + +/** + * struct ethosu_uapi_device_hw_cfg - Device hardware configuration + * @macs_per_cc: MACs per clock cycle + * @cmd_stream_version: NPU command stream version + * @custom_dma: Custom DMA enabled + */ +struct ethosu_uapi_device_hw_cfg { + __u32 macs_per_cc; + __u32 cmd_stream_version; + __u32 custom_dma; +}; + +/** + * struct ethosu_uapi_device_capabilities - Device capabilities + * @hw_id: Hardware identification + * @hw_cfg: Hardware configuration + * @driver_patch_rev: Driver version patch + * @driver_minor_rev: Driver version minor + * @driver_major_rev: Driver version major + */ +struct ethosu_uapi_device_capabilities { + struct ethosu_uapi_device_hw_id hw_id; + struct ethosu_uapi_device_hw_cfg hw_cfg; + __u32 driver_patch_rev; + __u32 driver_minor_rev; + __u32 driver_major_rev; +}; + +/** + * struct ethosu_uapi_inference_create - Create inference request + * @ifm_count: Number of IFM file descriptors + * @ifm_fd: IFM buffer file descriptors + * @ofm_count: Number of OFM file descriptors + * @ofm_fd: OFM buffer file descriptors + */ +struct ethosu_uapi_inference_create { + __u32 ifm_count; + __u32 ifm_fd[ETHOSU_FD_MAX]; + __u32 ofm_count; + __u32 ofm_fd[ETHOSU_FD_MAX]; + struct ethosu_uapi_pmu_config pmu_config; +}; + +/** + * struct ethosu_uapi_result_status - Status of inference + * @status Status of run inference. + * @pmu_config Configured performance counters. + * @pmu_count Perfomance counters values, when status is + * ETHOSU_UAPI_STATUS_OK. + */ +struct ethosu_uapi_result_status { + __u32 status; + struct ethosu_uapi_pmu_config pmu_config; + struct ethosu_uapi_pmu_counts pmu_count; +}; + +/** + * struct ethosu_uapi_cancel_inference_status - Status of inference + * cancellation. + * @status OK if inference cancellation was performed, ERROR otherwise. + */ +struct ethosu_uapi_cancel_inference_status { + __u32 status; +}; + +#ifdef __cplusplus +} /* namespace EthosU */ +#endif +#endif diff --git a/kernel/rpmsg/ethosu_rpmsg_cancel_inference.c b/kernel/rpmsg/ethosu_rpmsg_cancel_inference.c new file mode 100644 index 0000000..5d46434 --- /dev/null +++ b/kernel/rpmsg/ethosu_rpmsg_cancel_inference.c @@ -0,0 +1,191 @@ +/* + * SPDX-FileCopyrightText: Copyright 2022-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include + +#include +#include +#include + +#include +#include + +/**************************************************************************** + * Defines + ****************************************************************************/ + +#define CANCEL_INFERENCE_RESP_TIMEOUT_MS 2000 + +/**************************************************************************** + * Functions + ****************************************************************************/ + +static int ethosu_cancel_inference_send( + struct ethosu_cancel_inference *cancellation, + struct ethosu_mailbox *mailbox) +{ + return ethosu_mailbox_cancel_inference(mailbox, + &cancellation->msg, + cancellation->inf->msg.id); +} + +static void ethosu_cancel_inference_fail(struct ethosu_mailbox_msg *msg) +{ + struct ethosu_cancel_inference *cancellation = + container_of(msg, typeof(*cancellation), msg); + + if (completion_done(&cancellation->done)) + return; + + cancellation->errno = -EFAULT; + cancellation->uapi->status = ETHOSU_UAPI_STATUS_ERROR; + complete(&cancellation->done); +} + +int ethosu_cancel_inference_request(struct device *dev, + struct ethosu_mailbox *mailbox, + struct ethosu_inference *inf, + struct ethosu_uapi_cancel_inference_status *uapi) +{ + struct ethosu_cancel_inference *cancellation; + int ret; + int timeout; + + if (inf->done) { + uapi->status = ETHOSU_UAPI_STATUS_ERROR; + + return 0; + } + + cancellation = + devm_kzalloc(dev, + sizeof(struct ethosu_cancel_inference), + GFP_KERNEL); + if (!cancellation) + return -ENOMEM; + + /* increase ref count on the inference we are refering to */ + ethosu_inference_get(inf); + /* mark inference ABORTING to avoid resending the inference message */ + inf->status = ETHOSU_UAPI_STATUS_ABORTING; + + cancellation->dev = dev; + cancellation->inf = inf; + cancellation->uapi = uapi; + init_completion(&cancellation->done); + cancellation->msg.fail = ethosu_cancel_inference_fail; + + ret = ethosu_mailbox_register(mailbox, + &cancellation->msg); + if (ret < 0) + goto kfree; + + dev_dbg(dev, + "Inference cancellation create. cancel=0x%pK, msg.id=%ddev", + cancellation, cancellation->msg.id); + + ret = ethosu_cancel_inference_send(cancellation, mailbox); + if (0 != ret) + goto deregister; + + /* Unlock the mutex before going to block on the condition */ + device_unlock(dev); + + /* wait for response to arrive back */ + timeout = wait_for_completion_timeout(&cancellation->done, + msecs_to_jiffies( + CANCEL_INFERENCE_RESP_TIMEOUT_MS)); + /* take back the mutex before resuming to do anything */ + ret = device_lock_interruptible(dev); + if (0 != ret) + goto deregister; + + if (0 == timeout /* timed out*/) { + dev_warn(dev, + "Msg: Cancel Inference response lost - timeoutdev"); + ret = -EIO; + + rproc_report_crash(rproc_get_by_child(dev), RPROC_FATAL_ERROR); + goto deregister; + } + + if (cancellation->errno) { + ret = cancellation->errno; + rproc_report_crash(rproc_get_by_child(dev), RPROC_FATAL_ERROR); + goto deregister; + } + + if (inf->status != ETHOSU_UAPI_STATUS_ABORTED) + inf->status = ETHOSU_UAPI_STATUS_ABORTED; + +deregister: + ethosu_mailbox_deregister(mailbox, + &cancellation->msg); + +kfree: + dev_dbg(dev, + "Cancel inference destroy. cancel=0x%pK", cancellation); + + /* decrease the reference on the inference we are refering to */ + ethosu_inference_put(cancellation->inf); + devm_kfree(dev, cancellation); + + return ret; +} + +void ethosu_cancel_inference_rsp(struct ethosu_mailbox *mailbox, + int msg_id, + struct ethosu_core_msg_cancel_inference_rsp *rsp) +{ + struct device *dev = mailbox->dev; + struct ethosu_mailbox_msg *msg; + struct ethosu_cancel_inference *cancellation; + + msg = ethosu_mailbox_find(mailbox, msg_id, + ETHOSU_CORE_MSG_CANCEL_INFERENCE_REQ); + if (IS_ERR(msg)) { + dev_warn(dev, + "Id for cancel inference msg not found. Id=0x%x: %ld", + msg_id, PTR_ERR(msg)); + + return; + } + + cancellation = container_of(msg, typeof(*cancellation), msg); + + if (completion_done(&cancellation->done)) + return; + + cancellation->errno = 0; + switch (rsp->status) { + case ETHOSU_CORE_STATUS_OK: + cancellation->uapi->status = ETHOSU_UAPI_STATUS_OK; + break; + case ETHOSU_CORE_STATUS_ERROR: + cancellation->uapi->status = ETHOSU_UAPI_STATUS_ERROR; + break; + } + + complete(&cancellation->done); +} diff --git a/kernel/rpmsg/ethosu_rpmsg_capabilities.c b/kernel/rpmsg/ethosu_rpmsg_capabilities.c new file mode 100644 index 0000000..73ce2dc --- /dev/null +++ b/kernel/rpmsg/ethosu_rpmsg_capabilities.c @@ -0,0 +1,164 @@ +/* + * SPDX-FileCopyrightText: Copyright 2022-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + * + */ + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include + +#include +#include + +#include +#include +#include + +/**************************************************************************** + * Defines + ****************************************************************************/ + +#define CAPABILITIES_RESP_TIMEOUT_MS 2000 + +/**************************************************************************** + * Functions + ****************************************************************************/ + +static inline int ethosu_capabilities_send(struct ethosu_capabilities *cap, + struct ethosu_mailbox *mailbox) +{ + return ethosu_mailbox_capabilities_request(mailbox, + &cap->msg); +} + +static void ethosu_capabilities_fail(struct ethosu_mailbox_msg *msg) +{ + struct ethosu_capabilities *cap = + container_of(msg, typeof(*cap), msg); + + if (completion_done(&cap->done)) + return; + + cap->errno = -EFAULT; + complete(&cap->done); +} + +void ethosu_capability_rsp(struct ethosu_mailbox *mailbox, + int msg_id, + struct ethosu_core_msg_capabilities_rsp *rsp) +{ + struct device *dev = mailbox->dev; + struct ethosu_mailbox_msg *msg; + struct ethosu_capabilities *cap; + + msg = ethosu_mailbox_find(mailbox, msg_id, + ETHOSU_CORE_MSG_CAPABILITIES_REQ); + if (IS_ERR(msg)) { + dev_warn(dev, + "Id for capabilities msg not found. Id=0x%0x: %ld\n", + msg_id, PTR_ERR(msg)); + + return; + } + + cap = container_of(msg, typeof(*cap), msg); + + if (completion_done(&cap->done)) + return; + + cap->uapi->hw_id.version_status = rsp->version_status; + cap->uapi->hw_id.version_minor = rsp->version_minor; + cap->uapi->hw_id.version_major = rsp->version_major; + cap->uapi->hw_id.product_major = rsp->product_major; + cap->uapi->hw_id.arch_patch_rev = rsp->arch_patch_rev; + cap->uapi->hw_id.arch_minor_rev = rsp->arch_minor_rev; + cap->uapi->hw_id.arch_major_rev = rsp->arch_major_rev; + cap->uapi->driver_patch_rev = rsp->driver_patch_rev; + cap->uapi->driver_minor_rev = rsp->driver_minor_rev; + cap->uapi->driver_major_rev = rsp->driver_major_rev; + cap->uapi->hw_cfg.macs_per_cc = rsp->macs_per_cc; + cap->uapi->hw_cfg.cmd_stream_version = rsp->cmd_stream_version; + cap->uapi->hw_cfg.custom_dma = rsp->custom_dma; + + cap->errno = 0; + complete(&cap->done); +} + +int ethosu_capabilities_request(struct device *dev, + struct ethosu_mailbox *mailbox, + struct ethosu_uapi_device_capabilities *uapi) +{ + struct ethosu_capabilities *cap; + int ret; + int timeout; + + cap = devm_kzalloc(dev, sizeof(struct ethosu_capabilities), + GFP_KERNEL); + if (!cap) + return -ENOMEM; + + cap->dev = dev; + cap->uapi = uapi; + init_completion(&cap->done); + cap->msg.fail = ethosu_capabilities_fail; + + ret = ethosu_mailbox_register(mailbox, &cap->msg); + if (ret < 0) + goto kfree; + + dev_dbg(dev, "Capabilities create. Id=%d, handle=0x%p\n", + cap->msg.id, cap); + + ret = ethosu_capabilities_send(cap, mailbox); + if (0 != ret) + goto deregister; + + /* Unlock the mutex before going to block on the condition */ + device_unlock(dev); + + /* wait for response to arrive back */ + timeout = wait_for_completion_timeout(&cap->done, + msecs_to_jiffies( + CAPABILITIES_RESP_TIMEOUT_MS)); + + /* take back the mutex before resuming to do anything */ + device_lock(dev); + + if (0 == timeout) { + dev_warn(dev, "Capabilities response timeout"); + ret = -ETIME; + goto deregister; + } + + if (cap->errno) { + ret = cap->errno; + goto deregister; + } + +deregister: + ethosu_mailbox_deregister(mailbox, &cap->msg); + +kfree: + dev_dbg(dev, "Capabilities destroy. Id=%d, handle=0x%p\n", + cap->msg.id, cap); + devm_kfree(dev, cap); + + return ret; +} diff --git a/kernel/rpmsg/ethosu_rpmsg_inference.c b/kernel/rpmsg/ethosu_rpmsg_inference.c new file mode 100644 index 0000000..b721fb7 --- /dev/null +++ b/kernel/rpmsg/ethosu_rpmsg_inference.c @@ -0,0 +1,486 @@ +/* + * SPDX-FileCopyrightText: Copyright 2020, 2022-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +/**************************************************************************** + * Variables + ****************************************************************************/ + +static int ethosu_inference_release(struct inode *inode, + struct file *file); + +static __poll_t ethosu_inference_poll(struct file *file, + poll_table *wait); + +static long ethosu_inference_ioctl(struct file *file, + unsigned int cmd, + unsigned long arg); + +static const struct file_operations ethosu_inference_fops = { + .release = ðosu_inference_release, + .poll = ðosu_inference_poll, + .unlocked_ioctl = ðosu_inference_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ðosu_inference_ioctl, +#endif +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +static const char *status_to_string(const enum ethosu_uapi_status status) +{ + switch (status) { + case ETHOSU_UAPI_STATUS_OK: { + return "Ok"; + } + case ETHOSU_UAPI_STATUS_ERROR: { + return "Error"; + } + case ETHOSU_UAPI_STATUS_RUNNING: { + return "Running"; + } + case ETHOSU_UAPI_STATUS_REJECTED: { + return "Rejected"; + } + case ETHOSU_UAPI_STATUS_ABORTED: { + return "Aborted"; + } + case ETHOSU_UAPI_STATUS_ABORTING: { + return "Aborting"; + } + default: { + return "Unknown"; + } + } +} + +static int ethosu_inference_send(struct ethosu_inference *inf) +{ + struct device *dev = inf->dev; + int ret; + + inf->status = ETHOSU_UAPI_STATUS_ERROR; + + ret = ethosu_mailbox_inference(inf->mailbox, &inf->msg, + inf->ifm_count, inf->ifm, + inf->ofm_count, inf->ofm, + inf->net, inf->pmu_event_config, + ETHOSU_PMU_EVENT_MAX, + inf->pmu_cycle_counter_enable); + if (ret) { + dev_warn(dev, + "Failed to send inference request. inf=0x%pK, ret=%d", + inf, ret); + + return ret; + } + + inf->status = ETHOSU_UAPI_STATUS_RUNNING; + + ethosu_inference_get(inf); + + return 0; +} + +static void ethosu_inference_fail(struct ethosu_mailbox_msg *msg) +{ + struct ethosu_inference *inf = + container_of(msg, typeof(*inf), msg); + int ret; + + if (inf->done) + return; + + /* Decrement reference count if inference was pending reponse */ + ret = ethosu_inference_put(inf); + if (ret) + return; + + /* Set status accordingly to the inference state */ + inf->status = inf->status == ETHOSU_UAPI_STATUS_ABORTING ? + ETHOSU_UAPI_STATUS_ABORTED : + ETHOSU_UAPI_STATUS_ERROR; + /* Mark it done and wake up the waiting process */ + inf->done = true; + wake_up_interruptible(&inf->waitq); +} + +static bool ethosu_inference_verify(struct file *file) +{ + return file->f_op == ðosu_inference_fops; +} + +static void ethosu_inference_kref_destroy(struct kref *kref) +{ + struct ethosu_inference *inf = + container_of(kref, struct ethosu_inference, kref); + struct device *dev = inf->dev; + + dev_dbg(dev, + "Inference destroy. inf=0x%pK, status=%d, ifm_count=%u, ofm_count=%u", + inf, inf->status, inf->ifm_count, inf->ofm_count); + + ethosu_mailbox_deregister(inf->mailbox, &inf->msg); + + while (inf->ifm_count-- > 0) + ethosu_buffer_put(inf->ifm[inf->ifm_count]); + + while (inf->ofm_count-- > 0) + ethosu_buffer_put(inf->ofm[inf->ofm_count]); + + ethosu_network_put(inf->net); + memset(inf, 0, sizeof(*inf)); + devm_kfree(dev, inf); +} + +static int ethosu_inference_release(struct inode *inode, + struct file *file) +{ + struct ethosu_inference *inf = file->private_data; + struct device *dev = inf->dev; + + dev_dbg(dev, + "Inference release. file=0x%pK, inf=0x%pK", + file, inf); + + device_lock(dev); + ethosu_inference_put(inf); + device_unlock(dev); + + return 0; +} + +static __poll_t ethosu_inference_poll(struct file *file, + poll_table *wait) +{ + struct ethosu_inference *inf = file->private_data; + __poll_t ret = 0; + + poll_wait(file, &inf->waitq, wait); + + if (inf->done) + ret |= EPOLLIN; + + return ret; +} + +static long ethosu_inference_ioctl(struct file *file, + unsigned int cmd, + unsigned long arg) +{ + struct ethosu_inference *inf = file->private_data; + struct device *dev = inf->dev; + void __user *udata = (void __user *)arg; + int ret; + + ret = device_lock_interruptible(dev); + if (ret) + return ret; + + switch (cmd) { + case ETHOSU_IOCTL_INFERENCE_STATUS: { + struct ethosu_uapi_result_status uapi = { 0 }; + int i; + + uapi.status = inf->status; + + for (i = 0; i < ETHOSU_PMU_EVENT_MAX; i++) { + uapi.pmu_config.events[i] = + inf->pmu_event_config[i]; + uapi.pmu_count.events[i] = + inf->pmu_event_count[i]; + } + + uapi.pmu_config.cycle_count = inf->pmu_cycle_counter_enable; + uapi.pmu_count.cycle_count = inf->pmu_cycle_counter_count; + + dev_dbg(dev, + "Inference ioctl: Inference status. status=%s (%d)\n", + status_to_string(uapi.status), uapi.status); + + ret = copy_to_user(udata, &uapi, sizeof(uapi)) ? -EFAULT : 0; + + break; + } + case ETHOSU_IOCTL_INFERENCE_CANCEL: { + struct ethosu_uapi_cancel_inference_status uapi = { 0 }; + + dev_dbg(dev, + "Inference ioctl: Cancel Inference. Handle=%p\n", + inf); + + ret = ethosu_cancel_inference_request(dev, inf->mailbox, inf, + &uapi); + if (ret) + break; + + ret = copy_to_user(udata, &uapi, sizeof(uapi)) ? -EFAULT : 0; + + break; + } + default: { + dev_err(dev, "Invalid ioctl. cmd=%u, arg=%lu\n", + cmd, arg); + ret = -ENOIOCTLCMD; + break; + } + } + + device_unlock(dev); + + return ret; +} + +int ethosu_inference_create(struct device *dev, + struct ethosu_mailbox *mailbox, + struct ethosu_network *net, + struct ethosu_uapi_inference_create *uapi) +{ + struct ethosu_inference *inf; + uint32_t i; + int fd; + int ret = -ENOMEM; + + if (uapi->ifm_count > ETHOSU_FD_MAX || + uapi->ofm_count > ETHOSU_FD_MAX) { + dev_warn(dev, + "Too many IFM and/or OFM buffers for inference. ifm_count=%u, ofm_count=%u", + uapi->ifm_count, uapi->ofm_count); + + return -EFAULT; + } + + inf = devm_kzalloc(dev, sizeof(*inf), GFP_KERNEL); + if (!inf) + return -ENOMEM; + + inf->dev = dev; + inf->mailbox = mailbox; + inf->net = net; + inf->done = false; + inf->status = ETHOSU_UAPI_STATUS_ERROR; + kref_init(&inf->kref); + init_waitqueue_head(&inf->waitq); + inf->msg.fail = ethosu_inference_fail; + + /* Add inference to pending list */ + ret = ethosu_mailbox_register(mailbox, &inf->msg); + if (ret < 0) + goto kfree; + + /* Get pointer to IFM buffers */ + for (i = 0; i < uapi->ifm_count; i++) { + inf->ifm[i] = ethosu_buffer_get_from_fd(uapi->ifm_fd[i]); + if (IS_ERR(inf->ifm[i])) { + ret = PTR_ERR(inf->ifm[i]); + goto put_ifm; + } + + inf->ifm_count++; + } + + /* Get pointer to OFM buffer */ + for (i = 0; i < uapi->ofm_count; i++) { + inf->ofm[i] = ethosu_buffer_get_from_fd(uapi->ofm_fd[i]); + if (IS_ERR(inf->ofm[i])) { + ret = PTR_ERR(inf->ofm[i]); + goto put_ofm; + } + + inf->ofm_count++; + } + + /* Configure PMU and cycle counter */ + dev_dbg(dev, + "Configuring events for PMU. events=[%u, %u, %u, %u]\n", + uapi->pmu_config.events[0], uapi->pmu_config.events[1], + uapi->pmu_config.events[2], uapi->pmu_config.events[3]); + + /* Configure events and reset count for all events */ + for (i = 0; i < ETHOSU_PMU_EVENT_MAX; i++) { + inf->pmu_event_config[i] = uapi->pmu_config.events[i]; + inf->pmu_event_count[i] = 0; + } + + /* Configure cycle counter and reset any previous count */ + inf->pmu_cycle_counter_enable = uapi->pmu_config.cycle_count; + inf->pmu_cycle_counter_count = 0; + + /* Increment network reference count */ + ethosu_network_get(net); + + /* Send inference request to Arm Ethos-U subsystem */ + ret = ethosu_inference_send(inf); + if (ret) + goto put_net; + + /* Create file descriptor */ + ret = fd = anon_inode_getfd("ethosu-inference", ðosu_inference_fops, + inf, O_RDWR | O_CLOEXEC); + if (ret < 0) + goto put_net; + + /* Store pointer to file structure */ + inf->file = fget(ret); + fput(inf->file); + + dev_dbg(dev, + "Inference create. file=0x%pK, fd=%d, inf=0x%p, net=0x%pK, msg.id=0x%x", + inf->file, fd, inf, inf->net, inf->msg.id); + + return fd; + +put_net: + ethosu_network_put(inf->net); + +put_ofm: + while (inf->ofm_count-- > 0) + ethosu_buffer_put(inf->ofm[inf->ofm_count]); + +put_ifm: + while (inf->ifm_count-- > 0) + ethosu_buffer_put(inf->ifm[inf->ifm_count]); + + ethosu_mailbox_deregister(mailbox, &inf->msg); + +kfree: + memset(inf, 0, sizeof(*inf)); + devm_kfree(dev, inf); + + return ret; +} + +struct ethosu_inference *ethosu_inference_get_from_fd(int fd) +{ + struct ethosu_inference *inf; + struct file *file; + + file = fget(fd); + if (!file) + return ERR_PTR(-EINVAL); + + if (!ethosu_inference_verify(file)) { + fput(file); + + return ERR_PTR(-EINVAL); + } + + inf = file->private_data; + ethosu_inference_get(inf); + fput(file); + + return inf; +} + +void ethosu_inference_get(struct ethosu_inference *inf) +{ + kref_get(&inf->kref); +} + +int ethosu_inference_put(struct ethosu_inference *inf) +{ + return kref_put(&inf->kref, ðosu_inference_kref_destroy); +} + +void ethosu_inference_rsp(struct ethosu_mailbox *mailbox, + int msg_id, + struct ethosu_core_msg_inference_rsp *rsp) +{ + struct device *dev = mailbox->dev; + struct ethosu_mailbox_msg *msg; + struct ethosu_inference *inf; + int i; + + msg = ethosu_mailbox_find(mailbox, msg_id, + ETHOSU_CORE_MSG_INFERENCE_REQ); + if (IS_ERR(msg)) { + dev_warn(dev, + "Id for inference msg not found. Id=0x%x: %ld\n", + msg_id, PTR_ERR(msg)); + + return; + } + + inf = container_of(msg, typeof(*inf), msg); + + /* + * Don't handle the response if the inference is aborted or + * in the process of being aborted + */ + if (inf->status == ETHOSU_UAPI_STATUS_ABORTED || + inf->status == ETHOSU_UAPI_STATUS_ABORTING) { + inf->status = ETHOSU_UAPI_STATUS_ABORTED; + goto done; + } + + if (rsp->status == ETHOSU_CORE_STATUS_OK && + inf->ofm_count <= ETHOSU_CORE_BUFFER_MAX) + inf->status = ETHOSU_UAPI_STATUS_OK; + else if (rsp->status == ETHOSU_CORE_STATUS_REJECTED) + inf->status = ETHOSU_UAPI_STATUS_REJECTED; + else if (rsp->status == ETHOSU_CORE_STATUS_ABORTED) + inf->status = ETHOSU_UAPI_STATUS_ABORTED; + else + inf->status = ETHOSU_UAPI_STATUS_ERROR; + + if (inf->status == ETHOSU_UAPI_STATUS_OK) { + for (i = 0; i < ETHOSU_CORE_PMU_MAX; i++) { + inf->pmu_event_config[i] = rsp->pmu_event_config[i]; + inf->pmu_event_count[i] = rsp->pmu_event_count[i]; + } + + inf->pmu_cycle_counter_enable = rsp->pmu_cycle_counter_enable; + inf->pmu_cycle_counter_count = rsp->pmu_cycle_counter_count; + + dev_dbg(dev, + "PMU events. config=[%u, %u, %u, %u], count=[%llu, %llu, %llu, %llu]\n", + inf->pmu_event_config[0], inf->pmu_event_config[1], + inf->pmu_event_config[2], inf->pmu_event_config[3], + inf->pmu_event_count[0], inf->pmu_event_count[1], + inf->pmu_event_count[2], inf->pmu_event_count[3]); + + if (inf->pmu_cycle_counter_enable) + dev_dbg(dev, + "PMU cycle counter: count=%llu\n", + inf->pmu_cycle_counter_count); + } + +done: + inf->done = true; + wake_up_interruptible(&inf->waitq); + ethosu_inference_put(inf); +} diff --git a/kernel/rpmsg/ethosu_rpmsg_mailbox.c b/kernel/rpmsg/ethosu_rpmsg_mailbox.c new file mode 100644 index 0000000..11ac414 --- /dev/null +++ b/kernel/rpmsg/ethosu_rpmsg_mailbox.c @@ -0,0 +1,363 @@ +/* + * SPDX-FileCopyrightText: Copyright 2020-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include + +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#ifndef fallthrough +#if __has_attribute(__fallthrough__) +#define fallthrough __attribute__((__fallthrough__)) +#else +#define fallthrough do {} while (0) /* fallthrough */ +#endif +#endif + +/**************************************************************************** + * Defines + ****************************************************************************/ + +#define MAILBOX_SEND_TIMEOUT_MS 15000 + +/**************************************************************************** + * Functions + ****************************************************************************/ + +/** + * ethosu_send_locked() - Blocking mailbox message sender + * + * Context: Can sleep and must be called with the device mutex locked. + * + * Return: 0 on success, else error code. + */ +static int ethosu_send_locked(struct ethosu_mailbox *mbox, + void *data, + size_t length) +{ + DEFINE_WAIT_FUNC(wait, woken_wake_function); + struct device *dev = mbox->dev; + long timeout = msecs_to_jiffies(MAILBOX_SEND_TIMEOUT_MS); + bool try_send = !wq_has_sleeper(&mbox->send_queue); + int ret; + + might_sleep(); + + /* Exclusive wait to only wake up one task at a time */ + add_wait_queue_exclusive(&mbox->send_queue, &wait); + for (;;) { + /* Stop if the mailbox is closing down */ + if (atomic_read(&mbox->done)) { + ret = -ENODEV; + break; + } + + /* Attempt to send if queue is empty or task was woken up */ + if (try_send) { + ret = rpmsg_trysend(mbox->ept, data, length); + if (ret != -ENOMEM) + break; + } else { + try_send = true; + } + + /* Unlock device mutex while waiting to not block other tasks */ + device_unlock(dev); + timeout = wait_woken(&wait, TASK_INTERRUPTIBLE, timeout); + device_lock(dev); + + /* Stop if the wait was interrupted */ + if (signal_pending(current)) { + ret = -EINTR; + break; + } + + if (!timeout) { + ret = -ETIME; + break; + } + } + + remove_wait_queue(&mbox->send_queue, &wait); + + /* + * If the message was sent successfully, there may be more TX buffers + * available so wake up the next waiting task. + */ + if (!ret && wq_has_sleeper(&mbox->send_queue)) + wake_up(&mbox->send_queue); + + return ret; +} + +static void ethosu_core_buffer_dma_mem_set(struct ethosu_dma_mem *dma_mem, + struct ethosu_core_buffer *cbuf) +{ + cbuf->ptr = (uint32_t)dma_mem->dma_addr; + cbuf->size = (uint32_t)dma_mem->size; +} + +int ethosu_mailbox_register(struct ethosu_mailbox *mbox, + struct ethosu_mailbox_msg *msg) +{ + WARN_ON_ONCE(!mutex_is_locked(&mbox->dev->mutex)); + msg->id = idr_alloc_cyclic(&mbox->msg_idr, msg, 0, INT_MAX, GFP_KERNEL); + if (msg->id < 0) + return msg->id; + + return 0; +} + +void ethosu_mailbox_deregister(struct ethosu_mailbox *mbox, + struct ethosu_mailbox_msg *msg) +{ + WARN_ON_ONCE(!mutex_is_locked(&mbox->dev->mutex)); + idr_remove(&mbox->msg_idr, msg->id); +} + +struct ethosu_mailbox_msg *ethosu_mailbox_find(struct ethosu_mailbox *mbox, + int msg_id, + uint32_t msg_type) +{ + struct ethosu_mailbox_msg *ptr; + + WARN_ON_ONCE(!mutex_is_locked(&mbox->dev->mutex)); + ptr = (struct ethosu_mailbox_msg *)idr_find(&mbox->msg_idr, msg_id); + + if (ptr == NULL) + return ERR_PTR(-ENOENT); + + if (ptr->type != msg_type) + return ERR_PTR(-EINVAL); + + return ptr; +} + +void ethosu_mailbox_fail(struct ethosu_mailbox *mbox) +{ + struct ethosu_mailbox_msg *cur; + int id; + + WARN_ON_ONCE(!mutex_is_locked(&mbox->dev->mutex)); + idr_for_each_entry(&mbox->msg_idr, cur, id) { + cur->fail(cur); + } +} + +int ethosu_mailbox_ping(struct ethosu_mailbox *mbox) +{ + struct ethosu_core_rpmsg rpmsg = { + .header = { + .magic = ETHOSU_CORE_MSG_MAGIC, + .type = ETHOSU_CORE_MSG_PING, + } + }; + + return ethosu_send_locked(mbox, &rpmsg, sizeof(rpmsg.header)); +} + +int ethosu_mailbox_pong(struct ethosu_mailbox *mbox) +{ + struct ethosu_core_rpmsg rpmsg = { + .header = { + .magic = ETHOSU_CORE_MSG_MAGIC, + .type = ETHOSU_CORE_MSG_PONG, + } + }; + + return ethosu_send_locked(mbox, &rpmsg, sizeof(rpmsg.header)); +} + +int ethosu_mailbox_version_request(struct ethosu_mailbox *mbox, + struct ethosu_mailbox_msg *msg) +{ + struct ethosu_core_rpmsg rpmsg = { + .header = { + .magic = ETHOSU_CORE_MSG_MAGIC, + .type = ETHOSU_CORE_MSG_VERSION_REQ, + .msg_id = msg->id + } + }; + + msg->type = rpmsg.header.type; + + return ethosu_send_locked(mbox, &rpmsg, sizeof(rpmsg.header)); +} + +int ethosu_mailbox_capabilities_request(struct ethosu_mailbox *mbox, + struct ethosu_mailbox_msg *msg) +{ + struct ethosu_core_rpmsg rpmsg = { + .header = { + .magic = ETHOSU_CORE_MSG_MAGIC, + .type = ETHOSU_CORE_MSG_CAPABILITIES_REQ, + .msg_id = msg->id + } + }; + + msg->type = rpmsg.header.type; + + return ethosu_send_locked(mbox, &rpmsg, sizeof(rpmsg.header)); +} + +int ethosu_mailbox_inference(struct ethosu_mailbox *mbox, + struct ethosu_mailbox_msg *msg, + uint32_t ifm_count, + struct ethosu_buffer **ifm, + uint32_t ofm_count, + struct ethosu_buffer **ofm, + struct ethosu_network *network, + uint8_t *pmu_event_config, + uint8_t pmu_event_config_count, + uint8_t pmu_cycle_counter_enable) +{ + struct ethosu_core_rpmsg rpmsg = { + .header = { + .magic = ETHOSU_CORE_MSG_MAGIC, + .type = ETHOSU_CORE_MSG_INFERENCE_REQ, + .msg_id = msg->id + } + }; + struct ethosu_core_msg_inference_req *inf_req = &rpmsg.inf_req; + uint32_t i; + + msg->type = rpmsg.header.type; + + /* Verify that the uapi and core has the same number of pmus */ + if (pmu_event_config_count != ETHOSU_CORE_PMU_MAX) { + dev_err(mbox->dev, "PMU count misconfigured."); + + return -EINVAL; + } + + inf_req->ifm_count = ifm_count; + inf_req->ofm_count = ofm_count; + inf_req->pmu_cycle_counter_enable = pmu_cycle_counter_enable; + + for (i = 0; i < ifm_count; i++) + ethosu_core_buffer_dma_mem_set(ifm[i]->dma_mem, + &inf_req->ifm[i]); + + for (i = 0; i < ofm_count; i++) + ethosu_core_buffer_dma_mem_set(ofm[i]->dma_mem, + &inf_req->ofm[i]); + + for (i = 0; i < ETHOSU_CORE_PMU_MAX; i++) + inf_req->pmu_event_config[i] = pmu_event_config[i]; + + if (network->dma_mem != NULL) { + inf_req->network.type = ETHOSU_CORE_NETWORK_BUFFER; + ethosu_core_buffer_dma_mem_set(network->dma_mem, + &inf_req->network.buffer); + } else { + inf_req->network.type = ETHOSU_CORE_NETWORK_INDEX; + inf_req->network.index = network->index; + } + + return ethosu_send_locked(mbox, &rpmsg, + sizeof(rpmsg.header) + sizeof(rpmsg.inf_req)); +} + +int ethosu_mailbox_network_info_request(struct ethosu_mailbox *mbox, + struct ethosu_mailbox_msg *msg, + struct ethosu_network *network) +{ + struct ethosu_core_rpmsg rpmsg = { + .header = { + .magic = ETHOSU_CORE_MSG_MAGIC, + .type = ETHOSU_CORE_MSG_NETWORK_INFO_REQ, + .msg_id = msg->id + } + }; + struct ethosu_core_msg_network_info_req *info_req = &rpmsg.net_info_req; + + msg->type = rpmsg.header.type; + + if (network->dma_mem != NULL) { + info_req->network.type = ETHOSU_CORE_NETWORK_BUFFER; + ethosu_core_buffer_dma_mem_set(network->dma_mem, + &info_req->network.buffer); + } else { + info_req->network.type = ETHOSU_CORE_NETWORK_INDEX; + info_req->network.index = network->index; + } + + return ethosu_send_locked(mbox, &rpmsg, + sizeof(rpmsg.header) + + sizeof(rpmsg.net_info_req)); +} + +int ethosu_mailbox_cancel_inference(struct ethosu_mailbox *mbox, + struct ethosu_mailbox_msg *msg, + int inference_handle) +{ + struct ethosu_core_rpmsg rpmsg = { + .header = { + .magic = ETHOSU_CORE_MSG_MAGIC, + .type = + ETHOSU_CORE_MSG_CANCEL_INFERENCE_REQ, + .msg_id = msg->id + }, + .cancel_req = { + .inference_handle = inference_handle + } + }; + + msg->type = rpmsg.header.type; + + return ethosu_send_locked(mbox, &rpmsg, + sizeof(rpmsg.header) + + sizeof(rpmsg.cancel_req)); +} + +int ethosu_mailbox_init(struct ethosu_mailbox *mbox, + struct device *dev, + struct rpmsg_endpoint *ept) +{ + mbox->dev = dev; + mbox->ept = ept; + idr_init(&mbox->msg_idr); + init_waitqueue_head(&mbox->send_queue); + + return 0; +} + +void ethosu_mailbox_deinit(struct ethosu_mailbox *mbox) +{ + atomic_set(&mbox->done, 1); + wake_up_all(&mbox->send_queue); +} diff --git a/kernel/rpmsg/ethosu_rpmsg_network.c b/kernel/rpmsg/ethosu_rpmsg_network.c new file mode 100644 index 0000000..f5b8265 --- /dev/null +++ b/kernel/rpmsg/ethosu_rpmsg_network.c @@ -0,0 +1,255 @@ +/* + * SPDX-FileCopyrightText: Copyright 2020, 2022-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +/**************************************************************************** + * Variables + ****************************************************************************/ + +static int ethosu_network_release(struct inode *inode, + struct file *file); + +static long ethosu_network_ioctl(struct file *file, + unsigned int cmd, + unsigned long arg); + +static const struct file_operations ethosu_network_fops = { + .release = ðosu_network_release, + .unlocked_ioctl = ðosu_network_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = ðosu_network_ioctl, +#endif +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +static bool ethosu_network_verify(struct file *file) +{ + return file->f_op == ðosu_network_fops; +} + +static void ethosu_network_destroy(struct kref *kref) +{ + struct ethosu_network *net = + container_of(kref, struct ethosu_network, kref); + struct device *dev = net->dev; + + dev_dbg(dev, "Network destroy. net=0x%pK\n", net); + + if (net->dma_mem != NULL) + ethosu_dma_mem_free(&net->dma_mem); + + memset(net, 0, sizeof(*net)); + devm_kfree(dev, net); +} + +static int ethosu_network_release(struct inode *inode, + struct file *file) +{ + struct ethosu_network *net = file->private_data; + struct device *dev = net->dev; + + dev_dbg(dev, "Network release. file=0x%pK, net=0x%pK\n", + file, net); + + ethosu_network_put(net); + + return 0; +} + +static long ethosu_network_ioctl(struct file *file, + unsigned int cmd, + unsigned long arg) +{ + struct ethosu_network *net = file->private_data; + struct device *dev = net->dev; + void __user *udata = (void __user *)arg; + int ret; + + ret = device_lock_interruptible(net->dev); + if (ret) + return ret; + + switch (cmd) { + case ETHOSU_IOCTL_NETWORK_INFO: { + struct ethosu_uapi_network_info uapi = { 0 }; + + dev_dbg(dev, "Network ioctl: Network info. net=0x%pK\n", net); + + ret = ethosu_network_info_request(dev, net->mailbox, net, + &uapi); + if (ret) + break; + + ret = copy_to_user(udata, &uapi, sizeof(uapi)) ? -EFAULT : 0; + break; + } + case ETHOSU_IOCTL_INFERENCE_CREATE: { + struct ethosu_uapi_inference_create uapi; + + if (copy_from_user(&uapi, udata, sizeof(uapi))) { + ret = -EFAULT; + break; + } + + dev_dbg(dev, + "Network ioctl: Inference. ifm_fd=%u, ofm_fd=%u\n", + uapi.ifm_fd[0], uapi.ofm_fd[0]); + + ret = ethosu_inference_create(dev, net->mailbox, net, &uapi); + break; + } + default: { + dev_err(dev, "Invalid ioctl. cmd=%u, arg=%lu", + cmd, arg); + ret = -ENOIOCTLCMD; + break; + } + } + + device_unlock(net->dev); + + return ret; +} + +int ethosu_network_create(struct device *dev, + struct ethosu_mailbox *mailbox, + struct ethosu_uapi_network_create *uapi) +{ + struct ethosu_network *net; + const void __user *data; + int ret; + + net = devm_kzalloc(dev, sizeof(*net), GFP_KERNEL); + if (!net) + return -ENOMEM; + + net->dev = dev; + net->mailbox = mailbox; + kref_init(&net->kref); + + switch (uapi->type) { + case ETHOSU_UAPI_NETWORK_USER_BUFFER: + if (!uapi->network.data_ptr) { + dev_err(dev, "Invalid network data ptr\n"); + ret = -EINVAL; + goto free_net; + } + + if (!uapi->network.size) { + dev_err(dev, "Invalid network data size\n"); + ret = -EINVAL; + goto free_net; + } + + net->dma_mem = ethosu_dma_mem_alloc(dev, uapi->network.size); + if (IS_ERR(net->dma_mem)) { + ret = PTR_ERR(net->dma_mem); + goto free_net; + } + + data = u64_to_user_ptr(uapi->network.data_ptr); + ret = copy_from_user(net->dma_mem->cpu_addr, data, + uapi->network.size); + if (ret) + goto free_dma_mem; + + break; + case ETHOSU_UAPI_NETWORK_INDEX: + net->index = uapi->index; + break; + default: + ret = -EINVAL; + goto free_net; + } + + ret = anon_inode_getfd("ethosu-network", ðosu_network_fops, net, + O_RDWR | O_CLOEXEC); + if (ret < 0) + goto free_dma_mem; + + net->file = fget(ret); + fput(net->file); + + dev_dbg(dev, + "Network create. file=0x%pK, fd=%d, net=0x%pK, buf=0x%pK, index=%u", + net->file, ret, net, net->dma_mem, net->index); + + return ret; + +free_dma_mem: + if (net->dma_mem != NULL) + ethosu_dma_mem_free(&net->dma_mem); + +free_net: + memset(net, 0, sizeof(*net)); + devm_kfree(dev, net); + + return ret; +} + +struct ethosu_network *ethosu_network_get_from_fd(int fd) +{ + struct ethosu_network *net; + struct file *file; + + file = fget(fd); + if (!file) + return ERR_PTR(-EINVAL); + + if (!ethosu_network_verify(file)) { + fput(file); + + return ERR_PTR(-EINVAL); + } + + net = file->private_data; + ethosu_network_get(net); + fput(file); + + return net; +} + +void ethosu_network_get(struct ethosu_network *net) +{ + kref_get(&net->kref); +} + +int ethosu_network_put(struct ethosu_network *net) +{ + return kref_put(&net->kref, ethosu_network_destroy); +} diff --git a/kernel/rpmsg/ethosu_rpmsg_network_info.c b/kernel/rpmsg/ethosu_rpmsg_network_info.c new file mode 100644 index 0000000..0e54d87 --- /dev/null +++ b/kernel/rpmsg/ethosu_rpmsg_network_info.c @@ -0,0 +1,187 @@ +/* + * SPDX-FileCopyrightText: Copyright 2022-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include + +#include +#include +#include + +#include + +#define NETWORK_INFO_RESP_TIMEOUT_MS 3000 + +static inline int ethosu_network_info_send(struct ethosu_network_info *info, + struct ethosu_mailbox *mailbox) +{ + /* Send network info request to firmware */ + return ethosu_mailbox_network_info_request(mailbox, + &info->msg, + info->net); +} + +static void ethosu_network_info_fail(struct ethosu_mailbox_msg *msg) +{ + struct ethosu_network_info *info = + container_of(msg, typeof(*info), msg); + + if (completion_done(&info->done)) + return; + + info->errno = -EFAULT; + complete(&info->done); +} + +int ethosu_network_info_request(struct device *dev, + struct ethosu_mailbox *mailbox, + struct ethosu_network *net, + struct ethosu_uapi_network_info *uapi) +{ + struct ethosu_network_info *info; + int ret; + int timeout; + + info = devm_kzalloc(dev, sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + info->dev = dev; + info->net = net; + info->uapi = uapi; + init_completion(&info->done); + info->msg.fail = ethosu_network_info_fail; + + ret = ethosu_mailbox_register(mailbox, &info->msg); + if (ret < 0) + goto kfree; + + /* Get reference to network */ + ethosu_network_get(info->net); + + ret = ethosu_network_info_send(info, mailbox); + if (ret) + goto deregister; + + dev_dbg(dev, + "Network info create. info=0x%pK, net=0x%pK, msg.id=0x%x\n", + info, info->net, info->msg.id); + + /* Unlock the device mutex and wait for completion */ + device_unlock(dev); + timeout = wait_for_completion_timeout(&info->done, + msecs_to_jiffies( + NETWORK_INFO_RESP_TIMEOUT_MS)); + device_lock(dev); + + if (0 == timeout) { + dev_warn(dev, "Network info timed out. info=0x%pK", + info); + + ret = -ETIME; + goto deregister; + } + + ret = info->errno; + +deregister: + ethosu_mailbox_deregister(mailbox, &info->msg); + ethosu_network_put(info->net); + +kfree: + dev_dbg(dev, + "Network info destroy. info=0x%pK, msg.id=0x%x\n", + info, info->msg.id); + devm_kfree(dev, info); + + return ret; +} + +void ethosu_network_info_rsp(struct ethosu_mailbox *mailbox, + int msg_id, + struct ethosu_core_msg_network_info_rsp *rsp) +{ + int ret; + struct device *dev = mailbox->dev; + struct ethosu_mailbox_msg *msg; + struct ethosu_network_info *info; + uint32_t i; + const size_t rsp_desc_size = sizeof(rsp->desc); + + BUILD_BUG_ON(rsp_desc_size != sizeof(info->uapi->desc)); + + msg = ethosu_mailbox_find(mailbox, msg_id, + ETHOSU_CORE_MSG_NETWORK_INFO_REQ); + if (IS_ERR(msg)) { + dev_warn(dev, + "Id for network info msg not found. Id=0x%x: %ld\n", + msg_id, PTR_ERR(msg)); + + return; + } + + info = container_of(msg, typeof(*info), msg); + + if (completion_done(&info->done)) + return; + + info->errno = 0; + + if (rsp->status != ETHOSU_CORE_STATUS_OK) { + dev_err(dev, "Failed to get information about the network\n"); + info->errno = -EBADF; + goto signal_complete; + } + + if (rsp->ifm_count > ETHOSU_FD_MAX || rsp->ofm_count > ETHOSU_FD_MAX) { + dev_err(dev, + "Invalid number of IFMs/OFMs in network info: IFMs=%u OFMs=%u\n", + rsp->ifm_count, rsp->ofm_count); + info->errno = -ENFILE; + goto signal_complete; + } + + if (strnlen(rsp->desc, rsp_desc_size) == rsp_desc_size) { + dev_err(dev, + "Description in network info is not null-terminated\n"); + info->errno = -EMSGSIZE; + goto signal_complete; + } + + ret = strscpy(info->uapi->desc, rsp->desc, sizeof(info->uapi->desc)); + if (ret < 0) { + dev_err(dev, "Failed to copy network info description\n"); + info->errno = ret; + goto signal_complete; + } + + info->uapi->ifm_count = rsp->ifm_count; + for (i = 0; i < rsp->ifm_count; i++) + info->uapi->ifm_size[i] = rsp->ifm_size[i]; + + info->uapi->ofm_count = rsp->ofm_count; + for (i = 0; i < rsp->ofm_count; i++) + info->uapi->ofm_size[i] = rsp->ofm_size[i]; + +signal_complete: + complete(&info->done); +} diff --git a/kernel/rpmsg/ethosu_rpmsg_version.c b/kernel/rpmsg/ethosu_rpmsg_version.c new file mode 100644 index 0000000..03e164b --- /dev/null +++ b/kernel/rpmsg/ethosu_rpmsg_version.c @@ -0,0 +1,147 @@ +/* + * SPDX-FileCopyrightText: Copyright 2023-2024 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include +#include + +#include + +/**************************************************************************** + * Defines + ****************************************************************************/ + +#define VERSION_RESP_TIMEOUT_MS 2000 + +/**************************************************************************** + * Functions + ****************************************************************************/ + +static void ethosu_version_fail(struct ethosu_mailbox_msg *msg) +{ + struct ethosu_version *version = + container_of(msg, typeof(*version), msg); + + if (completion_done(&version->done)) + return; + + version->errno = -EFAULT; + complete(&version->done); +} + +void ethosu_version_rsp(struct ethosu_mailbox *mailbox, + int msg_id, + struct ethosu_core_msg_version_rsp *rsp) +{ + struct device *dev = mailbox->dev; + struct ethosu_mailbox_msg *msg; + struct ethosu_version *version; + + msg = ethosu_mailbox_find(mailbox, msg_id, + ETHOSU_CORE_MSG_VERSION_REQ); + if (IS_ERR(msg)) { + dev_warn(dev, + "Id for version msg not found. Id=0x%0x: %ld\n", + msg_id, PTR_ERR(msg)); + + return; + } + + version = container_of(msg, typeof(*version), msg); + + if (completion_done(&version->done)) + return; + + if (rsp->major != ETHOSU_CORE_MSG_VERSION_MAJOR || + rsp->minor != ETHOSU_CORE_MSG_VERSION_MINOR) { + dev_warn(dev, + "Msg: Protocol version mismatch. Expected %u.%u.X but got %u.%u.%u", + ETHOSU_CORE_MSG_VERSION_MAJOR, + ETHOSU_CORE_MSG_VERSION_MINOR, + rsp->major, rsp->minor, rsp->patch); + version->errno = -EPROTO; + } else { + version->errno = 0; + } + + complete(&version->done); +} + +int ethosu_version_check_request(struct device *dev, + struct ethosu_mailbox *mailbox) +{ + struct ethosu_version *version; + int ret; + int timeout; + + version = devm_kzalloc(dev, sizeof(*version), GFP_KERNEL); + if (!version) + return -ENOMEM; + + version->dev = dev; + init_completion(&version->done); + version->msg.fail = ethosu_version_fail; + + ret = ethosu_mailbox_register(mailbox, &version->msg); + if (ret < 0) + goto free_version; + + dev_dbg(dev, "Protocol version request created. Id=0x%x, handle=%pK\n", + version->msg.id, version); + + ret = ethosu_mailbox_version_request(mailbox, &version->msg); + if (ret) + goto deregister; + + /* Unlock the mutex to not block other messages while waiting */ + device_unlock(dev); + + /* Wait for version response */ + timeout = wait_for_completion_timeout(&version->done, + msecs_to_jiffies( + VERSION_RESP_TIMEOUT_MS)); + + /* Take back the mutex before resuming to do anything */ + device_lock(dev); + + if (0 == timeout) { + dev_warn(dev, "Protocol version response timeout"); + ret = -ETIME; + goto deregister; + } + + if (version->errno) { + ret = version->errno; + goto deregister; + } + +deregister: + ethosu_mailbox_deregister(mailbox, &version->msg); + +free_version: + dev_dbg(dev, "Protocol version destroy. Id=0x%x, handle=%pK\n", + version->msg.id, + version); + devm_kfree(dev, version); + + return ret; +} diff --git a/kernel/uapi/ethosu.h b/kernel/uapi/ethosu.h deleted file mode 100644 index 8240ad9..0000000 --- a/kernel/uapi/ethosu.h +++ /dev/null @@ -1,264 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright 2020-2023 Arm Limited and/or its affiliates - * SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note - * - * This program is free software and is provided to you under the terms of the - * GNU General Public License version 2 as published by the Free Software - * Foundation, and any use by you of this program is subject to the terms - * of such GNU licence. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, you can access it online at - * http://www.gnu.org/licenses/gpl-2.0.html. - * - */ - -#ifndef ETHOSU_H -#define ETHOSU_H - -/**************************************************************************** - * Includes - ****************************************************************************/ - -#include -#include - -#ifdef __cplusplus -namespace EthosU { -#endif - -/**************************************************************************** - * Defines - ****************************************************************************/ - -#define ETHOSU_IOCTL_BASE 0x01 -#define ETHOSU_IO(nr) _IO(ETHOSU_IOCTL_BASE, nr) -#define ETHOSU_IOR(nr, type) _IOR(ETHOSU_IOCTL_BASE, nr, type) -#define ETHOSU_IOW(nr, type) _IOW(ETHOSU_IOCTL_BASE, nr, type) -#define ETHOSU_IOWR(nr, type) _IOWR(ETHOSU_IOCTL_BASE, nr, type) - -#define ETHOSU_IOCTL_PING ETHOSU_IO(0x00) -#define ETHOSU_IOCTL_CAPABILITIES_REQ ETHOSU_IOR(0x02, \ - struct ethosu_uapi_device_capabilities) -#define ETHOSU_IOCTL_DRIVER_VERSION_GET ETHOSU_IOR(0x03, \ - struct ethosu_uapi_kernel_driver_version) -#define ETHOSU_IOCTL_BUFFER_CREATE ETHOSU_IOR(0x10, \ - struct ethosu_uapi_buffer_create) -#define ETHOSU_IOCTL_NETWORK_CREATE ETHOSU_IOR(0x20, \ - struct ethosu_uapi_network_create) -#define ETHOSU_IOCTL_NETWORK_INFO ETHOSU_IOR(0x21, \ - struct ethosu_uapi_network_info) -#define ETHOSU_IOCTL_INFERENCE_CREATE ETHOSU_IOR(0x30, \ - struct ethosu_uapi_inference_create) -#define ETHOSU_IOCTL_INFERENCE_STATUS ETHOSU_IOR(0x31, \ - struct ethosu_uapi_result_status) -#define ETHOSU_IOCTL_INFERENCE_CANCEL ETHOSU_IOR(0x32, \ - struct ethosu_uapi_cancel_inference_status) - -/* Maximum number of IFM/OFM file descriptors per network */ -#define ETHOSU_FD_MAX 16 - -/* Maximum number of PMUs available */ -#define ETHOSU_PMU_EVENT_MAX 8 - -/* Kernel driver version */ -#define ETHOSU_KERNEL_DRIVER_VERSION_MAJOR 3 -#define ETHOSU_KERNEL_DRIVER_VERSION_MINOR 0 -#define ETHOSU_KERNEL_DRIVER_VERSION_PATCH 0 - -/**************************************************************************** - * Types - ****************************************************************************/ - -/** - * enum ethosu_uapi_status - Status - */ -enum ethosu_uapi_status { - ETHOSU_UAPI_STATUS_OK, - ETHOSU_UAPI_STATUS_ERROR, - ETHOSU_UAPI_STATUS_RUNNING, - ETHOSU_UAPI_STATUS_REJECTED, - ETHOSU_UAPI_STATUS_ABORTED, - ETHOSU_UAPI_STATUS_ABORTING, -}; - -/** - * struct ethosu_uapi_kernel_driver_version - Kernel driver version - * @major: Major version - * @minor: Minor version - * @patch: Patch version - */ -struct ethosu_uapi_kernel_driver_version { - __u32 major; - __u32 minor; - __u32 patch; -}; - -/** - * struct ethosu_uapi_buffer_create - Create buffer request - * @size: Size of the requested buffer - */ -struct ethosu_uapi_buffer_create { - __u32 size; -}; - -/** - * enum ethosu_uapi_network_type - Network buffer type. - * @ETHOSU_UAPI_NETWORK_USER_BUFFER: Network data is provided in a user - * buffer. - * @ETHOSU_UAPI_NETWORK_INDEX: Network is built into firmware and - * referenced by index. - */ -enum ethosu_uapi_network_type { - ETHOSU_UAPI_NETWORK_USER_BUFFER = 1, - ETHOSU_UAPI_NETWORK_INDEX, -}; - -/** - * struct ethosu_uapi_network_create - Create network request - * @type: Buffer type. See @ethosu_uapi_network_type. - * @fd: Buffer file descriptor - * @index: Buffer index compiled into firmware binary. - */ -struct ethosu_uapi_network_create { - uint32_t type; - union { - struct { - __u64 data_ptr; - __u32 size; - } network; - __u32 index; - }; -}; - -/** - * struct ethosu_uapi_network_info - Network info - * @desc: Network description - * @ifm_count: Number of IFM buffers - * @ifm_size: IFM buffer sizes - * @ofm_count: Number of OFM buffers - * @ofm_size: OFM buffer sizes - */ -struct ethosu_uapi_network_info { - char desc[32]; - __u32 ifm_count; - __u32 ifm_size[ETHOSU_FD_MAX]; - __u32 ofm_count; - __u32 ofm_size[ETHOSU_FD_MAX]; -}; - -/** - * struct ethosu_uapi_pmu_config - Configure performance counters - * @events: Array of counters to configure, set to non-zero for - * each counter to enable corresponding event. - * @cycle_count: Set to enable the cycle counter. - */ -struct ethosu_uapi_pmu_config { - __u32 events[ETHOSU_PMU_EVENT_MAX]; - __u32 cycle_count; -}; - -/** - * struct ethosu_uapi_pmu_counts - Status of performance counters - * @events: Count for respective configured events. - * @cycle_count: Count for cycle counter. - */ -struct ethosu_uapi_pmu_counts { - __u64 events[ETHOSU_PMU_EVENT_MAX]; - __u64 cycle_count; -}; - -/** - * struct ethosu_uapi_device_hw_id - Device hardware identification - * @version_status: Version status - * @version_minor: Version minor - * @version_major: Version major - * @product_major: Product major - * @arch_patch_rev: Architecture version patch - * @arch_minor_rev: Architecture version minor - * @arch_major_rev: Architecture version major - */ -struct ethosu_uapi_device_hw_id { - __u32 version_status; - __u32 version_minor; - __u32 version_major; - __u32 product_major; - __u32 arch_patch_rev; - __u32 arch_minor_rev; - __u32 arch_major_rev; -}; - -/** - * struct ethosu_uapi_device_hw_cfg - Device hardware configuration - * @macs_per_cc: MACs per clock cycle - * @cmd_stream_version: NPU command stream version - * @custom_dma: Custom DMA enabled - */ -struct ethosu_uapi_device_hw_cfg { - __u32 macs_per_cc; - __u32 cmd_stream_version; - __u32 custom_dma; -}; - -/** - * struct ethosu_uapi_device_capabilities - Device capabilities - * @hw_id: Hardware identification - * @hw_cfg: Hardware configuration - * @driver_patch_rev: Driver version patch - * @driver_minor_rev: Driver version minor - * @driver_major_rev: Driver version major - */ -struct ethosu_uapi_device_capabilities { - struct ethosu_uapi_device_hw_id hw_id; - struct ethosu_uapi_device_hw_cfg hw_cfg; - __u32 driver_patch_rev; - __u32 driver_minor_rev; - __u32 driver_major_rev; -}; - -/** - * struct ethosu_uapi_inference_create - Create inference request - * @ifm_count: Number of IFM file descriptors - * @ifm_fd: IFM buffer file descriptors - * @ofm_count: Number of OFM file descriptors - * @ofm_fd: OFM buffer file descriptors - */ -struct ethosu_uapi_inference_create { - __u32 ifm_count; - __u32 ifm_fd[ETHOSU_FD_MAX]; - __u32 ofm_count; - __u32 ofm_fd[ETHOSU_FD_MAX]; - struct ethosu_uapi_pmu_config pmu_config; -}; - -/** - * struct ethosu_uapi_result_status - Status of inference - * @status Status of run inference. - * @pmu_config Configured performance counters. - * @pmu_count Perfomance counters values, when status is - * ETHOSU_UAPI_STATUS_OK. - */ -struct ethosu_uapi_result_status { - __u32 status; - struct ethosu_uapi_pmu_config pmu_config; - struct ethosu_uapi_pmu_counts pmu_count; -}; - -/** - * struct ethosu_uapi_cancel_inference_status - Status of inference - * cancellation. - * @status OK if inference cancellation was performed, ERROR otherwise. - */ -struct ethosu_uapi_cancel_inference_status { - __u32 status; -}; - -#ifdef __cplusplus -} /* namespace EthosU */ -#endif -#endif -- cgit v1.2.1