From 9c999fdd40c0bf2ae420f6f3bfe013dc6baa73c1 Mon Sep 17 00:00:00 2001 From: Mikael Olsson Date: Mon, 30 Oct 2023 11:05:39 +0100 Subject: Split DMA memory and buffer setup in kernel driver To allow the NPU kernel driver to allocate and use DMA memory internally without creating a buffer instance, the DMA memory management has been split out from the buffer code. Change-Id: I46fdeee51b5ef786a54b8e7c866d137d91222724 Signed-off-by: Mikael Olsson --- kernel/Kbuild | 1 + kernel/ethosu_buffer.c | 26 ++++++++--------- kernel/ethosu_buffer.h | 23 +++++++-------- kernel/ethosu_dma_mem.c | 77 +++++++++++++++++++++++++++++++++++++++++++++++++ kernel/ethosu_dma_mem.h | 58 +++++++++++++++++++++++++++++++++++++ kernel/ethosu_mailbox.c | 22 +++++++++----- 6 files changed, 172 insertions(+), 35 deletions(-) create mode 100644 kernel/ethosu_dma_mem.c create mode 100644 kernel/ethosu_dma_mem.h diff --git a/kernel/Kbuild b/kernel/Kbuild index a83400f..98bece3 100644 --- a/kernel/Kbuild +++ b/kernel/Kbuild @@ -23,6 +23,7 @@ obj-$(CONFIG_ETHOSU) = ethosu.o ethosu-objs := ethosu_driver.o \ ethosu_buffer.o \ + ethosu_dma_mem.o \ ethosu_device.o \ ethosu_inference.o \ ethosu_mailbox.o \ diff --git a/kernel/ethosu_buffer.c b/kernel/ethosu_buffer.c index bf7d745..69b5007 100644 --- a/kernel/ethosu_buffer.c +++ b/kernel/ethosu_buffer.c @@ -24,6 +24,7 @@ #include "ethosu_buffer.h" #include "ethosu_device.h" +#include "ethosu_dma_mem.h" #include "uapi/ethosu.h" #include @@ -71,9 +72,7 @@ static void ethosu_buffer_destroy(struct kref *kref) dev_dbg(dev, "Buffer destroy. buf=0x%pK", buf); - memset(buf->cpu_addr, 0, buf->size); - dma_free_coherent(dev, buf->size, buf->cpu_addr, - buf->dma_addr); + ethosu_dma_mem_free(&buf->dma_mem); memset(buf, 0, sizeof(*buf)); devm_kfree(dev, buf); @@ -103,8 +102,8 @@ static int ethosu_buffer_mmap(struct file *file, dev_dbg(dev, "Buffer mmap. file=0x%pK, buf=0x%pK\n", file, buf); - ret = dma_mmap_coherent(dev, vma, buf->cpu_addr, - buf->dma_addr, buf->size); + ret = dma_mmap_coherent(dev, vma, buf->dma_mem->cpu_addr, + buf->dma_mem->dma_addr, buf->dma_mem->size); return ret; } @@ -126,7 +125,7 @@ static loff_t ethosu_buffer_llseek(struct file *file, */ switch (whence) { case SEEK_END: - return buf->size; + return buf->dma_mem->size; case SEEK_SET: return 0; default: @@ -148,13 +147,13 @@ int ethosu_buffer_create(struct device *dev, return -ENOMEM; buf->dev = dev; - buf->size = size; kref_init(&buf->kref); - buf->cpu_addr = dma_alloc_coherent(dev, size, &buf->dma_addr, - GFP_KERNEL); - if (!buf->cpu_addr) + buf->dma_mem = ethosu_dma_mem_alloc(dev, size); + if (IS_ERR(buf->dma_mem)) { + ret = PTR_ERR(buf->dma_mem); goto free_buf; + } ret = anon_inode_getfd("ethosu-buffer", ðosu_buffer_fops, buf, O_RDWR | O_CLOEXEC); @@ -168,14 +167,13 @@ int ethosu_buffer_create(struct device *dev, dev_dbg(dev, "Buffer create. file=0x%pK, fd=%d, buf=0x%pK, size=%zu, cpu_addr=0x%pK, dma_addr=0x%llx, phys_addr=0x%llx\n", - buf->file, ret, buf, size, buf->cpu_addr, buf->dma_addr, - virt_to_phys(buf->cpu_addr)); + buf->file, ret, buf, size, buf->dma_mem->cpu_addr, + buf->dma_mem->dma_addr, virt_to_phys(buf->dma_mem->cpu_addr)); return ret; free_dma: - dma_free_coherent(dev, buf->size, buf->cpu_addr, - buf->dma_addr); + ethosu_dma_mem_free(&buf->dma_mem); free_buf: memset(buf, 0, sizeof(*buf)); diff --git a/kernel/ethosu_buffer.h b/kernel/ethosu_buffer.h index 1829fbe..8bef2d2 100644 --- a/kernel/ethosu_buffer.h +++ b/kernel/ethosu_buffer.h @@ -31,25 +31,22 @@ * Types ****************************************************************************/ +struct ethosu_dma_mem; struct ethosu_device; struct device; /** - * struct ethosu_buffer - Buffer - * @dev: Device - * @file: File - * @kref: Reference counting - * @size: Size of the buffer - * @cpu_addr: Kernel mapped address - * @dma_addr: DMA address + * struct ethosu_buffer - User data buffer + * @dev: Device + * @file: File + * @kref: Reference counting + * @dma_mem: DMA memory allocated for the buffer */ struct ethosu_buffer { - struct device *dev; - struct file *file; - struct kref kref; - size_t size; - void *cpu_addr; - dma_addr_t dma_addr; + struct device *dev; + struct file *file; + struct kref kref; + struct ethosu_dma_mem *dma_mem; }; /**************************************************************************** diff --git a/kernel/ethosu_dma_mem.c b/kernel/ethosu_dma_mem.c new file mode 100644 index 0000000..cb63081 --- /dev/null +++ b/kernel/ethosu_dma_mem.c @@ -0,0 +1,77 @@ +/* + * SPDX-FileCopyrightText: Copyright 2023 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include "ethosu_dma_mem.h" + +#include +#include + +/**************************************************************************** + * Functions + ****************************************************************************/ + +struct ethosu_dma_mem *ethosu_dma_mem_alloc(struct device *dev, + size_t size) +{ + struct ethosu_dma_mem *dma_mem; + + if (!size) + return ERR_PTR(-EINVAL); + + dma_mem = devm_kzalloc(dev, sizeof(*dma_mem), GFP_KERNEL); + if (!dma_mem) + return ERR_PTR(-ENOMEM); + + dma_mem->dev = dev; + dma_mem->size = size; + dma_mem->cpu_addr = dma_alloc_coherent(dev, size, &dma_mem->dma_addr, + GFP_KERNEL); + if (!dma_mem->cpu_addr) { + memset(dma_mem, 0, sizeof(*dma_mem)); + devm_kfree(dev, dma_mem); + + return ERR_PTR(-ENOMEM); + } + + return dma_mem; +} + +void ethosu_dma_mem_free(struct ethosu_dma_mem **dma_mem) +{ + struct device *dev; + struct ethosu_dma_mem *mem; + + if (!dma_mem || !*dma_mem) + return; + + mem = *dma_mem; + dev = mem->dev; + + memset(mem->cpu_addr, 0, mem->size); + dma_free_coherent(dev, mem->size, mem->cpu_addr, mem->dma_addr); + + memset(mem, 0, sizeof(*mem)); + devm_kfree(dev, mem); + + *dma_mem = NULL; +} diff --git a/kernel/ethosu_dma_mem.h b/kernel/ethosu_dma_mem.h new file mode 100644 index 0000000..11cdd39 --- /dev/null +++ b/kernel/ethosu_dma_mem.h @@ -0,0 +1,58 @@ +/* + * SPDX-FileCopyrightText: Copyright 2023 Arm Limited and/or its affiliates + * SPDX-License-Identifier: GPL-2.0-only + * + * This program is free software and is provided to you under the terms of the + * GNU General Public License version 2 as published by the Free Software + * Foundation, and any use by you of this program is subject to the terms + * of such GNU licence. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, you can access it online at + * http://www.gnu.org/licenses/gpl-2.0.html. + */ + +#ifndef ETHOSU_DMA_MEM_H +#define ETHOSU_DMA_MEM_H + +/**************************************************************************** + * Includes + ****************************************************************************/ + +#include + +/**************************************************************************** + * Types + ****************************************************************************/ + +struct device; + +/** + * struct ethosu_dma_mem - DMA memory allocation + * @dev: Device + * @size: Size of the allocation + * @cpu_addr: Kernel mapped address + * @dma_addr: DMA address + */ +struct ethosu_dma_mem { + struct device *dev; + size_t size; + void *cpu_addr; + dma_addr_t dma_addr; +}; + +/**************************************************************************** + * Functions + ****************************************************************************/ + +struct ethosu_dma_mem *ethosu_dma_mem_alloc(struct device *dev, + size_t size); + +void ethosu_dma_mem_free(struct ethosu_dma_mem **dma_mem); + +#endif /* ETHOSU_DMA_MEM_H */ diff --git a/kernel/ethosu_mailbox.c b/kernel/ethosu_mailbox.c index e499860..9b9cd18 100644 --- a/kernel/ethosu_mailbox.c +++ b/kernel/ethosu_mailbox.c @@ -26,6 +26,8 @@ #include "ethosu_buffer.h" #include "ethosu_core_rpmsg.h" #include "ethosu_device.h" +#include "ethosu_dma_mem.h" +#include "ethosu_network.h" #include #include @@ -121,11 +123,11 @@ static int ethosu_send_locked(struct ethosu_mailbox *mbox, return ret; } -static void ethosu_core_set_size(struct ethosu_buffer *buf, - struct ethosu_core_buffer *cbuf) +static void ethosu_core_buffer_dma_mem_set(struct ethosu_dma_mem *dma_mem, + struct ethosu_core_buffer *cbuf) { - cbuf->ptr = (uint32_t)buf->dma_addr; - cbuf->size = (uint32_t)buf->size; + cbuf->ptr = (uint32_t)dma_mem->dma_addr; + cbuf->size = (uint32_t)dma_mem->size; } int ethosu_mailbox_register(struct ethosu_mailbox *mbox, @@ -267,17 +269,20 @@ int ethosu_mailbox_inference(struct ethosu_mailbox *mbox, inf_req->pmu_cycle_counter_enable = pmu_cycle_counter_enable; for (i = 0; i < ifm_count; i++) - ethosu_core_set_size(ifm[i], &inf_req->ifm[i]); + ethosu_core_buffer_dma_mem_set(ifm[i]->dma_mem, + &inf_req->ifm[i]); for (i = 0; i < ofm_count; i++) - ethosu_core_set_size(ofm[i], &inf_req->ofm[i]); + ethosu_core_buffer_dma_mem_set(ofm[i]->dma_mem, + &inf_req->ofm[i]); for (i = 0; i < ETHOSU_CORE_PMU_MAX; i++) inf_req->pmu_event_config[i] = pmu_event_config[i]; if (network != NULL) { inf_req->network.type = ETHOSU_CORE_NETWORK_BUFFER; - ethosu_core_set_size(network, &inf_req->network.buffer); + ethosu_core_buffer_dma_mem_set(network->dma_mem, + &inf_req->network.buffer); } else { inf_req->network.type = ETHOSU_CORE_NETWORK_INDEX; inf_req->network.index = network_index; @@ -305,7 +310,8 @@ int ethosu_mailbox_network_info_request(struct ethosu_mailbox *mbox, if (network != NULL) { info_req->network.type = ETHOSU_CORE_NETWORK_BUFFER; - ethosu_core_set_size(network, &info_req->network.buffer); + ethosu_core_buffer_dma_mem_set(network->dma_mem, + &info_req->network.buffer); } else { info_req->network.type = ETHOSU_CORE_NETWORK_INDEX; info_req->network.index = network_index; -- cgit v1.2.1