Split DMA memory and buffer setup in kernel driver

To allow the NPU kernel driver to allocate and use DMA memory internally
without creating a buffer instance, the DMA memory management has been
split out from the buffer code.

Change-Id: I46fdeee51b5ef786a54b8e7c866d137d91222724
Signed-off-by: Mikael Olsson <mikael.olsson@arm.com>

6 files changed, 172 insertions, 35 deletions

diff --git a/kernel/Kbuild b/kernel/Kbuild
index a83400f..98bece3 100644
--- a/kernel/Kbuild
+++ b/kernel/Kbuild
@@ -23,6 +23,7 @@ obj-$(CONFIG_ETHOSU) = ethosu.o
 
 ethosu-objs := ethosu_driver.o \
                ethosu_buffer.o \
+               ethosu_dma_mem.o \
                ethosu_device.o \
                ethosu_inference.o \
                ethosu_mailbox.o \
diff --git a/kernel/ethosu_buffer.c b/kernel/ethosu_buffer.c
index bf7d745..69b5007 100644
--- a/kernel/ethosu_buffer.c
+++ b/kernel/ethosu_buffer.c
@@ -24,6 +24,7 @@
 #include "ethosu_buffer.h"
 
 #include "ethosu_device.h"
+#include "ethosu_dma_mem.h"
 #include "uapi/ethosu.h"
 
 #include <linux/anon_inodes.h>
@@ -71,9 +72,7 @@ static void ethosu_buffer_destroy(struct kref *kref)
 
 	dev_dbg(dev, "Buffer destroy. buf=0x%pK", buf);
 
-	memset(buf->cpu_addr, 0, buf->size);
-	dma_free_coherent(dev, buf->size, buf->cpu_addr,
-			  buf->dma_addr);
+	ethosu_dma_mem_free(&buf->dma_mem);
 
 	memset(buf, 0, sizeof(*buf));
 	devm_kfree(dev, buf);
@@ -103,8 +102,8 @@ static int ethosu_buffer_mmap(struct file *file,
 	dev_dbg(dev, "Buffer mmap. file=0x%pK, buf=0x%pK\n",
 		file, buf);
 
-	ret = dma_mmap_coherent(dev, vma, buf->cpu_addr,
-				buf->dma_addr, buf->size);
+	ret = dma_mmap_coherent(dev, vma, buf->dma_mem->cpu_addr,
+				buf->dma_mem->dma_addr, buf->dma_mem->size);
 
 	return ret;
 }
@@ -126,7 +125,7 @@ static loff_t ethosu_buffer_llseek(struct file *file,
 	 */
 	switch (whence) {
 	case SEEK_END:
-		return buf->size;
+		return buf->dma_mem->size;
 	case SEEK_SET:
 		return 0;
 	default:
@@ -148,13 +147,13 @@ int ethosu_buffer_create(struct device *dev,
 		return -ENOMEM;
 
 	buf->dev = dev;
-	buf->size = size;
 	kref_init(&buf->kref);
 
-	buf->cpu_addr = dma_alloc_coherent(dev, size, &buf->dma_addr,
-					   GFP_KERNEL);
-	if (!buf->cpu_addr)
+	buf->dma_mem = ethosu_dma_mem_alloc(dev, size);
+	if (IS_ERR(buf->dma_mem)) {
+		ret = PTR_ERR(buf->dma_mem);
 		goto free_buf;
+	}
 
 	ret = anon_inode_getfd("ethosu-buffer", &ethosu_buffer_fops, buf,
 			       O_RDWR | O_CLOEXEC);
@@ -168,14 +167,13 @@ int ethosu_buffer_create(struct device *dev,
 
 	dev_dbg(dev,
 		"Buffer create. file=0x%pK, fd=%d, buf=0x%pK, size=%zu, cpu_addr=0x%pK, dma_addr=0x%llx, phys_addr=0x%llx\n",
-		buf->file, ret, buf, size, buf->cpu_addr, buf->dma_addr,
-		virt_to_phys(buf->cpu_addr));
+		buf->file, ret, buf, size, buf->dma_mem->cpu_addr,
+		buf->dma_mem->dma_addr, virt_to_phys(buf->dma_mem->cpu_addr));
 
 	return ret;
 
 free_dma:
-	dma_free_coherent(dev, buf->size, buf->cpu_addr,
-			  buf->dma_addr);
+	ethosu_dma_mem_free(&buf->dma_mem);
 
 free_buf:
 	memset(buf, 0, sizeof(*buf));
diff --git a/kernel/ethosu_buffer.h b/kernel/ethosu_buffer.h
index 1829fbe..8bef2d2 100644
--- a/kernel/ethosu_buffer.h
+++ b/kernel/ethosu_buffer.h
@@ -31,25 +31,22 @@
  * Types
  ****************************************************************************/
 
+struct ethosu_dma_mem;
 struct ethosu_device;
 struct device;
 
 /**
- * struct ethosu_buffer - Buffer
- * @dev:		Device
- * @file:		File
- * @kref:		Reference counting
- * @size:		Size of the buffer
- * @cpu_addr:		Kernel mapped address
- * @dma_addr:		DMA address
+ * struct ethosu_buffer - User data buffer
+ * @dev:	Device
+ * @file:	File
+ * @kref:	Reference counting
+ * @dma_mem:	DMA memory allocated for the buffer
  */
 struct ethosu_buffer {
-	struct device *dev;
-	struct file   *file;
-	struct kref   kref;
-	size_t        size;
-	void          *cpu_addr;
-	dma_addr_t    dma_addr;
+	struct device         *dev;
+	struct file           *file;
+	struct kref           kref;
+	struct ethosu_dma_mem *dma_mem;
 };
 
 /****************************************************************************
diff --git a/kernel/ethosu_dma_mem.c b/kernel/ethosu_dma_mem.c
new file mode 100644
index 0000000..cb63081
--- /dev/null
+++ b/kernel/ethosu_dma_mem.c
@@ -0,0 +1,77 @@
+/*
+ * SPDX-FileCopyrightText: Copyright 2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ */
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include "ethosu_dma_mem.h"
+
+#include <linux/err.h>
+#include <linux/dma-mapping.h>
+
+/****************************************************************************
+ * Functions
+ ****************************************************************************/
+
+struct ethosu_dma_mem *ethosu_dma_mem_alloc(struct device *dev,
+					    size_t size)
+{
+	struct ethosu_dma_mem *dma_mem;
+
+	if (!size)
+		return ERR_PTR(-EINVAL);
+
+	dma_mem = devm_kzalloc(dev, sizeof(*dma_mem), GFP_KERNEL);
+	if (!dma_mem)
+		return ERR_PTR(-ENOMEM);
+
+	dma_mem->dev = dev;
+	dma_mem->size = size;
+	dma_mem->cpu_addr = dma_alloc_coherent(dev, size, &dma_mem->dma_addr,
+					       GFP_KERNEL);
+	if (!dma_mem->cpu_addr) {
+		memset(dma_mem, 0, sizeof(*dma_mem));
+		devm_kfree(dev, dma_mem);
+
+		return ERR_PTR(-ENOMEM);
+	}
+
+	return dma_mem;
+}
+
+void ethosu_dma_mem_free(struct ethosu_dma_mem **dma_mem)
+{
+	struct device *dev;
+	struct ethosu_dma_mem *mem;
+
+	if (!dma_mem || !*dma_mem)
+		return;
+
+	mem = *dma_mem;
+	dev = mem->dev;
+
+	memset(mem->cpu_addr, 0, mem->size);
+	dma_free_coherent(dev, mem->size, mem->cpu_addr, mem->dma_addr);
+
+	memset(mem, 0, sizeof(*mem));
+	devm_kfree(dev, mem);
+
+	*dma_mem = NULL;
+}
diff --git a/kernel/ethosu_dma_mem.h b/kernel/ethosu_dma_mem.h
new file mode 100644
index 0000000..11cdd39
--- /dev/null
+++ b/kernel/ethosu_dma_mem.h
@@ -0,0 +1,58 @@
+/*
+ * SPDX-FileCopyrightText: Copyright 2023 Arm Limited and/or its affiliates <open-source-office@arm.com>
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, you can access it online at
+ * http://www.gnu.org/licenses/gpl-2.0.html.
+ */
+
+#ifndef ETHOSU_DMA_MEM_H
+#define ETHOSU_DMA_MEM_H
+
+/****************************************************************************
+ * Includes
+ ****************************************************************************/
+
+#include <linux/types.h>
+
+/****************************************************************************
+ * Types
+ ****************************************************************************/
+
+struct device;
+
+/**
+ * struct ethosu_dma_mem - DMA memory allocation
+ * @dev:	Device
+ * @size:	Size of the allocation
+ * @cpu_addr:	Kernel mapped address
+ * @dma_addr:	DMA address
+ */
+struct ethosu_dma_mem {
+	struct device *dev;
+	size_t        size;
+	void          *cpu_addr;
+	dma_addr_t    dma_addr;
+};
+
+/****************************************************************************
+ * Functions
+ ****************************************************************************/
+
+struct ethosu_dma_mem *ethosu_dma_mem_alloc(struct device *dev,
+					    size_t size);
+
+void ethosu_dma_mem_free(struct ethosu_dma_mem **dma_mem);
+
+#endif /* ETHOSU_DMA_MEM_H */
diff --git a/kernel/ethosu_mailbox.c b/kernel/ethosu_mailbox.c
index e499860..9b9cd18 100644
--- a/kernel/ethosu_mailbox.c
+++ b/kernel/ethosu_mailbox.c
@@ -26,6 +26,8 @@
 #include "ethosu_buffer.h"
 #include "ethosu_core_rpmsg.h"
 #include "ethosu_device.h"
+#include "ethosu_dma_mem.h"
+#include "ethosu_network.h"
 
 #include <linux/atomic.h>
 #include <linux/jiffies.h>
@@ -121,11 +123,11 @@ static int ethosu_send_locked(struct ethosu_mailbox *mbox,
 	return ret;
 }
 
-static void ethosu_core_set_size(struct ethosu_buffer *buf,
-				 struct ethosu_core_buffer *cbuf)
+static void ethosu_core_buffer_dma_mem_set(struct ethosu_dma_mem *dma_mem,
+					   struct ethosu_core_buffer *cbuf)
 {
-	cbuf->ptr = (uint32_t)buf->dma_addr;
-	cbuf->size = (uint32_t)buf->size;
+	cbuf->ptr = (uint32_t)dma_mem->dma_addr;
+	cbuf->size = (uint32_t)dma_mem->size;
 }
 
 int ethosu_mailbox_register(struct ethosu_mailbox *mbox,
@@ -267,17 +269,20 @@ int ethosu_mailbox_inference(struct ethosu_mailbox *mbox,
 	inf_req->pmu_cycle_counter_enable = pmu_cycle_counter_enable;
 
 	for (i = 0; i < ifm_count; i++)
-		ethosu_core_set_size(ifm[i], &inf_req->ifm[i]);
+		ethosu_core_buffer_dma_mem_set(ifm[i]->dma_mem,
+					       &inf_req->ifm[i]);
 
 	for (i = 0; i < ofm_count; i++)
-		ethosu_core_set_size(ofm[i], &inf_req->ofm[i]);
+		ethosu_core_buffer_dma_mem_set(ofm[i]->dma_mem,
+					       &inf_req->ofm[i]);
 
 	for (i = 0; i < ETHOSU_CORE_PMU_MAX; i++)
 		inf_req->pmu_event_config[i] = pmu_event_config[i];
 
 	if (network != NULL) {
 		inf_req->network.type = ETHOSU_CORE_NETWORK_BUFFER;
-		ethosu_core_set_size(network, &inf_req->network.buffer);
+		ethosu_core_buffer_dma_mem_set(network->dma_mem,
+					       &inf_req->network.buffer);
 	} else {
 		inf_req->network.type = ETHOSU_CORE_NETWORK_INDEX;
 		inf_req->network.index = network_index;
@@ -305,7 +310,8 @@ int ethosu_mailbox_network_info_request(struct ethosu_mailbox *mbox,
 
 	if (network != NULL) {
 		info_req->network.type = ETHOSU_CORE_NETWORK_BUFFER;
-		ethosu_core_set_size(network, &info_req->network.buffer);
+		ethosu_core_buffer_dma_mem_set(network->dma_mem,
+					       &info_req->network.buffer);
 	} else {
 		info_req->network.type = ETHOSU_CORE_NETWORK_INDEX;
 		info_req->network.index = network_index;