143 lines
4.0 KiB
Diff
143 lines
4.0 KiB
Diff
From ace17dfbf605c4eab9eb68c897ace0708d466bca Mon Sep 17 00:00:00 2001
|
|
From: Pavel Begunkov <asml.silence@gmail.com>
|
|
Date: Wed, 2 Jul 2025 15:29:09 +0100
|
|
Subject: io_uring/zcrx: prepare fallback for larger pages
|
|
|
|
io_zcrx_copy_chunk() processes one page at a time, which won't be
|
|
sufficient when the net_iov size grows. Introduce a structure keeping
|
|
the target niov page and other parameters, it's more convenient and can
|
|
be reused later. And add a helper function that can efficient copy
|
|
buffers of an arbitrary length. For 64bit archs the loop inside should
|
|
be compiled out.
|
|
|
|
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
|
|
Link: https://lore.kernel.org/r/e84bc705a4e1edeb9aefff470d96558d8232388f.1751466461.git.asml.silence@gmail.com
|
|
Signed-off-by: Jens Axboe <axboe@kernel.dk>
|
|
---
|
|
io_uring/zcrx.c | 83 +++++++++++++++++++++++++++++++++----------------
|
|
1 file changed, 56 insertions(+), 27 deletions(-)
|
|
|
|
--- a/io_uring/zcrx.c
|
|
+++ b/io_uring/zcrx.c
|
|
@@ -969,6 +969,51 @@ static struct net_iov *io_zcrx_alloc_fal
|
|
return niov;
|
|
}
|
|
|
|
+struct io_copy_cache {
|
|
+ struct page *page;
|
|
+ unsigned long offset;
|
|
+ size_t size;
|
|
+};
|
|
+
|
|
+static ssize_t io_copy_page(struct io_copy_cache *cc, struct page *src_page,
|
|
+ unsigned int src_offset, size_t len)
|
|
+{
|
|
+ size_t copied = 0;
|
|
+
|
|
+ len = min(len, cc->size);
|
|
+
|
|
+ while (len) {
|
|
+ void *src_addr, *dst_addr;
|
|
+ struct page *dst_page = cc->page;
|
|
+ unsigned dst_offset = cc->offset;
|
|
+ size_t n = len;
|
|
+
|
|
+ if (folio_test_partial_kmap(page_folio(dst_page)) ||
|
|
+ folio_test_partial_kmap(page_folio(src_page))) {
|
|
+ dst_page = nth_page(dst_page, dst_offset / PAGE_SIZE);
|
|
+ dst_offset = offset_in_page(dst_offset);
|
|
+ src_page = nth_page(src_page, src_offset / PAGE_SIZE);
|
|
+ src_offset = offset_in_page(src_offset);
|
|
+ n = min(PAGE_SIZE - src_offset, PAGE_SIZE - dst_offset);
|
|
+ n = min(n, len);
|
|
+ }
|
|
+
|
|
+ dst_addr = kmap_local_page(dst_page) + dst_offset;
|
|
+ src_addr = kmap_local_page(src_page) + src_offset;
|
|
+
|
|
+ memcpy(dst_addr, src_addr, n);
|
|
+
|
|
+ kunmap_local(src_addr);
|
|
+ kunmap_local(dst_addr);
|
|
+
|
|
+ cc->size -= n;
|
|
+ cc->offset += n;
|
|
+ len -= n;
|
|
+ copied += n;
|
|
+ }
|
|
+ return copied;
|
|
+}
|
|
+
|
|
static ssize_t io_zcrx_copy_chunk(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
|
|
struct page *src_page, unsigned int src_offset,
|
|
size_t len)
|
|
@@ -981,11 +1026,9 @@ static ssize_t io_zcrx_copy_chunk(struct
|
|
return -EFAULT;
|
|
|
|
while (len) {
|
|
- size_t copy_size = min_t(size_t, PAGE_SIZE, len);
|
|
- const int dst_off = 0;
|
|
+ struct io_copy_cache cc;
|
|
struct net_iov *niov;
|
|
- struct page *dst_page;
|
|
- void *dst_addr, *src_addr;
|
|
+ size_t n;
|
|
|
|
niov = io_zcrx_alloc_fallback(area);
|
|
if (!niov) {
|
|
@@ -993,25 +1036,22 @@ static ssize_t io_zcrx_copy_chunk(struct
|
|
break;
|
|
}
|
|
|
|
- dst_page = io_zcrx_iov_page(niov);
|
|
- dst_addr = kmap_local_page(dst_page);
|
|
- src_addr = kmap_local_page(src_page);
|
|
-
|
|
- memcpy(dst_addr, src_addr + src_offset, copy_size);
|
|
+ cc.page = io_zcrx_iov_page(niov);
|
|
+ cc.offset = 0;
|
|
+ cc.size = PAGE_SIZE;
|
|
|
|
- kunmap_local(src_addr);
|
|
- kunmap_local(dst_addr);
|
|
+ n = io_copy_page(&cc, src_page, src_offset, len);
|
|
|
|
- if (!io_zcrx_queue_cqe(req, niov, ifq, dst_off, copy_size)) {
|
|
+ if (!io_zcrx_queue_cqe(req, niov, ifq, 0, n)) {
|
|
io_zcrx_return_niov(niov);
|
|
ret = -ENOSPC;
|
|
break;
|
|
}
|
|
|
|
io_zcrx_get_niov_uref(niov);
|
|
- src_offset += copy_size;
|
|
- len -= copy_size;
|
|
- copied += copy_size;
|
|
+ src_offset += n;
|
|
+ len -= n;
|
|
+ copied += n;
|
|
}
|
|
|
|
return copied ? copied : ret;
|
|
@@ -1021,19 +1061,8 @@ static int io_zcrx_copy_frag(struct io_k
|
|
const skb_frag_t *frag, int off, int len)
|
|
{
|
|
struct page *page = skb_frag_page(frag);
|
|
- u32 p_off, p_len, t, copied = 0;
|
|
- int ret = 0;
|
|
|
|
- off += skb_frag_off(frag);
|
|
-
|
|
- skb_frag_foreach_page(frag, off, len,
|
|
- page, p_off, p_len, t) {
|
|
- ret = io_zcrx_copy_chunk(req, ifq, page, p_off, p_len);
|
|
- if (ret < 0)
|
|
- return copied ? copied : ret;
|
|
- copied += ret;
|
|
- }
|
|
- return copied;
|
|
+ return io_zcrx_copy_chunk(req, ifq, page, off + skb_frag_off(frag), len);
|
|
}
|
|
|
|
static int io_zcrx_recv_frag(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
|