@@ -22,7 +22,9 @@ struct ifq_region {
int count;
int nr_pages;
u16 id;
- struct page *freelist[];
+
+ struct io_zctap_buf *buf;
+ struct io_zctap_buf *freelist[];
};
typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf);
@@ -62,35 +64,65 @@ static void io_remove_ifq_region(struct ifq_region *ifr)
set_page_private(page, 0);
}
+ kvfree(ifr->buf);
kvfree(ifr);
}
-static int io_zctap_map_region(struct ifq_region *ifr)
+static inline struct device *
+netdev2device(struct net_device *dev)
+{
+ return dev->dev.parent; /* from SET_NETDEV_DEV() */
+}
+
+static int io_zctap_map_region(struct ifq_region *ifr, struct device *device)
{
struct io_mapped_ubuf *imu;
+ struct io_zctap_buf *buf;
struct page *page;
+ dma_addr_t addr;
+ int i, err;
u64 info;
- int i;
imu = ifr->imu;
for (i = 0; i < ifr->nr_pages; i++) {
page = imu->bvec[i].bv_page;
- if (PagePrivate(page))
+
+ if (PagePrivate(page)) {
+ err = -EEXIST;
goto out;
+ }
+
SetPagePrivate(page);
info = zctap_mk_page_info(ifr->id, i);
zctap_set_page_info(page, info);
- ifr->freelist[i] = page;
+
+ buf = &ifr->buf[i];
+ addr = dma_map_page_attrs(device, page, 0, PAGE_SIZE,
+ DMA_BIDIRECTIONAL,
+ DMA_ATTR_SKIP_CPU_SYNC);
+ if (dma_mapping_error(device, addr)) {
+ err = -ENOMEM;
+ goto out;
+ }
+ buf->dma = addr;
+ buf->page = page;
+ atomic_set(&buf->refcount, 0);
+
+ ifr->freelist[i] = buf;
}
return 0;
out:
while (i--) {
page = imu->bvec[i].bv_page;
- ClearPagePrivate(page);
set_page_private(page, 0);
+ ClearPagePrivate(page);
+ buf = &ifr->buf[i];
+ dma_unmap_page_attrs(device, buf->dma, PAGE_SIZE,
+ DMA_BIDIRECTIONAL,
+ DMA_ATTR_SKIP_CPU_SYNC);
}
- return -EEXIST;
+ return err;
}
int io_provide_ifq_region(struct io_zctap_ifq *ifq, u16 id)
@@ -119,13 +151,20 @@ int io_provide_ifq_region(struct io_zctap_ifq *ifq, u16 id)
if (!ifr)
return -ENOMEM;
+ ifr->buf = kvmalloc_array(nr_pages, sizeof(*ifr->buf), GFP_KERNEL);
+ if (!ifr->buf) {
+ kvfree(ifr);
+ return -ENOMEM;
+ }
+
ifr->nr_pages = nr_pages;
ifr->imu = imu;
ifr->count = nr_pages;
ifr->id = id;
- err = io_zctap_map_region(ifr);
+ err = io_zctap_map_region(ifr, netdev2device(ifq->dev));
if (err) {
+ kvfree(ifr->buf);
kvfree(ifr);
return err;
}
The idea is to register a memory region with the device, and later specify the desired packet buffer size. The code currently assumes a page size. Create the desired number of zctap buffers and DMA map them to the target device, recording the dma address for later use. Signed-off-by: Jonathan Lemon <jonathan.lemon@gmail.com> --- io_uring/zctap.c | 55 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 8 deletions(-)