1 // SPDX-License-Identifier: GPL-2.0
2 #ifndef IOU_ZC_RX_H
3 #define IOU_ZC_RX_H
4
5 #include <linux/io_uring_types.h>
6 #include <linux/dma-buf.h>
7 #include <linux/socket.h>
8 #include <net/page_pool/types.h>
9 #include <net/net_trackers.h>
10
11 struct io_zcrx_mem {
12 unsigned long size;
13 bool is_dmabuf;
14
15 struct page **pages;
16 unsigned long nr_folios;
17 struct sg_table page_sg_table;
18 unsigned long account_pages;
19
20 struct dma_buf_attachment *attach;
21 struct dma_buf *dmabuf;
22 struct sg_table *sgt;
23 unsigned long dmabuf_offset;
24 };
25
26 struct io_zcrx_area {
27 struct net_iov_area nia;
28 struct io_zcrx_ifq *ifq;
29 atomic_t *user_refs;
30
31 bool is_mapped;
32 u16 area_id;
33
34 /* freelist */
35 spinlock_t freelist_lock ____cacheline_aligned_in_smp;
36 u32 free_count;
37 u32 *freelist;
38
39 struct io_zcrx_mem mem;
40 };
41
42 struct io_zcrx_ifq {
43 struct io_ring_ctx *ctx;
44 struct io_zcrx_area *area;
45
46 spinlock_t rq_lock ____cacheline_aligned_in_smp;
47 struct io_uring *rq_ring;
48 struct io_uring_zcrx_rqe *rqes;
49 u32 cached_rq_head;
50 u32 rq_entries;
51
52 u32 if_rxq;
53 struct device *dev;
54 struct net_device *netdev;
55 netdevice_tracker netdev_tracker;
56 spinlock_t lock;
57 struct mutex dma_lock;
58 struct io_mapped_region region;
59 };
60
61 #if defined(CONFIG_IO_URING_ZCRX)
62 int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
63 struct io_uring_zcrx_ifq_reg __user *arg);
64 void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx);
65 void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx);
66 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
67 struct socket *sock, unsigned int flags,
68 unsigned issue_flags, unsigned int *len);
69 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
70 unsigned int id);
71 #else
io_register_zcrx_ifq(struct io_ring_ctx * ctx,struct io_uring_zcrx_ifq_reg __user * arg)72 static inline int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
73 struct io_uring_zcrx_ifq_reg __user *arg)
74 {
75 return -EOPNOTSUPP;
76 }
io_unregister_zcrx_ifqs(struct io_ring_ctx * ctx)77 static inline void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx)
78 {
79 }
io_shutdown_zcrx_ifqs(struct io_ring_ctx * ctx)80 static inline void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx)
81 {
82 }
io_zcrx_recv(struct io_kiocb * req,struct io_zcrx_ifq * ifq,struct socket * sock,unsigned int flags,unsigned issue_flags,unsigned int * len)83 static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
84 struct socket *sock, unsigned int flags,
85 unsigned issue_flags, unsigned int *len)
86 {
87 return -EOPNOTSUPP;
88 }
io_zcrx_get_region(struct io_ring_ctx * ctx,unsigned int id)89 static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
90 unsigned int id)
91 {
92 return NULL;
93 }
94 #endif
95
96 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags);
97 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
98
99 #endif
100