Lines Matching refs:nd
16 struct io_notif_data *nd = io_notif_to_data(notif); in io_notif_tw_complete() local
19 notif = cmd_to_io_kiocb(nd); in io_notif_tw_complete()
21 lockdep_assert(refcount_read(&nd->uarg.refcnt) == 0); in io_notif_tw_complete()
23 if (unlikely(nd->zc_report) && (nd->zc_copied || !nd->zc_used)) in io_notif_tw_complete()
26 if (nd->account_pages && notif->ctx->user) { in io_notif_tw_complete()
27 __io_unaccount_mem(notif->ctx->user, nd->account_pages); in io_notif_tw_complete()
28 nd->account_pages = 0; in io_notif_tw_complete()
31 nd = nd->next; in io_notif_tw_complete()
33 } while (nd); in io_notif_tw_complete()
39 struct io_notif_data *nd = container_of(uarg, struct io_notif_data, uarg); in io_tx_ubuf_complete() local
40 struct io_kiocb *notif = cmd_to_io_kiocb(nd); in io_tx_ubuf_complete()
43 if (nd->zc_report) { in io_tx_ubuf_complete()
44 if (success && !nd->zc_used && skb) in io_tx_ubuf_complete()
45 WRITE_ONCE(nd->zc_used, true); in io_tx_ubuf_complete()
46 else if (!success && !nd->zc_copied) in io_tx_ubuf_complete()
47 WRITE_ONCE(nd->zc_copied, true); in io_tx_ubuf_complete()
53 if (nd->head != nd) { in io_tx_ubuf_complete()
54 io_tx_ubuf_complete(skb, &nd->head->uarg, success); in io_tx_ubuf_complete()
58 tw_flags = nd->next ? 0 : IOU_F_TWQ_LAZY_WAKE; in io_tx_ubuf_complete()
65 struct io_notif_data *nd, *prev_nd; in io_link_skb() local
69 nd = container_of(uarg, struct io_notif_data, uarg); in io_link_skb()
70 notif = cmd_to_io_kiocb(nd); in io_link_skb()
73 net_zcopy_get(&nd->uarg); in io_link_skb()
74 skb_zcopy_init(skb, &nd->uarg); in io_link_skb()
78 if (unlikely(prev_uarg == &nd->uarg)) in io_link_skb()
81 if (unlikely(nd->head != nd || nd->next)) in io_link_skb()
88 prev_notif = cmd_to_io_kiocb(nd); in io_link_skb()
95 nd->head = prev_nd->head; in io_link_skb()
96 nd->next = prev_nd->next; in io_link_skb()
97 prev_nd->next = nd; in io_link_skb()
98 net_zcopy_get(&nd->head->uarg); in io_link_skb()
111 struct io_notif_data *nd; in io_alloc_notif() local
122 nd = io_notif_to_data(notif); in io_alloc_notif()
123 nd->zc_report = false; in io_alloc_notif()
124 nd->account_pages = 0; in io_alloc_notif()
125 nd->next = NULL; in io_alloc_notif()
126 nd->head = nd; in io_alloc_notif()
128 nd->uarg.flags = IO_NOTIF_UBUF_FLAGS; in io_alloc_notif()
129 nd->uarg.ops = &io_ubuf_ops; in io_alloc_notif()
130 refcount_set(&nd->uarg.refcnt, 1); in io_alloc_notif()