Lines Matching refs:uarg
1171 struct ubuf_info *uarg; in msg_zerocopy_alloc() local
1180 BUILD_BUG_ON(sizeof(*uarg) > sizeof(skb->cb)); in msg_zerocopy_alloc()
1181 uarg = (void *)skb->cb; in msg_zerocopy_alloc()
1182 uarg->mmp.user = NULL; in msg_zerocopy_alloc()
1184 if (mm_account_pinned_pages(&uarg->mmp, size)) { in msg_zerocopy_alloc()
1189 uarg->callback = msg_zerocopy_callback; in msg_zerocopy_alloc()
1190 uarg->id = ((u32)atomic_inc_return(&sk->sk_zckey)) - 1; in msg_zerocopy_alloc()
1191 uarg->len = 1; in msg_zerocopy_alloc()
1192 uarg->bytelen = size; in msg_zerocopy_alloc()
1193 uarg->zerocopy = 1; in msg_zerocopy_alloc()
1194 uarg->flags = SKBFL_ZEROCOPY_FRAG; in msg_zerocopy_alloc()
1195 refcount_set(&uarg->refcnt, 1); in msg_zerocopy_alloc()
1198 return uarg; in msg_zerocopy_alloc()
1202 static inline struct sk_buff *skb_from_uarg(struct ubuf_info *uarg) in skb_from_uarg() argument
1204 return container_of((void *)uarg, struct sk_buff, cb); in skb_from_uarg()
1208 struct ubuf_info *uarg) in msg_zerocopy_realloc() argument
1210 if (uarg) { in msg_zerocopy_realloc()
1222 bytelen = uarg->bytelen + size; in msg_zerocopy_realloc()
1223 if (uarg->len == USHRT_MAX - 1 || bytelen > byte_limit) { in msg_zerocopy_realloc()
1231 if ((u32)(uarg->id + uarg->len) == next) { in msg_zerocopy_realloc()
1232 if (mm_account_pinned_pages(&uarg->mmp, size)) in msg_zerocopy_realloc()
1234 uarg->len++; in msg_zerocopy_realloc()
1235 uarg->bytelen = bytelen; in msg_zerocopy_realloc()
1240 net_zcopy_get(uarg); in msg_zerocopy_realloc()
1242 return uarg; in msg_zerocopy_realloc()
1271 static void __msg_zerocopy_callback(struct ubuf_info *uarg) in __msg_zerocopy_callback() argument
1273 struct sk_buff *tail, *skb = skb_from_uarg(uarg); in __msg_zerocopy_callback()
1282 mm_unaccount_pinned_pages(&uarg->mmp); in __msg_zerocopy_callback()
1287 if (!uarg->len || sock_flag(sk, SOCK_DEAD)) in __msg_zerocopy_callback()
1290 len = uarg->len; in __msg_zerocopy_callback()
1291 lo = uarg->id; in __msg_zerocopy_callback()
1292 hi = uarg->id + len - 1; in __msg_zerocopy_callback()
1293 is_zerocopy = uarg->zerocopy; in __msg_zerocopy_callback()
1321 void msg_zerocopy_callback(struct sk_buff *skb, struct ubuf_info *uarg, in msg_zerocopy_callback() argument
1324 uarg->zerocopy = uarg->zerocopy & success; in msg_zerocopy_callback()
1326 if (refcount_dec_and_test(&uarg->refcnt)) in msg_zerocopy_callback()
1327 __msg_zerocopy_callback(uarg); in msg_zerocopy_callback()
1331 void msg_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref) in msg_zerocopy_put_abort() argument
1333 struct sock *sk = skb_from_uarg(uarg)->sk; in msg_zerocopy_put_abort()
1336 uarg->len--; in msg_zerocopy_put_abort()
1339 msg_zerocopy_callback(NULL, uarg, true); in msg_zerocopy_put_abort()
1351 struct ubuf_info *uarg) in skb_zerocopy_iter_stream() argument
1360 if (orig_uarg && uarg != orig_uarg) in skb_zerocopy_iter_stream()
1375 skb_zcopy_set(skb, uarg, NULL); in skb_zerocopy_iter_stream()