Lines Matching refs:msg
33 struct sk_msg *msg, u32 apply_bytes) in bpf_tcp_ingress() argument
46 tmp->sg.start = msg->sg.start; in bpf_tcp_ingress()
47 i = msg->sg.start; in bpf_tcp_ingress()
49 sge = sk_msg_elem(msg, i); in bpf_tcp_ingress()
60 sk_msg_xfer(tmp, msg, i, size); in bpf_tcp_ingress()
74 } while (i != msg->sg.end); in bpf_tcp_ingress()
77 msg->sg.start = i; in bpf_tcp_ingress()
90 static int tcp_bpf_push(struct sock *sk, struct sk_msg *msg, u32 apply_bytes, in tcp_bpf_push() argument
104 sge = sk_msg_elem(msg, msg->sg.start); in tcp_bpf_push()
117 if (size < sge->length && msg->sg.start != msg->sg.end) in tcp_bpf_push()
128 msg->sg.size -= ret; in tcp_bpf_push()
140 sk_msg_iter_next(msg, start); in tcp_bpf_push()
142 if (msg->sg.start == msg->sg.end) in tcp_bpf_push()
152 static int tcp_bpf_push_locked(struct sock *sk, struct sk_msg *msg, in tcp_bpf_push_locked() argument
158 ret = tcp_bpf_push(sk, msg, apply_bytes, flags, uncharge); in tcp_bpf_push_locked()
164 struct sk_msg *msg, u32 bytes, int flags) in tcp_bpf_sendmsg_redir() argument
172 ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes) : in tcp_bpf_sendmsg_redir()
173 tcp_bpf_push_locked(sk, msg, bytes, flags, false); in tcp_bpf_sendmsg_redir()
221 struct msghdr *msg, in tcp_bpf_recvmsg_parser() argument
233 return inet_recv_error(sk, msg, len, addr_len); in tcp_bpf_recvmsg_parser()
240 return tcp_recvmsg(sk, msg, len, flags, addr_len); in tcp_bpf_recvmsg_parser()
265 copied = sk_msg_recvmsg(sk, psock, msg, len, flags); in tcp_bpf_recvmsg_parser()
334 static int tcp_bpf_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, in tcp_bpf_recvmsg() argument
341 return inet_recv_error(sk, msg, len, addr_len); in tcp_bpf_recvmsg()
348 return tcp_recvmsg(sk, msg, len, flags, addr_len); in tcp_bpf_recvmsg()
352 return tcp_recvmsg(sk, msg, len, flags, addr_len); in tcp_bpf_recvmsg()
356 copied = sk_msg_recvmsg(sk, psock, msg, len, flags); in tcp_bpf_recvmsg()
372 return tcp_recvmsg(sk, msg, len, flags, addr_len); in tcp_bpf_recvmsg()
385 struct sk_msg *msg, int *copied, int flags) in tcp_bpf_send_verdict() argument
387 bool cork = false, enospc = sk_msg_full(msg), redir_ingress; in tcp_bpf_send_verdict()
400 delta = msg->sg.size; in tcp_bpf_send_verdict()
401 psock->eval = sk_psock_msg_verdict(sk, psock, msg); in tcp_bpf_send_verdict()
402 delta -= msg->sg.size; in tcp_bpf_send_verdict()
405 if (msg->cork_bytes && in tcp_bpf_send_verdict()
406 msg->cork_bytes > msg->sg.size && !enospc) { in tcp_bpf_send_verdict()
407 psock->cork_bytes = msg->cork_bytes - msg->sg.size; in tcp_bpf_send_verdict()
414 memcpy(psock->cork, msg, sizeof(*msg)); in tcp_bpf_send_verdict()
418 tosend = msg->sg.size; in tcp_bpf_send_verdict()
425 ret = tcp_bpf_push(sk, msg, tosend, flags, true); in tcp_bpf_send_verdict()
427 *copied -= sk_msg_free(sk, msg); in tcp_bpf_send_verdict()
448 origsize = msg->sg.size; in tcp_bpf_send_verdict()
450 msg, tosend, flags); in tcp_bpf_send_verdict()
451 sent = origsize - msg->sg.size; in tcp_bpf_send_verdict()
459 int free = sk_msg_free(sk, msg); in tcp_bpf_send_verdict()
465 sk_msg_free(sk, msg); in tcp_bpf_send_verdict()
466 kfree(msg); in tcp_bpf_send_verdict()
467 msg = NULL; in tcp_bpf_send_verdict()
473 sk_msg_free(sk, msg); in tcp_bpf_send_verdict()
487 if (msg && in tcp_bpf_send_verdict()
488 msg->sg.data[msg->sg.start].page_link && in tcp_bpf_send_verdict()
489 msg->sg.data[msg->sg.start].length) in tcp_bpf_send_verdict()
495 static int tcp_bpf_sendmsg(struct sock *sk, struct msghdr *msg, size_t size) in tcp_bpf_sendmsg() argument
504 flags = (msg->msg_flags & ~MSG_SENDPAGE_DECRYPTED); in tcp_bpf_sendmsg()
509 return tcp_sendmsg(sk, msg, size); in tcp_bpf_sendmsg()
512 timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); in tcp_bpf_sendmsg()
513 while (msg_data_left(msg)) { in tcp_bpf_sendmsg()
522 copy = msg_data_left(msg); in tcp_bpf_sendmsg()
541 ret = sk_msg_memcopy_from_iter(sk, &msg->msg_iter, msg_tx, in tcp_bpf_sendmsg()
577 err = sk_stream_error(sk, msg->msg_flags, err); in tcp_bpf_sendmsg()