Lines Matching refs:sk
59 void tcp_time_wait(struct sock *sk, int state, int timeo);
276 static inline bool tcp_under_memory_pressure(const struct sock *sk) in tcp_under_memory_pressure() argument
278 if (mem_cgroup_sockets_enabled && sk->sk_memcg && in tcp_under_memory_pressure()
279 mem_cgroup_under_socket_pressure(sk->sk_memcg)) in tcp_under_memory_pressure()
301 static inline void tcp_wmem_free_skb(struct sock *sk, struct sk_buff *skb) in tcp_wmem_free_skb() argument
303 sk_wmem_queued_add(sk, -skb->truesize); in tcp_wmem_free_skb()
305 sk_mem_uncharge(sk, skb->truesize); in tcp_wmem_free_skb()
307 sk_mem_uncharge(sk, SKB_TRUESIZE(skb_end_offset(skb))); in tcp_wmem_free_skb()
311 void sk_forced_mem_schedule(struct sock *sk, int size);
313 bool tcp_check_oom(const struct sock *sk, int shift);
327 void tcp_shutdown(struct sock *sk, int how);
332 void tcp_remove_empty_skb(struct sock *sk);
333 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
334 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size);
335 int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied,
338 int tcp_send_mss(struct sock *sk, int *size_goal, int flags);
339 int tcp_wmem_schedule(struct sock *sk, int copy);
340 void tcp_push(struct sock *sk, int flags, int mss_now, int nonagle,
342 void tcp_release_cb(struct sock *sk);
344 void tcp_write_timer_handler(struct sock *sk);
345 void tcp_delack_timer_handler(struct sock *sk);
346 int tcp_ioctl(struct sock *sk, int cmd, int *karg);
347 enum skb_drop_reason tcp_rcv_state_process(struct sock *sk, struct sk_buff *skb);
348 void tcp_rcv_established(struct sock *sk, struct sk_buff *skb);
349 void tcp_rcv_space_adjust(struct sock *sk);
350 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp);
351 void tcp_twsk_destructor(struct sock *sk);
353 ssize_t tcp_splice_read(struct socket *sk, loff_t *ppos,
356 struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp,
359 static inline void tcp_dec_quickack_mode(struct sock *sk) in tcp_dec_quickack_mode() argument
361 struct inet_connection_sock *icsk = inet_csk(sk); in tcp_dec_quickack_mode()
365 const unsigned int pkts = inet_csk_ack_scheduled(sk) ? 1 : 0; in tcp_dec_quickack_mode()
431 struct sock *tcp_check_req(struct sock *sk, struct sk_buff *skb,
436 void tcp_enter_loss(struct sock *sk);
437 void tcp_cwnd_reduction(struct sock *sk, int newly_acked_sacked, int newly_lost, int flag);
439 void tcp_update_metrics(struct sock *sk);
440 void tcp_init_metrics(struct sock *sk);
443 void __tcp_close(struct sock *sk, long timeout);
444 void tcp_close(struct sock *sk, long timeout);
445 void tcp_init_sock(struct sock *sk);
446 void tcp_init_transfer(struct sock *sk, int bpf_op, struct sk_buff *skb);
449 int do_tcp_getsockopt(struct sock *sk, int level,
451 int tcp_getsockopt(struct sock *sk, int level, int optname,
454 int do_tcp_setsockopt(struct sock *sk, int level, int optname,
456 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
458 void tcp_reset_keepalive_timer(struct sock *sk, unsigned long timeout);
459 void tcp_set_keepalive(struct sock *sk, int val);
461 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
463 int tcp_set_rcvlowat(struct sock *sk, int val);
464 int tcp_set_window_clamp(struct sock *sk, int val);
467 void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
469 void tcp_data_ready(struct sock *sk);
481 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
483 u16 tcp_v6_get_syncookie(struct sock *sk, struct ipv6hdr *iph,
488 struct sock *sk, struct tcphdr *th);
493 void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb);
494 void tcp_v4_mtu_reduced(struct sock *sk);
495 void tcp_req_err(struct sock *sk, u32 seq, bool abort);
496 void tcp_ld_RTO_revert(struct sock *sk, u32 seq);
497 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb);
498 struct sock *tcp_create_openreq_child(const struct sock *sk,
501 void tcp_ca_openreq_child(struct sock *sk, const struct dst_entry *dst);
502 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
507 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb);
508 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len);
509 int tcp_connect(struct sock *sk);
515 struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
520 int tcp_disconnect(struct sock *sk, int flags);
522 void tcp_finish_connect(struct sock *sk, struct sk_buff *skb);
523 int tcp_send_rcvq(struct sock *sk, struct msghdr *msg, size_t size);
524 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb);
527 struct sock *tcp_get_cookie_sock(struct sock *sk, struct sk_buff *skb,
531 struct sock *cookie_v4_check(struct sock *sk, struct sk_buff *skb);
533 struct sock *sk, struct sk_buff *skb,
570 static inline void tcp_synq_overflow(const struct sock *sk) in tcp_synq_overflow() argument
575 if (sk->sk_reuseport) { in tcp_synq_overflow()
578 reuse = rcu_dereference(sk->sk_reuseport_cb); in tcp_synq_overflow()
588 last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp); in tcp_synq_overflow()
590 WRITE_ONCE(tcp_sk_rw(sk)->rx_opt.ts_recent_stamp, now); in tcp_synq_overflow()
594 static inline bool tcp_synq_no_recent_overflow(const struct sock *sk) in tcp_synq_no_recent_overflow() argument
599 if (sk->sk_reuseport) { in tcp_synq_no_recent_overflow()
602 reuse = rcu_dereference(sk->sk_reuseport_cb); in tcp_synq_no_recent_overflow()
611 last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp); in tcp_synq_no_recent_overflow()
657 return skb->sk; in cookie_bpf_ok()
660 struct request_sock *cookie_bpf_check(struct sock *sk, struct sk_buff *skb);
667 static inline struct request_sock *cookie_bpf_check(struct net *net, struct sock *sk, in cookie_bpf_check() argument
676 struct sock *cookie_v6_check(struct sock *sk, struct sk_buff *skb);
684 void tcp_skb_entail(struct sock *sk, struct sk_buff *skb);
686 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
688 int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
689 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs);
690 void tcp_retransmit_timer(struct sock *sk);
693 void tcp_enter_recovery(struct sock *sk, bool ece_ack);
699 int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
705 void tcp_send_fin(struct sock *sk);
706 void tcp_send_active_reset(struct sock *sk, gfp_t priority,
710 void __tcp_send_ack(struct sock *sk, u32 rcv_nxt, u16 flags);
711 void tcp_send_ack(struct sock *sk);
712 void tcp_send_delayed_ack(struct sock *sk);
713 void tcp_send_loss_probe(struct sock *sk);
714 bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto);
719 void tcp_rearm_rto(struct sock *sk);
720 void tcp_synack_rtt_meas(struct sock *sk, struct request_sock *req);
721 void tcp_done_with_error(struct sock *sk, int err);
722 void tcp_reset(struct sock *sk, struct sk_buff *skb);
723 void tcp_fin(struct sock *sk);
724 void tcp_check_space(struct sock *sk);
725 void tcp_sack_compress_send_ack(struct sock *sk);
733 static inline void tcp_add_receive_queue(struct sock *sk, struct sk_buff *skb) in tcp_add_receive_queue() argument
737 __skb_queue_tail(&sk->sk_receive_queue, skb); in tcp_add_receive_queue()
742 static inline void tcp_clear_xmit_timers(struct sock *sk) in tcp_clear_xmit_timers() argument
744 if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1) in tcp_clear_xmit_timers()
745 __sock_put(sk); in tcp_clear_xmit_timers()
747 if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1) in tcp_clear_xmit_timers()
748 __sock_put(sk); in tcp_clear_xmit_timers()
750 inet_csk_clear_xmit_timers(sk); in tcp_clear_xmit_timers()
753 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu);
754 unsigned int tcp_current_mss(struct sock *sk);
755 u32 tcp_clamp_probe0_to_user_timeout(const struct sock *sk, u32 when);
784 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
786 int tcp_read_sock_noack(struct sock *sk, read_descriptor_t *desc,
789 int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor);
790 struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off);
791 void tcp_read_done(struct sock *sk, size_t len);
793 void tcp_initialize_rcv_mss(struct sock *sk);
795 int tcp_mtu_to_mss(struct sock *sk, int pmtu);
796 int tcp_mss_to_mtu(struct sock *sk, int mss);
797 void tcp_mtup_init(struct sock *sk);
799 static inline unsigned int tcp_rto_max(const struct sock *sk) in tcp_rto_max() argument
801 return READ_ONCE(inet_csk(sk)->icsk_rto_max); in tcp_rto_max()
804 static inline void tcp_bound_rto(struct sock *sk) in tcp_bound_rto() argument
806 inet_csk(sk)->icsk_rto = min(inet_csk(sk)->icsk_rto, tcp_rto_max(sk)); in tcp_bound_rto()
830 static inline void tcp_fast_path_check(struct sock *sk) in tcp_fast_path_check() argument
832 struct tcp_sock *tp = tcp_sk(sk); in tcp_fast_path_check()
836 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf && in tcp_fast_path_check()
841 u32 tcp_delack_max(const struct sock *sk);
844 static inline u32 tcp_rto_min(const struct sock *sk) in tcp_rto_min() argument
846 const struct dst_entry *dst = __sk_dst_get(sk); in tcp_rto_min()
847 u32 rto_min = READ_ONCE(inet_csk(sk)->icsk_rto_min); in tcp_rto_min()
854 static inline u32 tcp_rto_min_us(const struct sock *sk) in tcp_rto_min_us() argument
856 return jiffies_to_usecs(tcp_rto_min(sk)); in tcp_rto_min_us()
887 u32 __tcp_select_window(struct sock *sk);
889 void tcp_send_window_probe(struct sock *sk);
1091 INDIRECT_CALLABLE_DECLARE(void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb));
1224 u32 (*ssthresh)(struct sock *sk);
1227 void (*cong_avoid)(struct sock *sk, u32 ack, u32 acked);
1230 void (*set_state)(struct sock *sk, u8 new_state);
1233 void (*cwnd_event)(struct sock *sk, enum tcp_ca_event ev);
1236 void (*in_ack_event)(struct sock *sk, u32 flags);
1239 void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
1242 u32 (*min_tso_segs)(struct sock *sk);
1247 void (*cong_control)(struct sock *sk, u32 ack, int flag, const struct rate_sample *rs);
1251 u32 (*undo_cwnd)(struct sock *sk);
1253 u32 (*sndbuf_expand)(struct sock *sk);
1257 size_t (*get_info)(struct sock *sk, u32 ext, int *attr,
1267 void (*init)(struct sock *sk);
1269 void (*release)(struct sock *sk);
1278 void tcp_assign_congestion_control(struct sock *sk);
1279 void tcp_init_congestion_control(struct sock *sk);
1280 void tcp_cleanup_congestion_control(struct sock *sk);
1286 int tcp_set_congestion_control(struct sock *sk, const char *name, bool load,
1291 u32 tcp_reno_ssthresh(struct sock *sk);
1292 u32 tcp_reno_undo_cwnd(struct sock *sk);
1293 void tcp_reno_cong_avoid(struct sock *sk, u32 ack, u32 acked);
1308 static inline bool tcp_ca_needs_ecn(const struct sock *sk) in tcp_ca_needs_ecn() argument
1310 const struct inet_connection_sock *icsk = inet_csk(sk); in tcp_ca_needs_ecn()
1315 static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event) in tcp_ca_event() argument
1317 const struct inet_connection_sock *icsk = inet_csk(sk); in tcp_ca_event()
1320 icsk->icsk_ca_ops->cwnd_event(sk, event); in tcp_ca_event()
1324 void tcp_set_ca_state(struct sock *sk, const u8 ca_state);
1327 void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb);
1328 void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
1330 void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
1332 void tcp_rate_check_app_limited(struct sock *sk);
1403 static inline bool tcp_in_cwnd_reduction(const struct sock *sk) in tcp_in_cwnd_reduction() argument
1406 (1 << inet_csk(sk)->icsk_ca_state); in tcp_in_cwnd_reduction()
1413 static inline __u32 tcp_current_ssthresh(const struct sock *sk) in tcp_current_ssthresh() argument
1415 const struct tcp_sock *tp = tcp_sk(sk); in tcp_current_ssthresh()
1417 if (tcp_in_cwnd_reduction(sk)) in tcp_current_ssthresh()
1428 void tcp_enter_cwr(struct sock *sk);
1458 static inline bool tcp_is_cwnd_limited(const struct sock *sk) in tcp_is_cwnd_limited() argument
1460 const struct tcp_sock *tp = tcp_sk(sk); in tcp_is_cwnd_limited()
1478 static inline bool tcp_needs_internal_pacing(const struct sock *sk) in tcp_needs_internal_pacing() argument
1480 return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED; in tcp_needs_internal_pacing()
1486 static inline unsigned long tcp_pacing_delay(const struct sock *sk) in tcp_pacing_delay() argument
1488 s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache; in tcp_pacing_delay()
1493 static inline void tcp_reset_xmit_timer(struct sock *sk, in tcp_reset_xmit_timer() argument
1499 when += tcp_pacing_delay(sk); in tcp_reset_xmit_timer()
1500 inet_csk_reset_xmit_timer(sk, what, when, in tcp_reset_xmit_timer()
1501 tcp_rto_max(sk)); in tcp_reset_xmit_timer()
1510 static inline unsigned long tcp_probe0_base(const struct sock *sk) in tcp_probe0_base() argument
1512 return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN); in tcp_probe0_base()
1516 static inline unsigned long tcp_probe0_when(const struct sock *sk, in tcp_probe0_when() argument
1520 inet_csk(sk)->icsk_backoff); in tcp_probe0_when()
1521 u64 when = (u64)tcp_probe0_base(sk) << backoff; in tcp_probe0_when()
1526 static inline void tcp_check_probe_timer(struct sock *sk) in tcp_check_probe_timer() argument
1528 if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending) in tcp_check_probe_timer()
1529 tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, in tcp_check_probe_timer()
1530 tcp_probe0_base(sk), true); in tcp_check_probe_timer()
1558 bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb,
1562 int tcp_filter(struct sock *sk, struct sk_buff *skb, enum skb_drop_reason *reason);
1563 void tcp_set_state(struct sock *sk, int state);
1564 void tcp_done(struct sock *sk);
1565 int tcp_abort(struct sock *sk, int err);
1573 void tcp_cwnd_restart(struct sock *sk, s32 delta);
1575 static inline void tcp_slow_start_after_idle_check(struct sock *sk) in tcp_slow_start_after_idle_check() argument
1577 const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops; in tcp_slow_start_after_idle_check()
1578 struct tcp_sock *tp = tcp_sk(sk); in tcp_slow_start_after_idle_check()
1581 if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) || in tcp_slow_start_after_idle_check()
1585 if (delta > inet_csk(sk)->icsk_rto) in tcp_slow_start_after_idle_check()
1586 tcp_cwnd_restart(sk, delta); in tcp_slow_start_after_idle_check()
1590 void tcp_select_initial_window(const struct sock *sk, int __space,
1602 static inline int tcp_win_from_space(const struct sock *sk, int space) in tcp_win_from_space() argument
1604 return __tcp_win_from_space(tcp_sk(sk)->scaling_ratio, space); in tcp_win_from_space()
1616 static inline int tcp_space_from_win(const struct sock *sk, int win) in tcp_space_from_win() argument
1618 return __tcp_space_from_win(tcp_sk(sk)->scaling_ratio, win); in tcp_space_from_win()
1626 static inline void tcp_scaling_ratio_init(struct sock *sk) in tcp_scaling_ratio_init() argument
1628 tcp_sk(sk)->scaling_ratio = TCP_DEFAULT_SCALING_RATIO; in tcp_scaling_ratio_init()
1632 static inline int tcp_space(const struct sock *sk) in tcp_space() argument
1634 return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) - in tcp_space()
1635 READ_ONCE(sk->sk_backlog.len) - in tcp_space()
1636 atomic_read(&sk->sk_rmem_alloc)); in tcp_space()
1639 static inline int tcp_full_space(const struct sock *sk) in tcp_full_space() argument
1641 return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf)); in tcp_full_space()
1644 static inline void __tcp_adjust_rcv_ssthresh(struct sock *sk, u32 new_ssthresh) in __tcp_adjust_rcv_ssthresh() argument
1646 int unused_mem = sk_unused_reserved_mem(sk); in __tcp_adjust_rcv_ssthresh()
1647 struct tcp_sock *tp = tcp_sk(sk); in __tcp_adjust_rcv_ssthresh()
1652 tcp_win_from_space(sk, unused_mem)); in __tcp_adjust_rcv_ssthresh()
1655 static inline void tcp_adjust_rcv_ssthresh(struct sock *sk) in tcp_adjust_rcv_ssthresh() argument
1657 __tcp_adjust_rcv_ssthresh(sk, 4U * tcp_sk(sk)->advmss); in tcp_adjust_rcv_ssthresh()
1660 void tcp_cleanup_rbuf(struct sock *sk, int copied);
1661 void __tcp_cleanup_rbuf(struct sock *sk, int copied);
1669 static inline bool tcp_rmem_pressure(const struct sock *sk) in tcp_rmem_pressure() argument
1673 if (tcp_under_memory_pressure(sk)) in tcp_rmem_pressure()
1676 rcvbuf = READ_ONCE(sk->sk_rcvbuf); in tcp_rmem_pressure()
1679 return atomic_read(&sk->sk_rmem_alloc) > threshold; in tcp_rmem_pressure()
1682 static inline bool tcp_epollin_ready(const struct sock *sk, int target) in tcp_epollin_ready() argument
1684 const struct tcp_sock *tp = tcp_sk(sk); in tcp_epollin_ready()
1690 return (avail >= target) || tcp_rmem_pressure(sk) || in tcp_epollin_ready()
1691 (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss); in tcp_epollin_ready()
1698 void tcp_enter_memory_pressure(struct sock *sk);
1699 void tcp_leave_memory_pressure(struct sock *sk);
1746 static inline int tcp_fin_time(const struct sock *sk) in tcp_fin_time() argument
1748 int fin_timeout = tcp_sk(sk)->linger2 ? : in tcp_fin_time()
1749 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout); in tcp_fin_time()
1750 const int rto = inet_csk(sk)->icsk_rto; in tcp_fin_time()
1897 const struct sock *sk, const struct sk_buff *skb);
1898 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1901 int tcp_md5_key_copy(struct sock *sk, const union tcp_md5_addr *addr,
1905 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr,
1907 void tcp_clear_md5_list(struct sock *sk);
1908 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1912 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
1916 tcp_md5_do_lookup(const struct sock *sk, int l3index, in tcp_md5_do_lookup() argument
1921 return __tcp_md5_do_lookup(sk, l3index, addr, family, false); in tcp_md5_do_lookup()
1925 tcp_md5_do_lookup_any_l3index(const struct sock *sk, in tcp_md5_do_lookup_any_l3index() argument
1930 return __tcp_md5_do_lookup(sk, 0, addr, family, true); in tcp_md5_do_lookup_any_l3index()
1936 tcp_md5_do_lookup(const struct sock *sk, int l3index, in tcp_md5_do_lookup() argument
1943 tcp_md5_do_lookup_any_l3index(const struct sock *sk, in tcp_md5_do_lookup_any_l3index() argument
1961 void tcp_fastopen_cache_get(struct sock *sk, u16 *mss,
1963 void tcp_fastopen_cache_set(struct sock *sk, u16 mss,
1975 void tcp_fastopen_destroy_cipher(struct sock *sk);
1977 int tcp_fastopen_reset_cipher(struct net *net, struct sock *sk,
1981 void tcp_fastopen_add_skb(struct sock *sk, struct sk_buff *skb);
1982 struct sock *tcp_try_fastopen(struct sock *sk, struct sk_buff *skb,
1987 bool tcp_fastopen_cookie_check(struct sock *sk, u16 *mss,
1989 bool tcp_fastopen_defer_connect(struct sock *sk, int *err);
2002 void tcp_fastopen_active_disable(struct sock *sk);
2003 bool tcp_fastopen_active_should_disable(struct sock *sk);
2004 void tcp_fastopen_active_disable_ofo_check(struct sock *sk);
2005 void tcp_fastopen_active_detect_blackhole(struct sock *sk, bool expired);
2009 struct tcp_fastopen_context *tcp_fastopen_get_ctx(const struct sock *sk) in tcp_fastopen_get_ctx() argument
2013 ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx); in tcp_fastopen_get_ctx()
2015 ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx); in tcp_fastopen_get_ctx()
2047 void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type);
2048 void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type);
2067 void tcp_write_queue_purge(struct sock *sk);
2069 static inline struct sk_buff *tcp_rtx_queue_head(const struct sock *sk) in tcp_rtx_queue_head() argument
2071 return skb_rb_first(&sk->tcp_rtx_queue); in tcp_rtx_queue_head()
2074 static inline struct sk_buff *tcp_rtx_queue_tail(const struct sock *sk) in tcp_rtx_queue_tail() argument
2076 return skb_rb_last(&sk->tcp_rtx_queue); in tcp_rtx_queue_tail()
2079 static inline struct sk_buff *tcp_write_queue_tail(const struct sock *sk) in tcp_write_queue_tail() argument
2081 return skb_peek_tail(&sk->sk_write_queue); in tcp_write_queue_tail()
2084 #define tcp_for_write_queue_from_safe(skb, tmp, sk) \ argument
2085 skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
2087 static inline struct sk_buff *tcp_send_head(const struct sock *sk) in tcp_send_head() argument
2089 return skb_peek(&sk->sk_write_queue); in tcp_send_head()
2092 static inline bool tcp_skb_is_last(const struct sock *sk, in tcp_skb_is_last() argument
2095 return skb_queue_is_last(&sk->sk_write_queue, skb); in tcp_skb_is_last()
2105 static inline bool tcp_write_queue_empty(const struct sock *sk) in tcp_write_queue_empty() argument
2107 const struct tcp_sock *tp = tcp_sk(sk); in tcp_write_queue_empty()
2112 static inline bool tcp_rtx_queue_empty(const struct sock *sk) in tcp_rtx_queue_empty() argument
2114 return RB_EMPTY_ROOT(&sk->tcp_rtx_queue); in tcp_rtx_queue_empty()
2117 static inline bool tcp_rtx_and_write_queues_empty(const struct sock *sk) in tcp_rtx_and_write_queues_empty() argument
2119 return tcp_rtx_queue_empty(sk) && tcp_write_queue_empty(sk); in tcp_rtx_and_write_queues_empty()
2122 static inline void tcp_add_write_queue_tail(struct sock *sk, struct sk_buff *skb) in tcp_add_write_queue_tail() argument
2124 __skb_queue_tail(&sk->sk_write_queue, skb); in tcp_add_write_queue_tail()
2127 if (sk->sk_write_queue.next == skb) in tcp_add_write_queue_tail()
2128 tcp_chrono_start(sk, TCP_CHRONO_BUSY); in tcp_add_write_queue_tail()
2134 struct sock *sk) in tcp_insert_write_queue_before() argument
2136 __skb_queue_before(&sk->sk_write_queue, skb, new); in tcp_insert_write_queue_before()
2139 static inline void tcp_unlink_write_queue(struct sk_buff *skb, struct sock *sk) in tcp_unlink_write_queue() argument
2142 __skb_unlink(skb, &sk->sk_write_queue); in tcp_unlink_write_queue()
2147 static inline void tcp_rtx_queue_unlink(struct sk_buff *skb, struct sock *sk) in tcp_rtx_queue_unlink() argument
2150 rb_erase(&skb->rbnode, &sk->tcp_rtx_queue); in tcp_rtx_queue_unlink()
2153 static inline void tcp_rtx_queue_unlink_and_free(struct sk_buff *skb, struct sock *sk) in tcp_rtx_queue_unlink_and_free() argument
2156 tcp_rtx_queue_unlink(skb, sk); in tcp_rtx_queue_unlink_and_free()
2157 tcp_wmem_free_skb(sk, skb); in tcp_rtx_queue_unlink_and_free()
2160 static inline void tcp_write_collapse_fence(struct sock *sk) in tcp_write_collapse_fence() argument
2162 struct sk_buff *skb = tcp_write_queue_tail(sk); in tcp_write_collapse_fence()
2168 static inline void tcp_push_pending_frames(struct sock *sk) in tcp_push_pending_frames() argument
2170 if (tcp_send_head(sk)) { in tcp_push_pending_frames()
2171 struct tcp_sock *tp = tcp_sk(sk); in tcp_push_pending_frames()
2173 __tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle); in tcp_push_pending_frames()
2192 static inline void tcp_advance_highest_sack(struct sock *sk, struct sk_buff *skb) in tcp_advance_highest_sack() argument
2194 tcp_sk(sk)->highest_sack = skb_rb_next(skb); in tcp_advance_highest_sack()
2197 static inline struct sk_buff *tcp_highest_sack(struct sock *sk) in tcp_highest_sack() argument
2199 return tcp_sk(sk)->highest_sack; in tcp_highest_sack()
2202 static inline void tcp_highest_sack_reset(struct sock *sk) in tcp_highest_sack_reset() argument
2204 tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk); in tcp_highest_sack_reset()
2208 static inline void tcp_highest_sack_replace(struct sock *sk, in tcp_highest_sack_replace() argument
2212 if (old == tcp_highest_sack(sk)) in tcp_highest_sack_replace()
2213 tcp_sk(sk)->highest_sack = new; in tcp_highest_sack_replace()
2217 static inline bool inet_sk_transparent(const struct sock *sk) in inet_sk_transparent() argument
2219 switch (sk->sk_state) { in inet_sk_transparent()
2221 return inet_twsk(sk)->tw_transparent; in inet_sk_transparent()
2223 return inet_rsk(inet_reqsk(sk))->no_srccheck; in inet_sk_transparent()
2225 return inet_test_bit(TRANSPARENT, sk); in inet_sk_transparent()
2261 void tcp_v4_destroy_sock(struct sock *sk);
2291 bool tcp_stream_memory_free(const struct sock *sk, int wake);
2298 int tcp_rtx_synack(const struct sock *sk, struct request_sock *req);
2301 struct sock *sk, struct sk_buff *skb);
2306 struct tcp_md5sig_key *(*md5_lookup) (const struct sock *sk,
2310 const struct sock *sk,
2312 int (*md5_parse)(struct sock *sk,
2318 int (*ao_parse)(struct sock *sk, int optname, sockptr_t optval, int optlen);
2319 struct tcp_ao_key *(*ao_lookup)(const struct sock *sk,
2323 const struct sock *sk,
2326 const struct sock *sk, const struct sk_buff *skb,
2334 struct tcp_md5sig_key *(*req_md5_lookup)(const struct sock *sk,
2338 const struct sock *sk,
2342 struct tcp_ao_key *(*ao_lookup)(const struct sock *sk,
2345 int (*ao_calc_key)(struct tcp_ao_key *mkt, u8 *key, struct request_sock *sk);
2354 struct dst_entry *(*route_req)(const struct sock *sk,
2361 int (*send_synack)(const struct sock *sk, struct dst_entry *dst,
2375 const struct sock *sk, struct sk_buff *skb, in cookie_init_sequence() argument
2378 tcp_synq_overflow(sk); in cookie_init_sequence()
2379 __NET_INC_STATS(sock_net(sk), LINUX_MIB_SYNCOOKIESSENT); in cookie_init_sequence()
2384 const struct sock *sk, struct sk_buff *skb, in cookie_init_sequence() argument
2408 static inline void tcp_get_current_key(const struct sock *sk, in tcp_get_current_key() argument
2412 const struct tcp_sock *tp = tcp_sk(sk); in tcp_get_current_key()
2420 lockdep_sock_is_held(sk)); in tcp_get_current_key()
2431 out->md5_key = tp->af_specific->md5_lookup(sk, sk); in tcp_get_current_key()
2461 void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb);
2462 void tcp_newreno_mark_lost(struct sock *sk, bool snd_una_advanced);
2465 extern bool tcp_rack_mark_lost(struct sock *sk);
2468 extern void tcp_rack_reo_timeout(struct sock *sk);
2469 extern void tcp_rack_update_reo_wnd(struct sock *sk, struct rate_sample *rs);
2488 static inline void tcp_plb_init(const struct sock *sk, in tcp_plb_init() argument
2494 void tcp_plb_update_state(const struct sock *sk, struct tcp_plb_state *plb,
2496 void tcp_plb_check_rehash(struct sock *sk, struct tcp_plb_state *plb);
2497 void tcp_plb_update_state_upon_rto(struct sock *sk, struct tcp_plb_state *plb);
2499 static inline void tcp_warn_once(const struct sock *sk, bool cond, const char *str) in tcp_warn_once() argument
2504 tcp_snd_cwnd(tcp_sk(sk)), in tcp_warn_once()
2505 tcp_sk(sk)->packets_out, tcp_sk(sk)->sacked_out, in tcp_warn_once()
2506 tcp_sk(sk)->lost_out, tcp_sk(sk)->retrans_out, in tcp_warn_once()
2507 tcp_sk(sk)->tlp_high_seq, sk->sk_state, in tcp_warn_once()
2508 inet_csk(sk)->icsk_ca_state, in tcp_warn_once()
2509 tcp_sk(sk)->advmss, tcp_sk(sk)->mss_cache, in tcp_warn_once()
2510 inet_csk(sk)->icsk_pmtu_cookie); in tcp_warn_once()
2514 static inline s64 tcp_rto_delta_us(const struct sock *sk) in tcp_rto_delta_us() argument
2516 const struct sk_buff *skb = tcp_rtx_queue_head(sk); in tcp_rto_delta_us()
2517 u32 rto = inet_csk(sk)->icsk_rto; in tcp_rto_delta_us()
2522 return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp; in tcp_rto_delta_us()
2524 tcp_warn_once(sk, 1, "rtx queue empty: "); in tcp_rto_delta_us()
2566 static inline int tcp_inq(struct sock *sk) in tcp_inq() argument
2568 struct tcp_sock *tp = tcp_sk(sk); in tcp_inq()
2571 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) { in tcp_inq()
2573 } else if (sock_flag(sk, SOCK_URGINLINE) || in tcp_inq()
2581 if (answ && sock_flag(sk, SOCK_DONE)) in tcp_inq()
2613 static inline void tcp_listendrop(const struct sock *sk) in tcp_listendrop() argument
2615 atomic_inc(&((struct sock *)sk)->sk_drops); in tcp_listendrop()
2616 __NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENDROPS); in tcp_listendrop()
2633 int (*init)(struct sock *sk);
2635 void (*update)(struct sock *sk, struct proto *p,
2636 void (*write_space)(struct sock *sk));
2638 void (*release)(struct sock *sk);
2640 int (*get_info)(struct sock *sk, struct sk_buff *skb, bool net_admin);
2641 size_t (*get_info_size)(const struct sock *sk, bool net_admin);
2651 int tcp_set_ulp(struct sock *sk, const char *name);
2653 void tcp_cleanup_ulp(struct sock *sk);
2654 void tcp_update_ulp(struct sock *sk, struct proto *p,
2655 void (*write_space)(struct sock *sk));
2666 int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore);
2667 void tcp_bpf_clone(const struct sock *sk, struct sock *newsk);
2676 void tcp_eat_skb(struct sock *sk, struct sk_buff *skb);
2678 static inline void tcp_eat_skb(struct sock *sk, struct sk_buff *skb) in tcp_eat_skb() argument
2683 int tcp_bpf_sendmsg_redir(struct sock *sk, bool ingress,
2688 static inline void tcp_bpf_clone(const struct sock *sk, struct sock *newsk) in tcp_bpf_clone() argument
2715 static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) in tcp_call_bpf() argument
2721 if (sk_fullsock(sk)) { in tcp_call_bpf()
2724 sock_owned_by_me(sk); in tcp_call_bpf()
2727 sock_ops.sk = sk; in tcp_call_bpf()
2740 static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) in tcp_call_bpf_2arg() argument
2744 return tcp_call_bpf(sk, op, 2, args); in tcp_call_bpf_2arg()
2747 static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, in tcp_call_bpf_3arg() argument
2752 return tcp_call_bpf(sk, op, 3, args); in tcp_call_bpf_3arg()
2756 static inline int tcp_call_bpf(struct sock *sk, int op, u32 nargs, u32 *args) in tcp_call_bpf() argument
2761 static inline int tcp_call_bpf_2arg(struct sock *sk, int op, u32 arg1, u32 arg2) in tcp_call_bpf_2arg() argument
2766 static inline int tcp_call_bpf_3arg(struct sock *sk, int op, u32 arg1, u32 arg2, in tcp_call_bpf_3arg() argument
2774 static inline u32 tcp_timeout_init(struct sock *sk) in tcp_timeout_init() argument
2778 timeout = tcp_call_bpf(sk, BPF_SOCK_OPS_TIMEOUT_INIT, 0, NULL); in tcp_timeout_init()
2785 static inline u32 tcp_rwnd_init_bpf(struct sock *sk) in tcp_rwnd_init_bpf() argument
2789 rwnd = tcp_call_bpf(sk, BPF_SOCK_OPS_RWND_INIT, 0, NULL); in tcp_rwnd_init_bpf()
2796 static inline bool tcp_bpf_ca_needs_ecn(struct sock *sk) in tcp_bpf_ca_needs_ecn() argument
2798 return (tcp_call_bpf(sk, BPF_SOCK_OPS_NEEDS_ECN, 0, NULL) == 1); in tcp_bpf_ca_needs_ecn()
2801 static inline void tcp_bpf_rtt(struct sock *sk, long mrtt, u32 srtt) in tcp_bpf_rtt() argument
2803 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_RTT_CB_FLAG)) in tcp_bpf_rtt()
2804 tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_RTT_CB, mrtt, srtt); in tcp_bpf_rtt()
2813 void (*cad)(struct sock *sk, u32 ack_seq));
2829 static inline u64 tcp_transmit_time(const struct sock *sk) in tcp_transmit_time() argument
2832 u32 delay = (sk->sk_state == TCP_TIME_WAIT) ? in tcp_transmit_time()
2833 tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay; in tcp_transmit_time()
2863 static inline bool tcp_ao_required(struct sock *sk, const void *saddr, in tcp_ao_required() argument
2873 ao_info = rcu_dereference_check(tcp_sk(sk)->ao_info, in tcp_ao_required()
2874 lockdep_sock_is_held(sk)); in tcp_ao_required()
2878 ao_key = tcp_ao_do_lookup(sk, l3index, saddr, family, -1, -1); in tcp_ao_required()
2881 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAOREQUIRED); in tcp_ao_required()
2890 enum skb_drop_reason tcp_inbound_hash(struct sock *sk,