Lines Matching refs:tm
54 static inline struct net *tm_net(struct tcp_metrics_block *tm) in tm_net() argument
56 return read_pnet(&tm->tcpm_net); in tm_net()
59 static bool tcp_metric_locked(struct tcp_metrics_block *tm, in tcp_metric_locked() argument
62 return tm->tcpm_lock & (1 << idx); in tcp_metric_locked()
65 static u32 tcp_metric_get(struct tcp_metrics_block *tm, in tcp_metric_get() argument
68 return tm->tcpm_vals[idx]; in tcp_metric_get()
71 static void tcp_metric_set(struct tcp_metrics_block *tm, in tcp_metric_set() argument
75 tm->tcpm_vals[idx] = val; in tcp_metric_set()
93 static void tcpm_suck_dst(struct tcp_metrics_block *tm, in tcpm_suck_dst() argument
100 tm->tcpm_stamp = jiffies; in tcpm_suck_dst()
113 tm->tcpm_lock = val; in tcpm_suck_dst()
116 tm->tcpm_vals[TCP_METRIC_RTT] = msval * USEC_PER_MSEC; in tcpm_suck_dst()
119 tm->tcpm_vals[TCP_METRIC_RTTVAR] = msval * USEC_PER_MSEC; in tcpm_suck_dst()
120 tm->tcpm_vals[TCP_METRIC_SSTHRESH] = dst_metric_raw(dst, RTAX_SSTHRESH); in tcpm_suck_dst()
121 tm->tcpm_vals[TCP_METRIC_CWND] = dst_metric_raw(dst, RTAX_CWND); in tcpm_suck_dst()
122 tm->tcpm_vals[TCP_METRIC_REORDERING] = dst_metric_raw(dst, RTAX_REORDERING); in tcpm_suck_dst()
124 tm->tcpm_fastopen.mss = 0; in tcpm_suck_dst()
125 tm->tcpm_fastopen.syn_loss = 0; in tcpm_suck_dst()
126 tm->tcpm_fastopen.try_exp = 0; in tcpm_suck_dst()
127 tm->tcpm_fastopen.cookie.exp = false; in tcpm_suck_dst()
128 tm->tcpm_fastopen.cookie.len = 0; in tcpm_suck_dst()
134 static void tcpm_check_stamp(struct tcp_metrics_block *tm, struct dst_entry *dst) in tcpm_check_stamp() argument
136 if (tm && unlikely(time_after(jiffies, tm->tcpm_stamp + TCP_METRICS_TIMEOUT))) in tcpm_check_stamp()
137 tcpm_suck_dst(tm, dst, false); in tcpm_check_stamp()
151 struct tcp_metrics_block *tm; in tcpm_new() local
161 tm = __tcp_get_metrics(saddr, daddr, net, hash); in tcpm_new()
162 if (tm == TCP_METRICS_RECLAIM_PTR) { in tcpm_new()
164 tm = NULL; in tcpm_new()
166 if (tm) { in tcpm_new()
167 tcpm_check_stamp(tm, dst); in tcpm_new()
175 for (tm = deref_locked(oldest->tcpm_next); tm; in tcpm_new()
176 tm = deref_locked(tm->tcpm_next)) { in tcpm_new()
177 if (time_before(tm->tcpm_stamp, oldest->tcpm_stamp)) in tcpm_new()
178 oldest = tm; in tcpm_new()
180 tm = oldest; in tcpm_new()
182 tm = kmalloc(sizeof(*tm), GFP_ATOMIC); in tcpm_new()
183 if (!tm) in tcpm_new()
186 write_pnet(&tm->tcpm_net, net); in tcpm_new()
187 tm->tcpm_saddr = *saddr; in tcpm_new()
188 tm->tcpm_daddr = *daddr; in tcpm_new()
190 tcpm_suck_dst(tm, dst, true); in tcpm_new()
193 tm->tcpm_next = tcp_metrics_hash[hash].chain; in tcpm_new()
194 rcu_assign_pointer(tcp_metrics_hash[hash].chain, tm); in tcpm_new()
199 return tm; in tcpm_new()
202 static struct tcp_metrics_block *tcp_get_encode(struct tcp_metrics_block *tm, int depth) in tcp_get_encode() argument
204 if (tm) in tcp_get_encode()
205 return tm; in tcp_get_encode()
215 struct tcp_metrics_block *tm; in __tcp_get_metrics() local
218 for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm; in __tcp_get_metrics()
219 tm = rcu_dereference(tm->tcpm_next)) { in __tcp_get_metrics()
220 if (addr_same(&tm->tcpm_saddr, saddr) && in __tcp_get_metrics()
221 addr_same(&tm->tcpm_daddr, daddr) && in __tcp_get_metrics()
222 net_eq(tm_net(tm), net)) in __tcp_get_metrics()
226 return tcp_get_encode(tm, depth); in __tcp_get_metrics()
232 struct tcp_metrics_block *tm; in __tcp_get_metrics_req() local
260 for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm; in __tcp_get_metrics_req()
261 tm = rcu_dereference(tm->tcpm_next)) { in __tcp_get_metrics_req()
262 if (addr_same(&tm->tcpm_saddr, &saddr) && in __tcp_get_metrics_req()
263 addr_same(&tm->tcpm_daddr, &daddr) && in __tcp_get_metrics_req()
264 net_eq(tm_net(tm), net)) in __tcp_get_metrics_req()
267 tcpm_check_stamp(tm, dst); in __tcp_get_metrics_req()
268 return tm; in __tcp_get_metrics_req()
275 struct tcp_metrics_block *tm; in tcp_get_metrics() local
305 tm = __tcp_get_metrics(&saddr, &daddr, net, hash); in tcp_get_metrics()
306 if (tm == TCP_METRICS_RECLAIM_PTR) in tcp_get_metrics()
307 tm = NULL; in tcp_get_metrics()
308 if (!tm && create) in tcp_get_metrics()
309 tm = tcpm_new(dst, &saddr, &daddr, hash); in tcp_get_metrics()
311 tcpm_check_stamp(tm, dst); in tcp_get_metrics()
313 return tm; in tcp_get_metrics()
326 struct tcp_metrics_block *tm; in tcp_update_metrics() local
341 tm = tcp_get_metrics(sk, dst, false); in tcp_update_metrics()
342 if (tm && !tcp_metric_locked(tm, TCP_METRIC_RTT)) in tcp_update_metrics()
343 tcp_metric_set(tm, TCP_METRIC_RTT, 0); in tcp_update_metrics()
346 tm = tcp_get_metrics(sk, dst, true); in tcp_update_metrics()
348 if (!tm) in tcp_update_metrics()
351 rtt = tcp_metric_get(tm, TCP_METRIC_RTT); in tcp_update_metrics()
358 if (!tcp_metric_locked(tm, TCP_METRIC_RTT)) { in tcp_update_metrics()
363 tcp_metric_set(tm, TCP_METRIC_RTT, rtt); in tcp_update_metrics()
366 if (!tcp_metric_locked(tm, TCP_METRIC_RTTVAR)) { in tcp_update_metrics()
377 var = tcp_metric_get(tm, TCP_METRIC_RTTVAR); in tcp_update_metrics()
383 tcp_metric_set(tm, TCP_METRIC_RTTVAR, var); in tcp_update_metrics()
389 !tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) { in tcp_update_metrics()
390 val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH); in tcp_update_metrics()
392 tcp_metric_set(tm, TCP_METRIC_SSTHRESH, in tcp_update_metrics()
395 if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) { in tcp_update_metrics()
396 val = tcp_metric_get(tm, TCP_METRIC_CWND); in tcp_update_metrics()
398 tcp_metric_set(tm, TCP_METRIC_CWND, in tcp_update_metrics()
405 !tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) in tcp_update_metrics()
406 tcp_metric_set(tm, TCP_METRIC_SSTHRESH, in tcp_update_metrics()
408 if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) { in tcp_update_metrics()
409 val = tcp_metric_get(tm, TCP_METRIC_CWND); in tcp_update_metrics()
410 tcp_metric_set(tm, TCP_METRIC_CWND, (val + tcp_snd_cwnd(tp)) >> 1); in tcp_update_metrics()
416 if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) { in tcp_update_metrics()
417 val = tcp_metric_get(tm, TCP_METRIC_CWND); in tcp_update_metrics()
418 tcp_metric_set(tm, TCP_METRIC_CWND, in tcp_update_metrics()
422 !tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) { in tcp_update_metrics()
423 val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH); in tcp_update_metrics()
425 tcp_metric_set(tm, TCP_METRIC_SSTHRESH, in tcp_update_metrics()
428 if (!tcp_metric_locked(tm, TCP_METRIC_REORDERING)) { in tcp_update_metrics()
429 val = tcp_metric_get(tm, TCP_METRIC_REORDERING); in tcp_update_metrics()
433 tcp_metric_set(tm, TCP_METRIC_REORDERING, in tcp_update_metrics()
437 tm->tcpm_stamp = jiffies; in tcp_update_metrics()
449 struct tcp_metrics_block *tm; in tcp_init_metrics() local
457 tm = tcp_get_metrics(sk, dst, true); in tcp_init_metrics()
458 if (!tm) { in tcp_init_metrics()
463 if (tcp_metric_locked(tm, TCP_METRIC_CWND)) in tcp_init_metrics()
464 tp->snd_cwnd_clamp = tcp_metric_get(tm, TCP_METRIC_CWND); in tcp_init_metrics()
467 0 : tcp_metric_get(tm, TCP_METRIC_SSTHRESH); in tcp_init_metrics()
478 val = tcp_metric_get(tm, TCP_METRIC_REORDERING); in tcp_init_metrics()
482 crtt = tcp_metric_get(tm, TCP_METRIC_RTT); in tcp_init_metrics()
525 struct tcp_metrics_block *tm; in tcp_peer_is_proven() local
532 tm = __tcp_get_metrics_req(req, dst); in tcp_peer_is_proven()
533 if (tm && tcp_metric_get(tm, TCP_METRIC_RTT)) in tcp_peer_is_proven()
547 struct tcp_metrics_block *tm; in tcp_fastopen_cache_get() local
550 tm = tcp_get_metrics(sk, __sk_dst_get(sk), false); in tcp_fastopen_cache_get()
551 if (tm) { in tcp_fastopen_cache_get()
552 struct tcp_fastopen_metrics *tfom = &tm->tcpm_fastopen; in tcp_fastopen_cache_get()
572 struct tcp_metrics_block *tm; in tcp_fastopen_cache_set() local
577 tm = tcp_get_metrics(sk, dst, true); in tcp_fastopen_cache_set()
578 if (tm) { in tcp_fastopen_cache_set()
579 struct tcp_fastopen_metrics *tfom = &tm->tcpm_fastopen; in tcp_fastopen_cache_set()
623 struct tcp_metrics_block *tm) in tcp_metrics_fill_info() argument
628 switch (tm->tcpm_daddr.family) { in tcp_metrics_fill_info()
631 inetpeer_get_addr_v4(&tm->tcpm_daddr)) < 0) in tcp_metrics_fill_info()
634 inetpeer_get_addr_v4(&tm->tcpm_saddr)) < 0) in tcp_metrics_fill_info()
639 inetpeer_get_addr_v6(&tm->tcpm_daddr)) < 0) in tcp_metrics_fill_info()
642 inetpeer_get_addr_v6(&tm->tcpm_saddr)) < 0) in tcp_metrics_fill_info()
650 jiffies - tm->tcpm_stamp, in tcp_metrics_fill_info()
661 u32 val = tm->tcpm_vals[i]; in tcp_metrics_fill_info()
695 tfom_copy[0] = tm->tcpm_fastopen; in tcp_metrics_fill_info()
724 struct tcp_metrics_block *tm) in tcp_metrics_dump_info() argument
734 if (tcp_metrics_fill_info(skb, tm) < 0) in tcp_metrics_dump_info()
754 struct tcp_metrics_block *tm; in tcp_metrics_nl_dump() local
758 for (col = 0, tm = rcu_dereference(hb->chain); tm; in tcp_metrics_nl_dump()
759 tm = rcu_dereference(tm->tcpm_next), col++) { in tcp_metrics_nl_dump()
760 if (!net_eq(tm_net(tm), net)) in tcp_metrics_nl_dump()
764 if (tcp_metrics_dump_info(skb, cb, tm) < 0) { in tcp_metrics_nl_dump()
822 struct tcp_metrics_block *tm; in tcp_metrics_nl_cmd_get() local
852 for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm; in tcp_metrics_nl_cmd_get()
853 tm = rcu_dereference(tm->tcpm_next)) { in tcp_metrics_nl_cmd_get()
854 if (addr_same(&tm->tcpm_daddr, &daddr) && in tcp_metrics_nl_cmd_get()
855 (!src || addr_same(&tm->tcpm_saddr, &saddr)) && in tcp_metrics_nl_cmd_get()
856 net_eq(tm_net(tm), net)) { in tcp_metrics_nl_cmd_get()
857 ret = tcp_metrics_fill_info(msg, tm); in tcp_metrics_nl_cmd_get()
880 struct tcp_metrics_block *tm; in tcp_metrics_flush_all() local
889 for (tm = deref_locked(*pp); tm; tm = deref_locked(*pp)) { in tcp_metrics_flush_all()
890 match = net ? net_eq(tm_net(tm), net) : in tcp_metrics_flush_all()
891 !refcount_read(&tm_net(tm)->ns.count); in tcp_metrics_flush_all()
893 *pp = tm->tcpm_next; in tcp_metrics_flush_all()
894 kfree_rcu(tm, rcu_head); in tcp_metrics_flush_all()
896 pp = &tm->tcpm_next; in tcp_metrics_flush_all()
906 struct tcp_metrics_block *tm; in tcp_metrics_nl_cmd_del() local
930 for (tm = deref_locked(*pp); tm; tm = deref_locked(*pp)) { in tcp_metrics_nl_cmd_del()
931 if (addr_same(&tm->tcpm_daddr, &daddr) && in tcp_metrics_nl_cmd_del()
932 (!src || addr_same(&tm->tcpm_saddr, &saddr)) && in tcp_metrics_nl_cmd_del()
933 net_eq(tm_net(tm), net)) { in tcp_metrics_nl_cmd_del()
934 *pp = tm->tcpm_next; in tcp_metrics_nl_cmd_del()
935 kfree_rcu(tm, rcu_head); in tcp_metrics_nl_cmd_del()
938 pp = &tm->tcpm_next; in tcp_metrics_nl_cmd_del()