Lines Matching refs:psock
56 static void kcm_abort_tx_psock(struct kcm_psock *psock, int err, in kcm_abort_tx_psock() argument
59 struct sock *csk = psock->sk; in kcm_abort_tx_psock()
60 struct kcm_mux *mux = psock->mux; in kcm_abort_tx_psock()
66 if (psock->tx_stopped) { in kcm_abort_tx_psock()
71 psock->tx_stopped = 1; in kcm_abort_tx_psock()
72 KCM_STATS_INCR(psock->stats.tx_aborts); in kcm_abort_tx_psock()
74 if (!psock->tx_kcm) { in kcm_abort_tx_psock()
76 list_del(&psock->psock_avail_list); in kcm_abort_tx_psock()
85 queue_work(kcm_wq, &psock->tx_kcm->tx_work); in kcm_abort_tx_psock()
96 struct kcm_psock *psock) in kcm_update_rx_mux_stats() argument
99 psock->strp.stats.bytes - in kcm_update_rx_mux_stats()
100 psock->saved_rx_bytes); in kcm_update_rx_mux_stats()
102 psock->strp.stats.msgs - psock->saved_rx_msgs; in kcm_update_rx_mux_stats()
103 psock->saved_rx_msgs = psock->strp.stats.msgs; in kcm_update_rx_mux_stats()
104 psock->saved_rx_bytes = psock->strp.stats.bytes; in kcm_update_rx_mux_stats()
108 struct kcm_psock *psock) in kcm_update_tx_mux_stats() argument
111 psock->stats.tx_bytes - psock->saved_tx_bytes); in kcm_update_tx_mux_stats()
113 psock->stats.tx_msgs - psock->saved_tx_msgs; in kcm_update_tx_mux_stats()
114 psock->saved_tx_msgs = psock->stats.tx_msgs; in kcm_update_tx_mux_stats()
115 psock->saved_tx_bytes = psock->stats.tx_bytes; in kcm_update_tx_mux_stats()
127 struct kcm_psock *psock; in kcm_rcv_ready() local
143 psock = list_first_entry(&mux->psocks_ready, struct kcm_psock, in kcm_rcv_ready()
146 if (kcm_queue_rcv_skb(&kcm->sk, psock->ready_rx_msg)) { in kcm_rcv_ready()
155 list_del(&psock->psock_ready_list); in kcm_rcv_ready()
156 psock->ready_rx_msg = NULL; in kcm_rcv_ready()
160 strp_unpause(&psock->strp); in kcm_rcv_ready()
161 strp_check_rcv(&psock->strp); in kcm_rcv_ready()
255 static struct kcm_sock *reserve_rx_kcm(struct kcm_psock *psock, in reserve_rx_kcm() argument
258 struct kcm_mux *mux = psock->mux; in reserve_rx_kcm()
261 WARN_ON(psock->ready_rx_msg); in reserve_rx_kcm()
263 if (psock->rx_kcm) in reserve_rx_kcm()
264 return psock->rx_kcm; in reserve_rx_kcm()
268 if (psock->rx_kcm) { in reserve_rx_kcm()
270 return psock->rx_kcm; in reserve_rx_kcm()
273 kcm_update_rx_mux_stats(mux, psock); in reserve_rx_kcm()
276 psock->ready_rx_msg = head; in reserve_rx_kcm()
277 strp_pause(&psock->strp); in reserve_rx_kcm()
278 list_add_tail(&psock->psock_ready_list, in reserve_rx_kcm()
290 psock->rx_kcm = kcm; in reserve_rx_kcm()
292 WRITE_ONCE(kcm->rx_psock, psock); in reserve_rx_kcm()
307 static void unreserve_rx_kcm(struct kcm_psock *psock, in unreserve_rx_kcm() argument
310 struct kcm_sock *kcm = psock->rx_kcm; in unreserve_rx_kcm()
311 struct kcm_mux *mux = psock->mux; in unreserve_rx_kcm()
318 psock->rx_kcm = NULL; in unreserve_rx_kcm()
352 struct kcm_psock *psock; in psock_data_ready() local
358 psock = (struct kcm_psock *)sk->sk_user_data; in psock_data_ready()
359 if (likely(psock)) in psock_data_ready()
360 strp_data_ready(&psock->strp); in psock_data_ready()
368 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_rcv_strparser() local
372 kcm = reserve_rx_kcm(psock, skb); in kcm_rcv_strparser()
382 unreserve_rx_kcm(psock, false); in kcm_rcv_strparser()
389 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_parse_func_strparser() local
390 struct bpf_prog *prog = psock->bpf_prog; in kcm_parse_func_strparser()
399 struct kcm_psock *psock = container_of(strp, struct kcm_psock, strp); in kcm_read_sock_done() local
401 unreserve_rx_kcm(psock, true); in kcm_read_sock_done()
418 struct kcm_psock *psock; in psock_write_space() local
424 psock = (struct kcm_psock *)sk->sk_user_data; in psock_write_space()
425 if (unlikely(!psock)) in psock_write_space()
427 mux = psock->mux; in psock_write_space()
432 kcm = psock->tx_kcm; in psock_write_space()
447 struct kcm_psock *psock; in reserve_psock() local
449 psock = kcm->tx_psock; in reserve_psock()
453 if (psock) { in reserve_psock()
455 if (unlikely(psock->tx_stopped)) in reserve_psock()
466 psock = kcm->tx_psock; in reserve_psock()
467 if (unlikely(psock)) { in reserve_psock()
474 psock = list_first_entry(&mux->psocks_avail, in reserve_psock()
477 list_del(&psock->psock_avail_list); in reserve_psock()
482 kcm->tx_psock = psock; in reserve_psock()
483 psock->tx_kcm = kcm; in reserve_psock()
484 KCM_STATS_INCR(psock->stats.reserved); in reserve_psock()
493 return psock; in reserve_psock()
497 static void psock_now_avail(struct kcm_psock *psock) in psock_now_avail() argument
499 struct kcm_mux *mux = psock->mux; in psock_now_avail()
503 list_add_tail(&psock->psock_avail_list, in psock_now_avail()
511 psock->tx_kcm = kcm; in psock_now_avail()
518 kcm->tx_psock = psock; in psock_now_avail()
519 KCM_STATS_INCR(psock->stats.reserved); in psock_now_avail()
527 struct kcm_psock *psock; in unreserve_psock() local
532 psock = kcm->tx_psock; in unreserve_psock()
534 if (WARN_ON(!psock)) { in unreserve_psock()
541 kcm_update_tx_mux_stats(mux, psock); in unreserve_psock()
546 psock->tx_kcm = NULL; in unreserve_psock()
547 KCM_STATS_INCR(psock->stats.unreserved); in unreserve_psock()
549 if (unlikely(psock->tx_stopped)) { in unreserve_psock()
550 if (psock->done) { in unreserve_psock()
552 list_del(&psock->psock_list); in unreserve_psock()
554 sock_put(psock->sk); in unreserve_psock()
555 fput(psock->sk->sk_socket->file); in unreserve_psock()
556 kmem_cache_free(kcm_psockp, psock); in unreserve_psock()
566 psock_now_avail(psock); in unreserve_psock()
587 struct kcm_psock *psock; in kcm_write_msgs() local
592 psock = kcm->tx_psock; in kcm_write_msgs()
593 if (unlikely(psock && psock->tx_stopped)) { in kcm_write_msgs()
616 psock = reserve_psock(kcm); in kcm_write_msgs()
617 if (!psock) in kcm_write_msgs()
624 if (WARN_ON(!psock)) { in kcm_write_msgs()
647 ret = sock_sendmsg(psock->sk->sk_socket, &msg); in kcm_write_msgs()
663 kcm_abort_tx_psock(psock, ret ? -ret : EPIPE, in kcm_write_msgs()
666 psock = NULL; in kcm_write_msgs()
676 KCM_STATS_ADD(psock->stats.tx_bytes, ret); in kcm_write_msgs()
696 KCM_STATS_INCR(psock->stats.tx_msgs); in kcm_write_msgs()
702 if (psock) in kcm_write_msgs()
1228 struct kcm_psock *psock = NULL, *tpsock; in kcm_attach() local
1257 psock = kmem_cache_zalloc(kcm_psockp, GFP_KERNEL); in kcm_attach()
1258 if (!psock) { in kcm_attach()
1263 psock->mux = mux; in kcm_attach()
1264 psock->sk = csk; in kcm_attach()
1265 psock->bpf_prog = prog; in kcm_attach()
1274 kmem_cache_free(kcm_psockp, psock); in kcm_attach()
1279 err = strp_init(&psock->strp, csk, &cb); in kcm_attach()
1282 kmem_cache_free(kcm_psockp, psock); in kcm_attach()
1286 psock->save_data_ready = csk->sk_data_ready; in kcm_attach()
1287 psock->save_write_space = csk->sk_write_space; in kcm_attach()
1288 psock->save_state_change = csk->sk_state_change; in kcm_attach()
1289 csk->sk_user_data = psock; in kcm_attach()
1308 list_add(&psock->psock_list, head); in kcm_attach()
1309 psock->index = index; in kcm_attach()
1313 psock_now_avail(psock); in kcm_attach()
1317 strp_check_rcv(&psock->strp); in kcm_attach()
1355 static void kcm_unattach(struct kcm_psock *psock) in kcm_unattach() argument
1357 struct sock *csk = psock->sk; in kcm_unattach()
1358 struct kcm_mux *mux = psock->mux; in kcm_unattach()
1367 csk->sk_data_ready = psock->save_data_ready; in kcm_unattach()
1368 csk->sk_write_space = psock->save_write_space; in kcm_unattach()
1369 csk->sk_state_change = psock->save_state_change; in kcm_unattach()
1370 strp_stop(&psock->strp); in kcm_unattach()
1372 if (WARN_ON(psock->rx_kcm)) { in kcm_unattach()
1383 if (psock->ready_rx_msg) { in kcm_unattach()
1384 list_del(&psock->psock_ready_list); in kcm_unattach()
1385 kfree_skb(psock->ready_rx_msg); in kcm_unattach()
1386 psock->ready_rx_msg = NULL; in kcm_unattach()
1396 strp_done(&psock->strp); in kcm_unattach()
1399 bpf_prog_put(psock->bpf_prog); in kcm_unattach()
1403 aggregate_psock_stats(&psock->stats, &mux->aggregate_psock_stats); in kcm_unattach()
1404 save_strp_stats(&psock->strp, &mux->aggregate_strp_stats); in kcm_unattach()
1408 if (psock->tx_kcm) { in kcm_unattach()
1420 kcm_abort_tx_psock(psock, EPIPE, false); in kcm_unattach()
1423 if (!psock->tx_kcm) { in kcm_unattach()
1427 psock->done = 1; in kcm_unattach()
1433 queue_work(kcm_wq, &psock->tx_kcm->tx_work); in kcm_unattach()
1437 if (!psock->tx_stopped) in kcm_unattach()
1438 list_del(&psock->psock_avail_list); in kcm_unattach()
1439 list_del(&psock->psock_list); in kcm_unattach()
1445 kmem_cache_free(kcm_psockp, psock); in kcm_unattach()
1455 struct kcm_psock *psock; in kcm_unattach_ioctl() local
1474 list_for_each_entry(psock, &mux->psocks, psock_list) { in kcm_unattach_ioctl()
1475 if (psock->sk != csk) in kcm_unattach_ioctl()
1480 if (psock->unattaching || WARN_ON(psock->done)) { in kcm_unattach_ioctl()
1485 psock->unattaching = 1; in kcm_unattach_ioctl()
1490 kcm_unattach(psock); in kcm_unattach_ioctl()
1595 struct kcm_psock *psock, *tmp_psock; in release_mux() local
1598 list_for_each_entry_safe(psock, tmp_psock, in release_mux()
1600 if (!WARN_ON(psock->unattaching)) in release_mux()
1601 kcm_unattach(psock); in release_mux()
1678 struct kcm_psock *psock; in kcm_release() local
1714 psock = kcm->tx_psock; in kcm_release()
1715 if (psock) { in kcm_release()
1720 kcm_abort_tx_psock(psock, EPIPE, false); in kcm_release()