Lines Matching refs:cep
39 static int siw_cm_upcall(struct siw_cep *cep, enum iw_cm_event_type reason,
54 struct siw_cep *cep = sk_to_cep(sk); in siw_sk_save_upcalls() local
57 cep->sk_state_change = sk->sk_state_change; in siw_sk_save_upcalls()
58 cep->sk_data_ready = sk->sk_data_ready; in siw_sk_save_upcalls()
59 cep->sk_write_space = sk->sk_write_space; in siw_sk_save_upcalls()
60 cep->sk_error_report = sk->sk_error_report; in siw_sk_save_upcalls()
64 static void siw_sk_restore_upcalls(struct sock *sk, struct siw_cep *cep) in siw_sk_restore_upcalls() argument
66 sk->sk_state_change = cep->sk_state_change; in siw_sk_restore_upcalls()
67 sk->sk_data_ready = cep->sk_data_ready; in siw_sk_restore_upcalls()
68 sk->sk_write_space = cep->sk_write_space; in siw_sk_restore_upcalls()
69 sk->sk_error_report = cep->sk_error_report; in siw_sk_restore_upcalls()
73 static void siw_qp_socket_assoc(struct siw_cep *cep, struct siw_qp *qp) in siw_qp_socket_assoc() argument
75 struct socket *s = cep->sock; in siw_qp_socket_assoc()
90 struct siw_cep *cep; in siw_socket_disassoc() local
94 cep = sk_to_cep(sk); in siw_socket_disassoc()
95 if (cep) { in siw_socket_disassoc()
96 siw_sk_restore_upcalls(sk, cep); in siw_socket_disassoc()
97 siw_cep_put(cep); in siw_socket_disassoc()
109 struct siw_cep *cep; in siw_rtr_data_ready() local
117 cep = sk_to_cep(sk); in siw_rtr_data_ready()
118 if (!cep) { in siw_rtr_data_ready()
136 siw_cm_upcall(cep, IW_CM_EVENT_ESTABLISHED, 0); in siw_rtr_data_ready()
140 siw_qp_socket_assoc(cep, qp); in siw_rtr_data_ready()
143 static void siw_sk_assign_rtr_upcalls(struct siw_cep *cep) in siw_sk_assign_rtr_upcalls() argument
145 struct sock *sk = cep->sock->sk; in siw_sk_assign_rtr_upcalls()
153 static void siw_cep_socket_assoc(struct siw_cep *cep, struct socket *s) in siw_cep_socket_assoc() argument
155 cep->sock = s; in siw_cep_socket_assoc()
156 siw_cep_get(cep); in siw_cep_socket_assoc()
157 s->sk->sk_user_data = cep; in siw_cep_socket_assoc()
165 struct siw_cep *cep = kzalloc(sizeof(*cep), GFP_KERNEL); in siw_cep_alloc() local
168 if (!cep) in siw_cep_alloc()
171 INIT_LIST_HEAD(&cep->listenq); in siw_cep_alloc()
172 INIT_LIST_HEAD(&cep->devq); in siw_cep_alloc()
173 INIT_LIST_HEAD(&cep->work_freelist); in siw_cep_alloc()
175 kref_init(&cep->ref); in siw_cep_alloc()
176 cep->state = SIW_EPSTATE_IDLE; in siw_cep_alloc()
177 init_waitqueue_head(&cep->waitq); in siw_cep_alloc()
178 spin_lock_init(&cep->lock); in siw_cep_alloc()
179 cep->sdev = sdev; in siw_cep_alloc()
180 cep->enhanced_rdma_conn_est = false; in siw_cep_alloc()
183 list_add_tail(&cep->devq, &sdev->cep_list); in siw_cep_alloc()
186 siw_dbg_cep(cep, "new endpoint\n"); in siw_cep_alloc()
187 return cep; in siw_cep_alloc()
190 static void siw_cm_free_work(struct siw_cep *cep) in siw_cm_free_work() argument
195 list_for_each_safe(w, tmp, &cep->work_freelist) { in siw_cm_free_work()
202 static void siw_cancel_mpatimer(struct siw_cep *cep) in siw_cancel_mpatimer() argument
204 spin_lock_bh(&cep->lock); in siw_cancel_mpatimer()
205 if (cep->mpa_timer) { in siw_cancel_mpatimer()
206 if (cancel_delayed_work(&cep->mpa_timer->work)) { in siw_cancel_mpatimer()
207 siw_cep_put(cep); in siw_cancel_mpatimer()
208 kfree(cep->mpa_timer); /* not needed again */ in siw_cancel_mpatimer()
210 cep->mpa_timer = NULL; in siw_cancel_mpatimer()
212 spin_unlock_bh(&cep->lock); in siw_cancel_mpatimer()
218 spin_lock_bh(&work->cep->lock); in siw_put_work()
219 list_add(&work->list, &work->cep->work_freelist); in siw_put_work()
220 spin_unlock_bh(&work->cep->lock); in siw_put_work()
223 static void siw_cep_set_inuse(struct siw_cep *cep) in siw_cep_set_inuse() argument
227 spin_lock_irqsave(&cep->lock, flags); in siw_cep_set_inuse()
229 if (cep->in_use) { in siw_cep_set_inuse()
230 spin_unlock_irqrestore(&cep->lock, flags); in siw_cep_set_inuse()
231 wait_event_interruptible(cep->waitq, !cep->in_use); in siw_cep_set_inuse()
236 cep->in_use = 1; in siw_cep_set_inuse()
237 spin_unlock_irqrestore(&cep->lock, flags); in siw_cep_set_inuse()
241 static void siw_cep_set_free(struct siw_cep *cep) in siw_cep_set_free() argument
245 spin_lock_irqsave(&cep->lock, flags); in siw_cep_set_free()
246 cep->in_use = 0; in siw_cep_set_free()
247 spin_unlock_irqrestore(&cep->lock, flags); in siw_cep_set_free()
249 wake_up(&cep->waitq); in siw_cep_set_free()
254 struct siw_cep *cep = container_of(ref, struct siw_cep, ref); in __siw_cep_dealloc() local
255 struct siw_device *sdev = cep->sdev; in __siw_cep_dealloc()
258 WARN_ON(cep->listen_cep); in __siw_cep_dealloc()
261 kfree(cep->mpa.pdata); in __siw_cep_dealloc()
262 spin_lock_bh(&cep->lock); in __siw_cep_dealloc()
263 if (!list_empty(&cep->work_freelist)) in __siw_cep_dealloc()
264 siw_cm_free_work(cep); in __siw_cep_dealloc()
265 spin_unlock_bh(&cep->lock); in __siw_cep_dealloc()
268 list_del(&cep->devq); in __siw_cep_dealloc()
271 siw_dbg_cep(cep, "free endpoint\n"); in __siw_cep_dealloc()
272 kfree(cep); in __siw_cep_dealloc()
275 static struct siw_cm_work *siw_get_work(struct siw_cep *cep) in siw_get_work() argument
279 spin_lock_bh(&cep->lock); in siw_get_work()
280 if (!list_empty(&cep->work_freelist)) { in siw_get_work()
281 work = list_entry(cep->work_freelist.next, struct siw_cm_work, in siw_get_work()
285 spin_unlock_bh(&cep->lock); in siw_get_work()
289 static int siw_cm_alloc_work(struct siw_cep *cep, int num) in siw_cm_alloc_work() argument
296 if (!(list_empty(&cep->work_freelist))) in siw_cm_alloc_work()
297 siw_cm_free_work(cep); in siw_cm_alloc_work()
300 work->cep = cep; in siw_cm_alloc_work()
302 list_add(&work->list, &cep->work_freelist); in siw_cm_alloc_work()
312 static int siw_cm_upcall(struct siw_cep *cep, enum iw_cm_event_type reason, in siw_cm_upcall() argument
323 event.provider_data = cep; in siw_cm_upcall()
324 id = cep->listen_cep->cm_id; in siw_cm_upcall()
326 id = cep->cm_id; in siw_cm_upcall()
332 event.ird = cep->ird; in siw_cm_upcall()
333 event.ord = cep->ord; in siw_cm_upcall()
335 event.ird = cep->ord; in siw_cm_upcall()
336 event.ord = cep->ird; in siw_cm_upcall()
341 u16 pd_len = be16_to_cpu(cep->mpa.hdr.params.pd_len); in siw_cm_upcall()
348 event.private_data = cep->mpa.pdata; in siw_cm_upcall()
351 if (cep->enhanced_rdma_conn_est) { in siw_cm_upcall()
358 getname_local(cep->sock, &event.local_addr); in siw_cm_upcall()
359 getname_peer(cep->sock, &event.remote_addr); in siw_cm_upcall()
361 siw_dbg_cep(cep, "[QP %u]: reason=%d, status=%d\n", in siw_cm_upcall()
362 cep->qp ? qp_id(cep->qp) : UINT_MAX, reason, status); in siw_cm_upcall()
377 struct siw_cep *cep = qp->cep; in siw_qp_cm_drop() local
382 if (!qp->cep) in siw_qp_cm_drop()
386 siw_cm_queue_work(cep, SIW_CM_WORK_CLOSE_LLP); in siw_qp_cm_drop()
388 siw_cep_set_inuse(cep); in siw_qp_cm_drop()
390 if (cep->state == SIW_EPSTATE_CLOSED) { in siw_qp_cm_drop()
391 siw_dbg_cep(cep, "already closed\n"); in siw_qp_cm_drop()
394 siw_dbg_cep(cep, "immediate close, state %d\n", cep->state); in siw_qp_cm_drop()
399 if (cep->cm_id) { in siw_qp_cm_drop()
400 switch (cep->state) { in siw_qp_cm_drop()
402 siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, in siw_qp_cm_drop()
407 siw_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0); in siw_qp_cm_drop()
419 cep->cm_id->rem_ref(cep->cm_id); in siw_qp_cm_drop()
420 cep->cm_id = NULL; in siw_qp_cm_drop()
421 siw_cep_put(cep); in siw_qp_cm_drop()
423 cep->state = SIW_EPSTATE_CLOSED; in siw_qp_cm_drop()
425 if (cep->sock) { in siw_qp_cm_drop()
426 siw_socket_disassoc(cep->sock); in siw_qp_cm_drop()
430 sock_release(cep->sock); in siw_qp_cm_drop()
431 cep->sock = NULL; in siw_qp_cm_drop()
433 if (cep->qp) { in siw_qp_cm_drop()
434 cep->qp = NULL; in siw_qp_cm_drop()
438 siw_cep_set_free(cep); in siw_qp_cm_drop()
442 void siw_cep_put(struct siw_cep *cep) in siw_cep_put() argument
444 WARN_ON(kref_read(&cep->ref) < 1); in siw_cep_put()
445 kref_put(&cep->ref, __siw_cep_dealloc); in siw_cep_put()
448 void siw_cep_get(struct siw_cep *cep) in siw_cep_get() argument
450 kref_get(&cep->ref); in siw_cep_get()
456 static int siw_send_mpareqrep(struct siw_cep *cep, const void *pdata, u8 pd_len) in siw_send_mpareqrep() argument
458 struct socket *s = cep->sock; in siw_send_mpareqrep()
459 struct mpa_rr *rr = &cep->mpa.hdr; in siw_send_mpareqrep()
472 if (cep->enhanced_rdma_conn_est) { in siw_send_mpareqrep()
474 iov[iovec_num].iov_base = &cep->mpa.v2_ctrl; in siw_send_mpareqrep()
475 iov[iovec_num].iov_len = sizeof(cep->mpa.v2_ctrl); in siw_send_mpareqrep()
476 mpa_len += sizeof(cep->mpa.v2_ctrl); in siw_send_mpareqrep()
484 if (cep->enhanced_rdma_conn_est) in siw_send_mpareqrep()
485 pd_len += sizeof(cep->mpa.v2_ctrl); in siw_send_mpareqrep()
503 static int siw_recv_mpa_rr(struct siw_cep *cep) in siw_recv_mpa_rr() argument
505 struct mpa_rr *hdr = &cep->mpa.hdr; in siw_recv_mpa_rr()
506 struct socket *s = cep->sock; in siw_recv_mpa_rr()
510 if (cep->mpa.bytes_rcvd < sizeof(struct mpa_rr)) { in siw_recv_mpa_rr()
511 rcvd = ksock_recv(s, (char *)hdr + cep->mpa.bytes_rcvd, in siw_recv_mpa_rr()
512 sizeof(struct mpa_rr) - cep->mpa.bytes_rcvd, in siw_recv_mpa_rr()
517 cep->mpa.bytes_rcvd += rcvd; in siw_recv_mpa_rr()
519 if (cep->mpa.bytes_rcvd < sizeof(struct mpa_rr)) in siw_recv_mpa_rr()
532 to_rcv = pd_len - (cep->mpa.bytes_rcvd - sizeof(struct mpa_rr)); in siw_recv_mpa_rr()
547 siw_dbg_cep(cep, "peer EOF\n"); in siw_recv_mpa_rr()
551 siw_dbg_cep(cep, "error: %d\n", rcvd); in siw_recv_mpa_rr()
554 siw_dbg_cep(cep, "peer sent extra data: %d\n", rcvd); in siw_recv_mpa_rr()
563 if (!cep->mpa.pdata) { in siw_recv_mpa_rr()
564 cep->mpa.pdata = kmalloc(pd_len + 4, GFP_KERNEL); in siw_recv_mpa_rr()
565 if (!cep->mpa.pdata) in siw_recv_mpa_rr()
569 s, cep->mpa.pdata + cep->mpa.bytes_rcvd - sizeof(struct mpa_rr), in siw_recv_mpa_rr()
578 cep->mpa.bytes_rcvd += rcvd; in siw_recv_mpa_rr()
581 siw_dbg_cep(cep, "%d bytes private data received\n", pd_len); in siw_recv_mpa_rr()
593 static int siw_proc_mpareq(struct siw_cep *cep) in siw_proc_mpareq() argument
599 rv = siw_recv_mpa_rr(cep); in siw_proc_mpareq()
603 req = &cep->mpa.hdr; in siw_proc_mpareq()
628 cep->enhanced_rdma_conn_est = true; in siw_proc_mpareq()
649 if (cep->enhanced_rdma_conn_est) { in siw_proc_mpareq()
650 struct mpa_v2_data *v2 = (struct mpa_v2_data *)cep->mpa.pdata; in siw_proc_mpareq()
657 cep->ord = ntohs(v2->ird) & MPA_IRD_ORD_MASK; in siw_proc_mpareq()
658 cep->ord = min(cep->ord, SIW_MAX_ORD_QP); in siw_proc_mpareq()
659 cep->ird = ntohs(v2->ord) & MPA_IRD_ORD_MASK; in siw_proc_mpareq()
660 cep->ird = min(cep->ird, SIW_MAX_IRD_QP); in siw_proc_mpareq()
663 cep->mpa.v2_ctrl.ird = htons(cep->ird); in siw_proc_mpareq()
664 cep->mpa.v2_ctrl.ord = htons(cep->ord); in siw_proc_mpareq()
675 cep->mpa.v2_ctrl.ird |= MPA_V2_PEER_TO_PEER; in siw_proc_mpareq()
678 cep->mpa.v2_ctrl.ord |= MPA_V2_RDMA_WRITE_RTR; in siw_proc_mpareq()
680 cep->mpa.v2_ctrl.ord |= MPA_V2_RDMA_READ_RTR; in siw_proc_mpareq()
682 cep->mpa.v2_ctrl.ord |= MPA_V2_RDMA_WRITE_RTR; in siw_proc_mpareq()
686 cep->state = SIW_EPSTATE_RECVD_MPAREQ; in siw_proc_mpareq()
689 siw_cep_get(cep); in siw_proc_mpareq()
690 rv = siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REQUEST, 0); in siw_proc_mpareq()
692 siw_cep_put(cep); in siw_proc_mpareq()
697 siw_dbg_cep(cep, "reject: crc %d:%d:%d, m %d:%d\n", in siw_proc_mpareq()
709 kfree(cep->mpa.pdata); in siw_proc_mpareq()
711 cep->mpa.pdata = NULL; in siw_proc_mpareq()
713 siw_send_mpareqrep(cep, NULL, 0); in siw_proc_mpareq()
718 static int siw_proc_mpareply(struct siw_cep *cep) in siw_proc_mpareply() argument
722 struct siw_qp *qp = cep->qp; in siw_proc_mpareply()
730 rv = siw_recv_mpa_rr(cep); in siw_proc_mpareply()
734 siw_cancel_mpatimer(cep); in siw_proc_mpareply()
736 rep = &cep->mpa.hdr; in siw_proc_mpareply()
751 siw_dbg_cep(cep, "got mpa reject\n"); in siw_proc_mpareply()
752 siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -ECONNRESET); in siw_proc_mpareply()
757 siw_dbg_cep(cep, "peer allows GSO on TX\n"); in siw_proc_mpareply()
764 siw_dbg_cep(cep, "reply unsupp: crc %d:%d:%d, m %d:%d\n", in siw_proc_mpareply()
769 siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED); in siw_proc_mpareply()
773 if (cep->enhanced_rdma_conn_est) { in siw_proc_mpareply()
782 siw_dbg_cep(cep, "mpa reply error: vers %d, enhcd %d\n", in siw_proc_mpareply()
788 siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, in siw_proc_mpareply()
792 v2 = (struct mpa_v2_data *)cep->mpa.pdata; in siw_proc_mpareply()
796 if (cep->ird < rep_ord && in siw_proc_mpareply()
798 rep_ord > cep->sdev->attrs.max_ird)) { in siw_proc_mpareply()
799 siw_dbg_cep(cep, "ird %d, rep_ord %d, max_ord %d\n", in siw_proc_mpareply()
800 cep->ird, rep_ord, in siw_proc_mpareply()
801 cep->sdev->attrs.max_ord); in siw_proc_mpareply()
804 if (cep->ord > rep_ird && relaxed_ird_negotiation == false) { in siw_proc_mpareply()
805 siw_dbg_cep(cep, "ord %d, rep_ird %d\n", cep->ord, in siw_proc_mpareply()
813 cep->ird = rep_ord; in siw_proc_mpareply()
814 cep->ord = rep_ird; in siw_proc_mpareply()
828 if (cep->mpa.v2_ctrl_req.ird & MPA_V2_PEER_TO_PEER) in siw_proc_mpareply()
830 cep->mpa.v2_ctrl_req.ord & in siw_proc_mpareply()
842 siw_dbg_cep(cep, in siw_proc_mpareply()
865 qp_attrs.irq_size = cep->ird; in siw_proc_mpareply()
866 qp_attrs.orq_size = cep->ord; in siw_proc_mpareply()
867 qp_attrs.sk = cep->sock; in siw_proc_mpareply()
882 siw_qp_socket_assoc(cep, qp); in siw_proc_mpareply()
893 rv = siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 0); in siw_proc_mpareply()
895 cep->state = SIW_EPSTATE_RDMA_MODE; in siw_proc_mpareply()
902 siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -EINVAL); in siw_proc_mpareply()
911 static void siw_accept_newconn(struct siw_cep *cep) in siw_accept_newconn() argument
913 struct socket *s = cep->sock; in siw_accept_newconn()
918 if (cep->state != SIW_EPSTATE_LISTENING) in siw_accept_newconn()
921 new_cep = siw_cep_alloc(cep->sdev); in siw_accept_newconn()
937 new_cep->sk_state_change = cep->sk_state_change; in siw_accept_newconn()
938 new_cep->sk_data_ready = cep->sk_data_ready; in siw_accept_newconn()
939 new_cep->sk_write_space = cep->sk_write_space; in siw_accept_newconn()
940 new_cep->sk_error_report = cep->sk_error_report; in siw_accept_newconn()
947 siw_dbg_cep(cep, "kernel_accept() error: %d\n", rv); in siw_accept_newconn()
964 new_cep->listen_cep = cep; in siw_accept_newconn()
965 siw_cep_get(cep); in siw_accept_newconn()
971 siw_dbg_cep(cep, "immediate mpa request\n"); in siw_accept_newconn()
976 siw_cep_put(cep); in siw_accept_newconn()
996 siw_dbg_cep(cep, "error %d\n", rv); in siw_accept_newconn()
1002 struct siw_cep *cep; in siw_cm_work_handler() local
1006 cep = work->cep; in siw_cm_work_handler()
1008 siw_dbg_cep(cep, "[QP %u]: work type: %d, state %d\n", in siw_cm_work_handler()
1009 cep->qp ? qp_id(cep->qp) : UINT_MAX, in siw_cm_work_handler()
1010 work->type, cep->state); in siw_cm_work_handler()
1012 siw_cep_set_inuse(cep); in siw_cm_work_handler()
1016 siw_accept_newconn(cep); in siw_cm_work_handler()
1020 if (cep->state == SIW_EPSTATE_AWAIT_MPAREQ) { in siw_cm_work_handler()
1021 if (cep->listen_cep) { in siw_cm_work_handler()
1022 siw_cep_set_inuse(cep->listen_cep); in siw_cm_work_handler()
1024 if (cep->listen_cep->state == in siw_cm_work_handler()
1026 rv = siw_proc_mpareq(cep); in siw_cm_work_handler()
1030 siw_cep_set_free(cep->listen_cep); in siw_cm_work_handler()
1033 siw_cep_put(cep->listen_cep); in siw_cm_work_handler()
1034 cep->listen_cep = NULL; in siw_cm_work_handler()
1036 siw_cep_put(cep); in siw_cm_work_handler()
1039 } else if (cep->state == SIW_EPSTATE_AWAIT_MPAREP) { in siw_cm_work_handler()
1040 rv = siw_proc_mpareply(cep); in siw_cm_work_handler()
1047 if (cep->state == SIW_EPSTATE_RDMA_MODE) { in siw_cm_work_handler()
1048 cep->sock->sk->sk_data_ready(cep->sock->sk); in siw_cm_work_handler()
1049 siw_dbg_cep(cep, "already in RDMA mode"); in siw_cm_work_handler()
1051 siw_dbg_cep(cep, "out of state: %d\n", in siw_cm_work_handler()
1052 cep->state); in siw_cm_work_handler()
1063 if (cep->qp && cep->qp->term_info.valid) in siw_cm_work_handler()
1064 siw_send_terminate(cep->qp); in siw_cm_work_handler()
1066 if (cep->cm_id) in siw_cm_work_handler()
1067 siw_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0); in siw_cm_work_handler()
1073 if (cep->cm_id) { in siw_cm_work_handler()
1074 if (cep->state == SIW_EPSTATE_AWAIT_MPAREP) { in siw_cm_work_handler()
1078 siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, in siw_cm_work_handler()
1080 } else if (cep->state == SIW_EPSTATE_RDMA_MODE) { in siw_cm_work_handler()
1085 siw_cm_upcall(cep, IW_CM_EVENT_DISCONNECT, 0); in siw_cm_work_handler()
1086 siw_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0); in siw_cm_work_handler()
1093 if (cep->state == SIW_EPSTATE_RECVD_MPAREQ) { in siw_cm_work_handler()
1097 siw_dbg_cep(cep, in siw_cm_work_handler()
1099 } else if (cep->state == SIW_EPSTATE_AWAIT_MPAREQ) { in siw_cm_work_handler()
1103 siw_dbg_cep(cep, "no mpareq: drop listener\n"); in siw_cm_work_handler()
1104 siw_cep_put(cep->listen_cep); in siw_cm_work_handler()
1105 cep->listen_cep = NULL; in siw_cm_work_handler()
1112 cep->mpa_timer = NULL; in siw_cm_work_handler()
1114 if (cep->state == SIW_EPSTATE_AWAIT_MPAREP) { in siw_cm_work_handler()
1120 cep->mpa.hdr.params.pd_len = 0; in siw_cm_work_handler()
1122 if (cep->cm_id) in siw_cm_work_handler()
1123 siw_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, in siw_cm_work_handler()
1127 } else if (cep->state == SIW_EPSTATE_AWAIT_MPAREQ) { in siw_cm_work_handler()
1131 if (cep->listen_cep) { in siw_cm_work_handler()
1132 siw_cep_put(cep->listen_cep); in siw_cm_work_handler()
1133 cep->listen_cep = NULL; in siw_cm_work_handler()
1143 siw_dbg_cep(cep, in siw_cm_work_handler()
1145 cep->mpa_timer ? "y" : "n", in siw_cm_work_handler()
1146 cep->qp ? qp_id(cep->qp) : UINT_MAX); in siw_cm_work_handler()
1148 siw_cancel_mpatimer(cep); in siw_cm_work_handler()
1150 cep->state = SIW_EPSTATE_CLOSED; in siw_cm_work_handler()
1152 if (cep->qp) { in siw_cm_work_handler()
1153 struct siw_qp *qp = cep->qp; in siw_cm_work_handler()
1159 siw_cep_set_free(cep); in siw_cm_work_handler()
1164 siw_cep_set_inuse(cep); in siw_cm_work_handler()
1165 cep->qp = NULL; in siw_cm_work_handler()
1168 if (cep->sock) { in siw_cm_work_handler()
1169 siw_socket_disassoc(cep->sock); in siw_cm_work_handler()
1170 sock_release(cep->sock); in siw_cm_work_handler()
1171 cep->sock = NULL; in siw_cm_work_handler()
1173 if (cep->cm_id) { in siw_cm_work_handler()
1174 cep->cm_id->rem_ref(cep->cm_id); in siw_cm_work_handler()
1175 cep->cm_id = NULL; in siw_cm_work_handler()
1176 siw_cep_put(cep); in siw_cm_work_handler()
1179 siw_cep_set_free(cep); in siw_cm_work_handler()
1181 siw_cep_put(cep); in siw_cm_work_handler()
1186 int siw_cm_queue_work(struct siw_cep *cep, enum siw_work_type type) in siw_cm_queue_work() argument
1188 struct siw_cm_work *work = siw_get_work(cep); in siw_cm_queue_work()
1192 siw_dbg_cep(cep, "failed with no work available\n"); in siw_cm_queue_work()
1196 work->cep = cep; in siw_cm_queue_work()
1198 siw_cep_get(cep); in siw_cm_queue_work()
1203 cep->mpa_timer = work; in siw_cm_queue_work()
1205 if (cep->state == SIW_EPSTATE_AWAIT_MPAREP) in siw_cm_queue_work()
1210 siw_dbg_cep(cep, "[QP %u]: work type: %d, timeout %lu\n", in siw_cm_queue_work()
1211 cep->qp ? qp_id(cep->qp) : -1, type, delay); in siw_cm_queue_work()
1220 struct siw_cep *cep; in siw_cm_llp_data_ready() local
1226 cep = sk_to_cep(sk); in siw_cm_llp_data_ready()
1227 if (!cep) in siw_cm_llp_data_ready()
1230 siw_dbg_cep(cep, "state: %d\n", cep->state); in siw_cm_llp_data_ready()
1232 switch (cep->state) { in siw_cm_llp_data_ready()
1239 siw_cm_queue_work(cep, SIW_CM_WORK_READ_MPAHDR); in siw_cm_llp_data_ready()
1243 siw_dbg_cep(cep, "unexpected data, state %d\n", cep->state); in siw_cm_llp_data_ready()
1252 struct siw_cep *cep = sk_to_cep(sk); in siw_cm_llp_write_space() local
1254 if (cep) in siw_cm_llp_write_space()
1255 siw_dbg_cep(cep, "state: %d\n", cep->state); in siw_cm_llp_write_space()
1260 struct siw_cep *cep = sk_to_cep(sk); in siw_cm_llp_error_report() local
1262 if (cep) { in siw_cm_llp_error_report()
1263 siw_dbg_cep(cep, "error %d, socket state: %d, cep state: %d\n", in siw_cm_llp_error_report()
1264 sk->sk_err, sk->sk_state, cep->state); in siw_cm_llp_error_report()
1265 cep->sk_error_report(sk); in siw_cm_llp_error_report()
1271 struct siw_cep *cep; in siw_cm_llp_state_change() local
1276 cep = sk_to_cep(sk); in siw_cm_llp_state_change()
1277 if (!cep) { in siw_cm_llp_state_change()
1282 orig_state_change = cep->sk_state_change; in siw_cm_llp_state_change()
1284 siw_dbg_cep(cep, "state: %d\n", cep->state); in siw_cm_llp_state_change()
1292 siw_cm_queue_work(cep, SIW_CM_WORK_ACCEPT); in siw_cm_llp_state_change()
1297 if (cep->qp) in siw_cm_llp_state_change()
1298 cep->qp->tx_ctx.tx_suspend = 1; in siw_cm_llp_state_change()
1299 siw_cm_queue_work(cep, SIW_CM_WORK_PEER_CLOSE); in siw_cm_llp_state_change()
1303 siw_dbg_cep(cep, "unexpected socket state %d\n", sk->sk_state); in siw_cm_llp_state_change()
1340 struct siw_cep *cep = NULL; in siw_connect() local
1394 cep = siw_cep_alloc(sdev); in siw_connect()
1395 if (!cep) { in siw_connect()
1399 siw_cep_set_inuse(cep); in siw_connect()
1402 siw_cep_get(cep); in siw_connect()
1403 qp->cep = cep; in siw_connect()
1406 cep->qp = qp; in siw_connect()
1409 cep->cm_id = id; in siw_connect()
1416 rv = siw_cm_alloc_work(cep, 4); in siw_connect()
1421 cep->ird = params->ird; in siw_connect()
1422 cep->ord = params->ord; in siw_connect()
1424 if (p2p_mode && cep->ord == 0) in siw_connect()
1425 cep->ord = 1; in siw_connect()
1427 cep->state = SIW_EPSTATE_CONNECTING; in siw_connect()
1432 siw_cep_socket_assoc(cep, s); in siw_connect()
1434 cep->state = SIW_EPSTATE_AWAIT_MPAREP; in siw_connect()
1440 cep->mpa.hdr.params.bits = 0; in siw_connect()
1447 __mpa_rr_set_revision(&cep->mpa.hdr.params.bits, version); in siw_connect()
1450 cep->mpa.hdr.params.bits |= MPA_RR_FLAG_GSO_EXP; in siw_connect()
1453 cep->mpa.hdr.params.bits |= MPA_RR_FLAG_CRC; in siw_connect()
1462 cep->enhanced_rdma_conn_est = true; in siw_connect()
1463 cep->mpa.hdr.params.bits |= MPA_RR_FLAG_ENHANCED; in siw_connect()
1465 cep->mpa.v2_ctrl.ird = htons(cep->ird); in siw_connect()
1466 cep->mpa.v2_ctrl.ord = htons(cep->ord); in siw_connect()
1469 cep->mpa.v2_ctrl.ird |= MPA_V2_PEER_TO_PEER; in siw_connect()
1470 cep->mpa.v2_ctrl.ord |= rtr_type; in siw_connect()
1473 cep->mpa.v2_ctrl_req.ird = cep->mpa.v2_ctrl.ird; in siw_connect()
1474 cep->mpa.v2_ctrl_req.ord = cep->mpa.v2_ctrl.ord; in siw_connect()
1476 memcpy(cep->mpa.hdr.key, MPA_KEY_REQ, 16); in siw_connect()
1478 rv = siw_send_mpareqrep(cep, params->private_data, pd_len); in siw_connect()
1482 cep->mpa.hdr.params.pd_len = 0; in siw_connect()
1485 rv = siw_cm_queue_work(cep, SIW_CM_WORK_MPATIMEOUT); in siw_connect()
1487 siw_dbg_cep(cep, "[QP %u]: exit\n", qp_id(qp)); in siw_connect()
1488 siw_cep_set_free(cep); in siw_connect()
1495 if (cep) { in siw_connect()
1498 cep->sock = NULL; in siw_connect()
1500 cep->qp = NULL; in siw_connect()
1502 cep->cm_id = NULL; in siw_connect()
1504 siw_cep_put(cep); in siw_connect()
1506 qp->cep = NULL; in siw_connect()
1507 siw_cep_put(cep); in siw_connect()
1509 cep->state = SIW_EPSTATE_CLOSED; in siw_connect()
1511 siw_cep_set_free(cep); in siw_connect()
1513 siw_cep_put(cep); in siw_connect()
1541 struct siw_cep *cep = (struct siw_cep *)id->provider_data; in siw_accept() local
1547 siw_cep_set_inuse(cep); in siw_accept()
1548 siw_cep_put(cep); in siw_accept()
1551 if (cep->mpa.hdr.params.pd_len) { in siw_accept()
1552 cep->mpa.hdr.params.pd_len = 0; in siw_accept()
1553 kfree(cep->mpa.pdata); in siw_accept()
1554 cep->mpa.pdata = NULL; in siw_accept()
1556 siw_cancel_mpatimer(cep); in siw_accept()
1558 if (cep->state != SIW_EPSTATE_RECVD_MPAREQ) { in siw_accept()
1559 siw_dbg_cep(cep, "out of state\n"); in siw_accept()
1561 siw_cep_set_free(cep); in siw_accept()
1562 siw_cep_put(cep); in siw_accept()
1569 siw_cep_set_free(cep); in siw_accept()
1570 siw_cep_put(cep); in siw_accept()
1580 siw_dbg_cep(cep, "[QP %d]\n", params->qpn); in siw_accept()
1582 if (try_gso && cep->mpa.hdr.params.bits & MPA_RR_FLAG_GSO_EXP) { in siw_accept()
1583 siw_dbg_cep(cep, "peer allows GSO on TX\n"); in siw_accept()
1589 cep, in siw_accept()
1597 if (cep->enhanced_rdma_conn_est) in siw_accept()
1602 cep, in siw_accept()
1609 if (cep->enhanced_rdma_conn_est) { in siw_accept()
1610 if (params->ord > cep->ord) { in siw_accept()
1612 params->ord = cep->ord; in siw_accept()
1614 cep->ird = params->ird; in siw_accept()
1615 cep->ord = params->ord; in siw_accept()
1621 if (params->ird < cep->ird) { in siw_accept()
1623 cep->ird <= sdev->attrs.max_ird) in siw_accept()
1624 params->ird = cep->ird; in siw_accept()
1631 if (cep->mpa.v2_ctrl.ord & in siw_accept()
1637 cep->mpa.v2_ctrl.ord = in siw_accept()
1639 (cep->mpa.v2_ctrl.ord & ~MPA_V2_MASK_IRD_ORD); in siw_accept()
1640 cep->mpa.v2_ctrl.ird = in siw_accept()
1642 (cep->mpa.v2_ctrl.ird & ~MPA_V2_MASK_IRD_ORD); in siw_accept()
1644 cep->ird = params->ird; in siw_accept()
1645 cep->ord = params->ord; in siw_accept()
1647 cep->cm_id = id; in siw_accept()
1651 qp_attrs.orq_size = cep->ord; in siw_accept()
1652 qp_attrs.irq_size = cep->ird; in siw_accept()
1653 qp_attrs.sk = cep->sock; in siw_accept()
1654 if (cep->mpa.hdr.params.bits & MPA_RR_FLAG_CRC) in siw_accept()
1658 siw_dbg_cep(cep, "[QP%u]: moving to rts\n", qp_id(qp)); in siw_accept()
1661 siw_cep_get(cep); in siw_accept()
1662 qp->cep = cep; in siw_accept()
1665 cep->qp = qp; in siw_accept()
1667 cep->state = SIW_EPSTATE_RDMA_MODE; in siw_accept()
1679 siw_dbg_cep(cep, "[QP %u]: send mpa reply, %d byte pdata\n", in siw_accept()
1682 rv = siw_send_mpareqrep(cep, params->private_data, in siw_accept()
1688 siw_sk_assign_rtr_upcalls(cep); in siw_accept()
1690 siw_qp_socket_assoc(cep, qp); in siw_accept()
1691 rv = siw_cm_upcall(cep, IW_CM_EVENT_ESTABLISHED, 0); in siw_accept()
1695 siw_cep_set_free(cep); in siw_accept()
1699 siw_socket_disassoc(cep->sock); in siw_accept()
1700 sock_release(cep->sock); in siw_accept()
1701 cep->sock = NULL; in siw_accept()
1703 cep->state = SIW_EPSTATE_CLOSED; in siw_accept()
1705 if (cep->cm_id) { in siw_accept()
1706 cep->cm_id->rem_ref(id); in siw_accept()
1707 cep->cm_id = NULL; in siw_accept()
1709 if (qp->cep) { in siw_accept()
1710 siw_cep_put(cep); in siw_accept()
1711 qp->cep = NULL; in siw_accept()
1713 cep->qp = NULL; in siw_accept()
1716 siw_cep_set_free(cep); in siw_accept()
1717 siw_cep_put(cep); in siw_accept()
1730 struct siw_cep *cep = (struct siw_cep *)id->provider_data; in siw_reject() local
1732 siw_cep_set_inuse(cep); in siw_reject()
1733 siw_cep_put(cep); in siw_reject()
1735 siw_cancel_mpatimer(cep); in siw_reject()
1737 if (cep->state != SIW_EPSTATE_RECVD_MPAREQ) { in siw_reject()
1738 siw_dbg_cep(cep, "out of state\n"); in siw_reject()
1740 siw_cep_set_free(cep); in siw_reject()
1741 siw_cep_put(cep); /* put last reference */ in siw_reject()
1745 siw_dbg_cep(cep, "cep->state %d, pd_len %d\n", cep->state, in siw_reject()
1748 if (__mpa_rr_revision(cep->mpa.hdr.params.bits) >= MPA_REVISION_1) { in siw_reject()
1749 cep->mpa.hdr.params.bits |= MPA_RR_FLAG_REJECT; /* reject */ in siw_reject()
1750 siw_send_mpareqrep(cep, pdata, pd_len); in siw_reject()
1752 siw_socket_disassoc(cep->sock); in siw_reject()
1753 sock_release(cep->sock); in siw_reject()
1754 cep->sock = NULL; in siw_reject()
1756 cep->state = SIW_EPSTATE_CLOSED; in siw_reject()
1758 siw_cep_set_free(cep); in siw_reject()
1759 siw_cep_put(cep); in siw_reject()
1773 struct siw_cep *cep = NULL; in siw_create_listen() local
1822 cep = siw_cep_alloc(sdev); in siw_create_listen()
1823 if (!cep) { in siw_create_listen()
1827 siw_cep_socket_assoc(cep, s); in siw_create_listen()
1829 rv = siw_cm_alloc_work(cep, backlog); in siw_create_listen()
1841 cep->cm_id = id; in siw_create_listen()
1873 list_add_tail(&cep->listenq, (struct list_head *)id->provider_data); in siw_create_listen()
1874 cep->state = SIW_EPSTATE_LISTENING; in siw_create_listen()
1883 if (cep) { in siw_create_listen()
1884 siw_cep_set_inuse(cep); in siw_create_listen()
1886 if (cep->cm_id) { in siw_create_listen()
1887 cep->cm_id->rem_ref(cep->cm_id); in siw_create_listen()
1888 cep->cm_id = NULL; in siw_create_listen()
1890 cep->sock = NULL; in siw_create_listen()
1892 cep->state = SIW_EPSTATE_CLOSED; in siw_create_listen()
1894 siw_cep_set_free(cep); in siw_create_listen()
1895 siw_cep_put(cep); in siw_create_listen()
1911 struct siw_cep *cep = list_entry(p, struct siw_cep, listenq); in siw_drop_listeners() local
1915 siw_dbg_cep(cep, "drop cep, state %d\n", cep->state); in siw_drop_listeners()
1917 siw_cep_set_inuse(cep); in siw_drop_listeners()
1919 if (cep->cm_id) { in siw_drop_listeners()
1920 cep->cm_id->rem_ref(cep->cm_id); in siw_drop_listeners()
1921 cep->cm_id = NULL; in siw_drop_listeners()
1923 if (cep->sock) { in siw_drop_listeners()
1924 siw_socket_disassoc(cep->sock); in siw_drop_listeners()
1925 sock_release(cep->sock); in siw_drop_listeners()
1926 cep->sock = NULL; in siw_drop_listeners()
1928 cep->state = SIW_EPSTATE_CLOSED; in siw_drop_listeners()
1929 siw_cep_set_free(cep); in siw_drop_listeners()
1930 siw_cep_put(cep); in siw_drop_listeners()