Lines Matching refs:link
44 struct smc_link *link; member
55 static inline bool smc_wr_is_tx_pend(struct smc_link *link) in smc_wr_is_tx_pend() argument
57 if (find_first_bit(link->wr_tx_mask, link->wr_tx_cnt) != in smc_wr_is_tx_pend()
58 link->wr_tx_cnt) { in smc_wr_is_tx_pend()
65 void smc_wr_tx_wait_no_pending_sends(struct smc_link *link) in smc_wr_tx_wait_no_pending_sends() argument
67 wait_event(link->wr_tx_wait, !smc_wr_is_tx_pend(link)); in smc_wr_tx_wait_no_pending_sends()
70 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id) in smc_wr_tx_find_pending_index() argument
74 for (i = 0; i < link->wr_tx_cnt; i++) { in smc_wr_tx_find_pending_index()
75 if (link->wr_tx_pends[i].wr_id == wr_id) in smc_wr_tx_find_pending_index()
78 return link->wr_tx_cnt; in smc_wr_tx_find_pending_index()
84 struct smc_link *link; in smc_wr_tx_process_cqe() local
87 link = wc->qp->qp_context; in smc_wr_tx_process_cqe()
91 link->wr_reg_state = FAILED; in smc_wr_tx_process_cqe()
93 link->wr_reg_state = CONFIRMED; in smc_wr_tx_process_cqe()
94 smc_wr_wakeup_reg_wait(link); in smc_wr_tx_process_cqe()
98 pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id); in smc_wr_tx_process_cqe()
99 if (pnd_snd_idx == link->wr_tx_cnt) { in smc_wr_tx_process_cqe()
100 if (link->lgr->smc_version != SMC_V2 || in smc_wr_tx_process_cqe()
101 link->wr_tx_v2_pend->wr_id != wc->wr_id) in smc_wr_tx_process_cqe()
103 link->wr_tx_v2_pend->wc_status = wc->status; in smc_wr_tx_process_cqe()
104 memcpy(&pnd_snd, link->wr_tx_v2_pend, sizeof(pnd_snd)); in smc_wr_tx_process_cqe()
106 memset(link->wr_tx_v2_pend, 0, in smc_wr_tx_process_cqe()
107 sizeof(*link->wr_tx_v2_pend)); in smc_wr_tx_process_cqe()
108 memset(link->lgr->wr_tx_buf_v2, 0, in smc_wr_tx_process_cqe()
109 sizeof(*link->lgr->wr_tx_buf_v2)); in smc_wr_tx_process_cqe()
111 link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status; in smc_wr_tx_process_cqe()
112 if (link->wr_tx_pends[pnd_snd_idx].compl_requested) in smc_wr_tx_process_cqe()
113 complete(&link->wr_tx_compl[pnd_snd_idx]); in smc_wr_tx_process_cqe()
114 memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], in smc_wr_tx_process_cqe()
117 memset(&link->wr_tx_pends[pnd_snd_idx], 0, in smc_wr_tx_process_cqe()
118 sizeof(link->wr_tx_pends[pnd_snd_idx])); in smc_wr_tx_process_cqe()
119 memset(&link->wr_tx_bufs[pnd_snd_idx], 0, in smc_wr_tx_process_cqe()
120 sizeof(link->wr_tx_bufs[pnd_snd_idx])); in smc_wr_tx_process_cqe()
121 if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask)) in smc_wr_tx_process_cqe()
126 if (link->lgr->smc_version == SMC_V2) { in smc_wr_tx_process_cqe()
127 memset(link->wr_tx_v2_pend, 0, in smc_wr_tx_process_cqe()
128 sizeof(*link->wr_tx_v2_pend)); in smc_wr_tx_process_cqe()
129 memset(link->lgr->wr_tx_buf_v2, 0, in smc_wr_tx_process_cqe()
130 sizeof(*link->lgr->wr_tx_buf_v2)); in smc_wr_tx_process_cqe()
133 smcr_link_down_cond_sched(link); in smc_wr_tx_process_cqe()
136 pnd_snd.handler(&pnd_snd.priv, link, wc->status); in smc_wr_tx_process_cqe()
137 wake_up(&link->wr_tx_wait); in smc_wr_tx_process_cqe()
175 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx) in smc_wr_tx_get_free_slot_index() argument
177 *idx = link->wr_tx_cnt; in smc_wr_tx_get_free_slot_index()
178 if (!smc_link_sendable(link)) in smc_wr_tx_get_free_slot_index()
180 for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) { in smc_wr_tx_get_free_slot_index()
181 if (!test_and_set_bit(*idx, link->wr_tx_mask)) in smc_wr_tx_get_free_slot_index()
184 *idx = link->wr_tx_cnt; in smc_wr_tx_get_free_slot_index()
199 int smc_wr_tx_get_free_slot(struct smc_link *link, in smc_wr_tx_get_free_slot() argument
205 struct smc_link_group *lgr = smc_get_lgr(link); in smc_wr_tx_get_free_slot()
207 u32 idx = link->wr_tx_cnt; in smc_wr_tx_get_free_slot()
215 rc = smc_wr_tx_get_free_slot_index(link, &idx); in smc_wr_tx_get_free_slot()
220 link->wr_tx_wait, in smc_wr_tx_get_free_slot()
221 !smc_link_sendable(link) || in smc_wr_tx_get_free_slot()
223 (smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY), in smc_wr_tx_get_free_slot()
227 smcr_link_down_cond_sched(link); in smc_wr_tx_get_free_slot()
230 if (idx == link->wr_tx_cnt) in smc_wr_tx_get_free_slot()
233 wr_id = smc_wr_tx_get_next_wr_id(link); in smc_wr_tx_get_free_slot()
234 wr_pend = &link->wr_tx_pends[idx]; in smc_wr_tx_get_free_slot()
237 wr_pend->link = link; in smc_wr_tx_get_free_slot()
239 wr_ib = &link->wr_tx_ibs[idx]; in smc_wr_tx_get_free_slot()
241 *wr_buf = &link->wr_tx_bufs[idx]; in smc_wr_tx_get_free_slot()
243 *wr_rdma_buf = &link->wr_tx_rdmas[idx]; in smc_wr_tx_get_free_slot()
248 int smc_wr_tx_get_v2_slot(struct smc_link *link, in smc_wr_tx_get_v2_slot() argument
257 if (link->wr_tx_v2_pend->idx == link->wr_tx_cnt) in smc_wr_tx_get_v2_slot()
262 wr_id = smc_wr_tx_get_next_wr_id(link); in smc_wr_tx_get_v2_slot()
263 wr_pend = link->wr_tx_v2_pend; in smc_wr_tx_get_v2_slot()
266 wr_pend->link = link; in smc_wr_tx_get_v2_slot()
267 wr_pend->idx = link->wr_tx_cnt; in smc_wr_tx_get_v2_slot()
268 wr_ib = link->wr_tx_v2_ib; in smc_wr_tx_get_v2_slot()
270 *wr_buf = link->lgr->wr_tx_buf_v2; in smc_wr_tx_get_v2_slot()
275 int smc_wr_tx_put_slot(struct smc_link *link, in smc_wr_tx_put_slot() argument
281 if (pend->idx < link->wr_tx_cnt) { in smc_wr_tx_put_slot()
285 memset(&link->wr_tx_pends[idx], 0, in smc_wr_tx_put_slot()
286 sizeof(link->wr_tx_pends[idx])); in smc_wr_tx_put_slot()
287 memset(&link->wr_tx_bufs[idx], 0, in smc_wr_tx_put_slot()
288 sizeof(link->wr_tx_bufs[idx])); in smc_wr_tx_put_slot()
289 test_and_clear_bit(idx, link->wr_tx_mask); in smc_wr_tx_put_slot()
290 wake_up(&link->wr_tx_wait); in smc_wr_tx_put_slot()
292 } else if (link->lgr->smc_version == SMC_V2 && in smc_wr_tx_put_slot()
293 pend->idx == link->wr_tx_cnt) { in smc_wr_tx_put_slot()
295 memset(&link->wr_tx_v2_pend, 0, in smc_wr_tx_put_slot()
296 sizeof(link->wr_tx_v2_pend)); in smc_wr_tx_put_slot()
297 memset(&link->lgr->wr_tx_buf_v2, 0, in smc_wr_tx_put_slot()
298 sizeof(link->lgr->wr_tx_buf_v2)); in smc_wr_tx_put_slot()
308 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv) in smc_wr_tx_send() argument
313 ib_req_notify_cq(link->smcibdev->roce_cq_send, in smc_wr_tx_send()
316 rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL); in smc_wr_tx_send()
318 smc_wr_tx_put_slot(link, priv); in smc_wr_tx_send()
319 smcr_link_down_cond_sched(link); in smc_wr_tx_send()
324 int smc_wr_tx_v2_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv, in smc_wr_tx_v2_send() argument
329 link->wr_tx_v2_ib->sg_list[0].length = len; in smc_wr_tx_v2_send()
330 ib_req_notify_cq(link->smcibdev->roce_cq_send, in smc_wr_tx_v2_send()
332 rc = ib_post_send(link->roce_qp, link->wr_tx_v2_ib, NULL); in smc_wr_tx_v2_send()
334 smc_wr_tx_put_slot(link, priv); in smc_wr_tx_v2_send()
335 smcr_link_down_cond_sched(link); in smc_wr_tx_v2_send()
344 int smc_wr_tx_send_wait(struct smc_link *link, struct smc_wr_tx_pend_priv *priv, in smc_wr_tx_send_wait() argument
354 init_completion(&link->wr_tx_compl[pnd_idx]); in smc_wr_tx_send_wait()
356 rc = smc_wr_tx_send(link, priv); in smc_wr_tx_send_wait()
361 &link->wr_tx_compl[pnd_idx], timeout); in smc_wr_tx_send_wait()
370 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr) in smc_wr_reg_send() argument
374 ib_req_notify_cq(link->smcibdev->roce_cq_send, in smc_wr_reg_send()
376 link->wr_reg_state = POSTED; in smc_wr_reg_send()
377 link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr; in smc_wr_reg_send()
378 link->wr_reg.mr = mr; in smc_wr_reg_send()
379 link->wr_reg.key = mr->rkey; in smc_wr_reg_send()
380 rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL); in smc_wr_reg_send()
384 atomic_inc(&link->wr_reg_refcnt); in smc_wr_reg_send()
385 rc = wait_event_interruptible_timeout(link->wr_reg_wait, in smc_wr_reg_send()
386 (link->wr_reg_state != POSTED), in smc_wr_reg_send()
388 if (atomic_dec_and_test(&link->wr_reg_refcnt)) in smc_wr_reg_send()
389 wake_up_all(&link->wr_reg_wait); in smc_wr_reg_send()
392 smcr_link_down_cond_sched(link); in smc_wr_reg_send()
397 switch (link->wr_reg_state) { in smc_wr_reg_send()
437 struct smc_link *link = (struct smc_link *)wc->qp->qp_context; in smc_wr_rx_demultiplex() local
446 index = do_div(temp_wr_id, link->wr_rx_cnt); in smc_wr_rx_demultiplex()
447 wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index]; in smc_wr_rx_demultiplex()
456 struct smc_link *link; in smc_wr_rx_process_cqes() local
460 link = wc[i].qp->qp_context; in smc_wr_rx_process_cqes()
462 link->wr_rx_tstamp = jiffies; in smc_wr_rx_process_cqes()
464 smc_wr_rx_post(link); /* refill WR RX */ in smc_wr_rx_process_cqes()
471 smcr_link_down_cond_sched(link); in smc_wr_rx_process_cqes()
474 smc_wr_rx_post(link); /* refill WR RX */ in smc_wr_rx_process_cqes()
513 int smc_wr_rx_post_init(struct smc_link *link) in smc_wr_rx_post_init() argument
518 for (i = 0; i < link->wr_rx_cnt; i++) in smc_wr_rx_post_init()
519 rc = smc_wr_rx_post(link); in smc_wr_rx_post_init()
735 int smc_wr_alloc_link_mem(struct smc_link *link) in smc_wr_alloc_link_mem() argument
737 int sges_per_buf = link->lgr->smc_version == SMC_V2 ? 2 : 1; in smc_wr_alloc_link_mem()
740 link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL); in smc_wr_alloc_link_mem()
741 if (!link->wr_tx_bufs) in smc_wr_alloc_link_mem()
743 link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE, in smc_wr_alloc_link_mem()
745 if (!link->wr_rx_bufs) in smc_wr_alloc_link_mem()
747 link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]), in smc_wr_alloc_link_mem()
749 if (!link->wr_tx_ibs) in smc_wr_alloc_link_mem()
751 link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3, in smc_wr_alloc_link_mem()
752 sizeof(link->wr_rx_ibs[0]), in smc_wr_alloc_link_mem()
754 if (!link->wr_rx_ibs) in smc_wr_alloc_link_mem()
756 link->wr_tx_rdmas = kcalloc(SMC_WR_BUF_CNT, in smc_wr_alloc_link_mem()
757 sizeof(link->wr_tx_rdmas[0]), in smc_wr_alloc_link_mem()
759 if (!link->wr_tx_rdmas) in smc_wr_alloc_link_mem()
761 link->wr_tx_rdma_sges = kcalloc(SMC_WR_BUF_CNT, in smc_wr_alloc_link_mem()
762 sizeof(link->wr_tx_rdma_sges[0]), in smc_wr_alloc_link_mem()
764 if (!link->wr_tx_rdma_sges) in smc_wr_alloc_link_mem()
766 link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]), in smc_wr_alloc_link_mem()
768 if (!link->wr_tx_sges) in smc_wr_alloc_link_mem()
770 link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3, in smc_wr_alloc_link_mem()
771 sizeof(link->wr_rx_sges[0]) * sges_per_buf, in smc_wr_alloc_link_mem()
773 if (!link->wr_rx_sges) in smc_wr_alloc_link_mem()
775 link->wr_tx_mask = kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT), in smc_wr_alloc_link_mem()
776 sizeof(*link->wr_tx_mask), in smc_wr_alloc_link_mem()
778 if (!link->wr_tx_mask) in smc_wr_alloc_link_mem()
780 link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT, in smc_wr_alloc_link_mem()
781 sizeof(link->wr_tx_pends[0]), in smc_wr_alloc_link_mem()
783 if (!link->wr_tx_pends) in smc_wr_alloc_link_mem()
785 link->wr_tx_compl = kcalloc(SMC_WR_BUF_CNT, in smc_wr_alloc_link_mem()
786 sizeof(link->wr_tx_compl[0]), in smc_wr_alloc_link_mem()
788 if (!link->wr_tx_compl) in smc_wr_alloc_link_mem()
791 if (link->lgr->smc_version == SMC_V2) { in smc_wr_alloc_link_mem()
792 link->wr_tx_v2_ib = kzalloc(sizeof(*link->wr_tx_v2_ib), in smc_wr_alloc_link_mem()
794 if (!link->wr_tx_v2_ib) in smc_wr_alloc_link_mem()
796 link->wr_tx_v2_sge = kzalloc(sizeof(*link->wr_tx_v2_sge), in smc_wr_alloc_link_mem()
798 if (!link->wr_tx_v2_sge) in smc_wr_alloc_link_mem()
800 link->wr_tx_v2_pend = kzalloc(sizeof(*link->wr_tx_v2_pend), in smc_wr_alloc_link_mem()
802 if (!link->wr_tx_v2_pend) in smc_wr_alloc_link_mem()
808 kfree(link->wr_tx_v2_sge); in smc_wr_alloc_link_mem()
810 kfree(link->wr_tx_v2_ib); in smc_wr_alloc_link_mem()
812 kfree(link->wr_tx_compl); in smc_wr_alloc_link_mem()
814 kfree(link->wr_tx_pends); in smc_wr_alloc_link_mem()
816 kfree(link->wr_tx_mask); in smc_wr_alloc_link_mem()
818 kfree(link->wr_rx_sges); in smc_wr_alloc_link_mem()
820 kfree(link->wr_tx_sges); in smc_wr_alloc_link_mem()
822 kfree(link->wr_tx_rdma_sges); in smc_wr_alloc_link_mem()
824 kfree(link->wr_tx_rdmas); in smc_wr_alloc_link_mem()
826 kfree(link->wr_rx_ibs); in smc_wr_alloc_link_mem()
828 kfree(link->wr_tx_ibs); in smc_wr_alloc_link_mem()
830 kfree(link->wr_rx_bufs); in smc_wr_alloc_link_mem()
832 kfree(link->wr_tx_bufs); in smc_wr_alloc_link_mem()