Lines Matching refs:sess
51 struct rtrs_clt_sess *sess; in rtrs_clt_is_connected() local
55 list_for_each_entry_rcu(sess, &clt->paths_list, s.entry) in rtrs_clt_is_connected()
56 connected |= READ_ONCE(sess->state) == RTRS_CLT_CONNECTED; in rtrs_clt_is_connected()
173 struct rtrs_clt_con *rtrs_permit_to_clt_con(struct rtrs_clt_sess *sess, in rtrs_permit_to_clt_con() argument
179 id = (permit->cpu_id % (sess->s.irq_con_num - 1)) + 1; in rtrs_permit_to_clt_con()
181 return to_clt_con(sess->s.con[id]); in rtrs_permit_to_clt_con()
196 static bool rtrs_clt_change_state(struct rtrs_clt_sess *sess, in rtrs_clt_change_state() argument
202 lockdep_assert_held(&sess->state_wq.lock); in rtrs_clt_change_state()
204 old_state = sess->state; in rtrs_clt_change_state()
278 sess->state = new_state; in rtrs_clt_change_state()
279 wake_up_locked(&sess->state_wq); in rtrs_clt_change_state()
285 static bool rtrs_clt_change_state_from_to(struct rtrs_clt_sess *sess, in rtrs_clt_change_state_from_to() argument
291 spin_lock_irq(&sess->state_wq.lock); in rtrs_clt_change_state_from_to()
292 if (sess->state == old_state) in rtrs_clt_change_state_from_to()
293 changed = rtrs_clt_change_state(sess, new_state); in rtrs_clt_change_state_from_to()
294 spin_unlock_irq(&sess->state_wq.lock); in rtrs_clt_change_state_from_to()
301 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in rtrs_rdma_error_recovery() local
303 if (rtrs_clt_change_state_from_to(sess, in rtrs_rdma_error_recovery()
306 struct rtrs_clt *clt = sess->clt; in rtrs_rdma_error_recovery()
313 queue_delayed_work(rtrs_wq, &sess->reconnect_dwork, in rtrs_rdma_error_recovery()
322 rtrs_clt_change_state_from_to(sess, in rtrs_rdma_error_recovery()
333 rtrs_err(con->c.sess, "Failed IB_WR_REG_MR: %s\n", in rtrs_clt_fast_reg_done()
353 rtrs_err(con->c.sess, "Failed IB_WR_LOCAL_INV: %s\n", in rtrs_clt_inv_rkey_done()
383 struct rtrs_clt_sess *sess; in complete_rdma_req() local
390 sess = to_clt_sess(con->c.sess); in complete_rdma_req()
420 rtrs_err(con->c.sess, "Send INV WR key=%#x: %d\n", in complete_rdma_req()
436 ib_dma_unmap_sg(sess->s.dev->ib_dev, req->sglist, in complete_rdma_req()
442 atomic_dec(&sess->stats->inflight); in complete_rdma_req()
448 rtrs_err_rl(con->c.sess, "IO request failed: error=%d path=%s [%s:%u] notify=%d\n", in complete_rdma_req()
449 errno, kobject_name(&sess->kobj), sess->hca_name, in complete_rdma_req()
450 sess->hca_port, notify); in complete_rdma_req()
462 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in rtrs_post_send_rdma() local
467 rtrs_wrn(con->c.sess, in rtrs_post_send_rdma()
475 sge.lkey = sess->s.dev->ib_pd->local_dma_lkey; in rtrs_post_send_rdma()
481 flags = atomic_inc_return(&con->c.wr_cnt) % sess->s.signal_interval ? in rtrs_post_send_rdma()
484 ib_dma_sync_single_for_device(sess->s.dev->ib_dev, req->iu->dma_addr, in rtrs_post_send_rdma()
492 static void process_io_rsp(struct rtrs_clt_sess *sess, u32 msg_id, in process_io_rsp() argument
497 if (WARN_ON(msg_id >= sess->queue_depth)) in process_io_rsp()
500 req = &sess->reqs[msg_id]; in process_io_rsp()
510 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in rtrs_clt_recv_done() local
512 WARN_ON((sess->flags & RTRS_MSG_NEW_RKEY_F) == 0); in rtrs_clt_recv_done()
517 rtrs_err(con->c.sess, "post iu failed %d\n", err); in rtrs_clt_recv_done()
524 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in rtrs_clt_rkey_rsp_done() local
532 WARN_ON((sess->flags & RTRS_MSG_NEW_RKEY_F) == 0); in rtrs_clt_rkey_rsp_done()
537 rtrs_err(con->c.sess, "rkey response is malformed: size %d\n", in rtrs_clt_rkey_rsp_done()
541 ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr, in rtrs_clt_rkey_rsp_done()
545 rtrs_err(sess->clt, "rkey response is malformed: type %d\n", in rtrs_clt_rkey_rsp_done()
550 if (WARN_ON(buf_id >= sess->queue_depth)) in rtrs_clt_rkey_rsp_done()
563 sess->rbufs[buf_id].rkey = le32_to_cpu(msg->rkey); in rtrs_clt_rkey_rsp_done()
564 process_io_rsp(sess, msg_id, err, w_inval); in rtrs_clt_rkey_rsp_done()
566 ib_dma_sync_single_for_device(sess->s.dev->ib_dev, iu->dma_addr, in rtrs_clt_rkey_rsp_done()
603 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in rtrs_clt_rdma_done() local
610 rtrs_err(sess->clt, "RDMA failed: %s\n", in rtrs_clt_rdma_done()
635 process_io_rsp(sess, msg_id, err, w_inval); in rtrs_clt_rdma_done()
638 rtrs_send_hb_ack(&sess->s); in rtrs_clt_rdma_done()
639 if (sess->flags & RTRS_MSG_NEW_RKEY_F) in rtrs_clt_rdma_done()
643 sess->s.hb_missed_cnt = 0; in rtrs_clt_rdma_done()
644 sess->s.hb_cur_latency = in rtrs_clt_rdma_done()
645 ktime_sub(ktime_get(), sess->s.hb_last_sent); in rtrs_clt_rdma_done()
646 if (sess->flags & RTRS_MSG_NEW_RKEY_F) in rtrs_clt_rdma_done()
649 rtrs_wrn(con->c.sess, "Unknown IMM type %u\n", in rtrs_clt_rdma_done()
661 rtrs_err(con->c.sess, "rtrs_post_recv_empty(): %d\n", in rtrs_clt_rdma_done()
673 if (sess->flags & RTRS_MSG_NEW_RKEY_F) { in rtrs_clt_rdma_done()
688 rtrs_wrn(sess->clt, "Unexpected WC type: %d\n", wc->opcode); in rtrs_clt_rdma_done()
696 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in post_recv_io() local
699 if (sess->flags & RTRS_MSG_NEW_RKEY_F) { in post_recv_io()
713 static int post_recv_sess(struct rtrs_clt_sess *sess) in post_recv_sess() argument
718 for (cid = 0; cid < sess->s.con_num; cid++) { in post_recv_sess()
722 q_size = sess->queue_depth; in post_recv_sess()
730 err = post_recv_io(to_clt_con(sess->s.con[cid]), q_size); in post_recv_sess()
732 rtrs_err(sess->clt, "post_recv_io(), err: %d\n", err); in post_recv_sess()
818 struct rtrs_clt_sess *sess; in get_next_path_min_inflight() local
822 list_for_each_entry_rcu(sess, &clt->paths_list, s.entry) { in get_next_path_min_inflight()
823 if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED) in get_next_path_min_inflight()
826 if (!list_empty(raw_cpu_ptr(sess->mp_skip_entry))) in get_next_path_min_inflight()
829 inflight = atomic_read(&sess->stats->inflight); in get_next_path_min_inflight()
833 min_path = sess; in get_next_path_min_inflight()
869 struct rtrs_clt_sess *sess; in get_next_path_min_latency() local
873 list_for_each_entry_rcu(sess, &clt->paths_list, s.entry) { in get_next_path_min_latency()
874 if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED) in get_next_path_min_latency()
877 if (!list_empty(raw_cpu_ptr(sess->mp_skip_entry))) in get_next_path_min_latency()
880 latency = sess->s.hb_cur_latency; in get_next_path_min_latency()
884 min_path = sess; in get_next_path_min_latency()
943 struct rtrs_clt_sess *sess, in rtrs_clt_init_req() argument
961 req->con = rtrs_permit_to_clt_con(sess, permit); in rtrs_clt_init_req()
967 req->mp_policy = sess->clt->mp_policy; in rtrs_clt_init_req()
977 rtrs_clt_get_req(struct rtrs_clt_sess *sess, in rtrs_clt_get_req() argument
986 req = &sess->reqs[permit->mem_id]; in rtrs_clt_get_req()
987 rtrs_clt_init_req(req, sess, conf, permit, priv, vec, usr_len, in rtrs_clt_get_req()
1016 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in rtrs_post_rdma_write_sg() local
1036 sge[i].lkey = sess->s.dev->ib_pd->local_dma_lkey; in rtrs_post_rdma_write_sg()
1042 sge[i].lkey = sess->s.dev->ib_pd->local_dma_lkey; in rtrs_post_rdma_write_sg()
1048 flags = atomic_inc_return(&con->c.wr_cnt) % sess->s.signal_interval ? in rtrs_post_rdma_write_sg()
1051 ib_dma_sync_single_for_device(sess->s.dev->ib_dev, req->iu->dma_addr, in rtrs_post_rdma_write_sg()
1077 struct rtrs_sess *s = con->c.sess; in rtrs_clt_write_req()
1078 struct rtrs_clt_sess *sess = to_clt_sess(s); in rtrs_clt_write_req() local
1091 if (tsize > sess->chunk_size) { in rtrs_clt_write_req()
1093 tsize, sess->chunk_size); in rtrs_clt_write_req()
1097 count = ib_dma_map_sg(sess->s.dev->ib_dev, req->sglist, in rtrs_clt_write_req()
1114 rbuf = &sess->rbufs[buf_id]; in rtrs_clt_write_req()
1122 ib_dma_unmap_sg(sess->s.dev->ib_dev, req->sglist, in rtrs_clt_write_req()
1156 ret, kobject_name(&sess->kobj), sess->hca_name, in rtrs_clt_write_req()
1157 sess->hca_port); in rtrs_clt_write_req()
1159 atomic_dec(&sess->stats->inflight); in rtrs_clt_write_req()
1161 ib_dma_unmap_sg(sess->s.dev->ib_dev, req->sglist, in rtrs_clt_write_req()
1171 struct rtrs_sess *s = con->c.sess; in rtrs_clt_read_req()
1172 struct rtrs_clt_sess *sess = to_clt_sess(s); in rtrs_clt_read_req() local
1174 struct rtrs_ib_dev *dev = sess->s.dev; in rtrs_clt_read_req()
1184 if (tsize > sess->chunk_size) { in rtrs_clt_read_req()
1187 tsize, sess->chunk_size); in rtrs_clt_read_req()
1257 ret = rtrs_post_send_rdma(req->con, req, &sess->rbufs[buf_id], in rtrs_clt_read_req()
1262 ret, kobject_name(&sess->kobj), sess->hca_name, in rtrs_clt_read_req()
1263 sess->hca_port); in rtrs_clt_read_req()
1265 atomic_dec(&sess->stats->inflight); in rtrs_clt_read_req()
1313 static void fail_all_outstanding_reqs(struct rtrs_clt_sess *sess) in fail_all_outstanding_reqs() argument
1315 struct rtrs_clt *clt = sess->clt; in fail_all_outstanding_reqs()
1319 if (!sess->reqs) in fail_all_outstanding_reqs()
1321 for (i = 0; i < sess->queue_depth; ++i) { in fail_all_outstanding_reqs()
1322 req = &sess->reqs[i]; in fail_all_outstanding_reqs()
1340 static void free_sess_reqs(struct rtrs_clt_sess *sess) in free_sess_reqs() argument
1345 if (!sess->reqs) in free_sess_reqs()
1347 for (i = 0; i < sess->queue_depth; ++i) { in free_sess_reqs()
1348 req = &sess->reqs[i]; in free_sess_reqs()
1352 rtrs_iu_free(req->iu, sess->s.dev->ib_dev, 1); in free_sess_reqs()
1354 kfree(sess->reqs); in free_sess_reqs()
1355 sess->reqs = NULL; in free_sess_reqs()
1358 static int alloc_sess_reqs(struct rtrs_clt_sess *sess) in alloc_sess_reqs() argument
1363 sess->reqs = kcalloc(sess->queue_depth, sizeof(*sess->reqs), in alloc_sess_reqs()
1365 if (!sess->reqs) in alloc_sess_reqs()
1368 for (i = 0; i < sess->queue_depth; ++i) { in alloc_sess_reqs()
1369 req = &sess->reqs[i]; in alloc_sess_reqs()
1370 req->iu = rtrs_iu_alloc(1, sess->max_hdr_size, GFP_KERNEL, in alloc_sess_reqs()
1371 sess->s.dev->ib_dev, in alloc_sess_reqs()
1381 req->mr = ib_alloc_mr(sess->s.dev->ib_pd, IB_MR_TYPE_MEM_REG, in alloc_sess_reqs()
1382 sess->max_pages_per_mr); in alloc_sess_reqs()
1387 sess->max_pages_per_mr); in alloc_sess_reqs()
1397 free_sess_reqs(sess); in alloc_sess_reqs()
1450 static void query_fast_reg_mode(struct rtrs_clt_sess *sess) in query_fast_reg_mode() argument
1456 ib_dev = sess->s.dev->ib_dev; in query_fast_reg_mode()
1466 sess->max_pages_per_mr = in query_fast_reg_mode()
1467 min3(sess->max_pages_per_mr, (u32)max_pages_per_mr, in query_fast_reg_mode()
1469 sess->clt->max_segments = in query_fast_reg_mode()
1470 min(sess->max_pages_per_mr, sess->clt->max_segments); in query_fast_reg_mode()
1473 static bool rtrs_clt_change_state_get_old(struct rtrs_clt_sess *sess, in rtrs_clt_change_state_get_old() argument
1479 spin_lock_irq(&sess->state_wq.lock); in rtrs_clt_change_state_get_old()
1481 *old_state = sess->state; in rtrs_clt_change_state_get_old()
1482 changed = rtrs_clt_change_state(sess, new_state); in rtrs_clt_change_state_get_old()
1483 spin_unlock_irq(&sess->state_wq.lock); in rtrs_clt_change_state_get_old()
1495 static void rtrs_clt_init_hb(struct rtrs_clt_sess *sess) in rtrs_clt_init_hb() argument
1497 rtrs_init_hb(&sess->s, &io_comp_cqe, in rtrs_clt_init_hb()
1511 struct rtrs_clt_sess *sess; in alloc_sess() local
1516 sess = kzalloc(sizeof(*sess), GFP_KERNEL); in alloc_sess()
1517 if (!sess) in alloc_sess()
1525 sess->s.con = kcalloc(total_con, sizeof(*sess->s.con), GFP_KERNEL); in alloc_sess()
1526 if (!sess->s.con) in alloc_sess()
1529 sess->s.con_num = total_con; in alloc_sess()
1530 sess->s.irq_con_num = con_num + 1; in alloc_sess()
1532 sess->stats = kzalloc(sizeof(*sess->stats), GFP_KERNEL); in alloc_sess()
1533 if (!sess->stats) in alloc_sess()
1536 mutex_init(&sess->init_mutex); in alloc_sess()
1537 uuid_gen(&sess->s.uuid); in alloc_sess()
1538 memcpy(&sess->s.dst_addr, path->dst, in alloc_sess()
1547 memcpy(&sess->s.src_addr, path->src, in alloc_sess()
1549 strscpy(sess->s.sessname, clt->sessname, sizeof(sess->s.sessname)); in alloc_sess()
1550 sess->clt = clt; in alloc_sess()
1551 sess->max_pages_per_mr = RTRS_MAX_SEGMENTS; in alloc_sess()
1552 init_waitqueue_head(&sess->state_wq); in alloc_sess()
1553 sess->state = RTRS_CLT_CONNECTING; in alloc_sess()
1554 atomic_set(&sess->connected_cnt, 0); in alloc_sess()
1555 INIT_WORK(&sess->close_work, rtrs_clt_close_work); in alloc_sess()
1556 INIT_DELAYED_WORK(&sess->reconnect_dwork, rtrs_clt_reconnect_work); in alloc_sess()
1557 rtrs_clt_init_hb(sess); in alloc_sess()
1559 sess->mp_skip_entry = alloc_percpu(typeof(*sess->mp_skip_entry)); in alloc_sess()
1560 if (!sess->mp_skip_entry) in alloc_sess()
1564 INIT_LIST_HEAD(per_cpu_ptr(sess->mp_skip_entry, cpu)); in alloc_sess()
1566 err = rtrs_clt_init_stats(sess->stats); in alloc_sess()
1570 return sess; in alloc_sess()
1573 free_percpu(sess->mp_skip_entry); in alloc_sess()
1575 kfree(sess->stats); in alloc_sess()
1577 kfree(sess->s.con); in alloc_sess()
1579 kfree(sess); in alloc_sess()
1584 void free_sess(struct rtrs_clt_sess *sess) in free_sess() argument
1586 free_percpu(sess->mp_skip_entry); in free_sess()
1587 mutex_destroy(&sess->init_mutex); in free_sess()
1588 kfree(sess->s.con); in free_sess()
1589 kfree(sess->rbufs); in free_sess()
1590 kfree(sess); in free_sess()
1593 static int create_con(struct rtrs_clt_sess *sess, unsigned int cid) in create_con() argument
1604 con->c.sess = &sess->s; in create_con()
1609 sess->s.con[cid] = &con->c; in create_con()
1616 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in destroy_con() local
1618 sess->s.con[con->c.cid] = NULL; in destroy_con()
1625 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in create_con_cq_qp() local
1634 if (WARN_ON(sess->s.dev)) in create_con_cq_qp()
1642 sess->s.dev = rtrs_ib_dev_find_or_add(con->c.cm_id->device, in create_con_cq_qp()
1644 if (!sess->s.dev) { in create_con_cq_qp()
1645 rtrs_wrn(sess->clt, in create_con_cq_qp()
1649 sess->s.dev_ref = 1; in create_con_cq_qp()
1650 query_fast_reg_mode(sess); in create_con_cq_qp()
1651 wr_limit = sess->s.dev->ib_dev->attrs.max_qp_wr; in create_con_cq_qp()
1668 if (WARN_ON(!sess->s.dev)) in create_con_cq_qp()
1670 if (WARN_ON(!sess->queue_depth)) in create_con_cq_qp()
1673 wr_limit = sess->s.dev->ib_dev->attrs.max_qp_wr; in create_con_cq_qp()
1675 sess->s.dev_ref++; in create_con_cq_qp()
1678 sess->queue_depth * 3 + 1); in create_con_cq_qp()
1680 sess->queue_depth * 3 + 1); in create_con_cq_qp()
1686 if (sess->flags & RTRS_MSG_NEW_RKEY_F || con->c.cid == 0) { in create_con_cq_qp()
1688 GFP_KERNEL, sess->s.dev->ib_dev, in create_con_cq_qp()
1696 cq_vector = con->cpu % sess->s.dev->ib_dev->num_comp_vectors; in create_con_cq_qp()
1697 if (con->c.cid >= sess->s.irq_con_num) in create_con_cq_qp()
1698 err = rtrs_cq_qp_create(&sess->s, &con->c, max_send_sge, in create_con_cq_qp()
1702 err = rtrs_cq_qp_create(&sess->s, &con->c, max_send_sge, in create_con_cq_qp()
1714 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in destroy_con_cq_qp() local
1723 rtrs_iu_free(con->rsp_ius, sess->s.dev->ib_dev, con->queue_num); in destroy_con_cq_qp()
1727 if (sess->s.dev_ref && !--sess->s.dev_ref) { in destroy_con_cq_qp()
1728 rtrs_ib_dev_put(sess->s.dev); in destroy_con_cq_qp()
1729 sess->s.dev = NULL; in destroy_con_cq_qp()
1748 struct rtrs_sess *s = con->c.sess; in rtrs_rdma_addr_resolved()
1767 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in rtrs_rdma_route_resolved() local
1768 struct rtrs_clt *clt = sess->clt; in rtrs_rdma_route_resolved()
1785 .cid_num = cpu_to_le16(sess->s.con_num), in rtrs_rdma_route_resolved()
1786 .recon_cnt = cpu_to_le16(sess->s.recon_cnt), in rtrs_rdma_route_resolved()
1788 msg.first_conn = sess->for_new_clt ? FIRST_CONN : 0; in rtrs_rdma_route_resolved()
1789 uuid_copy(&msg.sess_uuid, &sess->s.uuid); in rtrs_rdma_route_resolved()
1802 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in rtrs_rdma_conn_established() local
1803 struct rtrs_clt *clt = sess->clt; in rtrs_rdma_conn_established()
1834 if (sess->queue_depth > 0 && queue_depth != sess->queue_depth) { in rtrs_rdma_conn_established()
1840 sess->reconnect_attempts = -1; in rtrs_rdma_conn_established()
1846 if (!sess->rbufs) { in rtrs_rdma_conn_established()
1847 sess->rbufs = kcalloc(queue_depth, sizeof(*sess->rbufs), in rtrs_rdma_conn_established()
1849 if (!sess->rbufs) in rtrs_rdma_conn_established()
1852 sess->queue_depth = queue_depth; in rtrs_rdma_conn_established()
1853 sess->s.signal_interval = min_not_zero(queue_depth, in rtrs_rdma_conn_established()
1855 sess->max_hdr_size = le32_to_cpu(msg->max_hdr_size); in rtrs_rdma_conn_established()
1856 sess->max_io_size = le32_to_cpu(msg->max_io_size); in rtrs_rdma_conn_established()
1857 sess->flags = le32_to_cpu(msg->flags); in rtrs_rdma_conn_established()
1858 sess->chunk_size = sess->max_io_size + sess->max_hdr_size; in rtrs_rdma_conn_established()
1869 clt->queue_depth = sess->queue_depth; in rtrs_rdma_conn_established()
1870 clt->max_io_size = min_not_zero(sess->max_io_size, in rtrs_rdma_conn_established()
1877 sess->hca_port = con->c.cm_id->port_num; in rtrs_rdma_conn_established()
1878 scnprintf(sess->hca_name, sizeof(sess->hca_name), in rtrs_rdma_conn_established()
1879 sess->s.dev->ib_dev->name); in rtrs_rdma_conn_established()
1880 sess->s.src_addr = con->c.cm_id->route.addr.src_addr; in rtrs_rdma_conn_established()
1882 sess->for_new_clt = 1; in rtrs_rdma_conn_established()
1890 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in flag_success_on_conn() local
1892 atomic_inc(&sess->connected_cnt); in flag_success_on_conn()
1899 struct rtrs_sess *s = con->c.sess; in rtrs_rdma_conn_rejected()
1927 void rtrs_clt_close_conns(struct rtrs_clt_sess *sess, bool wait) in rtrs_clt_close_conns() argument
1929 if (rtrs_clt_change_state_get_old(sess, RTRS_CLT_CLOSING, NULL)) in rtrs_clt_close_conns()
1930 queue_work(rtrs_wq, &sess->close_work); in rtrs_clt_close_conns()
1932 flush_work(&sess->close_work); in rtrs_clt_close_conns()
1938 struct rtrs_clt_sess *sess; in flag_error_on_conn() local
1940 sess = to_clt_sess(con->c.sess); in flag_error_on_conn()
1941 if (atomic_dec_and_test(&sess->connected_cnt)) in flag_error_on_conn()
1943 wake_up(&sess->state_wq); in flag_error_on_conn()
1952 struct rtrs_sess *s = con->c.sess; in rtrs_clt_rdma_cm_handler()
1953 struct rtrs_clt_sess *sess = to_clt_sess(s); in rtrs_clt_rdma_cm_handler() local
1971 wake_up(&sess->state_wq); in rtrs_clt_rdma_cm_handler()
2000 rtrs_clt_close_conns(sess, false); in rtrs_clt_rdma_cm_handler()
2023 struct rtrs_sess *s = con->c.sess; in create_cm()
2024 struct rtrs_clt_sess *sess = to_clt_sess(s); in create_cm() local
2029 sess->s.dst_addr.ss_family == AF_IB ? in create_cm()
2045 err = rdma_resolve_addr(cm_id, (struct sockaddr *)&sess->s.src_addr, in create_cm()
2046 (struct sockaddr *)&sess->s.dst_addr, in create_cm()
2058 sess->state_wq, in create_cm()
2059 con->cm_err || sess->state != RTRS_CLT_CONNECTING, in create_cm()
2071 if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTING) { in create_cm()
2090 static void rtrs_clt_sess_up(struct rtrs_clt_sess *sess) in rtrs_clt_sess_up() argument
2092 struct rtrs_clt *clt = sess->clt; in rtrs_clt_sess_up()
2116 sess->established = true; in rtrs_clt_sess_up()
2117 sess->reconnect_attempts = 0; in rtrs_clt_sess_up()
2118 sess->stats->reconnects.successful_cnt++; in rtrs_clt_sess_up()
2121 static void rtrs_clt_sess_down(struct rtrs_clt_sess *sess) in rtrs_clt_sess_down() argument
2123 struct rtrs_clt *clt = sess->clt; in rtrs_clt_sess_down()
2125 if (!sess->established) in rtrs_clt_sess_down()
2128 sess->established = false; in rtrs_clt_sess_down()
2136 static void rtrs_clt_stop_and_destroy_conns(struct rtrs_clt_sess *sess) in rtrs_clt_stop_and_destroy_conns() argument
2141 WARN_ON(READ_ONCE(sess->state) == RTRS_CLT_CONNECTED); in rtrs_clt_stop_and_destroy_conns()
2147 mutex_lock(&sess->init_mutex); in rtrs_clt_stop_and_destroy_conns()
2148 mutex_unlock(&sess->init_mutex); in rtrs_clt_stop_and_destroy_conns()
2156 rtrs_stop_hb(&sess->s); in rtrs_clt_stop_and_destroy_conns()
2165 for (cid = 0; cid < sess->s.con_num; cid++) { in rtrs_clt_stop_and_destroy_conns()
2166 if (!sess->s.con[cid]) in rtrs_clt_stop_and_destroy_conns()
2168 con = to_clt_con(sess->s.con[cid]); in rtrs_clt_stop_and_destroy_conns()
2171 fail_all_outstanding_reqs(sess); in rtrs_clt_stop_and_destroy_conns()
2172 free_sess_reqs(sess); in rtrs_clt_stop_and_destroy_conns()
2173 rtrs_clt_sess_down(sess); in rtrs_clt_stop_and_destroy_conns()
2183 wait_event_timeout(sess->state_wq, !atomic_read(&sess->connected_cnt), in rtrs_clt_stop_and_destroy_conns()
2186 for (cid = 0; cid < sess->s.con_num; cid++) { in rtrs_clt_stop_and_destroy_conns()
2187 if (!sess->s.con[cid]) in rtrs_clt_stop_and_destroy_conns()
2189 con = to_clt_con(sess->s.con[cid]); in rtrs_clt_stop_and_destroy_conns()
2199 struct rtrs_clt_sess *sess, in xchg_sessions() argument
2206 return sess == cmpxchg(ppcpu_path, sess, next); in xchg_sessions()
2209 static void rtrs_clt_remove_path_from_arr(struct rtrs_clt_sess *sess) in rtrs_clt_remove_path_from_arr() argument
2211 struct rtrs_clt *clt = sess->clt; in rtrs_clt_remove_path_from_arr()
2217 list_del_rcu(&sess->s.entry); in rtrs_clt_remove_path_from_arr()
2258 next = list_next_or_null_rr_rcu(&clt->paths_list, &sess->s.entry, in rtrs_clt_remove_path_from_arr()
2271 lockdep_is_held(&clt->paths_mutex)) != sess) in rtrs_clt_remove_path_from_arr()
2284 if (xchg_sessions(ppcpu_path, sess, next)) in rtrs_clt_remove_path_from_arr()
2299 static void rtrs_clt_add_path_to_arr(struct rtrs_clt_sess *sess) in rtrs_clt_add_path_to_arr() argument
2301 struct rtrs_clt *clt = sess->clt; in rtrs_clt_add_path_to_arr()
2306 list_add_tail_rcu(&sess->s.entry, &clt->paths_list); in rtrs_clt_add_path_to_arr()
2312 struct rtrs_clt_sess *sess; in rtrs_clt_close_work() local
2314 sess = container_of(work, struct rtrs_clt_sess, close_work); in rtrs_clt_close_work()
2316 cancel_delayed_work_sync(&sess->reconnect_dwork); in rtrs_clt_close_work()
2317 rtrs_clt_stop_and_destroy_conns(sess); in rtrs_clt_close_work()
2318 rtrs_clt_change_state_get_old(sess, RTRS_CLT_CLOSED, NULL); in rtrs_clt_close_work()
2321 static int init_conns(struct rtrs_clt_sess *sess) in init_conns() argument
2331 sess->s.recon_cnt++; in init_conns()
2334 for (cid = 0; cid < sess->s.con_num; cid++) { in init_conns()
2335 err = create_con(sess, cid); in init_conns()
2339 err = create_cm(to_clt_con(sess->s.con[cid])); in init_conns()
2341 destroy_con(to_clt_con(sess->s.con[cid])); in init_conns()
2345 err = alloc_sess_reqs(sess); in init_conns()
2349 rtrs_start_hb(&sess->s); in init_conns()
2355 struct rtrs_clt_con *con = to_clt_con(sess->s.con[cid]); in init_conns()
2370 rtrs_clt_change_state_get_old(sess, RTRS_CLT_CONNECTING_ERR, NULL); in init_conns()
2378 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in rtrs_clt_info_req_done() local
2382 rtrs_iu_free(iu, sess->s.dev->ib_dev, 1); in rtrs_clt_info_req_done()
2385 rtrs_err(sess->clt, "Sess info request send failed: %s\n", in rtrs_clt_info_req_done()
2387 rtrs_clt_change_state_get_old(sess, RTRS_CLT_CONNECTING_ERR, NULL); in rtrs_clt_info_req_done()
2394 static int process_info_rsp(struct rtrs_clt_sess *sess, in process_info_rsp() argument
2401 if (!sg_cnt || (sess->queue_depth % sg_cnt)) { in process_info_rsp()
2402 rtrs_err(sess->clt, "Incorrect sg_cnt %d, is not multiple\n", in process_info_rsp()
2411 if ((ilog2(sg_cnt - 1) + 1) + (ilog2(sess->chunk_size - 1) + 1) > in process_info_rsp()
2413 rtrs_err(sess->clt, in process_info_rsp()
2415 MAX_IMM_PAYL_BITS, sg_cnt, sess->chunk_size); in process_info_rsp()
2419 for (sgi = 0, i = 0; sgi < sg_cnt && i < sess->queue_depth; sgi++) { in process_info_rsp()
2430 if (!len || (len % sess->chunk_size)) { in process_info_rsp()
2431 rtrs_err(sess->clt, "Incorrect [%d].len %d\n", sgi, in process_info_rsp()
2435 for ( ; len && i < sess->queue_depth; i++) { in process_info_rsp()
2436 sess->rbufs[i].addr = addr; in process_info_rsp()
2437 sess->rbufs[i].rkey = rkey; in process_info_rsp()
2439 len -= sess->chunk_size; in process_info_rsp()
2440 addr += sess->chunk_size; in process_info_rsp()
2444 if (sgi != sg_cnt || i != sess->queue_depth) { in process_info_rsp()
2445 rtrs_err(sess->clt, "Incorrect sg vector, not fully mapped\n"); in process_info_rsp()
2448 if (total_len != sess->chunk_size * sess->queue_depth) { in process_info_rsp()
2449 rtrs_err(sess->clt, "Incorrect total_len %d\n", total_len); in process_info_rsp()
2459 struct rtrs_clt_sess *sess = to_clt_sess(con->c.sess); in rtrs_clt_info_rsp_done() local
2471 rtrs_err(sess->clt, "Sess info response recv failed: %s\n", in rtrs_clt_info_rsp_done()
2478 rtrs_err(sess->clt, "Sess info response is malformed: size %d\n", in rtrs_clt_info_rsp_done()
2482 ib_dma_sync_single_for_cpu(sess->s.dev->ib_dev, iu->dma_addr, in rtrs_clt_info_rsp_done()
2486 rtrs_err(sess->clt, "Sess info response is malformed: type %d\n", in rtrs_clt_info_rsp_done()
2493 rtrs_err(sess->clt, "Sess info response is malformed: size %d\n", in rtrs_clt_info_rsp_done()
2497 err = process_info_rsp(sess, msg); in rtrs_clt_info_rsp_done()
2501 err = post_recv_sess(sess); in rtrs_clt_info_rsp_done()
2509 rtrs_iu_free(iu, sess->s.dev->ib_dev, 1); in rtrs_clt_info_rsp_done()
2510 rtrs_clt_change_state_get_old(sess, state, NULL); in rtrs_clt_info_rsp_done()
2513 static int rtrs_send_sess_info(struct rtrs_clt_sess *sess) in rtrs_send_sess_info() argument
2515 struct rtrs_clt_con *usr_con = to_clt_con(sess->s.con[0]); in rtrs_send_sess_info()
2522 rx_sz += sizeof(struct rtrs_sg_desc) * sess->queue_depth; in rtrs_send_sess_info()
2525 sess->s.dev->ib_dev, DMA_TO_DEVICE, in rtrs_send_sess_info()
2527 rx_iu = rtrs_iu_alloc(1, rx_sz, GFP_KERNEL, sess->s.dev->ib_dev, in rtrs_send_sess_info()
2536 rtrs_err(sess->clt, "rtrs_iu_post_recv(), err: %d\n", err); in rtrs_send_sess_info()
2543 memcpy(msg->sessname, sess->s.sessname, sizeof(msg->sessname)); in rtrs_send_sess_info()
2545 ib_dma_sync_single_for_device(sess->s.dev->ib_dev, tx_iu->dma_addr, in rtrs_send_sess_info()
2551 rtrs_err(sess->clt, "rtrs_iu_post_send(), err: %d\n", err); in rtrs_send_sess_info()
2557 wait_event_interruptible_timeout(sess->state_wq, in rtrs_send_sess_info()
2558 sess->state != RTRS_CLT_CONNECTING, in rtrs_send_sess_info()
2561 if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED) { in rtrs_send_sess_info()
2562 if (READ_ONCE(sess->state) == RTRS_CLT_CONNECTING_ERR) in rtrs_send_sess_info()
2570 rtrs_iu_free(tx_iu, sess->s.dev->ib_dev, 1); in rtrs_send_sess_info()
2572 rtrs_iu_free(rx_iu, sess->s.dev->ib_dev, 1); in rtrs_send_sess_info()
2575 rtrs_clt_change_state_get_old(sess, RTRS_CLT_CONNECTING_ERR, NULL); in rtrs_send_sess_info()
2586 static int init_sess(struct rtrs_clt_sess *sess) in init_sess() argument
2591 .src = &sess->s.src_addr, in init_sess()
2592 .dst = &sess->s.dst_addr, in init_sess()
2597 mutex_lock(&sess->init_mutex); in init_sess()
2598 err = init_conns(sess); in init_sess()
2600 rtrs_err(sess->clt, in init_sess()
2602 str, sess->hca_name, sess->hca_port); in init_sess()
2605 err = rtrs_send_sess_info(sess); in init_sess()
2608 sess->clt, in init_sess()
2610 err, str, sess->hca_name, sess->hca_port); in init_sess()
2613 rtrs_clt_sess_up(sess); in init_sess()
2615 mutex_unlock(&sess->init_mutex); in init_sess()
2622 struct rtrs_clt_sess *sess; in rtrs_clt_reconnect_work() local
2627 sess = container_of(to_delayed_work(work), struct rtrs_clt_sess, in rtrs_clt_reconnect_work()
2629 clt = sess->clt; in rtrs_clt_reconnect_work()
2631 if (READ_ONCE(sess->state) != RTRS_CLT_RECONNECTING) in rtrs_clt_reconnect_work()
2634 if (sess->reconnect_attempts >= clt->max_reconnect_attempts) { in rtrs_clt_reconnect_work()
2636 rtrs_clt_close_conns(sess, false); in rtrs_clt_reconnect_work()
2639 sess->reconnect_attempts++; in rtrs_clt_reconnect_work()
2642 rtrs_clt_stop_and_destroy_conns(sess); in rtrs_clt_reconnect_work()
2644 if (rtrs_clt_change_state_get_old(sess, RTRS_CLT_CONNECTING, NULL)) { in rtrs_clt_reconnect_work()
2645 err = init_sess(sess); in rtrs_clt_reconnect_work()
2653 if (rtrs_clt_change_state_get_old(sess, RTRS_CLT_RECONNECTING, NULL)) { in rtrs_clt_reconnect_work()
2654 sess->stats->reconnects.fail_cnt++; in rtrs_clt_reconnect_work()
2656 queue_delayed_work(rtrs_wq, &sess->reconnect_dwork, in rtrs_clt_reconnect_work()
2787 struct rtrs_clt_sess *sess, *tmp; in rtrs_clt_open() local
2806 struct rtrs_clt_sess *sess; in rtrs_clt_open() local
2808 sess = alloc_sess(clt, &paths[i], nr_cpu_ids, in rtrs_clt_open()
2810 if (IS_ERR(sess)) { in rtrs_clt_open()
2811 err = PTR_ERR(sess); in rtrs_clt_open()
2815 sess->for_new_clt = 1; in rtrs_clt_open()
2816 list_add_tail_rcu(&sess->s.entry, &clt->paths_list); in rtrs_clt_open()
2818 err = init_sess(sess); in rtrs_clt_open()
2820 list_del_rcu(&sess->s.entry); in rtrs_clt_open()
2821 rtrs_clt_close_conns(sess, true); in rtrs_clt_open()
2822 free_percpu(sess->stats->pcpu_stats); in rtrs_clt_open()
2823 kfree(sess->stats); in rtrs_clt_open()
2824 free_sess(sess); in rtrs_clt_open()
2828 err = rtrs_clt_create_sess_files(sess); in rtrs_clt_open()
2830 list_del_rcu(&sess->s.entry); in rtrs_clt_open()
2831 rtrs_clt_close_conns(sess, true); in rtrs_clt_open()
2832 free_percpu(sess->stats->pcpu_stats); in rtrs_clt_open()
2833 kfree(sess->stats); in rtrs_clt_open()
2834 free_sess(sess); in rtrs_clt_open()
2845 list_for_each_entry_safe(sess, tmp, &clt->paths_list, s.entry) { in rtrs_clt_open()
2846 rtrs_clt_destroy_sess_files(sess, NULL); in rtrs_clt_open()
2847 rtrs_clt_close_conns(sess, true); in rtrs_clt_open()
2848 kobject_put(&sess->kobj); in rtrs_clt_open()
2864 struct rtrs_clt_sess *sess, *tmp; in rtrs_clt_close() local
2870 list_for_each_entry_safe(sess, tmp, &clt->paths_list, s.entry) { in rtrs_clt_close()
2871 rtrs_clt_close_conns(sess, true); in rtrs_clt_close()
2872 rtrs_clt_destroy_sess_files(sess, NULL); in rtrs_clt_close()
2873 kobject_put(&sess->kobj); in rtrs_clt_close()
2879 int rtrs_clt_reconnect_from_sysfs(struct rtrs_clt_sess *sess) in rtrs_clt_reconnect_from_sysfs() argument
2885 changed = rtrs_clt_change_state_get_old(sess, RTRS_CLT_RECONNECTING, in rtrs_clt_reconnect_from_sysfs()
2888 sess->reconnect_attempts = 0; in rtrs_clt_reconnect_from_sysfs()
2889 queue_delayed_work(rtrs_wq, &sess->reconnect_dwork, 0); in rtrs_clt_reconnect_from_sysfs()
2897 flush_delayed_work(&sess->reconnect_dwork); in rtrs_clt_reconnect_from_sysfs()
2898 err = (READ_ONCE(sess->state) == in rtrs_clt_reconnect_from_sysfs()
2905 int rtrs_clt_remove_path_from_sysfs(struct rtrs_clt_sess *sess, in rtrs_clt_remove_path_from_sysfs() argument
2921 rtrs_clt_close_conns(sess, true); in rtrs_clt_remove_path_from_sysfs()
2922 changed = rtrs_clt_change_state_get_old(sess, in rtrs_clt_remove_path_from_sysfs()
2928 rtrs_clt_remove_path_from_arr(sess); in rtrs_clt_remove_path_from_sysfs()
2929 rtrs_clt_destroy_sess_files(sess, sysfs_self); in rtrs_clt_remove_path_from_sysfs()
2930 kobject_put(&sess->kobj); in rtrs_clt_remove_path_from_sysfs()
2976 struct rtrs_clt_sess *sess; in rtrs_clt_request() local
2998 (sess = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) { in rtrs_clt_request()
2999 if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED) in rtrs_clt_request()
3002 if (usr_len + hdr_len > sess->max_hdr_size) { in rtrs_clt_request()
3003 rtrs_wrn_rl(sess->clt, in rtrs_clt_request()
3006 usr_len, hdr_len, sess->max_hdr_size); in rtrs_clt_request()
3010 req = rtrs_clt_get_req(sess, ops->conf_fn, permit, ops->priv, in rtrs_clt_request()
3036 struct rtrs_clt_sess *sess; in rtrs_clt_rdma_cq_direct() local
3041 (sess = it.next_path(&it)) && it.i < it.clt->paths_num; it.i++) { in rtrs_clt_rdma_cq_direct()
3042 if (READ_ONCE(sess->state) != RTRS_CLT_CONNECTED) in rtrs_clt_rdma_cq_direct()
3045 con = sess->s.con[index + 1]; in rtrs_clt_rdma_cq_direct()
3083 struct rtrs_clt_sess *sess; in rtrs_clt_create_path_from_sysfs() local
3086 sess = alloc_sess(clt, addr, nr_cpu_ids, 0); in rtrs_clt_create_path_from_sysfs()
3087 if (IS_ERR(sess)) in rtrs_clt_create_path_from_sysfs()
3088 return PTR_ERR(sess); in rtrs_clt_create_path_from_sysfs()
3097 sess->for_new_clt = 1; in rtrs_clt_create_path_from_sysfs()
3107 rtrs_clt_add_path_to_arr(sess); in rtrs_clt_create_path_from_sysfs()
3109 err = init_sess(sess); in rtrs_clt_create_path_from_sysfs()
3113 err = rtrs_clt_create_sess_files(sess); in rtrs_clt_create_path_from_sysfs()
3120 rtrs_clt_remove_path_from_arr(sess); in rtrs_clt_create_path_from_sysfs()
3121 rtrs_clt_close_conns(sess, true); in rtrs_clt_create_path_from_sysfs()
3122 free_percpu(sess->stats->pcpu_stats); in rtrs_clt_create_path_from_sysfs()
3123 kfree(sess->stats); in rtrs_clt_create_path_from_sysfs()
3124 free_sess(sess); in rtrs_clt_create_path_from_sysfs()