1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2022, Microsoft Corporation. All rights reserved.
4 */
5
6 #include "mana_ib.h"
7 #include <net/mana/mana_auxiliary.h>
8 #include <net/addrconf.h>
9
10 MODULE_DESCRIPTION("Microsoft Azure Network Adapter IB driver");
11 MODULE_LICENSE("GPL");
12 MODULE_IMPORT_NS("NET_MANA");
13
14 static const struct ib_device_ops mana_ib_dev_ops = {
15 .owner = THIS_MODULE,
16 .driver_id = RDMA_DRIVER_MANA,
17 .uverbs_abi_ver = MANA_IB_UVERBS_ABI_VERSION,
18
19 .add_gid = mana_ib_gd_add_gid,
20 .alloc_pd = mana_ib_alloc_pd,
21 .alloc_ucontext = mana_ib_alloc_ucontext,
22 .create_ah = mana_ib_create_ah,
23 .create_cq = mana_ib_create_cq,
24 .create_qp = mana_ib_create_qp,
25 .create_rwq_ind_table = mana_ib_create_rwq_ind_table,
26 .create_wq = mana_ib_create_wq,
27 .dealloc_pd = mana_ib_dealloc_pd,
28 .dealloc_ucontext = mana_ib_dealloc_ucontext,
29 .del_gid = mana_ib_gd_del_gid,
30 .dereg_mr = mana_ib_dereg_mr,
31 .destroy_ah = mana_ib_destroy_ah,
32 .destroy_cq = mana_ib_destroy_cq,
33 .destroy_qp = mana_ib_destroy_qp,
34 .destroy_rwq_ind_table = mana_ib_destroy_rwq_ind_table,
35 .destroy_wq = mana_ib_destroy_wq,
36 .disassociate_ucontext = mana_ib_disassociate_ucontext,
37 .get_dma_mr = mana_ib_get_dma_mr,
38 .get_link_layer = mana_ib_get_link_layer,
39 .get_port_immutable = mana_ib_get_port_immutable,
40 .mmap = mana_ib_mmap,
41 .modify_qp = mana_ib_modify_qp,
42 .modify_wq = mana_ib_modify_wq,
43 .poll_cq = mana_ib_poll_cq,
44 .post_recv = mana_ib_post_recv,
45 .post_send = mana_ib_post_send,
46 .query_device = mana_ib_query_device,
47 .query_gid = mana_ib_query_gid,
48 .query_pkey = mana_ib_query_pkey,
49 .query_port = mana_ib_query_port,
50 .reg_user_mr = mana_ib_reg_user_mr,
51 .reg_user_mr_dmabuf = mana_ib_reg_user_mr_dmabuf,
52 .req_notify_cq = mana_ib_arm_cq,
53
54 INIT_RDMA_OBJ_SIZE(ib_ah, mana_ib_ah, ibah),
55 INIT_RDMA_OBJ_SIZE(ib_cq, mana_ib_cq, ibcq),
56 INIT_RDMA_OBJ_SIZE(ib_pd, mana_ib_pd, ibpd),
57 INIT_RDMA_OBJ_SIZE(ib_qp, mana_ib_qp, ibqp),
58 INIT_RDMA_OBJ_SIZE(ib_ucontext, mana_ib_ucontext, ibucontext),
59 INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mana_ib_rwq_ind_table,
60 ib_ind_table),
61 };
62
63 static const struct ib_device_ops mana_ib_stats_ops = {
64 .alloc_hw_port_stats = mana_ib_alloc_hw_port_stats,
65 .get_hw_stats = mana_ib_get_hw_stats,
66 };
67
68 static const struct ib_device_ops mana_ib_device_stats_ops = {
69 .alloc_hw_device_stats = mana_ib_alloc_hw_device_stats,
70 };
71
mana_ib_netdev_event(struct notifier_block * this,unsigned long event,void * ptr)72 static int mana_ib_netdev_event(struct notifier_block *this,
73 unsigned long event, void *ptr)
74 {
75 struct mana_ib_dev *dev = container_of(this, struct mana_ib_dev, nb);
76 struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
77 struct gdma_context *gc = dev->gdma_dev->gdma_context;
78 struct mana_context *mc = gc->mana.driver_data;
79 struct net_device *ndev;
80 int i;
81
82 /* Only process events from our parent device */
83 for (i = 0; i < dev->ib_dev.phys_port_cnt; i++)
84 if (event_dev == mc->ports[i]) {
85 switch (event) {
86 case NETDEV_CHANGEUPPER:
87 ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
88 /*
89 * RDMA core will setup GID based on updated netdev.
90 * It's not possible to race with the core as rtnl lock is being
91 * held.
92 */
93 ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
94
95 /* mana_get_primary_netdev() returns ndev with refcount held */
96 if (ndev)
97 netdev_put(ndev, &dev->dev_tracker);
98
99 return NOTIFY_OK;
100 default:
101 return NOTIFY_DONE;
102 }
103 }
104 return NOTIFY_DONE;
105 }
106
mana_ib_probe(struct auxiliary_device * adev,const struct auxiliary_device_id * id)107 static int mana_ib_probe(struct auxiliary_device *adev,
108 const struct auxiliary_device_id *id)
109 {
110 struct mana_adev *madev = container_of(adev, struct mana_adev, adev);
111 struct gdma_context *gc = madev->mdev->gdma_context;
112 struct mana_context *mc = gc->mana.driver_data;
113 struct gdma_dev *mdev = madev->mdev;
114 struct net_device *ndev;
115 struct mana_ib_dev *dev;
116 u8 mac_addr[ETH_ALEN];
117 int ret, i;
118
119 dev = ib_alloc_device(mana_ib_dev, ib_dev);
120 if (!dev)
121 return -ENOMEM;
122
123 ib_set_device_ops(&dev->ib_dev, &mana_ib_dev_ops);
124 dev->ib_dev.node_type = RDMA_NODE_IB_CA;
125 dev->ib_dev.num_comp_vectors = gc->max_num_queues;
126 dev->ib_dev.dev.parent = gc->dev;
127 dev->gdma_dev = mdev;
128 xa_init_flags(&dev->qp_table_wq, XA_FLAGS_LOCK_IRQ);
129
130 if (mana_ib_is_rnic(dev)) {
131 dev->ib_dev.phys_port_cnt = 1;
132 addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, mc->ports[0]->dev_addr);
133 ret = mana_ib_gd_query_adapter_caps(dev);
134 if (ret) {
135 ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", ret);
136 goto free_ib_device;
137 }
138
139 ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops);
140 if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_DEV_COUNTERS_SUPPORT)
141 ib_set_device_ops(&dev->ib_dev, &mana_ib_device_stats_ops);
142
143 ret = mana_ib_create_eqs(dev);
144 if (ret) {
145 ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret);
146 goto free_ib_device;
147 }
148
149 ret = mana_ib_gd_create_rnic_adapter(dev);
150 if (ret)
151 goto destroy_eqs;
152
153 if (dev->adapter_caps.feature_flags & MANA_IB_FEATURE_MULTI_PORTS_SUPPORT)
154 dev->ib_dev.phys_port_cnt = mc->num_ports;
155
156 for (i = 0; i < dev->ib_dev.phys_port_cnt; i++) {
157 ndev = mana_get_primary_netdev(mc, i, &dev->dev_tracker);
158 if (!ndev) {
159 ret = -ENODEV;
160 ibdev_err(&dev->ib_dev,
161 "Failed to get netdev for IB port %d", i + 1);
162 goto destroy_rnic;
163 }
164 ether_addr_copy(mac_addr, ndev->dev_addr);
165 ret = ib_device_set_netdev(&dev->ib_dev, ndev, i + 1);
166 /* mana_get_primary_netdev() returns ndev with refcount held */
167 netdev_put(ndev, &dev->dev_tracker);
168 if (ret) {
169 ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret);
170 goto destroy_rnic;
171 }
172 ret = mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr);
173 if (ret) {
174 ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret);
175 goto destroy_rnic;
176 }
177 }
178 dev->nb.notifier_call = mana_ib_netdev_event;
179 ret = register_netdevice_notifier(&dev->nb);
180 if (ret) {
181 ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", ret);
182 goto destroy_rnic;
183 }
184 } else {
185 dev->ib_dev.phys_port_cnt = mc->num_ports;
186 ret = mana_eth_query_adapter_caps(dev);
187 if (ret) {
188 ibdev_err(&dev->ib_dev, "Failed to query ETH device caps, ret %d", ret);
189 goto free_ib_device;
190 }
191 }
192
193 dev->av_pool = dma_pool_create("mana_ib_av", gc->dev, MANA_AV_BUFFER_SIZE,
194 MANA_AV_BUFFER_SIZE, 0);
195 if (!dev->av_pool) {
196 ret = -ENOMEM;
197 goto deregister_net_notifier;
198 }
199
200 ibdev_dbg(&dev->ib_dev, "mdev=%p id=%d num_ports=%d\n", mdev,
201 mdev->dev_id.as_uint32, dev->ib_dev.phys_port_cnt);
202
203 ret = ib_register_device(&dev->ib_dev, mana_ib_is_rnic(dev) ? "mana_%d" : "manae_%d",
204 gc->dev);
205 if (ret)
206 goto deallocate_pool;
207
208 dev_set_drvdata(&adev->dev, dev);
209
210 return 0;
211
212 deallocate_pool:
213 dma_pool_destroy(dev->av_pool);
214 deregister_net_notifier:
215 if (mana_ib_is_rnic(dev))
216 unregister_netdevice_notifier(&dev->nb);
217 destroy_rnic:
218 if (mana_ib_is_rnic(dev))
219 mana_ib_gd_destroy_rnic_adapter(dev);
220 destroy_eqs:
221 if (mana_ib_is_rnic(dev))
222 mana_ib_destroy_eqs(dev);
223 free_ib_device:
224 xa_destroy(&dev->qp_table_wq);
225 ib_dealloc_device(&dev->ib_dev);
226 return ret;
227 }
228
mana_ib_remove(struct auxiliary_device * adev)229 static void mana_ib_remove(struct auxiliary_device *adev)
230 {
231 struct mana_ib_dev *dev = dev_get_drvdata(&adev->dev);
232
233 ib_unregister_device(&dev->ib_dev);
234 dma_pool_destroy(dev->av_pool);
235 if (mana_ib_is_rnic(dev)) {
236 unregister_netdevice_notifier(&dev->nb);
237 mana_ib_gd_destroy_rnic_adapter(dev);
238 mana_ib_destroy_eqs(dev);
239 }
240 xa_destroy(&dev->qp_table_wq);
241 ib_dealloc_device(&dev->ib_dev);
242 }
243
244 static const struct auxiliary_device_id mana_id_table[] = {
245 { .name = "mana.rdma", },
246 { .name = "mana.eth", },
247 {},
248 };
249
250 MODULE_DEVICE_TABLE(auxiliary, mana_id_table);
251
252 static struct auxiliary_driver mana_driver = {
253 .probe = mana_ib_probe,
254 .remove = mana_ib_remove,
255 .id_table = mana_id_table,
256 };
257
258 module_auxiliary_driver(mana_driver);
259