1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
2 /*
3 * Copyright(c) 2020 Intel Corporation.
4 *
5 */
6
7 /*
8 * This file contains HFI1 support for IPOIB SDMA functionality
9 */
10
11 #include <linux/log2.h>
12 #include <linux/circ_buf.h>
13
14 #include "sdma.h"
15 #include "verbs.h"
16 #include "trace_ibhdrs.h"
17 #include "ipoib.h"
18 #include "trace_tx.h"
19
20 /* Add a convenience helper */
21 #define CIRC_ADD(val, add, size) (((val) + (add)) & ((size) - 1))
22 #define CIRC_NEXT(val, size) CIRC_ADD(val, 1, size)
23 #define CIRC_PREV(val, size) CIRC_ADD(val, -1, size)
24
25 struct ipoib_txparms {
26 struct hfi1_devdata *dd;
27 struct rdma_ah_attr *ah_attr;
28 struct hfi1_ibport *ibp;
29 struct hfi1_ipoib_txq *txq;
30 union hfi1_ipoib_flow flow;
31 u32 dqpn;
32 u8 hdr_dwords;
33 u8 entropy;
34 };
35
36 static struct ipoib_txreq *
hfi1_txreq_from_idx(struct hfi1_ipoib_circ_buf * r,u32 idx)37 hfi1_txreq_from_idx(struct hfi1_ipoib_circ_buf *r, u32 idx)
38 {
39 return (struct ipoib_txreq *)(r->items + (idx << r->shift));
40 }
41
hfi1_ipoib_txreqs(const u64 sent,const u64 completed)42 static u32 hfi1_ipoib_txreqs(const u64 sent, const u64 completed)
43 {
44 return sent - completed;
45 }
46
hfi1_ipoib_used(struct hfi1_ipoib_txq * txq)47 static u64 hfi1_ipoib_used(struct hfi1_ipoib_txq *txq)
48 {
49 return hfi1_ipoib_txreqs(txq->tx_ring.sent_txreqs,
50 txq->tx_ring.complete_txreqs);
51 }
52
hfi1_ipoib_stop_txq(struct hfi1_ipoib_txq * txq)53 static void hfi1_ipoib_stop_txq(struct hfi1_ipoib_txq *txq)
54 {
55 trace_hfi1_txq_stop(txq);
56 if (atomic_inc_return(&txq->tx_ring.stops) == 1)
57 netif_stop_subqueue(txq->priv->netdev, txq->q_idx);
58 }
59
hfi1_ipoib_wake_txq(struct hfi1_ipoib_txq * txq)60 static void hfi1_ipoib_wake_txq(struct hfi1_ipoib_txq *txq)
61 {
62 trace_hfi1_txq_wake(txq);
63 if (atomic_dec_and_test(&txq->tx_ring.stops))
64 netif_wake_subqueue(txq->priv->netdev, txq->q_idx);
65 }
66
hfi1_ipoib_ring_hwat(struct hfi1_ipoib_txq * txq)67 static uint hfi1_ipoib_ring_hwat(struct hfi1_ipoib_txq *txq)
68 {
69 return min_t(uint, txq->priv->netdev->tx_queue_len,
70 txq->tx_ring.max_items - 1);
71 }
72
hfi1_ipoib_ring_lwat(struct hfi1_ipoib_txq * txq)73 static uint hfi1_ipoib_ring_lwat(struct hfi1_ipoib_txq *txq)
74 {
75 return min_t(uint, txq->priv->netdev->tx_queue_len,
76 txq->tx_ring.max_items) >> 1;
77 }
78
hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq * txq)79 static void hfi1_ipoib_check_queue_depth(struct hfi1_ipoib_txq *txq)
80 {
81 ++txq->tx_ring.sent_txreqs;
82 if (hfi1_ipoib_used(txq) >= hfi1_ipoib_ring_hwat(txq) &&
83 !atomic_xchg(&txq->tx_ring.ring_full, 1)) {
84 trace_hfi1_txq_full(txq);
85 hfi1_ipoib_stop_txq(txq);
86 }
87 }
88
hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq * txq)89 static void hfi1_ipoib_check_queue_stopped(struct hfi1_ipoib_txq *txq)
90 {
91 struct net_device *dev = txq->priv->netdev;
92
93 /* If shutting down just return as queue state is irrelevant */
94 if (unlikely(dev->reg_state != NETREG_REGISTERED))
95 return;
96
97 /*
98 * When the queue has been drained to less than half full it will be
99 * restarted.
100 * The size of the txreq ring is fixed at initialization.
101 * The tx queue len can be adjusted upward while the interface is
102 * running.
103 * The tx queue len can be large enough to overflow the txreq_ring.
104 * Use the minimum of the current tx_queue_len or the rings max txreqs
105 * to protect against ring overflow.
106 */
107 if (hfi1_ipoib_used(txq) < hfi1_ipoib_ring_lwat(txq) &&
108 atomic_xchg(&txq->tx_ring.ring_full, 0)) {
109 trace_hfi1_txq_xmit_unstopped(txq);
110 hfi1_ipoib_wake_txq(txq);
111 }
112 }
113
hfi1_ipoib_free_tx(struct ipoib_txreq * tx,int budget)114 static void hfi1_ipoib_free_tx(struct ipoib_txreq *tx, int budget)
115 {
116 struct hfi1_ipoib_dev_priv *priv = tx->txq->priv;
117
118 if (likely(!tx->sdma_status)) {
119 dev_sw_netstats_tx_add(priv->netdev, 1, tx->skb->len);
120 } else {
121 ++priv->netdev->stats.tx_errors;
122 dd_dev_warn(priv->dd,
123 "%s: Status = 0x%x pbc 0x%llx txq = %d sde = %d\n",
124 __func__, tx->sdma_status,
125 le64_to_cpu(tx->sdma_hdr->pbc), tx->txq->q_idx,
126 tx->txq->sde->this_idx);
127 }
128
129 napi_consume_skb(tx->skb, budget);
130 tx->skb = NULL;
131 sdma_txclean(priv->dd, &tx->txreq);
132 }
133
hfi1_ipoib_drain_tx_ring(struct hfi1_ipoib_txq * txq)134 static void hfi1_ipoib_drain_tx_ring(struct hfi1_ipoib_txq *txq)
135 {
136 struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;
137 int i;
138 struct ipoib_txreq *tx;
139
140 for (i = 0; i < tx_ring->max_items; i++) {
141 tx = hfi1_txreq_from_idx(tx_ring, i);
142 tx->complete = 0;
143 dev_kfree_skb_any(tx->skb);
144 tx->skb = NULL;
145 sdma_txclean(txq->priv->dd, &tx->txreq);
146 }
147 tx_ring->head = 0;
148 tx_ring->tail = 0;
149 tx_ring->complete_txreqs = 0;
150 tx_ring->sent_txreqs = 0;
151 tx_ring->avail = hfi1_ipoib_ring_hwat(txq);
152 }
153
hfi1_ipoib_poll_tx_ring(struct napi_struct * napi,int budget)154 static int hfi1_ipoib_poll_tx_ring(struct napi_struct *napi, int budget)
155 {
156 struct hfi1_ipoib_txq *txq =
157 container_of(napi, struct hfi1_ipoib_txq, napi);
158 struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;
159 u32 head = tx_ring->head;
160 u32 max_tx = tx_ring->max_items;
161 int work_done;
162 struct ipoib_txreq *tx = hfi1_txreq_from_idx(tx_ring, head);
163
164 trace_hfi1_txq_poll(txq);
165 for (work_done = 0; work_done < budget; work_done++) {
166 /* See hfi1_ipoib_sdma_complete() */
167 if (!smp_load_acquire(&tx->complete))
168 break;
169 tx->complete = 0;
170 trace_hfi1_tx_produce(tx, head);
171 hfi1_ipoib_free_tx(tx, budget);
172 head = CIRC_NEXT(head, max_tx);
173 tx = hfi1_txreq_from_idx(tx_ring, head);
174 }
175 tx_ring->complete_txreqs += work_done;
176
177 /* Finished freeing tx items so store the head value. */
178 smp_store_release(&tx_ring->head, head);
179
180 hfi1_ipoib_check_queue_stopped(txq);
181
182 if (work_done < budget)
183 napi_complete_done(napi, work_done);
184
185 return work_done;
186 }
187
hfi1_ipoib_sdma_complete(struct sdma_txreq * txreq,int status)188 static void hfi1_ipoib_sdma_complete(struct sdma_txreq *txreq, int status)
189 {
190 struct ipoib_txreq *tx = container_of(txreq, struct ipoib_txreq, txreq);
191
192 trace_hfi1_txq_complete(tx->txq);
193 tx->sdma_status = status;
194 /* see hfi1_ipoib_poll_tx_ring */
195 smp_store_release(&tx->complete, 1);
196 napi_schedule_irqoff(&tx->txq->napi);
197 }
198
hfi1_ipoib_build_ulp_payload(struct ipoib_txreq * tx,struct ipoib_txparms * txp)199 static int hfi1_ipoib_build_ulp_payload(struct ipoib_txreq *tx,
200 struct ipoib_txparms *txp)
201 {
202 struct hfi1_devdata *dd = txp->dd;
203 struct sdma_txreq *txreq = &tx->txreq;
204 struct sk_buff *skb = tx->skb;
205 int ret = 0;
206 int i;
207
208 if (skb_headlen(skb)) {
209 ret = sdma_txadd_kvaddr(dd, txreq, skb->data, skb_headlen(skb));
210 if (unlikely(ret))
211 return ret;
212 }
213
214 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
215 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
216
217 ret = sdma_txadd_page(dd,
218 txreq,
219 skb_frag_page(frag),
220 frag->bv_offset,
221 skb_frag_size(frag));
222 if (unlikely(ret))
223 break;
224 }
225
226 return ret;
227 }
228
hfi1_ipoib_build_tx_desc(struct ipoib_txreq * tx,struct ipoib_txparms * txp)229 static int hfi1_ipoib_build_tx_desc(struct ipoib_txreq *tx,
230 struct ipoib_txparms *txp)
231 {
232 struct hfi1_devdata *dd = txp->dd;
233 struct sdma_txreq *txreq = &tx->txreq;
234 struct hfi1_sdma_header *sdma_hdr = tx->sdma_hdr;
235 u16 pkt_bytes =
236 sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2) + tx->skb->len;
237 int ret;
238
239 ret = sdma_txinit(txreq, 0, pkt_bytes, hfi1_ipoib_sdma_complete);
240 if (unlikely(ret))
241 return ret;
242
243 /* add pbc + headers */
244 ret = sdma_txadd_kvaddr(dd,
245 txreq,
246 sdma_hdr,
247 sizeof(sdma_hdr->pbc) + (txp->hdr_dwords << 2));
248 if (unlikely(ret))
249 return ret;
250
251 /* add the ulp payload */
252 return hfi1_ipoib_build_ulp_payload(tx, txp);
253 }
254
hfi1_ipoib_build_ib_tx_headers(struct ipoib_txreq * tx,struct ipoib_txparms * txp)255 static void hfi1_ipoib_build_ib_tx_headers(struct ipoib_txreq *tx,
256 struct ipoib_txparms *txp)
257 {
258 struct hfi1_ipoib_dev_priv *priv = tx->txq->priv;
259 struct hfi1_sdma_header *sdma_hdr = tx->sdma_hdr;
260 struct sk_buff *skb = tx->skb;
261 struct hfi1_pportdata *ppd = ppd_from_ibp(txp->ibp);
262 struct rdma_ah_attr *ah_attr = txp->ah_attr;
263 struct ib_other_headers *ohdr;
264 struct ib_grh *grh;
265 u16 dwords;
266 u16 slid;
267 u16 dlid;
268 u16 lrh0;
269 u32 bth0;
270 u32 sqpn = (u32)(priv->netdev->dev_addr[1] << 16 |
271 priv->netdev->dev_addr[2] << 8 |
272 priv->netdev->dev_addr[3]);
273 u16 payload_dwords;
274 u8 pad_cnt;
275
276 pad_cnt = -skb->len & 3;
277
278 /* Includes ICRC */
279 payload_dwords = ((skb->len + pad_cnt) >> 2) + SIZE_OF_CRC;
280
281 /* header size in dwords LRH+BTH+DETH = (8+12+8)/4. */
282 txp->hdr_dwords = 7;
283
284 if (rdma_ah_get_ah_flags(ah_attr) & IB_AH_GRH) {
285 grh = &sdma_hdr->hdr.ibh.u.l.grh;
286 txp->hdr_dwords +=
287 hfi1_make_grh(txp->ibp,
288 grh,
289 rdma_ah_read_grh(ah_attr),
290 txp->hdr_dwords - LRH_9B_DWORDS,
291 payload_dwords);
292 lrh0 = HFI1_LRH_GRH;
293 ohdr = &sdma_hdr->hdr.ibh.u.l.oth;
294 } else {
295 lrh0 = HFI1_LRH_BTH;
296 ohdr = &sdma_hdr->hdr.ibh.u.oth;
297 }
298
299 lrh0 |= (rdma_ah_get_sl(ah_attr) & 0xf) << 4;
300 lrh0 |= (txp->flow.sc5 & 0xf) << 12;
301
302 dlid = opa_get_lid(rdma_ah_get_dlid(ah_attr), 9B);
303 if (dlid == be16_to_cpu(IB_LID_PERMISSIVE)) {
304 slid = be16_to_cpu(IB_LID_PERMISSIVE);
305 } else {
306 u16 lid = (u16)ppd->lid;
307
308 if (lid) {
309 lid |= rdma_ah_get_path_bits(ah_attr) &
310 ((1 << ppd->lmc) - 1);
311 slid = lid;
312 } else {
313 slid = be16_to_cpu(IB_LID_PERMISSIVE);
314 }
315 }
316
317 /* Includes ICRC */
318 dwords = txp->hdr_dwords + payload_dwords;
319
320 /* Build the lrh */
321 sdma_hdr->hdr.hdr_type = HFI1_PKT_TYPE_9B;
322 hfi1_make_ib_hdr(&sdma_hdr->hdr.ibh, lrh0, dwords, dlid, slid);
323
324 /* Build the bth */
325 bth0 = (IB_OPCODE_UD_SEND_ONLY << 24) | (pad_cnt << 20) | priv->pkey;
326
327 ohdr->bth[0] = cpu_to_be32(bth0);
328 ohdr->bth[1] = cpu_to_be32(txp->dqpn);
329 ohdr->bth[2] = cpu_to_be32(mask_psn((u32)txp->txq->tx_ring.sent_txreqs));
330
331 /* Build the deth */
332 ohdr->u.ud.deth[0] = cpu_to_be32(priv->qkey);
333 ohdr->u.ud.deth[1] = cpu_to_be32((txp->entropy <<
334 HFI1_IPOIB_ENTROPY_SHIFT) | sqpn);
335
336 /* Construct the pbc. */
337 sdma_hdr->pbc =
338 cpu_to_le64(create_pbc(ppd,
339 ib_is_sc5(txp->flow.sc5) <<
340 PBC_DC_INFO_SHIFT,
341 0,
342 sc_to_vlt(priv->dd, txp->flow.sc5),
343 dwords - SIZE_OF_CRC +
344 (sizeof(sdma_hdr->pbc) >> 2)));
345 }
346
hfi1_ipoib_send_dma_common(struct net_device * dev,struct sk_buff * skb,struct ipoib_txparms * txp)347 static struct ipoib_txreq *hfi1_ipoib_send_dma_common(struct net_device *dev,
348 struct sk_buff *skb,
349 struct ipoib_txparms *txp)
350 {
351 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
352 struct hfi1_ipoib_txq *txq = txp->txq;
353 struct ipoib_txreq *tx;
354 struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;
355 u32 tail = tx_ring->tail;
356 int ret;
357
358 if (unlikely(!tx_ring->avail)) {
359 u32 head;
360
361 if (hfi1_ipoib_used(txq) >= hfi1_ipoib_ring_hwat(txq))
362 /* This shouldn't happen with a stopped queue */
363 return ERR_PTR(-ENOMEM);
364 /* See hfi1_ipoib_poll_tx_ring() */
365 head = smp_load_acquire(&tx_ring->head);
366 tx_ring->avail =
367 min_t(u32, hfi1_ipoib_ring_hwat(txq),
368 CIRC_CNT(head, tail, tx_ring->max_items));
369 } else {
370 tx_ring->avail--;
371 }
372 tx = hfi1_txreq_from_idx(tx_ring, tail);
373 trace_hfi1_txq_alloc_tx(txq);
374
375 /* so that we can test if the sdma descriptors are there */
376 tx->txreq.num_desc = 0;
377 tx->txq = txq;
378 tx->skb = skb;
379 INIT_LIST_HEAD(&tx->txreq.list);
380
381 hfi1_ipoib_build_ib_tx_headers(tx, txp);
382
383 ret = hfi1_ipoib_build_tx_desc(tx, txp);
384 if (likely(!ret)) {
385 if (txq->flow.as_int != txp->flow.as_int) {
386 txq->flow.tx_queue = txp->flow.tx_queue;
387 txq->flow.sc5 = txp->flow.sc5;
388 txq->sde =
389 sdma_select_engine_sc(priv->dd,
390 txp->flow.tx_queue,
391 txp->flow.sc5);
392 trace_hfi1_flow_switch(txq);
393 }
394
395 return tx;
396 }
397
398 sdma_txclean(priv->dd, &tx->txreq);
399
400 return ERR_PTR(ret);
401 }
402
hfi1_ipoib_submit_tx_list(struct net_device * dev,struct hfi1_ipoib_txq * txq)403 static int hfi1_ipoib_submit_tx_list(struct net_device *dev,
404 struct hfi1_ipoib_txq *txq)
405 {
406 int ret;
407 u16 count_out;
408
409 ret = sdma_send_txlist(txq->sde,
410 iowait_get_ib_work(&txq->wait),
411 &txq->tx_list,
412 &count_out);
413 if (likely(!ret) || ret == -EBUSY || ret == -ECOMM)
414 return ret;
415
416 dd_dev_warn(txq->priv->dd, "cannot send skb tx list, err %d.\n", ret);
417
418 return ret;
419 }
420
hfi1_ipoib_flush_tx_list(struct net_device * dev,struct hfi1_ipoib_txq * txq)421 static int hfi1_ipoib_flush_tx_list(struct net_device *dev,
422 struct hfi1_ipoib_txq *txq)
423 {
424 int ret = 0;
425
426 if (!list_empty(&txq->tx_list)) {
427 /* Flush the current list */
428 ret = hfi1_ipoib_submit_tx_list(dev, txq);
429
430 if (unlikely(ret))
431 if (ret != -EBUSY)
432 ++dev->stats.tx_carrier_errors;
433 }
434
435 return ret;
436 }
437
hfi1_ipoib_submit_tx(struct hfi1_ipoib_txq * txq,struct ipoib_txreq * tx)438 static int hfi1_ipoib_submit_tx(struct hfi1_ipoib_txq *txq,
439 struct ipoib_txreq *tx)
440 {
441 int ret;
442
443 ret = sdma_send_txreq(txq->sde,
444 iowait_get_ib_work(&txq->wait),
445 &tx->txreq,
446 txq->pkts_sent);
447 if (likely(!ret)) {
448 txq->pkts_sent = true;
449 iowait_starve_clear(txq->pkts_sent, &txq->wait);
450 }
451
452 return ret;
453 }
454
hfi1_ipoib_send_dma_single(struct net_device * dev,struct sk_buff * skb,struct ipoib_txparms * txp)455 static int hfi1_ipoib_send_dma_single(struct net_device *dev,
456 struct sk_buff *skb,
457 struct ipoib_txparms *txp)
458 {
459 struct hfi1_ipoib_txq *txq = txp->txq;
460 struct hfi1_ipoib_circ_buf *tx_ring;
461 struct ipoib_txreq *tx;
462 int ret;
463
464 tx = hfi1_ipoib_send_dma_common(dev, skb, txp);
465 if (IS_ERR(tx)) {
466 int ret = PTR_ERR(tx);
467
468 dev_kfree_skb_any(skb);
469
470 if (ret == -ENOMEM)
471 ++dev->stats.tx_errors;
472 else
473 ++dev->stats.tx_carrier_errors;
474
475 return NETDEV_TX_OK;
476 }
477
478 tx_ring = &txq->tx_ring;
479 trace_hfi1_tx_consume(tx, tx_ring->tail);
480 /* consume tx */
481 smp_store_release(&tx_ring->tail, CIRC_NEXT(tx_ring->tail, tx_ring->max_items));
482 ret = hfi1_ipoib_submit_tx(txq, tx);
483 if (likely(!ret)) {
484 tx_ok:
485 trace_sdma_output_ibhdr(txq->priv->dd,
486 &tx->sdma_hdr->hdr,
487 ib_is_sc5(txp->flow.sc5));
488 hfi1_ipoib_check_queue_depth(txq);
489 return NETDEV_TX_OK;
490 }
491
492 txq->pkts_sent = false;
493
494 if (ret == -EBUSY || ret == -ECOMM)
495 goto tx_ok;
496
497 /* mark complete and kick napi tx */
498 smp_store_release(&tx->complete, 1);
499 napi_schedule(&tx->txq->napi);
500
501 ++dev->stats.tx_carrier_errors;
502
503 return NETDEV_TX_OK;
504 }
505
hfi1_ipoib_send_dma_list(struct net_device * dev,struct sk_buff * skb,struct ipoib_txparms * txp)506 static int hfi1_ipoib_send_dma_list(struct net_device *dev,
507 struct sk_buff *skb,
508 struct ipoib_txparms *txp)
509 {
510 struct hfi1_ipoib_txq *txq = txp->txq;
511 struct hfi1_ipoib_circ_buf *tx_ring;
512 struct ipoib_txreq *tx;
513
514 /* Has the flow change ? */
515 if (txq->flow.as_int != txp->flow.as_int) {
516 int ret;
517
518 trace_hfi1_flow_flush(txq);
519 ret = hfi1_ipoib_flush_tx_list(dev, txq);
520 if (unlikely(ret)) {
521 if (ret == -EBUSY)
522 ++dev->stats.tx_dropped;
523 dev_kfree_skb_any(skb);
524 return NETDEV_TX_OK;
525 }
526 }
527 tx = hfi1_ipoib_send_dma_common(dev, skb, txp);
528 if (IS_ERR(tx)) {
529 int ret = PTR_ERR(tx);
530
531 dev_kfree_skb_any(skb);
532
533 if (ret == -ENOMEM)
534 ++dev->stats.tx_errors;
535 else
536 ++dev->stats.tx_carrier_errors;
537
538 return NETDEV_TX_OK;
539 }
540
541 tx_ring = &txq->tx_ring;
542 trace_hfi1_tx_consume(tx, tx_ring->tail);
543 /* consume tx */
544 smp_store_release(&tx_ring->tail, CIRC_NEXT(tx_ring->tail, tx_ring->max_items));
545 list_add_tail(&tx->txreq.list, &txq->tx_list);
546
547 hfi1_ipoib_check_queue_depth(txq);
548
549 trace_sdma_output_ibhdr(txq->priv->dd,
550 &tx->sdma_hdr->hdr,
551 ib_is_sc5(txp->flow.sc5));
552
553 if (!netdev_xmit_more())
554 (void)hfi1_ipoib_flush_tx_list(dev, txq);
555
556 return NETDEV_TX_OK;
557 }
558
hfi1_ipoib_calc_entropy(struct sk_buff * skb)559 static u8 hfi1_ipoib_calc_entropy(struct sk_buff *skb)
560 {
561 if (skb_transport_header_was_set(skb)) {
562 u8 *hdr = (u8 *)skb_transport_header(skb);
563
564 return (hdr[0] ^ hdr[1] ^ hdr[2] ^ hdr[3]);
565 }
566
567 return (u8)skb_get_queue_mapping(skb);
568 }
569
hfi1_ipoib_send(struct net_device * dev,struct sk_buff * skb,struct ib_ah * address,u32 dqpn)570 int hfi1_ipoib_send(struct net_device *dev,
571 struct sk_buff *skb,
572 struct ib_ah *address,
573 u32 dqpn)
574 {
575 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
576 struct ipoib_txparms txp;
577 struct rdma_netdev *rn = netdev_priv(dev);
578
579 if (unlikely(skb->len > rn->mtu + HFI1_IPOIB_ENCAP_LEN)) {
580 dd_dev_warn(priv->dd, "packet len %d (> %d) too long to send, dropping\n",
581 skb->len,
582 rn->mtu + HFI1_IPOIB_ENCAP_LEN);
583 ++dev->stats.tx_dropped;
584 ++dev->stats.tx_errors;
585 dev_kfree_skb_any(skb);
586 return NETDEV_TX_OK;
587 }
588
589 txp.dd = priv->dd;
590 txp.ah_attr = &ibah_to_rvtah(address)->attr;
591 txp.ibp = to_iport(priv->device, priv->port_num);
592 txp.txq = &priv->txqs[skb_get_queue_mapping(skb)];
593 txp.dqpn = dqpn;
594 txp.flow.sc5 = txp.ibp->sl_to_sc[rdma_ah_get_sl(txp.ah_attr)];
595 txp.flow.tx_queue = (u8)skb_get_queue_mapping(skb);
596 txp.entropy = hfi1_ipoib_calc_entropy(skb);
597
598 if (netdev_xmit_more() || !list_empty(&txp.txq->tx_list))
599 return hfi1_ipoib_send_dma_list(dev, skb, &txp);
600
601 return hfi1_ipoib_send_dma_single(dev, skb, &txp);
602 }
603
604 /*
605 * hfi1_ipoib_sdma_sleep - ipoib sdma sleep function
606 *
607 * This function gets called from sdma_send_txreq() when there are not enough
608 * sdma descriptors available to send the packet. It adds Tx queue's wait
609 * structure to sdma engine's dmawait list to be woken up when descriptors
610 * become available.
611 */
hfi1_ipoib_sdma_sleep(struct sdma_engine * sde,struct iowait_work * wait,struct sdma_txreq * txreq,uint seq,bool pkts_sent)612 static int hfi1_ipoib_sdma_sleep(struct sdma_engine *sde,
613 struct iowait_work *wait,
614 struct sdma_txreq *txreq,
615 uint seq,
616 bool pkts_sent)
617 {
618 struct hfi1_ipoib_txq *txq =
619 container_of(wait->iow, struct hfi1_ipoib_txq, wait);
620
621 write_seqlock(&sde->waitlock);
622
623 if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED)) {
624 if (sdma_progress(sde, seq, txreq)) {
625 write_sequnlock(&sde->waitlock);
626 return -EAGAIN;
627 }
628
629 if (list_empty(&txreq->list))
630 /* came from non-list submit */
631 list_add_tail(&txreq->list, &txq->tx_list);
632 if (list_empty(&txq->wait.list)) {
633 struct hfi1_ibport *ibp = &sde->ppd->ibport_data;
634
635 if (!atomic_xchg(&txq->tx_ring.no_desc, 1)) {
636 trace_hfi1_txq_queued(txq);
637 hfi1_ipoib_stop_txq(txq);
638 }
639 ibp->rvp.n_dmawait++;
640 iowait_queue(pkts_sent, wait->iow, &sde->dmawait);
641 }
642
643 write_sequnlock(&sde->waitlock);
644 return -EBUSY;
645 }
646
647 write_sequnlock(&sde->waitlock);
648 return -EINVAL;
649 }
650
651 /*
652 * hfi1_ipoib_sdma_wakeup - ipoib sdma wakeup function
653 *
654 * This function gets called when SDMA descriptors becomes available and Tx
655 * queue's wait structure was previously added to sdma engine's dmawait list.
656 */
hfi1_ipoib_sdma_wakeup(struct iowait * wait,int reason)657 static void hfi1_ipoib_sdma_wakeup(struct iowait *wait, int reason)
658 {
659 struct hfi1_ipoib_txq *txq =
660 container_of(wait, struct hfi1_ipoib_txq, wait);
661
662 trace_hfi1_txq_wakeup(txq);
663 if (likely(txq->priv->netdev->reg_state == NETREG_REGISTERED))
664 iowait_schedule(wait, system_highpri_wq, WORK_CPU_UNBOUND);
665 }
666
hfi1_ipoib_flush_txq(struct work_struct * work)667 static void hfi1_ipoib_flush_txq(struct work_struct *work)
668 {
669 struct iowait_work *ioww =
670 container_of(work, struct iowait_work, iowork);
671 struct iowait *wait = iowait_ioww_to_iow(ioww);
672 struct hfi1_ipoib_txq *txq =
673 container_of(wait, struct hfi1_ipoib_txq, wait);
674 struct net_device *dev = txq->priv->netdev;
675
676 if (likely(dev->reg_state == NETREG_REGISTERED) &&
677 likely(!hfi1_ipoib_flush_tx_list(dev, txq)))
678 if (atomic_xchg(&txq->tx_ring.no_desc, 0))
679 hfi1_ipoib_wake_txq(txq);
680 }
681
hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv * priv)682 int hfi1_ipoib_txreq_init(struct hfi1_ipoib_dev_priv *priv)
683 {
684 struct net_device *dev = priv->netdev;
685 u32 tx_ring_size, tx_item_size;
686 struct hfi1_ipoib_circ_buf *tx_ring;
687 int i, j;
688
689 /*
690 * Ring holds 1 less than tx_ring_size
691 * Round up to next power of 2 in order to hold at least tx_queue_len
692 */
693 tx_ring_size = roundup_pow_of_two(dev->tx_queue_len + 1);
694 tx_item_size = roundup_pow_of_two(sizeof(struct ipoib_txreq));
695
696 priv->txqs = kcalloc_node(dev->num_tx_queues,
697 sizeof(struct hfi1_ipoib_txq),
698 GFP_KERNEL,
699 priv->dd->node);
700 if (!priv->txqs)
701 return -ENOMEM;
702
703 for (i = 0; i < dev->num_tx_queues; i++) {
704 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
705 struct ipoib_txreq *tx;
706
707 tx_ring = &txq->tx_ring;
708 iowait_init(&txq->wait,
709 0,
710 hfi1_ipoib_flush_txq,
711 NULL,
712 hfi1_ipoib_sdma_sleep,
713 hfi1_ipoib_sdma_wakeup,
714 NULL,
715 NULL);
716 txq->priv = priv;
717 txq->sde = NULL;
718 INIT_LIST_HEAD(&txq->tx_list);
719 atomic_set(&txq->tx_ring.stops, 0);
720 atomic_set(&txq->tx_ring.ring_full, 0);
721 atomic_set(&txq->tx_ring.no_desc, 0);
722 txq->q_idx = i;
723 txq->flow.tx_queue = 0xff;
724 txq->flow.sc5 = 0xff;
725 txq->pkts_sent = false;
726
727 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i),
728 priv->dd->node);
729
730 txq->tx_ring.items =
731 kvzalloc_node(array_size(tx_ring_size, tx_item_size),
732 GFP_KERNEL, priv->dd->node);
733 if (!txq->tx_ring.items)
734 goto free_txqs;
735
736 txq->tx_ring.max_items = tx_ring_size;
737 txq->tx_ring.shift = ilog2(tx_item_size);
738 txq->tx_ring.avail = hfi1_ipoib_ring_hwat(txq);
739 tx_ring = &txq->tx_ring;
740 for (j = 0; j < tx_ring_size; j++)
741 hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr =
742 kzalloc_node(sizeof(*tx->sdma_hdr),
743 GFP_KERNEL, priv->dd->node);
744
745 netif_napi_add_tx(dev, &txq->napi, hfi1_ipoib_poll_tx_ring);
746 }
747
748 return 0;
749
750 free_txqs:
751 for (i--; i >= 0; i--) {
752 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
753
754 netif_napi_del(&txq->napi);
755 tx_ring = &txq->tx_ring;
756 for (j = 0; j < tx_ring_size; j++)
757 kfree(hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr);
758 kvfree(tx_ring->items);
759 }
760
761 kfree(priv->txqs);
762 priv->txqs = NULL;
763 return -ENOMEM;
764 }
765
hfi1_ipoib_drain_tx_list(struct hfi1_ipoib_txq * txq)766 static void hfi1_ipoib_drain_tx_list(struct hfi1_ipoib_txq *txq)
767 {
768 struct sdma_txreq *txreq;
769 struct sdma_txreq *txreq_tmp;
770
771 list_for_each_entry_safe(txreq, txreq_tmp, &txq->tx_list, list) {
772 struct ipoib_txreq *tx =
773 container_of(txreq, struct ipoib_txreq, txreq);
774
775 list_del(&txreq->list);
776 sdma_txclean(txq->priv->dd, &tx->txreq);
777 dev_kfree_skb_any(tx->skb);
778 tx->skb = NULL;
779 txq->tx_ring.complete_txreqs++;
780 }
781
782 if (hfi1_ipoib_used(txq))
783 dd_dev_warn(txq->priv->dd,
784 "txq %d not empty found %u requests\n",
785 txq->q_idx,
786 hfi1_ipoib_txreqs(txq->tx_ring.sent_txreqs,
787 txq->tx_ring.complete_txreqs));
788 }
789
hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv * priv)790 void hfi1_ipoib_txreq_deinit(struct hfi1_ipoib_dev_priv *priv)
791 {
792 int i, j;
793
794 for (i = 0; i < priv->netdev->num_tx_queues; i++) {
795 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
796 struct hfi1_ipoib_circ_buf *tx_ring = &txq->tx_ring;
797
798 iowait_cancel_work(&txq->wait);
799 iowait_sdma_drain(&txq->wait);
800 hfi1_ipoib_drain_tx_list(txq);
801 netif_napi_del(&txq->napi);
802 hfi1_ipoib_drain_tx_ring(txq);
803 for (j = 0; j < tx_ring->max_items; j++)
804 kfree(hfi1_txreq_from_idx(tx_ring, j)->sdma_hdr);
805 kvfree(tx_ring->items);
806 }
807
808 kfree(priv->txqs);
809 priv->txqs = NULL;
810 }
811
hfi1_ipoib_napi_tx_enable(struct net_device * dev)812 void hfi1_ipoib_napi_tx_enable(struct net_device *dev)
813 {
814 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
815 int i;
816
817 for (i = 0; i < dev->num_tx_queues; i++) {
818 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
819
820 napi_enable(&txq->napi);
821 }
822 }
823
hfi1_ipoib_napi_tx_disable(struct net_device * dev)824 void hfi1_ipoib_napi_tx_disable(struct net_device *dev)
825 {
826 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
827 int i;
828
829 for (i = 0; i < dev->num_tx_queues; i++) {
830 struct hfi1_ipoib_txq *txq = &priv->txqs[i];
831
832 napi_disable(&txq->napi);
833 hfi1_ipoib_drain_tx_ring(txq);
834 }
835 }
836
hfi1_ipoib_tx_timeout(struct net_device * dev,unsigned int q)837 void hfi1_ipoib_tx_timeout(struct net_device *dev, unsigned int q)
838 {
839 struct hfi1_ipoib_dev_priv *priv = hfi1_ipoib_priv(dev);
840 struct hfi1_ipoib_txq *txq = &priv->txqs[q];
841
842 dd_dev_info(priv->dd, "timeout txq %p q %u stopped %u stops %d no_desc %d ring_full %d\n",
843 txq, q,
844 __netif_subqueue_stopped(dev, txq->q_idx),
845 atomic_read(&txq->tx_ring.stops),
846 atomic_read(&txq->tx_ring.no_desc),
847 atomic_read(&txq->tx_ring.ring_full));
848 dd_dev_info(priv->dd, "sde %p engine %u\n",
849 txq->sde,
850 txq->sde ? txq->sde->this_idx : 0);
851 dd_dev_info(priv->dd, "flow %x\n", txq->flow.as_int);
852 dd_dev_info(priv->dd, "sent %llu completed %llu used %llu\n",
853 txq->tx_ring.sent_txreqs, txq->tx_ring.complete_txreqs,
854 hfi1_ipoib_used(txq));
855 dd_dev_info(priv->dd, "tx_queue_len %u max_items %u\n",
856 dev->tx_queue_len, txq->tx_ring.max_items);
857 dd_dev_info(priv->dd, "head %u tail %u\n",
858 txq->tx_ring.head, txq->tx_ring.tail);
859 dd_dev_info(priv->dd, "wait queued %u\n",
860 !list_empty(&txq->wait.list));
861 dd_dev_info(priv->dd, "tx_list empty %u\n",
862 list_empty(&txq->tx_list));
863 }
864
865