| /linux-6.3-rc2/net/core/ |
| A D | xdp.c | 425 __xdp_return(xdpf->data, &xdpf->mem, false, NULL); in xdp_return_frame()
444 __xdp_return(xdpf->data, &xdpf->mem, true, NULL); in xdp_return_frame_rx_napi()
579 xdpf = addr; in xdp_convert_zc_to_xdp_frame()
580 memset(xdpf, 0, sizeof(*xdpf)); in xdp_convert_zc_to_xdp_frame()
582 addr += sizeof(*xdpf); in xdp_convert_zc_to_xdp_frame()
588 xdpf->headroom = 0; in xdp_convert_zc_to_xdp_frame()
594 return xdpf; in xdp_convert_zc_to_xdp_frame()
629 headroom = sizeof(*xdpf) + xdpf->headroom; in __xdp_build_skb_from_frame()
643 if (xdpf->metasize) in __xdp_build_skb_from_frame()
665 xdp_scrub_frame(xdpf); in __xdp_build_skb_from_frame()
[all …]
|
| /linux-6.3-rc2/kernel/bpf/ |
| A D | cpumap.c | 152 struct xdp_frame *xdpf; in __cpu_map_ring_cleanup() local
155 if (WARN_ON_ONCE(xdpf)) in __cpu_map_ring_cleanup()
156 xdp_return_frame(xdpf); in __cpu_map_ring_cleanup()
228 rxq.dev = xdpf->dev_rx; in cpu_map_bpf_prog_run_xdp()
229 rxq.mem = xdpf->mem; in cpu_map_bpf_prog_run_xdp()
239 xdp_return_frame(xdpf); in cpu_map_bpf_prog_run_xdp()
250 xdp_return_frame(xdpf); in cpu_map_bpf_prog_run_xdp()
260 xdp_return_frame(xdpf); in cpu_map_bpf_prog_run_xdp()
378 xdpf->dev_rx); in cpu_map_kthread_run()
754 xdpf->dev_rx = dev_rx; in cpu_map_enqueue()
[all …]
|
| A D | devmap.c | 347 xdp_return_frame_rx_napi(xdpf); in dev_map_bpf_prog_run()
349 frames[nframes++] = xdpf; in dev_map_bpf_prog_run()
358 xdp_return_frame_rx_napi(xdpf); in dev_map_bpf_prog_run()
379 prefetch(xdpf); in bq_xmit_all()
468 bq->q[bq->count++] = xdpf; in bq_enqueue()
481 xdp_frame_has_frags(xdpf))) in __xdp_enqueue()
546 xdp_frame_has_frags(xdpf))) in is_valid_dst()
557 struct xdp_frame *xdpf) in dev_map_enqueue_clone() argument
561 nxdpf = xdpf_clone(xdpf); in dev_map_enqueue_clone()
615 if (!is_valid_dst(dst, xdpf)) in dev_map_enqueue_multi()
[all …]
|
| /linux-6.3-rc2/include/net/ |
| A D | xdp.h | 249 struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf);
313 void xdp_return_frame(struct xdp_frame *xdpf);
314 void xdp_return_frame_rx_napi(struct xdp_frame *xdpf);
317 void xdp_return_frame_bulk(struct xdp_frame *xdpf,
328 struct xdp_mem_info *mem = &xdpf->mem; in xdp_release_frame()
336 if (likely(!xdp_frame_has_frags(xdpf))) in xdp_release_frame()
339 sinfo = xdp_get_shared_info_from_frame(xdpf); in xdp_release_frame()
346 __xdp_release_frame(xdpf->data, mem); in xdp_release_frame()
352 unsigned int len = xdpf->len; in xdp_get_frame_len()
354 if (likely(!xdp_frame_has_frags(xdpf))) in xdp_get_frame_len()
[all …]
|
| /linux-6.3-rc2/drivers/net/ethernet/mellanox/mlx5/core/en/ |
| A D | xdp.c | 66 struct xdp_frame *xdpf; in mlx5e_xmit_xdp_buff() local
70 xdpf = xdp_convert_buff_to_frame(xdp); in mlx5e_xmit_xdp_buff()
71 if (unlikely(!xdpf)) in mlx5e_xmit_xdp_buff()
74 xdptxd.data = xdpf->data; in mlx5e_xmit_xdp_buff()
75 xdptxd.len = xdpf->len; in mlx5e_xmit_xdp_buff()
95 xdp_return_frame(xdpf); in mlx5e_xmit_xdp_buff()
100 xdpi.frame.xdpf = xdpf; in mlx5e_xmit_xdp_buff()
120 dma_addr = page_pool_get_dma_addr(page) + (xdpf->data - (void *)xdpf); in mlx5e_xmit_xdp_buff()
668 xdptxd.data = xdpf->data; in mlx5e_xdp_xmit()
669 xdptxd.len = xdpf->len; in mlx5e_xdp_xmit()
[all …]
|
| /linux-6.3-rc2/drivers/net/ethernet/microchip/lan966x/ |
| A D | lan966x_xdp.c | 62 struct xdp_frame *xdpf = frames[i]; in lan966x_xdp_xmit() local
65 err = lan966x_fdma_xmit_xdpf(port, xdpf, NULL, true); in lan966x_xdp_xmit()
79 struct xdp_frame *xdpf; in lan966x_xdp_run() local
93 xdpf = xdp_convert_buff_to_frame(&xdp); in lan966x_xdp_run()
94 if (!xdpf) in lan966x_xdp_run()
97 return lan966x_fdma_xmit_xdpf(port, xdpf, page, false) ? in lan966x_xdp_run()
|
| A D | lan966x_fdma.c | 435 xdp_return_frame_rx_napi(dcb_buf->data.xdpf); in lan966x_fdma_tx_clear_buf()
703 struct xdp_frame *xdpf, in lan966x_fdma_xmit_xdpf() argument
727 if (xdpf->headroom < IFH_LEN_BYTES) { in lan966x_fdma_xmit_xdpf()
732 ifh = xdpf->data - IFH_LEN_BYTES; in lan966x_fdma_xmit_xdpf()
738 xdpf->data - IFH_LEN_BYTES, in lan966x_fdma_xmit_xdpf()
739 xdpf->len + IFH_LEN_BYTES, in lan966x_fdma_xmit_xdpf()
748 xdpf->len + IFH_LEN_BYTES, in lan966x_fdma_xmit_xdpf()
759 xdpf->len + IFH_LEN_BYTES, in lan966x_fdma_xmit_xdpf()
764 xdpf->len + IFH_LEN_BYTES, in lan966x_fdma_xmit_xdpf()
771 next_dcb_buf->data.xdpf = xdpf; in lan966x_fdma_xmit_xdpf()
[all …]
|
| /linux-6.3-rc2/drivers/net/ethernet/intel/ixgbe/ |
| A D | ixgbe_xsk.c | 104 struct xdp_frame *xdpf; in ixgbe_run_xdp_zc() local
125 xdpf = xdp_convert_buff_to_frame(xdp); in ixgbe_run_xdp_zc()
126 if (unlikely(!xdpf)) in ixgbe_run_xdp_zc()
131 result = ixgbe_xmit_xdp_ring(ring, xdpf); in ixgbe_run_xdp_zc()
423 tx_bi->xdpf = NULL; in ixgbe_xmit_zc()
454 xdp_return_frame(tx_bi->xdpf); in ixgbe_clean_xdp_tx_buffer()
481 if (tx_bi->xdpf) in ixgbe_clean_xdp_tx_irq()
486 tx_bi->xdpf = NULL; in ixgbe_clean_xdp_tx_irq()
560 if (tx_bi->xdpf) in ixgbe_xsk_clean_tx_ring()
565 tx_bi->xdpf = NULL; in ixgbe_xsk_clean_tx_ring()
|
| A D | ixgbe_txrx_common.h | 17 struct xdp_frame *xdpf);
|
| /linux-6.3-rc2/drivers/net/ethernet/sfc/siena/ |
| A D | tx.c | 213 struct xdp_frame *xdpf; in efx_siena_xdp_tx_buffers() local
255 xdpf = xdpfs[i]; in efx_siena_xdp_tx_buffers()
263 len = xdpf->len; in efx_siena_xdp_tx_buffers()
267 xdpf->data, len, in efx_siena_xdp_tx_buffers()
274 tx_buffer->xdpf = xdpf; in efx_siena_xdp_tx_buffers()
|
| A D | rx.c | 257 struct xdp_frame *xdpf; in efx_do_xdp() local
312 xdpf = xdp_convert_buff_to_frame(&xdp); in efx_do_xdp()
313 err = efx_siena_xdp_tx_buffers(efx, 1, &xdpf, true); in efx_do_xdp()
|
| /linux-6.3-rc2/drivers/net/ethernet/freescale/dpaa/ |
| A D | dpaa_eth.c | 2232 memmove(aligned_data, xdpf->data, xdpf->len); in dpaa_a050385_wa_xdpf()
2259 memcpy(new_buff + headroom, xdpf->data, xdpf->len); in dpaa_a050385_wa_xdpf()
2266 new_xdpf->len = xdpf->len; in dpaa_a050385_wa_xdpf()
2527 buff_start = xdpf->data - xdpf->headroom; in dpaa_xdp_xmit_frame()
2534 swbp->xdpf = xdpf; in dpaa_xdp_xmit_frame()
2539 qm_fd_set_contig(&fd, xdpf->headroom, xdpf->len); in dpaa_xdp_xmit_frame()
2542 xdpf->headroom + xdpf->len, in dpaa_xdp_xmit_frame()
2575 struct xdp_frame *xdpf; in dpaa_run_xdp() local
2624 if (unlikely(!xdpf)) { in dpaa_run_xdp()
3051 struct xdp_frame *xdpf; in dpaa_xdp_xmit() local
[all …]
|
| /linux-6.3-rc2/drivers/net/ethernet/fungible/funeth/ |
| A D | funeth_tx.c | 501 xdp_return_frame(q->info[reclaim_idx].xdpf); in fun_xdpq_clean()
515 bool fun_xdp_tx(struct funeth_txq *q, struct xdp_frame *xdpf) in fun_xdp_tx() argument
517 unsigned int idx, nfrags = 1, ndesc = 1, tot_len = xdpf->len; in fun_xdp_tx()
526 if (unlikely(xdp_frame_has_frags(xdpf))) { in fun_xdp_tx()
527 si = xdp_get_shared_info_from_frame(xdpf); in fun_xdp_tx()
528 tot_len = xdp_get_frame_len(xdpf); in fun_xdp_tx()
540 if (unlikely(fun_map_pkt(q->dma_dev, si, xdpf->data, xdpf->len, dma, in fun_xdp_tx()
559 q->info[idx].xdpf = xdpf; in fun_xdp_tx()
619 xdp_return_frame(q->info[idx].xdpf); in fun_xdpq_purge()
|
| A D | funeth_txrx.h | 100 struct xdp_frame *xdpf; /* associated XDP frame (XDP path) */ member
249 bool fun_xdp_tx(struct funeth_txq *q, struct xdp_frame *xdpf);
|
| /linux-6.3-rc2/drivers/net/ethernet/ti/ |
| A D | cpsw_priv.c | 55 struct xdp_frame *xdpf; in cpsw_tx_handler() local
62 xdpf = cpsw_handle_to_xdpf(token); in cpsw_tx_handler()
63 xmeta = (void *)xdpf + CPSW_XMETA_OFFSET; in cpsw_tx_handler()
66 xdp_return_frame(xdpf); in cpsw_tx_handler()
1309 dma, xdpf->len, port); in cpsw_xdp_tx_frame()
1311 if (sizeof(*xmeta) > xdpf->headroom) in cpsw_xdp_tx_frame()
1315 xdpf->data, xdpf->len, port); in cpsw_xdp_tx_frame()
1330 struct xdp_frame *xdpf; in cpsw_run_xdp() local
1347 xdpf = xdp_convert_buff_to_frame(xdp); in cpsw_run_xdp()
1348 if (unlikely(!xdpf)) in cpsw_run_xdp()
[all …]
|
| /linux-6.3-rc2/drivers/net/ethernet/aquantia/atlantic/ |
| A D | aq_ring.c | 349 } else if (buff->xdpf) { in aq_ring_tx_clean()
354 xdp_return_frame_rx_napi(buff->xdpf); in aq_ring_tx_clean()
359 buff->xdpf = NULL; in aq_ring_tx_clean()
406 struct xdp_frame *xdpf = frames[i]; in aq_xdp_xmit() local
422 struct xdp_frame *xdpf; in aq_xdp_run_prog() local
446 xdpf = xdp_convert_buff_to_frame(xdp); in aq_xdp_run_prog()
447 if (unlikely(!xdpf)) in aq_xdp_run_prog()
449 skb = xdp_build_skb_from_frame(xdpf, aq_nic->ndev); in aq_xdp_run_prog()
458 xdpf = xdp_convert_buff_to_frame(xdp); in aq_xdp_run_prog()
459 if (unlikely(!xdpf)) in aq_xdp_run_prog()
[all …]
|
| A D | aq_nic.c | 576 struct xdp_frame *xdpf, in aq_nic_map_xdp() argument
592 sinfo = xdp_get_shared_info_from_frame(xdpf); in aq_nic_map_xdp()
593 total_len = xdpf->len; in aq_nic_map_xdp()
595 if (xdp_frame_has_frags(xdpf)) { in aq_nic_map_xdp()
599 dx_buff->pa = dma_map_single(dev, xdpf->data, dx_buff->len, in aq_nic_map_xdp()
649 dx_buff->xdpf = xdpf; in aq_nic_map_xdp()
800 dx_buff->xdpf = NULL; in aq_nic_map_skb()
830 struct xdp_frame *xdpf) in aq_nic_xmit_xdpf() argument
840 if (xdp_frame_has_frags(xdpf)) { in aq_nic_xmit_xdpf()
841 sinfo = xdp_get_shared_info_from_frame(xdpf); in aq_nic_xmit_xdpf()
[all …]
|
| /linux-6.3-rc2/drivers/net/ethernet/socionext/ |
| A D | netsec.c | 272 struct xdp_frame *xdpf; member
675 bytes += desc->xdpf->len; in netsec_clean_tx_dring()
677 xdp_return_frame_rx_napi(desc->xdpf); in netsec_clean_tx_dring()
821 dring->desc[idx].xdpf = buf; in netsec_set_tx_de()
829 struct xdp_frame *xdpf, bool is_ndo) in netsec_xdp_queue_one() argument
851 dma_handle = dma_map_single(priv->dev, xdpf->data, xdpf->len, in netsec_xdp_queue_one()
866 sizeof(*xdpf); in netsec_xdp_queue_one()
873 tx_desc.addr = xdpf->data; in netsec_xdp_queue_one()
874 tx_desc.len = xdpf->len; in netsec_xdp_queue_one()
888 if (unlikely(!xdpf)) in netsec_xdp_xmit_back()
[all …]
|
| /linux-6.3-rc2/drivers/net/ethernet/sfc/ |
| A D | tx.c | 426 struct xdp_frame *xdpf; in efx_xdp_tx_buffers() local
468 xdpf = xdpfs[i]; in efx_xdp_tx_buffers()
476 len = xdpf->len; in efx_xdp_tx_buffers()
480 xdpf->data, len, in efx_xdp_tx_buffers()
487 tx_buffer->xdpf = xdpf; in efx_xdp_tx_buffers()
|
| A D | rx.c | 257 struct xdp_frame *xdpf; in efx_do_xdp() local
312 xdpf = xdp_convert_buff_to_frame(&xdp); in efx_do_xdp()
313 err = efx_xdp_tx_buffers(efx, 1, &xdpf, true); in efx_do_xdp()
|
| /linux-6.3-rc2/drivers/net/ethernet/qlogic/qede/ |
| A D | qede_fp.c | 332 xdp->xdpf = xdpf; in qede_xdp_xmit()
345 struct xdp_frame *xdpf; in qede_xdp_transmit() local
362 xdpf = frames[i]; in qede_xdp_transmit()
364 mapping = dma_map_single(dmadev, xdpf->data, xdpf->len, in qede_xdp_transmit()
370 NULL, xdpf))) in qede_xdp_transmit()
404 struct xdp_frame *xdpf; in qede_xdp_tx_int() local
412 xdpf = xdp_info->xdpf; in qede_xdp_tx_int()
414 if (xdpf) { in qede_xdp_tx_int()
415 dma_unmap_single(dev, xdp_info->mapping, xdpf->len, in qede_xdp_tx_int()
417 xdp_return_frame(xdpf); in qede_xdp_tx_int()
[all …]
|
| /linux-6.3-rc2/drivers/net/ethernet/engleder/ |
| A D | tsnep_main.c | 518 len = xdpf->len; in tsnep_xdp_tx_map()
523 xdpf->data; in tsnep_xdp_tx_map()
531 virt_to_page(xdpf->data); in tsnep_xdp_tx_map()
536 dma += sizeof(*xdpf) + xdpf->headroom; in tsnep_xdp_tx_map()
568 if (unlikely(xdp_frame_has_frags(xdpf))) in tsnep_xdp_xmit_frame_ring()
579 entry->xdpf = xdpf; in tsnep_xdp_xmit_frame_ring()
584 entry->xdpf = NULL; in tsnep_xdp_xmit_frame_ring()
612 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp); in tsnep_xdp_xmit_back() local
615 if (unlikely(!xdpf)) in tsnep_xdp_xmit_back()
662 xdp_frame_has_frags(entry->xdpf)) in tsnep_tx_poll()
[all …]
|
| /linux-6.3-rc2/drivers/net/ethernet/broadcom/bnxt/ |
| A D | bnxt_xdp.c | 117 struct xdp_frame *xdpf) in __bnxt_xmit_xdp_redirect() argument
123 tx_buf->xdpf = xdpf; in __bnxt_xmit_xdp_redirect()
148 xdp_return_frame(tx_buf->xdpf); in bnxt_tx_int_xdp()
150 tx_buf->xdpf = NULL; in bnxt_tx_int_xdp()
|
| /linux-6.3-rc2/drivers/net/ethernet/intel/i40e/ |
| A D | i40e_xsk.c | 572 xdp_return_frame(tx_bi->xdpf); in i40e_clean_xdp_tx_buffer()
612 if (tx_bi->xdpf) { in i40e_clean_xdp_tx_irq()
614 tx_bi->xdpf = NULL; in i40e_clean_xdp_tx_irq()
711 if (tx_bi->xdpf) in i40e_xsk_clean_tx_ring()
716 tx_bi->xdpf = NULL; in i40e_xsk_clean_tx_ring()
|
| A D | i40e_txrx.c | 763 xdp_return_frame(tx_buffer->xdpf); in i40e_unmap_and_free_tx_resource()
971 xdp_return_frame(tx_buf->xdpf); in i40e_clean_tx_irq()
2274 static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf,
2281 if (unlikely(!xdpf)) in i40e_xmit_xdp_tx_ring()
2284 return i40e_xmit_xdp_ring(xdpf, xdp_ring); in i40e_xmit_xdp_tx_ring()
3736 void *data = xdpf->data; in i40e_xmit_xdp_ring()
3737 u32 size = xdpf->len; in i40e_xmit_xdp_ring()
3744 tx_head->bytecount = xdp_get_frame_len(xdpf); in i40e_xmit_xdp_ring()
3746 tx_head->xdpf = xdpf; in i40e_xmit_xdp_ring()
3979 struct xdp_frame *xdpf = frames[i]; in i40e_xdp_xmit() local
[all …]
|