Lines Matching refs:xdp
58 offsetof(struct xdp_buff_xsk, xdp.data));
589 void libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp);
599 const struct libeth_xdp_buff *xdp) in libeth_xdp_tx_queue_head() argument
601 const struct xdp_buff *base = &xdp->base; in libeth_xdp_tx_queue_head()
604 .data = xdp->data, in libeth_xdp_tx_queue_head()
605 .len_fl = (base->data_end - xdp->data) | LIBETH_XDP_TX_FIRST, in libeth_xdp_tx_queue_head()
606 .soff = xdp_data_hard_end(base) - xdp->data, in libeth_xdp_tx_queue_head()
638 struct libeth_xdp_buff *xdp, in libeth_xdp_tx_queue_bulk() argument
648 libeth_xdp_return_buff_slow(xdp); in libeth_xdp_tx_queue_bulk()
652 if (!libeth_xdp_tx_queue_head(bq, xdp)) in libeth_xdp_tx_queue_bulk()
655 sinfo = xdp_get_shared_info_from_buff(&xdp->base); in libeth_xdp_tx_queue_bulk()
670 xdp->data = NULL; in libeth_xdp_tx_queue_bulk()
1248 #define libeth_xdp_return_buff(xdp) __libeth_xdp_return_buff(xdp, true) argument
1250 static inline void __libeth_xdp_return_buff(struct libeth_xdp_buff *xdp, in __libeth_xdp_return_buff() argument
1253 if (!xdp_buff_has_frags(&xdp->base)) in __libeth_xdp_return_buff()
1256 libeth_xdp_return_frags(xdp_get_shared_info_from_buff(&xdp->base), in __libeth_xdp_return_buff()
1260 libeth_xdp_return_va(xdp->data, napi); in __libeth_xdp_return_buff()
1261 xdp->data = NULL; in __libeth_xdp_return_buff()
1264 bool libeth_xdp_buff_add_frag(struct libeth_xdp_buff *xdp,
1279 static inline void libeth_xdp_prepare_buff(struct libeth_xdp_buff *xdp, in libeth_xdp_prepare_buff() argument
1286 static_assert(offsetofend(typeof(xdp->base), flags) - in libeth_xdp_prepare_buff()
1287 offsetof(typeof(xdp->base), frame_sz) == in libeth_xdp_prepare_buff()
1290 *(u64 *)&xdp->base.frame_sz = fqe->truesize; in libeth_xdp_prepare_buff()
1292 xdp_init_buff(&xdp->base, fqe->truesize, xdp->base.rxq); in libeth_xdp_prepare_buff()
1294 xdp_prepare_buff(&xdp->base, page_address(page) + fqe->offset, in libeth_xdp_prepare_buff()
1312 static inline bool libeth_xdp_process_buff(struct libeth_xdp_buff *xdp, in libeth_xdp_process_buff() argument
1319 if (xdp->data) in libeth_xdp_process_buff()
1320 return libeth_xdp_buff_add_frag(xdp, fqe, len); in libeth_xdp_process_buff()
1322 libeth_xdp_prepare_buff(xdp, fqe, len); in libeth_xdp_process_buff()
1324 prefetch(xdp->data); in libeth_xdp_process_buff()
1339 const struct libeth_xdp_buff *xdp) in libeth_xdp_buff_stats_frags() argument
1343 sinfo = xdp_get_shared_info_from_buff(&xdp->base); in libeth_xdp_buff_stats_frags()
1349 struct libeth_xdp_buff *xdp,
1364 __libeth_xdp_run_prog(struct libeth_xdp_buff *xdp, in __libeth_xdp_run_prog() argument
1369 act = bpf_prog_run_xdp(bq->prog, &xdp->base); in __libeth_xdp_run_prog()
1377 libeth_xdp_return_buff(xdp); in __libeth_xdp_run_prog()
1383 if (unlikely(xdp_do_redirect(bq->dev, &xdp->base, bq->prog))) in __libeth_xdp_run_prog()
1386 xdp->data = NULL; in __libeth_xdp_run_prog()
1394 return libeth_xdp_prog_exception(bq, xdp, act, 0); in __libeth_xdp_run_prog()
1412 __libeth_xdp_run_flush(struct libeth_xdp_buff *xdp, in __libeth_xdp_run_flush() argument
1414 u32 (*run)(struct libeth_xdp_buff *xdp, in __libeth_xdp_run_flush() argument
1417 struct libeth_xdp_buff *xdp, in __libeth_xdp_run_flush()
1426 act = run(xdp, bq); in __libeth_xdp_run_flush()
1427 if (act == LIBETH_XDP_TX && unlikely(!queue(bq, xdp, flush_bulk))) in __libeth_xdp_run_flush()
1448 #define libeth_xdp_run_prog(xdp, bq, fl) \ argument
1449 (__libeth_xdp_run_flush(xdp, bq, __libeth_xdp_run_prog, \
1478 __libeth_xdp_run_pass(struct libeth_xdp_buff *xdp, in __libeth_xdp_run_pass() argument
1481 void (*prep)(struct libeth_xdp_buff *xdp, in __libeth_xdp_run_pass() argument
1483 bool (*run)(struct libeth_xdp_buff *xdp, in __libeth_xdp_run_pass() argument
1486 const struct libeth_xdp_buff *xdp, in __libeth_xdp_run_pass() argument
1491 rs->bytes += xdp->base.data_end - xdp->data; in __libeth_xdp_run_pass()
1494 if (xdp_buff_has_frags(&xdp->base)) in __libeth_xdp_run_pass()
1495 libeth_xdp_buff_stats_frags(rs, xdp); in __libeth_xdp_run_pass()
1498 prep(xdp, md); in __libeth_xdp_run_pass()
1503 if (!run(xdp, bq)) in __libeth_xdp_run_pass()
1507 skb = xdp_build_skb_from_buff(&xdp->base); in __libeth_xdp_run_pass()
1509 libeth_xdp_return_buff_slow(xdp); in __libeth_xdp_run_pass()
1513 xdp->data = NULL; in __libeth_xdp_run_pass()
1515 if (unlikely(!populate(skb, xdp, rs))) { in __libeth_xdp_run_pass()
1523 static inline void libeth_xdp_prep_desc(struct libeth_xdp_buff *xdp, in libeth_xdp_prep_desc() argument
1526 xdp->desc = desc; in libeth_xdp_prep_desc()
1542 #define libeth_xdp_run_pass(xdp, bq, napi, ss, desc, run, populate) \ argument
1543 __libeth_xdp_run_pass(xdp, bq, napi, ss, desc, libeth_xdp_prep_desc, \
1623 __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, xdp)
1649 bool __LIBETH_XDP_DEFINE_RUN_PROG(name, flush, xdp)
1652 name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq) \
1654 return libeth_##pfx##_run_prog(xdp, bq, flush); \
1664 void __LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate, xdp)
1667 name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq, \
1671 return libeth_##pfx##_run_pass(xdp, bq, napi, ss, desc, run, \
1696 __LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize, xdp)
1719 #define libeth_xdp_buff_to_rq(xdp, type, member) \ argument
1720 container_of_const((xdp)->base.rxq, type, member)
1755 void libeth_xsk_buff_free_slow(struct libeth_xdp_buff *xdp);