Lines Matching refs:rx_queue
76 ef4_rx_buf_next(struct ef4_rx_queue *rx_queue, struct ef4_rx_buffer *rx_buf) in ef4_rx_buf_next() argument
78 if (unlikely(rx_buf == ef4_rx_buffer(rx_queue, rx_queue->ptr_mask))) in ef4_rx_buf_next()
79 return ef4_rx_buffer(rx_queue, 0); in ef4_rx_buf_next()
106 static struct page *ef4_reuse_page(struct ef4_rx_queue *rx_queue) in ef4_reuse_page() argument
108 struct ef4_nic *efx = rx_queue->efx; in ef4_reuse_page()
113 if (unlikely(!rx_queue->page_ring)) in ef4_reuse_page()
115 index = rx_queue->page_remove & rx_queue->page_ptr_mask; in ef4_reuse_page()
116 page = rx_queue->page_ring[index]; in ef4_reuse_page()
120 rx_queue->page_ring[index] = NULL; in ef4_reuse_page()
122 if (rx_queue->page_remove != rx_queue->page_add) in ef4_reuse_page()
123 ++rx_queue->page_remove; in ef4_reuse_page()
127 ++rx_queue->page_recycle_count; in ef4_reuse_page()
135 ++rx_queue->page_recycle_failed; in ef4_reuse_page()
152 static int ef4_init_rx_buffers(struct ef4_rx_queue *rx_queue, bool atomic) in ef4_init_rx_buffers() argument
154 struct ef4_nic *efx = rx_queue->efx; in ef4_init_rx_buffers()
164 page = ef4_reuse_page(rx_queue); in ef4_init_rx_buffers()
191 index = rx_queue->added_count & rx_queue->ptr_mask; in ef4_init_rx_buffers()
192 rx_buf = ef4_rx_buffer(rx_queue, index); in ef4_init_rx_buffers()
198 ++rx_queue->added_count; in ef4_init_rx_buffers()
227 static void ef4_free_rx_buffers(struct ef4_rx_queue *rx_queue, in ef4_free_rx_buffers() argument
236 rx_buf = ef4_rx_buf_next(rx_queue, rx_buf); in ef4_free_rx_buffers()
248 struct ef4_rx_queue *rx_queue = ef4_channel_get_rx_queue(channel); in ef4_recycle_rx_page() local
249 struct ef4_nic *efx = rx_queue->efx; in ef4_recycle_rx_page()
256 index = rx_queue->page_add & rx_queue->page_ptr_mask; in ef4_recycle_rx_page()
257 if (rx_queue->page_ring[index] == NULL) { in ef4_recycle_rx_page()
258 unsigned read_index = rx_queue->page_remove & in ef4_recycle_rx_page()
259 rx_queue->page_ptr_mask; in ef4_recycle_rx_page()
266 ++rx_queue->page_remove; in ef4_recycle_rx_page()
267 rx_queue->page_ring[index] = page; in ef4_recycle_rx_page()
268 ++rx_queue->page_add; in ef4_recycle_rx_page()
271 ++rx_queue->page_recycle_full; in ef4_recycle_rx_page()
276 static void ef4_fini_rx_buffer(struct ef4_rx_queue *rx_queue, in ef4_fini_rx_buffer() argument
285 ef4_unmap_rx_buffer(rx_queue->efx, rx_buf); in ef4_fini_rx_buffer()
286 ef4_free_rx_buffers(rx_queue, rx_buf, 1); in ef4_fini_rx_buffer()
296 struct ef4_rx_queue *rx_queue = ef4_channel_get_rx_queue(channel); in ef4_recycle_rx_pages() local
298 if (unlikely(!rx_queue->page_ring)) in ef4_recycle_rx_pages()
303 rx_buf = ef4_rx_buf_next(rx_queue, rx_buf); in ef4_recycle_rx_pages()
311 struct ef4_rx_queue *rx_queue = ef4_channel_get_rx_queue(channel); in ef4_discard_rx_packet() local
315 ef4_free_rx_buffers(rx_queue, rx_buf, n_frags); in ef4_discard_rx_packet()
331 void ef4_fast_push_rx_descriptors(struct ef4_rx_queue *rx_queue, bool atomic) in ef4_fast_push_rx_descriptors() argument
333 struct ef4_nic *efx = rx_queue->efx; in ef4_fast_push_rx_descriptors()
337 if (!rx_queue->refill_enabled) in ef4_fast_push_rx_descriptors()
341 fill_level = (rx_queue->added_count - rx_queue->removed_count); in ef4_fast_push_rx_descriptors()
342 EF4_BUG_ON_PARANOID(fill_level > rx_queue->efx->rxq_entries); in ef4_fast_push_rx_descriptors()
343 if (fill_level >= rx_queue->fast_fill_trigger) in ef4_fast_push_rx_descriptors()
347 if (unlikely(fill_level < rx_queue->min_fill)) { in ef4_fast_push_rx_descriptors()
349 rx_queue->min_fill = fill_level; in ef4_fast_push_rx_descriptors()
353 space = rx_queue->max_fill - fill_level; in ef4_fast_push_rx_descriptors()
356 netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev, in ef4_fast_push_rx_descriptors()
359 ef4_rx_queue_index(rx_queue), fill_level, in ef4_fast_push_rx_descriptors()
360 rx_queue->max_fill); in ef4_fast_push_rx_descriptors()
364 rc = ef4_init_rx_buffers(rx_queue, atomic); in ef4_fast_push_rx_descriptors()
367 if (rx_queue->added_count == rx_queue->removed_count) in ef4_fast_push_rx_descriptors()
368 ef4_schedule_slow_fill(rx_queue); in ef4_fast_push_rx_descriptors()
373 netif_vdbg(rx_queue->efx, rx_status, rx_queue->efx->net_dev, in ef4_fast_push_rx_descriptors()
375 "to level %d\n", ef4_rx_queue_index(rx_queue), in ef4_fast_push_rx_descriptors()
376 rx_queue->added_count - rx_queue->removed_count); in ef4_fast_push_rx_descriptors()
379 if (rx_queue->notified_count != rx_queue->added_count) in ef4_fast_push_rx_descriptors()
380 ef4_nic_notify_rx_desc(rx_queue); in ef4_fast_push_rx_descriptors()
385 struct ef4_rx_queue *rx_queue = from_timer(rx_queue, t, slow_fill); in ef4_rx_slow_fill() local
388 ef4_nic_generate_fill_event(rx_queue); in ef4_rx_slow_fill()
389 ++rx_queue->slow_fill_count; in ef4_rx_slow_fill()
392 static void ef4_rx_packet__check_len(struct ef4_rx_queue *rx_queue, in ef4_rx_packet__check_len() argument
396 struct ef4_nic *efx = rx_queue->efx; in ef4_rx_packet__check_len()
412 ef4_rx_queue_index(rx_queue), len, max_len, in ef4_rx_packet__check_len()
420 ef4_rx_queue_index(rx_queue), len, max_len); in ef4_rx_packet__check_len()
423 ef4_rx_queue_channel(rx_queue)->n_rx_overlength++; in ef4_rx_packet__check_len()
439 struct ef4_rx_queue *rx_queue; in ef4_rx_packet_gro() local
441 rx_queue = ef4_channel_get_rx_queue(channel); in ef4_rx_packet_gro()
442 ef4_free_rx_buffers(rx_queue, rx_buf, n_frags); in ef4_rx_packet_gro()
461 rx_buf = ef4_rx_buf_next(&channel->rx_queue, rx_buf); in ef4_rx_packet_gro()
467 skb_record_rx_queue(skb, channel->rx_queue.core_index); in ef4_rx_packet_gro()
512 rx_buf = ef4_rx_buf_next(&channel->rx_queue, rx_buf); in ef4_rx_mk_skb()
530 void ef4_rx_packet(struct ef4_rx_queue *rx_queue, unsigned int index, in ef4_rx_packet() argument
533 struct ef4_nic *efx = rx_queue->efx; in ef4_rx_packet()
534 struct ef4_channel *channel = ef4_rx_queue_channel(rx_queue); in ef4_rx_packet()
537 rx_queue->rx_packets++; in ef4_rx_packet()
539 rx_buf = ef4_rx_buffer(rx_queue, index); in ef4_rx_packet()
545 ef4_rx_packet__check_len(rx_queue, rx_buf, len); in ef4_rx_packet()
559 ef4_rx_queue_index(rx_queue), index, in ef4_rx_packet()
560 (index + n_frags - 1) & rx_queue->ptr_mask, len, in ef4_rx_packet()
596 rx_buf = ef4_rx_buf_next(rx_queue, rx_buf); in ef4_rx_packet()
606 rx_buf = ef4_rx_buffer(rx_queue, index); in ef4_rx_packet()
626 struct ef4_rx_queue *rx_queue; in ef4_rx_deliver() local
628 rx_queue = ef4_channel_get_rx_queue(channel); in ef4_rx_deliver()
629 ef4_free_rx_buffers(rx_queue, rx_buf, n_frags); in ef4_rx_deliver()
632 skb_record_rx_queue(skb, channel->rx_queue.core_index); in ef4_rx_deliver()
652 ef4_rx_buffer(&channel->rx_queue, channel->rx_pkt_index); in __ef4_rx_packet()
666 struct ef4_rx_queue *rx_queue; in __ef4_rx_packet() local
669 rx_queue = ef4_channel_get_rx_queue(channel); in __ef4_rx_packet()
670 ef4_free_rx_buffers(rx_queue, rx_buf, in __ef4_rx_packet()
686 int ef4_probe_rx_queue(struct ef4_rx_queue *rx_queue) in ef4_probe_rx_queue() argument
688 struct ef4_nic *efx = rx_queue->efx; in ef4_probe_rx_queue()
695 rx_queue->ptr_mask = entries - 1; in ef4_probe_rx_queue()
699 ef4_rx_queue_index(rx_queue), efx->rxq_entries, in ef4_probe_rx_queue()
700 rx_queue->ptr_mask); in ef4_probe_rx_queue()
703 rx_queue->buffer = kcalloc(entries, sizeof(*rx_queue->buffer), in ef4_probe_rx_queue()
705 if (!rx_queue->buffer) in ef4_probe_rx_queue()
708 rc = ef4_nic_probe_rx(rx_queue); in ef4_probe_rx_queue()
710 kfree(rx_queue->buffer); in ef4_probe_rx_queue()
711 rx_queue->buffer = NULL; in ef4_probe_rx_queue()
718 struct ef4_rx_queue *rx_queue) in ef4_init_rx_recycle_ring() argument
736 rx_queue->page_ring = kcalloc(page_ring_size, in ef4_init_rx_recycle_ring()
737 sizeof(*rx_queue->page_ring), GFP_KERNEL); in ef4_init_rx_recycle_ring()
738 if (!rx_queue->page_ring) in ef4_init_rx_recycle_ring()
739 rx_queue->page_ptr_mask = 0; in ef4_init_rx_recycle_ring()
741 rx_queue->page_ptr_mask = page_ring_size - 1; in ef4_init_rx_recycle_ring()
744 void ef4_init_rx_queue(struct ef4_rx_queue *rx_queue) in ef4_init_rx_queue() argument
746 struct ef4_nic *efx = rx_queue->efx; in ef4_init_rx_queue()
749 netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, in ef4_init_rx_queue()
750 "initialising RX queue %d\n", ef4_rx_queue_index(rx_queue)); in ef4_init_rx_queue()
753 rx_queue->added_count = 0; in ef4_init_rx_queue()
754 rx_queue->notified_count = 0; in ef4_init_rx_queue()
755 rx_queue->removed_count = 0; in ef4_init_rx_queue()
756 rx_queue->min_fill = -1U; in ef4_init_rx_queue()
757 ef4_init_rx_recycle_ring(efx, rx_queue); in ef4_init_rx_queue()
759 rx_queue->page_remove = 0; in ef4_init_rx_queue()
760 rx_queue->page_add = rx_queue->page_ptr_mask + 1; in ef4_init_rx_queue()
761 rx_queue->page_recycle_count = 0; in ef4_init_rx_queue()
762 rx_queue->page_recycle_failed = 0; in ef4_init_rx_queue()
763 rx_queue->page_recycle_full = 0; in ef4_init_rx_queue()
777 rx_queue->max_fill = max_fill; in ef4_init_rx_queue()
778 rx_queue->fast_fill_trigger = trigger; in ef4_init_rx_queue()
779 rx_queue->refill_enabled = true; in ef4_init_rx_queue()
782 ef4_nic_init_rx(rx_queue); in ef4_init_rx_queue()
785 void ef4_fini_rx_queue(struct ef4_rx_queue *rx_queue) in ef4_fini_rx_queue() argument
788 struct ef4_nic *efx = rx_queue->efx; in ef4_fini_rx_queue()
791 netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, in ef4_fini_rx_queue()
792 "shutting down RX queue %d\n", ef4_rx_queue_index(rx_queue)); in ef4_fini_rx_queue()
794 del_timer_sync(&rx_queue->slow_fill); in ef4_fini_rx_queue()
797 if (rx_queue->buffer) { in ef4_fini_rx_queue()
798 for (i = rx_queue->removed_count; i < rx_queue->added_count; in ef4_fini_rx_queue()
800 unsigned index = i & rx_queue->ptr_mask; in ef4_fini_rx_queue()
801 rx_buf = ef4_rx_buffer(rx_queue, index); in ef4_fini_rx_queue()
802 ef4_fini_rx_buffer(rx_queue, rx_buf); in ef4_fini_rx_queue()
807 for (i = 0; i <= rx_queue->page_ptr_mask; i++) { in ef4_fini_rx_queue()
808 struct page *page = rx_queue->page_ring[i]; in ef4_fini_rx_queue()
820 kfree(rx_queue->page_ring); in ef4_fini_rx_queue()
821 rx_queue->page_ring = NULL; in ef4_fini_rx_queue()
824 void ef4_remove_rx_queue(struct ef4_rx_queue *rx_queue) in ef4_remove_rx_queue() argument
826 netif_dbg(rx_queue->efx, drv, rx_queue->efx->net_dev, in ef4_remove_rx_queue()
827 "destroying RX queue %d\n", ef4_rx_queue_index(rx_queue)); in ef4_remove_rx_queue()
829 ef4_nic_remove_rx(rx_queue); in ef4_remove_rx_queue()
831 kfree(rx_queue->buffer); in ef4_remove_rx_queue()
832 rx_queue->buffer = NULL; in ef4_remove_rx_queue()