Lines Matching refs:vring
185 struct mlxbf_tmfifo_vring *vring[2]; member
230 struct mlxbf_tmfifo_vring *vring; in mlxbf_tmfifo_free_vrings() local
234 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_free_vrings()
235 if (vring->va) { in mlxbf_tmfifo_free_vrings()
236 size = vring_size(vring->num, vring->align); in mlxbf_tmfifo_free_vrings()
238 vring->va, vring->dma); in mlxbf_tmfifo_free_vrings()
239 vring->va = NULL; in mlxbf_tmfifo_free_vrings()
240 if (vring->vq) { in mlxbf_tmfifo_free_vrings()
241 vring_del_virtqueue(vring->vq); in mlxbf_tmfifo_free_vrings()
242 vring->vq = NULL; in mlxbf_tmfifo_free_vrings()
252 struct mlxbf_tmfifo_vring *vring; in mlxbf_tmfifo_alloc_vrings() local
259 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_alloc_vrings()
260 vring->fifo = fifo; in mlxbf_tmfifo_alloc_vrings()
261 vring->num = MLXBF_TMFIFO_VRING_SIZE; in mlxbf_tmfifo_alloc_vrings()
262 vring->align = SMP_CACHE_BYTES; in mlxbf_tmfifo_alloc_vrings()
263 vring->index = i; in mlxbf_tmfifo_alloc_vrings()
264 vring->vdev_id = tm_vdev->vdev.id.device; in mlxbf_tmfifo_alloc_vrings()
267 size = vring_size(vring->num, vring->align); in mlxbf_tmfifo_alloc_vrings()
275 vring->va = va; in mlxbf_tmfifo_alloc_vrings()
276 vring->dma = dma; in mlxbf_tmfifo_alloc_vrings()
307 mlxbf_tmfifo_get_next_desc(struct mlxbf_tmfifo_vring *vring) in mlxbf_tmfifo_get_next_desc() argument
309 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_get_next_desc()
310 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_get_next_desc()
313 if (vring->next_avail == virtio16_to_cpu(vdev, vr->avail->idx)) in mlxbf_tmfifo_get_next_desc()
319 idx = vring->next_avail % vr->num; in mlxbf_tmfifo_get_next_desc()
324 vring->next_avail++; in mlxbf_tmfifo_get_next_desc()
330 static void mlxbf_tmfifo_release_desc(struct mlxbf_tmfifo_vring *vring, in mlxbf_tmfifo_release_desc() argument
333 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_release_desc()
334 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_release_desc()
352 static u32 mlxbf_tmfifo_get_pkt_len(struct mlxbf_tmfifo_vring *vring, in mlxbf_tmfifo_get_pkt_len() argument
355 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_get_pkt_len()
356 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_get_pkt_len()
370 static void mlxbf_tmfifo_release_pending_pkt(struct mlxbf_tmfifo_vring *vring) in mlxbf_tmfifo_release_pending_pkt() argument
375 if (vring->desc_head) { in mlxbf_tmfifo_release_pending_pkt()
376 desc_head = vring->desc_head; in mlxbf_tmfifo_release_pending_pkt()
377 len = vring->pkt_len; in mlxbf_tmfifo_release_pending_pkt()
379 desc_head = mlxbf_tmfifo_get_next_desc(vring); in mlxbf_tmfifo_release_pending_pkt()
380 len = mlxbf_tmfifo_get_pkt_len(vring, desc_head); in mlxbf_tmfifo_release_pending_pkt()
384 mlxbf_tmfifo_release_desc(vring, desc_head, len); in mlxbf_tmfifo_release_pending_pkt()
386 vring->pkt_len = 0; in mlxbf_tmfifo_release_pending_pkt()
387 vring->desc = NULL; in mlxbf_tmfifo_release_pending_pkt()
388 vring->desc_head = NULL; in mlxbf_tmfifo_release_pending_pkt()
391 static void mlxbf_tmfifo_init_net_desc(struct mlxbf_tmfifo_vring *vring, in mlxbf_tmfifo_init_net_desc() argument
394 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_init_net_desc()
403 mlxbf_tmfifo_get_next_pkt(struct mlxbf_tmfifo_vring *vring, bool is_rx) in mlxbf_tmfifo_get_next_pkt() argument
407 desc = mlxbf_tmfifo_get_next_desc(vring); in mlxbf_tmfifo_get_next_pkt()
408 if (desc && is_rx && vring->vdev_id == VIRTIO_ID_NET) in mlxbf_tmfifo_get_next_pkt()
409 mlxbf_tmfifo_init_net_desc(vring, desc, is_rx); in mlxbf_tmfifo_get_next_pkt()
411 vring->desc_head = desc; in mlxbf_tmfifo_get_next_pkt()
412 vring->desc = desc; in mlxbf_tmfifo_get_next_pkt()
434 struct mlxbf_tmfifo_vring *vring, in mlxbf_tmfifo_console_output_one() argument
437 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_console_output_one()
467 struct mlxbf_tmfifo_vring *vring) in mlxbf_tmfifo_console_output() argument
472 desc = mlxbf_tmfifo_get_next_desc(vring); in mlxbf_tmfifo_console_output()
475 len = mlxbf_tmfifo_get_pkt_len(vring, desc); in mlxbf_tmfifo_console_output()
479 mlxbf_tmfifo_release_desc(vring, desc, len); in mlxbf_tmfifo_console_output()
483 mlxbf_tmfifo_console_output_one(cons, vring, desc); in mlxbf_tmfifo_console_output()
484 mlxbf_tmfifo_release_desc(vring, desc, len); in mlxbf_tmfifo_console_output()
485 desc = mlxbf_tmfifo_get_next_desc(vring); in mlxbf_tmfifo_console_output()
581 static void mlxbf_tmfifo_rxtx_word(struct mlxbf_tmfifo_vring *vring, in mlxbf_tmfifo_rxtx_word() argument
585 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_rxtx_word()
586 struct mlxbf_tmfifo *fifo = vring->fifo; in mlxbf_tmfifo_rxtx_word()
597 if (vring->cur_len + sizeof(u64) <= len) { in mlxbf_tmfifo_rxtx_word()
600 memcpy(addr + vring->cur_len, &data, sizeof(u64)); in mlxbf_tmfifo_rxtx_word()
602 memcpy(&data, addr + vring->cur_len, sizeof(u64)); in mlxbf_tmfifo_rxtx_word()
603 vring->cur_len += sizeof(u64); in mlxbf_tmfifo_rxtx_word()
607 memcpy(addr + vring->cur_len, &data, in mlxbf_tmfifo_rxtx_word()
608 len - vring->cur_len); in mlxbf_tmfifo_rxtx_word()
610 memcpy(&data, addr + vring->cur_len, in mlxbf_tmfifo_rxtx_word()
611 len - vring->cur_len); in mlxbf_tmfifo_rxtx_word()
612 vring->cur_len = len; in mlxbf_tmfifo_rxtx_word()
627 static void mlxbf_tmfifo_rxtx_header(struct mlxbf_tmfifo_vring *vring, in mlxbf_tmfifo_rxtx_header() argument
631 struct mlxbf_tmfifo *fifo = vring->fifo; in mlxbf_tmfifo_rxtx_header()
665 if (vdev_id != vring->vdev_id) { in mlxbf_tmfifo_rxtx_header()
670 vring->desc = desc; in mlxbf_tmfifo_rxtx_header()
671 vring = &tm_dev2->vrings[MLXBF_TMFIFO_VRING_RX]; in mlxbf_tmfifo_rxtx_header()
674 vring->pkt_len = ntohs(hdr.len) + hdr_len; in mlxbf_tmfifo_rxtx_header()
677 hdr_len = (vring->vdev_id == VIRTIO_ID_NET) ? in mlxbf_tmfifo_rxtx_header()
679 vring->pkt_len = mlxbf_tmfifo_get_pkt_len(vring, desc); in mlxbf_tmfifo_rxtx_header()
680 hdr.type = (vring->vdev_id == VIRTIO_ID_NET) ? in mlxbf_tmfifo_rxtx_header()
682 hdr.len = htons(vring->pkt_len - hdr_len); in mlxbf_tmfifo_rxtx_header()
686 vring->cur_len = hdr_len; in mlxbf_tmfifo_rxtx_header()
687 vring->rem_len = vring->pkt_len; in mlxbf_tmfifo_rxtx_header()
688 fifo->vring[is_rx] = vring; in mlxbf_tmfifo_rxtx_header()
696 static bool mlxbf_tmfifo_rxtx_one_desc(struct mlxbf_tmfifo_vring *vring, in mlxbf_tmfifo_rxtx_one_desc() argument
699 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_rxtx_one_desc()
700 struct mlxbf_tmfifo *fifo = vring->fifo; in mlxbf_tmfifo_rxtx_one_desc()
707 vdev = &fifo->vdev[vring->vdev_id]->vdev; in mlxbf_tmfifo_rxtx_one_desc()
710 if (!vring->desc) { in mlxbf_tmfifo_rxtx_one_desc()
711 desc = mlxbf_tmfifo_get_next_pkt(vring, is_rx); in mlxbf_tmfifo_rxtx_one_desc()
715 desc = vring->desc; in mlxbf_tmfifo_rxtx_one_desc()
719 if (vring->pkt_len == 0) { in mlxbf_tmfifo_rxtx_one_desc()
720 mlxbf_tmfifo_rxtx_header(vring, desc, is_rx, &vring_change); in mlxbf_tmfifo_rxtx_one_desc()
731 if (len > vring->rem_len) in mlxbf_tmfifo_rxtx_one_desc()
732 len = vring->rem_len; in mlxbf_tmfifo_rxtx_one_desc()
735 if (vring->cur_len < len) { in mlxbf_tmfifo_rxtx_one_desc()
736 mlxbf_tmfifo_rxtx_word(vring, desc, is_rx, len); in mlxbf_tmfifo_rxtx_one_desc()
741 if (vring->cur_len == len) { in mlxbf_tmfifo_rxtx_one_desc()
742 vring->cur_len = 0; in mlxbf_tmfifo_rxtx_one_desc()
743 vring->rem_len -= len; in mlxbf_tmfifo_rxtx_one_desc()
746 if (vring->rem_len > 0 && in mlxbf_tmfifo_rxtx_one_desc()
754 mlxbf_tmfifo_release_pending_pkt(vring); in mlxbf_tmfifo_rxtx_one_desc()
756 fifo->vring[is_rx] = NULL; in mlxbf_tmfifo_rxtx_one_desc()
766 vring_interrupt(0, vring->vq); in mlxbf_tmfifo_rxtx_one_desc()
772 vring->desc = desc; in mlxbf_tmfifo_rxtx_one_desc()
778 static void mlxbf_tmfifo_rxtx(struct mlxbf_tmfifo_vring *vring, bool is_rx) in mlxbf_tmfifo_rxtx() argument
780 int avail = 0, devid = vring->vdev_id; in mlxbf_tmfifo_rxtx()
784 fifo = vring->fifo; in mlxbf_tmfifo_rxtx()
791 if (fifo->vring[is_rx] && fifo->vring[is_rx] != vring) in mlxbf_tmfifo_rxtx()
816 more = mlxbf_tmfifo_rxtx_one_desc(vring, is_rx, &avail); in mlxbf_tmfifo_rxtx()
825 struct mlxbf_tmfifo_vring *vring; in mlxbf_tmfifo_work_rxtx() local
835 vring = &tm_vdev->vrings[queue_id]; in mlxbf_tmfifo_work_rxtx()
836 if (vring->vq) in mlxbf_tmfifo_work_rxtx()
837 mlxbf_tmfifo_rxtx(vring, is_rx); in mlxbf_tmfifo_work_rxtx()
867 struct mlxbf_tmfifo_vring *vring = vq->priv; in mlxbf_tmfifo_virtio_notify() local
872 fifo = vring->fifo; in mlxbf_tmfifo_virtio_notify()
878 if (vring->index & BIT(0)) { in mlxbf_tmfifo_virtio_notify()
885 if (vring->vdev_id == VIRTIO_ID_CONSOLE) { in mlxbf_tmfifo_virtio_notify()
888 mlxbf_tmfifo_console_output(tm_vdev, vring); in mlxbf_tmfifo_virtio_notify()
926 struct mlxbf_tmfifo_vring *vring; in mlxbf_tmfifo_virtio_del_vqs() local
931 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_virtio_del_vqs()
934 if (vring->desc) in mlxbf_tmfifo_virtio_del_vqs()
935 mlxbf_tmfifo_release_pending_pkt(vring); in mlxbf_tmfifo_virtio_del_vqs()
936 vq = vring->vq; in mlxbf_tmfifo_virtio_del_vqs()
938 vring->vq = NULL; in mlxbf_tmfifo_virtio_del_vqs()
954 struct mlxbf_tmfifo_vring *vring; in mlxbf_tmfifo_virtio_find_vqs() local
966 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_virtio_find_vqs()
969 size = vring_size(vring->num, vring->align); in mlxbf_tmfifo_virtio_find_vqs()
970 memset(vring->va, 0, size); in mlxbf_tmfifo_virtio_find_vqs()
971 vq = vring_new_virtqueue(i, vring->num, vring->align, vdev, in mlxbf_tmfifo_virtio_find_vqs()
972 false, false, vring->va, in mlxbf_tmfifo_virtio_find_vqs()
981 vq->num_max = vring->num; in mlxbf_tmfifo_virtio_find_vqs()
984 vring->vq = vq; in mlxbf_tmfifo_virtio_find_vqs()
985 vq->priv = vring; in mlxbf_tmfifo_virtio_find_vqs()