Lines Matching refs:_vq
20 #define BAD_RING(_vq, fmt, args...) \ argument
22 dev_err(&(_vq)->vq.vdev->dev, \
23 "%s:"fmt, (_vq)->vq.name, ##args); \
27 #define START_USE(_vq) \ argument
29 if ((_vq)->in_use) \
31 (_vq)->vq.name, (_vq)->in_use); \
32 (_vq)->in_use = __LINE__; \
34 #define END_USE(_vq) \ argument
35 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
36 #define LAST_ADD_TIME_UPDATE(_vq) \ argument
41 if ((_vq)->last_add_time_valid) \
43 (_vq)->last_add_time)) > 100); \
44 (_vq)->last_add_time = now; \
45 (_vq)->last_add_time_valid = true; \
47 #define LAST_ADD_TIME_CHECK(_vq) \ argument
49 if ((_vq)->last_add_time_valid) { \
51 (_vq)->last_add_time)) > 100); \
54 #define LAST_ADD_TIME_INVALID(_vq) \ argument
55 ((_vq)->last_add_time_valid = false)
57 #define BAD_RING(_vq, fmt, args...) \ argument
59 dev_err(&_vq->vq.vdev->dev, \
60 "%s:"fmt, (_vq)->vq.name, ##args); \
61 (_vq)->broken = true; \
228 static void vring_free(struct virtqueue *_vq);
234 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq) argument
471 static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq, in alloc_indirect_split() argument
490 desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1); in alloc_indirect_split()
523 static inline int virtqueue_add_split(struct virtqueue *_vq, in virtqueue_add_split() argument
532 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_add_split()
556 desc = alloc_indirect_split(_vq, total_sg, gfp); in virtqueue_add_split()
599 i = virtqueue_add_desc_split(_vq, desc, i, addr, sg->length, in virtqueue_add_split()
614 i = virtqueue_add_desc_split(_vq, desc, i, addr, in virtqueue_add_split()
622 desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT); in virtqueue_add_split()
635 virtqueue_add_desc_split(_vq, vq->split.vring.desc, in virtqueue_add_split()
661 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head); in virtqueue_add_split()
667 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, in virtqueue_add_split()
677 virtqueue_kick(_vq); in virtqueue_add_split()
694 i = virtio16_to_cpu(_vq->vdev, desc[i].next); in virtqueue_add_split()
706 static bool virtqueue_kick_prepare_split(struct virtqueue *_vq) in virtqueue_kick_prepare_split() argument
708 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_kick_prepare_split()
725 needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, in virtqueue_kick_prepare_split()
730 cpu_to_virtio16(_vq->vdev, in virtqueue_kick_prepare_split()
793 static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq, in virtqueue_get_buf_ctx_split() argument
797 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_buf_ctx_split()
819 i = virtio32_to_cpu(_vq->vdev, in virtqueue_get_buf_ctx_split()
821 *len = virtio32_to_cpu(_vq->vdev, in virtqueue_get_buf_ctx_split()
843 cpu_to_virtio16(_vq->vdev, vq->last_used_idx)); in virtqueue_get_buf_ctx_split()
851 static void virtqueue_disable_cb_split(struct virtqueue *_vq) in virtqueue_disable_cb_split() argument
853 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_disable_cb_split()
862 cpu_to_virtio16(_vq->vdev, in virtqueue_disable_cb_split()
867 static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq) in virtqueue_enable_cb_prepare_split() argument
869 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_prepare_split()
883 cpu_to_virtio16(_vq->vdev, in virtqueue_enable_cb_prepare_split()
886 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev, in virtqueue_enable_cb_prepare_split()
892 static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned int last_used_idx) in virtqueue_poll_split() argument
894 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_poll_split()
896 return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, in virtqueue_poll_split()
900 static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq) in virtqueue_enable_cb_delayed_split() argument
902 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_delayed_split()
916 cpu_to_virtio16(_vq->vdev, in virtqueue_enable_cb_delayed_split()
924 cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs)); in virtqueue_enable_cb_delayed_split()
926 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx) in virtqueue_enable_cb_delayed_split()
936 static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq) in virtqueue_detach_unused_buf_split() argument
938 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_detach_unused_buf_split()
951 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, in virtqueue_detach_unused_buf_split()
1137 static int virtqueue_resize_split(struct virtqueue *_vq, u32 num) in virtqueue_resize_split() argument
1140 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_resize_split()
1141 struct virtio_device *vdev = _vq->vdev; in virtqueue_resize_split()
1356 static inline int virtqueue_add_packed(struct virtqueue *_vq, in virtqueue_add_packed() argument
1365 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_add_packed()
1507 static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq) in virtqueue_kick_prepare_packed() argument
1509 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_kick_prepare_packed()
1627 static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq, in virtqueue_get_buf_ctx_packed() argument
1631 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_buf_ctx_packed()
1696 static void virtqueue_disable_cb_packed(struct virtqueue *_vq) in virtqueue_disable_cb_packed() argument
1698 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_disable_cb_packed()
1707 static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq) in virtqueue_enable_cb_prepare_packed() argument
1709 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_prepare_packed()
1740 static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap) in virtqueue_poll_packed() argument
1742 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_poll_packed()
1752 static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq) in virtqueue_enable_cb_delayed_packed() argument
1754 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_delayed_packed()
1813 static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq) in virtqueue_detach_unused_buf_packed() argument
1815 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_detach_unused_buf_packed()
2069 static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num) in virtqueue_resize_packed() argument
2072 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_resize_packed()
2073 struct virtio_device *vdev = _vq->vdev; in virtqueue_resize_packed()
2104 static inline int virtqueue_add(struct virtqueue *_vq, in virtqueue_add() argument
2113 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_add()
2115 return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg, in virtqueue_add()
2117 virtqueue_add_split(_vq, sgs, total_sg, in virtqueue_add()
2135 int virtqueue_add_sgs(struct virtqueue *_vq, in virtqueue_add_sgs() argument
2151 return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs, in virtqueue_add_sgs()
2235 bool virtqueue_kick_prepare(struct virtqueue *_vq) in virtqueue_kick_prepare() argument
2237 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_kick_prepare()
2239 return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) : in virtqueue_kick_prepare()
2240 virtqueue_kick_prepare_split(_vq); in virtqueue_kick_prepare()
2252 bool virtqueue_notify(struct virtqueue *_vq) in virtqueue_notify() argument
2254 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_notify()
2260 if (!vq->notify(_vq)) { in virtqueue_notify()
2305 void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len, in virtqueue_get_buf_ctx() argument
2308 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_buf_ctx()
2310 return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) : in virtqueue_get_buf_ctx()
2311 virtqueue_get_buf_ctx_split(_vq, len, ctx); in virtqueue_get_buf_ctx()
2315 void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len) in virtqueue_get_buf() argument
2317 return virtqueue_get_buf_ctx(_vq, len, NULL); in virtqueue_get_buf()
2329 void virtqueue_disable_cb(struct virtqueue *_vq) in virtqueue_disable_cb() argument
2331 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_disable_cb()
2340 virtqueue_disable_cb_packed(_vq); in virtqueue_disable_cb()
2342 virtqueue_disable_cb_split(_vq); in virtqueue_disable_cb()
2358 unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq) in virtqueue_enable_cb_prepare() argument
2360 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_prepare()
2365 return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) : in virtqueue_enable_cb_prepare()
2366 virtqueue_enable_cb_prepare_split(_vq); in virtqueue_enable_cb_prepare()
2379 bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx) in virtqueue_poll() argument
2381 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_poll()
2387 return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) : in virtqueue_poll()
2388 virtqueue_poll_split(_vq, last_used_idx); in virtqueue_poll()
2403 bool virtqueue_enable_cb(struct virtqueue *_vq) in virtqueue_enable_cb() argument
2405 unsigned int last_used_idx = virtqueue_enable_cb_prepare(_vq); in virtqueue_enable_cb()
2407 return !virtqueue_poll(_vq, last_used_idx); in virtqueue_enable_cb()
2424 bool virtqueue_enable_cb_delayed(struct virtqueue *_vq) in virtqueue_enable_cb_delayed() argument
2426 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_enable_cb_delayed()
2431 return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) : in virtqueue_enable_cb_delayed()
2432 virtqueue_enable_cb_delayed_split(_vq); in virtqueue_enable_cb_delayed()
2444 void *virtqueue_detach_unused_buf(struct virtqueue *_vq) in virtqueue_detach_unused_buf() argument
2446 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_detach_unused_buf()
2448 return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) : in virtqueue_detach_unused_buf()
2449 virtqueue_detach_unused_buf_split(_vq); in virtqueue_detach_unused_buf()
2466 irqreturn_t vring_interrupt(int irq, void *_vq) in vring_interrupt() argument
2468 struct vring_virtqueue *vq = to_vvq(_vq); in vring_interrupt()
2632 int virtqueue_resize(struct virtqueue *_vq, u32 num, in virtqueue_resize() argument
2635 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_resize()
2658 err = vdev->config->disable_vq_and_reset(_vq); in virtqueue_resize()
2662 while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL) in virtqueue_resize()
2663 recycle(_vq, buf); in virtqueue_resize()
2666 err = virtqueue_resize_packed(_vq, num); in virtqueue_resize()
2668 err = virtqueue_resize_split(_vq, num); in virtqueue_resize()
2670 if (vdev->config->enable_vq_after_reset(_vq)) in virtqueue_resize()
2701 static void vring_free(struct virtqueue *_vq) in vring_free() argument
2703 struct vring_virtqueue *vq = to_vvq(_vq); in vring_free()
2741 void vring_del_virtqueue(struct virtqueue *_vq) in vring_del_virtqueue() argument
2743 struct vring_virtqueue *vq = to_vvq(_vq); in vring_del_virtqueue()
2746 list_del(&_vq->list); in vring_del_virtqueue()
2749 vring_free(_vq); in vring_del_virtqueue()
2789 unsigned int virtqueue_get_vring_size(struct virtqueue *_vq) in virtqueue_get_vring_size() argument
2792 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_vring_size()
2801 void __virtqueue_break(struct virtqueue *_vq) in __virtqueue_break() argument
2803 struct vring_virtqueue *vq = to_vvq(_vq); in __virtqueue_break()
2813 void __virtqueue_unbreak(struct virtqueue *_vq) in __virtqueue_unbreak() argument
2815 struct vring_virtqueue *vq = to_vvq(_vq); in __virtqueue_unbreak()
2822 bool virtqueue_is_broken(struct virtqueue *_vq) in virtqueue_is_broken() argument
2824 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_is_broken()
2836 struct virtqueue *_vq; in virtio_break_device() local
2839 list_for_each_entry(_vq, &dev->vqs, list) { in virtio_break_device()
2840 struct vring_virtqueue *vq = to_vvq(_vq); in virtio_break_device()
2858 struct virtqueue *_vq; in __virtio_unbreak_device() local
2861 list_for_each_entry(_vq, &dev->vqs, list) { in __virtio_unbreak_device()
2862 struct vring_virtqueue *vq = to_vvq(_vq); in __virtio_unbreak_device()
2871 dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq) in virtqueue_get_desc_addr() argument
2873 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_desc_addr()
2884 dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq) in virtqueue_get_avail_addr() argument
2886 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_avail_addr()
2898 dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq) in virtqueue_get_used_addr() argument
2900 struct vring_virtqueue *vq = to_vvq(_vq); in virtqueue_get_used_addr()