Lines Matching refs:kit
6088 static bool scx_dispatch_from_dsq(struct bpf_iter_scx_dsq_kern *kit, in scx_dispatch_from_dsq() argument
6092 struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq; in scx_dispatch_from_dsq()
6129 u32_before(kit->cursor.priv, p->scx.dsq_seq) || in scx_dispatch_from_dsq()
6176 if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME) in scx_dispatch_from_dsq()
6177 p->scx.dsq_vtime = kit->vtime; in scx_dispatch_from_dsq()
6181 if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE) in scx_dispatch_from_dsq()
6182 p->scx.slice = kit->slice; in scx_dispatch_from_dsq()
6195 kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE | in scx_dispatch_from_dsq()
6291 struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; in scx_bpf_dispatch_from_dsq_set_slice() local
6293 kit->slice = slice; in scx_bpf_dispatch_from_dsq_set_slice()
6294 kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE; in scx_bpf_dispatch_from_dsq_set_slice()
6310 struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter; in scx_bpf_dispatch_from_dsq_set_vtime() local
6312 kit->vtime = vtime; in scx_bpf_dispatch_from_dsq_set_vtime()
6313 kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME; in scx_bpf_dispatch_from_dsq_set_vtime()
6619 struct bpf_iter_scx_dsq_kern *kit = (void *)it; in bpf_iter_scx_dsq_new() local
6629 kit->dsq = find_user_dsq(dsq_id); in bpf_iter_scx_dsq_new()
6630 if (!kit->dsq) in bpf_iter_scx_dsq_new()
6633 INIT_LIST_HEAD(&kit->cursor.node); in bpf_iter_scx_dsq_new()
6634 kit->cursor.flags |= SCX_DSQ_LNODE_ITER_CURSOR | flags; in bpf_iter_scx_dsq_new()
6635 kit->cursor.priv = READ_ONCE(kit->dsq->seq); in bpf_iter_scx_dsq_new()
6648 struct bpf_iter_scx_dsq_kern *kit = (void *)it; in bpf_iter_scx_dsq_next() local
6649 bool rev = kit->cursor.flags & SCX_DSQ_ITER_REV; in bpf_iter_scx_dsq_next()
6653 if (!kit->dsq) in bpf_iter_scx_dsq_next()
6656 raw_spin_lock_irqsave(&kit->dsq->lock, flags); in bpf_iter_scx_dsq_next()
6658 if (list_empty(&kit->cursor.node)) in bpf_iter_scx_dsq_next()
6661 p = container_of(&kit->cursor, struct task_struct, scx.dsq_list); in bpf_iter_scx_dsq_next()
6669 p = nldsq_next_task(kit->dsq, p, rev); in bpf_iter_scx_dsq_next()
6670 } while (p && unlikely(u32_before(kit->cursor.priv, p->scx.dsq_seq))); in bpf_iter_scx_dsq_next()
6674 list_move_tail(&kit->cursor.node, &p->scx.dsq_list.node); in bpf_iter_scx_dsq_next()
6676 list_move(&kit->cursor.node, &p->scx.dsq_list.node); in bpf_iter_scx_dsq_next()
6678 list_del_init(&kit->cursor.node); in bpf_iter_scx_dsq_next()
6681 raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); in bpf_iter_scx_dsq_next()
6694 struct bpf_iter_scx_dsq_kern *kit = (void *)it; in bpf_iter_scx_dsq_destroy() local
6696 if (!kit->dsq) in bpf_iter_scx_dsq_destroy()
6699 if (!list_empty(&kit->cursor.node)) { in bpf_iter_scx_dsq_destroy()
6702 raw_spin_lock_irqsave(&kit->dsq->lock, flags); in bpf_iter_scx_dsq_destroy()
6703 list_del_init(&kit->cursor.node); in bpf_iter_scx_dsq_destroy()
6704 raw_spin_unlock_irqrestore(&kit->dsq->lock, flags); in bpf_iter_scx_dsq_destroy()
6706 kit->dsq = NULL; in bpf_iter_scx_dsq_destroy()