Lines Matching refs:hdr
388 struct bpf_ringbuf_hdr *hdr) in bpf_ringbuf_rec_pg_off() argument
390 return ((void *)hdr - (void *)rb) >> PAGE_SHIFT; in bpf_ringbuf_rec_pg_off()
397 bpf_ringbuf_restore_from_rec(struct bpf_ringbuf_hdr *hdr) in bpf_ringbuf_restore_from_rec() argument
399 unsigned long addr = (unsigned long)(void *)hdr; in bpf_ringbuf_restore_from_rec()
400 unsigned long off = (unsigned long)hdr->pg_off << PAGE_SHIFT; in bpf_ringbuf_restore_from_rec()
408 struct bpf_ringbuf_hdr *hdr; in __bpf_ringbuf_reserve() local
428 hdr = (void *)rb->data + (pend_pos & rb->mask); in __bpf_ringbuf_reserve()
429 hdr_len = READ_ONCE(hdr->len); in __bpf_ringbuf_reserve()
450 hdr = (void *)rb->data + (prod_pos & rb->mask); in __bpf_ringbuf_reserve()
451 pg_off = bpf_ringbuf_rec_pg_off(rb, hdr); in __bpf_ringbuf_reserve()
452 hdr->len = size | BPF_RINGBUF_BUSY_BIT; in __bpf_ringbuf_reserve()
453 hdr->pg_off = pg_off; in __bpf_ringbuf_reserve()
460 return (void *)hdr + BPF_RINGBUF_HDR_SZ; in __bpf_ringbuf_reserve()
485 struct bpf_ringbuf_hdr *hdr; in bpf_ringbuf_commit() local
489 hdr = sample - BPF_RINGBUF_HDR_SZ; in bpf_ringbuf_commit()
490 rb = bpf_ringbuf_restore_from_rec(hdr); in bpf_ringbuf_commit()
491 new_len = hdr->len ^ BPF_RINGBUF_BUSY_BIT; in bpf_ringbuf_commit()
496 xchg(&hdr->len, new_len); in bpf_ringbuf_commit()
501 rec_pos = (void *)hdr - (void *)rb->data; in bpf_ringbuf_commit()
672 u32 hdr_len, sample_len, total_len, flags, *hdr; in __bpf_user_ringbuf_peek() local
685 hdr = (u32 *)((uintptr_t)rb->data + (uintptr_t)(cons_pos & rb->mask)); in __bpf_user_ringbuf_peek()
687 hdr_len = smp_load_acquire(hdr); in __bpf_user_ringbuf_peek()