Lines Matching refs:rcpu
124 static void get_cpu_map_entry(struct bpf_cpu_map_entry *rcpu) in get_cpu_map_entry() argument
126 atomic_inc(&rcpu->refcnt); in get_cpu_map_entry()
132 struct bpf_cpu_map_entry *rcpu; in cpu_map_kthread_stop() local
134 rcpu = container_of(work, struct bpf_cpu_map_entry, kthread_stop_wq); in cpu_map_kthread_stop()
142 kthread_stop(rcpu->kthread); in cpu_map_kthread_stop()
159 static void put_cpu_map_entry(struct bpf_cpu_map_entry *rcpu) in put_cpu_map_entry() argument
161 if (atomic_dec_and_test(&rcpu->refcnt)) { in put_cpu_map_entry()
162 if (rcpu->prog) in put_cpu_map_entry()
163 bpf_prog_put(rcpu->prog); in put_cpu_map_entry()
165 __cpu_map_ring_cleanup(rcpu->queue); in put_cpu_map_entry()
166 ptr_ring_cleanup(rcpu->queue, NULL); in put_cpu_map_entry()
167 kfree(rcpu->queue); in put_cpu_map_entry()
168 kfree(rcpu); in put_cpu_map_entry()
172 static void cpu_map_bpf_prog_run_skb(struct bpf_cpu_map_entry *rcpu, in cpu_map_bpf_prog_run_skb() argument
182 act = bpf_prog_run_generic_xdp(skb, &xdp, rcpu->prog); in cpu_map_bpf_prog_run_skb()
189 rcpu->prog); in cpu_map_bpf_prog_run_skb()
198 bpf_warn_invalid_xdp_action(NULL, rcpu->prog, act); in cpu_map_bpf_prog_run_skb()
201 trace_xdp_exception(skb->dev, rcpu->prog, act); in cpu_map_bpf_prog_run_skb()
212 static int cpu_map_bpf_prog_run_xdp(struct bpf_cpu_map_entry *rcpu, in cpu_map_bpf_prog_run_xdp() argument
234 act = bpf_prog_run_xdp(rcpu->prog, &xdp); in cpu_map_bpf_prog_run_xdp()
248 rcpu->prog); in cpu_map_bpf_prog_run_xdp()
257 bpf_warn_invalid_xdp_action(NULL, rcpu->prog, act); in cpu_map_bpf_prog_run_xdp()
273 static int cpu_map_bpf_prog_run(struct bpf_cpu_map_entry *rcpu, void **frames, in cpu_map_bpf_prog_run() argument
279 if (!rcpu->prog) in cpu_map_bpf_prog_run()
284 nframes = cpu_map_bpf_prog_run_xdp(rcpu, frames, xdp_n, stats); in cpu_map_bpf_prog_run()
290 cpu_map_bpf_prog_run_skb(rcpu, list, stats); in cpu_map_bpf_prog_run()
300 struct bpf_cpu_map_entry *rcpu = data; in cpu_map_kthread_run() local
309 while (!kthread_should_stop() || !__ptr_ring_empty(rcpu->queue)) { in cpu_map_kthread_run()
319 if (__ptr_ring_empty(rcpu->queue)) { in cpu_map_kthread_run()
322 if (__ptr_ring_empty(rcpu->queue)) { in cpu_map_kthread_run()
337 n = __ptr_ring_consume_batched(rcpu->queue, frames, in cpu_map_kthread_run()
362 nframes = cpu_map_bpf_prog_run(rcpu, frames, xdp_n, &stats, &list); in cpu_map_kthread_run()
389 trace_xdp_cpumap_kthread(rcpu->map_id, n, kmem_alloc_drops, in cpu_map_kthread_run()
396 put_cpu_map_entry(rcpu); in cpu_map_kthread_run()
400 static int __cpu_map_load_bpf_program(struct bpf_cpu_map_entry *rcpu, in __cpu_map_load_bpf_program() argument
415 rcpu->value.bpf_prog.id = prog->aux->id; in __cpu_map_load_bpf_program()
416 rcpu->prog = prog; in __cpu_map_load_bpf_program()
427 struct bpf_cpu_map_entry *rcpu; in __cpu_map_entry_alloc() local
433 rcpu = bpf_map_kmalloc_node(map, sizeof(*rcpu), gfp | __GFP_ZERO, numa); in __cpu_map_entry_alloc()
434 if (!rcpu) in __cpu_map_entry_alloc()
438 rcpu->bulkq = bpf_map_alloc_percpu(map, sizeof(*rcpu->bulkq), in __cpu_map_entry_alloc()
440 if (!rcpu->bulkq) in __cpu_map_entry_alloc()
444 bq = per_cpu_ptr(rcpu->bulkq, i); in __cpu_map_entry_alloc()
445 bq->obj = rcpu; in __cpu_map_entry_alloc()
449 rcpu->queue = bpf_map_kmalloc_node(map, sizeof(*rcpu->queue), gfp, in __cpu_map_entry_alloc()
451 if (!rcpu->queue) in __cpu_map_entry_alloc()
454 err = ptr_ring_init(rcpu->queue, value->qsize, gfp); in __cpu_map_entry_alloc()
458 rcpu->cpu = cpu; in __cpu_map_entry_alloc()
459 rcpu->map_id = map->id; in __cpu_map_entry_alloc()
460 rcpu->value.qsize = value->qsize; in __cpu_map_entry_alloc()
462 if (fd > 0 && __cpu_map_load_bpf_program(rcpu, map, fd)) in __cpu_map_entry_alloc()
466 rcpu->kthread = kthread_create_on_node(cpu_map_kthread_run, rcpu, numa, in __cpu_map_entry_alloc()
469 if (IS_ERR(rcpu->kthread)) in __cpu_map_entry_alloc()
472 get_cpu_map_entry(rcpu); /* 1-refcnt for being in cmap->cpu_map[] */ in __cpu_map_entry_alloc()
473 get_cpu_map_entry(rcpu); /* 1-refcnt for kthread */ in __cpu_map_entry_alloc()
476 kthread_bind(rcpu->kthread, cpu); in __cpu_map_entry_alloc()
477 wake_up_process(rcpu->kthread); in __cpu_map_entry_alloc()
479 return rcpu; in __cpu_map_entry_alloc()
482 if (rcpu->prog) in __cpu_map_entry_alloc()
483 bpf_prog_put(rcpu->prog); in __cpu_map_entry_alloc()
485 ptr_ring_cleanup(rcpu->queue, NULL); in __cpu_map_entry_alloc()
487 kfree(rcpu->queue); in __cpu_map_entry_alloc()
489 free_percpu(rcpu->bulkq); in __cpu_map_entry_alloc()
491 kfree(rcpu); in __cpu_map_entry_alloc()
497 struct bpf_cpu_map_entry *rcpu; in __cpu_map_entry_free() local
504 rcpu = container_of(rcu, struct bpf_cpu_map_entry, rcu); in __cpu_map_entry_free()
506 free_percpu(rcpu->bulkq); in __cpu_map_entry_free()
508 put_cpu_map_entry(rcpu); in __cpu_map_entry_free()
531 u32 key_cpu, struct bpf_cpu_map_entry *rcpu) in __cpu_map_entry_replace() argument
535 old_rcpu = unrcu_pointer(xchg(&cmap->cpu_map[key_cpu], RCU_INITIALIZER(rcpu))); in __cpu_map_entry_replace()
561 struct bpf_cpu_map_entry *rcpu; in cpu_map_update_elem() local
581 rcpu = NULL; /* Same as deleting */ in cpu_map_update_elem()
584 rcpu = __cpu_map_entry_alloc(map, &cpumap_value, key_cpu); in cpu_map_update_elem()
585 if (!rcpu) in cpu_map_update_elem()
587 rcpu->cmap = cmap; in cpu_map_update_elem()
590 __cpu_map_entry_replace(cmap, key_cpu, rcpu); in cpu_map_update_elem()
615 struct bpf_cpu_map_entry *rcpu; in cpu_map_free() local
617 rcpu = rcu_dereference_raw(cmap->cpu_map[i]); in cpu_map_free()
618 if (!rcpu) in cpu_map_free()
635 struct bpf_cpu_map_entry *rcpu; in __cpu_map_lookup_elem() local
640 rcpu = rcu_dereference_check(cmap->cpu_map[key], in __cpu_map_lookup_elem()
642 return rcpu; in __cpu_map_lookup_elem()
647 struct bpf_cpu_map_entry *rcpu = in cpu_map_lookup_elem() local
650 return rcpu ? &rcpu->value : NULL; in cpu_map_lookup_elem()
692 struct bpf_cpu_map_entry *rcpu = bq->obj; in bq_flush_to_queue() local
694 const int to_cpu = rcpu->cpu; in bq_flush_to_queue()
701 q = rcpu->queue; in bq_flush_to_queue()
721 trace_xdp_cpumap_enqueue(rcpu->map_id, processed, drops, to_cpu); in bq_flush_to_queue()
727 static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf) in bq_enqueue() argument
730 struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq); in bq_enqueue()
750 int cpu_map_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf, in cpu_map_enqueue() argument
756 bq_enqueue(rcpu, xdpf); in cpu_map_enqueue()
760 int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu, in cpu_map_generic_redirect() argument
769 ret = ptr_ring_produce(rcpu->queue, skb); in cpu_map_generic_redirect()
773 wake_up_process(rcpu->kthread); in cpu_map_generic_redirect()
775 trace_xdp_cpumap_enqueue(rcpu->map_id, !ret, !!ret, rcpu->cpu); in cpu_map_generic_redirect()