Lines Matching refs:smap
64 static int prealloc_elems_and_freelist(struct bpf_stack_map *smap) in prealloc_elems_and_freelist() argument
67 (u64)smap->map.value_size; in prealloc_elems_and_freelist()
70 smap->elems = bpf_map_area_alloc(elem_size * smap->map.max_entries, in prealloc_elems_and_freelist()
71 smap->map.numa_node); in prealloc_elems_and_freelist()
72 if (!smap->elems) in prealloc_elems_and_freelist()
75 err = pcpu_freelist_init(&smap->freelist); in prealloc_elems_and_freelist()
79 pcpu_freelist_populate(&smap->freelist, smap->elems, elem_size, in prealloc_elems_and_freelist()
80 smap->map.max_entries); in prealloc_elems_and_freelist()
84 bpf_map_area_free(smap->elems); in prealloc_elems_and_freelist()
92 struct bpf_stack_map *smap; in stack_map_alloc() local
121 cost = n_buckets * sizeof(struct stack_map_bucket *) + sizeof(*smap); in stack_map_alloc()
123 smap = bpf_map_area_alloc(cost, bpf_map_attr_numa_node(attr)); in stack_map_alloc()
124 if (!smap) in stack_map_alloc()
127 bpf_map_init_from_attr(&smap->map, attr); in stack_map_alloc()
128 smap->map.value_size = value_size; in stack_map_alloc()
129 smap->n_buckets = n_buckets; in stack_map_alloc()
135 err = prealloc_elems_and_freelist(smap); in stack_map_alloc()
139 return &smap->map; in stack_map_alloc()
144 bpf_map_area_free(smap); in stack_map_alloc()
262 struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); in __bpf_get_stackid() local
286 id = hash & (smap->n_buckets - 1); in __bpf_get_stackid()
287 bucket = READ_ONCE(smap->buckets[id]); in __bpf_get_stackid()
297 pcpu_freelist_pop(&smap->freelist); in __bpf_get_stackid()
307 pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); in __bpf_get_stackid()
311 pcpu_freelist_push(&smap->freelist, &new_bucket->fnode); in __bpf_get_stackid()
322 pcpu_freelist_pop(&smap->freelist); in __bpf_get_stackid()
331 old_bucket = xchg(&smap->buckets[id], new_bucket); in __bpf_get_stackid()
333 pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); in __bpf_get_stackid()
623 struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); in bpf_stackmap_copy() local
627 if (unlikely(id >= smap->n_buckets)) in bpf_stackmap_copy()
630 bucket = xchg(&smap->buckets[id], NULL); in bpf_stackmap_copy()
638 old_bucket = xchg(&smap->buckets[id], bucket); in bpf_stackmap_copy()
640 pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); in bpf_stackmap_copy()
647 struct bpf_stack_map *smap = container_of(map, in stack_map_get_next_key() local
657 if (id >= smap->n_buckets || !smap->buckets[id]) in stack_map_get_next_key()
663 while (id < smap->n_buckets && !smap->buckets[id]) in stack_map_get_next_key()
666 if (id >= smap->n_buckets) in stack_map_get_next_key()
682 struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); in stack_map_delete_elem() local
686 if (unlikely(id >= smap->n_buckets)) in stack_map_delete_elem()
689 old_bucket = xchg(&smap->buckets[id], NULL); in stack_map_delete_elem()
691 pcpu_freelist_push(&smap->freelist, &old_bucket->fnode); in stack_map_delete_elem()
701 struct bpf_stack_map *smap = container_of(map, struct bpf_stack_map, map); in stack_map_free() local
703 bpf_map_area_free(smap->elems); in stack_map_free()
704 pcpu_freelist_destroy(&smap->freelist); in stack_map_free()
705 bpf_map_area_free(smap); in stack_map_free()