Lines Matching refs:smap
18 select_bucket(struct bpf_local_storage_map *smap, in select_bucket() argument
21 return &smap->buckets[hash_ptr(selem, smap->bucket_log)]; in select_bucket()
24 static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size) in mem_charge() argument
26 struct bpf_map *map = &smap->map; in mem_charge()
31 return map->ops->map_local_storage_charge(smap, owner, size); in mem_charge()
34 static void mem_uncharge(struct bpf_local_storage_map *smap, void *owner, in mem_uncharge() argument
37 struct bpf_map *map = &smap->map; in mem_uncharge()
40 map->ops->map_local_storage_uncharge(smap, owner, size); in mem_uncharge()
44 owner_storage(struct bpf_local_storage_map *smap, void *owner) in owner_storage() argument
46 struct bpf_map *map = &smap->map; in owner_storage()
62 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, in bpf_selem_alloc() argument
67 if (charge_mem && mem_charge(smap, owner, smap->elem_size)) in bpf_selem_alloc()
70 selem = bpf_map_kzalloc(&smap->map, smap->elem_size, in bpf_selem_alloc()
74 memcpy(SDATA(selem)->data, value, smap->map.value_size); in bpf_selem_alloc()
79 mem_uncharge(smap, owner, smap->elem_size); in bpf_selem_alloc()
92 struct bpf_local_storage_map *smap; in bpf_selem_unlink_storage_nolock() local
96 smap = rcu_dereference(SDATA(selem)->smap); in bpf_selem_unlink_storage_nolock()
104 mem_uncharge(smap, owner, smap->elem_size); in bpf_selem_unlink_storage_nolock()
109 mem_uncharge(smap, owner, sizeof(struct bpf_local_storage)); in bpf_selem_unlink_storage_nolock()
113 RCU_INIT_POINTER(*owner_storage(smap, owner), NULL); in bpf_selem_unlink_storage_nolock()
130 if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) == in bpf_selem_unlink_storage_nolock()
132 RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL); in bpf_selem_unlink_storage_nolock()
169 struct bpf_local_storage_map *smap; in bpf_selem_unlink_map() local
177 smap = rcu_dereference(SDATA(selem)->smap); in bpf_selem_unlink_map()
178 b = select_bucket(smap, selem); in bpf_selem_unlink_map()
185 void bpf_selem_link_map(struct bpf_local_storage_map *smap, in bpf_selem_link_map() argument
188 struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem); in bpf_selem_link_map()
192 RCU_INIT_POINTER(SDATA(selem)->smap, smap); in bpf_selem_link_map()
209 struct bpf_local_storage_map *smap, in bpf_local_storage_lookup() argument
216 sdata = rcu_dereference(local_storage->cache[smap->cache_idx]); in bpf_local_storage_lookup()
217 if (sdata && rcu_access_pointer(sdata->smap) == smap) in bpf_local_storage_lookup()
222 if (rcu_access_pointer(SDATA(selem)->smap) == smap) in bpf_local_storage_lookup()
239 rcu_assign_pointer(local_storage->cache[smap->cache_idx], in bpf_local_storage_lookup()
262 struct bpf_local_storage_map *smap, in bpf_local_storage_alloc() argument
269 err = mem_charge(smap, owner, sizeof(*storage)); in bpf_local_storage_alloc()
273 storage = bpf_map_kzalloc(&smap->map, sizeof(*storage), in bpf_local_storage_alloc()
285 bpf_selem_link_map(smap, first_selem); in bpf_local_storage_alloc()
288 (struct bpf_local_storage **)owner_storage(smap, owner); in bpf_local_storage_alloc()
319 mem_uncharge(smap, owner, sizeof(*storage)); in bpf_local_storage_alloc()
329 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, in bpf_local_storage_update() argument
342 !map_value_has_spin_lock(&smap->map))) in bpf_local_storage_update()
345 local_storage = rcu_dereference(*owner_storage(smap, owner)); in bpf_local_storage_update()
352 selem = bpf_selem_alloc(smap, owner, value, true); in bpf_local_storage_update()
356 err = bpf_local_storage_alloc(owner, smap, selem); in bpf_local_storage_update()
359 mem_uncharge(smap, owner, smap->elem_size); in bpf_local_storage_update()
372 bpf_local_storage_lookup(local_storage, smap, false); in bpf_local_storage_update()
377 copy_map_value_locked(&smap->map, old_sdata->data, in bpf_local_storage_update()
396 old_sdata = bpf_local_storage_lookup(local_storage, smap, false); in bpf_local_storage_update()
402 copy_map_value_locked(&smap->map, old_sdata->data, value, in bpf_local_storage_update()
417 selem = bpf_selem_alloc(smap, owner, value, !old_sdata); in bpf_local_storage_update()
424 bpf_selem_link_map(smap, selem); in bpf_local_storage_update()
477 void bpf_local_storage_map_free(struct bpf_local_storage_map *smap, in bpf_local_storage_map_free() argument
499 for (i = 0; i < (1U << smap->bucket_log); i++) { in bpf_local_storage_map_free()
500 b = &smap->buckets[i]; in bpf_local_storage_map_free()
535 kvfree(smap->buckets); in bpf_local_storage_map_free()
536 kfree(smap); in bpf_local_storage_map_free()
560 struct bpf_local_storage_map *smap; in bpf_local_storage_map_alloc() local
564 smap = kzalloc(sizeof(*smap), GFP_USER | __GFP_NOWARN | __GFP_ACCOUNT); in bpf_local_storage_map_alloc()
565 if (!smap) in bpf_local_storage_map_alloc()
567 bpf_map_init_from_attr(&smap->map, attr); in bpf_local_storage_map_alloc()
572 smap->bucket_log = ilog2(nbuckets); in bpf_local_storage_map_alloc()
574 smap->buckets = kvcalloc(sizeof(*smap->buckets), nbuckets, in bpf_local_storage_map_alloc()
576 if (!smap->buckets) { in bpf_local_storage_map_alloc()
577 kfree(smap); in bpf_local_storage_map_alloc()
582 INIT_HLIST_HEAD(&smap->buckets[i].list); in bpf_local_storage_map_alloc()
583 raw_spin_lock_init(&smap->buckets[i].lock); in bpf_local_storage_map_alloc()
586 smap->elem_size = in bpf_local_storage_map_alloc()
589 return smap; in bpf_local_storage_map_alloc()