Lines Matching refs:smap
21 select_bucket(struct bpf_local_storage_map *smap, in select_bucket() argument
24 return &smap->buckets[hash_ptr(selem, smap->bucket_log)]; in select_bucket()
27 static int mem_charge(struct bpf_local_storage_map *smap, void *owner, u32 size) in mem_charge() argument
29 struct bpf_map *map = &smap->map; in mem_charge()
34 return map->ops->map_local_storage_charge(smap, owner, size); in mem_charge()
37 static void mem_uncharge(struct bpf_local_storage_map *smap, void *owner, in mem_uncharge() argument
40 struct bpf_map *map = &smap->map; in mem_uncharge()
43 map->ops->map_local_storage_uncharge(smap, owner, size); in mem_uncharge()
47 owner_storage(struct bpf_local_storage_map *smap, void *owner) in owner_storage() argument
49 struct bpf_map *map = &smap->map; in owner_storage()
65 bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, in bpf_selem_alloc() argument
70 if (charge_mem && mem_charge(smap, owner, smap->elem_size)) in bpf_selem_alloc()
73 selem = bpf_map_kzalloc(&smap->map, smap->elem_size, in bpf_selem_alloc()
77 copy_map_value(&smap->map, SDATA(selem)->data, value); in bpf_selem_alloc()
82 mem_uncharge(smap, owner, smap->elem_size); in bpf_selem_alloc()
120 struct bpf_local_storage_map *smap; in bpf_selem_unlink_storage_nolock() local
124 smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); in bpf_selem_unlink_storage_nolock()
132 mem_uncharge(smap, owner, smap->elem_size); in bpf_selem_unlink_storage_nolock()
137 mem_uncharge(smap, owner, sizeof(struct bpf_local_storage)); in bpf_selem_unlink_storage_nolock()
141 RCU_INIT_POINTER(*owner_storage(smap, owner), NULL); in bpf_selem_unlink_storage_nolock()
158 if (rcu_access_pointer(local_storage->cache[smap->cache_idx]) == in bpf_selem_unlink_storage_nolock()
160 RCU_INIT_POINTER(local_storage->cache[smap->cache_idx], NULL); in bpf_selem_unlink_storage_nolock()
207 struct bpf_local_storage_map *smap; in bpf_selem_unlink_map() local
215 smap = rcu_dereference_check(SDATA(selem)->smap, bpf_rcu_lock_held()); in bpf_selem_unlink_map()
216 b = select_bucket(smap, selem); in bpf_selem_unlink_map()
223 void bpf_selem_link_map(struct bpf_local_storage_map *smap, in bpf_selem_link_map() argument
226 struct bpf_local_storage_map_bucket *b = select_bucket(smap, selem); in bpf_selem_link_map()
230 RCU_INIT_POINTER(SDATA(selem)->smap, smap); in bpf_selem_link_map()
248 struct bpf_local_storage_map *smap, in bpf_local_storage_lookup() argument
255 sdata = rcu_dereference_check(local_storage->cache[smap->cache_idx], in bpf_local_storage_lookup()
257 if (sdata && rcu_access_pointer(sdata->smap) == smap) in bpf_local_storage_lookup()
263 if (rcu_access_pointer(SDATA(selem)->smap) == smap) in bpf_local_storage_lookup()
280 rcu_assign_pointer(local_storage->cache[smap->cache_idx], in bpf_local_storage_lookup()
303 struct bpf_local_storage_map *smap, in bpf_local_storage_alloc() argument
311 err = mem_charge(smap, owner, sizeof(*storage)); in bpf_local_storage_alloc()
315 storage = bpf_map_kzalloc(&smap->map, sizeof(*storage), in bpf_local_storage_alloc()
327 bpf_selem_link_map(smap, first_selem); in bpf_local_storage_alloc()
330 (struct bpf_local_storage **)owner_storage(smap, owner); in bpf_local_storage_alloc()
362 mem_uncharge(smap, owner, sizeof(*storage)); in bpf_local_storage_alloc()
372 bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap, in bpf_local_storage_update() argument
385 !btf_record_has_field(smap->map.record, BPF_SPIN_LOCK))) in bpf_local_storage_update()
391 local_storage = rcu_dereference_check(*owner_storage(smap, owner), in bpf_local_storage_update()
399 selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags); in bpf_local_storage_update()
403 err = bpf_local_storage_alloc(owner, smap, selem, gfp_flags); in bpf_local_storage_update()
406 mem_uncharge(smap, owner, smap->elem_size); in bpf_local_storage_update()
419 bpf_local_storage_lookup(local_storage, smap, false); in bpf_local_storage_update()
424 copy_map_value_locked(&smap->map, old_sdata->data, in bpf_local_storage_update()
431 selem = bpf_selem_alloc(smap, owner, value, true, gfp_flags); in bpf_local_storage_update()
449 old_sdata = bpf_local_storage_lookup(local_storage, smap, false); in bpf_local_storage_update()
455 copy_map_value_locked(&smap->map, old_sdata->data, value, in bpf_local_storage_update()
471 selem = bpf_selem_alloc(smap, owner, value, !old_sdata, gfp_flags); in bpf_local_storage_update()
479 bpf_selem_link_map(smap, selem); in bpf_local_storage_update()
498 mem_uncharge(smap, owner, smap->elem_size); in bpf_local_storage_update()
557 struct bpf_local_storage_map *smap; in __bpf_local_storage_map_alloc() local
561 smap = bpf_map_area_alloc(sizeof(*smap), NUMA_NO_NODE); in __bpf_local_storage_map_alloc()
562 if (!smap) in __bpf_local_storage_map_alloc()
564 bpf_map_init_from_attr(&smap->map, attr); in __bpf_local_storage_map_alloc()
569 smap->bucket_log = ilog2(nbuckets); in __bpf_local_storage_map_alloc()
571 smap->buckets = bpf_map_kvcalloc(&smap->map, sizeof(*smap->buckets), in __bpf_local_storage_map_alloc()
573 if (!smap->buckets) { in __bpf_local_storage_map_alloc()
574 bpf_map_area_free(smap); in __bpf_local_storage_map_alloc()
579 INIT_HLIST_HEAD(&smap->buckets[i].list); in __bpf_local_storage_map_alloc()
580 raw_spin_lock_init(&smap->buckets[i].lock); in __bpf_local_storage_map_alloc()
583 smap->elem_size = offsetof(struct bpf_local_storage_elem, in __bpf_local_storage_map_alloc()
586 return smap; in __bpf_local_storage_map_alloc()
643 struct bpf_local_storage_map *smap; in bpf_local_storage_map_alloc() local
645 smap = __bpf_local_storage_map_alloc(attr); in bpf_local_storage_map_alloc()
646 if (IS_ERR(smap)) in bpf_local_storage_map_alloc()
647 return ERR_CAST(smap); in bpf_local_storage_map_alloc()
649 smap->cache_idx = bpf_local_storage_cache_idx_get(cache); in bpf_local_storage_map_alloc()
650 return &smap->map; in bpf_local_storage_map_alloc()
659 struct bpf_local_storage_map *smap; in bpf_local_storage_map_free() local
662 smap = (struct bpf_local_storage_map *)map; in bpf_local_storage_map_free()
663 bpf_local_storage_cache_idx_free(cache, smap->cache_idx); in bpf_local_storage_map_free()
680 for (i = 0; i < (1U << smap->bucket_log); i++) { in bpf_local_storage_map_free()
681 b = &smap->buckets[i]; in bpf_local_storage_map_free()
716 kvfree(smap->buckets); in bpf_local_storage_map_free()
717 bpf_map_area_free(smap); in bpf_local_storage_map_free()