Lines Matching refs:l
183 static inline void *htab_elem_value(struct htab_elem *l, u32 key_size) in htab_elem_value() argument
185 return l->key + round_up(key_size, 8); in htab_elem_value()
188 static inline void htab_elem_set_ptr(struct htab_elem *l, u32 key_size, in htab_elem_set_ptr() argument
191 *(void __percpu **)htab_elem_value(l, key_size) = pptr; in htab_elem_set_ptr()
194 static inline void __percpu *htab_elem_get_ptr(struct htab_elem *l, u32 key_size) in htab_elem_get_ptr() argument
196 return *(void __percpu **)htab_elem_value(l, key_size); in htab_elem_get_ptr()
199 static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l) in fd_htab_map_get_ptr() argument
201 return *(void **)htab_elem_value(l, map->key_size); in fd_htab_map_get_ptr()
304 struct htab_elem *l; in prealloc_lru_pop() local
308 l = container_of(node, struct htab_elem, lru_node); in prealloc_lru_pop()
309 memcpy(l->key, key, htab->map.key_size); in prealloc_lru_pop()
310 return l; in prealloc_lru_pop()
388 struct pcpu_freelist_node *l; in alloc_extra_elems() local
397 l = pcpu_freelist_pop(&htab->freelist); in alloc_extra_elems()
401 l_new = container_of(l, struct htab_elem, fnode); in alloc_extra_elems()
621 struct htab_elem *l; in lookup_elem_raw() local
623 hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) in lookup_elem_raw()
624 if (l->hash == hash && !memcmp(&l->key, key, key_size)) in lookup_elem_raw()
625 return l; in lookup_elem_raw()
639 struct htab_elem *l; in lookup_nulls_elem_raw() local
642 hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) in lookup_nulls_elem_raw()
643 if (l->hash == hash && !memcmp(&l->key, key, key_size)) in lookup_nulls_elem_raw()
644 return l; in lookup_nulls_elem_raw()
661 struct htab_elem *l; in __htab_map_lookup_elem() local
673 l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); in __htab_map_lookup_elem()
675 return l; in __htab_map_lookup_elem()
680 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_map_lookup_elem() local
682 if (l) in htab_map_lookup_elem()
683 return htab_elem_value(l, map->key_size); in htab_map_lookup_elem()
717 struct htab_elem *l = __htab_map_lookup_elem(map, key); in __htab_lru_map_lookup_elem() local
719 if (l) { in __htab_lru_map_lookup_elem()
721 bpf_lru_node_set_ref(&l->lru_node); in __htab_lru_map_lookup_elem()
722 return htab_elem_value(l, map->key_size); in __htab_lru_map_lookup_elem()
788 struct htab_elem *l = NULL, *tgt_l; in htab_lru_map_delete_node() local
803 hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) in htab_lru_map_delete_node()
804 if (l == tgt_l) { in htab_lru_map_delete_node()
805 hlist_nulls_del_rcu(&l->hash_node); in htab_lru_map_delete_node()
812 if (l == tgt_l) in htab_lru_map_delete_node()
813 check_and_free_fields(htab, l); in htab_lru_map_delete_node()
814 return l == tgt_l; in htab_lru_map_delete_node()
822 struct htab_elem *l, *next_l; in htab_map_get_next_key() local
838 l = lookup_nulls_elem_raw(head, hash, key, key_size, htab->n_buckets); in htab_map_get_next_key()
840 if (!l) in htab_map_get_next_key()
844 next_l = hlist_nulls_entry_safe(rcu_dereference_raw(hlist_nulls_next_rcu(&l->hash_node)), in htab_map_get_next_key()
876 static void htab_elem_free(struct bpf_htab *htab, struct htab_elem *l) in htab_elem_free() argument
878 check_and_free_fields(htab, l); in htab_elem_free()
881 bpf_mem_cache_free(&htab->pcpu_ma, l->ptr_to_pptr); in htab_elem_free()
882 bpf_mem_cache_free(&htab->ma, l); in htab_elem_free()
885 static void htab_put_fd_value(struct bpf_htab *htab, struct htab_elem *l) in htab_put_fd_value() argument
891 ptr = fd_htab_map_get_ptr(map, l); in htab_put_fd_value()
925 static void free_htab_elem(struct bpf_htab *htab, struct htab_elem *l) in free_htab_elem() argument
927 htab_put_fd_value(htab, l); in free_htab_elem()
931 check_and_free_fields(htab, l); in free_htab_elem()
932 pcpu_freelist_push(&htab->freelist, &l->fnode); in free_htab_elem()
935 htab_elem_free(htab, l); in free_htab_elem()
1003 struct pcpu_freelist_node *l; in alloc_htab_elem() local
1005 l = __pcpu_freelist_pop(&htab->freelist); in alloc_htab_elem()
1006 if (!l) in alloc_htab_elem()
1008 l_new = container_of(l, struct htab_elem, fnode); in alloc_htab_elem()
1406 struct htab_elem *l; in htab_map_delete_elem() local
1424 l = lookup_elem_raw(head, hash, key, key_size); in htab_map_delete_elem()
1425 if (l) in htab_map_delete_elem()
1426 hlist_nulls_del_rcu(&l->hash_node); in htab_map_delete_elem()
1432 if (l) in htab_map_delete_elem()
1433 free_htab_elem(htab, l); in htab_map_delete_elem()
1442 struct htab_elem *l; in htab_lru_map_delete_elem() local
1460 l = lookup_elem_raw(head, hash, key, key_size); in htab_lru_map_delete_elem()
1462 if (l) in htab_lru_map_delete_elem()
1463 hlist_nulls_del_rcu(&l->hash_node); in htab_lru_map_delete_elem()
1468 if (l) in htab_lru_map_delete_elem()
1469 htab_lru_push_free(htab, l); in htab_lru_map_delete_elem()
1483 struct htab_elem *l; in delete_all_elements() local
1485 hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { in delete_all_elements()
1486 hlist_nulls_del_rcu(&l->hash_node); in delete_all_elements()
1487 htab_elem_free(htab, l); in delete_all_elements()
1501 struct htab_elem *l; in htab_free_malloced_timers_and_wq() local
1503 hlist_nulls_for_each_entry(l, n, head, hash_node) { in htab_free_malloced_timers_and_wq()
1507 htab_elem_value(l, htab->map.key_size)); in htab_free_malloced_timers_and_wq()
1510 htab_elem_value(l, htab->map.key_size)); in htab_free_malloced_timers_and_wq()
1589 struct htab_elem *l; in __htab_map_lookup_and_delete_elem() local
1604 l = lookup_elem_raw(head, hash, key, key_size); in __htab_map_lookup_and_delete_elem()
1605 if (!l) { in __htab_map_lookup_and_delete_elem()
1615 pptr = htab_elem_get_ptr(l, key_size); in __htab_map_lookup_and_delete_elem()
1622 void *src = htab_elem_value(l, map->key_size); in __htab_map_lookup_and_delete_elem()
1631 hlist_nulls_del_rcu(&l->hash_node); in __htab_map_lookup_and_delete_elem()
1636 if (l) { in __htab_map_lookup_and_delete_elem()
1638 htab_lru_push_free(htab, l); in __htab_map_lookup_and_delete_elem()
1640 free_htab_elem(htab, l); in __htab_map_lookup_and_delete_elem()
1696 struct htab_elem *l; in __htab_map_lookup_and_delete_batch() local
1764 hlist_nulls_for_each_entry_rcu(l, n, head, hash_node) in __htab_map_lookup_and_delete_batch()
1801 hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { in __htab_map_lookup_and_delete_batch()
1802 memcpy(dst_key, l->key, key_size); in __htab_map_lookup_and_delete_batch()
1808 pptr = htab_elem_get_ptr(l, map->key_size); in __htab_map_lookup_and_delete_batch()
1815 value = htab_elem_value(l, key_size); in __htab_map_lookup_and_delete_batch()
1833 hlist_nulls_del_rcu(&l->hash_node); in __htab_map_lookup_and_delete_batch()
1845 l->batch_flink = node_to_free; in __htab_map_lookup_and_delete_batch()
1846 node_to_free = l; in __htab_map_lookup_and_delete_batch()
1856 l = node_to_free; in __htab_map_lookup_and_delete_batch()
1859 htab_lru_push_free(htab, l); in __htab_map_lookup_and_delete_batch()
1861 free_htab_elem(htab, l); in __htab_map_lookup_and_delete_batch()
2298 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_percpu_map_lookup_elem() local
2300 if (l) in htab_percpu_map_lookup_elem()
2301 return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); in htab_percpu_map_lookup_elem()
2328 struct htab_elem *l; in htab_percpu_map_lookup_percpu_elem() local
2333 l = __htab_map_lookup_elem(map, key); in htab_percpu_map_lookup_percpu_elem()
2334 if (l) in htab_percpu_map_lookup_percpu_elem()
2335 return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); in htab_percpu_map_lookup_percpu_elem()
2342 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_lru_percpu_map_lookup_elem() local
2344 if (l) { in htab_lru_percpu_map_lookup_elem()
2345 bpf_lru_node_set_ref(&l->lru_node); in htab_lru_percpu_map_lookup_elem()
2346 return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); in htab_lru_percpu_map_lookup_elem()
2354 struct htab_elem *l; in htab_lru_percpu_map_lookup_percpu_elem() local
2359 l = __htab_map_lookup_elem(map, key); in htab_lru_percpu_map_lookup_percpu_elem()
2360 if (l) { in htab_lru_percpu_map_lookup_percpu_elem()
2361 bpf_lru_node_set_ref(&l->lru_node); in htab_lru_percpu_map_lookup_percpu_elem()
2362 return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); in htab_lru_percpu_map_lookup_percpu_elem()
2370 struct htab_elem *l; in bpf_percpu_hash_copy() local
2382 l = __htab_map_lookup_elem(map, key); in bpf_percpu_hash_copy()
2383 if (!l) in bpf_percpu_hash_copy()
2388 pptr = htab_elem_get_ptr(l, map->key_size); in bpf_percpu_hash_copy()
2421 struct htab_elem *l; in htab_percpu_map_seq_show_elem() local
2427 l = __htab_map_lookup_elem(map, key); in htab_percpu_map_seq_show_elem()
2428 if (!l) { in htab_percpu_map_seq_show_elem()
2435 pptr = htab_elem_get_ptr(l, map->key_size); in htab_percpu_map_seq_show_elem()
2500 struct htab_elem *l; in fd_htab_map_free() local
2506 hlist_nulls_for_each_entry_safe(l, n, head, hash_node) { in fd_htab_map_free()
2507 void *ptr = fd_htab_map_get_ptr(map, l); in fd_htab_map_free()