Lines Matching refs:map
89 struct bpf_map map; member
133 return !(htab->map.map_flags & BPF_F_NO_PREALLOC); in htab_is_prealloc()
168 return htab->map.map_type == BPF_MAP_TYPE_LRU_HASH || in htab_is_lru()
169 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_lru()
174 return htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH || in htab_is_percpu()
175 htab->map.map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH; in htab_is_percpu()
180 return htab->map.map_type == BPF_MAP_TYPE_HASH_OF_MAPS; in is_fd_htab()
199 static void *fd_htab_map_get_ptr(const struct bpf_map *map, struct htab_elem *l) in fd_htab_map_get_ptr() argument
201 return *(void **)htab_elem_value(l, map->key_size); in fd_htab_map_get_ptr()
220 u32 num_entries = htab->map.max_entries; in htab_free_prealloced_timers_and_wq()
230 if (btf_record_has_field(htab->map.record, BPF_TIMER)) in htab_free_prealloced_timers_and_wq()
231 bpf_obj_free_timer(htab->map.record, in htab_free_prealloced_timers_and_wq()
232 htab_elem_value(elem, htab->map.key_size)); in htab_free_prealloced_timers_and_wq()
233 if (btf_record_has_field(htab->map.record, BPF_WORKQUEUE)) in htab_free_prealloced_timers_and_wq()
234 bpf_obj_free_workqueue(htab->map.record, in htab_free_prealloced_timers_and_wq()
235 htab_elem_value(elem, htab->map.key_size)); in htab_free_prealloced_timers_and_wq()
242 u32 num_entries = htab->map.max_entries; in htab_free_prealloced_fields()
245 if (IS_ERR_OR_NULL(htab->map.record)) in htab_free_prealloced_fields()
254 void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size); in htab_free_prealloced_fields()
258 bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu)); in htab_free_prealloced_fields()
262 bpf_obj_free_fields(htab->map.record, in htab_free_prealloced_fields()
263 htab_elem_value(elem, htab->map.key_size)); in htab_free_prealloced_fields()
277 for (i = 0; i < htab->map.max_entries; i++) { in htab_free_elems()
281 htab->map.key_size); in htab_free_elems()
307 bpf_map_inc_elem_count(&htab->map); in prealloc_lru_pop()
309 memcpy(l->key, key, htab->map.key_size); in prealloc_lru_pop()
318 u32 num_entries = htab->map.max_entries; in prealloc_init()
325 htab->map.numa_node); in prealloc_init()
333 u32 size = round_up(htab->map.value_size, 8); in prealloc_init()
336 pptr = bpf_map_alloc_percpu(&htab->map, size, 8, in prealloc_init()
340 htab_elem_set_ptr(get_htab_elem(htab, i), htab->map.key_size, in prealloc_init()
348 htab->map.map_flags & BPF_F_NO_COMMON_LRU, in prealloc_init()
391 pptr = bpf_map_alloc_percpu(&htab->map, sizeof(struct htab_elem *), 8, in alloc_extra_elems()
485 bpf_map_init_from_attr(&htab->map, attr); in htab_map_alloc()
492 htab->map.max_entries = roundup(attr->max_entries, in htab_map_alloc()
494 if (htab->map.max_entries < attr->max_entries) in htab_map_alloc()
495 htab->map.max_entries = rounddown(attr->max_entries, in htab_map_alloc()
503 if (htab->map.max_entries > 1UL << 31) in htab_map_alloc()
506 htab->n_buckets = roundup_pow_of_two(htab->map.max_entries); in htab_map_alloc()
509 round_up(htab->map.key_size, 8); in htab_map_alloc()
513 htab->elem_size += round_up(htab->map.value_size, 8); in htab_map_alloc()
519 err = bpf_map_init_elem_count(&htab->map); in htab_map_alloc()
526 htab->map.numa_node); in htab_map_alloc()
530 if (htab->map.map_flags & BPF_F_ZERO_SEED) in htab_map_alloc()
576 round_up(htab->map.value_size, 8), true); in htab_map_alloc()
582 return &htab->map; in htab_map_alloc()
593 bpf_map_free_elem_count(&htab->map); in htab_map_alloc()
657 static void *__htab_map_lookup_elem(struct bpf_map *map, void *key) in __htab_map_lookup_elem() argument
659 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_elem()
667 key_size = map->key_size; in __htab_map_lookup_elem()
678 static void *htab_map_lookup_elem(struct bpf_map *map, void *key) in htab_map_lookup_elem() argument
680 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_map_lookup_elem()
683 return htab_elem_value(l, map->key_size); in htab_map_lookup_elem()
699 static int htab_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) in htab_map_gen_lookup() argument
705 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_map_gen_lookup()
710 round_up(map->key_size, 8)); in htab_map_gen_lookup()
714 static __always_inline void *__htab_lru_map_lookup_elem(struct bpf_map *map, in __htab_lru_map_lookup_elem() argument
717 struct htab_elem *l = __htab_map_lookup_elem(map, key); in __htab_lru_map_lookup_elem()
722 return htab_elem_value(l, map->key_size); in __htab_lru_map_lookup_elem()
728 static void *htab_lru_map_lookup_elem(struct bpf_map *map, void *key) in htab_lru_map_lookup_elem() argument
730 return __htab_lru_map_lookup_elem(map, key, true); in htab_lru_map_lookup_elem()
733 static void *htab_lru_map_lookup_elem_sys(struct bpf_map *map, void *key) in htab_lru_map_lookup_elem_sys() argument
735 return __htab_lru_map_lookup_elem(map, key, false); in htab_lru_map_lookup_elem_sys()
738 static int htab_lru_map_gen_lookup(struct bpf_map *map, in htab_lru_map_gen_lookup() argument
746 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_lru_map_gen_lookup()
759 round_up(map->key_size, 8)); in htab_lru_map_gen_lookup()
766 if (IS_ERR_OR_NULL(htab->map.record)) in check_and_free_fields()
770 void __percpu *pptr = htab_elem_get_ptr(elem, htab->map.key_size); in check_and_free_fields()
774 bpf_obj_free_fields(htab->map.record, per_cpu_ptr(pptr, cpu)); in check_and_free_fields()
776 void *map_value = htab_elem_value(elem, htab->map.key_size); in check_and_free_fields()
778 bpf_obj_free_fields(htab->map.record, map_value); in check_and_free_fields()
806 bpf_map_dec_elem_count(&htab->map); in htab_lru_map_delete_node()
818 static int htab_map_get_next_key(struct bpf_map *map, void *key, void *next_key) in htab_map_get_next_key() argument
820 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_get_next_key()
828 key_size = map->key_size; in htab_map_get_next_key()
880 if (htab->map.map_type == BPF_MAP_TYPE_PERCPU_HASH) in htab_elem_free()
887 struct bpf_map *map = &htab->map; in htab_put_fd_value() local
890 if (map->ops->map_fd_put_ptr) { in htab_put_fd_value()
891 ptr = fd_htab_map_get_ptr(map, l); in htab_put_fd_value()
892 map->ops->map_fd_put_ptr(map, ptr, true); in htab_put_fd_value()
899 return __percpu_counter_compare(&htab->pcount, htab->map.max_entries, in is_map_full()
901 return atomic_read(&htab->count) >= htab->map.max_entries; in is_map_full()
906 bpf_map_inc_elem_count(&htab->map); in inc_elem_count()
916 bpf_map_dec_elem_count(&htab->map); in dec_elem_count()
930 bpf_map_dec_elem_count(&htab->map); in free_htab_elem()
944 copy_map_value(&htab->map, this_cpu_ptr(pptr), value); in pcpu_copy_value()
946 u32 size = round_up(htab->map.value_size, 8); in pcpu_copy_value()
950 copy_map_value_long(&htab->map, per_cpu_ptr(pptr, cpu), value + off); in pcpu_copy_value()
970 copy_map_value_long(&htab->map, per_cpu_ptr(pptr, cpu), value); in pcpu_init_value()
972 zero_map_value(&htab->map, per_cpu_ptr(pptr, cpu)); in pcpu_init_value()
989 u32 size = htab->map.value_size; in alloc_htab_elem()
1009 bpf_map_inc_elem_count(&htab->map); in alloc_htab_elem()
1053 copy_map_value(&htab->map, htab_elem_value(l_new, key_size), value); in alloc_htab_elem()
1078 static long htab_map_update_elem(struct bpf_map *map, void *key, void *value, in htab_map_update_elem() argument
1081 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_update_elem()
1096 key_size = map->key_size; in htab_map_update_elem()
1104 if (unlikely(!btf_record_has_field(map->record, BPF_SPIN_LOCK))) in htab_map_update_elem()
1114 copy_map_value_locked(map, in htab_map_update_elem()
1142 copy_map_value_locked(map, in htab_map_update_elem()
1182 bpf_map_dec_elem_count(&htab->map); in htab_lru_push_free()
1186 static long htab_lru_map_update_elem(struct bpf_map *map, void *key, void *value, in htab_lru_map_update_elem() argument
1189 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_update_elem()
1204 key_size = map->key_size; in htab_lru_map_update_elem()
1219 copy_map_value(&htab->map, htab_elem_value(l_new, map->key_size), value); in htab_lru_map_update_elem()
1253 static long htab_map_update_elem_in_place(struct bpf_map *map, void *key, in htab_map_update_elem_in_place() argument
1257 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_update_elem_in_place()
1273 key_size = map->key_size; in htab_map_update_elem_in_place()
1313 map->ops->map_fd_put_ptr(map, old_map_ptr, true); in htab_map_update_elem_in_place()
1317 static long __htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, in __htab_lru_percpu_map_update_elem() argument
1321 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_lru_percpu_map_update_elem()
1336 key_size = map->key_size; in __htab_lru_percpu_map_update_elem()
1381 bpf_map_dec_elem_count(&htab->map); in __htab_lru_percpu_map_update_elem()
1387 static long htab_percpu_map_update_elem(struct bpf_map *map, void *key, in htab_percpu_map_update_elem() argument
1390 return htab_map_update_elem_in_place(map, key, value, map_flags, true, false); in htab_percpu_map_update_elem()
1393 static long htab_lru_percpu_map_update_elem(struct bpf_map *map, void *key, in htab_lru_percpu_map_update_elem() argument
1396 return __htab_lru_percpu_map_update_elem(map, key, value, map_flags, in htab_lru_percpu_map_update_elem()
1401 static long htab_map_delete_elem(struct bpf_map *map, void *key) in htab_map_delete_elem() argument
1403 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_delete_elem()
1414 key_size = map->key_size; in htab_map_delete_elem()
1437 static long htab_lru_map_delete_elem(struct bpf_map *map, void *key) in htab_lru_map_delete_elem() argument
1439 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_lru_map_delete_elem()
1450 key_size = map->key_size; in htab_lru_map_delete_elem()
1505 if (btf_record_has_field(htab->map.record, BPF_TIMER)) in htab_free_malloced_timers_and_wq()
1506 bpf_obj_free_timer(htab->map.record, in htab_free_malloced_timers_and_wq()
1507 htab_elem_value(l, htab->map.key_size)); in htab_free_malloced_timers_and_wq()
1508 if (btf_record_has_field(htab->map.record, BPF_WORKQUEUE)) in htab_free_malloced_timers_and_wq()
1509 bpf_obj_free_workqueue(htab->map.record, in htab_free_malloced_timers_and_wq()
1510 htab_elem_value(l, htab->map.key_size)); in htab_free_malloced_timers_and_wq()
1517 static void htab_map_free_timers_and_wq(struct bpf_map *map) in htab_map_free_timers_and_wq() argument
1519 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free_timers_and_wq()
1522 if (btf_record_has_field(htab->map.record, BPF_TIMER | BPF_WORKQUEUE)) { in htab_map_free_timers_and_wq()
1531 static void htab_map_free(struct bpf_map *map) in htab_map_free() argument
1533 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_free()
1551 bpf_map_free_elem_count(map); in htab_map_free()
1561 static void htab_map_seq_show_elem(struct bpf_map *map, void *key, in htab_map_seq_show_elem() argument
1568 value = htab_map_lookup_elem(map, key); in htab_map_seq_show_elem()
1574 btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); in htab_map_seq_show_elem()
1576 btf_type_seq_show(map->btf, map->btf_value_type_id, value, m); in htab_map_seq_show_elem()
1582 static int __htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, in __htab_map_lookup_and_delete_elem() argument
1586 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_and_delete_elem()
1594 key_size = map->key_size; in __htab_map_lookup_and_delete_elem()
1611 u32 roundup_value_size = round_up(map->value_size, 8); in __htab_map_lookup_and_delete_elem()
1617 copy_map_value_long(&htab->map, value + off, per_cpu_ptr(pptr, cpu)); in __htab_map_lookup_and_delete_elem()
1618 check_and_init_map_value(&htab->map, value + off); in __htab_map_lookup_and_delete_elem()
1622 void *src = htab_elem_value(l, map->key_size); in __htab_map_lookup_and_delete_elem()
1625 copy_map_value_locked(map, value, src, true); in __htab_map_lookup_and_delete_elem()
1627 copy_map_value(map, value, src); in __htab_map_lookup_and_delete_elem()
1629 check_and_init_map_value(map, value); in __htab_map_lookup_and_delete_elem()
1646 static int htab_map_lookup_and_delete_elem(struct bpf_map *map, void *key, in htab_map_lookup_and_delete_elem() argument
1649 return __htab_map_lookup_and_delete_elem(map, key, value, false, false, in htab_map_lookup_and_delete_elem()
1653 static int htab_percpu_map_lookup_and_delete_elem(struct bpf_map *map, in htab_percpu_map_lookup_and_delete_elem() argument
1657 return __htab_map_lookup_and_delete_elem(map, key, value, false, true, in htab_percpu_map_lookup_and_delete_elem()
1661 static int htab_lru_map_lookup_and_delete_elem(struct bpf_map *map, void *key, in htab_lru_map_lookup_and_delete_elem() argument
1664 return __htab_map_lookup_and_delete_elem(map, key, value, true, false, in htab_lru_map_lookup_and_delete_elem()
1668 static int htab_lru_percpu_map_lookup_and_delete_elem(struct bpf_map *map, in htab_lru_percpu_map_lookup_and_delete_elem() argument
1672 return __htab_map_lookup_and_delete_elem(map, key, value, true, true, in htab_lru_percpu_map_lookup_and_delete_elem()
1677 __htab_map_lookup_and_delete_batch(struct bpf_map *map, in __htab_map_lookup_and_delete_batch() argument
1683 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in __htab_map_lookup_and_delete_batch()
1702 ((elem_map_flags & BPF_F_LOCK) && !btf_record_has_field(map->record, BPF_SPIN_LOCK))) in __htab_map_lookup_and_delete_batch()
1723 key_size = htab->map.key_size; in __htab_map_lookup_and_delete_batch()
1724 value_size = htab->map.value_size; in __htab_map_lookup_and_delete_batch()
1808 pptr = htab_elem_get_ptr(l, map->key_size); in __htab_map_lookup_and_delete_batch()
1810 copy_map_value_long(&htab->map, dst_val + off, per_cpu_ptr(pptr, cpu)); in __htab_map_lookup_and_delete_batch()
1811 check_and_init_map_value(&htab->map, dst_val + off); in __htab_map_lookup_and_delete_batch()
1820 map_id = map->ops->map_fd_sys_lookup_elem(*inner_map); in __htab_map_lookup_and_delete_batch()
1825 copy_map_value_locked(map, dst_val, value, in __htab_map_lookup_and_delete_batch()
1828 copy_map_value(map, dst_val, value); in __htab_map_lookup_and_delete_batch()
1830 check_and_init_map_value(map, dst_val); in __htab_map_lookup_and_delete_batch()
1908 htab_percpu_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, in htab_percpu_map_lookup_batch() argument
1911 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_percpu_map_lookup_batch()
1916 htab_percpu_map_lookup_and_delete_batch(struct bpf_map *map, in htab_percpu_map_lookup_and_delete_batch() argument
1920 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_percpu_map_lookup_and_delete_batch()
1925 htab_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, in htab_map_lookup_batch() argument
1928 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_map_lookup_batch()
1933 htab_map_lookup_and_delete_batch(struct bpf_map *map, in htab_map_lookup_and_delete_batch() argument
1937 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_map_lookup_and_delete_batch()
1942 htab_lru_percpu_map_lookup_batch(struct bpf_map *map, in htab_lru_percpu_map_lookup_batch() argument
1946 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_lru_percpu_map_lookup_batch()
1951 htab_lru_percpu_map_lookup_and_delete_batch(struct bpf_map *map, in htab_lru_percpu_map_lookup_and_delete_batch() argument
1955 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_lru_percpu_map_lookup_and_delete_batch()
1960 htab_lru_map_lookup_batch(struct bpf_map *map, const union bpf_attr *attr, in htab_lru_map_lookup_batch() argument
1963 return __htab_map_lookup_and_delete_batch(map, attr, uattr, false, in htab_lru_map_lookup_batch()
1968 htab_lru_map_lookup_and_delete_batch(struct bpf_map *map, in htab_lru_map_lookup_and_delete_batch() argument
1972 return __htab_map_lookup_and_delete_batch(map, attr, uattr, true, in htab_lru_map_lookup_and_delete_batch()
1977 struct bpf_map *map; member
2067 struct bpf_map *map = info->map; in __bpf_hash_map_seq_show() local
2078 ctx.map = info->map; in __bpf_hash_map_seq_show()
2082 ctx.value = htab_elem_value(elem, map->key_size); in __bpf_hash_map_seq_show()
2084 roundup_value_size = round_up(map->value_size, 8); in __bpf_hash_map_seq_show()
2085 pptr = htab_elem_get_ptr(elem, map->key_size); in __bpf_hash_map_seq_show()
2087 copy_map_value_long(map, info->percpu_value_buf + off, in __bpf_hash_map_seq_show()
2089 check_and_init_map_value(map, info->percpu_value_buf + off); in __bpf_hash_map_seq_show()
2118 struct bpf_map *map = aux->map; in bpf_iter_init_hash_map() local
2122 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || in bpf_iter_init_hash_map()
2123 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH) { in bpf_iter_init_hash_map()
2124 buf_size = round_up(map->value_size, 8) * num_possible_cpus(); in bpf_iter_init_hash_map()
2132 bpf_map_inc_with_uref(map); in bpf_iter_init_hash_map()
2133 seq_info->map = map; in bpf_iter_init_hash_map()
2134 seq_info->htab = container_of(map, struct bpf_htab, map); in bpf_iter_init_hash_map()
2142 bpf_map_put_with_uref(seq_info->map); in bpf_iter_fini_hash_map()
2160 static long bpf_for_each_hash_elem(struct bpf_map *map, bpf_callback_t callback_fn, in bpf_for_each_hash_elem() argument
2163 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_for_each_hash_elem()
2193 pptr = htab_elem_get_ptr(elem, map->key_size); in bpf_for_each_hash_elem()
2196 val = htab_elem_value(elem, map->key_size); in bpf_for_each_hash_elem()
2199 ret = callback_fn((u64)(long)map, (u64)(long)key, in bpf_for_each_hash_elem()
2213 static u64 htab_map_mem_usage(const struct bpf_map *map) in htab_map_mem_usage() argument
2215 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in htab_map_mem_usage()
2216 u32 value_size = round_up(htab->map.value_size, 8); in htab_map_mem_usage()
2226 num_entries = map->max_entries; in htab_map_mem_usage()
2296 static void *htab_percpu_map_lookup_elem(struct bpf_map *map, void *key) in htab_percpu_map_lookup_elem() argument
2298 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_percpu_map_lookup_elem()
2301 return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); in htab_percpu_map_lookup_elem()
2307 static int htab_percpu_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) in htab_percpu_map_gen_lookup() argument
2315 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_percpu_map_gen_lookup()
2319 offsetof(struct htab_elem, key) + roundup(map->key_size, 8)); in htab_percpu_map_gen_lookup()
2326 static void *htab_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu) in htab_percpu_map_lookup_percpu_elem() argument
2333 l = __htab_map_lookup_elem(map, key); in htab_percpu_map_lookup_percpu_elem()
2335 return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); in htab_percpu_map_lookup_percpu_elem()
2340 static void *htab_lru_percpu_map_lookup_elem(struct bpf_map *map, void *key) in htab_lru_percpu_map_lookup_elem() argument
2342 struct htab_elem *l = __htab_map_lookup_elem(map, key); in htab_lru_percpu_map_lookup_elem()
2346 return this_cpu_ptr(htab_elem_get_ptr(l, map->key_size)); in htab_lru_percpu_map_lookup_elem()
2352 static void *htab_lru_percpu_map_lookup_percpu_elem(struct bpf_map *map, void *key, u32 cpu) in htab_lru_percpu_map_lookup_percpu_elem() argument
2359 l = __htab_map_lookup_elem(map, key); in htab_lru_percpu_map_lookup_percpu_elem()
2362 return per_cpu_ptr(htab_elem_get_ptr(l, map->key_size), cpu); in htab_lru_percpu_map_lookup_percpu_elem()
2368 int bpf_percpu_hash_copy(struct bpf_map *map, void *key, void *value) in bpf_percpu_hash_copy() argument
2380 size = round_up(map->value_size, 8); in bpf_percpu_hash_copy()
2382 l = __htab_map_lookup_elem(map, key); in bpf_percpu_hash_copy()
2388 pptr = htab_elem_get_ptr(l, map->key_size); in bpf_percpu_hash_copy()
2390 copy_map_value_long(map, value + off, per_cpu_ptr(pptr, cpu)); in bpf_percpu_hash_copy()
2391 check_and_init_map_value(map, value + off); in bpf_percpu_hash_copy()
2400 int bpf_percpu_hash_update(struct bpf_map *map, void *key, void *value, in bpf_percpu_hash_update() argument
2403 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in bpf_percpu_hash_update()
2408 ret = __htab_lru_percpu_map_update_elem(map, key, value, in bpf_percpu_hash_update()
2411 ret = htab_map_update_elem_in_place(map, key, value, map_flags, in bpf_percpu_hash_update()
2418 static void htab_percpu_map_seq_show_elem(struct bpf_map *map, void *key, in htab_percpu_map_seq_show_elem() argument
2427 l = __htab_map_lookup_elem(map, key); in htab_percpu_map_seq_show_elem()
2433 btf_type_seq_show(map->btf, map->btf_key_type_id, key, m); in htab_percpu_map_seq_show_elem()
2435 pptr = htab_elem_get_ptr(l, map->key_size); in htab_percpu_map_seq_show_elem()
2438 btf_type_seq_show(map->btf, map->btf_value_type_id, in htab_percpu_map_seq_show_elem()
2495 static void fd_htab_map_free(struct bpf_map *map) in fd_htab_map_free() argument
2497 struct bpf_htab *htab = container_of(map, struct bpf_htab, map); in fd_htab_map_free()
2507 void *ptr = fd_htab_map_get_ptr(map, l); in fd_htab_map_free()
2509 map->ops->map_fd_put_ptr(map, ptr, false); in fd_htab_map_free()
2513 htab_map_free(map); in fd_htab_map_free()
2517 int bpf_fd_htab_map_lookup_elem(struct bpf_map *map, void *key, u32 *value) in bpf_fd_htab_map_lookup_elem() argument
2522 if (!map->ops->map_fd_sys_lookup_elem) in bpf_fd_htab_map_lookup_elem()
2526 ptr = htab_map_lookup_elem(map, key); in bpf_fd_htab_map_lookup_elem()
2528 *value = map->ops->map_fd_sys_lookup_elem(READ_ONCE(*ptr)); in bpf_fd_htab_map_lookup_elem()
2537 int bpf_fd_htab_map_update_elem(struct bpf_map *map, struct file *map_file, in bpf_fd_htab_map_update_elem() argument
2543 ptr = map->ops->map_fd_get_ptr(map, map_file, *(int *)value); in bpf_fd_htab_map_update_elem()
2552 ret = htab_map_update_elem_in_place(map, key, &ptr, map_flags, false, false); in bpf_fd_htab_map_update_elem()
2555 map->ops->map_fd_put_ptr(map, ptr, false); in bpf_fd_htab_map_update_elem()
2562 struct bpf_map *map, *inner_map_meta; in htab_of_map_alloc() local
2568 map = htab_map_alloc(attr); in htab_of_map_alloc()
2569 if (IS_ERR(map)) { in htab_of_map_alloc()
2571 return map; in htab_of_map_alloc()
2574 map->inner_map_meta = inner_map_meta; in htab_of_map_alloc()
2576 return map; in htab_of_map_alloc()
2579 static void *htab_of_map_lookup_elem(struct bpf_map *map, void *key) in htab_of_map_lookup_elem() argument
2581 struct bpf_map **inner_map = htab_map_lookup_elem(map, key); in htab_of_map_lookup_elem()
2589 static int htab_of_map_gen_lookup(struct bpf_map *map, in htab_of_map_gen_lookup() argument
2596 (void *(*)(struct bpf_map *map, void *key))NULL)); in htab_of_map_gen_lookup()
2601 round_up(map->key_size, 8)); in htab_of_map_gen_lookup()
2607 static void htab_of_map_free(struct bpf_map *map) in htab_of_map_free() argument
2609 bpf_map_meta_free(map->inner_map_meta); in htab_of_map_free()
2610 fd_htab_map_free(map); in htab_of_map_free()