Lines Matching refs:b
34 struct bset_tree *bch2_bkey_to_bset(struct btree *b, struct bkey_packed *k) in bch2_bkey_to_bset() argument
36 return bch2_bkey_to_bset_inlined(b, k); in bch2_bkey_to_bset()
55 void bch2_dump_bset(struct bch_fs *c, struct btree *b, in bch2_dump_bset() argument
77 k = bkey_disassemble(b, _k, &uk); in bch2_dump_bset()
90 n = bkey_unpack_key(b, _n); in bch2_dump_bset()
104 void bch2_dump_btree_node(struct bch_fs *c, struct btree *b) in bch2_dump_btree_node() argument
107 for_each_bset(b, t) in bch2_dump_btree_node()
108 bch2_dump_bset(c, b, bset(b, t), t - b->set); in bch2_dump_btree_node()
112 void bch2_dump_btree_node_iter(struct btree *b, in bch2_dump_btree_node_iter() argument
119 __btree_node_iter_used(iter), b->nsets); in bch2_dump_btree_node_iter()
122 struct bkey_packed *k = __btree_node_offset_to_key(b, set->k); in bch2_dump_btree_node_iter()
123 struct bset_tree *t = bch2_bkey_to_bset(b, k); in bch2_dump_btree_node_iter()
124 struct bkey uk = bkey_unpack_key(b, k); in bch2_dump_btree_node_iter()
129 t - b->set, set->k, buf.buf); in bch2_dump_btree_node_iter()
135 struct btree_nr_keys bch2_btree_node_count_keys(struct btree *b) in bch2_btree_node_count_keys() argument
140 for_each_bset(b, t) in bch2_btree_node_count_keys()
141 bset_tree_for_each_key(b, t, k) in bch2_btree_node_count_keys()
143 btree_keys_account_key_add(&nr, t - b->set, k); in bch2_btree_node_count_keys()
147 void __bch2_verify_btree_nr_keys(struct btree *b) in __bch2_verify_btree_nr_keys() argument
149 struct btree_nr_keys nr = bch2_btree_node_count_keys(b); in __bch2_verify_btree_nr_keys()
151 BUG_ON(memcmp(&nr, &b->nr, sizeof(nr))); in __bch2_verify_btree_nr_keys()
155 struct btree *b) in __bch2_btree_node_iter_next_check() argument
160 k = bch2_btree_node_iter_peek_all(&iter, b); in __bch2_btree_node_iter_next_check()
161 __bch2_btree_node_iter_advance(&iter, b); in __bch2_btree_node_iter_next_check()
162 n = bch2_btree_node_iter_peek_all(&iter, b); in __bch2_btree_node_iter_next_check()
164 bkey_unpack_key(b, k); in __bch2_btree_node_iter_next_check()
167 bkey_iter_cmp(b, k, n) > 0) { in __bch2_btree_node_iter_next_check()
169 struct bkey ku = bkey_unpack_key(b, k); in __bch2_btree_node_iter_next_check()
170 struct bkey nu = bkey_unpack_key(b, n); in __bch2_btree_node_iter_next_check()
174 bch2_dump_btree_node(NULL, b); in __bch2_btree_node_iter_next_check()
182 struct bkey_packed *k2 = __btree_node_offset_to_key(b, set->k); in __bch2_btree_node_iter_next_check()
183 struct bset_tree *t = bch2_bkey_to_bset(b, k2); in __bch2_btree_node_iter_next_check()
184 printk(" [%zi %zi]", t - b->set, in __bch2_btree_node_iter_next_check()
185 k2->_data - bset(b, t)->_data); in __bch2_btree_node_iter_next_check()
192 struct btree *b) in __bch2_btree_node_iter_verify() argument
209 for_each_bset(b, t) in __bch2_btree_node_iter_verify()
223 btree_node_iter_cmp(b, set[-1], set[0]) > 0); in __bch2_btree_node_iter_verify()
225 k = bch2_btree_node_iter_peek_all(iter, b); in __bch2_btree_node_iter_verify()
227 for_each_bset(b, t) { in __bch2_btree_node_iter_verify()
231 p = bch2_bkey_prev_all(b, t, in __bch2_btree_node_iter_verify()
232 bch2_btree_node_iter_bset_pos(iter, b, t)); in __bch2_btree_node_iter_verify()
234 BUG_ON(p && bkey_iter_cmp(b, k, p) < 0); in __bch2_btree_node_iter_verify()
238 static void __bch2_verify_insert_pos(struct btree *b, struct bkey_packed *where, in __bch2_verify_insert_pos() argument
241 struct bset_tree *t = bch2_bkey_to_bset(b, where); in __bch2_verify_insert_pos()
242 struct bkey_packed *prev = bch2_bkey_prev_all(b, t, where); in __bch2_verify_insert_pos()
248 bkey_iter_cmp(b, prev, insert) > 0); in __bch2_verify_insert_pos()
251 bkey_iter_cmp(b, prev, insert) > 0) { in __bch2_verify_insert_pos()
252 struct bkey k1 = bkey_unpack_key(b, prev); in __bch2_verify_insert_pos()
253 struct bkey k2 = bkey_unpack_key(b, insert); in __bch2_verify_insert_pos()
255 bch2_dump_btree_node(NULL, b); in __bch2_verify_insert_pos()
266 BUG_ON(next != btree_bkey_last(b, t) && in __bch2_verify_insert_pos()
267 bkey_iter_cmp(b, insert, next) > 0); in __bch2_verify_insert_pos()
269 if (next != btree_bkey_last(b, t) && in __bch2_verify_insert_pos()
270 bkey_iter_cmp(b, insert, next) > 0) { in __bch2_verify_insert_pos()
271 struct bkey k1 = bkey_unpack_key(b, insert); in __bch2_verify_insert_pos()
272 struct bkey k2 = bkey_unpack_key(b, next); in __bch2_verify_insert_pos()
274 bch2_dump_btree_node(NULL, b); in __bch2_verify_insert_pos()
286 static inline void bch2_verify_insert_pos(struct btree *b, in bch2_verify_insert_pos() argument
292 __bch2_verify_insert_pos(b, where, insert, clobber_u64s); in bch2_verify_insert_pos()
336 static unsigned bset_aux_tree_buf_start(const struct btree *b, in bset_aux_tree_buf_start() argument
339 return t == b->set in bset_aux_tree_buf_start()
340 ? DIV_ROUND_UP(b->unpack_fn_len, 8) in bset_aux_tree_buf_start()
344 static void *__aux_tree_base(const struct btree *b, in __aux_tree_base() argument
347 return b->aux_data + t->aux_data_offset * 8; in __aux_tree_base()
350 static struct ro_aux_tree *ro_aux_tree_base(const struct btree *b, in ro_aux_tree_base() argument
355 return __aux_tree_base(b, t); in ro_aux_tree_base()
358 static struct bkey_float *bkey_float(const struct btree *b, in bkey_float() argument
362 return ro_aux_tree_base(b, t)->f + idx; in bkey_float()
365 static void __bset_aux_tree_verify(struct btree *b) in __bset_aux_tree_verify() argument
367 for_each_bset(b, t) { in __bset_aux_tree_verify()
371 BUG_ON(t != b->set && in __bset_aux_tree_verify()
374 BUG_ON(t->aux_data_offset < bset_aux_tree_buf_start(b, t)); in __bset_aux_tree_verify()
375 BUG_ON(t->aux_data_offset > btree_aux_data_u64s(b)); in __bset_aux_tree_verify()
376 BUG_ON(bset_aux_tree_buf_end(t) > btree_aux_data_u64s(b)); in __bset_aux_tree_verify()
380 static inline void bset_aux_tree_verify(struct btree *b) in bset_aux_tree_verify() argument
383 __bset_aux_tree_verify(b); in bset_aux_tree_verify()
386 void bch2_btree_keys_init(struct btree *b) in bch2_btree_keys_init() argument
390 b->nsets = 0; in bch2_btree_keys_init()
391 memset(&b->nr, 0, sizeof(b->nr)); in bch2_btree_keys_init()
394 b->set[i].data_offset = U16_MAX; in bch2_btree_keys_init()
396 bch2_bset_set_no_aux_tree(b, b->set); in bch2_btree_keys_init()
421 static inline void *bset_cacheline(const struct btree *b, in bset_cacheline() argument
425 return (void *) round_down((unsigned long) btree_bkey_first(b, t), in bset_cacheline()
430 static struct bkey_packed *cacheline_to_bkey(const struct btree *b, in cacheline_to_bkey() argument
435 return bset_cacheline(b, t, cacheline) + offset * 8; in cacheline_to_bkey()
438 static unsigned bkey_to_cacheline(const struct btree *b, in bkey_to_cacheline() argument
442 return ((void *) k - bset_cacheline(b, t, 0)) / BSET_CACHELINE; in bkey_to_cacheline()
445 static ssize_t __bkey_to_cacheline_offset(const struct btree *b, in __bkey_to_cacheline_offset() argument
450 return (u64 *) k - (u64 *) bset_cacheline(b, t, cacheline); in __bkey_to_cacheline_offset()
453 static unsigned bkey_to_cacheline_offset(const struct btree *b, in bkey_to_cacheline_offset() argument
458 size_t m = __bkey_to_cacheline_offset(b, t, cacheline, k); in bkey_to_cacheline_offset()
464 static inline struct bkey_packed *tree_to_bkey(const struct btree *b, in tree_to_bkey() argument
468 return cacheline_to_bkey(b, t, in tree_to_bkey()
470 bkey_float(b, t, j)->key_offset); in tree_to_bkey()
473 static struct rw_aux_tree *rw_aux_tree(const struct btree *b, in rw_aux_tree() argument
478 return __aux_tree_base(b, t); in rw_aux_tree()
485 static struct bkey_packed *rw_aux_to_bkey(const struct btree *b, in rw_aux_to_bkey() argument
489 return __btree_node_offset_to_key(b, rw_aux_tree(b, t)[j].offset); in rw_aux_to_bkey()
492 static void rw_aux_tree_set(const struct btree *b, struct bset_tree *t, in rw_aux_tree_set() argument
495 EBUG_ON(k >= btree_bkey_last(b, t)); in rw_aux_tree_set()
497 rw_aux_tree(b, t)[j] = (struct rw_aux_tree) { in rw_aux_tree_set()
498 .offset = __btree_node_key_to_offset(b, k), in rw_aux_tree_set()
499 .k = bkey_unpack_pos(b, k), in rw_aux_tree_set()
503 static void __bch2_bset_verify_rw_aux_tree(struct btree *b, struct bset_tree *t) in __bch2_bset_verify_rw_aux_tree() argument
505 struct bkey_packed *k = btree_bkey_first(b, t); in __bch2_bset_verify_rw_aux_tree()
514 BUG_ON(rw_aux_to_bkey(b, t, j) != k); in __bch2_bset_verify_rw_aux_tree()
518 if (rw_aux_to_bkey(b, t, j) == k) { in __bch2_bset_verify_rw_aux_tree()
519 BUG_ON(!bpos_eq(rw_aux_tree(b, t)[j].k, in __bch2_bset_verify_rw_aux_tree()
520 bkey_unpack_pos(b, k))); in __bch2_bset_verify_rw_aux_tree()
525 BUG_ON(rw_aux_tree(b, t)[j].offset <= in __bch2_bset_verify_rw_aux_tree()
526 rw_aux_tree(b, t)[j - 1].offset); in __bch2_bset_verify_rw_aux_tree()
530 BUG_ON(k >= btree_bkey_last(b, t)); in __bch2_bset_verify_rw_aux_tree()
534 static inline void bch2_bset_verify_rw_aux_tree(struct btree *b, in bch2_bset_verify_rw_aux_tree() argument
538 __bch2_bset_verify_rw_aux_tree(b, t); in bch2_bset_verify_rw_aux_tree()
542 static unsigned rw_aux_tree_bsearch(struct btree *b, in rw_aux_tree_bsearch() argument
555 rw_aux_tree(b, t)[idx].offset < offset) in rw_aux_tree_bsearch()
559 rw_aux_tree(b, t)[idx - 1].offset >= offset) in rw_aux_tree_bsearch()
563 rw_aux_tree(b, t)[idx].offset < offset); in rw_aux_tree_bsearch()
564 EBUG_ON(idx && rw_aux_tree(b, t)[idx - 1].offset >= offset); in rw_aux_tree_bsearch()
566 rw_aux_tree(b, t)[idx].offset == in rw_aux_tree_bsearch()
567 rw_aux_tree(b, t)[idx + 1].offset); in rw_aux_tree_bsearch()
595 static __always_inline void make_bfloat(struct btree *b, struct bset_tree *t, in make_bfloat() argument
600 struct bkey_float *f = bkey_float(b, t, j); in make_bfloat()
601 struct bkey_packed *m = tree_to_bkey(b, t, j); in make_bfloat()
604 : tree_to_bkey(b, t, j >> ffs(j)); in make_bfloat()
607 : tree_to_bkey(b, t, j >> (ffz(j) + 1)); in make_bfloat()
617 !b->nr_key_bits) { in make_bfloat()
632 high_bit = max(bch2_bkey_greatest_differing_bit(b, l, r), in make_bfloat()
633 min_t(unsigned, BKEY_MANTISSA_BITS, b->nr_key_bits) - 1); in make_bfloat()
641 shift = (int) (b->format.key_u64s * 64 - b->nr_key_bits) + exponent; in make_bfloat()
643 EBUG_ON(shift + BKEY_MANTISSA_BITS > b->format.key_u64s * 64); in make_bfloat()
646 b->nr_key_bits - in make_bfloat()
668 static unsigned __bset_tree_capacity(struct btree *b, const struct bset_tree *t) in __bset_tree_capacity() argument
670 bset_aux_tree_verify(b); in __bset_tree_capacity()
672 return btree_aux_data_bytes(b) - t->aux_data_offset * sizeof(u64); in __bset_tree_capacity()
675 static unsigned bset_ro_tree_capacity(struct btree *b, const struct bset_tree *t) in bset_ro_tree_capacity() argument
677 return __bset_tree_capacity(b, t) / sizeof(struct bkey_float); in bset_ro_tree_capacity()
680 static unsigned bset_rw_tree_capacity(struct btree *b, const struct bset_tree *t) in bset_rw_tree_capacity() argument
682 return __bset_tree_capacity(b, t) / sizeof(struct rw_aux_tree); in bset_rw_tree_capacity()
685 static noinline void __build_rw_aux_tree(struct btree *b, struct bset_tree *t) in __build_rw_aux_tree() argument
691 rw_aux_tree(b, t)[0].offset = in __build_rw_aux_tree() local
692 __btree_node_key_to_offset(b, btree_bkey_first(b, t)); in __build_rw_aux_tree()
694 bset_tree_for_each_key(b, t, k) { in __build_rw_aux_tree()
695 if (t->size == bset_rw_tree_capacity(b, t)) in __build_rw_aux_tree()
698 if ((void *) k - (void *) rw_aux_to_bkey(b, t, t->size - 1) > in __build_rw_aux_tree()
700 rw_aux_tree_set(b, t, t->size++, k); in __build_rw_aux_tree()
704 static noinline void __build_ro_aux_tree(struct btree *b, struct bset_tree *t) in __build_ro_aux_tree() argument
706 struct bkey_packed *k = btree_bkey_first(b, t); in __build_ro_aux_tree()
710 t->size = min(bkey_to_cacheline(b, t, btree_bkey_last(b, t)), in __build_ro_aux_tree()
711 bset_ro_tree_capacity(b, t)); in __build_ro_aux_tree()
723 while (bkey_to_cacheline(b, t, k) < cacheline) in __build_ro_aux_tree()
726 if (k >= btree_bkey_last(b, t)) { in __build_ro_aux_tree()
732 bkey_float(b, t, j)->key_offset = in __build_ro_aux_tree()
733 bkey_to_cacheline_offset(b, t, cacheline++, k); in __build_ro_aux_tree()
735 EBUG_ON(tree_to_bkey(b, t, j) != k); in __build_ro_aux_tree()
738 if (!bkey_pack_pos(bkey_to_packed(&min_key), b->data->min_key, b)) { in __build_ro_aux_tree()
740 min_key.k.p = b->data->min_key; in __build_ro_aux_tree()
743 if (!bkey_pack_pos(bkey_to_packed(&max_key), b->data->max_key, b)) { in __build_ro_aux_tree()
745 max_key.k.p = b->data->max_key; in __build_ro_aux_tree()
750 make_bfloat(b, t, j, in __build_ro_aux_tree()
755 static void bset_alloc_tree(struct btree *b, struct bset_tree *t) in bset_alloc_tree() argument
759 for (i = b->set; i != t; i++) in bset_alloc_tree()
762 bch2_bset_set_no_aux_tree(b, t); in bset_alloc_tree()
765 t->aux_data_offset = round_up(bset_aux_tree_buf_start(b, t), in bset_alloc_tree()
768 bset_aux_tree_verify(b); in bset_alloc_tree()
771 void bch2_bset_build_aux_tree(struct btree *b, struct bset_tree *t, in bch2_bset_build_aux_tree() argument
779 bset_alloc_tree(b, t); in bch2_bset_build_aux_tree()
781 if (!__bset_tree_capacity(b, t)) in bch2_bset_build_aux_tree()
785 __build_rw_aux_tree(b, t); in bch2_bset_build_aux_tree()
787 __build_ro_aux_tree(b, t); in bch2_bset_build_aux_tree()
789 bset_aux_tree_verify(b); in bch2_bset_build_aux_tree()
792 void bch2_bset_init_first(struct btree *b, struct bset *i) in bch2_bset_init_first() argument
796 BUG_ON(b->nsets); in bch2_bset_init_first()
802 t = &b->set[b->nsets++]; in bch2_bset_init_first()
803 set_btree_bset(b, t, i); in bch2_bset_init_first()
806 void bch2_bset_init_next(struct btree *b, struct btree_node_entry *bne) in bch2_bset_init_next() argument
811 BUG_ON(bset_byte_offset(b, bne) >= btree_buf_bytes(b)); in bch2_bset_init_next()
812 BUG_ON((void *) bne < (void *) btree_bkey_last(b, bset_tree_last(b))); in bch2_bset_init_next()
813 BUG_ON(b->nsets >= MAX_BSETS); in bch2_bset_init_next()
816 i->seq = btree_bset_first(b)->seq; in bch2_bset_init_next()
819 t = &b->set[b->nsets++]; in bch2_bset_init_next()
820 set_btree_bset(b, t, i); in bch2_bset_init_next()
827 static struct bkey_packed *__bkey_prev(struct btree *b, struct bset_tree *t, in __bkey_prev() argument
834 EBUG_ON(k < btree_bkey_first(b, t) || in __bkey_prev()
835 k > btree_bkey_last(b, t)); in __bkey_prev()
837 if (k == btree_bkey_first(b, t)) in __bkey_prev()
842 p = btree_bkey_first(b, t); in __bkey_prev()
845 j = min_t(unsigned, t->size - 1, bkey_to_cacheline(b, t, k)); in __bkey_prev()
848 p = j ? tree_to_bkey(b, t, in __bkey_prev()
851 : btree_bkey_first(b, t); in __bkey_prev()
855 offset = __btree_node_key_to_offset(b, k); in __bkey_prev()
856 j = rw_aux_tree_bsearch(b, t, offset); in __bkey_prev()
857 p = j ? rw_aux_to_bkey(b, t, j - 1) in __bkey_prev()
858 : btree_bkey_first(b, t); in __bkey_prev()
865 struct bkey_packed *bch2_bkey_prev_filter(struct btree *b, in bch2_bkey_prev_filter() argument
872 while ((p = __bkey_prev(b, t, k)) && !ret) { in bch2_bkey_prev_filter()
885 : btree_bkey_first(b, t); in bch2_bkey_prev_filter()
896 static void rw_aux_tree_insert_entry(struct btree *b, in rw_aux_tree_insert_entry() argument
901 struct bkey_packed *start = rw_aux_to_bkey(b, t, idx - 1); in rw_aux_tree_insert_entry()
903 ? rw_aux_to_bkey(b, t, idx) in rw_aux_tree_insert_entry()
904 : btree_bkey_last(b, t); in rw_aux_tree_insert_entry()
906 if (t->size < bset_rw_tree_capacity(b, t) && in rw_aux_tree_insert_entry()
916 memmove(&rw_aux_tree(b, t)[idx + 1], in rw_aux_tree_insert_entry()
917 &rw_aux_tree(b, t)[idx], in rw_aux_tree_insert_entry()
918 (void *) &rw_aux_tree(b, t)[t->size] - in rw_aux_tree_insert_entry()
919 (void *) &rw_aux_tree(b, t)[idx]); in rw_aux_tree_insert_entry()
921 rw_aux_tree_set(b, t, idx, k); in rw_aux_tree_insert_entry()
928 static void bch2_bset_fix_lookup_table(struct btree *b, in bch2_bset_fix_lookup_table() argument
935 unsigned idx, j, where = __btree_node_key_to_offset(b, _where); in bch2_bset_fix_lookup_table()
942 if (where > rw_aux_tree(b, t)[t->size - 1].offset) { in bch2_bset_fix_lookup_table()
943 rw_aux_tree_insert_entry(b, t, t->size); in bch2_bset_fix_lookup_table()
948 idx = rw_aux_tree_bsearch(b, t, where); in bch2_bset_fix_lookup_table()
950 if (rw_aux_tree(b, t)[idx].offset == where) { in bch2_bset_fix_lookup_table() local
954 rw_aux_tree_set(b, t, idx++, _where); in bch2_bset_fix_lookup_table()
957 rw_aux_tree_insert_entry(b, t, --t->size); in bch2_bset_fix_lookup_table()
962 EBUG_ON(idx < t->size && rw_aux_tree(b, t)[idx].offset <= where); in bch2_bset_fix_lookup_table()
964 rw_aux_tree(b, t)[idx].offset + shift == in bch2_bset_fix_lookup_table()
965 rw_aux_tree(b, t)[idx - 1].offset) { in bch2_bset_fix_lookup_table()
966 memmove(&rw_aux_tree(b, t)[idx], in bch2_bset_fix_lookup_table()
967 &rw_aux_tree(b, t)[idx + 1], in bch2_bset_fix_lookup_table()
968 (void *) &rw_aux_tree(b, t)[t->size] - in bch2_bset_fix_lookup_table()
969 (void *) &rw_aux_tree(b, t)[idx + 1]); in bch2_bset_fix_lookup_table()
974 rw_aux_tree(b, t)[j].offset += shift; in bch2_bset_fix_lookup_table() local
977 rw_aux_tree(b, t)[idx].offset == in bch2_bset_fix_lookup_table()
978 rw_aux_tree(b, t)[idx - 1].offset); in bch2_bset_fix_lookup_table()
980 rw_aux_tree_insert_entry(b, t, idx); in bch2_bset_fix_lookup_table()
983 bch2_bset_verify_rw_aux_tree(b, t); in bch2_bset_fix_lookup_table()
984 bset_aux_tree_verify(b); in bch2_bset_fix_lookup_table()
987 void bch2_bset_insert(struct btree *b, in bch2_bset_insert() argument
992 struct bkey_format *f = &b->format; in bch2_bset_insert()
993 struct bset_tree *t = bset_tree_last(b); in bch2_bset_insert()
996 bch2_bset_verify_rw_aux_tree(b, t); in bch2_bset_insert()
997 bch2_verify_insert_pos(b, where, bkey_to_packed(insert), clobber_u64s); in bch2_bset_insert()
1003 btree_keys_account_key_add(&b->nr, t - b->set, src); in bch2_bset_insert()
1009 EBUG_ON((int) le16_to_cpu(bset(b, t)->u64s) < in bch2_bset_insert()
1012 memmove_u64s(dst_p, src_p, btree_bkey_last(b, t)->_data - src_p); in bch2_bset_insert()
1013 le16_add_cpu(&bset(b, t)->u64s, src->u64s - clobber_u64s); in bch2_bset_insert()
1014 set_btree_bset_end(b, t); in bch2_bset_insert()
1023 bch2_bset_fix_lookup_table(b, t, where, clobber_u64s, src->u64s); in bch2_bset_insert()
1025 bch2_verify_btree_nr_keys(b); in bch2_bset_insert()
1028 void bch2_bset_delete(struct btree *b, in bch2_bset_delete() argument
1032 struct bset_tree *t = bset_tree_last(b); in bch2_bset_delete()
1036 bch2_bset_verify_rw_aux_tree(b, t); in bch2_bset_delete()
1038 EBUG_ON(le16_to_cpu(bset(b, t)->u64s) < clobber_u64s); in bch2_bset_delete()
1040 memmove_u64s_down(dst_p, src_p, btree_bkey_last(b, t)->_data - src_p); in bch2_bset_delete()
1041 le16_add_cpu(&bset(b, t)->u64s, -clobber_u64s); in bch2_bset_delete()
1042 set_btree_bset_end(b, t); in bch2_bset_delete()
1044 bch2_bset_fix_lookup_table(b, t, where, clobber_u64s, 0); in bch2_bset_delete()
1050 static struct bkey_packed *bset_search_write_set(const struct btree *b, in bset_search_write_set() argument
1059 if (bpos_lt(rw_aux_tree(b, t)[m].k, *search)) in bset_search_write_set()
1065 return rw_aux_to_bkey(b, t, l); in bset_search_write_set()
1085 static inline bool bkey_mantissa_bits_dropped(const struct btree *b, in bkey_mantissa_bits_dropped() argument
1089 unsigned key_bits_start = b->format.key_u64s * 64 - b->nr_key_bits; in bkey_mantissa_bits_dropped()
1093 unsigned key_bits_end = high_bit_offset + b->nr_key_bits; in bkey_mantissa_bits_dropped()
1100 static struct bkey_packed *bset_search_tree(const struct btree *b, in bset_search_tree() argument
1105 struct ro_aux_tree *base = ro_aux_tree_base(b, t); in bset_search_tree()
1122 if (unlikely(l == r) && bkey_mantissa_bits_dropped(b, f)) in bset_search_tree()
1128 k = tree_to_bkey(b, t, n); in bset_search_tree()
1129 cmp = bkey_cmp_p_or_unp(b, k, packed_search, search); in bset_search_tree()
1145 return btree_bkey_first(b, t); in bset_search_tree()
1150 return cacheline_to_bkey(b, t, inorder, f->key_offset); in bset_search_tree()
1154 struct bkey_packed *__bch2_bset_search(struct btree *b, in __bch2_bset_search() argument
1177 return btree_bkey_first(b, t); in __bch2_bset_search()
1179 return bset_search_write_set(b, t, search); in __bch2_bset_search()
1181 return bset_search_tree(b, t, search, lossy_packed_search); in __bch2_bset_search()
1188 struct bkey_packed *bch2_bset_search_linear(struct btree *b, in bch2_bset_search_linear() argument
1196 while (m != btree_bkey_last(b, t) && in bch2_bset_search_linear()
1197 bkey_iter_cmp_p_or_unp(b, m, in bch2_bset_search_linear()
1202 while (m != btree_bkey_last(b, t) && in bch2_bset_search_linear()
1203 bkey_iter_pos_cmp(b, m, search) < 0) in bch2_bset_search_linear()
1207 struct bkey_packed *prev = bch2_bkey_prev_all(b, t, m); in bch2_bset_search_linear()
1210 bkey_iter_cmp_p_or_unp(b, prev, in bch2_bset_search_linear()
1220 struct btree *b, in __bch2_btree_node_iter_push() argument
1232 __btree_node_key_to_offset(b, k), in __bch2_btree_node_iter_push()
1233 __btree_node_key_to_offset(b, end) in __bch2_btree_node_iter_push()
1239 struct btree *b, in bch2_btree_node_iter_push() argument
1243 __bch2_btree_node_iter_push(iter, b, k, end); in bch2_btree_node_iter_push()
1244 bch2_btree_node_iter_sort(iter, b); in bch2_btree_node_iter_push()
1249 struct btree *b, struct bpos *search) in btree_node_iter_init_pack_failed() argument
1255 bch2_btree_node_iter_init_from_start(iter, b); in btree_node_iter_init_pack_failed()
1257 while ((k = bch2_btree_node_iter_peek(iter, b)) && in btree_node_iter_init_pack_failed()
1258 bkey_iter_pos_cmp(b, k, search) < 0) in btree_node_iter_init_pack_failed()
1259 bch2_btree_node_iter_advance(iter, b); in btree_node_iter_init_pack_failed()
1308 struct btree *b, struct bpos *search) in bch2_btree_node_iter_init() argument
1315 EBUG_ON(bpos_lt(*search, b->data->min_key)); in bch2_btree_node_iter_init()
1316 EBUG_ON(bpos_gt(*search, b->data->max_key)); in bch2_btree_node_iter_init()
1317 bset_aux_tree_verify(b); in bch2_btree_node_iter_init()
1321 switch (bch2_bkey_pack_pos_lossy(&p, *search, b)) { in bch2_btree_node_iter_init()
1329 btree_node_iter_init_pack_failed(iter, b, search); in bch2_btree_node_iter_init()
1333 for (i = 0; i < b->nsets; i++) { in bch2_btree_node_iter_init()
1334 k[i] = __bch2_bset_search(b, b->set + i, search, &p); in bch2_btree_node_iter_init()
1338 for (i = 0; i < b->nsets; i++) { in bch2_btree_node_iter_init()
1339 struct bset_tree *t = b->set + i; in bch2_btree_node_iter_init()
1340 struct bkey_packed *end = btree_bkey_last(b, t); in bch2_btree_node_iter_init()
1342 k[i] = bch2_bset_search_linear(b, t, search, in bch2_btree_node_iter_init()
1346 __btree_node_key_to_offset(b, k[i]), in bch2_btree_node_iter_init()
1347 __btree_node_key_to_offset(b, end) in bch2_btree_node_iter_init()
1351 bch2_btree_node_iter_sort(iter, b); in bch2_btree_node_iter_init()
1355 struct btree *b) in bch2_btree_node_iter_init_from_start() argument
1359 for_each_bset(b, t) in bch2_btree_node_iter_init_from_start()
1360 __bch2_btree_node_iter_push(iter, b, in bch2_btree_node_iter_init_from_start()
1361 btree_bkey_first(b, t), in bch2_btree_node_iter_init_from_start()
1362 btree_bkey_last(b, t)); in bch2_btree_node_iter_init_from_start()
1363 bch2_btree_node_iter_sort(iter, b); in bch2_btree_node_iter_init_from_start()
1367 struct btree *b, in bch2_btree_node_iter_bset_pos() argument
1374 return __btree_node_offset_to_key(b, set->k); in bch2_btree_node_iter_bset_pos()
1376 return btree_bkey_last(b, t); in bch2_btree_node_iter_bset_pos()
1380 struct btree *b, in btree_node_iter_sort_two() argument
1385 if ((ret = (btree_node_iter_cmp(b, in btree_node_iter_sort_two()
1393 struct btree *b) in bch2_btree_node_iter_sort() argument
1398 btree_node_iter_sort_two(iter, b, 0); in bch2_btree_node_iter_sort()
1399 btree_node_iter_sort_two(iter, b, 1); in bch2_btree_node_iter_sort()
1403 btree_node_iter_sort_two(iter, b, 0); in bch2_btree_node_iter_sort()
1417 struct btree *b) in __bch2_btree_node_iter_advance() argument
1419 iter->data->k += __bch2_btree_node_iter_peek_all(iter, b)->u64s; in __bch2_btree_node_iter_advance()
1434 if (!btree_node_iter_sort_two(iter, b, 0)) in __bch2_btree_node_iter_advance()
1440 btree_node_iter_sort_two(iter, b, 1); in __bch2_btree_node_iter_advance()
1444 struct btree *b) in bch2_btree_node_iter_advance() argument
1447 __bch2_btree_node_iter_verify(iter, b); in bch2_btree_node_iter_advance()
1448 __bch2_btree_node_iter_next_check(iter, b); in bch2_btree_node_iter_advance()
1451 __bch2_btree_node_iter_advance(iter, b); in bch2_btree_node_iter_advance()
1458 struct btree *b) in bch2_btree_node_iter_prev_all() argument
1464 bch2_btree_node_iter_verify(iter, b); in bch2_btree_node_iter_prev_all()
1466 for_each_bset(b, t) { in bch2_btree_node_iter_prev_all()
1467 k = bch2_bkey_prev_all(b, t, in bch2_btree_node_iter_prev_all()
1468 bch2_btree_node_iter_bset_pos(iter, b, t)); in bch2_btree_node_iter_prev_all()
1470 (!prev || bkey_iter_cmp(b, k, prev) > 0)) { in bch2_btree_node_iter_prev_all()
1496 iter->data[0].k = __btree_node_key_to_offset(b, prev); in bch2_btree_node_iter_prev_all()
1499 bch2_btree_node_iter_verify(iter, b); in bch2_btree_node_iter_prev_all()
1504 struct btree *b) in bch2_btree_node_iter_prev() argument
1509 prev = bch2_btree_node_iter_prev_all(iter, b); in bch2_btree_node_iter_prev()
1516 struct btree *b, in bch2_btree_node_iter_peek_unpack() argument
1519 struct bkey_packed *k = bch2_btree_node_iter_peek(iter, b); in bch2_btree_node_iter_peek_unpack()
1521 return k ? bkey_disassemble(b, k, u) : bkey_s_c_null; in bch2_btree_node_iter_peek_unpack()
1526 void bch2_btree_keys_stats(const struct btree *b, struct bset_stats *stats) in bch2_btree_keys_stats() argument
1528 for_each_bset_c(b, t) { in bch2_btree_keys_stats()
1533 stats->sets[type].bytes += le16_to_cpu(bset(b, t)->u64s) * in bch2_btree_keys_stats()
1541 bkey_float(b, t, j)->exponent == in bch2_btree_keys_stats()
1547 void bch2_bfloat_to_text(struct printbuf *out, struct btree *b, in bch2_bfloat_to_text() argument
1550 struct bset_tree *t = bch2_bkey_to_bset(b, k); in bch2_bfloat_to_text()
1557 inorder = bkey_to_cacheline(b, t, k); in bch2_bfloat_to_text()
1562 if (k != tree_to_bkey(b, t, j)) in bch2_bfloat_to_text()
1565 switch (bkey_float(b, t, j)->exponent) { in bch2_bfloat_to_text()
1567 uk = bkey_unpack_key(b, k); in bch2_bfloat_to_text()