Lines Matching refs:iter
29 static inline unsigned long btree_iter_ip_allocated(struct btree_iter *iter) in btree_iter_ip_allocated() argument
32 return iter->ip_allocated; in btree_iter_ip_allocated()
62 static inline struct bpos bkey_successor(struct btree_iter *iter, struct bpos p) in bkey_successor() argument
65 if (iter->flags & BTREE_ITER_all_snapshots) { in bkey_successor()
69 p.snapshot = iter->snapshot; in bkey_successor()
75 static inline struct bpos bkey_predecessor(struct btree_iter *iter, struct bpos p) in bkey_predecessor() argument
78 if (iter->flags & BTREE_ITER_all_snapshots) { in bkey_predecessor()
82 p.snapshot = iter->snapshot; in bkey_predecessor()
88 static inline struct bpos btree_iter_search_key(struct btree_iter *iter) in btree_iter_search_key() argument
90 struct bpos pos = iter->pos; in btree_iter_search_key()
92 if ((iter->flags & BTREE_ITER_is_extents) && in btree_iter_search_key()
94 pos = bkey_successor(iter, pos); in btree_iter_search_key()
150 tmp = l->iter; in __bch2_btree_path_verify_level()
167 bch2_btree_node_iter_verify(&l->iter, l->b); in __bch2_btree_path_verify_level()
175 k = bch2_btree_node_iter_peek_all(&l->iter, l->b); in __bch2_btree_path_verify_level()
237 unsigned iter; in __bch2_trans_verify_paths() local
239 trans_for_each_path(trans, path, iter) in __bch2_trans_verify_paths()
243 static void __bch2_btree_iter_verify(struct btree_trans *trans, struct btree_iter *iter) in __bch2_btree_iter_verify() argument
245 BUG_ON(!!(iter->flags & BTREE_ITER_cached) != btree_iter_path(trans, iter)->cached); in __bch2_btree_iter_verify()
247 BUG_ON((iter->flags & BTREE_ITER_is_extents) && in __bch2_btree_iter_verify()
248 (iter->flags & BTREE_ITER_all_snapshots)); in __bch2_btree_iter_verify()
250 BUG_ON(!(iter->flags & BTREE_ITER_snapshot_field) && in __bch2_btree_iter_verify()
251 (iter->flags & BTREE_ITER_all_snapshots) && in __bch2_btree_iter_verify()
252 !btree_type_has_snapshot_field(iter->btree_id)); in __bch2_btree_iter_verify()
254 if (iter->update_path) in __bch2_btree_iter_verify()
255 __bch2_btree_path_verify(trans, &trans->paths[iter->update_path]); in __bch2_btree_iter_verify()
256 __bch2_btree_path_verify(trans, btree_iter_path(trans, iter)); in __bch2_btree_iter_verify()
259 static void __bch2_btree_iter_verify_entry_exit(struct btree_iter *iter) in __bch2_btree_iter_verify_entry_exit() argument
261 BUG_ON((iter->flags & BTREE_ITER_filter_snapshots) && in __bch2_btree_iter_verify_entry_exit()
262 !iter->pos.snapshot); in __bch2_btree_iter_verify_entry_exit()
264 BUG_ON(!(iter->flags & BTREE_ITER_all_snapshots) && in __bch2_btree_iter_verify_entry_exit()
265 iter->pos.snapshot != iter->snapshot); in __bch2_btree_iter_verify_entry_exit()
267 BUG_ON(iter->flags & BTREE_ITER_all_snapshots ? !bpos_eq(iter->pos, iter->k.p) : in __bch2_btree_iter_verify_entry_exit()
268 !(iter->flags & BTREE_ITER_is_extents) ? !bkey_eq(iter->pos, iter->k.p) : in __bch2_btree_iter_verify_entry_exit()
269 (bkey_lt(iter->pos, bkey_start_pos(&iter->k)) || in __bch2_btree_iter_verify_entry_exit()
270 bkey_gt(iter->pos, iter->k.p))); in __bch2_btree_iter_verify_entry_exit()
274 struct btree_iter *iter, struct bkey_s_c k) in __bch2_btree_iter_verify_ret() argument
280 if (!(iter->flags & BTREE_ITER_filter_snapshots)) in __bch2_btree_iter_verify_ret()
287 iter->snapshot, in __bch2_btree_iter_verify_ret()
290 bch2_trans_iter_init(trans, ©, iter->btree_id, iter->pos, in __bch2_btree_iter_verify_ret()
302 bch2_snapshot_is_ancestor(trans->c, iter->snapshot, in __bch2_btree_iter_verify_ret()
312 iter->snapshot, in __bch2_btree_iter_verify_ret()
326 struct trans_for_each_path_inorder_iter iter; in __bch2_assert_pos_locked() local
331 trans_for_each_path_inorder(trans, path, iter) { in __bch2_assert_pos_locked()
368 struct btree_iter *iter) in bch2_btree_iter_verify() argument
371 __bch2_btree_iter_verify(trans, iter); in bch2_btree_iter_verify()
374 static inline void bch2_btree_iter_verify_entry_exit(struct btree_iter *iter) in bch2_btree_iter_verify_entry_exit() argument
377 __bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_verify_entry_exit()
380 static inline int bch2_btree_iter_verify_ret(struct btree_trans *trans, struct btree_iter *iter, in bch2_btree_iter_verify_ret() argument
384 ? __bch2_btree_iter_verify_ret(trans, iter, k) in bch2_btree_iter_verify_ret()
390 static void btree_node_iter_set_set_pos(struct btree_node_iter *iter, in btree_node_iter_set_set_pos() argument
397 btree_node_iter_for_each(iter, set) in btree_node_iter_set_set_pos()
400 bch2_btree_node_iter_sort(iter, b); in btree_node_iter_set_set_pos()
404 bch2_btree_node_iter_push(iter, b, k, btree_bkey_last(b, t)); in btree_node_iter_set_set_pos()
413 if (where != bch2_btree_node_iter_peek_all(&l->iter, l->b)) in __bch2_btree_path_fix_key_modified()
417 bch2_btree_node_iter_advance(&l->iter, l->b); in __bch2_btree_path_fix_key_modified()
536 if (node_iter != &path->l[b->c.level].iter) { in bch2_btree_node_iter_fix()
546 &linked->l[b->c.level].iter, t, in bch2_btree_node_iter_fix()
576 bch2_btree_node_iter_peek_all(&l->iter, l->b)); in btree_path_level_peek_all()
585 bch2_btree_node_iter_prev(&l->iter, l->b)); in btree_path_level_prev()
600 while ((k = bch2_btree_node_iter_peek_all(&l->iter, l->b)) && in btree_path_advance_to_pos()
605 bch2_btree_node_iter_advance(&l->iter, l->b); in btree_path_advance_to_pos()
617 bch2_btree_node_iter_init(&l->iter, l->b, &path->pos); in __btree_path_level_init()
624 bch2_btree_node_iter_peek(&l->iter, l->b); in __btree_path_level_init()
800 struct btree_node_iter node_iter = l->iter; in btree_path_prefetch()
881 k = bch2_btree_node_iter_peek_all(&l->iter, l->b); in btree_node_mem_ptr_set()
901 __bch2_btree_and_journal_iter_init_node_iter(trans, &jiter, l->b, l->iter, path->pos); in btree_node_iter_and_journal_peek()
963 struct bkey_packed *k = bch2_btree_node_iter_peek(&l->iter, l->b); in btree_path_down()
1223 unsigned iter; in bch2_btree_path_traverse_one() local
1225 trans_for_each_path_with_node(trans, path_l(path)->b, linked, iter) in bch2_btree_path_traverse_one()
1322 bch2_btree_node_iter_init(&l->iter, l->b, &path->pos); in __bch2_btree_path_set_pos()
1329 bch2_btree_node_iter_peek(&l->iter, l->b); in __bch2_btree_path_set_pos()
1583 struct trans_for_each_path_inorder_iter iter; in __bch2_trans_paths_to_text() local
1588 trans_for_each_path_idx_inorder(trans, iter) { in __bch2_trans_paths_to_text()
1589 bch2_btree_path_to_text_short(out, trans, iter.path_idx); in __bch2_trans_paths_to_text()
1744 struct trans_for_each_path_inorder_iter iter; in bch2_path_get() local
1756 trans_for_each_path_inorder(trans, path, iter) { in bch2_path_get()
1764 path_pos = iter.path_idx; in bch2_path_get()
1844 _k = bch2_btree_node_iter_peek_all(&l->iter, l->b); in bch2_btree_path_peek_slot()
1870 void bch2_set_btree_iter_dontneed(struct btree_trans *trans, struct btree_iter *iter) in bch2_set_btree_iter_dontneed() argument
1872 if (!iter->path || trans->restarted) in bch2_set_btree_iter_dontneed()
1875 struct btree_path *path = btree_iter_path(trans, iter); in bch2_set_btree_iter_dontneed()
1883 __bch2_btree_iter_traverse(struct btree_trans *trans, struct btree_iter *iter) in __bch2_btree_iter_traverse() argument
1885 return bch2_btree_path_traverse(trans, iter->path, iter->flags); in __bch2_btree_iter_traverse()
1889 bch2_btree_iter_traverse(struct btree_trans *trans, struct btree_iter *iter) in bch2_btree_iter_traverse() argument
1893 iter->path = bch2_btree_path_set_pos(trans, iter->path, in bch2_btree_iter_traverse()
1894 btree_iter_search_key(iter), in bch2_btree_iter_traverse()
1895 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_traverse()
1896 btree_iter_ip_allocated(iter)); in bch2_btree_iter_traverse()
1898 int ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); in bch2_btree_iter_traverse()
1902 struct btree_path *path = btree_iter_path(trans, iter); in bch2_btree_iter_traverse()
1911 struct btree_iter *iter) in bch2_btree_iter_peek_node() argument
1916 EBUG_ON(trans->paths[iter->path].cached); in bch2_btree_iter_peek_node()
1917 bch2_btree_iter_verify(trans, iter); in bch2_btree_iter_peek_node()
1919 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); in bch2_btree_iter_peek_node()
1923 struct btree_path *path = btree_iter_path(trans, iter); in bch2_btree_iter_peek_node()
1928 BUG_ON(bpos_lt(b->key.k.p, iter->pos)); in bch2_btree_iter_peek_node()
1930 bkey_init(&iter->k); in bch2_btree_iter_peek_node()
1931 iter->k.p = iter->pos = b->key.k.p; in bch2_btree_iter_peek_node()
1933 iter->path = bch2_btree_path_set_pos(trans, iter->path, b->key.k.p, in bch2_btree_iter_peek_node()
1934 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_peek_node()
1935 btree_iter_ip_allocated(iter)); in bch2_btree_iter_peek_node()
1936 btree_path_set_should_be_locked(trans, btree_iter_path(trans, iter)); in bch2_btree_iter_peek_node()
1938 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_peek_node()
1939 bch2_btree_iter_verify(trans, iter); in bch2_btree_iter_peek_node()
1949 struct btree_iter *iter) in bch2_btree_iter_peek_node_and_restart() argument
1953 while (b = bch2_btree_iter_peek_node(trans, iter), in bch2_btree_iter_peek_node_and_restart()
1960 struct btree *bch2_btree_iter_next_node(struct btree_trans *trans, struct btree_iter *iter) in bch2_btree_iter_next_node() argument
1965 EBUG_ON(trans->paths[iter->path].cached); in bch2_btree_iter_next_node()
1967 bch2_btree_iter_verify(trans, iter); in bch2_btree_iter_next_node()
1969 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); in bch2_btree_iter_next_node()
1974 struct btree_path *path = btree_iter_path(trans, iter); in bch2_btree_iter_next_node()
2005 if (bpos_eq(iter->pos, b->key.k.p)) { in bch2_btree_iter_next_node()
2015 iter->path = bch2_btree_path_set_pos(trans, iter->path, in bch2_btree_iter_next_node()
2016 bpos_successor(iter->pos), in bch2_btree_iter_next_node()
2017 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_next_node()
2018 btree_iter_ip_allocated(iter)); in bch2_btree_iter_next_node()
2020 path = btree_iter_path(trans, iter); in bch2_btree_iter_next_node()
2021 btree_path_set_level_down(trans, path, iter->min_depth); in bch2_btree_iter_next_node()
2023 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); in bch2_btree_iter_next_node()
2027 path = btree_iter_path(trans, iter); in bch2_btree_iter_next_node()
2031 bkey_init(&iter->k); in bch2_btree_iter_next_node()
2032 iter->k.p = iter->pos = b->key.k.p; in bch2_btree_iter_next_node()
2034 iter->path = bch2_btree_path_set_pos(trans, iter->path, b->key.k.p, in bch2_btree_iter_next_node()
2035 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_next_node()
2036 btree_iter_ip_allocated(iter)); in bch2_btree_iter_next_node()
2037 btree_path_set_should_be_locked(trans, btree_iter_path(trans, iter)); in bch2_btree_iter_next_node()
2038 EBUG_ON(btree_iter_path(trans, iter)->uptodate); in bch2_btree_iter_next_node()
2040 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_next_node()
2041 bch2_btree_iter_verify(trans, iter); in bch2_btree_iter_next_node()
2051 inline bool bch2_btree_iter_advance(struct btree_trans *trans, struct btree_iter *iter) in bch2_btree_iter_advance() argument
2053 struct bpos pos = iter->k.p; in bch2_btree_iter_advance()
2054 bool ret = !(iter->flags & BTREE_ITER_all_snapshots in bch2_btree_iter_advance()
2058 if (ret && !(iter->flags & BTREE_ITER_is_extents)) in bch2_btree_iter_advance()
2059 pos = bkey_successor(iter, pos); in bch2_btree_iter_advance()
2060 bch2_btree_iter_set_pos(trans, iter, pos); in bch2_btree_iter_advance()
2064 inline bool bch2_btree_iter_rewind(struct btree_trans *trans, struct btree_iter *iter) in bch2_btree_iter_rewind() argument
2066 struct bpos pos = bkey_start_pos(&iter->k); in bch2_btree_iter_rewind()
2067 bool ret = !(iter->flags & BTREE_ITER_all_snapshots in bch2_btree_iter_rewind()
2071 if (ret && !(iter->flags & BTREE_ITER_is_extents)) in bch2_btree_iter_rewind()
2072 pos = bkey_predecessor(iter, pos); in bch2_btree_iter_rewind()
2073 bch2_btree_iter_set_pos(trans, iter, pos); in bch2_btree_iter_rewind()
2078 void bch2_btree_trans_peek_prev_updates(struct btree_trans *trans, struct btree_iter *iter, in bch2_btree_trans_peek_prev_updates() argument
2081 struct bpos end = path_l(btree_iter_path(trans, iter))->b->data->min_key; in bch2_btree_trans_peek_prev_updates()
2085 i->btree_id == iter->btree_id && in bch2_btree_trans_peek_prev_updates()
2088 iter->k = i->k->k; in bch2_btree_trans_peek_prev_updates()
2094 void bch2_btree_trans_peek_updates(struct btree_trans *trans, struct btree_iter *iter, in bch2_btree_trans_peek_updates() argument
2098 struct btree_path *path = btree_iter_path(trans, iter); in bch2_btree_trans_peek_updates()
2103 i->btree_id == iter->btree_id && in bch2_btree_trans_peek_updates()
2106 iter->k = i->k->k; in bch2_btree_trans_peek_updates()
2112 void bch2_btree_trans_peek_slot_updates(struct btree_trans *trans, struct btree_iter *iter, in bch2_btree_trans_peek_slot_updates() argument
2117 i->btree_id == iter->btree_id && in bch2_btree_trans_peek_slot_updates()
2118 bpos_eq(i->k->k.p, iter->pos)) { in bch2_btree_trans_peek_slot_updates()
2119 iter->k = i->k->k; in bch2_btree_trans_peek_slot_updates()
2125 struct btree_iter *iter, in bch2_btree_journal_peek() argument
2129 struct btree_path *path = btree_iter_path(trans, iter); in bch2_btree_journal_peek()
2131 return bch2_journal_keys_peek_max(trans->c, iter->btree_id, in bch2_btree_journal_peek()
2135 &iter->journal_idx); in bch2_btree_journal_peek()
2140 struct btree_iter *iter) in btree_trans_peek_slot_journal() argument
2142 struct btree_path *path = btree_iter_path(trans, iter); in btree_trans_peek_slot_journal()
2143 struct bkey_i *k = bch2_btree_journal_peek(trans, iter, path->pos, path->pos); in btree_trans_peek_slot_journal()
2146 iter->k = k->k; in btree_trans_peek_slot_journal()
2155 struct btree_iter *iter, in btree_trans_peek_journal() argument
2159 struct btree_path *path = btree_iter_path(trans, iter); in btree_trans_peek_journal()
2161 bch2_btree_journal_peek(trans, iter, search_key, in btree_trans_peek_journal()
2164 iter->k = next_journal->k; in btree_trans_peek_journal()
2170 struct btree_iter *iter, in bch2_btree_journal_peek_prev() argument
2174 struct btree_path *path = btree_iter_path(trans, iter); in bch2_btree_journal_peek_prev()
2176 return bch2_journal_keys_peek_prev_min(trans->c, iter->btree_id, in bch2_btree_journal_peek_prev()
2180 &iter->journal_idx); in bch2_btree_journal_peek_prev()
2185 struct btree_iter *iter, in btree_trans_peek_prev_journal() argument
2189 struct btree_path *path = btree_iter_path(trans, iter); in btree_trans_peek_prev_journal()
2191 bch2_btree_journal_peek_prev(trans, iter, search_key, in btree_trans_peek_prev_journal()
2195 iter->k = next_journal->k; in btree_trans_peek_prev_journal()
2205 struct bkey_s_c btree_trans_peek_key_cache(struct btree_trans *trans, struct btree_iter *iter, in btree_trans_peek_key_cache() argument
2215 if ((iter->flags & BTREE_ITER_key_cache_fill) && in btree_trans_peek_key_cache()
2216 bpos_eq(iter->pos, pos)) in btree_trans_peek_key_cache()
2219 if (!bch2_btree_key_cache_find(c, iter->btree_id, pos)) in btree_trans_peek_key_cache()
2222 if (!iter->key_cache_path) in btree_trans_peek_key_cache()
2223 iter->key_cache_path = bch2_path_get(trans, iter->btree_id, pos, in btree_trans_peek_key_cache()
2224 iter->flags & BTREE_ITER_intent, 0, in btree_trans_peek_key_cache()
2225 iter->flags|BTREE_ITER_cached| in btree_trans_peek_key_cache()
2229 iter->key_cache_path = bch2_btree_path_set_pos(trans, iter->key_cache_path, pos, in btree_trans_peek_key_cache()
2230 iter->flags & BTREE_ITER_intent, in btree_trans_peek_key_cache()
2231 btree_iter_ip_allocated(iter)); in btree_trans_peek_key_cache()
2233 ret = bch2_btree_path_traverse(trans, iter->key_cache_path, in btree_trans_peek_key_cache()
2234 iter->flags|BTREE_ITER_cached) ?: in btree_trans_peek_key_cache()
2235 bch2_btree_path_relock(trans, btree_iter_path(trans, iter), _THIS_IP_); in btree_trans_peek_key_cache()
2239 k = bch2_btree_path_peek_slot(trans->paths + iter->key_cache_path, &u); in btree_trans_peek_key_cache()
2243 if ((iter->flags & BTREE_ITER_all_snapshots) && in btree_trans_peek_key_cache()
2247 iter->k = u; in btree_trans_peek_key_cache()
2248 k.k = &iter->k; in btree_trans_peek_key_cache()
2249 btree_path_set_should_be_locked(trans, trans->paths + iter->key_cache_path); in btree_trans_peek_key_cache()
2253 static struct bkey_s_c __bch2_btree_iter_peek(struct btree_trans *trans, struct btree_iter *iter, in __bch2_btree_iter_peek() argument
2259 EBUG_ON(btree_iter_path(trans, iter)->cached); in __bch2_btree_iter_peek()
2260 bch2_btree_iter_verify(trans, iter); in __bch2_btree_iter_peek()
2263 iter->path = bch2_btree_path_set_pos(trans, iter->path, search_key, in __bch2_btree_iter_peek()
2264 iter->flags & BTREE_ITER_intent, in __bch2_btree_iter_peek()
2265 btree_iter_ip_allocated(iter)); in __bch2_btree_iter_peek()
2267 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); in __bch2_btree_iter_peek()
2270 bch2_btree_iter_set_pos(trans, iter, iter->pos); in __bch2_btree_iter_peek()
2275 struct btree_path *path = btree_iter_path(trans, iter); in __bch2_btree_iter_peek()
2280 bch2_btree_iter_set_pos(trans, iter, SPOS_MAX); in __bch2_btree_iter_peek()
2287 k = btree_path_level_peek_all(trans->c, l, &iter->k); in __bch2_btree_iter_peek()
2289 if (unlikely(iter->flags & BTREE_ITER_with_key_cache) && in __bch2_btree_iter_peek()
2291 (k2 = btree_trans_peek_key_cache(trans, iter, k.k->p)).k) { in __bch2_btree_iter_peek()
2294 bch2_btree_iter_set_pos(trans, iter, iter->pos); in __bch2_btree_iter_peek()
2299 if (unlikely(iter->flags & BTREE_ITER_with_journal)) in __bch2_btree_iter_peek()
2300 btree_trans_peek_journal(trans, iter, search_key, &k); in __bch2_btree_iter_peek()
2302 if (unlikely((iter->flags & BTREE_ITER_with_updates) && in __bch2_btree_iter_peek()
2304 bch2_btree_trans_peek_updates(trans, iter, search_key, &k); in __bch2_btree_iter_peek()
2327 bch2_btree_iter_set_pos(trans, iter, SPOS_MAX); in __bch2_btree_iter_peek()
2333 bch2_btree_iter_verify(trans, iter); in __bch2_btree_iter_peek()
2360 struct bkey_s_c bch2_btree_iter_peek_max(struct btree_trans *trans, struct btree_iter *iter, in bch2_btree_iter_peek_max() argument
2363 struct bpos search_key = btree_iter_search_key(iter); in bch2_btree_iter_peek_max()
2365 struct bpos iter_pos = iter->pos; in bch2_btree_iter_peek_max()
2369 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_peek_max()
2370 EBUG_ON((iter->flags & BTREE_ITER_filter_snapshots) && bkey_eq(end, POS_MAX)); in bch2_btree_iter_peek_max()
2378 if (iter->update_path) { in bch2_btree_iter_peek_max()
2379 bch2_path_put(trans, iter->update_path, iter->flags & BTREE_ITER_intent); in bch2_btree_iter_peek_max()
2380 iter->update_path = 0; in bch2_btree_iter_peek_max()
2384 k = __bch2_btree_iter_peek(trans, iter, search_key); in bch2_btree_iter_peek_max()
2390 if (iter->flags & BTREE_ITER_filter_snapshots) { in bch2_btree_iter_peek_max()
2401 if (unlikely(!(iter->flags & BTREE_ITER_is_extents) in bch2_btree_iter_peek_max()
2406 if (iter->update_path && in bch2_btree_iter_peek_max()
2407 !bkey_eq(trans->paths[iter->update_path].pos, k.k->p)) { in bch2_btree_iter_peek_max()
2408 bch2_path_put(trans, iter->update_path, in bch2_btree_iter_peek_max()
2409 iter->flags & BTREE_ITER_intent); in bch2_btree_iter_peek_max()
2410 iter->update_path = 0; in bch2_btree_iter_peek_max()
2413 if ((iter->flags & BTREE_ITER_intent) && in bch2_btree_iter_peek_max()
2414 !(iter->flags & BTREE_ITER_is_extents) && in bch2_btree_iter_peek_max()
2415 !iter->update_path) { in bch2_btree_iter_peek_max()
2418 if (pos.snapshot < iter->snapshot) { in bch2_btree_iter_peek_max()
2423 pos.snapshot = iter->snapshot; in bch2_btree_iter_peek_max()
2429 __btree_path_get(trans, trans->paths + iter->path, iter->flags & BTREE_ITER_intent); in bch2_btree_iter_peek_max()
2430 iter->update_path = iter->path; in bch2_btree_iter_peek_max()
2432 iter->update_path = bch2_btree_path_set_pos(trans, in bch2_btree_iter_peek_max()
2433 iter->update_path, pos, in bch2_btree_iter_peek_max()
2434 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_peek_max()
2436 ret = bch2_btree_path_traverse(trans, iter->update_path, iter->flags); in bch2_btree_iter_peek_max()
2448 iter->snapshot, in bch2_btree_iter_peek_max()
2455 !(iter->flags & BTREE_ITER_key_cache_fill)) { in bch2_btree_iter_peek_max()
2456 search_key = bkey_successor(iter, k.k->p); in bch2_btree_iter_peek_max()
2466 if (!(iter->flags & BTREE_ITER_is_extents)) in bch2_btree_iter_peek_max()
2469 iter_pos = bkey_max(iter->pos, bkey_start_pos(k.k)); in bch2_btree_iter_peek_max()
2471 if (unlikely(iter->flags & BTREE_ITER_all_snapshots ? bpos_gt(iter_pos, end) : in bch2_btree_iter_peek_max()
2472 iter->flags & BTREE_ITER_is_extents ? bkey_ge(iter_pos, end) : in bch2_btree_iter_peek_max()
2479 iter->pos = iter_pos; in bch2_btree_iter_peek_max()
2481 iter->path = bch2_btree_path_set_pos(trans, iter->path, k.k->p, in bch2_btree_iter_peek_max()
2482 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_peek_max()
2483 btree_iter_ip_allocated(iter)); in bch2_btree_iter_peek_max()
2485 btree_path_set_should_be_locked(trans, btree_iter_path(trans, iter)); in bch2_btree_iter_peek_max()
2487 if (iter->update_path) { in bch2_btree_iter_peek_max()
2488 ret = bch2_btree_path_relock(trans, trans->paths + iter->update_path, _THIS_IP_); in bch2_btree_iter_peek_max()
2492 btree_path_set_should_be_locked(trans, trans->paths + iter->update_path); in bch2_btree_iter_peek_max()
2495 if (!(iter->flags & BTREE_ITER_all_snapshots)) in bch2_btree_iter_peek_max()
2496 iter->pos.snapshot = iter->snapshot; in bch2_btree_iter_peek_max()
2498 ret = bch2_btree_iter_verify_ret(trans, iter, k); in bch2_btree_iter_peek_max()
2500 bch2_btree_iter_set_pos(trans, iter, iter->pos); in bch2_btree_iter_peek_max()
2504 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_peek_max()
2521 bch2_btree_iter_set_pos(trans, iter, end); in bch2_btree_iter_peek_max()
2534 struct bkey_s_c bch2_btree_iter_next(struct btree_trans *trans, struct btree_iter *iter) in bch2_btree_iter_next() argument
2536 if (!bch2_btree_iter_advance(trans, iter)) in bch2_btree_iter_next()
2539 return bch2_btree_iter_peek(trans, iter); in bch2_btree_iter_next()
2542 …tic struct bkey_s_c __bch2_btree_iter_peek_prev(struct btree_trans *trans, struct btree_iter *iter, in __bch2_btree_iter_peek_prev() argument
2547 bch2_btree_iter_verify(trans, iter); in __bch2_btree_iter_peek_prev()
2550 iter->path = bch2_btree_path_set_pos(trans, iter->path, search_key, in __bch2_btree_iter_peek_prev()
2551 iter->flags & BTREE_ITER_intent, in __bch2_btree_iter_peek_prev()
2552 btree_iter_ip_allocated(iter)); in __bch2_btree_iter_peek_prev()
2554 int ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); in __bch2_btree_iter_peek_prev()
2557 bch2_btree_iter_set_pos(trans, iter, iter->pos); in __bch2_btree_iter_peek_prev()
2562 struct btree_path *path = btree_iter_path(trans, iter); in __bch2_btree_iter_peek_prev()
2567 bch2_btree_iter_set_pos(trans, iter, SPOS_MAX); in __bch2_btree_iter_peek_prev()
2574 k = btree_path_level_peek_all(trans->c, l, &iter->k); in __bch2_btree_iter_peek_prev()
2576 k = btree_path_level_prev(trans, path, l, &iter->k); in __bch2_btree_iter_peek_prev()
2581 if (unlikely(iter->flags & BTREE_ITER_with_key_cache) && in __bch2_btree_iter_peek_prev()
2583 (k2 = btree_trans_peek_key_cache(trans, iter, k.k->p)).k) { in __bch2_btree_iter_peek_prev()
2586 bch2_btree_iter_set_pos(trans, iter, iter->pos); in __bch2_btree_iter_peek_prev()
2591 if (unlikely(iter->flags & BTREE_ITER_with_journal)) in __bch2_btree_iter_peek_prev()
2592 btree_trans_peek_prev_journal(trans, iter, search_key, &k); in __bch2_btree_iter_peek_prev()
2594 if (unlikely((iter->flags & BTREE_ITER_with_updates) && in __bch2_btree_iter_peek_prev()
2596 bch2_btree_trans_peek_prev_updates(trans, iter, search_key, &k); in __bch2_btree_iter_peek_prev()
2607 bch2_btree_iter_set_pos(trans, iter, POS_MIN); in __bch2_btree_iter_peek_prev()
2613 bch2_btree_iter_verify(trans, iter); in __bch2_btree_iter_peek_prev()
2626 struct bkey_s_c bch2_btree_iter_peek_prev_min(struct btree_trans *trans, struct btree_iter *iter, in bch2_btree_iter_peek_prev_min() argument
2629 if ((iter->flags & (BTREE_ITER_is_extents|BTREE_ITER_filter_snapshots)) && in bch2_btree_iter_peek_prev_min()
2630 !bkey_eq(iter->pos, POS_MAX) && in bch2_btree_iter_peek_prev_min()
2631 !((iter->flags & BTREE_ITER_is_extents) && in bch2_btree_iter_peek_prev_min()
2632 iter->pos.offset == U64_MAX)) { in bch2_btree_iter_peek_prev_min()
2642 struct bkey_s_c k = bch2_btree_iter_peek_slot(trans, iter); in bch2_btree_iter_peek_prev_min()
2647 (!(iter->flags & BTREE_ITER_is_extents) || in bch2_btree_iter_peek_prev_min()
2648 bkey_lt(bkey_start_pos(k.k), iter->pos))) in bch2_btree_iter_peek_prev_min()
2652 struct bpos search_key = iter->pos; in bch2_btree_iter_peek_prev_min()
2657 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_peek_prev_min()
2658 EBUG_ON((iter->flags & BTREE_ITER_filter_snapshots) && iter->pos.inode != end.inode); in bch2_btree_iter_peek_prev_min()
2667 k = __bch2_btree_iter_peek_prev(trans, iter, search_key); in bch2_btree_iter_peek_prev_min()
2673 if (iter->flags & BTREE_ITER_filter_snapshots) { in bch2_btree_iter_peek_prev_min()
2675 if (s && bpos_lt(k.k->p, SPOS(s->pos.inode, s->pos.offset, iter->snapshot))) { in bch2_btree_iter_peek_prev_min()
2681 bch2_path_put(trans, iter->path, in bch2_btree_iter_peek_prev_min()
2682 iter->flags & BTREE_ITER_intent); in bch2_btree_iter_peek_prev_min()
2683 iter->path = saved_path; in bch2_btree_iter_peek_prev_min()
2685 k = bch2_btree_path_peek_slot(btree_iter_path(trans, iter), &iter->k); in bch2_btree_iter_peek_prev_min()
2698 if (!bch2_snapshot_is_ancestor(trans->c, iter->snapshot, k.k->p.snapshot)) { in bch2_btree_iter_peek_prev_min()
2703 if (k.k->p.snapshot != iter->snapshot) { in bch2_btree_iter_peek_prev_min()
2712 iter->flags & BTREE_ITER_intent); in bch2_btree_iter_peek_prev_min()
2717 saved_path = btree_path_clone(trans, iter->path, in bch2_btree_iter_peek_prev_min()
2718 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_peek_prev_min()
2721 trans->paths + iter->path, in bch2_btree_iter_peek_prev_min()
2730 search_key = bkey_predecessor(iter, k.k->p); in bch2_btree_iter_peek_prev_min()
2736 EBUG_ON(iter->flags & BTREE_ITER_all_snapshots ? bpos_gt(k.k->p, iter->pos) : in bch2_btree_iter_peek_prev_min()
2737 iter->flags & BTREE_ITER_is_extents ? bkey_ge(bkey_start_pos(k.k), iter->pos) : in bch2_btree_iter_peek_prev_min()
2738 bkey_gt(k.k->p, iter->pos)); in bch2_btree_iter_peek_prev_min()
2740 if (unlikely(iter->flags & BTREE_ITER_all_snapshots ? bpos_lt(k.k->p, end) : in bch2_btree_iter_peek_prev_min()
2741 iter->flags & BTREE_ITER_is_extents ? bkey_le(k.k->p, end) : in bch2_btree_iter_peek_prev_min()
2749 iter->pos = bpos_min(iter->pos, k.k->p);; in bch2_btree_iter_peek_prev_min()
2751 if (iter->flags & BTREE_ITER_filter_snapshots) in bch2_btree_iter_peek_prev_min()
2752 iter->pos.snapshot = iter->snapshot; in bch2_btree_iter_peek_prev_min()
2755 bch2_path_put(trans, saved_path, iter->flags & BTREE_ITER_intent); in bch2_btree_iter_peek_prev_min()
2757 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_peek_prev_min()
2758 bch2_btree_iter_verify(trans, iter); in bch2_btree_iter_peek_prev_min()
2774 bch2_btree_iter_set_pos(trans, iter, end); in bch2_btree_iter_peek_prev_min()
2787 struct bkey_s_c bch2_btree_iter_prev(struct btree_trans *trans, struct btree_iter *iter) in bch2_btree_iter_prev() argument
2789 if (!bch2_btree_iter_rewind(trans, iter)) in bch2_btree_iter_prev()
2792 return bch2_btree_iter_peek_prev(trans, iter); in bch2_btree_iter_prev()
2795 struct bkey_s_c bch2_btree_iter_peek_slot(struct btree_trans *trans, struct btree_iter *iter) in bch2_btree_iter_peek_slot() argument
2802 bch2_btree_iter_verify(trans, iter); in bch2_btree_iter_peek_slot()
2803 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_peek_slot()
2804 EBUG_ON(btree_iter_path(trans, iter)->level && (iter->flags & BTREE_ITER_with_key_cache)); in bch2_btree_iter_peek_slot()
2813 if ((iter->flags & BTREE_ITER_is_extents) && in bch2_btree_iter_peek_slot()
2814 unlikely(iter->pos.offset == KEY_OFFSET_MAX)) { in bch2_btree_iter_peek_slot()
2815 if (iter->pos.inode == KEY_INODE_MAX) { in bch2_btree_iter_peek_slot()
2820 bch2_btree_iter_set_pos(trans, iter, bpos_nosnap_successor(iter->pos)); in bch2_btree_iter_peek_slot()
2823 search_key = btree_iter_search_key(iter); in bch2_btree_iter_peek_slot()
2824 iter->path = bch2_btree_path_set_pos(trans, iter->path, search_key, in bch2_btree_iter_peek_slot()
2825 iter->flags & BTREE_ITER_intent, in bch2_btree_iter_peek_slot()
2826 btree_iter_ip_allocated(iter)); in bch2_btree_iter_peek_slot()
2828 ret = bch2_btree_path_traverse(trans, iter->path, iter->flags); in bch2_btree_iter_peek_slot()
2834 struct btree_path *path = btree_iter_path(trans, iter); in bch2_btree_iter_peek_slot()
2842 if ((iter->flags & BTREE_ITER_cached) || in bch2_btree_iter_peek_slot()
2843 !(iter->flags & (BTREE_ITER_is_extents|BTREE_ITER_filter_snapshots))) { in bch2_btree_iter_peek_slot()
2846 if (unlikely((iter->flags & BTREE_ITER_with_updates) && in bch2_btree_iter_peek_slot()
2848 bch2_btree_trans_peek_slot_updates(trans, iter, &k); in bch2_btree_iter_peek_slot()
2853 if (unlikely(iter->flags & BTREE_ITER_with_journal) && in bch2_btree_iter_peek_slot()
2854 (k = btree_trans_peek_slot_journal(trans, iter)).k) in bch2_btree_iter_peek_slot()
2857 if (unlikely(iter->flags & BTREE_ITER_with_key_cache) && in bch2_btree_iter_peek_slot()
2858 (k = btree_trans_peek_key_cache(trans, iter, iter->pos)).k) { in bch2_btree_iter_peek_slot()
2860 iter->k = *k.k; in bch2_btree_iter_peek_slot()
2865 k = bch2_btree_path_peek_slot(btree_iter_path(trans, iter), &iter->k); in bch2_btree_iter_peek_slot()
2870 (iter->flags & BTREE_ITER_filter_snapshots) && in bch2_btree_iter_peek_slot()
2871 !(iter->flags & BTREE_ITER_key_cache_fill))) in bch2_btree_iter_peek_slot()
2872 iter->k.type = KEY_TYPE_deleted; in bch2_btree_iter_peek_slot()
2875 struct bpos end = iter->pos; in bch2_btree_iter_peek_slot()
2877 if (iter->flags & BTREE_ITER_is_extents) in bch2_btree_iter_peek_slot()
2880 EBUG_ON(btree_iter_path(trans, iter)->level); in bch2_btree_iter_peek_slot()
2882 if (iter->flags & BTREE_ITER_intent) { in bch2_btree_iter_peek_slot()
2885 bch2_trans_copy_iter(trans, &iter2, iter); in bch2_btree_iter_peek_slot()
2889 swap(iter->key_cache_path, iter2.key_cache_path); in bch2_btree_iter_peek_slot()
2890 iter->k = iter2.k; in bch2_btree_iter_peek_slot()
2891 k.k = &iter->k; in bch2_btree_iter_peek_slot()
2895 struct bpos pos = iter->pos; in bch2_btree_iter_peek_slot()
2897 k = bch2_btree_iter_peek_max(trans, iter, end); in bch2_btree_iter_peek_slot()
2899 bch2_btree_iter_set_pos(trans, iter, pos); in bch2_btree_iter_peek_slot()
2901 iter->pos = pos; in bch2_btree_iter_peek_slot()
2909 if (bkey_lt(iter->pos, next)) { in bch2_btree_iter_peek_slot()
2910 bkey_init(&iter->k); in bch2_btree_iter_peek_slot()
2911 iter->k.p = iter->pos; in bch2_btree_iter_peek_slot()
2913 if (iter->flags & BTREE_ITER_is_extents) { in bch2_btree_iter_peek_slot()
2914 bch2_key_resize(&iter->k, in bch2_btree_iter_peek_slot()
2916 (next.inode == iter->pos.inode in bch2_btree_iter_peek_slot()
2919 iter->pos.offset)); in bch2_btree_iter_peek_slot()
2920 EBUG_ON(!iter->k.size); in bch2_btree_iter_peek_slot()
2923 k = (struct bkey_s_c) { &iter->k, NULL }; in bch2_btree_iter_peek_slot()
2927 bch2_btree_iter_verify_entry_exit(iter); in bch2_btree_iter_peek_slot()
2928 bch2_btree_iter_verify(trans, iter); in bch2_btree_iter_peek_slot()
2929 ret = bch2_btree_iter_verify_ret(trans, iter, k); in bch2_btree_iter_peek_slot()
2949 struct bkey_s_c bch2_btree_iter_next_slot(struct btree_trans *trans, struct btree_iter *iter) in bch2_btree_iter_next_slot() argument
2951 if (!bch2_btree_iter_advance(trans, iter)) in bch2_btree_iter_next_slot()
2954 return bch2_btree_iter_peek_slot(trans, iter); in bch2_btree_iter_next_slot()
2957 struct bkey_s_c bch2_btree_iter_prev_slot(struct btree_trans *trans, struct btree_iter *iter) in bch2_btree_iter_prev_slot() argument
2959 if (!bch2_btree_iter_rewind(trans, iter)) in bch2_btree_iter_prev_slot()
2962 return bch2_btree_iter_peek_slot(trans, iter); in bch2_btree_iter_prev_slot()
2966 …y_s_c bch2_btree_iter_peek_and_restart_outlined(struct btree_trans *trans, struct btree_iter *iter) in bch2_btree_iter_peek_and_restart_outlined() argument
2971 (k = bch2_btree_iter_peek_type(trans, iter, iter->flags), in bch2_btree_iter_peek_and_restart_outlined()
3004 struct trans_for_each_path_inorder_iter iter; in btree_trans_verify_sorted() local
3009 trans_for_each_path_inorder(trans, path, iter) { in btree_trans_verify_sorted()
3105 void bch2_trans_iter_exit(struct btree_trans *trans, struct btree_iter *iter) in bch2_trans_iter_exit() argument
3107 if (iter->update_path) in bch2_trans_iter_exit()
3108 bch2_path_put(trans, iter->update_path, in bch2_trans_iter_exit()
3109 iter->flags & BTREE_ITER_intent); in bch2_trans_iter_exit()
3110 if (iter->path) in bch2_trans_iter_exit()
3111 bch2_path_put(trans, iter->path, in bch2_trans_iter_exit()
3112 iter->flags & BTREE_ITER_intent); in bch2_trans_iter_exit()
3113 if (iter->key_cache_path) in bch2_trans_iter_exit()
3114 bch2_path_put(trans, iter->key_cache_path, in bch2_trans_iter_exit()
3115 iter->flags & BTREE_ITER_intent); in bch2_trans_iter_exit()
3116 iter->path = 0; in bch2_trans_iter_exit()
3117 iter->update_path = 0; in bch2_trans_iter_exit()
3118 iter->key_cache_path = 0; in bch2_trans_iter_exit()
3122 struct btree_iter *iter, in bch2_trans_iter_init_outlined() argument
3126 bch2_trans_iter_init_common(trans, iter, btree_id, pos, 0, 0, in bch2_trans_iter_init_outlined()
3132 struct btree_iter *iter, in bch2_trans_node_iter_init() argument
3146 bch2_trans_iter_init_common(trans, iter, btree_id, pos, locks_want, depth, in bch2_trans_node_iter_init()
3150 iter->min_depth = depth; in bch2_trans_node_iter_init()
3152 struct btree_path *path = btree_iter_path(trans, iter); in bch2_trans_node_iter_init()
3155 BUG_ON(iter->min_depth != depth); in bch2_trans_node_iter_init()