Lines Matching refs:trans
46 static void verify_update_old_key(struct btree_trans *trans, struct btree_insert_entry *i) in verify_update_old_key() argument
49 struct bch_fs *c = trans->c; in verify_update_old_key()
51 struct bkey_s_c k = bch2_btree_path_peek_slot_exact(trans->paths + i->path, &u); in verify_update_old_key()
53 if (unlikely(trans->journal_replay_not_finished)) { in verify_update_old_key()
69 static inline struct btree_path_level *insert_l(struct btree_trans *trans, struct btree_insert_entr… in insert_l() argument
71 return (trans->paths + i->path)->l + i->level; in insert_l()
74 static inline bool same_leaf_as_prev(struct btree_trans *trans, in same_leaf_as_prev() argument
77 return i != trans->updates && in same_leaf_as_prev()
78 insert_l(trans, &i[0])->b == insert_l(trans, &i[-1])->b; in same_leaf_as_prev()
81 static inline bool same_leaf_as_next(struct btree_trans *trans, in same_leaf_as_next() argument
84 return i + 1 < trans->updates + trans->nr_updates && in same_leaf_as_next()
85 insert_l(trans, &i[0])->b == insert_l(trans, &i[1])->b; in same_leaf_as_next()
88 inline void bch2_btree_node_prep_for_write(struct btree_trans *trans, in bch2_btree_node_prep_for_write() argument
92 struct bch_fs *c = trans->c; in bch2_btree_node_prep_for_write()
96 bch2_trans_node_reinit_iter(trans, b); in bch2_btree_node_prep_for_write()
103 bch2_btree_init_next(trans, b); in bch2_btree_node_prep_for_write()
106 static noinline int trans_lock_write_fail(struct btree_trans *trans, struct btree_insert_entry *i) in trans_lock_write_fail() argument
108 while (--i >= trans->updates) { in trans_lock_write_fail()
109 if (same_leaf_as_prev(trans, i)) in trans_lock_write_fail()
112 bch2_btree_node_unlock_write(trans, trans->paths + i->path, insert_l(trans, i)->b); in trans_lock_write_fail()
115 trace_and_count(trans->c, trans_restart_would_deadlock_write, trans); in trans_lock_write_fail()
116 return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock_write); in trans_lock_write_fail()
119 static inline int bch2_trans_lock_write(struct btree_trans *trans) in bch2_trans_lock_write() argument
121 EBUG_ON(trans->write_locked); in bch2_trans_lock_write()
123 trans_for_each_update(trans, i) { in bch2_trans_lock_write()
124 if (same_leaf_as_prev(trans, i)) in bch2_trans_lock_write()
127 if (bch2_btree_node_lock_write(trans, trans->paths + i->path, &insert_l(trans, i)->b->c)) in bch2_trans_lock_write()
128 return trans_lock_write_fail(trans, i); in bch2_trans_lock_write()
131 bch2_btree_node_prep_for_write(trans, trans->paths + i->path, insert_l(trans, i)->b); in bch2_trans_lock_write()
134 trans->write_locked = true; in bch2_trans_lock_write()
138 static inline void bch2_trans_unlock_updates_write(struct btree_trans *trans) in bch2_trans_unlock_updates_write() argument
140 if (likely(trans->write_locked)) { in bch2_trans_unlock_updates_write()
141 trans_for_each_update(trans, i) in bch2_trans_unlock_updates_write()
142 if (btree_node_locked_type(trans->paths + i->path, i->level) == in bch2_trans_unlock_updates_write()
144 bch2_btree_node_unlock_write_inlined(trans, in bch2_trans_unlock_updates_write()
145 trans->paths + i->path, insert_l(trans, i)->b); in bch2_trans_unlock_updates_write()
146 trans->write_locked = false; in bch2_trans_unlock_updates_write()
153 bool bch2_btree_bset_insert_key(struct btree_trans *trans, in bch2_btree_bset_insert_key() argument
196 bch2_btree_path_fix_key_modified(trans, b, k); in bch2_btree_bset_insert_key()
214 bch2_btree_path_fix_key_modified(trans, b, k); in bch2_btree_bset_insert_key()
224 bch2_btree_node_iter_fix(trans, path, b, node_iter, k, in bch2_btree_bset_insert_key()
235 struct btree_trans *trans = bch2_trans_get(c); in __btree_node_flush() local
239 btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read); in __btree_node_flush()
255 btree_node_write_if_need(trans, b, SIX_LOCK_read); in __btree_node_flush()
258 bch2_trans_put(trans); in __btree_node_flush()
290 inline void bch2_btree_insert_key_leaf(struct btree_trans *trans, in bch2_btree_insert_key_leaf() argument
295 struct bch_fs *c = trans->c; in bch2_btree_insert_key_leaf()
303 if (unlikely(!bch2_btree_bset_insert_key(trans, path, b, in bch2_btree_insert_key_leaf()
326 bch2_trans_node_reinit_iter(trans, b); in bch2_btree_insert_key_leaf()
333 static inline void btree_insert_entry_checks(struct btree_trans *trans, in btree_insert_entry_checks() argument
336 struct btree_path *path = trans->paths + i->path; in btree_insert_entry_checks()
346 test_bit(JOURNAL_replay_done, &trans->c->journal.flags) && in btree_insert_entry_checks()
348 bch2_snapshot_is_internal_node(trans->c, i->k->k.p.snapshot) > 0); in btree_insert_entry_checks()
351 static __always_inline int bch2_trans_journal_res_get(struct btree_trans *trans, in bch2_trans_journal_res_get() argument
354 return bch2_journal_res_get(&trans->c->journal, &trans->journal_res, in bch2_trans_journal_res_get()
355 trans->journal_u64s, flags, trans); in bch2_trans_journal_res_get()
360 static noinline void journal_transaction_name(struct btree_trans *trans) in journal_transaction_name() argument
362 struct bch_fs *c = trans->c; in journal_transaction_name()
365 bch2_journal_add_entry(j, &trans->journal_res, in journal_transaction_name()
372 trans->fn, strlen(trans->fn), 0); in journal_transaction_name()
375 static inline int btree_key_can_insert(struct btree_trans *trans, in btree_key_can_insert() argument
379 return bch_err_throw(trans->c, btree_insert_btree_node_full); in btree_key_can_insert()
385 btree_key_can_insert_cached_slowpath(struct btree_trans *trans, unsigned flags, in btree_key_can_insert_cached_slowpath() argument
392 bch2_trans_unlock_updates_write(trans); in btree_key_can_insert_cached_slowpath()
393 bch2_trans_unlock(trans); in btree_key_can_insert_cached_slowpath()
397 struct bch_fs *c = trans->c; in btree_key_can_insert_cached_slowpath()
403 ret = bch2_trans_relock(trans) ?: in btree_key_can_insert_cached_slowpath()
404 bch2_trans_lock_write(trans); in btree_key_can_insert_cached_slowpath()
412 trans_for_each_update(trans, i) in btree_key_can_insert_cached_slowpath()
422 static int btree_key_can_insert_cached(struct btree_trans *trans, unsigned flags, in btree_key_can_insert_cached() argument
425 struct bch_fs *c = trans->c; in btree_key_can_insert_cached()
450 return btree_key_can_insert_cached_slowpath(trans, flags, path, new_u64s); in btree_key_can_insert_cached()
452 trans_for_each_update(trans, i) in btree_key_can_insert_cached()
463 static int run_one_mem_trigger(struct btree_trans *trans, in run_one_mem_trigger() argument
467 verify_update_old_key(trans, i); in run_one_mem_trigger()
478 return bch2_key_trigger(trans, i->btree_id, i->level, in run_one_mem_trigger()
482 return bch2_key_trigger_new(trans, i->btree_id, i->level, in run_one_mem_trigger()
484 bch2_key_trigger_old(trans, i->btree_id, i->level, in run_one_mem_trigger()
488 static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_entry *i) in run_one_trans_trigger() argument
490 verify_update_old_key(trans, i); in run_one_trans_trigger()
512 return bch2_key_trigger(trans, i->btree_id, i->level, old, bkey_i_to_s(i->k), in run_one_trans_trigger()
517 return bch2_key_trigger_old(trans, i->btree_id, i->level, old, flags) ?: 1; in run_one_trans_trigger()
520 return bch2_key_trigger_new(trans, i->btree_id, i->level, bkey_i_to_s(i->k), flags) ?: 1; in run_one_trans_trigger()
526 static int bch2_trans_commit_run_triggers(struct btree_trans *trans) in bch2_trans_commit_run_triggers() argument
530 while (sort_id_start < trans->nr_updates) { in bch2_trans_commit_run_triggers()
531 unsigned i, sort_id = trans->updates[sort_id_start].sort_order; in bch2_trans_commit_run_triggers()
547 i < trans->nr_updates && trans->updates[i].sort_order <= sort_id; in bch2_trans_commit_run_triggers()
549 if (trans->updates[i].sort_order < sort_id) { in bch2_trans_commit_run_triggers()
554 int ret = run_one_trans_trigger(trans, trans->updates + i); in bch2_trans_commit_run_triggers()
566 trans_for_each_update(trans, i) in bch2_trans_commit_run_triggers()
574 static noinline int bch2_trans_commit_run_gc_triggers(struct btree_trans *trans) in bch2_trans_commit_run_gc_triggers() argument
576 trans_for_each_update(trans, i) in bch2_trans_commit_run_gc_triggers()
578 gc_visited(trans->c, gc_pos_btree(i->btree_id, i->level, i->k->k.p))) { in bch2_trans_commit_run_gc_triggers()
579 int ret = run_one_mem_trigger(trans, i, i->flags|BTREE_TRIGGER_gc); in bch2_trans_commit_run_gc_triggers()
588 bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags, in bch2_trans_commit_write_locked() argument
592 struct bch_fs *c = trans->c; in bch2_trans_commit_write_locked()
597 bch2_trans_verify_not_unlocked_or_in_restart(trans); in bch2_trans_commit_write_locked()
601 trace_and_count(c, trans_restart_fault_inject, trans, trace_ip); in bch2_trans_commit_write_locked()
602 return btree_trans_restart(trans, BCH_ERR_transaction_restart_fault_inject); in bch2_trans_commit_write_locked()
611 prefetch(&trans->c->journal.flags); in bch2_trans_commit_write_locked()
613 trans_for_each_update(trans, i) { in bch2_trans_commit_write_locked()
615 if (!same_leaf_as_prev(trans, i)) in bch2_trans_commit_write_locked()
620 ? btree_key_can_insert(trans, insert_l(trans, i)->b, u64s) in bch2_trans_commit_write_locked()
621 : btree_key_can_insert_cached(trans, flags, trans->paths + i->path, u64s); in bch2_trans_commit_write_locked()
635 ret = bch2_trans_journal_res_get(trans, in bch2_trans_commit_write_locked()
641 if (unlikely(trans->journal_transaction_names)) in bch2_trans_commit_write_locked()
642 journal_transaction_name(trans); in bch2_trans_commit_write_locked()
653 trans_for_each_update(trans, i) in bch2_trans_commit_write_locked()
654 i->k->k.bversion.lo = trans->journal_res.seq; in bch2_trans_commit_write_locked()
656 trans_for_each_update(trans, i) in bch2_trans_commit_write_locked()
660 h = trans->hooks; in bch2_trans_commit_write_locked()
662 ret = h->fn(trans, h); in bch2_trans_commit_write_locked()
671 for (accounting = btree_trans_subbuf_base(trans, &trans->accounting); in bch2_trans_commit_write_locked()
672 accounting != btree_trans_subbuf_top(trans, &trans->accounting); in bch2_trans_commit_write_locked()
674 ret = bch2_accounting_trans_commit_hook(trans, in bch2_trans_commit_write_locked()
682 bch2_trans_account_disk_usage_change(trans); in bch2_trans_commit_write_locked()
684 trans_for_each_update(trans, i) in bch2_trans_commit_write_locked()
686 ret = run_one_mem_trigger(trans, i, BTREE_TRIGGER_atomic|i->flags); in bch2_trans_commit_write_locked()
692 ret = bch2_trans_commit_run_gc_triggers(trans); in bch2_trans_commit_write_locked()
702 for (struct jset_entry *i = btree_trans_journal_entries_start(trans); in bch2_trans_commit_write_locked()
703 i != btree_trans_journal_entries_top(trans); in bch2_trans_commit_write_locked()
709 bch2_trans_inconsistent(trans, "invalid journal entry on insert from %s\n", in bch2_trans_commit_write_locked()
710 trans->fn); in bch2_trans_commit_write_locked()
715 trans_for_each_update(trans, i) { in bch2_trans_commit_write_locked()
721 bch2_trans_inconsistent(trans, "invalid bkey on insert from %s -> %ps\n", in bch2_trans_commit_write_locked()
722 trans->fn, (void *) i->ip_allocated); in bch2_trans_commit_write_locked()
725 btree_insert_entry_checks(trans, i); in bch2_trans_commit_write_locked()
732 trans_for_each_update(trans, i) { in bch2_trans_commit_write_locked()
739 verify_update_old_key(trans, i); in bch2_trans_commit_write_locked()
741 if (trans->journal_transaction_names) { in bch2_trans_commit_write_locked()
742 entry = bch2_journal_add_entry(j, &trans->journal_res, in bch2_trans_commit_write_locked()
750 entry = bch2_journal_add_entry(j, &trans->journal_res, in bch2_trans_commit_write_locked()
757 memcpy_u64s_small(journal_res_entry(&c->journal, &trans->journal_res), in bch2_trans_commit_write_locked()
758 btree_trans_journal_entries_start(trans), in bch2_trans_commit_write_locked()
759 trans->journal_entries.u64s); in bch2_trans_commit_write_locked()
761 EBUG_ON(trans->journal_res.u64s < trans->journal_entries.u64s); in bch2_trans_commit_write_locked()
763 trans->journal_res.offset += trans->journal_entries.u64s; in bch2_trans_commit_write_locked()
764 trans->journal_res.u64s -= trans->journal_entries.u64s; in bch2_trans_commit_write_locked()
766 memcpy_u64s_small(bch2_journal_add_entry(j, &trans->journal_res, in bch2_trans_commit_write_locked()
769 trans->accounting.u64s)->_data, in bch2_trans_commit_write_locked()
770 btree_trans_subbuf_base(trans, &trans->accounting), in bch2_trans_commit_write_locked()
771 trans->accounting.u64s); in bch2_trans_commit_write_locked()
773 if (trans->journal_seq) in bch2_trans_commit_write_locked()
774 *trans->journal_seq = trans->journal_res.seq; in bch2_trans_commit_write_locked()
777 trans_for_each_update(trans, i) { in bch2_trans_commit_write_locked()
778 struct btree_path *path = trans->paths + i->path; in bch2_trans_commit_write_locked()
781 bch2_btree_insert_key_leaf(trans, path, i->k, trans->journal_res.seq); in bch2_trans_commit_write_locked()
783 bch2_btree_insert_key_cached(trans, flags, i); in bch2_trans_commit_write_locked()
785 bch2_btree_key_cache_drop(trans, path); in bch2_trans_commit_write_locked()
793 for (struct bkey_i *i = btree_trans_subbuf_base(trans, &trans->accounting); in bch2_trans_commit_write_locked()
796 bch2_accounting_trans_commit_revert(trans, bkey_i_to_accounting(i), flags); in bch2_trans_commit_write_locked()
801 static noinline void bch2_drop_overwrites_from_journal(struct btree_trans *trans) in bch2_drop_overwrites_from_journal() argument
809 trans_for_each_update(trans, i) in bch2_drop_overwrites_from_journal()
811 bch2_journal_key_overwritten(trans->c, i->btree_id, i->level, i->k->k.p); in bch2_drop_overwrites_from_journal()
823 static inline int do_bch2_trans_commit(struct btree_trans *trans, unsigned flags, in do_bch2_trans_commit() argument
827 struct bch_fs *c = trans->c; in do_bch2_trans_commit()
830 for (unsigned idx = 0; idx < trans->nr_updates; idx++) { in do_bch2_trans_commit()
831 struct btree_insert_entry *i = trans->updates + idx; in do_bch2_trans_commit()
838 if (!same_leaf_as_next(trans, i)) { in do_bch2_trans_commit()
840 ret = bch2_foreground_maybe_merge(trans, i->path, in do_bch2_trans_commit()
850 ret = bch2_trans_lock_write(trans); in do_bch2_trans_commit()
854 ret = bch2_trans_commit_write_locked(trans, flags, stopped_at, trace_ip); in do_bch2_trans_commit()
856 if (!ret && unlikely(trans->journal_replay_not_finished)) in do_bch2_trans_commit()
857 bch2_drop_overwrites_from_journal(trans); in do_bch2_trans_commit()
859 bch2_trans_unlock_updates_write(trans); in do_bch2_trans_commit()
861 if (!ret && trans->journal_pin) in do_bch2_trans_commit()
862 bch2_journal_pin_add(&c->journal, trans->journal_res.seq, in do_bch2_trans_commit()
863 trans->journal_pin, in do_bch2_trans_commit()
871 bch2_journal_res_put(&c->journal, &trans->journal_res); in do_bch2_trans_commit()
887 int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags, in bch2_trans_commit_error() argument
891 struct bch_fs *c = trans->c; in bch2_trans_commit_error()
905 ret = drop_locks_do(trans, in bch2_trans_commit_error()
906 bch2_trans_journal_res_get(trans, in bch2_trans_commit_error()
914 ret = bch2_btree_split_leaf(trans, i->path, flags); in bch2_trans_commit_error()
916 trace_and_count(c, trans_restart_btree_node_split, trans, in bch2_trans_commit_error()
917 trace_ip, trans->paths + i->path); in bch2_trans_commit_error()
920 ret = drop_locks_do(trans, in bch2_trans_commit_error()
921 bch2_accounting_update_sb(trans)); in bch2_trans_commit_error()
924 bch2_trans_unlock(trans); in bch2_trans_commit_error()
926 trace_and_count(c, trans_blocked_journal_reclaim, trans, trace_ip); in bch2_trans_commit_error()
937 ret = bch2_trans_relock(trans); in bch2_trans_commit_error()
944 BUG_ON(bch2_err_matches(ret, BCH_ERR_transaction_restart) != !!trans->restarted); in bch2_trans_commit_error()
959 do_bch2_trans_commit_to_journal_replay(struct btree_trans *trans) in do_bch2_trans_commit_to_journal_replay() argument
961 struct bch_fs *c = trans->c; in do_bch2_trans_commit_to_journal_replay()
965 trans_for_each_update(trans, i) { in do_bch2_trans_commit_to_journal_replay()
971 for (struct jset_entry *i = btree_trans_journal_entries_start(trans); in do_bch2_trans_commit_to_journal_replay()
972 i != btree_trans_journal_entries_top(trans); in do_bch2_trans_commit_to_journal_replay()
994 for (struct bkey_i *i = btree_trans_subbuf_base(trans, &trans->accounting); in do_bch2_trans_commit_to_journal_replay()
995 i != btree_trans_subbuf_top(trans, &trans->accounting); in do_bch2_trans_commit_to_journal_replay()
1005 int __bch2_trans_commit(struct btree_trans *trans, unsigned flags) in __bch2_trans_commit() argument
1008 struct bch_fs *c = trans->c; in __bch2_trans_commit()
1012 bch2_trans_verify_not_unlocked_or_in_restart(trans); in __bch2_trans_commit()
1014 ret = trans_maybe_inject_restart(trans, _RET_IP_); in __bch2_trans_commit()
1018 if (!trans->nr_updates && in __bch2_trans_commit()
1019 !trans->journal_entries.u64s && in __bch2_trans_commit()
1020 !trans->accounting.u64s) in __bch2_trans_commit()
1023 ret = bch2_trans_commit_run_triggers(trans); in __bch2_trans_commit()
1030 ret = do_bch2_trans_commit_to_journal_replay(trans); in __bch2_trans_commit()
1038 journal_u64s = jset_u64s(trans->accounting.u64s); in __bch2_trans_commit()
1039 trans->journal_transaction_names = READ_ONCE(c->opts.journal_transaction_names); in __bch2_trans_commit()
1040 if (trans->journal_transaction_names) in __bch2_trans_commit()
1043 trans_for_each_update(trans, i) { in __bch2_trans_commit()
1044 struct btree_path *path = trans->paths + i->path; in __bch2_trans_commit()
1048 ret = bch2_btree_path_upgrade(trans, path, i->level + 1); in __bch2_trans_commit()
1064 if (trans->journal_transaction_names) in __bch2_trans_commit()
1068 if (trans->extra_disk_res) { in __bch2_trans_commit()
1069 ret = bch2_disk_reservation_add(c, trans->disk_res, in __bch2_trans_commit()
1070 trans->extra_disk_res, in __bch2_trans_commit()
1078 bch2_trans_verify_not_unlocked_or_in_restart(trans); in __bch2_trans_commit()
1080 memset(&trans->journal_res, 0, sizeof(trans->journal_res)); in __bch2_trans_commit()
1081 memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta)); in __bch2_trans_commit()
1083 trans->journal_u64s = journal_u64s + trans->journal_entries.u64s; in __bch2_trans_commit()
1085 ret = do_bch2_trans_commit(trans, flags, &errored_at, _RET_IP_); in __bch2_trans_commit()
1088 bch2_trans_verify_locks(trans); in __bch2_trans_commit()
1093 trace_and_count(c, transaction_commit, trans, _RET_IP_); in __bch2_trans_commit()
1099 bch2_trans_downgrade(trans); in __bch2_trans_commit()
1100 bch2_trans_reset_updates(trans); in __bch2_trans_commit()
1104 ret = bch2_trans_commit_error(trans, flags, errored_at, ret, _RET_IP_); in __bch2_trans_commit()