Lines Matching refs:trans
44 static void verify_update_old_key(struct btree_trans *trans, struct btree_insert_entry *i) in verify_update_old_key() argument
47 struct bch_fs *c = trans->c; in verify_update_old_key()
49 struct bkey_s_c k = bch2_btree_path_peek_slot_exact(trans->paths + i->path, &u); in verify_update_old_key()
51 if (unlikely(trans->journal_replay_not_finished)) { in verify_update_old_key()
67 static inline struct btree_path_level *insert_l(struct btree_trans *trans, struct btree_insert_entr… in insert_l() argument
69 return (trans->paths + i->path)->l + i->level; in insert_l()
72 static inline bool same_leaf_as_prev(struct btree_trans *trans, in same_leaf_as_prev() argument
75 return i != trans->updates && in same_leaf_as_prev()
76 insert_l(trans, &i[0])->b == insert_l(trans, &i[-1])->b; in same_leaf_as_prev()
79 static inline bool same_leaf_as_next(struct btree_trans *trans, in same_leaf_as_next() argument
82 return i + 1 < trans->updates + trans->nr_updates && in same_leaf_as_next()
83 insert_l(trans, &i[0])->b == insert_l(trans, &i[1])->b; in same_leaf_as_next()
86 inline void bch2_btree_node_prep_for_write(struct btree_trans *trans, in bch2_btree_node_prep_for_write() argument
90 struct bch_fs *c = trans->c; in bch2_btree_node_prep_for_write()
94 bch2_trans_node_reinit_iter(trans, b); in bch2_btree_node_prep_for_write()
101 bch2_btree_init_next(trans, b); in bch2_btree_node_prep_for_write()
104 static noinline int trans_lock_write_fail(struct btree_trans *trans, struct btree_insert_entry *i) in trans_lock_write_fail() argument
106 while (--i >= trans->updates) { in trans_lock_write_fail()
107 if (same_leaf_as_prev(trans, i)) in trans_lock_write_fail()
110 bch2_btree_node_unlock_write(trans, trans->paths + i->path, insert_l(trans, i)->b); in trans_lock_write_fail()
113 trace_and_count(trans->c, trans_restart_would_deadlock_write, trans); in trans_lock_write_fail()
114 return btree_trans_restart(trans, BCH_ERR_transaction_restart_would_deadlock_write); in trans_lock_write_fail()
117 static inline int bch2_trans_lock_write(struct btree_trans *trans) in bch2_trans_lock_write() argument
119 EBUG_ON(trans->write_locked); in bch2_trans_lock_write()
121 trans_for_each_update(trans, i) { in bch2_trans_lock_write()
122 if (same_leaf_as_prev(trans, i)) in bch2_trans_lock_write()
125 if (bch2_btree_node_lock_write(trans, trans->paths + i->path, &insert_l(trans, i)->b->c)) in bch2_trans_lock_write()
126 return trans_lock_write_fail(trans, i); in bch2_trans_lock_write()
129 bch2_btree_node_prep_for_write(trans, trans->paths + i->path, insert_l(trans, i)->b); in bch2_trans_lock_write()
132 trans->write_locked = true; in bch2_trans_lock_write()
136 static inline void bch2_trans_unlock_write(struct btree_trans *trans) in bch2_trans_unlock_write() argument
138 if (likely(trans->write_locked)) { in bch2_trans_unlock_write()
139 trans_for_each_update(trans, i) in bch2_trans_unlock_write()
140 if (btree_node_locked_type(trans->paths + i->path, i->level) == in bch2_trans_unlock_write()
142 bch2_btree_node_unlock_write_inlined(trans, in bch2_trans_unlock_write()
143 trans->paths + i->path, insert_l(trans, i)->b); in bch2_trans_unlock_write()
144 trans->write_locked = false; in bch2_trans_unlock_write()
151 bool bch2_btree_bset_insert_key(struct btree_trans *trans, in bch2_btree_bset_insert_key() argument
193 bch2_btree_path_fix_key_modified(trans, b, k); in bch2_btree_bset_insert_key()
211 bch2_btree_path_fix_key_modified(trans, b, k); in bch2_btree_bset_insert_key()
221 bch2_btree_node_iter_fix(trans, path, b, node_iter, k, in bch2_btree_bset_insert_key()
232 struct btree_trans *trans = bch2_trans_get(c); in __btree_node_flush() local
236 btree_node_lock_nopath_nofail(trans, &b->c, SIX_LOCK_read); in __btree_node_flush()
255 bch2_trans_put(trans); in __btree_node_flush()
287 inline void bch2_btree_insert_key_leaf(struct btree_trans *trans, in bch2_btree_insert_key_leaf() argument
292 struct bch_fs *c = trans->c; in bch2_btree_insert_key_leaf()
300 if (unlikely(!bch2_btree_bset_insert_key(trans, path, b, in bch2_btree_insert_key_leaf()
323 bch2_trans_node_reinit_iter(trans, b); in bch2_btree_insert_key_leaf()
330 static inline void btree_insert_entry_checks(struct btree_trans *trans, in btree_insert_entry_checks() argument
333 struct btree_path *path = trans->paths + i->path; in btree_insert_entry_checks()
342 test_bit(JOURNAL_replay_done, &trans->c->journal.flags) && in btree_insert_entry_checks()
344 bch2_snapshot_is_internal_node(trans->c, i->k->k.p.snapshot) > 0); in btree_insert_entry_checks()
347 static __always_inline int bch2_trans_journal_res_get(struct btree_trans *trans, in bch2_trans_journal_res_get() argument
350 return bch2_journal_res_get(&trans->c->journal, &trans->journal_res, in bch2_trans_journal_res_get()
351 trans->journal_u64s, flags); in bch2_trans_journal_res_get()
356 static noinline void journal_transaction_name(struct btree_trans *trans) in journal_transaction_name() argument
358 struct bch_fs *c = trans->c; in journal_transaction_name()
361 bch2_journal_add_entry(j, &trans->journal_res, in journal_transaction_name()
367 strncpy(l->d, trans->fn, JSET_ENTRY_LOG_U64s * sizeof(u64)); in journal_transaction_name()
370 static inline int btree_key_can_insert(struct btree_trans *trans, in btree_key_can_insert() argument
380 btree_key_can_insert_cached_slowpath(struct btree_trans *trans, unsigned flags, in btree_key_can_insert_cached_slowpath() argument
387 bch2_trans_unlock_write(trans); in btree_key_can_insert_cached_slowpath()
388 bch2_trans_unlock(trans); in btree_key_can_insert_cached_slowpath()
392 bch_err(trans->c, "error allocating memory for key cache key, btree %s u64s %u", in btree_key_can_insert_cached_slowpath()
397 ret = bch2_trans_relock(trans) ?: in btree_key_can_insert_cached_slowpath()
398 bch2_trans_lock_write(trans); in btree_key_can_insert_cached_slowpath()
406 trans_for_each_update(trans, i) in btree_key_can_insert_cached_slowpath()
416 static int btree_key_can_insert_cached(struct btree_trans *trans, unsigned flags, in btree_key_can_insert_cached() argument
419 struct bch_fs *c = trans->c; in btree_key_can_insert_cached()
444 return btree_key_can_insert_cached_slowpath(trans, flags, path, new_u64s); in btree_key_can_insert_cached()
446 trans_for_each_update(trans, i) in btree_key_can_insert_cached()
457 static int run_one_mem_trigger(struct btree_trans *trans, in run_one_mem_trigger() argument
461 verify_update_old_key(trans, i); in run_one_mem_trigger()
472 return bch2_key_trigger(trans, i->btree_id, i->level, in run_one_mem_trigger()
476 return bch2_key_trigger_new(trans, i->btree_id, i->level, in run_one_mem_trigger()
478 bch2_key_trigger_old(trans, i->btree_id, i->level, in run_one_mem_trigger()
482 static int run_one_trans_trigger(struct btree_trans *trans, struct btree_insert_entry *i, in run_one_trans_trigger() argument
485 verify_update_old_key(trans, i); in run_one_trans_trigger()
507 return bch2_key_trigger(trans, i->btree_id, i->level, old, bkey_i_to_s(i->k), in run_one_trans_trigger()
512 return bch2_key_trigger_old(trans, i->btree_id, i->level, old, flags) ?: 1; in run_one_trans_trigger()
515 return bch2_key_trigger_new(trans, i->btree_id, i->level, bkey_i_to_s(i->k), flags) ?: 1; in run_one_trans_trigger()
521 static int run_btree_triggers(struct btree_trans *trans, enum btree_id btree_id, in run_btree_triggers() argument
535 i < trans->nr_updates && trans->updates[i].btree_id <= btree_id; in run_btree_triggers()
537 if (trans->updates[i].btree_id != btree_id) in run_btree_triggers()
540 int ret = run_one_trans_trigger(trans, trans->updates + i, overwrite); in run_btree_triggers()
552 static int bch2_trans_commit_run_triggers(struct btree_trans *trans) in bch2_trans_commit_run_triggers() argument
568 while (btree_id_start < trans->nr_updates && in bch2_trans_commit_run_triggers()
569 trans->updates[btree_id_start].btree_id < btree_id) in bch2_trans_commit_run_triggers()
572 ret = run_btree_triggers(trans, btree_id, btree_id_start); in bch2_trans_commit_run_triggers()
577 for (unsigned idx = 0; idx < trans->nr_updates; idx++) { in bch2_trans_commit_run_triggers()
578 struct btree_insert_entry *i = trans->updates + idx; in bch2_trans_commit_run_triggers()
583 ret = run_btree_triggers(trans, BTREE_ID_alloc, idx); in bch2_trans_commit_run_triggers()
591 trans_for_each_update(trans, i) in bch2_trans_commit_run_triggers()
599 static noinline int bch2_trans_commit_run_gc_triggers(struct btree_trans *trans) in bch2_trans_commit_run_gc_triggers() argument
601 trans_for_each_update(trans, i) in bch2_trans_commit_run_gc_triggers()
603 gc_visited(trans->c, gc_pos_btree(i->btree_id, i->level, i->k->k.p))) { in bch2_trans_commit_run_gc_triggers()
604 int ret = run_one_mem_trigger(trans, i, i->flags|BTREE_TRIGGER_gc); in bch2_trans_commit_run_gc_triggers()
621 bch2_trans_commit_write_locked(struct btree_trans *trans, unsigned flags, in bch2_trans_commit_write_locked() argument
625 struct bch_fs *c = trans->c; in bch2_trans_commit_write_locked()
630 bch2_trans_verify_not_unlocked(trans); in bch2_trans_commit_write_locked()
631 bch2_trans_verify_not_in_restart(trans); in bch2_trans_commit_write_locked()
634 trace_and_count(c, trans_restart_fault_inject, trans, trace_ip); in bch2_trans_commit_write_locked()
635 return btree_trans_restart_nounlock(trans, BCH_ERR_transaction_restart_fault_inject); in bch2_trans_commit_write_locked()
644 prefetch(&trans->c->journal.flags); in bch2_trans_commit_write_locked()
646 trans_for_each_update(trans, i) { in bch2_trans_commit_write_locked()
648 if (!same_leaf_as_prev(trans, i)) in bch2_trans_commit_write_locked()
653 ? btree_key_can_insert(trans, insert_l(trans, i)->b, u64s) in bch2_trans_commit_write_locked()
654 : btree_key_can_insert_cached(trans, flags, trans->paths + i->path, u64s); in bch2_trans_commit_write_locked()
668 ret = bch2_trans_journal_res_get(trans, in bch2_trans_commit_write_locked()
674 if (unlikely(trans->journal_transaction_names)) in bch2_trans_commit_write_locked()
675 journal_transaction_name(trans); in bch2_trans_commit_write_locked()
686 trans_for_each_update(trans, i) in bch2_trans_commit_write_locked()
687 i->k->k.bversion.lo = trans->journal_res.seq; in bch2_trans_commit_write_locked()
689 trans_for_each_update(trans, i) in bch2_trans_commit_write_locked()
693 h = trans->hooks; in bch2_trans_commit_write_locked()
695 ret = h->fn(trans, h); in bch2_trans_commit_write_locked()
701 struct jset_entry *entry = trans->journal_entries; in bch2_trans_commit_write_locked()
705 for (entry = trans->journal_entries; in bch2_trans_commit_write_locked()
706 entry != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s); in bch2_trans_commit_write_locked()
710 BUG_ON(!trans->journal_res.ref); in bch2_trans_commit_write_locked()
714 a->k.bversion = journal_pos_to_bversion(&trans->journal_res, in bch2_trans_commit_write_locked()
715 (u64 *) entry - (u64 *) trans->journal_entries); in bch2_trans_commit_write_locked()
719 ret = bch2_accounting_mem_mod_locked(trans, accounting_i_to_s_c(a), BCH_ACCOUNTING_normal); in bch2_trans_commit_write_locked()
727 bch2_trans_account_disk_usage_change(trans); in bch2_trans_commit_write_locked()
729 trans_for_each_update(trans, i) in bch2_trans_commit_write_locked()
731 ret = run_one_mem_trigger(trans, i, BTREE_TRIGGER_atomic|i->flags); in bch2_trans_commit_write_locked()
737 ret = bch2_trans_commit_run_gc_triggers(trans); in bch2_trans_commit_write_locked()
742 trans_for_each_update(trans, i) { in bch2_trans_commit_write_locked()
751 bch2_trans_inconsistent(trans, "invalid bkey on insert from %s -> %ps\n", in bch2_trans_commit_write_locked()
752 trans->fn, (void *) i->ip_allocated); in bch2_trans_commit_write_locked()
755 btree_insert_entry_checks(trans, i); in bch2_trans_commit_write_locked()
758 for (struct jset_entry *i = trans->journal_entries; in bch2_trans_commit_write_locked()
759 i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s); in bch2_trans_commit_write_locked()
770 bch2_trans_inconsistent(trans, "invalid journal entry on insert from %s\n", in bch2_trans_commit_write_locked()
771 trans->fn); in bch2_trans_commit_write_locked()
780 trans_for_each_update(trans, i) { in bch2_trans_commit_write_locked()
787 verify_update_old_key(trans, i); in bch2_trans_commit_write_locked()
789 if (trans->journal_transaction_names) { in bch2_trans_commit_write_locked()
790 entry = bch2_journal_add_entry(j, &trans->journal_res, in bch2_trans_commit_write_locked()
798 entry = bch2_journal_add_entry(j, &trans->journal_res, in bch2_trans_commit_write_locked()
805 memcpy_u64s_small(journal_res_entry(&c->journal, &trans->journal_res), in bch2_trans_commit_write_locked()
806 trans->journal_entries, in bch2_trans_commit_write_locked()
807 trans->journal_entries_u64s); in bch2_trans_commit_write_locked()
809 trans->journal_res.offset += trans->journal_entries_u64s; in bch2_trans_commit_write_locked()
810 trans->journal_res.u64s -= trans->journal_entries_u64s; in bch2_trans_commit_write_locked()
812 if (trans->journal_seq) in bch2_trans_commit_write_locked()
813 *trans->journal_seq = trans->journal_res.seq; in bch2_trans_commit_write_locked()
816 trans_for_each_update(trans, i) { in bch2_trans_commit_write_locked()
817 struct btree_path *path = trans->paths + i->path; in bch2_trans_commit_write_locked()
820 bch2_btree_insert_key_leaf(trans, path, i->k, trans->journal_res.seq); in bch2_trans_commit_write_locked()
822 bch2_btree_insert_key_cached(trans, flags, i); in bch2_trans_commit_write_locked()
824 bch2_btree_key_cache_drop(trans, path); in bch2_trans_commit_write_locked()
832 for (struct jset_entry *entry2 = trans->journal_entries; in bch2_trans_commit_write_locked()
840 bch2_accounting_mem_mod_locked(trans, a.c, BCH_ACCOUNTING_normal); in bch2_trans_commit_write_locked()
847 static noinline void bch2_drop_overwrites_from_journal(struct btree_trans *trans) in bch2_drop_overwrites_from_journal() argument
855 trans_for_each_update(trans, i) in bch2_drop_overwrites_from_journal()
857 bch2_journal_key_overwritten(trans->c, i->btree_id, i->level, i->k->k.p); in bch2_drop_overwrites_from_journal()
869 static inline int do_bch2_trans_commit(struct btree_trans *trans, unsigned flags, in do_bch2_trans_commit() argument
873 struct bch_fs *c = trans->c; in do_bch2_trans_commit()
876 for (unsigned idx = 0; idx < trans->nr_updates; idx++) { in do_bch2_trans_commit()
877 struct btree_insert_entry *i = trans->updates + idx; in do_bch2_trans_commit()
884 if (!same_leaf_as_next(trans, i)) { in do_bch2_trans_commit()
886 ret = bch2_foreground_maybe_merge(trans, i->path, in do_bch2_trans_commit()
896 ret = bch2_trans_lock_write(trans); in do_bch2_trans_commit()
900 ret = bch2_trans_commit_write_locked(trans, flags, stopped_at, trace_ip); in do_bch2_trans_commit()
902 if (!ret && unlikely(trans->journal_replay_not_finished)) in do_bch2_trans_commit()
903 bch2_drop_overwrites_from_journal(trans); in do_bch2_trans_commit()
905 bch2_trans_unlock_write(trans); in do_bch2_trans_commit()
907 if (!ret && trans->journal_pin) in do_bch2_trans_commit()
908 bch2_journal_pin_add(&c->journal, trans->journal_res.seq, in do_bch2_trans_commit()
909 trans->journal_pin, in do_bch2_trans_commit()
917 bch2_journal_res_put(&c->journal, &trans->journal_res); in do_bch2_trans_commit()
933 int bch2_trans_commit_error(struct btree_trans *trans, unsigned flags, in bch2_trans_commit_error() argument
937 struct bch_fs *c = trans->c; in bch2_trans_commit_error()
942 ret = bch2_btree_split_leaf(trans, i->path, flags); in bch2_trans_commit_error()
944 trace_and_count(c, trans_restart_btree_node_split, trans, in bch2_trans_commit_error()
945 trace_ip, trans->paths + i->path); in bch2_trans_commit_error()
948 ret = drop_locks_do(trans, in bch2_trans_commit_error()
949 bch2_accounting_update_sb(trans)); in bch2_trans_commit_error()
962 ret = drop_locks_do(trans, in bch2_trans_commit_error()
963 bch2_trans_journal_res_get(trans, in bch2_trans_commit_error()
968 bch2_trans_unlock(trans); in bch2_trans_commit_error()
970 trace_and_count(c, trans_blocked_journal_reclaim, trans, trace_ip); in bch2_trans_commit_error()
981 ret = bch2_trans_relock(trans); in bch2_trans_commit_error()
988 BUG_ON(bch2_err_matches(ret, BCH_ERR_transaction_restart) != !!trans->restarted); in bch2_trans_commit_error()
998 bch2_trans_commit_get_rw_cold(struct btree_trans *trans, unsigned flags) in bch2_trans_commit_get_rw_cold() argument
1000 struct bch_fs *c = trans->c; in bch2_trans_commit_get_rw_cold()
1007 ret = drop_locks_do(trans, bch2_fs_read_write_early(c)); in bch2_trans_commit_get_rw_cold()
1021 do_bch2_trans_commit_to_journal_replay(struct btree_trans *trans) in do_bch2_trans_commit_to_journal_replay() argument
1023 struct bch_fs *c = trans->c; in do_bch2_trans_commit_to_journal_replay()
1025 trans_for_each_update(trans, i) { in do_bch2_trans_commit_to_journal_replay()
1031 for (struct jset_entry *i = trans->journal_entries; in do_bch2_trans_commit_to_journal_replay()
1032 i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s); in do_bch2_trans_commit_to_journal_replay()
1044 int __bch2_trans_commit(struct btree_trans *trans, unsigned flags) in __bch2_trans_commit() argument
1047 struct bch_fs *c = trans->c; in __bch2_trans_commit()
1050 bch2_trans_verify_not_unlocked(trans); in __bch2_trans_commit()
1051 bch2_trans_verify_not_in_restart(trans); in __bch2_trans_commit()
1053 if (!trans->nr_updates && in __bch2_trans_commit()
1054 !trans->journal_entries_u64s) in __bch2_trans_commit()
1057 ret = bch2_trans_commit_run_triggers(trans); in __bch2_trans_commit()
1062 ret = do_bch2_trans_commit_to_journal_replay(trans); in __bch2_trans_commit()
1068 ret = bch2_trans_commit_get_rw_cold(trans, flags); in __bch2_trans_commit()
1075 trans->journal_u64s = trans->journal_entries_u64s; in __bch2_trans_commit()
1076 trans->journal_transaction_names = READ_ONCE(c->opts.journal_transaction_names); in __bch2_trans_commit()
1077 if (trans->journal_transaction_names) in __bch2_trans_commit()
1078 trans->journal_u64s += jset_u64s(JSET_ENTRY_LOG_U64s); in __bch2_trans_commit()
1080 trans_for_each_update(trans, i) { in __bch2_trans_commit()
1081 struct btree_path *path = trans->paths + i->path; in __bch2_trans_commit()
1085 ret = bch2_btree_path_upgrade(trans, path, i->level + 1); in __bch2_trans_commit()
1098 trans->journal_u64s += jset_u64s(i->k->k.u64s); in __bch2_trans_commit()
1101 if (trans->journal_transaction_names) in __bch2_trans_commit()
1102 trans->journal_u64s += jset_u64s(i->old_k.u64s); in __bch2_trans_commit()
1105 if (trans->extra_disk_res) { in __bch2_trans_commit()
1106 ret = bch2_disk_reservation_add(c, trans->disk_res, in __bch2_trans_commit()
1107 trans->extra_disk_res, in __bch2_trans_commit()
1115 bch2_trans_verify_not_unlocked(trans); in __bch2_trans_commit()
1116 bch2_trans_verify_not_in_restart(trans); in __bch2_trans_commit()
1118 memset(&trans->journal_res, 0, sizeof(trans->journal_res)); in __bch2_trans_commit()
1119 memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta)); in __bch2_trans_commit()
1121 ret = do_bch2_trans_commit(trans, flags, &errored_at, _RET_IP_); in __bch2_trans_commit()
1124 bch2_trans_verify_locks(trans); in __bch2_trans_commit()
1129 trace_and_count(c, transaction_commit, trans, _RET_IP_); in __bch2_trans_commit()
1135 bch2_trans_downgrade(trans); in __bch2_trans_commit()
1136 bch2_trans_reset_updates(trans); in __bch2_trans_commit()
1140 ret = bch2_trans_commit_error(trans, flags, errored_at, ret, _RET_IP_); in __bch2_trans_commit()