Lines Matching refs:dentry
125 static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent, in in_lookup_hash()
301 static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount) in dentry_cmp() argument
319 const unsigned char *cs = READ_ONCE(dentry->d_name.name); in dentry_cmp()
336 static inline struct external_name *external_name(struct dentry *dentry) in external_name() argument
338 return container_of(dentry->d_name.name, struct external_name, name[0]); in external_name()
343 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu); in __d_free() local
345 kmem_cache_free(dentry_cache, dentry); in __d_free()
350 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu); in __d_free_external() local
351 kfree(external_name(dentry)); in __d_free_external()
352 kmem_cache_free(dentry_cache, dentry); in __d_free_external()
355 static inline int dname_external(const struct dentry *dentry) in dname_external() argument
357 return dentry->d_name.name != dentry->d_shortname.string; in dname_external()
360 void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry) in take_dentry_name_snapshot() argument
367 seq = read_seqcount_begin(&dentry->d_seq); in take_dentry_name_snapshot()
368 s = READ_ONCE(dentry->d_name.name); in take_dentry_name_snapshot()
369 name->name.hash_len = dentry->d_name.hash_len; in take_dentry_name_snapshot()
371 if (likely(s == dentry->d_shortname.string)) { in take_dentry_name_snapshot()
372 name->inline_name = dentry->d_shortname; in take_dentry_name_snapshot()
381 if (read_seqcount_retry(&dentry->d_seq, seq)) { in take_dentry_name_snapshot()
400 static inline void __d_set_inode_and_type(struct dentry *dentry, in __d_set_inode_and_type() argument
406 dentry->d_inode = inode; in __d_set_inode_and_type()
407 flags = READ_ONCE(dentry->d_flags); in __d_set_inode_and_type()
410 smp_store_release(&dentry->d_flags, flags); in __d_set_inode_and_type()
413 static inline void __d_clear_type_and_inode(struct dentry *dentry) in __d_clear_type_and_inode() argument
415 unsigned flags = READ_ONCE(dentry->d_flags); in __d_clear_type_and_inode()
418 WRITE_ONCE(dentry->d_flags, flags); in __d_clear_type_and_inode()
419 dentry->d_inode = NULL; in __d_clear_type_and_inode()
428 static void dentry_free(struct dentry *dentry) in dentry_free() argument
430 WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias)); in dentry_free()
431 if (unlikely(dname_external(dentry))) { in dentry_free()
432 struct external_name *p = external_name(dentry); in dentry_free()
434 call_rcu(&dentry->d_u.d_rcu, __d_free_external); in dentry_free()
439 if (dentry->d_flags & DCACHE_NORCU) in dentry_free()
440 __d_free(&dentry->d_u.d_rcu); in dentry_free()
442 call_rcu(&dentry->d_u.d_rcu, __d_free); in dentry_free()
449 static void dentry_unlink_inode(struct dentry * dentry) in dentry_unlink_inode() argument
450 __releases(dentry->d_lock) in dentry_unlink_inode()
451 __releases(dentry->d_inode->i_lock) in dentry_unlink_inode()
453 struct inode *inode = dentry->d_inode; in dentry_unlink_inode()
455 raw_write_seqcount_begin(&dentry->d_seq); in dentry_unlink_inode()
456 __d_clear_type_and_inode(dentry); in dentry_unlink_inode()
457 hlist_del_init(&dentry->d_u.d_alias); in dentry_unlink_inode()
458 raw_write_seqcount_end(&dentry->d_seq); in dentry_unlink_inode()
459 spin_unlock(&dentry->d_lock); in dentry_unlink_inode()
463 if (dentry->d_op && dentry->d_op->d_iput) in dentry_unlink_inode()
464 dentry->d_op->d_iput(dentry, inode); in dentry_unlink_inode()
488 #define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_… argument
489 static void d_lru_add(struct dentry *dentry) in d_lru_add() argument
491 D_FLAG_VERIFY(dentry, 0); in d_lru_add()
492 dentry->d_flags |= DCACHE_LRU_LIST; in d_lru_add()
494 if (d_is_negative(dentry)) in d_lru_add()
497 &dentry->d_sb->s_dentry_lru, &dentry->d_lru)); in d_lru_add()
500 static void d_lru_del(struct dentry *dentry) in d_lru_del() argument
502 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); in d_lru_del()
503 dentry->d_flags &= ~DCACHE_LRU_LIST; in d_lru_del()
505 if (d_is_negative(dentry)) in d_lru_del()
508 &dentry->d_sb->s_dentry_lru, &dentry->d_lru)); in d_lru_del()
511 static void d_shrink_del(struct dentry *dentry) in d_shrink_del() argument
513 D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST); in d_shrink_del()
514 list_del_init(&dentry->d_lru); in d_shrink_del()
515 dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST); in d_shrink_del()
519 static void d_shrink_add(struct dentry *dentry, struct list_head *list) in d_shrink_add() argument
521 D_FLAG_VERIFY(dentry, 0); in d_shrink_add()
522 list_add(&dentry->d_lru, list); in d_shrink_add()
523 dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST; in d_shrink_add()
533 static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry) in d_lru_isolate() argument
535 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); in d_lru_isolate()
536 dentry->d_flags &= ~DCACHE_LRU_LIST; in d_lru_isolate()
538 if (d_is_negative(dentry)) in d_lru_isolate()
540 list_lru_isolate(lru, &dentry->d_lru); in d_lru_isolate()
543 static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry, in d_lru_shrink_move() argument
546 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); in d_lru_shrink_move()
547 dentry->d_flags |= DCACHE_SHRINK_LIST; in d_lru_shrink_move()
548 if (d_is_negative(dentry)) in d_lru_shrink_move()
550 list_lru_isolate_move(lru, &dentry->d_lru, list); in d_lru_shrink_move()
553 static void ___d_drop(struct dentry *dentry) in ___d_drop() argument
561 if (unlikely(IS_ROOT(dentry))) in ___d_drop()
562 b = &dentry->d_sb->s_roots; in ___d_drop()
564 b = d_hash(dentry->d_name.hash); in ___d_drop()
567 __hlist_bl_del(&dentry->d_hash); in ___d_drop()
571 void __d_drop(struct dentry *dentry) in __d_drop() argument
573 if (!d_unhashed(dentry)) { in __d_drop()
574 ___d_drop(dentry); in __d_drop()
575 dentry->d_hash.pprev = NULL; in __d_drop()
576 write_seqcount_invalidate(&dentry->d_seq); in __d_drop()
599 void d_drop(struct dentry *dentry) in d_drop() argument
601 spin_lock(&dentry->d_lock); in d_drop()
602 __d_drop(dentry); in d_drop()
603 spin_unlock(&dentry->d_lock); in d_drop()
607 static inline void dentry_unlist(struct dentry *dentry) in dentry_unlist() argument
609 struct dentry *next; in dentry_unlist()
614 dentry->d_flags |= DCACHE_DENTRY_KILLED; in dentry_unlist()
615 if (unlikely(hlist_unhashed(&dentry->d_sib))) in dentry_unlist()
617 __hlist_del(&dentry->d_sib); in dentry_unlist()
637 while (dentry->d_sib.next) { in dentry_unlist()
638 next = hlist_entry(dentry->d_sib.next, struct dentry, d_sib); in dentry_unlist()
641 dentry->d_sib.next = next->d_sib.next; in dentry_unlist()
645 static struct dentry *__dentry_kill(struct dentry *dentry) in __dentry_kill() argument
647 struct dentry *parent = NULL; in __dentry_kill()
653 lockref_mark_dead(&dentry->d_lockref); in __dentry_kill()
659 if (dentry->d_flags & DCACHE_OP_PRUNE) in __dentry_kill()
660 dentry->d_op->d_prune(dentry); in __dentry_kill()
662 if (dentry->d_flags & DCACHE_LRU_LIST) { in __dentry_kill()
663 if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) in __dentry_kill()
664 d_lru_del(dentry); in __dentry_kill()
667 __d_drop(dentry); in __dentry_kill()
668 if (dentry->d_inode) in __dentry_kill()
669 dentry_unlink_inode(dentry); in __dentry_kill()
671 spin_unlock(&dentry->d_lock); in __dentry_kill()
673 if (dentry->d_op && dentry->d_op->d_release) in __dentry_kill()
674 dentry->d_op->d_release(dentry); in __dentry_kill()
678 if (!IS_ROOT(dentry)) { in __dentry_kill()
679 parent = dentry->d_parent; in __dentry_kill()
682 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); in __dentry_kill()
683 dentry_unlist(dentry); in __dentry_kill()
684 if (dentry->d_flags & DCACHE_SHRINK_LIST) in __dentry_kill()
686 spin_unlock(&dentry->d_lock); in __dentry_kill()
688 dentry_free(dentry); in __dentry_kill()
707 static bool lock_for_kill(struct dentry *dentry) in lock_for_kill() argument
709 struct inode *inode = dentry->d_inode; in lock_for_kill()
711 if (unlikely(dentry->d_lockref.count)) in lock_for_kill()
718 spin_unlock(&dentry->d_lock); in lock_for_kill()
720 spin_lock(&dentry->d_lock); in lock_for_kill()
721 if (likely(inode == dentry->d_inode)) in lock_for_kill()
724 inode = dentry->d_inode; in lock_for_kill()
726 if (likely(!dentry->d_lockref.count)) in lock_for_kill()
744 static inline bool retain_dentry(struct dentry *dentry, bool locked) in retain_dentry() argument
749 d_flags = READ_ONCE(dentry->d_flags); in retain_dentry()
752 if (unlikely(d_unhashed(dentry))) in retain_dentry()
762 if (!locked || dentry->d_op->d_delete(dentry)) in retain_dentry()
778 d_lru_add(dentry); in retain_dentry()
782 dentry->d_flags |= DCACHE_REFERENCED; in retain_dentry()
789 struct dentry *de; in d_mark_dontcache()
812 static inline bool fast_dput(struct dentry *dentry) in fast_dput() argument
819 ret = lockref_put_return(&dentry->d_lockref); in fast_dput()
827 spin_lock(&dentry->d_lock); in fast_dput()
828 if (WARN_ON_ONCE(dentry->d_lockref.count <= 0)) { in fast_dput()
829 spin_unlock(&dentry->d_lock); in fast_dput()
832 dentry->d_lockref.count--; in fast_dput()
848 if (retain_dentry(dentry, false)) in fast_dput()
856 spin_lock(&dentry->d_lock); in fast_dput()
865 if (dentry->d_lockref.count || retain_dentry(dentry, true)) { in fast_dput()
866 spin_unlock(&dentry->d_lock); in fast_dput()
899 void dput(struct dentry *dentry) in dput() argument
901 if (!dentry) in dput()
905 if (likely(fast_dput(dentry))) { in dput()
909 while (lock_for_kill(dentry)) { in dput()
911 dentry = __dentry_kill(dentry); in dput()
912 if (!dentry) in dput()
914 if (retain_dentry(dentry, true)) { in dput()
915 spin_unlock(&dentry->d_lock); in dput()
921 spin_unlock(&dentry->d_lock); in dput()
925 static void to_shrink_list(struct dentry *dentry, struct list_head *list) in to_shrink_list() argument
926 __must_hold(&dentry->d_lock) in to_shrink_list()
928 if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) { in to_shrink_list()
929 if (dentry->d_flags & DCACHE_LRU_LIST) in to_shrink_list()
930 d_lru_del(dentry); in to_shrink_list()
931 d_shrink_add(dentry, list); in to_shrink_list()
935 void dput_to_list(struct dentry *dentry, struct list_head *list) in dput_to_list() argument
938 if (likely(fast_dput(dentry))) { in dput_to_list()
943 to_shrink_list(dentry, list); in dput_to_list()
944 spin_unlock(&dentry->d_lock); in dput_to_list()
947 struct dentry *dget_parent(struct dentry *dentry) in dget_parent() argument
950 struct dentry *ret; in dget_parent()
958 seq = raw_seqcount_begin(&dentry->d_seq); in dget_parent()
959 ret = READ_ONCE(dentry->d_parent); in dget_parent()
963 if (!read_seqcount_retry(&dentry->d_seq, seq)) in dget_parent()
974 ret = dentry->d_parent; in dget_parent()
976 if (unlikely(ret != dentry->d_parent)) { in dget_parent()
989 static struct dentry * __d_find_any_alias(struct inode *inode) in __d_find_any_alias()
991 struct dentry *alias; in __d_find_any_alias()
995 alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias); in __d_find_any_alias()
1007 struct dentry *d_find_any_alias(struct inode *inode) in d_find_any_alias()
1009 struct dentry *de; in d_find_any_alias()
1018 static struct dentry *__d_find_alias(struct inode *inode) in __d_find_alias()
1020 struct dentry *alias; in __d_find_alias()
1051 struct dentry *d_find_alias(struct inode *inode) in d_find_alias()
1053 struct dentry *de = NULL; in d_find_alias()
1068 struct dentry *d_find_alias_rcu(struct inode *inode) in d_find_alias_rcu()
1071 struct dentry *de = NULL; in d_find_alias_rcu()
1078 de = hlist_entry(l->first, struct dentry, d_u.d_alias); in d_find_alias_rcu()
1096 struct dentry *dentry; in d_prune_aliases() local
1099 hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) { in d_prune_aliases()
1100 spin_lock(&dentry->d_lock); in d_prune_aliases()
1101 if (!dentry->d_lockref.count) in d_prune_aliases()
1102 to_shrink_list(dentry, &dispose); in d_prune_aliases()
1103 spin_unlock(&dentry->d_lock); in d_prune_aliases()
1110 static inline void shrink_kill(struct dentry *victim) in shrink_kill()
1125 struct dentry *dentry; in shrink_dentry_list() local
1127 dentry = list_entry(list->prev, struct dentry, d_lru); in shrink_dentry_list()
1128 spin_lock(&dentry->d_lock); in shrink_dentry_list()
1130 if (!lock_for_kill(dentry)) { in shrink_dentry_list()
1133 d_shrink_del(dentry); in shrink_dentry_list()
1134 can_free = dentry->d_flags & DCACHE_DENTRY_KILLED; in shrink_dentry_list()
1135 spin_unlock(&dentry->d_lock); in shrink_dentry_list()
1137 dentry_free(dentry); in shrink_dentry_list()
1140 d_shrink_del(dentry); in shrink_dentry_list()
1141 shrink_kill(dentry); in shrink_dentry_list()
1149 struct dentry *dentry = container_of(item, struct dentry, d_lru); in dentry_lru_isolate() local
1157 if (!spin_trylock(&dentry->d_lock)) in dentry_lru_isolate()
1165 if (dentry->d_lockref.count) { in dentry_lru_isolate()
1166 d_lru_isolate(lru, dentry); in dentry_lru_isolate()
1167 spin_unlock(&dentry->d_lock); in dentry_lru_isolate()
1171 if (dentry->d_flags & DCACHE_REFERENCED) { in dentry_lru_isolate()
1172 dentry->d_flags &= ~DCACHE_REFERENCED; in dentry_lru_isolate()
1173 spin_unlock(&dentry->d_lock); in dentry_lru_isolate()
1197 d_lru_shrink_move(lru, dentry, freeable); in dentry_lru_isolate()
1198 spin_unlock(&dentry->d_lock); in dentry_lru_isolate()
1230 struct dentry *dentry = container_of(item, struct dentry, d_lru); in dentry_lru_isolate_shrink() local
1237 if (!spin_trylock(&dentry->d_lock)) in dentry_lru_isolate_shrink()
1240 d_lru_shrink_move(lru, dentry, freeable); in dentry_lru_isolate_shrink()
1241 spin_unlock(&dentry->d_lock); in dentry_lru_isolate_shrink()
1288 static void d_walk(struct dentry *parent, void *data, in d_walk() argument
1289 enum d_walk_ret (*enter)(void *, struct dentry *)) in d_walk()
1291 struct dentry *this_parent, *dentry; in d_walk() local
1313 dentry = d_first_child(this_parent); in d_walk()
1315 hlist_for_each_entry_from(dentry, d_sib) { in d_walk()
1316 if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR)) in d_walk()
1319 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); in d_walk()
1321 ret = enter(data, dentry); in d_walk()
1326 spin_unlock(&dentry->d_lock); in d_walk()
1332 spin_unlock(&dentry->d_lock); in d_walk()
1336 if (!hlist_empty(&dentry->d_children)) { in d_walk()
1338 spin_release(&dentry->d_lock.dep_map, _RET_IP_); in d_walk()
1339 this_parent = dentry; in d_walk()
1343 spin_unlock(&dentry->d_lock); in d_walk()
1351 dentry = this_parent; in d_walk()
1352 this_parent = dentry->d_parent; in d_walk()
1354 spin_unlock(&dentry->d_lock); in d_walk()
1361 hlist_for_each_entry_continue(dentry, d_sib) { in d_walk()
1362 if (likely(!(dentry->d_flags & DCACHE_DENTRY_KILLED))) { in d_walk()
1393 static enum d_walk_ret path_check_mount(void *data, struct dentry *dentry) in path_check_mount() argument
1396 struct path path = { .mnt = info->mnt, .dentry = dentry }; in path_check_mount()
1398 if (likely(!d_mountpoint(dentry))) in path_check_mount()
1420 d_walk(parent->dentry, &data, path_check_mount); in path_has_submounts()
1435 int d_set_mounted(struct dentry *dentry) in d_set_mounted() argument
1437 struct dentry *p; in d_set_mounted()
1440 for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) { in d_set_mounted()
1449 spin_lock(&dentry->d_lock); in d_set_mounted()
1450 if (!d_unlinked(dentry)) { in d_set_mounted()
1452 if (!d_mountpoint(dentry)) { in d_set_mounted()
1453 dentry->d_flags |= DCACHE_MOUNTED; in d_set_mounted()
1457 spin_unlock(&dentry->d_lock); in d_set_mounted()
1479 struct dentry *start;
1482 struct dentry *victim;
1487 static enum d_walk_ret select_collect(void *_data, struct dentry *dentry) in select_collect() argument
1492 if (data->start == dentry) in select_collect()
1495 if (dentry->d_flags & DCACHE_SHRINK_LIST) { in select_collect()
1497 } else if (!dentry->d_lockref.count) { in select_collect()
1498 to_shrink_list(dentry, &data->dispose); in select_collect()
1500 } else if (dentry->d_lockref.count < 0) { in select_collect()
1514 static enum d_walk_ret select_collect2(void *_data, struct dentry *dentry) in select_collect2() argument
1519 if (data->start == dentry) in select_collect2()
1522 if (!dentry->d_lockref.count) { in select_collect2()
1523 if (dentry->d_flags & DCACHE_SHRINK_LIST) { in select_collect2()
1525 data->victim = dentry; in select_collect2()
1528 to_shrink_list(dentry, &data->dispose); in select_collect2()
1547 void shrink_dcache_parent(struct dentry *parent) in shrink_dcache_parent()
1580 static enum d_walk_ret umount_check(void *_data, struct dentry *dentry) in umount_check() argument
1583 if (!hlist_empty(&dentry->d_children)) in umount_check()
1587 if (dentry == _data && dentry->d_lockref.count == 1) in umount_check()
1592 dentry, in umount_check()
1593 dentry->d_inode ? in umount_check()
1594 dentry->d_inode->i_ino : 0UL, in umount_check()
1595 dentry, in umount_check()
1596 dentry->d_lockref.count, in umount_check()
1597 dentry->d_sb->s_type->name, in umount_check()
1598 dentry->d_sb->s_id); in umount_check()
1602 static void do_one_tree(struct dentry *dentry) in do_one_tree() argument
1604 shrink_dcache_parent(dentry); in do_one_tree()
1605 d_walk(dentry, dentry, umount_check); in do_one_tree()
1606 d_drop(dentry); in do_one_tree()
1607 dput(dentry); in do_one_tree()
1615 struct dentry *dentry; in shrink_dcache_for_umount() local
1619 dentry = sb->s_root; in shrink_dcache_for_umount()
1621 do_one_tree(dentry); in shrink_dcache_for_umount()
1624 dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_roots), struct dentry, d_hash)); in shrink_dcache_for_umount()
1625 do_one_tree(dentry); in shrink_dcache_for_umount()
1629 static enum d_walk_ret find_submount(void *_data, struct dentry *dentry) in find_submount() argument
1631 struct dentry **victim = _data; in find_submount()
1632 if (d_mountpoint(dentry)) { in find_submount()
1633 *victim = dget_dlock(dentry); in find_submount()
1643 void d_invalidate(struct dentry *dentry) in d_invalidate() argument
1646 spin_lock(&dentry->d_lock); in d_invalidate()
1647 if (d_unhashed(dentry)) { in d_invalidate()
1648 spin_unlock(&dentry->d_lock); in d_invalidate()
1651 __d_drop(dentry); in d_invalidate()
1652 spin_unlock(&dentry->d_lock); in d_invalidate()
1655 if (!dentry->d_inode) in d_invalidate()
1658 shrink_dcache_parent(dentry); in d_invalidate()
1660 struct dentry *victim = NULL; in d_invalidate()
1661 d_walk(dentry, &victim, find_submount); in d_invalidate()
1664 shrink_dcache_parent(dentry); in d_invalidate()
1684 static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name) in __d_alloc()
1686 struct dentry *dentry; in __d_alloc() local
1690 dentry = kmem_cache_alloc_lru(dentry_cache, &sb->s_dentry_lru, in __d_alloc()
1692 if (!dentry) in __d_alloc()
1701 dentry->d_shortname.string[DNAME_INLINE_LEN-1] = 0; in __d_alloc()
1704 dname = dentry->d_shortname.string; in __d_alloc()
1711 kmem_cache_free(dentry_cache, dentry); in __d_alloc()
1717 dname = dentry->d_shortname.string; in __d_alloc()
1720 dentry->d_name.len = name->len; in __d_alloc()
1721 dentry->d_name.hash = name->hash; in __d_alloc()
1726 smp_store_release(&dentry->d_name.name, dname); /* ^^^ */ in __d_alloc()
1728 dentry->d_flags = 0; in __d_alloc()
1729 lockref_init(&dentry->d_lockref); in __d_alloc()
1730 seqcount_spinlock_init(&dentry->d_seq, &dentry->d_lock); in __d_alloc()
1731 dentry->d_inode = NULL; in __d_alloc()
1732 dentry->d_parent = dentry; in __d_alloc()
1733 dentry->d_sb = sb; in __d_alloc()
1734 dentry->d_op = sb->__s_d_op; in __d_alloc()
1735 dentry->d_flags = sb->s_d_flags; in __d_alloc()
1736 dentry->d_fsdata = NULL; in __d_alloc()
1737 INIT_HLIST_BL_NODE(&dentry->d_hash); in __d_alloc()
1738 INIT_LIST_HEAD(&dentry->d_lru); in __d_alloc()
1739 INIT_HLIST_HEAD(&dentry->d_children); in __d_alloc()
1740 INIT_HLIST_NODE(&dentry->d_u.d_alias); in __d_alloc()
1741 INIT_HLIST_NODE(&dentry->d_sib); in __d_alloc()
1743 if (dentry->d_op && dentry->d_op->d_init) { in __d_alloc()
1744 err = dentry->d_op->d_init(dentry); in __d_alloc()
1746 if (dname_external(dentry)) in __d_alloc()
1747 kfree(external_name(dentry)); in __d_alloc()
1748 kmem_cache_free(dentry_cache, dentry); in __d_alloc()
1755 return dentry; in __d_alloc()
1767 struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) in d_alloc()
1769 struct dentry *dentry = __d_alloc(parent->d_sb, name); in d_alloc() local
1770 if (!dentry) in d_alloc()
1777 dentry->d_parent = dget_dlock(parent); in d_alloc()
1778 hlist_add_head(&dentry->d_sib, &parent->d_children); in d_alloc()
1781 return dentry; in d_alloc()
1785 struct dentry *d_alloc_anon(struct super_block *sb) in d_alloc_anon()
1791 struct dentry *d_alloc_cursor(struct dentry * parent) in d_alloc_cursor()
1793 struct dentry *dentry = d_alloc_anon(parent->d_sb); in d_alloc_cursor() local
1794 if (dentry) { in d_alloc_cursor()
1795 dentry->d_flags |= DCACHE_DENTRY_CURSOR; in d_alloc_cursor()
1796 dentry->d_parent = dget(parent); in d_alloc_cursor()
1798 return dentry; in d_alloc_cursor()
1816 struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name) in d_alloc_pseudo()
1821 struct dentry *dentry = __d_alloc(sb, name); in d_alloc_pseudo() local
1822 if (likely(dentry)) { in d_alloc_pseudo()
1823 dentry->d_flags |= DCACHE_NORCU; in d_alloc_pseudo()
1825 if (!dentry->d_op) in d_alloc_pseudo()
1826 dentry->d_op = &anon_ops; in d_alloc_pseudo()
1828 return dentry; in d_alloc_pseudo()
1831 struct dentry *d_alloc_name(struct dentry *parent, const char *name) in d_alloc_name()
1868 static void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op) in d_set_d_op() argument
1871 WARN_ON_ONCE(dentry->d_op); in d_set_d_op()
1872 WARN_ON_ONCE(dentry->d_flags & DCACHE_OP_FLAGS); in d_set_d_op()
1873 dentry->d_op = op; in d_set_d_op()
1875 dentry->d_flags |= flags; in d_set_d_op()
1921 static void __d_instantiate(struct dentry *dentry, struct inode *inode) in __d_instantiate() argument
1924 WARN_ON(d_in_lookup(dentry)); in __d_instantiate()
1926 spin_lock(&dentry->d_lock); in __d_instantiate()
1931 if ((dentry->d_flags & in __d_instantiate()
1934 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry); in __d_instantiate()
1935 raw_write_seqcount_begin(&dentry->d_seq); in __d_instantiate()
1936 __d_set_inode_and_type(dentry, inode, add_flags); in __d_instantiate()
1937 raw_write_seqcount_end(&dentry->d_seq); in __d_instantiate()
1938 fsnotify_update_flags(dentry); in __d_instantiate()
1939 spin_unlock(&dentry->d_lock); in __d_instantiate()
1957 void d_instantiate(struct dentry *entry, struct inode * inode) in d_instantiate()
1975 void d_instantiate_new(struct dentry *entry, struct inode *inode) in d_instantiate_new()
1996 struct dentry *d_make_root(struct inode *root_inode) in d_make_root()
1998 struct dentry *res = NULL; in d_make_root()
2011 static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected) in __d_obtain_alias()
2014 struct dentry *new, *res; in __d_obtain_alias()
2082 struct dentry *d_obtain_alias(struct inode *inode) in d_obtain_alias()
2103 struct dentry *d_obtain_root(struct inode *inode) in d_obtain_root()
2125 struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode, in d_add_ci() argument
2128 struct dentry *found, *res; in d_add_ci()
2134 found = d_hash_and_lookup(dentry->d_parent, name); in d_add_ci()
2139 if (d_in_lookup(dentry)) { in d_add_ci()
2140 found = d_alloc_parallel(dentry->d_parent, name, in d_add_ci()
2141 dentry->d_wait); in d_add_ci()
2147 found = d_alloc(dentry->d_parent, name); in d_add_ci()
2171 bool d_same_name(const struct dentry *dentry, const struct dentry *parent, in d_same_name() argument
2175 if (dentry->d_name.len != name->len) in d_same_name()
2177 return dentry_cmp(dentry, name->name, name->len) == 0; in d_same_name()
2179 return parent->d_op->d_compare(dentry, in d_same_name()
2180 dentry->d_name.len, dentry->d_name.name, in d_same_name()
2189 static noinline struct dentry *__d_lookup_rcu_op_compare( in __d_lookup_rcu_op_compare()
2190 const struct dentry *parent, in __d_lookup_rcu_op_compare()
2197 struct dentry *dentry; in __d_lookup_rcu_op_compare() local
2199 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { in __d_lookup_rcu_op_compare()
2205 seq = raw_seqcount_begin(&dentry->d_seq); in __d_lookup_rcu_op_compare()
2206 if (dentry->d_parent != parent) in __d_lookup_rcu_op_compare()
2208 if (d_unhashed(dentry)) in __d_lookup_rcu_op_compare()
2210 if (dentry->d_name.hash != hashlen_hash(hashlen)) in __d_lookup_rcu_op_compare()
2212 tlen = dentry->d_name.len; in __d_lookup_rcu_op_compare()
2213 tname = dentry->d_name.name; in __d_lookup_rcu_op_compare()
2215 if (read_seqcount_retry(&dentry->d_seq, seq)) { in __d_lookup_rcu_op_compare()
2219 if (parent->d_op->d_compare(dentry, tlen, tname, name) != 0) in __d_lookup_rcu_op_compare()
2222 return dentry; in __d_lookup_rcu_op_compare()
2253 struct dentry *__d_lookup_rcu(const struct dentry *parent, in __d_lookup_rcu()
2261 struct dentry *dentry; in __d_lookup_rcu() local
2286 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { in __d_lookup_rcu()
2306 seq = raw_seqcount_begin(&dentry->d_seq); in __d_lookup_rcu()
2307 if (dentry->d_parent != parent) in __d_lookup_rcu()
2309 if (d_unhashed(dentry)) in __d_lookup_rcu()
2311 if (dentry->d_name.hash_len != hashlen) in __d_lookup_rcu()
2313 if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0) in __d_lookup_rcu()
2316 return dentry; in __d_lookup_rcu()
2332 struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name) in d_lookup()
2334 struct dentry *dentry; in d_lookup() local
2339 dentry = __d_lookup(parent, name); in d_lookup()
2340 if (dentry) in d_lookup()
2343 return dentry; in d_lookup()
2362 struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name) in __d_lookup()
2367 struct dentry *found = NULL; in __d_lookup()
2368 struct dentry *dentry; in __d_lookup() local
2392 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { in __d_lookup()
2394 if (dentry->d_name.hash != hash) in __d_lookup()
2397 spin_lock(&dentry->d_lock); in __d_lookup()
2398 if (dentry->d_parent != parent) in __d_lookup()
2400 if (d_unhashed(dentry)) in __d_lookup()
2403 if (!d_same_name(dentry, parent, name)) in __d_lookup()
2406 dentry->d_lockref.count++; in __d_lookup()
2407 found = dentry; in __d_lookup()
2408 spin_unlock(&dentry->d_lock); in __d_lookup()
2411 spin_unlock(&dentry->d_lock); in __d_lookup()
2425 struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name) in d_hash_and_lookup()
2462 void d_delete(struct dentry * dentry) in d_delete() argument
2464 struct inode *inode = dentry->d_inode; in d_delete()
2467 spin_lock(&dentry->d_lock); in d_delete()
2471 if (dentry->d_lockref.count == 1) { in d_delete()
2473 __d_drop(dentry); in d_delete()
2474 dentry->d_flags &= ~DCACHE_CANT_MOUNT; in d_delete()
2475 dentry_unlink_inode(dentry); in d_delete()
2477 __d_drop(dentry); in d_delete()
2478 spin_unlock(&dentry->d_lock); in d_delete()
2484 static void __d_rehash(struct dentry *entry) in __d_rehash()
2500 void d_rehash(struct dentry * entry) in d_rehash()
2528 static void d_wait_lookup(struct dentry *dentry) in d_wait_lookup() argument
2530 if (d_in_lookup(dentry)) { in d_wait_lookup()
2532 add_wait_queue(dentry->d_wait, &wait); in d_wait_lookup()
2535 spin_unlock(&dentry->d_lock); in d_wait_lookup()
2537 spin_lock(&dentry->d_lock); in d_wait_lookup()
2538 } while (d_in_lookup(dentry)); in d_wait_lookup()
2542 struct dentry *d_alloc_parallel(struct dentry *parent, in d_alloc_parallel()
2549 struct dentry *new = __d_alloc(parent->d_sb, name); in d_alloc_parallel()
2550 struct dentry *dentry; in d_alloc_parallel() local
2566 dentry = __d_lookup_rcu(parent, name, &d_seq); in d_alloc_parallel()
2567 if (unlikely(dentry)) { in d_alloc_parallel()
2568 if (!lockref_get_not_dead(&dentry->d_lockref)) { in d_alloc_parallel()
2572 if (read_seqcount_retry(&dentry->d_seq, d_seq)) { in d_alloc_parallel()
2574 dput(dentry); in d_alloc_parallel()
2579 return dentry; in d_alloc_parallel()
2604 hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) { in d_alloc_parallel()
2605 if (dentry->d_name.hash != hash) in d_alloc_parallel()
2607 if (dentry->d_parent != parent) in d_alloc_parallel()
2609 if (!d_same_name(dentry, parent, name)) in d_alloc_parallel()
2613 if (!lockref_get_not_dead(&dentry->d_lockref)) { in d_alloc_parallel()
2623 spin_lock(&dentry->d_lock); in d_alloc_parallel()
2624 d_wait_lookup(dentry); in d_alloc_parallel()
2631 if (unlikely(dentry->d_name.hash != hash)) in d_alloc_parallel()
2633 if (unlikely(dentry->d_parent != parent)) in d_alloc_parallel()
2635 if (unlikely(d_unhashed(dentry))) in d_alloc_parallel()
2637 if (unlikely(!d_same_name(dentry, parent, name))) in d_alloc_parallel()
2640 spin_unlock(&dentry->d_lock); in d_alloc_parallel()
2642 return dentry; in d_alloc_parallel()
2650 spin_unlock(&dentry->d_lock); in d_alloc_parallel()
2651 dput(dentry); in d_alloc_parallel()
2661 static wait_queue_head_t *__d_lookup_unhash(struct dentry *dentry) in __d_lookup_unhash() argument
2666 lockdep_assert_held(&dentry->d_lock); in __d_lookup_unhash()
2668 b = in_lookup_hash(dentry->d_parent, dentry->d_name.hash); in __d_lookup_unhash()
2670 dentry->d_flags &= ~DCACHE_PAR_LOOKUP; in __d_lookup_unhash()
2671 __hlist_bl_del(&dentry->d_u.d_in_lookup_hash); in __d_lookup_unhash()
2672 d_wait = dentry->d_wait; in __d_lookup_unhash()
2673 dentry->d_wait = NULL; in __d_lookup_unhash()
2675 INIT_HLIST_NODE(&dentry->d_u.d_alias); in __d_lookup_unhash()
2676 INIT_LIST_HEAD(&dentry->d_lru); in __d_lookup_unhash()
2680 void __d_lookup_unhash_wake(struct dentry *dentry) in __d_lookup_unhash_wake() argument
2682 spin_lock(&dentry->d_lock); in __d_lookup_unhash_wake()
2683 wake_up_all(__d_lookup_unhash(dentry)); in __d_lookup_unhash_wake()
2684 spin_unlock(&dentry->d_lock); in __d_lookup_unhash_wake()
2690 static inline void __d_add(struct dentry *dentry, struct inode *inode, in __d_add() argument
2696 spin_lock(&dentry->d_lock); in __d_add()
2697 if (unlikely(d_in_lookup(dentry))) { in __d_add()
2698 dir = dentry->d_parent->d_inode; in __d_add()
2700 d_wait = __d_lookup_unhash(dentry); in __d_add()
2703 d_set_d_op(dentry, ops); in __d_add()
2706 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry); in __d_add()
2707 raw_write_seqcount_begin(&dentry->d_seq); in __d_add()
2708 __d_set_inode_and_type(dentry, inode, add_flags); in __d_add()
2709 raw_write_seqcount_end(&dentry->d_seq); in __d_add()
2710 fsnotify_update_flags(dentry); in __d_add()
2712 __d_rehash(dentry); in __d_add()
2715 spin_unlock(&dentry->d_lock); in __d_add()
2729 void d_add(struct dentry *entry, struct inode *inode) in d_add()
2739 static void swap_names(struct dentry *dentry, struct dentry *target) in swap_names() argument
2742 if (unlikely(dname_external(dentry))) { in swap_names()
2746 swap(target->d_name.name, dentry->d_name.name); in swap_names()
2752 dentry->d_name.name = target->d_name.name; in swap_names()
2753 target->d_shortname = dentry->d_shortname; in swap_names()
2757 if (unlikely(dname_external(dentry))) { in swap_names()
2762 target->d_name.name = dentry->d_name.name; in swap_names()
2763 dentry->d_shortname = target->d_shortname; in swap_names()
2764 dentry->d_name.name = dentry->d_shortname.string; in swap_names()
2770 swap(dentry->d_shortname.words[i], in swap_names()
2774 swap(dentry->d_name.hash_len, target->d_name.hash_len); in swap_names()
2777 static void copy_name(struct dentry *dentry, struct dentry *target) in copy_name() argument
2780 if (unlikely(dname_external(dentry))) in copy_name()
2781 old_name = external_name(dentry); in copy_name()
2784 dentry->d_name = target->d_name; in copy_name()
2786 dentry->d_shortname = target->d_shortname; in copy_name()
2787 dentry->d_name.name = dentry->d_shortname.string; in copy_name()
2788 dentry->d_name.hash_len = target->d_name.hash_len; in copy_name()
2805 static void __d_move(struct dentry *dentry, struct dentry *target, in __d_move() argument
2808 struct dentry *old_parent, *p; in __d_move()
2813 WARN_ON(!dentry->d_inode); in __d_move()
2814 if (WARN_ON(dentry == target)) in __d_move()
2817 BUG_ON(d_ancestor(target, dentry)); in __d_move()
2818 old_parent = dentry->d_parent; in __d_move()
2820 if (IS_ROOT(dentry)) { in __d_move()
2828 BUG_ON(p == dentry); in __d_move()
2834 spin_lock_nested(&dentry->d_lock, 2); in __d_move()
2843 write_seqcount_begin(&dentry->d_seq); in __d_move()
2847 if (!d_unhashed(dentry)) in __d_move()
2848 ___d_drop(dentry); in __d_move()
2853 dentry->d_parent = target->d_parent; in __d_move()
2855 copy_name(dentry, target); in __d_move()
2857 dentry->d_parent->d_lockref.count++; in __d_move()
2858 if (dentry != old_parent) /* wasn't IS_ROOT */ in __d_move()
2862 swap_names(dentry, target); in __d_move()
2869 if (!hlist_unhashed(&dentry->d_sib)) in __d_move()
2870 __hlist_del(&dentry->d_sib); in __d_move()
2871 hlist_add_head(&dentry->d_sib, &dentry->d_parent->d_children); in __d_move()
2872 __d_rehash(dentry); in __d_move()
2873 fsnotify_update_flags(dentry); in __d_move()
2874 fscrypt_handle_d_move(dentry); in __d_move()
2877 write_seqcount_end(&dentry->d_seq); in __d_move()
2882 if (dentry->d_parent != old_parent) in __d_move()
2883 spin_unlock(&dentry->d_parent->d_lock); in __d_move()
2884 if (dentry != old_parent) in __d_move()
2887 spin_unlock(&dentry->d_lock); in __d_move()
2899 void d_move(struct dentry *dentry, struct dentry *target) in d_move() argument
2902 __d_move(dentry, target, false); in d_move()
2912 void d_exchange(struct dentry *dentry1, struct dentry *dentry2) in d_exchange()
2934 struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2) in d_ancestor()
2936 struct dentry *p; in d_ancestor()
2954 static int __d_unalias(struct dentry *dentry, struct dentry *alias) in __d_unalias() argument
2961 if (alias->d_parent == dentry->d_parent) in __d_unalias()
2965 if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex)) in __d_unalias()
2967 m1 = &dentry->d_sb->s_vfs_rename_mutex; in __d_unalias()
2975 __d_move(alias, dentry, false); in __d_unalias()
2987 struct dentry *d_splice_alias_ops(struct inode *inode, struct dentry *dentry, in d_splice_alias_ops() argument
2993 BUG_ON(!d_unhashed(dentry)); in d_splice_alias_ops()
2998 security_d_instantiate(dentry, inode); in d_splice_alias_ops()
3001 struct dentry *new = __d_find_any_alias(inode); in d_splice_alias_ops()
3006 if (unlikely(d_ancestor(new, dentry))) { in d_splice_alias_ops()
3013 dentry->d_name.name, in d_splice_alias_ops()
3017 struct dentry *old_parent = dget(new->d_parent); in d_splice_alias_ops()
3018 int err = __d_unalias(dentry, new); in d_splice_alias_ops()
3026 __d_move(new, dentry, false); in d_splice_alias_ops()
3034 __d_add(dentry, inode, ops); in d_splice_alias_ops()
3061 struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry) in d_splice_alias() argument
3063 return d_splice_alias_ops(inode, dentry, NULL); in d_splice_alias()
3083 bool is_subdir(struct dentry *new_dentry, struct dentry *old_dentry) in is_subdir()
3107 static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry) in d_genocide_kill() argument
3109 struct dentry *root = data; in d_genocide_kill()
3110 if (dentry != root) { in d_genocide_kill()
3111 if (d_unhashed(dentry) || !dentry->d_inode) in d_genocide_kill()
3114 if (!(dentry->d_flags & DCACHE_GENOCIDE)) { in d_genocide_kill()
3115 dentry->d_flags |= DCACHE_GENOCIDE; in d_genocide_kill()
3116 dentry->d_lockref.count--; in d_genocide_kill()
3122 void d_genocide(struct dentry *parent) in d_genocide()
3129 struct dentry *dentry = file->f_path.dentry; in d_mark_tmpfile() local
3131 BUG_ON(dname_external(dentry) || in d_mark_tmpfile()
3132 !hlist_unhashed(&dentry->d_u.d_alias) || in d_mark_tmpfile()
3133 !d_unlinked(dentry)); in d_mark_tmpfile()
3134 spin_lock(&dentry->d_parent->d_lock); in d_mark_tmpfile()
3135 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); in d_mark_tmpfile()
3136 dentry->d_name.len = sprintf(dentry->d_shortname.string, "#%llu", in d_mark_tmpfile()
3138 spin_unlock(&dentry->d_lock); in d_mark_tmpfile()
3139 spin_unlock(&dentry->d_parent->d_lock); in d_mark_tmpfile()
3145 struct dentry *dentry = file->f_path.dentry; in d_tmpfile() local
3149 d_instantiate(dentry, inode); in d_tmpfile()
3156 ino_t d_parent_ino(struct dentry *dentry) in d_parent_ino() argument
3158 struct dentry *parent; in d_parent_ino()
3164 seq = raw_seqcount_begin(&dentry->d_seq); in d_parent_ino()
3165 parent = READ_ONCE(dentry->d_parent); in d_parent_ino()
3169 if (!read_seqcount_retry(&dentry->d_seq, seq)) in d_parent_ino()
3174 spin_lock(&dentry->d_lock); in d_parent_ino()
3175 ret = dentry->d_parent->d_inode->i_ino; in d_parent_ino()
3176 spin_unlock(&dentry->d_lock); in d_parent_ino()
3222 dentry_cache = KMEM_CACHE_USERCOPY(dentry, in dcache_init()