Lines Matching refs:inode
212 static int no_open(struct inode *inode, struct file *file) in no_open() argument
227 int inode_init_always_gfp(struct super_block *sb, struct inode *inode, gfp_t gfp) in inode_init_always_gfp() argument
231 struct address_space *const mapping = &inode->i_data; in inode_init_always_gfp()
233 inode->i_sb = sb; in inode_init_always_gfp()
234 inode->i_blkbits = sb->s_blocksize_bits; in inode_init_always_gfp()
235 inode->i_flags = 0; in inode_init_always_gfp()
236 inode->i_state = 0; in inode_init_always_gfp()
237 atomic64_set(&inode->i_sequence, 0); in inode_init_always_gfp()
238 atomic_set(&inode->i_count, 1); in inode_init_always_gfp()
239 inode->i_op = &empty_iops; in inode_init_always_gfp()
240 inode->i_fop = &no_open_fops; in inode_init_always_gfp()
241 inode->i_ino = 0; in inode_init_always_gfp()
242 inode->__i_nlink = 1; in inode_init_always_gfp()
243 inode->i_opflags = 0; in inode_init_always_gfp()
245 inode->i_opflags |= IOP_XATTR; in inode_init_always_gfp()
247 inode->i_opflags |= IOP_MGTIME; in inode_init_always_gfp()
248 i_uid_write(inode, 0); in inode_init_always_gfp()
249 i_gid_write(inode, 0); in inode_init_always_gfp()
250 atomic_set(&inode->i_writecount, 0); in inode_init_always_gfp()
251 inode->i_size = 0; in inode_init_always_gfp()
252 inode->i_write_hint = WRITE_LIFE_NOT_SET; in inode_init_always_gfp()
253 inode->i_blocks = 0; in inode_init_always_gfp()
254 inode->i_bytes = 0; in inode_init_always_gfp()
255 inode->i_generation = 0; in inode_init_always_gfp()
256 inode->i_pipe = NULL; in inode_init_always_gfp()
257 inode->i_cdev = NULL; in inode_init_always_gfp()
258 inode->i_link = NULL; in inode_init_always_gfp()
259 inode->i_dir_seq = 0; in inode_init_always_gfp()
260 inode->i_rdev = 0; in inode_init_always_gfp()
261 inode->dirtied_when = 0; in inode_init_always_gfp()
264 inode->i_wb_frn_winner = 0; in inode_init_always_gfp()
265 inode->i_wb_frn_avg_time = 0; in inode_init_always_gfp()
266 inode->i_wb_frn_history = 0; in inode_init_always_gfp()
269 spin_lock_init(&inode->i_lock); in inode_init_always_gfp()
270 lockdep_set_class(&inode->i_lock, &sb->s_type->i_lock_key); in inode_init_always_gfp()
272 init_rwsem(&inode->i_rwsem); in inode_init_always_gfp()
273 lockdep_set_class(&inode->i_rwsem, &sb->s_type->i_mutex_key); in inode_init_always_gfp()
275 atomic_set(&inode->i_dio_count, 0); in inode_init_always_gfp()
278 mapping->host = inode; in inode_init_always_gfp()
294 inode->i_private = NULL; in inode_init_always_gfp()
295 inode->i_mapping = mapping; in inode_init_always_gfp()
296 INIT_HLIST_HEAD(&inode->i_dentry); /* buggered by rcu freeing */ in inode_init_always_gfp()
298 inode->i_acl = inode->i_default_acl = ACL_NOT_CACHED; in inode_init_always_gfp()
302 inode->i_fsnotify_mask = 0; in inode_init_always_gfp()
304 inode->i_flctx = NULL; in inode_init_always_gfp()
306 if (unlikely(security_inode_alloc(inode, gfp))) in inode_init_always_gfp()
315 void free_inode_nonrcu(struct inode *inode) in free_inode_nonrcu() argument
317 kmem_cache_free(inode_cachep, inode); in free_inode_nonrcu()
323 struct inode *inode = container_of(head, struct inode, i_rcu); in i_callback() local
324 if (inode->free_inode) in i_callback()
325 inode->free_inode(inode); in i_callback()
327 free_inode_nonrcu(inode); in i_callback()
340 struct inode *alloc_inode(struct super_block *sb) in alloc_inode()
343 struct inode *inode; in alloc_inode() local
346 inode = ops->alloc_inode(sb); in alloc_inode()
348 inode = alloc_inode_sb(sb, inode_cachep, GFP_KERNEL); in alloc_inode()
350 if (!inode) in alloc_inode()
353 if (unlikely(inode_init_always(sb, inode))) { in alloc_inode()
355 ops->destroy_inode(inode); in alloc_inode()
359 inode->free_inode = ops->free_inode; in alloc_inode()
360 i_callback(&inode->i_rcu); in alloc_inode()
364 return inode; in alloc_inode()
367 void __destroy_inode(struct inode *inode) in __destroy_inode() argument
369 BUG_ON(inode_has_buffers(inode)); in __destroy_inode()
370 inode_detach_wb(inode); in __destroy_inode()
371 security_inode_free(inode); in __destroy_inode()
372 fsnotify_inode_delete(inode); in __destroy_inode()
373 locks_free_lock_context(inode); in __destroy_inode()
374 if (!inode->i_nlink) { in __destroy_inode()
375 WARN_ON(atomic_long_read(&inode->i_sb->s_remove_count) == 0); in __destroy_inode()
376 atomic_long_dec(&inode->i_sb->s_remove_count); in __destroy_inode()
380 if (inode->i_acl && !is_uncached_acl(inode->i_acl)) in __destroy_inode()
381 posix_acl_release(inode->i_acl); in __destroy_inode()
382 if (inode->i_default_acl && !is_uncached_acl(inode->i_default_acl)) in __destroy_inode()
383 posix_acl_release(inode->i_default_acl); in __destroy_inode()
389 static void destroy_inode(struct inode *inode) in destroy_inode() argument
391 const struct super_operations *ops = inode->i_sb->s_op; in destroy_inode()
393 BUG_ON(!list_empty(&inode->i_lru)); in destroy_inode()
394 __destroy_inode(inode); in destroy_inode()
396 ops->destroy_inode(inode); in destroy_inode()
400 inode->free_inode = ops->free_inode; in destroy_inode()
401 call_rcu(&inode->i_rcu, i_callback); in destroy_inode()
415 void drop_nlink(struct inode *inode) in drop_nlink() argument
417 WARN_ON(inode->i_nlink == 0); in drop_nlink()
418 inode->__i_nlink--; in drop_nlink()
419 if (!inode->i_nlink) in drop_nlink()
420 atomic_long_inc(&inode->i_sb->s_remove_count); in drop_nlink()
432 void clear_nlink(struct inode *inode) in clear_nlink() argument
434 if (inode->i_nlink) { in clear_nlink()
435 inode->__i_nlink = 0; in clear_nlink()
436 atomic_long_inc(&inode->i_sb->s_remove_count); in clear_nlink()
449 void set_nlink(struct inode *inode, unsigned int nlink) in set_nlink() argument
452 clear_nlink(inode); in set_nlink()
455 if (inode->i_nlink == 0) in set_nlink()
456 atomic_long_dec(&inode->i_sb->s_remove_count); in set_nlink()
458 inode->__i_nlink = nlink; in set_nlink()
471 void inc_nlink(struct inode *inode) in inc_nlink() argument
473 if (unlikely(inode->i_nlink == 0)) { in inc_nlink()
474 WARN_ON(!(inode->i_state & I_LINKABLE)); in inc_nlink()
475 atomic_long_dec(&inode->i_sb->s_remove_count); in inc_nlink()
478 inode->__i_nlink++; in inc_nlink()
503 void inode_init_once(struct inode *inode) in inode_init_once() argument
505 memset(inode, 0, sizeof(*inode)); in inode_init_once()
506 INIT_HLIST_NODE(&inode->i_hash); in inode_init_once()
507 INIT_LIST_HEAD(&inode->i_devices); in inode_init_once()
508 INIT_LIST_HEAD(&inode->i_io_list); in inode_init_once()
509 INIT_LIST_HEAD(&inode->i_wb_list); in inode_init_once()
510 INIT_LIST_HEAD(&inode->i_lru); in inode_init_once()
511 INIT_LIST_HEAD(&inode->i_sb_list); in inode_init_once()
512 __address_space_init_once(&inode->i_data); in inode_init_once()
513 i_size_ordered_init(inode); in inode_init_once()
519 struct inode *inode = (struct inode *) foo; in init_once() local
521 inode_init_once(inode); in init_once()
527 void ihold(struct inode *inode) in ihold() argument
529 WARN_ON(atomic_inc_return(&inode->i_count) < 2); in ihold()
533 static void __inode_add_lru(struct inode *inode, bool rotate) in __inode_add_lru() argument
535 if (inode->i_state & (I_DIRTY_ALL | I_SYNC | I_FREEING | I_WILL_FREE)) in __inode_add_lru()
537 if (atomic_read(&inode->i_count)) in __inode_add_lru()
539 if (!(inode->i_sb->s_flags & SB_ACTIVE)) in __inode_add_lru()
541 if (!mapping_shrinkable(&inode->i_data)) in __inode_add_lru()
544 if (list_lru_add_obj(&inode->i_sb->s_inode_lru, &inode->i_lru)) in __inode_add_lru()
547 inode->i_state |= I_REFERENCED; in __inode_add_lru()
551 struct inode *inode, u32 bit) in inode_bit_waitqueue() argument
555 bit_address = inode_state_wait_address(inode, bit); in inode_bit_waitqueue()
566 void inode_add_lru(struct inode *inode) in inode_add_lru() argument
568 __inode_add_lru(inode, false); in inode_add_lru()
571 static void inode_lru_list_del(struct inode *inode) in inode_lru_list_del() argument
573 if (list_lru_del_obj(&inode->i_sb->s_inode_lru, &inode->i_lru)) in inode_lru_list_del()
577 static void inode_pin_lru_isolating(struct inode *inode) in inode_pin_lru_isolating() argument
579 lockdep_assert_held(&inode->i_lock); in inode_pin_lru_isolating()
580 WARN_ON(inode->i_state & (I_LRU_ISOLATING | I_FREEING | I_WILL_FREE)); in inode_pin_lru_isolating()
581 inode->i_state |= I_LRU_ISOLATING; in inode_pin_lru_isolating()
584 static void inode_unpin_lru_isolating(struct inode *inode) in inode_unpin_lru_isolating() argument
586 spin_lock(&inode->i_lock); in inode_unpin_lru_isolating()
587 WARN_ON(!(inode->i_state & I_LRU_ISOLATING)); in inode_unpin_lru_isolating()
588 inode->i_state &= ~I_LRU_ISOLATING; in inode_unpin_lru_isolating()
590 inode_wake_up_bit(inode, __I_LRU_ISOLATING); in inode_unpin_lru_isolating()
591 spin_unlock(&inode->i_lock); in inode_unpin_lru_isolating()
594 static void inode_wait_for_lru_isolating(struct inode *inode) in inode_wait_for_lru_isolating() argument
599 lockdep_assert_held(&inode->i_lock); in inode_wait_for_lru_isolating()
600 if (!(inode->i_state & I_LRU_ISOLATING)) in inode_wait_for_lru_isolating()
603 wq_head = inode_bit_waitqueue(&wqe, inode, __I_LRU_ISOLATING); in inode_wait_for_lru_isolating()
610 if (!(inode->i_state & I_LRU_ISOLATING)) in inode_wait_for_lru_isolating()
612 spin_unlock(&inode->i_lock); in inode_wait_for_lru_isolating()
614 spin_lock(&inode->i_lock); in inode_wait_for_lru_isolating()
617 WARN_ON(inode->i_state & I_LRU_ISOLATING); in inode_wait_for_lru_isolating()
624 void inode_sb_list_add(struct inode *inode) in inode_sb_list_add() argument
626 struct super_block *sb = inode->i_sb; in inode_sb_list_add()
629 list_add(&inode->i_sb_list, &sb->s_inodes); in inode_sb_list_add()
634 static inline void inode_sb_list_del(struct inode *inode) in inode_sb_list_del() argument
636 struct super_block *sb = inode->i_sb; in inode_sb_list_del()
638 if (!list_empty(&inode->i_sb_list)) { in inode_sb_list_del()
640 list_del_init(&inode->i_sb_list); in inode_sb_list_del()
663 void __insert_inode_hash(struct inode *inode, unsigned long hashval) in __insert_inode_hash() argument
665 struct hlist_head *b = inode_hashtable + hash(inode->i_sb, hashval); in __insert_inode_hash()
668 spin_lock(&inode->i_lock); in __insert_inode_hash()
669 hlist_add_head_rcu(&inode->i_hash, b); in __insert_inode_hash()
670 spin_unlock(&inode->i_lock); in __insert_inode_hash()
681 void __remove_inode_hash(struct inode *inode) in __remove_inode_hash() argument
684 spin_lock(&inode->i_lock); in __remove_inode_hash()
685 hlist_del_init_rcu(&inode->i_hash); in __remove_inode_hash()
686 spin_unlock(&inode->i_lock); in __remove_inode_hash()
693 struct inode *host; in dump_mapping()
745 void clear_inode(struct inode *inode) in clear_inode() argument
752 xa_lock_irq(&inode->i_data.i_pages); in clear_inode()
753 BUG_ON(inode->i_data.nrpages); in clear_inode()
762 xa_unlock_irq(&inode->i_data.i_pages); in clear_inode()
763 BUG_ON(!list_empty(&inode->i_data.i_private_list)); in clear_inode()
764 BUG_ON(!(inode->i_state & I_FREEING)); in clear_inode()
765 BUG_ON(inode->i_state & I_CLEAR); in clear_inode()
766 BUG_ON(!list_empty(&inode->i_wb_list)); in clear_inode()
768 inode->i_state = I_FREEING | I_CLEAR; in clear_inode()
785 static void evict(struct inode *inode) in evict() argument
787 const struct super_operations *op = inode->i_sb->s_op; in evict()
789 BUG_ON(!(inode->i_state & I_FREEING)); in evict()
790 BUG_ON(!list_empty(&inode->i_lru)); in evict()
792 if (!list_empty(&inode->i_io_list)) in evict()
793 inode_io_list_del(inode); in evict()
795 inode_sb_list_del(inode); in evict()
797 spin_lock(&inode->i_lock); in evict()
798 inode_wait_for_lru_isolating(inode); in evict()
806 inode_wait_for_writeback(inode); in evict()
807 spin_unlock(&inode->i_lock); in evict()
810 op->evict_inode(inode); in evict()
812 truncate_inode_pages_final(&inode->i_data); in evict()
813 clear_inode(inode); in evict()
815 if (S_ISCHR(inode->i_mode) && inode->i_cdev) in evict()
816 cd_forget(inode); in evict()
818 remove_inode_hash(inode); in evict()
831 inode_wake_up_bit(inode, __I_NEW); in evict()
832 BUG_ON(inode->i_state != (I_FREEING | I_CLEAR)); in evict()
834 destroy_inode(inode); in evict()
847 struct inode *inode; in dispose_list() local
849 inode = list_first_entry(head, struct inode, i_lru); in dispose_list()
850 list_del_init(&inode->i_lru); in dispose_list()
852 evict(inode); in dispose_list()
868 struct inode *inode; in evict_inodes() local
873 list_for_each_entry(inode, &sb->s_inodes, i_sb_list) { in evict_inodes()
874 if (atomic_read(&inode->i_count)) in evict_inodes()
877 spin_lock(&inode->i_lock); in evict_inodes()
878 if (atomic_read(&inode->i_count)) { in evict_inodes()
879 spin_unlock(&inode->i_lock); in evict_inodes()
882 if (inode->i_state & (I_NEW | I_FREEING | I_WILL_FREE)) { in evict_inodes()
883 spin_unlock(&inode->i_lock); in evict_inodes()
887 inode->i_state |= I_FREEING; in evict_inodes()
888 inode_lru_list_del(inode); in evict_inodes()
889 spin_unlock(&inode->i_lock); in evict_inodes()
890 list_add(&inode->i_lru, &dispose); in evict_inodes()
925 struct inode *inode = container_of(item, struct inode, i_lru); in inode_lru_isolate() local
931 if (!spin_trylock(&inode->i_lock)) in inode_lru_isolate()
940 if (atomic_read(&inode->i_count) || in inode_lru_isolate()
941 (inode->i_state & ~I_REFERENCED) || in inode_lru_isolate()
942 !mapping_shrinkable(&inode->i_data)) { in inode_lru_isolate()
943 list_lru_isolate(lru, &inode->i_lru); in inode_lru_isolate()
944 spin_unlock(&inode->i_lock); in inode_lru_isolate()
950 if (inode->i_state & I_REFERENCED) { in inode_lru_isolate()
951 inode->i_state &= ~I_REFERENCED; in inode_lru_isolate()
952 spin_unlock(&inode->i_lock); in inode_lru_isolate()
961 if (inode_has_buffers(inode) || !mapping_empty(&inode->i_data)) { in inode_lru_isolate()
962 inode_pin_lru_isolating(inode); in inode_lru_isolate()
963 spin_unlock(&inode->i_lock); in inode_lru_isolate()
965 if (remove_inode_buffers(inode)) { in inode_lru_isolate()
967 reap = invalidate_mapping_pages(&inode->i_data, 0, -1); in inode_lru_isolate()
974 inode_unpin_lru_isolating(inode); in inode_lru_isolate()
978 WARN_ON(inode->i_state & I_NEW); in inode_lru_isolate()
979 inode->i_state |= I_FREEING; in inode_lru_isolate()
980 list_lru_isolate_move(lru, &inode->i_lru, freeable); in inode_lru_isolate()
981 spin_unlock(&inode->i_lock); in inode_lru_isolate()
1004 static void __wait_on_freeing_inode(struct inode *inode, bool is_inode_hash_locked);
1008 static struct inode *find_inode(struct super_block *sb, in find_inode()
1010 int (*test)(struct inode *, void *), in find_inode() argument
1013 struct inode *inode = NULL; in find_inode() local
1022 hlist_for_each_entry_rcu(inode, head, i_hash) { in find_inode()
1023 if (inode->i_sb != sb) in find_inode()
1025 if (!test(inode, data)) in find_inode()
1027 spin_lock(&inode->i_lock); in find_inode()
1028 if (inode->i_state & (I_FREEING|I_WILL_FREE)) { in find_inode()
1029 __wait_on_freeing_inode(inode, is_inode_hash_locked); in find_inode()
1032 if (unlikely(inode->i_state & I_CREATING)) { in find_inode()
1033 spin_unlock(&inode->i_lock); in find_inode()
1037 __iget(inode); in find_inode()
1038 spin_unlock(&inode->i_lock); in find_inode()
1040 return inode; in find_inode()
1050 static struct inode *find_inode_fast(struct super_block *sb, in find_inode_fast()
1054 struct inode *inode = NULL; in find_inode_fast() local
1063 hlist_for_each_entry_rcu(inode, head, i_hash) { in find_inode_fast()
1064 if (inode->i_ino != ino) in find_inode_fast()
1066 if (inode->i_sb != sb) in find_inode_fast()
1068 spin_lock(&inode->i_lock); in find_inode_fast()
1069 if (inode->i_state & (I_FREEING|I_WILL_FREE)) { in find_inode_fast()
1070 __wait_on_freeing_inode(inode, is_inode_hash_locked); in find_inode_fast()
1073 if (unlikely(inode->i_state & I_CREATING)) { in find_inode_fast()
1074 spin_unlock(&inode->i_lock); in find_inode_fast()
1078 __iget(inode); in find_inode_fast()
1079 spin_unlock(&inode->i_lock); in find_inode_fast()
1081 return inode; in find_inode_fast()
1141 struct inode *new_inode(struct super_block *sb) in new_inode()
1143 struct inode *inode; in new_inode() local
1145 inode = alloc_inode(sb); in new_inode()
1146 if (inode) in new_inode()
1147 inode_sb_list_add(inode); in new_inode()
1148 return inode; in new_inode()
1153 void lockdep_annotate_inode_mutex_key(struct inode *inode) in lockdep_annotate_inode_mutex_key() argument
1155 if (S_ISDIR(inode->i_mode)) { in lockdep_annotate_inode_mutex_key()
1156 struct file_system_type *type = inode->i_sb->s_type; in lockdep_annotate_inode_mutex_key()
1159 if (lockdep_match_class(&inode->i_rwsem, &type->i_mutex_key)) { in lockdep_annotate_inode_mutex_key()
1163 init_rwsem(&inode->i_rwsem); in lockdep_annotate_inode_mutex_key()
1164 lockdep_set_class(&inode->i_rwsem, in lockdep_annotate_inode_mutex_key()
1179 void unlock_new_inode(struct inode *inode) in unlock_new_inode() argument
1181 lockdep_annotate_inode_mutex_key(inode); in unlock_new_inode()
1182 spin_lock(&inode->i_lock); in unlock_new_inode()
1183 WARN_ON(!(inode->i_state & I_NEW)); in unlock_new_inode()
1184 inode->i_state &= ~I_NEW & ~I_CREATING; in unlock_new_inode()
1191 inode_wake_up_bit(inode, __I_NEW); in unlock_new_inode()
1192 spin_unlock(&inode->i_lock); in unlock_new_inode()
1196 void discard_new_inode(struct inode *inode) in discard_new_inode() argument
1198 lockdep_annotate_inode_mutex_key(inode); in discard_new_inode()
1199 spin_lock(&inode->i_lock); in discard_new_inode()
1200 WARN_ON(!(inode->i_state & I_NEW)); in discard_new_inode()
1201 inode->i_state &= ~I_NEW; in discard_new_inode()
1208 inode_wake_up_bit(inode, __I_NEW); in discard_new_inode()
1209 spin_unlock(&inode->i_lock); in discard_new_inode()
1210 iput(inode); in discard_new_inode()
1223 void lock_two_nondirectories(struct inode *inode1, struct inode *inode2) in lock_two_nondirectories()
1243 void unlock_two_nondirectories(struct inode *inode1, struct inode *inode2) in unlock_two_nondirectories()
1275 struct inode *inode_insert5(struct inode *inode, unsigned long hashval, in inode_insert5() argument
1276 int (*test)(struct inode *, void *), in inode_insert5() argument
1277 int (*set)(struct inode *, void *), void *data) in inode_insert5() argument
1279 struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval); in inode_insert5()
1280 struct inode *old; in inode_insert5()
1284 old = find_inode(inode->i_sb, head, test, data, true); in inode_insert5()
1301 if (set && unlikely(set(inode, data))) { in inode_insert5()
1310 spin_lock(&inode->i_lock); in inode_insert5()
1311 inode->i_state |= I_NEW; in inode_insert5()
1312 hlist_add_head_rcu(&inode->i_hash, head); in inode_insert5()
1313 spin_unlock(&inode->i_lock); in inode_insert5()
1321 if (list_empty(&inode->i_sb_list)) in inode_insert5()
1322 inode_sb_list_add(inode); in inode_insert5()
1324 return inode; in inode_insert5()
1348 struct inode *iget5_locked(struct super_block *sb, unsigned long hashval, in iget5_locked()
1349 int (*test)(struct inode *, void *), in iget5_locked() argument
1350 int (*set)(struct inode *, void *), void *data) in iget5_locked() argument
1352 struct inode *inode = ilookup5(sb, hashval, test, data); in iget5_locked() local
1354 if (!inode) { in iget5_locked()
1355 struct inode *new = alloc_inode(sb); in iget5_locked()
1358 inode = inode_insert5(new, hashval, test, set, data); in iget5_locked()
1359 if (unlikely(inode != new)) in iget5_locked()
1363 return inode; in iget5_locked()
1378 struct inode *iget5_locked_rcu(struct super_block *sb, unsigned long hashval, in iget5_locked_rcu()
1379 int (*test)(struct inode *, void *), in iget5_locked_rcu() argument
1380 int (*set)(struct inode *, void *), void *data) in iget5_locked_rcu() argument
1383 struct inode *inode, *new; in iget5_locked_rcu() local
1386 inode = find_inode(sb, head, test, data, false); in iget5_locked_rcu()
1387 if (inode) { in iget5_locked_rcu()
1388 if (IS_ERR(inode)) in iget5_locked_rcu()
1390 wait_on_inode(inode); in iget5_locked_rcu()
1391 if (unlikely(inode_unhashed(inode))) { in iget5_locked_rcu()
1392 iput(inode); in iget5_locked_rcu()
1395 return inode; in iget5_locked_rcu()
1400 inode = inode_insert5(new, hashval, test, set, data); in iget5_locked_rcu()
1401 if (unlikely(inode != new)) in iget5_locked_rcu()
1404 return inode; in iget5_locked_rcu()
1421 struct inode *iget_locked(struct super_block *sb, unsigned long ino) in iget_locked()
1424 struct inode *inode; in iget_locked() local
1426 inode = find_inode_fast(sb, head, ino, false); in iget_locked()
1427 if (inode) { in iget_locked()
1428 if (IS_ERR(inode)) in iget_locked()
1430 wait_on_inode(inode); in iget_locked()
1431 if (unlikely(inode_unhashed(inode))) { in iget_locked()
1432 iput(inode); in iget_locked()
1435 return inode; in iget_locked()
1438 inode = alloc_inode(sb); in iget_locked()
1439 if (inode) { in iget_locked()
1440 struct inode *old; in iget_locked()
1446 inode->i_ino = ino; in iget_locked()
1447 spin_lock(&inode->i_lock); in iget_locked()
1448 inode->i_state = I_NEW; in iget_locked()
1449 hlist_add_head_rcu(&inode->i_hash, head); in iget_locked()
1450 spin_unlock(&inode->i_lock); in iget_locked()
1452 inode_sb_list_add(inode); in iget_locked()
1457 return inode; in iget_locked()
1466 destroy_inode(inode); in iget_locked()
1469 inode = old; in iget_locked()
1470 wait_on_inode(inode); in iget_locked()
1471 if (unlikely(inode_unhashed(inode))) { in iget_locked()
1472 iput(inode); in iget_locked()
1476 return inode; in iget_locked()
1490 struct inode *inode; in test_inode_iunique() local
1492 hlist_for_each_entry_rcu(inode, b, i_hash) { in test_inode_iunique()
1493 if (inode->i_ino == ino && inode->i_sb == sb) in test_inode_iunique()
1538 struct inode *igrab(struct inode *inode) in igrab() argument
1540 spin_lock(&inode->i_lock); in igrab()
1541 if (!(inode->i_state & (I_FREEING|I_WILL_FREE))) { in igrab()
1542 __iget(inode); in igrab()
1543 spin_unlock(&inode->i_lock); in igrab()
1545 spin_unlock(&inode->i_lock); in igrab()
1551 inode = NULL; in igrab()
1553 return inode; in igrab()
1573 struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval, in ilookup5_nowait()
1574 int (*test)(struct inode *, void *), void *data) in ilookup5_nowait() argument
1577 struct inode *inode; in ilookup5_nowait() local
1580 inode = find_inode(sb, head, test, data, true); in ilookup5_nowait()
1583 return IS_ERR(inode) ? NULL : inode; in ilookup5_nowait()
1604 struct inode *ilookup5(struct super_block *sb, unsigned long hashval, in ilookup5()
1605 int (*test)(struct inode *, void *), void *data) in ilookup5() argument
1607 struct inode *inode; in ilookup5() local
1609 inode = ilookup5_nowait(sb, hashval, test, data); in ilookup5()
1610 if (inode) { in ilookup5()
1611 wait_on_inode(inode); in ilookup5()
1612 if (unlikely(inode_unhashed(inode))) { in ilookup5()
1613 iput(inode); in ilookup5()
1617 return inode; in ilookup5()
1629 struct inode *ilookup(struct super_block *sb, unsigned long ino) in ilookup()
1632 struct inode *inode; in ilookup() local
1634 inode = find_inode_fast(sb, head, ino, false); in ilookup()
1636 if (inode) { in ilookup()
1637 if (IS_ERR(inode)) in ilookup()
1639 wait_on_inode(inode); in ilookup()
1640 if (unlikely(inode_unhashed(inode))) { in ilookup()
1641 iput(inode); in ilookup()
1645 return inode; in ilookup()
1672 struct inode *find_inode_nowait(struct super_block *sb, in find_inode_nowait()
1674 int (*match)(struct inode *, unsigned long, in find_inode_nowait() argument
1679 struct inode *inode, *ret_inode = NULL; in find_inode_nowait() local
1683 hlist_for_each_entry(inode, head, i_hash) { in find_inode_nowait()
1684 if (inode->i_sb != sb) in find_inode_nowait()
1686 mval = match(inode, hashval, data); in find_inode_nowait()
1690 ret_inode = inode; in find_inode_nowait()
1720 struct inode *find_inode_rcu(struct super_block *sb, unsigned long hashval, in find_inode_rcu()
1721 int (*test)(struct inode *, void *), void *data) in find_inode_rcu() argument
1724 struct inode *inode; in find_inode_rcu() local
1729 hlist_for_each_entry_rcu(inode, head, i_hash) { in find_inode_rcu()
1730 if (inode->i_sb == sb && in find_inode_rcu()
1731 !(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE)) && in find_inode_rcu()
1732 test(inode, data)) in find_inode_rcu()
1733 return inode; in find_inode_rcu()
1758 struct inode *find_inode_by_ino_rcu(struct super_block *sb, in find_inode_by_ino_rcu()
1762 struct inode *inode; in find_inode_by_ino_rcu() local
1767 hlist_for_each_entry_rcu(inode, head, i_hash) { in find_inode_by_ino_rcu()
1768 if (inode->i_ino == ino && in find_inode_by_ino_rcu()
1769 inode->i_sb == sb && in find_inode_by_ino_rcu()
1770 !(READ_ONCE(inode->i_state) & (I_FREEING | I_WILL_FREE))) in find_inode_by_ino_rcu()
1771 return inode; in find_inode_by_ino_rcu()
1777 int insert_inode_locked(struct inode *inode) in insert_inode_locked() argument
1779 struct super_block *sb = inode->i_sb; in insert_inode_locked()
1780 ino_t ino = inode->i_ino; in insert_inode_locked()
1784 struct inode *old = NULL; in insert_inode_locked()
1799 spin_lock(&inode->i_lock); in insert_inode_locked()
1800 inode->i_state |= I_NEW | I_CREATING; in insert_inode_locked()
1801 hlist_add_head_rcu(&inode->i_hash, head); in insert_inode_locked()
1802 spin_unlock(&inode->i_lock); in insert_inode_locked()
1824 int insert_inode_locked4(struct inode *inode, unsigned long hashval, in insert_inode_locked4() argument
1825 int (*test)(struct inode *, void *), void *data) in insert_inode_locked4() argument
1827 struct inode *old; in insert_inode_locked4()
1829 inode->i_state |= I_CREATING; in insert_inode_locked4()
1830 old = inode_insert5(inode, hashval, test, NULL, data); in insert_inode_locked4()
1832 if (old != inode) { in insert_inode_locked4()
1841 int generic_delete_inode(struct inode *inode) in generic_delete_inode() argument
1857 static void iput_final(struct inode *inode) in iput_final() argument
1859 struct super_block *sb = inode->i_sb; in iput_final()
1860 const struct super_operations *op = inode->i_sb->s_op; in iput_final()
1864 WARN_ON(inode->i_state & I_NEW); in iput_final()
1867 drop = op->drop_inode(inode); in iput_final()
1869 drop = generic_drop_inode(inode); in iput_final()
1872 !(inode->i_state & I_DONTCACHE) && in iput_final()
1874 __inode_add_lru(inode, true); in iput_final()
1875 spin_unlock(&inode->i_lock); in iput_final()
1879 state = inode->i_state; in iput_final()
1881 WRITE_ONCE(inode->i_state, state | I_WILL_FREE); in iput_final()
1882 spin_unlock(&inode->i_lock); in iput_final()
1884 write_inode_now(inode, 1); in iput_final()
1886 spin_lock(&inode->i_lock); in iput_final()
1887 state = inode->i_state; in iput_final()
1892 WRITE_ONCE(inode->i_state, state | I_FREEING); in iput_final()
1893 if (!list_empty(&inode->i_lru)) in iput_final()
1894 inode_lru_list_del(inode); in iput_final()
1895 spin_unlock(&inode->i_lock); in iput_final()
1897 evict(inode); in iput_final()
1909 void iput(struct inode *inode) in iput() argument
1911 if (!inode) in iput()
1913 BUG_ON(inode->i_state & I_CLEAR); in iput()
1915 if (atomic_dec_and_lock(&inode->i_count, &inode->i_lock)) { in iput()
1916 if (inode->i_nlink && (inode->i_state & I_DIRTY_TIME)) { in iput()
1917 atomic_inc(&inode->i_count); in iput()
1918 spin_unlock(&inode->i_lock); in iput()
1919 trace_writeback_lazytime_iput(inode); in iput()
1920 mark_inode_dirty_sync(inode); in iput()
1923 iput_final(inode); in iput()
1943 int bmap(struct inode *inode, sector_t *block) in bmap() argument
1945 if (!inode->i_mapping->a_ops->bmap) in bmap()
1948 *block = inode->i_mapping->a_ops->bmap(inode->i_mapping, *block); in bmap()
1959 static bool relatime_need_update(struct vfsmount *mnt, struct inode *inode, in relatime_need_update() argument
1969 atime = inode_get_atime(inode); in relatime_need_update()
1970 mtime = inode_get_mtime(inode); in relatime_need_update()
1976 ctime = inode_get_ctime(inode); in relatime_need_update()
2008 int inode_update_timestamps(struct inode *inode, int flags) in inode_update_timestamps() argument
2014 struct timespec64 ctime = inode_get_ctime(inode); in inode_update_timestamps()
2015 struct timespec64 mtime = inode_get_mtime(inode); in inode_update_timestamps()
2017 now = inode_set_ctime_current(inode); in inode_update_timestamps()
2021 inode_set_mtime_to_ts(inode, now); in inode_update_timestamps()
2024 if (IS_I_VERSION(inode) && inode_maybe_inc_iversion(inode, updated)) in inode_update_timestamps()
2027 now = current_time(inode); in inode_update_timestamps()
2031 struct timespec64 atime = inode_get_atime(inode); in inode_update_timestamps()
2034 inode_set_atime_to_ts(inode, now); in inode_update_timestamps()
2054 int generic_update_time(struct inode *inode, int flags) in generic_update_time() argument
2056 int updated = inode_update_timestamps(inode, flags); in generic_update_time()
2060 dirty_flags = inode->i_sb->s_flags & SB_LAZYTIME ? I_DIRTY_TIME : I_DIRTY_SYNC; in generic_update_time()
2063 __mark_inode_dirty(inode, dirty_flags); in generic_update_time()
2072 int inode_update_time(struct inode *inode, int flags) in inode_update_time() argument
2074 if (inode->i_op->update_time) in inode_update_time()
2075 return inode->i_op->update_time(inode, flags); in inode_update_time()
2076 generic_update_time(inode, flags); in inode_update_time()
2090 bool atime_needs_update(const struct path *path, struct inode *inode) in atime_needs_update() argument
2095 if (inode->i_flags & S_NOATIME) in atime_needs_update()
2101 if (HAS_UNMAPPED_ID(mnt_idmap(mnt), inode)) in atime_needs_update()
2104 if (IS_NOATIME(inode)) in atime_needs_update()
2106 if ((inode->i_sb->s_flags & SB_NODIRATIME) && S_ISDIR(inode->i_mode)) in atime_needs_update()
2111 if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode)) in atime_needs_update()
2114 now = current_time(inode); in atime_needs_update()
2116 if (!relatime_need_update(mnt, inode, now)) in atime_needs_update()
2119 atime = inode_get_atime(inode); in atime_needs_update()
2129 struct inode *inode = d_inode(path->dentry); in touch_atime() local
2131 if (!atime_needs_update(path, inode)) in touch_atime()
2134 if (!sb_start_write_trylock(inode->i_sb)) in touch_atime()
2148 inode_update_time(inode, S_ATIME); in touch_atime()
2151 sb_end_write(inode->i_sb); in touch_atime()
2163 struct inode *inode = d_inode(dentry); in dentry_needs_remove_privs() local
2167 if (IS_NOSEC(inode)) in dentry_needs_remove_privs()
2170 mask = setattr_should_drop_suidgid(idmap, inode); in dentry_needs_remove_privs()
2195 struct inode *inode = file_inode(file); in file_remove_privs_flags() local
2199 if (IS_NOSEC(inode) || !S_ISREG(inode->i_mode)) in file_remove_privs_flags()
2214 inode_has_no_xattr(inode); in file_remove_privs_flags()
2247 struct timespec64 current_time(struct inode *inode) in current_time() argument
2254 if (!is_mgtime(inode)) in current_time()
2258 cns = smp_load_acquire(&inode->i_ctime_nsec); in current_time()
2268 return timestamp_truncate(now, inode); in current_time()
2272 static int inode_needs_update_time(struct inode *inode) in inode_needs_update_time() argument
2278 if (IS_NOCMTIME(inode)) in inode_needs_update_time()
2281 now = current_time(inode); in inode_needs_update_time()
2283 ts = inode_get_mtime(inode); in inode_needs_update_time()
2287 ts = inode_get_ctime(inode); in inode_needs_update_time()
2291 if (IS_I_VERSION(inode) && inode_iversion_need_inc(inode)) in inode_needs_update_time()
2300 struct inode *inode = file_inode(file); in __file_update_time() local
2304 ret = inode_update_time(inode, sync_mode); in __file_update_time()
2328 struct inode *inode = file_inode(file); in file_update_time() local
2330 ret = inode_needs_update_time(inode); in file_update_time()
2356 struct inode *inode = file_inode(file); in file_modified_flags() local
2369 ret = inode_needs_update_time(inode); in file_modified_flags()
2412 int inode_needs_sync(struct inode *inode) in inode_needs_sync() argument
2414 if (IS_SYNC(inode)) in inode_needs_sync()
2416 if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode)) in inode_needs_sync()
2433 static void __wait_on_freeing_inode(struct inode *inode, bool is_inode_hash_locked) in __wait_on_freeing_inode() argument
2441 if (unlikely(inode_unhashed(inode))) { in __wait_on_freeing_inode()
2443 spin_unlock(&inode->i_lock); in __wait_on_freeing_inode()
2447 wq_head = inode_bit_waitqueue(&wqe, inode, __I_NEW); in __wait_on_freeing_inode()
2449 spin_unlock(&inode->i_lock); in __wait_on_freeing_inode()
2497 sizeof(struct inode), in inode_init()
2519 void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev) in init_special_inode() argument
2521 inode->i_mode = mode; in init_special_inode()
2523 inode->i_fop = &def_chr_fops; in init_special_inode()
2524 inode->i_rdev = rdev; in init_special_inode()
2527 inode->i_fop = &def_blk_fops; in init_special_inode()
2528 inode->i_rdev = rdev; in init_special_inode()
2530 inode->i_fop = &pipefifo_fops; in init_special_inode()
2535 " inode %s:%lu\n", mode, inode->i_sb->s_id, in init_special_inode()
2536 inode->i_ino); in init_special_inode()
2553 void inode_init_owner(struct mnt_idmap *idmap, struct inode *inode, in inode_init_owner() argument
2554 const struct inode *dir, umode_t mode) in inode_init_owner()
2556 inode_fsuid_set(inode, idmap); in inode_init_owner()
2558 inode->i_gid = dir->i_gid; in inode_init_owner()
2564 inode_fsgid_set(inode, idmap); in inode_init_owner()
2565 inode->i_mode = mode; in inode_init_owner()
2584 const struct inode *inode) in inode_owner_or_capable() argument
2589 vfsuid = i_uid_into_vfsuid(idmap, inode); in inode_owner_or_capable()
2603 bool inode_dio_finished(const struct inode *inode) in inode_dio_finished() argument
2605 return atomic_read(&inode->i_dio_count) == 0; in inode_dio_finished()
2619 void inode_dio_wait(struct inode *inode) in inode_dio_wait() argument
2621 wait_var_event(&inode->i_dio_count, inode_dio_finished(inode)); in inode_dio_wait()
2625 void inode_dio_wait_interruptible(struct inode *inode) in inode_dio_wait_interruptible() argument
2627 wait_var_event_interruptible(&inode->i_dio_count, in inode_dio_wait_interruptible()
2628 inode_dio_finished(inode)); in inode_dio_wait_interruptible()
2648 void inode_set_flags(struct inode *inode, unsigned int flags, in inode_set_flags() argument
2652 set_mask_bits(&inode->i_flags, mask, flags); in inode_set_flags()
2656 void inode_nohighmem(struct inode *inode) in inode_nohighmem() argument
2658 mapping_set_gfp_mask(inode->i_mapping, GFP_USER); in inode_nohighmem()
2662 struct timespec64 inode_set_ctime_to_ts(struct inode *inode, struct timespec64 ts) in inode_set_ctime_to_ts() argument
2664 trace_inode_set_ctime_to_ts(inode, &ts); in inode_set_ctime_to_ts()
2666 inode->i_ctime_sec = ts.tv_sec; in inode_set_ctime_to_ts()
2667 inode->i_ctime_nsec = ts.tv_nsec; in inode_set_ctime_to_ts()
2681 struct timespec64 timestamp_truncate(struct timespec64 t, struct inode *inode) in timestamp_truncate() argument
2683 struct super_block *sb = inode->i_sb; in timestamp_truncate()
2720 struct timespec64 inode_set_ctime_current(struct inode *inode) in inode_set_ctime_current() argument
2726 now = timestamp_truncate(now, inode); in inode_set_ctime_current()
2729 if (!is_mgtime(inode)) { in inode_set_ctime_current()
2730 inode_set_ctime_to_ts(inode, now); in inode_set_ctime_current()
2739 cns = smp_load_acquire(&inode->i_ctime_nsec); in inode_set_ctime_current()
2741 struct timespec64 ctime = { .tv_sec = inode->i_ctime_sec, in inode_set_ctime_current()
2746 now = timestamp_truncate(now, inode); in inode_set_ctime_current()
2753 if (cns == now.tv_nsec && inode->i_ctime_sec == now.tv_sec) { in inode_set_ctime_current()
2754 trace_ctime_xchg_skip(inode, &now); in inode_set_ctime_current()
2760 if (try_cmpxchg(&inode->i_ctime_nsec, &cur, now.tv_nsec)) { in inode_set_ctime_current()
2762 inode->i_ctime_sec = now.tv_sec; in inode_set_ctime_current()
2763 trace_ctime_ns_xchg(inode, cns, now.tv_nsec, cur); in inode_set_ctime_current()
2777 now.tv_sec = inode->i_ctime_sec; in inode_set_ctime_current()
2802 struct timespec64 inode_set_ctime_deleg(struct inode *inode, struct timespec64 update) in inode_set_ctime_deleg() argument
2808 cur = smp_load_acquire(&inode->i_ctime_nsec); in inode_set_ctime_deleg()
2810 cur_ts.tv_sec = inode->i_ctime_sec; in inode_set_ctime_deleg()
2822 update = timestamp_truncate(update, inode); in inode_set_ctime_deleg()
2835 if (try_cmpxchg(&inode->i_ctime_nsec, &cur, update.tv_nsec)) { in inode_set_ctime_deleg()
2836 inode->i_ctime_sec = update.tv_sec; in inode_set_ctime_deleg()
2852 cur_ts.tv_sec = inode->i_ctime_sec; in inode_set_ctime_deleg()
2871 const struct inode *inode, vfsgid_t vfsgid) in in_group_or_capable() argument
2875 if (capable_wrt_inode_uidgid(idmap, inode, CAP_FSETID)) in in_group_or_capable()
2896 const struct inode *dir, umode_t mode) in mode_strip_sgid()
2915 void dump_inode(struct inode *inode, const char *reason) in dump_inode() argument
2917 pr_warn("%s encountered for inode %px", reason, inode); in dump_inode()