Lines Matching refs:conf
47 static void allow_barrier(struct r1conf *conf, sector_t sector_nr);
48 static void lower_barrier(struct r1conf *conf, sector_t sector_nr);
130 static void *r1bio_pool_alloc(gfp_t gfp_flags, struct r1conf *conf) in r1bio_pool_alloc() argument
132 int size = offsetof(struct r1bio, bios[conf->raid_disks * 2]); in r1bio_pool_alloc()
147 struct r1conf *conf = data; in r1buf_pool_alloc() local
154 r1_bio = r1bio_pool_alloc(gfp_flags, conf); in r1buf_pool_alloc()
158 rps = kmalloc_array(conf->raid_disks * 2, sizeof(struct resync_pages), in r1buf_pool_alloc()
166 for (j = conf->raid_disks * 2; j-- ; ) { in r1buf_pool_alloc()
179 if (test_bit(MD_RECOVERY_REQUESTED, &conf->mddev->recovery)) in r1buf_pool_alloc()
180 need_pages = conf->raid_disks * 2; in r1buf_pool_alloc()
183 for (j = 0; j < conf->raid_disks * 2; j++) { in r1buf_pool_alloc()
209 while (++j < conf->raid_disks * 2) { in r1buf_pool_alloc()
222 struct r1conf *conf = data; in r1buf_pool_free() local
227 for (i = conf->raid_disks * 2; i--; ) { in r1buf_pool_free()
240 static void put_all_bios(struct r1conf *conf, struct r1bio *r1_bio) in put_all_bios() argument
244 for (i = 0; i < conf->raid_disks * 2; i++) { in put_all_bios()
254 struct r1conf *conf = r1_bio->mddev->private; in free_r1bio() local
256 put_all_bios(conf, r1_bio); in free_r1bio()
257 mempool_free(r1_bio, conf->r1bio_pool); in free_r1bio()
262 struct r1conf *conf = r1_bio->mddev->private; in put_buf() local
266 for (i = 0; i < conf->raid_disks * 2; i++) { in put_buf()
269 rdev_dec_pending(conf->mirrors[i].rdev, r1_bio->mddev); in put_buf()
272 mempool_free(r1_bio, &conf->r1buf_pool); in put_buf()
274 lower_barrier(conf, sect); in put_buf()
281 struct r1conf *conf = mddev->private; in reschedule_retry() local
285 spin_lock_irqsave(&conf->device_lock, flags); in reschedule_retry()
286 list_add(&r1_bio->retry_list, &conf->retry_list); in reschedule_retry()
287 atomic_inc(&conf->nr_queued[idx]); in reschedule_retry()
288 spin_unlock_irqrestore(&conf->device_lock, flags); in reschedule_retry()
290 wake_up(&conf->wait_barrier); in reschedule_retry()
312 struct r1conf *conf = r1_bio->mddev->private; in raid_end_bio_io() local
330 allow_barrier(conf, sector); in raid_end_bio_io()
338 struct r1conf *conf = r1_bio->mddev->private; in update_head_pos() local
340 conf->mirrors[disk].head_position = in update_head_pos()
350 struct r1conf *conf = r1_bio->mddev->private; in find_bio_disk() local
351 int raid_disks = conf->raid_disks; in find_bio_disk()
367 struct r1conf *conf = r1_bio->mddev->private; in raid1_end_read_request() local
368 struct md_rdev *rdev = conf->mirrors[r1_bio->read_disk].rdev; in raid1_end_read_request()
390 spin_lock_irqsave(&conf->device_lock, flags); in raid1_end_read_request()
391 if (r1_bio->mddev->degraded == conf->raid_disks || in raid1_end_read_request()
392 (r1_bio->mddev->degraded == conf->raid_disks-1 && in raid1_end_read_request()
395 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1_end_read_request()
400 rdev_dec_pending(rdev, conf->mddev); in raid1_end_read_request()
406 mdname(conf->mddev), in raid1_end_read_request()
451 struct r1conf *conf = r1_bio->mddev->private; in raid1_end_write_request() local
454 struct md_rdev *rdev = conf->mirrors[mirror].rdev; in raid1_end_write_request()
467 conf->mddev->recovery); in raid1_end_write_request()
548 rdev_dec_pending(rdev, conf->mddev); in raid1_end_write_request()
579 static void update_read_sectors(struct r1conf *conf, int disk, in update_read_sectors() argument
582 struct raid1_info *info = &conf->mirrors[disk]; in update_read_sectors()
590 static int choose_first_rdev(struct r1conf *conf, struct r1bio *r1_bio, in choose_first_rdev() argument
597 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) { in choose_first_rdev()
604 rdev = conf->mirrors[disk].rdev; in choose_first_rdev()
611 update_read_sectors(conf, disk, this_sector, read_len); in choose_first_rdev()
626 static int choose_bb_rdev(struct r1conf *conf, struct r1bio *r1_bio, in choose_bb_rdev() argument
634 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) { in choose_bb_rdev()
642 rdev = conf->mirrors[disk].rdev; in choose_bb_rdev()
659 update_read_sectors(conf, best_disk, this_sector, best_len); in choose_bb_rdev()
665 static int choose_slow_rdev(struct r1conf *conf, struct r1bio *r1_bio, in choose_slow_rdev() argument
673 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) { in choose_slow_rdev()
681 rdev = conf->mirrors[disk].rdev; in choose_slow_rdev()
692 update_read_sectors(conf, disk, this_sector, read_len); in choose_slow_rdev()
708 update_read_sectors(conf, bb_disk, this_sector, bb_read_len); in choose_slow_rdev()
714 static bool is_sequential(struct r1conf *conf, int disk, struct r1bio *r1_bio) in is_sequential() argument
717 return conf->mirrors[disk].next_seq_sect == r1_bio->sector || in is_sequential()
718 conf->mirrors[disk].head_position == r1_bio->sector; in is_sequential()
725 static bool should_choose_next(struct r1conf *conf, int disk) in should_choose_next() argument
727 struct raid1_info *mirror = &conf->mirrors[disk]; in should_choose_next()
767 static int choose_best_rdev(struct r1conf *conf, struct r1bio *r1_bio) in choose_best_rdev() argument
778 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) { in choose_best_rdev()
786 rdev = conf->mirrors[disk].rdev; in choose_best_rdev()
795 dist = abs(r1_bio->sector - conf->mirrors[disk].head_position); in choose_best_rdev()
798 if (is_sequential(conf, disk, r1_bio)) { in choose_best_rdev()
799 if (!should_choose_next(conf, disk)) in choose_best_rdev()
839 (READ_ONCE(conf->nonrot_disks) || ctl.min_pending == 0)) in choose_best_rdev()
863 static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, in read_balance() argument
870 if (raid1_should_read_first(conf->mddev, r1_bio->sector, in read_balance()
872 return choose_first_rdev(conf, r1_bio, max_sectors); in read_balance()
874 disk = choose_best_rdev(conf, r1_bio); in read_balance()
877 update_read_sectors(conf, disk, r1_bio->sector, in read_balance()
887 disk = choose_bb_rdev(conf, r1_bio, max_sectors); in read_balance()
891 return choose_slow_rdev(conf, r1_bio, max_sectors); in read_balance()
894 static void wake_up_barrier(struct r1conf *conf) in wake_up_barrier() argument
896 if (wq_has_sleeper(&conf->wait_barrier)) in wake_up_barrier()
897 wake_up(&conf->wait_barrier); in wake_up_barrier()
900 static void flush_bio_list(struct r1conf *conf, struct bio *bio) in flush_bio_list() argument
903 raid1_prepare_flush_writes(conf->mddev); in flush_bio_list()
904 wake_up_barrier(conf); in flush_bio_list()
915 static void flush_pending_writes(struct r1conf *conf) in flush_pending_writes() argument
920 spin_lock_irq(&conf->device_lock); in flush_pending_writes()
922 if (conf->pending_bio_list.head) { in flush_pending_writes()
926 bio = bio_list_get(&conf->pending_bio_list); in flush_pending_writes()
927 spin_unlock_irq(&conf->device_lock); in flush_pending_writes()
940 flush_bio_list(conf, bio); in flush_pending_writes()
943 spin_unlock_irq(&conf->device_lock); in flush_pending_writes()
970 static int raise_barrier(struct r1conf *conf, sector_t sector_nr) in raise_barrier() argument
974 spin_lock_irq(&conf->resync_lock); in raise_barrier()
977 wait_event_lock_irq(conf->wait_barrier, in raise_barrier()
978 !atomic_read(&conf->nr_waiting[idx]), in raise_barrier()
979 conf->resync_lock); in raise_barrier()
982 atomic_inc(&conf->barrier[idx]); in raise_barrier()
1000 wait_event_lock_irq(conf->wait_barrier, in raise_barrier()
1001 (!conf->array_frozen && in raise_barrier()
1002 !atomic_read(&conf->nr_pending[idx]) && in raise_barrier()
1003 atomic_read(&conf->barrier[idx]) < RESYNC_DEPTH) || in raise_barrier()
1004 test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery), in raise_barrier()
1005 conf->resync_lock); in raise_barrier()
1007 if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { in raise_barrier()
1008 atomic_dec(&conf->barrier[idx]); in raise_barrier()
1009 spin_unlock_irq(&conf->resync_lock); in raise_barrier()
1010 wake_up(&conf->wait_barrier); in raise_barrier()
1014 atomic_inc(&conf->nr_sync_pending); in raise_barrier()
1015 spin_unlock_irq(&conf->resync_lock); in raise_barrier()
1020 static void lower_barrier(struct r1conf *conf, sector_t sector_nr) in lower_barrier() argument
1024 BUG_ON(atomic_read(&conf->barrier[idx]) <= 0); in lower_barrier()
1026 atomic_dec(&conf->barrier[idx]); in lower_barrier()
1027 atomic_dec(&conf->nr_sync_pending); in lower_barrier()
1028 wake_up(&conf->wait_barrier); in lower_barrier()
1031 static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) in _wait_barrier() argument
1043 atomic_inc(&conf->nr_pending[idx]); in _wait_barrier()
1063 if (!READ_ONCE(conf->array_frozen) && in _wait_barrier()
1064 !atomic_read(&conf->barrier[idx])) in _wait_barrier()
1074 spin_lock_irq(&conf->resync_lock); in _wait_barrier()
1075 atomic_inc(&conf->nr_waiting[idx]); in _wait_barrier()
1076 atomic_dec(&conf->nr_pending[idx]); in _wait_barrier()
1081 wake_up_barrier(conf); in _wait_barrier()
1088 wait_event_lock_irq(conf->wait_barrier, in _wait_barrier()
1089 !conf->array_frozen && in _wait_barrier()
1090 !atomic_read(&conf->barrier[idx]), in _wait_barrier()
1091 conf->resync_lock); in _wait_barrier()
1092 atomic_inc(&conf->nr_pending[idx]); in _wait_barrier()
1095 atomic_dec(&conf->nr_waiting[idx]); in _wait_barrier()
1096 spin_unlock_irq(&conf->resync_lock); in _wait_barrier()
1100 static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait) in wait_read_barrier() argument
1112 atomic_inc(&conf->nr_pending[idx]); in wait_read_barrier()
1114 if (!READ_ONCE(conf->array_frozen)) in wait_read_barrier()
1117 spin_lock_irq(&conf->resync_lock); in wait_read_barrier()
1118 atomic_inc(&conf->nr_waiting[idx]); in wait_read_barrier()
1119 atomic_dec(&conf->nr_pending[idx]); in wait_read_barrier()
1124 wake_up_barrier(conf); in wait_read_barrier()
1132 wait_event_lock_irq(conf->wait_barrier, in wait_read_barrier()
1133 !conf->array_frozen, in wait_read_barrier()
1134 conf->resync_lock); in wait_read_barrier()
1135 atomic_inc(&conf->nr_pending[idx]); in wait_read_barrier()
1138 atomic_dec(&conf->nr_waiting[idx]); in wait_read_barrier()
1139 spin_unlock_irq(&conf->resync_lock); in wait_read_barrier()
1143 static bool wait_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait) in wait_barrier() argument
1147 return _wait_barrier(conf, idx, nowait); in wait_barrier()
1150 static void _allow_barrier(struct r1conf *conf, int idx) in _allow_barrier() argument
1152 atomic_dec(&conf->nr_pending[idx]); in _allow_barrier()
1153 wake_up_barrier(conf); in _allow_barrier()
1156 static void allow_barrier(struct r1conf *conf, sector_t sector_nr) in allow_barrier() argument
1160 _allow_barrier(conf, idx); in allow_barrier()
1164 static int get_unqueued_pending(struct r1conf *conf) in get_unqueued_pending() argument
1168 ret = atomic_read(&conf->nr_sync_pending); in get_unqueued_pending()
1170 ret += atomic_read(&conf->nr_pending[idx]) - in get_unqueued_pending()
1171 atomic_read(&conf->nr_queued[idx]); in get_unqueued_pending()
1176 static void freeze_array(struct r1conf *conf, int extra) in freeze_array() argument
1201 spin_lock_irq(&conf->resync_lock); in freeze_array()
1202 conf->array_frozen = 1; in freeze_array()
1203 mddev_add_trace_msg(conf->mddev, "raid1 wait freeze"); in freeze_array()
1205 conf->wait_barrier, in freeze_array()
1206 get_unqueued_pending(conf) == extra, in freeze_array()
1207 conf->resync_lock, in freeze_array()
1208 flush_pending_writes(conf)); in freeze_array()
1209 spin_unlock_irq(&conf->resync_lock); in freeze_array()
1211 static void unfreeze_array(struct r1conf *conf) in unfreeze_array() argument
1214 spin_lock_irq(&conf->resync_lock); in unfreeze_array()
1215 conf->array_frozen = 0; in unfreeze_array()
1216 spin_unlock_irq(&conf->resync_lock); in unfreeze_array()
1217 wake_up(&conf->wait_barrier); in unfreeze_array()
1273 struct r1conf *conf = mddev->private; in raid1_unplug() local
1277 spin_lock_irq(&conf->device_lock); in raid1_unplug()
1278 bio_list_merge(&conf->pending_bio_list, &plug->pending); in raid1_unplug()
1279 spin_unlock_irq(&conf->device_lock); in raid1_unplug()
1280 wake_up_barrier(conf); in raid1_unplug()
1288 flush_bio_list(conf, bio); in raid1_unplug()
1304 struct r1conf *conf = mddev->private; in alloc_r1bio() local
1307 r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO); in alloc_r1bio()
1308 memset(r1_bio, 0, offsetof(struct r1bio, bios[conf->raid_disks * 2])); in alloc_r1bio()
1316 struct r1conf *conf = mddev->private; in raid1_read_request() local
1334 if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, in raid1_read_request()
1350 rdisk = read_balance(conf, r1_bio, &max_sectors); in raid1_read_request()
1356 conf->mirrors[r1_bio->read_disk].rdev->bdev, in raid1_read_request()
1361 mirror = conf->mirrors + rdisk; in raid1_read_request()
1380 gfp, &conf->bio_split); in raid1_read_request()
1423 struct r1conf *conf = mddev->private; in wait_blocked_rdev() local
1424 int disks = conf->raid_disks * 2; in wait_blocked_rdev()
1429 struct md_rdev *rdev = conf->mirrors[i].rdev; in wait_blocked_rdev()
1458 struct r1conf *conf = mddev->private; in raid1_write_request() local
1477 prepare_to_wait(&conf->wait_barrier, in raid1_write_request()
1485 finish_wait(&conf->wait_barrier, &w); in raid1_write_request()
1493 if (!wait_barrier(conf, bio->bi_iter.bi_sector, in raid1_write_request()
1518 disks = conf->raid_disks * 2; in raid1_write_request()
1521 struct md_rdev *rdev = conf->mirrors[i].rdev; in raid1_write_request()
1588 GFP_NOIO, &conf->bio_split); in raid1_write_request()
1610 struct md_rdev *rdev = conf->mirrors[i].rdev; in raid1_write_request()
1657 conf->raid_disks - mddev->degraded > 1) in raid1_write_request()
1666 spin_lock_irqsave(&conf->device_lock, flags); in raid1_write_request()
1667 bio_list_add(&conf->pending_bio_list, mbio); in raid1_write_request()
1668 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1_write_request()
1676 wake_up_barrier(conf); in raid1_write_request()
1681 rdev_dec_pending(conf->mirrors[k].rdev, mddev); in raid1_write_request()
1720 struct r1conf *conf = mddev->private; in raid1_status() local
1725 seq_printf(seq, " [%d/%d] [", conf->raid_disks, in raid1_status()
1726 conf->raid_disks - mddev->degraded); in raid1_status()
1727 for (i = 0; i < conf->raid_disks; i++) { in raid1_status()
1728 struct md_rdev *rdev = READ_ONCE(conf->mirrors[i].rdev); in raid1_status()
1754 struct r1conf *conf = mddev->private; in raid1_error() local
1757 spin_lock_irqsave(&conf->device_lock, flags); in raid1_error()
1760 (conf->raid_disks - mddev->degraded) == 1) { in raid1_error()
1764 conf->recovery_disabled = mddev->recovery_disabled; in raid1_error()
1765 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1_error()
1773 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1_error()
1783 mdname(mddev), conf->raid_disks - mddev->degraded); in raid1_error()
1786 static void print_conf(struct r1conf *conf) in print_conf() argument
1791 if (!conf) { in print_conf()
1795 pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded, in print_conf()
1796 conf->raid_disks); in print_conf()
1798 lockdep_assert_held(&conf->mddev->reconfig_mutex); in print_conf()
1799 for (i = 0; i < conf->raid_disks; i++) { in print_conf()
1800 struct md_rdev *rdev = conf->mirrors[i].rdev; in print_conf()
1809 static void close_sync(struct r1conf *conf) in close_sync() argument
1814 _wait_barrier(conf, idx, false); in close_sync()
1815 _allow_barrier(conf, idx); in close_sync()
1818 mempool_exit(&conf->r1buf_pool); in close_sync()
1824 struct r1conf *conf = mddev->private; in raid1_spare_active() local
1835 spin_lock_irqsave(&conf->device_lock, flags); in raid1_spare_active()
1836 for (i = 0; i < conf->raid_disks; i++) { in raid1_spare_active()
1837 struct md_rdev *rdev = conf->mirrors[i].rdev; in raid1_spare_active()
1838 struct md_rdev *repl = conf->mirrors[conf->raid_disks + i].rdev; in raid1_spare_active()
1867 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1_spare_active()
1869 print_conf(conf); in raid1_spare_active()
1873 static bool raid1_add_conf(struct r1conf *conf, struct md_rdev *rdev, int disk, in raid1_add_conf() argument
1876 struct raid1_info *info = conf->mirrors + disk; in raid1_add_conf()
1879 info += conf->raid_disks; in raid1_add_conf()
1886 WRITE_ONCE(conf->nonrot_disks, conf->nonrot_disks + 1); in raid1_add_conf()
1897 static bool raid1_remove_conf(struct r1conf *conf, int disk) in raid1_remove_conf() argument
1899 struct raid1_info *info = conf->mirrors + disk; in raid1_remove_conf()
1908 rdev->mddev->recovery_disabled != conf->recovery_disabled && in raid1_remove_conf()
1909 rdev->mddev->degraded < conf->raid_disks) in raid1_remove_conf()
1913 WRITE_ONCE(conf->nonrot_disks, conf->nonrot_disks - 1); in raid1_remove_conf()
1921 struct r1conf *conf = mddev->private; in raid1_add_disk() local
1926 int last = conf->raid_disks - 1; in raid1_add_disk()
1928 if (mddev->recovery_disabled == conf->recovery_disabled) in raid1_add_disk()
1940 rdev->saved_raid_disk < conf->raid_disks && in raid1_add_disk()
1941 conf->mirrors[rdev->saved_raid_disk].rdev == NULL) in raid1_add_disk()
1945 p = conf->mirrors + mirror; in raid1_add_disk()
1951 raid1_add_conf(conf, rdev, mirror, false); in raid1_add_disk()
1956 conf->fullsync = 1; in raid1_add_disk()
1960 p[conf->raid_disks].rdev == NULL && repl_slot < 0) in raid1_add_disk()
1968 raid1_add_conf(conf, rdev, repl_slot, true); in raid1_add_disk()
1970 conf->fullsync = 1; in raid1_add_disk()
1973 print_conf(conf); in raid1_add_disk()
1979 struct r1conf *conf = mddev->private; in raid1_remove_disk() local
1982 struct raid1_info *p = conf->mirrors + number; in raid1_remove_disk()
1984 if (unlikely(number >= conf->raid_disks)) in raid1_remove_disk()
1988 number += conf->raid_disks; in raid1_remove_disk()
1989 p = conf->mirrors + number; in raid1_remove_disk()
1992 print_conf(conf); in raid1_remove_disk()
1994 if (!raid1_remove_conf(conf, number)) { in raid1_remove_disk()
1999 if (number < conf->raid_disks && in raid1_remove_disk()
2000 conf->mirrors[conf->raid_disks + number].rdev) { in raid1_remove_disk()
2006 conf->mirrors[conf->raid_disks + number].rdev; in raid1_remove_disk()
2007 freeze_array(conf, 0); in raid1_remove_disk()
2016 unfreeze_array(conf); in raid1_remove_disk()
2021 conf->mirrors[conf->raid_disks + number].rdev = NULL; in raid1_remove_disk()
2022 unfreeze_array(conf); in raid1_remove_disk()
2030 print_conf(conf); in raid1_remove_disk()
2087 struct r1conf *conf = mddev->private; in end_sync_write() local
2088 struct md_rdev *rdev = conf->mirrors[find_bio_disk(r1_bio, bio)].rdev; in end_sync_write()
2098 !rdev_has_badblock(conf->mirrors[r1_bio->read_disk].rdev, in end_sync_write()
2139 struct r1conf *conf = mddev->private; in fix_sync_read_error() local
2147 rdev = conf->mirrors[r1_bio->read_disk].rdev; in fix_sync_read_error()
2173 rdev = conf->mirrors[d].rdev; in fix_sync_read_error()
2182 if (d == conf->raid_disks * 2) in fix_sync_read_error()
2196 for (d = 0; d < conf->raid_disks * 2; d++) { in fix_sync_read_error()
2197 rdev = conf->mirrors[d].rdev; in fix_sync_read_error()
2217 d = conf->raid_disks * 2; in fix_sync_read_error()
2221 rdev = conf->mirrors[d].rdev; in fix_sync_read_error()
2232 d = conf->raid_disks * 2; in fix_sync_read_error()
2236 rdev = conf->mirrors[d].rdev; in fix_sync_read_error()
2261 struct r1conf *conf = mddev->private; in process_checks() local
2268 for (i = 0; i < conf->raid_disks * 2; i++) { in process_checks()
2276 bio_reset(b, conf->mirrors[i].rdev->bdev, REQ_OP_READ); in process_checks()
2279 conf->mirrors[i].rdev->data_offset; in process_checks()
2287 for (primary = 0; primary < conf->raid_disks * 2; primary++) in process_checks()
2291 rdev_dec_pending(conf->mirrors[primary].rdev, mddev); in process_checks()
2295 for (i = 0; i < conf->raid_disks * 2; i++) { in process_checks()
2329 rdev_dec_pending(conf->mirrors[i].rdev, mddev); in process_checks()
2339 struct r1conf *conf = mddev->private; in sync_request_write() local
2341 int disks = conf->raid_disks * 2; in sync_request_write()
2352 conf->recovery_disabled = mddev->recovery_disabled; in sync_request_write()
2374 if (test_bit(Faulty, &conf->mirrors[i].rdev->flags)) { in sync_request_write()
2380 if (test_bit(FailFast, &conf->mirrors[i].rdev->flags)) in sync_request_write()
2400 static void fix_read_error(struct r1conf *conf, struct r1bio *r1_bio) in fix_read_error() argument
2405 struct mddev *mddev = conf->mddev; in fix_read_error()
2406 struct md_rdev *rdev = conf->mirrors[read_disk].rdev; in fix_read_error()
2423 rdev = conf->mirrors[d].rdev; in fix_read_error()
2431 conf->tmppage, REQ_OP_READ, false)) in fix_read_error()
2439 if (d == conf->raid_disks * 2) in fix_read_error()
2445 struct md_rdev *rdev = conf->mirrors[read_disk].rdev; in fix_read_error()
2454 d = conf->raid_disks * 2; in fix_read_error()
2456 rdev = conf->mirrors[d].rdev; in fix_read_error()
2461 conf->tmppage, REQ_OP_WRITE); in fix_read_error()
2468 d = conf->raid_disks * 2; in fix_read_error()
2470 rdev = conf->mirrors[d].rdev; in fix_read_error()
2475 conf->tmppage, REQ_OP_READ)) { in fix_read_error()
2494 struct r1conf *conf = mddev->private; in narrow_write_error() local
2495 struct md_rdev *rdev = conf->mirrors[i].rdev; in narrow_write_error()
2560 static void handle_sync_write_finished(struct r1conf *conf, struct r1bio *r1_bio) in handle_sync_write_finished() argument
2564 for (m = 0; m < conf->raid_disks * 2 ; m++) { in handle_sync_write_finished()
2565 struct md_rdev *rdev = conf->mirrors[m].rdev; in handle_sync_write_finished()
2576 md_error(conf->mddev, rdev); in handle_sync_write_finished()
2580 md_done_sync(conf->mddev, s, 1); in handle_sync_write_finished()
2583 static void handle_write_finished(struct r1conf *conf, struct r1bio *r1_bio) in handle_write_finished() argument
2588 for (m = 0; m < conf->raid_disks * 2 ; m++) in handle_write_finished()
2590 struct md_rdev *rdev = conf->mirrors[m].rdev; in handle_write_finished()
2594 rdev_dec_pending(rdev, conf->mddev); in handle_write_finished()
2602 md_error(conf->mddev, in handle_write_finished()
2603 conf->mirrors[m].rdev); in handle_write_finished()
2605 rdev_dec_pending(conf->mirrors[m].rdev, in handle_write_finished()
2606 conf->mddev); in handle_write_finished()
2609 spin_lock_irq(&conf->device_lock); in handle_write_finished()
2610 list_add(&r1_bio->retry_list, &conf->bio_end_io_list); in handle_write_finished()
2612 atomic_inc(&conf->nr_queued[idx]); in handle_write_finished()
2613 spin_unlock_irq(&conf->device_lock); in handle_write_finished()
2618 wake_up(&conf->wait_barrier); in handle_write_finished()
2619 md_wakeup_thread(conf->mddev->thread); in handle_write_finished()
2627 static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio) in handle_read_error() argument
2629 struct mddev *mddev = conf->mddev; in handle_read_error()
2648 rdev = conf->mirrors[r1_bio->read_disk].rdev; in handle_read_error()
2651 freeze_array(conf, 1); in handle_read_error()
2652 fix_read_error(conf, r1_bio); in handle_read_error()
2653 unfreeze_array(conf); in handle_read_error()
2660 rdev_dec_pending(rdev, conf->mddev); in handle_read_error()
2667 allow_barrier(conf, sector); in handle_read_error()
2675 struct r1conf *conf = mddev->private; in raid1d() local
2676 struct list_head *head = &conf->retry_list; in raid1d()
2682 if (!list_empty_careful(&conf->bio_end_io_list) && in raid1d()
2685 spin_lock_irqsave(&conf->device_lock, flags); in raid1d()
2687 list_splice_init(&conf->bio_end_io_list, &tmp); in raid1d()
2688 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1d()
2694 atomic_dec(&conf->nr_queued[idx]); in raid1d()
2704 flush_pending_writes(conf); in raid1d()
2706 spin_lock_irqsave(&conf->device_lock, flags); in raid1d()
2708 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1d()
2714 atomic_dec(&conf->nr_queued[idx]); in raid1d()
2715 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1d()
2718 conf = mddev->private; in raid1d()
2722 handle_sync_write_finished(conf, r1_bio); in raid1d()
2727 handle_write_finished(conf, r1_bio); in raid1d()
2729 handle_read_error(conf, r1_bio); in raid1d()
2740 static int init_resync(struct r1conf *conf) in init_resync() argument
2745 BUG_ON(mempool_initialized(&conf->r1buf_pool)); in init_resync()
2747 return mempool_init(&conf->r1buf_pool, buffs, r1buf_pool_alloc, in init_resync()
2748 r1buf_pool_free, conf); in init_resync()
2751 static struct r1bio *raid1_alloc_init_r1buf(struct r1conf *conf) in raid1_alloc_init_r1buf() argument
2753 struct r1bio *r1bio = mempool_alloc(&conf->r1buf_pool, GFP_NOIO); in raid1_alloc_init_r1buf()
2758 for (i = conf->raid_disks * 2; i--; ) { in raid1_alloc_init_r1buf()
2781 struct r1conf *conf = mddev->private; in raid1_sync_request() local
2796 if (!mempool_initialized(&conf->r1buf_pool)) in raid1_sync_request()
2797 if (init_resync(conf)) in raid1_sync_request()
2810 conf->fullsync = 0; in raid1_sync_request()
2813 close_sync(conf); in raid1_sync_request()
2816 conf->cluster_sync_low = 0; in raid1_sync_request()
2817 conf->cluster_sync_high = 0; in raid1_sync_request()
2825 conf->fullsync == 0) { in raid1_sync_request()
2833 !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) { in raid1_sync_request()
2843 if (atomic_read(&conf->nr_waiting[idx])) in raid1_sync_request()
2852 (sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high)); in raid1_sync_request()
2854 if (raise_barrier(conf, sector_nr)) in raid1_sync_request()
2857 r1_bio = raid1_alloc_init_r1buf(conf); in raid1_sync_request()
2875 for (i = 0; i < conf->raid_disks * 2; i++) { in raid1_sync_request()
2879 rdev = conf->mirrors[i].rdev; in raid1_sync_request()
2882 if (i < conf->raid_disks) in raid1_sync_request()
2946 for (i = 0 ; i < conf->raid_disks * 2 ; i++) in raid1_sync_request()
2948 struct md_rdev *rdev = conf->mirrors[i].rdev; in raid1_sync_request()
2963 conf->recovery_disabled = mddev->recovery_disabled; in raid1_sync_request()
3009 !conf->fullsync && in raid1_sync_request()
3016 for (i = 0 ; i < conf->raid_disks * 2; i++) { in raid1_sync_request()
3039 conf->cluster_sync_high < sector_nr + nr_sectors) { in raid1_sync_request()
3040 conf->cluster_sync_low = mddev->curr_resync_completed; in raid1_sync_request()
3041 conf->cluster_sync_high = conf->cluster_sync_low + CLUSTER_RESYNC_WINDOW_SECTORS; in raid1_sync_request()
3044 conf->cluster_sync_low, in raid1_sync_request()
3045 conf->cluster_sync_high); in raid1_sync_request()
3053 for (i = 0; i < conf->raid_disks * 2 && read_targets; i++) { in raid1_sync_request()
3082 struct r1conf *conf; in setup_conf() local
3089 conf = kzalloc(sizeof(struct r1conf), GFP_KERNEL); in setup_conf()
3090 if (!conf) in setup_conf()
3093 conf->nr_pending = kcalloc(BARRIER_BUCKETS_NR, in setup_conf()
3095 if (!conf->nr_pending) in setup_conf()
3098 conf->nr_waiting = kcalloc(BARRIER_BUCKETS_NR, in setup_conf()
3100 if (!conf->nr_waiting) in setup_conf()
3103 conf->nr_queued = kcalloc(BARRIER_BUCKETS_NR, in setup_conf()
3105 if (!conf->nr_queued) in setup_conf()
3108 conf->barrier = kcalloc(BARRIER_BUCKETS_NR, in setup_conf()
3110 if (!conf->barrier) in setup_conf()
3113 conf->mirrors = kzalloc(array3_size(sizeof(struct raid1_info), in setup_conf()
3116 if (!conf->mirrors) in setup_conf()
3119 conf->tmppage = alloc_page(GFP_KERNEL); in setup_conf()
3120 if (!conf->tmppage) in setup_conf()
3124 conf->r1bio_pool = mempool_create_kmalloc_pool(NR_RAID_BIOS, r1bio_size); in setup_conf()
3125 if (!conf->r1bio_pool) in setup_conf()
3128 err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0); in setup_conf()
3133 spin_lock_init(&conf->device_lock); in setup_conf()
3134 conf->raid_disks = mddev->raid_disks; in setup_conf()
3138 if (disk_idx >= conf->raid_disks || disk_idx < 0) in setup_conf()
3141 if (!raid1_add_conf(conf, rdev, disk_idx, in setup_conf()
3145 conf->mddev = mddev; in setup_conf()
3146 INIT_LIST_HEAD(&conf->retry_list); in setup_conf()
3147 INIT_LIST_HEAD(&conf->bio_end_io_list); in setup_conf()
3149 spin_lock_init(&conf->resync_lock); in setup_conf()
3150 init_waitqueue_head(&conf->wait_barrier); in setup_conf()
3152 bio_list_init(&conf->pending_bio_list); in setup_conf()
3153 conf->recovery_disabled = mddev->recovery_disabled - 1; in setup_conf()
3156 for (i = 0; i < conf->raid_disks * 2; i++) { in setup_conf()
3158 disk = conf->mirrors + i; in setup_conf()
3160 if (i < conf->raid_disks && in setup_conf()
3161 disk[conf->raid_disks].rdev) { in setup_conf()
3168 disk[conf->raid_disks].rdev; in setup_conf()
3169 disk[conf->raid_disks].rdev = NULL; in setup_conf()
3180 conf->fullsync = 1; in setup_conf()
3185 rcu_assign_pointer(conf->thread, in setup_conf()
3187 if (!conf->thread) in setup_conf()
3190 return conf; in setup_conf()
3193 if (conf) { in setup_conf()
3194 mempool_destroy(conf->r1bio_pool); in setup_conf()
3195 kfree(conf->mirrors); in setup_conf()
3196 safe_put_page(conf->tmppage); in setup_conf()
3197 kfree(conf->nr_pending); in setup_conf()
3198 kfree(conf->nr_waiting); in setup_conf()
3199 kfree(conf->nr_queued); in setup_conf()
3200 kfree(conf->barrier); in setup_conf()
3201 bioset_exit(&conf->bio_split); in setup_conf()
3202 kfree(conf); in setup_conf()
3223 struct r1conf *conf; in raid1_run() local
3244 conf = setup_conf(mddev); in raid1_run()
3246 conf = mddev->private; in raid1_run()
3248 if (IS_ERR(conf)) in raid1_run()
3249 return PTR_ERR(conf); in raid1_run()
3255 raid1_free(mddev, conf); in raid1_run()
3261 for (i = 0; i < conf->raid_disks; i++) in raid1_run()
3262 if (conf->mirrors[i].rdev == NULL || in raid1_run()
3263 !test_bit(In_sync, &conf->mirrors[i].rdev->flags) || in raid1_run()
3264 test_bit(Faulty, &conf->mirrors[i].rdev->flags)) in raid1_run()
3269 if (conf->raid_disks - mddev->degraded < 1) { in raid1_run()
3270 md_unregister_thread(mddev, &conf->thread); in raid1_run()
3272 raid1_free(mddev, conf); in raid1_run()
3276 if (conf->raid_disks - mddev->degraded == 1) in raid1_run()
3289 rcu_assign_pointer(mddev->thread, conf->thread); in raid1_run()
3290 rcu_assign_pointer(conf->thread, NULL); in raid1_run()
3291 mddev->private = conf; in raid1_run()
3304 struct r1conf *conf = priv; in raid1_free() local
3306 mempool_destroy(conf->r1bio_pool); in raid1_free()
3307 kfree(conf->mirrors); in raid1_free()
3308 safe_put_page(conf->tmppage); in raid1_free()
3309 kfree(conf->nr_pending); in raid1_free()
3310 kfree(conf->nr_waiting); in raid1_free()
3311 kfree(conf->nr_queued); in raid1_free()
3312 kfree(conf->barrier); in raid1_free()
3313 bioset_exit(&conf->bio_split); in raid1_free()
3314 kfree(conf); in raid1_free()
3364 struct r1conf *conf = mddev->private; in raid1_reshape() local
3384 if (raid_disks < conf->raid_disks) { in raid1_reshape()
3386 for (d= 0; d < conf->raid_disks; d++) in raid1_reshape()
3387 if (conf->mirrors[d].rdev) in raid1_reshape()
3406 freeze_array(conf, 0); in raid1_reshape()
3409 oldpool = conf->r1bio_pool; in raid1_reshape()
3410 conf->r1bio_pool = newpool; in raid1_reshape()
3412 for (d = d2 = 0; d < conf->raid_disks; d++) { in raid1_reshape()
3413 struct md_rdev *rdev = conf->mirrors[d].rdev; in raid1_reshape()
3425 kfree(conf->mirrors); in raid1_reshape()
3426 conf->mirrors = newmirrors; in raid1_reshape()
3428 spin_lock_irqsave(&conf->device_lock, flags); in raid1_reshape()
3429 mddev->degraded += (raid_disks - conf->raid_disks); in raid1_reshape()
3430 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1_reshape()
3431 conf->raid_disks = mddev->raid_disks = raid_disks; in raid1_reshape()
3434 unfreeze_array(conf); in raid1_reshape()
3446 struct r1conf *conf = mddev->private; in raid1_quiesce() local
3449 freeze_array(conf, 0); in raid1_quiesce()
3451 unfreeze_array(conf); in raid1_quiesce()
3460 struct r1conf *conf; in raid1_takeover() local
3464 conf = setup_conf(mddev); in raid1_takeover()
3465 if (!IS_ERR(conf)) { in raid1_takeover()
3467 conf->array_frozen = 1; in raid1_takeover()
3471 return conf; in raid1_takeover()