Lines Matching full:conf
46 static void allow_barrier(struct r1conf *conf, sector_t sector_nr);
47 static void lower_barrier(struct r1conf *conf, sector_t sector_nr);
239 static void put_all_bios(struct r1conf *conf, struct r1bio *r1_bio) in put_all_bios() argument
243 for (i = 0; i < conf->raid_disks * 2; i++) { in put_all_bios()
253 struct r1conf *conf = r1_bio->mddev->private; in free_r1bio() local
255 put_all_bios(conf, r1_bio); in free_r1bio()
256 mempool_free(r1_bio, &conf->r1bio_pool); in free_r1bio()
261 struct r1conf *conf = r1_bio->mddev->private; in put_buf() local
265 for (i = 0; i < conf->raid_disks * 2; i++) { in put_buf()
268 rdev_dec_pending(conf->mirrors[i].rdev, r1_bio->mddev); in put_buf()
271 mempool_free(r1_bio, &conf->r1buf_pool); in put_buf()
273 lower_barrier(conf, sect); in put_buf()
280 struct r1conf *conf = mddev->private; in reschedule_retry() local
284 spin_lock_irqsave(&conf->device_lock, flags); in reschedule_retry()
285 list_add(&r1_bio->retry_list, &conf->retry_list); in reschedule_retry()
286 atomic_inc(&conf->nr_queued[idx]); in reschedule_retry()
287 spin_unlock_irqrestore(&conf->device_lock, flags); in reschedule_retry()
289 wake_up(&conf->wait_barrier); in reschedule_retry()
311 struct r1conf *conf = r1_bio->mddev->private; in raid_end_bio_io() local
329 allow_barrier(conf, sector); in raid_end_bio_io()
337 struct r1conf *conf = r1_bio->mddev->private; in update_head_pos() local
339 conf->mirrors[disk].head_position = in update_head_pos()
349 struct r1conf *conf = r1_bio->mddev->private; in find_bio_disk() local
350 int raid_disks = conf->raid_disks; in find_bio_disk()
366 struct r1conf *conf = r1_bio->mddev->private; in raid1_end_read_request() local
367 struct md_rdev *rdev = conf->mirrors[r1_bio->read_disk].rdev; in raid1_end_read_request()
387 spin_lock_irqsave(&conf->device_lock, flags); in raid1_end_read_request()
388 if (r1_bio->mddev->degraded == conf->raid_disks || in raid1_end_read_request()
389 (r1_bio->mddev->degraded == conf->raid_disks-1 && in raid1_end_read_request()
392 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1_end_read_request()
397 rdev_dec_pending(rdev, conf->mddev); in raid1_end_read_request()
403 mdname(conf->mddev), in raid1_end_read_request()
450 struct r1conf *conf = r1_bio->mddev->private; in raid1_end_write_request() local
453 struct md_rdev *rdev = conf->mirrors[mirror].rdev; in raid1_end_write_request()
467 conf->mddev->recovery); in raid1_end_write_request()
550 rdev_dec_pending(rdev, conf->mddev); in raid1_end_write_request()
581 static void update_read_sectors(struct r1conf *conf, int disk, in update_read_sectors() argument
584 struct raid1_info *info = &conf->mirrors[disk]; in update_read_sectors()
592 static int choose_first_rdev(struct r1conf *conf, struct r1bio *r1_bio, in choose_first_rdev() argument
599 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) { in choose_first_rdev()
606 rdev = conf->mirrors[disk].rdev; in choose_first_rdev()
613 update_read_sectors(conf, disk, this_sector, read_len); in choose_first_rdev()
628 static int choose_bb_rdev(struct r1conf *conf, struct r1bio *r1_bio, in choose_bb_rdev() argument
636 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) { in choose_bb_rdev()
644 rdev = conf->mirrors[disk].rdev; in choose_bb_rdev()
661 update_read_sectors(conf, best_disk, this_sector, best_len); in choose_bb_rdev()
667 static int choose_slow_rdev(struct r1conf *conf, struct r1bio *r1_bio, in choose_slow_rdev() argument
675 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) { in choose_slow_rdev()
683 rdev = conf->mirrors[disk].rdev; in choose_slow_rdev()
694 update_read_sectors(conf, disk, this_sector, read_len); in choose_slow_rdev()
710 update_read_sectors(conf, bb_disk, this_sector, bb_read_len); in choose_slow_rdev()
716 static bool is_sequential(struct r1conf *conf, int disk, struct r1bio *r1_bio) in is_sequential() argument
719 return conf->mirrors[disk].next_seq_sect == r1_bio->sector || in is_sequential()
720 conf->mirrors[disk].head_position == r1_bio->sector; in is_sequential()
727 static bool should_choose_next(struct r1conf *conf, int disk) in should_choose_next() argument
729 struct raid1_info *mirror = &conf->mirrors[disk]; in should_choose_next()
769 static int choose_best_rdev(struct r1conf *conf, struct r1bio *r1_bio) in choose_best_rdev() argument
780 for (disk = 0 ; disk < conf->raid_disks * 2 ; disk++) { in choose_best_rdev()
788 rdev = conf->mirrors[disk].rdev; in choose_best_rdev()
797 dist = abs(r1_bio->sector - conf->mirrors[disk].head_position); in choose_best_rdev()
800 if (is_sequential(conf, disk, r1_bio)) { in choose_best_rdev()
801 if (!should_choose_next(conf, disk)) in choose_best_rdev()
841 (READ_ONCE(conf->nonrot_disks) || ctl.min_pending == 0)) in choose_best_rdev()
865 static int read_balance(struct r1conf *conf, struct r1bio *r1_bio, in read_balance() argument
872 if (raid1_should_read_first(conf->mddev, r1_bio->sector, in read_balance()
874 return choose_first_rdev(conf, r1_bio, max_sectors); in read_balance()
876 disk = choose_best_rdev(conf, r1_bio); in read_balance()
879 update_read_sectors(conf, disk, r1_bio->sector, in read_balance()
889 disk = choose_bb_rdev(conf, r1_bio, max_sectors); in read_balance()
893 return choose_slow_rdev(conf, r1_bio, max_sectors); in read_balance()
896 static void wake_up_barrier(struct r1conf *conf) in wake_up_barrier() argument
898 if (wq_has_sleeper(&conf->wait_barrier)) in wake_up_barrier()
899 wake_up(&conf->wait_barrier); in wake_up_barrier()
902 static void flush_bio_list(struct r1conf *conf, struct bio *bio) in flush_bio_list() argument
905 raid1_prepare_flush_writes(conf->mddev); in flush_bio_list()
906 wake_up_barrier(conf); in flush_bio_list()
917 static void flush_pending_writes(struct r1conf *conf) in flush_pending_writes() argument
922 spin_lock_irq(&conf->device_lock); in flush_pending_writes()
924 if (conf->pending_bio_list.head) { in flush_pending_writes()
928 bio = bio_list_get(&conf->pending_bio_list); in flush_pending_writes()
929 spin_unlock_irq(&conf->device_lock); in flush_pending_writes()
942 flush_bio_list(conf, bio); in flush_pending_writes()
945 spin_unlock_irq(&conf->device_lock); in flush_pending_writes()
972 static int raise_barrier(struct r1conf *conf, sector_t sector_nr) in raise_barrier() argument
976 spin_lock_irq(&conf->resync_lock); in raise_barrier()
979 wait_event_lock_irq(conf->wait_barrier, in raise_barrier()
980 !atomic_read(&conf->nr_waiting[idx]), in raise_barrier()
981 conf->resync_lock); in raise_barrier()
984 atomic_inc(&conf->barrier[idx]); in raise_barrier()
986 * In raise_barrier() we firstly increase conf->barrier[idx] then in raise_barrier()
987 * check conf->nr_pending[idx]. In _wait_barrier() we firstly in raise_barrier()
988 * increase conf->nr_pending[idx] then check conf->barrier[idx]. in raise_barrier()
989 * A memory barrier here to make sure conf->nr_pending[idx] won't in raise_barrier()
990 * be fetched before conf->barrier[idx] is increased. Otherwise in raise_barrier()
997 * B: while conf->nr_pending[idx] is not 0, meaning regular I/O in raise_barrier()
999 * C: while conf->barrier[idx] >= RESYNC_DEPTH, meaning reaches in raise_barrier()
1002 wait_event_lock_irq(conf->wait_barrier, in raise_barrier()
1003 (!conf->array_frozen && in raise_barrier()
1004 !atomic_read(&conf->nr_pending[idx]) && in raise_barrier()
1005 atomic_read(&conf->barrier[idx]) < RESYNC_DEPTH) || in raise_barrier()
1006 test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery), in raise_barrier()
1007 conf->resync_lock); in raise_barrier()
1009 if (test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { in raise_barrier()
1010 atomic_dec(&conf->barrier[idx]); in raise_barrier()
1011 spin_unlock_irq(&conf->resync_lock); in raise_barrier()
1012 wake_up(&conf->wait_barrier); in raise_barrier()
1016 atomic_inc(&conf->nr_sync_pending); in raise_barrier()
1017 spin_unlock_irq(&conf->resync_lock); in raise_barrier()
1022 static void lower_barrier(struct r1conf *conf, sector_t sector_nr) in lower_barrier() argument
1026 BUG_ON(atomic_read(&conf->barrier[idx]) <= 0); in lower_barrier()
1028 atomic_dec(&conf->barrier[idx]); in lower_barrier()
1029 atomic_dec(&conf->nr_sync_pending); in lower_barrier()
1030 wake_up(&conf->wait_barrier); in lower_barrier()
1033 static bool _wait_barrier(struct r1conf *conf, int idx, bool nowait) in _wait_barrier() argument
1038 * We need to increase conf->nr_pending[idx] very early here, in _wait_barrier()
1040 * conf->nr_pending[idx] to be 0. Then we can avoid holding in _wait_barrier()
1041 * conf->resync_lock when there is no barrier raised in same in _wait_barrier()
1045 atomic_inc(&conf->nr_pending[idx]); in _wait_barrier()
1047 * In _wait_barrier() we firstly increase conf->nr_pending[idx], then in _wait_barrier()
1048 * check conf->barrier[idx]. In raise_barrier() we firstly increase in _wait_barrier()
1049 * conf->barrier[idx], then check conf->nr_pending[idx]. A memory in _wait_barrier()
1050 * barrier is necessary here to make sure conf->barrier[idx] won't be in _wait_barrier()
1051 * fetched before conf->nr_pending[idx] is increased. Otherwise there in _wait_barrier()
1058 * here. If during we check conf->barrier[idx], the array is in _wait_barrier()
1059 * frozen (conf->array_frozen is 1), and chonf->barrier[idx] is in _wait_barrier()
1065 if (!READ_ONCE(conf->array_frozen) && in _wait_barrier()
1066 !atomic_read(&conf->barrier[idx])) in _wait_barrier()
1070 * After holding conf->resync_lock, conf->nr_pending[idx] in _wait_barrier()
1073 * raise_barrer() might be waiting for conf->nr_pending[idx] in _wait_barrier()
1076 spin_lock_irq(&conf->resync_lock); in _wait_barrier()
1077 atomic_inc(&conf->nr_waiting[idx]); in _wait_barrier()
1078 atomic_dec(&conf->nr_pending[idx]); in _wait_barrier()
1083 wake_up_barrier(conf); in _wait_barrier()
1090 wait_event_lock_irq(conf->wait_barrier, in _wait_barrier()
1091 !conf->array_frozen && in _wait_barrier()
1092 !atomic_read(&conf->barrier[idx]), in _wait_barrier()
1093 conf->resync_lock); in _wait_barrier()
1094 atomic_inc(&conf->nr_pending[idx]); in _wait_barrier()
1097 atomic_dec(&conf->nr_waiting[idx]); in _wait_barrier()
1098 spin_unlock_irq(&conf->resync_lock); in _wait_barrier()
1102 static bool wait_read_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait) in wait_read_barrier() argument
1112 * conf->barrier[idx] here, memory barrier is unnecessary as well. in wait_read_barrier()
1114 atomic_inc(&conf->nr_pending[idx]); in wait_read_barrier()
1116 if (!READ_ONCE(conf->array_frozen)) in wait_read_barrier()
1119 spin_lock_irq(&conf->resync_lock); in wait_read_barrier()
1120 atomic_inc(&conf->nr_waiting[idx]); in wait_read_barrier()
1121 atomic_dec(&conf->nr_pending[idx]); in wait_read_barrier()
1126 wake_up_barrier(conf); in wait_read_barrier()
1134 wait_event_lock_irq(conf->wait_barrier, in wait_read_barrier()
1135 !conf->array_frozen, in wait_read_barrier()
1136 conf->resync_lock); in wait_read_barrier()
1137 atomic_inc(&conf->nr_pending[idx]); in wait_read_barrier()
1140 atomic_dec(&conf->nr_waiting[idx]); in wait_read_barrier()
1141 spin_unlock_irq(&conf->resync_lock); in wait_read_barrier()
1145 static bool wait_barrier(struct r1conf *conf, sector_t sector_nr, bool nowait) in wait_barrier() argument
1149 return _wait_barrier(conf, idx, nowait); in wait_barrier()
1152 static void _allow_barrier(struct r1conf *conf, int idx) in _allow_barrier() argument
1154 atomic_dec(&conf->nr_pending[idx]); in _allow_barrier()
1155 wake_up_barrier(conf); in _allow_barrier()
1158 static void allow_barrier(struct r1conf *conf, sector_t sector_nr) in allow_barrier() argument
1162 _allow_barrier(conf, idx); in allow_barrier()
1165 /* conf->resync_lock should be held */
1166 static int get_unqueued_pending(struct r1conf *conf) in get_unqueued_pending() argument
1170 ret = atomic_read(&conf->nr_sync_pending); in get_unqueued_pending()
1172 ret += atomic_read(&conf->nr_pending[idx]) - in get_unqueued_pending()
1173 atomic_read(&conf->nr_queued[idx]); in get_unqueued_pending()
1178 static void freeze_array(struct r1conf *conf, int extra) in freeze_array() argument
1191 * Every flying I/O contributes to a conf->nr_pending[idx], idx is the in freeze_array()
1193 * normal I/O are queued, sum of all conf->nr_pending[] will match sum in freeze_array()
1194 * of all conf->nr_queued[]. But normal I/O failure is an exception, in freeze_array()
1200 * get_unqueued_pendings(conf) gets equal to extra. For in freeze_array()
1203 spin_lock_irq(&conf->resync_lock); in freeze_array()
1204 conf->array_frozen = 1; in freeze_array()
1205 mddev_add_trace_msg(conf->mddev, "raid1 wait freeze"); in freeze_array()
1207 conf->wait_barrier, in freeze_array()
1208 get_unqueued_pending(conf) == extra, in freeze_array()
1209 conf->resync_lock, in freeze_array()
1210 flush_pending_writes(conf)); in freeze_array()
1211 spin_unlock_irq(&conf->resync_lock); in freeze_array()
1213 static void unfreeze_array(struct r1conf *conf) in unfreeze_array() argument
1216 spin_lock_irq(&conf->resync_lock); in unfreeze_array()
1217 conf->array_frozen = 0; in unfreeze_array()
1218 spin_unlock_irq(&conf->resync_lock); in unfreeze_array()
1219 wake_up(&conf->wait_barrier); in unfreeze_array()
1275 struct r1conf *conf = mddev->private; in raid1_unplug() local
1279 spin_lock_irq(&conf->device_lock); in raid1_unplug()
1280 bio_list_merge(&conf->pending_bio_list, &plug->pending); in raid1_unplug()
1281 spin_unlock_irq(&conf->device_lock); in raid1_unplug()
1282 wake_up_barrier(conf); in raid1_unplug()
1290 flush_bio_list(conf, bio); in raid1_unplug()
1306 struct r1conf *conf = mddev->private; in alloc_r1bio() local
1309 r1_bio = mempool_alloc(&conf->r1bio_pool, GFP_NOIO); in alloc_r1bio()
1311 memset(r1_bio->bios, 0, conf->raid_disks * sizeof(r1_bio->bios[0])); in alloc_r1bio()
1319 struct r1conf *conf = mddev->private; in raid1_read_request() local
1339 if (!wait_read_barrier(conf, bio->bi_iter.bi_sector, in raid1_read_request()
1355 rdisk = read_balance(conf, r1_bio, &max_sectors); in raid1_read_request()
1361 conf->mirrors[r1_bio->read_disk].rdev->bdev, in raid1_read_request()
1366 mirror = conf->mirrors + rdisk; in raid1_read_request()
1385 gfp, &conf->bio_split); in raid1_read_request()
1418 struct r1conf *conf = mddev->private; in raid1_write_request() local
1438 prepare_to_wait(&conf->wait_barrier, in raid1_write_request()
1446 finish_wait(&conf->wait_barrier, &w); in raid1_write_request()
1454 if (!wait_barrier(conf, bio->bi_iter.bi_sector, in raid1_write_request()
1475 disks = conf->raid_disks * 2; in raid1_write_request()
1479 struct md_rdev *rdev = conf->mirrors[i].rdev; in raid1_write_request()
1496 if (i < conf->raid_disks) in raid1_write_request()
1552 rdev_dec_pending(conf->mirrors[j].rdev, mddev); in raid1_write_request()
1553 mempool_free(r1_bio, &conf->r1bio_pool); in raid1_write_request()
1554 allow_barrier(conf, bio->bi_iter.bi_sector); in raid1_write_request()
1563 wait_barrier(conf, bio->bi_iter.bi_sector, false); in raid1_write_request()
1578 GFP_NOIO, &conf->bio_split); in raid1_write_request()
1595 struct md_rdev *rdev = conf->mirrors[i].rdev; in raid1_write_request()
1643 conf->raid_disks - mddev->degraded > 1) in raid1_write_request()
1652 spin_lock_irqsave(&conf->device_lock, flags); in raid1_write_request()
1653 bio_list_add(&conf->pending_bio_list, mbio); in raid1_write_request()
1654 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1_write_request()
1662 wake_up_barrier(conf); in raid1_write_request()
1694 struct r1conf *conf = mddev->private; in raid1_status() local
1699 seq_printf(seq, " [%d/%d] [", conf->raid_disks, in raid1_status()
1700 conf->raid_disks - mddev->degraded); in raid1_status()
1701 for (i = 0; i < conf->raid_disks; i++) { in raid1_status()
1702 struct md_rdev *rdev = READ_ONCE(conf->mirrors[i].rdev); in raid1_status()
1728 struct r1conf *conf = mddev->private; in raid1_error() local
1731 spin_lock_irqsave(&conf->device_lock, flags); in raid1_error()
1734 (conf->raid_disks - mddev->degraded) == 1) { in raid1_error()
1738 conf->recovery_disabled = mddev->recovery_disabled; in raid1_error()
1739 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1_error()
1747 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1_error()
1757 mdname(mddev), conf->raid_disks - mddev->degraded); in raid1_error()
1760 static void print_conf(struct r1conf *conf) in print_conf() argument
1764 pr_debug("RAID1 conf printout:\n"); in print_conf()
1765 if (!conf) { in print_conf()
1766 pr_debug("(!conf)\n"); in print_conf()
1769 pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded, in print_conf()
1770 conf->raid_disks); in print_conf()
1772 lockdep_assert_held(&conf->mddev->reconfig_mutex); in print_conf()
1773 for (i = 0; i < conf->raid_disks; i++) { in print_conf()
1774 struct md_rdev *rdev = conf->mirrors[i].rdev; in print_conf()
1783 static void close_sync(struct r1conf *conf) in close_sync() argument
1788 _wait_barrier(conf, idx, false); in close_sync()
1789 _allow_barrier(conf, idx); in close_sync()
1792 mempool_exit(&conf->r1buf_pool); in close_sync()
1798 struct r1conf *conf = mddev->private; in raid1_spare_active() local
1809 spin_lock_irqsave(&conf->device_lock, flags); in raid1_spare_active()
1810 for (i = 0; i < conf->raid_disks; i++) { in raid1_spare_active()
1811 struct md_rdev *rdev = conf->mirrors[i].rdev; in raid1_spare_active()
1812 struct md_rdev *repl = conf->mirrors[conf->raid_disks + i].rdev; in raid1_spare_active()
1841 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1_spare_active()
1843 print_conf(conf); in raid1_spare_active()
1847 static bool raid1_add_conf(struct r1conf *conf, struct md_rdev *rdev, int disk, in raid1_add_conf() argument
1850 struct raid1_info *info = conf->mirrors + disk; in raid1_add_conf()
1853 info += conf->raid_disks; in raid1_add_conf()
1860 WRITE_ONCE(conf->nonrot_disks, conf->nonrot_disks + 1); in raid1_add_conf()
1871 static bool raid1_remove_conf(struct r1conf *conf, int disk) in raid1_remove_conf() argument
1873 struct raid1_info *info = conf->mirrors + disk; in raid1_remove_conf()
1882 rdev->mddev->recovery_disabled != conf->recovery_disabled && in raid1_remove_conf()
1883 rdev->mddev->degraded < conf->raid_disks) in raid1_remove_conf()
1887 WRITE_ONCE(conf->nonrot_disks, conf->nonrot_disks - 1); in raid1_remove_conf()
1895 struct r1conf *conf = mddev->private; in raid1_add_disk() local
1900 int last = conf->raid_disks - 1; in raid1_add_disk()
1902 if (mddev->recovery_disabled == conf->recovery_disabled) in raid1_add_disk()
1914 rdev->saved_raid_disk < conf->raid_disks && in raid1_add_disk()
1915 conf->mirrors[rdev->saved_raid_disk].rdev == NULL) in raid1_add_disk()
1919 p = conf->mirrors + mirror; in raid1_add_disk()
1925 raid1_add_conf(conf, rdev, mirror, false); in raid1_add_disk()
1930 conf->fullsync = 1; in raid1_add_disk()
1934 p[conf->raid_disks].rdev == NULL && repl_slot < 0) in raid1_add_disk()
1942 raid1_add_conf(conf, rdev, repl_slot, true); in raid1_add_disk()
1944 conf->fullsync = 1; in raid1_add_disk()
1947 print_conf(conf); in raid1_add_disk()
1953 struct r1conf *conf = mddev->private; in raid1_remove_disk() local
1956 struct raid1_info *p = conf->mirrors + number; in raid1_remove_disk()
1958 if (unlikely(number >= conf->raid_disks)) in raid1_remove_disk()
1962 number += conf->raid_disks; in raid1_remove_disk()
1963 p = conf->mirrors + number; in raid1_remove_disk()
1966 print_conf(conf); in raid1_remove_disk()
1968 if (!raid1_remove_conf(conf, number)) { in raid1_remove_disk()
1973 if (number < conf->raid_disks && in raid1_remove_disk()
1974 conf->mirrors[conf->raid_disks + number].rdev) { in raid1_remove_disk()
1980 conf->mirrors[conf->raid_disks + number].rdev; in raid1_remove_disk()
1981 freeze_array(conf, 0); in raid1_remove_disk()
1990 unfreeze_array(conf); in raid1_remove_disk()
1995 conf->mirrors[conf->raid_disks + number].rdev = NULL; in raid1_remove_disk()
1996 unfreeze_array(conf); in raid1_remove_disk()
2004 print_conf(conf); in raid1_remove_disk()
2061 struct r1conf *conf = mddev->private; in end_sync_write() local
2062 struct md_rdev *rdev = conf->mirrors[find_bio_disk(r1_bio, bio)].rdev; in end_sync_write()
2072 !rdev_has_badblock(conf->mirrors[r1_bio->read_disk].rdev, in end_sync_write()
2113 struct r1conf *conf = mddev->private; in fix_sync_read_error() local
2121 rdev = conf->mirrors[r1_bio->read_disk].rdev; in fix_sync_read_error()
2147 rdev = conf->mirrors[d].rdev; in fix_sync_read_error()
2156 if (d == conf->raid_disks * 2) in fix_sync_read_error()
2170 for (d = 0; d < conf->raid_disks * 2; d++) { in fix_sync_read_error()
2171 rdev = conf->mirrors[d].rdev; in fix_sync_read_error()
2178 conf->recovery_disabled = in fix_sync_read_error()
2196 d = conf->raid_disks * 2; in fix_sync_read_error()
2200 rdev = conf->mirrors[d].rdev; in fix_sync_read_error()
2211 d = conf->raid_disks * 2; in fix_sync_read_error()
2215 rdev = conf->mirrors[d].rdev; in fix_sync_read_error()
2240 struct r1conf *conf = mddev->private; in process_checks() local
2247 for (i = 0; i < conf->raid_disks * 2; i++) { in process_checks()
2255 bio_reset(b, conf->mirrors[i].rdev->bdev, REQ_OP_READ); in process_checks()
2258 conf->mirrors[i].rdev->data_offset; in process_checks()
2266 for (primary = 0; primary < conf->raid_disks * 2; primary++) in process_checks()
2270 rdev_dec_pending(conf->mirrors[primary].rdev, mddev); in process_checks()
2274 for (i = 0; i < conf->raid_disks * 2; i++) { in process_checks()
2308 rdev_dec_pending(conf->mirrors[i].rdev, mddev); in process_checks()
2318 struct r1conf *conf = mddev->private; in sync_request_write() local
2320 int disks = conf->raid_disks * 2; in sync_request_write()
2342 if (test_bit(Faulty, &conf->mirrors[i].rdev->flags)) { in sync_request_write()
2348 if (test_bit(FailFast, &conf->mirrors[i].rdev->flags)) in sync_request_write()
2353 md_sync_acct(conf->mirrors[i].rdev->bdev, bio_sectors(wbio)); in sync_request_write()
2369 static void fix_read_error(struct r1conf *conf, struct r1bio *r1_bio) in fix_read_error() argument
2374 struct mddev *mddev = conf->mddev; in fix_read_error()
2375 struct md_rdev *rdev = conf->mirrors[read_disk].rdev; in fix_read_error()
2392 rdev = conf->mirrors[d].rdev; in fix_read_error()
2400 conf->tmppage, REQ_OP_READ, false)) in fix_read_error()
2408 if (d == conf->raid_disks * 2) in fix_read_error()
2414 struct md_rdev *rdev = conf->mirrors[read_disk].rdev; in fix_read_error()
2423 d = conf->raid_disks * 2; in fix_read_error()
2425 rdev = conf->mirrors[d].rdev; in fix_read_error()
2430 conf->tmppage, REQ_OP_WRITE); in fix_read_error()
2437 d = conf->raid_disks * 2; in fix_read_error()
2439 rdev = conf->mirrors[d].rdev; in fix_read_error()
2444 conf->tmppage, REQ_OP_READ)) { in fix_read_error()
2463 struct r1conf *conf = mddev->private; in narrow_write_error() local
2464 struct md_rdev *rdev = conf->mirrors[i].rdev; in narrow_write_error()
2529 static void handle_sync_write_finished(struct r1conf *conf, struct r1bio *r1_bio) in handle_sync_write_finished() argument
2533 for (m = 0; m < conf->raid_disks * 2 ; m++) { in handle_sync_write_finished()
2534 struct md_rdev *rdev = conf->mirrors[m].rdev; in handle_sync_write_finished()
2545 md_error(conf->mddev, rdev); in handle_sync_write_finished()
2549 md_done_sync(conf->mddev, s, 1); in handle_sync_write_finished()
2552 static void handle_write_finished(struct r1conf *conf, struct r1bio *r1_bio) in handle_write_finished() argument
2557 for (m = 0; m < conf->raid_disks * 2 ; m++) in handle_write_finished()
2559 struct md_rdev *rdev = conf->mirrors[m].rdev; in handle_write_finished()
2563 rdev_dec_pending(rdev, conf->mddev); in handle_write_finished()
2571 md_error(conf->mddev, in handle_write_finished()
2572 conf->mirrors[m].rdev); in handle_write_finished()
2576 rdev_dec_pending(conf->mirrors[m].rdev, in handle_write_finished()
2577 conf->mddev); in handle_write_finished()
2580 spin_lock_irq(&conf->device_lock); in handle_write_finished()
2581 list_add(&r1_bio->retry_list, &conf->bio_end_io_list); in handle_write_finished()
2583 atomic_inc(&conf->nr_queued[idx]); in handle_write_finished()
2584 spin_unlock_irq(&conf->device_lock); in handle_write_finished()
2589 wake_up(&conf->wait_barrier); in handle_write_finished()
2590 md_wakeup_thread(conf->mddev->thread); in handle_write_finished()
2598 static void handle_read_error(struct r1conf *conf, struct r1bio *r1_bio) in handle_read_error() argument
2600 struct mddev *mddev = conf->mddev; in handle_read_error()
2619 rdev = conf->mirrors[r1_bio->read_disk].rdev; in handle_read_error()
2622 freeze_array(conf, 1); in handle_read_error()
2623 fix_read_error(conf, r1_bio); in handle_read_error()
2624 unfreeze_array(conf); in handle_read_error()
2631 rdev_dec_pending(rdev, conf->mddev); in handle_read_error()
2638 allow_barrier(conf, sector); in handle_read_error()
2646 struct r1conf *conf = mddev->private; in raid1d() local
2647 struct list_head *head = &conf->retry_list; in raid1d()
2653 if (!list_empty_careful(&conf->bio_end_io_list) && in raid1d()
2656 spin_lock_irqsave(&conf->device_lock, flags); in raid1d()
2658 list_splice_init(&conf->bio_end_io_list, &tmp); in raid1d()
2659 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1d()
2665 atomic_dec(&conf->nr_queued[idx]); in raid1d()
2677 flush_pending_writes(conf); in raid1d()
2679 spin_lock_irqsave(&conf->device_lock, flags); in raid1d()
2681 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1d()
2687 atomic_dec(&conf->nr_queued[idx]); in raid1d()
2688 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1d()
2691 conf = mddev->private; in raid1d()
2695 handle_sync_write_finished(conf, r1_bio); in raid1d()
2700 handle_write_finished(conf, r1_bio); in raid1d()
2702 handle_read_error(conf, r1_bio); in raid1d()
2713 static int init_resync(struct r1conf *conf) in init_resync() argument
2718 BUG_ON(mempool_initialized(&conf->r1buf_pool)); in init_resync()
2720 return mempool_init(&conf->r1buf_pool, buffs, r1buf_pool_alloc, in init_resync()
2721 r1buf_pool_free, conf->poolinfo); in init_resync()
2724 static struct r1bio *raid1_alloc_init_r1buf(struct r1conf *conf) in raid1_alloc_init_r1buf() argument
2726 struct r1bio *r1bio = mempool_alloc(&conf->r1buf_pool, GFP_NOIO); in raid1_alloc_init_r1buf()
2731 for (i = conf->poolinfo->raid_disks; i--; ) { in raid1_alloc_init_r1buf()
2754 struct r1conf *conf = mddev->private; in raid1_sync_request() local
2769 if (!mempool_initialized(&conf->r1buf_pool)) in raid1_sync_request()
2770 if (init_resync(conf)) in raid1_sync_request()
2783 conf->fullsync = 0; in raid1_sync_request()
2786 close_sync(conf); in raid1_sync_request()
2789 conf->cluster_sync_low = 0; in raid1_sync_request()
2790 conf->cluster_sync_high = 0; in raid1_sync_request()
2798 conf->fullsync == 0) { in raid1_sync_request()
2806 !conf->fullsync && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery)) { in raid1_sync_request()
2816 if (atomic_read(&conf->nr_waiting[idx])) in raid1_sync_request()
2825 (sector_nr + 2 * RESYNC_SECTORS > conf->cluster_sync_high)); in raid1_sync_request()
2827 if (raise_barrier(conf, sector_nr)) in raid1_sync_request()
2830 r1_bio = raid1_alloc_init_r1buf(conf); in raid1_sync_request()
2848 for (i = 0; i < conf->raid_disks * 2; i++) { in raid1_sync_request()
2852 rdev = conf->mirrors[i].rdev; in raid1_sync_request()
2855 if (i < conf->raid_disks) in raid1_sync_request()
2919 for (i = 0 ; i < conf->raid_disks * 2 ; i++) in raid1_sync_request()
2921 struct md_rdev *rdev = conf->mirrors[i].rdev; in raid1_sync_request()
2936 conf->recovery_disabled = mddev->recovery_disabled; in raid1_sync_request()
2982 !conf->fullsync && in raid1_sync_request()
2989 for (i = 0 ; i < conf->raid_disks * 2; i++) { in raid1_sync_request()
3012 conf->cluster_sync_high < sector_nr + nr_sectors) { in raid1_sync_request()
3013 conf->cluster_sync_low = mddev->curr_resync_completed; in raid1_sync_request()
3014 conf->cluster_sync_high = conf->cluster_sync_low + CLUSTER_RESYNC_WINDOW_SECTORS; in raid1_sync_request()
3017 conf->cluster_sync_low, in raid1_sync_request()
3018 conf->cluster_sync_high); in raid1_sync_request()
3026 for (i = 0; i < conf->raid_disks * 2 && read_targets; i++) { in raid1_sync_request()
3057 struct r1conf *conf; in setup_conf() local
3063 conf = kzalloc(sizeof(struct r1conf), GFP_KERNEL); in setup_conf()
3064 if (!conf) in setup_conf()
3067 conf->nr_pending = kcalloc(BARRIER_BUCKETS_NR, in setup_conf()
3069 if (!conf->nr_pending) in setup_conf()
3072 conf->nr_waiting = kcalloc(BARRIER_BUCKETS_NR, in setup_conf()
3074 if (!conf->nr_waiting) in setup_conf()
3077 conf->nr_queued = kcalloc(BARRIER_BUCKETS_NR, in setup_conf()
3079 if (!conf->nr_queued) in setup_conf()
3082 conf->barrier = kcalloc(BARRIER_BUCKETS_NR, in setup_conf()
3084 if (!conf->barrier) in setup_conf()
3087 conf->mirrors = kzalloc(array3_size(sizeof(struct raid1_info), in setup_conf()
3090 if (!conf->mirrors) in setup_conf()
3093 conf->tmppage = alloc_page(GFP_KERNEL); in setup_conf()
3094 if (!conf->tmppage) in setup_conf()
3097 conf->poolinfo = kzalloc(sizeof(*conf->poolinfo), GFP_KERNEL); in setup_conf()
3098 if (!conf->poolinfo) in setup_conf()
3100 conf->poolinfo->raid_disks = mddev->raid_disks * 2; in setup_conf()
3101 err = mempool_init(&conf->r1bio_pool, NR_RAID_BIOS, r1bio_pool_alloc, in setup_conf()
3102 rbio_pool_free, conf->poolinfo); in setup_conf()
3106 err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0); in setup_conf()
3110 conf->poolinfo->mddev = mddev; in setup_conf()
3113 spin_lock_init(&conf->device_lock); in setup_conf()
3114 conf->raid_disks = mddev->raid_disks; in setup_conf()
3118 if (disk_idx >= conf->raid_disks || disk_idx < 0) in setup_conf()
3121 if (!raid1_add_conf(conf, rdev, disk_idx, in setup_conf()
3125 conf->mddev = mddev; in setup_conf()
3126 INIT_LIST_HEAD(&conf->retry_list); in setup_conf()
3127 INIT_LIST_HEAD(&conf->bio_end_io_list); in setup_conf()
3129 spin_lock_init(&conf->resync_lock); in setup_conf()
3130 init_waitqueue_head(&conf->wait_barrier); in setup_conf()
3132 bio_list_init(&conf->pending_bio_list); in setup_conf()
3133 conf->recovery_disabled = mddev->recovery_disabled - 1; in setup_conf()
3136 for (i = 0; i < conf->raid_disks * 2; i++) { in setup_conf()
3138 disk = conf->mirrors + i; in setup_conf()
3140 if (i < conf->raid_disks && in setup_conf()
3141 disk[conf->raid_disks].rdev) { in setup_conf()
3148 disk[conf->raid_disks].rdev; in setup_conf()
3149 disk[conf->raid_disks].rdev = NULL; in setup_conf()
3160 conf->fullsync = 1; in setup_conf()
3165 rcu_assign_pointer(conf->thread, in setup_conf()
3167 if (!conf->thread) in setup_conf()
3170 return conf; in setup_conf()
3173 if (conf) { in setup_conf()
3174 mempool_exit(&conf->r1bio_pool); in setup_conf()
3175 kfree(conf->mirrors); in setup_conf()
3176 safe_put_page(conf->tmppage); in setup_conf()
3177 kfree(conf->poolinfo); in setup_conf()
3178 kfree(conf->nr_pending); in setup_conf()
3179 kfree(conf->nr_waiting); in setup_conf()
3180 kfree(conf->nr_queued); in setup_conf()
3181 kfree(conf->barrier); in setup_conf()
3182 bioset_exit(&conf->bio_split); in setup_conf()
3183 kfree(conf); in setup_conf()
3205 struct r1conf *conf; in raid1_run() local
3226 conf = setup_conf(mddev); in raid1_run()
3228 conf = mddev->private; in raid1_run()
3230 if (IS_ERR(conf)) in raid1_run()
3231 return PTR_ERR(conf); in raid1_run()
3240 for (i = 0; i < conf->raid_disks; i++) in raid1_run()
3241 if (conf->mirrors[i].rdev == NULL || in raid1_run()
3242 !test_bit(In_sync, &conf->mirrors[i].rdev->flags) || in raid1_run()
3243 test_bit(Faulty, &conf->mirrors[i].rdev->flags)) in raid1_run()
3248 if (conf->raid_disks - mddev->degraded < 1) { in raid1_run()
3249 md_unregister_thread(mddev, &conf->thread); in raid1_run()
3253 if (conf->raid_disks - mddev->degraded == 1) in raid1_run()
3266 rcu_assign_pointer(mddev->thread, conf->thread); in raid1_run()
3267 rcu_assign_pointer(conf->thread, NULL); in raid1_run()
3268 mddev->private = conf; in raid1_run()
3281 struct r1conf *conf = priv; in raid1_free() local
3283 mempool_exit(&conf->r1bio_pool); in raid1_free()
3284 kfree(conf->mirrors); in raid1_free()
3285 safe_put_page(conf->tmppage); in raid1_free()
3286 kfree(conf->poolinfo); in raid1_free()
3287 kfree(conf->nr_pending); in raid1_free()
3288 kfree(conf->nr_waiting); in raid1_free()
3289 kfree(conf->nr_queued); in raid1_free()
3290 kfree(conf->barrier); in raid1_free()
3291 bioset_exit(&conf->bio_split); in raid1_free()
3292 kfree(conf); in raid1_free()
3330 * 2/ resize conf->mirrors in raid1_reshape()
3334 * Then resize conf->mirrors and swap in the new r1bio pool. in raid1_reshape()
3342 struct r1conf *conf = mddev->private; in raid1_reshape() local
3366 if (raid_disks < conf->raid_disks) { in raid1_reshape()
3368 for (d= 0; d < conf->raid_disks; d++) in raid1_reshape()
3369 if (conf->mirrors[d].rdev) in raid1_reshape()
3396 freeze_array(conf, 0); in raid1_reshape()
3399 oldpool = conf->r1bio_pool; in raid1_reshape()
3400 conf->r1bio_pool = newpool; in raid1_reshape()
3402 for (d = d2 = 0; d < conf->raid_disks; d++) { in raid1_reshape()
3403 struct md_rdev *rdev = conf->mirrors[d].rdev; in raid1_reshape()
3415 kfree(conf->mirrors); in raid1_reshape()
3416 conf->mirrors = newmirrors; in raid1_reshape()
3417 kfree(conf->poolinfo); in raid1_reshape()
3418 conf->poolinfo = newpoolinfo; in raid1_reshape()
3420 spin_lock_irqsave(&conf->device_lock, flags); in raid1_reshape()
3421 mddev->degraded += (raid_disks - conf->raid_disks); in raid1_reshape()
3422 spin_unlock_irqrestore(&conf->device_lock, flags); in raid1_reshape()
3423 conf->raid_disks = mddev->raid_disks = raid_disks; in raid1_reshape()
3426 unfreeze_array(conf); in raid1_reshape()
3438 struct r1conf *conf = mddev->private; in raid1_quiesce() local
3441 freeze_array(conf, 0); in raid1_quiesce()
3443 unfreeze_array(conf); in raid1_quiesce()
3452 struct r1conf *conf; in raid1_takeover() local
3456 conf = setup_conf(mddev); in raid1_takeover()
3457 if (!IS_ERR(conf)) { in raid1_takeover()
3459 conf->array_frozen = 1; in raid1_takeover()
3463 return conf; in raid1_takeover()