Lines Matching +full:system +full:- +full:ctl
1 // SPDX-License-Identifier: GPL-2.0
17 #include "disk-io.h"
21 #include "rcu-string.h"
22 #include "dev-replace.h"
24 #include "tree-checker.h"
25 #include "space-info.h"
26 #include "block-group.h"
31 #include "uuid-tree.h"
36 #include "raid-stripe-tree.h"
227 size_bp -= ret; \ in btrfs_describe_block_groups()
234 DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_SYSTEM, "system"); in btrfs_describe_block_groups()
245 size_bp -= ret; in btrfs_describe_block_groups()
249 buf[size_buf - size_bp - 1] = '\0'; /* remove last | */ in btrfs_describe_block_groups()
266 * There are several mutexes that protect manipulation of devices and low-level
270 * ------------------------
271 * protects the fs_uuids list that tracks all per-fs fs_devices, resulting from
275 * the mutex can be very coarse and can cover long-running operations
280 * global::fs_devs - add, remove, updates to the global list
286 * btrfs_device::name - renames (write side), read is RCU
288 * fs_devices::device_list_mutex (per-fs, with RCU)
289 * ------------------------------------------------
292 * simple list traversal with read-only actions can be done with RCU protection
301 * -------------
306 * -----------
313 * -------------
333 * - Balance (*)
334 * - Device add
335 * - Device remove
336 * - Device replace (*)
337 * - Resize
341 * - Running state
342 * - Paused state
343 * - Completed state
348 * - ioctl (only Balance can be Paused through ioctl)
349 * - filesystem remounted as read-only
350 * - filesystem unmounted and mounted as read-only
351 * - system power-cycle and filesystem mounted as read-only
352 * - filesystem or device errors leading to forced read-only
357 * either by ioctl (Balance only) or when remounted as read-write.
385 return ERR_PTR(-ENOMEM); in alloc_fs_devices()
387 mutex_init(&fs_devs->device_list_mutex); in alloc_fs_devices()
389 INIT_LIST_HEAD(&fs_devs->devices); in alloc_fs_devices()
390 INIT_LIST_HEAD(&fs_devs->alloc_list); in alloc_fs_devices()
391 INIT_LIST_HEAD(&fs_devs->fs_list); in alloc_fs_devices()
392 INIT_LIST_HEAD(&fs_devs->seed_list); in alloc_fs_devices()
395 memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE); in alloc_fs_devices()
396 memcpy(fs_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE); in alloc_fs_devices()
404 WARN_ON(!list_empty(&device->post_commit_list)); in btrfs_free_device()
405 rcu_string_free(device->name); in btrfs_free_device()
406 extent_io_tree_release(&device->alloc_state); in btrfs_free_device()
415 WARN_ON(fs_devices->opened); in free_fs_devices()
416 while (!list_empty(&fs_devices->devices)) { in free_fs_devices()
417 device = list_entry(fs_devices->devices.next, in free_fs_devices()
419 list_del(&device->dev_list); in free_fs_devices()
432 list_del(&fs_devices->fs_list); in btrfs_cleanup_fs_uuids()
440 if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) != 0) in match_fsid_fs_devices()
446 if (memcmp(metadata_fsid, fs_devices->metadata_uuid, BTRFS_FSID_SIZE) != 0) in match_fsid_fs_devices()
459 /* Handle non-split brain cases */ in find_fsid()
520 * -EBUSY if @devt is a mounted device.
521 * -ENOENT if @devt does not match any device in the list.
536 mutex_lock(&fs_devices->device_list_mutex); in btrfs_free_stale_devices()
538 &fs_devices->devices, dev_list) { in btrfs_free_stale_devices()
541 if (devt && devt != device->devt) in btrfs_free_stale_devices()
543 if (fs_devices->opened) { in btrfs_free_stale_devices()
545 ret = -EBUSY; in btrfs_free_stale_devices()
550 fs_devices->num_devices--; in btrfs_free_stale_devices()
551 list_del(&device->dev_list); in btrfs_free_stale_devices()
556 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_free_stale_devices()
558 if (fs_devices->num_devices == 0) { in btrfs_free_stale_devices()
560 list_del(&fs_devices->fs_list); in btrfs_free_stale_devices()
583 fsid_fs_devices = find_fsid(disk_super->fsid, in find_fsid_by_device()
584 has_metadata_uuid ? disk_super->metadata_uuid : NULL); in find_fsid_by_device()
592 * functions as a multi-device filesystem. So, temp-fsid feature is in find_fsid_by_device()
602 list_for_each_entry(device, &devt_fs_devices->devices, dev_list) { in find_fsid_by_device()
603 if (device->devt == devt) { in find_fsid_by_device()
615 if (devt_fs_devices->opened == 0) { in find_fsid_by_device()
643 * fs_devices->device_list_mutex here.
654 if (device->bdev) in btrfs_open_one_device()
655 return -EINVAL; in btrfs_open_one_device()
656 if (!device->name) in btrfs_open_one_device()
657 return -EINVAL; in btrfs_open_one_device()
659 ret = btrfs_get_bdev_and_sb(device->name->str, flags, holder, 1, in btrfs_open_one_device()
664 devid = btrfs_stack_device_id(&disk_super->dev_item); in btrfs_open_one_device()
665 if (devid != device->devid) in btrfs_open_one_device()
668 if (memcmp(device->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE)) in btrfs_open_one_device()
671 device->generation = btrfs_super_generation(disk_super); in btrfs_open_one_device()
677 "BTRFS: Invalid seeding and uuid-changed device detected\n"); in btrfs_open_one_device()
681 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); in btrfs_open_one_device()
682 fs_devices->seeding = true; in btrfs_open_one_device()
685 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); in btrfs_open_one_device()
687 set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); in btrfs_open_one_device()
691 fs_devices->rotating = true; in btrfs_open_one_device()
694 fs_devices->discardable = true; in btrfs_open_one_device()
696 device->bdev_file = bdev_file; in btrfs_open_one_device()
697 device->bdev = file_bdev(bdev_file); in btrfs_open_one_device()
698 clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); in btrfs_open_one_device()
700 if (device->devt != device->bdev->bd_dev) { in btrfs_open_one_device()
703 device->name->str, MAJOR(device->devt), in btrfs_open_one_device()
704 MINOR(device->devt), MAJOR(device->bdev->bd_dev), in btrfs_open_one_device()
705 MINOR(device->bdev->bd_dev)); in btrfs_open_one_device()
707 device->devt = device->bdev->bd_dev; in btrfs_open_one_device()
710 fs_devices->open_devices++; in btrfs_open_one_device()
711 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) && in btrfs_open_one_device()
712 device->devid != BTRFS_DEV_REPLACE_DEVID) { in btrfs_open_one_device()
713 fs_devices->rw_devices++; in btrfs_open_one_device()
714 list_add_tail(&device->dev_alloc_list, &fs_devices->alloc_list); in btrfs_open_one_device()
724 return -EINVAL; in btrfs_open_one_device()
732 return has_metadata_uuid ? sb->metadata_uuid : sb->fsid; in btrfs_sb_fsid_ptr()
750 u64 devid = btrfs_stack_device_id(&disk_super->dev_item); in device_list_add()
761 return ERR_PTR(-EAGAIN); in device_list_add()
774 fs_devices = alloc_fs_devices(disk_super->fsid); in device_list_add()
779 memcpy(fs_devices->metadata_uuid, in device_list_add()
780 disk_super->metadata_uuid, BTRFS_FSID_SIZE); in device_list_add()
783 generate_random_uuid(fs_devices->fsid); in device_list_add()
784 fs_devices->temp_fsid = true; in device_list_add()
785 pr_info("BTRFS: device %s (%d:%d) using temp-fsid %pU\n", in device_list_add()
787 fs_devices->fsid); in device_list_add()
790 mutex_lock(&fs_devices->device_list_mutex); in device_list_add()
791 list_add(&fs_devices->fs_list, &fs_uuids); in device_list_add()
797 .uuid = disk_super->dev_item.uuid, in device_list_add()
800 mutex_lock(&fs_devices->device_list_mutex); in device_list_add()
803 if (found_transid > fs_devices->latest_generation) { in device_list_add()
804 memcpy(fs_devices->fsid, disk_super->fsid, in device_list_add()
806 memcpy(fs_devices->metadata_uuid, in device_list_add()
814 if (fs_devices->opened) { in device_list_add()
818 fs_devices->fsid, current->comm, in device_list_add()
820 mutex_unlock(&fs_devices->device_list_mutex); in device_list_add()
821 return ERR_PTR(-EBUSY); in device_list_add()
826 disk_super->dev_item.uuid, path); in device_list_add()
829 mutex_unlock(&fs_devices->device_list_mutex); in device_list_add()
834 device->devt = path_devt; in device_list_add()
836 list_add_rcu(&device->dev_list, &fs_devices->devices); in device_list_add()
837 fs_devices->num_devices++; in device_list_add()
839 device->fs_devices = fs_devices; in device_list_add()
842 if (disk_super->label[0]) in device_list_add()
845 disk_super->label, devid, found_transid, path, in device_list_add()
847 current->comm, task_pid_nr(current)); in device_list_add()
851 disk_super->fsid, devid, found_transid, path, in device_list_add()
853 current->comm, task_pid_nr(current)); in device_list_add()
855 } else if (!device->name || strcmp(device->name->str, path)) { in device_list_add()
858 * 1. If you are here and if the device->name is NULL that in device_list_add()
860 * 2. If you are here and if the device->name is different in device_list_add()
864 * b. The missing-disk-which-was-replaced, has in device_list_add()
882 if (!fs_devices->opened && found_transid < device->generation) { in device_list_add()
887 * with larger generation number or the last-in if in device_list_add()
890 mutex_unlock(&fs_devices->device_list_mutex); in device_list_add()
893 path, found_transid, device->generation); in device_list_add()
894 return ERR_PTR(-EEXIST); in device_list_add()
901 * NOTE: the device->fs_info may not be reliable here so pass in device_list_add()
903 * use-after-free when the fs_info and fs_info->sb are already in device_list_add()
906 if (device->bdev) { in device_list_add()
907 if (device->devt != path_devt) { in device_list_add()
908 mutex_unlock(&fs_devices->device_list_mutex); in device_list_add()
912 current->comm, in device_list_add()
914 return ERR_PTR(-EEXIST); in device_list_add()
919 path, current->comm, in device_list_add()
925 mutex_unlock(&fs_devices->device_list_mutex); in device_list_add()
926 return ERR_PTR(-ENOMEM); in device_list_add()
928 rcu_string_free(device->name); in device_list_add()
929 rcu_assign_pointer(device->name, name); in device_list_add()
930 if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) { in device_list_add()
931 fs_devices->missing_devices--; in device_list_add()
932 clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state); in device_list_add()
934 device->devt = path_devt; in device_list_add()
943 if (!fs_devices->opened) { in device_list_add()
944 device->generation = found_transid; in device_list_add()
945 fs_devices->latest_generation = max_t(u64, found_transid, in device_list_add()
946 fs_devices->latest_generation); in device_list_add()
949 fs_devices->total_devices = btrfs_super_num_devices(disk_super); in device_list_add()
951 mutex_unlock(&fs_devices->device_list_mutex); in device_list_add()
964 fs_devices = alloc_fs_devices(orig->fsid); in clone_fs_devices()
968 fs_devices->total_devices = orig->total_devices; in clone_fs_devices()
970 list_for_each_entry(orig_dev, &orig->devices, dev_list) { in clone_fs_devices()
977 if (orig_dev->name) in clone_fs_devices()
978 dev_path = orig_dev->name->str; in clone_fs_devices()
980 device = btrfs_alloc_device(NULL, &orig_dev->devid, in clone_fs_devices()
981 orig_dev->uuid, dev_path); in clone_fs_devices()
987 if (orig_dev->zone_info) { in clone_fs_devices()
993 ret = -ENOMEM; in clone_fs_devices()
996 device->zone_info = zone_info; in clone_fs_devices()
999 list_add(&device->dev_list, &fs_devices->devices); in clone_fs_devices()
1000 device->fs_devices = fs_devices; in clone_fs_devices()
1001 fs_devices->num_devices++; in clone_fs_devices()
1015 list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) { in __btrfs_free_extra_devids()
1016 if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state)) { in __btrfs_free_extra_devids()
1018 &device->dev_state) && in __btrfs_free_extra_devids()
1020 &device->dev_state) && in __btrfs_free_extra_devids()
1022 device->generation > (*latest_dev)->generation)) { in __btrfs_free_extra_devids()
1032 if (device->devid == BTRFS_DEV_REPLACE_DEVID) in __btrfs_free_extra_devids()
1035 if (device->bdev_file) { in __btrfs_free_extra_devids()
1036 fput(device->bdev_file); in __btrfs_free_extra_devids()
1037 device->bdev = NULL; in __btrfs_free_extra_devids()
1038 device->bdev_file = NULL; in __btrfs_free_extra_devids()
1039 fs_devices->open_devices--; in __btrfs_free_extra_devids()
1041 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in __btrfs_free_extra_devids()
1042 list_del_init(&device->dev_alloc_list); in __btrfs_free_extra_devids()
1043 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); in __btrfs_free_extra_devids()
1044 fs_devices->rw_devices--; in __btrfs_free_extra_devids()
1046 list_del_init(&device->dev_list); in __btrfs_free_extra_devids()
1047 fs_devices->num_devices--; in __btrfs_free_extra_devids()
1054 * After we have read the system tree and know devids belonging to this
1065 list_for_each_entry(seed_dev, &fs_devices->seed_list, seed_list) in btrfs_free_extra_devids()
1068 fs_devices->latest_dev = latest_dev; in btrfs_free_extra_devids()
1075 if (!device->bdev) in btrfs_close_bdev()
1078 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in btrfs_close_bdev()
1079 sync_blockdev(device->bdev); in btrfs_close_bdev()
1080 invalidate_bdev(device->bdev); in btrfs_close_bdev()
1083 fput(device->bdev_file); in btrfs_close_bdev()
1088 struct btrfs_fs_devices *fs_devices = device->fs_devices; in btrfs_close_one_device()
1090 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) && in btrfs_close_one_device()
1091 device->devid != BTRFS_DEV_REPLACE_DEVID) { in btrfs_close_one_device()
1092 list_del_init(&device->dev_alloc_list); in btrfs_close_one_device()
1093 fs_devices->rw_devices--; in btrfs_close_one_device()
1096 if (device->devid == BTRFS_DEV_REPLACE_DEVID) in btrfs_close_one_device()
1097 clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); in btrfs_close_one_device()
1099 if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) { in btrfs_close_one_device()
1100 clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state); in btrfs_close_one_device()
1101 fs_devices->missing_devices--; in btrfs_close_one_device()
1105 if (device->bdev) { in btrfs_close_one_device()
1106 fs_devices->open_devices--; in btrfs_close_one_device()
1107 device->bdev = NULL; in btrfs_close_one_device()
1108 device->bdev_file = NULL; in btrfs_close_one_device()
1110 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); in btrfs_close_one_device()
1113 device->fs_info = NULL; in btrfs_close_one_device()
1114 atomic_set(&device->dev_stats_ccnt, 0); in btrfs_close_one_device()
1115 extent_io_tree_release(&device->alloc_state); in btrfs_close_one_device()
1123 * (btrfs_check_rw_degradable() should not fail) - if after mounting the in btrfs_close_one_device()
1128 device->last_flush_error = 0; in btrfs_close_one_device()
1131 WARN_ON(test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state)); in btrfs_close_one_device()
1132 WARN_ON(test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)); in btrfs_close_one_device()
1133 WARN_ON(!list_empty(&device->dev_alloc_list)); in btrfs_close_one_device()
1134 WARN_ON(!list_empty(&device->post_commit_list)); in btrfs_close_one_device()
1143 if (--fs_devices->opened > 0) in close_fs_devices()
1146 list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list) in close_fs_devices()
1149 WARN_ON(fs_devices->open_devices); in close_fs_devices()
1150 WARN_ON(fs_devices->rw_devices); in close_fs_devices()
1151 fs_devices->opened = 0; in close_fs_devices()
1152 fs_devices->seeding = false; in close_fs_devices()
1153 fs_devices->fs_info = NULL; in close_fs_devices()
1163 if (!fs_devices->opened) { in btrfs_close_devices()
1164 list_splice_init(&fs_devices->seed_list, &list); in btrfs_close_devices()
1168 * other device, it can be re-initialized during the next mount in btrfs_close_devices()
1169 * without the needing device-scan step. Therefore, it can be in btrfs_close_devices()
1172 if (fs_devices->num_devices == 1) { in btrfs_close_devices()
1173 list_del(&fs_devices->fs_list); in btrfs_close_devices()
1181 list_del(&fs_devices->seed_list); in btrfs_close_devices()
1195 list_for_each_entry_safe(device, tmp_device, &fs_devices->devices, in open_fs_devices()
1201 (!latest_dev || device->generation > latest_dev->generation)) { in open_fs_devices()
1203 } else if (ret2 == -ENODATA) { in open_fs_devices()
1204 fs_devices->num_devices--; in open_fs_devices()
1205 list_del(&device->dev_list); in open_fs_devices()
1212 if (fs_devices->open_devices == 0) { in open_fs_devices()
1215 return -EINVAL; in open_fs_devices()
1218 fs_devices->opened = 1; in open_fs_devices()
1219 fs_devices->latest_dev = latest_dev; in open_fs_devices()
1220 fs_devices->total_rw_bytes = 0; in open_fs_devices()
1221 fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_REGULAR; in open_fs_devices()
1222 fs_devices->read_policy = BTRFS_READ_POLICY_PID; in open_fs_devices()
1235 if (dev1->devid < dev2->devid) in devid_cmp()
1236 return -1; in devid_cmp()
1237 else if (dev1->devid > dev2->devid) in devid_cmp()
1256 if (fs_devices->opened) { in btrfs_open_devices()
1257 fs_devices->opened++; in btrfs_open_devices()
1260 list_sort(NULL, &fs_devices->devices, devid_cmp); in btrfs_open_devices()
1284 return ERR_PTR(-EINVAL); in btrfs_read_disk_super()
1288 return ERR_PTR(-EINVAL); in btrfs_read_disk_super()
1292 if ((bytenr + sizeof(*disk_super) - 1) >> PAGE_SHIFT != index) in btrfs_read_disk_super()
1293 return ERR_PTR(-EINVAL); in btrfs_read_disk_super()
1296 page = read_cache_page_gfp(bdev->bd_mapping, index, GFP_KERNEL); in btrfs_read_disk_super()
1309 return ERR_PTR(-EINVAL); in btrfs_read_disk_super()
1312 if (disk_super->label[0] && disk_super->label[BTRFS_LABEL_SIZE - 1]) in btrfs_read_disk_super()
1313 disk_super->label[BTRFS_LABEL_SIZE - 1] = 0; in btrfs_read_disk_super()
1339 * A successful scan ensures grub2-probe selects the correct device. in btrfs_skip_registration()
1344 mutex_lock(&fs_devices->device_list_mutex); in btrfs_skip_registration()
1346 if (!fs_devices->opened) { in btrfs_skip_registration()
1347 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_skip_registration()
1351 list_for_each_entry(device, &fs_devices->devices, dev_list) { in btrfs_skip_registration()
1352 if (device->bdev && (device->bdev->bd_dev == devt) && in btrfs_skip_registration()
1353 strcmp(device->name->str, path) != 0) { in btrfs_skip_registration()
1354 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_skip_registration()
1360 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_skip_registration()
1376 * the device or return an error. Multi-device and seeding devices are registered
1393 * Avoid an exclusive open here, as the systemd-udev may initiate the in btrfs_scan_one_device()
1410 * using the userspace command 'btrfs check --super'. in btrfs_scan_one_device()
1425 devt = file_bdev(bdev_file)->bd_dev; in btrfs_scan_one_device()
1427 pr_debug("BTRFS: skip registering single non-seed device %s (%d:%d)\n", in btrfs_scan_one_device()
1438 btrfs_free_stale_devices(device->devt, device); in btrfs_scan_one_device()
1458 lockdep_assert_held(&device->fs_info->chunk_mutex); in contains_pending_extent()
1460 if (find_first_extent_bit(&device->alloc_state, *start, in contains_pending_extent()
1466 physical_end + 1 - physical_start)) { in contains_pending_extent()
1476 switch (device->fs_devices->chunk_alloc_policy) { in dev_extent_search_start()
1495 u64 zone_size = device->zone_info->zone_size; in dev_extent_hole_check_zoned()
1507 *hole_size = *hole_start + *hole_size - pos; in dev_extent_hole_check_zoned()
1521 if (ret == -ERANGE) { in dev_extent_hole_check_zoned()
1528 *hole_size -= zone_size; in dev_extent_hole_check_zoned()
1559 *hole_size = hole_end - *hole_start; in dev_extent_hole_check()
1565 switch (device->fs_devices->chunk_alloc_policy) { in dev_extent_hole_check()
1620 struct btrfs_fs_info *fs_info = device->fs_info; in find_free_dev_extent()
1621 struct btrfs_root *root = fs_info->dev_root; in find_free_dev_extent()
1630 u64 search_end = device->total_bytes; in find_free_dev_extent()
1638 WARN_ON(device->zone_info && in find_free_dev_extent()
1639 !IS_ALIGNED(num_bytes, device->zone_info->zone_size)); in find_free_dev_extent()
1643 ret = -ENOMEM; in find_free_dev_extent()
1648 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { in find_free_dev_extent()
1649 ret = -ENOSPC; in find_free_dev_extent()
1653 path->reada = READA_FORWARD; in find_free_dev_extent()
1654 path->search_commit_root = 1; in find_free_dev_extent()
1655 path->skip_locking = 1; in find_free_dev_extent()
1657 key.objectid = device->devid; in find_free_dev_extent()
1666 l = path->nodes[0]; in find_free_dev_extent()
1667 slot = path->slots[0]; in find_free_dev_extent()
1679 if (key.objectid < device->devid) in find_free_dev_extent()
1682 if (key.objectid > device->devid) in find_free_dev_extent()
1692 hole_size = key.offset - search_start; in find_free_dev_extent()
1722 path->slots[0]++; in find_free_dev_extent()
1732 hole_size = search_end - search_start; in find_free_dev_extent()
1747 ret = -ENOSPC; in find_free_dev_extent()
1764 struct btrfs_fs_info *fs_info = device->fs_info; in btrfs_free_dev_extent()
1765 struct btrfs_root *root = fs_info->dev_root; in btrfs_free_dev_extent()
1775 return -ENOMEM; in btrfs_free_dev_extent()
1777 key.objectid = device->devid; in btrfs_free_dev_extent()
1781 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); in btrfs_free_dev_extent()
1787 leaf = path->nodes[0]; in btrfs_free_dev_extent()
1788 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); in btrfs_free_dev_extent()
1789 extent = btrfs_item_ptr(leaf, path->slots[0], in btrfs_free_dev_extent()
1797 leaf = path->nodes[0]; in btrfs_free_dev_extent()
1798 extent = btrfs_item_ptr(leaf, path->slots[0], in btrfs_free_dev_extent()
1808 set_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags); in btrfs_free_dev_extent()
1819 read_lock(&fs_info->mapping_tree_lock); in find_next_chunk()
1820 n = rb_last(&fs_info->mapping_tree.rb_root); in find_next_chunk()
1825 ret = map->start + map->chunk_len; in find_next_chunk()
1827 read_unlock(&fs_info->mapping_tree_lock); in find_next_chunk()
1842 return -ENOMEM; in find_next_devid()
1846 key.offset = (u64)-1; in find_next_devid()
1848 ret = btrfs_search_slot(NULL, fs_info->chunk_root, &key, path, 0, 0); in find_next_devid()
1854 btrfs_err(fs_info, "corrupted chunk tree devid -1 matched"); in find_next_devid()
1855 ret = -EUCLEAN; in find_next_devid()
1859 ret = btrfs_previous_item(fs_info->chunk_root, path, in find_next_devid()
1865 btrfs_item_key_to_cpu(path->nodes[0], &found_key, in find_next_devid()
1866 path->slots[0]); in find_next_devid()
1891 return -ENOMEM; in btrfs_add_dev_item()
1895 key.offset = device->devid; in btrfs_add_dev_item()
1898 ret = btrfs_insert_empty_item(trans, trans->fs_info->chunk_root, path, in btrfs_add_dev_item()
1904 leaf = path->nodes[0]; in btrfs_add_dev_item()
1905 dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item); in btrfs_add_dev_item()
1907 btrfs_set_device_id(leaf, dev_item, device->devid); in btrfs_add_dev_item()
1909 btrfs_set_device_type(leaf, dev_item, device->type); in btrfs_add_dev_item()
1910 btrfs_set_device_io_align(leaf, dev_item, device->io_align); in btrfs_add_dev_item()
1911 btrfs_set_device_io_width(leaf, dev_item, device->io_width); in btrfs_add_dev_item()
1912 btrfs_set_device_sector_size(leaf, dev_item, device->sector_size); in btrfs_add_dev_item()
1923 write_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE); in btrfs_add_dev_item()
1925 write_extent_buffer(leaf, trans->fs_info->fs_devices->metadata_uuid, in btrfs_add_dev_item()
1957 struct btrfs_root *root = device->fs_info->chunk_root; in btrfs_rm_dev_item()
1964 return -ENOMEM; in btrfs_rm_dev_item()
1968 key.offset = device->devid; in btrfs_rm_dev_item()
1971 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); in btrfs_rm_dev_item()
1975 ret = -ENOENT; in btrfs_rm_dev_item()
1998 seq = read_seqbegin(&fs_info->profiles_lock); in btrfs_check_raid_min_devices()
2000 all_avail = fs_info->avail_data_alloc_bits | in btrfs_check_raid_min_devices()
2001 fs_info->avail_system_alloc_bits | in btrfs_check_raid_min_devices()
2002 fs_info->avail_metadata_alloc_bits; in btrfs_check_raid_min_devices()
2003 } while (read_seqretry(&fs_info->profiles_lock, seq)); in btrfs_check_raid_min_devices()
2021 list_for_each_entry(next_device, &fs_devs->devices, dev_list) { in btrfs_find_next_active_device()
2023 !test_bit(BTRFS_DEV_STATE_MISSING, &next_device->dev_state) in btrfs_find_next_active_device()
2024 && next_device->bdev) in btrfs_find_next_active_device()
2040 struct btrfs_fs_info *fs_info = device->fs_info; in btrfs_assign_next_active_device()
2043 next_device = btrfs_find_next_active_device(fs_info->fs_devices, in btrfs_assign_next_active_device()
2047 if (fs_info->sb->s_bdev && in btrfs_assign_next_active_device()
2048 (fs_info->sb->s_bdev == device->bdev)) in btrfs_assign_next_active_device()
2049 fs_info->sb->s_bdev = next_device->bdev; in btrfs_assign_next_active_device()
2051 if (fs_info->fs_devices->latest_dev->bdev == device->bdev) in btrfs_assign_next_active_device()
2052 fs_info->fs_devices->latest_dev = next_device; in btrfs_assign_next_active_device()
2061 u64 num_devices = fs_info->fs_devices->num_devices; in btrfs_num_devices()
2063 down_read(&fs_info->dev_replace.rwsem); in btrfs_num_devices()
2064 if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace)) { in btrfs_num_devices()
2066 num_devices--; in btrfs_num_devices()
2068 up_read(&fs_info->dev_replace.rwsem); in btrfs_num_devices()
2077 const size_t len = sizeof(disk_super->magic); in btrfs_scratch_superblock()
2085 memset(&disk_super->magic, 0, len); in btrfs_scratch_superblock()
2089 ret = sync_blockdev_range(bdev, bytenr, bytenr + len - 1); in btrfs_scratch_superblock()
2098 struct block_device *bdev = device->bdev; in btrfs_scratch_superblocks()
2114 update_dev_time(device->name->str); in btrfs_scratch_superblocks()
2124 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_rm_device()
2130 return -EINVAL; in btrfs_rm_device()
2140 ret = btrfs_check_raid_min_devices(fs_info, num_devices - 1); in btrfs_rm_device()
2144 device = btrfs_find_device(fs_info->fs_devices, args); in btrfs_rm_device()
2146 if (args->missing) in btrfs_rm_device()
2149 ret = -ENOENT; in btrfs_rm_device()
2156 btrfs_dev_name(device), device->devid); in btrfs_rm_device()
2157 return -ETXTBSY; in btrfs_rm_device()
2160 if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) in btrfs_rm_device()
2163 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) && in btrfs_rm_device()
2164 fs_info->fs_devices->rw_devices == 1) in btrfs_rm_device()
2167 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in btrfs_rm_device()
2168 mutex_lock(&fs_info->chunk_mutex); in btrfs_rm_device()
2169 list_del_init(&device->dev_alloc_list); in btrfs_rm_device()
2170 device->fs_devices->rw_devices--; in btrfs_rm_device()
2171 mutex_unlock(&fs_info->chunk_mutex); in btrfs_rm_device()
2178 trans = btrfs_start_transaction(fs_info->chunk_root, 0); in btrfs_rm_device()
2189 device->devid, ret); in btrfs_rm_device()
2195 clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); in btrfs_rm_device()
2211 * its own fs_devices listed under the fs_devices->seed_list. in btrfs_rm_device()
2213 cur_devices = device->fs_devices; in btrfs_rm_device()
2214 mutex_lock(&fs_devices->device_list_mutex); in btrfs_rm_device()
2215 list_del_rcu(&device->dev_list); in btrfs_rm_device()
2217 cur_devices->num_devices--; in btrfs_rm_device()
2218 cur_devices->total_devices--; in btrfs_rm_device()
2221 fs_devices->total_devices--; in btrfs_rm_device()
2223 if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) in btrfs_rm_device()
2224 cur_devices->missing_devices--; in btrfs_rm_device()
2228 if (device->bdev_file) { in btrfs_rm_device()
2229 cur_devices->open_devices--; in btrfs_rm_device()
2234 num_devices = btrfs_super_num_devices(fs_info->super_copy) - 1; in btrfs_rm_device()
2235 btrfs_set_super_num_devices(fs_info->super_copy, num_devices); in btrfs_rm_device()
2236 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_rm_device()
2245 * ->open_mutex on the block device and it's dependencies. Instead in btrfs_rm_device()
2248 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in btrfs_rm_device()
2250 if (device->bdev) { in btrfs_rm_device()
2251 sync_blockdev(device->bdev); in btrfs_rm_device()
2252 invalidate_bdev(device->bdev); in btrfs_rm_device()
2256 *bdev_file = device->bdev_file; in btrfs_rm_device()
2264 * seed_devices, we can simply decrement cur_devices->opened and then in btrfs_rm_device()
2267 if (cur_devices->num_devices == 0) { in btrfs_rm_device()
2268 list_del_init(&cur_devices->seed_list); in btrfs_rm_device()
2269 ASSERT(cur_devices->opened == 1); in btrfs_rm_device()
2270 cur_devices->opened--; in btrfs_rm_device()
2279 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in btrfs_rm_device()
2280 mutex_lock(&fs_info->chunk_mutex); in btrfs_rm_device()
2281 list_add(&device->dev_alloc_list, in btrfs_rm_device()
2282 &fs_devices->alloc_list); in btrfs_rm_device()
2283 device->fs_devices->rw_devices++; in btrfs_rm_device()
2284 mutex_unlock(&fs_info->chunk_mutex); in btrfs_rm_device()
2293 lockdep_assert_held(&srcdev->fs_info->fs_devices->device_list_mutex); in btrfs_rm_dev_replace_remove_srcdev()
2296 * in case of fs with no seed, srcdev->fs_devices will point in btrfs_rm_dev_replace_remove_srcdev()
2301 fs_devices = srcdev->fs_devices; in btrfs_rm_dev_replace_remove_srcdev()
2303 list_del_rcu(&srcdev->dev_list); in btrfs_rm_dev_replace_remove_srcdev()
2304 list_del(&srcdev->dev_alloc_list); in btrfs_rm_dev_replace_remove_srcdev()
2305 fs_devices->num_devices--; in btrfs_rm_dev_replace_remove_srcdev()
2306 if (test_bit(BTRFS_DEV_STATE_MISSING, &srcdev->dev_state)) in btrfs_rm_dev_replace_remove_srcdev()
2307 fs_devices->missing_devices--; in btrfs_rm_dev_replace_remove_srcdev()
2309 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state)) in btrfs_rm_dev_replace_remove_srcdev()
2310 fs_devices->rw_devices--; in btrfs_rm_dev_replace_remove_srcdev()
2312 if (srcdev->bdev) in btrfs_rm_dev_replace_remove_srcdev()
2313 fs_devices->open_devices--; in btrfs_rm_dev_replace_remove_srcdev()
2318 struct btrfs_fs_devices *fs_devices = srcdev->fs_devices; in btrfs_rm_dev_replace_free_srcdev()
2327 if (!fs_devices->num_devices) { in btrfs_rm_dev_replace_free_srcdev()
2334 ASSERT(fs_devices->seeding); in btrfs_rm_dev_replace_free_srcdev()
2336 list_del_init(&fs_devices->seed_list); in btrfs_rm_dev_replace_free_srcdev()
2345 struct btrfs_fs_devices *fs_devices = tgtdev->fs_info->fs_devices; in btrfs_destroy_dev_replace_tgtdev()
2347 mutex_lock(&fs_devices->device_list_mutex); in btrfs_destroy_dev_replace_tgtdev()
2351 if (tgtdev->bdev) in btrfs_destroy_dev_replace_tgtdev()
2352 fs_devices->open_devices--; in btrfs_destroy_dev_replace_tgtdev()
2354 fs_devices->num_devices--; in btrfs_destroy_dev_replace_tgtdev()
2358 list_del_rcu(&tgtdev->dev_list); in btrfs_destroy_dev_replace_tgtdev()
2360 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_destroy_dev_replace_tgtdev()
2362 btrfs_scratch_superblocks(tgtdev->fs_info, tgtdev); in btrfs_destroy_dev_replace_tgtdev()
2385 * Return: 0 for success, -errno for failure
2396 return -EINVAL; in btrfs_get_dev_args_from_path()
2398 args->missing = true; in btrfs_get_dev_args_from_path()
2402 args->uuid = kzalloc(BTRFS_UUID_SIZE, GFP_KERNEL); in btrfs_get_dev_args_from_path()
2403 args->fsid = kzalloc(BTRFS_FSID_SIZE, GFP_KERNEL); in btrfs_get_dev_args_from_path()
2404 if (!args->uuid || !args->fsid) { in btrfs_get_dev_args_from_path()
2406 return -ENOMEM; in btrfs_get_dev_args_from_path()
2416 args->devid = btrfs_stack_device_id(&disk_super->dev_item); in btrfs_get_dev_args_from_path()
2417 memcpy(args->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE); in btrfs_get_dev_args_from_path()
2419 memcpy(args->fsid, disk_super->metadata_uuid, BTRFS_FSID_SIZE); in btrfs_get_dev_args_from_path()
2421 memcpy(args->fsid, disk_super->fsid, BTRFS_FSID_SIZE); in btrfs_get_dev_args_from_path()
2429 * allocate our ->uuid and ->fsid pointers, everybody else uses local variables
2434 kfree(args->uuid); in btrfs_put_dev_args_from_path()
2435 kfree(args->fsid); in btrfs_put_dev_args_from_path()
2436 args->uuid = NULL; in btrfs_put_dev_args_from_path()
2437 args->fsid = NULL; in btrfs_put_dev_args_from_path()
2450 device = btrfs_find_device(fs_info->fs_devices, &args); in btrfs_find_device_by_devspec()
2452 return ERR_PTR(-ENOENT); in btrfs_find_device_by_devspec()
2459 device = btrfs_find_device(fs_info->fs_devices, &args); in btrfs_find_device_by_devspec()
2462 return ERR_PTR(-ENOENT); in btrfs_find_device_by_devspec()
2468 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_init_sprout()
2473 if (!fs_devices->seeding) in btrfs_init_sprout()
2474 return ERR_PTR(-EINVAL); in btrfs_init_sprout()
2478 * fs_info->fs_devices->seed_list in btrfs_init_sprout()
2496 list_add(&old_devices->fs_list, &fs_uuids); in btrfs_init_sprout()
2499 seed_devices->opened = 1; in btrfs_init_sprout()
2500 INIT_LIST_HEAD(&seed_devices->devices); in btrfs_init_sprout()
2501 INIT_LIST_HEAD(&seed_devices->alloc_list); in btrfs_init_sprout()
2502 mutex_init(&seed_devices->device_list_mutex); in btrfs_init_sprout()
2509 * Generate a new fsid for the sprouted read-write filesystem.
2514 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_setup_sprout()
2515 struct btrfs_super_block *disk_super = fs_info->super_copy; in btrfs_setup_sprout()
2528 * - All device ops and balance - as we are in btrfs_exclop_start. in btrfs_setup_sprout()
2529 * - Various dev_list readers - are using RCU. in btrfs_setup_sprout()
2530 * - btrfs_ioctl_fitrim() - is using RCU. in btrfs_setup_sprout()
2532 * For-read threads as below are using device_list_mutex: in btrfs_setup_sprout()
2533 * - Readonly scrub btrfs_scrub_dev() in btrfs_setup_sprout()
2534 * - Readonly scrub btrfs_scrub_progress() in btrfs_setup_sprout()
2535 * - btrfs_get_dev_stats() in btrfs_setup_sprout()
2537 lockdep_assert_held(&fs_devices->device_list_mutex); in btrfs_setup_sprout()
2539 list_splice_init_rcu(&fs_devices->devices, &seed_devices->devices, in btrfs_setup_sprout()
2541 list_for_each_entry(device, &seed_devices->devices, dev_list) in btrfs_setup_sprout()
2542 device->fs_devices = seed_devices; in btrfs_setup_sprout()
2544 fs_devices->seeding = false; in btrfs_setup_sprout()
2545 fs_devices->num_devices = 0; in btrfs_setup_sprout()
2546 fs_devices->open_devices = 0; in btrfs_setup_sprout()
2547 fs_devices->missing_devices = 0; in btrfs_setup_sprout()
2548 fs_devices->rotating = false; in btrfs_setup_sprout()
2549 list_add(&seed_devices->seed_list, &fs_devices->seed_list); in btrfs_setup_sprout()
2551 generate_random_uuid(fs_devices->fsid); in btrfs_setup_sprout()
2552 memcpy(fs_devices->metadata_uuid, fs_devices->fsid, BTRFS_FSID_SIZE); in btrfs_setup_sprout()
2553 memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE); in btrfs_setup_sprout()
2566 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_finish_sprout()
2567 struct btrfs_root *root = fs_info->chunk_root; in btrfs_finish_sprout()
2579 return -ENOMEM; in btrfs_finish_sprout()
2592 leaf = path->nodes[0]; in btrfs_finish_sprout()
2594 if (path->slots[0] >= btrfs_header_nritems(leaf)) { in btrfs_finish_sprout()
2600 leaf = path->nodes[0]; in btrfs_finish_sprout()
2601 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); in btrfs_finish_sprout()
2606 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); in btrfs_finish_sprout()
2611 dev_item = btrfs_item_ptr(leaf, path->slots[0], in btrfs_finish_sprout()
2620 device = btrfs_find_device(fs_info->fs_devices, &args); in btrfs_finish_sprout()
2623 if (device->fs_devices->seeding) { in btrfs_finish_sprout()
2625 device->generation); in btrfs_finish_sprout()
2629 path->slots[0]++; in btrfs_finish_sprout()
2640 struct btrfs_root *root = fs_info->dev_root; in btrfs_init_new_device()
2644 struct super_block *sb = fs_info->sb; in btrfs_init_new_device()
2645 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_init_new_device()
2653 if (sb_rdonly(sb) && !fs_devices->seeding) in btrfs_init_new_device()
2654 return -EROFS; in btrfs_init_new_device()
2657 fs_info->bdev_holder, NULL); in btrfs_init_new_device()
2662 ret = -EINVAL; in btrfs_init_new_device()
2666 if (fs_devices->seeding) { in btrfs_init_new_device()
2668 down_write(&sb->s_umount); in btrfs_init_new_device()
2676 list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) { in btrfs_init_new_device()
2677 if (device->bdev == file_bdev(bdev_file)) { in btrfs_init_new_device()
2678 ret = -EEXIST; in btrfs_init_new_device()
2692 device->fs_info = fs_info; in btrfs_init_new_device()
2693 device->bdev_file = bdev_file; in btrfs_init_new_device()
2694 device->bdev = file_bdev(bdev_file); in btrfs_init_new_device()
2695 ret = lookup_bdev(device_path, &device->devt); in btrfs_init_new_device()
2709 set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state); in btrfs_init_new_device()
2710 device->generation = trans->transid; in btrfs_init_new_device()
2711 device->io_width = fs_info->sectorsize; in btrfs_init_new_device()
2712 device->io_align = fs_info->sectorsize; in btrfs_init_new_device()
2713 device->sector_size = fs_info->sectorsize; in btrfs_init_new_device()
2714 device->total_bytes = in btrfs_init_new_device()
2715 round_down(bdev_nr_bytes(device->bdev), fs_info->sectorsize); in btrfs_init_new_device()
2716 device->disk_total_bytes = device->total_bytes; in btrfs_init_new_device()
2717 device->commit_total_bytes = device->total_bytes; in btrfs_init_new_device()
2718 set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); in btrfs_init_new_device()
2719 clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); in btrfs_init_new_device()
2720 device->dev_stats_valid = 1; in btrfs_init_new_device()
2721 set_blocksize(device->bdev_file, BTRFS_BDEV_BLOCKSIZE); in btrfs_init_new_device()
2735 mutex_lock(&fs_devices->device_list_mutex); in btrfs_init_new_device()
2738 btrfs_assign_next_active_device(fs_info->fs_devices->latest_dev, in btrfs_init_new_device()
2742 device->fs_devices = fs_devices; in btrfs_init_new_device()
2744 mutex_lock(&fs_info->chunk_mutex); in btrfs_init_new_device()
2745 list_add_rcu(&device->dev_list, &fs_devices->devices); in btrfs_init_new_device()
2746 list_add(&device->dev_alloc_list, &fs_devices->alloc_list); in btrfs_init_new_device()
2747 fs_devices->num_devices++; in btrfs_init_new_device()
2748 fs_devices->open_devices++; in btrfs_init_new_device()
2749 fs_devices->rw_devices++; in btrfs_init_new_device()
2750 fs_devices->total_devices++; in btrfs_init_new_device()
2751 fs_devices->total_rw_bytes += device->total_bytes; in btrfs_init_new_device()
2753 atomic64_add(device->total_bytes, &fs_info->free_chunk_space); in btrfs_init_new_device()
2755 if (!bdev_nonrot(device->bdev)) in btrfs_init_new_device()
2756 fs_devices->rotating = true; in btrfs_init_new_device()
2758 orig_super_total_bytes = btrfs_super_total_bytes(fs_info->super_copy); in btrfs_init_new_device()
2759 btrfs_set_super_total_bytes(fs_info->super_copy, in btrfs_init_new_device()
2760 round_down(orig_super_total_bytes + device->total_bytes, in btrfs_init_new_device()
2761 fs_info->sectorsize)); in btrfs_init_new_device()
2763 orig_super_num_devices = btrfs_super_num_devices(fs_info->super_copy); in btrfs_init_new_device()
2764 btrfs_set_super_num_devices(fs_info->super_copy, in btrfs_init_new_device()
2773 mutex_unlock(&fs_info->chunk_mutex); in btrfs_init_new_device()
2778 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_init_new_device()
2781 mutex_lock(&fs_info->chunk_mutex); in btrfs_init_new_device()
2783 mutex_unlock(&fs_info->chunk_mutex); in btrfs_init_new_device()
2814 up_write(&sb->s_umount); in btrfs_init_new_device()
2826 if (PTR_ERR(trans) == -ENOENT) in btrfs_init_new_device()
2839 * We can ignore the return value as it typically returns -EINVAL and in btrfs_init_new_device()
2842 btrfs_forget_devices(device->devt); in btrfs_init_new_device()
2851 mutex_lock(&fs_info->fs_devices->device_list_mutex); in btrfs_init_new_device()
2852 mutex_lock(&fs_info->chunk_mutex); in btrfs_init_new_device()
2853 list_del_rcu(&device->dev_list); in btrfs_init_new_device()
2854 list_del(&device->dev_alloc_list); in btrfs_init_new_device()
2855 fs_info->fs_devices->num_devices--; in btrfs_init_new_device()
2856 fs_info->fs_devices->open_devices--; in btrfs_init_new_device()
2857 fs_info->fs_devices->rw_devices--; in btrfs_init_new_device()
2858 fs_info->fs_devices->total_devices--; in btrfs_init_new_device()
2859 fs_info->fs_devices->total_rw_bytes -= device->total_bytes; in btrfs_init_new_device()
2860 atomic64_sub(device->total_bytes, &fs_info->free_chunk_space); in btrfs_init_new_device()
2861 btrfs_set_super_total_bytes(fs_info->super_copy, in btrfs_init_new_device()
2863 btrfs_set_super_num_devices(fs_info->super_copy, in btrfs_init_new_device()
2865 mutex_unlock(&fs_info->chunk_mutex); in btrfs_init_new_device()
2866 mutex_unlock(&fs_info->fs_devices->device_list_mutex); in btrfs_init_new_device()
2880 up_write(&sb->s_umount); in btrfs_init_new_device()
2890 struct btrfs_root *root = device->fs_info->chunk_root; in btrfs_update_device()
2897 return -ENOMEM; in btrfs_update_device()
2901 key.offset = device->devid; in btrfs_update_device()
2908 ret = -ENOENT; in btrfs_update_device()
2912 leaf = path->nodes[0]; in btrfs_update_device()
2913 dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item); in btrfs_update_device()
2915 btrfs_set_device_id(leaf, dev_item, device->devid); in btrfs_update_device()
2916 btrfs_set_device_type(leaf, dev_item, device->type); in btrfs_update_device()
2917 btrfs_set_device_io_align(leaf, dev_item, device->io_align); in btrfs_update_device()
2918 btrfs_set_device_io_width(leaf, dev_item, device->io_width); in btrfs_update_device()
2919 btrfs_set_device_sector_size(leaf, dev_item, device->sector_size); in btrfs_update_device()
2934 struct btrfs_fs_info *fs_info = device->fs_info; in btrfs_grow_device()
2935 struct btrfs_super_block *super_copy = fs_info->super_copy; in btrfs_grow_device()
2940 if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) in btrfs_grow_device()
2941 return -EACCES; in btrfs_grow_device()
2943 new_size = round_down(new_size, fs_info->sectorsize); in btrfs_grow_device()
2945 mutex_lock(&fs_info->chunk_mutex); in btrfs_grow_device()
2947 diff = round_down(new_size - device->total_bytes, fs_info->sectorsize); in btrfs_grow_device()
2949 if (new_size <= device->total_bytes || in btrfs_grow_device()
2950 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { in btrfs_grow_device()
2951 mutex_unlock(&fs_info->chunk_mutex); in btrfs_grow_device()
2952 return -EINVAL; in btrfs_grow_device()
2956 round_down(old_total + diff, fs_info->sectorsize)); in btrfs_grow_device()
2957 device->fs_devices->total_rw_bytes += diff; in btrfs_grow_device()
2958 atomic64_add(diff, &fs_info->free_chunk_space); in btrfs_grow_device()
2962 btrfs_clear_space_info_full(device->fs_info); in btrfs_grow_device()
2963 if (list_empty(&device->post_commit_list)) in btrfs_grow_device()
2964 list_add_tail(&device->post_commit_list, in btrfs_grow_device()
2965 &trans->transaction->dev_update_list); in btrfs_grow_device()
2966 mutex_unlock(&fs_info->chunk_mutex); in btrfs_grow_device()
2977 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_free_chunk()
2978 struct btrfs_root *root = fs_info->chunk_root; in btrfs_free_chunk()
2985 return -ENOMEM; in btrfs_free_chunk()
2991 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); in btrfs_free_chunk()
2997 btrfs_abort_transaction(trans, -ENOENT); in btrfs_free_chunk()
2998 ret = -EUCLEAN; in btrfs_free_chunk()
3015 struct btrfs_super_block *super_copy = fs_info->super_copy; in btrfs_del_sys_chunk()
3026 lockdep_assert_held(&fs_info->chunk_mutex); in btrfs_del_sys_chunk()
3029 ptr = super_copy->sys_chunk_array; in btrfs_del_sys_chunk()
3043 ret = -EIO; in btrfs_del_sys_chunk()
3048 memmove(ptr, ptr + len, array_size - (cur + len)); in btrfs_del_sys_chunk()
3049 array_size -= len; in btrfs_del_sys_chunk()
3062 struct rb_node *node = fs_info->mapping_tree.rb_root.rb_node; in btrfs_find_chunk_map_nolock()
3073 if (logical < map->start) { in btrfs_find_chunk_map_nolock()
3074 node = node->rb_left; in btrfs_find_chunk_map_nolock()
3075 } else if (logical >= map->start + map->chunk_len) { in btrfs_find_chunk_map_nolock()
3076 node = node->rb_right; in btrfs_find_chunk_map_nolock()
3078 refcount_inc(&map->refs); in btrfs_find_chunk_map_nolock()
3087 while (prev && logical >= prev_map->start + prev_map->chunk_len) { in btrfs_find_chunk_map_nolock()
3095 while (prev && logical < prev_map->start) { in btrfs_find_chunk_map_nolock()
3112 if (end > prev_map->start && in btrfs_find_chunk_map_nolock()
3113 logical < prev_map->start + prev_map->chunk_len) { in btrfs_find_chunk_map_nolock()
3114 refcount_inc(&prev_map->refs); in btrfs_find_chunk_map_nolock()
3127 read_lock(&fs_info->mapping_tree_lock); in btrfs_find_chunk_map()
3129 read_unlock(&fs_info->mapping_tree_lock); in btrfs_find_chunk_map()
3153 return ERR_PTR(-EINVAL); in btrfs_get_chunk_map()
3156 if (unlikely(map->start > logical || map->start + map->chunk_len <= logical)) { in btrfs_get_chunk_map()
3158 "found a bad chunk map, wanted %llu-%llu, found %llu-%llu", in btrfs_get_chunk_map()
3159 logical, logical + length, map->start, in btrfs_get_chunk_map()
3160 map->start + map->chunk_len); in btrfs_get_chunk_map()
3162 return ERR_PTR(-EINVAL); in btrfs_get_chunk_map()
3179 lockdep_assert_held(&trans->fs_info->chunk_mutex); in remove_chunk_item()
3181 for (i = 0; i < map->num_stripes; i++) { in remove_chunk_item()
3184 ret = btrfs_update_device(trans, map->stripes[i].dev); in remove_chunk_item()
3194 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_remove_chunk()
3198 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_remove_chunk()
3215 * fs_info->chunk_mutex. Note that here we do not acquire the chunk_mutex in btrfs_remove_chunk()
3217 * items from the devices btree - COWing an extent buffer from the btree in btrfs_remove_chunk()
3219 * lock again fs_info->chunk_mutex. in btrfs_remove_chunk()
3221 mutex_lock(&fs_devices->device_list_mutex); in btrfs_remove_chunk()
3222 for (i = 0; i < map->num_stripes; i++) { in btrfs_remove_chunk()
3223 struct btrfs_device *device = map->stripes[i].dev; in btrfs_remove_chunk()
3225 map->stripes[i].physical, in btrfs_remove_chunk()
3228 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_remove_chunk()
3233 if (device->bytes_used > 0) { in btrfs_remove_chunk()
3234 mutex_lock(&fs_info->chunk_mutex); in btrfs_remove_chunk()
3236 device->bytes_used - dev_extent_len); in btrfs_remove_chunk()
3237 atomic64_add(dev_extent_len, &fs_info->free_chunk_space); in btrfs_remove_chunk()
3239 mutex_unlock(&fs_info->chunk_mutex); in btrfs_remove_chunk()
3242 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_remove_chunk()
3245 * We acquire fs_info->chunk_mutex for 2 reasons: in btrfs_remove_chunk()
3248 * reserve system space, do all chunk btree updates and deletions, and in btrfs_remove_chunk()
3249 * update the system chunk array in the superblock while holding this in btrfs_remove_chunk()
3257 * (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the in btrfs_remove_chunk()
3265 trans->removing_chunk = true; in btrfs_remove_chunk()
3266 mutex_lock(&fs_info->chunk_mutex); in btrfs_remove_chunk()
3268 check_system_chunk(trans, map->type); in btrfs_remove_chunk()
3272 * Normally we should not get -ENOSPC since we reserved space before in btrfs_remove_chunk()
3275 * Despite our system space_info having enough free space, we may not in btrfs_remove_chunk()
3277 * an incompatible profile, which will force us to allocate a new system in btrfs_remove_chunk()
3279 * check_system_space() above, a scrub turned the only system block group in btrfs_remove_chunk()
3283 * So if we get -ENOSPC, allocate a new system chunk and retry once. in btrfs_remove_chunk()
3285 if (ret == -ENOSPC) { in btrfs_remove_chunk()
3312 trace_btrfs_chunk_free(fs_info, map, chunk_offset, map->chunk_len); in btrfs_remove_chunk()
3314 if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) { in btrfs_remove_chunk()
3322 mutex_unlock(&fs_info->chunk_mutex); in btrfs_remove_chunk()
3323 trans->removing_chunk = false; in btrfs_remove_chunk()
3327 * system space we previously reserved (with check_system_chunk()). in btrfs_remove_chunk()
3338 if (trans->removing_chunk) { in btrfs_remove_chunk()
3339 mutex_unlock(&fs_info->chunk_mutex); in btrfs_remove_chunk()
3340 trans->removing_chunk = false; in btrfs_remove_chunk()
3349 struct btrfs_root *root = fs_info->chunk_root; in btrfs_relocate_chunk()
3358 return -EINVAL; in btrfs_relocate_chunk()
3373 lockdep_assert_held(&fs_info->reclaim_bgs_lock); in btrfs_relocate_chunk()
3391 return -ENOENT; in btrfs_relocate_chunk()
3392 btrfs_discard_cancel_work(&fs_info->discard_ctl, block_group); in btrfs_relocate_chunk()
3393 length = block_group->length; in btrfs_relocate_chunk()
3397 * On a zoned file system, discard the whole block group, this will in btrfs_relocate_chunk()
3410 trans = btrfs_start_trans_remove_block_group(root->fs_info, in btrfs_relocate_chunk()
3414 btrfs_handle_fs_error(root->fs_info, ret, NULL); in btrfs_relocate_chunk()
3429 struct btrfs_root *chunk_root = fs_info->chunk_root; in btrfs_relocate_sys_chunks()
3442 return -ENOMEM; in btrfs_relocate_sys_chunks()
3446 key.offset = (u64)-1; in btrfs_relocate_sys_chunks()
3450 mutex_lock(&fs_info->reclaim_bgs_lock); in btrfs_relocate_sys_chunks()
3453 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_relocate_sys_chunks()
3459 * offset -1, which is not possible. On subsequent in btrfs_relocate_sys_chunks()
3464 ret = -EUCLEAN; in btrfs_relocate_sys_chunks()
3465 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_relocate_sys_chunks()
3472 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_relocate_sys_chunks()
3478 leaf = path->nodes[0]; in btrfs_relocate_sys_chunks()
3479 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]); in btrfs_relocate_sys_chunks()
3481 chunk = btrfs_item_ptr(leaf, path->slots[0], in btrfs_relocate_sys_chunks()
3488 if (ret == -ENOSPC) in btrfs_relocate_sys_chunks()
3493 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_relocate_sys_chunks()
3497 key.offset = found_key.offset - 1; in btrfs_relocate_sys_chunks()
3505 ret = -ENOSPC; in btrfs_relocate_sys_chunks()
3526 chunk_type = cache->flags; in btrfs_may_alloc_data_chunk()
3532 spin_lock(&fs_info->data_sinfo->lock); in btrfs_may_alloc_data_chunk()
3533 bytes_used = fs_info->data_sinfo->bytes_used; in btrfs_may_alloc_data_chunk()
3534 spin_unlock(&fs_info->data_sinfo->lock); in btrfs_may_alloc_data_chunk()
3540 trans = btrfs_join_transaction(fs_info->tree_root); in btrfs_may_alloc_data_chunk()
3559 cpu->profiles = le64_to_cpu(disk->profiles); in btrfs_disk_balance_args_to_cpu()
3560 cpu->usage = le64_to_cpu(disk->usage); in btrfs_disk_balance_args_to_cpu()
3561 cpu->devid = le64_to_cpu(disk->devid); in btrfs_disk_balance_args_to_cpu()
3562 cpu->pstart = le64_to_cpu(disk->pstart); in btrfs_disk_balance_args_to_cpu()
3563 cpu->pend = le64_to_cpu(disk->pend); in btrfs_disk_balance_args_to_cpu()
3564 cpu->vstart = le64_to_cpu(disk->vstart); in btrfs_disk_balance_args_to_cpu()
3565 cpu->vend = le64_to_cpu(disk->vend); in btrfs_disk_balance_args_to_cpu()
3566 cpu->target = le64_to_cpu(disk->target); in btrfs_disk_balance_args_to_cpu()
3567 cpu->flags = le64_to_cpu(disk->flags); in btrfs_disk_balance_args_to_cpu()
3568 cpu->limit = le64_to_cpu(disk->limit); in btrfs_disk_balance_args_to_cpu()
3569 cpu->stripes_min = le32_to_cpu(disk->stripes_min); in btrfs_disk_balance_args_to_cpu()
3570 cpu->stripes_max = le32_to_cpu(disk->stripes_max); in btrfs_disk_balance_args_to_cpu()
3578 disk->profiles = cpu_to_le64(cpu->profiles); in btrfs_cpu_balance_args_to_disk()
3579 disk->usage = cpu_to_le64(cpu->usage); in btrfs_cpu_balance_args_to_disk()
3580 disk->devid = cpu_to_le64(cpu->devid); in btrfs_cpu_balance_args_to_disk()
3581 disk->pstart = cpu_to_le64(cpu->pstart); in btrfs_cpu_balance_args_to_disk()
3582 disk->pend = cpu_to_le64(cpu->pend); in btrfs_cpu_balance_args_to_disk()
3583 disk->vstart = cpu_to_le64(cpu->vstart); in btrfs_cpu_balance_args_to_disk()
3584 disk->vend = cpu_to_le64(cpu->vend); in btrfs_cpu_balance_args_to_disk()
3585 disk->target = cpu_to_le64(cpu->target); in btrfs_cpu_balance_args_to_disk()
3586 disk->flags = cpu_to_le64(cpu->flags); in btrfs_cpu_balance_args_to_disk()
3587 disk->limit = cpu_to_le64(cpu->limit); in btrfs_cpu_balance_args_to_disk()
3588 disk->stripes_min = cpu_to_le32(cpu->stripes_min); in btrfs_cpu_balance_args_to_disk()
3589 disk->stripes_max = cpu_to_le32(cpu->stripes_max); in btrfs_cpu_balance_args_to_disk()
3595 struct btrfs_root *root = fs_info->tree_root; in insert_balance_item()
3606 return -ENOMEM; in insert_balance_item()
3623 leaf = path->nodes[0]; in insert_balance_item()
3624 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item); in insert_balance_item()
3628 btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->data); in insert_balance_item()
3630 btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->meta); in insert_balance_item()
3632 btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->sys); in insert_balance_item()
3635 btrfs_set_balance_flags(leaf, item, bctl->flags); in insert_balance_item()
3648 struct btrfs_root *root = fs_info->tree_root; in del_balance_item()
3656 return -ENOMEM; in del_balance_item()
3668 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); in del_balance_item()
3672 ret = -ENOENT; in del_balance_item()
3694 if (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) in update_balance_args()
3695 bctl->data.flags |= BTRFS_BALANCE_ARGS_SOFT; in update_balance_args()
3696 if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) in update_balance_args()
3697 bctl->sys.flags |= BTRFS_BALANCE_ARGS_SOFT; in update_balance_args()
3698 if (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) in update_balance_args()
3699 bctl->meta.flags |= BTRFS_BALANCE_ARGS_SOFT; in update_balance_args()
3705 * converted - that will keep us from relocating unconverted in update_balance_args()
3708 if (!(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE) && in update_balance_args()
3709 !(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) && in update_balance_args()
3710 !(bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)) { in update_balance_args()
3711 bctl->data.flags |= BTRFS_BALANCE_ARGS_USAGE; in update_balance_args()
3712 bctl->data.usage = 90; in update_balance_args()
3714 if (!(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE) && in update_balance_args()
3715 !(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) && in update_balance_args()
3716 !(bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)) { in update_balance_args()
3717 bctl->sys.flags |= BTRFS_BALANCE_ARGS_USAGE; in update_balance_args()
3718 bctl->sys.usage = 90; in update_balance_args()
3720 if (!(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE) && in update_balance_args()
3721 !(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) && in update_balance_args()
3722 !(bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)) { in update_balance_args()
3723 bctl->meta.flags |= BTRFS_BALANCE_ARGS_USAGE; in update_balance_args()
3724 bctl->meta.usage = 90; in update_balance_args()
3733 struct btrfs_balance_control *bctl = fs_info->balance_ctl; in reset_balance_state()
3736 ASSERT(fs_info->balance_ctl); in reset_balance_state()
3738 spin_lock(&fs_info->balance_lock); in reset_balance_state()
3739 fs_info->balance_ctl = NULL; in reset_balance_state()
3740 spin_unlock(&fs_info->balance_lock); in reset_balance_state()
3758 if (bargs->profiles & chunk_type) in chunk_profiles_filter()
3774 chunk_used = cache->used; in chunk_usage_range_filter()
3776 if (bargs->usage_min == 0) in chunk_usage_range_filter()
3779 user_thresh_min = mult_perc(cache->length, bargs->usage_min); in chunk_usage_range_filter()
3781 if (bargs->usage_max == 0) in chunk_usage_range_filter()
3783 else if (bargs->usage_max > 100) in chunk_usage_range_filter()
3784 user_thresh_max = cache->length; in chunk_usage_range_filter()
3786 user_thresh_max = mult_perc(cache->length, bargs->usage_max); in chunk_usage_range_filter()
3803 chunk_used = cache->used; in chunk_usage_filter()
3805 if (bargs->usage_min == 0) in chunk_usage_filter()
3807 else if (bargs->usage > 100) in chunk_usage_filter()
3808 user_thresh = cache->length; in chunk_usage_filter()
3810 user_thresh = mult_perc(cache->length, bargs->usage); in chunk_usage_filter()
3829 if (btrfs_stripe_devid(leaf, stripe) == bargs->devid) in chunk_devid_filter()
3842 return (num_stripes - nparity) / ncopies; in calc_data_stripes()
3858 if (!(bargs->flags & BTRFS_BALANCE_ARGS_DEVID)) in chunk_drange_filter()
3866 if (btrfs_stripe_devid(leaf, stripe) != bargs->devid) in chunk_drange_filter()
3873 if (stripe_offset < bargs->pend && in chunk_drange_filter()
3874 stripe_offset + stripe_length > bargs->pstart) in chunk_drange_filter()
3887 if (chunk_offset < bargs->vend && in chunk_vrange_filter()
3888 chunk_offset + btrfs_chunk_length(leaf, chunk) > bargs->vstart) in chunk_vrange_filter()
3901 if (bargs->stripes_min <= num_stripes in chunk_stripes_range_filter()
3902 && num_stripes <= bargs->stripes_max) in chunk_stripes_range_filter()
3911 if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT)) in chunk_soft_convert_filter()
3917 if (bargs->target == chunk_type) in chunk_soft_convert_filter()
3926 struct btrfs_fs_info *fs_info = leaf->fs_info; in should_balance_chunk()
3927 struct btrfs_balance_control *bctl = fs_info->balance_ctl; in should_balance_chunk()
3933 (bctl->flags & BTRFS_BALANCE_TYPE_MASK))) { in should_balance_chunk()
3938 bargs = &bctl->data; in should_balance_chunk()
3940 bargs = &bctl->sys; in should_balance_chunk()
3942 bargs = &bctl->meta; in should_balance_chunk()
3945 if ((bargs->flags & BTRFS_BALANCE_ARGS_PROFILES) && in should_balance_chunk()
3951 if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE) && in should_balance_chunk()
3954 } else if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) && in should_balance_chunk()
3960 if ((bargs->flags & BTRFS_BALANCE_ARGS_DEVID) && in should_balance_chunk()
3966 if ((bargs->flags & BTRFS_BALANCE_ARGS_DRANGE) && in should_balance_chunk()
3972 if ((bargs->flags & BTRFS_BALANCE_ARGS_VRANGE) && in should_balance_chunk()
3978 if ((bargs->flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE) && in should_balance_chunk()
3984 if ((bargs->flags & BTRFS_BALANCE_ARGS_SOFT) && in should_balance_chunk()
3992 if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT)) { in should_balance_chunk()
3993 if (bargs->limit == 0) in should_balance_chunk()
3996 bargs->limit--; in should_balance_chunk()
3997 } else if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)) { in should_balance_chunk()
4003 if (bargs->limit_max == 0) in should_balance_chunk()
4006 bargs->limit_max--; in should_balance_chunk()
4014 struct btrfs_balance_control *bctl = fs_info->balance_ctl; in __btrfs_balance()
4015 struct btrfs_root *chunk_root = fs_info->chunk_root; in __btrfs_balance()
4027 u64 limit_data = bctl->data.limit; in __btrfs_balance()
4028 u64 limit_meta = bctl->meta.limit; in __btrfs_balance()
4029 u64 limit_sys = bctl->sys.limit; in __btrfs_balance()
4037 ret = -ENOMEM; in __btrfs_balance()
4042 spin_lock(&fs_info->balance_lock); in __btrfs_balance()
4043 memset(&bctl->stat, 0, sizeof(bctl->stat)); in __btrfs_balance()
4044 spin_unlock(&fs_info->balance_lock); in __btrfs_balance()
4051 bctl->data.limit = limit_data; in __btrfs_balance()
4052 bctl->meta.limit = limit_meta; in __btrfs_balance()
4053 bctl->sys.limit = limit_sys; in __btrfs_balance()
4056 key.offset = (u64)-1; in __btrfs_balance()
4060 if ((!counting && atomic_read(&fs_info->balance_pause_req)) || in __btrfs_balance()
4061 atomic_read(&fs_info->balance_cancel_req)) { in __btrfs_balance()
4062 ret = -ECANCELED; in __btrfs_balance()
4066 mutex_lock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4069 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4083 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4088 leaf = path->nodes[0]; in __btrfs_balance()
4089 slot = path->slots[0]; in __btrfs_balance()
4093 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4101 spin_lock(&fs_info->balance_lock); in __btrfs_balance()
4102 bctl->stat.considered++; in __btrfs_balance()
4103 spin_unlock(&fs_info->balance_lock); in __btrfs_balance()
4110 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4115 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4116 spin_lock(&fs_info->balance_lock); in __btrfs_balance()
4117 bctl->stat.expected++; in __btrfs_balance()
4118 spin_unlock(&fs_info->balance_lock); in __btrfs_balance()
4135 count_data < bctl->data.limit_min) in __btrfs_balance()
4137 count_meta < bctl->meta.limit_min) in __btrfs_balance()
4139 count_sys < bctl->sys.limit_min)) { in __btrfs_balance()
4140 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4154 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4162 mutex_unlock(&fs_info->reclaim_bgs_lock); in __btrfs_balance()
4163 if (ret == -ENOSPC) { in __btrfs_balance()
4165 } else if (ret == -ETXTBSY) { in __btrfs_balance()
4173 spin_lock(&fs_info->balance_lock); in __btrfs_balance()
4174 bctl->stat.completed++; in __btrfs_balance()
4175 spin_unlock(&fs_info->balance_lock); in __btrfs_balance()
4180 key.offset = found_key.offset - 1; in __btrfs_balance()
4194 ret = -ENOSPC; in __btrfs_balance()
4232 if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT)) in validate_convert_profile()
4236 if (alloc_profile_is_valid(bargs->target, 1) && in validate_convert_profile()
4237 (bargs->target & ~allowed) == 0) in validate_convert_profile()
4241 type, btrfs_bg_type_to_raid_name(bargs->target)); in validate_convert_profile()
4256 u64 flags = bargs->flags; in describe_balance_args()
4267 size_bp -= ret; \ in describe_balance_args()
4276 size_bp -= ret; \ in describe_balance_args()
4285 size_bp -= ret; \ in describe_balance_args()
4291 btrfs_bg_type_to_raid_name(bargs->target)); in describe_balance_args()
4297 btrfs_describe_block_groups(bargs->profiles, tmp_buf, in describe_balance_args()
4303 CHECK_APPEND_1ARG("usage=%llu,", bargs->usage); in describe_balance_args()
4307 bargs->usage_min, bargs->usage_max); in describe_balance_args()
4310 CHECK_APPEND_1ARG("devid=%llu,", bargs->devid); in describe_balance_args()
4314 bargs->pstart, bargs->pend); in describe_balance_args()
4318 bargs->vstart, bargs->vend); in describe_balance_args()
4321 CHECK_APPEND_1ARG("limit=%llu,", bargs->limit); in describe_balance_args()
4325 bargs->limit_min, bargs->limit_max); in describe_balance_args()
4329 bargs->stripes_min, bargs->stripes_max); in describe_balance_args()
4338 buf[size_buf - size_bp - 1] = '\0'; /* remove last , */ in describe_balance_args()
4351 struct btrfs_balance_control *bctl = fs_info->balance_ctl; in describe_balance_start_or_resume()
4364 size_bp -= ret; \ in describe_balance_start_or_resume()
4368 if (bctl->flags & BTRFS_BALANCE_FORCE) in describe_balance_start_or_resume()
4369 CHECK_APPEND_1ARG("%s", "-f "); in describe_balance_start_or_resume()
4371 if (bctl->flags & BTRFS_BALANCE_DATA) { in describe_balance_start_or_resume()
4372 describe_balance_args(&bctl->data, tmp_buf, sizeof(tmp_buf)); in describe_balance_start_or_resume()
4373 CHECK_APPEND_1ARG("-d%s ", tmp_buf); in describe_balance_start_or_resume()
4376 if (bctl->flags & BTRFS_BALANCE_METADATA) { in describe_balance_start_or_resume()
4377 describe_balance_args(&bctl->meta, tmp_buf, sizeof(tmp_buf)); in describe_balance_start_or_resume()
4378 CHECK_APPEND_1ARG("-m%s ", tmp_buf); in describe_balance_start_or_resume()
4381 if (bctl->flags & BTRFS_BALANCE_SYSTEM) { in describe_balance_start_or_resume()
4382 describe_balance_args(&bctl->sys, tmp_buf, sizeof(tmp_buf)); in describe_balance_start_or_resume()
4383 CHECK_APPEND_1ARG("-s%s ", tmp_buf); in describe_balance_start_or_resume()
4391 buf[size_buf - size_bp - 1] = '\0'; /* remove last " " */ in describe_balance_start_or_resume()
4393 (bctl->flags & BTRFS_BALANCE_RESUME) ? in describe_balance_start_or_resume()
4417 atomic_read(&fs_info->balance_pause_req) || in btrfs_balance()
4419 ret = -EINVAL; in btrfs_balance()
4423 allowed = btrfs_super_incompat_flags(fs_info->super_copy); in btrfs_balance()
4432 if (mixed && (bctl->flags & allowed)) { in btrfs_balance()
4433 if (!(bctl->flags & BTRFS_BALANCE_DATA) || in btrfs_balance()
4434 !(bctl->flags & BTRFS_BALANCE_METADATA) || in btrfs_balance()
4435 memcmp(&bctl->data, &bctl->meta, sizeof(bctl->data))) { in btrfs_balance()
4438 ret = -EINVAL; in btrfs_balance()
4447 num_devices = fs_info->fs_devices->rw_devices; in btrfs_balance()
4450 * SINGLE profile on-disk has no profile bit, but in-memory we have a in btrfs_balance()
4459 if (!validate_convert_profile(fs_info, &bctl->data, allowed, "data") || in btrfs_balance()
4460 !validate_convert_profile(fs_info, &bctl->meta, allowed, "metadata") || in btrfs_balance()
4461 !validate_convert_profile(fs_info, &bctl->sys, allowed, "system")) { in btrfs_balance()
4462 ret = -EINVAL; in btrfs_balance()
4467 * Allow to reduce metadata or system integrity only if force set for in btrfs_balance()
4477 seq = read_seqbegin(&fs_info->profiles_lock); in btrfs_balance()
4479 if (((bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) && in btrfs_balance()
4480 (fs_info->avail_system_alloc_bits & allowed) && in btrfs_balance()
4481 !(bctl->sys.target & allowed)) || in btrfs_balance()
4482 ((bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) && in btrfs_balance()
4483 (fs_info->avail_metadata_alloc_bits & allowed) && in btrfs_balance()
4484 !(bctl->meta.target & allowed))) in btrfs_balance()
4490 meta_target = (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) ? in btrfs_balance()
4491 bctl->meta.target : fs_info->avail_metadata_alloc_bits; in btrfs_balance()
4492 data_target = (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) ? in btrfs_balance()
4493 bctl->data.target : fs_info->avail_data_alloc_bits; in btrfs_balance()
4494 } while (read_seqretry(&fs_info->profiles_lock, seq)); in btrfs_balance()
4497 if (bctl->flags & BTRFS_BALANCE_FORCE) { in btrfs_balance()
4502 "balance: reduces metadata redundancy, use --force if you want this"); in btrfs_balance()
4503 ret = -EINVAL; in btrfs_balance()
4517 if (ret && ret != -EEXIST) in btrfs_balance()
4520 if (!(bctl->flags & BTRFS_BALANCE_RESUME)) { in btrfs_balance()
4521 BUG_ON(ret == -EEXIST); in btrfs_balance()
4522 BUG_ON(fs_info->balance_ctl); in btrfs_balance()
4523 spin_lock(&fs_info->balance_lock); in btrfs_balance()
4524 fs_info->balance_ctl = bctl; in btrfs_balance()
4525 spin_unlock(&fs_info->balance_lock); in btrfs_balance()
4527 BUG_ON(ret != -EEXIST); in btrfs_balance()
4528 spin_lock(&fs_info->balance_lock); in btrfs_balance()
4530 spin_unlock(&fs_info->balance_lock); in btrfs_balance()
4533 ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); in btrfs_balance()
4534 set_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags); in btrfs_balance()
4536 mutex_unlock(&fs_info->balance_mutex); in btrfs_balance()
4540 mutex_lock(&fs_info->balance_mutex); in btrfs_balance()
4541 if (ret == -ECANCELED && atomic_read(&fs_info->balance_pause_req)) { in btrfs_balance()
4549 * - Regular cancel request in btrfs_balance()
4550 * Then ret == -ECANCELED and balance_cancel_req > 0 in btrfs_balance()
4552 * - Fatal signal to "btrfs" process in btrfs_balance()
4554 * got -EINTR, or caught by btrfs_should_cancel_balance() and in btrfs_balance()
4555 * got -ECANCELED. in btrfs_balance()
4557 * ret == -EINTR or ret == -ECANCELED. in btrfs_balance()
4561 else if (ret == -ECANCELED || ret == -EINTR) in btrfs_balance()
4566 clear_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags); in btrfs_balance()
4579 wake_up(&fs_info->balance_wait_q); in btrfs_balance()
4583 if (bctl->flags & BTRFS_BALANCE_RESUME) in btrfs_balance()
4597 sb_start_write(fs_info->sb); in balance_kthread()
4598 mutex_lock(&fs_info->balance_mutex); in balance_kthread()
4599 if (fs_info->balance_ctl) in balance_kthread()
4600 ret = btrfs_balance(fs_info, fs_info->balance_ctl, NULL); in balance_kthread()
4601 mutex_unlock(&fs_info->balance_mutex); in balance_kthread()
4602 sb_end_write(fs_info->sb); in balance_kthread()
4611 mutex_lock(&fs_info->balance_mutex); in btrfs_resume_balance_async()
4612 if (!fs_info->balance_ctl) { in btrfs_resume_balance_async()
4613 mutex_unlock(&fs_info->balance_mutex); in btrfs_resume_balance_async()
4616 mutex_unlock(&fs_info->balance_mutex); in btrfs_resume_balance_async()
4623 spin_lock(&fs_info->super_lock); in btrfs_resume_balance_async()
4624 ASSERT(fs_info->exclusive_operation == BTRFS_EXCLOP_BALANCE_PAUSED); in btrfs_resume_balance_async()
4625 fs_info->exclusive_operation = BTRFS_EXCLOP_BALANCE; in btrfs_resume_balance_async()
4626 spin_unlock(&fs_info->super_lock); in btrfs_resume_balance_async()
4628 * A ro->rw remount sequence should continue with the paused balance in btrfs_resume_balance_async()
4629 * regardless of who pauses it, system or the user as of now, so set in btrfs_resume_balance_async()
4632 spin_lock(&fs_info->balance_lock); in btrfs_resume_balance_async()
4633 fs_info->balance_ctl->flags |= BTRFS_BALANCE_RESUME; in btrfs_resume_balance_async()
4634 spin_unlock(&fs_info->balance_lock); in btrfs_resume_balance_async()
4636 tsk = kthread_run(balance_kthread, fs_info, "btrfs-balance"); in btrfs_resume_balance_async()
4652 return -ENOMEM; in btrfs_recover_balance()
4658 ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0); in btrfs_recover_balance()
4661 if (ret > 0) { /* ret = -ENOENT; */ in btrfs_recover_balance()
4668 ret = -ENOMEM; in btrfs_recover_balance()
4672 leaf = path->nodes[0]; in btrfs_recover_balance()
4673 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item); in btrfs_recover_balance()
4675 bctl->flags = btrfs_balance_flags(leaf, item); in btrfs_recover_balance()
4676 bctl->flags |= BTRFS_BALANCE_RESUME; in btrfs_recover_balance()
4679 btrfs_disk_balance_args_to_cpu(&bctl->data, &disk_bargs); in btrfs_recover_balance()
4681 btrfs_disk_balance_args_to_cpu(&bctl->meta, &disk_bargs); in btrfs_recover_balance()
4683 btrfs_disk_balance_args_to_cpu(&bctl->sys, &disk_bargs); in btrfs_recover_balance()
4701 mutex_lock(&fs_info->balance_mutex); in btrfs_recover_balance()
4702 BUG_ON(fs_info->balance_ctl); in btrfs_recover_balance()
4703 spin_lock(&fs_info->balance_lock); in btrfs_recover_balance()
4704 fs_info->balance_ctl = bctl; in btrfs_recover_balance()
4705 spin_unlock(&fs_info->balance_lock); in btrfs_recover_balance()
4706 mutex_unlock(&fs_info->balance_mutex); in btrfs_recover_balance()
4716 mutex_lock(&fs_info->balance_mutex); in btrfs_pause_balance()
4717 if (!fs_info->balance_ctl) { in btrfs_pause_balance()
4718 mutex_unlock(&fs_info->balance_mutex); in btrfs_pause_balance()
4719 return -ENOTCONN; in btrfs_pause_balance()
4722 if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) { in btrfs_pause_balance()
4723 atomic_inc(&fs_info->balance_pause_req); in btrfs_pause_balance()
4724 mutex_unlock(&fs_info->balance_mutex); in btrfs_pause_balance()
4726 wait_event(fs_info->balance_wait_q, in btrfs_pause_balance()
4727 !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); in btrfs_pause_balance()
4729 mutex_lock(&fs_info->balance_mutex); in btrfs_pause_balance()
4731 BUG_ON(test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); in btrfs_pause_balance()
4732 atomic_dec(&fs_info->balance_pause_req); in btrfs_pause_balance()
4734 ret = -ENOTCONN; in btrfs_pause_balance()
4737 mutex_unlock(&fs_info->balance_mutex); in btrfs_pause_balance()
4743 mutex_lock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4744 if (!fs_info->balance_ctl) { in btrfs_cancel_balance()
4745 mutex_unlock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4746 return -ENOTCONN; in btrfs_cancel_balance()
4751 * mount time if the mount is read-write. Otherwise it's still paused in btrfs_cancel_balance()
4754 if (sb_rdonly(fs_info->sb)) { in btrfs_cancel_balance()
4755 mutex_unlock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4756 return -EROFS; in btrfs_cancel_balance()
4759 atomic_inc(&fs_info->balance_cancel_req); in btrfs_cancel_balance()
4764 if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) { in btrfs_cancel_balance()
4765 mutex_unlock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4766 wait_event(fs_info->balance_wait_q, in btrfs_cancel_balance()
4767 !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); in btrfs_cancel_balance()
4768 mutex_lock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4770 mutex_unlock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4775 mutex_lock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4777 if (fs_info->balance_ctl) { in btrfs_cancel_balance()
4784 ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)); in btrfs_cancel_balance()
4785 atomic_dec(&fs_info->balance_cancel_req); in btrfs_cancel_balance()
4786 mutex_unlock(&fs_info->balance_mutex); in btrfs_cancel_balance()
4797 struct btrfs_fs_info *fs_info = device->fs_info; in btrfs_shrink_device()
4798 struct btrfs_root *root = fs_info->dev_root; in btrfs_shrink_device()
4810 struct btrfs_super_block *super_copy = fs_info->super_copy; in btrfs_shrink_device()
4817 new_size = round_down(new_size, fs_info->sectorsize); in btrfs_shrink_device()
4819 diff = round_down(old_size - new_size, fs_info->sectorsize); in btrfs_shrink_device()
4821 if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) in btrfs_shrink_device()
4822 return -EINVAL; in btrfs_shrink_device()
4826 return -ENOMEM; in btrfs_shrink_device()
4828 path->reada = READA_BACK; in btrfs_shrink_device()
4836 mutex_lock(&fs_info->chunk_mutex); in btrfs_shrink_device()
4839 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in btrfs_shrink_device()
4840 device->fs_devices->total_rw_bytes -= diff; in btrfs_shrink_device()
4843 * The new free_chunk_space is new_size - used, so we have to in btrfs_shrink_device()
4845 * old_size - used. If used > new_size then just subtract this in btrfs_shrink_device()
4848 if (device->bytes_used < new_size) in btrfs_shrink_device()
4849 free_diff = (old_size - device->bytes_used) - in btrfs_shrink_device()
4850 (new_size - device->bytes_used); in btrfs_shrink_device()
4852 free_diff = old_size - device->bytes_used; in btrfs_shrink_device()
4853 atomic64_sub(free_diff, &fs_info->free_chunk_space); in btrfs_shrink_device()
4858 * in-memory chunks are synced to disk so that the loop below sees them in btrfs_shrink_device()
4862 mutex_unlock(&fs_info->chunk_mutex); in btrfs_shrink_device()
4867 mutex_unlock(&fs_info->chunk_mutex); in btrfs_shrink_device()
4872 key.objectid = device->devid; in btrfs_shrink_device()
4873 key.offset = (u64)-1; in btrfs_shrink_device()
4877 mutex_lock(&fs_info->reclaim_bgs_lock); in btrfs_shrink_device()
4880 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_shrink_device()
4886 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_shrink_device()
4894 l = path->nodes[0]; in btrfs_shrink_device()
4895 slot = path->slots[0]; in btrfs_shrink_device()
4896 btrfs_item_key_to_cpu(l, &key, path->slots[0]); in btrfs_shrink_device()
4898 if (key.objectid != device->devid) { in btrfs_shrink_device()
4899 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_shrink_device()
4908 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_shrink_device()
4924 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_shrink_device()
4929 mutex_unlock(&fs_info->reclaim_bgs_lock); in btrfs_shrink_device()
4930 if (ret == -ENOSPC) { in btrfs_shrink_device()
4933 if (ret == -ETXTBSY) { in btrfs_shrink_device()
4940 } while (key.offset-- > 0); in btrfs_shrink_device()
4947 ret = -ENOSPC; in btrfs_shrink_device()
4958 mutex_lock(&fs_info->chunk_mutex); in btrfs_shrink_device()
4960 clear_extent_bits(&device->alloc_state, new_size, (u64)-1, in btrfs_shrink_device()
4964 if (list_empty(&device->post_commit_list)) in btrfs_shrink_device()
4965 list_add_tail(&device->post_commit_list, in btrfs_shrink_device()
4966 &trans->transaction->dev_update_list); in btrfs_shrink_device()
4970 round_down(old_total - diff, fs_info->sectorsize)); in btrfs_shrink_device()
4971 mutex_unlock(&fs_info->chunk_mutex); in btrfs_shrink_device()
4974 /* Now btrfs_update_device() will change the on-disk size. */ in btrfs_shrink_device()
4986 mutex_lock(&fs_info->chunk_mutex); in btrfs_shrink_device()
4988 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in btrfs_shrink_device()
4989 device->fs_devices->total_rw_bytes += diff; in btrfs_shrink_device()
4990 atomic64_add(free_diff, &fs_info->free_chunk_space); in btrfs_shrink_device()
4992 mutex_unlock(&fs_info->chunk_mutex); in btrfs_shrink_device()
5001 struct btrfs_super_block *super_copy = fs_info->super_copy; in btrfs_add_system_chunk()
5006 lockdep_assert_held(&fs_info->chunk_mutex); in btrfs_add_system_chunk()
5011 return -EFBIG; in btrfs_add_system_chunk()
5013 ptr = super_copy->sys_chunk_array + array_size; in btrfs_add_system_chunk()
5032 if (di_a->max_avail > di_b->max_avail) in btrfs_cmp_device_info()
5033 return -1; in btrfs_cmp_device_info()
5034 if (di_a->max_avail < di_b->max_avail) in btrfs_cmp_device_info()
5036 if (di_a->total_avail > di_b->total_avail) in btrfs_cmp_device_info()
5037 return -1; in btrfs_cmp_device_info()
5038 if (di_a->total_avail < di_b->total_avail) in btrfs_cmp_device_info()
5092 struct alloc_chunk_ctl *ctl) in init_alloc_chunk_ctl_policy_regular() argument
5096 space_info = btrfs_find_space_info(fs_devices->fs_info, ctl->type); in init_alloc_chunk_ctl_policy_regular()
5099 ctl->max_chunk_size = READ_ONCE(space_info->chunk_size); in init_alloc_chunk_ctl_policy_regular()
5100 ctl->max_stripe_size = min_t(u64, ctl->max_chunk_size, SZ_1G); in init_alloc_chunk_ctl_policy_regular()
5102 if (ctl->type & BTRFS_BLOCK_GROUP_SYSTEM) in init_alloc_chunk_ctl_policy_regular()
5103 ctl->devs_max = min_t(int, ctl->devs_max, BTRFS_MAX_DEVS_SYS_CHUNK); in init_alloc_chunk_ctl_policy_regular()
5106 ctl->max_chunk_size = min(mult_perc(fs_devices->total_rw_bytes, 10), in init_alloc_chunk_ctl_policy_regular()
5107 ctl->max_chunk_size); in init_alloc_chunk_ctl_policy_regular()
5108 ctl->dev_extent_min = btrfs_stripe_nr_to_offset(ctl->dev_stripes); in init_alloc_chunk_ctl_policy_regular()
5113 struct alloc_chunk_ctl *ctl) in init_alloc_chunk_ctl_policy_zoned() argument
5115 u64 zone_size = fs_devices->fs_info->zone_size; in init_alloc_chunk_ctl_policy_zoned()
5117 int min_num_stripes = ctl->devs_min * ctl->dev_stripes; in init_alloc_chunk_ctl_policy_zoned()
5118 int min_data_stripes = (min_num_stripes - ctl->nparity) / ctl->ncopies; in init_alloc_chunk_ctl_policy_zoned()
5120 u64 type = ctl->type; in init_alloc_chunk_ctl_policy_zoned()
5122 ctl->max_stripe_size = zone_size; in init_alloc_chunk_ctl_policy_zoned()
5124 ctl->max_chunk_size = round_down(BTRFS_MAX_DATA_CHUNK_SIZE, in init_alloc_chunk_ctl_policy_zoned()
5127 ctl->max_chunk_size = ctl->max_stripe_size; in init_alloc_chunk_ctl_policy_zoned()
5129 ctl->max_chunk_size = 2 * ctl->max_stripe_size; in init_alloc_chunk_ctl_policy_zoned()
5130 ctl->devs_max = min_t(int, ctl->devs_max, in init_alloc_chunk_ctl_policy_zoned()
5137 limit = max(round_down(mult_perc(fs_devices->total_rw_bytes, 10), in init_alloc_chunk_ctl_policy_zoned()
5140 ctl->max_chunk_size = min(limit, ctl->max_chunk_size); in init_alloc_chunk_ctl_policy_zoned()
5141 ctl->dev_extent_min = zone_size * ctl->dev_stripes; in init_alloc_chunk_ctl_policy_zoned()
5145 struct alloc_chunk_ctl *ctl) in init_alloc_chunk_ctl() argument
5147 int index = btrfs_bg_flags_to_raid_index(ctl->type); in init_alloc_chunk_ctl()
5149 ctl->sub_stripes = btrfs_raid_array[index].sub_stripes; in init_alloc_chunk_ctl()
5150 ctl->dev_stripes = btrfs_raid_array[index].dev_stripes; in init_alloc_chunk_ctl()
5151 ctl->devs_max = btrfs_raid_array[index].devs_max; in init_alloc_chunk_ctl()
5152 if (!ctl->devs_max) in init_alloc_chunk_ctl()
5153 ctl->devs_max = BTRFS_MAX_DEVS(fs_devices->fs_info); in init_alloc_chunk_ctl()
5154 ctl->devs_min = btrfs_raid_array[index].devs_min; in init_alloc_chunk_ctl()
5155 ctl->devs_increment = btrfs_raid_array[index].devs_increment; in init_alloc_chunk_ctl()
5156 ctl->ncopies = btrfs_raid_array[index].ncopies; in init_alloc_chunk_ctl()
5157 ctl->nparity = btrfs_raid_array[index].nparity; in init_alloc_chunk_ctl()
5158 ctl->ndevs = 0; in init_alloc_chunk_ctl()
5160 switch (fs_devices->chunk_alloc_policy) { in init_alloc_chunk_ctl()
5162 init_alloc_chunk_ctl_policy_regular(fs_devices, ctl); in init_alloc_chunk_ctl()
5165 init_alloc_chunk_ctl_policy_zoned(fs_devices, ctl); in init_alloc_chunk_ctl()
5173 struct alloc_chunk_ctl *ctl, in gather_device_info() argument
5176 struct btrfs_fs_info *info = fs_devices->fs_info; in gather_device_info()
5179 u64 dev_extent_want = ctl->max_stripe_size * ctl->dev_stripes; in gather_device_info()
5189 list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) { in gather_device_info()
5190 if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { in gather_device_info()
5192 "BTRFS: read-only device in alloc_list\n"); in gather_device_info()
5197 &device->dev_state) || in gather_device_info()
5198 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) in gather_device_info()
5201 if (device->total_bytes > device->bytes_used) in gather_device_info()
5202 total_avail = device->total_bytes - device->bytes_used; in gather_device_info()
5207 if (total_avail < ctl->dev_extent_min) in gather_device_info()
5212 if (ret && ret != -ENOSPC) in gather_device_info()
5218 if (max_avail < ctl->dev_extent_min) { in gather_device_info()
5222 __func__, device->devid, max_avail, in gather_device_info()
5223 ctl->dev_extent_min); in gather_device_info()
5227 if (ndevs == fs_devices->rw_devices) { in gather_device_info()
5229 __func__, fs_devices->rw_devices); in gather_device_info()
5238 ctl->ndevs = ndevs; in gather_device_info()
5249 static int decide_stripe_size_regular(struct alloc_chunk_ctl *ctl, in decide_stripe_size_regular() argument
5262 ctl->stripe_size = div_u64(devices_info[ctl->ndevs - 1].max_avail, in decide_stripe_size_regular()
5263 ctl->dev_stripes); in decide_stripe_size_regular()
5264 ctl->num_stripes = ctl->ndevs * ctl->dev_stripes; in decide_stripe_size_regular()
5267 data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies; in decide_stripe_size_regular()
5275 if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) { in decide_stripe_size_regular()
5281 ctl->stripe_size = min(round_up(div_u64(ctl->max_chunk_size, in decide_stripe_size_regular()
5283 ctl->stripe_size); in decide_stripe_size_regular()
5287 ctl->stripe_size = min_t(u64, ctl->stripe_size, SZ_1G); in decide_stripe_size_regular()
5290 ctl->stripe_size = round_down(ctl->stripe_size, BTRFS_STRIPE_LEN); in decide_stripe_size_regular()
5291 ctl->chunk_size = ctl->stripe_size * data_stripes; in decide_stripe_size_regular()
5296 static int decide_stripe_size_zoned(struct alloc_chunk_ctl *ctl, in decide_stripe_size_zoned() argument
5299 u64 zone_size = devices_info[0].dev->zone_info->zone_size; in decide_stripe_size_zoned()
5307 ASSERT(devices_info[ctl->ndevs - 1].max_avail == ctl->dev_extent_min); in decide_stripe_size_zoned()
5309 ctl->stripe_size = zone_size; in decide_stripe_size_zoned()
5310 ctl->num_stripes = ctl->ndevs * ctl->dev_stripes; in decide_stripe_size_zoned()
5311 data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies; in decide_stripe_size_zoned()
5314 if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) { in decide_stripe_size_zoned()
5315 ctl->ndevs = div_u64(div_u64(ctl->max_chunk_size * ctl->ncopies, in decide_stripe_size_zoned()
5316 ctl->stripe_size) + ctl->nparity, in decide_stripe_size_zoned()
5317 ctl->dev_stripes); in decide_stripe_size_zoned()
5318 ctl->num_stripes = ctl->ndevs * ctl->dev_stripes; in decide_stripe_size_zoned()
5319 data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies; in decide_stripe_size_zoned()
5320 ASSERT(ctl->stripe_size * data_stripes <= ctl->max_chunk_size); in decide_stripe_size_zoned()
5323 ctl->chunk_size = ctl->stripe_size * data_stripes; in decide_stripe_size_zoned()
5329 struct alloc_chunk_ctl *ctl, in decide_stripe_size() argument
5332 struct btrfs_fs_info *info = fs_devices->fs_info; in decide_stripe_size()
5339 ctl->ndevs = rounddown(ctl->ndevs, ctl->devs_increment); in decide_stripe_size()
5341 if (ctl->ndevs < ctl->devs_min) { in decide_stripe_size()
5345 __func__, ctl->ndevs, ctl->devs_min); in decide_stripe_size()
5347 return -ENOSPC; in decide_stripe_size()
5350 ctl->ndevs = min(ctl->ndevs, ctl->devs_max); in decide_stripe_size()
5352 switch (fs_devices->chunk_alloc_policy) { in decide_stripe_size()
5354 return decide_stripe_size_regular(ctl, devices_info); in decide_stripe_size()
5356 return decide_stripe_size_zoned(ctl, devices_info); in decide_stripe_size()
5364 for (int i = 0; i < map->num_stripes; i++) { in chunk_map_device_set_bits()
5365 struct btrfs_io_stripe *stripe = &map->stripes[i]; in chunk_map_device_set_bits()
5366 struct btrfs_device *device = stripe->dev; in chunk_map_device_set_bits()
5368 set_extent_bit(&device->alloc_state, stripe->physical, in chunk_map_device_set_bits()
5369 stripe->physical + map->stripe_size - 1, in chunk_map_device_set_bits()
5376 for (int i = 0; i < map->num_stripes; i++) { in chunk_map_device_clear_bits()
5377 struct btrfs_io_stripe *stripe = &map->stripes[i]; in chunk_map_device_clear_bits()
5378 struct btrfs_device *device = stripe->dev; in chunk_map_device_clear_bits()
5380 __clear_extent_bit(&device->alloc_state, stripe->physical, in chunk_map_device_clear_bits()
5381 stripe->physical + map->stripe_size - 1, in chunk_map_device_clear_bits()
5389 write_lock(&fs_info->mapping_tree_lock); in btrfs_remove_chunk_map()
5390 rb_erase_cached(&map->rb_node, &fs_info->mapping_tree); in btrfs_remove_chunk_map()
5391 RB_CLEAR_NODE(&map->rb_node); in btrfs_remove_chunk_map()
5393 write_unlock(&fs_info->mapping_tree_lock); in btrfs_remove_chunk_map()
5406 write_lock(&fs_info->mapping_tree_lock); in btrfs_add_chunk_map()
5407 p = &fs_info->mapping_tree.rb_root.rb_node; in btrfs_add_chunk_map()
5414 if (map->start < entry->start) { in btrfs_add_chunk_map()
5415 p = &(*p)->rb_left; in btrfs_add_chunk_map()
5416 } else if (map->start > entry->start) { in btrfs_add_chunk_map()
5417 p = &(*p)->rb_right; in btrfs_add_chunk_map()
5420 write_unlock(&fs_info->mapping_tree_lock); in btrfs_add_chunk_map()
5421 return -EEXIST; in btrfs_add_chunk_map()
5424 rb_link_node(&map->rb_node, parent, p); in btrfs_add_chunk_map()
5425 rb_insert_color_cached(&map->rb_node, &fs_info->mapping_tree, leftmost); in btrfs_add_chunk_map()
5428 write_unlock(&fs_info->mapping_tree_lock); in btrfs_add_chunk_map()
5442 refcount_set(&map->refs, 1); in btrfs_alloc_chunk_map()
5443 RB_CLEAR_NODE(&map->rb_node); in btrfs_alloc_chunk_map()
5449 struct alloc_chunk_ctl *ctl, in create_chunk() argument
5452 struct btrfs_fs_info *info = trans->fs_info; in create_chunk()
5455 u64 start = ctl->start; in create_chunk()
5456 u64 type = ctl->type; in create_chunk()
5459 map = btrfs_alloc_chunk_map(ctl->num_stripes, GFP_NOFS); in create_chunk()
5461 return ERR_PTR(-ENOMEM); in create_chunk()
5463 map->start = start; in create_chunk()
5464 map->chunk_len = ctl->chunk_size; in create_chunk()
5465 map->stripe_size = ctl->stripe_size; in create_chunk()
5466 map->type = type; in create_chunk()
5467 map->io_align = BTRFS_STRIPE_LEN; in create_chunk()
5468 map->io_width = BTRFS_STRIPE_LEN; in create_chunk()
5469 map->sub_stripes = ctl->sub_stripes; in create_chunk()
5470 map->num_stripes = ctl->num_stripes; in create_chunk()
5472 for (int i = 0; i < ctl->ndevs; i++) { in create_chunk()
5473 for (int j = 0; j < ctl->dev_stripes; j++) { in create_chunk()
5474 int s = i * ctl->dev_stripes + j; in create_chunk()
5475 map->stripes[s].dev = devices_info[i].dev; in create_chunk()
5476 map->stripes[s].physical = devices_info[i].dev_offset + in create_chunk()
5477 j * ctl->stripe_size; in create_chunk()
5481 trace_btrfs_chunk_alloc(info, map, start, ctl->chunk_size); in create_chunk()
5489 block_group = btrfs_make_block_group(trans, type, start, ctl->chunk_size); in create_chunk()
5495 for (int i = 0; i < map->num_stripes; i++) { in create_chunk()
5496 struct btrfs_device *dev = map->stripes[i].dev; in create_chunk()
5499 dev->bytes_used + ctl->stripe_size); in create_chunk()
5500 if (list_empty(&dev->post_commit_list)) in create_chunk()
5501 list_add_tail(&dev->post_commit_list, in create_chunk()
5502 &trans->transaction->dev_update_list); in create_chunk()
5505 atomic64_sub(ctl->stripe_size * map->num_stripes, in create_chunk()
5506 &info->free_chunk_space); in create_chunk()
5517 struct btrfs_fs_info *info = trans->fs_info; in btrfs_create_chunk()
5518 struct btrfs_fs_devices *fs_devices = info->fs_devices; in btrfs_create_chunk()
5520 struct alloc_chunk_ctl ctl; in btrfs_create_chunk() local
5524 lockdep_assert_held(&info->chunk_mutex); in btrfs_create_chunk()
5528 return ERR_PTR(-EINVAL); in btrfs_create_chunk()
5531 if (list_empty(&fs_devices->alloc_list)) { in btrfs_create_chunk()
5534 return ERR_PTR(-ENOSPC); in btrfs_create_chunk()
5540 return ERR_PTR(-EINVAL); in btrfs_create_chunk()
5543 ctl.start = find_next_chunk(info); in btrfs_create_chunk()
5544 ctl.type = type; in btrfs_create_chunk()
5545 init_alloc_chunk_ctl(fs_devices, &ctl); in btrfs_create_chunk()
5547 devices_info = kcalloc(fs_devices->rw_devices, sizeof(*devices_info), in btrfs_create_chunk()
5550 return ERR_PTR(-ENOMEM); in btrfs_create_chunk()
5552 ret = gather_device_info(fs_devices, &ctl, devices_info); in btrfs_create_chunk()
5558 ret = decide_stripe_size(fs_devices, &ctl, devices_info); in btrfs_create_chunk()
5564 block_group = create_chunk(trans, &ctl, devices_info); in btrfs_create_chunk()
5573 * phase 1 of chunk allocation. It belongs to phase 2 only when allocating system
5582 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_chunk_alloc_add_chunk_item()
5583 struct btrfs_root *chunk_root = fs_info->chunk_root; in btrfs_chunk_alloc_add_chunk_item()
5596 * the chunk_mutex, as well as updating the system chunk array in the in btrfs_chunk_alloc_add_chunk_item()
5604 * (dev-replace.c:btrfs_dev_replace_finishing()), so we could grab the in btrfs_chunk_alloc_add_chunk_item()
5610 * acquires both mutexes - first the device_list_mutex and then the in btrfs_chunk_alloc_add_chunk_item()
5614 lockdep_assert_held(&fs_info->chunk_mutex); in btrfs_chunk_alloc_add_chunk_item()
5616 map = btrfs_get_chunk_map(fs_info, bg->start, bg->length); in btrfs_chunk_alloc_add_chunk_item()
5623 item_size = btrfs_chunk_item_size(map->num_stripes); in btrfs_chunk_alloc_add_chunk_item()
5627 ret = -ENOMEM; in btrfs_chunk_alloc_add_chunk_item()
5632 for (i = 0; i < map->num_stripes; i++) { in btrfs_chunk_alloc_add_chunk_item()
5633 struct btrfs_device *device = map->stripes[i].dev; in btrfs_chunk_alloc_add_chunk_item()
5640 stripe = &chunk->stripe; in btrfs_chunk_alloc_add_chunk_item()
5641 for (i = 0; i < map->num_stripes; i++) { in btrfs_chunk_alloc_add_chunk_item()
5642 struct btrfs_device *device = map->stripes[i].dev; in btrfs_chunk_alloc_add_chunk_item()
5643 const u64 dev_offset = map->stripes[i].physical; in btrfs_chunk_alloc_add_chunk_item()
5645 btrfs_set_stack_stripe_devid(stripe, device->devid); in btrfs_chunk_alloc_add_chunk_item()
5647 memcpy(stripe->dev_uuid, device->uuid, BTRFS_UUID_SIZE); in btrfs_chunk_alloc_add_chunk_item()
5651 btrfs_set_stack_chunk_length(chunk, bg->length); in btrfs_chunk_alloc_add_chunk_item()
5654 btrfs_set_stack_chunk_type(chunk, map->type); in btrfs_chunk_alloc_add_chunk_item()
5655 btrfs_set_stack_chunk_num_stripes(chunk, map->num_stripes); in btrfs_chunk_alloc_add_chunk_item()
5658 btrfs_set_stack_chunk_sector_size(chunk, fs_info->sectorsize); in btrfs_chunk_alloc_add_chunk_item()
5659 btrfs_set_stack_chunk_sub_stripes(chunk, map->sub_stripes); in btrfs_chunk_alloc_add_chunk_item()
5663 key.offset = bg->start; in btrfs_chunk_alloc_add_chunk_item()
5669 set_bit(BLOCK_GROUP_FLAG_CHUNK_ITEM_INSERTED, &bg->runtime_flags); in btrfs_chunk_alloc_add_chunk_item()
5671 if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) { in btrfs_chunk_alloc_add_chunk_item()
5685 struct btrfs_fs_info *fs_info = trans->fs_info; in init_first_rw_device()
5691 * When adding a new device for sprouting, the seed device is read-only in init_first_rw_device()
5692 * so we must first allocate a metadata and a system chunk. But before in init_first_rw_device()
5697 * otherwise we would get -ENOSPC since the block groups from the in init_first_rw_device()
5698 * seed device are read-only; in init_first_rw_device()
5700 * 2) Add the device item for the new sprout device - finishing the setup in init_first_rw_device()
5703 * ensures this does not fail with -ENOSPC. in init_first_rw_device()
5726 const int index = btrfs_bg_flags_to_raid_index(map->type); in btrfs_chunk_max_errors()
5742 for (i = 0; i < map->num_stripes; i++) { in btrfs_chunk_writeable()
5744 &map->stripes[i].dev->dev_state)) { in btrfs_chunk_writeable()
5749 &map->stripes[i].dev->dev_state)) { in btrfs_chunk_writeable()
5768 write_lock(&fs_info->mapping_tree_lock); in btrfs_mapping_tree_free()
5769 while (!RB_EMPTY_ROOT(&fs_info->mapping_tree.rb_root)) { in btrfs_mapping_tree_free()
5773 node = rb_first_cached(&fs_info->mapping_tree); in btrfs_mapping_tree_free()
5775 rb_erase_cached(&map->rb_node, &fs_info->mapping_tree); in btrfs_mapping_tree_free()
5776 RB_CLEAR_NODE(&map->rb_node); in btrfs_mapping_tree_free()
5780 cond_resched_rwlock_write(&fs_info->mapping_tree_lock); in btrfs_mapping_tree_free()
5782 write_unlock(&fs_info->mapping_tree_lock); in btrfs_mapping_tree_free()
5787 enum btrfs_raid_types index = btrfs_bg_flags_to_raid_index(map->type); in btrfs_chunk_map_num_copies()
5789 if (map->type & BTRFS_BLOCK_GROUP_RAID5) in btrfs_chunk_map_num_copies()
5799 if (map->type & BTRFS_BLOCK_GROUP_RAID6) in btrfs_chunk_map_num_copies()
5800 return map->num_stripes; in btrfs_chunk_map_num_copies()
5802 /* Non-RAID56, use their ncopies from btrfs_raid_array. */ in btrfs_chunk_map_num_copies()
5830 unsigned long len = fs_info->sectorsize; in btrfs_full_stripe_len()
5838 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) in btrfs_full_stripe_len()
5856 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) in btrfs_is_parity_mirror()
5867 const enum btrfs_read_policy policy = READ_ONCE(fs_info->fs_devices->read_policy); in find_live_mirror()
5874 ASSERT((map->type & in find_live_mirror()
5877 if (map->type & BTRFS_BLOCK_GROUP_RAID10) in find_live_mirror()
5878 num_stripes = map->sub_stripes; in find_live_mirror()
5880 num_stripes = map->num_stripes; in find_live_mirror()
5887 WRITE_ONCE(fs_info->fs_devices->read_policy, BTRFS_READ_POLICY_PID); in find_live_mirror()
5890 preferred_mirror = first + (current->pid % num_stripes); in find_live_mirror()
5895 fs_info->dev_replace.cont_reading_from_srcdev_mode == in find_live_mirror()
5897 srcdev = fs_info->dev_replace.srcdev; in find_live_mirror()
5903 * dev-replace procedure, only choose it if no other non-missing in find_live_mirror()
5907 if (map->stripes[preferred_mirror].dev->bdev && in find_live_mirror()
5908 (tolerance || map->stripes[preferred_mirror].dev != srcdev)) in find_live_mirror()
5911 if (map->stripes[i].dev->bdev && in find_live_mirror()
5912 (tolerance || map->stripes[i].dev != srcdev)) in find_live_mirror()
5939 refcount_set(&bioc->refs, 1); in alloc_btrfs_io_context()
5941 bioc->fs_info = fs_info; in alloc_btrfs_io_context()
5942 bioc->replace_stripe_src = -1; in alloc_btrfs_io_context()
5943 bioc->full_stripe_logical = (u64)-1; in alloc_btrfs_io_context()
5944 bioc->logical = logical; in alloc_btrfs_io_context()
5951 WARN_ON(!refcount_read(&bioc->refs)); in btrfs_get_bioc()
5952 refcount_inc(&bioc->refs); in btrfs_get_bioc()
5959 if (refcount_dec_and_test(&bioc->refs)) in btrfs_put_bioc()
5994 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { in btrfs_map_discard()
5995 ret = -EOPNOTSUPP; in btrfs_map_discard()
5999 offset = logical - map->start; in btrfs_map_discard()
6000 length = min_t(u64, map->start + map->chunk_len - logical, length); in btrfs_map_discard()
6010 stripe_offset = offset - btrfs_stripe_nr_to_offset(stripe_nr); in btrfs_map_discard()
6014 stripe_cnt = stripe_nr_end - stripe_nr; in btrfs_map_discard()
6015 stripe_end_offset = btrfs_stripe_nr_to_offset(stripe_nr_end) - in btrfs_map_discard()
6024 if (map->type & (BTRFS_BLOCK_GROUP_RAID0 | in btrfs_map_discard()
6026 if (map->type & BTRFS_BLOCK_GROUP_RAID0) in btrfs_map_discard()
6029 sub_stripes = map->sub_stripes; in btrfs_map_discard()
6031 factor = map->num_stripes / sub_stripes; in btrfs_map_discard()
6032 *num_stripes = min_t(u64, map->num_stripes, in btrfs_map_discard()
6040 last_stripe = ((stripe_nr_end - 1) % factor) * sub_stripes; in btrfs_map_discard()
6041 } else if (map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK | in btrfs_map_discard()
6043 *num_stripes = map->num_stripes; in btrfs_map_discard()
6045 stripe_index = stripe_nr % map->num_stripes; in btrfs_map_discard()
6046 stripe_nr /= map->num_stripes; in btrfs_map_discard()
6051 ret = -ENOMEM; in btrfs_map_discard()
6057 map->stripes[stripe_index].physical + in btrfs_map_discard()
6059 stripes[i].dev = map->stripes[stripe_index].dev; in btrfs_map_discard()
6061 if (map->type & (BTRFS_BLOCK_GROUP_RAID0 | in btrfs_map_discard()
6072 * |-------|...|-------| in btrfs_map_discard()
6073 * |----------| in btrfs_map_discard()
6077 stripes[i].length -= stripe_offset; in btrfs_map_discard()
6081 sub_stripes - 1)) in btrfs_map_discard()
6082 stripes[i].length -= stripe_end_offset; in btrfs_map_discard()
6084 if (i == sub_stripes - 1) in btrfs_map_discard()
6091 if (stripe_index == map->num_stripes) { in btrfs_map_discard()
6115 ret = test_bit(BLOCK_GROUP_FLAG_TO_COPY, &cache->runtime_flags); in is_block_group_to_copy()
6126 u64 srcdev_devid = dev_replace->srcdev->devid; in handle_ops_on_dev_replace()
6131 int num_stripes = io_geom->num_stripes; in handle_ops_on_dev_replace()
6132 int max_errors = io_geom->max_errors; in handle_ops_on_dev_replace()
6138 * the dev-replace process. We can avoid cloning IO here. in handle_ops_on_dev_replace()
6140 if (is_block_group_to_copy(dev_replace->srcdev->fs_info, logical)) in handle_ops_on_dev_replace()
6144 * Duplicate the write operations while the dev-replace procedure is in handle_ops_on_dev_replace()
6150 * Note that device->missing is handled by the caller, and that the in handle_ops_on_dev_replace()
6154 struct btrfs_io_stripe *old = &bioc->stripes[i]; in handle_ops_on_dev_replace()
6155 struct btrfs_io_stripe *new = &bioc->stripes[num_stripes + nr_extra_stripes]; in handle_ops_on_dev_replace()
6157 if (old->dev->devid != srcdev_devid) in handle_ops_on_dev_replace()
6160 new->physical = old->physical; in handle_ops_on_dev_replace()
6161 new->dev = dev_replace->tgtdev; in handle_ops_on_dev_replace()
6162 if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) in handle_ops_on_dev_replace()
6163 bioc->replace_stripe_src = i; in handle_ops_on_dev_replace()
6174 if (io_geom->op == BTRFS_MAP_GET_READ_MIRRORS && nr_extra_stripes == 2) { in handle_ops_on_dev_replace()
6175 struct btrfs_io_stripe *first = &bioc->stripes[num_stripes]; in handle_ops_on_dev_replace()
6176 struct btrfs_io_stripe *second = &bioc->stripes[num_stripes + 1]; in handle_ops_on_dev_replace()
6179 ASSERT(bioc->map_type & BTRFS_BLOCK_GROUP_DUP); in handle_ops_on_dev_replace()
6185 if (first->physical > second->physical) { in handle_ops_on_dev_replace()
6186 swap(second->physical, first->physical); in handle_ops_on_dev_replace()
6187 swap(second->dev, first->dev); in handle_ops_on_dev_replace()
6188 nr_extra_stripes--; in handle_ops_on_dev_replace()
6192 io_geom->num_stripes = num_stripes + nr_extra_stripes; in handle_ops_on_dev_replace()
6193 io_geom->max_errors = max_errors + nr_extra_stripes; in handle_ops_on_dev_replace()
6194 bioc->replace_nr_stripes = nr_extra_stripes; in handle_ops_on_dev_replace()
6204 io_geom->stripe_offset = offset & BTRFS_STRIPE_LEN_MASK; in btrfs_max_io_len()
6205 io_geom->stripe_nr = offset >> BTRFS_STRIPE_LEN_SHIFT; in btrfs_max_io_len()
6206 ASSERT(io_geom->stripe_offset < U32_MAX); in btrfs_max_io_len()
6208 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { in btrfs_max_io_len()
6221 io_geom->raid56_full_stripe_start = btrfs_stripe_nr_to_offset( in btrfs_max_io_len()
6222 rounddown(io_geom->stripe_nr, nr_data_stripes(map))); in btrfs_max_io_len()
6224 ASSERT(io_geom->raid56_full_stripe_start + full_stripe_len > offset); in btrfs_max_io_len()
6225 ASSERT(io_geom->raid56_full_stripe_start <= offset); in btrfs_max_io_len()
6230 if (io_geom->op == BTRFS_MAP_WRITE) in btrfs_max_io_len()
6231 return full_stripe_len - (offset - io_geom->raid56_full_stripe_start); in btrfs_max_io_len()
6238 if (map->type & BTRFS_BLOCK_GROUP_STRIPE_MASK) in btrfs_max_io_len()
6239 return BTRFS_STRIPE_LEN - io_geom->stripe_offset; in btrfs_max_io_len()
6248 dst->dev = map->stripes[io_geom->stripe_index].dev; in set_io_stripe()
6250 if (io_geom->op == BTRFS_MAP_READ && in set_io_stripe()
6251 btrfs_need_stripe_tree_update(fs_info, map->type)) in set_io_stripe()
6253 map->type, in set_io_stripe()
6254 io_geom->stripe_index, dst); in set_io_stripe()
6256 dst->physical = map->stripes[io_geom->stripe_index].physical + in set_io_stripe()
6257 io_geom->stripe_offset + in set_io_stripe()
6258 btrfs_stripe_nr_to_offset(io_geom->stripe_nr); in set_io_stripe()
6274 if (btrfs_need_stripe_tree_update(fs_info, map->type) && op != BTRFS_MAP_READ) in is_single_device_io()
6277 if ((map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) && mirror_num > 1) in is_single_device_io()
6286 io_geom->stripe_index = io_geom->stripe_nr % map->num_stripes; in map_blocks_raid0()
6287 io_geom->stripe_nr /= map->num_stripes; in map_blocks_raid0()
6288 if (io_geom->op == BTRFS_MAP_READ) in map_blocks_raid0()
6289 io_geom->mirror_num = 1; in map_blocks_raid0()
6297 if (io_geom->op != BTRFS_MAP_READ) { in map_blocks_raid1()
6298 io_geom->num_stripes = map->num_stripes; in map_blocks_raid1()
6302 if (io_geom->mirror_num) { in map_blocks_raid1()
6303 io_geom->stripe_index = io_geom->mirror_num - 1; in map_blocks_raid1()
6307 io_geom->stripe_index = find_live_mirror(fs_info, map, 0, in map_blocks_raid1()
6309 io_geom->mirror_num = io_geom->stripe_index + 1; in map_blocks_raid1()
6315 if (io_geom->op != BTRFS_MAP_READ) { in map_blocks_dup()
6316 io_geom->num_stripes = map->num_stripes; in map_blocks_dup()
6320 if (io_geom->mirror_num) { in map_blocks_dup()
6321 io_geom->stripe_index = io_geom->mirror_num - 1; in map_blocks_dup()
6325 io_geom->mirror_num = 1; in map_blocks_dup()
6333 u32 factor = map->num_stripes / map->sub_stripes; in map_blocks_raid10()
6336 io_geom->stripe_index = (io_geom->stripe_nr % factor) * map->sub_stripes; in map_blocks_raid10()
6337 io_geom->stripe_nr /= factor; in map_blocks_raid10()
6339 if (io_geom->op != BTRFS_MAP_READ) { in map_blocks_raid10()
6340 io_geom->num_stripes = map->sub_stripes; in map_blocks_raid10()
6344 if (io_geom->mirror_num) { in map_blocks_raid10()
6345 io_geom->stripe_index += io_geom->mirror_num - 1; in map_blocks_raid10()
6349 old_stripe_index = io_geom->stripe_index; in map_blocks_raid10()
6350 io_geom->stripe_index = find_live_mirror(fs_info, map, in map_blocks_raid10()
6351 io_geom->stripe_index, in map_blocks_raid10()
6353 io_geom->mirror_num = io_geom->stripe_index - old_stripe_index + 1; in map_blocks_raid10()
6372 io_geom->stripe_nr /= data_stripes; in map_blocks_raid56_write()
6375 io_geom->num_stripes = map->num_stripes; in map_blocks_raid56_write()
6376 io_geom->max_errors = btrfs_chunk_max_errors(map); in map_blocks_raid56_write()
6380 io_geom->raid56_full_stripe_start + map->start + in map_blocks_raid56_write()
6381 btrfs_stripe_nr_to_offset(data_stripes)) - in map_blocks_raid56_write()
6383 io_geom->stripe_index = 0; in map_blocks_raid56_write()
6384 io_geom->stripe_offset = 0; in map_blocks_raid56_write()
6392 ASSERT(io_geom->mirror_num <= 1); in map_blocks_raid56_read()
6394 io_geom->stripe_index = io_geom->stripe_nr % data_stripes; in map_blocks_raid56_read()
6395 io_geom->stripe_nr /= data_stripes; in map_blocks_raid56_read()
6398 io_geom->stripe_index = in map_blocks_raid56_read()
6399 (io_geom->stripe_nr + io_geom->stripe_index) % map->num_stripes; in map_blocks_raid56_read()
6401 if (io_geom->op == BTRFS_MAP_READ && io_geom->mirror_num < 1) in map_blocks_raid56_read()
6402 io_geom->mirror_num = 1; in map_blocks_raid56_read()
6408 io_geom->stripe_index = io_geom->stripe_nr % map->num_stripes; in map_blocks_single()
6409 io_geom->stripe_nr /= map->num_stripes; in map_blocks_single()
6410 io_geom->mirror_num = io_geom->stripe_index + 1; in map_blocks_single()
6437 * For non-RAID56 profiles, non-zero mirror_num means
6459 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; in btrfs_map_block()
6477 return -EINVAL; in btrfs_map_block()
6479 map_offset = logical - map->start; in btrfs_map_block()
6480 io_geom.raid56_full_stripe_start = (u64)-1; in btrfs_map_block()
6482 *length = min_t(u64, map->chunk_len - map_offset, max_len); in btrfs_map_block()
6484 down_read(&dev_replace->rwsem); in btrfs_map_block()
6491 up_read(&dev_replace->rwsem); in btrfs_map_block()
6493 switch (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) { in btrfs_map_block()
6524 if (io_geom.stripe_index >= map->num_stripes) { in btrfs_map_block()
6527 io_geom.stripe_index, map->num_stripes); in btrfs_map_block()
6528 ret = -EINVAL; in btrfs_map_block()
6533 if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL && in btrfs_map_block()
6560 ret = -ENOMEM; in btrfs_map_block()
6563 bioc->map_type = map->type; in btrfs_map_block()
6572 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK && in btrfs_map_block()
6582 bioc->full_stripe_logical = map->start + in btrfs_map_block()
6586 struct btrfs_io_stripe *dst = &bioc->stripes[i]; in btrfs_map_block()
6590 dst->dev = map->stripes[stripe_index].dev; in btrfs_map_block()
6591 dst->physical = in btrfs_map_block()
6592 map->stripes[stripe_index].physical + in btrfs_map_block()
6598 * For all other non-RAID56 profiles, just copy the target in btrfs_map_block()
6603 &bioc->stripes[i], map, &io_geom); in btrfs_map_block()
6619 if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL && in btrfs_map_block()
6625 bioc->num_stripes = io_geom.num_stripes; in btrfs_map_block()
6626 bioc->max_errors = io_geom.max_errors; in btrfs_map_block()
6627 bioc->mirror_num = io_geom.mirror_num; in btrfs_map_block()
6631 lockdep_assert_held(&dev_replace->rwsem); in btrfs_map_block()
6633 up_read(&dev_replace->rwsem); in btrfs_map_block()
6642 if (args->fsid == NULL) in dev_args_match_fs_devices()
6644 if (memcmp(fs_devices->metadata_uuid, args->fsid, BTRFS_FSID_SIZE) == 0) in dev_args_match_fs_devices()
6652 if (args->missing) { in dev_args_match_device()
6653 if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state) && in dev_args_match_device()
6654 !device->bdev) in dev_args_match_device()
6659 if (device->devid != args->devid) in dev_args_match_device()
6661 if (args->uuid && memcmp(device->uuid, args->uuid, BTRFS_UUID_SIZE) != 0) in dev_args_match_device()
6680 list_for_each_entry(device, &fs_devices->devices, dev_list) { in btrfs_find_device()
6686 list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) { in btrfs_find_device()
6689 list_for_each_entry(device, &seed_devs->devices, dev_list) { in btrfs_find_device()
6717 list_add(&device->dev_list, &fs_devices->devices); in add_missing_dev()
6718 device->fs_devices = fs_devices; in add_missing_dev()
6719 fs_devices->num_devices++; in add_missing_dev()
6721 set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state); in add_missing_dev()
6722 fs_devices->missing_devices++; in add_missing_dev()
6750 return ERR_PTR(-EINVAL); in btrfs_alloc_device()
6754 return ERR_PTR(-ENOMEM); in btrfs_alloc_device()
6756 INIT_LIST_HEAD(&dev->dev_list); in btrfs_alloc_device()
6757 INIT_LIST_HEAD(&dev->dev_alloc_list); in btrfs_alloc_device()
6758 INIT_LIST_HEAD(&dev->post_commit_list); in btrfs_alloc_device()
6760 atomic_set(&dev->dev_stats_ccnt, 0); in btrfs_alloc_device()
6762 extent_io_tree_init(fs_info, &dev->alloc_state, IO_TREE_DEVICE_ALLOC_STATE); in btrfs_alloc_device()
6775 dev->devid = tmp; in btrfs_alloc_device()
6778 memcpy(dev->uuid, uuid, BTRFS_UUID_SIZE); in btrfs_alloc_device()
6780 generate_random_uuid(dev->uuid); in btrfs_alloc_device()
6788 return ERR_PTR(-ENOMEM); in btrfs_alloc_device()
6790 rcu_assign_pointer(dev->name, name); in btrfs_alloc_device()
6809 const int data_stripes = calc_data_stripes(map->type, map->num_stripes); in btrfs_calc_stripe_length()
6811 return div_u64(map->chunk_len, data_stripes); in btrfs_calc_stripe_length()
6832 return -EOVERFLOW; in check_32bit_meta_chunk()
6861 return ERR_PTR(-ENOENT); in handle_missing_device()
6864 dev = add_missing_dev(fs_info->fs_devices, devid, uuid); in handle_missing_device()
6879 struct btrfs_fs_info *fs_info = leaf->fs_info; in read_one_chunk()
6891 logical = key->offset; in read_one_chunk()
6906 * as chunk item in tree block is already verified by tree-checker. in read_one_chunk()
6908 if (leaf->start == BTRFS_SUPER_INFO_OFFSET) { in read_one_chunk()
6917 if (map && map->start <= logical && map->start + map->chunk_len > logical) { in read_one_chunk()
6926 return -ENOMEM; in read_one_chunk()
6928 map->start = logical; in read_one_chunk()
6929 map->chunk_len = length; in read_one_chunk()
6930 map->num_stripes = num_stripes; in read_one_chunk()
6931 map->io_width = btrfs_chunk_io_width(leaf, chunk); in read_one_chunk()
6932 map->io_align = btrfs_chunk_io_align(leaf, chunk); in read_one_chunk()
6933 map->type = type; in read_one_chunk()
6938 * In that case, it can cause divide-by-zero errors later. in read_one_chunk()
6942 map->sub_stripes = btrfs_raid_array[index].sub_stripes; in read_one_chunk()
6943 map->verified_stripes = 0; in read_one_chunk()
6944 map->stripe_size = btrfs_calc_stripe_length(map); in read_one_chunk()
6946 map->stripes[i].physical = in read_one_chunk()
6954 map->stripes[i].dev = btrfs_find_device(fs_info->fs_devices, &args); in read_one_chunk()
6955 if (!map->stripes[i].dev) { in read_one_chunk()
6956 map->stripes[i].dev = handle_missing_device(fs_info, in read_one_chunk()
6958 if (IS_ERR(map->stripes[i].dev)) { in read_one_chunk()
6959 ret = PTR_ERR(map->stripes[i].dev); in read_one_chunk()
6966 &(map->stripes[i].dev->dev_state)); in read_one_chunk()
6973 map->start, map->chunk_len, ret); in read_one_chunk()
6985 device->devid = btrfs_device_id(leaf, dev_item); in fill_device_from_item()
6986 device->disk_total_bytes = btrfs_device_total_bytes(leaf, dev_item); in fill_device_from_item()
6987 device->total_bytes = device->disk_total_bytes; in fill_device_from_item()
6988 device->commit_total_bytes = device->disk_total_bytes; in fill_device_from_item()
6989 device->bytes_used = btrfs_device_bytes_used(leaf, dev_item); in fill_device_from_item()
6990 device->commit_bytes_used = device->bytes_used; in fill_device_from_item()
6991 device->type = btrfs_device_type(leaf, dev_item); in fill_device_from_item()
6992 device->io_align = btrfs_device_io_align(leaf, dev_item); in fill_device_from_item()
6993 device->io_width = btrfs_device_io_width(leaf, dev_item); in fill_device_from_item()
6994 device->sector_size = btrfs_device_sector_size(leaf, dev_item); in fill_device_from_item()
6995 WARN_ON(device->devid == BTRFS_DEV_REPLACE_DEVID); in fill_device_from_item()
6996 clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state); in fill_device_from_item()
6999 read_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE); in fill_device_from_item()
7011 /* This will match only for multi-device seed fs */ in open_seed_devices()
7012 list_for_each_entry(fs_devices, &fs_info->fs_devices->seed_list, seed_list) in open_seed_devices()
7013 if (!memcmp(fs_devices->fsid, fsid, BTRFS_FSID_SIZE)) in open_seed_devices()
7020 return ERR_PTR(-ENOENT); in open_seed_devices()
7026 fs_devices->seeding = true; in open_seed_devices()
7027 fs_devices->opened = 1; in open_seed_devices()
7033 * respective fs_devices and anchor it at fs_info->fs_devices->seed_list in open_seed_devices()
7039 ret = open_fs_devices(fs_devices, BLK_OPEN_READ, fs_info->bdev_holder); in open_seed_devices()
7045 if (!fs_devices->seeding) { in open_seed_devices()
7048 return ERR_PTR(-EINVAL); in open_seed_devices()
7051 list_add(&fs_devices->seed_list, &fs_info->fs_devices->seed_list); in open_seed_devices()
7060 struct btrfs_fs_info *fs_info = leaf->fs_info; in read_one_dev()
7061 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in read_one_dev()
7077 if (memcmp(fs_uuid, fs_devices->metadata_uuid, BTRFS_FSID_SIZE)) { in read_one_dev()
7083 device = btrfs_find_device(fs_info->fs_devices, &args); in read_one_dev()
7088 return -ENOENT; in read_one_dev()
7100 if (!device->bdev) { in read_one_dev()
7104 return -ENOENT; in read_one_dev()
7110 if (!device->bdev && in read_one_dev()
7111 !test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) { in read_one_dev()
7115 * device->bdev is NULL, and so we have to set in read_one_dev()
7116 * device->missing to one here in read_one_dev()
7118 device->fs_devices->missing_devices++; in read_one_dev()
7119 set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state); in read_one_dev()
7123 if (device->fs_devices != fs_devices) { in read_one_dev()
7125 &device->dev_state)); in read_one_dev()
7127 list_move(&device->dev_list, &fs_devices->devices); in read_one_dev()
7128 device->fs_devices->num_devices--; in read_one_dev()
7129 fs_devices->num_devices++; in read_one_dev()
7131 device->fs_devices->missing_devices--; in read_one_dev()
7132 fs_devices->missing_devices++; in read_one_dev()
7134 device->fs_devices = fs_devices; in read_one_dev()
7138 if (device->fs_devices != fs_info->fs_devices) { in read_one_dev()
7139 BUG_ON(test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)); in read_one_dev()
7140 if (device->generation != in read_one_dev()
7142 return -EINVAL; in read_one_dev()
7146 if (device->bdev) { in read_one_dev()
7147 u64 max_total_bytes = bdev_nr_bytes(device->bdev); in read_one_dev()
7149 if (device->total_bytes > max_total_bytes) { in read_one_dev()
7152 max_total_bytes, device->total_bytes); in read_one_dev()
7153 return -EINVAL; in read_one_dev()
7156 set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state); in read_one_dev()
7157 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) && in read_one_dev()
7158 !test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { in read_one_dev()
7159 device->fs_devices->total_rw_bytes += device->total_bytes; in read_one_dev()
7160 atomic64_add(device->total_bytes - device->bytes_used, in read_one_dev()
7161 &fs_info->free_chunk_space); in read_one_dev()
7169 struct btrfs_super_block *super_copy = fs_info->super_copy; in btrfs_read_sys_array()
7183 ASSERT(BTRFS_SUPER_INFO_SIZE <= fs_info->nodesize); in btrfs_read_sys_array()
7188 * that's fine, we will not go beyond system chunk array anyway. in btrfs_read_sys_array()
7192 return -ENOMEM; in btrfs_read_sys_array()
7198 array_ptr = super_copy->sys_chunk_array; in btrfs_read_sys_array()
7218 ret = -EIO; in btrfs_read_sys_array()
7236 ret = -EIO; in btrfs_read_sys_array()
7245 ret = -EIO; in btrfs_read_sys_array()
7270 return -EIO; in btrfs_read_sys_array()
7274 * Check if all chunks in the fs are OK for read-write degraded mount
7301 map->type); in btrfs_check_rw_degradable()
7302 for (i = 0; i < map->num_stripes; i++) { in btrfs_check_rw_degradable()
7303 struct btrfs_device *dev = map->stripes[i].dev; in btrfs_check_rw_degradable()
7305 if (!dev || !dev->bdev || in btrfs_check_rw_degradable()
7306 test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) || in btrfs_check_rw_degradable()
7307 dev->last_flush_error) in btrfs_check_rw_degradable()
7316 map->start, missing, max_tolerated); in btrfs_check_rw_degradable()
7321 next_start = map->start + map->chunk_len; in btrfs_check_rw_degradable()
7324 map = btrfs_find_chunk_map(fs_info, next_start, U64_MAX - next_start); in btrfs_check_rw_degradable()
7341 struct btrfs_root *root = fs_info->chunk_root; in btrfs_read_chunk_tree()
7354 return -ENOMEM; in btrfs_read_chunk_tree()
7368 fs_info->fs_devices->total_rw_bytes = 0; in btrfs_read_chunk_tree()
7380 ASSERT(!test_bit(BTRFS_FS_OPEN, &fs_info->flags)); in btrfs_read_chunk_tree()
7381 path->skip_locking = 1; in btrfs_read_chunk_tree()
7387 * item - BTRFS_FIRST_CHUNK_TREE_OBJECTID). in btrfs_read_chunk_tree()
7393 struct extent_buffer *node = path->nodes[1]; in btrfs_read_chunk_tree()
7395 leaf = path->nodes[0]; in btrfs_read_chunk_tree()
7396 slot = path->slots[0]; in btrfs_read_chunk_tree()
7399 if (last_ra_node != node->start) { in btrfs_read_chunk_tree()
7401 last_ra_node = node->start; in btrfs_read_chunk_tree()
7417 * fs_info->chunk_mutex. Plus, to avoid lockdep warnings, in btrfs_read_chunk_tree()
7418 * we always lock first fs_info->chunk_mutex before in btrfs_read_chunk_tree()
7439 if (total_dev != fs_info->fs_devices->total_devices) { in btrfs_read_chunk_tree()
7442 btrfs_super_num_devices(fs_info->super_copy), in btrfs_read_chunk_tree()
7444 fs_info->fs_devices->total_devices = total_dev; in btrfs_read_chunk_tree()
7445 btrfs_set_super_num_devices(fs_info->super_copy, total_dev); in btrfs_read_chunk_tree()
7447 if (btrfs_super_total_bytes(fs_info->super_copy) < in btrfs_read_chunk_tree()
7448 fs_info->fs_devices->total_rw_bytes) { in btrfs_read_chunk_tree()
7451 btrfs_super_total_bytes(fs_info->super_copy), in btrfs_read_chunk_tree()
7452 fs_info->fs_devices->total_rw_bytes); in btrfs_read_chunk_tree()
7453 ret = -EINVAL; in btrfs_read_chunk_tree()
7466 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs; in btrfs_init_devices_late()
7470 fs_devices->fs_info = fs_info; in btrfs_init_devices_late()
7472 mutex_lock(&fs_devices->device_list_mutex); in btrfs_init_devices_late()
7473 list_for_each_entry(device, &fs_devices->devices, dev_list) in btrfs_init_devices_late()
7474 device->fs_info = fs_info; in btrfs_init_devices_late()
7476 list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) { in btrfs_init_devices_late()
7477 list_for_each_entry(device, &seed_devs->devices, dev_list) { in btrfs_init_devices_late()
7478 device->fs_info = fs_info; in btrfs_init_devices_late()
7484 seed_devs->fs_info = fs_info; in btrfs_init_devices_late()
7486 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_init_devices_late()
7523 if (!device->fs_info->dev_root) in btrfs_device_init_dev_stats()
7528 key.offset = device->devid; in btrfs_device_init_dev_stats()
7529 ret = btrfs_search_slot(NULL, device->fs_info->dev_root, &key, path, 0, 0); in btrfs_device_init_dev_stats()
7533 device->dev_stats_valid = 1; in btrfs_device_init_dev_stats()
7537 slot = path->slots[0]; in btrfs_device_init_dev_stats()
7538 eb = path->nodes[0]; in btrfs_device_init_dev_stats()
7551 device->dev_stats_valid = 1; in btrfs_device_init_dev_stats()
7560 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs; in btrfs_init_dev_stats()
7567 return -ENOMEM; in btrfs_init_dev_stats()
7569 mutex_lock(&fs_devices->device_list_mutex); in btrfs_init_dev_stats()
7570 list_for_each_entry(device, &fs_devices->devices, dev_list) { in btrfs_init_dev_stats()
7575 list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) { in btrfs_init_dev_stats()
7576 list_for_each_entry(device, &seed_devs->devices, dev_list) { in btrfs_init_dev_stats()
7583 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_init_dev_stats()
7592 struct btrfs_fs_info *fs_info = trans->fs_info; in update_dev_stat_item()
7593 struct btrfs_root *dev_root = fs_info->dev_root; in update_dev_stat_item()
7603 key.offset = device->devid; in update_dev_stat_item()
7607 return -ENOMEM; in update_dev_stat_item()
7608 ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1); in update_dev_stat_item()
7617 btrfs_item_size(path->nodes[0], path->slots[0]) < sizeof(*ptr)) { in update_dev_stat_item()
7642 eb = path->nodes[0]; in update_dev_stat_item()
7643 ptr = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dev_stats_item); in update_dev_stat_item()
7659 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_run_dev_stats()
7660 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_run_dev_stats()
7665 mutex_lock(&fs_devices->device_list_mutex); in btrfs_run_dev_stats()
7666 list_for_each_entry(device, &fs_devices->devices, dev_list) { in btrfs_run_dev_stats()
7667 stats_cnt = atomic_read(&device->dev_stats_ccnt); in btrfs_run_dev_stats()
7668 if (!device->dev_stats_valid || stats_cnt == 0) in btrfs_run_dev_stats()
7673 * There is a LOAD-LOAD control dependency between the value of in btrfs_run_dev_stats()
7674 * dev_stats_ccnt and updating the on-disk values which requires in btrfs_run_dev_stats()
7675 * reading the in-memory counters. Such control dependencies in btrfs_run_dev_stats()
7687 atomic_sub(stats_cnt, &device->dev_stats_ccnt); in btrfs_run_dev_stats()
7689 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_run_dev_stats()
7698 if (!dev->dev_stats_valid) in btrfs_dev_stat_inc_and_print()
7700 btrfs_err_rl_in_rcu(dev->fs_info, in btrfs_dev_stat_inc_and_print()
7720 btrfs_info_in_rcu(dev->fs_info, in btrfs_dev_stat_print_on_load()
7735 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_get_dev_stats()
7738 mutex_lock(&fs_devices->device_list_mutex); in btrfs_get_dev_stats()
7739 args.devid = stats->devid; in btrfs_get_dev_stats()
7740 dev = btrfs_find_device(fs_info->fs_devices, &args); in btrfs_get_dev_stats()
7741 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_get_dev_stats()
7745 return -ENODEV; in btrfs_get_dev_stats()
7746 } else if (!dev->dev_stats_valid) { in btrfs_get_dev_stats()
7748 return -ENODEV; in btrfs_get_dev_stats()
7749 } else if (stats->flags & BTRFS_DEV_STATS_RESET) { in btrfs_get_dev_stats()
7751 if (stats->nr_items > i) in btrfs_get_dev_stats()
7752 stats->values[i] = in btrfs_get_dev_stats()
7758 current->comm, task_pid_nr(current)); in btrfs_get_dev_stats()
7761 if (stats->nr_items > i) in btrfs_get_dev_stats()
7762 stats->values[i] = btrfs_dev_stat_read(dev, i); in btrfs_get_dev_stats()
7764 if (stats->nr_items > BTRFS_DEV_STAT_VALUES_MAX) in btrfs_get_dev_stats()
7765 stats->nr_items = BTRFS_DEV_STAT_VALUES_MAX; in btrfs_get_dev_stats()
7780 ASSERT(trans->state == TRANS_STATE_COMMIT_DOING); in btrfs_commit_device_sizes()
7782 if (list_empty(&trans->dev_update_list)) in btrfs_commit_device_sizes()
7790 mutex_lock(&trans->fs_info->chunk_mutex); in btrfs_commit_device_sizes()
7791 list_for_each_entry_safe(curr, next, &trans->dev_update_list, in btrfs_commit_device_sizes()
7793 list_del_init(&curr->post_commit_list); in btrfs_commit_device_sizes()
7794 curr->commit_total_bytes = curr->disk_total_bytes; in btrfs_commit_device_sizes()
7795 curr->commit_bytes_used = curr->bytes_used; in btrfs_commit_device_sizes()
7797 mutex_unlock(&trans->fs_info->chunk_mutex); in btrfs_commit_device_sizes()
7801 * Multiplicity factor for simple profiles: DUP, RAID1-like and RAID10.
7829 ret = -EUCLEAN; in verify_one_dev_extent()
7837 physical_offset, devid, map->start, physical_len, in verify_one_dev_extent()
7839 ret = -EUCLEAN; in verify_one_dev_extent()
7853 for (i = 0; i < map->num_stripes; i++) { in verify_one_dev_extent()
7854 if (map->stripes[i].dev->devid == devid && in verify_one_dev_extent()
7855 map->stripes[i].physical == physical_offset) { in verify_one_dev_extent()
7857 if (map->verified_stripes >= map->num_stripes) { in verify_one_dev_extent()
7860 map->start); in verify_one_dev_extent()
7861 ret = -EUCLEAN; in verify_one_dev_extent()
7864 map->verified_stripes++; in verify_one_dev_extent()
7872 ret = -EUCLEAN; in verify_one_dev_extent()
7876 dev = btrfs_find_device(fs_info->fs_devices, &args); in verify_one_dev_extent()
7879 ret = -EUCLEAN; in verify_one_dev_extent()
7883 if (physical_offset + physical_len > dev->disk_total_bytes) { in verify_one_dev_extent()
7887 dev->disk_total_bytes); in verify_one_dev_extent()
7888 ret = -EUCLEAN; in verify_one_dev_extent()
7892 if (dev->zone_info) { in verify_one_dev_extent()
7893 u64 zone_size = dev->zone_info->zone_size; in verify_one_dev_extent()
7900 ret = -EUCLEAN; in verify_one_dev_extent()
7915 read_lock(&fs_info->mapping_tree_lock); in verify_chunk_dev_extent_mapping()
7916 for (node = rb_first_cached(&fs_info->mapping_tree); node; node = rb_next(node)) { in verify_chunk_dev_extent_mapping()
7920 if (map->num_stripes != map->verified_stripes) { in verify_chunk_dev_extent_mapping()
7923 map->start, map->verified_stripes, map->num_stripes); in verify_chunk_dev_extent_mapping()
7924 ret = -EUCLEAN; in verify_chunk_dev_extent_mapping()
7929 read_unlock(&fs_info->mapping_tree_lock); in verify_chunk_dev_extent_mapping()
7943 struct btrfs_root *root = fs_info->dev_root; in btrfs_verify_dev_extents()
7968 return -ENOMEM; in btrfs_verify_dev_extents()
7970 path->reada = READA_FORWARD; in btrfs_verify_dev_extents()
7975 if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { in btrfs_verify_dev_extents()
7981 ret = -EUCLEAN; in btrfs_verify_dev_extents()
7986 struct extent_buffer *leaf = path->nodes[0]; in btrfs_verify_dev_extents()
7988 int slot = path->slots[0]; in btrfs_verify_dev_extents()
8009 ret = -EUCLEAN; in btrfs_verify_dev_extents()
8045 spin_lock(&fs_info->swapfile_pins_lock); in btrfs_pinned_by_swapfile()
8046 node = fs_info->swapfile_pins.rb_node; in btrfs_pinned_by_swapfile()
8049 if (ptr < sp->ptr) in btrfs_pinned_by_swapfile()
8050 node = node->rb_left; in btrfs_pinned_by_swapfile()
8051 else if (ptr > sp->ptr) in btrfs_pinned_by_swapfile()
8052 node = node->rb_right; in btrfs_pinned_by_swapfile()
8056 spin_unlock(&fs_info->swapfile_pins_lock); in btrfs_pinned_by_swapfile()
8063 struct btrfs_fs_info *fs_info = cache->fs_info; in relocating_repair_kthread()
8067 target = cache->start; in relocating_repair_kthread()
8070 sb_start_write(fs_info->sb); in relocating_repair_kthread()
8075 sb_end_write(fs_info->sb); in relocating_repair_kthread()
8076 return -EBUSY; in relocating_repair_kthread()
8079 mutex_lock(&fs_info->reclaim_bgs_lock); in relocating_repair_kthread()
8086 if (!test_bit(BLOCK_GROUP_FLAG_RELOCATING_REPAIR, &cache->runtime_flags)) in relocating_repair_kthread()
8101 mutex_unlock(&fs_info->reclaim_bgs_lock); in relocating_repair_kthread()
8103 sb_end_write(fs_info->sb); in relocating_repair_kthread()
8123 if (test_and_set_bit(BLOCK_GROUP_FLAG_RELOCATING_REPAIR, &cache->runtime_flags)) { in btrfs_repair_one_zone()
8129 "btrfs-relocating-repair"); in btrfs_repair_one_zone()
8142 u64 stripe_start = bioc->full_stripe_logical + in map_raid56_repair_block()
8150 smap->dev = bioc->stripes[i].dev; in map_raid56_repair_block()
8151 smap->physical = bioc->stripes[i].physical + in map_raid56_repair_block()
8152 ((logical - bioc->full_stripe_logical) & in map_raid56_repair_block()
8165 * - Call btrfs_bio_counter_inc_blocked() first
8166 * - The range does not cross stripe boundary
8167 * - Has a valid @mirror_num passed in.
8192 /* Map the RAID56 multi-stripe writes to a single one. */ in btrfs_map_repair_block()
8193 if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) { in btrfs_map_repair_block()
8198 ASSERT(mirror_num <= bioc->num_stripes); in btrfs_map_repair_block()
8199 smap->dev = bioc->stripes[mirror_num - 1].dev; in btrfs_map_repair_block()
8200 smap->physical = bioc->stripes[mirror_num - 1].physical; in btrfs_map_repair_block()
8203 ASSERT(smap->dev); in btrfs_map_repair_block()