Lines Matching +full:ref2 +full:-
1 // SPDX-License-Identifier: GPL-2.0
20 #include "extent-tree.h"
22 #include "disk-io.h"
23 #include "print-tree.h"
27 #include "free-space-cache.h"
28 #include "free-space-tree.h"
30 #include "ref-verify.h"
31 #include "space-info.h"
32 #include "block-rsv.h"
35 #include "dev-replace.h"
38 #include "root-tree.h"
39 #include "file-item.h"
41 #include "tree-checker.h"
42 #include "raid-stripe-tree.h"
66 return (cache->flags & bits) == bits; in block_group_bits()
79 return -ENOMEM; in btrfs_lookup_data_extent()
118 offset = fs_info->nodesize; in btrfs_lookup_extent_info()
124 return -ENOMEM; in btrfs_lookup_extent_info()
140 if (path->slots[0]) { in btrfs_lookup_extent_info()
141 path->slots[0]--; in btrfs_lookup_extent_info()
142 btrfs_item_key_to_cpu(path->nodes[0], &key, in btrfs_lookup_extent_info()
143 path->slots[0]); in btrfs_lookup_extent_info()
146 key.offset == fs_info->nodesize) in btrfs_lookup_extent_info()
152 struct extent_buffer *leaf = path->nodes[0]; in btrfs_lookup_extent_info()
154 const u32 item_size = btrfs_item_size(leaf, path->slots[0]); in btrfs_lookup_extent_info()
157 ret = -EUCLEAN; in btrfs_lookup_extent_info()
165 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); in btrfs_lookup_extent_info()
168 ret = -EUCLEAN; in btrfs_lookup_extent_info()
176 owner = btrfs_get_extent_owner_root(fs_info, leaf, path->slots[0]); in btrfs_lookup_extent_info()
183 delayed_refs = &trans->transaction->delayed_refs; in btrfs_lookup_extent_info()
184 spin_lock(&delayed_refs->lock); in btrfs_lookup_extent_info()
187 if (!mutex_trylock(&head->mutex)) { in btrfs_lookup_extent_info()
188 refcount_inc(&head->refs); in btrfs_lookup_extent_info()
189 spin_unlock(&delayed_refs->lock); in btrfs_lookup_extent_info()
197 mutex_lock(&head->mutex); in btrfs_lookup_extent_info()
198 mutex_unlock(&head->mutex); in btrfs_lookup_extent_info()
202 spin_lock(&head->lock); in btrfs_lookup_extent_info()
203 if (head->extent_op && head->extent_op->update_flags) in btrfs_lookup_extent_info()
204 extent_flags |= head->extent_op->flags_to_set; in btrfs_lookup_extent_info()
206 num_refs += head->ref_mod; in btrfs_lookup_extent_info()
207 spin_unlock(&head->lock); in btrfs_lookup_extent_info()
208 mutex_unlock(&head->mutex); in btrfs_lookup_extent_info()
210 spin_unlock(&delayed_refs->lock); in btrfs_lookup_extent_info()
239 * for pointers in non-shared tree blocks. For a given pointer in a block,
242 * b-tree searching. The full back refs is for pointers in tree blocks not
285 * - multiple snapshots, subvolumes, or different generations in one subvol
286 * - different files inside a single subvolume
287 * - different offsets inside a file (bookend extents in file.c)
291 * - Objectid of the subvolume root
292 * - objectid of the file holding the reference
293 * - original offset in the file
294 * - how many bookend extents
301 * - number of pointers in the tree leaf
318 * - Different subvolumes
339 struct btrfs_fs_info *fs_info = eb->fs_info; in btrfs_get_extent_inline_ref_type()
361 if (offset && IS_ALIGNED(offset, fs_info->sectorsize)) in btrfs_get_extent_inline_ref_type()
374 IS_ALIGNED(offset, fs_info->sectorsize)) in btrfs_get_extent_inline_ref_type()
387 eb->start, (unsigned long)iref, type); in btrfs_get_extent_inline_ref_type()
433 struct btrfs_root *root = btrfs_extent_root(trans->fs_info, bytenr); in lookup_extent_data_ref()
452 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); in lookup_extent_data_ref()
458 return -ENOENT; in lookup_extent_data_ref()
462 ret = -ENOENT; in lookup_extent_data_ref()
463 leaf = path->nodes[0]; in lookup_extent_data_ref()
466 if (path->slots[0] >= nritems) { in lookup_extent_data_ref()
470 return -ENOENT; in lookup_extent_data_ref()
474 leaf = path->nodes[0]; in lookup_extent_data_ref()
479 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); in lookup_extent_data_ref()
484 ref = btrfs_item_ptr(leaf, path->slots[0], in lookup_extent_data_ref()
496 path->slots[0]++; in lookup_extent_data_ref()
507 struct btrfs_root *root = btrfs_extent_root(trans->fs_info, bytenr); in insert_extent_data_ref()
517 if (node->parent) { in insert_extent_data_ref()
519 key.offset = node->parent; in insert_extent_data_ref()
523 key.offset = hash_extent_data_ref(node->ref_root, owner, offset); in insert_extent_data_ref()
528 if (ret && ret != -EEXIST) in insert_extent_data_ref()
531 leaf = path->nodes[0]; in insert_extent_data_ref()
532 if (node->parent) { in insert_extent_data_ref()
534 ref = btrfs_item_ptr(leaf, path->slots[0], in insert_extent_data_ref()
537 btrfs_set_shared_data_ref_count(leaf, ref, node->ref_mod); in insert_extent_data_ref()
540 num_refs += node->ref_mod; in insert_extent_data_ref()
545 while (ret == -EEXIST) { in insert_extent_data_ref()
546 ref = btrfs_item_ptr(leaf, path->slots[0], in insert_extent_data_ref()
548 if (match_extent_data_ref(leaf, ref, node->ref_root, in insert_extent_data_ref()
555 if (ret && ret != -EEXIST) in insert_extent_data_ref()
558 leaf = path->nodes[0]; in insert_extent_data_ref()
560 ref = btrfs_item_ptr(leaf, path->slots[0], in insert_extent_data_ref()
563 btrfs_set_extent_data_ref_root(leaf, ref, node->ref_root); in insert_extent_data_ref()
566 btrfs_set_extent_data_ref_count(leaf, ref, node->ref_mod); in insert_extent_data_ref()
569 num_refs += node->ref_mod; in insert_extent_data_ref()
587 struct btrfs_shared_data_ref *ref2 = NULL; in remove_extent_data_ref() local
592 leaf = path->nodes[0]; in remove_extent_data_ref()
593 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); in remove_extent_data_ref()
596 ref1 = btrfs_item_ptr(leaf, path->slots[0], in remove_extent_data_ref()
600 ref2 = btrfs_item_ptr(leaf, path->slots[0], in remove_extent_data_ref()
602 num_refs = btrfs_shared_data_ref_count(leaf, ref2); in remove_extent_data_ref()
604 btrfs_err(trans->fs_info, in remove_extent_data_ref()
607 btrfs_abort_transaction(trans, -EUCLEAN); in remove_extent_data_ref()
608 return -EUCLEAN; in remove_extent_data_ref()
612 num_refs -= refs_to_drop; in remove_extent_data_ref()
620 btrfs_set_shared_data_ref_count(leaf, ref2, num_refs); in remove_extent_data_ref()
632 struct btrfs_shared_data_ref *ref2; in extent_data_ref_count() local
636 leaf = path->nodes[0]; in extent_data_ref_count()
637 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); in extent_data_ref_count()
647 ref1 = (struct btrfs_extent_data_ref *)(&iref->offset); in extent_data_ref_count()
650 ref2 = (struct btrfs_shared_data_ref *)(iref + 1); in extent_data_ref_count()
651 num_refs = btrfs_shared_data_ref_count(leaf, ref2); in extent_data_ref_count()
654 ref1 = btrfs_item_ptr(leaf, path->slots[0], in extent_data_ref_count()
658 ref2 = btrfs_item_ptr(leaf, path->slots[0], in extent_data_ref_count()
660 num_refs = btrfs_shared_data_ref_count(leaf, ref2); in extent_data_ref_count()
672 struct btrfs_root *root = btrfs_extent_root(trans->fs_info, bytenr); in lookup_tree_block_ref()
685 ret = btrfs_search_slot(trans, root, &key, path, -1, 1); in lookup_tree_block_ref()
687 ret = -ENOENT; in lookup_tree_block_ref()
696 struct btrfs_root *root = btrfs_extent_root(trans->fs_info, bytenr); in insert_tree_block_ref()
701 if (node->parent) { in insert_tree_block_ref()
703 key.offset = node->parent; in insert_tree_block_ref()
706 key.offset = node->ref_root; in insert_tree_block_ref()
736 if (!path->nodes[level]) in find_next_key()
738 if (path->slots[level] + 1 >= in find_next_key()
739 btrfs_header_nritems(path->nodes[level])) in find_next_key()
742 btrfs_item_key_to_cpu(path->nodes[level], key, in find_next_key()
743 path->slots[level] + 1); in find_next_key()
745 btrfs_node_key_to_cpu(path->nodes[level], key, in find_next_key()
746 path->slots[level] + 1); in find_next_key()
757 * should be inserted, and -ENOENT is returned.
760 * points to the extent item, and -EAGAIN is returned.
773 struct btrfs_fs_info *fs_info = trans->fs_info; in lookup_inline_extent_backref()
797 path->search_for_extension = 1; in lookup_inline_extent_backref()
798 path->keep_locks = 1; in lookup_inline_extent_backref()
800 extra_size = -1; in lookup_inline_extent_backref()
822 if (path->slots[0]) { in lookup_inline_extent_backref()
823 path->slots[0]--; in lookup_inline_extent_backref()
824 btrfs_item_key_to_cpu(path->nodes[0], &key, in lookup_inline_extent_backref()
825 path->slots[0]); in lookup_inline_extent_backref()
841 ret = -ENOENT; in lookup_inline_extent_backref()
844 btrfs_print_leaf(path->nodes[0]); in lookup_inline_extent_backref()
849 ret = -EUCLEAN; in lookup_inline_extent_backref()
853 leaf = path->nodes[0]; in lookup_inline_extent_backref()
854 item_size = btrfs_item_size(leaf, path->slots[0]); in lookup_inline_extent_backref()
856 ret = -EUCLEAN; in lookup_inline_extent_backref()
864 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); in lookup_inline_extent_backref()
880 ret = -ENOENT; in lookup_inline_extent_backref()
890 ret = -EUCLEAN; in lookup_inline_extent_backref()
903 dref = (struct btrfs_extent_data_ref *)(&iref->offset); in lookup_inline_extent_backref()
935 ret = -EUCLEAN; in lookup_inline_extent_backref()
936 btrfs_print_leaf(path->nodes[0]); in lookup_inline_extent_backref()
939 path->slots[0], root_objectid, owner, offset, parent); in lookup_inline_extent_backref()
943 if (ret == -ENOENT && insert) { in lookup_inline_extent_backref()
946 ret = -EAGAIN; in lookup_inline_extent_backref()
958 ret = -EAGAIN; in lookup_inline_extent_backref()
965 path->keep_locks = 0; in lookup_inline_extent_backref()
966 path->search_for_extension = 0; in lookup_inline_extent_backref()
992 leaf = path->nodes[0]; in setup_inline_extent_backref()
993 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); in setup_inline_extent_backref()
994 item_offset = (unsigned long)iref - (unsigned long)ei; in setup_inline_extent_backref()
1001 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); in setup_inline_extent_backref()
1009 end = (unsigned long)ei + btrfs_item_size(leaf, path->slots[0]); in setup_inline_extent_backref()
1010 if (ptr < end - size) in setup_inline_extent_backref()
1012 end - size - ptr); in setup_inline_extent_backref()
1018 dref = (struct btrfs_extent_data_ref *)(&iref->offset); in setup_inline_extent_backref()
1047 if (ret != -ENOENT) in lookup_extent_backref()
1073 struct extent_buffer *leaf = path->nodes[0]; in update_inline_extent_backref()
1074 struct btrfs_fs_info *fs_info = leaf->fs_info; in update_inline_extent_backref()
1085 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); in update_inline_extent_backref()
1091 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); in update_inline_extent_backref()
1093 extent_size = fs_info->nodesize; in update_inline_extent_backref()
1098 "invalid refs_to_mod for extent %llu num_bytes %u, has %d expect >= -%llu", in update_inline_extent_backref()
1100 return -EUCLEAN; in update_inline_extent_backref()
1113 return -EUCLEAN; in update_inline_extent_backref()
1116 dref = (struct btrfs_extent_data_ref *)(&iref->offset); in update_inline_extent_backref()
1131 if (unlikely(refs_to_mod != -1)) { in update_inline_extent_backref()
1134 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); in update_inline_extent_backref()
1138 "invalid refs_to_mod for tree block %llu, has %d expect -1", in update_inline_extent_backref()
1140 return -EUCLEAN; in update_inline_extent_backref()
1144 if (unlikely(refs_to_mod < 0 && refs < -refs_to_mod)) { in update_inline_extent_backref()
1148 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); in update_inline_extent_backref()
1150 extent_size = fs_info->nodesize; in update_inline_extent_backref()
1155 "invalid refs_to_mod for backref entry, iref %lu extent %llu num_bytes %u, has %d expect >= -%llu", in update_inline_extent_backref()
1158 return -EUCLEAN; in update_inline_extent_backref()
1169 item_size = btrfs_item_size(leaf, path->slots[0]); in update_inline_extent_backref()
1174 end - ptr - size); in update_inline_extent_backref()
1175 item_size -= size; in update_inline_extent_backref()
1202 btrfs_print_leaf(path->nodes[0]); in insert_inline_extent_backref()
1203 btrfs_crit(trans->fs_info, in insert_inline_extent_backref()
1205 bytenr, num_bytes, root_objectid, path->slots[0]); in insert_inline_extent_backref()
1206 return -EUCLEAN; in insert_inline_extent_backref()
1210 } else if (ret == -ENOENT) { in insert_inline_extent_backref()
1230 -refs_to_drop, NULL); in remove_extent_backref()
1247 len -= aligned_start - start; in btrfs_issue_discard()
1264 u64 size = sb_start - start; in btrfs_issue_discard()
1276 start += sb_end - start; in btrfs_issue_discard()
1281 bytes_left = end - start; in btrfs_issue_discard()
1291 else if (ret != -EOPNOTSUPP) in btrfs_issue_discard()
1300 bytes_left = end - start; in btrfs_issue_discard()
1311 if (ret != -EOPNOTSUPP) in btrfs_issue_discard()
1317 bytes_left -= bytes_to_discard; in btrfs_issue_discard()
1321 ret = -ERESTARTSYS; in btrfs_issue_discard()
1331 struct btrfs_device *dev = stripe->dev; in do_discard_extent()
1332 struct btrfs_fs_info *fs_info = dev->fs_info; in do_discard_extent()
1333 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; in do_discard_extent()
1334 u64 phys = stripe->physical; in do_discard_extent()
1335 u64 len = stripe->length; in do_discard_extent()
1348 dev != dev_replace->srcdev) in do_discard_extent()
1354 ret = btrfs_reset_device_zone(dev_replace->tgtdev, phys, len, in do_discard_extent()
1357 } else if (bdev_max_discard_sectors(stripe->dev->bdev)) { in do_discard_extent()
1358 ret = btrfs_issue_discard(dev->bdev, phys, len, &discarded); in do_discard_extent()
1387 num_bytes = end - cur; in btrfs_discard_extent()
1391 if (ret == -EOPNOTSUPP) in btrfs_discard_extent()
1400 if (!stripe->dev->bdev) { in btrfs_discard_extent()
1406 &stripe->dev->dev_state)) in btrfs_discard_extent()
1415 if (ret != -EOPNOTSUPP) in btrfs_discard_extent()
1433 /* Can return -ENOMEM */
1437 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_inc_extent_ref()
1440 ASSERT(generic_ref->type != BTRFS_REF_NOT_SET && in btrfs_inc_extent_ref()
1441 generic_ref->action); in btrfs_inc_extent_ref()
1442 BUG_ON(generic_ref->type == BTRFS_REF_METADATA && in btrfs_inc_extent_ref()
1443 generic_ref->ref_root == BTRFS_TREE_LOG_OBJECTID); in btrfs_inc_extent_ref()
1445 if (generic_ref->type == BTRFS_REF_METADATA) in btrfs_inc_extent_ref()
1478 u64 bytenr = node->bytenr; in __btrfs_inc_extent_ref()
1479 u64 num_bytes = node->num_bytes; in __btrfs_inc_extent_ref()
1483 int refs_to_add = node->ref_mod; in __btrfs_inc_extent_ref()
1488 return -ENOMEM; in __btrfs_inc_extent_ref()
1492 node->parent, node->ref_root, owner, in __btrfs_inc_extent_ref()
1494 if ((ret < 0 && ret != -EAGAIN) || !ret) in __btrfs_inc_extent_ref()
1498 * Ok we had -EAGAIN which means we didn't have space to insert and in __btrfs_inc_extent_ref()
1502 leaf = path->nodes[0]; in __btrfs_inc_extent_ref()
1503 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); in __btrfs_inc_extent_ref()
1504 item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); in __btrfs_inc_extent_ref()
1529 u64 root = href->owning_root; in free_head_ref_squota_rsv()
1536 !href->is_data || !is_fstree(root)) in free_head_ref_squota_rsv()
1539 btrfs_qgroup_free_refroot(fs_info, root, href->reserved_bytes, in free_head_ref_squota_rsv()
1553 trace_run_delayed_data_ref(trans->fs_info, node); in run_delayed_data_ref()
1555 if (node->type == BTRFS_SHARED_DATA_REF_KEY) in run_delayed_data_ref()
1556 parent = node->parent; in run_delayed_data_ref()
1558 if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) { in run_delayed_data_ref()
1561 .root = href->owning_root, in run_delayed_data_ref()
1562 .num_bytes = node->num_bytes, in run_delayed_data_ref()
1565 .generation = trans->transid, in run_delayed_data_ref()
1571 flags |= extent_op->flags_to_set; in run_delayed_data_ref()
1573 key.objectid = node->bytenr; in run_delayed_data_ref()
1575 key.offset = node->num_bytes; in run_delayed_data_ref()
1577 ret = alloc_reserved_file_extent(trans, parent, node->ref_root, in run_delayed_data_ref()
1579 node->ref_mod, in run_delayed_data_ref()
1580 href->owning_root); in run_delayed_data_ref()
1581 free_head_ref_squota_rsv(trans->fs_info, href); in run_delayed_data_ref()
1583 ret = btrfs_record_squota_delta(trans->fs_info, &delta); in run_delayed_data_ref()
1584 } else if (node->action == BTRFS_ADD_DELAYED_REF) { in run_delayed_data_ref()
1586 } else if (node->action == BTRFS_DROP_DELAYED_REF) { in run_delayed_data_ref()
1599 if (extent_op->update_flags) { in __run_delayed_extent_op()
1600 flags |= extent_op->flags_to_set; in __run_delayed_extent_op()
1604 if (extent_op->update_key) { in __run_delayed_extent_op()
1608 btrfs_set_tree_block_key(leaf, bi, &extent_op->key); in __run_delayed_extent_op()
1616 struct btrfs_fs_info *fs_info = trans->fs_info; in run_delayed_extent_op()
1634 return -ENOMEM; in run_delayed_extent_op()
1636 key.objectid = head->bytenr; in run_delayed_extent_op()
1640 key.offset = head->level; in run_delayed_extent_op()
1643 key.offset = head->num_bytes; in run_delayed_extent_op()
1653 if (path->slots[0] > 0) { in run_delayed_extent_op()
1654 path->slots[0]--; in run_delayed_extent_op()
1655 btrfs_item_key_to_cpu(path->nodes[0], &key, in run_delayed_extent_op()
1656 path->slots[0]); in run_delayed_extent_op()
1657 if (key.objectid == head->bytenr && in run_delayed_extent_op()
1659 key.offset == head->num_bytes) in run_delayed_extent_op()
1666 key.objectid = head->bytenr; in run_delayed_extent_op()
1667 key.offset = head->num_bytes; in run_delayed_extent_op()
1672 ret = -EUCLEAN; in run_delayed_extent_op()
1675 head->bytenr, head->num_bytes, head->level); in run_delayed_extent_op()
1680 leaf = path->nodes[0]; in run_delayed_extent_op()
1681 item_size = btrfs_item_size(leaf, path->slots[0]); in run_delayed_extent_op()
1684 ret = -EUCLEAN; in run_delayed_extent_op()
1692 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); in run_delayed_extent_op()
1708 struct btrfs_fs_info *fs_info = trans->fs_info; in run_delayed_tree_ref()
1712 trace_run_delayed_tree_ref(trans->fs_info, node); in run_delayed_tree_ref()
1714 if (node->type == BTRFS_SHARED_BLOCK_REF_KEY) in run_delayed_tree_ref()
1715 parent = node->parent; in run_delayed_tree_ref()
1716 ref_root = node->ref_root; in run_delayed_tree_ref()
1718 if (unlikely(node->ref_mod != 1)) { in run_delayed_tree_ref()
1719 btrfs_err(trans->fs_info, in run_delayed_tree_ref()
1721 node->bytenr, node->ref_mod, node->action, ref_root, in run_delayed_tree_ref()
1723 return -EUCLEAN; in run_delayed_tree_ref()
1725 if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) { in run_delayed_tree_ref()
1727 .root = href->owning_root, in run_delayed_tree_ref()
1728 .num_bytes = fs_info->nodesize, in run_delayed_tree_ref()
1731 .generation = trans->transid, in run_delayed_tree_ref()
1737 } else if (node->action == BTRFS_ADD_DELAYED_REF) { in run_delayed_tree_ref()
1739 } else if (node->action == BTRFS_DROP_DELAYED_REF) { in run_delayed_tree_ref()
1758 btrfs_pin_extent(trans, node->bytenr, node->num_bytes, 1); in run_one_delayed_ref()
1759 free_head_ref_squota_rsv(trans->fs_info, href); in run_one_delayed_ref()
1764 if (node->type == BTRFS_TREE_BLOCK_REF_KEY || in run_one_delayed_ref()
1765 node->type == BTRFS_SHARED_BLOCK_REF_KEY) in run_one_delayed_ref()
1768 else if (node->type == BTRFS_EXTENT_DATA_REF_KEY || in run_one_delayed_ref()
1769 node->type == BTRFS_SHARED_DATA_REF_KEY) in run_one_delayed_ref()
1772 else if (node->type == BTRFS_EXTENT_OWNER_REF_KEY) in run_one_delayed_ref()
1777 btrfs_pin_extent(trans, node->bytenr, node->num_bytes, 1); in run_one_delayed_ref()
1779 btrfs_err(trans->fs_info, in run_one_delayed_ref()
1781 node->bytenr, node->num_bytes, node->type, in run_one_delayed_ref()
1782 node->action, node->ref_mod, ret); in run_one_delayed_ref()
1791 if (RB_EMPTY_ROOT(&head->ref_tree.rb_root)) in select_delayed_ref()
1800 if (!list_empty(&head->ref_add_list)) in select_delayed_ref()
1801 return list_first_entry(&head->ref_add_list, in select_delayed_ref()
1804 ref = rb_entry(rb_first_cached(&head->ref_tree), in select_delayed_ref()
1806 ASSERT(list_empty(&ref->add_list)); in select_delayed_ref()
1813 spin_lock(&delayed_refs->lock); in unselect_delayed_ref_head()
1814 head->processing = false; in unselect_delayed_ref_head()
1815 delayed_refs->num_heads_ready++; in unselect_delayed_ref_head()
1816 spin_unlock(&delayed_refs->lock); in unselect_delayed_ref_head()
1823 struct btrfs_delayed_extent_op *extent_op = head->extent_op; in cleanup_extent_op()
1828 if (head->must_insert_reserved) { in cleanup_extent_op()
1829 head->extent_op = NULL; in cleanup_extent_op()
1845 head->extent_op = NULL; in run_and_cleanup_extent_op()
1846 spin_unlock(&head->lock); in run_and_cleanup_extent_op()
1862 if (head->total_ref_mod < 0 && head->is_data) { in btrfs_cleanup_ref_head_accounting()
1865 spin_lock(&delayed_refs->lock); in btrfs_cleanup_ref_head_accounting()
1866 delayed_refs->pending_csums -= head->num_bytes; in btrfs_cleanup_ref_head_accounting()
1867 spin_unlock(&delayed_refs->lock); in btrfs_cleanup_ref_head_accounting()
1868 nr_csums = btrfs_csum_bytes_to_leaves(fs_info, head->num_bytes); in btrfs_cleanup_ref_head_accounting()
1875 if (head->must_insert_reserved) in btrfs_cleanup_ref_head_accounting()
1886 struct btrfs_fs_info *fs_info = trans->fs_info; in cleanup_ref_head()
1890 delayed_refs = &trans->transaction->delayed_refs; in cleanup_ref_head()
1902 * Need to drop our head ref lock and re-acquire the delayed ref lock in cleanup_ref_head()
1903 * and then re-check to make sure nobody got added. in cleanup_ref_head()
1905 spin_unlock(&head->lock); in cleanup_ref_head()
1906 spin_lock(&delayed_refs->lock); in cleanup_ref_head()
1907 spin_lock(&head->lock); in cleanup_ref_head()
1908 if (!RB_EMPTY_ROOT(&head->ref_tree.rb_root) || head->extent_op) { in cleanup_ref_head()
1909 spin_unlock(&head->lock); in cleanup_ref_head()
1910 spin_unlock(&delayed_refs->lock); in cleanup_ref_head()
1914 spin_unlock(&head->lock); in cleanup_ref_head()
1915 spin_unlock(&delayed_refs->lock); in cleanup_ref_head()
1917 if (head->must_insert_reserved) { in cleanup_ref_head()
1918 btrfs_pin_extent(trans, head->bytenr, head->num_bytes, 1); in cleanup_ref_head()
1919 if (head->is_data) { in cleanup_ref_head()
1922 csum_root = btrfs_csum_root(fs_info, head->bytenr); in cleanup_ref_head()
1923 ret = btrfs_del_csums(trans, csum_root, head->bytenr, in cleanup_ref_head()
1924 head->num_bytes); in cleanup_ref_head()
1940 &trans->transaction->delayed_refs; in btrfs_obtain_ref_head()
1944 spin_lock(&delayed_refs->lock); in btrfs_obtain_ref_head()
1947 spin_unlock(&delayed_refs->lock); in btrfs_obtain_ref_head()
1956 spin_unlock(&delayed_refs->lock); in btrfs_obtain_ref_head()
1963 if (ret == -EAGAIN) in btrfs_obtain_ref_head()
1964 head = ERR_PTR(-EAGAIN); in btrfs_obtain_ref_head()
1973 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_run_delayed_refs_for_head()
1980 delayed_refs = &trans->transaction->delayed_refs; in btrfs_run_delayed_refs_for_head()
1982 lockdep_assert_held(&locked_ref->mutex); in btrfs_run_delayed_refs_for_head()
1983 lockdep_assert_held(&locked_ref->lock); in btrfs_run_delayed_refs_for_head()
1986 if (ref->seq && in btrfs_run_delayed_refs_for_head()
1987 btrfs_check_delayed_seq(fs_info, ref->seq)) { in btrfs_run_delayed_refs_for_head()
1988 spin_unlock(&locked_ref->lock); in btrfs_run_delayed_refs_for_head()
1990 return -EAGAIN; in btrfs_run_delayed_refs_for_head()
1993 rb_erase_cached(&ref->ref_node, &locked_ref->ref_tree); in btrfs_run_delayed_refs_for_head()
1994 RB_CLEAR_NODE(&ref->ref_node); in btrfs_run_delayed_refs_for_head()
1995 if (!list_empty(&ref->add_list)) in btrfs_run_delayed_refs_for_head()
1996 list_del(&ref->add_list); in btrfs_run_delayed_refs_for_head()
2001 switch (ref->action) { in btrfs_run_delayed_refs_for_head()
2004 locked_ref->ref_mod -= ref->ref_mod; in btrfs_run_delayed_refs_for_head()
2007 locked_ref->ref_mod += ref->ref_mod; in btrfs_run_delayed_refs_for_head()
2012 atomic_dec(&delayed_refs->num_entries); in btrfs_run_delayed_refs_for_head()
2018 must_insert_reserved = locked_ref->must_insert_reserved; in btrfs_run_delayed_refs_for_head()
2025 locked_ref->must_insert_reserved = false; in btrfs_run_delayed_refs_for_head()
2027 extent_op = locked_ref->extent_op; in btrfs_run_delayed_refs_for_head()
2028 locked_ref->extent_op = NULL; in btrfs_run_delayed_refs_for_head()
2029 spin_unlock(&locked_ref->lock); in btrfs_run_delayed_refs_for_head()
2046 spin_lock(&locked_ref->lock); in btrfs_run_delayed_refs_for_head()
2055 * Returns -ENOMEM or -EIO on failure and will abort the transaction.
2060 struct btrfs_fs_info *fs_info = trans->fs_info; in __btrfs_run_delayed_refs()
2068 delayed_refs = &trans->transaction->delayed_refs; in __btrfs_run_delayed_refs()
2070 max_count = delayed_refs->num_heads_ready; in __btrfs_run_delayed_refs()
2078 if (PTR_ERR(locked_ref) == -EAGAIN) { in __btrfs_run_delayed_refs()
2090 * finish. If we merged anything we need to re-loop so we can in __btrfs_run_delayed_refs()
2098 spin_lock(&locked_ref->lock); in __btrfs_run_delayed_refs()
2102 if (ret < 0 && ret != -EAGAIN) { in __btrfs_run_delayed_refs()
2125 * returned -EAGAIN, meaning we need to select another head in __btrfs_run_delayed_refs()
2145 struct rb_node *n = root->rb_node; in find_middle()
2154 first = entry->bytenr; in find_middle()
2159 last = entry->bytenr; in find_middle()
2161 n = root->rb_node; in find_middle()
2165 WARN_ON(!entry->in_tree); in find_middle()
2167 middle = entry->bytenr; in find_middle()
2170 n = n->rb_left; in find_middle()
2172 n = n->rb_right; in find_middle()
2174 alt = 1 - alt; in find_middle()
2189 * Use (u64)-1 (U64_MAX) to run all existing delayed references
2197 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_run_delayed_refs()
2205 if (test_bit(BTRFS_FS_CREATING_FREE_SPACE_TREE, &fs_info->flags)) in btrfs_run_delayed_refs()
2208 delayed_refs = &trans->transaction->delayed_refs; in btrfs_run_delayed_refs()
2211 delayed_refs->run_delayed_start = find_middle(&delayed_refs->root); in btrfs_run_delayed_refs()
2222 spin_lock(&delayed_refs->lock); in btrfs_run_delayed_refs()
2223 if (RB_EMPTY_ROOT(&delayed_refs->href_root.rb_root)) { in btrfs_run_delayed_refs()
2224 spin_unlock(&delayed_refs->lock); in btrfs_run_delayed_refs()
2227 spin_unlock(&delayed_refs->lock); in btrfs_run_delayed_refs()
2244 return -ENOMEM; in btrfs_set_disk_extent_flags()
2246 extent_op->flags_to_set = flags; in btrfs_set_disk_extent_flags()
2247 extent_op->update_flags = true; in btrfs_set_disk_extent_flags()
2248 extent_op->update_key = false; in btrfs_set_disk_extent_flags()
2250 ret = btrfs_add_delayed_extent_op(trans, eb->start, eb->len, in btrfs_set_disk_extent_flags()
2268 spin_lock(&root->fs_info->trans_lock); in check_delayed_ref()
2269 cur_trans = root->fs_info->running_transaction; in check_delayed_ref()
2271 refcount_inc(&cur_trans->use_count); in check_delayed_ref()
2272 spin_unlock(&root->fs_info->trans_lock); in check_delayed_ref()
2276 delayed_refs = &cur_trans->delayed_refs; in check_delayed_ref()
2277 spin_lock(&delayed_refs->lock); in check_delayed_ref()
2280 spin_unlock(&delayed_refs->lock); in check_delayed_ref()
2285 if (!mutex_trylock(&head->mutex)) { in check_delayed_ref()
2286 if (path->nowait) { in check_delayed_ref()
2287 spin_unlock(&delayed_refs->lock); in check_delayed_ref()
2289 return -EAGAIN; in check_delayed_ref()
2292 refcount_inc(&head->refs); in check_delayed_ref()
2293 spin_unlock(&delayed_refs->lock); in check_delayed_ref()
2301 mutex_lock(&head->mutex); in check_delayed_ref()
2302 mutex_unlock(&head->mutex); in check_delayed_ref()
2305 return -EAGAIN; in check_delayed_ref()
2307 spin_unlock(&delayed_refs->lock); in check_delayed_ref()
2309 spin_lock(&head->lock); in check_delayed_ref()
2314 for (node = rb_first_cached(&head->ref_tree); node; in check_delayed_ref()
2321 if (ref->type != BTRFS_EXTENT_DATA_REF_KEY) { in check_delayed_ref()
2333 if (ref->ref_root != btrfs_root_id(root) || in check_delayed_ref()
2339 spin_unlock(&head->lock); in check_delayed_ref()
2340 mutex_unlock(&head->mutex); in check_delayed_ref()
2350 struct btrfs_fs_info *fs_info = root->fs_info; in check_committed_ref()
2363 key.offset = (u64)-1; in check_committed_ref()
2371 * Key with offset -1 found, there would have to exist an extent in check_committed_ref()
2374 ret = -EUCLEAN; in check_committed_ref()
2378 ret = -ENOENT; in check_committed_ref()
2379 if (path->slots[0] == 0) in check_committed_ref()
2382 path->slots[0]--; in check_committed_ref()
2383 leaf = path->nodes[0]; in check_committed_ref()
2384 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]); in check_committed_ref()
2390 item_size = btrfs_item_size(leaf, path->slots[0]); in check_committed_ref()
2391 ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item); in check_committed_ref()
2416 btrfs_root_last_snapshot(&root->root_item))) in check_committed_ref()
2424 ref = (struct btrfs_extent_data_ref *)(&iref->offset); in check_committed_ref()
2445 if (ret && ret != -ENOENT) in btrfs_cross_ref_exist()
2449 } while (ret == -EAGAIN); in btrfs_cross_ref_exist()
2463 struct btrfs_fs_info *fs_info = root->fs_info; in __btrfs_mod_ref()
2482 if (!test_bit(BTRFS_ROOT_SHAREABLE, &root->state) && level == 0) in __btrfs_mod_ref()
2486 parent = buf->start; in __btrfs_mod_ref()
2517 key.offset -= btrfs_file_extent_offset(buf, fi); in __btrfs_mod_ref()
2529 ref.num_bytes = fs_info->nodesize; in __btrfs_mod_ref()
2531 btrfs_init_tree_ref(&ref, level - 1, in __btrfs_mod_ref()
2560 struct btrfs_fs_info *fs_info = root->fs_info; in get_alloc_profile_by_root()
2566 else if (root == fs_info->chunk_root) in get_alloc_profile_by_root()
2580 read_lock(&fs_info->block_group_cache_lock); in first_logical_byte()
2582 leftmost = rb_first_cached(&fs_info->block_group_cache_tree); in first_logical_byte()
2587 bytenr = bg->start; in first_logical_byte()
2589 read_unlock(&fs_info->block_group_cache_lock); in first_logical_byte()
2598 struct btrfs_fs_info *fs_info = cache->fs_info; in pin_down_extent()
2600 spin_lock(&cache->space_info->lock); in pin_down_extent()
2601 spin_lock(&cache->lock); in pin_down_extent()
2602 cache->pinned += num_bytes; in pin_down_extent()
2603 btrfs_space_info_update_bytes_pinned(fs_info, cache->space_info, in pin_down_extent()
2606 cache->reserved -= num_bytes; in pin_down_extent()
2607 cache->space_info->bytes_reserved -= num_bytes; in pin_down_extent()
2609 spin_unlock(&cache->lock); in pin_down_extent()
2610 spin_unlock(&cache->space_info->lock); in pin_down_extent()
2612 set_extent_bit(&trans->transaction->pinned_extents, bytenr, in pin_down_extent()
2613 bytenr + num_bytes - 1, EXTENT_DIRTY, NULL); in pin_down_extent()
2622 cache = btrfs_lookup_block_group(trans->fs_info, bytenr); in btrfs_pin_extent()
2637 cache = btrfs_lookup_block_group(trans->fs_info, eb->start); in btrfs_pin_extent_for_log_replay()
2639 return -EINVAL; in btrfs_pin_extent_for_log_replay()
2649 pin_down_extent(trans, cache, eb->start, eb->len, 0); in btrfs_pin_extent_for_log_replay()
2652 ret = btrfs_remove_free_space(cache, eb->start, eb->len); in btrfs_pin_extent_for_log_replay()
2666 return -EINVAL; in __exclude_logged_extent()
2680 struct btrfs_fs_info *fs_info = eb->fs_info; in btrfs_exclude_logged_extents()
2713 atomic_inc(&bg->reservations); in btrfs_inc_block_group_reservations()
2730 if (space_info->flags & BTRFS_BLOCK_GROUP_METADATA) { in fetch_cluster_info()
2731 ret = &fs_info->meta_alloc_cluster; in fetch_cluster_info()
2736 } else if ((space_info->flags & BTRFS_BLOCK_GROUP_DATA) && in fetch_cluster_info()
2739 ret = &fs_info->data_alloc_cluster; in fetch_cluster_info()
2751 struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv; in unpin_extent_range()
2762 start >= cache->start + cache->length) { in unpin_extent_range()
2769 ret = -EUCLEAN; in unpin_extent_range()
2774 cache->space_info, in unpin_extent_range()
2779 len = cache->start + cache->length - start; in unpin_extent_range()
2780 len = min(len, end + 1 - start); in unpin_extent_range()
2787 space_info = cache->space_info; in unpin_extent_range()
2795 if (cluster && cluster->fragmented && in unpin_extent_range()
2797 spin_lock(&cluster->lock); in unpin_extent_range()
2798 cluster->fragmented = 0; in unpin_extent_range()
2799 spin_unlock(&cluster->lock); in unpin_extent_range()
2802 spin_lock(&space_info->lock); in unpin_extent_range()
2803 spin_lock(&cache->lock); in unpin_extent_range()
2804 cache->pinned -= len; in unpin_extent_range()
2805 btrfs_space_info_update_bytes_pinned(fs_info, space_info, -len); in unpin_extent_range()
2806 space_info->max_extent_size = 0; in unpin_extent_range()
2807 if (cache->ro) { in unpin_extent_range()
2808 space_info->bytes_readonly += len; in unpin_extent_range()
2816 spin_unlock(&cache->lock); in unpin_extent_range()
2818 global_rsv->space_info == space_info) { in unpin_extent_range()
2819 spin_lock(&global_rsv->lock); in unpin_extent_range()
2820 if (!global_rsv->full) { in unpin_extent_range()
2821 u64 to_add = min(len, global_rsv->size - in unpin_extent_range()
2822 global_rsv->reserved); in unpin_extent_range()
2824 global_rsv->reserved += to_add; in unpin_extent_range()
2827 if (global_rsv->reserved >= global_rsv->size) in unpin_extent_range()
2828 global_rsv->full = 1; in unpin_extent_range()
2829 len -= to_add; in unpin_extent_range()
2831 spin_unlock(&global_rsv->lock); in unpin_extent_range()
2836 spin_unlock(&space_info->lock); in unpin_extent_range()
2847 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_finish_extent_commit()
2855 unpin = &trans->transaction->pinned_extents; in btrfs_finish_extent_commit()
2860 mutex_lock(&fs_info->unused_bg_unpin_mutex); in btrfs_finish_extent_commit()
2863 mutex_unlock(&fs_info->unused_bg_unpin_mutex); in btrfs_finish_extent_commit()
2869 end + 1 - start, NULL); in btrfs_finish_extent_commit()
2874 mutex_unlock(&fs_info->unused_bg_unpin_mutex); in btrfs_finish_extent_commit()
2880 btrfs_discard_calc_delay(&fs_info->discard_ctl); in btrfs_finish_extent_commit()
2881 btrfs_discard_schedule_work(&fs_info->discard_ctl, true); in btrfs_finish_extent_commit()
2889 deleted_bgs = &trans->transaction->deleted_bgs; in btrfs_finish_extent_commit()
2893 ret = -EROFS; in btrfs_finish_extent_commit()
2896 block_group->start, in btrfs_finish_extent_commit()
2897 block_group->length, in btrfs_finish_extent_commit()
2900 list_del_init(&block_group->bg_list); in btrfs_finish_extent_commit()
2955 oref = (struct btrfs_extent_owner_ref *)(&iref->offset); in btrfs_get_extent_owner_root()
2967 u64 num_bytes = delta->num_bytes; in do_free_extent_accounting()
2969 if (delta->is_data) { in do_free_extent_accounting()
2972 csum_root = btrfs_csum_root(trans->fs_info, bytenr); in do_free_extent_accounting()
2986 ret = btrfs_record_squota_delta(trans->fs_info, delta); in do_free_extent_accounting()
3007 btrfs_abort_transaction(trans, -EUCLEAN); \
3008 btrfs_print_leaf(path->nodes[0]); \
3009 btrfs_crit(trans->fs_info, fmt, ##args); \
3032 * node->bytenr = 13631488
3033 * node->num_bytes = 1048576
3057 * node->bytenr = 13631488
3058 * node->num_bytes = 1048576
3076 struct btrfs_fs_info *info = trans->fs_info; in __btrfs_free_extent()
3088 int refs_to_drop = node->ref_mod; in __btrfs_free_extent()
3091 u64 bytenr = node->bytenr; in __btrfs_free_extent()
3092 u64 num_bytes = node->num_bytes; in __btrfs_free_extent()
3096 u64 delayed_ref_root = href->owning_root; in __btrfs_free_extent()
3103 return -ENOMEM; in __btrfs_free_extent()
3110 node->bytenr, refs_to_drop); in __btrfs_free_extent()
3111 ret = -EINVAL; in __btrfs_free_extent()
3120 node->parent, node->ref_root, owner_objectid, in __btrfs_free_extent()
3130 extent_slot = path->slots[0]; in __btrfs_free_extent()
3132 btrfs_item_key_to_cpu(path->nodes[0], &key, in __btrfs_free_extent()
3148 if (path->slots[0] - extent_slot > 5) in __btrfs_free_extent()
3150 extent_slot--; in __btrfs_free_extent()
3157 path->slots[0]); in __btrfs_free_extent()
3158 ret = -EUCLEAN; in __btrfs_free_extent()
3181 &key, path, -1, 1); in __btrfs_free_extent()
3182 if (ret > 0 && skinny_metadata && path->slots[0]) { in __btrfs_free_extent()
3187 path->slots[0]--; in __btrfs_free_extent()
3188 btrfs_item_key_to_cpu(path->nodes[0], &key, in __btrfs_free_extent()
3189 path->slots[0]); in __btrfs_free_extent()
3203 &key, path, -1, 1); in __btrfs_free_extent()
3208 btrfs_print_leaf(path->nodes[0]); in __btrfs_free_extent()
3211 ret, bytenr, path->slots[0]); in __btrfs_free_extent()
3217 extent_slot = path->slots[0]; in __btrfs_free_extent()
3219 } else if (WARN_ON(ret == -ENOENT)) { in __btrfs_free_extent()
3222 bytenr, node->parent, node->ref_root, owner_objectid, in __btrfs_free_extent()
3223 owner_offset, path->slots[0]); in __btrfs_free_extent()
3230 leaf = path->nodes[0]; in __btrfs_free_extent()
3233 ret = -EUCLEAN; in __btrfs_free_extent()
3234 btrfs_err(trans->fs_info, in __btrfs_free_extent()
3250 path->slots[0], owner_objectid, item_size, in __btrfs_free_extent()
3252 ret = -EUCLEAN; in __btrfs_free_extent()
3263 refs_to_drop, refs, bytenr, path->slots[0]); in __btrfs_free_extent()
3264 ret = -EUCLEAN; in __btrfs_free_extent()
3267 refs -= refs_to_drop; in __btrfs_free_extent()
3280 path->slots[0]); in __btrfs_free_extent()
3281 ret = -EUCLEAN; in __btrfs_free_extent()
3312 refs_to_drop, path->slots[0]); in __btrfs_free_extent()
3313 ret = -EUCLEAN; in __btrfs_free_extent()
3317 if (path->slots[0] != extent_slot) { in __btrfs_free_extent()
3321 key.offset, path->slots[0]); in __btrfs_free_extent()
3322 ret = -EUCLEAN; in __btrfs_free_extent()
3332 if (path->slots[0] != extent_slot + 1) { in __btrfs_free_extent()
3335 path->slots[0]); in __btrfs_free_extent()
3336 ret = -EUCLEAN; in __btrfs_free_extent()
3339 path->slots[0] = extent_slot; in __btrfs_free_extent()
3351 delta.root = btrfs_get_extent_owner_root(trans->fs_info, in __btrfs_free_extent()
3354 ret = btrfs_del_items(trans, extent_root, path, path->slots[0], in __btrfs_free_extent()
3384 delayed_refs = &trans->transaction->delayed_refs; in check_ref_cleanup()
3385 spin_lock(&delayed_refs->lock); in check_ref_cleanup()
3390 spin_lock(&head->lock); in check_ref_cleanup()
3391 if (!RB_EMPTY_ROOT(&head->ref_tree.rb_root)) in check_ref_cleanup()
3401 if (!mutex_trylock(&head->mutex)) in check_ref_cleanup()
3405 head->processing = false; in check_ref_cleanup()
3407 spin_unlock(&head->lock); in check_ref_cleanup()
3408 spin_unlock(&delayed_refs->lock); in check_ref_cleanup()
3410 BUG_ON(head->extent_op); in check_ref_cleanup()
3411 if (head->must_insert_reserved) in check_ref_cleanup()
3414 btrfs_cleanup_ref_head_accounting(trans->fs_info, delayed_refs, head); in check_ref_cleanup()
3415 mutex_unlock(&head->mutex); in check_ref_cleanup()
3419 spin_unlock(&head->lock); in check_ref_cleanup()
3422 spin_unlock(&delayed_refs->lock); in check_ref_cleanup()
3431 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_free_tree_block()
3438 .bytenr = buf->start, in btrfs_free_tree_block()
3439 .num_bytes = buf->len, in btrfs_free_tree_block()
3463 if (btrfs_header_generation(buf) != trans->transid) in btrfs_free_tree_block()
3467 ret = check_ref_cleanup(trans, buf->start); in btrfs_free_tree_block()
3472 bg = btrfs_lookup_block_group(fs_info, buf->start); in btrfs_free_tree_block()
3475 pin_down_extent(trans, bg, buf->start, buf->len, 1); in btrfs_free_tree_block()
3482 * operations for this node. If we re-allocate this node we in btrfs_free_tree_block()
3497 if (test_bit(BTRFS_FS_TREE_MOD_LOG_USERS, &fs_info->flags) in btrfs_free_tree_block()
3499 pin_down_extent(trans, bg, buf->start, buf->len, 1); in btrfs_free_tree_block()
3504 WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags)); in btrfs_free_tree_block()
3506 btrfs_add_free_space(bg, buf->start, buf->len); in btrfs_free_tree_block()
3507 btrfs_free_reserved_bytes(bg, buf->len, 0); in btrfs_free_tree_block()
3509 trace_btrfs_reserved_extent_free(fs_info, buf->start, buf->len); in btrfs_free_tree_block()
3517 clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags); in btrfs_free_tree_block()
3521 /* Can return -ENOMEM */
3524 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_free_extent()
3534 if (ref->ref_root == BTRFS_TREE_LOG_OBJECTID) { in btrfs_free_extent()
3535 btrfs_pin_extent(trans, ref->bytenr, ref->num_bytes, 1); in btrfs_free_extent()
3537 } else if (ref->type == BTRFS_REF_METADATA) { in btrfs_free_extent()
3543 if (ref->ref_root != BTRFS_TREE_LOG_OBJECTID) in btrfs_free_extent()
3590 down_read(&cache->data_rwsem); in btrfs_lock_block_group()
3598 down_read(&cache->data_rwsem); in btrfs_grab_block_group()
3605 __acquires(&cluster->refill_lock) in btrfs_lock_cluster()
3609 spin_lock(&cluster->refill_lock); in btrfs_lock_cluster()
3611 used_bg = cluster->block_group; in btrfs_lock_cluster()
3623 if (down_read_trylock(&used_bg->data_rwsem)) in btrfs_lock_cluster()
3626 spin_unlock(&cluster->refill_lock); in btrfs_lock_cluster()
3628 /* We should only have one-level nested. */ in btrfs_lock_cluster()
3629 down_read_nested(&used_bg->data_rwsem, SINGLE_DEPTH_NESTING); in btrfs_lock_cluster()
3631 spin_lock(&cluster->refill_lock); in btrfs_lock_cluster()
3632 if (used_bg == cluster->block_group) in btrfs_lock_cluster()
3635 up_read(&used_bg->data_rwsem); in btrfs_lock_cluster()
3645 up_read(&cache->data_rwsem); in btrfs_release_block_group()
3652 * Return -ENOENT to inform caller that we need fallback to unclustered mode.
3654 * Return 0 means we have found a location and set ffe_ctl->found_offset.
3661 struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr; in find_free_extent_clustered()
3666 cluster_bg = btrfs_lock_cluster(bg, last_ptr, ffe_ctl->delalloc); in find_free_extent_clustered()
3669 if (cluster_bg != bg && (cluster_bg->ro || in find_free_extent_clustered()
3670 !block_group_bits(cluster_bg, ffe_ctl->flags))) in find_free_extent_clustered()
3674 ffe_ctl->num_bytes, cluster_bg->start, in find_free_extent_clustered()
3675 &ffe_ctl->max_extent_size); in find_free_extent_clustered()
3678 spin_unlock(&last_ptr->refill_lock); in find_free_extent_clustered()
3681 ffe_ctl->found_offset = offset; in find_free_extent_clustered()
3684 WARN_ON(last_ptr->block_group != cluster_bg); in find_free_extent_clustered()
3698 if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE && cluster_bg != bg) { in find_free_extent_clustered()
3699 spin_unlock(&last_ptr->refill_lock); in find_free_extent_clustered()
3700 btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc); in find_free_extent_clustered()
3701 return -ENOENT; in find_free_extent_clustered()
3708 btrfs_release_block_group(cluster_bg, ffe_ctl->delalloc); in find_free_extent_clustered()
3711 if (ffe_ctl->loop >= LOOP_NO_EMPTY_SIZE) { in find_free_extent_clustered()
3712 spin_unlock(&last_ptr->refill_lock); in find_free_extent_clustered()
3713 return -ENOENT; in find_free_extent_clustered()
3717 ffe_ctl->empty_cluster + ffe_ctl->empty_size, in find_free_extent_clustered()
3718 bg->full_stripe_len); in find_free_extent_clustered()
3719 ret = btrfs_find_space_cluster(bg, last_ptr, ffe_ctl->search_start, in find_free_extent_clustered()
3720 ffe_ctl->num_bytes, aligned_cluster); in find_free_extent_clustered()
3724 ffe_ctl->num_bytes, ffe_ctl->search_start, in find_free_extent_clustered()
3725 &ffe_ctl->max_extent_size); in find_free_extent_clustered()
3728 spin_unlock(&last_ptr->refill_lock); in find_free_extent_clustered()
3729 ffe_ctl->found_offset = offset; in find_free_extent_clustered()
3740 spin_unlock(&last_ptr->refill_lock); in find_free_extent_clustered()
3746 * Return 0 when we found an free extent and set ffe_ctrl->found_offset
3751 struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr; in find_free_extent_unclustered()
3760 spin_lock(&last_ptr->lock); in find_free_extent_unclustered()
3761 last_ptr->fragmented = 1; in find_free_extent_unclustered()
3762 spin_unlock(&last_ptr->lock); in find_free_extent_unclustered()
3764 if (ffe_ctl->cached) { in find_free_extent_unclustered()
3767 free_space_ctl = bg->free_space_ctl; in find_free_extent_unclustered()
3768 spin_lock(&free_space_ctl->tree_lock); in find_free_extent_unclustered()
3769 if (free_space_ctl->free_space < in find_free_extent_unclustered()
3770 ffe_ctl->num_bytes + ffe_ctl->empty_cluster + in find_free_extent_unclustered()
3771 ffe_ctl->empty_size) { in find_free_extent_unclustered()
3772 ffe_ctl->total_free_space = max_t(u64, in find_free_extent_unclustered()
3773 ffe_ctl->total_free_space, in find_free_extent_unclustered()
3774 free_space_ctl->free_space); in find_free_extent_unclustered()
3775 spin_unlock(&free_space_ctl->tree_lock); in find_free_extent_unclustered()
3778 spin_unlock(&free_space_ctl->tree_lock); in find_free_extent_unclustered()
3781 offset = btrfs_find_space_for_alloc(bg, ffe_ctl->search_start, in find_free_extent_unclustered()
3782 ffe_ctl->num_bytes, ffe_ctl->empty_size, in find_free_extent_unclustered()
3783 &ffe_ctl->max_extent_size); in find_free_extent_unclustered()
3786 ffe_ctl->found_offset = offset; in find_free_extent_unclustered()
3797 if (ffe_ctl->last_ptr && ffe_ctl->use_cluster) { in do_allocation_clustered()
3801 /* ret == -ENOENT case falls through */ in do_allocation_clustered()
3808 * Tree-log block group locking
3813 * for tree-log metadata.
3824 * Simple allocator for sequential-only block group. It only allows sequential
3832 struct btrfs_fs_info *fs_info = block_group->fs_info; in do_allocation_zoned()
3833 struct btrfs_space_info *space_info = block_group->space_info; in do_allocation_zoned()
3834 struct btrfs_free_space_ctl *ctl = block_group->free_space_ctl; in do_allocation_zoned()
3835 u64 start = block_group->start; in do_allocation_zoned()
3836 u64 num_bytes = ffe_ctl->num_bytes; in do_allocation_zoned()
3838 u64 bytenr = block_group->start; in do_allocation_zoned()
3844 ASSERT(btrfs_is_zoned(block_group->fs_info)); in do_allocation_zoned()
3847 * Do not allow non-tree-log blocks in the dedicated tree-log block in do_allocation_zoned()
3850 spin_lock(&fs_info->treelog_bg_lock); in do_allocation_zoned()
3851 log_bytenr = fs_info->treelog_bg; in do_allocation_zoned()
3852 if (log_bytenr && ((ffe_ctl->for_treelog && bytenr != log_bytenr) || in do_allocation_zoned()
3853 (!ffe_ctl->for_treelog && bytenr == log_bytenr))) in do_allocation_zoned()
3855 spin_unlock(&fs_info->treelog_bg_lock); in do_allocation_zoned()
3860 * Do not allow non-relocation blocks in the dedicated relocation block in do_allocation_zoned()
3863 spin_lock(&fs_info->relocation_bg_lock); in do_allocation_zoned()
3864 data_reloc_bytenr = fs_info->data_reloc_bg; in do_allocation_zoned()
3866 ((ffe_ctl->for_data_reloc && bytenr != data_reloc_bytenr) || in do_allocation_zoned()
3867 (!ffe_ctl->for_data_reloc && bytenr == data_reloc_bytenr))) in do_allocation_zoned()
3869 spin_unlock(&fs_info->relocation_bg_lock); in do_allocation_zoned()
3874 spin_lock(&block_group->lock); in do_allocation_zoned()
3875 if (block_group->ro || btrfs_zoned_bg_is_full(block_group)) { in do_allocation_zoned()
3878 * May need to clear fs_info->{treelog,data_reloc}_bg. in do_allocation_zoned()
3882 spin_unlock(&block_group->lock); in do_allocation_zoned()
3885 if (!ret && (block_group->flags & BTRFS_BLOCK_GROUP_DATA) && in do_allocation_zoned()
3889 * May need to clear fs_info->{treelog,data_reloc}_bg. in do_allocation_zoned()
3894 spin_lock(&space_info->lock); in do_allocation_zoned()
3895 spin_lock(&block_group->lock); in do_allocation_zoned()
3896 spin_lock(&fs_info->treelog_bg_lock); in do_allocation_zoned()
3897 spin_lock(&fs_info->relocation_bg_lock); in do_allocation_zoned()
3902 ASSERT(!ffe_ctl->for_treelog || in do_allocation_zoned()
3903 block_group->start == fs_info->treelog_bg || in do_allocation_zoned()
3904 fs_info->treelog_bg == 0); in do_allocation_zoned()
3905 ASSERT(!ffe_ctl->for_data_reloc || in do_allocation_zoned()
3906 block_group->start == fs_info->data_reloc_bg || in do_allocation_zoned()
3907 fs_info->data_reloc_bg == 0); in do_allocation_zoned()
3909 if (block_group->ro || in do_allocation_zoned()
3910 (!ffe_ctl->for_data_reloc && in do_allocation_zoned()
3911 test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags))) { in do_allocation_zoned()
3917 * Do not allow currently using block group to be tree-log dedicated in do_allocation_zoned()
3920 if (ffe_ctl->for_treelog && !fs_info->treelog_bg && in do_allocation_zoned()
3921 (block_group->used || block_group->reserved)) { in do_allocation_zoned()
3930 if (ffe_ctl->for_data_reloc && !fs_info->data_reloc_bg && in do_allocation_zoned()
3931 (block_group->used || block_group->reserved)) { in do_allocation_zoned()
3936 WARN_ON_ONCE(block_group->alloc_offset > block_group->zone_capacity); in do_allocation_zoned()
3937 avail = block_group->zone_capacity - block_group->alloc_offset; in do_allocation_zoned()
3939 if (ffe_ctl->max_extent_size < avail) { in do_allocation_zoned()
3944 ffe_ctl->max_extent_size = avail; in do_allocation_zoned()
3945 ffe_ctl->total_free_space = avail; in do_allocation_zoned()
3951 if (ffe_ctl->for_treelog && !fs_info->treelog_bg) in do_allocation_zoned()
3952 fs_info->treelog_bg = block_group->start; in do_allocation_zoned()
3954 if (ffe_ctl->for_data_reloc) { in do_allocation_zoned()
3955 if (!fs_info->data_reloc_bg) in do_allocation_zoned()
3956 fs_info->data_reloc_bg = block_group->start; in do_allocation_zoned()
3959 * for data relocation. Compared to increasing the ->ro, setting in do_allocation_zoned()
3960 * the ->zoned_data_reloc_ongoing flag still allows nocow in do_allocation_zoned()
3964 * regular (non-relocation data) extent. With mix of relocation in do_allocation_zoned()
3972 set_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags); in do_allocation_zoned()
3975 ffe_ctl->found_offset = start + block_group->alloc_offset; in do_allocation_zoned()
3976 block_group->alloc_offset += num_bytes; in do_allocation_zoned()
3977 spin_lock(&ctl->tree_lock); in do_allocation_zoned()
3978 ctl->free_space -= num_bytes; in do_allocation_zoned()
3979 spin_unlock(&ctl->tree_lock); in do_allocation_zoned()
3986 ffe_ctl->search_start = ffe_ctl->found_offset; in do_allocation_zoned()
3989 if (ret && ffe_ctl->for_treelog) in do_allocation_zoned()
3990 fs_info->treelog_bg = 0; in do_allocation_zoned()
3991 if (ret && ffe_ctl->for_data_reloc) in do_allocation_zoned()
3992 fs_info->data_reloc_bg = 0; in do_allocation_zoned()
3993 spin_unlock(&fs_info->relocation_bg_lock); in do_allocation_zoned()
3994 spin_unlock(&fs_info->treelog_bg_lock); in do_allocation_zoned()
3995 spin_unlock(&block_group->lock); in do_allocation_zoned()
3996 spin_unlock(&space_info->lock); in do_allocation_zoned()
4004 switch (ffe_ctl->policy) { in do_allocation()
4018 switch (ffe_ctl->policy) { in release_block_group()
4020 ffe_ctl->retry_uncached = false; in release_block_group()
4029 BUG_ON(btrfs_bg_flags_to_raid_index(block_group->flags) != in release_block_group()
4030 ffe_ctl->index); in release_block_group()
4037 struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr; in found_extent_clustered()
4039 if (!ffe_ctl->use_cluster && last_ptr) { in found_extent_clustered()
4040 spin_lock(&last_ptr->lock); in found_extent_clustered()
4041 last_ptr->window_start = ins->objectid; in found_extent_clustered()
4042 spin_unlock(&last_ptr->lock); in found_extent_clustered()
4049 switch (ffe_ctl->policy) { in found_extent()
4065 if (!(ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA)) in can_allocate_chunk_zoned()
4069 if (btrfs_can_activate_zone(fs_info->fs_devices, ffe_ctl->flags)) in can_allocate_chunk_zoned()
4079 if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA) { in can_allocate_chunk_zoned()
4093 if (ffe_ctl->max_extent_size >= ffe_ctl->min_alloc_size) in can_allocate_chunk_zoned()
4094 return -ENOSPC; in can_allocate_chunk_zoned()
4103 if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA) in can_allocate_chunk_zoned()
4104 return -EAGAIN; in can_allocate_chunk_zoned()
4117 switch (ffe_ctl->policy) { in can_allocate_chunk()
4128 * Return >0 means caller needs to re-search for free extent
4137 struct btrfs_root *root = fs_info->chunk_root; in find_free_extent_update_loop()
4140 if ((ffe_ctl->loop == LOOP_CACHING_NOWAIT) && in find_free_extent_update_loop()
4141 ffe_ctl->have_caching_bg && !ffe_ctl->orig_have_caching_bg) in find_free_extent_update_loop()
4142 ffe_ctl->orig_have_caching_bg = true; in find_free_extent_update_loop()
4144 if (ins->objectid) { in find_free_extent_update_loop()
4149 if (ffe_ctl->loop >= LOOP_CACHING_WAIT && ffe_ctl->have_caching_bg) in find_free_extent_update_loop()
4152 ffe_ctl->index++; in find_free_extent_update_loop()
4153 if (ffe_ctl->index < BTRFS_NR_RAID_TYPES) in find_free_extent_update_loop()
4157 if (ffe_ctl->loop < LOOP_NO_EMPTY_SIZE) { in find_free_extent_update_loop()
4158 ffe_ctl->index = 0; in find_free_extent_update_loop()
4164 if (ffe_ctl->loop == LOOP_CACHING_NOWAIT && in find_free_extent_update_loop()
4165 (!ffe_ctl->orig_have_caching_bg && full_search)) in find_free_extent_update_loop()
4166 ffe_ctl->loop++; in find_free_extent_update_loop()
4167 ffe_ctl->loop++; in find_free_extent_update_loop()
4169 if (ffe_ctl->loop == LOOP_ALLOC_CHUNK) { in find_free_extent_update_loop()
4178 trans = current->journal_info; in find_free_extent_update_loop()
4189 ret = btrfs_chunk_alloc(trans, ffe_ctl->flags, in find_free_extent_update_loop()
4193 if (ret == -ENOSPC) { in find_free_extent_update_loop()
4195 ffe_ctl->loop++; in find_free_extent_update_loop()
4207 if (ffe_ctl->loop == LOOP_NO_EMPTY_SIZE) { in find_free_extent_update_loop()
4208 if (ffe_ctl->policy != BTRFS_EXTENT_ALLOC_CLUSTERED) in find_free_extent_update_loop()
4209 return -ENOSPC; in find_free_extent_update_loop()
4215 if (ffe_ctl->empty_size == 0 && in find_free_extent_update_loop()
4216 ffe_ctl->empty_cluster == 0) in find_free_extent_update_loop()
4217 return -ENOSPC; in find_free_extent_update_loop()
4218 ffe_ctl->empty_size = 0; in find_free_extent_update_loop()
4219 ffe_ctl->empty_cluster = 0; in find_free_extent_update_loop()
4223 return -ENOSPC; in find_free_extent_update_loop()
4229 if (ffe_ctl->policy == BTRFS_EXTENT_ALLOC_ZONED) in find_free_extent_check_size_class()
4233 if (ffe_ctl->loop >= LOOP_WRONG_SIZE_CLASS) in find_free_extent_check_size_class()
4235 if (ffe_ctl->loop >= LOOP_UNSET_SIZE_CLASS && in find_free_extent_check_size_class()
4236 bg->size_class == BTRFS_BG_SZ_NONE) in find_free_extent_check_size_class()
4238 return ffe_ctl->size_class == bg->size_class; in find_free_extent_check_size_class()
4256 if (space_info->max_extent_size) { in prepare_allocation_clustered()
4257 spin_lock(&space_info->lock); in prepare_allocation_clustered()
4258 if (space_info->max_extent_size && in prepare_allocation_clustered()
4259 ffe_ctl->num_bytes > space_info->max_extent_size) { in prepare_allocation_clustered()
4260 ins->offset = space_info->max_extent_size; in prepare_allocation_clustered()
4261 spin_unlock(&space_info->lock); in prepare_allocation_clustered()
4262 return -ENOSPC; in prepare_allocation_clustered()
4263 } else if (space_info->max_extent_size) { in prepare_allocation_clustered()
4264 ffe_ctl->use_cluster = false; in prepare_allocation_clustered()
4266 spin_unlock(&space_info->lock); in prepare_allocation_clustered()
4269 ffe_ctl->last_ptr = fetch_cluster_info(fs_info, space_info, in prepare_allocation_clustered()
4270 &ffe_ctl->empty_cluster); in prepare_allocation_clustered()
4271 if (ffe_ctl->last_ptr) { in prepare_allocation_clustered()
4272 struct btrfs_free_cluster *last_ptr = ffe_ctl->last_ptr; in prepare_allocation_clustered()
4274 spin_lock(&last_ptr->lock); in prepare_allocation_clustered()
4275 if (last_ptr->block_group) in prepare_allocation_clustered()
4276 ffe_ctl->hint_byte = last_ptr->window_start; in prepare_allocation_clustered()
4277 if (last_ptr->fragmented) { in prepare_allocation_clustered()
4283 ffe_ctl->hint_byte = last_ptr->window_start; in prepare_allocation_clustered()
4284 ffe_ctl->use_cluster = false; in prepare_allocation_clustered()
4286 spin_unlock(&last_ptr->lock); in prepare_allocation_clustered()
4295 if (ffe_ctl->for_treelog) { in prepare_allocation_zoned()
4296 spin_lock(&fs_info->treelog_bg_lock); in prepare_allocation_zoned()
4297 if (fs_info->treelog_bg) in prepare_allocation_zoned()
4298 ffe_ctl->hint_byte = fs_info->treelog_bg; in prepare_allocation_zoned()
4299 spin_unlock(&fs_info->treelog_bg_lock); in prepare_allocation_zoned()
4300 } else if (ffe_ctl->for_data_reloc) { in prepare_allocation_zoned()
4301 spin_lock(&fs_info->relocation_bg_lock); in prepare_allocation_zoned()
4302 if (fs_info->data_reloc_bg) in prepare_allocation_zoned()
4303 ffe_ctl->hint_byte = fs_info->data_reloc_bg; in prepare_allocation_zoned()
4304 spin_unlock(&fs_info->relocation_bg_lock); in prepare_allocation_zoned()
4305 } else if (ffe_ctl->flags & BTRFS_BLOCK_GROUP_DATA) { in prepare_allocation_zoned()
4308 spin_lock(&fs_info->zone_active_bgs_lock); in prepare_allocation_zoned()
4309 list_for_each_entry(block_group, &fs_info->zone_active_bgs, active_bg_list) { in prepare_allocation_zoned()
4314 u64 avail = block_group->zone_capacity - block_group->alloc_offset; in prepare_allocation_zoned()
4316 if (block_group_bits(block_group, ffe_ctl->flags) && in prepare_allocation_zoned()
4317 avail >= ffe_ctl->num_bytes) { in prepare_allocation_zoned()
4318 ffe_ctl->hint_byte = block_group->start; in prepare_allocation_zoned()
4322 spin_unlock(&fs_info->zone_active_bgs_lock); in prepare_allocation_zoned()
4333 switch (ffe_ctl->policy) { in prepare_allocation()
4347 * ins->objectid == start position
4348 * ins->flags = BTRFS_EXTENT_ITEM_KEY
4349 * ins->offset == the size of the hole.
4358 * |- Iterate through all block groups
4359 * | |- Get a valid block group
4360 * | |- Try to do clustered allocation in that block group
4361 * | |- Try to do unclustered allocation in that block group
4362 * | |- Check if the result is valid
4363 * | | |- If valid, then exit
4364 * | |- Jump to next block group
4366 * |- Push harder to find free extents
4367 * |- If not found, re-iterate all block groups
4373 struct btrfs_fs_info *fs_info = root->fs_info; in find_free_extent()
4380 WARN_ON(ffe_ctl->num_bytes < fs_info->sectorsize); in find_free_extent()
4382 ffe_ctl->search_start = 0; in find_free_extent()
4384 ffe_ctl->empty_cluster = 0; in find_free_extent()
4385 ffe_ctl->last_ptr = NULL; in find_free_extent()
4386 ffe_ctl->use_cluster = true; in find_free_extent()
4387 ffe_ctl->have_caching_bg = false; in find_free_extent()
4388 ffe_ctl->orig_have_caching_bg = false; in find_free_extent()
4389 ffe_ctl->index = btrfs_bg_flags_to_raid_index(ffe_ctl->flags); in find_free_extent()
4390 ffe_ctl->loop = 0; in find_free_extent()
4391 ffe_ctl->retry_uncached = false; in find_free_extent()
4392 ffe_ctl->cached = 0; in find_free_extent()
4393 ffe_ctl->max_extent_size = 0; in find_free_extent()
4394 ffe_ctl->total_free_space = 0; in find_free_extent()
4395 ffe_ctl->found_offset = 0; in find_free_extent()
4396 ffe_ctl->policy = BTRFS_EXTENT_ALLOC_CLUSTERED; in find_free_extent()
4397 ffe_ctl->size_class = btrfs_calc_block_group_size_class(ffe_ctl->num_bytes); in find_free_extent()
4400 ffe_ctl->policy = BTRFS_EXTENT_ALLOC_ZONED; in find_free_extent()
4402 ins->type = BTRFS_EXTENT_ITEM_KEY; in find_free_extent()
4403 ins->objectid = 0; in find_free_extent()
4404 ins->offset = 0; in find_free_extent()
4408 space_info = btrfs_find_space_info(fs_info, ffe_ctl->flags); in find_free_extent()
4410 btrfs_err(fs_info, "No space info for %llu", ffe_ctl->flags); in find_free_extent()
4411 return -ENOSPC; in find_free_extent()
4418 ffe_ctl->search_start = max(ffe_ctl->search_start, in find_free_extent()
4420 ffe_ctl->search_start = max(ffe_ctl->search_start, ffe_ctl->hint_byte); in find_free_extent()
4421 if (ffe_ctl->search_start == ffe_ctl->hint_byte) { in find_free_extent()
4423 ffe_ctl->search_start); in find_free_extent()
4428 * However if we are re-searching with an ideal block group in find_free_extent()
4431 if (block_group && block_group_bits(block_group, ffe_ctl->flags) && in find_free_extent()
4432 block_group->cached != BTRFS_CACHE_NO) { in find_free_extent()
4433 down_read(&space_info->groups_sem); in find_free_extent()
4434 if (list_empty(&block_group->list) || in find_free_extent()
4435 block_group->ro) { in find_free_extent()
4443 up_read(&space_info->groups_sem); in find_free_extent()
4445 ffe_ctl->index = btrfs_bg_flags_to_raid_index( in find_free_extent()
4446 block_group->flags); in find_free_extent()
4448 ffe_ctl->delalloc); in find_free_extent()
4449 ffe_ctl->hinted = true; in find_free_extent()
4458 ffe_ctl->have_caching_bg = false; in find_free_extent()
4459 if (ffe_ctl->index == btrfs_bg_flags_to_raid_index(ffe_ctl->flags) || in find_free_extent()
4460 ffe_ctl->index == 0) in find_free_extent()
4462 down_read(&space_info->groups_sem); in find_free_extent()
4464 &space_info->block_groups[ffe_ctl->index], list) { in find_free_extent()
4467 ffe_ctl->hinted = false; in find_free_extent()
4468 /* If the block group is read-only, we can skip it entirely. */ in find_free_extent()
4469 if (unlikely(block_group->ro)) { in find_free_extent()
4470 if (ffe_ctl->for_treelog) in find_free_extent()
4472 if (ffe_ctl->for_data_reloc) in find_free_extent()
4477 btrfs_grab_block_group(block_group, ffe_ctl->delalloc); in find_free_extent()
4478 ffe_ctl->search_start = block_group->start; in find_free_extent()
4485 if (!block_group_bits(block_group, ffe_ctl->flags)) { in find_free_extent()
4496 if ((ffe_ctl->flags & extra) && !(block_group->flags & extra)) in find_free_extent()
4504 btrfs_release_block_group(block_group, ffe_ctl->delalloc); in find_free_extent()
4510 ffe_ctl->cached = btrfs_block_group_done(block_group); in find_free_extent()
4511 if (unlikely(!ffe_ctl->cached)) { in find_free_extent()
4512 ffe_ctl->have_caching_bg = true; in find_free_extent()
4531 if (unlikely(block_group->cached == BTRFS_CACHE_ERROR)) { in find_free_extent()
4533 cache_block_group_error = -EIO; in find_free_extent()
4546 btrfs_release_block_group(block_group, ffe_ctl->delalloc); in find_free_extent()
4551 ffe_ctl->search_start = round_up(ffe_ctl->found_offset, in find_free_extent()
4552 fs_info->stripesize); in find_free_extent()
4555 if (ffe_ctl->search_start + ffe_ctl->num_bytes > in find_free_extent()
4556 block_group->start + block_group->length) { in find_free_extent()
4558 ffe_ctl->found_offset, in find_free_extent()
4559 ffe_ctl->num_bytes); in find_free_extent()
4563 if (ffe_ctl->found_offset < ffe_ctl->search_start) in find_free_extent()
4565 ffe_ctl->found_offset, in find_free_extent()
4566 ffe_ctl->search_start - ffe_ctl->found_offset); in find_free_extent()
4568 ret = btrfs_add_reserved_bytes(block_group, ffe_ctl->ram_bytes, in find_free_extent()
4569 ffe_ctl->num_bytes, in find_free_extent()
4570 ffe_ctl->delalloc, in find_free_extent()
4571 ffe_ctl->loop >= LOOP_WRONG_SIZE_CLASS); in find_free_extent()
4572 if (ret == -EAGAIN) { in find_free_extent()
4574 ffe_ctl->found_offset, in find_free_extent()
4575 ffe_ctl->num_bytes); in find_free_extent()
4581 ins->objectid = ffe_ctl->search_start; in find_free_extent()
4582 ins->offset = ffe_ctl->num_bytes; in find_free_extent()
4585 btrfs_release_block_group(block_group, ffe_ctl->delalloc); in find_free_extent()
4588 if (!ffe_ctl->cached && ffe_ctl->loop > LOOP_CACHING_NOWAIT && in find_free_extent()
4589 !ffe_ctl->retry_uncached) { in find_free_extent()
4590 ffe_ctl->retry_uncached = true; in find_free_extent()
4592 ffe_ctl->num_bytes + in find_free_extent()
4593 ffe_ctl->empty_cluster + in find_free_extent()
4594 ffe_ctl->empty_size); in find_free_extent()
4597 release_block_group(block_group, ffe_ctl, ffe_ctl->delalloc); in find_free_extent()
4600 up_read(&space_info->groups_sem); in find_free_extent()
4606 if (ret == -ENOSPC && !cache_block_group_error) { in find_free_extent()
4608 * Use ffe_ctl->total_free_space as fallback if we can't find in find_free_extent()
4611 if (!ffe_ctl->max_extent_size) in find_free_extent()
4612 ffe_ctl->max_extent_size = ffe_ctl->total_free_space; in find_free_extent()
4613 spin_lock(&space_info->lock); in find_free_extent()
4614 space_info->max_extent_size = ffe_ctl->max_extent_size; in find_free_extent()
4615 spin_unlock(&space_info->lock); in find_free_extent()
4616 ins->offset = ffe_ctl->max_extent_size; in find_free_extent()
4617 } else if (ret == -ENOSPC) { in find_free_extent()
4627 * @root - The root that will contain this extent
4629 * @ram_bytes - The amount of space in ram that @num_bytes take. This
4633 * @num_bytes - Number of bytes to allocate on-disk.
4635 * @min_alloc_size - Indicates the minimum amount of space that the
4642 * @empty_size - A hint that you plan on doing more COW. This is the
4647 * @hint_byte - Hint to the allocator to start searching above the byte
4650 * @ins - This key is modified to record the found hole. It will
4652 * ins->objectid == start position
4653 * ins->flags = BTRFS_EXTENT_ITEM_KEY
4654 * ins->offset == the size of the hole.
4656 * @is_data - Boolean flag indicating whether an extent is
4659 * @delalloc - Boolean flag indicating whether this allocation is for
4665 * case -ENOSPC is returned then @ins->offset will contain the size of the
4673 struct btrfs_fs_info *fs_info = root->fs_info; in btrfs_reserve_extent()
4683 WARN_ON(num_bytes < fs_info->sectorsize); in btrfs_reserve_extent()
4697 btrfs_dec_block_group_reservations(fs_info, ins->objectid); in btrfs_reserve_extent()
4698 } else if (ret == -ENOSPC) { in btrfs_reserve_extent()
4699 if (!final_tried && ins->offset) { in btrfs_reserve_extent()
4700 num_bytes = min(num_bytes >> 1, ins->offset); in btrfs_reserve_extent()
4702 fs_info->sectorsize); in btrfs_reserve_extent()
4713 "allocation failed flags %llu, wanted %llu tree-log %d, relocation: %d", in btrfs_reserve_extent()
4733 return -ENOSPC; in btrfs_free_reserved_extent()
4750 cache = btrfs_lookup_block_group(trans->fs_info, eb->start); in btrfs_pin_reserved_extent()
4752 btrfs_err(trans->fs_info, "unable to find block group for %llu", in btrfs_pin_reserved_extent()
4753 eb->start); in btrfs_pin_reserved_extent()
4754 return -ENOSPC; in btrfs_pin_reserved_extent()
4757 ret = pin_down_extent(trans, cache, eb->start, eb->len, 1); in btrfs_pin_reserved_extent()
4765 struct btrfs_fs_info *fs_info = trans->fs_info; in alloc_reserved_extent()
4789 struct btrfs_fs_info *fs_info = trans->fs_info; in alloc_reserved_file_extent()
4813 return -ENOMEM; in alloc_reserved_file_extent()
4815 extent_root = btrfs_extent_root(fs_info, ins->objectid); in alloc_reserved_file_extent()
4822 leaf = path->nodes[0]; in alloc_reserved_file_extent()
4823 extent_item = btrfs_item_ptr(leaf, path->slots[0], in alloc_reserved_file_extent()
4826 btrfs_set_extent_generation(leaf, extent_item, trans->transid); in alloc_reserved_file_extent()
4833 oref = (struct btrfs_extent_owner_ref *)(&iref->offset); in alloc_reserved_file_extent()
4846 ref = (struct btrfs_extent_data_ref *)(&iref->offset); in alloc_reserved_file_extent()
4853 btrfs_mark_buffer_dirty(trans, path->nodes[0]); in alloc_reserved_file_extent()
4856 return alloc_reserved_extent(trans, ins->objectid, ins->offset); in alloc_reserved_file_extent()
4863 struct btrfs_fs_info *fs_info = trans->fs_info; in alloc_reserved_tree_block()
4873 const u64 flags = (extent_op ? extent_op->flags_to_set : 0); in alloc_reserved_tree_block()
4878 extent_key.objectid = node->bytenr; in alloc_reserved_tree_block()
4884 extent_key.offset = node->num_bytes; in alloc_reserved_tree_block()
4891 return -ENOMEM; in alloc_reserved_tree_block()
4901 leaf = path->nodes[0]; in alloc_reserved_tree_block()
4902 extent_item = btrfs_item_ptr(leaf, path->slots[0], in alloc_reserved_tree_block()
4905 btrfs_set_extent_generation(leaf, extent_item, trans->transid); in alloc_reserved_tree_block()
4913 btrfs_set_tree_block_key(leaf, block_info, &extent_op->key); in alloc_reserved_tree_block()
4918 if (node->type == BTRFS_SHARED_BLOCK_REF_KEY) { in alloc_reserved_tree_block()
4921 btrfs_set_extent_inline_ref_offset(leaf, iref, node->parent); in alloc_reserved_tree_block()
4925 btrfs_set_extent_inline_ref_offset(leaf, iref, node->ref_root); in alloc_reserved_tree_block()
4931 return alloc_reserved_extent(trans, node->bytenr, fs_info->nodesize); in alloc_reserved_tree_block()
4941 .bytenr = ins->objectid, in btrfs_alloc_reserved_file_extent()
4942 .num_bytes = ins->offset, in btrfs_alloc_reserved_file_extent()
4949 if (btrfs_is_data_reloc_root(root) && is_fstree(root->relocation_src_root)) in btrfs_alloc_reserved_file_extent()
4950 generic_ref.owning_root = root->relocation_src_root; in btrfs_alloc_reserved_file_extent()
4953 btrfs_ref_tree_mod(root->fs_info, &generic_ref); in btrfs_alloc_reserved_file_extent()
4967 struct btrfs_fs_info *fs_info = trans->fs_info; in btrfs_alloc_logged_file_extent()
4973 .num_bytes = ins->offset, in btrfs_alloc_logged_file_extent()
4974 .generation = trans->transid, in btrfs_alloc_logged_file_extent()
4984 ret = __exclude_logged_extent(fs_info, ins->objectid, in btrfs_alloc_logged_file_extent()
4985 ins->offset); in btrfs_alloc_logged_file_extent()
4990 block_group = btrfs_lookup_block_group(fs_info, ins->objectid); in btrfs_alloc_logged_file_extent()
4992 return -EINVAL; in btrfs_alloc_logged_file_extent()
4994 space_info = block_group->space_info; in btrfs_alloc_logged_file_extent()
4995 spin_lock(&space_info->lock); in btrfs_alloc_logged_file_extent()
4996 spin_lock(&block_group->lock); in btrfs_alloc_logged_file_extent()
4997 space_info->bytes_reserved += ins->offset; in btrfs_alloc_logged_file_extent()
4998 block_group->reserved += ins->offset; in btrfs_alloc_logged_file_extent()
4999 spin_unlock(&block_group->lock); in btrfs_alloc_logged_file_extent()
5000 spin_unlock(&space_info->lock); in btrfs_alloc_logged_file_extent()
5005 btrfs_pin_extent(trans, ins->objectid, ins->offset, 1); in btrfs_alloc_logged_file_extent()
5018 if (eb->lock_owner == current->pid) { in check_eb_lock_owner()
5019 btrfs_err_rl(eb->fs_info, in check_eb_lock_owner()
5021 eb->start, btrfs_header_owner(eb), current->pid); in check_eb_lock_owner()
5038 struct btrfs_fs_info *fs_info = root->fs_info; in btrfs_init_new_buffer()
5048 return ERR_PTR(-EUCLEAN); in btrfs_init_new_buffer()
5062 !test_bit(BTRFS_ROOT_RESET_LOCKDEP_CLASS, &root->state)) in btrfs_init_new_buffer()
5066 btrfs_set_header_generation(buf, trans->transid); in btrfs_init_new_buffer()
5077 clear_bit(EXTENT_BUFFER_STALE, &buf->bflags); in btrfs_init_new_buffer()
5078 clear_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &buf->bflags); in btrfs_init_new_buffer()
5084 btrfs_set_header_bytenr(buf, buf->start); in btrfs_init_new_buffer()
5085 btrfs_set_header_generation(buf, trans->transid); in btrfs_init_new_buffer()
5088 write_extent_buffer_fsid(buf, fs_info->fs_devices->metadata_uuid); in btrfs_init_new_buffer()
5089 write_extent_buffer_chunk_tree_uuid(buf, fs_info->chunk_tree_uuid); in btrfs_init_new_buffer()
5091 buf->log_index = root->log_transid % 2; in btrfs_init_new_buffer()
5096 if (buf->log_index == 0) in btrfs_init_new_buffer()
5097 set_extent_bit(&root->dirty_log_pages, buf->start, in btrfs_init_new_buffer()
5098 buf->start + buf->len - 1, in btrfs_init_new_buffer()
5101 set_extent_bit(&root->dirty_log_pages, buf->start, in btrfs_init_new_buffer()
5102 buf->start + buf->len - 1, in btrfs_init_new_buffer()
5105 buf->log_index = -1; in btrfs_init_new_buffer()
5106 set_extent_bit(&trans->transaction->dirty_pages, buf->start, in btrfs_init_new_buffer()
5107 buf->start + buf->len - 1, EXTENT_DIRTY, NULL); in btrfs_init_new_buffer()
5126 struct btrfs_fs_info *fs_info = root->fs_info; in btrfs_alloc_tree_block()
5132 u32 blocksize = fs_info->nodesize; in btrfs_alloc_tree_block()
5138 buf = btrfs_init_new_buffer(trans, root, root->alloc_bytenr, in btrfs_alloc_tree_block()
5141 root->alloc_bytenr += blocksize; in btrfs_alloc_tree_block()
5185 ret = -ENOMEM; in btrfs_alloc_tree_block()
5189 memcpy(&extent_op->key, key, sizeof(extent_op->key)); in btrfs_alloc_tree_block()
5191 memset(&extent_op->key, 0, sizeof(extent_op->key)); in btrfs_alloc_tree_block()
5192 extent_op->flags_to_set = flags; in btrfs_alloc_tree_block()
5193 extent_op->update_key = (skinny_metadata ? false : true); in btrfs_alloc_tree_block()
5194 extent_op->update_flags = (flags != 0); in btrfs_alloc_tree_block()
5259 * @refs: the number of refs for wc->level - 1
5260 * @flags: the flags for wc->level - 1
5264 * wc->level should be read and walked into, or if we can simply delete our
5277 int level = wc->level; in visit_node_for_delete()
5280 ASSERT(wc->refs[level - 1] > 0); in visit_node_for_delete()
5286 if (wc->stage == UPDATE_BACKREF) { in visit_node_for_delete()
5296 if (wc->refs[level - 1] == 1) in visit_node_for_delete()
5311 if (!wc->update_ref || generation <= root->root_key.offset) in visit_node_for_delete()
5319 if (btrfs_comp_cpu_keys(&key, &wc->update_progress) < 0) in visit_node_for_delete()
5331 struct btrfs_fs_info *fs_info = root->fs_info; in reada_walk_down()
5342 if (path->slots[wc->level] < wc->reada_slot) { in reada_walk_down()
5343 wc->reada_count = wc->reada_count * 2 / 3; in reada_walk_down()
5344 wc->reada_count = max(wc->reada_count, 2); in reada_walk_down()
5346 wc->reada_count = wc->reada_count * 3 / 2; in reada_walk_down()
5347 wc->reada_count = min_t(int, wc->reada_count, in reada_walk_down()
5351 eb = path->nodes[wc->level]; in reada_walk_down()
5354 for (slot = path->slots[wc->level]; slot < nritems; slot++) { in reada_walk_down()
5355 if (nread >= wc->reada_count) in reada_walk_down()
5362 if (slot == path->slots[wc->level]) in reada_walk_down()
5365 if (wc->stage == UPDATE_BACKREF && in reada_walk_down()
5366 generation <= root->root_key.offset) in reada_walk_down()
5371 wc->level - 1, 1, &refs, in reada_walk_down()
5393 wc->reada_slot = slot; in reada_walk_down()
5399 * when wc->stage == UPDATE_BACKREF, this function updates
5409 struct btrfs_fs_info *fs_info = root->fs_info; in walk_down_proc()
5410 int level = wc->level; in walk_down_proc()
5411 struct extent_buffer *eb = path->nodes[level]; in walk_down_proc()
5415 if (wc->stage == UPDATE_BACKREF && btrfs_header_owner(eb) != btrfs_root_id(root)) in walk_down_proc()
5422 if (wc->lookup_info && in walk_down_proc()
5423 ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) || in walk_down_proc()
5424 (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) { in walk_down_proc()
5425 ASSERT(path->locks[level]); in walk_down_proc()
5427 eb->start, level, 1, in walk_down_proc()
5428 &wc->refs[level], in walk_down_proc()
5429 &wc->flags[level], in walk_down_proc()
5433 if (unlikely(wc->refs[level] == 0)) { in walk_down_proc()
5435 eb->start); in walk_down_proc()
5436 return -EUCLEAN; in walk_down_proc()
5440 if (wc->stage == DROP_REFERENCE) { in walk_down_proc()
5441 if (wc->refs[level] > 1) in walk_down_proc()
5444 if (path->locks[level] && !wc->keep_locks) { in walk_down_proc()
5445 btrfs_tree_unlock_rw(eb, path->locks[level]); in walk_down_proc()
5446 path->locks[level] = 0; in walk_down_proc()
5451 /* wc->stage == UPDATE_BACKREF */ in walk_down_proc()
5452 if (!(wc->flags[level] & flag)) { in walk_down_proc()
5453 ASSERT(path->locks[level]); in walk_down_proc()
5469 wc->flags[level] |= flag; in walk_down_proc()
5476 if (path->locks[level] && level > 0) { in walk_down_proc()
5477 btrfs_tree_unlock_rw(eb, path->locks[level]); in walk_down_proc()
5478 path->locks[level] = 0; in walk_down_proc()
5500 return -ENOMEM; in check_ref_exists()
5503 root->fs_info->nodesize, parent, in check_ref_exists()
5505 if (ret != -ENOENT) { in check_ref_exists()
5508 * return the error if it's not -ENOENT; in check_ref_exists()
5519 delayed_refs = &trans->transaction->delayed_refs; in check_ref_exists()
5520 spin_lock(&delayed_refs->lock); in check_ref_exists()
5524 if (!mutex_trylock(&head->mutex)) { in check_ref_exists()
5530 refcount_inc(&head->refs); in check_ref_exists()
5531 spin_unlock(&delayed_refs->lock); in check_ref_exists()
5535 mutex_lock(&head->mutex); in check_ref_exists()
5536 mutex_unlock(&head->mutex); in check_ref_exists()
5541 exists = btrfs_find_delayed_tree_ref(head, root->root_key.objectid, parent); in check_ref_exists()
5542 mutex_unlock(&head->mutex); in check_ref_exists()
5544 spin_unlock(&delayed_refs->lock); in check_ref_exists()
5551 * block we need to drop the lock, read it off of the disk, re-lock it and
5562 int level = wc->level; in check_next_block_uptodate()
5567 generation = btrfs_node_ptr_generation(path->nodes[level], path->slots[level]); in check_next_block_uptodate()
5572 check.level = level - 1; in check_next_block_uptodate()
5576 btrfs_node_key_to_cpu(path->nodes[level], &check.first_key, path->slots[level]); in check_next_block_uptodate()
5587 wc->lookup_info = 1; in check_next_block_uptodate()
5592 * If we determine that we don't have to visit wc->level - 1 then we need to
5607 .bytenr = next->start, in maybe_drop_reference()
5608 .num_bytes = root->fs_info->nodesize, in maybe_drop_reference()
5612 int level = wc->level; in maybe_drop_reference()
5616 if (wc->stage == UPDATE_BACKREF) in maybe_drop_reference()
5619 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) { in maybe_drop_reference()
5620 ref.parent = path->nodes[level]->start; in maybe_drop_reference()
5622 ASSERT(btrfs_root_id(root) == btrfs_header_owner(path->nodes[level])); in maybe_drop_reference()
5623 if (btrfs_root_id(root) != btrfs_header_owner(path->nodes[level])) { in maybe_drop_reference()
5624 btrfs_err(root->fs_info, "mismatched block owner"); in maybe_drop_reference()
5625 return -EIO; in maybe_drop_reference()
5633 * ->restarted flag. in maybe_drop_reference()
5635 if (wc->restarted) { in maybe_drop_reference()
5636 ret = check_ref_exists(trans, root, next->start, ref.parent, in maybe_drop_reference()
5637 level - 1); in maybe_drop_reference()
5641 wc->restarted = 0; in maybe_drop_reference()
5650 wc->refs[level - 1] > 1) { in maybe_drop_reference()
5651 u64 generation = btrfs_node_ptr_generation(path->nodes[level], in maybe_drop_reference()
5652 path->slots[level]); in maybe_drop_reference()
5654 ret = btrfs_qgroup_trace_subtree(trans, next, generation, level - 1); in maybe_drop_reference()
5656 btrfs_err_rl(root->fs_info, in maybe_drop_reference()
5668 wc->drop_level = level; in maybe_drop_reference()
5669 find_next_key(path, level, &wc->drop_progress); in maybe_drop_reference()
5671 btrfs_init_tree_ref(&ref, level - 1, 0, false); in maybe_drop_reference()
5678 * when wc->stage == DROP_REFERENCE, this function checks
5681 * rooted at the block, this function changes wc->stage to
5693 struct btrfs_fs_info *fs_info = root->fs_info; in do_walk_down()
5698 int level = wc->level; in do_walk_down()
5701 generation = btrfs_node_ptr_generation(path->nodes[level], in do_walk_down()
5702 path->slots[level]); in do_walk_down()
5708 if (wc->stage == UPDATE_BACKREF && in do_walk_down()
5709 generation <= root->root_key.offset) { in do_walk_down()
5710 wc->lookup_info = 1; in do_walk_down()
5714 bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]); in do_walk_down()
5717 level - 1); in do_walk_down()
5723 ret = btrfs_lookup_extent_info(trans, fs_info, bytenr, level - 1, 1, in do_walk_down()
5724 &wc->refs[level - 1], in do_walk_down()
5725 &wc->flags[level - 1], in do_walk_down()
5730 if (unlikely(wc->refs[level - 1] == 0)) { in do_walk_down()
5733 ret = -EUCLEAN; in do_walk_down()
5736 wc->lookup_info = 0; in do_walk_down()
5739 if (!visit_node_for_delete(root, wc, path->nodes[level], in do_walk_down()
5740 wc->refs[level - 1], wc->flags[level - 1], in do_walk_down()
5741 path->slots[level])) in do_walk_down()
5749 if (wc->stage == DROP_REFERENCE && wc->refs[level - 1] > 1) { in do_walk_down()
5750 wc->stage = UPDATE_BACKREF; in do_walk_down()
5751 wc->shared_level = level - 1; in do_walk_down()
5758 level--; in do_walk_down()
5761 btrfs_err(root->fs_info, "mismatched level"); in do_walk_down()
5762 ret = -EIO; in do_walk_down()
5765 path->nodes[level] = next; in do_walk_down()
5766 path->slots[level] = 0; in do_walk_down()
5767 path->locks[level] = BTRFS_WRITE_LOCK; in do_walk_down()
5768 wc->level = level; in do_walk_down()
5769 if (wc->level == 1) in do_walk_down()
5770 wc->reada_slot = 0; in do_walk_down()
5776 wc->refs[level - 1] = 0; in do_walk_down()
5777 wc->flags[level - 1] = 0; in do_walk_down()
5778 wc->lookup_info = 1; in do_walk_down()
5791 * when wc->stage == DROP_REFERENCE, this function drops
5794 * when wc->stage == UPDATE_BACKREF, this function changes
5795 * wc->stage back to DROP_REFERENCE if we changed wc->stage
5805 struct btrfs_fs_info *fs_info = root->fs_info; in walk_up_proc()
5807 int level = wc->level; in walk_up_proc()
5808 struct extent_buffer *eb = path->nodes[level]; in walk_up_proc()
5811 if (wc->stage == UPDATE_BACKREF) { in walk_up_proc()
5812 ASSERT(wc->shared_level >= level); in walk_up_proc()
5813 if (level < wc->shared_level) in walk_up_proc()
5816 ret = find_next_key(path, level + 1, &wc->update_progress); in walk_up_proc()
5818 wc->update_ref = 0; in walk_up_proc()
5820 wc->stage = DROP_REFERENCE; in walk_up_proc()
5821 wc->shared_level = -1; in walk_up_proc()
5822 path->slots[level] = 0; in walk_up_proc()
5829 if (!path->locks[level]) { in walk_up_proc()
5832 path->locks[level] = BTRFS_WRITE_LOCK; in walk_up_proc()
5835 eb->start, level, 1, in walk_up_proc()
5836 &wc->refs[level], in walk_up_proc()
5837 &wc->flags[level], in walk_up_proc()
5840 btrfs_tree_unlock_rw(eb, path->locks[level]); in walk_up_proc()
5841 path->locks[level] = 0; in walk_up_proc()
5844 if (unlikely(wc->refs[level] == 0)) { in walk_up_proc()
5845 btrfs_tree_unlock_rw(eb, path->locks[level]); in walk_up_proc()
5847 eb->start); in walk_up_proc()
5848 return -EUCLEAN; in walk_up_proc()
5850 if (wc->refs[level] == 1) { in walk_up_proc()
5851 btrfs_tree_unlock_rw(eb, path->locks[level]); in walk_up_proc()
5852 path->locks[level] = 0; in walk_up_proc()
5858 /* wc->stage == DROP_REFERENCE */ in walk_up_proc()
5859 ASSERT(path->locks[level] || wc->refs[level] == 1); in walk_up_proc()
5861 if (wc->refs[level] == 1) { in walk_up_proc()
5863 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) in walk_up_proc()
5881 if (!path->locks[level]) { in walk_up_proc()
5883 path->locks[level] = BTRFS_WRITE_LOCK; in walk_up_proc()
5888 if (eb == root->node) { in walk_up_proc()
5889 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) in walk_up_proc()
5890 parent = eb->start; in walk_up_proc()
5894 if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF) in walk_up_proc()
5895 parent = path->nodes[level + 1]->start; in walk_up_proc()
5897 btrfs_header_owner(path->nodes[level + 1])) in walk_up_proc()
5902 wc->refs[level] == 1); in walk_up_proc()
5906 wc->refs[level] = 0; in walk_up_proc()
5907 wc->flags[level] = 0; in walk_up_proc()
5913 return -EUCLEAN; in walk_up_proc()
5920 * wc->level. At this point path->nodes[wc->level] should be populated and
5929 * our current path->nodes[wc->level]. For DROP_REFERENCE that means dropping
5942 int level = wc->level; in walk_down_tree()
5945 wc->lookup_info = 1; in walk_down_tree()
5954 if (path->slots[level] >= in walk_down_tree()
5955 btrfs_header_nritems(path->nodes[level])) in walk_down_tree()
5960 path->slots[level]++; in walk_down_tree()
5964 level = wc->level; in walk_down_tree()
5975 * UPDATE_BACKREF. If we wc->level is currently less than our wc->shared_level
5978 * wc->shared_level. Once we're at or above our wc->shared_level we can switch
5991 int level = wc->level; in walk_up_tree()
5994 path->slots[level] = btrfs_header_nritems(path->nodes[level]); in walk_up_tree()
5995 while (level < max_level && path->nodes[level]) { in walk_up_tree()
5996 wc->level = level; in walk_up_tree()
5997 if (path->slots[level] + 1 < in walk_up_tree()
5998 btrfs_header_nritems(path->nodes[level])) { in walk_up_tree()
5999 path->slots[level]++; in walk_up_tree()
6008 if (path->locks[level]) { in walk_up_tree()
6009 btrfs_tree_unlock_rw(path->nodes[level], in walk_up_tree()
6010 path->locks[level]); in walk_up_tree()
6011 path->locks[level] = 0; in walk_up_tree()
6013 free_extent_buffer(path->nodes[level]); in walk_up_tree()
6014 path->nodes[level] = NULL; in walk_up_tree()
6032 * If called with for_reloc == 0, may exit early with -EAGAIN
6037 struct btrfs_fs_info *fs_info = root->fs_info; in btrfs_drop_snapshot()
6040 struct btrfs_root *tree_root = fs_info->tree_root; in btrfs_drop_snapshot()
6041 struct btrfs_root_item *root_item = &root->root_item; in btrfs_drop_snapshot()
6054 ret = -ENOMEM; in btrfs_drop_snapshot()
6061 ret = -ENOMEM; in btrfs_drop_snapshot()
6090 set_bit(BTRFS_ROOT_DELETING, &root->state); in btrfs_drop_snapshot()
6091 unfinished_drop = test_bit(BTRFS_ROOT_UNFINISHED_DROP, &root->state); in btrfs_drop_snapshot()
6093 if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) { in btrfs_drop_snapshot()
6094 level = btrfs_header_level(root->node); in btrfs_drop_snapshot()
6095 path->nodes[level] = btrfs_lock_root_node(root); in btrfs_drop_snapshot()
6096 path->slots[level] = 0; in btrfs_drop_snapshot()
6097 path->locks[level] = BTRFS_WRITE_LOCK; in btrfs_drop_snapshot()
6098 memset(&wc->update_progress, 0, in btrfs_drop_snapshot()
6099 sizeof(wc->update_progress)); in btrfs_drop_snapshot()
6101 btrfs_disk_key_to_cpu(&key, &root_item->drop_progress); in btrfs_drop_snapshot()
6102 memcpy(&wc->update_progress, &key, in btrfs_drop_snapshot()
6103 sizeof(wc->update_progress)); in btrfs_drop_snapshot()
6107 path->lowest_level = level; in btrfs_drop_snapshot()
6109 path->lowest_level = 0; in btrfs_drop_snapshot()
6122 level = btrfs_header_level(root->node); in btrfs_drop_snapshot()
6124 btrfs_tree_lock(path->nodes[level]); in btrfs_drop_snapshot()
6125 path->locks[level] = BTRFS_WRITE_LOCK; in btrfs_drop_snapshot()
6132 path->nodes[level]->start, in btrfs_drop_snapshot()
6133 level, 1, &wc->refs[level], in btrfs_drop_snapshot()
6134 &wc->flags[level], NULL); in btrfs_drop_snapshot()
6138 BUG_ON(wc->refs[level] == 0); in btrfs_drop_snapshot()
6143 btrfs_tree_unlock(path->nodes[level]); in btrfs_drop_snapshot()
6144 path->locks[level] = 0; in btrfs_drop_snapshot()
6145 WARN_ON(wc->refs[level] != 1); in btrfs_drop_snapshot()
6146 level--; in btrfs_drop_snapshot()
6150 wc->restarted = test_bit(BTRFS_ROOT_DEAD_TREE, &root->state); in btrfs_drop_snapshot()
6151 wc->level = level; in btrfs_drop_snapshot()
6152 wc->shared_level = -1; in btrfs_drop_snapshot()
6153 wc->stage = DROP_REFERENCE; in btrfs_drop_snapshot()
6154 wc->update_ref = update_ref; in btrfs_drop_snapshot()
6155 wc->keep_locks = 0; in btrfs_drop_snapshot()
6156 wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(fs_info); in btrfs_drop_snapshot()
6173 BUG_ON(wc->stage != DROP_REFERENCE); in btrfs_drop_snapshot()
6178 if (wc->stage == DROP_REFERENCE) { in btrfs_drop_snapshot()
6179 wc->drop_level = wc->level; in btrfs_drop_snapshot()
6180 btrfs_node_key_to_cpu(path->nodes[wc->drop_level], in btrfs_drop_snapshot()
6181 &wc->drop_progress, in btrfs_drop_snapshot()
6182 path->slots[wc->drop_level]); in btrfs_drop_snapshot()
6184 btrfs_cpu_key_to_disk(&root_item->drop_progress, in btrfs_drop_snapshot()
6185 &wc->drop_progress); in btrfs_drop_snapshot()
6186 btrfs_set_root_drop_level(root_item, wc->drop_level); in btrfs_drop_snapshot()
6188 BUG_ON(wc->level == 0); in btrfs_drop_snapshot()
6192 &root->root_key, in btrfs_drop_snapshot()
6200 btrfs_set_last_root_drop_gen(fs_info, trans->transid); in btrfs_drop_snapshot()
6206 ret = -EAGAIN; in btrfs_drop_snapshot()
6229 ret = btrfs_del_root(trans, &root->root_key); in btrfs_drop_snapshot()
6236 ret = btrfs_find_root(tree_root, &root->root_key, path, in btrfs_drop_snapshot()
6247 * The most common failure here is just -ENOENT. in btrfs_drop_snapshot()
6261 if (test_bit(BTRFS_ROOT_IN_RADIX, &root->state)) in btrfs_drop_snapshot()
6268 btrfs_set_last_root_drop_gen(fs_info, trans->transid); in btrfs_drop_snapshot()
6313 struct btrfs_fs_info *fs_info = root->fs_info; in btrfs_drop_subtree()
6324 return -ENOMEM; in btrfs_drop_subtree()
6329 return -ENOMEM; in btrfs_drop_subtree()
6334 atomic_inc(&parent->refs); in btrfs_drop_subtree()
6335 path->nodes[parent_level] = parent; in btrfs_drop_subtree()
6336 path->slots[parent_level] = btrfs_header_nritems(parent); in btrfs_drop_subtree()
6340 path->nodes[level] = node; in btrfs_drop_subtree()
6341 path->slots[level] = 0; in btrfs_drop_subtree()
6342 path->locks[level] = BTRFS_WRITE_LOCK; in btrfs_drop_subtree()
6344 wc->refs[parent_level] = 1; in btrfs_drop_subtree()
6345 wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF; in btrfs_drop_subtree()
6346 wc->level = level; in btrfs_drop_subtree()
6347 wc->shared_level = -1; in btrfs_drop_subtree()
6348 wc->stage = DROP_REFERENCE; in btrfs_drop_subtree()
6349 wc->update_ref = 0; in btrfs_drop_subtree()
6350 wc->keep_locks = 1; in btrfs_drop_subtree()
6351 wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(fs_info); in btrfs_drop_subtree()
6408 if (!bdev_max_discard_sectors(device->bdev)) in btrfs_trim_free_extents()
6412 if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) in btrfs_trim_free_extents()
6416 if (device->total_bytes <= device->bytes_used) in btrfs_trim_free_extents()
6422 struct btrfs_fs_info *fs_info = device->fs_info; in btrfs_trim_free_extents()
6425 ret = mutex_lock_interruptible(&fs_info->chunk_mutex); in btrfs_trim_free_extents()
6429 find_first_clear_extent_bit(&device->alloc_state, start, in btrfs_trim_free_extents()
6434 if (start > device->total_bytes) { in btrfs_trim_free_extents()
6438 start, end - start + 1, in btrfs_trim_free_extents()
6440 device->total_bytes); in btrfs_trim_free_extents()
6441 mutex_unlock(&fs_info->chunk_mutex); in btrfs_trim_free_extents()
6451 * end of the device it will set end to -1, in this case it's up in btrfs_trim_free_extents()
6454 end = min(end, device->total_bytes - 1); in btrfs_trim_free_extents()
6456 len = end - start + 1; in btrfs_trim_free_extents()
6460 mutex_unlock(&fs_info->chunk_mutex); in btrfs_trim_free_extents()
6465 ret = btrfs_issue_discard(device->bdev, start, len, in btrfs_trim_free_extents()
6468 set_extent_bit(&device->alloc_state, start, in btrfs_trim_free_extents()
6469 start + bytes - 1, CHUNK_TRIMMED, NULL); in btrfs_trim_free_extents()
6470 mutex_unlock(&fs_info->chunk_mutex); in btrfs_trim_free_extents()
6479 ret = -ERESTARTSYS; in btrfs_trim_free_extents()
6500 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; in btrfs_trim_fs()
6514 if (range->start == U64_MAX) in btrfs_trim_fs()
6515 return -EINVAL; in btrfs_trim_fs()
6518 * Check range overflow if range->len is set. in btrfs_trim_fs()
6519 * The default range->len is U64_MAX. in btrfs_trim_fs()
6521 if (range->len != U64_MAX && in btrfs_trim_fs()
6522 check_add_overflow(range->start, range->len, &range_end)) in btrfs_trim_fs()
6523 return -EINVAL; in btrfs_trim_fs()
6525 cache = btrfs_lookup_first_block_group(fs_info, range->start); in btrfs_trim_fs()
6527 if (cache->start >= range_end) { in btrfs_trim_fs()
6532 start = max(range->start, cache->start); in btrfs_trim_fs()
6533 end = min(range_end, cache->start + cache->length); in btrfs_trim_fs()
6535 if (end - start >= range->minlen) { in btrfs_trim_fs()
6548 range->minlen); in btrfs_trim_fs()
6564 mutex_lock(&fs_devices->device_list_mutex); in btrfs_trim_fs()
6565 list_for_each_entry(device, &fs_devices->devices, dev_list) { in btrfs_trim_fs()
6566 if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) in btrfs_trim_fs()
6578 mutex_unlock(&fs_devices->device_list_mutex); in btrfs_trim_fs()
6584 range->len = trimmed; in btrfs_trim_fs()