/linux-6.12.1/lib/ |
D | xarray.c | 38 static inline void xas_lock_type(struct xa_state *xas, unsigned int lock_type) in xas_lock_type() argument 41 xas_lock_irq(xas); in xas_lock_type() 43 xas_lock_bh(xas); in xas_lock_type() 45 xas_lock(xas); in xas_lock_type() 48 static inline void xas_unlock_type(struct xa_state *xas, unsigned int lock_type) in xas_unlock_type() argument 51 xas_unlock_irq(xas); in xas_unlock_type() 53 xas_unlock_bh(xas); in xas_unlock_type() 55 xas_unlock(xas); in xas_unlock_type() 126 static void xas_squash_marks(const struct xa_state *xas) in xas_squash_marks() argument 129 unsigned int limit = xas->xa_offset + xas->xa_sibs + 1; in xas_squash_marks() [all …]
|
D | test_xarray.c | 74 XA_STATE_ORDER(xas, xa, index, order); in xa_store_order() 78 xas_lock(&xas); in xa_store_order() 79 curr = xas_store(&xas, entry); in xa_store_order() 80 xas_unlock(&xas); in xa_store_order() 81 } while (xas_nomem(&xas, gfp)); in xa_store_order() 104 XA_STATE(xas, xa, 0); in check_xas_retry() 111 XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != xa_mk_value(0)); in check_xas_retry() 113 XA_BUG_ON(xa, !xa_is_retry(xas_reload(&xas))); in check_xas_retry() 114 XA_BUG_ON(xa, xas_retry(&xas, NULL)); in check_xas_retry() 115 XA_BUG_ON(xa, xas_retry(&xas, xa_mk_value(0))); in check_xas_retry() [all …]
|
D | idr.c | 383 XA_STATE(xas, &ida->xa, min / IDA_BITMAP_BITS); in ida_alloc_range() 395 xas_lock_irqsave(&xas, flags); in ida_alloc_range() 397 bitmap = xas_find_marked(&xas, max / IDA_BITMAP_BITS, XA_FREE_MARK); in ida_alloc_range() 398 if (xas.xa_index > min / IDA_BITMAP_BITS) in ida_alloc_range() 400 if (xas.xa_index * IDA_BITMAP_BITS + bit > max) in ida_alloc_range() 408 if (xas.xa_index * IDA_BITMAP_BITS + bit > max) in ida_alloc_range() 412 xas_store(&xas, xa_mk_value(tmp)); in ida_alloc_range() 422 xas_store(&xas, bitmap); in ida_alloc_range() 423 if (xas_error(&xas)) { in ida_alloc_range() 431 if (xas.xa_index * IDA_BITMAP_BITS + bit > max) in ida_alloc_range() [all …]
|
D | iov_iter.c | 1061 XA_STATE(xas, xa, index); in iter_xarray_populate_pages() 1066 for (page = xas_load(&xas); page; page = xas_next(&xas)) { in iter_xarray_populate_pages() 1067 if (xas_retry(&xas, page)) in iter_xarray_populate_pages() 1071 if (unlikely(page != xas_reload(&xas))) { in iter_xarray_populate_pages() 1072 xas_reset(&xas); in iter_xarray_populate_pages() 1076 pages[ret] = find_subpage(page, xas.xa_index); in iter_xarray_populate_pages() 1652 XA_STATE(xas, i->xarray, index); in iov_iter_extract_xarray_pages() 1663 for (page = xas_load(&xas); page; page = xas_next(&xas)) { in iov_iter_extract_xarray_pages() 1664 if (xas_retry(&xas, page)) in iov_iter_extract_xarray_pages() 1668 if (unlikely(page != xas_reload(&xas))) { in iov_iter_extract_xarray_pages() [all …]
|
/linux-6.12.1/include/linux/ |
D | xarray.h | 1410 #define xas_marked(xas, mark) xa_marked((xas)->xa, (mark)) argument 1411 #define xas_trylock(xas) xa_trylock((xas)->xa) argument 1412 #define xas_lock(xas) xa_lock((xas)->xa) argument 1413 #define xas_unlock(xas) xa_unlock((xas)->xa) argument 1414 #define xas_lock_bh(xas) xa_lock_bh((xas)->xa) argument 1415 #define xas_unlock_bh(xas) xa_unlock_bh((xas)->xa) argument 1416 #define xas_lock_irq(xas) xa_lock_irq((xas)->xa) argument 1417 #define xas_unlock_irq(xas) xa_unlock_irq((xas)->xa) argument 1418 #define xas_lock_irqsave(xas, flags) \ argument 1419 xa_lock_irqsave((xas)->xa, flags) [all …]
|
/linux-6.12.1/fs/ |
D | dax.c | 143 static wait_queue_head_t *dax_entry_waitqueue(struct xa_state *xas, in dax_entry_waitqueue() argument 147 unsigned long index = xas->xa_index; in dax_entry_waitqueue() 156 key->xa = xas->xa; in dax_entry_waitqueue() 159 hash = hash_long((unsigned long)xas->xa ^ index, DAX_WAIT_TABLE_BITS); in dax_entry_waitqueue() 181 static void dax_wake_entry(struct xa_state *xas, void *entry, in dax_wake_entry() argument 187 wq = dax_entry_waitqueue(xas, entry, &key); in dax_wake_entry() 209 static void *get_unlocked_entry(struct xa_state *xas, unsigned int order) in get_unlocked_entry() argument 219 entry = xas_find_conflict(xas); in get_unlocked_entry() 227 wq = dax_entry_waitqueue(xas, entry, &ewait.key); in get_unlocked_entry() 230 xas_unlock_irq(xas); in get_unlocked_entry() [all …]
|
/linux-6.12.1/tools/testing/radix-tree/ |
D | iteration_check.c | 23 XA_STATE(xas, xa, index); in my_item_insert() 28 xas_lock(&xas); in my_item_insert() 30 xas_set_order(&xas, index, order); in my_item_insert() 32 if (xas_find_conflict(&xas)) in my_item_insert() 34 xas_store(&xas, item); in my_item_insert() 35 xas_set_mark(&xas, TAG); in my_item_insert() 38 xas_unlock(&xas); in my_item_insert() 39 if (xas_nomem(&xas, GFP_KERNEL)) in my_item_insert() 69 XA_STATE(xas, &array, 0); in tagged_iteration_fn() 75 xas_set(&xas, 0); in tagged_iteration_fn() [all …]
|
D | multiorder.c | 18 XA_STATE_ORDER(xas, xa, index, order); in item_insert_order() 22 xas_lock(&xas); in item_insert_order() 23 xas_store(&xas, item); in item_insert_order() 24 xas_unlock(&xas); in item_insert_order() 25 } while (xas_nomem(&xas, GFP_KERNEL)); in item_insert_order() 27 if (!xas_error(&xas)) in item_insert_order() 31 return xas_error(&xas); in item_insert_order() 36 XA_STATE(xas, xa, 0); in multiorder_iteration() 56 xas_set(&xas, j); in multiorder_iteration() 57 xas_for_each(&xas, item, ULONG_MAX) { in multiorder_iteration() [all …]
|
D | test.c | 176 XA_STATE(xas, xa, start); in tag_tagged_items() 183 xas_lock_irq(&xas); in tag_tagged_items() 184 xas_for_each_marked(&xas, item, end, iftag) { in tag_tagged_items() 185 xas_set_mark(&xas, thentag); in tag_tagged_items() 189 xas_pause(&xas); in tag_tagged_items() 190 xas_unlock_irq(&xas); in tag_tagged_items() 192 xas_lock_irq(&xas); in tag_tagged_items() 194 xas_unlock_irq(&xas); in tag_tagged_items() 257 XA_STATE(xas, xa, 0); in item_kill_tree() 260 xas_for_each(&xas, entry, ULONG_MAX) { in item_kill_tree() [all …]
|
D | iteration_check_2.c | 15 XA_STATE(xas, arg, 0); in iterator() 21 xas_set(&xas, 0); in iterator() 23 xas_for_each_marked(&xas, entry, ULONG_MAX, XA_MARK_0) in iterator() 26 assert(xas.xa_index >= 100); in iterator()
|
D | regression1.c | 82 XA_STATE(xas, &mt_tree, start); in find_get_pages() 87 xas_for_each(&xas, page, ULONG_MAX) { in find_get_pages() 88 if (xas_retry(&xas, page)) in find_get_pages() 99 if (unlikely(page != xas_reload(&xas))) in find_get_pages() 108 xas_reset(&xas); in find_get_pages()
|
/linux-6.12.1/fs/cachefiles/ |
D | ondemand.c | 25 XA_STATE(xas, NULL, 0); in cachefiles_ondemand_fd_release() 32 xas.xa = &cache->reqs; in cachefiles_ondemand_fd_release() 42 xas_for_each_marked(&xas, req, ULONG_MAX, CACHEFILES_REQ_NEW) { in cachefiles_ondemand_fd_release() 46 xas_store(&xas, NULL); in cachefiles_ondemand_fd_release() 104 XA_STATE(xas, &cache->reqs, id); in cachefiles_ondemand_fd_ioctl() 113 req = xas_load(&xas); in cachefiles_ondemand_fd_ioctl() 119 xas_store(&xas, NULL); in cachefiles_ondemand_fd_ioctl() 149 XA_STATE(xas, &cache->reqs, 0); in cachefiles_ondemand_copen() 174 xas.xa_index = id; in cachefiles_ondemand_copen() 175 req = xas_load(&xas); in cachefiles_ondemand_copen() [all …]
|
/linux-6.12.1/mm/ |
D | memfd.c | 38 static void memfd_tag_pins(struct xa_state *xas) in memfd_tag_pins() argument 45 xas_lock_irq(xas); in memfd_tag_pins() 46 xas_for_each(xas, folio, ULONG_MAX) { in memfd_tag_pins() 48 xas_set_mark(xas, MEMFD_TAG_PINNED); in memfd_tag_pins() 54 xas_pause(xas); in memfd_tag_pins() 55 xas_unlock_irq(xas); in memfd_tag_pins() 57 xas_lock_irq(xas); in memfd_tag_pins() 59 xas_unlock_irq(xas); in memfd_tag_pins() 120 XA_STATE(xas, &mapping->i_pages, 0); in memfd_wait_for_pins() 124 memfd_tag_pins(&xas); in memfd_wait_for_pins() [all …]
|
D | filemap.c | 128 static void mapping_set_update(struct xa_state *xas, in mapping_set_update() argument 133 xas_set_update(xas, workingset_update_node); in mapping_set_update() 134 xas_set_lru(xas, &shadow_nodes); in mapping_set_update() 140 XA_STATE(xas, &mapping->i_pages, folio->index); in page_cache_delete() 143 mapping_set_update(&xas, mapping); in page_cache_delete() 145 xas_set_order(&xas, folio->index, folio_order(folio)); in page_cache_delete() 150 xas_store(&xas, shadow); in page_cache_delete() 151 xas_init_marks(&xas); in page_cache_delete() 290 XA_STATE(xas, &mapping->i_pages, fbatch->folios[0]->index); in page_cache_delete_batch() 295 mapping_set_update(&xas, mapping); in page_cache_delete_batch() [all …]
|
D | list_lru.c | 382 XA_STATE(xas, &lru->xa, 0); in memcg_destroy_list_lru() 388 xas_lock_irq(&xas); in memcg_destroy_list_lru() 389 xas_for_each(&xas, mlru, ULONG_MAX) { in memcg_destroy_list_lru() 391 xas_store(&xas, NULL); in memcg_destroy_list_lru() 393 xas_unlock_irq(&xas); in memcg_destroy_list_lru() 487 XA_STATE(xas, &lru->xa, 0); in memcg_list_lru_alloc() 516 xas_lock_irqsave(&xas, flags); in memcg_list_lru_alloc() 521 xas_set(&xas, index); in memcg_list_lru_alloc() 523 if (unlikely(index < 0 || xas_error(&xas) || xas_load(&xas))) { in memcg_list_lru_alloc() 526 xas_store(&xas, mlru); in memcg_list_lru_alloc() [all …]
|
D | swap_state.c | 95 XA_STATE_ORDER(xas, &address_space->i_pages, idx, folio_order(folio)); in add_to_swap_cache() 99 xas_set_update(&xas, workingset_update_node); in add_to_swap_cache() 110 xas_lock_irq(&xas); in add_to_swap_cache() 111 xas_create_range(&xas); in add_to_swap_cache() 112 if (xas_error(&xas)) in add_to_swap_cache() 115 VM_BUG_ON_FOLIO(xas.xa_index != idx + i, folio); in add_to_swap_cache() 117 old = xas_load(&xas); in add_to_swap_cache() 121 xas_store(&xas, folio); in add_to_swap_cache() 122 xas_next(&xas); in add_to_swap_cache() 128 xas_unlock_irq(&xas); in add_to_swap_cache() [all …]
|
D | khugepaged.c | 1814 XA_STATE_ORDER(xas, &mapping->i_pages, start, HPAGE_PMD_ORDER); in collapse_file() 1836 xas_lock_irq(&xas); in collapse_file() 1837 xas_create_range(&xas); in collapse_file() 1838 if (!xas_error(&xas)) in collapse_file() 1840 xas_unlock_irq(&xas); in collapse_file() 1841 if (!xas_nomem(&xas, GFP_KERNEL)) { in collapse_file() 1848 xas_set(&xas, index); in collapse_file() 1849 folio = xas_load(&xas); in collapse_file() 1851 VM_BUG_ON(index != xas.xa_index); in collapse_file() 1860 if (!xas_next_entry(&xas, end - 1)) { in collapse_file() [all …]
|
D | shmem.c | 489 XA_STATE(xas, &mapping->i_pages, index); in shmem_replace_entry() 494 item = xas_load(&xas); in shmem_replace_entry() 497 xas_store(&xas, replacement); in shmem_replace_entry() 789 XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio)); in shmem_add_to_page_cache() 804 xas_lock_irq(&xas); in shmem_add_to_page_cache() 805 if (expected != xas_find_conflict(&xas)) { in shmem_add_to_page_cache() 806 xas_set_err(&xas, -EEXIST); in shmem_add_to_page_cache() 809 if (expected && xas_find_conflict(&xas)) { in shmem_add_to_page_cache() 810 xas_set_err(&xas, -EEXIST); in shmem_add_to_page_cache() 813 xas_store(&xas, folio); in shmem_add_to_page_cache() [all …]
|
D | page-writeback.c | 2414 XA_STATE(xas, &mapping->i_pages, start); in tag_pages_for_writeback() 2418 xas_lock_irq(&xas); in tag_pages_for_writeback() 2419 xas_for_each_marked(&xas, page, end, PAGECACHE_TAG_DIRTY) { in tag_pages_for_writeback() 2420 xas_set_mark(&xas, PAGECACHE_TAG_TOWRITE); in tag_pages_for_writeback() 2424 xas_pause(&xas); in tag_pages_for_writeback() 2425 xas_unlock_irq(&xas); in tag_pages_for_writeback() 2427 xas_lock_irq(&xas); in tag_pages_for_writeback() 2429 xas_unlock_irq(&xas); in tag_pages_for_writeback() 3123 XA_STATE(xas, &mapping->i_pages, folio_index(folio)); in __folio_start_writeback() 3129 xas_lock_irqsave(&xas, flags); in __folio_start_writeback() [all …]
|
D | truncate.c | 34 XA_STATE(xas, &mapping->i_pages, index); in __clear_shadow_entry() 36 xas_set_update(&xas, workingset_update_node); in __clear_shadow_entry() 37 if (xas_load(&xas) != entry) in __clear_shadow_entry() 39 xas_store(&xas, NULL); in __clear_shadow_entry()
|
/linux-6.12.1/drivers/infiniband/sw/rxe/ |
D | rxe_mr.c | 96 XA_STATE(xas, &mr->page_list, 0); in rxe_mr_fill_pages_from_sgt() 106 xas_lock(&xas); in rxe_mr_fill_pages_from_sgt() 112 xas_set_err(&xas, -EINVAL); in rxe_mr_fill_pages_from_sgt() 116 xas_store(&xas, page); in rxe_mr_fill_pages_from_sgt() 117 if (xas_error(&xas)) in rxe_mr_fill_pages_from_sgt() 119 xas_next(&xas); in rxe_mr_fill_pages_from_sgt() 123 xas_unlock(&xas); in rxe_mr_fill_pages_from_sgt() 124 } while (xas_nomem(&xas, GFP_KERNEL)); in rxe_mr_fill_pages_from_sgt() 126 return xas_error(&xas); in rxe_mr_fill_pages_from_sgt() 161 XA_STATE(xas, &mr->page_list, 0); in rxe_mr_alloc() [all …]
|
/linux-6.12.1/drivers/infiniband/core/ |
D | ib_core_uverbs.c | 268 XA_STATE(xas, &ucontext->mmap_xa, min_pgoff); in rdma_user_mmap_entry_insert_range() 294 xas_find_marked(&xas, max_pgoff, XA_FREE_MARK); in rdma_user_mmap_entry_insert_range() 295 if (xas.xa_node == XAS_RESTART) in rdma_user_mmap_entry_insert_range() 298 xa_first = xas.xa_index; in rdma_user_mmap_entry_insert_range() 308 xas_next_entry(&xas, xa_last - 1); in rdma_user_mmap_entry_insert_range() 309 if (xas.xa_node == XAS_BOUNDS || xas.xa_index >= xa_last) in rdma_user_mmap_entry_insert_range()
|
/linux-6.12.1/drivers/iommu/iommufd/ |
D | pages.c | 523 XA_STATE(xas, xa, start_index); in batch_from_xarray() 528 entry = xas_next(&xas); in batch_from_xarray() 529 if (xas_retry(&xas, entry)) in batch_from_xarray() 544 XA_STATE(xas, xa, start_index); in batch_from_xarray_clear() 547 xas_lock(&xas); in batch_from_xarray_clear() 549 entry = xas_next(&xas); in batch_from_xarray_clear() 550 if (xas_retry(&xas, entry)) in batch_from_xarray_clear() 555 xas_store(&xas, NULL); in batch_from_xarray_clear() 560 xas_unlock(&xas); in batch_from_xarray_clear() 566 XA_STATE(xas, xa, start_index); in clear_xarray() [all …]
|
/linux-6.12.1/arch/x86/kernel/cpu/sgx/ |
D | encl.c | 530 XA_STATE(xas, &encl->page_array, PFN_DOWN(start)); in sgx_encl_may_map() 545 xas_lock(&xas); in sgx_encl_may_map() 546 xas_for_each(&xas, page, PFN_DOWN(end - 1)) { in sgx_encl_may_map() 554 xas_pause(&xas); in sgx_encl_may_map() 555 xas_unlock(&xas); in sgx_encl_may_map() 561 xas_lock(&xas); in sgx_encl_may_map() 564 xas_unlock(&xas); in sgx_encl_may_map() 708 XA_STATE(xas, &encl->page_array, PFN_DOWN(encl->base)); in sgx_encl_release() 710 xas_lock(&xas); in sgx_encl_release() 711 xas_for_each(&xas, entry, max_page_index) { in sgx_encl_release() [all …]
|
/linux-6.12.1/fs/netfs/ |
D | iterator.c | 160 XA_STATE(xas, iter->xarray, index); in netfs_limit_xarray() 169 xas_for_each(&xas, folio, ULONG_MAX) { in netfs_limit_xarray() 171 if (xas_retry(&xas, folio)) in netfs_limit_xarray()
|