Lines Matching refs:vn
1058 struct vmap_node *vn; in find_vmap_area_exceed_addr_lock() local
1063 vn = &vmap_nodes[i]; in find_vmap_area_exceed_addr_lock()
1065 spin_lock(&vn->busy.lock); in find_vmap_area_exceed_addr_lock()
1066 *va = __find_vmap_area_exceed_addr(addr, &vn->busy.root); in find_vmap_area_exceed_addr_lock()
1071 spin_unlock(&vn->busy.lock); in find_vmap_area_exceed_addr_lock()
1080 vn = addr_to_node(va_start_lowest); in find_vmap_area_exceed_addr_lock()
1082 spin_lock(&vn->busy.lock); in find_vmap_area_exceed_addr_lock()
1083 *va = __find_vmap_area(va_start_lowest, &vn->busy.root); in find_vmap_area_exceed_addr_lock()
1086 return vn; in find_vmap_area_exceed_addr_lock()
1088 spin_unlock(&vn->busy.lock); in find_vmap_area_exceed_addr_lock()
1799 struct vmap_node *vn = addr_to_node(va->va_start); in free_vmap_area() local
1804 spin_lock(&vn->busy.lock); in free_vmap_area()
1805 unlink_va(va, &vn->busy.root); in free_vmap_area()
1806 spin_unlock(&vn->busy.lock); in free_vmap_area()
1841 size_to_va_pool(struct vmap_node *vn, unsigned long size) in size_to_va_pool() argument
1846 return &vn->pool[idx]; in size_to_va_pool()
1869 node_pool_del_va(struct vmap_node *vn, unsigned long size, in node_pool_del_va() argument
1877 vp = size_to_va_pool(vn, size); in node_pool_del_va()
1881 spin_lock(&vn->pool_lock); in node_pool_del_va()
1905 spin_unlock(&vn->pool_lock); in node_pool_del_va()
1958 struct vmap_node *vn; in alloc_vmap_area() local
2025 vn = addr_to_node(va->va_start); in alloc_vmap_area()
2027 spin_lock(&vn->busy.lock); in alloc_vmap_area()
2028 insert_vmap_area(va, &vn->busy.root, &vn->busy.head); in alloc_vmap_area()
2029 spin_unlock(&vn->busy.lock); in alloc_vmap_area()
2132 decay_va_pool_node(struct vmap_node *vn, bool full_decay) in decay_va_pool_node() argument
2143 if (list_empty(&vn->pool[i].head)) in decay_va_pool_node()
2147 spin_lock(&vn->pool_lock); in decay_va_pool_node()
2148 list_replace_init(&vn->pool[i].head, &tmp_list); in decay_va_pool_node()
2149 spin_unlock(&vn->pool_lock); in decay_va_pool_node()
2152 WRITE_ONCE(vn->pool[i].len, 0); in decay_va_pool_node()
2155 n_decay = vn->pool[i].len >> 2; in decay_va_pool_node()
2162 WRITE_ONCE(vn->pool[i].len, vn->pool[i].len - 1); in decay_va_pool_node()
2176 spin_lock(&vn->pool_lock); in decay_va_pool_node()
2177 list_replace_init(&tmp_list, &vn->pool[i].head); in decay_va_pool_node()
2178 spin_unlock(&vn->pool_lock); in decay_va_pool_node()
2187 struct vmap_node *vn = container_of(work, in purge_vmap_node() local
2193 vn->nr_purged = 0; in purge_vmap_node()
2195 list_for_each_entry_safe(va, n_va, &vn->purge_list, list) { in purge_vmap_node()
2208 vn->nr_purged++; in purge_vmap_node()
2210 if (is_vn_id_valid(vn_id) && !vn->skip_populate) in purge_vmap_node()
2211 if (node_pool_add_va(vn, va)) in purge_vmap_node()
2232 struct vmap_node *vn; in __purge_vmap_area_lazy() local
2243 vn = &vmap_nodes[i]; in __purge_vmap_area_lazy()
2245 INIT_LIST_HEAD(&vn->purge_list); in __purge_vmap_area_lazy()
2246 vn->skip_populate = full_pool_decay; in __purge_vmap_area_lazy()
2247 decay_va_pool_node(vn, full_pool_decay); in __purge_vmap_area_lazy()
2249 if (RB_EMPTY_ROOT(&vn->lazy.root)) in __purge_vmap_area_lazy()
2252 spin_lock(&vn->lazy.lock); in __purge_vmap_area_lazy()
2253 WRITE_ONCE(vn->lazy.root.rb_node, NULL); in __purge_vmap_area_lazy()
2254 list_replace_init(&vn->lazy.head, &vn->purge_list); in __purge_vmap_area_lazy()
2255 spin_unlock(&vn->lazy.lock); in __purge_vmap_area_lazy()
2257 start = min(start, list_first_entry(&vn->purge_list, in __purge_vmap_area_lazy()
2260 end = max(end, list_last_entry(&vn->purge_list, in __purge_vmap_area_lazy()
2275 vn = &vmap_nodes[i]; in __purge_vmap_area_lazy()
2278 INIT_WORK(&vn->purge_work, purge_vmap_node); in __purge_vmap_area_lazy()
2281 schedule_work_on(i, &vn->purge_work); in __purge_vmap_area_lazy()
2283 schedule_work(&vn->purge_work); in __purge_vmap_area_lazy()
2287 vn->purge_work.func = NULL; in __purge_vmap_area_lazy()
2288 purge_vmap_node(&vn->purge_work); in __purge_vmap_area_lazy()
2289 nr_purged_areas += vn->nr_purged; in __purge_vmap_area_lazy()
2294 vn = &vmap_nodes[i]; in __purge_vmap_area_lazy()
2296 if (vn->purge_work.func) { in __purge_vmap_area_lazy()
2297 flush_work(&vn->purge_work); in __purge_vmap_area_lazy()
2298 nr_purged_areas += vn->nr_purged; in __purge_vmap_area_lazy()
2336 struct vmap_node *vn; in free_vmap_area_noflush() local
2349 vn = is_vn_id_valid(vn_id) ? in free_vmap_area_noflush()
2352 spin_lock(&vn->lazy.lock); in free_vmap_area_noflush()
2353 insert_vmap_area(va, &vn->lazy.root, &vn->lazy.head); in free_vmap_area_noflush()
2354 spin_unlock(&vn->lazy.lock); in free_vmap_area_noflush()
2378 struct vmap_node *vn; in find_vmap_area() local
2400 vn = &vmap_nodes[i]; in find_vmap_area()
2402 spin_lock(&vn->busy.lock); in find_vmap_area()
2403 va = __find_vmap_area(addr, &vn->busy.root); in find_vmap_area()
2404 spin_unlock(&vn->busy.lock); in find_vmap_area()
2415 struct vmap_node *vn; in find_unlink_vmap_area() local
2424 vn = &vmap_nodes[i]; in find_unlink_vmap_area()
2426 spin_lock(&vn->busy.lock); in find_unlink_vmap_area()
2427 va = __find_vmap_area(addr, &vn->busy.root); in find_unlink_vmap_area()
2429 unlink_va(va, &vn->busy.root); in find_unlink_vmap_area()
2430 spin_unlock(&vn->busy.lock); in find_unlink_vmap_area()
2654 struct vmap_node *vn; in free_vmap_block() local
2662 vn = addr_to_node(vb->va->va_start); in free_vmap_block()
2663 spin_lock(&vn->busy.lock); in free_vmap_block()
2664 unlink_va(vb->va, &vn->busy.root); in free_vmap_block()
2665 spin_unlock(&vn->busy.lock); in free_vmap_block()
4322 struct vmap_node *vn; in vread_iter() local
4337 vn = find_vmap_area_exceed_addr_lock((unsigned long) addr, &va); in vread_iter()
4338 if (!vn) in vread_iter()
4404 spin_unlock(&vn->busy.lock); in vread_iter()
4405 } while ((vn = find_vmap_area_exceed_addr_lock(next, &va))); in vread_iter()
4408 if (vn) in vread_iter()
4409 spin_unlock(&vn->busy.lock); in vread_iter()
4415 if (vn) in vread_iter()
4416 spin_unlock(&vn->busy.lock); in vread_iter()
4751 struct vmap_node *vn = addr_to_node(vas[area]->va_start); in pcpu_get_vm_areas() local
4753 spin_lock(&vn->busy.lock); in pcpu_get_vm_areas()
4754 insert_vmap_area(vas[area], &vn->busy.root, &vn->busy.head); in pcpu_get_vm_areas()
4757 spin_unlock(&vn->busy.lock); in pcpu_get_vm_areas()
4870 struct vmap_node *vn; in vmalloc_dump_obj() local
4875 vn = addr_to_node(addr); in vmalloc_dump_obj()
4877 if (!spin_trylock(&vn->busy.lock)) in vmalloc_dump_obj()
4880 va = __find_vmap_area(addr, &vn->busy.root); in vmalloc_dump_obj()
4882 spin_unlock(&vn->busy.lock); in vmalloc_dump_obj()
4890 spin_unlock(&vn->busy.lock); in vmalloc_dump_obj()
4926 struct vmap_node *vn; in show_purge_info() local
4931 vn = &vmap_nodes[i]; in show_purge_info()
4933 spin_lock(&vn->lazy.lock); in show_purge_info()
4934 list_for_each_entry(va, &vn->lazy.head, list) { in show_purge_info()
4939 spin_unlock(&vn->lazy.lock); in show_purge_info()
4945 struct vmap_node *vn; in vmalloc_info_show() local
4951 vn = &vmap_nodes[i]; in vmalloc_info_show()
4953 spin_lock(&vn->busy.lock); in vmalloc_info_show()
4954 list_for_each_entry(va, &vn->busy.head, list) { in vmalloc_info_show()
5002 spin_unlock(&vn->busy.lock); in vmalloc_info_show()
5072 struct vmap_node *vn; in vmap_init_nodes() local
5093 vn = kmalloc_array(n, sizeof(*vn), GFP_NOWAIT | __GFP_NOWARN); in vmap_init_nodes()
5094 if (vn) { in vmap_init_nodes()
5098 vmap_nodes = vn; in vmap_init_nodes()
5106 vn = &vmap_nodes[n]; in vmap_init_nodes()
5107 vn->busy.root = RB_ROOT; in vmap_init_nodes()
5108 INIT_LIST_HEAD(&vn->busy.head); in vmap_init_nodes()
5109 spin_lock_init(&vn->busy.lock); in vmap_init_nodes()
5111 vn->lazy.root = RB_ROOT; in vmap_init_nodes()
5112 INIT_LIST_HEAD(&vn->lazy.head); in vmap_init_nodes()
5113 spin_lock_init(&vn->lazy.lock); in vmap_init_nodes()
5116 INIT_LIST_HEAD(&vn->pool[i].head); in vmap_init_nodes()
5117 WRITE_ONCE(vn->pool[i].len, 0); in vmap_init_nodes()
5120 spin_lock_init(&vn->pool_lock); in vmap_init_nodes()
5128 struct vmap_node *vn; in vmap_node_shrink_count() local
5132 vn = &vmap_nodes[i]; in vmap_node_shrink_count()
5135 count += READ_ONCE(vn->pool[j].len); in vmap_node_shrink_count()
5156 struct vmap_node *vn; in vmalloc_init() local
5193 vn = addr_to_node(va->va_start); in vmalloc_init()
5194 insert_vmap_area(va, &vn->busy.root, &vn->busy.head); in vmalloc_init()