Lines Matching refs:phys_pg_pack
92 struct hl_vm_phys_pg_pack *phys_pg_pack; in alloc_device_memory() local
128 phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL); in alloc_device_memory()
129 if (!phys_pg_pack) { in alloc_device_memory()
134 phys_pg_pack->vm_type = VM_TYPE_PHYS_PACK; in alloc_device_memory()
135 phys_pg_pack->asid = ctx->asid; in alloc_device_memory()
136 phys_pg_pack->npages = num_pgs; in alloc_device_memory()
137 phys_pg_pack->page_size = page_size; in alloc_device_memory()
138 phys_pg_pack->total_size = total_size; in alloc_device_memory()
139 phys_pg_pack->flags = args->flags; in alloc_device_memory()
140 phys_pg_pack->contiguous = contiguous; in alloc_device_memory()
142 phys_pg_pack->pages = kvmalloc_array(num_pgs, sizeof(u64), GFP_KERNEL); in alloc_device_memory()
143 if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) { in alloc_device_memory()
148 if (phys_pg_pack->contiguous) { in alloc_device_memory()
150 phys_pg_pack->pages[i] = paddr + i * page_size; in alloc_device_memory()
154 phys_pg_pack->pages[i] = in alloc_device_memory()
159 phys_pg_pack->pages[i] = gen_pool_alloc(vm->dram_pg_pool, in alloc_device_memory()
162 if (!phys_pg_pack->pages[i]) { in alloc_device_memory()
174 handle = idr_alloc(&vm->phys_pg_pack_handles, phys_pg_pack, 1, 0, in alloc_device_memory()
187 phys_pg_pack->handle = handle; in alloc_device_memory()
189 atomic64_add(phys_pg_pack->total_size, &ctx->dram_phys_mem); in alloc_device_memory()
190 atomic64_add(phys_pg_pack->total_size, &hdev->dram_used_mem); in alloc_device_memory()
198 if (!phys_pg_pack->contiguous) in alloc_device_memory()
200 gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[i], in alloc_device_memory()
203 kvfree(phys_pg_pack->pages); in alloc_device_memory()
205 kfree(phys_pg_pack); in alloc_device_memory()
313 struct hl_vm_phys_pg_pack *phys_pg_pack) in free_phys_pg_pack() argument
318 if (phys_pg_pack->created_from_userptr) in free_phys_pg_pack()
321 if (phys_pg_pack->contiguous) { in free_phys_pg_pack()
322 gen_pool_free(vm->dram_pg_pool, phys_pg_pack->pages[0], in free_phys_pg_pack()
323 phys_pg_pack->total_size); in free_phys_pg_pack()
325 for (i = 0; i < phys_pg_pack->npages ; i++) in free_phys_pg_pack()
329 for (i = 0 ; i < phys_pg_pack->npages ; i++) { in free_phys_pg_pack()
331 phys_pg_pack->pages[i], in free_phys_pg_pack()
332 phys_pg_pack->page_size); in free_phys_pg_pack()
339 kvfree(phys_pg_pack->pages); in free_phys_pg_pack()
340 kfree(phys_pg_pack); in free_phys_pg_pack()
357 struct hl_vm_phys_pg_pack *phys_pg_pack; in free_device_memory() local
361 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle); in free_device_memory()
362 if (!phys_pg_pack) { in free_device_memory()
368 if (atomic_read(&phys_pg_pack->mapping_cnt) > 0) { in free_device_memory()
380 atomic64_sub(phys_pg_pack->total_size, &ctx->dram_phys_mem); in free_device_memory()
381 atomic64_sub(phys_pg_pack->total_size, &hdev->dram_used_mem); in free_device_memory()
383 free_phys_pg_pack(hdev, phys_pg_pack); in free_device_memory()
846 struct hl_vm_phys_pg_pack *phys_pg_pack; in init_phys_pg_pack_from_userptr() local
853 phys_pg_pack = kzalloc(sizeof(*phys_pg_pack), GFP_KERNEL); in init_phys_pg_pack_from_userptr()
854 if (!phys_pg_pack) in init_phys_pg_pack_from_userptr()
857 phys_pg_pack->vm_type = userptr->vm_type; in init_phys_pg_pack_from_userptr()
858 phys_pg_pack->created_from_userptr = true; in init_phys_pg_pack_from_userptr()
859 phys_pg_pack->asid = ctx->asid; in init_phys_pg_pack_from_userptr()
860 atomic_set(&phys_pg_pack->mapping_cnt, 1); in init_phys_pg_pack_from_userptr()
888 phys_pg_pack->pages = kvmalloc_array(total_npages, sizeof(u64), in init_phys_pg_pack_from_userptr()
890 if (ZERO_OR_NULL_PTR(phys_pg_pack->pages)) { in init_phys_pg_pack_from_userptr()
895 phys_pg_pack->npages = total_npages; in init_phys_pg_pack_from_userptr()
896 phys_pg_pack->page_size = page_size; in init_phys_pg_pack_from_userptr()
897 phys_pg_pack->total_size = total_npages * page_size; in init_phys_pg_pack_from_userptr()
906 phys_pg_pack->offset = dma_addr & (page_size - 1); in init_phys_pg_pack_from_userptr()
911 phys_pg_pack->pages[j++] = dma_addr; in init_phys_pg_pack_from_userptr()
921 *pphys_pg_pack = phys_pg_pack; in init_phys_pg_pack_from_userptr()
926 kfree(phys_pg_pack); in init_phys_pg_pack_from_userptr()
943 struct hl_vm_phys_pg_pack *phys_pg_pack) in map_phys_pg_pack() argument
947 u32 page_size = phys_pg_pack->page_size; in map_phys_pg_pack()
951 for (i = 0 ; i < phys_pg_pack->npages ; i++) { in map_phys_pg_pack()
952 paddr = phys_pg_pack->pages[i]; in map_phys_pg_pack()
955 (i + 1) == phys_pg_pack->npages); in map_phys_pg_pack()
959 rc, phys_pg_pack->handle, phys_pg_pack->npages, in map_phys_pg_pack()
979 phys_pg_pack->handle, next_vaddr, in map_phys_pg_pack()
980 phys_pg_pack->pages[i], page_size); in map_phys_pg_pack()
1006 struct hl_vm_phys_pg_pack *phys_pg_pack) in unmap_phys_pg_pack() argument
1014 page_size = phys_pg_pack->page_size; in unmap_phys_pg_pack()
1017 for (i = 0 ; i < phys_pg_pack->npages ; i++, next_vaddr += page_size) { in unmap_phys_pg_pack()
1019 (i + 1) == phys_pg_pack->npages)) in unmap_phys_pg_pack()
1051 struct hl_vm_phys_pg_pack *phys_pg_pack; in map_device_va() local
1082 &phys_pg_pack, false); in map_device_va()
1092 handle = phys_pg_pack->handle; in map_device_va()
1095 if (phys_pg_pack->page_size == page_size) { in map_device_va()
1119 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, handle); in map_device_va()
1120 if (!phys_pg_pack) { in map_device_va()
1128 atomic_inc(&phys_pg_pack->mapping_cnt); in map_device_va()
1132 vm_type = (enum vm_type *) phys_pg_pack; in map_device_va()
1146 if (!is_userptr && !(phys_pg_pack->flags & HL_MEM_SHARED) && in map_device_va()
1147 phys_pg_pack->asid != ctx->asid) { in map_device_va()
1161 if (hint_addr && phys_pg_pack->offset) { in map_device_va()
1166 hint_addr, phys_pg_pack->offset); in map_device_va()
1172 hint_addr, phys_pg_pack->offset); in map_device_va()
1175 ret_vaddr = get_va_block(hdev, va_range, phys_pg_pack->total_size, in map_device_va()
1187 rc = map_phys_pg_pack(ctx, ret_vaddr, phys_pg_pack); in map_device_va()
1196 ctx->asid, ret_vaddr, phys_pg_pack->total_size); in map_device_va()
1207 phys_pg_pack->total_size); in map_device_va()
1212 ret_vaddr += phys_pg_pack->offset; in map_device_va()
1225 free_phys_pg_pack(hdev, phys_pg_pack); in map_device_va()
1231 ret_vaddr + phys_pg_pack->total_size - 1)) in map_device_va()
1240 atomic_dec(&phys_pg_pack->mapping_cnt); in map_device_va()
1242 free_phys_pg_pack(hdev, phys_pg_pack); in map_device_va()
1275 struct hl_vm_phys_pg_pack *phys_pg_pack = NULL; in unmap_device_va() local
1312 rc = init_phys_pg_pack_from_userptr(ctx, userptr, &phys_pg_pack, in unmap_device_va()
1321 if (phys_pg_pack->page_size == in unmap_device_va()
1329 phys_pg_pack = hnode->ptr; in unmap_device_va()
1338 if (atomic_read(&phys_pg_pack->mapping_cnt) == 0) { in unmap_device_va()
1344 if (!is_userptr && !is_power_of_2(phys_pg_pack->page_size)) in unmap_device_va()
1347 phys_pg_pack->page_size) * in unmap_device_va()
1348 phys_pg_pack->page_size; in unmap_device_va()
1350 vaddr &= ~(((u64) phys_pg_pack->page_size) - 1); in unmap_device_va()
1354 unmap_phys_pg_pack(ctx, vaddr, phys_pg_pack); in unmap_device_va()
1363 phys_pg_pack->total_size); in unmap_device_va()
1377 vaddr + phys_pg_pack->total_size - 1); in unmap_device_va()
1387 atomic_dec(&phys_pg_pack->mapping_cnt); in unmap_device_va()
1391 free_phys_pg_pack(hdev, phys_pg_pack); in unmap_device_va()
1399 free_phys_pg_pack(hdev, phys_pg_pack); in unmap_device_va()
1729 struct hl_vm_phys_pg_pack *phys_pg_pack; in hl_map_dmabuf() local
1744 phys_pg_pack = hl_dmabuf->phys_pg_pack; in hl_map_dmabuf()
1746 if (phys_pg_pack) { in hl_map_dmabuf()
1747 pages = phys_pg_pack->pages; in hl_map_dmabuf()
1748 npages = phys_pg_pack->npages; in hl_map_dmabuf()
1749 page_size = phys_pg_pack->page_size; in hl_map_dmabuf()
1961 struct hl_vm_phys_pg_pack *phys_pg_pack) in validate_export_params() argument
1971 if ((offset + size) > phys_pg_pack->total_size) { in validate_export_params()
1973 offset, size, phys_pg_pack->total_size); in validate_export_params()
1977 for (i = 0 ; i < phys_pg_pack->npages ; i++) { in validate_export_params()
1979 (phys_pg_pack->pages[i] - prop->dram_base_address); in validate_export_params()
1981 if ((bar_address + phys_pg_pack->page_size) > in validate_export_params()
1983 (bar_address + phys_pg_pack->page_size) < bar_address) { in validate_export_params()
1986 phys_pg_pack->pages[i], phys_pg_pack->page_size); in validate_export_params()
1997 struct hl_vm_phys_pg_pack *phys_pg_pack; in get_phys_pg_pack_from_hash_node() local
2001 phys_pg_pack = idr_find(&vm->phys_pg_pack_handles, (u32) hnode->handle); in get_phys_pg_pack_from_hash_node()
2002 if (!phys_pg_pack) { in get_phys_pg_pack_from_hash_node()
2010 if (phys_pg_pack->vm_type != VM_TYPE_PHYS_PACK) { in get_phys_pg_pack_from_hash_node()
2015 return phys_pg_pack; in get_phys_pg_pack_from_hash_node()
2037 struct hl_vm_phys_pg_pack *phys_pg_pack = NULL; in export_dmabuf_from_addr() local
2063 phys_pg_pack = get_phys_pg_pack_from_hash_node(hdev, hnode); in export_dmabuf_from_addr()
2064 if (IS_ERR(phys_pg_pack)) { in export_dmabuf_from_addr()
2065 rc = PTR_ERR(phys_pg_pack); in export_dmabuf_from_addr()
2068 rc = validate_export_params(hdev, addr, size, offset, phys_pg_pack); in export_dmabuf_from_addr()
2072 hl_dmabuf->phys_pg_pack = phys_pg_pack; in export_dmabuf_from_addr()