Lines Matching +full:ts +full:- +full:inv
1 // SPDX-License-Identifier: GPL-2.0 OR MIT
3 * Copyright 2020-2021 Advanced Micro Devices, Inc.
84 * svm_range_unlink - unlink svm_range from lists and interval tree
90 * Context: The caller must hold svms->lock
94 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, in svm_range_unlink()
95 prange, prange->start, prange->last); in svm_range_unlink()
97 if (prange->svm_bo) { in svm_range_unlink()
98 spin_lock(&prange->svm_bo->list_lock); in svm_range_unlink()
99 list_del(&prange->svm_bo_list); in svm_range_unlink()
100 spin_unlock(&prange->svm_bo->list_lock); in svm_range_unlink()
103 list_del(&prange->list); in svm_range_unlink()
104 if (prange->it_node.start != 0 && prange->it_node.last != 0) in svm_range_unlink()
105 interval_tree_remove(&prange->it_node, &prange->svms->objects); in svm_range_unlink()
111 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, in svm_range_add_notifier_locked()
112 prange, prange->start, prange->last); in svm_range_add_notifier_locked()
114 mmu_interval_notifier_insert_locked(&prange->notifier, mm, in svm_range_add_notifier_locked()
115 prange->start << PAGE_SHIFT, in svm_range_add_notifier_locked()
116 prange->npages << PAGE_SHIFT, in svm_range_add_notifier_locked()
121 * svm_range_add_to_svms - add svm range to svms
126 * Context: The caller must hold svms->lock
130 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, in svm_range_add_to_svms()
131 prange, prange->start, prange->last); in svm_range_add_to_svms()
133 list_move_tail(&prange->list, &prange->svms->list); in svm_range_add_to_svms()
134 prange->it_node.start = prange->start; in svm_range_add_to_svms()
135 prange->it_node.last = prange->last; in svm_range_add_to_svms()
136 interval_tree_insert(&prange->it_node, &prange->svms->objects); in svm_range_add_to_svms()
142 prange->svms, prange, in svm_range_remove_notifier()
143 prange->notifier.interval_tree.start >> PAGE_SHIFT, in svm_range_remove_notifier()
144 prange->notifier.interval_tree.last >> PAGE_SHIFT); in svm_range_remove_notifier()
146 if (prange->notifier.interval_tree.start != 0 && in svm_range_remove_notifier()
147 prange->notifier.interval_tree.last != 0) in svm_range_remove_notifier()
148 mmu_interval_notifier_remove(&prange->notifier); in svm_range_remove_notifier()
164 dma_addr_t *addr = prange->dma_addr[gpuidx]; in svm_range_dma_map_dev()
165 struct device *dev = adev->dev; in svm_range_dma_map_dev()
170 addr = kvcalloc(prange->npages, sizeof(*addr), GFP_KERNEL); in svm_range_dma_map_dev()
172 return -ENOMEM; in svm_range_dma_map_dev()
173 prange->dma_addr[gpuidx] = addr; in svm_range_dma_map_dev()
183 struct amdgpu_device *bo_adev = prange->svm_bo->node->adev; in svm_range_dma_map_dev()
186 bo_adev->vm_manager.vram_base_offset - in svm_range_dma_map_dev()
187 bo_adev->kfd.pgmap.range.start; in svm_range_dma_map_dev()
214 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_dma_map()
223 return -EINVAL; in svm_range_dma_map()
226 r = svm_range_dma_map_dev(pdd->dev->adev, prange, offset, npages, in svm_range_dma_map()
261 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_dma_unmap()
264 dma_addr = prange->dma_addr[gpuidx]; in svm_range_dma_unmap()
273 dev = &pdd->dev->adev->pdev->dev; in svm_range_dma_unmap()
275 svm_range_dma_unmap_dev(dev, dma_addr, 0, prange->npages); in svm_range_dma_unmap()
281 uint64_t size = (prange->last - prange->start + 1) << PAGE_SHIFT; in svm_range_free()
282 struct kfd_process *p = container_of(prange->svms, struct kfd_process, svms); in svm_range_free()
285 pr_debug("svms 0x%p prange 0x%p [0x%lx 0x%lx]\n", prange->svms, prange, in svm_range_free()
286 prange->start, prange->last); in svm_range_free()
292 if (do_unmap && !p->xnack_enabled) { in svm_range_free()
300 if (prange->dma_addr[gpuidx]) { in svm_range_free()
301 kvfree(prange->dma_addr[gpuidx]); in svm_range_free()
302 prange->dma_addr[gpuidx] = NULL; in svm_range_free()
306 mutex_destroy(&prange->lock); in svm_range_free()
307 mutex_destroy(&prange->migrate_mutex); in svm_range_free()
318 *granularity = svms->default_granularity; in svm_range_set_default_attributes()
327 uint64_t size = last - start + 1; in svm_range_new()
336 if (!p->xnack_enabled && update_mem_usage && in svm_range_new()
343 prange->npages = size; in svm_range_new()
344 prange->svms = svms; in svm_range_new()
345 prange->start = start; in svm_range_new()
346 prange->last = last; in svm_range_new()
347 INIT_LIST_HEAD(&prange->list); in svm_range_new()
348 INIT_LIST_HEAD(&prange->update_list); in svm_range_new()
349 INIT_LIST_HEAD(&prange->svm_bo_list); in svm_range_new()
350 INIT_LIST_HEAD(&prange->deferred_list); in svm_range_new()
351 INIT_LIST_HEAD(&prange->child_list); in svm_range_new()
352 atomic_set(&prange->invalid, 0); in svm_range_new()
353 prange->validate_timestamp = 0; in svm_range_new()
354 prange->vram_pages = 0; in svm_range_new()
355 mutex_init(&prange->migrate_mutex); in svm_range_new()
356 mutex_init(&prange->lock); in svm_range_new()
358 if (p->xnack_enabled) in svm_range_new()
359 bitmap_copy(prange->bitmap_access, svms->bitmap_supported, in svm_range_new()
362 svm_range_set_default_attributes(svms, &prange->preferred_loc, in svm_range_new()
363 &prange->prefetch_loc, in svm_range_new()
364 &prange->granularity, &prange->flags); in svm_range_new()
373 if (!svm_bo || !kref_get_unless_zero(&svm_bo->kref)) in svm_bo_ref_unless_zero()
386 spin_lock(&svm_bo->list_lock); in svm_range_bo_release()
387 while (!list_empty(&svm_bo->range_list)) { in svm_range_bo_release()
389 list_first_entry(&svm_bo->range_list, in svm_range_bo_release()
394 list_del_init(&prange->svm_bo_list); in svm_range_bo_release()
395 spin_unlock(&svm_bo->list_lock); in svm_range_bo_release()
397 pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, in svm_range_bo_release()
398 prange->start, prange->last); in svm_range_bo_release()
399 mutex_lock(&prange->lock); in svm_range_bo_release()
400 prange->svm_bo = NULL; in svm_range_bo_release()
402 WARN_ONCE(prange->actual_loc, "prange should not hold vram page"); in svm_range_bo_release()
403 mutex_unlock(&prange->lock); in svm_range_bo_release()
405 spin_lock(&svm_bo->list_lock); in svm_range_bo_release()
407 spin_unlock(&svm_bo->list_lock); in svm_range_bo_release()
409 if (mmget_not_zero(svm_bo->eviction_fence->mm)) { in svm_range_bo_release()
414 mm = svm_bo->eviction_fence->mm; in svm_range_bo_release()
421 pdd = kfd_get_process_device_data(svm_bo->node, p); in svm_range_bo_release()
423 atomic64_sub(amdgpu_bo_size(svm_bo->bo), &pdd->vram_usage); in svm_range_bo_release()
429 if (!dma_fence_is_signaled(&svm_bo->eviction_fence->base)) in svm_range_bo_release()
431 dma_fence_signal(&svm_bo->eviction_fence->base); in svm_range_bo_release()
432 dma_fence_put(&svm_bo->eviction_fence->base); in svm_range_bo_release()
433 amdgpu_bo_unref(&svm_bo->bo); in svm_range_bo_release()
442 svm_range_bo_release(&svm_bo->kref); in svm_range_bo_wq_release()
451 INIT_WORK(&svm_bo->release_work, svm_range_bo_wq_release); in svm_range_bo_release_async()
452 schedule_work(&svm_bo->release_work); in svm_range_bo_release_async()
457 kref_put(&svm_bo->kref, svm_range_bo_release_async); in svm_range_bo_unref_async()
463 kref_put(&svm_bo->kref, svm_range_bo_release); in svm_range_bo_unref()
469 mutex_lock(&prange->lock); in svm_range_validate_svm_bo()
470 if (!prange->svm_bo) { in svm_range_validate_svm_bo()
471 mutex_unlock(&prange->lock); in svm_range_validate_svm_bo()
474 if (prange->ttm_res) { in svm_range_validate_svm_bo()
476 mutex_unlock(&prange->lock); in svm_range_validate_svm_bo()
479 if (svm_bo_ref_unless_zero(prange->svm_bo)) { in svm_range_validate_svm_bo()
481 * Migrate from GPU to GPU, remove range from source svm_bo->node in svm_range_validate_svm_bo()
485 if (prange->svm_bo->node != node) { in svm_range_validate_svm_bo()
486 mutex_unlock(&prange->lock); in svm_range_validate_svm_bo()
488 spin_lock(&prange->svm_bo->list_lock); in svm_range_validate_svm_bo()
489 list_del_init(&prange->svm_bo_list); in svm_range_validate_svm_bo()
490 spin_unlock(&prange->svm_bo->list_lock); in svm_range_validate_svm_bo()
492 svm_range_bo_unref(prange->svm_bo); in svm_range_validate_svm_bo()
495 if (READ_ONCE(prange->svm_bo->evicting)) { in svm_range_validate_svm_bo()
501 mutex_unlock(&prange->lock); in svm_range_validate_svm_bo()
502 svm_bo = prange->svm_bo; in svm_range_validate_svm_bo()
503 f = dma_fence_get(&svm_bo->eviction_fence->base); in svm_range_validate_svm_bo()
504 svm_range_bo_unref(prange->svm_bo); in svm_range_validate_svm_bo()
505 /* wait for the fence to avoid long spin-loop in svm_range_validate_svm_bo()
514 mutex_unlock(&prange->lock); in svm_range_validate_svm_bo()
516 prange->svms, prange->start, prange->last); in svm_range_validate_svm_bo()
518 prange->ttm_res = prange->svm_bo->bo->tbo.resource; in svm_range_validate_svm_bo()
523 mutex_unlock(&prange->lock); in svm_range_validate_svm_bo()
526 /* We need a new svm_bo. Spin-loop to wait for concurrent in svm_range_validate_svm_bo()
528 * its range list and set prange->svm_bo to null. After this, in svm_range_validate_svm_bo()
531 while (!list_empty_careful(&prange->svm_bo_list) || prange->svm_bo) in svm_range_validate_svm_bo()
545 kref_init(&svm_bo->kref); in svm_range_bo_new()
546 INIT_LIST_HEAD(&svm_bo->range_list); in svm_range_bo_new()
547 spin_lock_init(&svm_bo->list_lock); in svm_range_bo_new()
565 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_vram_node_new()
566 pr_debug("pasid: %x svms 0x%p [0x%lx 0x%lx]\n", p->pasid, prange->svms, in svm_range_vram_node_new()
567 prange->start, prange->last); in svm_range_vram_node_new()
575 return -ENOMEM; in svm_range_vram_node_new()
577 mm = get_task_mm(p->lead_thread); in svm_range_vram_node_new()
581 return -ESRCH; in svm_range_vram_node_new()
583 svm_bo->node = node; in svm_range_vram_node_new()
584 svm_bo->eviction_fence = in svm_range_vram_node_new()
589 INIT_WORK(&svm_bo->eviction_work, svm_range_evict_svm_bo_worker); in svm_range_vram_node_new()
590 svm_bo->evicting = 0; in svm_range_vram_node_new()
592 bp.size = prange->npages * PAGE_SIZE; in svm_range_vram_node_new()
600 if (node->xcp) in svm_range_vram_node_new()
601 bp.xcp_id_plus1 = node->xcp->id + 1; in svm_range_vram_node_new()
603 r = amdgpu_bo_create_user(node->adev, &bp, &ubo); in svm_range_vram_node_new()
608 bo = &ubo->bo; in svm_range_vram_node_new()
611 bo->tbo.resource->start << PAGE_SHIFT, bp.size, in svm_range_vram_node_new()
612 bp.xcp_id_plus1 - 1); in svm_range_vram_node_new()
629 r = dma_resv_reserve_fences(bo->tbo.base.resv, 1); in svm_range_vram_node_new()
635 amdgpu_bo_fence(bo, &svm_bo->eviction_fence->base, true); in svm_range_vram_node_new()
639 svm_bo->bo = bo; in svm_range_vram_node_new()
640 prange->svm_bo = svm_bo; in svm_range_vram_node_new()
641 prange->ttm_res = bo->tbo.resource; in svm_range_vram_node_new()
642 prange->offset = 0; in svm_range_vram_node_new()
644 spin_lock(&svm_bo->list_lock); in svm_range_vram_node_new()
645 list_add(&prange->svm_bo_list, &svm_bo->range_list); in svm_range_vram_node_new()
646 spin_unlock(&svm_bo->list_lock); in svm_range_vram_node_new()
650 atomic64_add(amdgpu_bo_size(bo), &pdd->vram_usage); in svm_range_vram_node_new()
657 dma_fence_put(&svm_bo->eviction_fence->base); in svm_range_vram_node_new()
659 prange->ttm_res = NULL; in svm_range_vram_node_new()
666 /* serialize prange->svm_bo unref */ in svm_range_vram_node_free()
667 mutex_lock(&prange->lock); in svm_range_vram_node_free()
668 /* prange->svm_bo has not been unref */ in svm_range_vram_node_free()
669 if (prange->ttm_res) { in svm_range_vram_node_free()
670 prange->ttm_res = NULL; in svm_range_vram_node_free()
671 mutex_unlock(&prange->lock); in svm_range_vram_node_free()
672 svm_range_bo_unref(prange->svm_bo); in svm_range_vram_node_free()
674 mutex_unlock(&prange->lock); in svm_range_vram_node_free()
683 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_get_node_by_id()
690 return pdd->dev; in svm_range_get_node_by_id()
698 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_get_pdd_by_node()
709 return ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); in svm_range_bo_validate()
745 return -EINVAL; in svm_range_check_attr()
750 return -EINVAL; in svm_range_check_attr()
752 !test_bit(gpuidx, p->svms.bitmap_supported)) { in svm_range_check_attr()
754 return -EINVAL; in svm_range_check_attr()
772 prange->preferred_loc = attrs[i].value; in svm_range_apply_attrs()
775 prange->prefetch_loc = attrs[i].value; in svm_range_apply_attrs()
780 if (!p->xnack_enabled) in svm_range_apply_attrs()
786 bitmap_clear(prange->bitmap_access, gpuidx, 1); in svm_range_apply_attrs()
787 bitmap_clear(prange->bitmap_aip, gpuidx, 1); in svm_range_apply_attrs()
789 bitmap_set(prange->bitmap_access, gpuidx, 1); in svm_range_apply_attrs()
790 bitmap_clear(prange->bitmap_aip, gpuidx, 1); in svm_range_apply_attrs()
792 bitmap_clear(prange->bitmap_access, gpuidx, 1); in svm_range_apply_attrs()
793 bitmap_set(prange->bitmap_aip, gpuidx, 1); in svm_range_apply_attrs()
798 prange->flags |= attrs[i].value; in svm_range_apply_attrs()
802 prange->flags &= ~attrs[i].value; in svm_range_apply_attrs()
805 prange->granularity = min_t(uint32_t, attrs[i].value, 0x3F); in svm_range_apply_attrs()
823 if (prange->preferred_loc != attrs[i].value) in svm_range_is_same_attrs()
837 if (test_bit(gpuidx, prange->bitmap_access) || in svm_range_is_same_attrs()
838 test_bit(gpuidx, prange->bitmap_aip)) in svm_range_is_same_attrs()
841 if (!test_bit(gpuidx, prange->bitmap_access)) in svm_range_is_same_attrs()
844 if (!test_bit(gpuidx, prange->bitmap_aip)) in svm_range_is_same_attrs()
849 if ((prange->flags & attrs[i].value) != attrs[i].value) in svm_range_is_same_attrs()
853 if ((prange->flags & attrs[i].value) != 0) in svm_range_is_same_attrs()
857 if (prange->granularity != attrs[i].value) in svm_range_is_same_attrs()
869 * svm_range_debug_dump - print all range information from svms
875 * Context: The caller must hold svms->lock
885 list_for_each_entry(prange, &svms->list, list) { in svm_range_debug_dump()
887 prange, prange->start, prange->npages, in svm_range_debug_dump()
888 prange->start + prange->npages - 1, in svm_range_debug_dump()
889 prange->actual_loc); in svm_range_debug_dump()
894 node = interval_tree_iter_first(&svms->objects, 0, ~0ULL); in svm_range_debug_dump()
898 prange, prange->start, prange->npages, in svm_range_debug_dump()
899 prange->start + prange->npages - 1, in svm_range_debug_dump()
900 prange->actual_loc); in svm_range_debug_dump()
940 if (!src->dma_addr[i]) in svm_range_copy_dma_addrs()
942 dst->dma_addr[i] = svm_range_copy_array(src->dma_addr[i], in svm_range_copy_dma_addrs()
943 sizeof(*src->dma_addr[i]), src->npages, 0, NULL); in svm_range_copy_dma_addrs()
944 if (!dst->dma_addr[i]) in svm_range_copy_dma_addrs()
945 return -ENOMEM; in svm_range_copy_dma_addrs()
965 d = (new_start - old_start) * size; in svm_range_split_array()
969 return -ENOMEM; in svm_range_split_array()
974 return -ENOMEM; in svm_range_split_array()
987 uint64_t npages = last - start + 1; in svm_range_split_pages()
991 r = svm_range_split_array(&new->dma_addr[i], &old->dma_addr[i], in svm_range_split_pages()
992 sizeof(*old->dma_addr[i]), old->start, in svm_range_split_pages()
993 npages, new->start, new->npages, in svm_range_split_pages()
994 old->actual_loc ? &new->vram_pages : NULL); in svm_range_split_pages()
998 if (old->actual_loc) in svm_range_split_pages()
999 old->vram_pages -= new->vram_pages; in svm_range_split_pages()
1008 uint64_t npages = last - start + 1; in svm_range_split_nodes()
1011 new->svms, new, new->start, start, last); in svm_range_split_nodes()
1013 if (new->start == old->start) { in svm_range_split_nodes()
1014 new->offset = old->offset; in svm_range_split_nodes()
1015 old->offset += new->npages; in svm_range_split_nodes()
1017 new->offset = old->offset + npages; in svm_range_split_nodes()
1020 new->svm_bo = svm_range_bo_ref(old->svm_bo); in svm_range_split_nodes()
1021 new->ttm_res = old->ttm_res; in svm_range_split_nodes()
1023 spin_lock(&new->svm_bo->list_lock); in svm_range_split_nodes()
1024 list_add(&new->svm_bo_list, &new->svm_bo->range_list); in svm_range_split_nodes()
1025 spin_unlock(&new->svm_bo->list_lock); in svm_range_split_nodes()
1031 * svm_range_split_adjust - split range and adjust
1039 * range from new_start up to size new->npages, the remaining old range is from
1043 * 0 - OK, -ENOMEM - out of memory
1052 new->svms, new->start, old->start, old->last, start, last); in svm_range_split_adjust()
1054 if (new->start < old->start || in svm_range_split_adjust()
1055 new->last > old->last) { in svm_range_split_adjust()
1057 return -EINVAL; in svm_range_split_adjust()
1064 if (old->actual_loc && old->ttm_res) { in svm_range_split_adjust()
1070 old->npages = last - start + 1; in svm_range_split_adjust()
1071 old->start = start; in svm_range_split_adjust()
1072 old->last = last; in svm_range_split_adjust()
1073 new->flags = old->flags; in svm_range_split_adjust()
1074 new->preferred_loc = old->preferred_loc; in svm_range_split_adjust()
1075 new->prefetch_loc = old->prefetch_loc; in svm_range_split_adjust()
1076 new->actual_loc = old->actual_loc; in svm_range_split_adjust()
1077 new->granularity = old->granularity; in svm_range_split_adjust()
1078 new->mapped_to_gpu = old->mapped_to_gpu; in svm_range_split_adjust()
1079 bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE); in svm_range_split_adjust()
1080 bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE); in svm_range_split_adjust()
1081 atomic_set(&new->queue_refcount, atomic_read(&old->queue_refcount)); in svm_range_split_adjust()
1087 * svm_range_split - split a range in 2 ranges
1095 * case 1: if start == prange->start
1097 * new range [last + 1, prange->last]
1099 * case 2: if last == prange->last
1101 * new range [prange->start, start - 1]
1104 * 0 - OK, -ENOMEM - out of memory, -EINVAL - invalid start, last
1110 uint64_t old_start = prange->start; in svm_range_split()
1111 uint64_t old_last = prange->last; in svm_range_split()
1115 pr_debug("svms 0x%p [0x%llx 0x%llx] to [0x%llx 0x%llx]\n", prange->svms, in svm_range_split()
1119 return -EINVAL; in svm_range_split()
1121 return -EINVAL; in svm_range_split()
1123 svms = prange->svms; in svm_range_split()
1127 *new = svm_range_new(svms, old_start, start - 1, false); in svm_range_split()
1129 return -ENOMEM; in svm_range_split()
1147 int r = svm_range_split(prange, prange->start, new_last, &tail); in svm_range_split_tail()
1150 list_add(&tail->list, insert_list); in svm_range_split_tail()
1151 if (!IS_ALIGNED(new_last + 1, 1UL << prange->granularity)) in svm_range_split_tail()
1152 list_add(&tail->update_list, remap_list); in svm_range_split_tail()
1162 int r = svm_range_split(prange, new_start, prange->last, &head); in svm_range_split_head()
1165 list_add(&head->list, insert_list); in svm_range_split_head()
1166 if (!IS_ALIGNED(new_start, 1UL << prange->granularity)) in svm_range_split_head()
1167 list_add(&head->update_list, remap_list); in svm_range_split_head()
1177 pchild, pchild->start, pchild->last, prange, op); in svm_range_add_child()
1179 pchild->work_item.mm = mm; in svm_range_add_child()
1180 pchild->work_item.op = op; in svm_range_add_child()
1181 list_add_tail(&pchild->child_list, &prange->child_list); in svm_range_add_child()
1187 return (node_a->adev == node_b->adev || in svm_nodes_in_same_hive()
1188 amdgpu_xgmi_same_hive(node_a->adev, node_b->adev)); in svm_nodes_in_same_hive()
1196 uint32_t flags = prange->flags; in svm_range_get_pte_flags()
1205 bo_node = prange->svm_bo->node; in svm_range_get_pte_flags()
1207 switch (amdgpu_ip_version(node->adev, GC_HWIP, 0)) { in svm_range_get_pte_flags()
1229 if (node->adev->gmc.xgmi.connected_to_cpu) in svm_range_get_pte_flags()
1245 mtype_local = node->adev->rev_id ? AMDGPU_VM_MTYPE_CC : AMDGPU_VM_MTYPE_UC; in svm_range_get_pte_flags()
1252 if (bo_node->adev == node->adev && in svm_range_get_pte_flags()
1253 (!bo_node->xcp || !node->xcp || bo_node->xcp->mem_id == node->xcp->mem_id)) in svm_range_get_pte_flags()
1264 } else if (node->adev->flags & AMD_IS_APU) { in svm_range_get_pte_flags()
1265 /* On NUMA systems, locality is determined per-page in svm_range_get_pte_flags()
1305 pte_flags |= amdgpu_gem_va_map_flags(node->adev, mapping_flags); in svm_range_get_pte_flags()
1334 if (!prange->mapped_to_gpu) { in svm_range_unmap_from_gpus()
1336 prange, prange->start, prange->last); in svm_range_unmap_from_gpus()
1340 if (prange->start == start && prange->last == last) { in svm_range_unmap_from_gpus()
1341 pr_debug("unmap svms 0x%p prange 0x%p\n", prange->svms, prange); in svm_range_unmap_from_gpus()
1342 prange->mapped_to_gpu = false; in svm_range_unmap_from_gpus()
1345 bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip, in svm_range_unmap_from_gpus()
1347 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_unmap_from_gpus()
1354 return -EINVAL; in svm_range_unmap_from_gpus()
1357 kfd_smi_event_unmap_from_gpu(pdd->dev, p->lead_thread->pid, in svm_range_unmap_from_gpus()
1360 r = svm_range_unmap_from_gpu(pdd->dev->adev, in svm_range_unmap_from_gpus()
1361 drm_priv_to_vm(pdd->drm_priv), in svm_range_unmap_from_gpus()
1385 struct amdgpu_device *adev = pdd->dev->adev; in svm_range_map_to_gpu()
1386 struct amdgpu_vm *vm = drm_priv_to_vm(pdd->drm_priv); in svm_range_map_to_gpu()
1393 last_start = prange->start + offset; in svm_range_map_to_gpu()
1395 pr_debug("svms 0x%p [0x%lx 0x%lx] readonly %d\n", prange->svms, in svm_range_map_to_gpu()
1396 last_start, last_start + npages - 1, readonly); in svm_range_map_to_gpu()
1405 if (i < offset + npages - 1 && in svm_range_map_to_gpu()
1410 last_start, prange->start + i, last_domain ? "GPU" : "CPU"); in svm_range_map_to_gpu()
1412 pte_flags = svm_range_get_pte_flags(pdd->dev, prange, last_domain); in svm_range_map_to_gpu()
1417 prange->svms, last_start, prange->start + i, in svm_range_map_to_gpu()
1426 NULL, last_start, prange->start + i, in svm_range_map_to_gpu()
1428 (last_start - prange->start) << PAGE_SHIFT, in svm_range_map_to_gpu()
1429 bo_adev ? bo_adev->vm_manager.vram_base_offset : 0, in svm_range_map_to_gpu()
1430 NULL, dma_addr, &vm->last_update); in svm_range_map_to_gpu()
1432 for (j = last_start - prange->start; j <= i; j++) in svm_range_map_to_gpu()
1436 pr_debug("failed %d to map to gpu 0x%lx\n", r, prange->start); in svm_range_map_to_gpu()
1439 last_start = prange->start + i + 1; in svm_range_map_to_gpu()
1445 prange->start); in svm_range_map_to_gpu()
1450 *fence = dma_fence_get(vm->last_update); in svm_range_map_to_gpu()
1468 if (prange->svm_bo && prange->ttm_res) in svm_range_map_to_gpus()
1469 bo_adev = prange->svm_bo->node->adev; in svm_range_map_to_gpus()
1471 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_map_to_gpus()
1477 return -EINVAL; in svm_range_map_to_gpus()
1480 pdd = kfd_bind_process_to_device(pdd->dev, p); in svm_range_map_to_gpus()
1482 return -EINVAL; in svm_range_map_to_gpus()
1484 if (bo_adev && pdd->dev->adev != bo_adev && in svm_range_map_to_gpus()
1485 !amdgpu_xgmi_same_hive(pdd->dev->adev, bo_adev)) { in svm_range_map_to_gpus()
1491 prange->dma_addr[gpuidx], in svm_range_map_to_gpus()
1528 drm_exec_init(&ctx->exec, intr ? DRM_EXEC_INTERRUPTIBLE_WAIT: 0, 0); in svm_range_reserve_bos()
1529 drm_exec_until_all_locked(&ctx->exec) { in svm_range_reserve_bos()
1530 for_each_set_bit(gpuidx, ctx->bitmap, MAX_GPU_INSTANCE) { in svm_range_reserve_bos()
1531 pdd = kfd_process_device_from_gpuidx(ctx->process, gpuidx); in svm_range_reserve_bos()
1534 r = -EINVAL; in svm_range_reserve_bos()
1537 vm = drm_priv_to_vm(pdd->drm_priv); in svm_range_reserve_bos()
1539 r = amdgpu_vm_lock_pd(vm, &ctx->exec, 2); in svm_range_reserve_bos()
1540 drm_exec_retry_on_contention(&ctx->exec); in svm_range_reserve_bos()
1548 for_each_set_bit(gpuidx, ctx->bitmap, MAX_GPU_INSTANCE) { in svm_range_reserve_bos()
1549 pdd = kfd_process_device_from_gpuidx(ctx->process, gpuidx); in svm_range_reserve_bos()
1552 r = -EINVAL; in svm_range_reserve_bos()
1556 r = amdgpu_vm_validate(pdd->dev->adev, in svm_range_reserve_bos()
1557 drm_priv_to_vm(pdd->drm_priv), NULL, in svm_range_reserve_bos()
1568 drm_exec_fini(&ctx->exec); in svm_range_reserve_bos()
1574 drm_exec_fini(&ctx->exec); in svm_range_unreserve_bos()
1585 return SVM_ADEV_PGMAP_OWNER(pdd->dev->adev); in kfd_svm_page_owner()
1595 * prange->migrate_mutex must be held.
1600 * The following sequence ensures race-free validation and GPU mapping:
1604 * 3. DMA-map pages (if system memory)
1605 * 4-a. Take notifier lock
1606 * 4-b. Check that pages still valid (mmu_interval_read_retry)
1607 * 4-c. Check that the range was not split or otherwise invalidated
1608 * 4-d. Update GPU page table
1626 return -ENOMEM; in svm_range_validate_and_map()
1627 ctx->process = container_of(prange->svms, struct kfd_process, svms); in svm_range_validate_and_map()
1628 ctx->prange = prange; in svm_range_validate_and_map()
1629 ctx->intr = intr; in svm_range_validate_and_map()
1632 bitmap_zero(ctx->bitmap, MAX_GPU_INSTANCE); in svm_range_validate_and_map()
1633 bitmap_set(ctx->bitmap, gpuidx, 1); in svm_range_validate_and_map()
1634 } else if (ctx->process->xnack_enabled) { in svm_range_validate_and_map()
1635 bitmap_copy(ctx->bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE); in svm_range_validate_and_map()
1641 if (prange->actual_loc) { in svm_range_validate_and_map()
1642 gpuidx = kfd_process_gpuidx_from_gpuid(ctx->process, in svm_range_validate_and_map()
1643 prange->actual_loc); in svm_range_validate_and_map()
1646 prange->actual_loc); in svm_range_validate_and_map()
1647 r = -EINVAL; in svm_range_validate_and_map()
1650 if (test_bit(gpuidx, prange->bitmap_access)) in svm_range_validate_and_map()
1651 bitmap_set(ctx->bitmap, gpuidx, 1); in svm_range_validate_and_map()
1658 if (bitmap_empty(ctx->bitmap, MAX_GPU_INSTANCE)) { in svm_range_validate_and_map()
1659 if (prange->mapped_to_gpu || in svm_range_validate_and_map()
1660 prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED) in svm_range_validate_and_map()
1661 bitmap_copy(ctx->bitmap, prange->bitmap_access, MAX_GPU_INSTANCE); in svm_range_validate_and_map()
1664 bitmap_or(ctx->bitmap, prange->bitmap_access, in svm_range_validate_and_map()
1665 prange->bitmap_aip, MAX_GPU_INSTANCE); in svm_range_validate_and_map()
1668 if (bitmap_empty(ctx->bitmap, MAX_GPU_INSTANCE)) { in svm_range_validate_and_map()
1673 if (prange->actual_loc && !prange->ttm_res) { in svm_range_validate_and_map()
1678 r = -EINVAL; in svm_range_validate_and_map()
1686 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_validate_and_map()
1687 owner = kfd_svm_page_owner(p, find_first_bit(ctx->bitmap, in svm_range_validate_and_map()
1689 for_each_set_bit(idx, ctx->bitmap, MAX_GPU_INSTANCE) { in svm_range_validate_and_map()
1710 readonly = !(vma->vm_flags & VM_WRITE); in svm_range_validate_and_map()
1712 next = min(vma->vm_end, end); in svm_range_validate_and_map()
1713 npages = (next - addr) >> PAGE_SHIFT; in svm_range_validate_and_map()
1714 WRITE_ONCE(p->svms.faulting_task, current); in svm_range_validate_and_map()
1715 r = amdgpu_hmm_range_get_pages(&prange->notifier, addr, npages, in svm_range_validate_and_map()
1718 WRITE_ONCE(p->svms.faulting_task, NULL); in svm_range_validate_and_map()
1722 r = -EFAULT; in svm_range_validate_and_map()
1726 offset = (addr >> PAGE_SHIFT) - prange->start; in svm_range_validate_and_map()
1727 r = svm_range_dma_map(prange, ctx->bitmap, offset, npages, in svm_range_validate_and_map()
1728 hmm_range->hmm_pfns); in svm_range_validate_and_map()
1741 r = -EAGAIN; in svm_range_validate_and_map()
1744 if (!r && !list_empty(&prange->child_list)) { in svm_range_validate_and_map()
1746 r = -EAGAIN; in svm_range_validate_and_map()
1750 map_start_vma = max(map_start, prange->start + offset); in svm_range_validate_and_map()
1751 map_last_vma = min(map_last, prange->start + offset + npages - 1); in svm_range_validate_and_map()
1753 offset = map_start_vma - prange->start; in svm_range_validate_and_map()
1754 npages = map_last_vma - map_start_vma + 1; in svm_range_validate_and_map()
1756 ctx->bitmap, wait, flush_tlb); in svm_range_validate_and_map()
1761 prange->mapped_to_gpu = true; in svm_range_validate_and_map()
1770 prange->validate_timestamp = ktime_get_boottime(); in svm_range_validate_and_map()
1779 * svm_range_list_lock_and_flush_work - flush pending deferred work
1792 flush_work(&svms->deferred_list_work); in svm_range_list_lock_and_flush_work()
1795 if (list_empty(&svms->deferred_range_list)) in svm_range_list_lock_and_flush_work()
1815 evicted_ranges = atomic_read(&svms->evicted_ranges); in svm_range_restore_work()
1822 process_info = p->kgd_process_info; in svm_range_restore_work()
1825 mm = get_task_mm(p->lead_thread); in svm_range_restore_work()
1831 mutex_lock(&process_info->lock); in svm_range_restore_work()
1833 mutex_lock(&svms->lock); in svm_range_restore_work()
1835 evicted_ranges = atomic_read(&svms->evicted_ranges); in svm_range_restore_work()
1837 list_for_each_entry(prange, &svms->list, list) { in svm_range_restore_work()
1838 invalid = atomic_read(&prange->invalid); in svm_range_restore_work()
1842 pr_debug("restoring svms 0x%p prange 0x%p [0x%lx %lx] inv %d\n", in svm_range_restore_work()
1843 prange->svms, prange, prange->start, prange->last, in svm_range_restore_work()
1849 mutex_lock(&prange->migrate_mutex); in svm_range_restore_work()
1851 r = svm_range_validate_and_map(mm, prange->start, prange->last, prange, in svm_range_restore_work()
1855 prange->start); in svm_range_restore_work()
1857 mutex_unlock(&prange->migrate_mutex); in svm_range_restore_work()
1861 if (atomic_cmpxchg(&prange->invalid, invalid, 0) != invalid) in svm_range_restore_work()
1865 if (atomic_cmpxchg(&svms->evicted_ranges, evicted_ranges, 0) != in svm_range_restore_work()
1882 mutex_unlock(&svms->lock); in svm_range_restore_work()
1884 mutex_unlock(&process_info->lock); in svm_range_restore_work()
1889 queue_delayed_work(system_freezable_wq, &svms->restore_work, in svm_range_restore_work()
1898 * svm_range_evict - evict svm range
1918 struct svm_range_list *svms = prange->svms; in svm_range_evict()
1926 svms, prange->start, prange->last, start, last); in svm_range_evict()
1928 if (!p->xnack_enabled || in svm_range_evict()
1929 (prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) { in svm_range_evict()
1931 bool mapped = prange->mapped_to_gpu; in svm_range_evict()
1933 list_for_each_entry(pchild, &prange->child_list, child_list) { in svm_range_evict()
1934 if (!pchild->mapped_to_gpu) in svm_range_evict()
1937 mutex_lock_nested(&pchild->lock, 1); in svm_range_evict()
1938 if (pchild->start <= last && pchild->last >= start) { in svm_range_evict()
1940 pchild->start, pchild->last); in svm_range_evict()
1941 atomic_inc(&pchild->invalid); in svm_range_evict()
1943 mutex_unlock(&pchild->lock); in svm_range_evict()
1949 if (prange->start <= last && prange->last >= start) in svm_range_evict()
1950 atomic_inc(&prange->invalid); in svm_range_evict()
1952 evicted_ranges = atomic_inc_return(&svms->evicted_ranges); in svm_range_evict()
1957 prange->svms, prange->start, prange->last); in svm_range_evict()
1965 queue_delayed_work(system_freezable_wq, &svms->restore_work, in svm_range_evict()
1977 prange->svms, start, last); in svm_range_evict()
1978 list_for_each_entry(pchild, &prange->child_list, child_list) { in svm_range_evict()
1979 mutex_lock_nested(&pchild->lock, 1); in svm_range_evict()
1980 s = max(start, pchild->start); in svm_range_evict()
1981 l = min(last, pchild->last); in svm_range_evict()
1984 mutex_unlock(&pchild->lock); in svm_range_evict()
1986 s = max(start, prange->start); in svm_range_evict()
1987 l = min(last, prange->last); in svm_range_evict()
1999 new = svm_range_new(old->svms, old->start, old->last, false); in svm_range_clone()
2006 if (old->svm_bo) { in svm_range_clone()
2007 new->ttm_res = old->ttm_res; in svm_range_clone()
2008 new->offset = old->offset; in svm_range_clone()
2009 new->svm_bo = svm_range_bo_ref(old->svm_bo); in svm_range_clone()
2010 spin_lock(&new->svm_bo->list_lock); in svm_range_clone()
2011 list_add(&new->svm_bo_list, &new->svm_bo->range_list); in svm_range_clone()
2012 spin_unlock(&new->svm_bo->list_lock); in svm_range_clone()
2014 new->flags = old->flags; in svm_range_clone()
2015 new->preferred_loc = old->preferred_loc; in svm_range_clone()
2016 new->prefetch_loc = old->prefetch_loc; in svm_range_clone()
2017 new->actual_loc = old->actual_loc; in svm_range_clone()
2018 new->granularity = old->granularity; in svm_range_clone()
2019 new->mapped_to_gpu = old->mapped_to_gpu; in svm_range_clone()
2020 new->vram_pages = old->vram_pages; in svm_range_clone()
2021 bitmap_copy(new->bitmap_access, old->bitmap_access, MAX_GPU_INSTANCE); in svm_range_clone()
2022 bitmap_copy(new->bitmap_aip, old->bitmap_aip, MAX_GPU_INSTANCE); in svm_range_clone()
2023 atomic_set(&new->queue_refcount, atomic_read(&old->queue_refcount)); in svm_range_clone()
2035 for (i = 0; i < adev->kfd.dev->num_nodes; i++) { in svm_range_set_max_pages()
2036 if (adev->kfd.dev->nodes[i]->xcp) in svm_range_set_max_pages()
2037 id = adev->kfd.dev->nodes[i]->xcp->id; in svm_range_set_max_pages()
2039 id = -1; in svm_range_set_max_pages()
2064 l = min(last, ALIGN_DOWN(start + max_pages, max_pages) - 1); in svm_range_split_new()
2068 return -ENOMEM; in svm_range_split_new()
2069 list_add(&prange->list, insert_list); in svm_range_split_new()
2070 list_add(&prange->update_list, update_list); in svm_range_split_new()
2078 * svm_range_add - add svm range and handle overlap
2102 * Context: Process context, caller must hold svms->lock
2105 * 0 - OK, otherwise error code
2113 unsigned long last = start + size - 1UL; in svm_range_add()
2114 struct svm_range_list *svms = &p->svms; in svm_range_add()
2121 pr_debug("svms 0x%p [0x%llx 0x%lx]\n", &p->svms, start, last); in svm_range_add()
2129 node = interval_tree_iter_first(&svms->objects, start, last); in svm_range_add()
2134 pr_debug("found overlap node [0x%lx 0x%lx]\n", node->start, in svm_range_add()
2135 node->last); in svm_range_add()
2139 next_start = min(node->last, last) + 1; in svm_range_add()
2142 prange->mapped_to_gpu) { in svm_range_add()
2144 } else if (node->start < start || node->last > last) { in svm_range_add()
2153 r = -ENOMEM; in svm_range_add()
2157 list_add(&old->update_list, remove_list); in svm_range_add()
2158 list_add(&prange->list, insert_list); in svm_range_add()
2159 list_add(&prange->update_list, update_list); in svm_range_add()
2161 if (node->start < start) { in svm_range_add()
2168 if (node->last > last) { in svm_range_add()
2179 list_add(&prange->update_list, update_list); in svm_range_add()
2183 if (node->start > start) { in svm_range_add()
2184 r = svm_range_split_new(svms, start, node->start - 1, in svm_range_add()
2221 start = prange->notifier.interval_tree.start >> PAGE_SHIFT; in svm_range_update_notifier_and_interval_tree()
2222 last = prange->notifier.interval_tree.last >> PAGE_SHIFT; in svm_range_update_notifier_and_interval_tree()
2224 if (prange->start == start && prange->last == last) in svm_range_update_notifier_and_interval_tree()
2228 prange->svms, prange, start, last, prange->start, in svm_range_update_notifier_and_interval_tree()
2229 prange->last); in svm_range_update_notifier_and_interval_tree()
2232 interval_tree_remove(&prange->it_node, &prange->svms->objects); in svm_range_update_notifier_and_interval_tree()
2235 prange->it_node.start = prange->start; in svm_range_update_notifier_and_interval_tree()
2236 prange->it_node.last = prange->last; in svm_range_update_notifier_and_interval_tree()
2238 interval_tree_insert(&prange->it_node, &prange->svms->objects); in svm_range_update_notifier_and_interval_tree()
2246 switch (prange->work_item.op) { in svm_range_handle_list_op()
2249 svms, prange, prange->start, prange->last); in svm_range_handle_list_op()
2253 svms, prange, prange->start, prange->last); in svm_range_handle_list_op()
2260 svms, prange, prange->start, prange->last); in svm_range_handle_list_op()
2265 svms, prange, prange->start, prange->last); in svm_range_handle_list_op()
2271 prange->start, prange->last); in svm_range_handle_list_op()
2277 prange, prange->start, prange->last); in svm_range_handle_list_op()
2284 prange->work_item.op); in svm_range_handle_list_op()
2296 for_each_set_bit(i, svms->bitmap_supported, p->n_pdds) { in svm_range_drain_retry_fault()
2297 pdd = p->pdds[i]; in svm_range_drain_retry_fault()
2303 amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev, in svm_range_drain_retry_fault()
2304 pdd->dev->adev->irq.retry_cam_enabled ? in svm_range_drain_retry_fault()
2305 &pdd->dev->adev->irq.ih : in svm_range_drain_retry_fault()
2306 &pdd->dev->adev->irq.ih1); in svm_range_drain_retry_fault()
2308 if (pdd->dev->adev->irq.retry_cam_enabled) in svm_range_drain_retry_fault()
2309 amdgpu_ih_wait_on_checkpoint_process_ts(pdd->dev->adev, in svm_range_drain_retry_fault()
2310 &pdd->dev->adev->irq.ih_soft); in svm_range_drain_retry_fault()
2326 spin_lock(&svms->deferred_list_lock); in svm_range_deferred_list_work()
2327 while (!list_empty(&svms->deferred_range_list)) { in svm_range_deferred_list_work()
2328 prange = list_first_entry(&svms->deferred_range_list, in svm_range_deferred_list_work()
2330 spin_unlock(&svms->deferred_list_lock); in svm_range_deferred_list_work()
2333 prange->start, prange->last, prange->work_item.op); in svm_range_deferred_list_work()
2335 mm = prange->work_item.mm; in svm_range_deferred_list_work()
2347 spin_lock(&svms->deferred_list_lock); in svm_range_deferred_list_work()
2348 list_del_init(&prange->deferred_list); in svm_range_deferred_list_work()
2349 spin_unlock(&svms->deferred_list_lock); in svm_range_deferred_list_work()
2351 mutex_lock(&svms->lock); in svm_range_deferred_list_work()
2352 mutex_lock(&prange->migrate_mutex); in svm_range_deferred_list_work()
2353 while (!list_empty(&prange->child_list)) { in svm_range_deferred_list_work()
2356 pchild = list_first_entry(&prange->child_list, in svm_range_deferred_list_work()
2359 pchild->work_item.op); in svm_range_deferred_list_work()
2360 list_del_init(&pchild->child_list); in svm_range_deferred_list_work()
2363 mutex_unlock(&prange->migrate_mutex); in svm_range_deferred_list_work()
2366 mutex_unlock(&svms->lock); in svm_range_deferred_list_work()
2374 spin_lock(&svms->deferred_list_lock); in svm_range_deferred_list_work()
2376 spin_unlock(&svms->deferred_list_lock); in svm_range_deferred_list_work()
2384 spin_lock(&svms->deferred_list_lock); in svm_range_add_list_work()
2386 if (!list_empty(&prange->deferred_list)) { in svm_range_add_list_work()
2388 WARN_ONCE(prange->work_item.mm != mm, "unmatch mm\n"); in svm_range_add_list_work()
2390 prange->work_item.op != SVM_OP_UNMAP_RANGE) in svm_range_add_list_work()
2391 prange->work_item.op = op; in svm_range_add_list_work()
2393 prange->work_item.op = op; in svm_range_add_list_work()
2397 prange->work_item.mm = mm; in svm_range_add_list_work()
2398 list_add_tail(&prange->deferred_list, in svm_range_add_list_work()
2399 &prange->svms->deferred_range_list); in svm_range_add_list_work()
2401 prange, prange->start, prange->last, op); in svm_range_add_list_work()
2403 spin_unlock(&svms->deferred_list_lock); in svm_range_add_list_work()
2408 spin_lock(&svms->deferred_list_lock); in schedule_deferred_list_work()
2409 if (!list_empty(&svms->deferred_range_list)) in schedule_deferred_list_work()
2410 schedule_work(&svms->deferred_list_work); in schedule_deferred_list_work()
2411 spin_unlock(&svms->deferred_list_lock); in schedule_deferred_list_work()
2422 if (prange->work_item.op == SVM_OP_UNMAP_RANGE) { in svm_range_unmap_split()
2424 prange->start, prange->last); in svm_range_unmap_split()
2427 if (start > prange->last || last < prange->start) in svm_range_unmap_split()
2431 if (start > prange->start) in svm_range_unmap_split()
2432 svm_range_split(prange, prange->start, start - 1, &tail); in svm_range_unmap_split()
2433 if (last < tail->last) in svm_range_unmap_split()
2434 svm_range_split(tail, last + 1, tail->last, &head); in svm_range_unmap_split()
2444 prange->work_item.op = SVM_OP_UNMAP_RANGE; in svm_range_unmap_split()
2460 if (atomic_read(&prange->queue_refcount)) { in svm_range_unmap_from_cpu()
2464 prange->start << PAGE_SHIFT); in svm_range_unmap_from_cpu()
2473 svms = &p->svms; in svm_range_unmap_from_cpu()
2476 prange, prange->start, prange->last, start, last); in svm_range_unmap_from_cpu()
2481 for_each_set_bit(i, svms->bitmap_supported, p->n_pdds) { in svm_range_unmap_from_cpu()
2487 pdd = p->pdds[i]; in svm_range_unmap_from_cpu()
2491 adev = pdd->dev->adev; in svm_range_unmap_from_cpu()
2494 if (adev->irq.ih1.ring_size) { in svm_range_unmap_from_cpu()
2495 ih = &adev->irq.ih1; in svm_range_unmap_from_cpu()
2497 if (ih->rptr != checkpoint_wptr) { in svm_range_unmap_from_cpu()
2498 svms->checkpoint_ts[i] = in svm_range_unmap_from_cpu()
2499 amdgpu_ih_decode_iv_ts(adev, ih, checkpoint_wptr, -1); in svm_range_unmap_from_cpu()
2504 /* check if dev->irq.ih_soft is not empty */ in svm_range_unmap_from_cpu()
2505 ih = &adev->irq.ih_soft; in svm_range_unmap_from_cpu()
2507 if (ih->rptr != checkpoint_wptr) in svm_range_unmap_from_cpu()
2508 svms->checkpoint_ts[i] = amdgpu_ih_decode_iv_ts(adev, ih, checkpoint_wptr, -1); in svm_range_unmap_from_cpu()
2511 unmap_parent = start <= prange->start && last >= prange->last; in svm_range_unmap_from_cpu()
2513 list_for_each_entry(pchild, &prange->child_list, child_list) { in svm_range_unmap_from_cpu()
2514 mutex_lock_nested(&pchild->lock, 1); in svm_range_unmap_from_cpu()
2515 s = max(start, pchild->start); in svm_range_unmap_from_cpu()
2516 l = min(last, pchild->last); in svm_range_unmap_from_cpu()
2520 mutex_unlock(&pchild->lock); in svm_range_unmap_from_cpu()
2522 s = max(start, prange->start); in svm_range_unmap_from_cpu()
2523 l = min(last, prange->last); in svm_range_unmap_from_cpu()
2539 * svm_range_cpu_invalidate_pagetables - interval notifier callback
2567 if (range->event == MMU_NOTIFY_RELEASE) in svm_range_cpu_invalidate_pagetables()
2569 if (!mmget_not_zero(mni->mm)) in svm_range_cpu_invalidate_pagetables()
2572 start = mni->interval_tree.start; in svm_range_cpu_invalidate_pagetables()
2573 last = mni->interval_tree.last; in svm_range_cpu_invalidate_pagetables()
2574 start = max(start, range->start) >> PAGE_SHIFT; in svm_range_cpu_invalidate_pagetables()
2575 last = min(last, range->end - 1) >> PAGE_SHIFT; in svm_range_cpu_invalidate_pagetables()
2577 start, last, range->start >> PAGE_SHIFT, in svm_range_cpu_invalidate_pagetables()
2578 (range->end - 1) >> PAGE_SHIFT, in svm_range_cpu_invalidate_pagetables()
2579 mni->interval_tree.start >> PAGE_SHIFT, in svm_range_cpu_invalidate_pagetables()
2580 mni->interval_tree.last >> PAGE_SHIFT, range->event); in svm_range_cpu_invalidate_pagetables()
2587 switch (range->event) { in svm_range_cpu_invalidate_pagetables()
2589 svm_range_unmap_from_cpu(mni->mm, prange, start, last); in svm_range_cpu_invalidate_pagetables()
2592 svm_range_evict(prange, mni->mm, start, last, range->event); in svm_range_cpu_invalidate_pagetables()
2597 mmput(mni->mm); in svm_range_cpu_invalidate_pagetables()
2603 * svm_range_from_addr - find svm range from fault address
2608 * Context: The caller must hold svms->lock
2620 node = interval_tree_iter_first(&svms->objects, addr, addr); in svm_range_from_addr()
2626 addr, prange->start, prange->last, node->start, node->last); in svm_range_from_addr()
2628 if (addr >= prange->start && addr <= prange->last) { in svm_range_from_addr()
2633 list_for_each_entry(pchild, &prange->child_list, child_list) in svm_range_from_addr()
2634 if (addr >= pchild->start && addr <= pchild->last) { in svm_range_from_addr()
2636 addr, pchild->start, pchild->last); in svm_range_from_addr()
2645 /* svm_range_best_restore_location - decide the best fault restore location
2660 * Otherwise, GPU no access, best_loc is -1.
2663 * -1 means vm fault GPU no access
2676 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_best_restore_location()
2681 return -1; in svm_range_best_restore_location()
2684 if (node->adev->flags & AMD_IS_APU) in svm_range_best_restore_location()
2687 if (prange->preferred_loc == gpuid || in svm_range_best_restore_location()
2688 prange->preferred_loc == KFD_IOCTL_SVM_LOCATION_SYSMEM) { in svm_range_best_restore_location()
2689 return prange->preferred_loc; in svm_range_best_restore_location()
2690 } else if (prange->preferred_loc != KFD_IOCTL_SVM_LOCATION_UNDEFINED) { in svm_range_best_restore_location()
2691 preferred_node = svm_range_get_node_by_id(prange, prange->preferred_loc); in svm_range_best_restore_location()
2693 return prange->preferred_loc; in svm_range_best_restore_location()
2697 if (test_bit(*gpuidx, prange->bitmap_access)) in svm_range_best_restore_location()
2700 if (test_bit(*gpuidx, prange->bitmap_aip)) { in svm_range_best_restore_location()
2701 if (!prange->actual_loc) in svm_range_best_restore_location()
2704 bo_node = svm_range_get_node_by_id(prange, prange->actual_loc); in svm_range_best_restore_location()
2706 return prange->actual_loc; in svm_range_best_restore_location()
2711 return -1; in svm_range_best_restore_location()
2724 vma = vma_lookup(p->mm, addr << PAGE_SHIFT); in svm_range_get_range_boundaries()
2727 return -EFAULT; in svm_range_get_range_boundaries()
2732 start_limit = max(vma->vm_start >> PAGE_SHIFT, in svm_range_get_range_boundaries()
2733 (unsigned long)ALIGN_DOWN(addr, 1UL << p->svms.default_granularity)); in svm_range_get_range_boundaries()
2734 end_limit = min(vma->vm_end >> PAGE_SHIFT, in svm_range_get_range_boundaries()
2735 (unsigned long)ALIGN(addr + 1, 1UL << p->svms.default_granularity)); in svm_range_get_range_boundaries()
2738 node = interval_tree_iter_first(&p->svms.objects, addr + 1, ULONG_MAX); in svm_range_get_range_boundaries()
2740 end_limit = min(end_limit, node->start); in svm_range_get_range_boundaries()
2742 rb_node = rb_prev(&node->rb); in svm_range_get_range_boundaries()
2747 rb_node = rb_last(&p->svms.objects.rb_root); in svm_range_get_range_boundaries()
2751 if (node->last >= addr) { in svm_range_get_range_boundaries()
2753 return -EFAULT; in svm_range_get_range_boundaries()
2755 start_limit = max(start_limit, node->last + 1); in svm_range_get_range_boundaries()
2759 *last = end_limit - 1; in svm_range_get_range_boundaries()
2762 vma->vm_start >> PAGE_SHIFT, vma->vm_end >> PAGE_SHIFT, in svm_range_get_range_boundaries()
2779 for (i = 0; i < p->n_pdds; i++) { in svm_range_check_vm_userptr()
2782 if (!p->pdds[i]->drm_priv) in svm_range_check_vm_userptr()
2785 vm = drm_priv_to_vm(p->pdds[i]->drm_priv); in svm_range_check_vm_userptr()
2786 r = amdgpu_bo_reserve(vm->root.bo, false); in svm_range_check_vm_userptr()
2790 /* Check userptr by searching entire vm->va interval tree */ in svm_range_check_vm_userptr()
2791 node = interval_tree_iter_first(&vm->va, 0, ~0ULL); in svm_range_check_vm_userptr()
2795 bo = mapping->bo_va->base.bo; in svm_range_check_vm_userptr()
2797 if (!amdgpu_ttm_tt_affect_userptr(bo->tbo.ttm, in svm_range_check_vm_userptr()
2809 *bo_l = *bo_s + bo->tbo.ttm->num_pages - 1; in svm_range_check_vm_userptr()
2811 amdgpu_bo_unreserve(vm->root.bo); in svm_range_check_vm_userptr()
2812 return -EADDRINUSE; in svm_range_check_vm_userptr()
2814 amdgpu_bo_unreserve(vm->root.bo); in svm_range_check_vm_userptr()
2838 if (r != -EADDRINUSE) in svm_range_create_unregistered_range()
2841 if (r == -EADDRINUSE) { in svm_range_create_unregistered_range()
2850 prange = svm_range_new(&p->svms, start, last, true); in svm_range_create_unregistered_range()
2862 prange->preferred_loc = KFD_IOCTL_SVM_LOCATION_SYSMEM; in svm_range_create_unregistered_range()
2870 /* svm_range_skip_recover - decide if prange can be recovered
2884 struct svm_range_list *svms = prange->svms; in svm_range_skip_recover()
2886 spin_lock(&svms->deferred_list_lock); in svm_range_skip_recover()
2887 if (list_empty(&prange->deferred_list) && in svm_range_skip_recover()
2888 list_empty(&prange->child_list)) { in svm_range_skip_recover()
2889 spin_unlock(&svms->deferred_list_lock); in svm_range_skip_recover()
2892 spin_unlock(&svms->deferred_list_lock); in svm_range_skip_recover()
2894 if (prange->work_item.op == SVM_OP_UNMAP_RANGE) { in svm_range_skip_recover()
2896 svms, prange, prange->start, prange->last); in svm_range_skip_recover()
2899 if (prange->work_item.op == SVM_OP_ADD_RANGE_AND_MAP || in svm_range_skip_recover()
2900 prange->work_item.op == SVM_OP_ADD_RANGE) { in svm_range_skip_recover()
2902 svms, prange, prange->start, prange->last); in svm_range_skip_recover()
2932 WRITE_ONCE(pdd->faults, pdd->faults + 1); in svm_range_count_fault()
2944 vma->vm_flags); in svm_fault_allowed()
2945 return (vma->vm_flags & requested) == requested; in svm_fault_allowed()
2951 uint64_t addr, uint64_t ts, bool write_fault) in svm_range_restore_pages() argument
2969 return -EFAULT; in svm_range_restore_pages()
2977 svms = &p->svms; in svm_range_restore_pages()
2981 if (atomic_read(&svms->drain_pagefaults)) { in svm_range_restore_pages()
2991 r = -EFAULT; in svm_range_restore_pages()
2997 r = -EFAULT; in svm_range_restore_pages()
3001 /* check if this page fault time stamp is before svms->checkpoint_ts */ in svm_range_restore_pages()
3002 if (svms->checkpoint_ts[gpuidx] != 0) { in svm_range_restore_pages()
3003 if (amdgpu_ih_ts_after(ts, svms->checkpoint_ts[gpuidx])) { in svm_range_restore_pages()
3008 /* ts is after svms->checkpoint_ts now, reset svms->checkpoint_ts in svm_range_restore_pages()
3009 * to zero to avoid following ts wrap around give wrong comparing in svm_range_restore_pages()
3011 svms->checkpoint_ts[gpuidx] = 0; in svm_range_restore_pages()
3014 if (!p->xnack_enabled) { in svm_range_restore_pages()
3016 r = -EFAULT; in svm_range_restore_pages()
3020 /* p->lead_thread is available as kfd_process_wq_release flush the work in svm_range_restore_pages()
3023 mm = get_task_mm(p->lead_thread); in svm_range_restore_pages()
3032 mutex_lock(&svms->lock); in svm_range_restore_pages()
3042 mutex_unlock(&svms->lock); in svm_range_restore_pages()
3053 r = -EFAULT; in svm_range_restore_pages()
3060 mutex_lock(&prange->migrate_mutex); in svm_range_restore_pages()
3063 amdgpu_gmc_filter_faults_remove(node->adev, addr, pasid); in svm_range_restore_pages()
3069 if (ktime_before(timestamp, ktime_add_ns(prange->validate_timestamp, in svm_range_restore_pages()
3072 svms, prange->start, prange->last); in svm_range_restore_pages()
3090 r = -EPERM; in svm_range_restore_pages()
3095 if (best_loc == -1) { in svm_range_restore_pages()
3097 svms, prange->start, prange->last); in svm_range_restore_pages()
3098 r = -EACCES; in svm_range_restore_pages()
3103 svms, prange->start, prange->last, best_loc, in svm_range_restore_pages()
3104 prange->actual_loc); in svm_range_restore_pages()
3106 kfd_smi_event_page_fault_start(node, p->lead_thread->pid, addr, in svm_range_restore_pages()
3110 size = 1UL << prange->granularity; in svm_range_restore_pages()
3111 start = max_t(unsigned long, ALIGN_DOWN(addr, size), prange->start); in svm_range_restore_pages()
3112 last = min_t(unsigned long, ALIGN(addr + 1, size) - 1, prange->last); in svm_range_restore_pages()
3113 if (prange->actual_loc != 0 || best_loc != 0) { in svm_range_restore_pages()
3125 if (prange->actual_loc && prange->actual_loc != best_loc) in svm_range_restore_pages()
3148 kfd_smi_event_page_fault_end(node, p->lead_thread->pid, addr, in svm_range_restore_pages()
3152 mutex_unlock(&prange->migrate_mutex); in svm_range_restore_pages()
3154 mutex_unlock(&svms->lock); in svm_range_restore_pages()
3163 if (r == -EAGAIN) { in svm_range_restore_pages()
3165 amdgpu_gmc_filter_faults_remove(node->adev, addr, pasid); in svm_range_restore_pages()
3179 pr_debug("switching xnack from %d to %d\n", p->xnack_enabled, xnack_enabled); in svm_range_switch_xnack_reserve_mem()
3181 mutex_lock(&p->svms.lock); in svm_range_switch_xnack_reserve_mem()
3183 list_for_each_entry(prange, &p->svms.list, list) { in svm_range_switch_xnack_reserve_mem()
3185 list_for_each_entry(pchild, &prange->child_list, child_list) { in svm_range_switch_xnack_reserve_mem()
3186 size = (pchild->last - pchild->start + 1) << PAGE_SHIFT; in svm_range_switch_xnack_reserve_mem()
3199 size = (prange->last - prange->start + 1) << PAGE_SHIFT; in svm_range_switch_xnack_reserve_mem()
3223 p->xnack_enabled = xnack_enabled; in svm_range_switch_xnack_reserve_mem()
3225 mutex_unlock(&p->svms.lock); in svm_range_switch_xnack_reserve_mem()
3234 pr_debug("pasid 0x%x svms 0x%p\n", p->pasid, &p->svms); in svm_range_list_fini()
3236 cancel_delayed_work_sync(&p->svms.restore_work); in svm_range_list_fini()
3239 flush_work(&p->svms.deferred_list_work); in svm_range_list_fini()
3246 atomic_set(&p->svms.drain_pagefaults, 1); in svm_range_list_fini()
3247 svm_range_drain_retry_fault(&p->svms); in svm_range_list_fini()
3249 list_for_each_entry_safe(prange, next, &p->svms.list, list) { in svm_range_list_fini()
3255 mutex_destroy(&p->svms.lock); in svm_range_list_fini()
3257 pr_debug("pasid 0x%x svms 0x%p done\n", p->pasid, &p->svms); in svm_range_list_fini()
3262 struct svm_range_list *svms = &p->svms; in svm_range_list_init()
3265 svms->objects = RB_ROOT_CACHED; in svm_range_list_init()
3266 mutex_init(&svms->lock); in svm_range_list_init()
3267 INIT_LIST_HEAD(&svms->list); in svm_range_list_init()
3268 atomic_set(&svms->evicted_ranges, 0); in svm_range_list_init()
3269 atomic_set(&svms->drain_pagefaults, 0); in svm_range_list_init()
3270 INIT_DELAYED_WORK(&svms->restore_work, svm_range_restore_work); in svm_range_list_init()
3271 INIT_WORK(&svms->deferred_list_work, svm_range_deferred_list_work); in svm_range_list_init()
3272 INIT_LIST_HEAD(&svms->deferred_range_list); in svm_range_list_init()
3273 INIT_LIST_HEAD(&svms->criu_svm_metadata_list); in svm_range_list_init()
3274 spin_lock_init(&svms->deferred_list_lock); in svm_range_list_init()
3276 for (i = 0; i < p->n_pdds; i++) in svm_range_list_init()
3277 if (KFD_IS_SVM_API_SUPPORTED(p->pdds[i]->dev->adev)) in svm_range_list_init()
3278 bitmap_set(svms->bitmap_supported, i, 1); in svm_range_list_init()
3281 * number of pages supported by a 4-level paging table in svm_range_list_init()
3283 svms->default_granularity = min_t(u8, amdgpu_svm_default_granularity, 0x1B); in svm_range_list_init()
3284 pr_debug("Default SVM Granularity to use: %d\n", svms->default_granularity); in svm_range_list_init()
3290 * svm_range_check_vm - check if virtual address range mapped already
3303 * Return 0 - OK, if the range is not mapped.
3305 * -EADDRINUSE - if address is mapped already by kfd_ioctl_alloc_memory_of_gpu
3306 * -ERESTARTSYS - A wait for the buffer to become unreserved was interrupted by
3307 * a signal. Release all buffer reservations and return to user-space.
3318 for (i = 0; i < p->n_pdds; i++) { in svm_range_check_vm()
3321 if (!p->pdds[i]->drm_priv) in svm_range_check_vm()
3324 vm = drm_priv_to_vm(p->pdds[i]->drm_priv); in svm_range_check_vm()
3325 r = amdgpu_bo_reserve(vm->root.bo, false); in svm_range_check_vm()
3329 node = interval_tree_iter_first(&vm->va, start, last); in svm_range_check_vm()
3336 *bo_s = mapping->start; in svm_range_check_vm()
3337 *bo_l = mapping->last; in svm_range_check_vm()
3339 amdgpu_bo_unreserve(vm->root.bo); in svm_range_check_vm()
3340 return -EADDRINUSE; in svm_range_check_vm()
3342 amdgpu_bo_unreserve(vm->root.bo); in svm_range_check_vm()
3349 * svm_range_is_valid - check if virtual address range is valid
3359 * 0 - OK, otherwise error code
3372 vma = vma_lookup(p->mm, start); in svm_range_is_valid()
3373 if (!vma || (vma->vm_flags & device_vma)) in svm_range_is_valid()
3374 return -EFAULT; in svm_range_is_valid()
3375 start = min(end, vma->vm_end); in svm_range_is_valid()
3378 return svm_range_check_vm(p, start_unchg, (end - 1) >> PAGE_SHIFT, NULL, in svm_range_is_valid()
3383 * svm_range_best_prefetch_location - decide the best prefetch location
3393 * mapping VRAM of other GPUs even with large-BAR PCIe connection.
3412 uint32_t best_loc = prange->prefetch_loc; in svm_range_best_prefetch_location()
3418 p = container_of(prange->svms, struct kfd_process, svms); in svm_range_best_prefetch_location()
3430 if (bo_node->adev->flags & AMD_IS_APU) { in svm_range_best_prefetch_location()
3435 if (p->xnack_enabled) in svm_range_best_prefetch_location()
3436 bitmap_copy(bitmap, prange->bitmap_aip, MAX_GPU_INSTANCE); in svm_range_best_prefetch_location()
3438 bitmap_or(bitmap, prange->bitmap_access, prange->bitmap_aip, in svm_range_best_prefetch_location()
3448 if (pdd->dev->adev == bo_node->adev) in svm_range_best_prefetch_location()
3451 if (!svm_nodes_in_same_hive(pdd->dev, bo_node)) { in svm_range_best_prefetch_location()
3459 p->xnack_enabled, &p->svms, prange->start, prange->last, in svm_range_best_prefetch_location()
3465 /* svm_range_trigger_migration - start page migration if prefetch loc changed
3480 * a. svm_range_validate_vram takes prange->migrate_mutex
3487 * 0 - OK, otherwise - error code of migration
3499 /* when best_loc is a gpu node and same as prange->actual_loc in svm_range_trigger_migration()
3500 * we still need do migration as prange->actual_loc !=0 does in svm_range_trigger_migration()
3505 (best_loc == 0 && prange->actual_loc == 0)) in svm_range_trigger_migration()
3509 r = svm_migrate_vram_to_ram(prange, mm, prange->start, prange->last, in svm_range_trigger_migration()
3515 r = svm_migrate_to_vram(prange, best_loc, prange->start, prange->last, in svm_range_trigger_migration()
3524 /* Dereferencing fence->svm_bo is safe here because the fence hasn't in svm_range_schedule_evict_svm_bo()
3525 * signaled yet and we're under the protection of the fence->lock. in svm_range_schedule_evict_svm_bo()
3531 if (svm_bo_ref_unless_zero(fence->svm_bo)) { in svm_range_schedule_evict_svm_bo()
3532 WRITE_ONCE(fence->svm_bo->evicting, 1); in svm_range_schedule_evict_svm_bo()
3533 schedule_work(&fence->svm_bo->eviction_work); in svm_range_schedule_evict_svm_bo()
3547 if (mmget_not_zero(svm_bo->eviction_fence->mm)) { in svm_range_evict_svm_bo_worker()
3548 mm = svm_bo->eviction_fence->mm; in svm_range_evict_svm_bo_worker()
3555 spin_lock(&svm_bo->list_lock); in svm_range_evict_svm_bo_worker()
3556 while (!list_empty(&svm_bo->range_list) && !r) { in svm_range_evict_svm_bo_worker()
3558 list_first_entry(&svm_bo->range_list, in svm_range_evict_svm_bo_worker()
3562 list_del_init(&prange->svm_bo_list); in svm_range_evict_svm_bo_worker()
3563 spin_unlock(&svm_bo->list_lock); in svm_range_evict_svm_bo_worker()
3565 pr_debug("svms 0x%p [0x%lx 0x%lx]\n", prange->svms, in svm_range_evict_svm_bo_worker()
3566 prange->start, prange->last); in svm_range_evict_svm_bo_worker()
3568 mutex_lock(&prange->migrate_mutex); in svm_range_evict_svm_bo_worker()
3571 * after that prange->actual_loc should be zero in svm_range_evict_svm_bo_worker()
3574 prange->start, prange->last, in svm_range_evict_svm_bo_worker()
3576 } while (!r && prange->actual_loc && --retries); in svm_range_evict_svm_bo_worker()
3578 if (!r && prange->actual_loc) in svm_range_evict_svm_bo_worker()
3581 if (!prange->actual_loc) { in svm_range_evict_svm_bo_worker()
3582 mutex_lock(&prange->lock); in svm_range_evict_svm_bo_worker()
3583 prange->svm_bo = NULL; in svm_range_evict_svm_bo_worker()
3584 mutex_unlock(&prange->lock); in svm_range_evict_svm_bo_worker()
3586 mutex_unlock(&prange->migrate_mutex); in svm_range_evict_svm_bo_worker()
3588 spin_lock(&svm_bo->list_lock); in svm_range_evict_svm_bo_worker()
3590 spin_unlock(&svm_bo->list_lock); in svm_range_evict_svm_bo_worker()
3594 dma_fence_signal(&svm_bo->eviction_fence->base); in svm_range_evict_svm_bo_worker()
3599 WARN_ONCE(!r && kref_read(&svm_bo->kref) != 1, "This was not the last reference\n"); in svm_range_evict_svm_bo_worker()
3608 struct amdkfd_process_info *process_info = p->kgd_process_info; in svm_range_set_attr()
3621 p->pasid, &p->svms, start, start + size - 1, size); in svm_range_set_attr()
3627 svms = &p->svms; in svm_range_set_attr()
3629 mutex_lock(&process_info->lock); in svm_range_set_attr()
3640 mutex_lock(&svms->lock); in svm_range_set_attr()
3646 mutex_unlock(&svms->lock); in svm_range_set_attr()
3661 prange->svms, prange, prange->start, in svm_range_set_attr()
3662 prange->last); in svm_range_set_attr()
3677 mutex_lock(&prange->migrate_mutex); in svm_range_set_attr()
3683 if (migrated && (!p->xnack_enabled || in svm_range_set_attr()
3684 (prange->flags & KFD_IOCTL_SVM_FLAG_GPU_ALWAYS_MAPPED)) && in svm_range_set_attr()
3685 prange->mapped_to_gpu) { in svm_range_set_attr()
3687 mutex_unlock(&prange->migrate_mutex); in svm_range_set_attr()
3692 mutex_unlock(&prange->migrate_mutex); in svm_range_set_attr()
3696 flush_tlb = !migrated && update_mapping && prange->mapped_to_gpu; in svm_range_set_attr()
3698 r = svm_range_validate_and_map(mm, prange->start, prange->last, prange, in svm_range_set_attr()
3704 mutex_unlock(&prange->migrate_mutex); in svm_range_set_attr()
3711 prange, prange->start, prange->last); in svm_range_set_attr()
3712 mutex_lock(&prange->migrate_mutex); in svm_range_set_attr()
3713 r = svm_range_validate_and_map(mm, prange->start, prange->last, prange, in svm_range_set_attr()
3714 MAX_GPU_INSTANCE, true, true, prange->mapped_to_gpu); in svm_range_set_attr()
3717 mutex_unlock(&prange->migrate_mutex); in svm_range_set_attr()
3724 mutex_unlock(&svms->lock); in svm_range_set_attr()
3727 mutex_unlock(&process_info->lock); in svm_range_set_attr()
3729 pr_debug("pasid 0x%x svms 0x%p [0x%llx 0x%llx] done, r=%d\n", p->pasid, in svm_range_set_attr()
3730 &p->svms, start, start + size - 1, r); in svm_range_set_attr()
3747 uint64_t last = start + size - 1UL; in svm_range_get_attr()
3760 pr_debug("svms 0x%p [0x%llx 0x%llx] nattr 0x%x\n", &p->svms, start, in svm_range_get_attr()
3761 start + size - 1, nattr); in svm_range_get_attr()
3769 flush_work(&p->svms.deferred_list_work); in svm_range_get_attr()
3802 return -EINVAL; in svm_range_get_attr()
3806 svms = &p->svms; in svm_range_get_attr()
3808 mutex_lock(&svms->lock); in svm_range_get_attr()
3810 node = interval_tree_iter_first(&svms->objects, start, last); in svm_range_get_attr()
3816 if (p->xnack_enabled) in svm_range_get_attr()
3817 bitmap_copy(bitmap_access, svms->bitmap_supported, in svm_range_get_attr()
3824 bitmap_copy(bitmap_access, svms->bitmap_supported, MAX_GPU_INSTANCE); in svm_range_get_attr()
3825 bitmap_copy(bitmap_aip, svms->bitmap_supported, MAX_GPU_INSTANCE); in svm_range_get_attr()
3834 if (prange->preferred_loc == in svm_range_get_attr()
3837 location != prange->preferred_loc)) { in svm_range_get_attr()
3841 location = prange->preferred_loc; in svm_range_get_attr()
3845 if (prange->prefetch_loc == in svm_range_get_attr()
3848 prefetch_loc != prange->prefetch_loc)) { in svm_range_get_attr()
3852 prefetch_loc = prange->prefetch_loc; in svm_range_get_attr()
3857 prange->bitmap_access, MAX_GPU_INSTANCE); in svm_range_get_attr()
3859 prange->bitmap_aip, MAX_GPU_INSTANCE); in svm_range_get_attr()
3862 flags_and &= prange->flags; in svm_range_get_attr()
3863 flags_or |= prange->flags; in svm_range_get_attr()
3866 if (get_granularity && prange->granularity < granularity) in svm_range_get_attr()
3867 granularity = prange->granularity; in svm_range_get_attr()
3872 mutex_unlock(&svms->lock); in svm_range_get_attr()
3887 return -EINVAL; in svm_range_get_attr()
3917 struct svm_range_list *svms = &p->svms; in kfd_criu_resume_svm()
3924 if (list_empty(&svms->criu_svm_metadata_list)) { in kfd_criu_resume_svm()
3929 mm = get_task_mm(p->lead_thread); in kfd_criu_resume_svm()
3932 return -ESRCH; in kfd_criu_resume_svm()
3935 num_attrs = nattr_common + (nattr_accessibility * p->n_pdds); in kfd_criu_resume_svm()
3938 list_for_each_entry(criu_svm_md, &svms->criu_svm_metadata_list, list) { in kfd_criu_resume_svm()
3940 i, criu_svm_md->data.start_addr, criu_svm_md->data.size); in kfd_criu_resume_svm()
3943 pr_debug("\ncriu_svm_md[%d]->attrs[%d].type : 0x%x\ncriu_svm_md[%d]->attrs[%d].value : 0x%x\n", in kfd_criu_resume_svm()
3944 i, j, criu_svm_md->data.attrs[j].type, in kfd_criu_resume_svm()
3945 i, j, criu_svm_md->data.attrs[j].value); in kfd_criu_resume_svm()
3946 switch (criu_svm_md->data.attrs[j].type) { in kfd_criu_resume_svm()
3958 if (criu_svm_md->data.attrs[j].value == in kfd_criu_resume_svm()
3960 criu_svm_md->data.attrs[j].type = in kfd_criu_resume_svm()
3962 criu_svm_md->data.attrs[j].value = 0; in kfd_criu_resume_svm()
3966 set_flags = criu_svm_md->data.attrs[j].value; in kfd_criu_resume_svm()
3982 ret = -ENOMEM; in kfd_criu_resume_svm()
3987 memcpy(set_attr, criu_svm_md->data.attrs, num_attrs * in kfd_criu_resume_svm()
3992 ret = svm_range_set_attr(p, mm, criu_svm_md->data.start_addr, in kfd_criu_resume_svm()
3993 criu_svm_md->data.size, num_attrs + 1, in kfd_criu_resume_svm()
4004 list_for_each_entry_safe(criu_svm_md, next, &svms->criu_svm_metadata_list, list) { in kfd_criu_resume_svm()
4006 criu_svm_md->data.start_addr); in kfd_criu_resume_svm()
4023 struct svm_range_list *svms = &p->svms; in kfd_criu_restore_svm()
4027 num_devices = p->n_pdds; in kfd_criu_restore_svm()
4043 return -ENOMEM; in kfd_criu_restore_svm()
4046 ret = -EINVAL; in kfd_criu_restore_svm()
4050 ret = copy_from_user(&criu_svm_md->data, user_priv_ptr + *priv_data_offset, in kfd_criu_restore_svm()
4053 ret = -EFAULT; in kfd_criu_restore_svm()
4058 list_add_tail(&criu_svm_md->list, &svms->criu_svm_metadata_list); in kfd_criu_restore_svm()
4073 int num_devices = p->n_pdds; in svm_range_get_info()
4080 svms = &p->svms; in svm_range_get_info()
4082 return -EINVAL; in svm_range_get_info()
4084 mutex_lock(&svms->lock); in svm_range_get_info()
4085 list_for_each_entry(prange, &svms->list, list) { in svm_range_get_info()
4087 prange, prange->start, prange->npages, in svm_range_get_info()
4088 prange->start + prange->npages - 1); in svm_range_get_info()
4091 mutex_unlock(&svms->lock); in svm_range_get_info()
4136 int num_devices = p->n_pdds; in kfd_criu_checkpoint_svm()
4140 svms = &p->svms; in kfd_criu_checkpoint_svm()
4142 return -EINVAL; in kfd_criu_checkpoint_svm()
4144 mm = get_task_mm(p->lead_thread); in kfd_criu_checkpoint_svm()
4147 return -ESRCH; in kfd_criu_checkpoint_svm()
4155 ret = -ENOMEM; in kfd_criu_checkpoint_svm()
4165 struct kfd_process_device *pdd = p->pdds[index]; in kfd_criu_checkpoint_svm()
4169 query_attr[index + nattr_common].value = pdd->user_gpu_id; in kfd_criu_checkpoint_svm()
4176 ret = -ENOMEM; in kfd_criu_checkpoint_svm()
4181 list_for_each_entry(prange, &svms->list, list) { in kfd_criu_checkpoint_svm()
4183 svm_priv->object_type = KFD_CRIU_OBJECT_TYPE_SVM_RANGE; in kfd_criu_checkpoint_svm()
4184 svm_priv->start_addr = prange->start; in kfd_criu_checkpoint_svm()
4185 svm_priv->size = prange->npages; in kfd_criu_checkpoint_svm()
4186 memcpy(&svm_priv->attrs, query_attr, query_attr_size); in kfd_criu_checkpoint_svm()
4188 prange, prange->start, prange->npages, in kfd_criu_checkpoint_svm()
4189 prange->start + prange->npages - 1, in kfd_criu_checkpoint_svm()
4190 prange->npages * PAGE_SIZE); in kfd_criu_checkpoint_svm()
4192 ret = svm_range_get_attr(p, mm, svm_priv->start_addr, in kfd_criu_checkpoint_svm()
4193 svm_priv->size, in kfd_criu_checkpoint_svm()
4195 svm_priv->attrs); in kfd_criu_checkpoint_svm()
4204 ret = -EFAULT; in kfd_criu_checkpoint_svm()
4226 struct mm_struct *mm = current->mm; in svm_ioctl()