554 	const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in _ppgtt_get_root_entry()
581 	const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in _ppgtt_set_root_entry()
597 	const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_get_guest_entry()
609 	const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_set_guest_entry()
620 	const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_get_host_entry()
630 	const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;  in ggtt_set_host_entry()
656 	const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in ppgtt_spt_get_entry()
685 	const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in ppgtt_spt_set_entry()
748 	radix_tree_delete(&spt->vgpu->gtt.spt_tree, spt->shadow_page.mfn);  in ppgtt_free_spt()
769 	radix_tree_for_each_slot(slot, &vgpu->gtt.spt_tree, &iter, 0) {  in ppgtt_free_all_spt()
817 	return radix_tree_lookup(&vgpu->gtt.spt_tree, mfn);  in intel_vgpu_find_spt_by_mfn()
859 	ret = radix_tree_insert(&vgpu->gtt.spt_tree, spt->shadow_page.mfn, spt);  in ppgtt_alloc_spt()
913 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
919 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
948 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_invalidate_spt_by_shadow_entry()
969 			vgpu->gtt.scratch_pt[cur_pt_type].page_mfn)  in ppgtt_invalidate_spt_by_shadow_entry()
985 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_invalidate_pte()
993 	if (!pfn || pfn == vgpu->gtt.scratch_pt[type].page_mfn)  in ppgtt_invalidate_pte()
1073 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_populate_spt_by_guest_entry()
1137 	const struct intel_gvt_gtt_pte_ops *ops = s->vgpu->gvt->gtt.pte_ops;  in ppgtt_generate_shadow_entry()
1153 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in split_2MB_gtt_entry()
1209 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in split_64KB_gtt_entry()
1240 	const struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_populate_shadow_entry()
1325 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_entry_removal()
1338 	    vgpu->gtt.scratch_pt[spt->shadow_page.type].page_mfn)  in ppgtt_handle_guest_entry_removal()
1406 	const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in sync_oos_page()
1458 	list_move_tail(&oos_page->list, &gvt->gtt.oos_page_free_list_head);  in detach_oos_page()
1478 	list_move_tail(&oos_page->list, &gvt->gtt.oos_page_use_list_head);  in attach_oos_page()
1504 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in ppgtt_allocate_oos_page()  local
1510 	if (list_empty(&gtt->oos_page_free_list_head)) {  in ppgtt_allocate_oos_page()
1511 		oos_page = container_of(gtt->oos_page_use_list_head.next,  in ppgtt_allocate_oos_page()
1520 		oos_page = container_of(gtt->oos_page_free_list_head.next,  in ppgtt_allocate_oos_page()
1535 	list_add_tail(&oos_page->vm_list, &spt->vgpu->gtt.oos_page_list_head);  in ppgtt_set_guest_page_oos()
1558 	list_for_each_safe(pos, n, &vgpu->gtt.oos_page_list_head) {  in intel_vgpu_sync_oos_pages()
1577 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_write_page_table()
1609 					vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1616 				     vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1620 				     vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table()
1649 			&spt->vgpu->gtt.post_shadow_list_head);  in ppgtt_set_post_shadow()
1670 	list_for_each_safe(pos, n, &vgpu->gtt.post_shadow_list_head) {  in intel_vgpu_flush_post_shadow()
1694 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_handle_guest_write_page_table_bytes()
1728 			ops->set_pfn(&se, vgpu->gtt.scratch_pt[type].page_mfn);  in ppgtt_handle_guest_write_page_table_bytes()
1758 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in invalidate_ppgtt_mm()  local
1759 	const struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;  in invalidate_ppgtt_mm()
1788 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in shadow_ppgtt_mm()  local
1789 	const struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;  in shadow_ppgtt_mm()
1895 	list_add_tail(&mm->ppgtt_mm.list, &vgpu->gtt.ppgtt_mm_list_head);  in intel_vgpu_create_ppgtt_mm()
1897 	mutex_lock(&gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_create_ppgtt_mm()
1898 	list_add_tail(&mm->ppgtt_mm.lru_list, &gvt->gtt.ppgtt_mm_lru_list_head);  in intel_vgpu_create_ppgtt_mm()
1899 	mutex_unlock(&gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_create_ppgtt_mm()
1959 		mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in _intel_vgpu_mm_release()
1961 		mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in _intel_vgpu_mm_release()
2006 		mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_pin_mm()
2008 			       &mm->vgpu->gvt->gtt.ppgtt_mm_lru_list_head);  in intel_vgpu_pin_mm()
2009 		mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_pin_mm()
2020 	mutex_lock(&gvt->gtt.ppgtt_mm_lock);  in reclaim_one_ppgtt_mm()
2022 	list_for_each_safe(pos, n, &gvt->gtt.ppgtt_mm_lru_list_head) {  in reclaim_one_ppgtt_mm()
2029 		mutex_unlock(&gvt->gtt.ppgtt_mm_lock);  in reclaim_one_ppgtt_mm()
2033 	mutex_unlock(&gvt->gtt.ppgtt_mm_lock);  in reclaim_one_ppgtt_mm()
2044 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in ppgtt_get_next_level_entry()
2073 	const struct intel_gvt_gtt_pte_ops *pte_ops = gvt->gtt.pte_ops;  in intel_vgpu_gma_to_gpa()
2074 	const struct intel_gvt_gtt_gma_ops *gma_ops = gvt->gtt.gma_ops;  in intel_vgpu_gma_to_gpa()
2146 	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;  in emulate_ggtt_mmio_read()
2198 	const struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in ggtt_invalidate_pte()
2202 	if (pfn != vgpu->gvt->gtt.scratch_mfn)  in ggtt_invalidate_pte()
2211 	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;  in emulate_ggtt_mmio_write()
2212 	const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;  in emulate_ggtt_mmio_write()
2294 			ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2298 		ops->set_pfn(&m, gvt->gtt.scratch_mfn);  in emulate_ggtt_mmio_write()
2357 	struct intel_vgpu_gtt *gtt = &vgpu->gtt;  in alloc_scratch_pages()  local
2358 	const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;  in alloc_scratch_pages()
2382 	gtt->scratch_pt[type].page_mfn =  in alloc_scratch_pages()
2384 	gtt->scratch_pt[type].page = virt_to_page(scratch_pt);  in alloc_scratch_pages()
2386 			vgpu->id, type, gtt->scratch_pt[type].page_mfn);  in alloc_scratch_pages()
2401 		ops->set_pfn(&se, gtt->scratch_pt[type - 1].page_mfn);  in alloc_scratch_pages()
2424 		if (vgpu->gtt.scratch_pt[i].page != NULL) {  in release_scratch_page_tree()
2425 			daddr = (dma_addr_t)(vgpu->gtt.scratch_pt[i].page_mfn <<  in release_scratch_page_tree()
2428 			__free_page(vgpu->gtt.scratch_pt[i].page);  in release_scratch_page_tree()
2429 			vgpu->gtt.scratch_pt[i].page = NULL;  in release_scratch_page_tree()
2430 			vgpu->gtt.scratch_pt[i].page_mfn = 0;  in release_scratch_page_tree()
2466 	struct intel_vgpu_gtt *gtt = &vgpu->gtt;  in intel_vgpu_init_gtt()  local
2468 	INIT_RADIX_TREE(&gtt->spt_tree, GFP_KERNEL);  in intel_vgpu_init_gtt()
2470 	INIT_LIST_HEAD(&gtt->ppgtt_mm_list_head);  in intel_vgpu_init_gtt()
2471 	INIT_LIST_HEAD(&gtt->oos_page_list_head);  in intel_vgpu_init_gtt()
2472 	INIT_LIST_HEAD(&gtt->post_shadow_list_head);  in intel_vgpu_init_gtt()
2474 	gtt->ggtt_mm = intel_vgpu_create_ggtt_mm(vgpu);  in intel_vgpu_init_gtt()
2475 	if (IS_ERR(gtt->ggtt_mm)) {  in intel_vgpu_init_gtt()
2477 		return PTR_ERR(gtt->ggtt_mm);  in intel_vgpu_init_gtt()
2482 	INIT_LIST_HEAD(&gtt->ggtt_mm->ggtt_mm.partial_pte_list);  in intel_vgpu_init_gtt()
2492 	list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_destroy_all_ppgtt_mm()
2497 	if (GEM_WARN_ON(!list_empty(&vgpu->gtt.ppgtt_mm_list_head)))  in intel_vgpu_destroy_all_ppgtt_mm()
2500 	if (GEM_WARN_ON(!radix_tree_empty(&vgpu->gtt.spt_tree))) {  in intel_vgpu_destroy_all_ppgtt_mm()
2511 				 &vgpu->gtt.ggtt_mm->ggtt_mm.partial_pte_list,  in intel_vgpu_destroy_ggtt_mm()
2517 	intel_vgpu_destroy_mm(vgpu->gtt.ggtt_mm);  in intel_vgpu_destroy_ggtt_mm()
2518 	vgpu->gtt.ggtt_mm = NULL;  in intel_vgpu_destroy_ggtt_mm()
2540 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in clean_spt_oos()  local
2544 	WARN(!list_empty(&gtt->oos_page_use_list_head),  in clean_spt_oos()
2547 	list_for_each_safe(pos, n, &gtt->oos_page_free_list_head) {  in clean_spt_oos()
2557 	struct intel_gvt_gtt *gtt = &gvt->gtt;  in setup_spt_oos()  local
2562 	INIT_LIST_HEAD(&gtt->oos_page_free_list_head);  in setup_spt_oos()
2563 	INIT_LIST_HEAD(&gtt->oos_page_use_list_head);  in setup_spt_oos()
2581 		list_add_tail(&oos_page->list, &gtt->oos_page_free_list_head);  in setup_spt_oos()
2608 	list_for_each(pos, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_find_ppgtt_mm()
2697 	gvt->gtt.pte_ops = &gen8_gtt_pte_ops;  in intel_gvt_init_gtt()
2698 	gvt->gtt.gma_ops = &gen8_gtt_gma_ops;  in intel_gvt_init_gtt()
2714 	gvt->gtt.scratch_page = virt_to_page(page);  in intel_gvt_init_gtt()
2715 	gvt->gtt.scratch_mfn = (unsigned long)(daddr >> I915_GTT_PAGE_SHIFT);  in intel_gvt_init_gtt()
2722 			__free_page(gvt->gtt.scratch_page);  in intel_gvt_init_gtt()
2726 	INIT_LIST_HEAD(&gvt->gtt.ppgtt_mm_lru_list_head);  in intel_gvt_init_gtt()
2727 	mutex_init(&gvt->gtt.ppgtt_mm_lock);  in intel_gvt_init_gtt()
2742 	dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_mfn <<  in intel_gvt_clean_gtt()
2747 	__free_page(gvt->gtt.scratch_page);  in intel_gvt_clean_gtt()
2765 	list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) {  in intel_vgpu_invalidate_ppgtt()
2768 			mutex_lock(&vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_invalidate_ppgtt()
2770 			mutex_unlock(&vgpu->gvt->gtt.ppgtt_mm_lock);  in intel_vgpu_invalidate_ppgtt()
2789 	const struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;  in intel_vgpu_reset_ggtt()
2795 	pte_ops->set_pfn(&entry, gvt->gtt.scratch_mfn);  in intel_vgpu_reset_ggtt()
2802 			ggtt_get_host_entry(vgpu->gtt.ggtt_mm, &old_entry, index);  in intel_vgpu_reset_ggtt()
2805 		ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++);  in intel_vgpu_reset_ggtt()
2812 			ggtt_get_host_entry(vgpu->gtt.ggtt_mm, &old_entry, index);  in intel_vgpu_reset_ggtt()
2815 		ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++);  in intel_vgpu_reset_ggtt()
2839 		mm = vgpu->gtt.ggtt_mm;  in intel_gvt_restore_ggtt()