50 	struct ttm_tt ttm;  member
182 				      struct ttm_tt *ttm,  in i915_ttm_tt_shmem_populate()  argument
187 	struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm);  in i915_ttm_tt_shmem_populate()
189 	const size_t size = (size_t)ttm->num_pages << PAGE_SHIFT;  in i915_ttm_tt_shmem_populate()
227 		ttm->pages[i++] = page;  in i915_ttm_tt_shmem_populate()
229 	if (ttm->page_flags & TTM_TT_FLAG_SWAPPED)  in i915_ttm_tt_shmem_populate()
230 		ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED;  in i915_ttm_tt_shmem_populate()
240 static void i915_ttm_tt_shmem_unpopulate(struct ttm_tt *ttm)  in i915_ttm_tt_shmem_unpopulate()  argument
242 	struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm);  in i915_ttm_tt_shmem_unpopulate()
243 	bool backup = ttm->page_flags & TTM_TT_FLAG_SWAPPED;  in i915_ttm_tt_shmem_unpopulate()
299 	ret = ttm_tt_init(&i915_tt->ttm, bo, page_flags, caching, ccs_pages);  in i915_ttm_tt_create()
308 	return &i915_tt->ttm;  in i915_ttm_tt_create()
316 				struct ttm_tt *ttm,  in i915_ttm_tt_populate()  argument
319 	struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm);  in i915_ttm_tt_populate()
322 		return i915_ttm_tt_shmem_populate(bdev, ttm, ctx);  in i915_ttm_tt_populate()
324 	return ttm_pool_alloc(&bdev->pool, ttm, ctx);  in i915_ttm_tt_populate()
327 static void i915_ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm)  in i915_ttm_tt_unpopulate()  argument
329 	struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm);  in i915_ttm_tt_unpopulate()
336 		i915_ttm_tt_shmem_unpopulate(ttm);  in i915_ttm_tt_unpopulate()
339 		ttm_pool_free(&bdev->pool, ttm);  in i915_ttm_tt_unpopulate()
343 static void i915_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *ttm)  in i915_ttm_tt_destroy()  argument
345 	struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm);  in i915_ttm_tt_destroy()
350 	ttm_tt_fini(ttm);  in i915_ttm_tt_destroy()
367 	if (bo->ttm && bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL)  in i915_ttm_eviction_valuable()
395 	if (!obj->ttm.cached_io_rsgt)  in i915_ttm_free_cached_io_rsgt()
399 	radix_tree_for_each_slot(slot, &obj->ttm.get_io_page.radix, &iter, 0)  in i915_ttm_free_cached_io_rsgt()
400 		radix_tree_delete(&obj->ttm.get_io_page.radix, iter.index);  in i915_ttm_free_cached_io_rsgt()
403 	i915_refct_sgt_put(obj->ttm.cached_io_rsgt);  in i915_ttm_free_cached_io_rsgt()
404 	obj->ttm.cached_io_rsgt = NULL;  in i915_ttm_free_cached_io_rsgt()
420 		container_of(bo->ttm, typeof(*i915_tt), ttm);  in i915_ttm_purge()
435 	if (bo->ttm && i915_tt->filp) {  in i915_ttm_purge()
460 		container_of(bo->ttm, typeof(*i915_tt), ttm);  in i915_ttm_shrink()
468 	if (!bo->ttm || i915_ttm_cpu_maps_iomem(bo->resource))  in i915_ttm_shrink()
487 	if (bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED)  in i915_ttm_shrink()
490 	bo->ttm->page_flags |= TTM_TT_FLAG_SWAPPED;  in i915_ttm_shrink()
493 		bo->ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED;  in i915_ttm_shrink()
513 	if ((bo->resource || bo->ttm) && !i915_ttm_is_ghost_object(bo)) {  in i915_ttm_delete_mem_notify()
519 static struct i915_refct_sgt *i915_ttm_tt_get_st(struct ttm_tt *ttm)  in i915_ttm_tt_get_st()  argument
521 	struct i915_ttm_tt *i915_tt = container_of(ttm, typeof(*i915_tt), ttm);  in i915_ttm_tt_get_st()
530 			ttm->pages, ttm->num_pages,  in i915_ttm_tt_get_st()
531 			0, (unsigned long)ttm->num_pages << PAGE_SHIFT,  in i915_ttm_tt_get_st()
569 		return i915_ttm_tt_get_st(bo->ttm);  in i915_ttm_resource_get_st()
581 		if (!obj->ttm.cached_io_rsgt) {  in i915_ttm_resource_get_st()
590 			obj->ttm.cached_io_rsgt = rsgt;  in i915_ttm_resource_get_st()
592 		return i915_refct_sgt_get(obj->ttm.cached_io_rsgt);  in i915_ttm_resource_get_st()
630 	GEM_WARN_ON(obj->ttm.cached_io_rsgt);  in i915_ttm_swap_notify()
692 	GEM_WARN_ON(bo->ttm);  in i915_ttm_io_mem_pfn()
695 	sg = i915_gem_object_page_iter_get_sg(obj, &obj->ttm.get_io_page, page_offset, &ofs);  in i915_ttm_io_mem_pfn()
810 	if (bo->ttm && !ttm_tt_is_populated(bo->ttm)) {  in __i915_ttm_get_pages()
811 		ret = ttm_tt_populate(bo->bdev, bo->ttm, &ctx);  in __i915_ttm_get_pages()
831 	GEM_BUG_ON(bo->ttm && ((obj->base.size >> PAGE_SHIFT) < bo->ttm->num_pages));  in __i915_ttm_get_pages()
927 		container_of(bo->ttm, typeof(*i915_tt), ttm);  in i915_ttm_adjust_lru()
929 		bo->ttm && i915_tt->filp && ttm_tt_is_populated(bo->ttm);  in i915_ttm_adjust_lru()
1030 	GEM_BUG_ON(!obj->ttm.created);  in i915_ttm_delayed_free()
1072 		GEM_BUG_ON(!bo->ttm || !(bo->ttm->page_flags & TTM_TT_FLAG_SWAPPED));  in vm_fault_ttm()
1246 	mutex_destroy(&obj->ttm.get_io_page.lock);  in i915_ttm_bo_destroy()
1248 	if (obj->ttm.created) {  in i915_ttm_bo_destroy()
1306 	INIT_RADIX_TREE(&obj->ttm.get_io_page.radix, GFP_KERNEL | __GFP_NOWARN);  in __i915_gem_ttm_object_init()
1307 	mutex_init(&obj->ttm.get_io_page.lock);  in __i915_gem_ttm_object_init()
1348 	obj->ttm.created = true;  in __i915_gem_ttm_object_init()