2  * SPDX-License-Identifier: MIT
4  * Copyright © 2008-2012 Intel Corporation
46 	if (!drm_mm_initialized(&i915->mm.stolen))  in i915_gem_stolen_insert_node_in_range()
47 		return -ENODEV;  in i915_gem_stolen_insert_node_in_range()
53 	mutex_lock(&i915->mm.stolen_lock);  in i915_gem_stolen_insert_node_in_range()
54 	ret = drm_mm_insert_node_in_range(&i915->mm.stolen, node,  in i915_gem_stolen_insert_node_in_range()
57 	mutex_unlock(&i915->mm.stolen_lock);  in i915_gem_stolen_insert_node_in_range()
75 	mutex_lock(&i915->mm.stolen_lock);  in i915_gem_stolen_remove_node()
77 	mutex_unlock(&i915->mm.stolen_lock);  in i915_gem_stolen_remove_node()
82 	return (dsm->start != 0 || HAS_LMEMBAR_SMEM_STOLEN(i915)) && dsm->end > dsm->start;  in valid_stolen_size()
88 	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;  in adjust_stolen()
89 	struct intel_uncore *uncore = ggtt->vm.gt->uncore;  in adjust_stolen()
92 		return -EINVAL;  in adjust_stolen()
128 			drm_dbg(&i915->drm,  in adjust_stolen()
131 			drm_dbg(&i915->drm, "Stolen memory adjusted to %pR\n",  in adjust_stolen()
137 		return -EINVAL;  in adjust_stolen()
163 	r = devm_request_mem_region(i915->drm.dev, dsm->start,  in request_smem_stolen()
172 		 * PCI bus, but have an off-by-one error. Hence retry the  in request_smem_stolen()
174 		 * There's also BIOS with off-by-one on the other end.  in request_smem_stolen()
176 		r = devm_request_mem_region(i915->drm.dev, dsm->start + 1,  in request_smem_stolen()
177 					    resource_size(dsm) - 2,  in request_smem_stolen()
184 			drm_err(&i915->drm,  in request_smem_stolen()
188 			return -EBUSY;  in request_smem_stolen()
197 	if (!drm_mm_initialized(&i915->mm.stolen))  in i915_gem_cleanup_stolen()
200 	drm_mm_takedown(&i915->mm.stolen);  in i915_gem_cleanup_stolen()
205 				    resource_size_t *base,  in g4x_get_stolen_reserved()  argument
212 	resource_size_t stolen_top = i915->dsm.stolen.end + 1;  in g4x_get_stolen_reserved()
214 	drm_dbg(&i915->drm, "%s_STOLEN_RESERVED = %08x\n",  in g4x_get_stolen_reserved()
224 	drm_WARN(&i915->drm, GRAPHICS_VER(i915) == 5,  in g4x_get_stolen_reserved()
231 	*base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16;  in g4x_get_stolen_reserved()
232 	drm_WARN_ON(&i915->drm,  in g4x_get_stolen_reserved()
233 		    (reg_val & G4X_STOLEN_RESERVED_ADDR1_MASK) < *base);  in g4x_get_stolen_reserved()
235 	*size = stolen_top - *base;  in g4x_get_stolen_reserved()
240 				     resource_size_t *base,  in gen6_get_stolen_reserved()  argument
245 	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);  in gen6_get_stolen_reserved()
250 	*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;  in gen6_get_stolen_reserved()
273 				    resource_size_t *base,  in vlv_get_stolen_reserved()  argument
277 	resource_size_t stolen_top = i915->dsm.stolen.end + 1;  in vlv_get_stolen_reserved()
279 	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);  in vlv_get_stolen_reserved()
295 	 * reserved location as (top - size).  in vlv_get_stolen_reserved()
297 	*base = stolen_top - *size;  in vlv_get_stolen_reserved()
302 				     resource_size_t *base,  in gen7_get_stolen_reserved()  argument
307 	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);  in gen7_get_stolen_reserved()
312 	*base = reg_val & GEN7_STOLEN_RESERVED_ADDR_MASK;  in gen7_get_stolen_reserved()
329 				    resource_size_t *base,  in chv_get_stolen_reserved()  argument
334 	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);  in chv_get_stolen_reserved()
339 	*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;  in chv_get_stolen_reserved()
362 				    resource_size_t *base,  in bdw_get_stolen_reserved()  argument
366 	resource_size_t stolen_top = i915->dsm.stolen.end + 1;  in bdw_get_stolen_reserved()
368 	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);  in bdw_get_stolen_reserved()
376 	*base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;  in bdw_get_stolen_reserved()
377 	*size = stolen_top - *base;  in bdw_get_stolen_reserved()
382 				    resource_size_t *base,  in icl_get_stolen_reserved()  argument
387 	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = 0x%016llx\n", reg_val);  in icl_get_stolen_reserved()
402 		if (gscpsmi_base >= i915->dsm.stolen.start &&  in icl_get_stolen_reserved()
403 		    gscpsmi_base < i915->dsm.stolen.end) {  in icl_get_stolen_reserved()
404 			*base = gscpsmi_base;  in icl_get_stolen_reserved()
405 			*size = i915->dsm.stolen.end - gscpsmi_base;  in icl_get_stolen_reserved()
429 		/* the base is initialized to stolen top so subtract size to get base */  in icl_get_stolen_reserved()
430 		*base -= *size;  in icl_get_stolen_reserved()
432 		*base = reg_val & GEN11_STOLEN_RESERVED_ADDR_MASK;  in icl_get_stolen_reserved()
436  * Initialize i915->dsm.reserved to contain the reserved space within the Data
447 	struct intel_uncore *uncore = &i915->uncore;  in init_reserved_stolen()
452 	stolen_top = i915->dsm.stolen.end + 1;  in init_reserved_stolen()
486 		drm_err(&i915->drm,  in init_reserved_stolen()
489 		ret = -EINVAL;  in init_reserved_stolen()
493 	i915->dsm.reserved = DEFINE_RES_MEM(reserved_base, reserved_size);  in init_reserved_stolen()
495 	if (!resource_contains(&i915->dsm.stolen, &i915->dsm.reserved)) {  in init_reserved_stolen()
496 		drm_err(&i915->drm,  in init_reserved_stolen()
498 			&i915->dsm.reserved, &i915->dsm.stolen);  in init_reserved_stolen()
499 		ret = -EINVAL;  in init_reserved_stolen()
506 	i915->dsm.reserved = DEFINE_RES_MEM(reserved_base, 0);  in init_reserved_stolen()
511 static int i915_gem_init_stolen(struct intel_memory_region *mem)  in i915_gem_init_stolen()  argument
513 	struct drm_i915_private *i915 = mem->i915;  in i915_gem_init_stolen()
515 	mutex_init(&i915->mm.stolen_lock);  in i915_gem_init_stolen()
518 		drm_notice(&i915->drm,  in i915_gem_init_stolen()
520 			   "iGVT-g active");  in i915_gem_init_stolen()
521 		return -ENOSPC;  in i915_gem_init_stolen()
525 		drm_notice(&i915->drm,  in i915_gem_init_stolen()
528 		return -ENOSPC;  in i915_gem_init_stolen()
531 	if (adjust_stolen(i915, &mem->region))  in i915_gem_init_stolen()
532 		return -ENOSPC;  in i915_gem_init_stolen()
534 	if (request_smem_stolen(i915, &mem->region))  in i915_gem_init_stolen()
535 		return -ENOSPC;  in i915_gem_init_stolen()
537 	i915->dsm.stolen = mem->region;  in i915_gem_init_stolen()
540 		return -ENOSPC;  in i915_gem_init_stolen()
543 	mem->region.end = i915->dsm.reserved.start - 1;  in i915_gem_init_stolen()
544 	mem->io = DEFINE_RES_MEM(mem->io.start,  in i915_gem_init_stolen()
545 				 min(resource_size(&mem->io),  in i915_gem_init_stolen()
546 				     resource_size(&mem->region)));  in i915_gem_init_stolen()
548 	i915->dsm.usable_size = resource_size(&mem->region);  in i915_gem_init_stolen()
550 	drm_dbg(&i915->drm,  in i915_gem_init_stolen()
552 		(u64)resource_size(&i915->dsm.stolen) >> 10,  in i915_gem_init_stolen()
553 		(u64)i915->dsm.usable_size >> 10);  in i915_gem_init_stolen()
555 	if (i915->dsm.usable_size == 0)  in i915_gem_init_stolen()
556 		return -ENOSPC;  in i915_gem_init_stolen()
559 	drm_mm_init(&i915->mm.stolen, 0, i915->dsm.usable_size);  in i915_gem_init_stolen()
567 		i915->dsm.usable_size = 0;  in i915_gem_init_stolen()
577 	if (!drm_mm_node_allocated(&ggtt->error_capture))  in dbg_poison()
580 	if (ggtt->vm.bind_async_flags & I915_VMA_GLOBAL_BIND)  in dbg_poison()
585 	mutex_lock(&ggtt->error_mutex);  in dbg_poison()
589 		ggtt->vm.insert_page(&ggtt->vm, addr,  in dbg_poison()
590 				     ggtt->error_capture.start,  in dbg_poison()
591 				     i915_gem_get_pat_index(ggtt->vm.i915,  in dbg_poison()
596 		s = io_mapping_map_wc(&ggtt->iomap,  in dbg_poison()
597 				      ggtt->error_capture.start,  in dbg_poison()
603 		size -= PAGE_SIZE;  in dbg_poison()
606 	ggtt->vm.clear_range(&ggtt->vm, ggtt->error_capture.start, PAGE_SIZE);  in dbg_poison()
607 	mutex_unlock(&ggtt->error_mutex);  in dbg_poison()
619 	GEM_BUG_ON(range_overflows(offset, size, resource_size(&i915->dsm.stolen)));  in i915_pages_create_for_stolen()
628 		return ERR_PTR(-ENOMEM);  in i915_pages_create_for_stolen()
632 		return ERR_PTR(-ENOMEM);  in i915_pages_create_for_stolen()
635 	sg = st->sgl;  in i915_pages_create_for_stolen()
636 	sg->offset = 0;  in i915_pages_create_for_stolen()
637 	sg->length = size;  in i915_pages_create_for_stolen()
639 	sg_dma_address(sg) = (dma_addr_t)i915->dsm.stolen.start + offset;  in i915_pages_create_for_stolen()
647 	struct drm_i915_private *i915 = to_i915(obj->base.dev);  in i915_gem_object_get_pages_stolen()
649 		i915_pages_create_for_stolen(obj->base.dev,  in i915_gem_object_get_pages_stolen()
650 					     obj->stolen->start,  in i915_gem_object_get_pages_stolen()
651 					     obj->stolen->size);  in i915_gem_object_get_pages_stolen()
655 	dbg_poison(to_gt(i915)->ggtt,  in i915_gem_object_get_pages_stolen()
656 		   sg_dma_address(pages->sgl),  in i915_gem_object_get_pages_stolen()
657 		   sg_dma_len(pages->sgl),  in i915_gem_object_get_pages_stolen()
668 	struct drm_i915_private *i915 = to_i915(obj->base.dev);  in i915_gem_object_put_pages_stolen()
671 	dbg_poison(to_gt(i915)->ggtt,  in i915_gem_object_put_pages_stolen()
672 		   sg_dma_address(pages->sgl),  in i915_gem_object_put_pages_stolen()
673 		   sg_dma_len(pages->sgl),  in i915_gem_object_put_pages_stolen()
683 	struct drm_i915_private *i915 = to_i915(obj->base.dev);  in i915_gem_object_release_stolen()
684 	struct drm_mm_node *stolen = fetch_and_zero(&obj->stolen);  in i915_gem_object_release_stolen()
700 static int __i915_gem_object_create_stolen(struct intel_memory_region *mem,  in __i915_gem_object_create_stolen()  argument
715 	drm_gem_private_object_init(&mem->i915->drm, &obj->base, stolen->size);  in __i915_gem_object_create_stolen()
718 	obj->stolen = stolen;  in __i915_gem_object_create_stolen()
719 	obj->read_domains = I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT;  in __i915_gem_object_create_stolen()
720 	cache_level = HAS_LLC(mem->i915) ? I915_CACHE_LLC : I915_CACHE_NONE;  in __i915_gem_object_create_stolen()
724 		return -EBUSY;  in __i915_gem_object_create_stolen()
726 	i915_gem_object_init_memory_region(obj, mem);  in __i915_gem_object_create_stolen()
736 static int _i915_gem_object_stolen_init(struct intel_memory_region *mem,  in _i915_gem_object_stolen_init()  argument
743 	struct drm_i915_private *i915 = mem->i915;  in _i915_gem_object_stolen_init()
747 	if (!drm_mm_initialized(&i915->mm.stolen))  in _i915_gem_object_stolen_init()
748 		return -ENODEV;  in _i915_gem_object_stolen_init()
751 		return -EINVAL;  in _i915_gem_object_stolen_init()
757 	if (mem->type == INTEL_MEMORY_STOLEN_LOCAL && !resource_size(&mem->io) &&  in _i915_gem_object_stolen_init()
759 		return -ENOSPC;  in _i915_gem_object_stolen_init()
763 		return -ENOMEM;  in _i915_gem_object_stolen_init()
766 		drm_dbg(&i915->drm,  in _i915_gem_object_stolen_init()
770 		stolen->start = offset;  in _i915_gem_object_stolen_init()
771 		stolen->size = size;  in _i915_gem_object_stolen_init()
772 		mutex_lock(&i915->mm.stolen_lock);  in _i915_gem_object_stolen_init()
773 		ret = drm_mm_reserve_node(&i915->mm.stolen, stolen);  in _i915_gem_object_stolen_init()
774 		mutex_unlock(&i915->mm.stolen_lock);  in _i915_gem_object_stolen_init()
777 						  mem->min_page_size);  in _i915_gem_object_stolen_init()
782 	ret = __i915_gem_object_create_stolen(mem, obj, stolen);  in _i915_gem_object_stolen_init()
799 	return i915_gem_object_create_region(i915->mm.stolen_region, size, 0, 0);  in i915_gem_object_create_stolen()
802 static int init_stolen_smem(struct intel_memory_region *mem)  in init_stolen_smem()  argument
810 	err = i915_gem_init_stolen(mem);  in init_stolen_smem()
812 		drm_dbg(&mem->i915->drm, "Skip stolen region: failed to setup\n");  in init_stolen_smem()
817 static int release_stolen_smem(struct intel_memory_region *mem)  in release_stolen_smem()  argument
819 	i915_gem_cleanup_stolen(mem->i915);  in release_stolen_smem()
829 static int init_stolen_lmem(struct intel_memory_region *mem)  in init_stolen_lmem()  argument
833 	if (GEM_WARN_ON(resource_size(&mem->region) == 0))  in init_stolen_lmem()
836 	err = i915_gem_init_stolen(mem);  in init_stolen_lmem()
838 		drm_dbg(&mem->i915->drm, "Skip stolen region: failed to setup\n");  in init_stolen_lmem()
842 	if (resource_size(&mem->io) &&  in init_stolen_lmem()
843 	    !io_mapping_init_wc(&mem->iomap, mem->io.start, resource_size(&mem->io)))  in init_stolen_lmem()
849 	i915_gem_cleanup_stolen(mem->i915);  in init_stolen_lmem()
853 static int release_stolen_lmem(struct intel_memory_region *mem)  in release_stolen_lmem()  argument
855 	if (resource_size(&mem->io))  in release_stolen_lmem()
856 		io_mapping_fini(&mem->iomap);  in release_stolen_lmem()
857 	i915_gem_cleanup_stolen(mem->i915);  in release_stolen_lmem()
875 		return -EIO;  in mtl_get_gms_size()
877 	/* return valid GMS value, -EIO if invalid */  in mtl_get_gms_size()
883 		return (gms - 0xf0 + 1) * 4;  in mtl_get_gms_size()
886 		return -EIO;  in mtl_get_gms_size()
894 	struct intel_uncore *uncore = &i915->uncore;  in i915_gem_stolen_lmem_setup()
895 	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);  in i915_gem_stolen_lmem_setup()
897 	struct intel_memory_region *mem;  in i915_gem_stolen_lmem_setup()  local
903 		return ERR_PTR(-ENODEV);  in i915_gem_stolen_lmem_setup()
906 		return ERR_PTR(-ENXIO);  in i915_gem_stolen_lmem_setup()
927 			drm_err(&i915->drm, "invalid MTL GGC register setting\n");  in i915_gem_stolen_lmem_setup()
937 		/* Use DSM base address instead for stolen memory */  in i915_gem_stolen_lmem_setup()
940 			drm_dbg(&i915->drm,  in i915_gem_stolen_lmem_setup()
945 		dsm_size = ALIGN_DOWN(lmem_size - dsm_base, SZ_1M);  in i915_gem_stolen_lmem_setup()
949 		drm_dbg(&i915->drm, "Using direct DSM access\n");  in i915_gem_stolen_lmem_setup()
963 	mem = intel_memory_region_create(i915, dsm_base, dsm_size,  in i915_gem_stolen_lmem_setup()
968 	if (IS_ERR(mem))  in i915_gem_stolen_lmem_setup()
969 		return mem;  in i915_gem_stolen_lmem_setup()
971 	intel_memory_region_set_name(mem, "stolen-local");  in i915_gem_stolen_lmem_setup()
973 	mem->private = true;  in i915_gem_stolen_lmem_setup()
975 	return mem;  in i915_gem_stolen_lmem_setup()
982 	struct intel_memory_region *mem;  in i915_gem_stolen_smem_setup()  local
984 	mem = intel_memory_region_create(i915,  in i915_gem_stolen_smem_setup()
989 	if (IS_ERR(mem))  in i915_gem_stolen_smem_setup()
990 		return mem;  in i915_gem_stolen_smem_setup()
992 	intel_memory_region_set_name(mem, "stolen-system");  in i915_gem_stolen_smem_setup()
994 	mem->private = true;  in i915_gem_stolen_smem_setup()
996 	return mem;  in i915_gem_stolen_smem_setup()
1001 	return obj->ops == &i915_gem_object_stolen_ops;  in i915_gem_object_is_stolen()
1006 	return drm_mm_initialized(&i915->mm.stolen);  in i915_gem_stolen_initialized()
1011 	return i915->dsm.stolen.start;  in i915_gem_stolen_area_address()
1016 	return resource_size(&i915->dsm.stolen);  in i915_gem_stolen_area_size()
1022 	return i915->dsm.stolen.start + i915_gem_stolen_node_offset(node);  in i915_gem_stolen_node_address()
1032 	return node->start;  in i915_gem_stolen_node_offset()
1037 	return node->size;  in i915_gem_stolen_node_size()