15  * /fairy-tale-mode off
27 #include "intel-agp.h"
28 #include <drm/intel/intel-gtt.h>
52 	/* This should undo anything done in ->setup() save the unmapping
57 	 * For chipsets that need to support old ums (non-gem) code, this
77 	struct page *scratch_page;
92 #define INTEL_GTT_GEN	intel_private.driver->gen
93 #define IS_G33		intel_private.driver->is_g33
94 #define IS_PINEVIEW	intel_private.driver->is_pineview
95 #define IS_IRONLAKE	intel_private.driver->is_ironlake
96 #define HAS_PGTBL_EN	intel_private.driver->has_pgtbl_enable
99 static int intel_gtt_map_memory(struct page **pages,  in intel_gtt_map_memory()
111 	for_each_sg(st->sgl, sg, num_entries, i)  in intel_gtt_map_memory()
114 	if (!dma_map_sg(&intel_private.pcidev->dev, st->sgl, st->nents,  in intel_gtt_map_memory()
122 	return -ENOMEM;  in intel_gtt_map_memory()
128 	DBG("try unmapping %lu pages\n", (unsigned long)mem->page_count);  in intel_gtt_unmap_memory()
130 	dma_unmap_sg(&intel_private.pcidev->dev, sg_list, num_sg,  in intel_gtt_unmap_memory()
145 static struct page *i8xx_alloc_pages(void)  in i8xx_alloc_pages()
147 	struct page *page;  in i8xx_alloc_pages()  local
149 	page = alloc_pages(GFP_KERNEL | GFP_DMA32, 2);  in i8xx_alloc_pages()
150 	if (page == NULL)  in i8xx_alloc_pages()
153 	if (set_pages_uc(page, 4) < 0) {  in i8xx_alloc_pages()
154 		set_pages_wb(page, 4);  in i8xx_alloc_pages()
155 		__free_pages(page, 2);  in i8xx_alloc_pages()
158 	atomic_inc(&agp_bridge->current_memory_agp);  in i8xx_alloc_pages()
159 	return page;  in i8xx_alloc_pages()
162 static void i8xx_destroy_pages(struct page *page)  in i8xx_destroy_pages()  argument
164 	if (page == NULL)  in i8xx_destroy_pages()
167 	set_pages_wb(page, 4);  in i8xx_destroy_pages()
168 	__free_pages(page, 2);  in i8xx_destroy_pages()
169 	atomic_dec(&agp_bridge->current_memory_agp);  in i8xx_destroy_pages()
173 #define I810_GTT_ORDER 4
179 	/* i81x does not preallocate the gtt. It's always 64kb in size. */  in i810_setup()
182 		return -ENOMEM;  in i810_setup()
187 	intel_private.registers = ioremap(reg_addr, KB(64));  in i810_setup()
189 		return -ENOMEM;  in i810_setup()
198 		dev_info(&intel_private.pcidev->dev,  in i810_setup()
199 			 "detected 4MB dedicated video ram\n");  in i810_setup()
218 	if ((pg_start + mem->page_count)  in i810_insert_dcache_entries()
220 		return -EINVAL;  in i810_insert_dcache_entries()
222 	if (!mem->is_flushed)  in i810_insert_dcache_entries()
225 	for (i = pg_start; i < (pg_start + mem->page_count); i++) {  in i810_insert_dcache_entries()
227 		intel_private.driver->write_entry(addr,  in i810_insert_dcache_entries()
243 	struct page *page;  in alloc_agpphysmem_i8xx()  local
246 	case 1: page = agp_bridge->driver->agp_alloc_page(agp_bridge);  in alloc_agpphysmem_i8xx()
248 	case 4:  in alloc_agpphysmem_i8xx()
249 		/* kludge to get 4 physical pages for ARGB cursor */  in alloc_agpphysmem_i8xx()
250 		page = i8xx_alloc_pages();  in alloc_agpphysmem_i8xx()
256 	if (page == NULL)  in alloc_agpphysmem_i8xx()
263 	new->pages[0] = page;  in alloc_agpphysmem_i8xx()
264 	if (pg_count == 4) {  in alloc_agpphysmem_i8xx()
265 		/* kludge to get 4 physical pages for ARGB cursor */  in alloc_agpphysmem_i8xx()
266 		new->pages[1] = new->pages[0] + 1;  in alloc_agpphysmem_i8xx()
267 		new->pages[2] = new->pages[1] + 1;  in alloc_agpphysmem_i8xx()
268 		new->pages[3] = new->pages[2] + 1;  in alloc_agpphysmem_i8xx()
270 	new->page_count = pg_count;  in alloc_agpphysmem_i8xx()
271 	new->num_scratch_pages = pg_count;  in alloc_agpphysmem_i8xx()
272 	new->type = AGP_PHYS_MEMORY;  in alloc_agpphysmem_i8xx()
273 	new->physical = page_to_phys(new->pages[0]);  in alloc_agpphysmem_i8xx()
279 	agp_free_key(curr->key);  in intel_i810_free_by_type()
280 	if (curr->type == AGP_PHYS_MEMORY) {  in intel_i810_free_by_type()
281 		if (curr->page_count == 4)  in intel_i810_free_by_type()
282 			i8xx_destroy_pages(curr->pages[0]);  in intel_i810_free_by_type()
284 			agp_bridge->driver->agp_destroy_page(curr->pages[0],  in intel_i810_free_by_type()
286 			agp_bridge->driver->agp_destroy_page(curr->pages[0],  in intel_i810_free_by_type()
297 	struct page *page;  in intel_gtt_setup_scratch_page()  local
300 	page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);  in intel_gtt_setup_scratch_page()
301 	if (page == NULL)  in intel_gtt_setup_scratch_page()
302 		return -ENOMEM;  in intel_gtt_setup_scratch_page()
303 	set_pages_uc(page, 1);  in intel_gtt_setup_scratch_page()
306 		dma_addr = dma_map_page(&intel_private.pcidev->dev, page, 0,  in intel_gtt_setup_scratch_page()
308 		if (dma_mapping_error(&intel_private.pcidev->dev, dma_addr)) {  in intel_gtt_setup_scratch_page()
309 			__free_page(page);  in intel_gtt_setup_scratch_page()
310 			return -EINVAL;  in intel_gtt_setup_scratch_page()
315 		intel_private.scratch_page_dma = page_to_phys(page);  in intel_gtt_setup_scratch_page()
317 	intel_private.scratch_page = page;  in intel_gtt_setup_scratch_page()
344 	static const int ddt[4] = { 0, 16, 32, 64 };  in intel_gtt_stolen_size()
353 	if (intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82830_HB ||  in intel_gtt_stolen_size()
354 	    intel_private.bridge_dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {  in intel_gtt_stolen_size()
357 			stolen_size = KB(512);  in intel_gtt_stolen_size()
381 			stolen_size = MB(4);  in intel_gtt_stolen_size()
423 		dev_info(&intel_private.bridge_dev->dev, "detected %lluK %s memory\n",  in intel_gtt_stolen_size()
424 		       (u64)stolen_size / KB(1), local ? "local" : "stolen");  in intel_gtt_stolen_size()
426 		dev_info(&intel_private.bridge_dev->dev,  in intel_gtt_stolen_size()
427 		       "no pre-allocated video memory detected\n");  in intel_gtt_stolen_size()
479 		size = KB(128);  in i965_gtt_total_entries()
482 		size = KB(256);  in i965_gtt_total_entries()
485 		size = KB(512);  in i965_gtt_total_entries()
487 	/* GTT pagetable sizes bigger than 512KB are not possible on G33! */  in i965_gtt_total_entries()
489 		size = KB(1024);  in i965_gtt_total_entries()
492 		size = KB(2048);  in i965_gtt_total_entries()
495 		size = KB(1024 + 512);  in i965_gtt_total_entries()
498 		dev_info(&intel_private.pcidev->dev,  in i965_gtt_total_entries()
499 			 "unknown page table size, assuming 512KB\n");  in i965_gtt_total_entries()
500 		size = KB(512);  in i965_gtt_total_entries()
503 	return size/4;  in i965_gtt_total_entries()
508 	if (IS_G33 || INTEL_GTT_GEN == 4 || INTEL_GTT_GEN == 5)  in intel_gtt_total_entries()
555 		dma_unmap_page(&intel_private.pcidev->dev,  in intel_gtt_teardown_scratch_page()
563 	intel_private.driver->cleanup();  in intel_gtt_cleanup()
572  * unmapping anything from the GTT when VT-d is enabled.
576 	const unsigned short gpu_devid = intel_private.pcidev->device;  in needs_ilk_vtd_wa()
584 		device_iommu_mapped(&intel_private.pcidev->dev));  in needs_ilk_vtd_wa()
607 	ret = intel_private.driver->setup();  in intel_gtt_init()
622 	dev_info(&intel_private.bridge_dev->dev,  in intel_gtt_init()
624 			intel_private.gtt_total_entries * 4,  in intel_gtt_init()
625 			intel_private.gtt_mappable_entries * 4);  in intel_gtt_init()
627 	gtt_map_size = intel_private.gtt_total_entries * 4;  in intel_gtt_init()
637 		intel_private.driver->cleanup();  in intel_gtt_init()
639 		return -ENOMEM;  in intel_gtt_init()
668 	{64, 16384, 4},
684 			agp_bridge->current_size =  in intel_fake_agp_fetch_size()
704  * specs, it's 64 octwords, or 1KB.  So, to get those previous things in
705  * that buffer out, we just fill 1KB and clflush it out, on the assumption
760 			dev_err(&intel_private.pcidev->dev,  in intel_gmch_enable_gtt()
776 		dev_err(&intel_private.pcidev->dev,  in intel_gmch_enable_gtt()
795 	intel_private.registers = ioremap(reg_addr, KB(64));  in i830_setup()
797 		return -ENOMEM;  in i830_setup()
807 	agp_bridge->gatt_table_real = NULL;  in intel_fake_agp_create_gatt_table()
808 	agp_bridge->gatt_table = NULL;  in intel_fake_agp_create_gatt_table()
809 	agp_bridge->gatt_bus_addr = 0;  in intel_fake_agp_create_gatt_table()
822 		return -EIO;  in intel_fake_agp_configure()
825 	agp_bridge->gart_bus_addr = intel_private.gma_bus_addr;  in intel_fake_agp_configure()
848 	intel_private.driver->write_entry(addr, pg, flags);  in intel_gmch_gtt_insert_page()
850 	if (intel_private.driver->chipset_flush)  in intel_gmch_gtt_insert_page()
851 		intel_private.driver->chipset_flush();  in intel_gmch_gtt_insert_page()
866 	 * per-page addr for GTT */  in intel_gmch_gtt_insert_sg_entries()
867 	for_each_sg(st->sgl, sg, st->nents, i) {  in intel_gmch_gtt_insert_sg_entries()
871 			intel_private.driver->write_entry(addr, j, flags);  in intel_gmch_gtt_insert_sg_entries()
875 	readl(intel_private.gtt + j - 1);  in intel_gmch_gtt_insert_sg_entries()
876 	if (intel_private.driver->chipset_flush)  in intel_gmch_gtt_insert_sg_entries()
877 		intel_private.driver->chipset_flush();  in intel_gmch_gtt_insert_sg_entries()
884 					struct page **pages,  in intel_gmch_gtt_insert_pages()
891 		intel_private.driver->write_entry(addr,  in intel_gmch_gtt_insert_pages()
900 	int ret = -EINVAL;  in intel_fake_agp_insert_entries()
905 		intel_gmch_gtt_clear_range(start, end - start);  in intel_fake_agp_insert_entries()
912 	if (mem->page_count == 0)  in intel_fake_agp_insert_entries()
915 	if (pg_start + mem->page_count > intel_private.gtt_total_entries)  in intel_fake_agp_insert_entries()
918 	if (type != mem->type)  in intel_fake_agp_insert_entries()
921 	if (!intel_private.driver->check_flags(type))  in intel_fake_agp_insert_entries()
924 	if (!mem->is_flushed)  in intel_fake_agp_insert_entries()
930 		ret = intel_gtt_map_memory(mem->pages, mem->page_count, &st);  in intel_fake_agp_insert_entries()
935 		mem->sg_list = st.sgl;  in intel_fake_agp_insert_entries()
936 		mem->num_sg = st.nents;  in intel_fake_agp_insert_entries()
938 		intel_gmch_gtt_insert_pages(pg_start, mem->page_count, mem->pages,  in intel_fake_agp_insert_entries()
944 	mem->is_flushed = true;  in intel_fake_agp_insert_entries()
954 		intel_private.driver->write_entry(intel_private.scratch_page_dma,  in intel_gmch_gtt_clear_range()
965 	if (mem->page_count == 0)  in intel_fake_agp_remove_entries()
968 	intel_gmch_gtt_clear_range(pg_start, mem->page_count);  in intel_fake_agp_remove_entries()
971 		intel_gtt_unmap_memory(mem->sg_list, mem->num_sg);  in intel_fake_agp_remove_entries()
972 		mem->sg_list = NULL;  in intel_fake_agp_remove_entries()
973 		mem->num_sg = 0;  in intel_fake_agp_remove_entries()
992 		new->type = AGP_DCACHE_MEMORY;  in intel_fake_agp_alloc_by_type()
993 		new->page_count = pg_count;  in intel_fake_agp_alloc_by_type()
994 		new->num_scratch_pages = 0;  in intel_fake_agp_alloc_by_type()
1008 	ret = pci_bus_alloc_resource(intel_private.bridge_dev->bus, &intel_private.ifp_resource, PAGE_SIZE,  in intel_alloc_chipset_flush_resource()
1043 	pci_read_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4, &temp_hi);  in intel_i965_g33_setup_chipset_flush()
1051 		pci_write_config_dword(intel_private.bridge_dev, I965_IFPADDR + 4,  in intel_i965_g33_setup_chipset_flush()
1080 	intel_private.ifp_resource.name = "Intel Flush Page";  in intel_i9xx_setup_flush()
1084 	if (IS_G33 || INTEL_GTT_GEN >= 4) {  in intel_i9xx_setup_flush()
1093 		dev_err(&intel_private.pcidev->dev,  in intel_i9xx_setup_flush()
1094 			"can't ioremap flush page - no chipset flushing\n");  in intel_i9xx_setup_flush()
1132 	int size = KB(512);  in i9xx_setup()
1138 		return -ENOMEM;  in i9xx_setup()
1149 		intel_private.gtt_phys_addr = reg_addr + KB(512);  in i9xx_setup()
1233 	.gen = 4,
1263  * driver and gmch_driver must be non-null, and find_gmch will determine
1349 	if (gmch_device && PCI_FUNC(gmch_device->devfn) != 0) {  in find_gmch()
1368 			if (gpu_pdev->device ==  in intel_gmch_probe()
1391 		bridge->driver = &intel_fake_agp_driver;  in intel_gmch_probe()
1392 		bridge->dev_private_data = &intel_private;  in intel_gmch_probe()
1393 		bridge->dev = bridge_pdev;  in intel_gmch_probe()
1408 	dev_info(&bridge_pdev->dev, "Intel %s Chipset\n", intel_gtt_chipsets[i].name);  in intel_gmch_probe()
1411 		mask = intel_private.driver->dma_mask_size;  in intel_gmch_probe()
1412 		if (dma_set_mask(&intel_private.pcidev->dev, DMA_BIT_MASK(mask)))  in intel_gmch_probe()
1413 			dev_err(&intel_private.pcidev->dev,  in intel_gmch_probe()
1414 				"set gfx device dma mask %d-bit failed!\n",  in intel_gmch_probe()
1417 			dma_set_coherent_mask(&intel_private.pcidev->dev,  in intel_gmch_probe()
1443 	if (intel_private.driver->chipset_flush)  in intel_gmch_gtt_flush()
1444 		intel_private.driver->chipset_flush();  in intel_gmch_gtt_flush()
1450 	if (--intel_private.refcount)  in intel_gmch_remove()