1 // SPDX-License-Identifier: GPL-2.0-only
2 /**************************************************************************
3 * Copyright (c) 2007, Intel Corporation.
4 *
5 **************************************************************************/
6
7 #include <linux/highmem.h>
8 #include <linux/vmalloc.h>
9
10 #include "mmu.h"
11 #include "psb_drv.h"
12 #include "psb_reg.h"
13
14 /*
15 * Code for the SGX MMU:
16 */
17
18 /*
19 * clflush on one processor only:
20 * clflush should apparently flush the cache line on all processors in an
21 * SMP system.
22 */
23
24 /*
25 * kmap atomic:
26 * The usage of the slots must be completely encapsulated within a spinlock, and
27 * no other functions that may be using the locks for other purposed may be
28 * called from within the locked region.
29 * Since the slots are per processor, this will guarantee that we are the only
30 * user.
31 */
32
33 /*
34 * TODO: Inserting ptes from an interrupt handler:
35 * This may be desirable for some SGX functionality where the GPU can fault in
36 * needed pages. For that, we need to make an atomic insert_pages function, that
37 * may fail.
38 * If it fails, the caller need to insert the page using a workqueue function,
39 * but on average it should be fast.
40 */
41
psb_mmu_pt_index(uint32_t offset)42 static inline uint32_t psb_mmu_pt_index(uint32_t offset)
43 {
44 return (offset >> PSB_PTE_SHIFT) & 0x3FF;
45 }
46
psb_mmu_pd_index(uint32_t offset)47 static inline uint32_t psb_mmu_pd_index(uint32_t offset)
48 {
49 return offset >> PSB_PDE_SHIFT;
50 }
51
psb_clflush(void * addr)52 static inline void psb_clflush(void *addr)
53 {
54 __asm__ __volatile__("clflush (%0)\n" : : "r"(addr) : "memory");
55 }
56
psb_mmu_clflush(struct psb_mmu_driver * driver,void * addr)57 static inline void psb_mmu_clflush(struct psb_mmu_driver *driver, void *addr)
58 {
59 if (!driver->has_clflush)
60 return;
61
62 mb();
63 psb_clflush(addr);
64 mb();
65 }
66
psb_mmu_flush_pd_locked(struct psb_mmu_driver * driver,int force)67 static void psb_mmu_flush_pd_locked(struct psb_mmu_driver *driver, int force)
68 {
69 struct drm_device *dev = driver->dev;
70 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
71
72 if (atomic_read(&driver->needs_tlbflush) || force) {
73 uint32_t val = PSB_RSGX32(PSB_CR_BIF_CTRL);
74 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
75
76 /* Make sure data cache is turned off before enabling it */
77 wmb();
78 PSB_WSGX32(val & ~_PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
79 (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
80 if (driver->msvdx_mmu_invaldc)
81 atomic_set(driver->msvdx_mmu_invaldc, 1);
82 }
83 atomic_set(&driver->needs_tlbflush, 0);
84 }
85
86 #if 0
87 static void psb_mmu_flush_pd(struct psb_mmu_driver *driver, int force)
88 {
89 down_write(&driver->sem);
90 psb_mmu_flush_pd_locked(driver, force);
91 up_write(&driver->sem);
92 }
93 #endif
94
psb_mmu_flush(struct psb_mmu_driver * driver)95 void psb_mmu_flush(struct psb_mmu_driver *driver)
96 {
97 struct drm_device *dev = driver->dev;
98 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
99 uint32_t val;
100
101 down_write(&driver->sem);
102 val = PSB_RSGX32(PSB_CR_BIF_CTRL);
103 if (atomic_read(&driver->needs_tlbflush))
104 PSB_WSGX32(val | _PSB_CB_CTRL_INVALDC, PSB_CR_BIF_CTRL);
105 else
106 PSB_WSGX32(val | _PSB_CB_CTRL_FLUSH, PSB_CR_BIF_CTRL);
107
108 /* Make sure data cache is turned off and MMU is flushed before
109 restoring bank interface control register */
110 wmb();
111 PSB_WSGX32(val & ~(_PSB_CB_CTRL_FLUSH | _PSB_CB_CTRL_INVALDC),
112 PSB_CR_BIF_CTRL);
113 (void)PSB_RSGX32(PSB_CR_BIF_CTRL);
114
115 atomic_set(&driver->needs_tlbflush, 0);
116 if (driver->msvdx_mmu_invaldc)
117 atomic_set(driver->msvdx_mmu_invaldc, 1);
118 up_write(&driver->sem);
119 }
120
psb_mmu_set_pd_context(struct psb_mmu_pd * pd,int hw_context)121 void psb_mmu_set_pd_context(struct psb_mmu_pd *pd, int hw_context)
122 {
123 struct drm_device *dev = pd->driver->dev;
124 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
125 uint32_t offset = (hw_context == 0) ? PSB_CR_BIF_DIR_LIST_BASE0 :
126 PSB_CR_BIF_DIR_LIST_BASE1 + hw_context * 4;
127
128 down_write(&pd->driver->sem);
129 PSB_WSGX32(page_to_pfn(pd->p) << PAGE_SHIFT, offset);
130 wmb();
131 psb_mmu_flush_pd_locked(pd->driver, 1);
132 pd->hw_context = hw_context;
133 up_write(&pd->driver->sem);
134
135 }
136
psb_pd_addr_end(unsigned long addr,unsigned long end)137 static inline unsigned long psb_pd_addr_end(unsigned long addr,
138 unsigned long end)
139 {
140 addr = (addr + PSB_PDE_MASK + 1) & ~PSB_PDE_MASK;
141 return (addr < end) ? addr : end;
142 }
143
psb_mmu_mask_pte(uint32_t pfn,int type)144 static inline uint32_t psb_mmu_mask_pte(uint32_t pfn, int type)
145 {
146 uint32_t mask = PSB_PTE_VALID;
147
148 if (type & PSB_MMU_CACHED_MEMORY)
149 mask |= PSB_PTE_CACHED;
150 if (type & PSB_MMU_RO_MEMORY)
151 mask |= PSB_PTE_RO;
152 if (type & PSB_MMU_WO_MEMORY)
153 mask |= PSB_PTE_WO;
154
155 return (pfn << PAGE_SHIFT) | mask;
156 }
157
psb_mmu_alloc_pd(struct psb_mmu_driver * driver,int trap_pagefaults,int invalid_type)158 struct psb_mmu_pd *psb_mmu_alloc_pd(struct psb_mmu_driver *driver,
159 int trap_pagefaults, int invalid_type)
160 {
161 struct psb_mmu_pd *pd = kmalloc(sizeof(*pd), GFP_KERNEL);
162 uint32_t *v;
163 int i;
164
165 if (!pd)
166 return NULL;
167
168 pd->p = alloc_page(GFP_DMA32);
169 if (!pd->p)
170 goto out_err1;
171 pd->dummy_pt = alloc_page(GFP_DMA32);
172 if (!pd->dummy_pt)
173 goto out_err2;
174 pd->dummy_page = alloc_page(GFP_DMA32);
175 if (!pd->dummy_page)
176 goto out_err3;
177
178 if (!trap_pagefaults) {
179 pd->invalid_pde = psb_mmu_mask_pte(page_to_pfn(pd->dummy_pt),
180 invalid_type);
181 pd->invalid_pte = psb_mmu_mask_pte(page_to_pfn(pd->dummy_page),
182 invalid_type);
183 } else {
184 pd->invalid_pde = 0;
185 pd->invalid_pte = 0;
186 }
187
188 v = kmap_local_page(pd->dummy_pt);
189 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
190 v[i] = pd->invalid_pte;
191
192 kunmap_local(v);
193
194 v = kmap_local_page(pd->p);
195 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
196 v[i] = pd->invalid_pde;
197
198 kunmap_local(v);
199
200 clear_page(kmap(pd->dummy_page));
201 kunmap(pd->dummy_page);
202
203 pd->tables = vmalloc_user(sizeof(struct psb_mmu_pt *) * 1024);
204 if (!pd->tables)
205 goto out_err4;
206
207 pd->hw_context = -1;
208 pd->pd_mask = PSB_PTE_VALID;
209 pd->driver = driver;
210
211 return pd;
212
213 out_err4:
214 __free_page(pd->dummy_page);
215 out_err3:
216 __free_page(pd->dummy_pt);
217 out_err2:
218 __free_page(pd->p);
219 out_err1:
220 kfree(pd);
221 return NULL;
222 }
223
psb_mmu_free_pt(struct psb_mmu_pt * pt)224 static void psb_mmu_free_pt(struct psb_mmu_pt *pt)
225 {
226 __free_page(pt->p);
227 kfree(pt);
228 }
229
psb_mmu_free_pagedir(struct psb_mmu_pd * pd)230 void psb_mmu_free_pagedir(struct psb_mmu_pd *pd)
231 {
232 struct psb_mmu_driver *driver = pd->driver;
233 struct drm_device *dev = driver->dev;
234 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
235 struct psb_mmu_pt *pt;
236 int i;
237
238 down_write(&driver->sem);
239 if (pd->hw_context != -1) {
240 PSB_WSGX32(0, PSB_CR_BIF_DIR_LIST_BASE0 + pd->hw_context * 4);
241 psb_mmu_flush_pd_locked(driver, 1);
242 }
243
244 /* Should take the spinlock here, but we don't need to do that
245 since we have the semaphore in write mode. */
246
247 for (i = 0; i < 1024; ++i) {
248 pt = pd->tables[i];
249 if (pt)
250 psb_mmu_free_pt(pt);
251 }
252
253 vfree(pd->tables);
254 __free_page(pd->dummy_page);
255 __free_page(pd->dummy_pt);
256 __free_page(pd->p);
257 kfree(pd);
258 up_write(&driver->sem);
259 }
260
psb_mmu_alloc_pt(struct psb_mmu_pd * pd)261 static struct psb_mmu_pt *psb_mmu_alloc_pt(struct psb_mmu_pd *pd)
262 {
263 struct psb_mmu_pt *pt = kmalloc(sizeof(*pt), GFP_KERNEL);
264 void *v;
265 uint32_t clflush_add = pd->driver->clflush_add >> PAGE_SHIFT;
266 uint32_t clflush_count = PAGE_SIZE / clflush_add;
267 spinlock_t *lock = &pd->driver->lock;
268 uint8_t *clf;
269 uint32_t *ptes;
270 int i;
271
272 if (!pt)
273 return NULL;
274
275 pt->p = alloc_page(GFP_DMA32);
276 if (!pt->p) {
277 kfree(pt);
278 return NULL;
279 }
280
281 spin_lock(lock);
282
283 v = kmap_atomic(pt->p);
284 clf = (uint8_t *) v;
285 ptes = (uint32_t *) v;
286 for (i = 0; i < (PAGE_SIZE / sizeof(uint32_t)); ++i)
287 *ptes++ = pd->invalid_pte;
288
289 if (pd->driver->has_clflush && pd->hw_context != -1) {
290 mb();
291 for (i = 0; i < clflush_count; ++i) {
292 psb_clflush(clf);
293 clf += clflush_add;
294 }
295 mb();
296 }
297 kunmap_atomic(v);
298 spin_unlock(lock);
299
300 pt->count = 0;
301 pt->pd = pd;
302 pt->index = 0;
303
304 return pt;
305 }
306
psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd * pd,unsigned long addr)307 static struct psb_mmu_pt *psb_mmu_pt_alloc_map_lock(struct psb_mmu_pd *pd,
308 unsigned long addr)
309 {
310 uint32_t index = psb_mmu_pd_index(addr);
311 struct psb_mmu_pt *pt;
312 uint32_t *v;
313 spinlock_t *lock = &pd->driver->lock;
314
315 spin_lock(lock);
316 pt = pd->tables[index];
317 while (!pt) {
318 spin_unlock(lock);
319 pt = psb_mmu_alloc_pt(pd);
320 if (!pt)
321 return NULL;
322 spin_lock(lock);
323
324 if (pd->tables[index]) {
325 spin_unlock(lock);
326 psb_mmu_free_pt(pt);
327 spin_lock(lock);
328 pt = pd->tables[index];
329 continue;
330 }
331
332 v = kmap_atomic(pd->p);
333 pd->tables[index] = pt;
334 v[index] = (page_to_pfn(pt->p) << 12) | pd->pd_mask;
335 pt->index = index;
336 kunmap_atomic((void *) v);
337
338 if (pd->hw_context != -1) {
339 psb_mmu_clflush(pd->driver, (void *)&v[index]);
340 atomic_set(&pd->driver->needs_tlbflush, 1);
341 }
342 }
343 pt->v = kmap_atomic(pt->p);
344 return pt;
345 }
346
psb_mmu_pt_map_lock(struct psb_mmu_pd * pd,unsigned long addr)347 static struct psb_mmu_pt *psb_mmu_pt_map_lock(struct psb_mmu_pd *pd,
348 unsigned long addr)
349 {
350 uint32_t index = psb_mmu_pd_index(addr);
351 struct psb_mmu_pt *pt;
352 spinlock_t *lock = &pd->driver->lock;
353
354 spin_lock(lock);
355 pt = pd->tables[index];
356 if (!pt) {
357 spin_unlock(lock);
358 return NULL;
359 }
360 pt->v = kmap_atomic(pt->p);
361 return pt;
362 }
363
psb_mmu_pt_unmap_unlock(struct psb_mmu_pt * pt)364 static void psb_mmu_pt_unmap_unlock(struct psb_mmu_pt *pt)
365 {
366 struct psb_mmu_pd *pd = pt->pd;
367 uint32_t *v;
368
369 kunmap_atomic(pt->v);
370 if (pt->count == 0) {
371 v = kmap_atomic(pd->p);
372 v[pt->index] = pd->invalid_pde;
373 pd->tables[pt->index] = NULL;
374
375 if (pd->hw_context != -1) {
376 psb_mmu_clflush(pd->driver, (void *)&v[pt->index]);
377 atomic_set(&pd->driver->needs_tlbflush, 1);
378 }
379 kunmap_atomic(v);
380 spin_unlock(&pd->driver->lock);
381 psb_mmu_free_pt(pt);
382 return;
383 }
384 spin_unlock(&pd->driver->lock);
385 }
386
psb_mmu_set_pte(struct psb_mmu_pt * pt,unsigned long addr,uint32_t pte)387 static inline void psb_mmu_set_pte(struct psb_mmu_pt *pt, unsigned long addr,
388 uint32_t pte)
389 {
390 pt->v[psb_mmu_pt_index(addr)] = pte;
391 }
392
psb_mmu_invalidate_pte(struct psb_mmu_pt * pt,unsigned long addr)393 static inline void psb_mmu_invalidate_pte(struct psb_mmu_pt *pt,
394 unsigned long addr)
395 {
396 pt->v[psb_mmu_pt_index(addr)] = pt->pd->invalid_pte;
397 }
398
psb_mmu_get_default_pd(struct psb_mmu_driver * driver)399 struct psb_mmu_pd *psb_mmu_get_default_pd(struct psb_mmu_driver *driver)
400 {
401 struct psb_mmu_pd *pd;
402
403 down_read(&driver->sem);
404 pd = driver->default_pd;
405 up_read(&driver->sem);
406
407 return pd;
408 }
409
psb_mmu_driver_takedown(struct psb_mmu_driver * driver)410 void psb_mmu_driver_takedown(struct psb_mmu_driver *driver)
411 {
412 struct drm_device *dev = driver->dev;
413 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
414
415 PSB_WSGX32(driver->bif_ctrl, PSB_CR_BIF_CTRL);
416 psb_mmu_free_pagedir(driver->default_pd);
417 kfree(driver);
418 }
419
psb_mmu_driver_init(struct drm_device * dev,int trap_pagefaults,int invalid_type,atomic_t * msvdx_mmu_invaldc)420 struct psb_mmu_driver *psb_mmu_driver_init(struct drm_device *dev,
421 int trap_pagefaults,
422 int invalid_type,
423 atomic_t *msvdx_mmu_invaldc)
424 {
425 struct psb_mmu_driver *driver;
426 struct drm_psb_private *dev_priv = to_drm_psb_private(dev);
427
428 driver = kmalloc(sizeof(*driver), GFP_KERNEL);
429
430 if (!driver)
431 return NULL;
432
433 driver->dev = dev;
434 driver->default_pd = psb_mmu_alloc_pd(driver, trap_pagefaults,
435 invalid_type);
436 if (!driver->default_pd)
437 goto out_err1;
438
439 spin_lock_init(&driver->lock);
440 init_rwsem(&driver->sem);
441 down_write(&driver->sem);
442 atomic_set(&driver->needs_tlbflush, 1);
443 driver->msvdx_mmu_invaldc = msvdx_mmu_invaldc;
444
445 driver->bif_ctrl = PSB_RSGX32(PSB_CR_BIF_CTRL);
446 PSB_WSGX32(driver->bif_ctrl | _PSB_CB_CTRL_CLEAR_FAULT,
447 PSB_CR_BIF_CTRL);
448 PSB_WSGX32(driver->bif_ctrl & ~_PSB_CB_CTRL_CLEAR_FAULT,
449 PSB_CR_BIF_CTRL);
450
451 driver->has_clflush = 0;
452
453 if (boot_cpu_has(X86_FEATURE_CLFLUSH)) {
454 uint32_t tfms, misc, cap0, cap4, clflush_size;
455
456 /*
457 * clflush size is determined at kernel setup for x86_64 but not
458 * for i386. We have to do it here.
459 */
460
461 cpuid(0x00000001, &tfms, &misc, &cap0, &cap4);
462 clflush_size = ((misc >> 8) & 0xff) * 8;
463 driver->has_clflush = 1;
464 driver->clflush_add =
465 PAGE_SIZE * clflush_size / sizeof(uint32_t);
466 driver->clflush_mask = driver->clflush_add - 1;
467 driver->clflush_mask = ~driver->clflush_mask;
468 }
469
470 up_write(&driver->sem);
471 return driver;
472
473 out_err1:
474 kfree(driver);
475 return NULL;
476 }
477
psb_mmu_flush_ptes(struct psb_mmu_pd * pd,unsigned long address,uint32_t num_pages,uint32_t desired_tile_stride,uint32_t hw_tile_stride)478 static void psb_mmu_flush_ptes(struct psb_mmu_pd *pd, unsigned long address,
479 uint32_t num_pages, uint32_t desired_tile_stride,
480 uint32_t hw_tile_stride)
481 {
482 struct psb_mmu_pt *pt;
483 uint32_t rows = 1;
484 uint32_t i;
485 unsigned long addr;
486 unsigned long end;
487 unsigned long next;
488 unsigned long add;
489 unsigned long row_add;
490 unsigned long clflush_add = pd->driver->clflush_add;
491 unsigned long clflush_mask = pd->driver->clflush_mask;
492
493 if (!pd->driver->has_clflush)
494 return;
495
496 if (hw_tile_stride)
497 rows = num_pages / desired_tile_stride;
498 else
499 desired_tile_stride = num_pages;
500
501 add = desired_tile_stride << PAGE_SHIFT;
502 row_add = hw_tile_stride << PAGE_SHIFT;
503 mb();
504 for (i = 0; i < rows; ++i) {
505
506 addr = address;
507 end = addr + add;
508
509 do {
510 next = psb_pd_addr_end(addr, end);
511 pt = psb_mmu_pt_map_lock(pd, addr);
512 if (!pt)
513 continue;
514 do {
515 psb_clflush(&pt->v[psb_mmu_pt_index(addr)]);
516 } while (addr += clflush_add,
517 (addr & clflush_mask) < next);
518
519 psb_mmu_pt_unmap_unlock(pt);
520 } while (addr = next, next != end);
521 address += row_add;
522 }
523 mb();
524 }
525
psb_mmu_remove_pfn_sequence(struct psb_mmu_pd * pd,unsigned long address,uint32_t num_pages)526 void psb_mmu_remove_pfn_sequence(struct psb_mmu_pd *pd,
527 unsigned long address, uint32_t num_pages)
528 {
529 struct psb_mmu_pt *pt;
530 unsigned long addr;
531 unsigned long end;
532 unsigned long next;
533 unsigned long f_address = address;
534
535 down_read(&pd->driver->sem);
536
537 addr = address;
538 end = addr + (num_pages << PAGE_SHIFT);
539
540 do {
541 next = psb_pd_addr_end(addr, end);
542 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
543 if (!pt)
544 goto out;
545 do {
546 psb_mmu_invalidate_pte(pt, addr);
547 --pt->count;
548 } while (addr += PAGE_SIZE, addr < next);
549 psb_mmu_pt_unmap_unlock(pt);
550
551 } while (addr = next, next != end);
552
553 out:
554 if (pd->hw_context != -1)
555 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
556
557 up_read(&pd->driver->sem);
558
559 if (pd->hw_context != -1)
560 psb_mmu_flush(pd->driver);
561
562 return;
563 }
564
psb_mmu_remove_pages(struct psb_mmu_pd * pd,unsigned long address,uint32_t num_pages,uint32_t desired_tile_stride,uint32_t hw_tile_stride)565 void psb_mmu_remove_pages(struct psb_mmu_pd *pd, unsigned long address,
566 uint32_t num_pages, uint32_t desired_tile_stride,
567 uint32_t hw_tile_stride)
568 {
569 struct psb_mmu_pt *pt;
570 uint32_t rows = 1;
571 uint32_t i;
572 unsigned long addr;
573 unsigned long end;
574 unsigned long next;
575 unsigned long add;
576 unsigned long row_add;
577 unsigned long f_address = address;
578
579 if (hw_tile_stride)
580 rows = num_pages / desired_tile_stride;
581 else
582 desired_tile_stride = num_pages;
583
584 add = desired_tile_stride << PAGE_SHIFT;
585 row_add = hw_tile_stride << PAGE_SHIFT;
586
587 down_read(&pd->driver->sem);
588
589 /* Make sure we only need to flush this processor's cache */
590
591 for (i = 0; i < rows; ++i) {
592
593 addr = address;
594 end = addr + add;
595
596 do {
597 next = psb_pd_addr_end(addr, end);
598 pt = psb_mmu_pt_map_lock(pd, addr);
599 if (!pt)
600 continue;
601 do {
602 psb_mmu_invalidate_pte(pt, addr);
603 --pt->count;
604
605 } while (addr += PAGE_SIZE, addr < next);
606 psb_mmu_pt_unmap_unlock(pt);
607
608 } while (addr = next, next != end);
609 address += row_add;
610 }
611 if (pd->hw_context != -1)
612 psb_mmu_flush_ptes(pd, f_address, num_pages,
613 desired_tile_stride, hw_tile_stride);
614
615 up_read(&pd->driver->sem);
616
617 if (pd->hw_context != -1)
618 psb_mmu_flush(pd->driver);
619 }
620
psb_mmu_insert_pfn_sequence(struct psb_mmu_pd * pd,uint32_t start_pfn,unsigned long address,uint32_t num_pages,int type)621 int psb_mmu_insert_pfn_sequence(struct psb_mmu_pd *pd, uint32_t start_pfn,
622 unsigned long address, uint32_t num_pages,
623 int type)
624 {
625 struct psb_mmu_pt *pt;
626 uint32_t pte;
627 unsigned long addr;
628 unsigned long end;
629 unsigned long next;
630 unsigned long f_address = address;
631 int ret = -ENOMEM;
632
633 down_read(&pd->driver->sem);
634
635 addr = address;
636 end = addr + (num_pages << PAGE_SHIFT);
637
638 do {
639 next = psb_pd_addr_end(addr, end);
640 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
641 if (!pt) {
642 ret = -ENOMEM;
643 goto out;
644 }
645 do {
646 pte = psb_mmu_mask_pte(start_pfn++, type);
647 psb_mmu_set_pte(pt, addr, pte);
648 pt->count++;
649 } while (addr += PAGE_SIZE, addr < next);
650 psb_mmu_pt_unmap_unlock(pt);
651
652 } while (addr = next, next != end);
653 ret = 0;
654
655 out:
656 if (pd->hw_context != -1)
657 psb_mmu_flush_ptes(pd, f_address, num_pages, 1, 1);
658
659 up_read(&pd->driver->sem);
660
661 if (pd->hw_context != -1)
662 psb_mmu_flush(pd->driver);
663
664 return ret;
665 }
666
psb_mmu_insert_pages(struct psb_mmu_pd * pd,struct page ** pages,unsigned long address,uint32_t num_pages,uint32_t desired_tile_stride,uint32_t hw_tile_stride,int type)667 int psb_mmu_insert_pages(struct psb_mmu_pd *pd, struct page **pages,
668 unsigned long address, uint32_t num_pages,
669 uint32_t desired_tile_stride, uint32_t hw_tile_stride,
670 int type)
671 {
672 struct psb_mmu_pt *pt;
673 uint32_t rows = 1;
674 uint32_t i;
675 uint32_t pte;
676 unsigned long addr;
677 unsigned long end;
678 unsigned long next;
679 unsigned long add;
680 unsigned long row_add;
681 unsigned long f_address = address;
682 int ret = -ENOMEM;
683
684 if (hw_tile_stride) {
685 if (num_pages % desired_tile_stride != 0)
686 return -EINVAL;
687 rows = num_pages / desired_tile_stride;
688 } else {
689 desired_tile_stride = num_pages;
690 }
691
692 add = desired_tile_stride << PAGE_SHIFT;
693 row_add = hw_tile_stride << PAGE_SHIFT;
694
695 down_read(&pd->driver->sem);
696
697 for (i = 0; i < rows; ++i) {
698
699 addr = address;
700 end = addr + add;
701
702 do {
703 next = psb_pd_addr_end(addr, end);
704 pt = psb_mmu_pt_alloc_map_lock(pd, addr);
705 if (!pt)
706 goto out;
707 do {
708 pte = psb_mmu_mask_pte(page_to_pfn(*pages++),
709 type);
710 psb_mmu_set_pte(pt, addr, pte);
711 pt->count++;
712 } while (addr += PAGE_SIZE, addr < next);
713 psb_mmu_pt_unmap_unlock(pt);
714
715 } while (addr = next, next != end);
716
717 address += row_add;
718 }
719
720 ret = 0;
721 out:
722 if (pd->hw_context != -1)
723 psb_mmu_flush_ptes(pd, f_address, num_pages,
724 desired_tile_stride, hw_tile_stride);
725
726 up_read(&pd->driver->sem);
727
728 if (pd->hw_context != -1)
729 psb_mmu_flush(pd->driver);
730
731 return ret;
732 }
733
psb_mmu_virtual_to_pfn(struct psb_mmu_pd * pd,uint32_t virtual,unsigned long * pfn)734 int psb_mmu_virtual_to_pfn(struct psb_mmu_pd *pd, uint32_t virtual,
735 unsigned long *pfn)
736 {
737 int ret;
738 struct psb_mmu_pt *pt;
739 uint32_t tmp;
740 spinlock_t *lock = &pd->driver->lock;
741
742 down_read(&pd->driver->sem);
743 pt = psb_mmu_pt_map_lock(pd, virtual);
744 if (!pt) {
745 uint32_t *v;
746
747 spin_lock(lock);
748 v = kmap_atomic(pd->p);
749 tmp = v[psb_mmu_pd_index(virtual)];
750 kunmap_atomic(v);
751 spin_unlock(lock);
752
753 if (tmp != pd->invalid_pde || !(tmp & PSB_PTE_VALID) ||
754 !(pd->invalid_pte & PSB_PTE_VALID)) {
755 ret = -EINVAL;
756 goto out;
757 }
758 ret = 0;
759 *pfn = pd->invalid_pte >> PAGE_SHIFT;
760 goto out;
761 }
762 tmp = pt->v[psb_mmu_pt_index(virtual)];
763 if (!(tmp & PSB_PTE_VALID)) {
764 ret = -EINVAL;
765 } else {
766 ret = 0;
767 *pfn = tmp >> PAGE_SHIFT;
768 }
769 psb_mmu_pt_unmap_unlock(pt);
770 out:
771 up_read(&pd->driver->sem);
772 return ret;
773 }
774