1 /*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2014-2016 Intel Corporation
5 */
6
7 #include "display/intel_display.h"
8 #include "gt/intel_gt.h"
9
10 #include "i915_drv.h"
11 #include "i915_gem_clflush.h"
12 #include "i915_gem_domain.h"
13 #include "i915_gem_gtt.h"
14 #include "i915_gem_ioctls.h"
15 #include "i915_gem_lmem.h"
16 #include "i915_gem_mman.h"
17 #include "i915_gem_object.h"
18 #include "i915_gem_object_frontbuffer.h"
19 #include "i915_vma.h"
20
21 #define VTD_GUARD (168u * I915_GTT_PAGE_SIZE) /* 168 or tile-row PTE padding */
22
gpu_write_needs_clflush(struct drm_i915_gem_object * obj)23 static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
24 {
25 struct drm_i915_private *i915 = to_i915(obj->base.dev);
26
27 if (IS_DGFX(i915))
28 return false;
29
30 /*
31 * For objects created by userspace through GEM_CREATE with pat_index
32 * set by set_pat extension, i915_gem_object_has_cache_level() will
33 * always return true, because the coherency of such object is managed
34 * by userspace. Othereise the call here would fall back to checking
35 * whether the object is un-cached or write-through.
36 */
37 return !(i915_gem_object_has_cache_level(obj, I915_CACHE_NONE) ||
38 i915_gem_object_has_cache_level(obj, I915_CACHE_WT));
39 }
40
i915_gem_cpu_write_needs_clflush(struct drm_i915_gem_object * obj)41 bool i915_gem_cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
42 {
43 struct drm_i915_private *i915 = to_i915(obj->base.dev);
44
45 if (obj->cache_dirty)
46 return false;
47
48 if (IS_DGFX(i915))
49 return false;
50
51 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
52 return true;
53
54 /* Currently in use by HW (display engine)? Keep flushed. */
55 return i915_gem_object_is_framebuffer(obj);
56 }
57
58 static void
flush_write_domain(struct drm_i915_gem_object * obj,unsigned int flush_domains)59 flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
60 {
61 struct i915_vma *vma;
62
63 assert_object_held(obj);
64
65 if (!(obj->write_domain & flush_domains))
66 return;
67
68 switch (obj->write_domain) {
69 case I915_GEM_DOMAIN_GTT:
70 spin_lock(&obj->vma.lock);
71 for_each_ggtt_vma(vma, obj)
72 i915_vma_flush_writes(vma);
73 spin_unlock(&obj->vma.lock);
74
75 i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
76 break;
77
78 case I915_GEM_DOMAIN_WC:
79 wmb();
80 break;
81
82 case I915_GEM_DOMAIN_CPU:
83 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
84 break;
85
86 case I915_GEM_DOMAIN_RENDER:
87 if (gpu_write_needs_clflush(obj))
88 obj->cache_dirty = true;
89 break;
90 }
91
92 obj->write_domain = 0;
93 }
94
__i915_gem_object_flush_for_display(struct drm_i915_gem_object * obj)95 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
96 {
97 /*
98 * We manually flush the CPU domain so that we can override and
99 * force the flush for the display, and perform it asyncrhonously.
100 */
101 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
102 if (obj->cache_dirty)
103 i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
104 obj->write_domain = 0;
105 }
106
i915_gem_object_flush_if_display(struct drm_i915_gem_object * obj)107 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
108 {
109 if (!i915_gem_object_is_framebuffer(obj))
110 return;
111
112 i915_gem_object_lock(obj, NULL);
113 __i915_gem_object_flush_for_display(obj);
114 i915_gem_object_unlock(obj);
115 }
116
i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object * obj)117 void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj)
118 {
119 if (i915_gem_object_is_framebuffer(obj))
120 __i915_gem_object_flush_for_display(obj);
121 }
122
123 /**
124 * i915_gem_object_set_to_wc_domain - Moves a single object to the WC read, and
125 * possibly write domain.
126 * @obj: object to act on
127 * @write: ask for write access or read only
128 *
129 * This function returns when the move is complete, including waiting on
130 * flushes to occur.
131 */
132 int
i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object * obj,bool write)133 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
134 {
135 int ret;
136
137 assert_object_held(obj);
138
139 ret = i915_gem_object_wait(obj,
140 I915_WAIT_INTERRUPTIBLE |
141 (write ? I915_WAIT_ALL : 0),
142 MAX_SCHEDULE_TIMEOUT);
143 if (ret)
144 return ret;
145
146 if (obj->write_domain == I915_GEM_DOMAIN_WC)
147 return 0;
148
149 /* Flush and acquire obj->pages so that we are coherent through
150 * direct access in memory with previous cached writes through
151 * shmemfs and that our cache domain tracking remains valid.
152 * For example, if the obj->filp was moved to swap without us
153 * being notified and releasing the pages, we would mistakenly
154 * continue to assume that the obj remained out of the CPU cached
155 * domain.
156 */
157 ret = i915_gem_object_pin_pages(obj);
158 if (ret)
159 return ret;
160
161 flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);
162
163 /* Serialise direct access to this object with the barriers for
164 * coherent writes from the GPU, by effectively invalidating the
165 * WC domain upon first access.
166 */
167 if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)
168 mb();
169
170 /* It should now be out of any other write domains, and we can update
171 * the domain values for our changes.
172 */
173 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);
174 obj->read_domains |= I915_GEM_DOMAIN_WC;
175 if (write) {
176 obj->read_domains = I915_GEM_DOMAIN_WC;
177 obj->write_domain = I915_GEM_DOMAIN_WC;
178 obj->mm.dirty = true;
179 }
180
181 i915_gem_object_unpin_pages(obj);
182 return 0;
183 }
184
185 /**
186 * i915_gem_object_set_to_gtt_domain - Moves a single object to the GTT read,
187 * and possibly write domain.
188 * @obj: object to act on
189 * @write: ask for write access or read only
190 *
191 * This function returns when the move is complete, including waiting on
192 * flushes to occur.
193 */
194 int
i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object * obj,bool write)195 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
196 {
197 int ret;
198
199 assert_object_held(obj);
200
201 ret = i915_gem_object_wait(obj,
202 I915_WAIT_INTERRUPTIBLE |
203 (write ? I915_WAIT_ALL : 0),
204 MAX_SCHEDULE_TIMEOUT);
205 if (ret)
206 return ret;
207
208 if (obj->write_domain == I915_GEM_DOMAIN_GTT)
209 return 0;
210
211 /* Flush and acquire obj->pages so that we are coherent through
212 * direct access in memory with previous cached writes through
213 * shmemfs and that our cache domain tracking remains valid.
214 * For example, if the obj->filp was moved to swap without us
215 * being notified and releasing the pages, we would mistakenly
216 * continue to assume that the obj remained out of the CPU cached
217 * domain.
218 */
219 ret = i915_gem_object_pin_pages(obj);
220 if (ret)
221 return ret;
222
223 flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);
224
225 /* Serialise direct access to this object with the barriers for
226 * coherent writes from the GPU, by effectively invalidating the
227 * GTT domain upon first access.
228 */
229 if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)
230 mb();
231
232 /* It should now be out of any other write domains, and we can update
233 * the domain values for our changes.
234 */
235 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
236 obj->read_domains |= I915_GEM_DOMAIN_GTT;
237 if (write) {
238 struct i915_vma *vma;
239
240 obj->read_domains = I915_GEM_DOMAIN_GTT;
241 obj->write_domain = I915_GEM_DOMAIN_GTT;
242 obj->mm.dirty = true;
243
244 spin_lock(&obj->vma.lock);
245 for_each_ggtt_vma(vma, obj)
246 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
247 i915_vma_set_ggtt_write(vma);
248 spin_unlock(&obj->vma.lock);
249 }
250
251 i915_gem_object_unpin_pages(obj);
252 return 0;
253 }
254
255 /**
256 * i915_gem_object_set_cache_level - Changes the cache-level of an object across all VMA.
257 * @obj: object to act on
258 * @cache_level: new cache level to set for the object
259 *
260 * After this function returns, the object will be in the new cache-level
261 * across all GTT and the contents of the backing storage will be coherent,
262 * with respect to the new cache-level. In order to keep the backing storage
263 * coherent for all users, we only allow a single cache level to be set
264 * globally on the object and prevent it from being changed whilst the
265 * hardware is reading from the object. That is if the object is currently
266 * on the scanout it will be set to uncached (or equivalent display
267 * cache coherency) and all non-MOCS GPU access will also be uncached so
268 * that all direct access to the scanout remains coherent.
269 */
i915_gem_object_set_cache_level(struct drm_i915_gem_object * obj,enum i915_cache_level cache_level)270 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
271 enum i915_cache_level cache_level)
272 {
273 int ret;
274
275 /*
276 * For objects created by userspace through GEM_CREATE with pat_index
277 * set by set_pat extension, simply return 0 here without touching
278 * the cache setting, because such objects should have an immutable
279 * cache setting by desgin and always managed by userspace.
280 */
281 if (i915_gem_object_has_cache_level(obj, cache_level))
282 return 0;
283
284 ret = i915_gem_object_wait(obj,
285 I915_WAIT_INTERRUPTIBLE |
286 I915_WAIT_ALL,
287 MAX_SCHEDULE_TIMEOUT);
288 if (ret)
289 return ret;
290
291 /* Always invalidate stale cachelines */
292 i915_gem_object_set_cache_coherency(obj, cache_level);
293 obj->cache_dirty = true;
294
295 /* The cache-level will be applied when each vma is rebound. */
296 return i915_gem_object_unbind(obj,
297 I915_GEM_OBJECT_UNBIND_ACTIVE |
298 I915_GEM_OBJECT_UNBIND_BARRIER);
299 }
300
i915_gem_get_caching_ioctl(struct drm_device * dev,void * data,struct drm_file * file)301 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
302 struct drm_file *file)
303 {
304 struct drm_i915_gem_caching *args = data;
305 struct drm_i915_gem_object *obj;
306 int err = 0;
307
308 if (IS_DGFX(to_i915(dev)))
309 return -ENODEV;
310
311 rcu_read_lock();
312 obj = i915_gem_object_lookup_rcu(file, args->handle);
313 if (!obj) {
314 err = -ENOENT;
315 goto out;
316 }
317
318 /*
319 * This ioctl should be disabled for the objects with pat_index
320 * set by user space.
321 */
322 if (obj->pat_set_by_user) {
323 err = -EOPNOTSUPP;
324 goto out;
325 }
326
327 if (i915_gem_object_has_cache_level(obj, I915_CACHE_LLC) ||
328 i915_gem_object_has_cache_level(obj, I915_CACHE_L3_LLC))
329 args->caching = I915_CACHING_CACHED;
330 else if (i915_gem_object_has_cache_level(obj, I915_CACHE_WT))
331 args->caching = I915_CACHING_DISPLAY;
332 else
333 args->caching = I915_CACHING_NONE;
334 out:
335 rcu_read_unlock();
336 return err;
337 }
338
i915_gem_set_caching_ioctl(struct drm_device * dev,void * data,struct drm_file * file)339 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
340 struct drm_file *file)
341 {
342 struct drm_i915_private *i915 = to_i915(dev);
343 struct drm_i915_gem_caching *args = data;
344 struct drm_i915_gem_object *obj;
345 enum i915_cache_level level;
346 int ret = 0;
347
348 if (IS_DGFX(i915))
349 return -ENODEV;
350
351 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70))
352 return -EOPNOTSUPP;
353
354 switch (args->caching) {
355 case I915_CACHING_NONE:
356 level = I915_CACHE_NONE;
357 break;
358 case I915_CACHING_CACHED:
359 /*
360 * Due to a HW issue on BXT A stepping, GPU stores via a
361 * snooped mapping may leave stale data in a corresponding CPU
362 * cacheline, whereas normally such cachelines would get
363 * invalidated.
364 */
365 if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
366 return -ENODEV;
367
368 level = I915_CACHE_LLC;
369 break;
370 case I915_CACHING_DISPLAY:
371 level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
372 break;
373 default:
374 return -EINVAL;
375 }
376
377 obj = i915_gem_object_lookup(file, args->handle);
378 if (!obj)
379 return -ENOENT;
380
381 /*
382 * This ioctl should be disabled for the objects with pat_index
383 * set by user space.
384 */
385 if (obj->pat_set_by_user) {
386 ret = -EOPNOTSUPP;
387 goto out;
388 }
389
390 /*
391 * The caching mode of proxy object is handled by its generator, and
392 * not allowed to be changed by userspace.
393 */
394 if (i915_gem_object_is_proxy(obj)) {
395 /*
396 * Silently allow cached for userptr; the vulkan driver
397 * sets all objects to cached
398 */
399 if (!i915_gem_object_is_userptr(obj) ||
400 args->caching != I915_CACHING_CACHED)
401 ret = -ENXIO;
402
403 goto out;
404 }
405
406 ret = i915_gem_object_lock_interruptible(obj, NULL);
407 if (ret)
408 goto out;
409
410 ret = i915_gem_object_set_cache_level(obj, level);
411 i915_gem_object_unlock(obj);
412
413 out:
414 i915_gem_object_put(obj);
415 return ret;
416 }
417
418 /*
419 * Prepare buffer for display plane (scanout, cursors, etc). Can be called from
420 * an uninterruptible phase (modesetting) and allows any flushes to be pipelined
421 * (for pageflips). We only flush the caches while preparing the buffer for
422 * display, the callers are responsible for frontbuffer flush.
423 */
424 struct i915_vma *
i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object * obj,struct i915_gem_ww_ctx * ww,u32 alignment,const struct i915_gtt_view * view,unsigned int flags)425 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
426 struct i915_gem_ww_ctx *ww,
427 u32 alignment,
428 const struct i915_gtt_view *view,
429 unsigned int flags)
430 {
431 struct drm_i915_private *i915 = to_i915(obj->base.dev);
432 struct i915_vma *vma;
433 int ret;
434
435 /* Frame buffer must be in LMEM */
436 if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj))
437 return ERR_PTR(-EINVAL);
438
439 /*
440 * The display engine is not coherent with the LLC cache on gen6. As
441 * a result, we make sure that the pinning that is about to occur is
442 * done with uncached PTEs. This is lowest common denominator for all
443 * chipsets.
444 *
445 * However for gen6+, we could do better by using the GFDT bit instead
446 * of uncaching, which would allow us to flush all the LLC-cached data
447 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
448 */
449 ret = i915_gem_object_set_cache_level(obj,
450 HAS_WT(i915) ?
451 I915_CACHE_WT : I915_CACHE_NONE);
452 if (ret)
453 return ERR_PTR(ret);
454
455 /* VT-d may overfetch before/after the vma, so pad with scratch */
456 if (intel_scanout_needs_vtd_wa(i915)) {
457 unsigned int guard = VTD_GUARD;
458
459 if (i915_gem_object_is_tiled(obj))
460 guard = max(guard,
461 i915_gem_object_get_tile_row_size(obj));
462
463 flags |= PIN_OFFSET_GUARD | guard;
464 }
465
466 /*
467 * As the user may map the buffer once pinned in the display plane
468 * (e.g. libkms for the bootup splash), we have to ensure that we
469 * always use map_and_fenceable for all scanout buffers. However,
470 * it may simply be too big to fit into mappable, in which case
471 * put it anyway and hope that userspace can cope (but always first
472 * try to preserve the existing ABI).
473 */
474 vma = ERR_PTR(-ENOSPC);
475 if ((flags & PIN_MAPPABLE) == 0 &&
476 (!view || view->type == I915_GTT_VIEW_NORMAL))
477 vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0, alignment,
478 flags | PIN_MAPPABLE |
479 PIN_NONBLOCK);
480 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK))
481 vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0,
482 alignment, flags);
483 if (IS_ERR(vma))
484 return vma;
485
486 vma->display_alignment = max(vma->display_alignment, alignment);
487 i915_vma_mark_scanout(vma);
488
489 i915_gem_object_flush_if_display_locked(obj);
490
491 return vma;
492 }
493
494 /**
495 * i915_gem_object_set_to_cpu_domain - Moves a single object to the CPU read,
496 * and possibly write domain.
497 * @obj: object to act on
498 * @write: requesting write or read-only access
499 *
500 * This function returns when the move is complete, including waiting on
501 * flushes to occur.
502 */
503 int
i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object * obj,bool write)504 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
505 {
506 int ret;
507
508 assert_object_held(obj);
509
510 ret = i915_gem_object_wait(obj,
511 I915_WAIT_INTERRUPTIBLE |
512 (write ? I915_WAIT_ALL : 0),
513 MAX_SCHEDULE_TIMEOUT);
514 if (ret)
515 return ret;
516
517 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
518
519 /* Flush the CPU cache if it's still invalid. */
520 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
521 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
522 obj->read_domains |= I915_GEM_DOMAIN_CPU;
523 }
524
525 /* It should now be out of any other write domains, and we can update
526 * the domain values for our changes.
527 */
528 GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);
529
530 /* If we're writing through the CPU, then the GPU read domains will
531 * need to be invalidated at next use.
532 */
533 if (write)
534 __start_cpu_write(obj);
535
536 return 0;
537 }
538
539 /**
540 * i915_gem_set_domain_ioctl - Called when user space prepares to use an
541 * object with the CPU, either
542 * through the mmap ioctl's mapping or a GTT mapping.
543 * @dev: drm device
544 * @data: ioctl data blob
545 * @file: drm file
546 */
547 int
i915_gem_set_domain_ioctl(struct drm_device * dev,void * data,struct drm_file * file)548 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
549 struct drm_file *file)
550 {
551 struct drm_i915_gem_set_domain *args = data;
552 struct drm_i915_gem_object *obj;
553 u32 read_domains = args->read_domains;
554 u32 write_domain = args->write_domain;
555 int err;
556
557 if (IS_DGFX(to_i915(dev)))
558 return -ENODEV;
559
560 /* Only handle setting domains to types used by the CPU. */
561 if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
562 return -EINVAL;
563
564 /*
565 * Having something in the write domain implies it's in the read
566 * domain, and only that read domain. Enforce that in the request.
567 */
568 if (write_domain && read_domains != write_domain)
569 return -EINVAL;
570
571 if (!read_domains)
572 return 0;
573
574 obj = i915_gem_object_lookup(file, args->handle);
575 if (!obj)
576 return -ENOENT;
577
578 /*
579 * Try to flush the object off the GPU without holding the lock.
580 * We will repeat the flush holding the lock in the normal manner
581 * to catch cases where we are gazumped.
582 */
583 err = i915_gem_object_wait(obj,
584 I915_WAIT_INTERRUPTIBLE |
585 I915_WAIT_PRIORITY |
586 (write_domain ? I915_WAIT_ALL : 0),
587 MAX_SCHEDULE_TIMEOUT);
588 if (err)
589 goto out;
590
591 if (i915_gem_object_is_userptr(obj)) {
592 /*
593 * Try to grab userptr pages, iris uses set_domain to check
594 * userptr validity
595 */
596 err = i915_gem_object_userptr_validate(obj);
597 if (!err)
598 err = i915_gem_object_wait(obj,
599 I915_WAIT_INTERRUPTIBLE |
600 I915_WAIT_PRIORITY |
601 (write_domain ? I915_WAIT_ALL : 0),
602 MAX_SCHEDULE_TIMEOUT);
603 goto out;
604 }
605
606 /*
607 * Proxy objects do not control access to the backing storage, ergo
608 * they cannot be used as a means to manipulate the cache domain
609 * tracking for that backing storage. The proxy object is always
610 * considered to be outside of any cache domain.
611 */
612 if (i915_gem_object_is_proxy(obj)) {
613 err = -ENXIO;
614 goto out;
615 }
616
617 err = i915_gem_object_lock_interruptible(obj, NULL);
618 if (err)
619 goto out;
620
621 /*
622 * Flush and acquire obj->pages so that we are coherent through
623 * direct access in memory with previous cached writes through
624 * shmemfs and that our cache domain tracking remains valid.
625 * For example, if the obj->filp was moved to swap without us
626 * being notified and releasing the pages, we would mistakenly
627 * continue to assume that the obj remained out of the CPU cached
628 * domain.
629 */
630 err = i915_gem_object_pin_pages(obj);
631 if (err)
632 goto out_unlock;
633
634 /*
635 * Already in the desired write domain? Nothing for us to do!
636 *
637 * We apply a little bit of cunning here to catch a broader set of
638 * no-ops. If obj->write_domain is set, we must be in the same
639 * obj->read_domains, and only that domain. Therefore, if that
640 * obj->write_domain matches the request read_domains, we are
641 * already in the same read/write domain and can skip the operation,
642 * without having to further check the requested write_domain.
643 */
644 if (READ_ONCE(obj->write_domain) == read_domains)
645 goto out_unpin;
646
647 if (read_domains & I915_GEM_DOMAIN_WC)
648 err = i915_gem_object_set_to_wc_domain(obj, write_domain);
649 else if (read_domains & I915_GEM_DOMAIN_GTT)
650 err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
651 else
652 err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
653
654 out_unpin:
655 i915_gem_object_unpin_pages(obj);
656
657 out_unlock:
658 i915_gem_object_unlock(obj);
659
660 if (!err && write_domain)
661 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
662
663 out:
664 i915_gem_object_put(obj);
665 return err;
666 }
667
668 /*
669 * Pins the specified object's pages and synchronizes the object with
670 * GPU accesses. Sets needs_clflush to non-zero if the caller should
671 * flush the object from the CPU cache.
672 */
i915_gem_object_prepare_read(struct drm_i915_gem_object * obj,unsigned int * needs_clflush)673 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,
674 unsigned int *needs_clflush)
675 {
676 int ret;
677
678 *needs_clflush = 0;
679 if (!i915_gem_object_has_struct_page(obj))
680 return -ENODEV;
681
682 assert_object_held(obj);
683
684 ret = i915_gem_object_wait(obj,
685 I915_WAIT_INTERRUPTIBLE,
686 MAX_SCHEDULE_TIMEOUT);
687 if (ret)
688 return ret;
689
690 ret = i915_gem_object_pin_pages(obj);
691 if (ret)
692 return ret;
693
694 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
695 !static_cpu_has(X86_FEATURE_CLFLUSH)) {
696 ret = i915_gem_object_set_to_cpu_domain(obj, false);
697 if (ret)
698 goto err_unpin;
699 else
700 goto out;
701 }
702
703 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
704
705 /* If we're not in the cpu read domain, set ourself into the gtt
706 * read domain and manually flush cachelines (if required). This
707 * optimizes for the case when the gpu will dirty the data
708 * anyway again before the next pread happens.
709 */
710 if (!obj->cache_dirty &&
711 !(obj->read_domains & I915_GEM_DOMAIN_CPU))
712 *needs_clflush = CLFLUSH_BEFORE;
713
714 out:
715 /* return with the pages pinned */
716 return 0;
717
718 err_unpin:
719 i915_gem_object_unpin_pages(obj);
720 return ret;
721 }
722
i915_gem_object_prepare_write(struct drm_i915_gem_object * obj,unsigned int * needs_clflush)723 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,
724 unsigned int *needs_clflush)
725 {
726 int ret;
727
728 *needs_clflush = 0;
729 if (!i915_gem_object_has_struct_page(obj))
730 return -ENODEV;
731
732 assert_object_held(obj);
733
734 ret = i915_gem_object_wait(obj,
735 I915_WAIT_INTERRUPTIBLE |
736 I915_WAIT_ALL,
737 MAX_SCHEDULE_TIMEOUT);
738 if (ret)
739 return ret;
740
741 ret = i915_gem_object_pin_pages(obj);
742 if (ret)
743 return ret;
744
745 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
746 !static_cpu_has(X86_FEATURE_CLFLUSH)) {
747 ret = i915_gem_object_set_to_cpu_domain(obj, true);
748 if (ret)
749 goto err_unpin;
750 else
751 goto out;
752 }
753
754 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
755
756 /* If we're not in the cpu write domain, set ourself into the
757 * gtt write domain and manually flush cachelines (as required).
758 * This optimizes for the case when the gpu will use the data
759 * right away and we therefore have to clflush anyway.
760 */
761 if (!obj->cache_dirty) {
762 *needs_clflush |= CLFLUSH_AFTER;
763
764 /*
765 * Same trick applies to invalidate partially written
766 * cachelines read before writing.
767 */
768 if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))
769 *needs_clflush |= CLFLUSH_BEFORE;
770 }
771
772 out:
773 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
774 obj->mm.dirty = true;
775 /* return with the pages pinned */
776 return 0;
777
778 err_unpin:
779 i915_gem_object_unpin_pages(obj);
780 return ret;
781 }
782