Lines Matching +full:cost +full:- +full:effective
2 * SPDX-License-Identifier: MIT
4 * Copyright © 2008-2015 Intel Corporation
13 #include <linux/dma-buf.h>
37 return swap_available() || obj->mm.madv == I915_MADV_DONTNEED; in can_release_pages()
61 if (obj->ops->shrink) { in try_to_writeback()
70 return obj->ops->shrink(obj, shrink_flags); in try_to_writeback()
77 * i915_gem_shrink - Shrink buffer object caches
90 * free system memory - the pages might still be in-used to due to other reasons
95 * Also note that any kind of pinning (both per-vma address space pins and
113 { &i915->mm.purge_list, ~0u }, in i915_gem_shrink()
115 &i915->mm.shrink_list, in i915_gem_shrink()
126 /* CHV + VTD workaround use stop_machine(); need to trylock vm->mutex */ in i915_gem_shrink()
134 * we will force the wake during oom-notifier. in i915_gem_shrink()
137 wakeref = intel_runtime_pm_get_if_in_use(&i915->runtime_pm); in i915_gem_shrink()
174 * dev->struct_mutex and so we won't ever be able to observe an in i915_gem_shrink()
177 for (phase = phases; phase->list; phase++) { in i915_gem_shrink()
182 if ((shrink & phase->bit) == 0) in i915_gem_shrink()
194 spin_lock_irqsave(&i915->mm.obj_lock, flags); in i915_gem_shrink()
196 (obj = list_first_entry_or_null(phase->list, in i915_gem_shrink()
199 list_move_tail(&obj->mm.link, &still_in_list); in i915_gem_shrink()
202 !is_vmalloc_addr(obj->mm.mapping)) in i915_gem_shrink()
212 if (!kref_get_unless_zero(&obj->base.refcount)) in i915_gem_shrink()
215 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); in i915_gem_shrink()
230 count += obj->base.size >> PAGE_SHIFT; in i915_gem_shrink()
235 scanned += obj->base.size >> PAGE_SHIFT; in i915_gem_shrink()
239 spin_lock_irqsave(&i915->mm.obj_lock, flags); in i915_gem_shrink()
243 list_splice_tail(&still_in_list, phase->list); in i915_gem_shrink()
244 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); in i915_gem_shrink()
250 intel_runtime_pm_put(&i915->runtime_pm, wakeref); in i915_gem_shrink()
261 * i915_gem_shrink_all - Shrink buffer object caches completely
269 * last-ditch effort when memory seems to have run out.
279 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { in i915_gem_shrink_all()
280 freed = i915_gem_shrink(NULL, i915, -1UL, NULL, in i915_gem_shrink_all()
291 struct drm_i915_private *i915 = shrinker->private_data; in i915_gem_shrinker_count()
295 count = READ_ONCE(i915->mm.shrink_memory) >> PAGE_SHIFT; in i915_gem_shrinker_count()
296 num_objects = READ_ONCE(i915->mm.shrink_count); in i915_gem_shrinker_count()
300 * Our rough guess for an effective batch size is roughly 2 in i915_gem_shrinker_count()
302 * the shrinker to fire, until it is worth the cost of freeing an in i915_gem_shrinker_count()
308 i915->mm.shrinker->batch = in i915_gem_shrinker_count()
309 max((i915->mm.shrinker->batch + avg) >> 1, in i915_gem_shrinker_count()
319 struct drm_i915_private *i915 = shrinker->private_data; in i915_gem_shrinker_scan()
322 sc->nr_scanned = 0; in i915_gem_shrinker_scan()
325 sc->nr_to_scan, in i915_gem_shrinker_scan()
326 &sc->nr_scanned, in i915_gem_shrinker_scan()
329 if (sc->nr_scanned < sc->nr_to_scan && current_is_kswapd()) { in i915_gem_shrinker_scan()
332 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { in i915_gem_shrinker_scan()
334 sc->nr_to_scan - sc->nr_scanned, in i915_gem_shrinker_scan()
335 &sc->nr_scanned, in i915_gem_shrinker_scan()
343 return sc->nr_scanned ? freed : SHRINK_STOP; in i915_gem_shrinker_scan()
357 with_intel_runtime_pm(&i915->runtime_pm, wakeref) in i915_gem_shrinker_oom()
358 freed_pages += i915_gem_shrink(NULL, i915, -1UL, NULL, in i915_gem_shrinker_oom()
368 spin_lock_irqsave(&i915->mm.obj_lock, flags); in i915_gem_shrinker_oom()
369 list_for_each_entry(obj, &i915->mm.shrink_list, mm.link) { in i915_gem_shrinker_oom()
371 unevictable += obj->base.size >> PAGE_SHIFT; in i915_gem_shrinker_oom()
373 available += obj->base.size >> PAGE_SHIFT; in i915_gem_shrinker_oom()
375 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); in i915_gem_shrinker_oom()
397 with_intel_runtime_pm(&i915->runtime_pm, wakeref) in i915_gem_shrinker_vmap()
398 freed_pages += i915_gem_shrink(NULL, i915, -1UL, NULL, in i915_gem_shrinker_vmap()
405 mutex_lock(>->ggtt->vm.mutex); in i915_gem_shrinker_vmap()
407 >->ggtt->vm.bound_list, vm_link) { in i915_gem_shrinker_vmap()
409 struct drm_i915_gem_object *obj = vma->obj; in i915_gem_shrinker_vmap()
411 if (!vma->iomap || i915_vma_is_active(vma)) in i915_gem_shrinker_vmap()
422 mutex_unlock(>->ggtt->vm.mutex); in i915_gem_shrinker_vmap()
431 i915->mm.shrinker = shrinker_alloc(0, "drm-i915_gem"); in i915_gem_driver_register__shrinker()
432 if (!i915->mm.shrinker) { in i915_gem_driver_register__shrinker()
433 drm_WARN_ON(&i915->drm, 1); in i915_gem_driver_register__shrinker()
435 i915->mm.shrinker->scan_objects = i915_gem_shrinker_scan; in i915_gem_driver_register__shrinker()
436 i915->mm.shrinker->count_objects = i915_gem_shrinker_count; in i915_gem_driver_register__shrinker()
437 i915->mm.shrinker->batch = 4096; in i915_gem_driver_register__shrinker()
438 i915->mm.shrinker->private_data = i915; in i915_gem_driver_register__shrinker()
440 shrinker_register(i915->mm.shrinker); in i915_gem_driver_register__shrinker()
443 i915->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom; in i915_gem_driver_register__shrinker()
444 drm_WARN_ON(&i915->drm, register_oom_notifier(&i915->mm.oom_notifier)); in i915_gem_driver_register__shrinker()
446 i915->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap; in i915_gem_driver_register__shrinker()
447 drm_WARN_ON(&i915->drm, in i915_gem_driver_register__shrinker()
448 register_vmap_purge_notifier(&i915->mm.vmap_notifier)); in i915_gem_driver_register__shrinker()
453 drm_WARN_ON(&i915->drm, in i915_gem_driver_unregister__shrinker()
454 unregister_vmap_purge_notifier(&i915->mm.vmap_notifier)); in i915_gem_driver_unregister__shrinker()
455 drm_WARN_ON(&i915->drm, in i915_gem_driver_unregister__shrinker()
456 unregister_oom_notifier(&i915->mm.oom_notifier)); in i915_gem_driver_unregister__shrinker()
457 shrinker_free(i915->mm.shrinker); in i915_gem_driver_unregister__shrinker()
468 mutex_acquire(&mutex->dep_map, 0, 0, _RET_IP_); in i915_gem_shrinker_taints_mutex()
469 mutex_release(&mutex->dep_map, _RET_IP_); in i915_gem_shrinker_taints_mutex()
475 * i915_gem_object_make_unshrinkable - Hide the object from the shrinker. By
482 * easily processed by the shrinker, like if they are perma-pinned.
495 if (atomic_add_unless(&obj->mm.shrink_pin, 1, 0)) in i915_gem_object_make_unshrinkable()
498 spin_lock_irqsave(&i915->mm.obj_lock, flags); in i915_gem_object_make_unshrinkable()
499 if (!atomic_fetch_inc(&obj->mm.shrink_pin) && in i915_gem_object_make_unshrinkable()
500 !list_empty(&obj->mm.link)) { in i915_gem_object_make_unshrinkable()
501 list_del_init(&obj->mm.link); in i915_gem_object_make_unshrinkable()
502 i915->mm.shrink_count--; in i915_gem_object_make_unshrinkable()
503 i915->mm.shrink_memory -= obj->base.size; in i915_gem_object_make_unshrinkable()
505 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); in i915_gem_object_make_unshrinkable()
517 if (atomic_add_unless(&obj->mm.shrink_pin, -1, 1)) in ___i915_gem_object_make_shrinkable()
520 spin_lock_irqsave(&i915->mm.obj_lock, flags); in ___i915_gem_object_make_shrinkable()
521 GEM_BUG_ON(!kref_read(&obj->base.refcount)); in ___i915_gem_object_make_shrinkable()
522 if (atomic_dec_and_test(&obj->mm.shrink_pin)) { in ___i915_gem_object_make_shrinkable()
523 GEM_BUG_ON(!list_empty(&obj->mm.link)); in ___i915_gem_object_make_shrinkable()
525 list_add_tail(&obj->mm.link, head); in ___i915_gem_object_make_shrinkable()
526 i915->mm.shrink_count++; in ___i915_gem_object_make_shrinkable()
527 i915->mm.shrink_memory += obj->base.size; in ___i915_gem_object_make_shrinkable()
530 spin_unlock_irqrestore(&i915->mm.obj_lock, flags); in ___i915_gem_object_make_shrinkable()
534 * __i915_gem_object_make_shrinkable - Move the object to the tail of the
546 &obj_to_i915(obj)->mm.shrink_list); in __i915_gem_object_make_shrinkable()
550 * __i915_gem_object_make_purgeable - Move the object to the tail of the
562 &obj_to_i915(obj)->mm.purge_list); in __i915_gem_object_make_purgeable()
566 * i915_gem_object_make_shrinkable - Move the object to the tail of the
582 * i915_gem_object_make_purgeable - Move the object to the tail of the purgeable