67 static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)  in radeon_fence_write()  argument
69 	struct radeon_fence_driver *drv = &rdev->fence_drv[ring];  in radeon_fence_write()
71 	if (likely(rdev->wb.enabled || !drv->scratch_reg)) {  in radeon_fence_write()
88 static u32 radeon_fence_read(struct radeon_device *rdev, int ring)  in radeon_fence_read()  argument
90 	struct radeon_fence_driver *drv = &rdev->fence_drv[ring];  in radeon_fence_read()
93 	if (likely(rdev->wb.enabled || !drv->scratch_reg)) {  in radeon_fence_read()
112 static void radeon_fence_schedule_check(struct radeon_device *rdev, int ring)  in radeon_fence_schedule_check()  argument
119 			   &rdev->fence_drv[ring].lockup_work,  in radeon_fence_schedule_check()
133 int radeon_fence_emit(struct radeon_device *rdev,  in radeon_fence_emit()  argument
144 	(*fence)->rdev = rdev;  in radeon_fence_emit()
145 	(*fence)->seq = seq = ++rdev->fence_drv[ring].sync_seq[ring];  in radeon_fence_emit()
149 		       &rdev->fence_queue.lock,  in radeon_fence_emit()
150 		       rdev->fence_context + ring,  in radeon_fence_emit()
152 	radeon_fence_ring_emit(rdev, ring, *fence);  in radeon_fence_emit()
153 	trace_radeon_fence_emit(rdev_to_drm(rdev), ring, (*fence)->seq);  in radeon_fence_emit()
154 	radeon_fence_schedule_check(rdev, ring);  in radeon_fence_emit()
177 	seq = atomic64_read(&fence->rdev->fence_drv[fence->ring].last_seq);  in radeon_fence_check_signaled()
180 		radeon_irq_kms_sw_irq_put(fence->rdev, fence->ring);  in radeon_fence_check_signaled()
181 		__remove_wait_queue(&fence->rdev->fence_queue, &fence->fence_wake);  in radeon_fence_check_signaled()
197 static bool radeon_fence_activity(struct radeon_device *rdev, int ring)  in radeon_fence_activity()  argument
224 	last_seq = atomic64_read(&rdev->fence_drv[ring].last_seq);  in radeon_fence_activity()
226 		last_emitted = rdev->fence_drv[ring].sync_seq[ring];  in radeon_fence_activity()
227 		seq = radeon_fence_read(rdev, ring);  in radeon_fence_activity()
251 	} while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq);  in radeon_fence_activity()
254 		radeon_fence_schedule_check(rdev, ring);  in radeon_fence_activity()
270 	struct radeon_device *rdev;  in radeon_fence_check_lockup()  local
275 	rdev = fence_drv->rdev;  in radeon_fence_check_lockup()
276 	ring = fence_drv - &rdev->fence_drv[0];  in radeon_fence_check_lockup()
278 	if (!down_read_trylock(&rdev->exclusive_lock)) {  in radeon_fence_check_lockup()
280 		radeon_fence_schedule_check(rdev, ring);  in radeon_fence_check_lockup()
284 	if (fence_drv->delayed_irq && rdev->irq.installed) {  in radeon_fence_check_lockup()
288 		spin_lock_irqsave(&rdev->irq.lock, irqflags);  in radeon_fence_check_lockup()
289 		radeon_irq_set(rdev);  in radeon_fence_check_lockup()
290 		spin_unlock_irqrestore(&rdev->irq.lock, irqflags);  in radeon_fence_check_lockup()
293 	if (radeon_fence_activity(rdev, ring))  in radeon_fence_check_lockup()
294 		wake_up_all(&rdev->fence_queue);  in radeon_fence_check_lockup()
296 	else if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) {  in radeon_fence_check_lockup()
299 …dev_warn(rdev->dev, "GPU lockup (current fence id 0x%016llx last fence id 0x%016llx on ring %d)\n",  in radeon_fence_check_lockup()
304 		rdev->needs_reset = true;  in radeon_fence_check_lockup()
305 		wake_up_all(&rdev->fence_queue);  in radeon_fence_check_lockup()
307 	up_read(&rdev->exclusive_lock);  in radeon_fence_check_lockup()
319 void radeon_fence_process(struct radeon_device *rdev, int ring)  in radeon_fence_process()  argument
321 	if (radeon_fence_activity(rdev, ring))  in radeon_fence_process()
322 		wake_up_all(&rdev->fence_queue);  in radeon_fence_process()
339 static bool radeon_fence_seq_signaled(struct radeon_device *rdev,  in radeon_fence_seq_signaled()  argument
342 	if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq)  in radeon_fence_seq_signaled()
346 	radeon_fence_process(rdev, ring);  in radeon_fence_seq_signaled()
347 	if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq)  in radeon_fence_seq_signaled()
356 	struct radeon_device *rdev = fence->rdev;  in radeon_fence_is_signaled()  local
360 	if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq)  in radeon_fence_is_signaled()
363 	if (down_read_trylock(&rdev->exclusive_lock)) {  in radeon_fence_is_signaled()
364 		radeon_fence_process(rdev, ring);  in radeon_fence_is_signaled()
365 		up_read(&rdev->exclusive_lock);  in radeon_fence_is_signaled()
367 		if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq)  in radeon_fence_is_signaled()
384 	struct radeon_device *rdev = fence->rdev;  in radeon_fence_enable_signaling()  local
386 	if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq)  in radeon_fence_enable_signaling()
389 	if (down_read_trylock(&rdev->exclusive_lock)) {  in radeon_fence_enable_signaling()
390 		radeon_irq_kms_sw_irq_get(rdev, fence->ring);  in radeon_fence_enable_signaling()
392 		if (radeon_fence_activity(rdev, fence->ring))  in radeon_fence_enable_signaling()
393 			wake_up_all_locked(&rdev->fence_queue);  in radeon_fence_enable_signaling()
396 		if (atomic64_read(&rdev->fence_drv[fence->ring].last_seq) >= fence->seq) {  in radeon_fence_enable_signaling()
397 			radeon_irq_kms_sw_irq_put(rdev, fence->ring);  in radeon_fence_enable_signaling()
398 			up_read(&rdev->exclusive_lock);  in radeon_fence_enable_signaling()
402 		up_read(&rdev->exclusive_lock);  in radeon_fence_enable_signaling()
405 		if (radeon_irq_kms_sw_irq_get_delayed(rdev, fence->ring))  in radeon_fence_enable_signaling()
406 			rdev->fence_drv[fence->ring].delayed_irq = true;  in radeon_fence_enable_signaling()
407 		radeon_fence_schedule_check(rdev, fence->ring);  in radeon_fence_enable_signaling()
413 	__add_wait_queue(&rdev->fence_queue, &fence->fence_wake);  in radeon_fence_enable_signaling()
431 	if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) {  in radeon_fence_signaled()
449 static bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq)  in radeon_fence_any_seq_signaled()  argument
454 		if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i))  in radeon_fence_any_seq_signaled()
477 static long radeon_fence_wait_seq_timeout(struct radeon_device *rdev,  in radeon_fence_wait_seq_timeout()  argument
484 	if (radeon_fence_any_seq_signaled(rdev, target_seq))  in radeon_fence_wait_seq_timeout()
492 		trace_radeon_fence_wait_begin(rdev_to_drm(rdev), i, target_seq[i]);  in radeon_fence_wait_seq_timeout()
493 		radeon_irq_kms_sw_irq_get(rdev, i);  in radeon_fence_wait_seq_timeout()
497 		r = wait_event_interruptible_timeout(rdev->fence_queue, (  in radeon_fence_wait_seq_timeout()
498 			radeon_fence_any_seq_signaled(rdev, target_seq)  in radeon_fence_wait_seq_timeout()
499 			 || rdev->needs_reset), timeout);  in radeon_fence_wait_seq_timeout()
501 		r = wait_event_timeout(rdev->fence_queue, (  in radeon_fence_wait_seq_timeout()
502 			radeon_fence_any_seq_signaled(rdev, target_seq)  in radeon_fence_wait_seq_timeout()
503 			 || rdev->needs_reset), timeout);  in radeon_fence_wait_seq_timeout()
506 	if (rdev->needs_reset)  in radeon_fence_wait_seq_timeout()
513 		radeon_irq_kms_sw_irq_put(rdev, i);  in radeon_fence_wait_seq_timeout()
514 		trace_radeon_fence_wait_end(rdev_to_drm(rdev), i, target_seq[i]);  in radeon_fence_wait_seq_timeout()
548 	r = radeon_fence_wait_seq_timeout(fence->rdev, seq, intr, timeout);  in radeon_fence_wait_timeout()
590 int radeon_fence_wait_any(struct radeon_device *rdev,  in radeon_fence_wait_any()  argument
612 	r = radeon_fence_wait_seq_timeout(rdev, seq, intr, MAX_SCHEDULE_TIMEOUT);  in radeon_fence_wait_any()
629 int radeon_fence_wait_next(struct radeon_device *rdev, int ring)  in radeon_fence_wait_next()  argument
634 	seq[ring] = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL;  in radeon_fence_wait_next()
635 	if (seq[ring] >= rdev->fence_drv[ring].sync_seq[ring]) {  in radeon_fence_wait_next()
642 	r = radeon_fence_wait_seq_timeout(rdev, seq, false, MAX_SCHEDULE_TIMEOUT);  in radeon_fence_wait_next()
659 int radeon_fence_wait_empty(struct radeon_device *rdev, int ring)  in radeon_fence_wait_empty()  argument
664 	seq[ring] = rdev->fence_drv[ring].sync_seq[ring];  in radeon_fence_wait_empty()
668 	r = radeon_fence_wait_seq_timeout(rdev, seq, false, MAX_SCHEDULE_TIMEOUT);  in radeon_fence_wait_empty()
673 		dev_err(rdev->dev, "error waiting for ring[%d] to become idle (%ld)\n",  in radeon_fence_wait_empty()
719 unsigned int radeon_fence_count_emitted(struct radeon_device *rdev, int ring)  in radeon_fence_count_emitted()  argument
726 	radeon_fence_process(rdev, ring);  in radeon_fence_count_emitted()
727 	emitted = rdev->fence_drv[ring].sync_seq[ring]  in radeon_fence_count_emitted()
728 		- atomic64_read(&rdev->fence_drv[ring].last_seq);  in radeon_fence_count_emitted()
758 	fdrv = &fence->rdev->fence_drv[dst_ring];  in radeon_fence_need_sync()
786 	src = &fence->rdev->fence_drv[fence->ring];  in radeon_fence_note_sync()
787 	dst = &fence->rdev->fence_drv[dst_ring];  in radeon_fence_note_sync()
808 int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring)  in radeon_fence_driver_start_ring()  argument
813 	radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);  in radeon_fence_driver_start_ring()
814 	if (rdev->wb.use_event || !radeon_ring_supports_scratch_reg(rdev, &rdev->ring[ring])) {  in radeon_fence_driver_start_ring()
815 		rdev->fence_drv[ring].scratch_reg = 0;  in radeon_fence_driver_start_ring()
818 			rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4];  in radeon_fence_driver_start_ring()
819 			rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr +  in radeon_fence_driver_start_ring()
824 			index = ALIGN(rdev->uvd_fw->size, 8);  in radeon_fence_driver_start_ring()
825 			rdev->fence_drv[ring].cpu_addr = rdev->uvd.cpu_addr + index;  in radeon_fence_driver_start_ring()
826 			rdev->fence_drv[ring].gpu_addr = rdev->uvd.gpu_addr + index;  in radeon_fence_driver_start_ring()
830 		r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg);  in radeon_fence_driver_start_ring()
832 			dev_err(rdev->dev, "fence failed to get scratch register\n");  in radeon_fence_driver_start_ring()
836 			rdev->fence_drv[ring].scratch_reg -  in radeon_fence_driver_start_ring()
837 			rdev->scratch.reg_base;  in radeon_fence_driver_start_ring()
838 		rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4];  in radeon_fence_driver_start_ring()
839 		rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index;  in radeon_fence_driver_start_ring()
841 	radeon_fence_write(rdev, atomic64_read(&rdev->fence_drv[ring].last_seq), ring);  in radeon_fence_driver_start_ring()
842 	rdev->fence_drv[ring].initialized = true;  in radeon_fence_driver_start_ring()
843 	dev_info(rdev->dev, "fence driver on ring %d use gpu addr 0x%016llx\n",  in radeon_fence_driver_start_ring()
844 		 ring, rdev->fence_drv[ring].gpu_addr);  in radeon_fence_driver_start_ring()
858 static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring)  in radeon_fence_driver_init_ring()  argument
862 	rdev->fence_drv[ring].scratch_reg = -1;  in radeon_fence_driver_init_ring()
863 	rdev->fence_drv[ring].cpu_addr = NULL;  in radeon_fence_driver_init_ring()
864 	rdev->fence_drv[ring].gpu_addr = 0;  in radeon_fence_driver_init_ring()
866 		rdev->fence_drv[ring].sync_seq[i] = 0;  in radeon_fence_driver_init_ring()
867 	atomic64_set(&rdev->fence_drv[ring].last_seq, 0);  in radeon_fence_driver_init_ring()
868 	rdev->fence_drv[ring].initialized = false;  in radeon_fence_driver_init_ring()
869 	INIT_DELAYED_WORK(&rdev->fence_drv[ring].lockup_work,  in radeon_fence_driver_init_ring()
871 	rdev->fence_drv[ring].rdev = rdev;  in radeon_fence_driver_init_ring()
885 void radeon_fence_driver_init(struct radeon_device *rdev)  in radeon_fence_driver_init()  argument
889 	init_waitqueue_head(&rdev->fence_queue);  in radeon_fence_driver_init()
891 		radeon_fence_driver_init_ring(rdev, ring);  in radeon_fence_driver_init()
893 	radeon_debugfs_fence_init(rdev);  in radeon_fence_driver_init()
904 void radeon_fence_driver_fini(struct radeon_device *rdev)  in radeon_fence_driver_fini()  argument
908 	mutex_lock(&rdev->ring_lock);  in radeon_fence_driver_fini()
910 		if (!rdev->fence_drv[ring].initialized)  in radeon_fence_driver_fini()
912 		r = radeon_fence_wait_empty(rdev, ring);  in radeon_fence_driver_fini()
915 			radeon_fence_driver_force_completion(rdev, ring);  in radeon_fence_driver_fini()
917 		cancel_delayed_work_sync(&rdev->fence_drv[ring].lockup_work);  in radeon_fence_driver_fini()
918 		wake_up_all(&rdev->fence_queue);  in radeon_fence_driver_fini()
919 		radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);  in radeon_fence_driver_fini()
920 		rdev->fence_drv[ring].initialized = false;  in radeon_fence_driver_fini()
922 	mutex_unlock(&rdev->ring_lock);  in radeon_fence_driver_fini()
934 void radeon_fence_driver_force_completion(struct radeon_device *rdev, int ring)  in radeon_fence_driver_force_completion()  argument
936 	if (rdev->fence_drv[ring].initialized) {  in radeon_fence_driver_force_completion()
937 		radeon_fence_write(rdev, rdev->fence_drv[ring].sync_seq[ring], ring);  in radeon_fence_driver_force_completion()
938 		cancel_delayed_work_sync(&rdev->fence_drv[ring].lockup_work);  in radeon_fence_driver_force_completion()
949 	struct radeon_device *rdev = m->private;  in radeon_debugfs_fence_info_show()  local
953 		if (!rdev->fence_drv[i].initialized)  in radeon_debugfs_fence_info_show()
956 		radeon_fence_process(rdev, i);  in radeon_debugfs_fence_info_show()
960 			   (unsigned long long)atomic64_read(&rdev->fence_drv[i].last_seq));  in radeon_debugfs_fence_info_show()
962 			   rdev->fence_drv[i].sync_seq[i]);  in radeon_debugfs_fence_info_show()
965 			if (i != j && rdev->fence_drv[j].initialized)  in radeon_debugfs_fence_info_show()
967 					   j, rdev->fence_drv[i].sync_seq[j]);  in radeon_debugfs_fence_info_show()
980 	struct radeon_device *rdev = (struct radeon_device *)data;  in radeon_debugfs_gpu_reset()  local
982 	down_read(&rdev->exclusive_lock);  in radeon_debugfs_gpu_reset()
983 	*val = rdev->needs_reset;  in radeon_debugfs_gpu_reset()
984 	rdev->needs_reset = true;  in radeon_debugfs_gpu_reset()
985 	wake_up_all(&rdev->fence_queue);  in radeon_debugfs_gpu_reset()
986 	up_read(&rdev->exclusive_lock);  in radeon_debugfs_gpu_reset()
995 void radeon_debugfs_fence_init(struct radeon_device *rdev)  in radeon_debugfs_fence_init()  argument
998 	struct dentry *root = rdev_to_drm(rdev)->primary->debugfs_root;  in radeon_debugfs_fence_init()
1000 	debugfs_create_file("radeon_gpu_reset", 0444, root, rdev,  in radeon_debugfs_fence_init()
1002 	debugfs_create_file("radeon_fence_info", 0444, root, rdev,  in radeon_debugfs_fence_init()
1056 	struct radeon_device *rdev = fence->rdev;  in radeon_fence_default_wait()  local
1077 		if (rdev->needs_reset) {  in radeon_fence_default_wait()