39  * Most engines on the GPU are fed via ring buffers.  Ring
45  * pointers are equal, the ring is idle.  When the host
46  * writes commands to the ring buffer, it increments the
50 static void radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring);
53  * radeon_ring_supports_scratch_reg - check if the ring supports
57  * @ring: radeon_ring structure holding ring information
59  * Check if a specific ring supports writing to scratch registers (all asics).
60  * Returns true if the ring supports writing to scratch regs, false if not.
63 				      struct radeon_ring *ring)  in radeon_ring_supports_scratch_reg()  argument
65 	switch (ring->idx) {  in radeon_ring_supports_scratch_reg()
76  * radeon_ring_free_size - update the free size
79  * @ring: radeon_ring structure holding ring information
81  * Update the free dw slots in the ring buffer (all asics).
83 void radeon_ring_free_size(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_ring_free_size()  argument
85 	uint32_t rptr = radeon_ring_get_rptr(rdev, ring);  in radeon_ring_free_size()
88 	ring->ring_free_dw = rptr + (ring->ring_size / 4);  in radeon_ring_free_size()
89 	ring->ring_free_dw -= ring->wptr;  in radeon_ring_free_size()
90 	ring->ring_free_dw &= ring->ptr_mask;  in radeon_ring_free_size()
91 	if (!ring->ring_free_dw) {  in radeon_ring_free_size()
92 		/* this is an empty ring */  in radeon_ring_free_size()
93 		ring->ring_free_dw = ring->ring_size / 4;  in radeon_ring_free_size()
95 		radeon_ring_lockup_update(rdev, ring);  in radeon_ring_free_size()
100  * radeon_ring_alloc - allocate space on the ring buffer
103  * @ring: radeon_ring structure holding ring information
104  * @ndw: number of dwords to allocate in the ring buffer
106  * Allocate @ndw dwords in the ring buffer (all asics).
109 int radeon_ring_alloc(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)  in radeon_ring_alloc()  argument
113 	/* make sure we aren't trying to allocate more space than there is on the ring */  in radeon_ring_alloc()
114 	if (ndw > (ring->ring_size / 4))  in radeon_ring_alloc()
115 		return -ENOMEM;  in radeon_ring_alloc()
118 	radeon_ring_free_size(rdev, ring);  in radeon_ring_alloc()
119 	ndw = (ndw + ring->align_mask) & ~ring->align_mask;  in radeon_ring_alloc()
120 	while (ndw > (ring->ring_free_dw - 1)) {  in radeon_ring_alloc()
121 		radeon_ring_free_size(rdev, ring);  in radeon_ring_alloc()
122 		if (ndw < ring->ring_free_dw) {  in radeon_ring_alloc()
125 		r = radeon_fence_wait_next(rdev, ring->idx);  in radeon_ring_alloc()
129 	ring->count_dw = ndw;  in radeon_ring_alloc()
130 	ring->wptr_old = ring->wptr;  in radeon_ring_alloc()
135  * radeon_ring_lock - lock the ring and allocate space on it
138  * @ring: radeon_ring structure holding ring information
139  * @ndw: number of dwords to allocate in the ring buffer
141  * Lock the ring and allocate @ndw dwords in the ring buffer
145 int radeon_ring_lock(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ndw)  in radeon_ring_lock()  argument
149 	mutex_lock(&rdev->ring_lock);  in radeon_ring_lock()
150 	r = radeon_ring_alloc(rdev, ring, ndw);  in radeon_ring_lock()
152 		mutex_unlock(&rdev->ring_lock);  in radeon_ring_lock()
159  * radeon_ring_commit - tell the GPU to execute the new
160  * commands on the ring buffer
163  * @ring: radeon_ring structure holding ring information
167  * execute new commands on the ring buffer (all asics).
169 void radeon_ring_commit(struct radeon_device *rdev, struct radeon_ring *ring,  in radeon_ring_commit()  argument
172 	/* If we are emitting the HDP flush via the ring buffer, we need to  in radeon_ring_commit()
175 	if (hdp_flush && rdev->asic->ring[ring->idx]->hdp_flush)  in radeon_ring_commit()
176 		rdev->asic->ring[ring->idx]->hdp_flush(rdev, ring);  in radeon_ring_commit()
178 	while (ring->wptr & ring->align_mask) {  in radeon_ring_commit()
179 		radeon_ring_write(ring, ring->nop);  in radeon_ring_commit()
185 	if (hdp_flush && rdev->asic->mmio_hdp_flush)  in radeon_ring_commit()
186 		rdev->asic->mmio_hdp_flush(rdev);  in radeon_ring_commit()
187 	radeon_ring_set_wptr(rdev, ring);  in radeon_ring_commit()
191  * radeon_ring_unlock_commit - tell the GPU to execute the new
192  * commands on the ring buffer and unlock it
195  * @ring: radeon_ring structure holding ring information
198  * Call radeon_ring_commit() then unlock the ring (all asics).
200 void radeon_ring_unlock_commit(struct radeon_device *rdev, struct radeon_ring *ring,  in radeon_ring_unlock_commit()  argument
203 	radeon_ring_commit(rdev, ring, hdp_flush);  in radeon_ring_unlock_commit()
204 	mutex_unlock(&rdev->ring_lock);  in radeon_ring_unlock_commit()
208  * radeon_ring_undo - reset the wptr
210  * @ring: radeon_ring structure holding ring information
214 void radeon_ring_undo(struct radeon_ring *ring)  in radeon_ring_undo()  argument
216 	ring->wptr = ring->wptr_old;  in radeon_ring_undo()
220  * radeon_ring_unlock_undo - reset the wptr and unlock the ring
223  * @ring: radeon_ring structure holding ring information
225  * Call radeon_ring_undo() then unlock the ring (all asics).
227 void radeon_ring_unlock_undo(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_ring_unlock_undo()  argument
229 	radeon_ring_undo(ring);  in radeon_ring_unlock_undo()
230 	mutex_unlock(&rdev->ring_lock);  in radeon_ring_unlock_undo()
234  * radeon_ring_lockup_update - update lockup variables
237  * @ring: radeon_ring structure holding ring information
242 			       struct radeon_ring *ring)  in radeon_ring_lockup_update()  argument
244 	atomic_set(&ring->last_rptr, radeon_ring_get_rptr(rdev, ring));  in radeon_ring_lockup_update()
245 	atomic64_set(&ring->last_activity, jiffies_64);  in radeon_ring_lockup_update()
249  * radeon_ring_test_lockup() - check if ring is lockedup by recording information
251  * @ring:       radeon_ring structure holding ring information
254 bool radeon_ring_test_lockup(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_ring_test_lockup()  argument
256 	uint32_t rptr = radeon_ring_get_rptr(rdev, ring);  in radeon_ring_test_lockup()
257 	uint64_t last = atomic64_read(&ring->last_activity);  in radeon_ring_test_lockup()
260 	if (rptr != atomic_read(&ring->last_rptr)) {  in radeon_ring_test_lockup()
261 		/* ring is still working, no lockup */  in radeon_ring_test_lockup()
262 		radeon_ring_lockup_update(rdev, ring);  in radeon_ring_test_lockup()
266 	elapsed = jiffies_to_msecs(jiffies_64 - last);  in radeon_ring_test_lockup()
268 		dev_err(rdev->dev, "ring %d stalled for more than %llumsec\n",  in radeon_ring_test_lockup()
269 			ring->idx, elapsed);  in radeon_ring_test_lockup()
277  * radeon_ring_backup - Back up the content of a ring
280  * @ring: the ring we want to back up
283  * Saves all unprocessed commits from a ring, returns the number of dwords saved.
285 unsigned radeon_ring_backup(struct radeon_device *rdev, struct radeon_ring *ring,  in radeon_ring_backup()  argument
290 	/* just in case lock the ring */  in radeon_ring_backup()
291 	mutex_lock(&rdev->ring_lock);  in radeon_ring_backup()
294 	if (ring->ring_obj == NULL) {  in radeon_ring_backup()
295 		mutex_unlock(&rdev->ring_lock);  in radeon_ring_backup()
300 	if (!radeon_fence_count_emitted(rdev, ring->idx)) {  in radeon_ring_backup()
301 		mutex_unlock(&rdev->ring_lock);  in radeon_ring_backup()
305 	/* calculate the number of dw on the ring */  in radeon_ring_backup()
306 	if (ring->rptr_save_reg)  in radeon_ring_backup()
307 		ptr = RREG32(ring->rptr_save_reg);  in radeon_ring_backup()
308 	else if (rdev->wb.enabled)  in radeon_ring_backup()
309 		ptr = le32_to_cpu(*ring->next_rptr_cpu_addr);  in radeon_ring_backup()
312 		mutex_unlock(&rdev->ring_lock);  in radeon_ring_backup()
316 	size = ring->wptr + (ring->ring_size / 4);  in radeon_ring_backup()
317 	size -= ptr;  in radeon_ring_backup()
318 	size &= ring->ptr_mask;  in radeon_ring_backup()
320 		mutex_unlock(&rdev->ring_lock);  in radeon_ring_backup()
324 	/* and then save the content of the ring */  in radeon_ring_backup()
327 		mutex_unlock(&rdev->ring_lock);  in radeon_ring_backup()
331 		(*data)[i] = ring->ring[ptr++];  in radeon_ring_backup()
332 		ptr &= ring->ptr_mask;  in radeon_ring_backup()
335 	mutex_unlock(&rdev->ring_lock);  in radeon_ring_backup()
340  * radeon_ring_restore - append saved commands to the ring again
343  * @ring: ring to append commands to
347  * Allocates space on the ring and restore the previously saved commands.
349 int radeon_ring_restore(struct radeon_device *rdev, struct radeon_ring *ring,  in radeon_ring_restore()  argument
357 	/* restore the saved ring content */  in radeon_ring_restore()
358 	r = radeon_ring_lock(rdev, ring, size);  in radeon_ring_restore()
363 		radeon_ring_write(ring, data[i]);  in radeon_ring_restore()
366 	radeon_ring_unlock_commit(rdev, ring, false);  in radeon_ring_restore()
372  * radeon_ring_init - init driver ring struct.
375  * @ring: radeon_ring structure holding ring information
376  * @ring_size: size of the ring
378  * @nop: nop packet for this ring
380  * Initialize the driver information for the selected ring (all asics).
383 int radeon_ring_init(struct radeon_device *rdev, struct radeon_ring *ring, unsigned ring_size,  in radeon_ring_init()  argument
388 	ring->ring_size = ring_size;  in radeon_ring_init()
389 	ring->rptr_offs = rptr_offs;  in radeon_ring_init()
390 	ring->nop = nop;  in radeon_ring_init()
391 	ring->rdev = rdev;  in radeon_ring_init()
392 	/* Allocate ring buffer */  in radeon_ring_init()
393 	if (ring->ring_obj == NULL) {  in radeon_ring_init()
394 		r = radeon_bo_create(rdev, ring->ring_size, PAGE_SIZE, true,  in radeon_ring_init()
396 				     NULL, &ring->ring_obj);  in radeon_ring_init()
398 			dev_err(rdev->dev, "(%d) ring create failed\n", r);  in radeon_ring_init()
401 		r = radeon_bo_reserve(ring->ring_obj, false);  in radeon_ring_init()
404 		r = radeon_bo_pin(ring->ring_obj, RADEON_GEM_DOMAIN_GTT,  in radeon_ring_init()
405 					&ring->gpu_addr);  in radeon_ring_init()
407 			radeon_bo_unreserve(ring->ring_obj);  in radeon_ring_init()
408 			dev_err(rdev->dev, "(%d) ring pin failed\n", r);  in radeon_ring_init()
411 		r = radeon_bo_kmap(ring->ring_obj,  in radeon_ring_init()
412 				       (void **)&ring->ring);  in radeon_ring_init()
413 		radeon_bo_unreserve(ring->ring_obj);  in radeon_ring_init()
415 			dev_err(rdev->dev, "(%d) ring map failed\n", r);  in radeon_ring_init()
418 		radeon_debugfs_ring_init(rdev, ring);  in radeon_ring_init()
420 	ring->ptr_mask = (ring->ring_size / 4) - 1;  in radeon_ring_init()
421 	ring->ring_free_dw = ring->ring_size / 4;  in radeon_ring_init()
422 	if (rdev->wb.enabled) {  in radeon_ring_init()
423 		u32 index = RADEON_WB_RING0_NEXT_RPTR + (ring->idx * 4);  in radeon_ring_init()
424 		ring->next_rptr_gpu_addr = rdev->wb.gpu_addr + index;  in radeon_ring_init()
425 		ring->next_rptr_cpu_addr = &rdev->wb.wb[index/4];  in radeon_ring_init()
427 	radeon_ring_lockup_update(rdev, ring);  in radeon_ring_init()
432  * radeon_ring_fini - tear down the driver ring struct.
435  * @ring: radeon_ring structure holding ring information
437  * Tear down the driver information for the selected ring (all asics).
439 void radeon_ring_fini(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_ring_fini()  argument
444 	mutex_lock(&rdev->ring_lock);  in radeon_ring_fini()
445 	ring_obj = ring->ring_obj;  in radeon_ring_fini()
446 	ring->ready = false;  in radeon_ring_fini()
447 	ring->ring = NULL;  in radeon_ring_fini()
448 	ring->ring_obj = NULL;  in radeon_ring_fini()
449 	mutex_unlock(&rdev->ring_lock);  in radeon_ring_fini()
469 	struct radeon_ring *ring = m->private;  in radeon_debugfs_ring_info_show()  local
470 	struct radeon_device *rdev = ring->rdev;  in radeon_debugfs_ring_info_show()
475 	radeon_ring_free_size(rdev, ring);  in radeon_debugfs_ring_info_show()
476 	count = (ring->ring_size / 4) - ring->ring_free_dw;  in radeon_debugfs_ring_info_show()
478 	wptr = radeon_ring_get_wptr(rdev, ring);  in radeon_debugfs_ring_info_show()
482 	rptr = radeon_ring_get_rptr(rdev, ring);  in radeon_debugfs_ring_info_show()
486 	if (ring->rptr_save_reg) {  in radeon_debugfs_ring_info_show()
487 		rptr_next = RREG32(ring->rptr_save_reg);  in radeon_debugfs_ring_info_show()
489 			   ring->rptr_save_reg, rptr_next, rptr_next);  in radeon_debugfs_ring_info_show()
494 		   ring->wptr, ring->wptr);  in radeon_debugfs_ring_info_show()
496 		   ring->last_semaphore_signal_addr);  in radeon_debugfs_ring_info_show()
498 		   ring->last_semaphore_wait_addr);  in radeon_debugfs_ring_info_show()
499 	seq_printf(m, "%u free dwords in ring\n", ring->ring_free_dw);  in radeon_debugfs_ring_info_show()
500 	seq_printf(m, "%u dwords in ring\n", count);  in radeon_debugfs_ring_info_show()
502 	if (!ring->ring)  in radeon_debugfs_ring_info_show()
508 	i = (rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask;  in radeon_debugfs_ring_info_show()
510 		seq_printf(m, "r[%5d]=0x%08x", i, ring->ring[i]);  in radeon_debugfs_ring_info_show()
516 		i = (i + 1) & ring->ptr_mask;  in radeon_debugfs_ring_info_show()
549 static void radeon_debugfs_ring_init(struct radeon_device *rdev, struct radeon_ring *ring)  in radeon_debugfs_ring_init()  argument
552 	const char *ring_name = radeon_debugfs_ring_idx_to_name(ring->idx);  in radeon_debugfs_ring_init()
553 	struct dentry *root = rdev_to_drm(rdev)->primary->debugfs_root;  in radeon_debugfs_ring_init()
556 		debugfs_create_file(ring_name, 0444, root, ring,  in radeon_debugfs_ring_init()