Lines Matching full:ring
44 struct amdgpu_ring *ring) in amdgpu_ring_mux_sw_entry() argument
46 return ring->entry_index < mux->ring_entry_size ? in amdgpu_ring_mux_sw_entry()
47 &mux->ring_entry[ring->entry_index] : NULL; in amdgpu_ring_mux_sw_entry()
50 /* copy packages on sw ring range[begin, end) */
52 struct amdgpu_ring *ring, in amdgpu_ring_mux_copy_pkt_from_sw_ring() argument
58 start = s_start & ring->buf_mask; in amdgpu_ring_mux_copy_pkt_from_sw_ring()
59 end = s_end & ring->buf_mask; in amdgpu_ring_mux_copy_pkt_from_sw_ring()
62 DRM_ERROR("no more data copied from sw ring\n"); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
66 amdgpu_ring_alloc(real_ring, (ring->ring_size >> 2) + end - start); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
67 amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[start], in amdgpu_ring_mux_copy_pkt_from_sw_ring()
68 (ring->ring_size >> 2) - start); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
69 amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[0], end); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
72 amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[start], end - start); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
88 if (mux->ring_entry[i].ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) { in amdgpu_mux_resubmit_chunks()
95 DRM_ERROR("%s no low priority ring found\n", __func__); in amdgpu_mux_resubmit_chunks()
99 last_seq = atomic_read(&e->ring->fence_drv.last_seq); in amdgpu_mux_resubmit_chunks()
105 amdgpu_fence_update_start_timestamp(e->ring, in amdgpu_mux_resubmit_chunks()
109 le32_to_cpu(*(e->ring->fence_drv.cpu_addr + 2))) { in amdgpu_mux_resubmit_chunks()
110 if (chunk->cntl_offset <= e->ring->buf_mask) in amdgpu_mux_resubmit_chunks()
111 amdgpu_ring_patch_cntl(e->ring, in amdgpu_mux_resubmit_chunks()
113 if (chunk->ce_offset <= e->ring->buf_mask) in amdgpu_mux_resubmit_chunks()
114 amdgpu_ring_patch_ce(e->ring, chunk->ce_offset); in amdgpu_mux_resubmit_chunks()
115 if (chunk->de_offset <= e->ring->buf_mask) in amdgpu_mux_resubmit_chunks()
116 amdgpu_ring_patch_de(e->ring, chunk->de_offset); in amdgpu_mux_resubmit_chunks()
118 amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, e->ring, in amdgpu_mux_resubmit_chunks()
149 int amdgpu_ring_mux_init(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring, in amdgpu_ring_mux_init() argument
152 mux->real_ring = ring; in amdgpu_ring_mux_init()
194 int amdgpu_ring_mux_add_sw_ring(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_add_sw_ring() argument
199 DRM_ERROR("add sw ring exceeding max entry size\n"); in amdgpu_ring_mux_add_sw_ring()
204 ring->entry_index = mux->num_ring_entries; in amdgpu_ring_mux_add_sw_ring()
205 e->ring = ring; in amdgpu_ring_mux_add_sw_ring()
212 void amdgpu_ring_mux_set_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring, u64 wptr) in amdgpu_ring_mux_set_wptr() argument
218 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) in amdgpu_ring_mux_set_wptr()
221 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_set_wptr()
223 DRM_ERROR("cannot find entry for sw ring\n"); in amdgpu_ring_mux_set_wptr()
229 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && mux->pending_trailing_fence_signaled) { in amdgpu_ring_mux_set_wptr()
236 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && e->sw_cptr < mux->wptr_resubmit) in amdgpu_ring_mux_set_wptr()
242 if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT || mux->wptr_resubmit < wptr) { in amdgpu_ring_mux_set_wptr()
243 amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, ring, e->sw_cptr, wptr); in amdgpu_ring_mux_set_wptr()
252 u64 amdgpu_ring_mux_get_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_get_wptr() argument
256 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_get_wptr()
258 DRM_ERROR("cannot find entry for sw ring\n"); in amdgpu_ring_mux_get_wptr()
266 * amdgpu_ring_mux_get_rptr - get the readptr of the software ring
268 * @ring: the software ring of which we calculate the readptr
271 * write data onto the real ring buffer.After overwriting on the real ring, we
281 u64 amdgpu_ring_mux_get_rptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_get_rptr() argument
286 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_get_rptr()
304 e->sw_rptr = (e->sw_cptr + offset) & ring->buf_mask; in amdgpu_ring_mux_get_rptr()
315 u64 amdgpu_sw_ring_get_rptr_gfx(struct amdgpu_ring *ring) in amdgpu_sw_ring_get_rptr_gfx() argument
317 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_get_rptr_gfx()
320 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_get_rptr_gfx()
321 return amdgpu_ring_mux_get_rptr(mux, ring); in amdgpu_sw_ring_get_rptr_gfx()
324 u64 amdgpu_sw_ring_get_wptr_gfx(struct amdgpu_ring *ring) in amdgpu_sw_ring_get_wptr_gfx() argument
326 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_get_wptr_gfx()
329 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_get_wptr_gfx()
330 return amdgpu_ring_mux_get_wptr(mux, ring); in amdgpu_sw_ring_get_wptr_gfx()
333 void amdgpu_sw_ring_set_wptr_gfx(struct amdgpu_ring *ring) in amdgpu_sw_ring_set_wptr_gfx() argument
335 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_set_wptr_gfx()
338 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_set_wptr_gfx()
339 amdgpu_ring_mux_set_wptr(mux, ring, ring->wptr); in amdgpu_sw_ring_set_wptr_gfx()
343 void amdgpu_sw_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count) in amdgpu_sw_ring_insert_nop() argument
345 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_insert_nop()
360 /*Scan on low prio rings to have unsignaled fence and high ring has no fence.*/
363 struct amdgpu_ring *ring; in amdgpu_mcbp_scan() local
368 ring = mux->ring_entry[i].ring; in amdgpu_mcbp_scan()
369 if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT && in amdgpu_mcbp_scan()
370 amdgpu_fence_count_emitted(ring) > 0) in amdgpu_mcbp_scan()
372 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && in amdgpu_mcbp_scan()
373 amdgpu_fence_last_unsignaled_time_us(ring) > in amdgpu_mcbp_scan()
392 void amdgpu_sw_ring_ib_begin(struct amdgpu_ring *ring) in amdgpu_sw_ring_ib_begin() argument
394 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_ib_begin()
397 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_ib_begin()
398 if (adev->gfx.mcbp && ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT) { in amdgpu_sw_ring_ib_begin()
404 amdgpu_ring_mux_start_ib(mux, ring); in amdgpu_sw_ring_ib_begin()
407 void amdgpu_sw_ring_ib_end(struct amdgpu_ring *ring) in amdgpu_sw_ring_ib_end() argument
409 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_ib_end()
412 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_ib_end()
413 if (adev->gfx.mcbp && ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT) in amdgpu_sw_ring_ib_end()
415 amdgpu_ring_mux_end_ib(mux, ring); in amdgpu_sw_ring_ib_end()
418 void amdgpu_sw_ring_ib_mark_offset(struct amdgpu_ring *ring, enum amdgpu_ring_mux_offset_type type) in amdgpu_sw_ring_ib_mark_offset() argument
420 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_ib_mark_offset()
424 if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT) in amdgpu_sw_ring_ib_mark_offset()
427 offset = ring->wptr & ring->buf_mask; in amdgpu_sw_ring_ib_mark_offset()
429 amdgpu_ring_mux_ib_mark_offset(mux, ring, offset, type); in amdgpu_sw_ring_ib_mark_offset()
432 void amdgpu_ring_mux_start_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_start_ib() argument
441 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_start_ib()
453 chunk->start = ring->wptr; in amdgpu_ring_mux_start_ib()
455 chunk->cntl_offset = ring->buf_mask + 1; in amdgpu_ring_mux_start_ib()
456 chunk->de_offset = ring->buf_mask + 1; in amdgpu_ring_mux_start_ib()
457 chunk->ce_offset = ring->buf_mask + 1; in amdgpu_ring_mux_start_ib()
461 static void scan_and_remove_signaled_chunk(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in scan_and_remove_signaled_chunk() argument
467 e = amdgpu_ring_mux_sw_entry(mux, ring); in scan_and_remove_signaled_chunk()
473 last_seq = atomic_read(&ring->fence_drv.last_seq); in scan_and_remove_signaled_chunk()
484 struct amdgpu_ring *ring, u64 offset, in amdgpu_ring_mux_ib_mark_offset() argument
490 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_ib_mark_offset()
518 void amdgpu_ring_mux_end_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_end_ib() argument
523 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_end_ib()
535 chunk->end = ring->wptr; in amdgpu_ring_mux_end_ib()
536 chunk->sync_seq = READ_ONCE(ring->fence_drv.sync_seq); in amdgpu_ring_mux_end_ib()
538 scan_and_remove_signaled_chunk(mux, ring); in amdgpu_ring_mux_end_ib()
544 struct amdgpu_ring *ring = NULL; in amdgpu_mcbp_handle_trailing_fence_irq() local
555 if (e->ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) { in amdgpu_mcbp_handle_trailing_fence_irq()
556 ring = e->ring; in amdgpu_mcbp_handle_trailing_fence_irq()
561 if (!ring) { in amdgpu_mcbp_handle_trailing_fence_irq()
562 DRM_ERROR("cannot find low priority ring\n"); in amdgpu_mcbp_handle_trailing_fence_irq()
566 amdgpu_fence_process(ring); in amdgpu_mcbp_handle_trailing_fence_irq()
567 if (amdgpu_fence_count_emitted(ring) > 0) { in amdgpu_mcbp_handle_trailing_fence_irq()
569 mux->seqno_to_resubmit = ring->fence_drv.sync_seq; in amdgpu_mcbp_handle_trailing_fence_irq()