43 static inline struct amdgpu_mux_entry *amdgpu_ring_mux_sw_entry(struct amdgpu_ring_mux *mux,  in amdgpu_ring_mux_sw_entry()  argument
46 	return ring->entry_index < mux->ring_entry_size ?  in amdgpu_ring_mux_sw_entry()
47 			&mux->ring_entry[ring->entry_index] : NULL;  in amdgpu_ring_mux_sw_entry()
51 static void amdgpu_ring_mux_copy_pkt_from_sw_ring(struct amdgpu_ring_mux *mux,  in amdgpu_ring_mux_copy_pkt_from_sw_ring()  argument
56 	struct amdgpu_ring *real_ring = mux->real_ring;  in amdgpu_ring_mux_copy_pkt_from_sw_ring()
76 static void amdgpu_mux_resubmit_chunks(struct amdgpu_ring_mux *mux)  in amdgpu_mux_resubmit_chunks()  argument
84 	if (!mux->s_resubmit)  in amdgpu_mux_resubmit_chunks()
87 	for (i = 0; i < mux->num_ring_entries; i++) {  in amdgpu_mux_resubmit_chunks()
88 		if (mux->ring_entry[i].ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) {  in amdgpu_mux_resubmit_chunks()
89 			e = &mux->ring_entry[i];  in amdgpu_mux_resubmit_chunks()
100 	seq = mux->seqno_to_resubmit;  in amdgpu_mux_resubmit_chunks()
118 				amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, e->ring,  in amdgpu_mux_resubmit_chunks()
121 				mux->wptr_resubmit = chunk->end;  in amdgpu_mux_resubmit_chunks()
122 				amdgpu_ring_commit(mux->real_ring);  in amdgpu_mux_resubmit_chunks()
127 	del_timer(&mux->resubmit_timer);  in amdgpu_mux_resubmit_chunks()
128 	mux->s_resubmit = false;  in amdgpu_mux_resubmit_chunks()
131 static void amdgpu_ring_mux_schedule_resubmit(struct amdgpu_ring_mux *mux)  in amdgpu_ring_mux_schedule_resubmit()  argument
133 	mod_timer(&mux->resubmit_timer, jiffies + AMDGPU_MUX_RESUBMIT_JIFFIES_TIMEOUT);  in amdgpu_ring_mux_schedule_resubmit()
138 	struct amdgpu_ring_mux *mux = from_timer(mux, t, resubmit_timer);  in amdgpu_mux_resubmit_fallback()  local
140 	if (!spin_trylock(&mux->lock)) {  in amdgpu_mux_resubmit_fallback()
141 		amdgpu_ring_mux_schedule_resubmit(mux);  in amdgpu_mux_resubmit_fallback()
145 	amdgpu_mux_resubmit_chunks(mux);  in amdgpu_mux_resubmit_fallback()
146 	spin_unlock(&mux->lock);  in amdgpu_mux_resubmit_fallback()
149 int amdgpu_ring_mux_init(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring,  in amdgpu_ring_mux_init()  argument
152 	mux->real_ring = ring;  in amdgpu_ring_mux_init()
153 	mux->num_ring_entries = 0;  in amdgpu_ring_mux_init()
155 	mux->ring_entry = kcalloc(entry_size, sizeof(struct amdgpu_mux_entry), GFP_KERNEL);  in amdgpu_ring_mux_init()
156 	if (!mux->ring_entry)  in amdgpu_ring_mux_init()
159 	mux->ring_entry_size = entry_size;  in amdgpu_ring_mux_init()
160 	mux->s_resubmit = false;  in amdgpu_ring_mux_init()
168 	spin_lock_init(&mux->lock);  in amdgpu_ring_mux_init()
169 	timer_setup(&mux->resubmit_timer, amdgpu_mux_resubmit_fallback, 0);  in amdgpu_ring_mux_init()
174 void amdgpu_ring_mux_fini(struct amdgpu_ring_mux *mux)  in amdgpu_ring_mux_fini()  argument
180 	for (i = 0; i < mux->num_ring_entries; i++) {  in amdgpu_ring_mux_fini()
181 		e = &mux->ring_entry[i];  in amdgpu_ring_mux_fini()
188 	kfree(mux->ring_entry);  in amdgpu_ring_mux_fini()
189 	mux->ring_entry = NULL;  in amdgpu_ring_mux_fini()
190 	mux->num_ring_entries = 0;  in amdgpu_ring_mux_fini()
191 	mux->ring_entry_size = 0;  in amdgpu_ring_mux_fini()
194 int amdgpu_ring_mux_add_sw_ring(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)  in amdgpu_ring_mux_add_sw_ring()  argument
198 	if (mux->num_ring_entries >= mux->ring_entry_size) {  in amdgpu_ring_mux_add_sw_ring()
203 	e = &mux->ring_entry[mux->num_ring_entries];  in amdgpu_ring_mux_add_sw_ring()
204 	ring->entry_index = mux->num_ring_entries;  in amdgpu_ring_mux_add_sw_ring()
208 	mux->num_ring_entries += 1;  in amdgpu_ring_mux_add_sw_ring()
212 void amdgpu_ring_mux_set_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring, u64 wptr)  in amdgpu_ring_mux_set_wptr()  argument
216 	spin_lock(&mux->lock);  in amdgpu_ring_mux_set_wptr()
219 		amdgpu_mux_resubmit_chunks(mux);  in amdgpu_ring_mux_set_wptr()
221 	e = amdgpu_ring_mux_sw_entry(mux, ring);  in amdgpu_ring_mux_set_wptr()
224 		spin_unlock(&mux->lock);  in amdgpu_ring_mux_set_wptr()
229 	if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && mux->pending_trailing_fence_signaled) {  in amdgpu_ring_mux_set_wptr()
230 		spin_unlock(&mux->lock);  in amdgpu_ring_mux_set_wptr()
236 	if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && e->sw_cptr < mux->wptr_resubmit)  in amdgpu_ring_mux_set_wptr()
237 		e->sw_cptr = mux->wptr_resubmit;  in amdgpu_ring_mux_set_wptr()
239 	e->start_ptr_in_hw_ring = mux->real_ring->wptr;  in amdgpu_ring_mux_set_wptr()
242 	if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT || mux->wptr_resubmit < wptr) {  in amdgpu_ring_mux_set_wptr()
243 		amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, ring, e->sw_cptr, wptr);  in amdgpu_ring_mux_set_wptr()
244 		e->end_ptr_in_hw_ring = mux->real_ring->wptr;  in amdgpu_ring_mux_set_wptr()
245 		amdgpu_ring_commit(mux->real_ring);  in amdgpu_ring_mux_set_wptr()
247 		e->end_ptr_in_hw_ring = mux->real_ring->wptr;  in amdgpu_ring_mux_set_wptr()
249 	spin_unlock(&mux->lock);  in amdgpu_ring_mux_set_wptr()
252 u64 amdgpu_ring_mux_get_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)  in amdgpu_ring_mux_get_wptr()  argument
256 	e = amdgpu_ring_mux_sw_entry(mux, ring);  in amdgpu_ring_mux_get_wptr()
267  * @mux: the multiplexer the software rings attach to
281 u64 amdgpu_ring_mux_get_rptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)  in amdgpu_ring_mux_get_rptr()  argument
286 	e = amdgpu_ring_mux_sw_entry(mux, ring);  in amdgpu_ring_mux_get_rptr()
292 	readp = amdgpu_ring_get_rptr(mux->real_ring);  in amdgpu_ring_mux_get_rptr()
294 	start = e->start_ptr_in_hw_ring & mux->real_ring->buf_mask;  in amdgpu_ring_mux_get_rptr()
295 	end = e->end_ptr_in_hw_ring & mux->real_ring->buf_mask;  in amdgpu_ring_mux_get_rptr()
298 			readp += mux->real_ring->ring_size >> 2;  in amdgpu_ring_mux_get_rptr()
299 		end += mux->real_ring->ring_size >> 2;  in amdgpu_ring_mux_get_rptr()
318 	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;  in amdgpu_sw_ring_get_rptr_gfx()  local
321 	return amdgpu_ring_mux_get_rptr(mux, ring);  in amdgpu_sw_ring_get_rptr_gfx()
327 	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;  in amdgpu_sw_ring_get_wptr_gfx()  local
330 	return amdgpu_ring_mux_get_wptr(mux, ring);  in amdgpu_sw_ring_get_wptr_gfx()
336 	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;  in amdgpu_sw_ring_set_wptr_gfx()  local
339 	amdgpu_ring_mux_set_wptr(mux, ring, ring->wptr);  in amdgpu_sw_ring_set_wptr_gfx()
361 static int amdgpu_mcbp_scan(struct amdgpu_ring_mux *mux)  in amdgpu_mcbp_scan()  argument
367 	for (i = 0; i < mux->num_ring_entries; i++) {  in amdgpu_mcbp_scan()
368 		ring = mux->ring_entry[i].ring;  in amdgpu_mcbp_scan()
377 	return need_preempt && !mux->s_resubmit;  in amdgpu_mcbp_scan()
381 static int amdgpu_mcbp_trigger_preempt(struct amdgpu_ring_mux *mux)  in amdgpu_mcbp_trigger_preempt()  argument
385 	spin_lock(&mux->lock);  in amdgpu_mcbp_trigger_preempt()
386 	mux->pending_trailing_fence_signaled = true;  in amdgpu_mcbp_trigger_preempt()
387 	r = amdgpu_ring_preempt_ib(mux->real_ring);  in amdgpu_mcbp_trigger_preempt()
388 	spin_unlock(&mux->lock);  in amdgpu_mcbp_trigger_preempt()
395 	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;  in amdgpu_sw_ring_ib_begin()  local
399 		if (amdgpu_mcbp_scan(mux) > 0)  in amdgpu_sw_ring_ib_begin()
400 			amdgpu_mcbp_trigger_preempt(mux);  in amdgpu_sw_ring_ib_begin()
404 	amdgpu_ring_mux_start_ib(mux, ring);  in amdgpu_sw_ring_ib_begin()
410 	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;  in amdgpu_sw_ring_ib_end()  local
415 	amdgpu_ring_mux_end_ib(mux, ring);  in amdgpu_sw_ring_ib_end()
421 	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;  in amdgpu_sw_ring_ib_mark_offset()  local
429 	amdgpu_ring_mux_ib_mark_offset(mux, ring, offset, type);  in amdgpu_sw_ring_ib_mark_offset()
432 void amdgpu_ring_mux_start_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)  in amdgpu_ring_mux_start_ib()  argument
437 	spin_lock(&mux->lock);  in amdgpu_ring_mux_start_ib()
438 	amdgpu_mux_resubmit_chunks(mux);  in amdgpu_ring_mux_start_ib()
439 	spin_unlock(&mux->lock);  in amdgpu_ring_mux_start_ib()
441 	e = amdgpu_ring_mux_sw_entry(mux, ring);  in amdgpu_ring_mux_start_ib()
461 static void scan_and_remove_signaled_chunk(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)  in scan_and_remove_signaled_chunk()  argument
467 	e = amdgpu_ring_mux_sw_entry(mux, ring);  in scan_and_remove_signaled_chunk()
483 void amdgpu_ring_mux_ib_mark_offset(struct amdgpu_ring_mux *mux,  in amdgpu_ring_mux_ib_mark_offset()  argument
490 	e = amdgpu_ring_mux_sw_entry(mux, ring);  in amdgpu_ring_mux_ib_mark_offset()
518 void amdgpu_ring_mux_end_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)  in amdgpu_ring_mux_end_ib()  argument
523 	e = amdgpu_ring_mux_sw_entry(mux, ring);  in amdgpu_ring_mux_end_ib()
538 	scan_and_remove_signaled_chunk(mux, ring);  in amdgpu_ring_mux_end_ib()
541 bool amdgpu_mcbp_handle_trailing_fence_irq(struct amdgpu_ring_mux *mux)  in amdgpu_mcbp_handle_trailing_fence_irq()  argument
547 	if (!mux->pending_trailing_fence_signaled)  in amdgpu_mcbp_handle_trailing_fence_irq()
550 	if (mux->real_ring->trail_seq != le32_to_cpu(*mux->real_ring->trail_fence_cpu_addr))  in amdgpu_mcbp_handle_trailing_fence_irq()
553 	for (i = 0; i < mux->num_ring_entries; i++) {  in amdgpu_mcbp_handle_trailing_fence_irq()
554 		e = &mux->ring_entry[i];  in amdgpu_mcbp_handle_trailing_fence_irq()
568 		mux->s_resubmit = true;  in amdgpu_mcbp_handle_trailing_fence_irq()
569 		mux->seqno_to_resubmit = ring->fence_drv.sync_seq;  in amdgpu_mcbp_handle_trailing_fence_irq()
570 		amdgpu_ring_mux_schedule_resubmit(mux);  in amdgpu_mcbp_handle_trailing_fence_irq()
573 	mux->pending_trailing_fence_signaled = false;  in amdgpu_mcbp_handle_trailing_fence_irq()