225 	req.log2_throttle_limit = queue->eq.log2_throttle_limit;  in mana_gd_create_hw_eq()
226 	req.eq_pci_msix_index = queue->eq.msix_index;  in mana_gd_create_hw_eq()
236 	queue->eq.disable_needed = true;  in mana_gd_create_hw_eq()
282 		e.eq.id = qid;  in mana_gd_ring_doorbell()
283 		e.eq.tail_ptr = tail_ptr;  in mana_gd_ring_doorbell()
284 		e.eq.arm = num_req;  in mana_gd_ring_doorbell()
344 static void mana_gd_process_eqe(struct gdma_queue *eq)  in mana_gd_process_eqe()  argument
346 	u32 head = eq->head % (eq->queue_size / GDMA_EQE_SIZE);  in mana_gd_process_eqe()
347 	struct gdma_context *gc = eq->gdma_dev->gdma_context;  in mana_gd_process_eqe()
348 	struct gdma_eqe *eq_eqe_ptr = eq->queue_mem_ptr;  in mana_gd_process_eqe()
376 		gc->test_event_eq_id = eq->id;  in mana_gd_process_eqe()
384 		if (!eq->eq.callback)  in mana_gd_process_eqe()
389 		eq->eq.callback(eq->eq.context, eq, &event);  in mana_gd_process_eqe()
402 	struct gdma_queue *eq = arg;  in mana_gd_process_eq_events()  local
408 	gc = eq->gdma_dev->gdma_context;  in mana_gd_process_eq_events()
410 	num_eqe = eq->queue_size / GDMA_EQE_SIZE;  in mana_gd_process_eq_events()
411 	eq_eqe_ptr = eq->queue_mem_ptr;  in mana_gd_process_eq_events()
415 		eqe = &eq_eqe_ptr[eq->head % num_eqe];  in mana_gd_process_eq_events()
419 		old_bits = (eq->head / num_eqe - 1) & GDMA_EQE_OWNER_MASK;  in mana_gd_process_eq_events()
428 		new_bits = (eq->head / num_eqe) & GDMA_EQE_OWNER_MASK;  in mana_gd_process_eq_events()
430 			dev_err(gc->dev, "EQ %d: overflow detected\n", eq->id);  in mana_gd_process_eq_events()
439 		mana_gd_process_eqe(eq);  in mana_gd_process_eq_events()
441 		eq->head++;  in mana_gd_process_eq_events()
444 	head = eq->head % (num_eqe << GDMA_EQE_OWNER_BITS);  in mana_gd_process_eq_events()
446 	mana_gd_ring_doorbell(gc, eq->gdma_dev->doorbell, eq->type, eq->id,  in mana_gd_process_eq_events()
463 	msi_index = spec->eq.msix_index;  in mana_gd_register_irq()
473 	queue->eq.msix_index = msi_index;  in mana_gd_register_irq()
490 	struct gdma_queue *eq;  in mana_gd_deregiser_irq()  local
495 	msix_index = queue->eq.msix_index;  in mana_gd_deregiser_irq()
501 	list_for_each_entry_rcu(eq, &gic->eq_list, entry) {  in mana_gd_deregiser_irq()
502 		if (queue == eq) {  in mana_gd_deregiser_irq()
503 			list_del_rcu(&eq->entry);  in mana_gd_deregiser_irq()
509 	queue->eq.msix_index = INVALID_PCI_MSIX_INDEX;  in mana_gd_deregiser_irq()
513 int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq)  in mana_gd_test_eq()  argument
528 	req.hdr.dev_id = eq->gdma_dev->dev_id;  in mana_gd_test_eq()
529 	req.queue_index = eq->id;  in mana_gd_test_eq()
545 		dev_err(dev, "test_eq timed out on queue %d\n", eq->id);  in mana_gd_test_eq()
549 	if (eq->id != gc->test_event_eq_id) {  in mana_gd_test_eq()
551 			gc->test_event_eq_id, eq->id);  in mana_gd_test_eq()
569 			dev_warn(gc->dev, "Failed to flush EQ: %d\n", err);  in mana_gd_destroy_eq()
574 	if (queue->eq.disable_needed)  in mana_gd_destroy_eq()
587 	queue->eq.msix_index = INVALID_PCI_MSIX_INDEX;  in mana_gd_create_eq()
592 	if (spec->eq.log2_throttle_limit > log2_num_entries) {  in mana_gd_create_eq()
593 		dev_err(dev, "EQ throttling limit (%lu) > maximum EQE (%u)\n",  in mana_gd_create_eq()
594 			spec->eq.log2_throttle_limit, log2_num_entries);  in mana_gd_create_eq()
604 	queue->eq.callback = spec->eq.callback;  in mana_gd_create_eq()
605 	queue->eq.context = spec->eq.context;  in mana_gd_create_eq()
607 	queue->eq.log2_throttle_limit = spec->eq.log2_throttle_limit ?: 1;  in mana_gd_create_eq()
621 	dev_err(dev, "Failed to create EQ: %d\n", err);  in mana_gd_create_eq()
874 		mana_gd_destroy_eq(gc, queue->eq.disable_needed, queue);  in mana_gd_destroy_queue()
1223 	struct gdma_queue *eq;  in mana_gd_intr()  local
1226 	list_for_each_entry_rcu(eq, eq_list, entry) {  in mana_gd_intr()
1227 		gic->handler(eq);  in mana_gd_intr()