11 void notify_eq(struct erdma_eq *eq)  in notify_eq()  argument
13 	u64 db_data = FIELD_PREP(ERDMA_EQDB_CI_MASK, eq->ci) |  in notify_eq()
16 	*eq->dbrec = db_data;  in notify_eq()
17 	writeq(db_data, eq->db);  in notify_eq()
19 	atomic64_inc(&eq->notify_num);  in notify_eq()
22 void *get_next_valid_eqe(struct erdma_eq *eq)  in get_next_valid_eqe()  argument
24 	u64 *eqe = get_queue_entry(eq->qbuf, eq->ci, eq->depth, EQE_SHIFT);  in get_next_valid_eqe()
27 	return owner ^ !!(eq->ci & eq->depth) ? eqe : NULL;  in get_next_valid_eqe()
83 int erdma_eq_common_init(struct erdma_dev *dev, struct erdma_eq *eq, u32 depth)  in erdma_eq_common_init()  argument
87 	eq->qbuf = dma_alloc_coherent(&dev->pdev->dev, buf_size,  in erdma_eq_common_init()
88 				      &eq->qbuf_dma_addr, GFP_KERNEL);  in erdma_eq_common_init()
89 	if (!eq->qbuf)  in erdma_eq_common_init()
92 	eq->dbrec = dma_pool_zalloc(dev->db_pool, GFP_KERNEL, &eq->dbrec_dma);  in erdma_eq_common_init()
93 	if (!eq->dbrec)  in erdma_eq_common_init()
96 	spin_lock_init(&eq->lock);  in erdma_eq_common_init()
97 	atomic64_set(&eq->event_num, 0);  in erdma_eq_common_init()
98 	atomic64_set(&eq->notify_num, 0);  in erdma_eq_common_init()
99 	eq->ci = 0;  in erdma_eq_common_init()
100 	eq->depth = depth;  in erdma_eq_common_init()
105 	dma_free_coherent(&dev->pdev->dev, buf_size, eq->qbuf,  in erdma_eq_common_init()
106 			  eq->qbuf_dma_addr);  in erdma_eq_common_init()
111 void erdma_eq_destroy(struct erdma_dev *dev, struct erdma_eq *eq)  in erdma_eq_destroy()  argument
113 	dma_pool_free(dev->db_pool, eq->dbrec, eq->dbrec_dma);  in erdma_eq_destroy()
114 	dma_free_coherent(&dev->pdev->dev, eq->depth << EQE_SHIFT, eq->qbuf,  in erdma_eq_destroy()
115 			  eq->qbuf_dma_addr);  in erdma_eq_destroy()
120 	struct erdma_eq *eq = &dev->aeq;  in erdma_aeq_init()  local
127 	eq->db = dev->func_bar + ERDMA_REGS_AEQ_DB_REG;  in erdma_aeq_init()
130 			  upper_32_bits(eq->qbuf_dma_addr));  in erdma_aeq_init()
132 			  lower_32_bits(eq->qbuf_dma_addr));  in erdma_aeq_init()
133 	erdma_reg_write32(dev, ERDMA_REGS_AEQ_DEPTH_REG, eq->depth);  in erdma_aeq_init()
134 	erdma_reg_write64(dev, ERDMA_AEQ_DB_HOST_ADDR_REG, eq->dbrec_dma);  in erdma_aeq_init()
151 		ceqe = get_next_valid_eqe(&ceq_cb->eq);  in erdma_ceq_completion_handler()
156 		ceq_cb->eq.ci++;  in erdma_ceq_completion_handler()
171 	notify_eq(&ceq_cb->eq);  in erdma_ceq_completion_handler()
224 static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq)  in create_eq_cmd()  argument
231 	req.depth = ilog2(eq->depth);  in create_eq_cmd()
232 	req.qbuf_addr = eq->qbuf_dma_addr;  in create_eq_cmd()
236 	req.db_dma_addr_l = lower_32_bits(eq->dbrec_dma);  in create_eq_cmd()
237 	req.db_dma_addr_h = upper_32_bits(eq->dbrec_dma);  in create_eq_cmd()
244 	struct erdma_eq *eq = &dev->ceqs[ceqn].eq;  in erdma_ceq_init_one()  local
247 	ret = erdma_eq_common_init(dev, eq, ERDMA_DEFAULT_EQ_DEPTH);  in erdma_ceq_init_one()
251 	eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG +  in erdma_ceq_init_one()
256 	/* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */  in erdma_ceq_init_one()
257 	ret = create_eq_cmd(dev, ceqn + 1, eq);  in erdma_ceq_init_one()
259 		erdma_eq_destroy(dev, eq);  in erdma_ceq_init_one()
268 	struct erdma_eq *eq = &dev->ceqs[ceqn].eq;  in erdma_ceq_uninit_one()  local
276 	/* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */  in erdma_ceq_uninit_one()
285 	erdma_eq_destroy(dev, eq);  in erdma_ceq_uninit_one()