58 static int alloc_cqn(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)  in alloc_cqn()  argument
76 	hr_cq->cqn = (id << CQ_BANKID_SHIFT) | bankid;  in alloc_cqn()
104 			       struct hns_roce_cq *hr_cq,  in hns_roce_create_cqc()  argument
117 	hr_dev->hw->write_cqc(hr_dev, hr_cq, mailbox->buf, mtts, dma_handle);  in hns_roce_create_cqc()
120 				     hr_cq->cqn);  in hns_roce_create_cqc()
124 			  hr_cq->cqn, ret);  in hns_roce_create_cqc()
131 static int alloc_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)  in alloc_cqc()  argument
138 	ret = hns_roce_mtr_find(hr_dev, &hr_cq->mtr, 0, mtts, ARRAY_SIZE(mtts));  in alloc_cqc()
145 	ret = hns_roce_table_get(hr_dev, &cq_table->table, hr_cq->cqn);  in alloc_cqc()
148 			  hr_cq->cqn, ret);  in alloc_cqc()
152 	ret = xa_err(xa_store_irq(&cq_table->array, hr_cq->cqn, hr_cq, GFP_KERNEL));  in alloc_cqc()
158 	ret = hns_roce_create_cqc(hr_dev, hr_cq, mtts,  in alloc_cqc()
159 				  hns_roce_get_mtr_ba(&hr_cq->mtr));  in alloc_cqc()
166 	xa_erase_irq(&cq_table->array, hr_cq->cqn);  in alloc_cqc()
168 	hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn);  in alloc_cqc()
173 static void free_cqc(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)  in free_cqc()  argument
180 				      hr_cq->cqn);  in free_cqc()
183 			hr_cq->cqn);  in free_cqc()
185 	xa_erase_irq(&cq_table->array, hr_cq->cqn);  in free_cqc()
188 	synchronize_irq(hr_dev->eq_table.eq[hr_cq->vector].irq);  in free_cqc()
191 	if (refcount_dec_and_test(&hr_cq->refcount))  in free_cqc()
192 		complete(&hr_cq->free);  in free_cqc()
193 	wait_for_completion(&hr_cq->free);  in free_cqc()
195 	hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn);  in free_cqc()
198 static int alloc_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,  in alloc_cq_buf()  argument
206 	buf_attr.region[0].size = hr_cq->cq_depth * hr_cq->cqe_size;  in alloc_cq_buf()
210 	ret = hns_roce_mtr_create(hr_dev, &hr_cq->mtr, &buf_attr,  in alloc_cq_buf()
219 static void free_cq_buf(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq)  in free_cq_buf()  argument
221 	hns_roce_mtr_destroy(hr_dev, &hr_cq->mtr);  in free_cq_buf()
224 static int alloc_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,  in alloc_cq_db()  argument
237 			err = hns_roce_db_map_user(uctx, addr, &hr_cq->db);  in alloc_cq_db()
240 			hr_cq->flags |= HNS_ROCE_CQ_FLAG_RECORD_DB;  in alloc_cq_db()
245 			err = hns_roce_alloc_db(hr_dev, &hr_cq->db, 1);  in alloc_cq_db()
248 			hr_cq->set_ci_db = hr_cq->db.db_record;  in alloc_cq_db()
249 			*hr_cq->set_ci_db = 0;  in alloc_cq_db()
250 			hr_cq->flags |= HNS_ROCE_CQ_FLAG_RECORD_DB;  in alloc_cq_db()
252 		hr_cq->db_reg = hr_dev->reg_base + hr_dev->odb_offset +  in alloc_cq_db()
259 static void free_cq_db(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq,  in free_cq_db()  argument
264 	if (!(hr_cq->flags & HNS_ROCE_CQ_FLAG_RECORD_DB))  in free_cq_db()
267 	hr_cq->flags &= ~HNS_ROCE_CQ_FLAG_RECORD_DB;  in free_cq_db()
272 		hns_roce_db_unmap_user(uctx, &hr_cq->db);  in free_cq_db()
274 		hns_roce_free_db(hr_dev, &hr_cq->db);  in free_cq_db()
298 static int get_cq_ucmd(struct hns_roce_cq *hr_cq, struct ib_udata *udata,  in get_cq_ucmd()  argument
301 	struct ib_device *ibdev = hr_cq->ib_cq.device;  in get_cq_ucmd()
313 static void set_cq_param(struct hns_roce_cq *hr_cq, u32 cq_entries, int vector,  in set_cq_param()  argument
316 	struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);  in set_cq_param()
320 	hr_cq->ib_cq.cqe = cq_entries - 1; /* used as cqe index */  in set_cq_param()
321 	hr_cq->cq_depth = cq_entries;  in set_cq_param()
322 	hr_cq->vector = vector;  in set_cq_param()
324 	spin_lock_init(&hr_cq->lock);  in set_cq_param()
325 	INIT_LIST_HEAD(&hr_cq->sq_list);  in set_cq_param()
326 	INIT_LIST_HEAD(&hr_cq->rq_list);  in set_cq_param()
329 static int set_cqe_size(struct hns_roce_cq *hr_cq, struct ib_udata *udata,  in set_cqe_size()  argument
332 	struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);  in set_cqe_size()
335 		hr_cq->cqe_size = hr_dev->caps.cqe_sz;  in set_cqe_size()
347 		hr_cq->cqe_size = ucmd->cqe_size;  in set_cqe_size()
349 		hr_cq->cqe_size = HNS_ROCE_V2_CQE_SIZE;  in set_cqe_size()
361 	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);  in hns_roce_create_cq()  local
376 		ret = get_cq_ucmd(hr_cq, udata, &ucmd);  in hns_roce_create_cq()
381 	set_cq_param(hr_cq, attr->cqe, attr->comp_vector, &ucmd);  in hns_roce_create_cq()
383 	ret = set_cqe_size(hr_cq, udata, &ucmd);  in hns_roce_create_cq()
387 	ret = alloc_cq_buf(hr_dev, hr_cq, udata, ucmd.buf_addr);  in hns_roce_create_cq()
393 	ret = alloc_cq_db(hr_dev, hr_cq, udata, ucmd.db_addr, &resp);  in hns_roce_create_cq()
399 	ret = alloc_cqn(hr_dev, hr_cq);  in hns_roce_create_cq()
405 	ret = alloc_cqc(hr_dev, hr_cq);  in hns_roce_create_cq()
413 		resp.cqn = hr_cq->cqn;  in hns_roce_create_cq()
420 	hr_cq->cons_index = 0;  in hns_roce_create_cq()
421 	hr_cq->arm_sn = 1;  in hns_roce_create_cq()
422 	refcount_set(&hr_cq->refcount, 1);  in hns_roce_create_cq()
423 	init_completion(&hr_cq->free);  in hns_roce_create_cq()
428 	free_cqc(hr_dev, hr_cq);  in hns_roce_create_cq()
430 	free_cqn(hr_dev, hr_cq->cqn);  in hns_roce_create_cq()
432 	free_cq_db(hr_dev, hr_cq, udata);  in hns_roce_create_cq()
434 	free_cq_buf(hr_dev, hr_cq);  in hns_roce_create_cq()
444 	struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);  in hns_roce_destroy_cq()  local
446 	free_cqc(hr_dev, hr_cq);  in hns_roce_destroy_cq()
447 	free_cqn(hr_dev, hr_cq->cqn);  in hns_roce_destroy_cq()
448 	free_cq_db(hr_dev, hr_cq, udata);  in hns_roce_destroy_cq()
449 	free_cq_buf(hr_dev, hr_cq);  in hns_roce_destroy_cq()
456 	struct hns_roce_cq *hr_cq;  in hns_roce_cq_completion()  local
459 	hr_cq = xa_load(&hr_dev->cq_table.array,  in hns_roce_cq_completion()
461 	if (!hr_cq) {  in hns_roce_cq_completion()
467 	++hr_cq->arm_sn;  in hns_roce_cq_completion()
468 	ibcq = &hr_cq->ib_cq;  in hns_roce_cq_completion()
476 	struct hns_roce_cq *hr_cq;  in hns_roce_cq_event()  local
489 	hr_cq = xa_load(&hr_dev->cq_table.array,  in hns_roce_cq_event()
491 	if (hr_cq)  in hns_roce_cq_event()
492 		refcount_inc(&hr_cq->refcount);  in hns_roce_cq_event()
494 	if (!hr_cq) {  in hns_roce_cq_event()
499 	ibcq = &hr_cq->ib_cq;  in hns_roce_cq_event()
507 	if (refcount_dec_and_test(&hr_cq->refcount))  in hns_roce_cq_event()
508 		complete(&hr_cq->free);  in hns_roce_cq_event()