23 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_init_pf_regs()  local
27 	gc->db_page_size = mana_gd_r32(gc, GDMA_PF_REG_DB_PAGE_SIZE) & 0xFFFF;  in mana_gd_init_pf_regs()
28 	gc->db_page_base = gc->bar0_va +  in mana_gd_init_pf_regs()
29 				mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF);  in mana_gd_init_pf_regs()
31 	sriov_base_off = mana_gd_r64(gc, GDMA_SRIOV_REG_CFG_BASE_OFF);  in mana_gd_init_pf_regs()
33 	sriov_base_va = gc->bar0_va + sriov_base_off;  in mana_gd_init_pf_regs()
34 	gc->shm_base = sriov_base_va +  in mana_gd_init_pf_regs()
35 			mana_gd_r64(gc, sriov_base_off + GDMA_PF_REG_SHM_OFF);  in mana_gd_init_pf_regs()
40 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_init_vf_regs()  local
42 	gc->db_page_size = mana_gd_r32(gc, GDMA_REG_DB_PAGE_SIZE) & 0xFFFF;  in mana_gd_init_vf_regs()
44 	gc->db_page_base = gc->bar0_va +  in mana_gd_init_vf_regs()
45 				mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET);  in mana_gd_init_vf_regs()
47 	gc->phys_db_page_base = gc->bar0_pa +  in mana_gd_init_vf_regs()
48 				mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET);  in mana_gd_init_vf_regs()
50 	gc->shm_base = gc->bar0_va + mana_gd_r64(gc, GDMA_REG_SHM_OFFSET);  in mana_gd_init_vf_regs()
55 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_init_registers()  local
57 	if (gc->is_pf)  in mana_gd_init_registers()
65 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_query_max_resources()  local
73 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_query_max_resources()
75 		dev_err(gc->dev, "Failed to query resource info: %d, 0x%x\n",  in mana_gd_query_max_resources()
80 	if (gc->num_msix_usable > resp.max_msix)  in mana_gd_query_max_resources()
81 		gc->num_msix_usable = resp.max_msix;  in mana_gd_query_max_resources()
83 	if (gc->num_msix_usable <= 1)  in mana_gd_query_max_resources()
86 	gc->max_num_queues = num_online_cpus();  in mana_gd_query_max_resources()
87 	if (gc->max_num_queues > MANA_MAX_NUM_QUEUES)  in mana_gd_query_max_resources()
88 		gc->max_num_queues = MANA_MAX_NUM_QUEUES;  in mana_gd_query_max_resources()
90 	if (gc->max_num_queues > resp.max_eq)  in mana_gd_query_max_resources()
91 		gc->max_num_queues = resp.max_eq;  in mana_gd_query_max_resources()
93 	if (gc->max_num_queues > resp.max_cq)  in mana_gd_query_max_resources()
94 		gc->max_num_queues = resp.max_cq;  in mana_gd_query_max_resources()
96 	if (gc->max_num_queues > resp.max_sq)  in mana_gd_query_max_resources()
97 		gc->max_num_queues = resp.max_sq;  in mana_gd_query_max_resources()
99 	if (gc->max_num_queues > resp.max_rq)  in mana_gd_query_max_resources()
100 		gc->max_num_queues = resp.max_rq;  in mana_gd_query_max_resources()
103 	if (gc->max_num_queues > gc->num_msix_usable - 1)  in mana_gd_query_max_resources()
104 		gc->max_num_queues = gc->num_msix_usable - 1;  in mana_gd_query_max_resources()
111 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_query_hwc_timeout()  local
119 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_query_hwc_timeout()
130 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_detect_devices()  local
141 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_detect_devices()
143 		dev_err(gc->dev, "Failed to detect devices: %d, 0x%x\n", err,  in mana_gd_detect_devices()
159 			gc->mana.gdma_context = gc;  in mana_gd_detect_devices()
160 			gc->mana.dev_id = dev;  in mana_gd_detect_devices()
162 			gc->mana_ib.dev_id = dev;  in mana_gd_detect_devices()
163 			gc->mana_ib.gdma_context = gc;  in mana_gd_detect_devices()
167 	return gc->mana.dev_id.type == 0 ? -ENODEV : 0;  in mana_gd_detect_devices()
170 int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,  in mana_gd_send_request()  argument
173 	struct hw_channel_context *hwc = gc->hwc.driver_data;  in mana_gd_send_request()
179 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,  in mana_gd_alloc_memory()  argument
188 	gmi->dev = gc->dev;  in mana_gd_alloc_memory()
206 static int mana_gd_create_hw_eq(struct gdma_context *gc,  in mana_gd_create_hw_eq()  argument
228 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_create_hw_eq()
230 		dev_err(gc->dev, "Failed to create queue: %d, 0x%x\n", err,  in mana_gd_create_hw_eq()
243 	struct gdma_context *gc = queue->gdma_dev->gdma_context;  in mana_gd_disable_queue()  local
258 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_disable_queue()
260 		dev_err(gc->dev, "Failed to disable queue: %d, 0x%x\n", err,  in mana_gd_disable_queue()
273 static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index,  in mana_gd_ring_doorbell()  argument
277 	void __iomem *addr = gc->db_page_base + gc->db_page_size * db_index;  in mana_gd_ring_doorbell()
323 void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue)  in mana_gd_wq_ring_doorbell()  argument
328 	mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type,  in mana_gd_wq_ring_doorbell()
334 	struct gdma_context *gc = cq->gdma_dev->gdma_context;  in mana_gd_ring_cq()  local
340 	mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, cq->type, cq->id,  in mana_gd_ring_cq()
347 	struct gdma_context *gc = eq->gdma_dev->gdma_context;  in mana_gd_process_eqe()  local
363 		if (WARN_ON_ONCE(cq_id >= gc->max_num_cqs))  in mana_gd_process_eqe()
366 		cq = gc->cq_table[cq_id];  in mana_gd_process_eqe()
376 		gc->test_event_eq_id = eq->id;  in mana_gd_process_eqe()
377 		complete(&gc->eq_test_event);  in mana_gd_process_eqe()
403 	struct gdma_context *gc;  in mana_gd_process_eq_events()  local
408 	gc = eq->gdma_dev->gdma_context;  in mana_gd_process_eq_events()
430 			dev_err(gc->dev, "EQ %d: overflow detected\n", eq->id);  in mana_gd_process_eq_events()
446 	mana_gd_ring_doorbell(gc, eq->gdma_dev->doorbell, eq->type, eq->id,  in mana_gd_process_eq_events()
455 	struct gdma_context *gc;  in mana_gd_register_irq()  local
461 	gc = gd->gdma_context;  in mana_gd_register_irq()
462 	dev = gc->dev;  in mana_gd_register_irq()
465 	if (msi_index >= gc->num_msix_usable) {  in mana_gd_register_irq()
468 			err, msi_index, gc->num_msix_usable);  in mana_gd_register_irq()
474 	gic = &gc->irq_contexts[msi_index];  in mana_gd_register_irq()
487 	struct gdma_context *gc;  in mana_gd_deregiser_irq()  local
492 	gc = gd->gdma_context;  in mana_gd_deregiser_irq()
496 	if (WARN_ON(msix_index >= gc->num_msix_usable))  in mana_gd_deregiser_irq()
499 	gic = &gc->irq_contexts[msix_index];  in mana_gd_deregiser_irq()
513 int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq)  in mana_gd_test_eq()  argument
517 	struct device *dev = gc->dev;  in mana_gd_test_eq()
520 	mutex_lock(&gc->eq_test_event_mutex);  in mana_gd_test_eq()
522 	init_completion(&gc->eq_test_event);  in mana_gd_test_eq()
523 	gc->test_event_eq_id = INVALID_QUEUE_ID;  in mana_gd_test_eq()
531 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_test_eq()
544 	if (!wait_for_completion_timeout(&gc->eq_test_event, 30 * HZ)) {  in mana_gd_test_eq()
549 	if (eq->id != gc->test_event_eq_id) {  in mana_gd_test_eq()
551 			gc->test_event_eq_id, eq->id);  in mana_gd_test_eq()
557 	mutex_unlock(&gc->eq_test_event_mutex);  in mana_gd_test_eq()
561 static void mana_gd_destroy_eq(struct gdma_context *gc, bool flush_evenets,  in mana_gd_destroy_eq()  argument
567 		err = mana_gd_test_eq(gc, queue);  in mana_gd_destroy_eq()
569 			dev_warn(gc->dev, "Failed to flush EQ: %d\n", err);  in mana_gd_destroy_eq()
582 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_eq()  local
583 	struct device *dev = gc->dev;  in mana_gd_create_eq()
610 		err = mana_gd_create_hw_eq(gc, queue);  in mana_gd_create_eq()
614 		err = mana_gd_test_eq(gc, queue);  in mana_gd_create_eq()
622 	mana_gd_destroy_eq(gc, false, queue);  in mana_gd_create_eq()
637 static void mana_gd_destroy_cq(struct gdma_context *gc,  in mana_gd_destroy_cq()  argument
642 	if (id >= gc->max_num_cqs)  in mana_gd_destroy_cq()
645 	if (!gc->cq_table[id])  in mana_gd_destroy_cq()
648 	gc->cq_table[id] = NULL;  in mana_gd_destroy_cq()
655 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_hwc_queue()  local
665 	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);  in mana_gd_create_hwc_queue()
694 int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle)  in mana_gd_destroy_dma_region()  argument
707 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_destroy_dma_region()
709 		dev_err(gc->dev, "Failed to destroy DMA region: %d, 0x%x\n",  in mana_gd_destroy_dma_region()
724 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_dma_region()  local
737 	hwc = gc->hwc.driver_data;  in mana_gd_create_dma_region()
757 	err = mana_gd_send_request(gc, req_msg_size, req, sizeof(resp), &resp);  in mana_gd_create_dma_region()
763 		dev_err(gc->dev, "Failed to create DMA region: 0x%x\n",  in mana_gd_create_dma_region()
779 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_mana_eq()  local
792 	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);  in mana_gd_create_mana_eq()
826 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_mana_wq_cq()  local
840 	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);  in mana_gd_create_mana_wq_cq()
868 void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue)  in mana_gd_destroy_queue()  argument
874 		mana_gd_destroy_eq(gc, queue->eq.disable_needed, queue);  in mana_gd_destroy_queue()
878 		mana_gd_destroy_cq(gc, queue);  in mana_gd_destroy_queue()
888 		dev_err(gc->dev, "Can't destroy unknown queue: type=%d\n",  in mana_gd_destroy_queue()
893 	mana_gd_destroy_dma_region(gc, gmi->dma_region_handle);  in mana_gd_destroy_queue()
901 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_verify_vf_version()  local
907 	hwc = gc->hwc.driver_data;  in mana_gd_verify_vf_version()
928 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_verify_vf_version()
930 		dev_err(gc->dev, "VfVerifyVersionOutput: %d, status=0x%x\n",  in mana_gd_verify_vf_version()
937 			dev_err(gc->dev, "Failed to set the hwc timeout %d\n", err);  in mana_gd_verify_vf_version()
940 		dev_dbg(gc->dev, "set the hwc timeout to %u\n", hwc->hwc_timeout);  in mana_gd_verify_vf_version()
947 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_register_device()  local
961 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_register_device()
963 		dev_err(gc->dev, "gdma_register_device_resp failed: %d, 0x%x\n",  in mana_gd_register_device()
978 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_deregister_device()  local
991 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_deregister_device()
993 		dev_err(gc->dev, "Failed to deregister device: %d, 0x%x\n",  in mana_gd_deregister_device()
1097 	struct gdma_context *gc;  in mana_gd_post_work_request()  local
1128 		gc = wq->gdma_dev->gdma_context;  in mana_gd_post_work_request()
1129 		dev_err(gc->dev, "unsuccessful flow control!\n");  in mana_gd_post_work_request()
1153 	struct gdma_context *gc = queue->gdma_dev->gdma_context;  in mana_gd_post_and_ring()  local
1160 	mana_gd_wq_ring_doorbell(gc, queue);  in mana_gd_post_and_ring()
1284 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_setup_irqs()  local
1314 	gc->irq_contexts = kcalloc(nvec, sizeof(struct gdma_irq_context),  in mana_gd_setup_irqs()
1316 	if (!gc->irq_contexts) {  in mana_gd_setup_irqs()
1322 		gic = &gc->irq_contexts[i];  in mana_gd_setup_irqs()
1354 				cpu = cpumask_local_spread(i, gc->numa_node);  in mana_gd_setup_irqs()
1368 	err = irq_setup(irqs, (nvec - start_irq_index), gc->numa_node);  in mana_gd_setup_irqs()
1372 	gc->max_num_msix = nvec;  in mana_gd_setup_irqs()
1373 	gc->num_msix_usable = nvec;  in mana_gd_setup_irqs()
1380 		gic = &gc->irq_contexts[j];  in mana_gd_setup_irqs()
1386 	kfree(gc->irq_contexts);  in mana_gd_setup_irqs()
1388 	gc->irq_contexts = NULL;  in mana_gd_setup_irqs()
1397 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_remove_irqs()  local
1401 	if (gc->max_num_msix < 1)  in mana_gd_remove_irqs()
1404 	for (i = 0; i < gc->max_num_msix; i++) {  in mana_gd_remove_irqs()
1409 		gic = &gc->irq_contexts[i];  in mana_gd_remove_irqs()
1418 	gc->max_num_msix = 0;  in mana_gd_remove_irqs()
1419 	gc->num_msix_usable = 0;  in mana_gd_remove_irqs()
1420 	kfree(gc->irq_contexts);  in mana_gd_remove_irqs()
1421 	gc->irq_contexts = NULL;  in mana_gd_remove_irqs()
1426 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_setup()  local
1430 	mana_smc_init(&gc->shm_channel, gc->dev, gc->shm_base);  in mana_gd_setup()
1436 	err = mana_hwc_create_channel(gc);  in mana_gd_setup()
1455 	mana_hwc_destroy_channel(gc);  in mana_gd_setup()
1463 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_cleanup()  local
1465 	mana_hwc_destroy_channel(gc);  in mana_gd_cleanup()
1477 	struct gdma_context *gc;  in mana_gd_probe()  local
1502 	gc = vzalloc(sizeof(*gc));  in mana_gd_probe()
1503 	if (!gc)  in mana_gd_probe()
1506 	mutex_init(&gc->eq_test_event_mutex);  in mana_gd_probe()
1507 	pci_set_drvdata(pdev, gc);  in mana_gd_probe()
1508 	gc->bar0_pa = pci_resource_start(pdev, 0);  in mana_gd_probe()
1514 	gc->numa_node = dev_to_node(&pdev->dev);  in mana_gd_probe()
1515 	gc->is_pf = mana_is_pf(pdev->device);  in mana_gd_probe()
1516 	gc->bar0_va = bar0_va;  in mana_gd_probe()
1517 	gc->dev = &pdev->dev;  in mana_gd_probe()
1523 	err = mana_probe(&gc->mana, false);  in mana_gd_probe()
1535 	vfree(gc);  in mana_gd_probe()
1546 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_remove()  local
1548 	mana_remove(&gc->mana, false);  in mana_gd_remove()
1552 	pci_iounmap(pdev, gc->bar0_va);  in mana_gd_remove()
1554 	vfree(gc);  in mana_gd_remove()
1563 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_suspend()  local
1565 	mana_remove(&gc->mana, true);  in mana_gd_suspend()
1578 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_resume()  local
1585 	err = mana_probe(&gc->mana, true);  in mana_gd_resume()
1595 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_shutdown()  local
1599 	mana_remove(&gc->mana, true);  in mana_gd_shutdown()