43 	struct hl_device *hdev = cs->ctx->hdev;  in hl_hw_queue_update_ci()  local
47 	if (hdev->disabled)  in hl_hw_queue_update_ci()
50 	q = &hdev->kernel_queues[0];  in hl_hw_queue_update_ci()
53 	if (!hdev->asic_prop.max_queues || q->queue_type == QUEUE_TYPE_HW)  in hl_hw_queue_update_ci()
61 	for (i = 0 ; i < hdev->asic_prop.max_queues ; i++, q++) {  in hl_hw_queue_update_ci()
83 void hl_hw_queue_submit_bd(struct hl_device *hdev, struct hl_hw_queue *q,  in hl_hw_queue_submit_bd()  argument
100 			hdev->asic_funcs->access_dev_mem(hdev, PCI_REGION_DRAM,	addr,  in hl_hw_queue_submit_bd()
106 	hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);  in hl_hw_queue_submit_bd()
127 static int ext_queue_sanity_checks(struct hl_device *hdev,  in ext_queue_sanity_checks()  argument
132 			&hdev->completion_queue[q->cq_id].free_slots_cnt;  in ext_queue_sanity_checks()
139 		dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",  in ext_queue_sanity_checks()
153 			dev_dbg(hdev->dev, "No space for %d on CQ %d\n",  in ext_queue_sanity_checks()
176 static int int_queue_sanity_checks(struct hl_device *hdev,  in int_queue_sanity_checks()  argument
183 		dev_err(hdev->dev,  in int_queue_sanity_checks()
193 		dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",  in int_queue_sanity_checks()
211 static int hw_queue_sanity_checks(struct hl_device *hdev, struct hl_hw_queue *q,  in hw_queue_sanity_checks()  argument
220 		dev_dbg(hdev->dev, "Queue %d doesn't have room for %d CBs\n",  in hw_queue_sanity_checks()
239 int hl_hw_queue_send_cb_no_cmpl(struct hl_device *hdev, u32 hw_queue_id,  in hl_hw_queue_send_cb_no_cmpl()  argument
242 	struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];  in hl_hw_queue_send_cb_no_cmpl()
245 	hdev->asic_funcs->hw_queues_lock(hdev);  in hl_hw_queue_send_cb_no_cmpl()
247 	if (hdev->disabled) {  in hl_hw_queue_send_cb_no_cmpl()
258 		rc = ext_queue_sanity_checks(hdev, q, 1, false);  in hl_hw_queue_send_cb_no_cmpl()
263 	hl_hw_queue_submit_bd(hdev, q, 0, cb_size, cb_ptr);  in hl_hw_queue_send_cb_no_cmpl()
266 	hdev->asic_funcs->hw_queues_unlock(hdev);  in hl_hw_queue_send_cb_no_cmpl()
281 	struct hl_device *hdev = job->cs->ctx->hdev;  in ext_queue_schedule_job()  local
282 	struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];  in ext_queue_schedule_job()
318 	cq = &hdev->completion_queue[q->cq_id];  in ext_queue_schedule_job()
321 	hdev->asic_funcs->add_end_of_cb_packets(hdev, cb->kernel_address, len,  in ext_queue_schedule_job()
333 	hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);  in ext_queue_schedule_job()
346 	struct hl_device *hdev = job->cs->ctx->hdev;  in int_queue_schedule_job()  local
347 	struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];  in int_queue_schedule_job()
367 	hdev->asic_funcs->pqe_write(hdev, pi, &bd);  in int_queue_schedule_job()
369 	hdev->asic_funcs->ring_doorbell(hdev, q->hw_queue_id, q->pi);  in int_queue_schedule_job()
382 	struct hl_device *hdev = job->cs->ctx->hdev;  in hw_queue_schedule_job()  local
383 	struct hl_hw_queue *q = &hdev->kernel_queues[job->hw_queue_id];  in hw_queue_schedule_job()
393 	offset = job->cs->sequence & (hdev->asic_prop.max_pending_cs - 1);  in hw_queue_schedule_job()
412 	hl_hw_queue_submit_bd(hdev, q, ctl, len, ptr);  in hw_queue_schedule_job()
415 static int init_signal_cs(struct hl_device *hdev,  in init_signal_cs()  argument
424 	prop = &hdev->kernel_queues[q_idx].sync_stream_prop;  in init_signal_cs()
430 	dev_dbg(hdev->dev,  in init_signal_cs()
438 	hdev->asic_funcs->gen_signal_cb(hdev, job->patched_cb,  in init_signal_cs()
441 	rc = hl_cs_signal_sob_wraparound_handler(hdev, q_idx, &hw_sob, 1,  in init_signal_cs()
450 void hl_hw_queue_encaps_sig_set_sob_info(struct hl_device *hdev,  in hl_hw_queue_encaps_sig_set_sob_info()  argument
474 static int init_wait_cs(struct hl_device *hdev, struct hl_cs *cs,  in init_wait_cs()  argument
483 	prop = &hdev->kernel_queues[q_idx].sync_stream_prop;  in init_wait_cs()
494 		hl_hw_queue_encaps_sig_set_sob_info(hdev, cs, job, cs_cmpl);  in init_wait_cs()
496 …dev_dbg(hdev->dev, "Wait for encaps signals handle, qidx(%u), CS sequence(%llu), sob val: 0x%x, of…  in init_wait_cs()
530 	dev_dbg(hdev->dev,  in init_wait_cs()
543 	hdev->asic_funcs->gen_wait_cb(hdev, &wait_prop);  in init_wait_cs()
561 	struct hl_device *hdev = ctx->hdev;  in init_signal_wait_cs()  local
572 		rc = init_signal_cs(hdev, job, cs_cmpl);  in init_signal_wait_cs()
574 		rc = init_wait_cs(hdev, cs, job, cs_cmpl);  in init_signal_wait_cs()
580 			(struct hl_device *hdev, struct hl_cs *cs)  in encaps_sig_first_staged_cs_handler()  argument
614 …dev_dbg(hdev->dev, "CS seq (%llu) added to encaps signal handler id (%u), count(%u), qidx(%u), sob…  in encaps_sig_first_staged_cs_handler()
622 		dev_err(hdev->dev, "encaps handle id(%u) wasn't found!\n",  in encaps_sig_first_staged_cs_handler()
641 	struct hl_device *hdev = ctx->hdev;  in hl_hw_queue_schedule_cs()  local
648 	cntr = &hdev->aggregated_cs_counters;  in hl_hw_queue_schedule_cs()
650 	hdev->asic_funcs->hw_queues_lock(hdev);  in hl_hw_queue_schedule_cs()
652 	if (!hl_device_operational(hdev, &status)) {  in hl_hw_queue_schedule_cs()
655 		dev_err(hdev->dev,  in hl_hw_queue_schedule_cs()
656 			"device is %s, CS rejected!\n", hdev->status[status]);  in hl_hw_queue_schedule_cs()
661 	max_queues = hdev->asic_prop.max_queues;  in hl_hw_queue_schedule_cs()
663 	q = &hdev->kernel_queues[0];  in hl_hw_queue_schedule_cs()
668 				rc = ext_queue_sanity_checks(hdev, q,  in hl_hw_queue_schedule_cs()
674 				rc = int_queue_sanity_checks(hdev, q,  in hl_hw_queue_schedule_cs()
678 				rc = hw_queue_sanity_checks(hdev, q,  in hl_hw_queue_schedule_cs()
682 				dev_err(hdev->dev, "Queue type %d is invalid\n",  in hl_hw_queue_schedule_cs()
705 		rc = hdev->asic_funcs->collective_wait_init_cs(cs);  in hl_hw_queue_schedule_cs()
710 	rc = hdev->asic_funcs->pre_schedule_cs(cs);  in hl_hw_queue_schedule_cs()
712 		dev_err(hdev->dev,  in hl_hw_queue_schedule_cs()
718 	hdev->shadow_cs_queue[cs->sequence &  in hl_hw_queue_schedule_cs()
719 				(hdev->asic_prop.max_pending_cs - 1)] = cs;  in hl_hw_queue_schedule_cs()
722 		rc = encaps_sig_first_staged_cs_handler(hdev, cs);  in hl_hw_queue_schedule_cs()
727 	spin_lock(&hdev->cs_mirror_lock);  in hl_hw_queue_schedule_cs()
733 		staged_cs = hl_staged_cs_find_first(hdev, cs->staged_sequence);  in hl_hw_queue_schedule_cs()
735 			dev_err(hdev->dev,  in hl_hw_queue_schedule_cs()
742 		if (is_staged_cs_last_exists(hdev, staged_cs)) {  in hl_hw_queue_schedule_cs()
743 			dev_err(hdev->dev,  in hl_hw_queue_schedule_cs()
753 		if (hdev->supports_wait_for_multi_cs)  in hl_hw_queue_schedule_cs()
758 	list_add_tail(&cs->mirror_node, &hdev->cs_mirror_list);  in hl_hw_queue_schedule_cs()
761 	first_entry = list_first_entry(&hdev->cs_mirror_list,  in hl_hw_queue_schedule_cs()
763 	if ((hdev->timeout_jiffies != MAX_SCHEDULE_TIMEOUT) &&  in hl_hw_queue_schedule_cs()
770 	spin_unlock(&hdev->cs_mirror_lock);  in hl_hw_queue_schedule_cs()
792 	spin_unlock(&hdev->cs_mirror_lock);  in hl_hw_queue_schedule_cs()
794 	q = &hdev->kernel_queues[0];  in hl_hw_queue_schedule_cs()
799 				&hdev->completion_queue[i].free_slots_cnt;  in hl_hw_queue_schedule_cs()
806 	hdev->asic_funcs->hw_queues_unlock(hdev);  in hl_hw_queue_schedule_cs()
817 void hl_hw_queue_inc_ci_kernel(struct hl_device *hdev, u32 hw_queue_id)  in hl_hw_queue_inc_ci_kernel()  argument
819 	struct hl_hw_queue *q = &hdev->kernel_queues[hw_queue_id];  in hl_hw_queue_inc_ci_kernel()
824 static int ext_and_cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q,  in ext_and_cpu_queue_init()  argument
831 		p = hl_cpu_accessible_dma_pool_alloc(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address);  in ext_and_cpu_queue_init()
833 		p = hl_asic_dma_alloc_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address,  in ext_and_cpu_queue_init()
842 		dev_err(hdev->dev,  in ext_and_cpu_queue_init()
857 		hl_cpu_accessible_dma_pool_free(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address);  in ext_and_cpu_queue_init()
859 		hl_asic_dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address,  in ext_and_cpu_queue_init()
865 static int int_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in int_queue_init()  argument
869 	p = hdev->asic_funcs->get_int_queue_base(hdev, q->hw_queue_id,  in int_queue_init()
872 		dev_err(hdev->dev,  in int_queue_init()
885 static int cpu_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in cpu_queue_init()  argument
887 	return ext_and_cpu_queue_init(hdev, q, true);  in cpu_queue_init()
890 static int ext_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in ext_queue_init()  argument
892 	return ext_and_cpu_queue_init(hdev, q, false);  in ext_queue_init()
895 static int hw_queue_init(struct hl_device *hdev, struct hl_hw_queue *q)  in hw_queue_init()  argument
899 	p = hl_asic_dma_alloc_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, &q->bus_address,  in hw_queue_init()
913 static void sync_stream_queue_init(struct hl_device *hdev, u32 q_idx)  in sync_stream_queue_init()  argument
916 	struct asic_fixed_properties *prop = &hdev->asic_prop;  in sync_stream_queue_init()
920 	sync_stream_prop = &hdev->kernel_queues[q_idx].sync_stream_prop;  in sync_stream_queue_init()
927 	if (hdev->kernel_queues[q_idx].collective_mode ==  in sync_stream_queue_init()
929 		reserved_mon_idx = hdev->collective_mon_idx;  in sync_stream_queue_init()
939 		hdev->collective_mon_idx += HL_COLLECTIVE_RSVD_MSTR_MONS;  in sync_stream_queue_init()
940 	} else if (hdev->kernel_queues[q_idx].collective_mode ==  in sync_stream_queue_init()
942 		reserved_mon_idx = hdev->collective_mon_idx++;  in sync_stream_queue_init()
949 	if (!hdev->kernel_queues[q_idx].supports_sync_stream)  in sync_stream_queue_init()
952 	queue_idx = hdev->sync_stream_queue_idx++;  in sync_stream_queue_init()
963 		hw_sob->hdev = hdev;  in sync_stream_queue_init()
966 			hdev->asic_funcs->get_sob_addr(hdev, hw_sob->sob_id);  in sync_stream_queue_init()
972 static void sync_stream_queue_reset(struct hl_device *hdev, u32 q_idx)  in sync_stream_queue_reset()  argument
975 			&hdev->kernel_queues[q_idx].sync_stream_prop;  in sync_stream_queue_reset()
996 static int queue_init(struct hl_device *hdev, struct hl_hw_queue *q,  in queue_init()  argument
1005 		rc = ext_queue_init(hdev, q);  in queue_init()
1008 		rc = int_queue_init(hdev, q);  in queue_init()
1011 		rc = cpu_queue_init(hdev, q);  in queue_init()
1014 		rc = hw_queue_init(hdev, q);  in queue_init()
1020 		dev_crit(hdev->dev, "wrong queue type %d during init\n",  in queue_init()
1026 	sync_stream_queue_init(hdev, q->hw_queue_id);  in queue_init()
1044 static void queue_fini(struct hl_device *hdev, struct hl_hw_queue *q)  in queue_fini()  argument
1073 		hl_cpu_accessible_dma_pool_free(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address);  in queue_fini()
1075 		hl_asic_dma_free_coherent(hdev, HL_QUEUE_SIZE_IN_BYTES, q->kernel_address,  in queue_fini()
1079 int hl_hw_queues_create(struct hl_device *hdev)  in hl_hw_queues_create()  argument
1081 	struct asic_fixed_properties *asic = &hdev->asic_prop;  in hl_hw_queues_create()
1085 	hdev->kernel_queues = kcalloc(asic->max_queues,  in hl_hw_queues_create()
1086 				sizeof(*hdev->kernel_queues), GFP_KERNEL);  in hl_hw_queues_create()
1088 	if (!hdev->kernel_queues) {  in hl_hw_queues_create()
1089 		dev_err(hdev->dev, "Not enough memory for H/W queues\n");  in hl_hw_queues_create()
1094 	for (i = 0, q_ready_cnt = 0, q = hdev->kernel_queues;  in hl_hw_queues_create()
1103 		rc = queue_init(hdev, q, i);  in hl_hw_queues_create()
1105 			dev_err(hdev->dev,  in hl_hw_queues_create()
1118 	for (i = 0, q = hdev->kernel_queues ; i < q_ready_cnt ; i++, q++)  in hl_hw_queues_create()
1119 		queue_fini(hdev, q);  in hl_hw_queues_create()
1121 	kfree(hdev->kernel_queues);  in hl_hw_queues_create()
1126 void hl_hw_queues_destroy(struct hl_device *hdev)  in hl_hw_queues_destroy()  argument
1129 	u32 max_queues = hdev->asic_prop.max_queues;  in hl_hw_queues_destroy()
1132 	for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++)  in hl_hw_queues_destroy()
1133 		queue_fini(hdev, q);  in hl_hw_queues_destroy()
1135 	kfree(hdev->kernel_queues);  in hl_hw_queues_destroy()
1138 void hl_hw_queue_reset(struct hl_device *hdev, bool hard_reset)  in hl_hw_queue_reset()  argument
1141 	u32 max_queues = hdev->asic_prop.max_queues;  in hl_hw_queue_reset()
1144 	for (i = 0, q = hdev->kernel_queues ; i < max_queues ; i++, q++) {  in hl_hw_queue_reset()
1152 			sync_stream_queue_reset(hdev, q->hw_queue_id);  in hl_hw_queue_reset()