25 static int ipc_imem_msg_send_device_sleep(struct iosm_imem *ipc_imem, u32 state)  in ipc_imem_msg_send_device_sleep()  argument
32 	ipc_imem->device_sleep = state;  in ipc_imem_msg_send_device_sleep()
34 	return ipc_protocol_tq_msg_send(ipc_imem->ipc_protocol,  in ipc_imem_msg_send_device_sleep()
38 static bool ipc_imem_dl_skb_alloc(struct iosm_imem *ipc_imem,  in ipc_imem_dl_skb_alloc()  argument
45 	return ipc_protocol_dl_td_prepare(ipc_imem->ipc_protocol, pipe);  in ipc_imem_dl_skb_alloc()
51 static int ipc_imem_tq_td_alloc_timer(struct iosm_imem *ipc_imem, int arg,  in ipc_imem_tq_td_alloc_timer()  argument
59 		struct ipc_pipe *pipe = &ipc_imem->channels[i].dl_pipe;  in ipc_imem_tq_td_alloc_timer()
64 		while (ipc_imem_dl_skb_alloc(ipc_imem, pipe))  in ipc_imem_tq_td_alloc_timer()
72 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_tq_td_alloc_timer()
76 		ipc_imem->hrtimer_period =  in ipc_imem_tq_td_alloc_timer()
78 		if (!hrtimer_active(&ipc_imem->td_alloc_timer))  in ipc_imem_tq_td_alloc_timer()
79 			hrtimer_start(&ipc_imem->td_alloc_timer,  in ipc_imem_tq_td_alloc_timer()
80 				      ipc_imem->hrtimer_period,  in ipc_imem_tq_td_alloc_timer()
88 	struct iosm_imem *ipc_imem =  in ipc_imem_td_alloc_timer_cb()  local
91 	ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_td_alloc_timer, 0, NULL,  in ipc_imem_td_alloc_timer_cb()
97 static int ipc_imem_tq_fast_update_timer_cb(struct iosm_imem *ipc_imem, int arg,  in ipc_imem_tq_fast_update_timer_cb()  argument
100 	ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_tq_fast_update_timer_cb()
109 	struct iosm_imem *ipc_imem =  in ipc_imem_fast_update_timer_cb()  local
112 	ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_fast_update_timer_cb, 0,  in ipc_imem_fast_update_timer_cb()
117 static int ipc_imem_tq_adb_timer_cb(struct iosm_imem *ipc_imem, int arg,  in ipc_imem_tq_adb_timer_cb()  argument
120 	ipc_mux_ul_adb_finish(ipc_imem->mux);  in ipc_imem_tq_adb_timer_cb()
127 	struct iosm_imem *ipc_imem =  in ipc_imem_adb_timer_cb()  local
130 	ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_adb_timer_cb, 0,  in ipc_imem_adb_timer_cb()
135 static int ipc_imem_setup_cp_mux_cap_init(struct iosm_imem *ipc_imem,  in ipc_imem_setup_cp_mux_cap_init()  argument
138 	ipc_mmio_update_cp_capability(ipc_imem->mmio);  in ipc_imem_setup_cp_mux_cap_init()
140 	if (ipc_imem->mmio->mux_protocol == MUX_UNKNOWN) {  in ipc_imem_setup_cp_mux_cap_init()
141 		dev_err(ipc_imem->dev, "Failed to get Mux capability.");  in ipc_imem_setup_cp_mux_cap_init()
145 	cfg->protocol = ipc_imem->mmio->mux_protocol;  in ipc_imem_setup_cp_mux_cap_init()
147 	cfg->ul_flow = (ipc_imem->mmio->has_ul_flow_credit == 1) ?  in ipc_imem_setup_cp_mux_cap_init()
159 void ipc_imem_msg_send_feature_set(struct iosm_imem *ipc_imem,  in ipc_imem_msg_send_feature_set()  argument
166 		ipc_protocol_tq_msg_send(ipc_imem->ipc_protocol,  in ipc_imem_msg_send_feature_set()
170 		ipc_protocol_msg_send(ipc_imem->ipc_protocol,  in ipc_imem_msg_send_feature_set()
178 void ipc_imem_td_update_timer_start(struct iosm_imem *ipc_imem)  in ipc_imem_td_update_timer_start()  argument
181 	if (!ipc_imem->enter_runtime || ipc_imem->td_update_timer_suspended) {  in ipc_imem_td_update_timer_start()
183 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_td_update_timer_start()
188 	if (!hrtimer_active(&ipc_imem->tdupdate_timer)) {  in ipc_imem_td_update_timer_start()
189 		ipc_imem->hrtimer_period =  in ipc_imem_td_update_timer_start()
191 		if (!hrtimer_active(&ipc_imem->tdupdate_timer))  in ipc_imem_td_update_timer_start()
192 			hrtimer_start(&ipc_imem->tdupdate_timer,  in ipc_imem_td_update_timer_start()
193 				      ipc_imem->hrtimer_period,  in ipc_imem_td_update_timer_start()
208 void ipc_imem_adb_timer_start(struct iosm_imem *ipc_imem)  in ipc_imem_adb_timer_start()  argument
210 	if (!hrtimer_active(&ipc_imem->adb_timer)) {  in ipc_imem_adb_timer_start()
211 		ipc_imem->hrtimer_period =  in ipc_imem_adb_timer_start()
213 		hrtimer_start(&ipc_imem->adb_timer,  in ipc_imem_adb_timer_start()
214 			      ipc_imem->hrtimer_period,  in ipc_imem_adb_timer_start()
219 bool ipc_imem_ul_write_td(struct iosm_imem *ipc_imem)  in ipc_imem_ul_write_td()  argument
229 	for (i = 0; i < ipc_imem->nr_of_channels; i++) {  in ipc_imem_ul_write_td()
230 		channel = &ipc_imem->channels[i];  in ipc_imem_ul_write_td()
243 				ipc_protocol_ul_td_send(ipc_imem->ipc_protocol,  in ipc_imem_ul_write_td()
247 				ipc_protocol_ul_td_send(ipc_imem->ipc_protocol,  in ipc_imem_ul_write_td()
255 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_ul_write_td()
262 void ipc_imem_ipc_init_check(struct iosm_imem *ipc_imem)  in ipc_imem_ipc_init_check()  argument
266 	ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_INIT;  in ipc_imem_ipc_init_check()
269 	ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_ipc_init_check()
273 		if (ipc_mmio_get_ipc_state(ipc_imem->mmio) ==  in ipc_imem_ipc_init_check()
274 		    ipc_imem->ipc_requested_state) {  in ipc_imem_ipc_init_check()
276 			ipc_mmio_config(ipc_imem->mmio);  in ipc_imem_ipc_init_check()
279 			ipc_imem->ipc_requested_state =  in ipc_imem_ipc_init_check()
281 			ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_ipc_init_check()
290 	dev_err(ipc_imem->dev, "%s: ipc_status(%d) ne. IPC_MEM_DEVICE_IPC_INIT",  in ipc_imem_ipc_init_check()
291 		ipc_imem_phase_get_string(ipc_imem->phase),  in ipc_imem_ipc_init_check()
292 		ipc_mmio_get_ipc_state(ipc_imem->mmio));  in ipc_imem_ipc_init_check()
294 	ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_TIMEOUT);  in ipc_imem_ipc_init_check()
298 static void ipc_imem_dl_skb_process(struct iosm_imem *ipc_imem,  in ipc_imem_dl_skb_process()  argument
310 		ipc_pcie_addr_unmap(ipc_imem->pcie, IPC_CB(skb)->len,  in ipc_imem_dl_skb_process()
314 			ipc_imem_sys_devlink_notify_rx(ipc_imem->ipc_devlink,  in ipc_imem_dl_skb_process()
316 		else if (ipc_is_trace_channel(ipc_imem, port_id))  in ipc_imem_dl_skb_process()
317 			ipc_trace_port_rx(ipc_imem, skb);  in ipc_imem_dl_skb_process()
319 			wwan_port_rx(ipc_imem->ipc_port[port_id]->iosm_port,  in ipc_imem_dl_skb_process()
325 			ipc_mux_dl_decode(ipc_imem->mux, skb);  in ipc_imem_dl_skb_process()
328 		dev_err(ipc_imem->dev, "Invalid channel type");  in ipc_imem_dl_skb_process()
334 static void ipc_imem_dl_pipe_process(struct iosm_imem *ipc_imem,  in ipc_imem_dl_pipe_process()  argument
345 	ipc_protocol_get_head_tail_index(ipc_imem->ipc_protocol, pipe, &head,  in ipc_imem_dl_pipe_process()
358 		skb = ipc_protocol_dl_td_process(ipc_imem->ipc_protocol, pipe);  in ipc_imem_dl_pipe_process()
361 		ipc_imem_dl_skb_process(ipc_imem, pipe, skb);  in ipc_imem_dl_pipe_process()
365 	while (ipc_imem_dl_skb_alloc(ipc_imem, pipe))  in ipc_imem_dl_pipe_process()
370 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_dl_pipe_process()
375 		ipc_imem_hrtimer_stop(&ipc_imem->fast_update_timer);  in ipc_imem_dl_pipe_process()
383 		ipc_imem->hrtimer_period =  in ipc_imem_dl_pipe_process()
385 		hrtimer_start(&ipc_imem->fast_update_timer,  in ipc_imem_dl_pipe_process()
386 			      ipc_imem->hrtimer_period, HRTIMER_MODE_REL);  in ipc_imem_dl_pipe_process()
389 	if (ipc_imem->app_notify_dl_pend)  in ipc_imem_dl_pipe_process()
390 		complete(&ipc_imem->dl_pend_sem);  in ipc_imem_dl_pipe_process()
394 static void ipc_imem_ul_pipe_process(struct iosm_imem *ipc_imem,  in ipc_imem_ul_pipe_process()  argument
405 	ipc_protocol_get_head_tail_index(ipc_imem->ipc_protocol, pipe, &head,  in ipc_imem_ul_pipe_process()
417 		skb = ipc_protocol_ul_td_process(ipc_imem->ipc_protocol, pipe);  in ipc_imem_ul_pipe_process()
431 				ipc_mux_ul_encoded_process(ipc_imem->mux, skb);  in ipc_imem_ul_pipe_process()
433 				dev_err(ipc_imem->dev,  in ipc_imem_ul_pipe_process()
437 			ipc_pcie_kfree_skb(ipc_imem->pcie, skb);  in ipc_imem_ul_pipe_process()
443 		ipc_mux_check_n_restart_tx(ipc_imem->mux);  in ipc_imem_ul_pipe_process()
445 	if (ipc_imem->app_notify_ul_pend)  in ipc_imem_ul_pipe_process()
446 		complete(&ipc_imem->ul_pend_sem);  in ipc_imem_ul_pipe_process()
450 static void ipc_imem_rom_irq_exec(struct iosm_imem *ipc_imem)  in ipc_imem_rom_irq_exec()  argument
454 	channel = ipc_imem->ipc_devlink->devlink_sio.channel;  in ipc_imem_rom_irq_exec()
455 	ipc_imem->rom_exit_code = ipc_mmio_get_rom_exit_code(ipc_imem->mmio);  in ipc_imem_rom_irq_exec()
460 static int ipc_imem_tq_td_update_timer_cb(struct iosm_imem *ipc_imem, int arg,  in ipc_imem_tq_td_update_timer_cb()  argument
463 	ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol,  in ipc_imem_tq_td_update_timer_cb()
469 static void ipc_imem_slp_control_exec(struct iosm_imem *ipc_imem)  in ipc_imem_slp_control_exec()  argument
472 	if (ipc_protocol_pm_dev_sleep_handle(ipc_imem->ipc_protocol) &&  in ipc_imem_slp_control_exec()
473 	    hrtimer_active(&ipc_imem->tdupdate_timer)) {  in ipc_imem_slp_control_exec()
475 		ipc_imem_tq_td_update_timer_cb(ipc_imem, 0, NULL, 0);  in ipc_imem_slp_control_exec()
477 		ipc_imem_hrtimer_stop(&ipc_imem->tdupdate_timer);  in ipc_imem_slp_control_exec()
479 		ipc_imem_hrtimer_stop(&ipc_imem->fast_update_timer);  in ipc_imem_slp_control_exec()
484 static int ipc_imem_tq_startup_timer_cb(struct iosm_imem *ipc_imem, int arg,  in ipc_imem_tq_startup_timer_cb()  argument
488 	if (ipc_imem_phase_update(ipc_imem) != IPC_P_RUN)  in ipc_imem_tq_startup_timer_cb()
491 	if (ipc_mmio_get_ipc_state(ipc_imem->mmio) ==  in ipc_imem_tq_startup_timer_cb()
493 		ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_INIT;  in ipc_imem_tq_startup_timer_cb()
495 		ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_tq_startup_timer_cb()
498 		ipc_imem->hrtimer_period = ktime_set(0, 100 * 1000UL * 1000ULL);  in ipc_imem_tq_startup_timer_cb()
500 		if (!hrtimer_active(&ipc_imem->startup_timer))  in ipc_imem_tq_startup_timer_cb()
501 			hrtimer_start(&ipc_imem->startup_timer,  in ipc_imem_tq_startup_timer_cb()
502 				      ipc_imem->hrtimer_period,  in ipc_imem_tq_startup_timer_cb()
504 	} else if (ipc_mmio_get_ipc_state(ipc_imem->mmio) ==  in ipc_imem_tq_startup_timer_cb()
507 		ipc_imem_hrtimer_stop(&ipc_imem->startup_timer);  in ipc_imem_tq_startup_timer_cb()
510 		ipc_mmio_config(ipc_imem->mmio);  in ipc_imem_tq_startup_timer_cb()
511 		ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_RUNNING;  in ipc_imem_tq_startup_timer_cb()
512 		ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_tq_startup_timer_cb()
522 	struct iosm_imem *ipc_imem =  in ipc_imem_startup_timer_cb()  local
525 	if (ktime_to_ns(ipc_imem->hrtimer_period)) {  in ipc_imem_startup_timer_cb()
526 		hrtimer_forward_now(&ipc_imem->startup_timer,  in ipc_imem_startup_timer_cb()
527 				    ipc_imem->hrtimer_period);  in ipc_imem_startup_timer_cb()
531 	ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_startup_timer_cb, 0,  in ipc_imem_startup_timer_cb()
538 ipc_imem_get_exec_stage_buffered(struct iosm_imem *ipc_imem)  in ipc_imem_get_exec_stage_buffered()  argument
540 	return (ipc_imem->phase == IPC_P_RUN &&  in ipc_imem_get_exec_stage_buffered()
541 		ipc_imem->ipc_status == IPC_MEM_DEVICE_IPC_RUNNING) ?  in ipc_imem_get_exec_stage_buffered()
542 		       ipc_protocol_get_ap_exec_stage(ipc_imem->ipc_protocol) :  in ipc_imem_get_exec_stage_buffered()
543 		       ipc_mmio_get_exec_stage(ipc_imem->mmio);  in ipc_imem_get_exec_stage_buffered()
547 static int ipc_imem_send_mdm_rdy_cb(struct iosm_imem *ipc_imem, int arg,  in ipc_imem_send_mdm_rdy_cb()  argument
551 		ipc_imem_get_exec_stage_buffered(ipc_imem);  in ipc_imem_send_mdm_rdy_cb()
554 		ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_READY);  in ipc_imem_send_mdm_rdy_cb()
566 	struct iosm_imem *ipc_imem;  in ipc_imem_run_state_worker()  local
570 	ipc_imem = container_of(instance, struct iosm_imem, run_state_worker);  in ipc_imem_run_state_worker()
572 	if (ipc_imem->phase != IPC_P_RUN) {  in ipc_imem_run_state_worker()
573 		dev_err(ipc_imem->dev,  in ipc_imem_run_state_worker()
578 	if (test_and_clear_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag))  in ipc_imem_run_state_worker()
579 		ipc_devlink_deinit(ipc_imem->ipc_devlink);  in ipc_imem_run_state_worker()
581 	ret = ipc_imem_setup_cp_mux_cap_init(ipc_imem, &mux_cfg);  in ipc_imem_run_state_worker()
585 	ipc_imem->mux = ipc_mux_init(&mux_cfg, ipc_imem);  in ipc_imem_run_state_worker()
586 	if (!ipc_imem->mux)  in ipc_imem_run_state_worker()
589 	ret = ipc_imem_wwan_channel_init(ipc_imem, mux_cfg.protocol);  in ipc_imem_run_state_worker()
593 	ipc_imem->mux->wwan = ipc_imem->wwan;  in ipc_imem_run_state_worker()
597 			ipc_imem->ipc_port[ctrl_chl_idx] = NULL;  in ipc_imem_run_state_worker()
599 			if (ipc_imem->pcie->pci->device == INTEL_CP_DEVICE_7560_ID &&  in ipc_imem_run_state_worker()
605 			if (ipc_imem->pcie->pci->device == INTEL_CP_DEVICE_7360_ID &&  in ipc_imem_run_state_worker()
611 				ipc_imem_channel_init(ipc_imem, IPC_CTYPE_CTRL,  in ipc_imem_run_state_worker()
614 				ipc_imem->ipc_port[ctrl_chl_idx] =  in ipc_imem_run_state_worker()
615 					ipc_port_init(ipc_imem, chnl_cfg_port);  in ipc_imem_run_state_worker()
621 	ipc_debugfs_init(ipc_imem);  in ipc_imem_run_state_worker()
623 	ipc_task_queue_send_task(ipc_imem, ipc_imem_send_mdm_rdy_cb, 0, NULL, 0,  in ipc_imem_run_state_worker()
629 	set_bit(FULLY_FUNCTIONAL, &ipc_imem->flag);  in ipc_imem_run_state_worker()
637 	ipc_mux_deinit(ipc_imem->mux);  in ipc_imem_run_state_worker()
639 	ipc_uevent_send(ipc_imem->dev, UEVENT_CD_READY_LINK_DOWN);  in ipc_imem_run_state_worker()
642 static void ipc_imem_handle_irq(struct iosm_imem *ipc_imem, int irq)  in ipc_imem_handle_irq()  argument
651 		ipc_imem->ev_irq_pending[irq] = false;  in ipc_imem_handle_irq()
654 	old_phase = ipc_imem->phase;  in ipc_imem_handle_irq()
657 		dev_dbg(ipc_imem->dev,  in ipc_imem_handle_irq()
664 	phase = ipc_imem_phase_update(ipc_imem);  in ipc_imem_handle_irq()
668 		if (!ipc_imem->enter_runtime) {  in ipc_imem_handle_irq()
670 			ipc_imem->enter_runtime = 1;  in ipc_imem_handle_irq()
675 			ipc_imem_msg_send_device_sleep(ipc_imem,  in ipc_imem_handle_irq()
676 						       ipc_imem->device_sleep);  in ipc_imem_handle_irq()
678 			ipc_imem_msg_send_feature_set(ipc_imem,  in ipc_imem_handle_irq()
684 			ipc_protocol_get_ipc_status(ipc_imem->ipc_protocol);  in ipc_imem_handle_irq()
687 		if (ipc_imem->ipc_status != curr_ipc_status) {  in ipc_imem_handle_irq()
688 			ipc_imem->ipc_status = curr_ipc_status;  in ipc_imem_handle_irq()
690 			if (ipc_imem->ipc_status ==  in ipc_imem_handle_irq()
692 				schedule_work(&ipc_imem->run_state_worker);  in ipc_imem_handle_irq()
697 		ipc_imem_slp_control_exec(ipc_imem);  in ipc_imem_handle_irq()
703 		dev_err(ipc_imem->dev, "confused phase %s",  in ipc_imem_handle_irq()
719 		ipc_imem_rom_irq_exec(ipc_imem);  in ipc_imem_handle_irq()
727 	ipc_protocol_msg_process(ipc_imem, irq);  in ipc_imem_handle_irq()
731 		struct ipc_pipe *ul_pipe = &ipc_imem->channels[i].ul_pipe;  in ipc_imem_handle_irq()
732 		struct ipc_pipe *dl_pipe = &ipc_imem->channels[i].dl_pipe;  in ipc_imem_handle_irq()
736 			ipc_imem_dl_pipe_process(ipc_imem, dl_pipe);  in ipc_imem_handle_irq()
743 			ipc_imem_ul_pipe_process(ipc_imem, ul_pipe);  in ipc_imem_handle_irq()
747 	if (ipc_mux_ul_data_encode(ipc_imem->mux)) {  in ipc_imem_handle_irq()
748 		ipc_imem_td_update_timer_start(ipc_imem);  in ipc_imem_handle_irq()
749 		if (ipc_imem->mux->protocol == MUX_AGGREGATION)  in ipc_imem_handle_irq()
750 			ipc_imem_adb_timer_start(ipc_imem);  in ipc_imem_handle_irq()
756 	ul_pending |= ipc_imem_ul_write_td(ipc_imem);  in ipc_imem_handle_irq()
760 		ipc_imem->hrtimer_period =  in ipc_imem_handle_irq()
762 		if (!hrtimer_active(&ipc_imem->tdupdate_timer))  in ipc_imem_handle_irq()
763 			hrtimer_start(&ipc_imem->tdupdate_timer,  in ipc_imem_handle_irq()
764 				      ipc_imem->hrtimer_period,  in ipc_imem_handle_irq()
773 	    ipc_imem->ipc_requested_state == IPC_MEM_DEVICE_IPC_RUNNING &&  in ipc_imem_handle_irq()
774 	    ipc_mmio_get_ipc_state(ipc_imem->mmio) ==  in ipc_imem_handle_irq()
776 		complete(&ipc_imem->ipc_devlink->devlink_sio.channel->ul_sem);  in ipc_imem_handle_irq()
780 	ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_DONT_CARE;  in ipc_imem_handle_irq()
783 		ipc_imem->hrtimer_period =  in ipc_imem_handle_irq()
785 		if (!hrtimer_active(&ipc_imem->td_alloc_timer))  in ipc_imem_handle_irq()
786 			hrtimer_start(&ipc_imem->td_alloc_timer,  in ipc_imem_handle_irq()
787 				      ipc_imem->hrtimer_period,  in ipc_imem_handle_irq()
793 static int ipc_imem_tq_irq_cb(struct iosm_imem *ipc_imem, int arg, void *msg,  in ipc_imem_tq_irq_cb()  argument
796 	ipc_imem_handle_irq(ipc_imem, arg);  in ipc_imem_tq_irq_cb()
801 void ipc_imem_ul_send(struct iosm_imem *ipc_imem)  in ipc_imem_ul_send()  argument
804 	if (ipc_imem_ul_write_td(ipc_imem))  in ipc_imem_ul_send()
805 		ipc_imem_td_update_timer_start(ipc_imem);  in ipc_imem_ul_send()
809 static enum ipc_phase ipc_imem_phase_update_check(struct iosm_imem *ipc_imem,  in ipc_imem_phase_update_check()  argument
814 		if (ipc_imem->phase != IPC_P_ROM) {  in ipc_imem_phase_update_check()
816 			ipc_uevent_send(ipc_imem->dev, UEVENT_ROM_READY);  in ipc_imem_phase_update_check()
819 		ipc_imem->phase = IPC_P_ROM;  in ipc_imem_phase_update_check()
823 		ipc_imem->phase = IPC_P_PSI;  in ipc_imem_phase_update_check()
827 		ipc_imem->phase = IPC_P_EBL;  in ipc_imem_phase_update_check()
831 		if (ipc_imem->phase != IPC_P_RUN &&  in ipc_imem_phase_update_check()
832 		    ipc_imem->ipc_status == IPC_MEM_DEVICE_IPC_RUNNING) {  in ipc_imem_phase_update_check()
833 			ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_READY);  in ipc_imem_phase_update_check()
835 		ipc_imem->phase = IPC_P_RUN;  in ipc_imem_phase_update_check()
839 		if (ipc_imem->phase != IPC_P_CRASH)  in ipc_imem_phase_update_check()
840 			ipc_uevent_send(ipc_imem->dev, UEVENT_CRASH);  in ipc_imem_phase_update_check()
842 		ipc_imem->phase = IPC_P_CRASH;  in ipc_imem_phase_update_check()
846 		if (ipc_imem->phase != IPC_P_CD_READY)  in ipc_imem_phase_update_check()
847 			ipc_uevent_send(ipc_imem->dev, UEVENT_CD_READY);  in ipc_imem_phase_update_check()
848 		ipc_imem->phase = IPC_P_CD_READY;  in ipc_imem_phase_update_check()
855 		ipc_uevent_send(ipc_imem->dev, UEVENT_CD_READY_LINK_DOWN);  in ipc_imem_phase_update_check()
859 	return ipc_imem->phase;  in ipc_imem_phase_update_check()
863 static bool ipc_imem_pipe_open(struct iosm_imem *ipc_imem,  in ipc_imem_pipe_open()  argument
870 	if (ipc_protocol_msg_send(ipc_imem->ipc_protocol,  in ipc_imem_pipe_open()
878 static int ipc_imem_tq_pipe_td_alloc(struct iosm_imem *ipc_imem, int arg,  in ipc_imem_tq_pipe_td_alloc()  argument
886 		processed |= ipc_imem_dl_skb_alloc(ipc_imem, dl_pipe);  in ipc_imem_tq_pipe_td_alloc()
892 		ipc_protocol_doorbell_trigger(ipc_imem->ipc_protocol, arg);  in ipc_imem_tq_pipe_td_alloc()
900 	struct iosm_imem *ipc_imem =  in ipc_imem_td_update_timer_cb()  local
903 	ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_td_update_timer_cb, 0,  in ipc_imem_td_update_timer_cb()
909 enum ipc_phase ipc_imem_phase_update(struct iosm_imem *ipc_imem)  in ipc_imem_phase_update()  argument
912 				ipc_imem_get_exec_stage_buffered(ipc_imem);  in ipc_imem_phase_update()
914 	return ipc_imem->phase == IPC_P_OFF_REQ ?  in ipc_imem_phase_update()
915 		       ipc_imem->phase :  in ipc_imem_phase_update()
916 		       ipc_imem_phase_update_check(ipc_imem, exec_stage);  in ipc_imem_phase_update()
951 void ipc_imem_pipe_close(struct iosm_imem *ipc_imem, struct ipc_pipe *pipe)  in ipc_imem_pipe_close()  argument
956 	ipc_protocol_msg_send(ipc_imem->ipc_protocol, IPC_MSG_PREP_PIPE_CLOSE,  in ipc_imem_pipe_close()
959 	ipc_imem_pipe_cleanup(ipc_imem, pipe);  in ipc_imem_pipe_close()
962 void ipc_imem_channel_close(struct iosm_imem *ipc_imem, int channel_id)  in ipc_imem_channel_close()  argument
966 	if (channel_id < 0 || channel_id >= ipc_imem->nr_of_channels) {  in ipc_imem_channel_close()
967 		dev_err(ipc_imem->dev, "invalid channel id %d", channel_id);  in ipc_imem_channel_close()
971 	channel = &ipc_imem->channels[channel_id];  in ipc_imem_channel_close()
974 		dev_err(ipc_imem->dev, "ch[%d]: invalid channel state %d",  in ipc_imem_channel_close()
984 	if (ipc_imem->phase == IPC_P_RUN) {  in ipc_imem_channel_close()
985 		ipc_imem_pipe_close(ipc_imem, &channel->ul_pipe);  in ipc_imem_channel_close()
986 		ipc_imem_pipe_close(ipc_imem, &channel->dl_pipe);  in ipc_imem_channel_close()
989 	ipc_imem_pipe_cleanup(ipc_imem, &channel->ul_pipe);  in ipc_imem_channel_close()
990 	ipc_imem_pipe_cleanup(ipc_imem, &channel->dl_pipe);  in ipc_imem_channel_close()
996 struct ipc_mem_channel *ipc_imem_channel_open(struct iosm_imem *ipc_imem,  in ipc_imem_channel_open()  argument
1002 		dev_err(ipc_imem->dev, "invalid channel ID: %d", channel_id);  in ipc_imem_channel_open()
1006 	channel = &ipc_imem->channels[channel_id];  in ipc_imem_channel_open()
1010 	if (!ipc_imem_pipe_open(ipc_imem, &channel->ul_pipe))  in ipc_imem_channel_open()
1013 	if (!ipc_imem_pipe_open(ipc_imem, &channel->dl_pipe))  in ipc_imem_channel_open()
1017 	if (ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_pipe_td_alloc, db_id,  in ipc_imem_channel_open()
1019 		dev_err(ipc_imem->dev, "td allocation failed : %d", channel_id);  in ipc_imem_channel_open()
1026 	ipc_imem_pipe_close(ipc_imem, &channel->dl_pipe);  in ipc_imem_channel_open()
1028 	ipc_imem_pipe_close(ipc_imem, &channel->ul_pipe);  in ipc_imem_channel_open()
1034 void ipc_imem_pm_suspend(struct iosm_imem *ipc_imem)  in ipc_imem_pm_suspend()  argument
1036 	ipc_protocol_suspend(ipc_imem->ipc_protocol);  in ipc_imem_pm_suspend()
1039 void ipc_imem_pm_s2idle_sleep(struct iosm_imem *ipc_imem, bool sleep)  in ipc_imem_pm_s2idle_sleep()  argument
1041 	ipc_protocol_s2idle_sleep(ipc_imem->ipc_protocol, sleep);  in ipc_imem_pm_s2idle_sleep()
1044 void ipc_imem_pm_resume(struct iosm_imem *ipc_imem)  in ipc_imem_pm_resume()  argument
1048 	if (ipc_protocol_resume(ipc_imem->ipc_protocol)) {  in ipc_imem_pm_resume()
1049 		stage = ipc_mmio_get_exec_stage(ipc_imem->mmio);  in ipc_imem_pm_resume()
1050 		ipc_imem_phase_update_check(ipc_imem, stage);  in ipc_imem_pm_resume()
1060 int ipc_imem_channel_alloc(struct iosm_imem *ipc_imem, int index,  in ipc_imem_channel_alloc()  argument
1067 	for (i = 0; i < ipc_imem->nr_of_channels; i++) {  in ipc_imem_channel_alloc()
1068 		channel = &ipc_imem->channels[i];  in ipc_imem_channel_alloc()
1073 	if (i >= ipc_imem->nr_of_channels) {  in ipc_imem_channel_alloc()
1074 		dev_dbg(ipc_imem->dev,  in ipc_imem_channel_alloc()
1080 	if (ipc_imem->channels[i].state != IMEM_CHANNEL_FREE) {  in ipc_imem_channel_alloc()
1081 		dev_dbg(ipc_imem->dev, "channel is in use");  in ipc_imem_channel_alloc()
1095 void ipc_imem_channel_init(struct iosm_imem *ipc_imem, enum ipc_ctype ctype,  in ipc_imem_channel_init()  argument
1102 		dev_err(ipc_imem->dev, "invalid pipe: ul_pipe=%d, dl_pipe=%d",  in ipc_imem_channel_init()
1107 	if (ipc_imem->nr_of_channels >= IPC_MEM_MAX_CHANNELS) {  in ipc_imem_channel_init()
1108 		dev_err(ipc_imem->dev, "too many channels");  in ipc_imem_channel_init()
1112 	channel = &ipc_imem->channels[ipc_imem->nr_of_channels];  in ipc_imem_channel_init()
1113 	channel->channel_id = ipc_imem->nr_of_channels;  in ipc_imem_channel_init()
1118 	ipc_imem->nr_of_channels++;  in ipc_imem_channel_init()
1120 	ipc_imem_channel_update(ipc_imem, channel->channel_id, chnl_cfg,  in ipc_imem_channel_init()
1128 void ipc_imem_channel_update(struct iosm_imem *ipc_imem, int id,  in ipc_imem_channel_update()  argument
1133 	if (id < 0 || id >= ipc_imem->nr_of_channels) {  in ipc_imem_channel_update()
1134 		dev_err(ipc_imem->dev, "invalid channel id %d", id);  in ipc_imem_channel_update()
1138 	channel = &ipc_imem->channels[id];  in ipc_imem_channel_update()
1142 		dev_err(ipc_imem->dev, "invalid channel state %d",  in ipc_imem_channel_update()
1168 static void ipc_imem_channel_reset(struct iosm_imem *ipc_imem)  in ipc_imem_channel_reset()  argument
1172 	for (i = 0; i < ipc_imem->nr_of_channels; i++) {  in ipc_imem_channel_reset()
1175 		channel = &ipc_imem->channels[i];  in ipc_imem_channel_reset()
1177 		ipc_imem_pipe_cleanup(ipc_imem, &channel->dl_pipe);  in ipc_imem_channel_reset()
1178 		ipc_imem_pipe_cleanup(ipc_imem, &channel->ul_pipe);  in ipc_imem_channel_reset()
1184 void ipc_imem_pipe_cleanup(struct iosm_imem *ipc_imem, struct ipc_pipe *pipe)  in ipc_imem_pipe_cleanup()  argument
1195 		ipc_pcie_kfree_skb(ipc_imem->pcie, skb);  in ipc_imem_pipe_cleanup()
1197 	ipc_protocol_pipe_cleanup(ipc_imem->ipc_protocol, pipe);  in ipc_imem_pipe_cleanup()
1201 static void ipc_imem_device_ipc_uninit(struct iosm_imem *ipc_imem)  in ipc_imem_device_ipc_uninit()  argument
1209 	if (ipc_pcie_check_data_link_active(ipc_imem->pcie)) {  in ipc_imem_device_ipc_uninit()
1214 		ipc_doorbell_fire(ipc_imem->pcie, IPC_DOORBELL_IRQ_IPC,  in ipc_imem_device_ipc_uninit()
1216 		ipc_state = ipc_mmio_get_ipc_state(ipc_imem->mmio);  in ipc_imem_device_ipc_uninit()
1226 			ipc_state = ipc_mmio_get_ipc_state(ipc_imem->mmio);  in ipc_imem_device_ipc_uninit()
1231 void ipc_imem_cleanup(struct iosm_imem *ipc_imem)  in ipc_imem_cleanup()  argument
1233 	ipc_imem->phase = IPC_P_OFF_REQ;  in ipc_imem_cleanup()
1236 	ipc_uevent_send(ipc_imem->dev, UEVENT_MDM_NOT_READY);  in ipc_imem_cleanup()
1238 	hrtimer_cancel(&ipc_imem->td_alloc_timer);  in ipc_imem_cleanup()
1239 	hrtimer_cancel(&ipc_imem->tdupdate_timer);  in ipc_imem_cleanup()
1240 	hrtimer_cancel(&ipc_imem->fast_update_timer);  in ipc_imem_cleanup()
1241 	hrtimer_cancel(&ipc_imem->startup_timer);  in ipc_imem_cleanup()
1244 	cancel_work_sync(&ipc_imem->run_state_worker);  in ipc_imem_cleanup()
1246 	if (test_and_clear_bit(FULLY_FUNCTIONAL, &ipc_imem->flag)) {  in ipc_imem_cleanup()
1247 		ipc_mux_deinit(ipc_imem->mux);  in ipc_imem_cleanup()
1248 		ipc_debugfs_deinit(ipc_imem);  in ipc_imem_cleanup()
1249 		ipc_wwan_deinit(ipc_imem->wwan);  in ipc_imem_cleanup()
1250 		ipc_port_deinit(ipc_imem->ipc_port);  in ipc_imem_cleanup()
1253 	if (test_and_clear_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag))  in ipc_imem_cleanup()
1254 		ipc_devlink_deinit(ipc_imem->ipc_devlink);  in ipc_imem_cleanup()
1256 	ipc_imem_device_ipc_uninit(ipc_imem);  in ipc_imem_cleanup()
1257 	ipc_imem_channel_reset(ipc_imem);  in ipc_imem_cleanup()
1259 	ipc_protocol_deinit(ipc_imem->ipc_protocol);  in ipc_imem_cleanup()
1260 	ipc_task_deinit(ipc_imem->ipc_task);  in ipc_imem_cleanup()
1262 	kfree(ipc_imem->ipc_task);  in ipc_imem_cleanup()
1263 	kfree(ipc_imem->mmio);  in ipc_imem_cleanup()
1265 	ipc_imem->phase = IPC_P_OFF;  in ipc_imem_cleanup()
1273 static int ipc_imem_config(struct iosm_imem *ipc_imem)  in ipc_imem_config()  argument
1278 	init_completion(&ipc_imem->ul_pend_sem);  in ipc_imem_config()
1280 	init_completion(&ipc_imem->dl_pend_sem);  in ipc_imem_config()
1283 	ipc_imem->ipc_status = IPC_MEM_DEVICE_IPC_UNINIT;  in ipc_imem_config()
1284 	ipc_imem->enter_runtime = 0;  in ipc_imem_config()
1286 	phase = ipc_imem_phase_update(ipc_imem);  in ipc_imem_config()
1291 		ipc_imem->hrtimer_period = ktime_set(0, 1000 * 1000 * 1000ULL);  in ipc_imem_config()
1293 		if (!hrtimer_active(&ipc_imem->startup_timer))  in ipc_imem_config()
1294 			hrtimer_start(&ipc_imem->startup_timer,  in ipc_imem_config()
1295 				      ipc_imem->hrtimer_period,  in ipc_imem_config()
1303 		ipc_imem->ipc_requested_state = IPC_MEM_DEVICE_IPC_UNINIT;  in ipc_imem_config()
1306 		if (ipc_imem->ipc_requested_state ==  in ipc_imem_config()
1307 		    ipc_mmio_get_ipc_state(ipc_imem->mmio)) {  in ipc_imem_config()
1308 			ipc_imem_ipc_init_check(ipc_imem);  in ipc_imem_config()
1312 		dev_err(ipc_imem->dev,  in ipc_imem_config()
1314 			ipc_mmio_get_ipc_state(ipc_imem->mmio));  in ipc_imem_config()
1318 		dev_dbg(ipc_imem->dev,  in ipc_imem_config()
1323 		dev_err(ipc_imem->dev, "unexpected operation phase %d", phase);  in ipc_imem_config()
1327 	complete(&ipc_imem->dl_pend_sem);  in ipc_imem_config()
1328 	complete(&ipc_imem->ul_pend_sem);  in ipc_imem_config()
1329 	ipc_imem->phase = IPC_P_OFF;  in ipc_imem_config()
1337 	struct iosm_imem *ipc_imem = kzalloc(sizeof(*pcie->imem), GFP_KERNEL);  in ipc_imem_init()  local
1340 	if (!ipc_imem)  in ipc_imem_init()
1344 	ipc_imem->pcie = pcie;  in ipc_imem_init()
1345 	ipc_imem->dev = dev;  in ipc_imem_init()
1347 	ipc_imem->pci_device_id = device_id;  in ipc_imem_init()
1349 	ipc_imem->cp_version = 0;  in ipc_imem_init()
1350 	ipc_imem->device_sleep = IPC_HOST_SLEEP_ENTER_SLEEP;  in ipc_imem_init()
1353 	ipc_imem->nr_of_channels = 0;  in ipc_imem_init()
1356 	ipc_imem->mmio = ipc_mmio_init(mmio, ipc_imem->dev);  in ipc_imem_init()
1357 	if (!ipc_imem->mmio) {  in ipc_imem_init()
1358 		dev_err(ipc_imem->dev, "failed to initialize mmio region");  in ipc_imem_init()
1362 	ipc_imem->ipc_task = kzalloc(sizeof(*ipc_imem->ipc_task),  in ipc_imem_init()
1366 	if (!ipc_imem->ipc_task)  in ipc_imem_init()
1369 	if (ipc_task_init(ipc_imem->ipc_task))  in ipc_imem_init()
1372 	ipc_imem->ipc_task->dev = ipc_imem->dev;  in ipc_imem_init()
1374 	INIT_WORK(&ipc_imem->run_state_worker, ipc_imem_run_state_worker);  in ipc_imem_init()
1376 	ipc_imem->ipc_protocol = ipc_protocol_init(ipc_imem);  in ipc_imem_init()
1378 	if (!ipc_imem->ipc_protocol)  in ipc_imem_init()
1382 	ipc_imem->phase = IPC_P_OFF;  in ipc_imem_init()
1384 	hrtimer_init(&ipc_imem->startup_timer, CLOCK_MONOTONIC,  in ipc_imem_init()
1386 	ipc_imem->startup_timer.function = ipc_imem_startup_timer_cb;  in ipc_imem_init()
1388 	hrtimer_init(&ipc_imem->tdupdate_timer, CLOCK_MONOTONIC,  in ipc_imem_init()
1390 	ipc_imem->tdupdate_timer.function = ipc_imem_td_update_timer_cb;  in ipc_imem_init()
1392 	hrtimer_init(&ipc_imem->fast_update_timer, CLOCK_MONOTONIC,  in ipc_imem_init()
1394 	ipc_imem->fast_update_timer.function = ipc_imem_fast_update_timer_cb;  in ipc_imem_init()
1396 	hrtimer_init(&ipc_imem->td_alloc_timer, CLOCK_MONOTONIC,  in ipc_imem_init()
1398 	ipc_imem->td_alloc_timer.function = ipc_imem_td_alloc_timer_cb;  in ipc_imem_init()
1400 	hrtimer_init(&ipc_imem->adb_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in ipc_imem_init()
1401 	ipc_imem->adb_timer.function = ipc_imem_adb_timer_cb;  in ipc_imem_init()
1403 	if (ipc_imem_config(ipc_imem)) {  in ipc_imem_init()
1404 		dev_err(ipc_imem->dev, "failed to initialize the imem");  in ipc_imem_init()
1408 	stage = ipc_mmio_get_exec_stage(ipc_imem->mmio);  in ipc_imem_init()
1411 		ipc_imem->ipc_devlink = ipc_devlink_init(ipc_imem);  in ipc_imem_init()
1412 		if (!ipc_imem->ipc_devlink) {  in ipc_imem_init()
1413 			dev_err(ipc_imem->dev, "Devlink register failed");  in ipc_imem_init()
1417 		if (ipc_flash_link_establish(ipc_imem))  in ipc_imem_init()
1420 		set_bit(IOSM_DEVLINK_INIT, &ipc_imem->flag);  in ipc_imem_init()
1422 	return ipc_imem;  in ipc_imem_init()
1424 	ipc_devlink_deinit(ipc_imem->ipc_devlink);  in ipc_imem_init()
1426 	hrtimer_cancel(&ipc_imem->td_alloc_timer);  in ipc_imem_init()
1427 	hrtimer_cancel(&ipc_imem->fast_update_timer);  in ipc_imem_init()
1428 	hrtimer_cancel(&ipc_imem->tdupdate_timer);  in ipc_imem_init()
1429 	hrtimer_cancel(&ipc_imem->startup_timer);  in ipc_imem_init()
1431 	cancel_work_sync(&ipc_imem->run_state_worker);  in ipc_imem_init()
1432 	ipc_task_deinit(ipc_imem->ipc_task);  in ipc_imem_init()
1434 	kfree(ipc_imem->ipc_task);  in ipc_imem_init()
1436 	kfree(ipc_imem->mmio);  in ipc_imem_init()
1438 	kfree(ipc_imem);  in ipc_imem_init()
1442 void ipc_imem_irq_process(struct iosm_imem *ipc_imem, int irq)  in ipc_imem_irq_process()  argument
1445 	if (ipc_imem && !ipc_imem->ev_irq_pending[irq]) {  in ipc_imem_irq_process()
1446 		ipc_imem->ev_irq_pending[irq] = true;  in ipc_imem_irq_process()
1447 		ipc_task_queue_send_task(ipc_imem, ipc_imem_tq_irq_cb, irq,  in ipc_imem_irq_process()
1452 void ipc_imem_td_update_timer_suspend(struct iosm_imem *ipc_imem, bool suspend)  in ipc_imem_td_update_timer_suspend()  argument
1454 	ipc_imem->td_update_timer_suspended = suspend;  in ipc_imem_td_update_timer_suspend()
1460 static int ipc_imem_devlink_trigger_chip_info_cb(struct iosm_imem *ipc_imem,  in ipc_imem_devlink_trigger_chip_info_cb()  argument
1470 	stage = ipc_mmio_get_exec_stage(ipc_imem->mmio);  in ipc_imem_devlink_trigger_chip_info_cb()
1472 		dev_err(ipc_imem->dev,  in ipc_imem_devlink_trigger_chip_info_cb()
1477 	size = ipc_imem->mmio->chip_info_size;  in ipc_imem_devlink_trigger_chip_info_cb()
1481 	skb = ipc_pcie_alloc_local_skb(ipc_imem->pcie, GFP_ATOMIC, size);  in ipc_imem_devlink_trigger_chip_info_cb()
1483 		dev_err(ipc_imem->dev, "exhausted skbuf kernel DL memory");  in ipc_imem_devlink_trigger_chip_info_cb()
1488 	ipc_mmio_copy_chip_info(ipc_imem->mmio, skb_put(skb, size), size);  in ipc_imem_devlink_trigger_chip_info_cb()
1490 	dev_dbg(ipc_imem->dev, "execution_stage[%X] eq. BOOT", stage);  in ipc_imem_devlink_trigger_chip_info_cb()
1491 	ipc_imem->phase = ipc_imem_phase_update(ipc_imem);  in ipc_imem_devlink_trigger_chip_info_cb()
1492 	ipc_imem_sys_devlink_notify_rx(ipc_imem->ipc_devlink, skb);  in ipc_imem_devlink_trigger_chip_info_cb()
1498 int ipc_imem_devlink_trigger_chip_info(struct iosm_imem *ipc_imem)  in ipc_imem_devlink_trigger_chip_info()  argument
1500 	return ipc_task_queue_send_task(ipc_imem,  in ipc_imem_devlink_trigger_chip_info()