Lines Matching +full:disable +full:- +full:cqe +full:- +full:dcmd
1 // SPDX-License-Identifier: GPL-2.0-only
10 #include <linux/dma-mapping.h>
21 #include "cqhci-crypto.h"
43 return cq_host->desc_base + (tag * cq_host->slot_sz); in get_desc()
50 return desc + cq_host->task_desc_len; in get_link_desc()
55 return cq_host->trans_desc_len * cq_host->mmc->max_segs * tag; in get_trans_desc_offset()
62 return cq_host->trans_desc_dma_base + offset; in get_trans_desc_dma()
69 return cq_host->trans_desc_base + offset; in get_trans_desc()
80 memset(link_temp, 0, cq_host->link_desc_len); in setup_trans_desc()
81 if (cq_host->link_desc_len > 8) in setup_trans_desc()
84 if (tag == DCMD_SLOT && (cq_host->mmc->caps2 & MMC_CAP2_CQE_DCMD)) { in setup_trans_desc()
91 if (cq_host->dma64) { in setup_trans_desc()
115 struct mmc_host *mmc = cq_host->mmc; in cqhci_dumpregs()
143 CQHCI_DUMP("SSC2: 0x%08x | DCMD rsp: 0x%08x\n", in cqhci_dumpregs()
153 if (cq_host->ops->dumpregs) in cqhci_dumpregs()
154 cq_host->ops->dumpregs(mmc); in cqhci_dumpregs()
162 * |----------|
163 * |task desc | |->|----------|
164 * |----------| | |trans desc|
165 * |link desc-|->| |----------|
166 * |----------| .
168 * no. of slots max-segs
169 * . |----------|
170 * |----------|
179 if (cq_host->caps & CQHCI_TASK_DESC_SZ_128) { in cqhci_host_alloc_tdl()
182 cq_host->task_desc_len = 16; in cqhci_host_alloc_tdl()
184 cq_host->task_desc_len = 8; in cqhci_host_alloc_tdl()
192 if (cq_host->dma64) { in cqhci_host_alloc_tdl()
193 if (cq_host->quirks & CQHCI_QUIRK_SHORT_TXFR_DESC_SZ) in cqhci_host_alloc_tdl()
194 cq_host->trans_desc_len = 12; in cqhci_host_alloc_tdl()
196 cq_host->trans_desc_len = 16; in cqhci_host_alloc_tdl()
197 cq_host->link_desc_len = 16; in cqhci_host_alloc_tdl()
199 cq_host->trans_desc_len = 8; in cqhci_host_alloc_tdl()
200 cq_host->link_desc_len = 8; in cqhci_host_alloc_tdl()
204 cq_host->slot_sz = cq_host->task_desc_len + cq_host->link_desc_len; in cqhci_host_alloc_tdl()
206 cq_host->desc_size = cq_host->slot_sz * cq_host->num_slots; in cqhci_host_alloc_tdl()
208 cq_host->data_size = get_trans_desc_offset(cq_host, cq_host->mmc->cqe_qdepth); in cqhci_host_alloc_tdl()
210 pr_debug("%s: cqhci: desc_size: %zu data_sz: %zu slot-sz: %d\n", in cqhci_host_alloc_tdl()
211 mmc_hostname(cq_host->mmc), cq_host->desc_size, cq_host->data_size, in cqhci_host_alloc_tdl()
212 cq_host->slot_sz); in cqhci_host_alloc_tdl()
215 * allocate a dma-mapped chunk of memory for the descriptors in cqhci_host_alloc_tdl()
216 * allocate a dma-mapped chunk of memory for link descriptors in cqhci_host_alloc_tdl()
217 * setup each link-desc memory offset per slot-number to in cqhci_host_alloc_tdl()
220 cq_host->desc_base = dmam_alloc_coherent(mmc_dev(cq_host->mmc), in cqhci_host_alloc_tdl()
221 cq_host->desc_size, in cqhci_host_alloc_tdl()
222 &cq_host->desc_dma_base, in cqhci_host_alloc_tdl()
224 if (!cq_host->desc_base) in cqhci_host_alloc_tdl()
225 return -ENOMEM; in cqhci_host_alloc_tdl()
227 cq_host->trans_desc_base = dmam_alloc_coherent(mmc_dev(cq_host->mmc), in cqhci_host_alloc_tdl()
228 cq_host->data_size, in cqhci_host_alloc_tdl()
229 &cq_host->trans_desc_dma_base, in cqhci_host_alloc_tdl()
231 if (!cq_host->trans_desc_base) { in cqhci_host_alloc_tdl()
232 dmam_free_coherent(mmc_dev(cq_host->mmc), cq_host->desc_size, in cqhci_host_alloc_tdl()
233 cq_host->desc_base, in cqhci_host_alloc_tdl()
234 cq_host->desc_dma_base); in cqhci_host_alloc_tdl()
235 cq_host->desc_base = NULL; in cqhci_host_alloc_tdl()
236 cq_host->desc_dma_base = 0; in cqhci_host_alloc_tdl()
237 return -ENOMEM; in cqhci_host_alloc_tdl()
240 pr_debug("%s: cqhci: desc-base: 0x%p trans-base: 0x%p\n desc_dma 0x%llx trans_dma: 0x%llx\n", in cqhci_host_alloc_tdl()
241 mmc_hostname(cq_host->mmc), cq_host->desc_base, cq_host->trans_desc_base, in cqhci_host_alloc_tdl()
242 (unsigned long long)cq_host->desc_dma_base, in cqhci_host_alloc_tdl()
243 (unsigned long long)cq_host->trans_desc_dma_base); in cqhci_host_alloc_tdl()
245 for (; i < (cq_host->num_slots); i++) in cqhci_host_alloc_tdl()
253 struct mmc_host *mmc = cq_host->mmc; in __cqhci_enable()
266 if (mmc->caps2 & MMC_CAP2_CQE_DCMD) in __cqhci_enable()
269 if (cq_host->caps & CQHCI_TASK_DESC_SZ_128) in __cqhci_enable()
272 if (mmc->caps2 & MMC_CAP2_CRYPTO) in __cqhci_enable()
277 cqhci_writel(cq_host, lower_32_bits(cq_host->desc_dma_base), in __cqhci_enable()
279 cqhci_writel(cq_host, upper_32_bits(cq_host->desc_dma_base), in __cqhci_enable()
282 cqhci_writel(cq_host, cq_host->rca, CQHCI_SSC2); in __cqhci_enable()
293 mmc->cqe_on = true; in __cqhci_enable()
295 if (cq_host->ops->enable) in __cqhci_enable()
296 cq_host->ops->enable(mmc); in __cqhci_enable()
303 cq_host->activated = true; in __cqhci_enable()
314 cq_host->mmc->cqe_on = false; in __cqhci_disable()
316 cq_host->activated = false; in __cqhci_disable()
321 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_deactivate()
323 if (cq_host->enabled && cq_host->activated) in cqhci_deactivate()
332 /* Re-enable is done upon first request */ in cqhci_resume()
339 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_enable()
342 if (!card->ext_csd.cmdq_en) in cqhci_enable()
343 return -EINVAL; in cqhci_enable()
345 if (cq_host->enabled) in cqhci_enable()
348 cq_host->rca = card->rca; in cqhci_enable()
352 pr_err("%s: Failed to enable CQE, error %d\n", in cqhci_enable()
359 cq_host->enabled = true; in cqhci_enable()
377 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_off()
381 if (!cq_host->enabled || !mmc->cqe_on || cq_host->recovery_halt) in cqhci_off()
384 if (cq_host->ops->disable) in cqhci_off()
385 cq_host->ops->disable(mmc, false); in cqhci_off()
392 pr_err("%s: cqhci: CQE stuck on\n", mmc_hostname(mmc)); in cqhci_off()
394 pr_debug("%s: cqhci: CQE off\n", mmc_hostname(mmc)); in cqhci_off()
396 if (cq_host->ops->post_disable) in cqhci_off()
397 cq_host->ops->post_disable(mmc); in cqhci_off()
399 mmc->cqe_on = false; in cqhci_off()
404 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_disable()
406 if (!cq_host->enabled) in cqhci_disable()
413 dmam_free_coherent(mmc_dev(mmc), cq_host->data_size, in cqhci_disable()
414 cq_host->trans_desc_base, in cqhci_disable()
415 cq_host->trans_desc_dma_base); in cqhci_disable()
417 dmam_free_coherent(mmc_dev(mmc), cq_host->desc_size, in cqhci_disable()
418 cq_host->desc_base, in cqhci_disable()
419 cq_host->desc_dma_base); in cqhci_disable()
421 cq_host->trans_desc_base = NULL; in cqhci_disable()
422 cq_host->desc_base = NULL; in cqhci_disable()
424 cq_host->enabled = false; in cqhci_disable()
431 u32 req_flags = mrq->data->flags; in cqhci_prep_task_desc()
444 CQHCI_BLK_COUNT(mrq->data->blocks) | in cqhci_prep_task_desc()
445 CQHCI_BLK_ADDR((u64)mrq->data->blk_addr); in cqhci_prep_task_desc()
449 if (cq_host->caps & CQHCI_TASK_DESC_SZ_128) { in cqhci_prep_task_desc()
455 mmc_hostname(mrq->host), mrq->tag, desc1, desc0); in cqhci_prep_task_desc()
458 mmc_hostname(mrq->host), mrq->tag, desc0); in cqhci_prep_task_desc()
465 struct mmc_data *data = mrq->data; in cqhci_dma_map()
468 return -EINVAL; in cqhci_dma_map()
470 sg_count = dma_map_sg(mmc_dev(host), data->sg, in cqhci_dma_map()
471 data->sg_len, in cqhci_dma_map()
472 (data->flags & MMC_DATA_WRITE) ? in cqhci_dma_map()
475 pr_err("%s: sg-len: %d\n", __func__, data->sg_len); in cqhci_dma_map()
476 return -ENOMEM; in cqhci_dma_map()
508 struct mmc_data *data = mrq->data; in cqhci_prep_tran_desc()
511 bool dma64 = cq_host->dma64; in cqhci_prep_tran_desc()
516 sg_count = cqhci_dma_map(mrq->host, mrq); in cqhci_prep_tran_desc()
519 mmc_hostname(mrq->host), __func__, sg_count); in cqhci_prep_tran_desc()
525 for_each_sg(data->sg, sg, sg_count, i) { in cqhci_prep_tran_desc()
531 if (cq_host->ops->set_tran_desc) in cqhci_prep_tran_desc()
532 cq_host->ops->set_tran_desc(cq_host, &desc, addr, len, end, dma64); in cqhci_prep_tran_desc()
536 desc += cq_host->trans_desc_len; in cqhci_prep_tran_desc()
550 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_prep_dcmd_desc()
553 if (!(mrq->cmd->flags & MMC_RSP_PRESENT)) { in cqhci_prep_dcmd_desc()
557 if (mrq->cmd->flags & MMC_RSP_R1B) { in cqhci_prep_dcmd_desc()
566 task_desc = (__le64 __force *)get_desc(cq_host, cq_host->dcmd_slot); in cqhci_prep_dcmd_desc()
567 memset(task_desc, 0, cq_host->task_desc_len); in cqhci_prep_dcmd_desc()
573 CQHCI_CMD_INDEX(mrq->cmd->opcode) | in cqhci_prep_dcmd_desc()
575 if (cq_host->ops->update_dcmd_desc) in cqhci_prep_dcmd_desc()
576 cq_host->ops->update_dcmd_desc(mmc, mrq, &data); in cqhci_prep_dcmd_desc()
579 pr_debug("%s: cqhci: dcmd: cmd: %d timing: %d resp: %d\n", in cqhci_prep_dcmd_desc()
580 mmc_hostname(mmc), mrq->cmd->opcode, timing, resp_type); in cqhci_prep_dcmd_desc()
582 dataddr[0] = cpu_to_le64((u64)mrq->cmd->arg); in cqhci_prep_dcmd_desc()
588 struct mmc_data *data = mrq->data; in cqhci_post_req()
591 dma_unmap_sg(mmc_dev(host), data->sg, data->sg_len, in cqhci_post_req()
592 (data->flags & MMC_DATA_READ) ? in cqhci_post_req()
599 return mrq->cmd ? DCMD_SLOT : mrq->tag; in cqhci_tag()
606 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_request()
609 if (!cq_host->enabled) { in cqhci_request()
611 return -EINVAL; in cqhci_request()
614 /* First request after resume has to re-enable */ in cqhci_request()
615 if (!cq_host->activated) in cqhci_request()
618 if (!mmc->cqe_on) { in cqhci_request()
619 if (cq_host->ops->pre_enable) in cqhci_request()
620 cq_host->ops->pre_enable(mmc); in cqhci_request()
623 mmc->cqe_on = true; in cqhci_request()
624 pr_debug("%s: cqhci: CQE on\n", mmc_hostname(mmc)); in cqhci_request()
626 pr_err("%s: cqhci: CQE failed to exit halt state\n", in cqhci_request()
629 if (cq_host->ops->enable) in cqhci_request()
630 cq_host->ops->enable(mmc); in cqhci_request()
633 if (mrq->data) { in cqhci_request()
646 spin_lock_irqsave(&cq_host->lock, flags); in cqhci_request()
648 if (cq_host->recovery_halt) { in cqhci_request()
649 err = -EBUSY; in cqhci_request()
653 cq_host->slot[tag].mrq = mrq; in cqhci_request()
654 cq_host->slot[tag].flags = 0; in cqhci_request()
656 cq_host->qcnt += 1; in cqhci_request()
664 spin_unlock_irqrestore(&cq_host->lock, flags); in cqhci_request()
675 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_recovery_needed()
677 if (!cq_host->recovery_halt) { in cqhci_recovery_needed()
678 cq_host->recovery_halt = true; in cqhci_recovery_needed()
680 wake_up(&cq_host->wait_queue); in cqhci_recovery_needed()
681 if (notify && mrq->recovery_notifier) in cqhci_recovery_needed()
682 mrq->recovery_notifier(mrq); in cqhci_recovery_needed()
691 case -EILSEQ: in cqhci_error_flags()
693 case -ETIMEDOUT: in cqhci_error_flags()
703 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_error_irq()
709 spin_lock(&cq_host->lock); in cqhci_error_irq()
717 if (cq_host->recovery_halt) in cqhci_error_irq()
720 if (!cq_host->qcnt) { in cqhci_error_irq()
729 slot = &cq_host->slot[tag]; in cqhci_error_irq()
730 if (slot->mrq) { in cqhci_error_irq()
731 slot->flags = cqhci_error_flags(cmd_error, data_error); in cqhci_error_irq()
732 cqhci_recovery_needed(mmc, slot->mrq, true); in cqhci_error_irq()
738 slot = &cq_host->slot[tag]; in cqhci_error_irq()
739 if (slot->mrq) { in cqhci_error_irq()
740 slot->flags = cqhci_error_flags(data_error, cmd_error); in cqhci_error_irq()
741 cqhci_recovery_needed(mmc, slot->mrq, true); in cqhci_error_irq()
747 * never happen, since the block layer ensures that all crypto-enabled in cqhci_error_irq()
761 slot = &cq_host->slot[tag]; in cqhci_error_irq()
762 if (!slot->mrq) in cqhci_error_irq()
764 slot->flags = cqhci_error_flags(data_error, cmd_error); in cqhci_error_irq()
765 cqhci_recovery_needed(mmc, slot->mrq, true); in cqhci_error_irq()
769 if (!cq_host->recovery_halt) { in cqhci_error_irq()
775 slot = &cq_host->slot[tag]; in cqhci_error_irq()
776 if (!slot->mrq) in cqhci_error_irq()
778 slot->flags = cqhci_error_flags(data_error, cmd_error); in cqhci_error_irq()
779 cqhci_recovery_needed(mmc, slot->mrq, true); in cqhci_error_irq()
785 spin_unlock(&cq_host->lock); in cqhci_error_irq()
790 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_finish_mrq()
791 struct cqhci_slot *slot = &cq_host->slot[tag]; in cqhci_finish_mrq()
792 struct mmc_request *mrq = slot->mrq; in cqhci_finish_mrq()
802 if (cq_host->recovery_halt) { in cqhci_finish_mrq()
803 slot->flags |= CQHCI_COMPLETED; in cqhci_finish_mrq()
807 slot->mrq = NULL; in cqhci_finish_mrq()
809 cq_host->qcnt -= 1; in cqhci_finish_mrq()
811 data = mrq->data; in cqhci_finish_mrq()
813 if (data->error) in cqhci_finish_mrq()
814 data->bytes_xfered = 0; in cqhci_finish_mrq()
816 data->bytes_xfered = data->blksz * data->blocks; in cqhci_finish_mrq()
827 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_irq()
852 spin_lock(&cq_host->lock); in cqhci_irq()
854 for_each_set_bit(tag, &comp_status, cq_host->num_slots) { in cqhci_irq()
861 if (cq_host->waiting_for_idle && !cq_host->qcnt) { in cqhci_irq()
862 cq_host->waiting_for_idle = false; in cqhci_irq()
863 wake_up(&cq_host->wait_queue); in cqhci_irq()
866 spin_unlock(&cq_host->lock); in cqhci_irq()
870 wake_up(&cq_host->wait_queue); in cqhci_irq()
873 wake_up(&cq_host->wait_queue); in cqhci_irq()
884 spin_lock_irqsave(&cq_host->lock, flags); in cqhci_is_idle()
885 is_idle = !cq_host->qcnt || cq_host->recovery_halt; in cqhci_is_idle()
886 *ret = cq_host->recovery_halt ? -EBUSY : 0; in cqhci_is_idle()
887 cq_host->waiting_for_idle = !is_idle; in cqhci_is_idle()
888 spin_unlock_irqrestore(&cq_host->lock, flags); in cqhci_is_idle()
895 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_wait_for_idle()
898 wait_event(cq_host->wait_queue, cqhci_is_idle(cq_host, &ret)); in cqhci_wait_for_idle()
906 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_timeout()
908 struct cqhci_slot *slot = &cq_host->slot[tag]; in cqhci_timeout()
912 spin_lock_irqsave(&cq_host->lock, flags); in cqhci_timeout()
913 timed_out = slot->mrq == mrq; in cqhci_timeout()
915 slot->flags |= CQHCI_EXTERNAL_TIMEOUT; in cqhci_timeout()
917 *recovery_needed = cq_host->recovery_halt; in cqhci_timeout()
919 spin_unlock_irqrestore(&cq_host->lock, flags); in cqhci_timeout()
923 mmc_hostname(mmc), tag, cq_host->qcnt); in cqhci_timeout()
937 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_clear_all_tasks()
947 wait_event_timeout(cq_host->wait_queue, cqhci_tasks_cleared(cq_host), in cqhci_clear_all_tasks()
963 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_halt()
976 wait_event_timeout(cq_host->wait_queue, cqhci_halted(cq_host), in cqhci_halt()
999 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_recovery_start()
1003 WARN_ON(!cq_host->recovery_halt); in cqhci_recovery_start()
1007 if (cq_host->ops->disable) in cqhci_recovery_start()
1008 cq_host->ops->disable(mmc, true); in cqhci_recovery_start()
1010 mmc->cqe_on = false; in cqhci_recovery_start()
1018 /* CRC errors might indicate re-tuning so prefer to report that */ in cqhci_error_from_flags()
1020 return -EILSEQ; in cqhci_error_from_flags()
1023 return -ETIMEDOUT; in cqhci_error_from_flags()
1025 return -EIO; in cqhci_error_from_flags()
1030 struct cqhci_slot *slot = &cq_host->slot[tag]; in cqhci_recover_mrq()
1031 struct mmc_request *mrq = slot->mrq; in cqhci_recover_mrq()
1037 slot->mrq = NULL; in cqhci_recover_mrq()
1039 cq_host->qcnt -= 1; in cqhci_recover_mrq()
1041 data = mrq->data; in cqhci_recover_mrq()
1043 data->bytes_xfered = 0; in cqhci_recover_mrq()
1044 data->error = cqhci_error_from_flags(slot->flags); in cqhci_recover_mrq()
1046 mrq->cmd->error = cqhci_error_from_flags(slot->flags); in cqhci_recover_mrq()
1049 mmc_cqe_request_done(cq_host->mmc, mrq); in cqhci_recover_mrq()
1056 for (i = 0; i < cq_host->num_slots; i++) in cqhci_recover_mrqs()
1072 struct cqhci_host *cq_host = mmc->cqe_private; in cqhci_recovery_finish()
1079 WARN_ON(!cq_host->recovery_halt); in cqhci_recovery_finish()
1086 * be disabled/re-enabled, but not to disable before clearing tasks. in cqhci_recovery_finish()
1092 /* Disable to make sure tasks really are cleared */ in cqhci_recovery_finish()
1108 WARN_ON(cq_host->qcnt); in cqhci_recovery_finish()
1110 spin_lock_irqsave(&cq_host->lock, flags); in cqhci_recovery_finish()
1111 cq_host->qcnt = 0; in cqhci_recovery_finish()
1112 cq_host->recovery_halt = false; in cqhci_recovery_finish()
1113 mmc->cqe_on = false; in cqhci_recovery_finish()
1114 spin_unlock_irqrestore(&cq_host->lock, flags); in cqhci_recovery_finish()
1116 /* Ensure all writes are done before interrupts are re-enabled */ in cqhci_recovery_finish()
1147 dev_dbg(&pdev->dev, "CMDQ not supported\n"); in cqhci_pltfm_init()
1148 return ERR_PTR(-EINVAL); in cqhci_pltfm_init()
1151 cq_host = devm_kzalloc(&pdev->dev, sizeof(*cq_host), GFP_KERNEL); in cqhci_pltfm_init()
1153 return ERR_PTR(-ENOMEM); in cqhci_pltfm_init()
1154 cq_host->mmio = devm_ioremap(&pdev->dev, in cqhci_pltfm_init()
1155 cqhci_memres->start, in cqhci_pltfm_init()
1157 if (!cq_host->mmio) { in cqhci_pltfm_init()
1158 dev_err(&pdev->dev, "failed to remap cqhci regs\n"); in cqhci_pltfm_init()
1159 return ERR_PTR(-EBUSY); in cqhci_pltfm_init()
1161 dev_dbg(&pdev->dev, "CMDQ ioremap: done\n"); in cqhci_pltfm_init()
1184 cq_host->dma64 = dma64; in cqhci_init()
1185 cq_host->mmc = mmc; in cqhci_init()
1186 cq_host->mmc->cqe_private = cq_host; in cqhci_init()
1188 cq_host->num_slots = NUM_SLOTS; in cqhci_init()
1189 cq_host->dcmd_slot = DCMD_SLOT; in cqhci_init()
1191 mmc->cqe_ops = &cqhci_cqe_ops; in cqhci_init()
1193 mmc->cqe_qdepth = NUM_SLOTS; in cqhci_init()
1194 if (mmc->caps2 & MMC_CAP2_CQE_DCMD) in cqhci_init()
1195 mmc->cqe_qdepth -= 1; in cqhci_init()
1197 cq_host->slot = devm_kcalloc(mmc_dev(mmc), cq_host->num_slots, in cqhci_init()
1198 sizeof(*cq_host->slot), GFP_KERNEL); in cqhci_init()
1199 if (!cq_host->slot) { in cqhci_init()
1200 err = -ENOMEM; in cqhci_init()
1211 spin_lock_init(&cq_host->lock); in cqhci_init()
1213 init_completion(&cq_host->halt_comp); in cqhci_init()
1214 init_waitqueue_head(&cq_host->wait_queue); in cqhci_init()