1 // SPDX-License-Identifier: GPL-2.0-only
4  * Copyright(c) 2004 - 2015 Intel Corporation.
10 #include <linux/dma-mapping.h>
36 	raw->field[xor_idx_to_field[idx]] = addr + offset;  in xor_set_src()
43 	return raw->field[pq_idx_to_field[idx]];  in pq_get_src()
50 	return raw->field[pq16_idx_to_field[idx]];  in pq16_get_src()
54 		       dma_addr_t addr, u32 offset, u8 coef, int idx)  in pq_set_src()  argument
59 	raw->field[pq_idx_to_field[idx]] = addr + offset;  in pq_set_src()
60 	pq->coef[idx] = coef;  in pq_set_src()
64 			dma_addr_t addr, u32 offset, u8 coef, unsigned idx)  in pq16_set_src()  argument
71 	raw->field[pq16_idx_to_field[idx]] = addr + offset;  in pq16_set_src()
74 		pq->coef[idx] = coef;  in pq16_set_src()
76 		pq16->coef[idx - 8] = coef;  in pq16_set_src()
89 	sed->hw_pool = hw_pool;  in ioat3_alloc_sed()
90 	sed->hw = dma_pool_alloc(ioat_dma->sed_hw_pool[hw_pool],  in ioat3_alloc_sed()
91 				 flags, &sed->dma);  in ioat3_alloc_sed()
92 	if (!sed->hw) {  in ioat3_alloc_sed()
108 	dma_addr_t src = dma_src;  in ioat_dma_prep_memcpy_lock()  local
112 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))  in ioat_dma_prep_memcpy_lock()
118 		idx = ioat_chan->head;  in ioat_dma_prep_memcpy_lock()
123 		size_t copy = min_t(size_t, len, 1 << ioat_chan->xfercap_log);  in ioat_dma_prep_memcpy_lock()
126 		hw = desc->hw;  in ioat_dma_prep_memcpy_lock()
128 		hw->size = copy;  in ioat_dma_prep_memcpy_lock()
129 		hw->ctl = 0;  in ioat_dma_prep_memcpy_lock()
130 		hw->src_addr = src;  in ioat_dma_prep_memcpy_lock()
131 		hw->dst_addr = dst;  in ioat_dma_prep_memcpy_lock()
133 		len -= copy;  in ioat_dma_prep_memcpy_lock()
135 		src += copy;  in ioat_dma_prep_memcpy_lock()
139 	desc->txd.flags = flags;  in ioat_dma_prep_memcpy_lock()
140 	desc->len = total_len;  in ioat_dma_prep_memcpy_lock()
141 	hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);  in ioat_dma_prep_memcpy_lock()
142 	hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);  in ioat_dma_prep_memcpy_lock()
143 	hw->ctl_f.compl_write = 1;  in ioat_dma_prep_memcpy_lock()
147 	return &desc->txd;  in ioat_dma_prep_memcpy_lock()
153 		      dma_addr_t dest, dma_addr_t *src, unsigned int src_cnt,  in __ioat_prep_xor_lock()  argument
187 		idx = ioat_chan->head;  in __ioat_prep_xor_lock()
194 					 len, 1 << ioat_chan->xfercap_log);  in __ioat_prep_xor_lock()
198 		xor = desc->xor;  in __ioat_prep_xor_lock()
205 		xor_ex = ext->xor_ex;  in __ioat_prep_xor_lock()
210 			xor_set_src(descs, src[s], offset, s);  in __ioat_prep_xor_lock()
211 		xor->size = xfer_size;  in __ioat_prep_xor_lock()
212 		xor->dst_addr = dest + offset;  in __ioat_prep_xor_lock()
213 		xor->ctl = 0;  in __ioat_prep_xor_lock()
214 		xor->ctl_f.op = op;  in __ioat_prep_xor_lock()
215 		xor->ctl_f.src_cnt = src_cnt_to_hw(src_cnt);  in __ioat_prep_xor_lock()
217 		len -= xfer_size;  in __ioat_prep_xor_lock()
223 	desc->txd.flags = flags;  in __ioat_prep_xor_lock()
224 	desc->len = total_len;  in __ioat_prep_xor_lock()
226 		desc->result = result;  in __ioat_prep_xor_lock()
227 	xor->ctl_f.fence = !!(flags & DMA_PREP_FENCE);  in __ioat_prep_xor_lock()
231 	compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;  in __ioat_prep_xor_lock()
232 	hw = compl_desc->hw;  in __ioat_prep_xor_lock()
233 	hw->ctl = 0;  in __ioat_prep_xor_lock()
234 	hw->ctl_f.null = 1;  in __ioat_prep_xor_lock()
235 	hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);  in __ioat_prep_xor_lock()
236 	hw->ctl_f.compl_write = 1;  in __ioat_prep_xor_lock()
237 	hw->size = NULL_DESC_BUFFER_SIZE;  in __ioat_prep_xor_lock()
241 	return &compl_desc->txd;  in __ioat_prep_xor_lock()
245 ioat_prep_xor(struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src,  in ioat_prep_xor()  argument
250 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))  in ioat_prep_xor()
253 	return __ioat_prep_xor_lock(chan, NULL, dest, src, src_cnt, len, flags);  in ioat_prep_xor()
257 ioat_prep_xor_val(struct dma_chan *chan, dma_addr_t *src,  in ioat_prep_xor_val()  argument
263 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))  in ioat_prep_xor_val()
271 	return __ioat_prep_xor_lock(chan, result, src[0], &src[1],  in ioat_prep_xor_val()
272 				     src_cnt - 1, len, flags);  in ioat_prep_xor_val()
280 	struct ioat_pq_descriptor *pq = desc->pq;  in dump_pq_desc_dbg()
281 	struct ioat_pq_ext_descriptor *pq_ex = ext ? ext->pq_ex : NULL;  in dump_pq_desc_dbg()
283 	int src_cnt = src_cnt_to_sw(pq->ctl_f.src_cnt);  in dump_pq_desc_dbg()
286 	dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"  in dump_pq_desc_dbg()
289 		desc_id(desc), (unsigned long long) desc->txd.phys,  in dump_pq_desc_dbg()
290 		(unsigned long long) (pq_ex ? pq_ex->next : pq->next),  in dump_pq_desc_dbg()
291 		desc->txd.flags, pq->size, pq->ctl, pq->ctl_f.op,  in dump_pq_desc_dbg()
292 		pq->ctl_f.int_en, pq->ctl_f.compl_write,  in dump_pq_desc_dbg()
293 		pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",  in dump_pq_desc_dbg()
294 		pq->ctl_f.src_cnt);  in dump_pq_desc_dbg()
296 		dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,  in dump_pq_desc_dbg()
297 			(unsigned long long) pq_get_src(descs, i), pq->coef[i]);  in dump_pq_desc_dbg()
298 	dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);  in dump_pq_desc_dbg()
299 	dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);  in dump_pq_desc_dbg()
300 	dev_dbg(dev, "\tNEXT: %#llx\n", pq->next);  in dump_pq_desc_dbg()
307 	struct ioat_pq_descriptor *pq = desc->pq;  in dump_pq16_desc_dbg()
311 	int src_cnt = src16_cnt_to_sw(pq->ctl_f.src_cnt);  in dump_pq16_desc_dbg()
314 	if (desc->sed) {  in dump_pq16_desc_dbg()
315 		descs[1] = (void *)desc->sed->hw;  in dump_pq16_desc_dbg()
316 		descs[2] = (void *)desc->sed->hw + 64;  in dump_pq16_desc_dbg()
319 	dev_dbg(dev, "desc[%d]: (%#llx->%#llx) flags: %#x"  in dump_pq16_desc_dbg()
322 		desc_id(desc), (unsigned long long) desc->txd.phys,  in dump_pq16_desc_dbg()
323 		(unsigned long long) pq->next,  in dump_pq16_desc_dbg()
324 		desc->txd.flags, pq->size, pq->ctl,  in dump_pq16_desc_dbg()
325 		pq->ctl_f.op, pq->ctl_f.int_en,  in dump_pq16_desc_dbg()
326 		pq->ctl_f.compl_write,  in dump_pq16_desc_dbg()
327 		pq->ctl_f.p_disable ? "" : "p", pq->ctl_f.q_disable ? "" : "q",  in dump_pq16_desc_dbg()
328 		pq->ctl_f.src_cnt);  in dump_pq16_desc_dbg()
330 		dev_dbg(dev, "\tsrc[%d]: %#llx coef: %#x\n", i,  in dump_pq16_desc_dbg()
332 			pq->coef[i]);  in dump_pq16_desc_dbg()
334 	dev_dbg(dev, "\tP: %#llx\n", pq->p_addr);  in dump_pq16_desc_dbg()
335 	dev_dbg(dev, "\tQ: %#llx\n", pq->q_addr);  in dump_pq16_desc_dbg()
340 		     const dma_addr_t *dst, const dma_addr_t *src,  in __ioat_prep_pq_lock()  argument
345 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;  in __ioat_prep_pq_lock()
356 	int cb32 = (ioat_dma->version < IOAT_VER_3_3) ? 1 : 0;  in __ioat_prep_pq_lock()
366 	 * sources (we need 1 extra source in the q-only continuation  in __ioat_prep_pq_lock()
383 		idx = ioat_chan->head;  in __ioat_prep_pq_lock()
390 					 1 << ioat_chan->xfercap_log);  in __ioat_prep_pq_lock()
393 		pq = desc->pq;  in __ioat_prep_pq_lock()
400 		pq_ex = ext->pq_ex;  in __ioat_prep_pq_lock()
406 			pq_set_src(descs, src[s], offset, scf[s], s);  in __ioat_prep_pq_lock()
416 		pq->size = xfer_size;  in __ioat_prep_pq_lock()
417 		pq->p_addr = dst[0] + offset;  in __ioat_prep_pq_lock()
418 		pq->q_addr = dst[1] + offset;  in __ioat_prep_pq_lock()
419 		pq->ctl = 0;  in __ioat_prep_pq_lock()
420 		pq->ctl_f.op = op;  in __ioat_prep_pq_lock()
422 		if (ioat_dma->cap & IOAT_CAP_DWBES)  in __ioat_prep_pq_lock()
423 			pq->ctl_f.wb_en = result ? 1 : 0;  in __ioat_prep_pq_lock()
424 		pq->ctl_f.src_cnt = src_cnt_to_hw(s);  in __ioat_prep_pq_lock()
425 		pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);  in __ioat_prep_pq_lock()
426 		pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);  in __ioat_prep_pq_lock()
428 		len -= xfer_size;  in __ioat_prep_pq_lock()
433 	desc->txd.flags = flags;  in __ioat_prep_pq_lock()
434 	desc->len = total_len;  in __ioat_prep_pq_lock()
436 		desc->result = result;  in __ioat_prep_pq_lock()
437 	pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);  in __ioat_prep_pq_lock()
441 		pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);  in __ioat_prep_pq_lock()
442 		pq->ctl_f.compl_write = 1;  in __ioat_prep_pq_lock()
447 		compl_desc->txd.flags = flags & DMA_PREP_INTERRUPT;  in __ioat_prep_pq_lock()
448 		hw = compl_desc->hw;  in __ioat_prep_pq_lock()
449 		hw->ctl = 0;  in __ioat_prep_pq_lock()
450 		hw->ctl_f.null = 1;  in __ioat_prep_pq_lock()
451 		hw->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);  in __ioat_prep_pq_lock()
452 		hw->ctl_f.compl_write = 1;  in __ioat_prep_pq_lock()
453 		hw->size = NULL_DESC_BUFFER_SIZE;  in __ioat_prep_pq_lock()
459 	return &compl_desc->txd;  in __ioat_prep_pq_lock()
464 		       const dma_addr_t *dst, const dma_addr_t *src,  in __ioat_prep_pq16_lock()  argument
469 	struct ioatdma_device *ioat_dma = ioat_chan->ioat_dma;  in __ioat_prep_pq16_lock()
477 	/* this function is only called with 9-16 sources */  in __ioat_prep_pq16_lock()
489 		idx = ioat_chan->head;  in __ioat_prep_pq16_lock()
498 					 1 << ioat_chan->xfercap_log);  in __ioat_prep_pq16_lock()
501 		pq = desc->pq;  in __ioat_prep_pq16_lock()
505 		desc->sed = ioat3_alloc_sed(ioat_dma, (src_cnt-2) >> 3);  in __ioat_prep_pq16_lock()
506 		if (!desc->sed) {  in __ioat_prep_pq16_lock()
512 		pq->sed_addr = desc->sed->dma;  in __ioat_prep_pq16_lock()
513 		desc->sed->parent = desc;  in __ioat_prep_pq16_lock()
515 		descs[1] = (struct ioat_raw_descriptor *)desc->sed->hw;  in __ioat_prep_pq16_lock()
519 			pq16_set_src(descs, src[s], offset, scf[s], s);  in __ioat_prep_pq16_lock()
530 		pq->size = xfer_size;  in __ioat_prep_pq16_lock()
531 		pq->p_addr = dst[0] + offset;  in __ioat_prep_pq16_lock()
532 		pq->q_addr = dst[1] + offset;  in __ioat_prep_pq16_lock()
533 		pq->ctl = 0;  in __ioat_prep_pq16_lock()
534 		pq->ctl_f.op = op;  in __ioat_prep_pq16_lock()
535 		pq->ctl_f.src_cnt = src16_cnt_to_hw(s);  in __ioat_prep_pq16_lock()
537 		if (ioat_dma->cap & IOAT_CAP_DWBES)  in __ioat_prep_pq16_lock()
538 			pq->ctl_f.wb_en = result ? 1 : 0;  in __ioat_prep_pq16_lock()
539 		pq->ctl_f.p_disable = !!(flags & DMA_PREP_PQ_DISABLE_P);  in __ioat_prep_pq16_lock()
540 		pq->ctl_f.q_disable = !!(flags & DMA_PREP_PQ_DISABLE_Q);  in __ioat_prep_pq16_lock()
542 		len -= xfer_size;  in __ioat_prep_pq16_lock()
547 	desc->txd.flags = flags;  in __ioat_prep_pq16_lock()
548 	desc->len = total_len;  in __ioat_prep_pq16_lock()
550 		desc->result = result;  in __ioat_prep_pq16_lock()
551 	pq->ctl_f.fence = !!(flags & DMA_PREP_FENCE);  in __ioat_prep_pq16_lock()
554 	pq->ctl_f.int_en = !!(flags & DMA_PREP_INTERRUPT);  in __ioat_prep_pq16_lock()
555 	pq->ctl_f.compl_write = 1;  in __ioat_prep_pq16_lock()
560 	return &desc->txd;  in __ioat_prep_pq16_lock()
574 ioat_prep_pq(struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src,  in ioat_prep_pq()  argument
580 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))  in ioat_prep_pq()
597 		single_source[0] = src[0];  in ioat_prep_pq()
598 		single_source[1] = src[0];  in ioat_prep_pq()
611 			__ioat_prep_pq16_lock(chan, NULL, dst, src, src_cnt,  in ioat_prep_pq()
613 			__ioat_prep_pq_lock(chan, NULL, dst, src, src_cnt,  in ioat_prep_pq()
619 ioat_prep_pq_val(struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src,  in ioat_prep_pq_val()  argument
625 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))  in ioat_prep_pq_val()
640 		__ioat_prep_pq16_lock(chan, pqres, pq, src, src_cnt, scf, len,  in ioat_prep_pq_val()
642 		__ioat_prep_pq_lock(chan, pqres, pq, src, src_cnt, scf, len,  in ioat_prep_pq_val()
647 ioat_prep_pqxor(struct dma_chan *chan, dma_addr_t dst, dma_addr_t *src,  in ioat_prep_pqxor()  argument
654 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))  in ioat_prep_pqxor()
666 		__ioat_prep_pq16_lock(chan, NULL, pq, src, src_cnt, scf, len,  in ioat_prep_pqxor()
668 		__ioat_prep_pq_lock(chan, NULL, pq, src, src_cnt, scf, len,  in ioat_prep_pqxor()
673 ioat_prep_pqxor_val(struct dma_chan *chan, dma_addr_t *src,  in ioat_prep_pqxor_val()  argument
681 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))  in ioat_prep_pqxor_val()
693 	pq[0] = src[0];  in ioat_prep_pqxor_val()
698 		__ioat_prep_pq16_lock(chan, result, pq, &src[1], src_cnt - 1,  in ioat_prep_pqxor_val()
700 		__ioat_prep_pq_lock(chan, result, pq, &src[1], src_cnt - 1,  in ioat_prep_pqxor_val()
711 	if (test_bit(IOAT_CHAN_DOWN, &ioat_chan->state))  in ioat_prep_interrupt_lock()
715 		desc = ioat_get_ring_ent(ioat_chan, ioat_chan->head);  in ioat_prep_interrupt_lock()
719 	hw = desc->hw;  in ioat_prep_interrupt_lock()
720 	hw->ctl = 0;  in ioat_prep_interrupt_lock()
721 	hw->ctl_f.null = 1;  in ioat_prep_interrupt_lock()
722 	hw->ctl_f.int_en = 1;  in ioat_prep_interrupt_lock()
723 	hw->ctl_f.fence = !!(flags & DMA_PREP_FENCE);  in ioat_prep_interrupt_lock()
724 	hw->ctl_f.compl_write = 1;  in ioat_prep_interrupt_lock()
725 	hw->size = NULL_DESC_BUFFER_SIZE;  in ioat_prep_interrupt_lock()
726 	hw->src_addr = 0;  in ioat_prep_interrupt_lock()
727 	hw->dst_addr = 0;  in ioat_prep_interrupt_lock()
729 	desc->txd.flags = flags;  in ioat_prep_interrupt_lock()
730 	desc->len = 1;  in ioat_prep_interrupt_lock()
735 	return &desc->txd;  in ioat_prep_interrupt_lock()