1 /* SPDX-License-Identifier: GPL-2.0 */
31 #define sg_dma_address(sg)	((sg)->dma_address)
34 #define sg_dma_len(sg)		((sg)->dma_length)
36 #define sg_dma_len(sg)		((sg)->length)
73  * Those low bits are there for everyone! (thanks mason :-)
79 	return sg->page_link & SG_PAGE_LINK_MASK;  in __sg_flags()
84 	return (struct scatterlist *)(sg->page_link & ~SG_PAGE_LINK_MASK);  in sg_chain_ptr()
98  * sg_assign_page - Assign a given page to an SG entry
109 	unsigned long page_link = sg->page_link & (SG_CHAIN | SG_END);  in sg_assign_page()
113 	 * must be aligned at a 32-bit boundary as a minimum.  in sg_assign_page()
119 	sg->page_link = page_link | (unsigned long) page;  in sg_assign_page()
123  * sg_set_page - Set sg entry to point at given page
140 	sg->offset = offset;  in sg_set_page()
141 	sg->length = len;  in sg_set_page()
145  * sg_set_folio - Set sg entry to point at given folio
163 	sg_assign_page(sg, &folio->page);  in sg_set_folio()
164 	sg->offset = offset;  in sg_set_folio()
165 	sg->length = len;  in sg_set_folio()
173 	return (struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK);  in sg_page()
177  * sg_set_buf - Set sg entry to point at given data
202 	for_each_sg((sgt)->sgl, sg, (sgt)->orig_nents, i)
210 	for_each_sg((sgt)->sgl, sg, (sgt)->nents, i)
218 	chain_sg->offset = 0;  in __sg_chain()
219 	chain_sg->length = 0;  in __sg_chain()
223 	 * the termination bit if it happens to be set.  in __sg_chain()
225 	chain_sg->page_link = ((unsigned long) sgl | SG_CHAIN) & ~SG_END;  in __sg_chain()
229  * sg_chain - Chain two sglists together
241 	__sg_chain(&prv[prv_nents - 1], sgl);  in sg_chain()
245  * sg_mark_end - Mark the end of the scatterlist
249  *   Marks the passed in sg entry as the termination point for the sg
256 	 * Set termination bit, clear potential chain bit  in sg_mark_end()
258 	sg->page_link |= SG_END;  in sg_mark_end()
259 	sg->page_link &= ~SG_CHAIN;  in sg_mark_end()
263  * sg_unmark_end - Undo setting the end of the scatterlist
267  *   Removes the termination marker from the given entry of the scatterlist.
272 	sg->page_link &= ~SG_END;  in sg_unmark_end()
276  * One 64-bit architectures there is a 4-byte padding in struct scatterlist
287  * sg_dma_is_bus_address - Return whether a given segment was marked
297 	return sg->dma_flags & SG_DMA_BUS_ADDRESS;  in sg_dma_is_bus_address()
301  * sg_dma_mark_bus_address - Mark the scatterlist entry as a bus address
312 	sg->dma_flags |= SG_DMA_BUS_ADDRESS;  in sg_dma_mark_bus_address()
316  * sg_unmark_bus_address - Unmark the scatterlist entry as a bus address
324 	sg->dma_flags &= ~SG_DMA_BUS_ADDRESS;  in sg_dma_unmark_bus_address()
328  * sg_dma_is_swiotlb - Return whether the scatterlist was marked for SWIOTLB
339 	return sg->dma_flags & SG_DMA_SWIOTLB;  in sg_dma_is_swiotlb()
343  * sg_dma_mark_swiotlb - Mark the scatterlist for SWIOTLB bouncing
352 	sg->dma_flags |= SG_DMA_SWIOTLB;  in sg_dma_mark_swiotlb()
378  * sg_phys - Return physical address of an sg entry
389 	return page_to_phys(sg_page(sg)) + sg->offset;  in sg_phys()
393  * sg_virt - Return virtual address of an sg entry
404 	return page_address(sg_page(sg)) + sg->offset;  in sg_virt()
408  * sg_init_marker - Initialize markers in sg table
416 	sg_mark_end(&sgl[nents - 1]);  in sg_init_marker()
452  * sg_alloc_table_from_pages - Allocate and initialize an sg table from
543  * Iterates over sg entries page-by-page.  On each successful iteration, you
544  * can call sg_page_iter_page(@piter) to get the current page.
545  * @piter->sg will point to the sg holding this page and @piter->sg_pgoffset to
577  * sg_page_iter_page - get the current page held by the page iterator
582 	return nth_page(sg_page(piter->sg), piter->sg_pgoffset);  in sg_page_iter_page()
586  * sg_page_iter_dma_address - get the dma address of the current page held by
593 	return sg_dma_address(dma_iter->base.sg) +  in sg_page_iter_dma_address()
594 	       (dma_iter->base.sg_pgoffset << PAGE_SHIFT);  in sg_page_iter_dma_address()
598  * for_each_sg_page - iterate over the pages of the given sg list
600  * @piter:	page iterator to hold current page, sg, sg_pgoffset
612  * for_each_sg_dma_page - iterate over the pages of the given sg list
614  * @dma_iter:	DMA page iterator to hold current page
623 	for (__sg_page_iter_start(&(dma_iter)->base, sglist, dma_nents,        \
628  * for_each_sgtable_page - iterate over all pages in the sg_table object
630  * @piter:	page iterator to hold current page
638 	for_each_sg_page((sgt)->sgl, piter, (sgt)->orig_nents, pgoffset)
641  * for_each_sgtable_dma_page - iterate over the DMA mapped sg_table object
643  * @dma_iter:	DMA page iterator to hold current page
652 	for_each_sg_dma_page((sgt)->sgl, dma_iter, (sgt)->nents, pgoffset)
658  * Iterates over sg entries mapping page-by-page.  On each successful
659  * iteration, @miter->page points to the mapped page and
660  * @miter->length bytes of data can be accessed at @miter->addr.  As
664  * @miter->consumed is set to @miter->length on each iteration.  It