1 /* SPDX-License-Identifier: GPL-2.0 */
13 	unsigned int	offset;  member
31 #define sg_dma_address(sg)	((sg)->dma_address)
34 #define sg_dma_len(sg)		((sg)->dma_length)
36 #define sg_dma_len(sg)		((sg)->length)
55  * the page pointer AND encode information about the sg table as well. The two
71  * We overload the LSB of the page pointer to indicate whether it's
73  * Those low bits are there for everyone! (thanks mason :-)
79 	return sg->page_link & SG_PAGE_LINK_MASK;  in __sg_flags()
84 	return (struct scatterlist *)(sg->page_link & ~SG_PAGE_LINK_MASK);  in sg_chain_ptr()
98  * sg_assign_page - Assign a given page to an SG entry
100  * @page:	    The page
103  *   Assign page to sg entry. Also see sg_set_page(), the most commonly used
107 static inline void sg_assign_page(struct scatterlist *sg, struct page *page)  in sg_assign_page()  argument
109 	unsigned long page_link = sg->page_link & (SG_CHAIN | SG_END);  in sg_assign_page()
113 	 * must be aligned at a 32-bit boundary as a minimum.  in sg_assign_page()
115 	BUG_ON((unsigned long)page & SG_PAGE_LINK_MASK);  in sg_assign_page()
119 	sg->page_link = page_link | (unsigned long) page;  in sg_assign_page()
123  * sg_set_page - Set sg entry to point at given page
125  * @page:	 The page
127  * @offset:	 Offset into page
130  *   Use this function to set an sg entry pointing at a page, never assign
131  *   the page directly. We encode sg table information in the lower bits
132  *   of the page pointer. See sg_page() for looking up the page belonging
136 static inline void sg_set_page(struct scatterlist *sg, struct page *page,  in sg_set_page()  argument
137 			       unsigned int len, unsigned int offset)  in sg_set_page()  argument
139 	sg_assign_page(sg, page);  in sg_set_page()
140 	sg->offset = offset;  in sg_set_page()
141 	sg->length = len;  in sg_set_page()
145  * sg_set_folio - Set sg entry to point at given folio
149  * @offset:	 Offset into folio
154  *   of the folio pointer. See sg_page() for looking up the page belonging
159 			       size_t len, size_t offset)  in sg_set_folio()  argument
162 	WARN_ON_ONCE(offset > UINT_MAX);  in sg_set_folio()
163 	sg_assign_page(sg, &folio->page);  in sg_set_folio()
164 	sg->offset = offset;  in sg_set_folio()
165 	sg->length = len;  in sg_set_folio()
168 static inline struct page *sg_page(struct scatterlist *sg)  in sg_page()
173 	return (struct page *)((sg)->page_link & ~SG_PAGE_LINK_MASK);  in sg_page()
177  * sg_set_buf - Set sg entry to point at given data
202 	for_each_sg((sgt)->sgl, sg, (sgt)->orig_nents, i)
210 	for_each_sg((sgt)->sgl, sg, (sgt)->nents, i)
216 	 * offset and length are unused for chain entry. Clear them.  in __sg_chain()
218 	chain_sg->offset = 0;  in __sg_chain()
219 	chain_sg->length = 0;  in __sg_chain()
225 	chain_sg->page_link = ((unsigned long) sgl | SG_CHAIN) & ~SG_END;  in __sg_chain()
229  * sg_chain - Chain two sglists together
241 	__sg_chain(&prv[prv_nents - 1], sgl);  in sg_chain()
245  * sg_mark_end - Mark the end of the scatterlist
258 	sg->page_link |= SG_END;  in sg_mark_end()
259 	sg->page_link &= ~SG_CHAIN;  in sg_mark_end()
263  * sg_unmark_end - Undo setting the end of the scatterlist
272 	sg->page_link &= ~SG_END;  in sg_unmark_end()
276  * One 64-bit architectures there is a 4-byte padding in struct scatterlist
287  * sg_dma_is_bus_address - Return whether a given segment was marked
297 	return sg->dma_flags & SG_DMA_BUS_ADDRESS;  in sg_dma_is_bus_address()
301  * sg_dma_mark_bus_address - Mark the scatterlist entry as a bus address
312 	sg->dma_flags |= SG_DMA_BUS_ADDRESS;  in sg_dma_mark_bus_address()
316  * sg_unmark_bus_address - Unmark the scatterlist entry as a bus address
324 	sg->dma_flags &= ~SG_DMA_BUS_ADDRESS;  in sg_dma_unmark_bus_address()
328  * sg_dma_is_swiotlb - Return whether the scatterlist was marked for SWIOTLB
339 	return sg->dma_flags & SG_DMA_SWIOTLB;  in sg_dma_is_swiotlb()
343  * sg_dma_mark_swiotlb - Mark the scatterlist for SWIOTLB bouncing
352 	sg->dma_flags |= SG_DMA_SWIOTLB;  in sg_dma_mark_swiotlb()
378  * sg_phys - Return physical address of an sg entry
382  *   This calls page_to_phys() on the page in this sg entry, and adds the
383  *   sg offset. The caller must know that it is legal to call page_to_phys()
384  *   on the sg page.
389 	return page_to_phys(sg_page(sg)) + sg->offset;  in sg_phys()
393  * sg_virt - Return virtual address of an sg entry
397  *   This calls page_address() on the page in this sg entry, and adds the
398  *   sg offset. The caller must know that the sg page has a valid virtual
404 	return page_address(sg_page(sg)) + sg->offset;  in sg_virt()
408  * sg_init_marker - Initialize markers in sg table
416 	sg_mark_end(&sgl[nents - 1]);  in sg_init_marker()
442 				     struct page **pages, unsigned int n_pages,
443 				     unsigned int offset, unsigned long size,
446 int sg_alloc_table_from_pages_segment(struct sg_table *sgt, struct page **pages,
447 				      unsigned int n_pages, unsigned int offset,
452  * sg_alloc_table_from_pages - Allocate and initialize an sg table from
455  * @pages:	 Pointer to an array of page pointers
457  * @offset:      Offset from start of the first page to the start of a buffer
458  * @size:        Number of valid bytes in the buffer (after offset)
464  *    may provide an offset at a start and a size of valid data in a buffer
465  *    specified by the page array. The returned sg table is released by
472 					    struct page **pages,  in sg_alloc_table_from_pages()
474 					    unsigned int offset,  in sg_alloc_table_from_pages()  argument
477 	return sg_alloc_table_from_pages_segment(sgt, pages, n_pages, offset,  in sg_alloc_table_from_pages()
516  * single page, to avoid a higher order allocation.  We could define this
541  * sg page iterator
543  * Iterates over sg entries page-by-page.  On each successful iteration, you
544  * can call sg_page_iter_page(@piter) to get the current page.
545  * @piter->sg will point to the sg holding this page and @piter->sg_pgoffset to
546  * the page's page offset within the sg. The iteration will stop either when a
551 	struct scatterlist	*sg;		/* sg holding the page */
552 	unsigned int		sg_pgoffset;	/* page offset within the sg */
561  * sg page iterator for DMA addresses
564  * sg_page_iter_dma_address(@dma_iter) to get the page's DMA
577  * sg_page_iter_page - get the current page held by the page iterator
578  * @piter:	page iterator holding the page
580 static inline struct page *sg_page_iter_page(struct sg_page_iter *piter)  in sg_page_iter_page()
582 	return nth_page(sg_page(piter->sg), piter->sg_pgoffset);  in sg_page_iter_page()
586  * sg_page_iter_dma_address - get the dma address of the current page held by
587  * the page iterator.
588  * @dma_iter:	page iterator holding the page
593 	return sg_dma_address(dma_iter->base.sg) +  in sg_page_iter_dma_address()
594 	       (dma_iter->base.sg_pgoffset << PAGE_SHIFT);  in sg_page_iter_dma_address()
598  * for_each_sg_page - iterate over the pages of the given sg list
600  * @piter:	page iterator to hold current page, sg, sg_pgoffset
602  * @pgoffset:	starting page offset (in pages)
604  * Callers may use sg_page_iter_page() to get each page pointer.
612  * for_each_sg_dma_page - iterate over the pages of the given sg list
614  * @dma_iter:	DMA page iterator to hold current page
617  * @pgoffset:	starting page offset (in pages)
619  * Callers may use sg_page_iter_dma_address() to get each page's DMA address.
623 	for (__sg_page_iter_start(&(dma_iter)->base, sglist, dma_nents,        \
628  * for_each_sgtable_page - iterate over all pages in the sg_table object
630  * @piter:	page iterator to hold current page
631  * @pgoffset:	starting page offset (in pages)
638 	for_each_sg_page((sgt)->sgl, piter, (sgt)->orig_nents, pgoffset)
641  * for_each_sgtable_dma_page - iterate over the DMA mapped sg_table object
643  * @dma_iter:	DMA page iterator to hold current page
644  * @pgoffset:	starting page offset (in pages)
652 	for_each_sg_dma_page((sgt)->sgl, dma_iter, (sgt)->nents, pgoffset)
658  * Iterates over sg entries mapping page-by-page.  On each successful
659  * iteration, @miter->page points to the mapped page and
660  * @miter->length bytes of data can be accessed at @miter->addr.  As
664  * @miter->consumed is set to @miter->length on each iteration.  It
677 	struct page		*page;		/* currently mapped page */  member
681 	struct sg_page_iter	piter;		/* page iterator */
684 	unsigned int		__offset;	/* offset within page */
685 	unsigned int		__remaining;	/* remaining bytes on page */
691 bool sg_miter_skip(struct sg_mapping_iter *miter, off_t offset);