18  * @sg:		The current sg entry
21  *   Usually the next entry will be @sg@ + 1, but if this sg element is part
26 struct scatterlist *sg_next(struct scatterlist *sg)  in sg_next()  argument
28 	if (sg_is_last(sg))  in sg_next()
31 	sg++;  in sg_next()
32 	if (unlikely(sg_is_chain(sg)))  in sg_next()
33 		sg = sg_chain_ptr(sg);  in sg_next()
35 	return sg;  in sg_next()
41  * @sg:		The scatterlist
44  * Allows to know how many entries are in sg, taking into account
48 int sg_nents(struct scatterlist *sg)  in sg_nents()  argument
51 	for (nents = 0; sg; sg = sg_next(sg))  in sg_nents()
60  * @sg:		The scatterlist
64  * Determines the number of entries in sg that are required to meet
68  *   the number of sg entries needed, negative error on failure
71 int sg_nents_for_len(struct scatterlist *sg, u64 len)  in sg_nents_for_len()  argument
79 	for (nents = 0, total = 0; sg; sg = sg_next(sg)) {  in sg_nents_for_len()
81 		total += sg->length;  in sg_nents_for_len()
106 	struct scatterlist *sg, *ret = NULL;  in sg_last()  local
109 	for_each_sg(sgl, sg, nents, i)  in sg_last()
110 		ret = sg;  in sg_last()
118  * sg_init_table - Initialize SG table
119  * @sgl:	   The SG table
123  *   If this is part of a chained sg table, sg_mark_end() should be
135  * sg_init_one - Initialize a single entry sg list
136  * @sg:		 SG entry
141 void sg_init_one(struct scatterlist *sg, const void *buf, unsigned int buflen)  in sg_init_one()  argument
143 	sg_init_table(sg, 1);  in sg_init_one()
144 	sg_set_buf(sg, buf, buflen);  in sg_init_one()
172 static void sg_kfree(struct scatterlist *sg, unsigned int nents)  in sg_kfree()  argument
175 		kmemleak_free(sg);  in sg_kfree()
176 		free_page((unsigned long) sg);  in sg_kfree()
178 		kfree(sg);  in sg_kfree()
182  * __sg_free_table - Free a previously mapped sg table
183  * @table:	The sg table header to use
191  *    Free an sg table previously allocated and setup with
213 		 * then assign 'next' to the sg table after the current one.  in __sg_free_table()
240  * sg_free_append_table - Free a previously allocated append sg table.
241  * @table:	 The mapped sg append table header
253  * sg_free_table - Free a previously allocated sg table
254  * @table:	The mapped sg table header
265  * __sg_alloc_table - Allocate and initialize an sg table with given allocator
266  * @table:	The sg table header to use
267  * @nents:	Number of entries in sg list
291 	struct scatterlist *sg, *prv;  in __sg_alloc_table()  local
319 			sg = first_chunk;  in __sg_alloc_table()
322 			sg = alloc_fn(alloc_size, gfp_mask);  in __sg_alloc_table()
324 		if (unlikely(!sg)) {  in __sg_alloc_table()
337 		sg_init_table(sg, alloc_size);  in __sg_alloc_table()
341 		 * If this is the first mapping, assign the sg table header.  in __sg_alloc_table()
345 			sg_chain(prv, prv_max_ents, sg);  in __sg_alloc_table()
347 			table->sgl = sg;  in __sg_alloc_table()
353 			sg_mark_end(&sg[sg_size - 1]);  in __sg_alloc_table()
355 		prv = sg;  in __sg_alloc_table()
365  * sg_alloc_table - Allocate and initialize an sg table
366  * @table:	The sg table header to use
367  * @nents:	Number of entries in sg list
371  *    Allocate and initialize an sg table. If @nents@ is larger than
372  *    SG_MAX_SINGLE_ALLOC a chained sg table will be setup.
427  * sg_alloc_append_table_from_pages - Allocate and initialize an append sg
429  * @sgt_append:  The sg append table to use
439  *    In the first call it allocate and initialize an sg table from a list of
444  *    returned sg table is released by sg_free_append_table
483 		/* Merge contiguous pages into the last SG */  in sg_alloc_append_table_from_pages()
557  * sg_alloc_table_from_pages_segment - Allocate and initialize an sg table from
560  * @sgt:	 The sg table header to use
569  *    Allocate and initialize an sg table from a list of pages. Contiguous
574  *    The returned sg table is released by sg_free_table.
616 	struct scatterlist *sgl, *sg;  in sgl_alloc_order()  local
638 	sg = sgl;  in sgl_alloc_order()
647 		sg_set_page(sg, page, elem_len, 0);  in sgl_alloc_order()
649 		sg = sg_next(sg);  in sgl_alloc_order()
688 	struct scatterlist *sg;  in sgl_free_n_order()  local
692 	for_each_sg(sgl, sg, nents, i) {  in sgl_free_n_order()
693 		if (!sg)  in sgl_free_n_order()
695 		page = sg_page(sg);  in sgl_free_n_order()
733 	piter->sg = sglist;  in __sg_page_iter_start()
738 static int sg_page_count(struct scatterlist *sg)  in sg_page_count()  argument
740 	return PAGE_ALIGN(sg->offset + sg->length) >> PAGE_SHIFT;  in sg_page_count()
745 	if (!piter->__nents || !piter->sg)  in __sg_page_iter_next()
751 	while (piter->sg_pgoffset >= sg_page_count(piter->sg)) {  in __sg_page_iter_next()
752 		piter->sg_pgoffset -= sg_page_count(piter->sg);  in __sg_page_iter_next()
753 		piter->sg = sg_next(piter->sg);  in __sg_page_iter_next()
754 		if (!--piter->__nents || !piter->sg)  in __sg_page_iter_next()
762 static int sg_dma_page_count(struct scatterlist *sg)  in sg_dma_page_count()  argument
764 	return PAGE_ALIGN(sg->offset + sg_dma_len(sg)) >> PAGE_SHIFT;  in sg_dma_page_count()
771 	if (!piter->__nents || !piter->sg)  in __sg_page_iter_dma_next()
777 	while (piter->sg_pgoffset >= sg_dma_page_count(piter->sg)) {  in __sg_page_iter_dma_next()
778 		piter->sg_pgoffset -= sg_dma_page_count(piter->sg);  in __sg_page_iter_dma_next()
779 		piter->sg = sg_next(piter->sg);  in __sg_page_iter_dma_next()
780 		if (!--piter->__nents || !piter->sg)  in __sg_page_iter_dma_next()
789  * sg_miter_start - start mapping iteration over a sg list
790  * @miter: sg mapping iter to be started
791  * @sgl: sg list to iterate over
792  * @nents: number of sg entries
793  * @flags: sg iterator flags
815 		struct scatterlist *sg;  in sg_miter_get_next_page()  local
820 		sg = miter->piter.sg;  in sg_miter_get_next_page()
822 		miter->__offset = miter->piter.sg_pgoffset ? 0 : sg->offset;  in sg_miter_get_next_page()
825 		miter->__remaining = sg->offset + sg->length -  in sg_miter_get_next_page()
837  * @miter: sg mapping iter to be skipped
849  *   true if @miter contains the valid mapping.  false if end of sg
874  * @miter: sg mapping iter to proceed
885  *   true if @miter contains the next mapping.  false if end of sg
913  * @miter: sg mapping iter to be stopped
951  * sg_copy_buffer - Copy data between a linear buffer and an SG list
952  * @sgl:		 The SG list
953  * @nents:		 Number of SG entries
957  * @to_buffer:		 transfer direction (true == from an sg list to a
958  *			 buffer, false == from a buffer to an sg list)
1000  * sg_copy_from_buffer - Copy from a linear buffer to an SG list
1001  * @sgl:		 The SG list
1002  * @nents:		 Number of SG entries
1017  * sg_copy_to_buffer - Copy from an SG list to a linear buffer
1018  * @sgl:		 The SG list
1019  * @nents:		 Number of SG entries
1034  * sg_pcopy_from_buffer - Copy from a linear buffer to an SG list
1035  * @sgl:		 The SG list
1036  * @nents:		 Number of SG entries
1052  * sg_pcopy_to_buffer - Copy from an SG list to a linear buffer
1053  * @sgl:		 The SG list
1054  * @nents:		 Number of SG entries
1070  * sg_zero_buffer - Zero-out a part of a SG list
1071  * @sgl:		 The SG list
1072  * @nents:		 Number of SG entries
1114 	struct scatterlist *sg = sgtable->sgl + sgtable->nents;  in extract_user_to_sg()  local
1142 			sg_set_page(sg, page, seg, off);  in extract_user_to_sg()
1144 			sg++;  in extract_user_to_sg()
1169 	struct scatterlist *sg = sgtable->sgl + sgtable->nents;  in extract_bvec_to_sg()  local
1186 		sg_set_page(sg, bv[i].bv_page, len, off);  in extract_bvec_to_sg()
1188 		sg++;  in extract_bvec_to_sg()
1215 	struct scatterlist *sg = sgtable->sgl + sgtable->nents;  in extract_kvec_to_sg()  local
1245 			sg_set_page(sg, page, len, off);  in extract_kvec_to_sg()
1247 			sg++;  in extract_kvec_to_sg()
1276 	struct scatterlist *sg = sgtable->sgl + sgtable->nents;  in extract_folioq_to_sg()  local
1297 			sg_set_page(sg, folio_page(folio, 0), part, offset);  in extract_folioq_to_sg()
1299 			sg++;  in extract_folioq_to_sg()
1336 	struct scatterlist *sg = sgtable->sgl + sgtable->nents;  in extract_xarray_to_sg()  local
1358 		sg_set_page(sg, folio_page(folio, 0), len, offset);  in extract_xarray_to_sg()
1360 		sg++;  in extract_xarray_to_sg()