73 static inline struct folio *try_get_folio(struct page *page, int refs)  in try_get_folio()  argument
81 	if (unlikely(!folio_ref_try_add(folio, refs)))  in try_get_folio()
94 		if (!put_devmap_managed_folio_refs(folio, refs))  in try_get_folio()
95 			folio_put_refs(folio, refs);  in try_get_folio()
102 static void gup_put_folio(struct folio *folio, int refs, unsigned int flags)  in gup_put_folio()  argument
107 		node_stat_mod_folio(folio, NR_FOLL_PIN_RELEASED, refs);  in gup_put_folio()
109 			atomic_sub(refs, &folio->_pincount);  in gup_put_folio()
111 			refs *= GUP_PIN_COUNTING_BIAS;  in gup_put_folio()
114 	if (!put_devmap_managed_folio_refs(folio, refs))  in gup_put_folio()
115 		folio_put_refs(folio, refs);  in gup_put_folio()
121  * @refs:     the value to (effectively) add to the folio's refcount
141 int __must_check try_grab_folio(struct folio *folio, int refs,  in try_grab_folio()  argument
151 		folio_ref_add(folio, refs);  in try_grab_folio()
165 			folio_ref_add(folio, refs);  in try_grab_folio()
166 			atomic_add(refs, &folio->_pincount);  in try_grab_folio()
168 			folio_ref_add(folio, refs * GUP_PIN_COUNTING_BIAS);  in try_grab_folio()
171 		node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs);  in try_grab_folio()
496  * @refs:  the value to (effectively) add to the folio's refcount
506  *    FOLL_GET: folio's refcount will be incremented by @refs.
509  *    @refs, and its pincount will be incremented by @refs.
512  *    @refs * GUP_PIN_COUNTING_BIAS.
522 static struct folio *try_grab_folio_fast(struct page *page, int refs,  in try_grab_folio_fast()  argument
537 		return try_get_folio(page, refs);  in try_grab_folio_fast()
548 	folio = try_get_folio(page, refs);  in try_grab_folio_fast()
559 		if (!put_devmap_managed_folio_refs(folio, refs))  in try_grab_folio_fast()
560 			folio_put_refs(folio, refs);  in try_grab_folio_fast()
573 		atomic_add(refs, &folio->_pincount);  in try_grab_folio_fast()
576 				refs * (GUP_PIN_COUNTING_BIAS - 1));  in try_grab_folio_fast()
584 	node_stat_mod_folio(folio, NR_FOLL_PIN_ACQUIRED, refs);  in try_grab_folio_fast()
3092 	int refs;  in gup_fast_pmd_leaf()  local
3108 	refs = record_subpages(page, PMD_SIZE, addr, end, pages + *nr);  in gup_fast_pmd_leaf()
3110 	folio = try_grab_folio_fast(page, refs, flags);  in gup_fast_pmd_leaf()
3115 		gup_put_folio(folio, refs, flags);  in gup_fast_pmd_leaf()
3120 		gup_put_folio(folio, refs, flags);  in gup_fast_pmd_leaf()
3124 		gup_put_folio(folio, refs, flags);  in gup_fast_pmd_leaf()
3128 	*nr += refs;  in gup_fast_pmd_leaf()
3139 	int refs;  in gup_fast_pud_leaf()  local
3155 	refs = record_subpages(page, PUD_SIZE, addr, end, pages + *nr);  in gup_fast_pud_leaf()
3157 	folio = try_grab_folio_fast(page, refs, flags);  in gup_fast_pud_leaf()
3162 		gup_put_folio(folio, refs, flags);  in gup_fast_pud_leaf()
3167 		gup_put_folio(folio, refs, flags);  in gup_fast_pud_leaf()
3172 		gup_put_folio(folio, refs, flags);  in gup_fast_pud_leaf()
3176 	*nr += refs;  in gup_fast_pud_leaf()
3185 	int refs;  in gup_fast_pgd_leaf()  local
3195 	refs = record_subpages(page, PGDIR_SIZE, addr, end, pages + *nr);  in gup_fast_pgd_leaf()
3197 	folio = try_grab_folio_fast(page, refs, flags);  in gup_fast_pgd_leaf()
3202 		gup_put_folio(folio, refs, flags);  in gup_fast_pgd_leaf()
3207 		gup_put_folio(folio, refs, flags);  in gup_fast_pgd_leaf()
3212 		gup_put_folio(folio, refs, flags);  in gup_fast_pgd_leaf()
3216 	*nr += refs;  in gup_fast_pgd_leaf()