41 	struct xfs_mount	*mp = args->mp;  in xfs_rtany_summary()  local
47 	if (mp->m_rsum_cache) {  in xfs_rtany_summary()
48 		high = min(high, mp->m_rsum_cache[bbno] - 1);  in xfs_rtany_summary()
80 	if (mp->m_rsum_cache && log + 1 < mp->m_rsum_cache[bbno])  in xfs_rtany_summary()
81 		mp->m_rsum_cache[bbno] = log + 1;  in xfs_rtany_summary()
100 	for (log = oargs->mp->m_rsumlevels - 1; log >= 0; log--) {  in xfs_rtcopy_summary()
101 		for (bbno = oargs->mp->m_sb.sb_rbmblocks - 1;  in xfs_rtcopy_summary()
133 	struct xfs_mount	*mp = args->mp;  in xfs_rtallocate_range()  local
152 	error = xfs_rtfind_forw(args, end, mp->m_sb.sb_rextents - 1,  in xfs_rtallocate_range()
163 			xfs_rtx_to_rbmblock(mp, preblock), -1);  in xfs_rtallocate_range()
174 				xfs_rtx_to_rbmblock(mp, preblock), 1);  in xfs_rtallocate_range()
186 				xfs_rtx_to_rbmblock(mp, end + 1), 1);  in xfs_rtallocate_range()
214 	struct xfs_mount	*mp,  in xfs_rtallocate_clamp_len()  argument
221 	ret = min(mp->m_sb.sb_rextents, startrtx + rtxlen) - startrtx;  in xfs_rtallocate_clamp_len()
242 	struct xfs_mount	*mp = args->mp;  in xfs_rtallocate_extent_block()  local
256 	end = min(mp->m_sb.sb_rextents, xfs_rbmblock_to_rtx(mp, bbno + 1)) - 1;  in xfs_rtallocate_extent_block()
257 	for (i = xfs_rbmblock_to_rtx(mp, bbno); i <= end; i++) {  in xfs_rtallocate_extent_block()
259 		scanlen = xfs_rtallocate_clamp_len(mp, i, maxlen, prod);  in xfs_rtallocate_extent_block()
344 	struct xfs_mount	*mp = args->mp;  in xfs_rtallocate_extent_exact()  local
355 	scanlen = xfs_rtallocate_clamp_len(mp, start, maxlen, prod);  in xfs_rtallocate_extent_exact()
403 	struct xfs_mount	*mp = args->mp;  in xfs_rtallocate_extent_near()  local
419 	if (start >= mp->m_sb.sb_rextents)  in xfs_rtallocate_extent_near()
420 		start = mp->m_sb.sb_rextents - 1;  in xfs_rtallocate_extent_near()
430 	bbno = xfs_rtx_to_rbmblock(mp, start);  in xfs_rtallocate_extent_near()
443 		error = xfs_rtany_summary(args, log2len, mp->m_rsumlevels - 1,  in xfs_rtallocate_extent_near()
486 				else if (maxlog < mp->m_blkbit_log)  in xfs_rtallocate_extent_near()
489 					maxblocks = 2 << (maxlog - mp->m_blkbit_log);  in xfs_rtallocate_extent_near()
518 		else if (i > 0 && (int)bbno + i < mp->m_sb.sb_rbmblocks - 1)  in xfs_rtallocate_extent_near()
524 		else if (i <= 0 && (int)bbno - i < mp->m_sb.sb_rbmblocks - 1)  in xfs_rtallocate_extent_near()
554 	for (i = 0; i < args->mp->m_sb.sb_rbmblocks; i++) {  in xfs_rtalloc_sumlevel()
580 		if (xfs_rtx_to_rbmblock(args->mp, n) > i + 1)  in xfs_rtalloc_sumlevel()
581 			i = xfs_rtx_to_rbmblock(args->mp, n) - 1;  in xfs_rtalloc_sumlevel()
617 	for (l = xfs_highbit32(maxlen); l < args->mp->m_rsumlevels; l++) {  in xfs_rtallocate_extent_size()
654 	struct xfs_mount	*mp,  in xfs_alloc_rsum_cache()  argument
661 	mp->m_rsum_cache = kvmalloc(rbmblocks, GFP_KERNEL);  in xfs_alloc_rsum_cache()
662 	if (!mp->m_rsum_cache)  in xfs_alloc_rsum_cache()
664 	memset(mp->m_rsum_cache, -1, rbmblocks);  in xfs_alloc_rsum_cache()
676 	struct xfs_mount	*mp)  in xfs_growfs_rt_fixup_extsize()  argument
678 	struct xfs_inode	*ip = mp->m_rootip;  in xfs_growfs_rt_fixup_extsize()
687 	error = xfs_trans_alloc_inode(ip, &M_RES(mp)->tr_ichange, 0, 0, false,  in xfs_growfs_rt_fixup_extsize()
703 	struct xfs_mount	*mp,  in xfs_growfs_rt_bmblock()  argument
708 	struct xfs_inode	*rbmip = mp->m_rbmip;  in xfs_growfs_rt_bmblock()
709 	struct xfs_inode	*rsumip = mp->m_rsumip;  in xfs_growfs_rt_bmblock()
711 		.mp		= mp,  in xfs_growfs_rt_bmblock()
721 	nrblocks_step = (bmbno + 1) * NBBY * mp->m_sb.sb_blocksize * rextsize;  in xfs_growfs_rt_bmblock()
723 	nmp = nargs.mp = kmemdup(mp, sizeof(*mp), GFP_KERNEL);  in xfs_growfs_rt_bmblock()
737 	nmp->m_rsumblocks = xfs_rtsummary_blockcount(mp, nmp->m_rsumlevels,  in xfs_growfs_rt_bmblock()
745 	error = xfs_trans_alloc(mp, &M_RES(nmp)->tr_growrtfree, 0, 0, 0,  in xfs_growfs_rt_bmblock()
751 	xfs_rtbitmap_lock(mp);  in xfs_growfs_rt_bmblock()
776 	if (mp->m_sb.sb_rbmblocks != nmp->m_sb.sb_rbmblocks ||  in xfs_growfs_rt_bmblock()
777 	    mp->m_rsumlevels != nmp->m_rsumlevels) {  in xfs_growfs_rt_bmblock()
786 	if (nmp->m_sb.sb_rextsize != mp->m_sb.sb_rextsize)  in xfs_growfs_rt_bmblock()
788 			nmp->m_sb.sb_rextsize - mp->m_sb.sb_rextsize);  in xfs_growfs_rt_bmblock()
789 	if (nmp->m_sb.sb_rbmblocks != mp->m_sb.sb_rbmblocks)  in xfs_growfs_rt_bmblock()
791 			nmp->m_sb.sb_rbmblocks - mp->m_sb.sb_rbmblocks);  in xfs_growfs_rt_bmblock()
792 	if (nmp->m_sb.sb_rblocks != mp->m_sb.sb_rblocks)  in xfs_growfs_rt_bmblock()
794 			nmp->m_sb.sb_rblocks - mp->m_sb.sb_rblocks);  in xfs_growfs_rt_bmblock()
795 	if (nmp->m_sb.sb_rextents != mp->m_sb.sb_rextents)  in xfs_growfs_rt_bmblock()
797 			nmp->m_sb.sb_rextents - mp->m_sb.sb_rextents);  in xfs_growfs_rt_bmblock()
798 	if (nmp->m_sb.sb_rextslog != mp->m_sb.sb_rextslog)  in xfs_growfs_rt_bmblock()
800 			nmp->m_sb.sb_rextslog - mp->m_sb.sb_rextslog);  in xfs_growfs_rt_bmblock()
805 	freed_rtx = nmp->m_sb.sb_rextents - mp->m_sb.sb_rextents;  in xfs_growfs_rt_bmblock()
806 	error = xfs_rtfree_range(&nargs, mp->m_sb.sb_rextents, freed_rtx);  in xfs_growfs_rt_bmblock()
819 	mp->m_rsumlevels = nmp->m_rsumlevels;  in xfs_growfs_rt_bmblock()
820 	mp->m_rsumblocks = nmp->m_rsumblocks;  in xfs_growfs_rt_bmblock()
821 	xfs_mount_sb_set_rextsize(mp, &mp->m_sb);  in xfs_growfs_rt_bmblock()
826 	xfs_trans_resv_calc(mp, &mp->m_resv);  in xfs_growfs_rt_bmblock()
835 	mp->m_features |= XFS_FEAT_REALTIME;  in xfs_growfs_rt_bmblock()
854 	struct xfs_mount	*mp)  in xfs_last_rt_bmblock()  argument
856 	xfs_fileoff_t		bmbno = mp->m_sb.sb_rbmblocks;  in xfs_last_rt_bmblock()
859 	if (xfs_rtx_to_rbmword(mp, mp->m_sb.sb_rextents) != 0)  in xfs_last_rt_bmblock()
869 	xfs_mount_t	*mp,		/* mount point for filesystem */  in xfs_growfs_rt()  argument
881 	xfs_agblock_t	old_rextsize = mp->m_sb.sb_rextsize;  in xfs_growfs_rt()
887 	if (!XFS_IS_REALTIME_MOUNT(mp))  in xfs_growfs_rt()
890 	if (!mutex_trylock(&mp->m_growlock))  in xfs_growfs_rt()
897 	if (!mp->m_rbmip || !mp->m_rsumip)  in xfs_growfs_rt()
901 	if (in->newblocks <= mp->m_sb.sb_rblocks)  in xfs_growfs_rt()
904 	if (mp->m_sb.sb_rblocks > 0 && in->extsize != mp->m_sb.sb_rextsize)  in xfs_growfs_rt()
908 	if (XFS_FSB_TO_B(mp, in->extsize) > XFS_MAX_RTEXTSIZE ||  in xfs_growfs_rt()
909 	    XFS_FSB_TO_B(mp, in->extsize) < XFS_MIN_RTEXTSIZE)  in xfs_growfs_rt()
914 	if (xfs_has_rmapbt(mp) || xfs_has_reflink(mp) || xfs_has_quota(mp))  in xfs_growfs_rt()
917 	error = xfs_sb_validate_fsb_count(&mp->m_sb, in->newblocks);  in xfs_growfs_rt()
923 	error = xfs_buf_read_uncached(mp->m_rtdev_targp,  in xfs_growfs_rt()
924 				XFS_FSB_TO_BB(mp, in->newblocks - 1),  in xfs_growfs_rt()
925 				XFS_FSB_TO_BB(mp, 1), 0, &bp, NULL);  in xfs_growfs_rt()
938 	nrbmblocks = xfs_rtbitmap_blockcount(mp, nrextents);  in xfs_growfs_rt()
939 	nrsumblocks = xfs_rtsummary_blockcount(mp,  in xfs_growfs_rt()
947 	if (nrsumblocks > (mp->m_sb.sb_logblocks >> 1)) {  in xfs_growfs_rt()
956 	rbmblocks = XFS_B_TO_FSB(mp, mp->m_rbmip->i_disk_size);  in xfs_growfs_rt()
957 	rsumblocks = XFS_B_TO_FSB(mp, mp->m_rsumip->i_disk_size);  in xfs_growfs_rt()
961 	error = xfs_rtfile_initialize_blocks(mp->m_rbmip, rbmblocks,  in xfs_growfs_rt()
965 	error = xfs_rtfile_initialize_blocks(mp->m_rsumip, rsumblocks,  in xfs_growfs_rt()
970 	rsum_cache = mp->m_rsum_cache;  in xfs_growfs_rt()
971 	if (nrbmblocks != mp->m_sb.sb_rbmblocks) {  in xfs_growfs_rt()
972 		error = xfs_alloc_rsum_cache(mp, nrbmblocks);  in xfs_growfs_rt()
978 	for (bmbno = xfs_last_rt_bmblock(mp); bmbno < nrbmblocks; bmbno++) {  in xfs_growfs_rt()
979 		error = xfs_growfs_rt_bmblock(mp, in->newblocks, in->extsize,  in xfs_growfs_rt()
986 		error = xfs_growfs_rt_fixup_extsize(mp);  in xfs_growfs_rt()
992 	error = xfs_update_secondary_sbs(mp);  in xfs_growfs_rt()
1000 	if (rsum_cache != mp->m_rsum_cache) {  in xfs_growfs_rt()
1002 			kvfree(mp->m_rsum_cache);  in xfs_growfs_rt()
1003 			mp->m_rsum_cache = rsum_cache;  in xfs_growfs_rt()
1010 	mutex_unlock(&mp->m_growlock);  in xfs_growfs_rt()
1019 	struct xfs_mount	*mp)	/* file system mount structure */  in xfs_rtmount_init()  argument
1026 	sbp = &mp->m_sb;  in xfs_rtmount_init()
1029 	if (mp->m_rtdev_targp == NULL) {  in xfs_rtmount_init()
1030 		xfs_warn(mp,  in xfs_rtmount_init()
1034 	mp->m_rsumlevels = sbp->sb_rextslog + 1;  in xfs_rtmount_init()
1035 	mp->m_rsumblocks = xfs_rtsummary_blockcount(mp, mp->m_rsumlevels,  in xfs_rtmount_init()
1036 			mp->m_sb.sb_rbmblocks);  in xfs_rtmount_init()
1037 	mp->m_rbmip = mp->m_rsumip = NULL;  in xfs_rtmount_init()
1041 	d = (xfs_daddr_t)XFS_FSB_TO_BB(mp, mp->m_sb.sb_rblocks);  in xfs_rtmount_init()
1042 	if (XFS_BB_TO_FSB(mp, d) != mp->m_sb.sb_rblocks) {  in xfs_rtmount_init()
1043 		xfs_warn(mp, "realtime mount -- %llu != %llu",  in xfs_rtmount_init()
1044 			(unsigned long long) XFS_BB_TO_FSB(mp, d),  in xfs_rtmount_init()
1045 			(unsigned long long) mp->m_sb.sb_rblocks);  in xfs_rtmount_init()
1048 	error = xfs_buf_read_uncached(mp->m_rtdev_targp,  in xfs_rtmount_init()
1049 					d - XFS_FSB_TO_BB(mp, 1),  in xfs_rtmount_init()
1050 					XFS_FSB_TO_BB(mp, 1), 0, &bp, NULL);  in xfs_rtmount_init()
1052 		xfs_warn(mp, "realtime device size check failed");  in xfs_rtmount_init()
1061 	struct xfs_mount		*mp,  in xfs_rtalloc_count_frextent()  argument
1078 	struct xfs_mount	*mp)  in xfs_rtalloc_reinit_frextents()  argument
1083 	xfs_rtbitmap_lock_shared(mp, XFS_RBMLOCK_BITMAP);  in xfs_rtalloc_reinit_frextents()
1084 	error = xfs_rtalloc_query_all(mp, NULL, xfs_rtalloc_count_frextent,  in xfs_rtalloc_reinit_frextents()
1086 	xfs_rtbitmap_unlock_shared(mp, XFS_RBMLOCK_BITMAP);  in xfs_rtalloc_reinit_frextents()
1090 	spin_lock(&mp->m_sb_lock);  in xfs_rtalloc_reinit_frextents()
1091 	mp->m_sb.sb_frextents = val;  in xfs_rtalloc_reinit_frextents()
1092 	spin_unlock(&mp->m_sb_lock);  in xfs_rtalloc_reinit_frextents()
1093 	percpu_counter_set(&mp->m_frextents, mp->m_sb.sb_frextents);  in xfs_rtalloc_reinit_frextents()
1138 	xfs_mount_t	*mp)		/* file system mount structure */  in xfs_rtmount_inodes()  argument
1143 	sbp = &mp->m_sb;  in xfs_rtmount_inodes()
1144 	error = xfs_iget(mp, NULL, sbp->sb_rbmino, 0, 0, &mp->m_rbmip);  in xfs_rtmount_inodes()
1146 		xfs_rt_mark_sick(mp, XFS_SICK_RT_BITMAP);  in xfs_rtmount_inodes()
1149 	ASSERT(mp->m_rbmip != NULL);  in xfs_rtmount_inodes()
1151 	error = xfs_rtmount_iread_extents(mp->m_rbmip, XFS_ILOCK_RTBITMAP);  in xfs_rtmount_inodes()
1155 	error = xfs_iget(mp, NULL, sbp->sb_rsumino, 0, 0, &mp->m_rsumip);  in xfs_rtmount_inodes()
1157 		xfs_rt_mark_sick(mp, XFS_SICK_RT_SUMMARY);  in xfs_rtmount_inodes()
1160 	ASSERT(mp->m_rsumip != NULL);  in xfs_rtmount_inodes()
1162 	error = xfs_rtmount_iread_extents(mp->m_rsumip, XFS_ILOCK_RTSUM);  in xfs_rtmount_inodes()
1166 	error = xfs_alloc_rsum_cache(mp, sbp->sb_rbmblocks);  in xfs_rtmount_inodes()
1172 	xfs_irele(mp->m_rsumip);  in xfs_rtmount_inodes()
1174 	xfs_irele(mp->m_rbmip);  in xfs_rtmount_inodes()
1180 	struct xfs_mount	*mp)  in xfs_rtunmount_inodes()  argument
1182 	kvfree(mp->m_rsum_cache);  in xfs_rtunmount_inodes()
1183 	if (mp->m_rbmip)  in xfs_rtunmount_inodes()
1184 		xfs_irele(mp->m_rbmip);  in xfs_rtunmount_inodes()
1185 	if (mp->m_rsumip)  in xfs_rtunmount_inodes()
1186 		xfs_irele(mp->m_rsumip);  in xfs_rtunmount_inodes()
1198 	xfs_mount_t		*mp,		/* file system mount point */  in xfs_rtpick_extent()  argument
1208 	xfs_assert_ilocked(mp->m_rbmip, XFS_ILOCK_EXCL);  in xfs_rtpick_extent()
1210 	ts = inode_get_atime(VFS_I(mp->m_rbmip));  in xfs_rtpick_extent()
1211 	if (!(mp->m_rbmip->i_diflags & XFS_DIFLAG_NEWRTBM)) {  in xfs_rtpick_extent()
1212 		mp->m_rbmip->i_diflags |= XFS_DIFLAG_NEWRTBM;  in xfs_rtpick_extent()
1221 		b = (mp->m_sb.sb_rextents * ((resid << 1) + 1ULL)) >>  in xfs_rtpick_extent()
1223 		if (b >= mp->m_sb.sb_rextents)  in xfs_rtpick_extent()
1224 			div64_u64_rem(b, mp->m_sb.sb_rextents, &b);  in xfs_rtpick_extent()
1225 		if (b + len > mp->m_sb.sb_rextents)  in xfs_rtpick_extent()
1226 			b = mp->m_sb.sb_rextents - len;  in xfs_rtpick_extent()
1229 	inode_set_atime_to_ts(VFS_I(mp->m_rbmip), ts);  in xfs_rtpick_extent()
1230 	xfs_trans_log_inode(tp, mp->m_rbmip, XFS_ILOG_CORE);  in xfs_rtpick_extent()
1277 		.mp		= tp->t_mountp,  in xfs_rtallocate()
1289 		xfs_rtbitmap_lock(args.mp);  in xfs_rtallocate()
1299 		start = xfs_rtb_to_rtx(args.mp, bno_hint);  in xfs_rtallocate()
1301 		start = xfs_rtpick_extent(args.mp, tp, maxlen);  in xfs_rtallocate()
1331 	*bno = xfs_rtx_to_rtb(args.mp, rtx);  in xfs_rtallocate()
1332 	*blen = xfs_rtxlen_to_extlen(args.mp, len);  in xfs_rtallocate()
1347 	struct xfs_mount	*mp = ap->ip->i_mount;  in xfs_rtallocate_align()  local
1349 	xfs_extlen_t		minlen = mp->m_sb.sb_rextsize;  in xfs_rtallocate_align()
1355 		align = mp->m_sb.sb_rextsize;  in xfs_rtallocate_align()
1360 		if (align == mp->m_sb.sb_rextsize)  in xfs_rtallocate_align()
1364 	error = xfs_bmap_extsize_align(mp, &ap->got, &ap->prev, align, 1,  in xfs_rtallocate_align()
1369 	ASSERT(xfs_extlen_to_rtxmod(mp, ap->length) == 0);  in xfs_rtallocate_align()
1389 	*ralen = xfs_extlen_to_rtxlen(mp, min(ap->length, XFS_MAX_BMBT_EXTLEN));  in xfs_rtallocate_align()
1390 	*raminlen = max_t(xfs_rtxlen_t, 1, xfs_extlen_to_rtxlen(mp, minlen));  in xfs_rtallocate_align()
1402 		*prod = xfs_extlen_to_rtxlen(mp, align);  in xfs_rtallocate_align()