254 	struct gmap *sg, *next;  in gmap_remove()  local
260 		list_for_each_entry_safe(sg, next, &gmap->children, list) {  in gmap_remove()
261 			list_del(&sg->list);  in gmap_remove()
262 			gmap_put(sg);  in gmap_remove()
975  * Expected to be called with sg->mm->mmap_lock in read and
1021  * Expected to be called with sg->mm->mmap_lock in read
1057  * Called with sg->mm->mmap_lock in read.
1190  * @sg: pointer to the shadow guest address space structure
1194  * Called with the sg->guest_table_lock
1196 static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr,  in gmap_insert_rmap()  argument
1202 	BUG_ON(!gmap_is_shadow(sg));  in gmap_insert_rmap()
1203 	slot = radix_tree_lookup_slot(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);  in gmap_insert_rmap()
1206 							&sg->guest_table_lock);  in gmap_insert_rmap()
1213 		radix_tree_replace_slot(&sg->host_to_rmap, slot, rmap);  in gmap_insert_rmap()
1216 		radix_tree_insert(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT,  in gmap_insert_rmap()
1223  * @sg: pointer to the shadow guest address space structure
1231 static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,  in gmap_protect_rmap()  argument
1241 	BUG_ON(!gmap_is_shadow(sg));  in gmap_protect_rmap()
1242 	parent = sg->parent;  in gmap_protect_rmap()
1259 			spin_lock(&sg->guest_table_lock);  in gmap_protect_rmap()
1263 				gmap_insert_rmap(sg, vmaddr, rmap);  in gmap_protect_rmap()
1264 			spin_unlock(&sg->guest_table_lock);  in gmap_protect_rmap()
1307  * @sg: pointer to the shadow guest address space structure
1310  * Called with the sg->guest_table_lock
1312 static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_page()  argument
1316 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_page()
1317 	table = gmap_table_walk(sg, raddr, 0); /* get page table pointer */  in gmap_unshadow_page()
1320 	gmap_call_notifier(sg, raddr, raddr + _PAGE_SIZE - 1);  in gmap_unshadow_page()
1321 	ptep_unshadow_pte(sg->mm, raddr, (pte_t *) table);  in gmap_unshadow_page()
1326  * @sg: pointer to the shadow guest address space structure
1330  * Called with the sg->guest_table_lock
1332 static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_pgt()  argument
1337 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_pgt()
1344  * @sg: pointer to the shadow guest address space structure
1347  * Called with the sg->guest_table_lock
1349 static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_pgt()  argument
1355 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_pgt()
1356 	ste = gmap_table_walk(sg, raddr, 1); /* get segment pointer */  in gmap_unshadow_pgt()
1359 	gmap_call_notifier(sg, raddr, raddr + _SEGMENT_SIZE - 1);  in gmap_unshadow_pgt()
1364 	__gmap_unshadow_pgt(sg, raddr, __va(pgt));  in gmap_unshadow_pgt()
1373  * @sg: pointer to the shadow guest address space structure
1377  * Called with the sg->guest_table_lock
1379 static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_sgt()  argument
1386 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_sgt()
1392 		__gmap_unshadow_pgt(sg, raddr, __va(pgt));  in __gmap_unshadow_sgt()
1402  * @sg: pointer to the shadow guest address space structure
1407 static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_sgt()  argument
1413 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_sgt()
1414 	r3e = gmap_table_walk(sg, raddr, 2); /* get region-3 pointer */  in gmap_unshadow_sgt()
1417 	gmap_call_notifier(sg, raddr, raddr + _REGION3_SIZE - 1);  in gmap_unshadow_sgt()
1422 	__gmap_unshadow_sgt(sg, raddr, __va(sgt));  in gmap_unshadow_sgt()
1431  * @sg: pointer to the shadow guest address space structure
1435  * Called with the sg->guest_table_lock
1437 static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_r3t()  argument
1444 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_r3t()
1450 		__gmap_unshadow_sgt(sg, raddr, __va(sgt));  in __gmap_unshadow_r3t()
1460  * @sg: pointer to the shadow guest address space structure
1463  * Called with the sg->guest_table_lock
1465 static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_r3t()  argument
1471 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_r3t()
1472 	r2e = gmap_table_walk(sg, raddr, 3); /* get region-2 pointer */  in gmap_unshadow_r3t()
1475 	gmap_call_notifier(sg, raddr, raddr + _REGION2_SIZE - 1);  in gmap_unshadow_r3t()
1480 	__gmap_unshadow_r3t(sg, raddr, __va(r3t));  in gmap_unshadow_r3t()
1489  * @sg: pointer to the shadow guest address space structure
1493  * Called with the sg->guest_table_lock
1495 static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_r2t()  argument
1502 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_r2t()
1508 		__gmap_unshadow_r3t(sg, raddr, __va(r3t));  in __gmap_unshadow_r2t()
1518  * @sg: pointer to the shadow guest address space structure
1521  * Called with the sg->guest_table_lock
1523 static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr)  in gmap_unshadow_r2t()  argument
1529 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow_r2t()
1530 	r1e = gmap_table_walk(sg, raddr, 4); /* get region-1 pointer */  in gmap_unshadow_r2t()
1533 	gmap_call_notifier(sg, raddr, raddr + _REGION1_SIZE - 1);  in gmap_unshadow_r2t()
1538 	__gmap_unshadow_r2t(sg, raddr, __va(r2t));  in gmap_unshadow_r2t()
1547  * @sg: pointer to the shadow guest address space structure
1553 static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr,  in __gmap_unshadow_r1t()  argument
1561 	BUG_ON(!gmap_is_shadow(sg));  in __gmap_unshadow_r1t()
1567 		__gmap_unshadow_r2t(sg, raddr, __va(r2t));  in __gmap_unshadow_r1t()
1580  * @sg: pointer to the shadow guest address space structure
1582  * Called with sg->guest_table_lock
1584 static void gmap_unshadow(struct gmap *sg)  in gmap_unshadow()  argument
1588 	BUG_ON(!gmap_is_shadow(sg));  in gmap_unshadow()
1589 	if (sg->removed)  in gmap_unshadow()
1591 	sg->removed = 1;  in gmap_unshadow()
1592 	gmap_call_notifier(sg, 0, -1UL);  in gmap_unshadow()
1593 	gmap_flush_tlb(sg);  in gmap_unshadow()
1594 	table = __va(sg->asce & _ASCE_ORIGIN);  in gmap_unshadow()
1595 	switch (sg->asce & _ASCE_TYPE_MASK) {  in gmap_unshadow()
1597 		__gmap_unshadow_r1t(sg, 0, table);  in gmap_unshadow()
1600 		__gmap_unshadow_r2t(sg, 0, table);  in gmap_unshadow()
1603 		__gmap_unshadow_r3t(sg, 0, table);  in gmap_unshadow()
1606 		__gmap_unshadow_sgt(sg, 0, table);  in gmap_unshadow()
1624 	struct gmap *sg;  in gmap_find_shadow()  local
1626 	list_for_each_entry(sg, &parent->children, list) {  in gmap_find_shadow()
1627 		if (sg->orig_asce != asce || sg->edat_level != edat_level ||  in gmap_find_shadow()
1628 		    sg->removed)  in gmap_find_shadow()
1630 		if (!sg->initialized)  in gmap_find_shadow()
1632 		refcount_inc(&sg->ref_count);  in gmap_find_shadow()
1633 		return sg;  in gmap_find_shadow()
1641  * @sg: pointer to the shadow guest address space structure
1650 int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level)  in gmap_shadow_valid()  argument
1652 	if (sg->removed)  in gmap_shadow_valid()
1654 	return sg->orig_asce == asce && sg->edat_level == edat_level;  in gmap_shadow_valid()
1676 	struct gmap *sg, *new;  in gmap_shadow()  local
1683 	sg = gmap_find_shadow(parent, asce, edat_level);  in gmap_shadow()
1685 	if (sg)  in gmap_shadow()
1686 		return sg;  in gmap_shadow()
1702 	sg = gmap_find_shadow(parent, asce, edat_level);  in gmap_shadow()
1703 	if (sg) {  in gmap_shadow()
1706 		return sg;  in gmap_shadow()
1710 		list_for_each_entry(sg, &parent->children, list) {  in gmap_shadow()
1711 			if (sg->orig_asce & _ASCE_REAL_SPACE) {  in gmap_shadow()
1712 				spin_lock(&sg->guest_table_lock);  in gmap_shadow()
1713 				gmap_unshadow(sg);  in gmap_shadow()
1714 				spin_unlock(&sg->guest_table_lock);  in gmap_shadow()
1715 				list_del(&sg->list);  in gmap_shadow()
1716 				gmap_put(sg);  in gmap_shadow()
1750  * @sg: pointer to the shadow guest address space structure
1764  * Called with sg->mm->mmap_lock in read.
1766 int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t,  in gmap_shadow_r2t()  argument
1775 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_r2t()
1785 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1786 	table = gmap_table_walk(sg, saddr, 4); /* get region-1 pointer */  in gmap_shadow_r2t()
1802 	if (sg->edat_level >= 1)  in gmap_shadow_r2t()
1804 	list_add(&page->lru, &sg->crst_list);  in gmap_shadow_r2t()
1808 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1811 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1817 	rc = gmap_protect_rmap(sg, raddr, origin + offset, len);  in gmap_shadow_r2t()
1818 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1820 		table = gmap_table_walk(sg, saddr, 4);  in gmap_shadow_r2t()
1826 		gmap_unshadow_r2t(sg, raddr);  in gmap_shadow_r2t()
1828 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1831 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r2t()
1839  * @sg: pointer to the shadow guest address space structure
1848  * Called with sg->mm->mmap_lock in read.
1850 int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t,  in gmap_shadow_r3t()  argument
1859 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_r3t()
1869 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1870 	table = gmap_table_walk(sg, saddr, 3); /* get region-2 pointer */  in gmap_shadow_r3t()
1886 	if (sg->edat_level >= 1)  in gmap_shadow_r3t()
1888 	list_add(&page->lru, &sg->crst_list);  in gmap_shadow_r3t()
1892 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1895 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1901 	rc = gmap_protect_rmap(sg, raddr, origin + offset, len);  in gmap_shadow_r3t()
1902 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1904 		table = gmap_table_walk(sg, saddr, 3);  in gmap_shadow_r3t()
1910 		gmap_unshadow_r3t(sg, raddr);  in gmap_shadow_r3t()
1912 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1915 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_r3t()
1923  * @sg: pointer to the shadow guest address space structure
1932  * Called with sg->mm->mmap_lock in read.
1934 int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt,  in gmap_shadow_sgt()  argument
1943 	BUG_ON(!gmap_is_shadow(sg) || (sgt & _REGION3_ENTRY_LARGE));  in gmap_shadow_sgt()
1953 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1954 	table = gmap_table_walk(sg, saddr, 2); /* get region-3 pointer */  in gmap_shadow_sgt()
1970 	if (sg->edat_level >= 1)  in gmap_shadow_sgt()
1972 	list_add(&page->lru, &sg->crst_list);  in gmap_shadow_sgt()
1976 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1979 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1985 	rc = gmap_protect_rmap(sg, raddr, origin + offset, len);  in gmap_shadow_sgt()
1986 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1988 		table = gmap_table_walk(sg, saddr, 2);  in gmap_shadow_sgt()
1994 		gmap_unshadow_sgt(sg, raddr);  in gmap_shadow_sgt()
1996 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_sgt()
1999 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_sgt()
2007  * @sg: pointer to the shadow guest address space structure
2016  * Called with sg->mm->mmap_lock in read.
2018 int gmap_shadow_pgt_lookup(struct gmap *sg, unsigned long saddr,  in gmap_shadow_pgt_lookup()  argument
2026 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_pgt_lookup()
2027 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_pgt_lookup()
2028 	table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */  in gmap_shadow_pgt_lookup()
2039 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt_lookup()
2047  * @sg: pointer to the shadow guest address space structure
2058 int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt,  in gmap_shadow_pgt()  argument
2067 	BUG_ON(!gmap_is_shadow(sg) || (pgt & _SEGMENT_ENTRY_LARGE));  in gmap_shadow_pgt()
2069 	ptdesc = page_table_alloc_pgste(sg->mm);  in gmap_shadow_pgt()
2077 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2078 	table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */  in gmap_shadow_pgt()
2093 	list_add(&ptdesc->pt_list, &sg->pt_list);  in gmap_shadow_pgt()
2097 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2100 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2104 	rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE);  in gmap_shadow_pgt()
2105 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2107 		table = gmap_table_walk(sg, saddr, 1);  in gmap_shadow_pgt()
2113 		gmap_unshadow_pgt(sg, raddr);  in gmap_shadow_pgt()
2115 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2118 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_pgt()
2127  * @sg: pointer to the shadow guest address space structure
2135  * Called with sg->mm->mmap_lock in read.
2137 int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte)  in gmap_shadow_page()  argument
2147 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_page()
2148 	parent = sg->parent;  in gmap_shadow_page()
2169 			spin_lock(&sg->guest_table_lock);  in gmap_shadow_page()
2171 			tptep = (pte_t *) gmap_table_walk(sg, saddr, 0);  in gmap_shadow_page()
2173 				spin_unlock(&sg->guest_table_lock);  in gmap_shadow_page()
2178 			rc = ptep_shadow_pte(sg->mm, saddr, sptep, tptep, pte);  in gmap_shadow_page()
2181 				gmap_insert_rmap(sg, vmaddr, rmap);  in gmap_shadow_page()
2186 			spin_unlock(&sg->guest_table_lock);  in gmap_shadow_page()
2203  * Called with sg->parent->shadow_lock.
2205 static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr,  in gmap_shadow_notify()  argument
2211 	BUG_ON(!gmap_is_shadow(sg));  in gmap_shadow_notify()
2213 	spin_lock(&sg->guest_table_lock);  in gmap_shadow_notify()
2214 	if (sg->removed) {  in gmap_shadow_notify()
2215 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_notify()
2219 	start = sg->orig_asce & _ASCE_ORIGIN;  in gmap_shadow_notify()
2220 	end = start + ((sg->orig_asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE;  in gmap_shadow_notify()
2221 	if (!(sg->orig_asce & _ASCE_REAL_SPACE) && gaddr >= start &&  in gmap_shadow_notify()
2224 		gmap_unshadow(sg);  in gmap_shadow_notify()
2225 		spin_unlock(&sg->guest_table_lock);  in gmap_shadow_notify()
2226 		list_del(&sg->list);  in gmap_shadow_notify()
2227 		gmap_put(sg);  in gmap_shadow_notify()
2231 	head = radix_tree_delete(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);  in gmap_shadow_notify()
2237 			gmap_unshadow_r2t(sg, raddr);  in gmap_shadow_notify()
2240 			gmap_unshadow_r3t(sg, raddr);  in gmap_shadow_notify()
2243 			gmap_unshadow_sgt(sg, raddr);  in gmap_shadow_notify()
2246 			gmap_unshadow_pgt(sg, raddr);  in gmap_shadow_notify()
2249 			gmap_unshadow_page(sg, raddr);  in gmap_shadow_notify()
2254 	spin_unlock(&sg->guest_table_lock);  in gmap_shadow_notify()
2272 	struct gmap *gmap, *sg, *next;  in ptep_notify()  local
2289 			list_for_each_entry_safe(sg, next,  in ptep_notify()
2291 				gmap_shadow_notify(sg, vmaddr, gaddr);  in ptep_notify()