36 	rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
65 	return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask);  in xfrm_dst_hash()
73 	return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask);  in xfrm_src_hash()
80 	return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask);  in xfrm_spi_hash()
85 	return __xfrm_seq_hash(seq, net->xfrm.state_hmask);  in xfrm_seq_hash()
153 	struct net *net = container_of(work, struct net, xfrm.state_hash_work);  in xfrm_hash_resize()
159 	nsize = xfrm_hash_new_size(net->xfrm.state_hmask);  in xfrm_hash_resize()
182 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_hash_resize()
183 	write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);  in xfrm_hash_resize()
186 	odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net);  in xfrm_hash_resize()
187 	for (i = net->xfrm.state_hmask; i >= 0; i--)  in xfrm_hash_resize()
190 	osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net);  in xfrm_hash_resize()
191 	ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net);  in xfrm_hash_resize()
192 	oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net);  in xfrm_hash_resize()
193 	ohashmask = net->xfrm.state_hmask;  in xfrm_hash_resize()
195 	rcu_assign_pointer(net->xfrm.state_bydst, ndst);  in xfrm_hash_resize()
196 	rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);  in xfrm_hash_resize()
197 	rcu_assign_pointer(net->xfrm.state_byspi, nspi);  in xfrm_hash_resize()
198 	rcu_assign_pointer(net->xfrm.state_byseq, nseq);  in xfrm_hash_resize()
199 	net->xfrm.state_hmask = nhashmask;  in xfrm_hash_resize()
201 	write_seqcount_end(&net->xfrm.xfrm_state_hash_generation);  in xfrm_hash_resize()
202 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_hash_resize()
746 		spin_lock(&net->xfrm.xfrm_state_lock);  in __xfrm_state_delete()
754 		net->xfrm.state_num--;  in __xfrm_state_delete()
756 		spin_unlock(&net->xfrm.xfrm_state_lock);  in __xfrm_state_delete()
793 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_flush_secctx_check()
796 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_state_flush_secctx_check()
813 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_dev_state_flush_secctx_check()
817 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_dev_state_flush_secctx_check()
848 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
854 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_flush()
857 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_state_flush()
861 				spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
873 				spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
879 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_flush()
894 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
900 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_dev_state_flush()
902 		hlist_for_each_entry(x, net->xfrm.state_bydst+i, bydst) {  in xfrm_dev_state_flush()
907 				spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
918 				spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
927 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_dev_state_flush()
952 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_sad_getinfo()
953 	si->sadcnt = net->xfrm.state_num;  in xfrm_sad_getinfo()
954 	si->sadhcnt = net->xfrm.state_hmask + 1;  in xfrm_sad_getinfo()
956 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_sad_getinfo()
1045 	hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) {  in __xfrm_state_lookup_all()
1087 	hlist_for_each_entry_rcu(x, net->xfrm.state_byspi + h, byspi) {  in __xfrm_state_lookup()
1112 	hlist_for_each_entry_rcu(x, net->xfrm.state_bysrc + h, bysrc) {  in __xfrm_state_lookup_byaddr()
1148 	    (net->xfrm.state_hmask + 1) < xfrm_state_hashmax &&  in xfrm_hash_grow_check()
1149 	    net->xfrm.state_num > net->xfrm.state_hmask)  in xfrm_hash_grow_check()
1150 		schedule_work(&net->xfrm.state_hash_work);  in xfrm_hash_grow_check()
1215 	sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);  in xfrm_state_find()
1219 	hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h, bydst) {  in xfrm_state_find()
1253 	hlist_for_each_entry_rcu(x, net->xfrm.state_bydst + h_wildcard, bydst) {  in xfrm_state_find()
1351 			spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
1354 			list_add(&x->km.all, &net->xfrm.state_all);  in xfrm_state_find()
1356 					  net->xfrm.state_bydst + h,  in xfrm_state_find()
1360 					  net->xfrm.state_bysrc + h,  in xfrm_state_find()
1365 						  net->xfrm.state_byspi + h,  in xfrm_state_find()
1371 						  net->xfrm.state_byseq + h,  in xfrm_state_find()
1374 			x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;  in xfrm_state_find()
1376 				      ktime_set(net->xfrm.sysctl_acq_expires, 0),  in xfrm_state_find()
1378 			net->xfrm.state_num++;  in xfrm_state_find()
1380 			spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_find()
1409 	if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) {  in xfrm_state_find()
1428 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_stateonly_find()
1430 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in xfrm_stateonly_find()
1447 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_stateonly_find()
1460 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1461 	list_for_each_entry(w, &net->xfrm.state_all, all) {  in xfrm_state_lookup_byspi()
1468 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1471 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byspi()
1481 	list_add(&x->km.all, &net->xfrm.state_all);  in __xfrm_state_insert()
1485 	XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,  in __xfrm_state_insert()
1489 	XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,  in __xfrm_state_insert()
1496 		XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,  in __xfrm_state_insert()
1503 		XFRM_STATE_INSERT(byseq, &x->byseq, net->xfrm.state_byseq + h,  in __xfrm_state_insert()
1511 	net->xfrm.state_num++;  in __xfrm_state_insert()
1529 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in __xfrm_state_bump_genids()
1544 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_insert()
1547 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_insert()
1564 	hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in __find_acq_core()
1613 		x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;  in __find_acq_core()
1616 			      ktime_set(net->xfrm.sysctl_acq_expires, 0),  in __find_acq_core()
1618 		list_add(&x->km.all, &net->xfrm.state_all);  in __find_acq_core()
1619 		XFRM_STATE_INSERT(bydst, &x->bydst, net->xfrm.state_bydst + h,  in __find_acq_core()
1622 		XFRM_STATE_INSERT(bysrc, &x->bysrc, net->xfrm.state_bysrc + h,  in __find_acq_core()
1625 		net->xfrm.state_num++;  in __find_acq_core()
1648 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_add()
1677 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_add()
1823 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_migrate_state_find()
1828 		hlist_for_each_entry(x, net->xfrm.state_bydst+h, bydst) {  in xfrm_migrate_state_find()
1847 		hlist_for_each_entry(x, net->xfrm.state_bysrc+h, bysrc) {  in xfrm_migrate_state_find()
1863 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_migrate_state_find()
1914 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
1940 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
1977 			spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
1986 			spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update()
2032 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update_stats()
2033 	for (i = 0; i <= net->xfrm.state_hmask; i++) {  in xfrm_state_update_stats()
2034 		hlist_for_each_entry(x, net->xfrm.state_bydst + i, bydst)  in xfrm_state_update_stats()
2037 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_update_stats()
2060 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byaddr()
2062 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_lookup_byaddr()
2074 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq()
2076 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq()
2215 	hlist_for_each_entry_rcu(x, net->xfrm.state_byseq + h, byseq) {  in __xfrm_find_acq_byseq()
2231 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq_byseq()
2233 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_find_acq_byseq()
2325 		spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_alloc_spi()
2328 		XFRM_STATE_INSERT(byspi, &x->byspi, net->xfrm.state_byspi + h,  in xfrm_alloc_spi()
2330 		spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_alloc_spi()
2372 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk()
2374 		x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all);  in xfrm_state_walk()
2377 	list_for_each_entry_from(x, &net->xfrm.state_all, all) {  in xfrm_state_walk()
2398 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk()
2421 	spin_lock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk_done()
2423 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);  in xfrm_state_walk_done()
2975 	INIT_LIST_HEAD(&net->xfrm.state_all);  in xfrm_state_init()
2979 	net->xfrm.state_bydst = xfrm_hash_alloc(sz);  in xfrm_state_init()
2980 	if (!net->xfrm.state_bydst)  in xfrm_state_init()
2982 	net->xfrm.state_bysrc = xfrm_hash_alloc(sz);  in xfrm_state_init()
2983 	if (!net->xfrm.state_bysrc)  in xfrm_state_init()
2985 	net->xfrm.state_byspi = xfrm_hash_alloc(sz);  in xfrm_state_init()
2986 	if (!net->xfrm.state_byspi)  in xfrm_state_init()
2988 	net->xfrm.state_byseq = xfrm_hash_alloc(sz);  in xfrm_state_init()
2989 	if (!net->xfrm.state_byseq)  in xfrm_state_init()
2991 	net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1);  in xfrm_state_init()
2993 	net->xfrm.state_num = 0;  in xfrm_state_init()
2994 	INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize);  in xfrm_state_init()
2995 	spin_lock_init(&net->xfrm.xfrm_state_lock);  in xfrm_state_init()
2996 	seqcount_spinlock_init(&net->xfrm.xfrm_state_hash_generation,  in xfrm_state_init()
2997 			       &net->xfrm.xfrm_state_lock);  in xfrm_state_init()
3001 	xfrm_hash_free(net->xfrm.state_byspi, sz);  in xfrm_state_init()
3003 	xfrm_hash_free(net->xfrm.state_bysrc, sz);  in xfrm_state_init()
3005 	xfrm_hash_free(net->xfrm.state_bydst, sz);  in xfrm_state_init()
3014 	flush_work(&net->xfrm.state_hash_work);  in xfrm_state_fini()
3018 	WARN_ON(!list_empty(&net->xfrm.state_all));  in xfrm_state_fini()
3020 	sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head);  in xfrm_state_fini()
3021 	WARN_ON(!hlist_empty(net->xfrm.state_byseq));  in xfrm_state_fini()
3022 	xfrm_hash_free(net->xfrm.state_byseq, sz);  in xfrm_state_fini()
3023 	WARN_ON(!hlist_empty(net->xfrm.state_byspi));  in xfrm_state_fini()
3024 	xfrm_hash_free(net->xfrm.state_byspi, sz);  in xfrm_state_fini()
3025 	WARN_ON(!hlist_empty(net->xfrm.state_bysrc));  in xfrm_state_fini()
3026 	xfrm_hash_free(net->xfrm.state_bysrc, sz);  in xfrm_state_fini()
3027 	WARN_ON(!hlist_empty(net->xfrm.state_bydst));  in xfrm_state_fini()
3028 	xfrm_hash_free(net->xfrm.state_bydst, sz);  in xfrm_state_fini()