134 				       struct nh_group *nhg)  in nh_notifier_mpath_info_init()  argument
136 	u16 num_nh = nhg->num_nh;  in nh_notifier_mpath_info_init()
146 	info->nh_grp->is_fdb = nhg->fdb_nh;  in nh_notifier_mpath_info_init()
147 	info->nh_grp->hw_stats = nhg->hw_stats;  in nh_notifier_mpath_info_init()
150 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nh_notifier_mpath_info_init()
163 					   struct nh_group *nhg)  in nh_notifier_res_table_info_init()  argument
165 	struct nh_res_table *res_table = rtnl_dereference(nhg->res_table);  in nh_notifier_res_table_info_init()
178 	info->nh_res_table->hw_stats = nhg->hw_stats;  in nh_notifier_res_table_info_init()
197 	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nh_notifier_grp_info_init()  local
199 	if (nhg->hash_threshold)  in nh_notifier_grp_info_init()
200 		return nh_notifier_mpath_info_init(info, nhg);  in nh_notifier_grp_info_init()
201 	else if (nhg->resilient)  in nh_notifier_grp_info_init()
202 		return nh_notifier_res_table_info_init(info, nhg);  in nh_notifier_grp_info_init()
209 	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nh_notifier_grp_info_fini()  local
211 	if (nhg->hash_threshold)  in nh_notifier_grp_info_fini()
213 	else if (nhg->resilient)  in nh_notifier_grp_info_fini()
271 	struct nh_group *nhg;  in nh_notifier_res_bucket_idle_timer_get()  local
297 	nhg = rcu_dereference(nh->nh_grp);  in nh_notifier_res_bucket_idle_timer_get()
298 	res_table = rcu_dereference(nhg->res_table);  in nh_notifier_res_bucket_idle_timer_get()
412 	struct nh_group *nhg;  in call_nexthop_res_table_notifiers()  local
424 	nhg = rtnl_dereference(nh->nh_grp);  in call_nexthop_res_table_notifiers()
425 	err = nh_notifier_mpath_info_init(&info, nhg);  in call_nexthop_res_table_notifiers()
484 	struct nh_group *nhg;  in nexthop_free_group()  local
487 	nhg = rcu_dereference_raw(nh->nh_grp);  in nexthop_free_group()
488 	for (i = 0; i < nhg->num_nh; ++i) {  in nexthop_free_group()
489 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nexthop_free_group()
496 	WARN_ON(nhg->spare == nhg);  in nexthop_free_group()
498 	if (nhg->resilient)  in nexthop_free_group()
499 		vfree(rcu_dereference_raw(nhg->res_table));  in nexthop_free_group()
501 	kfree(nhg->spare);  in nexthop_free_group()
502 	kfree(nhg);  in nexthop_free_group()
550 	struct nh_group *nhg;  in nexthop_grp_alloc()  local
552 	nhg = kzalloc(struct_size(nhg, nh_entries, num_nh), GFP_KERNEL);  in nexthop_grp_alloc()
553 	if (nhg)  in nexthop_grp_alloc()
554 		nhg->num_nh = num_nh;  in nexthop_grp_alloc()
556 	return nhg;  in nexthop_grp_alloc()
645 static int nla_put_nh_group_res(struct sk_buff *skb, struct nh_group *nhg)  in nla_put_nh_group_res()  argument
647 	struct nh_res_table *res_table = rtnl_dereference(nhg->res_table);  in nla_put_nh_group_res()
709 	struct nh_group *nhg;  in nh_notifier_grp_hw_stats_init()  local
713 	nhg = rtnl_dereference(nh->nh_grp);  in nh_notifier_grp_hw_stats_init()
718 						    stats, nhg->num_nh),  in nh_notifier_grp_hw_stats_init()
723 	info->nh_grp_hw_stats->num_nh = nhg->num_nh;  in nh_notifier_grp_hw_stats_init()
724 	for (i = 0; i < nhg->num_nh; i++) {  in nh_notifier_grp_hw_stats_init()
725 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nh_notifier_grp_hw_stats_init()
750 	struct nh_group *nhg;  in nh_grp_hw_stats_apply_update()  local
754 	nhg = rtnl_dereference(nh->nh_grp);  in nh_grp_hw_stats_apply_update()
756 	for (i = 0; i < nhg->num_nh; i++) {  in nh_grp_hw_stats_apply_update()
757 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nh_grp_hw_stats_apply_update()
828 	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nla_put_nh_group_stats()  local
834 	if (nla_put_u32(skb, NHA_HW_STATS_ENABLE, nhg->hw_stats))  in nla_put_nh_group_stats()
838 	    nhg->hw_stats) {  in nla_put_nh_group_stats()
851 	for (i = 0; i < nhg->num_nh; i++)  in nla_put_nh_group_stats()
852 		if (nla_put_nh_group_stats_entry(skb, &nhg->nh_entries[i],  in nla_put_nh_group_stats()
870 	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nla_put_nh_group()  local
872 	size_t len = nhg->num_nh * sizeof(*p);  in nla_put_nh_group()
880 	if (nhg->hash_threshold)  in nla_put_nh_group()
882 	else if (nhg->resilient)  in nla_put_nh_group()
893 	for (i = 0; i < nhg->num_nh; ++i) {  in nla_put_nh_group()
894 		weight = nhg->nh_entries[i].weight - 1;  in nla_put_nh_group()
897 			.id = nhg->nh_entries[i].nh->id,  in nla_put_nh_group()
903 	if (nhg->resilient && nla_put_nh_group_res(skb, nhg))  in nla_put_nh_group()
907 	    (nla_put_u32(skb, NHA_HW_STATS_ENABLE, nhg->hw_stats) ||  in nla_put_nh_group()
942 		struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nh_fill_node()  local
945 		if (nhg->fdb_nh && nla_put_flag(skb, NHA_FDB))  in nh_fill_node()
1001 static size_t nh_nlmsg_size_grp_res(struct nh_group *nhg)  in nh_nlmsg_size_grp_res()  argument
1012 	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nh_nlmsg_size_grp()  local
1013 	size_t sz = sizeof(struct nexthop_grp) * nhg->num_nh;  in nh_nlmsg_size_grp()
1017 	if (nhg->resilient)  in nh_nlmsg_size_grp()
1018 		tot += nh_nlmsg_size_grp_res(nhg);  in nh_nlmsg_size_grp()
1220 		struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in valid_group_nh()  local
1223 		if (nhg->hash_threshold) {  in valid_group_nh()
1228 		if (nhg->resilient) {  in valid_group_nh()
1233 		*is_fdb = nhg->fdb_nh;  in valid_group_nh()
1276 	struct nexthop_grp *nhg;  in nh_check_attr_group()  local
1287 	len /= sizeof(*nhg);  in nh_check_attr_group()
1289 	nhg = nla_data(tb[NHA_GROUP]);  in nh_check_attr_group()
1291 		if (nhg[i].resvd2) {  in nh_check_attr_group()
1295 		if (nexthop_grp_weight(&nhg[i]) == 0) {  in nh_check_attr_group()
1303 			if (nhg[i].id == nhg[j].id) {  in nh_check_attr_group()
1312 	nhg = nla_data(tb[NHA_GROUP]);  in nh_check_attr_group()
1317 		nh = nexthop_find_by_id(net, nhg[i].id);  in nh_check_attr_group()
1400 static struct nexthop *nexthop_select_path_fdb(struct nh_group *nhg, int hash)  in nexthop_select_path_fdb()  argument
1404 	for (i = 0; i < nhg->num_nh; i++) {  in nexthop_select_path_fdb()
1405 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nexthop_select_path_fdb()
1418 static struct nexthop *nexthop_select_path_hthr(struct nh_group *nhg, int hash)  in nexthop_select_path_hthr()  argument
1423 	if (nhg->fdb_nh)  in nexthop_select_path_hthr()
1424 		return nexthop_select_path_fdb(nhg, hash);  in nexthop_select_path_hthr()
1426 	for (i = 0; i < nhg->num_nh; ++i) {  in nexthop_select_path_hthr()
1427 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nexthop_select_path_hthr()
1446 		nhge0 = &nhg->nh_entries[0];  in nexthop_select_path_hthr()
1451 static struct nexthop *nexthop_select_path_res(struct nh_group *nhg, int hash)  in nexthop_select_path_res()  argument
1453 	struct nh_res_table *res_table = rcu_dereference(nhg->res_table);  in nexthop_select_path_res()
1470 	struct nh_group *nhg;  in nexthop_select_path()  local
1475 	nhg = rcu_dereference(nh->nh_grp);  in nexthop_select_path()
1476 	if (nhg->hash_threshold)  in nexthop_select_path()
1477 		return nexthop_select_path_hthr(nhg, hash);  in nexthop_select_path()
1478 	else if (nhg->resilient)  in nexthop_select_path()
1479 		return nexthop_select_path_res(nhg, hash);  in nexthop_select_path()
1494 		struct nh_group *nhg;  in nexthop_for_each_fib6_nh()  local
1497 		nhg = rcu_dereference_rtnl(nh->nh_grp);  in nexthop_for_each_fib6_nh()
1498 		for (i = 0; i < nhg->num_nh; i++) {  in nexthop_for_each_fib6_nh()
1499 			struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nexthop_for_each_fib6_nh()
1543 		struct nh_group *nhg;  in fib6_check_nexthop()  local
1545 		nhg = rtnl_dereference(nh->nh_grp);  in fib6_check_nexthop()
1546 		if (nhg->has_v4)  in fib6_check_nexthop()
1548 		is_fdb_nh = nhg->fdb_nh;  in fib6_check_nexthop()
1615 		struct nh_group *nhg;  in fib_check_nexthop()  local
1617 		nhg = rtnl_dereference(nh->nh_grp);  in fib_check_nexthop()
1618 		if (nhg->fdb_nh) {  in fib_check_nexthop()
1631 		nhi = rtnl_dereference(nhg->nh_entries[0].nh->nh_info);  in fib_check_nexthop()
1891 static void nh_res_group_rebalance(struct nh_group *nhg,  in nh_res_group_rebalance()  argument
1901 	for (i = 0; i < nhg->num_nh; ++i)  in nh_res_group_rebalance()
1902 		total += nhg->nh_entries[i].weight;  in nh_res_group_rebalance()
1904 	for (i = 0; i < nhg->num_nh; ++i) {  in nh_res_group_rebalance()
1905 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nh_res_group_rebalance()
1929 					 struct nh_group *nhg)  in nh_res_table_migrate_buckets()  argument
1939 		for (j = 0; j < nhg->num_nh; j++) {  in nh_res_table_migrate_buckets()
1940 			struct nh_grp_entry *nhge = &nhg->nh_entries[j];  in nh_res_table_migrate_buckets()
1975 static void nh_hthr_group_rebalance(struct nh_group *nhg)  in nh_hthr_group_rebalance()  argument
1981 	for (i = 0; i < nhg->num_nh; ++i)  in nh_hthr_group_rebalance()
1982 		total += nhg->nh_entries[i].weight;  in nh_hthr_group_rebalance()
1984 	for (i = 0; i < nhg->num_nh; ++i) {  in nh_hthr_group_rebalance()
1985 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nh_hthr_group_rebalance()
2001 	struct nh_group *nhg, *newg;  in remove_nh_grp_entry()  local
2006 	nhg = rtnl_dereference(nhp->nh_grp);  in remove_nh_grp_entry()
2007 	newg = nhg->spare;  in remove_nh_grp_entry()
2010 	if (nhg->num_nh == 1) {  in remove_nh_grp_entry()
2016 	newg->is_multipath = nhg->is_multipath;  in remove_nh_grp_entry()
2017 	newg->hash_threshold = nhg->hash_threshold;  in remove_nh_grp_entry()
2018 	newg->resilient = nhg->resilient;  in remove_nh_grp_entry()
2019 	newg->fdb_nh = nhg->fdb_nh;  in remove_nh_grp_entry()
2020 	newg->num_nh = nhg->num_nh;  in remove_nh_grp_entry()
2023 	nhges = nhg->nh_entries;  in remove_nh_grp_entry()
2025 	for (i = 0, j = 0; i < nhg->num_nh; ++i) {  in remove_nh_grp_entry()
2029 		if (nhg->nh_entries[i].nh == nh) {  in remove_nh_grp_entry()
2050 		replace_nexthop_grp_res(nhg, newg);  in remove_nh_grp_entry()
2086 	struct nh_group *nhg = rcu_dereference_rtnl(nh->nh_grp);  in remove_nexthop_group()  local
2088 	int i, num_nh = nhg->num_nh;  in remove_nexthop_group()
2091 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in remove_nexthop_group()
2099 	if (nhg->resilient) {  in remove_nexthop_group()
2100 		res_table = rtnl_dereference(nhg->res_table);  in remove_nexthop_group()
2170 	struct nh_group *nhg;  in nh_rt_cache_flush()  local
2185 	nhg = rtnl_dereference(replaced_nh->nh_grp);  in nh_rt_cache_flush()
2186 	for (i = 0; i < nhg->num_nh; i++) {  in nh_rt_cache_flush()
2187 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nh_rt_cache_flush()
2282 static void nh_group_v4_update(struct nh_group *nhg)  in nh_group_v4_update()  argument
2288 	nhges = nhg->nh_entries;  in nh_group_v4_update()
2289 	for (i = 0; i < nhg->num_nh; i++) {  in nh_group_v4_update()
2296 	nhg->has_v4 = has_v4;  in nh_group_v4_update()
2348 	struct nh_group *nhg = rtnl_dereference(group_nh->nh_grp);  in replace_nexthop_single_notify()  local
2351 	if (nhg->hash_threshold) {  in replace_nexthop_single_notify()
2354 	} else if (nhg->resilient) {  in replace_nexthop_single_notify()
2355 		res_table = rtnl_dereference(nhg->res_table);  in replace_nexthop_single_notify()
2418 			struct nh_group *nhg;  in replace_nexthop_single()  local
2420 			nhg = rtnl_dereference(nhp->nh_grp);  in replace_nexthop_single()
2421 			nh_group_v4_update(nhg);  in replace_nexthop_single()
2589 		struct nh_group *nhg = rtnl_dereference(new_nh->nh_grp);  in insert_nexthop()  local
2592 		if (nhg->resilient) {  in insert_nexthop()
2593 			res_table = rtnl_dereference(nhg->res_table);  in insert_nexthop()
2604 			nh_res_group_rebalance(nhg, res_table);  in insert_nexthop()
2677 	struct nh_group *nhg;  in nexthop_create_group()  local
2691 	nhg = nexthop_grp_alloc(num_nh);  in nexthop_create_group()
2692 	if (!nhg) {  in nexthop_create_group()
2698 	nhg->spare = nexthop_grp_alloc(num_nh);  in nexthop_create_group()
2699 	if (!nhg->spare) {  in nexthop_create_group()
2700 		kfree(nhg);  in nexthop_create_group()
2704 	nhg->spare->spare = nhg;  in nexthop_create_group()
2706 	for (i = 0; i < nhg->num_nh; ++i) {  in nexthop_create_group()
2718 			nhg->has_v4 = true;  in nexthop_create_group()
2720 		nhg->nh_entries[i].stats =  in nexthop_create_group()
2722 		if (!nhg->nh_entries[i].stats) {  in nexthop_create_group()
2727 		nhg->nh_entries[i].nh = nhe;  in nexthop_create_group()
2728 		nhg->nh_entries[i].weight = nexthop_grp_weight(&entry[i]);  in nexthop_create_group()
2730 		list_add(&nhg->nh_entries[i].nh_list, &nhe->grp_list);  in nexthop_create_group()
2731 		nhg->nh_entries[i].nh_parent = nh;  in nexthop_create_group()
2735 		nhg->hash_threshold = 1;  in nexthop_create_group()
2736 		nhg->is_multipath = true;  in nexthop_create_group()
2746 		rcu_assign_pointer(nhg->spare->res_table, res_table);  in nexthop_create_group()
2747 		rcu_assign_pointer(nhg->res_table, res_table);  in nexthop_create_group()
2748 		nhg->resilient = true;  in nexthop_create_group()
2749 		nhg->is_multipath = true;  in nexthop_create_group()
2752 	WARN_ON_ONCE(nhg->hash_threshold + nhg->resilient != 1);  in nexthop_create_group()
2754 	if (nhg->hash_threshold)  in nexthop_create_group()
2755 		nh_hthr_group_rebalance(nhg);  in nexthop_create_group()
2758 		nhg->fdb_nh = 1;  in nexthop_create_group()
2761 		nhg->hw_stats = true;  in nexthop_create_group()
2763 	rcu_assign_pointer(nh->nh_grp, nhg);  in nexthop_create_group()
2769 		list_del(&nhg->nh_entries[i].nh_list);  in nexthop_create_group()
2770 		free_percpu(nhg->nh_entries[i].stats);  in nexthop_create_group()
2771 		nexthop_put(nhg->nh_entries[i].nh);  in nexthop_create_group()
2774 	kfree(nhg->spare);  in nexthop_create_group()
2775 	kfree(nhg);  in nexthop_create_group()
3526 	struct nh_group *nhg;  in nexthop_find_group_resilient()  local
3538 	nhg = rtnl_dereference(nh->nh_grp);  in nexthop_find_group_resilient()
3539 	if (!nhg->resilient) {  in nexthop_find_group_resilient()
3631 	struct nh_group *nhg;  in rtm_dump_nexthop_bucket_nh()  local
3635 	nhg = rtnl_dereference(nh->nh_grp);  in rtm_dump_nexthop_bucket_nh()
3636 	res_table = rtnl_dereference(nhg->res_table);  in rtm_dump_nexthop_bucket_nh()
3671 	struct nh_group *nhg;  in rtm_dump_nexthop_bucket_cb()  local
3676 	nhg = rtnl_dereference(nh->nh_grp);  in rtm_dump_nexthop_bucket_cb()
3677 	if (!nhg->resilient)  in rtm_dump_nexthop_bucket_cb()
3773 	struct nh_group *nhg;  in rtm_get_nexthop_bucket()  local
3787 	nhg = rtnl_dereference(nh->nh_grp);  in rtm_get_nexthop_bucket()
3788 	res_table = rtnl_dereference(nhg->res_table);  in rtm_get_nexthop_bucket()
3947 	struct nh_group *nhg;  in nexthop_bucket_set_hw_flags()  local
3955 	nhg = rcu_dereference(nexthop->nh_grp);  in nexthop_bucket_set_hw_flags()
3956 	if (!nhg->resilient)  in nexthop_bucket_set_hw_flags()
3959 	if (bucket_index >= nhg->res_table->num_nh_buckets)  in nexthop_bucket_set_hw_flags()
3962 	res_table = rcu_dereference(nhg->res_table);  in nexthop_bucket_set_hw_flags()
3980 	struct nh_group *nhg;  in nexthop_res_grp_activity_update()  local
3989 	nhg = rcu_dereference(nexthop->nh_grp);  in nexthop_res_grp_activity_update()
3990 	if (!nhg->resilient)  in nexthop_res_grp_activity_update()
3996 	res_table = rcu_dereference(nhg->res_table);  in nexthop_res_grp_activity_update()