133 	old = rhashtable_lookup_get_insert_fast(&efx->tc->mac_ht,  in efx_tc_flower_get_mac()
157 	rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage,  in efx_tc_flower_get_mac()
168 	rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage,  in efx_tc_flower_put_mac()
482 	rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage,  in efx_tc_flower_release_encap_match()
595 	old = rhashtable_lookup_get_insert_fast(&efx->tc->encap_match_ht,  in efx_tc_flower_record_encap_match()
681 	rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage,  in efx_tc_flower_record_encap_match()
706 	old = rhashtable_lookup_get_insert_fast(&efx->tc->recirc_ht,  in efx_tc_get_recirc_id()
719 		rc = ida_alloc_range(&efx->tc->recirc_ida, 1, U8_MAX, GFP_USER);  in efx_tc_get_recirc_id()
721 			rhashtable_remove_fast(&efx->tc->recirc_ht,  in efx_tc_get_recirc_id()
737 	rhashtable_remove_fast(&efx->tc->recirc_ht, &rid->linkage,  in efx_tc_put_recirc_id()
739 	ida_free(&efx->tc->recirc_ida, rid->fw_id);  in efx_tc_put_recirc_id()
968 					    struct flow_cls_offload *tc,  in efx_tc_flower_handle_lhs_actions()  argument
974 	struct netlink_ext_ack *extack = tc->common.extack;  in efx_tc_flower_handle_lhs_actions()
1014 				cnt = efx_tc_flower_get_counter_index(efx, tc->cookie,  in efx_tc_flower_handle_lhs_actions()
1453 						struct flow_cls_offload *tc,  in efx_tc_flower_replace_foreign_lhs_ar()  argument
1458 	struct netlink_ext_ack *extack = tc->common.extack;  in efx_tc_flower_replace_foreign_lhs_ar()
1513 	rule->cookie = tc->cookie;  in efx_tc_flower_replace_foreign_lhs_ar()
1515 	old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,  in efx_tc_flower_replace_foreign_lhs_ar()
1520 			  "Already offloaded rule (cookie %lx)\n", tc->cookie);  in efx_tc_flower_replace_foreign_lhs_ar()
1527 	rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);  in efx_tc_flower_replace_foreign_lhs_ar()
1541 		  tc->cookie);  in efx_tc_flower_replace_foreign_lhs_ar()
1547 		rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,  in efx_tc_flower_replace_foreign_lhs_ar()
1557 					     struct flow_cls_offload *tc,  in efx_tc_flower_replace_foreign_lhs()  argument
1562 	struct netlink_ext_ack *extack = tc->common.extack;  in efx_tc_flower_replace_foreign_lhs()
1567 	if (tc->common.chain_index) {  in efx_tc_flower_replace_foreign_lhs()
1579 		return efx_tc_flower_replace_foreign_lhs_ar(efx, tc, fr, match,  in efx_tc_flower_replace_foreign_lhs()
1626 	rule->cookie = tc->cookie;  in efx_tc_flower_replace_foreign_lhs()
1627 	old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,  in efx_tc_flower_replace_foreign_lhs()
1632 			  "Already offloaded rule (cookie %lx)\n", tc->cookie);  in efx_tc_flower_replace_foreign_lhs()
1639 	rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);  in efx_tc_flower_replace_foreign_lhs()
1653 		  tc->cookie);  in efx_tc_flower_replace_foreign_lhs()
1659 		rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,  in efx_tc_flower_replace_foreign_lhs()
1670 					 struct flow_cls_offload *tc)  in efx_tc_flower_replace_foreign()  argument
1672 	struct flow_rule *fr = flow_cls_offload_flow_rule(tc);  in efx_tc_flower_replace_foreign()
1673 	struct netlink_ext_ack *extack = tc->common.extack;  in efx_tc_flower_replace_foreign()
1702 		return efx_tc_flower_replace_foreign_lhs(efx, tc, fr, &match,  in efx_tc_flower_replace_foreign()
1705 	if (tc->common.chain_index) {  in efx_tc_flower_replace_foreign()
1708 		rid = efx_tc_get_recirc_id(efx, tc->common.chain_index, net_dev);  in efx_tc_flower_replace_foreign()
1712 					       tc->common.chain_index);  in efx_tc_flower_replace_foreign()
1789 	} else if (!tc->common.chain_index) {  in efx_tc_flower_replace_foreign()
1803 	rule->cookie = tc->cookie;  in efx_tc_flower_replace_foreign()
1804 	old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht,  in efx_tc_flower_replace_foreign()
1813 			  tc->cookie);  in efx_tc_flower_replace_foreign()
1855 								      tc->cookie,  in efx_tc_flower_replace_foreign()
1953 		  tc->cookie);  in efx_tc_flower_replace_foreign()
1980 			rhashtable_remove_fast(&efx->tc->match_action_ht,  in efx_tc_flower_replace_foreign()
1992 				     struct flow_cls_offload *tc,  in efx_tc_flower_replace_lhs()  argument
1998 	struct netlink_ext_ack *extack = tc->common.extack;  in efx_tc_flower_replace_lhs()
2002 	if (tc->common.chain_index) {  in efx_tc_flower_replace_lhs()
2022 	rule->cookie = tc->cookie;  in efx_tc_flower_replace_lhs()
2023 	old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,  in efx_tc_flower_replace_lhs()
2031 			  "Already offloaded rule (cookie %lx)\n", tc->cookie);  in efx_tc_flower_replace_lhs()
2041 	rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, efx->net_dev, rule);  in efx_tc_flower_replace_lhs()
2054 		  tc->cookie);  in efx_tc_flower_replace_lhs()
2060 		rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,  in efx_tc_flower_replace_lhs()
2068 				 struct flow_cls_offload *tc,  in efx_tc_flower_replace()  argument
2071 	struct flow_rule *fr = flow_cls_offload_flow_rule(tc);  in efx_tc_flower_replace()
2072 	struct netlink_ext_ack *extack = tc->common.extack;  in efx_tc_flower_replace()
2086 	if (WARN_ON(!efx->tc))  in efx_tc_flower_replace()
2088 	if (WARN_ON(!efx->tc->up))  in efx_tc_flower_replace()
2094 		return efx_tc_flower_replace_foreign(efx, net_dev, tc);  in efx_tc_flower_replace()
2123 		return efx_tc_flower_replace_lhs(efx, tc, fr, &match, efv,  in efx_tc_flower_replace()
2130 	if (tc->common.chain_index) {  in efx_tc_flower_replace()
2143 		rid = efx_tc_get_recirc_id(efx, tc->common.chain_index,  in efx_tc_flower_replace()
2148 					       tc->common.chain_index);  in efx_tc_flower_replace()
2186 	rule->cookie = tc->cookie;  in efx_tc_flower_replace()
2187 	old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht,  in efx_tc_flower_replace()
2195 			  "Already offloaded rule (cookie %lx)\n", tc->cookie);  in efx_tc_flower_replace()
2274 			ctr = efx_tc_flower_get_counter_index(efx, tc->cookie,  in efx_tc_flower_replace()
2487 			efx_mae_mport_mport(efx, efx->tc->reps_mport_id,  in efx_tc_flower_replace()
2501 		  tc->cookie);  in efx_tc_flower_replace()
2512 		rule->fallback = &efx->tc->facts.pf;  in efx_tc_flower_replace()
2515 		rule->fallback = &efx->tc->facts.reps;  in efx_tc_flower_replace()
2543 			rhashtable_remove_fast(&efx->tc->match_action_ht,  in efx_tc_flower_replace()
2554 				 struct flow_cls_offload *tc)  in efx_tc_flower_destroy()  argument
2556 	struct netlink_ext_ack *extack = tc->common.extack;  in efx_tc_flower_destroy()
2560 	lhs_rule = rhashtable_lookup_fast(&efx->tc->lhs_rule_ht, &tc->cookie,  in efx_tc_flower_destroy()
2567 		rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &lhs_rule->linkage,  in efx_tc_flower_destroy()
2577 	rule = rhashtable_lookup_fast(&efx->tc->match_action_ht, &tc->cookie,  in efx_tc_flower_destroy()
2587 				   "Filter %lx not found to remove\n", tc->cookie);  in efx_tc_flower_destroy()
2595 	rhashtable_remove_fast(&efx->tc->match_action_ht, &rule->linkage,  in efx_tc_flower_destroy()
2603 			       struct flow_cls_offload *tc)  in efx_tc_flower_stats()  argument
2605 	struct netlink_ext_ack *extack = tc->common.extack;  in efx_tc_flower_stats()
2610 	ctr = efx_tc_flower_find_counter_index(efx, tc->cookie);  in efx_tc_flower_stats()
2617 					   tc->cookie);  in efx_tc_flower_stats()
2629 	flow_stats_update(&tc->stats, bytes - cnt->old_bytes,  in efx_tc_flower_stats()
2639 		  struct flow_cls_offload *tc, struct efx_rep *efv)  in efx_tc_flower()  argument
2643 	if (!efx->tc)  in efx_tc_flower()
2646 	mutex_lock(&efx->tc->mutex);  in efx_tc_flower()
2647 	switch (tc->command) {  in efx_tc_flower()
2649 		rc = efx_tc_flower_replace(efx, net_dev, tc, efv);  in efx_tc_flower()
2652 		rc = efx_tc_flower_destroy(efx, net_dev, tc);  in efx_tc_flower()
2655 		rc = efx_tc_flower_stats(efx, net_dev, tc);  in efx_tc_flower()
2661 	mutex_unlock(&efx->tc->mutex);  in efx_tc_flower()
2705 	struct efx_tc_flow_rule *rule = &efx->tc->dflt.pf;  in efx_tc_configure_default_rule_pf()
2715 	struct efx_tc_flow_rule *rule = &efx->tc->dflt.wire;  in efx_tc_configure_default_rule_wire()
2730 	efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port);  in efx_tc_configure_default_rule_rep()
2772 	struct efx_tc_action_set_list *acts = &efx->tc->facts.pf;  in efx_tc_configure_fallback_acts_pf()
2781 	struct efx_tc_action_set_list *acts = &efx->tc->facts.reps;  in efx_tc_configure_fallback_acts_reps()
2784 	efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port);  in efx_tc_configure_fallback_acts_reps()
2799 	rc = efx_mae_allocate_mport(efx, &efx->tc->reps_mport_id, &rep_mport_label);  in efx_tc_configure_rep_mport()
2803 		efx->tc->reps_mport_id, rep_mport_label);  in efx_tc_configure_rep_mport()
2805 	efx_mae_mport_mport(efx, efx->tc->reps_mport_id,  in efx_tc_configure_rep_mport()
2806 			    &efx->tc->reps_mport_vport_id);  in efx_tc_configure_rep_mport()
2812 	efx_mae_free_mport(efx, efx->tc->reps_mport_id);  in efx_tc_deconfigure_rep_mport()
2813 	efx->tc->reps_mport_id = MAE_MPORT_SELECTOR_NULL;  in efx_tc_deconfigure_rep_mport()
2823 	if (!efx->tc)  in efx_tc_insert_rep_filters()
2827 	efx_filter_set_vport_id(&promisc, efx->tc->reps_mport_vport_id);  in efx_tc_insert_rep_filters()
2831 	efx->tc->reps_filter_uc = rc;  in efx_tc_insert_rep_filters()
2834 	efx_filter_set_vport_id(&allmulti, efx->tc->reps_mport_vport_id);  in efx_tc_insert_rep_filters()
2838 	efx->tc->reps_filter_mc = rc;  in efx_tc_insert_rep_filters()
2846 	if (!efx->tc)  in efx_tc_remove_rep_filters()
2848 	if (efx->tc->reps_filter_mc >= 0)  in efx_tc_remove_rep_filters()
2849 		efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_mc);  in efx_tc_remove_rep_filters()
2850 	efx->tc->reps_filter_mc = -1;  in efx_tc_remove_rep_filters()
2851 	if (efx->tc->reps_filter_uc >= 0)  in efx_tc_remove_rep_filters()
2852 		efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_uc);  in efx_tc_remove_rep_filters()
2853 	efx->tc->reps_filter_uc = -1;  in efx_tc_remove_rep_filters()
2860 	rc = efx_mae_get_caps(efx, efx->tc->caps);  in efx_init_tc()
2863 	if (efx->tc->caps->match_field_count > MAE_NUM_FIELDS)  in efx_init_tc()
2870 			   efx->tc->caps->match_field_count);  in efx_init_tc()
2871 	if (efx->tc->caps->action_prios < EFX_TC_PRIO__NUM) {  in efx_init_tc()
2874 			  efx->tc->caps->action_prios, EFX_TC_PRIO__NUM);  in efx_init_tc()
2898 	efx->tc->up = true;  in efx_init_tc()
2908 	if (!efx->tc)  in efx_fini_tc()
2910 	if (efx->tc->up)  in efx_fini_tc()
2913 	efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.pf);  in efx_fini_tc()
2914 	efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.wire);  in efx_fini_tc()
2915 	efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.pf);  in efx_fini_tc()
2916 	efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.reps);  in efx_fini_tc()
2917 	efx->tc->up = false;  in efx_fini_tc()
2939 	ida_free(&efx->tc->recirc_ida, rid->fw_id);  in efx_tc_recirc_free()
2991 	efx->tc = kzalloc(sizeof(*efx->tc), GFP_KERNEL);  in efx_init_struct_tc()
2992 	if (!efx->tc)  in efx_init_struct_tc()
2994 	efx->tc->caps = kzalloc(sizeof(struct mae_caps), GFP_KERNEL);  in efx_init_struct_tc()
2995 	if (!efx->tc->caps) {  in efx_init_struct_tc()
2999 	INIT_LIST_HEAD(&efx->tc->block_list);  in efx_init_struct_tc()
3001 	mutex_init(&efx->tc->mutex);  in efx_init_struct_tc()
3002 	init_waitqueue_head(&efx->tc->flush_wq);  in efx_init_struct_tc()
3009 	rc = rhashtable_init(&efx->tc->mac_ht, &efx_tc_mac_ht_params);  in efx_init_struct_tc()
3012 	rc = rhashtable_init(&efx->tc->encap_match_ht, &efx_tc_encap_match_ht_params);  in efx_init_struct_tc()
3015 	rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params);  in efx_init_struct_tc()
3018 	rc = rhashtable_init(&efx->tc->lhs_rule_ht, &efx_tc_lhs_rule_ht_params);  in efx_init_struct_tc()
3024 	rc = rhashtable_init(&efx->tc->recirc_ht, &efx_tc_recirc_ht_params);  in efx_init_struct_tc()
3027 	ida_init(&efx->tc->recirc_ida);  in efx_init_struct_tc()
3028 	efx->tc->reps_filter_uc = -1;  in efx_init_struct_tc()
3029 	efx->tc->reps_filter_mc = -1;  in efx_init_struct_tc()
3030 	INIT_LIST_HEAD(&efx->tc->dflt.pf.acts.list);  in efx_init_struct_tc()
3031 	efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;  in efx_init_struct_tc()
3032 	INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list);  in efx_init_struct_tc()
3033 	efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;  in efx_init_struct_tc()
3034 	INIT_LIST_HEAD(&efx->tc->facts.pf.list);  in efx_init_struct_tc()
3035 	efx->tc->facts.pf.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL;  in efx_init_struct_tc()
3036 	INIT_LIST_HEAD(&efx->tc->facts.reps.list);  in efx_init_struct_tc()
3037 	efx->tc->facts.reps.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL;  in efx_init_struct_tc()
3043 	rhashtable_destroy(&efx->tc->lhs_rule_ht);  in efx_init_struct_tc()
3045 	rhashtable_destroy(&efx->tc->match_action_ht);  in efx_init_struct_tc()
3047 	rhashtable_destroy(&efx->tc->encap_match_ht);  in efx_init_struct_tc()
3049 	rhashtable_destroy(&efx->tc->mac_ht);  in efx_init_struct_tc()
3055 	mutex_destroy(&efx->tc->mutex);  in efx_init_struct_tc()
3056 	kfree(efx->tc->caps);  in efx_init_struct_tc()
3058 	kfree(efx->tc);  in efx_init_struct_tc()
3059 	efx->tc = NULL;  in efx_init_struct_tc()
3065 	if (!efx->tc)  in efx_fini_struct_tc()
3068 	mutex_lock(&efx->tc->mutex);  in efx_fini_struct_tc()
3069 	EFX_WARN_ON_PARANOID(efx->tc->dflt.pf.fw_id !=  in efx_fini_struct_tc()
3071 	EFX_WARN_ON_PARANOID(efx->tc->dflt.wire.fw_id !=  in efx_fini_struct_tc()
3073 	EFX_WARN_ON_PARANOID(efx->tc->facts.pf.fw_id !=  in efx_fini_struct_tc()
3075 	EFX_WARN_ON_PARANOID(efx->tc->facts.reps.fw_id !=  in efx_fini_struct_tc()
3077 	rhashtable_free_and_destroy(&efx->tc->lhs_rule_ht, efx_tc_lhs_free, efx);  in efx_fini_struct_tc()
3078 	rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free,  in efx_fini_struct_tc()
3080 	rhashtable_free_and_destroy(&efx->tc->encap_match_ht,  in efx_fini_struct_tc()
3083 	rhashtable_free_and_destroy(&efx->tc->recirc_ht, efx_tc_recirc_free, efx);  in efx_fini_struct_tc()
3084 	WARN_ON(!ida_is_empty(&efx->tc->recirc_ida));  in efx_fini_struct_tc()
3085 	ida_destroy(&efx->tc->recirc_ida);  in efx_fini_struct_tc()
3086 	rhashtable_free_and_destroy(&efx->tc->mac_ht, efx_tc_mac_free, NULL);  in efx_fini_struct_tc()
3089 	mutex_unlock(&efx->tc->mutex);  in efx_fini_struct_tc()
3090 	mutex_destroy(&efx->tc->mutex);  in efx_fini_struct_tc()
3091 	kfree(efx->tc->caps);  in efx_fini_struct_tc()
3092 	kfree(efx->tc);  in efx_fini_struct_tc()
3093 	efx->tc = NULL;  in efx_fini_struct_tc()