226 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_##_name)) {		\
229 	flow_rule_match_##_tcget(rule, &fm);				\
239 				     struct flow_rule *rule,  in efx_tc_flower_parse_match()  argument
243 	struct flow_dissector *dissector = rule->match.dissector;  in efx_tc_flower_parse_match()
252 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {  in efx_tc_flower_parse_match()
255 		flow_rule_match_control(rule, &fm);  in efx_tc_flower_parse_match()
321 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {  in efx_tc_flower_parse_match()
324 		flow_rule_match_vlan(rule, &fm);  in efx_tc_flower_parse_match()
335 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)) {  in efx_tc_flower_parse_match()
338 		flow_rule_match_cvlan(rule, &fm);  in efx_tc_flower_parse_match()
349 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {  in efx_tc_flower_parse_match()
352 		flow_rule_match_eth_addrs(rule, &fm);  in efx_tc_flower_parse_match()
386 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {  in efx_tc_flower_parse_match()
389 		flow_rule_match_enc_control(rule, &fm);  in efx_tc_flower_parse_match()
435 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CT)) {  in efx_tc_flower_parse_match()
438 		flow_rule_match_ct(rule, &fm);  in efx_tc_flower_parse_match()
702 	 * a rule on the net_dev that's been offloaded to us, then the net_dev  in efx_tc_get_recirc_id()
703 	 * can't go away until the rule has been deoffloaded.  in efx_tc_get_recirc_id()
743 static void efx_tc_delete_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rule)  in efx_tc_delete_rule()  argument
745 	efx_mae_delete_rule(efx, rule->fw_id);  in efx_tc_delete_rule()
748 	efx_tc_free_action_set_list(efx, &rule->acts, true);  in efx_tc_delete_rule()
749 	if (rule->match.rid)  in efx_tc_delete_rule()
750 		efx_tc_put_recirc_id(efx, rule->match.rid);  in efx_tc_delete_rule()
751 	if (rule->match.encap)  in efx_tc_delete_rule()
752 		efx_tc_flower_release_encap_match(efx, rule->match.encap);  in efx_tc_delete_rule()
753 	rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;  in efx_tc_delete_rule()
839  * The MAE hardware can handle at most two rounds of action rule matching,
841  * rule".  This is a rule which typically contains only the actions "ct" and
844  * RHS rules go in the Action Rule table as normal but with a nonzero recirc_id
846  * the Action Rule or the Outer Rule table, the latter being preferred for
864 			/* If rule is -trk, or doesn't mention trk at all, then  in efx_tc_rule_is_lhs_rule()
866 			 * LHS rule).  If rule is +trk, then a CT action could  in efx_tc_rule_is_lhs_rule()
880 /* A foreign LHS rule has matches on enc_ keys at the TC layer (including an
886  * efx_tc_match_fields and (b) semantically an LHS rule doesn't have inner
889  * would imply a rule that needed a triple lookup.  (Hardware can do that,
938 /* If a foreign LHS rule wants to match on keys that are only available after
940  * Rule lookup, because that lookup determines the encap type used to parse
971 					    struct efx_tc_lhs_rule *rule)  in efx_tc_flower_handle_lhs_actions()  argument
975 	struct efx_tc_lhs_action *act = &rule->lhs_act;  in efx_tc_flower_handle_lhs_actions()
981 	ctype = rule->is_ar ? EFX_TC_COUNTER_TYPE_AR : EFX_TC_COUNTER_TYPE_OR;  in efx_tc_flower_handle_lhs_actions()
1036 				NL_SET_ERR_MSG_MOD(extack, "Can't clear ct in LHS rule");  in efx_tc_flower_handle_lhs_actions()
1042 				NL_SET_ERR_MSG_MOD(extack, "Can't perform NAT in LHS rule - packet isn't conntracked yet");  in efx_tc_flower_handle_lhs_actions()
1046 				NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled ct.action %u for LHS rule\n",  in efx_tc_flower_handle_lhs_actions()
1059 			NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u for LHS rule\n",  in efx_tc_flower_handle_lhs_actions()
1066 		NL_SET_ERR_MSG_MOD(extack, "Missing goto chain in LHS rule");  in efx_tc_flower_handle_lhs_actions()
1108  * @efx:	NIC we're installing a flow rule on
1226  * @efx:	NIC we're installing a flow rule on
1420 		NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled mangle htype %u for action rule",  in efx_tc_mangle()
1459 	struct efx_tc_lhs_rule *rule, *old;  in efx_tc_flower_replace_foreign_lhs_ar()  local
1476 	/* This is an Action Rule, so it needs a separate Encap Match in the  in efx_tc_flower_replace_foreign_lhs_ar()
1477 	 * Outer Rule table.  Insert that now.  in efx_tc_flower_replace_foreign_lhs_ar()
1486 		NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");  in efx_tc_flower_replace_foreign_lhs_ar()
1508 	rule = kzalloc(sizeof(*rule), GFP_USER);  in efx_tc_flower_replace_foreign_lhs_ar()
1509 	if (!rule) {  in efx_tc_flower_replace_foreign_lhs_ar()
1513 	rule->cookie = tc->cookie;  in efx_tc_flower_replace_foreign_lhs_ar()
1514 	rule->is_ar = true;  in efx_tc_flower_replace_foreign_lhs_ar()
1516 						&rule->linkage,  in efx_tc_flower_replace_foreign_lhs_ar()
1520 			  "Already offloaded rule (cookie %lx)\n", tc->cookie);  in efx_tc_flower_replace_foreign_lhs_ar()
1522 		NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");  in efx_tc_flower_replace_foreign_lhs_ar()
1527 	rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);  in efx_tc_flower_replace_foreign_lhs_ar()
1531 	rule->match = *match;  in efx_tc_flower_replace_foreign_lhs_ar()
1532 	rule->lhs_act.tun_type = type;  in efx_tc_flower_replace_foreign_lhs_ar()
1534 	rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);  in efx_tc_flower_replace_foreign_lhs_ar()
1536 		NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");  in efx_tc_flower_replace_foreign_lhs_ar()
1540 		  "Successfully parsed lhs rule (cookie %lx)\n",  in efx_tc_flower_replace_foreign_lhs_ar()
1545 	efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);  in efx_tc_flower_replace_foreign_lhs_ar()
1547 		rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,  in efx_tc_flower_replace_foreign_lhs_ar()
1549 	kfree(rule);  in efx_tc_flower_replace_foreign_lhs_ar()
1563 	struct efx_tc_lhs_rule *rule, *old;  in efx_tc_flower_replace_foreign_lhs()  local
1568 		NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0");  in efx_tc_flower_replace_foreign_lhs()
1573 		/* This is not a tunnel decap rule, ignore it */  in efx_tc_flower_replace_foreign_lhs()
1603 		NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");  in efx_tc_flower_replace_foreign_lhs()
1613 		NL_SET_ERR_MSG_MOD(extack, "LHS rule cannot match on inner fields");  in efx_tc_flower_replace_foreign_lhs()
1621 	rule = kzalloc(sizeof(*rule), GFP_USER);  in efx_tc_flower_replace_foreign_lhs()
1622 	if (!rule) {  in efx_tc_flower_replace_foreign_lhs()
1626 	rule->cookie = tc->cookie;  in efx_tc_flower_replace_foreign_lhs()
1628 						&rule->linkage,  in efx_tc_flower_replace_foreign_lhs()
1632 			  "Already offloaded rule (cookie %lx)\n", tc->cookie);  in efx_tc_flower_replace_foreign_lhs()
1634 		NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");  in efx_tc_flower_replace_foreign_lhs()
1639 	rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);  in efx_tc_flower_replace_foreign_lhs()
1643 	rule->match = *match;  in efx_tc_flower_replace_foreign_lhs()
1644 	rule->lhs_act.tun_type = type;  in efx_tc_flower_replace_foreign_lhs()
1646 	rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);  in efx_tc_flower_replace_foreign_lhs()
1648 		NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");  in efx_tc_flower_replace_foreign_lhs()
1652 		  "Successfully parsed lhs rule (cookie %lx)\n",  in efx_tc_flower_replace_foreign_lhs()
1657 	efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);  in efx_tc_flower_replace_foreign_lhs()
1659 		rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,  in efx_tc_flower_replace_foreign_lhs()
1661 	kfree(rule);  in efx_tc_flower_replace_foreign_lhs()
1674 	struct efx_tc_flow_rule *rule = NULL, *old = NULL;  in efx_tc_flower_replace_foreign()  local
1688 	/* The rule as given to us doesn't specify a source netdevice.  in efx_tc_flower_replace_foreign()
1790 		/* This is not a tunnel decap rule, ignore it */  in efx_tc_flower_replace_foreign()
1797 	rule = kzalloc(sizeof(*rule), GFP_USER);  in efx_tc_flower_replace_foreign()
1798 	if (!rule) {  in efx_tc_flower_replace_foreign()
1802 	INIT_LIST_HEAD(&rule->acts.list);  in efx_tc_flower_replace_foreign()
1803 	rule->cookie = tc->cookie;  in efx_tc_flower_replace_foreign()
1805 						&rule->linkage,  in efx_tc_flower_replace_foreign()
1812 			  "Ignoring already-offloaded rule (cookie %lx)\n",  in efx_tc_flower_replace_foreign()
1899 			list_add_tail(&act->list, &rule->acts.list);  in efx_tc_flower_replace_foreign()
1945 		list_add_tail(&act->list, &rule->acts.list);  in efx_tc_flower_replace_foreign()
1949 	rule->match = match;  in efx_tc_flower_replace_foreign()
1955 	rc = efx_mae_alloc_action_set_list(efx, &rule->acts);  in efx_tc_flower_replace_foreign()
1960 	rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,  in efx_tc_flower_replace_foreign()
1961 				 rule->acts.fw_id, &rule->fw_id);  in efx_tc_flower_replace_foreign()
1963 		NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");  in efx_tc_flower_replace_foreign()
1969 	efx_mae_free_action_set_list(efx, &rule->acts);  in efx_tc_flower_replace_foreign()
1971 	/* We failed to insert the rule, so free up any entries we created in  in efx_tc_flower_replace_foreign()
1978 	if (rule) {  in efx_tc_flower_replace_foreign()
1981 					       &rule->linkage,  in efx_tc_flower_replace_foreign()
1983 		efx_tc_free_action_set_list(efx, &rule->acts, false);  in efx_tc_flower_replace_foreign()
1985 	kfree(rule);  in efx_tc_flower_replace_foreign()
1999 	struct efx_tc_lhs_rule *rule, *old;  in efx_tc_flower_replace_lhs()  local
2003 		NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0");  in efx_tc_flower_replace_lhs()
2008 		NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");  in efx_tc_flower_replace_lhs()
2019 	rule = kzalloc(sizeof(*rule), GFP_USER);  in efx_tc_flower_replace_lhs()
2020 	if (!rule)  in efx_tc_flower_replace_lhs()
2022 	rule->cookie = tc->cookie;  in efx_tc_flower_replace_lhs()
2024 						&rule->linkage,  in efx_tc_flower_replace_lhs()
2031 			  "Already offloaded rule (cookie %lx)\n", tc->cookie);  in efx_tc_flower_replace_lhs()
2033 		NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");  in efx_tc_flower_replace_lhs()
2041 	rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, efx->net_dev, rule);  in efx_tc_flower_replace_lhs()
2045 	rule->match = *match;  in efx_tc_flower_replace_lhs()
2047 	rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);  in efx_tc_flower_replace_lhs()
2049 		NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");  in efx_tc_flower_replace_lhs()
2053 		  "Successfully parsed lhs rule (cookie %lx)\n",  in efx_tc_flower_replace_lhs()
2058 	efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);  in efx_tc_flower_replace_lhs()
2060 		rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,  in efx_tc_flower_replace_lhs()
2062 	kfree(rule);  in efx_tc_flower_replace_lhs()
2074 	struct efx_tc_flow_rule *rule = NULL, *old;  in efx_tc_flower_replace()  local
2180 	rule = kzalloc(sizeof(*rule), GFP_USER);  in efx_tc_flower_replace()
2181 	if (!rule) {  in efx_tc_flower_replace()
2185 	INIT_LIST_HEAD(&rule->acts.list);  in efx_tc_flower_replace()
2186 	rule->cookie = tc->cookie;  in efx_tc_flower_replace()
2188 						&rule->linkage,  in efx_tc_flower_replace()
2195 			  "Already offloaded rule (cookie %lx)\n", tc->cookie);  in efx_tc_flower_replace()
2196 		NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");  in efx_tc_flower_replace()
2220 	 * append @act to the action-set list (@rule->acts); if this is a pipe  in efx_tc_flower_replace()
2225 	 * @rule->acts or pointed to by @act (and never both), and that only  in efx_tc_flower_replace()
2226 	 * those action-sets in @rule->acts exist in hardware.  Consequently,  in efx_tc_flower_replace()
2228 	 * for @rule->acts we remove each action-set from hardware before  in efx_tc_flower_replace()
2292 			list_add_tail(&act->list, &rule->acts.list);  in efx_tc_flower_replace()
2336 				list_add_tail(&act->list, &rule->acts.list);  in efx_tc_flower_replace()
2337 				act->user = &rule->acts;  in efx_tc_flower_replace()
2377 			list_add_tail(&act->list, &rule->acts.list);  in efx_tc_flower_replace()
2478 			/* Rule applies to traffic from the wire,  in efx_tc_flower_replace()
2483 			/* Representor, so rule applies to traffic from  in efx_tc_flower_replace()
2495 		list_add_tail(&act->list, &rule->acts.list);  in efx_tc_flower_replace()
2503 	rule->match = match;  in efx_tc_flower_replace()
2505 	rc = efx_mae_alloc_action_set_list(efx, &rule->acts);  in efx_tc_flower_replace()
2511 		/* PF netdev, so rule applies to traffic from wire */  in efx_tc_flower_replace()
2512 		rule->fallback = &efx->tc->facts.pf;  in efx_tc_flower_replace()
2514 		/* repdev, so rule applies to traffic from representee */  in efx_tc_flower_replace()
2515 		rule->fallback = &efx->tc->facts.reps;  in efx_tc_flower_replace()
2516 	if (!efx_tc_check_ready(efx, rule)) {  in efx_tc_flower_replace()
2518 		acts_id = rule->fallback->fw_id;  in efx_tc_flower_replace()
2521 		acts_id = rule->acts.fw_id;  in efx_tc_flower_replace()
2523 	rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,  in efx_tc_flower_replace()
2524 				 acts_id, &rule->fw_id);  in efx_tc_flower_replace()
2526 		NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");  in efx_tc_flower_replace()
2532 	efx_mae_free_action_set_list(efx, &rule->acts);  in efx_tc_flower_replace()
2534 	/* We failed to insert the rule, so free up any entries we created in  in efx_tc_flower_replace()
2541 	if (rule) {  in efx_tc_flower_replace()
2544 					       &rule->linkage,  in efx_tc_flower_replace()
2546 		efx_tc_free_action_set_list(efx, &rule->acts, false);  in efx_tc_flower_replace()
2548 	kfree(rule);  in efx_tc_flower_replace()
2558 	struct efx_tc_flow_rule *rule;  in efx_tc_flower_destroy()  local
2577 	rule = rhashtable_lookup_fast(&efx->tc->match_action_ht, &tc->cookie,  in efx_tc_flower_destroy()
2579 	if (!rule) {  in efx_tc_flower_destroy()
2593 	efx_tc_delete_rule(efx, rule);  in efx_tc_flower_destroy()
2595 	rhashtable_remove_fast(&efx->tc->match_action_ht, &rule->linkage,  in efx_tc_flower_destroy()
2597 	netif_dbg(efx, drv, efx->net_dev, "Removed filter %lx\n", rule->cookie);  in efx_tc_flower_destroy()
2598 	kfree(rule);  in efx_tc_flower_destroy()
2666 					 u32 eg_port, struct efx_tc_flow_rule *rule)  in efx_tc_configure_default_rule()  argument
2668 	struct efx_tc_action_set_list *acts = &rule->acts;  in efx_tc_configure_default_rule()
2669 	struct efx_tc_match *match = &rule->match;  in efx_tc_configure_default_rule()
2689 				 acts->fw_id, &rule->fw_id);  in efx_tc_configure_default_rule()
2705 	struct efx_tc_flow_rule *rule = &efx->tc->dflt.pf;  in efx_tc_configure_default_rule_pf()  local
2710 	return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);  in efx_tc_configure_default_rule_pf()
2715 	struct efx_tc_flow_rule *rule = &efx->tc->dflt.wire;  in efx_tc_configure_default_rule_wire()  local
2720 	return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);  in efx_tc_configure_default_rule_wire()
2725 	struct efx_tc_flow_rule *rule = &efv->dflt;  in efx_tc_configure_default_rule_rep()  local
2731 	return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);  in efx_tc_configure_default_rule_rep()
2735 				     struct efx_tc_flow_rule *rule)  in efx_tc_deconfigure_default_rule()  argument
2737 	if (rule->fw_id != MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL)  in efx_tc_deconfigure_default_rule()
2738 		efx_tc_delete_rule(efx, rule);  in efx_tc_deconfigure_default_rule()
2739 	rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;  in efx_tc_deconfigure_default_rule()
2945 	struct efx_tc_lhs_rule *rule = ptr;  in efx_tc_lhs_free()  local
2950 		  rule->cookie);  in efx_tc_lhs_free()
2952 	if (rule->lhs_act.zone)  in efx_tc_lhs_free()
2953 		efx_tc_ct_unregister_zone(efx, rule->lhs_act.zone);  in efx_tc_lhs_free()
2954 	if (rule->lhs_act.count)  in efx_tc_lhs_free()
2955 		efx_tc_flower_put_counter_index(efx, rule->lhs_act.count);  in efx_tc_lhs_free()
2956 	efx_mae_remove_lhs_rule(efx, rule);  in efx_tc_lhs_free()
2958 	kfree(rule);  in efx_tc_lhs_free()
2971 	struct efx_tc_flow_rule *rule = ptr;  in efx_tc_flow_free()  local
2975 		  "tc rule %lx still present at teardown, removing\n",  in efx_tc_flow_free()
2976 		  rule->cookie);  in efx_tc_flow_free()
2979 	efx_tc_delete_rule(efx, rule);  in efx_tc_flow_free()
2981 	kfree(rule);  in efx_tc_flow_free()