230 	match->value._field = fm.key->_tcfield;				\
231 	match->mask._field = fm.mask->_tcfield;				\
240 				     struct efx_tc_match *match,  in efx_tc_flower_parse_match()  argument
243 	struct flow_dissector *dissector = rule->match.dissector;  in efx_tc_flower_parse_match()
269 			match->value.ip_frag = fm.key->flags & FLOW_DIS_IS_FRAGMENT;  in efx_tc_flower_parse_match()
270 			match->mask.ip_frag = true;  in efx_tc_flower_parse_match()
273 			match->value.ip_firstfrag = fm.key->flags & FLOW_DIS_FIRST_FRAG;  in efx_tc_flower_parse_match()
274 			match->mask.ip_firstfrag = true;  in efx_tc_flower_parse_match()
306 	if (!IS_ALL_ONES(match->mask.eth_proto) ||  in efx_tc_flower_parse_match()
307 	    !(match->value.eth_proto == htons(ETH_P_IP) ||  in efx_tc_flower_parse_match()
308 	      match->value.eth_proto == htons(ETH_P_IPV6)))  in efx_tc_flower_parse_match()
326 			match->value.vlan_proto[0] = fm.key->vlan_tpid;  in efx_tc_flower_parse_match()
327 			match->mask.vlan_proto[0] = fm.mask->vlan_tpid;  in efx_tc_flower_parse_match()
328 			match->value.vlan_tci[0] = cpu_to_be16(fm.key->vlan_priority << 13 |  in efx_tc_flower_parse_match()
330 			match->mask.vlan_tci[0] = cpu_to_be16(fm.mask->vlan_priority << 13 |  in efx_tc_flower_parse_match()
340 			match->value.vlan_proto[1] = fm.key->vlan_tpid;  in efx_tc_flower_parse_match()
341 			match->mask.vlan_proto[1] = fm.mask->vlan_tpid;  in efx_tc_flower_parse_match()
342 			match->value.vlan_tci[1] = cpu_to_be16(fm.key->vlan_priority << 13 |  in efx_tc_flower_parse_match()
344 			match->mask.vlan_tci[1] = cpu_to_be16(fm.mask->vlan_priority << 13 |  in efx_tc_flower_parse_match()
353 		ether_addr_copy(match->value.eth_saddr, fm.key->src);  in efx_tc_flower_parse_match()
354 		ether_addr_copy(match->value.eth_daddr, fm.key->dst);  in efx_tc_flower_parse_match()
355 		ether_addr_copy(match->mask.eth_saddr, fm.mask->src);  in efx_tc_flower_parse_match()
356 		ether_addr_copy(match->mask.eth_daddr, fm.mask->dst);  in efx_tc_flower_parse_match()
361 	if ((match->value.ip_proto != IPPROTO_UDP &&  in efx_tc_flower_parse_match()
362 	     match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto))  in efx_tc_flower_parse_match()
439 		match->value.ct_state_trk = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED);  in efx_tc_flower_parse_match()
440 		match->mask.ct_state_trk = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED);  in efx_tc_flower_parse_match()
441 		match->value.ct_state_est = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED);  in efx_tc_flower_parse_match()
442 		match->mask.ct_state_est = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED);  in efx_tc_flower_parse_match()
450 		match->value.ct_mark = fm.key->ct_mark;  in efx_tc_flower_parse_match()
451 		match->mask.ct_mark = fm.mask->ct_mark;  in efx_tc_flower_parse_match()
452 		match->value.ct_zone = fm.key->ct_zone;  in efx_tc_flower_parse_match()
453 		match->mask.ct_zone = fm.mask->ct_zone;  in efx_tc_flower_parse_match()
490 					    struct efx_tc_match *match,  in efx_tc_flower_record_encap_match()  argument
507 	if (match->mask.enc_dst_ip | match->mask.enc_src_ip) {  in efx_tc_flower_record_encap_match()
508 		if (!IS_ALL_ONES(match->mask.enc_dst_ip)) {  in efx_tc_flower_record_encap_match()
513 		if (!IS_ALL_ONES(match->mask.enc_src_ip)) {  in efx_tc_flower_record_encap_match()
519 		if (!ipv6_addr_any(&match->mask.enc_dst_ip6) ||  in efx_tc_flower_record_encap_match()
520 		    !ipv6_addr_any(&match->mask.enc_src_ip6)) {  in efx_tc_flower_record_encap_match()
527 		if (!efx_ipv6_addr_all_ones(&match->mask.enc_dst_ip6)) {  in efx_tc_flower_record_encap_match()
532 		if (!efx_ipv6_addr_all_ones(&match->mask.enc_src_ip6)) {  in efx_tc_flower_record_encap_match()
539 	if (!IS_ALL_ONES(match->mask.enc_dport)) {  in efx_tc_flower_record_encap_match()
543 	if (match->mask.enc_sport || match->mask.enc_ip_tos) {  in efx_tc_flower_record_encap_match()
544 		struct efx_tc_match pmatch = *match;  in efx_tc_flower_record_encap_match()
556 						      match->mask.enc_ip_tos,  in efx_tc_flower_record_encap_match()
557 						      match->mask.enc_sport,  in efx_tc_flower_record_encap_match()
563 	if (match->mask.enc_ip_ttl) {  in efx_tc_flower_record_encap_match()
569 	rc = efx_mae_check_encap_match_caps(efx, ipv6, match->mask.enc_ip_tos,  in efx_tc_flower_record_encap_match()
570 					    match->mask.enc_sport, extack);  in efx_tc_flower_record_encap_match()
579 	encap->src_ip = match->value.enc_src_ip;  in efx_tc_flower_record_encap_match()
580 	encap->dst_ip = match->value.enc_dst_ip;  in efx_tc_flower_record_encap_match()
582 	encap->src_ip6 = match->value.enc_src_ip6;  in efx_tc_flower_record_encap_match()
583 	encap->dst_ip6 = match->value.enc_dst_ip6;  in efx_tc_flower_record_encap_match()
585 	encap->udp_dport = match->value.enc_dport;  in efx_tc_flower_record_encap_match()
587 	encap->ip_tos = match->value.enc_ip_tos;  in efx_tc_flower_record_encap_match()
588 	encap->ip_tos_mask = match->mask.enc_ip_tos;  in efx_tc_flower_record_encap_match()
590 	encap->udp_sport = match->value.enc_sport;  in efx_tc_flower_record_encap_match()
591 	encap->udp_sport_mask = match->mask.enc_sport;  in efx_tc_flower_record_encap_match()
678 	match->encap = encap;  in efx_tc_flower_record_encap_match()
749 	if (rule->match.rid)  in efx_tc_delete_rule()
750 		efx_tc_put_recirc_id(efx, rule->match.rid);  in efx_tc_delete_rule()
751 	if (rule->match.encap)  in efx_tc_delete_rule()
752 		efx_tc_flower_release_encap_match(efx, rule->match.encap);  in efx_tc_delete_rule()
854 				    struct efx_tc_match *match)  in efx_tc_rule_is_lhs_rule()  argument
870 			if (!match->mask.ct_state_trk || !match->value.ct_state_trk)  in efx_tc_rule_is_lhs_rule()
893 static int efx_tc_flower_translate_flhs_match(struct efx_tc_match *match)  in efx_tc_flower_translate_flhs_match()  argument
898 	if (match->mask._key) {				\  in efx_tc_flower_translate_flhs_match()
901 		match->mask._key = match->mask._ekey;	\  in efx_tc_flower_translate_flhs_match()
902 		match->mask._ekey = 0;			\  in efx_tc_flower_translate_flhs_match()
903 		match->value._key = match->value._ekey;	\  in efx_tc_flower_translate_flhs_match()
904 		match->value._ekey = 0;			\  in efx_tc_flower_translate_flhs_match()
909 	if (match->mask.ip_proto)  in efx_tc_flower_translate_flhs_match()
911 	match->mask.ip_proto = ~0;  in efx_tc_flower_translate_flhs_match()
912 	match->value.ip_proto = IPPROTO_UDP;  in efx_tc_flower_translate_flhs_match()
916 	if (!ipv6_addr_any(&match->mask.src_ip6))  in efx_tc_flower_translate_flhs_match()
918 	match->mask.src_ip6 = match->mask.enc_src_ip6;  in efx_tc_flower_translate_flhs_match()
919 	memset(&match->mask.enc_src_ip6, 0, sizeof(struct in6_addr));  in efx_tc_flower_translate_flhs_match()
920 	if (!ipv6_addr_any(&match->mask.dst_ip6))  in efx_tc_flower_translate_flhs_match()
922 	match->mask.dst_ip6 = match->mask.enc_dst_ip6;  in efx_tc_flower_translate_flhs_match()
923 	memset(&match->mask.enc_dst_ip6, 0, sizeof(struct in6_addr));  in efx_tc_flower_translate_flhs_match()
945 static bool efx_tc_flower_flhs_needs_ar(struct efx_tc_match *match)  in efx_tc_flower_flhs_needs_ar()  argument
948 	return match->mask.eth_proto ||  in efx_tc_flower_flhs_needs_ar()
949 	       match->mask.vlan_tci[0] || match->mask.vlan_tci[1] ||  in efx_tc_flower_flhs_needs_ar()
950 	       match->mask.vlan_proto[0] || match->mask.vlan_proto[1] ||  in efx_tc_flower_flhs_needs_ar()
951 	       memchr_inv(match->mask.eth_saddr, 0, ETH_ALEN) ||  in efx_tc_flower_flhs_needs_ar()
952 	       memchr_inv(match->mask.eth_daddr, 0, ETH_ALEN) ||  in efx_tc_flower_flhs_needs_ar()
953 	       match->mask.ip_proto ||  in efx_tc_flower_flhs_needs_ar()
954 	       match->mask.ip_tos || match->mask.ip_ttl ||  in efx_tc_flower_flhs_needs_ar()
955 	       match->mask.src_ip || match->mask.dst_ip ||  in efx_tc_flower_flhs_needs_ar()
957 	       !ipv6_addr_any(&match->mask.src_ip6) ||  in efx_tc_flower_flhs_needs_ar()
958 	       !ipv6_addr_any(&match->mask.dst_ip6) ||  in efx_tc_flower_flhs_needs_ar()
960 	       match->mask.ip_frag || match->mask.ip_firstfrag ||  in efx_tc_flower_flhs_needs_ar()
961 	       match->mask.l4_sport || match->mask.l4_dport ||  in efx_tc_flower_flhs_needs_ar()
962 	       match->mask.tcp_flags ||  in efx_tc_flower_flhs_needs_ar()
964 	       match->mask.enc_keyid;  in efx_tc_flower_flhs_needs_ar()
1243 			 struct efx_tc_match *match)  in efx_tc_mangle()  argument
1323 			if (match->mask.ip_ttl != U8_MAX) {  in efx_tc_mangle()
1326 						       match->mask.ip_ttl);  in efx_tc_mangle()
1333 			if (match->value.ip_ttl == 0) {  in efx_tc_mangle()
1348 			tr_ttl = match->value.ip_ttl - 1;  in efx_tc_mangle()
1382 			if (match->mask.ip_ttl != U8_MAX) {  in efx_tc_mangle()
1385 						       match->mask.ip_ttl);  in efx_tc_mangle()
1392 			if (match->value.ip_ttl == 0) {  in efx_tc_mangle()
1407 			tr_ttl = match->value.ip_ttl - 1;  in efx_tc_mangle()
1455 						struct efx_tc_match *match,  in efx_tc_flower_replace_foreign_lhs_ar()  argument
1479 	rc = efx_tc_flower_record_encap_match(efx, match, type,  in efx_tc_flower_replace_foreign_lhs_ar()
1484 	match->mask.recirc_id = 0xff;  in efx_tc_flower_replace_foreign_lhs_ar()
1485 	if (match->mask.ct_state_trk && match->value.ct_state_trk) {  in efx_tc_flower_replace_foreign_lhs_ar()
1491 	match->mask.ct_state_trk = 0;  in efx_tc_flower_replace_foreign_lhs_ar()
1492 	match->value.ct_state_trk = 0;  in efx_tc_flower_replace_foreign_lhs_ar()
1502 	match->mask.tcp_syn_fin_rst = true;  in efx_tc_flower_replace_foreign_lhs_ar()
1504 	rc = efx_mae_match_check_caps(efx, &match->mask, extack);  in efx_tc_flower_replace_foreign_lhs_ar()
1531 	rule->match = *match;  in efx_tc_flower_replace_foreign_lhs_ar()
1551 	if (match->encap)  in efx_tc_flower_replace_foreign_lhs_ar()
1552 		efx_tc_flower_release_encap_match(efx, match->encap);  in efx_tc_flower_replace_foreign_lhs_ar()
1559 					     struct efx_tc_match *match,  in efx_tc_flower_replace_foreign_lhs()  argument
1572 	if (!efx_tc_match_is_encap(&match->mask)) {  in efx_tc_flower_replace_foreign_lhs()
1578 	if (efx_tc_flower_flhs_needs_ar(match))  in efx_tc_flower_replace_foreign_lhs()
1579 		return efx_tc_flower_replace_foreign_lhs_ar(efx, tc, fr, match,  in efx_tc_flower_replace_foreign_lhs()
1596 	rc = efx_tc_flower_record_encap_match(efx, match, type,  in efx_tc_flower_replace_foreign_lhs()
1602 	if (match->mask.ct_state_trk && match->value.ct_state_trk) {  in efx_tc_flower_replace_foreign_lhs()
1608 	match->mask.ct_state_trk = 0;  in efx_tc_flower_replace_foreign_lhs()
1609 	match->value.ct_state_trk = 0;  in efx_tc_flower_replace_foreign_lhs()
1611 	rc = efx_tc_flower_translate_flhs_match(match);  in efx_tc_flower_replace_foreign_lhs()
1617 	rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack);  in efx_tc_flower_replace_foreign_lhs()
1643 	rule->match = *match;  in efx_tc_flower_replace_foreign_lhs()
1663 	if (match->encap)  in efx_tc_flower_replace_foreign_lhs()
1664 		efx_tc_flower_release_encap_match(efx, match->encap);  in efx_tc_flower_replace_foreign_lhs()
1678 	struct efx_tc_match match;  in efx_tc_flower_replace_foreign()  local
1684 	memset(&match, 0, sizeof(match));  in efx_tc_flower_replace_foreign()
1685 	rc = efx_tc_flower_parse_match(efx, fr, &match, extack);  in efx_tc_flower_replace_foreign()
1698 	match.value.ingress_port = rc;  in efx_tc_flower_replace_foreign()
1699 	match.mask.ingress_port = ~0;  in efx_tc_flower_replace_foreign()
1701 	if (efx_tc_rule_is_lhs_rule(fr, &match))  in efx_tc_flower_replace_foreign()
1702 		return efx_tc_flower_replace_foreign_lhs(efx, tc, fr, &match,  in efx_tc_flower_replace_foreign()
1715 		match.rid = rid;  in efx_tc_flower_replace_foreign()
1716 		match.value.recirc_id = rid->fw_id;  in efx_tc_flower_replace_foreign()
1718 	match.mask.recirc_id = 0xff;  in efx_tc_flower_replace_foreign()
1723 	if (match.mask.ct_state_trk && match.value.ct_state_trk &&  in efx_tc_flower_replace_foreign()
1724 	    match.mask.ct_state_est && match.value.ct_state_est)  in efx_tc_flower_replace_foreign()
1725 		match.mask.ct_state_trk = 0;  in efx_tc_flower_replace_foreign()
1731 	if (match.mask.ct_state_est && !match.value.ct_state_est) {  in efx_tc_flower_replace_foreign()
1732 		if (match.value.tcp_syn_fin_rst) {  in efx_tc_flower_replace_foreign()
1738 		match.mask.tcp_syn_fin_rst = true;  in efx_tc_flower_replace_foreign()
1761 	rc = efx_mae_match_check_caps(efx, &match.mask, extack);  in efx_tc_flower_replace_foreign()
1765 	if (efx_tc_match_is_encap(&match.mask)) {  in efx_tc_flower_replace_foreign()
1784 		rc = efx_tc_flower_record_encap_match(efx, &match, type,  in efx_tc_flower_replace_foreign()
1949 	rule->match = match;  in efx_tc_flower_replace_foreign()
1960 	rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,  in efx_tc_flower_replace_foreign()
1974 	if (match.rid)  in efx_tc_flower_replace_foreign()
1975 		efx_tc_put_recirc_id(efx, match.rid);  in efx_tc_flower_replace_foreign()
1986 	if (match.encap)  in efx_tc_flower_replace_foreign()
1987 		efx_tc_flower_release_encap_match(efx, match.encap);  in efx_tc_flower_replace_foreign()
1994 				     struct efx_tc_match *match,  in efx_tc_flower_replace_lhs()  argument
2007 	if (match->mask.ct_state_trk && match->value.ct_state_trk) {  in efx_tc_flower_replace_lhs()
2012 	match->mask.ct_state_trk = 0;  in efx_tc_flower_replace_lhs()
2013 	match->value.ct_state_trk = 0;  in efx_tc_flower_replace_lhs()
2015 	rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack);  in efx_tc_flower_replace_lhs()
2045 	rule->match = *match;  in efx_tc_flower_replace_lhs()
2079 	struct efx_tc_match match;  in efx_tc_flower_replace()  local
2106 	memset(&match, 0, sizeof(match));  in efx_tc_flower_replace()
2112 	match.value.ingress_port = rc;  in efx_tc_flower_replace()
2113 	match.mask.ingress_port = ~0;  in efx_tc_flower_replace()
2114 	rc = efx_tc_flower_parse_match(efx, fr, &match, extack);  in efx_tc_flower_replace()
2117 	if (efx_tc_match_is_encap(&match.mask)) {  in efx_tc_flower_replace()
2122 	if (efx_tc_rule_is_lhs_rule(fr, &match))  in efx_tc_flower_replace()
2123 		return efx_tc_flower_replace_lhs(efx, tc, fr, &match, efv,  in efx_tc_flower_replace()
2151 		match.rid = rid;  in efx_tc_flower_replace()
2152 		match.value.recirc_id = rid->fw_id;  in efx_tc_flower_replace()
2154 	match.mask.recirc_id = 0xff;  in efx_tc_flower_replace()
2159 	if (match.mask.ct_state_trk && match.value.ct_state_trk &&  in efx_tc_flower_replace()
2160 	    match.mask.ct_state_est && match.value.ct_state_est)  in efx_tc_flower_replace()
2161 		match.mask.ct_state_trk = 0;  in efx_tc_flower_replace()
2167 	if (match.mask.ct_state_est && !match.value.ct_state_est) {  in efx_tc_flower_replace()
2168 		if (match.value.tcp_syn_fin_rst) {  in efx_tc_flower_replace()
2173 		match.mask.tcp_syn_fin_rst = true;  in efx_tc_flower_replace()
2176 	rc = efx_mae_match_check_caps(efx, &match.mask, extack);  in efx_tc_flower_replace()
2421 			rc = efx_tc_mangle(efx, act, fa, &mung, extack, &match);  in efx_tc_flower_replace()
2503 	rule->match = match;  in efx_tc_flower_replace()
2523 	rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,  in efx_tc_flower_replace()
2537 	if (match.rid)  in efx_tc_flower_replace()
2538 		efx_tc_put_recirc_id(efx, match.rid);  in efx_tc_flower_replace()
2569 		if (lhs_rule->match.encap)  in efx_tc_flower_destroy()
2570 			efx_tc_flower_release_encap_match(efx, lhs_rule->match.encap);  in efx_tc_flower_destroy()
2669 	struct efx_tc_match *match = &rule->match;  in efx_tc_configure_default_rule()  local
2673 	match->value.ingress_port = ing_port;  in efx_tc_configure_default_rule()
2674 	match->mask.ingress_port = ~0;  in efx_tc_configure_default_rule()
2688 	rc = efx_mae_insert_rule(efx, match, EFX_TC_PRIO_DFLT,  in efx_tc_configure_default_rule()