1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net> 4 * Copyright (c) 2016 Pablo Neira Ayuso <pablo@netfilter.org> 5 * 6 * Development of this code funded by Astaro AG (http://www.astaro.com/) 7 */ 8 9 #include <linux/kernel.h> 10 #include <linux/if_vlan.h> 11 #include <linux/init.h> 12 #include <linux/module.h> 13 #include <linux/netlink.h> 14 #include <linux/netfilter.h> 15 #include <linux/netfilter/nf_tables.h> 16 #include <net/netfilter/nf_tables_core.h> 17 #include <net/netfilter/nf_tables.h> 18 #include <net/netfilter/nf_tables_offload.h> 19 /* For layer 4 checksum field offset. */ 20 #include <linux/tcp.h> 21 #include <linux/udp.h> 22 #include <net/gre.h> 23 #include <linux/icmpv6.h> 24 #include <linux/ip.h> 25 #include <linux/ipv6.h> 26 #include <net/sctp/checksum.h> 27 nft_payload_rebuild_vlan_hdr(const struct sk_buff * skb,int mac_off,struct vlan_ethhdr * veth)28 static bool nft_payload_rebuild_vlan_hdr(const struct sk_buff *skb, int mac_off, 29 struct vlan_ethhdr *veth) 30 { 31 if (skb_copy_bits(skb, mac_off, veth, ETH_HLEN)) 32 return false; 33 34 veth->h_vlan_proto = skb->vlan_proto; 35 veth->h_vlan_TCI = htons(skb_vlan_tag_get(skb)); 36 veth->h_vlan_encapsulated_proto = skb->protocol; 37 38 return true; 39 } 40 41 /* add vlan header into the user buffer for if tag was removed by offloads */ 42 static bool nft_payload_copy_vlan(u32 * d,const struct sk_buff * skb,u8 offset,u8 len)43 nft_payload_copy_vlan(u32 *d, const struct sk_buff *skb, u8 offset, u8 len) 44 { 45 int mac_off = skb_mac_header(skb) - skb->data; 46 u8 *vlanh, *dst_u8 = (u8 *) d; 47 struct vlan_ethhdr veth; 48 49 vlanh = (u8 *) &veth; 50 if (offset < VLAN_ETH_HLEN) { 51 u8 ethlen = len; 52 53 if (!nft_payload_rebuild_vlan_hdr(skb, mac_off, &veth)) 54 return false; 55 56 if (offset + len > VLAN_ETH_HLEN) 57 ethlen -= offset + len - VLAN_ETH_HLEN; 58 59 memcpy(dst_u8, vlanh + offset, ethlen); 60 61 len -= ethlen; 62 if (len == 0) 63 return true; 64 65 dst_u8 += ethlen; 66 offset = ETH_HLEN; 67 } else { 68 offset -= VLAN_HLEN; 69 } 70 71 return skb_copy_bits(skb, offset + mac_off, dst_u8, len) == 0; 72 } 73 __nft_payload_inner_offset(struct nft_pktinfo * pkt)74 static int __nft_payload_inner_offset(struct nft_pktinfo *pkt) 75 { 76 unsigned int thoff = nft_thoff(pkt); 77 78 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff) 79 return -1; 80 81 switch (pkt->tprot) { 82 case IPPROTO_UDP: 83 pkt->inneroff = thoff + sizeof(struct udphdr); 84 break; 85 case IPPROTO_TCP: { 86 struct tcphdr *th, _tcph; 87 88 th = skb_header_pointer(pkt->skb, thoff, sizeof(_tcph), &_tcph); 89 if (!th) 90 return -1; 91 92 pkt->inneroff = thoff + __tcp_hdrlen(th); 93 } 94 break; 95 case IPPROTO_GRE: { 96 u32 offset = sizeof(struct gre_base_hdr); 97 struct gre_base_hdr *gre, _gre; 98 __be16 version; 99 100 gre = skb_header_pointer(pkt->skb, thoff, sizeof(_gre), &_gre); 101 if (!gre) 102 return -1; 103 104 version = gre->flags & GRE_VERSION; 105 switch (version) { 106 case GRE_VERSION_0: 107 if (gre->flags & GRE_ROUTING) 108 return -1; 109 110 if (gre->flags & GRE_CSUM) { 111 offset += sizeof_field(struct gre_full_hdr, csum) + 112 sizeof_field(struct gre_full_hdr, reserved1); 113 } 114 if (gre->flags & GRE_KEY) 115 offset += sizeof_field(struct gre_full_hdr, key); 116 117 if (gre->flags & GRE_SEQ) 118 offset += sizeof_field(struct gre_full_hdr, seq); 119 break; 120 default: 121 return -1; 122 } 123 124 pkt->inneroff = thoff + offset; 125 } 126 break; 127 case IPPROTO_IPIP: 128 pkt->inneroff = thoff; 129 break; 130 default: 131 return -1; 132 } 133 134 pkt->flags |= NFT_PKTINFO_INNER; 135 136 return 0; 137 } 138 nft_payload_inner_offset(const struct nft_pktinfo * pkt)139 int nft_payload_inner_offset(const struct nft_pktinfo *pkt) 140 { 141 if (!(pkt->flags & NFT_PKTINFO_INNER) && 142 __nft_payload_inner_offset((struct nft_pktinfo *)pkt) < 0) 143 return -1; 144 145 return pkt->inneroff; 146 } 147 nft_payload_need_vlan_adjust(u32 offset,u32 len)148 static bool nft_payload_need_vlan_adjust(u32 offset, u32 len) 149 { 150 unsigned int boundary = offset + len; 151 152 /* data past ether src/dst requested, copy needed */ 153 if (boundary > offsetof(struct ethhdr, h_proto)) 154 return true; 155 156 return false; 157 } 158 nft_payload_eval(const struct nft_expr * expr,struct nft_regs * regs,const struct nft_pktinfo * pkt)159 void nft_payload_eval(const struct nft_expr *expr, 160 struct nft_regs *regs, 161 const struct nft_pktinfo *pkt) 162 { 163 const struct nft_payload *priv = nft_expr_priv(expr); 164 const struct sk_buff *skb = pkt->skb; 165 u32 *dest = ®s->data[priv->dreg]; 166 int offset; 167 168 if (priv->len % NFT_REG32_SIZE) 169 dest[priv->len / NFT_REG32_SIZE] = 0; 170 171 switch (priv->base) { 172 case NFT_PAYLOAD_LL_HEADER: 173 if (!skb_mac_header_was_set(skb) || skb_mac_header_len(skb) == 0) 174 goto err; 175 176 if (skb_vlan_tag_present(skb) && 177 nft_payload_need_vlan_adjust(priv->offset, priv->len)) { 178 if (!nft_payload_copy_vlan(dest, skb, 179 priv->offset, priv->len)) 180 goto err; 181 return; 182 } 183 offset = skb_mac_header(skb) - skb->data; 184 break; 185 case NFT_PAYLOAD_NETWORK_HEADER: 186 offset = skb_network_offset(skb); 187 break; 188 case NFT_PAYLOAD_TRANSPORT_HEADER: 189 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff) 190 goto err; 191 offset = nft_thoff(pkt); 192 break; 193 case NFT_PAYLOAD_INNER_HEADER: 194 offset = nft_payload_inner_offset(pkt); 195 if (offset < 0) 196 goto err; 197 break; 198 default: 199 WARN_ON_ONCE(1); 200 goto err; 201 } 202 offset += priv->offset; 203 204 if (skb_copy_bits(skb, offset, dest, priv->len) < 0) 205 goto err; 206 return; 207 err: 208 regs->verdict.code = NFT_BREAK; 209 } 210 211 static const struct nla_policy nft_payload_policy[NFTA_PAYLOAD_MAX + 1] = { 212 [NFTA_PAYLOAD_SREG] = { .type = NLA_U32 }, 213 [NFTA_PAYLOAD_DREG] = { .type = NLA_U32 }, 214 [NFTA_PAYLOAD_BASE] = { .type = NLA_U32 }, 215 [NFTA_PAYLOAD_OFFSET] = NLA_POLICY_MAX(NLA_BE32, 255), 216 [NFTA_PAYLOAD_LEN] = NLA_POLICY_MAX(NLA_BE32, 255), 217 [NFTA_PAYLOAD_CSUM_TYPE] = { .type = NLA_U32 }, 218 [NFTA_PAYLOAD_CSUM_OFFSET] = NLA_POLICY_MAX(NLA_BE32, 255), 219 [NFTA_PAYLOAD_CSUM_FLAGS] = { .type = NLA_U32 }, 220 }; 221 nft_payload_init(const struct nft_ctx * ctx,const struct nft_expr * expr,const struct nlattr * const tb[])222 static int nft_payload_init(const struct nft_ctx *ctx, 223 const struct nft_expr *expr, 224 const struct nlattr * const tb[]) 225 { 226 struct nft_payload *priv = nft_expr_priv(expr); 227 228 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); 229 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET])); 230 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); 231 232 return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG], 233 &priv->dreg, NULL, NFT_DATA_VALUE, 234 priv->len); 235 } 236 nft_payload_dump(struct sk_buff * skb,const struct nft_expr * expr,bool reset)237 static int nft_payload_dump(struct sk_buff *skb, 238 const struct nft_expr *expr, bool reset) 239 { 240 const struct nft_payload *priv = nft_expr_priv(expr); 241 242 if (nft_dump_register(skb, NFTA_PAYLOAD_DREG, priv->dreg) || 243 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) || 244 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) || 245 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len))) 246 goto nla_put_failure; 247 return 0; 248 249 nla_put_failure: 250 return -1; 251 } 252 nft_payload_reduce(struct nft_regs_track * track,const struct nft_expr * expr)253 static bool nft_payload_reduce(struct nft_regs_track *track, 254 const struct nft_expr *expr) 255 { 256 const struct nft_payload *priv = nft_expr_priv(expr); 257 const struct nft_payload *payload; 258 259 if (!nft_reg_track_cmp(track, expr, priv->dreg)) { 260 nft_reg_track_update(track, expr, priv->dreg, priv->len); 261 return false; 262 } 263 264 payload = nft_expr_priv(track->regs[priv->dreg].selector); 265 if (priv->base != payload->base || 266 priv->offset != payload->offset || 267 priv->len != payload->len) { 268 nft_reg_track_update(track, expr, priv->dreg, priv->len); 269 return false; 270 } 271 272 if (!track->regs[priv->dreg].bitwise) 273 return true; 274 275 return nft_expr_reduce_bitwise(track, expr); 276 } 277 nft_payload_offload_mask(struct nft_offload_reg * reg,u32 priv_len,u32 field_len)278 static bool nft_payload_offload_mask(struct nft_offload_reg *reg, 279 u32 priv_len, u32 field_len) 280 { 281 unsigned int remainder, delta, k; 282 struct nft_data mask = {}; 283 __be32 remainder_mask; 284 285 if (priv_len == field_len) { 286 memset(®->mask, 0xff, priv_len); 287 return true; 288 } else if (priv_len > field_len) { 289 return false; 290 } 291 292 memset(&mask, 0xff, field_len); 293 remainder = priv_len % sizeof(u32); 294 if (remainder) { 295 k = priv_len / sizeof(u32); 296 delta = field_len - priv_len; 297 remainder_mask = htonl(~((1 << (delta * BITS_PER_BYTE)) - 1)); 298 mask.data[k] = (__force u32)remainder_mask; 299 } 300 301 memcpy(®->mask, &mask, field_len); 302 303 return true; 304 } 305 nft_payload_offload_ll(struct nft_offload_ctx * ctx,struct nft_flow_rule * flow,const struct nft_payload * priv)306 static int nft_payload_offload_ll(struct nft_offload_ctx *ctx, 307 struct nft_flow_rule *flow, 308 const struct nft_payload *priv) 309 { 310 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 311 312 switch (priv->offset) { 313 case offsetof(struct ethhdr, h_source): 314 if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN)) 315 return -EOPNOTSUPP; 316 317 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs, 318 src, ETH_ALEN, reg); 319 break; 320 case offsetof(struct ethhdr, h_dest): 321 if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN)) 322 return -EOPNOTSUPP; 323 324 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs, 325 dst, ETH_ALEN, reg); 326 break; 327 case offsetof(struct ethhdr, h_proto): 328 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 329 return -EOPNOTSUPP; 330 331 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, 332 n_proto, sizeof(__be16), reg); 333 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK); 334 break; 335 case offsetof(struct vlan_ethhdr, h_vlan_TCI): 336 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 337 return -EOPNOTSUPP; 338 339 NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_VLAN, vlan, 340 vlan_tci, sizeof(__be16), reg, 341 NFT_OFFLOAD_F_NETWORK2HOST); 342 break; 343 case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto): 344 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 345 return -EOPNOTSUPP; 346 347 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_VLAN, vlan, 348 vlan_tpid, sizeof(__be16), reg); 349 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK); 350 break; 351 case offsetof(struct vlan_ethhdr, h_vlan_TCI) + sizeof(struct vlan_hdr): 352 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 353 return -EOPNOTSUPP; 354 355 NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_CVLAN, cvlan, 356 vlan_tci, sizeof(__be16), reg, 357 NFT_OFFLOAD_F_NETWORK2HOST); 358 break; 359 case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto) + 360 sizeof(struct vlan_hdr): 361 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 362 return -EOPNOTSUPP; 363 364 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_CVLAN, cvlan, 365 vlan_tpid, sizeof(__be16), reg); 366 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK); 367 break; 368 default: 369 return -EOPNOTSUPP; 370 } 371 372 return 0; 373 } 374 nft_payload_offload_ip(struct nft_offload_ctx * ctx,struct nft_flow_rule * flow,const struct nft_payload * priv)375 static int nft_payload_offload_ip(struct nft_offload_ctx *ctx, 376 struct nft_flow_rule *flow, 377 const struct nft_payload *priv) 378 { 379 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 380 381 switch (priv->offset) { 382 case offsetof(struct iphdr, saddr): 383 if (!nft_payload_offload_mask(reg, priv->len, 384 sizeof(struct in_addr))) 385 return -EOPNOTSUPP; 386 387 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, src, 388 sizeof(struct in_addr), reg); 389 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS); 390 break; 391 case offsetof(struct iphdr, daddr): 392 if (!nft_payload_offload_mask(reg, priv->len, 393 sizeof(struct in_addr))) 394 return -EOPNOTSUPP; 395 396 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, dst, 397 sizeof(struct in_addr), reg); 398 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS); 399 break; 400 case offsetof(struct iphdr, protocol): 401 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8))) 402 return -EOPNOTSUPP; 403 404 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto, 405 sizeof(__u8), reg); 406 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT); 407 break; 408 default: 409 return -EOPNOTSUPP; 410 } 411 412 return 0; 413 } 414 nft_payload_offload_ip6(struct nft_offload_ctx * ctx,struct nft_flow_rule * flow,const struct nft_payload * priv)415 static int nft_payload_offload_ip6(struct nft_offload_ctx *ctx, 416 struct nft_flow_rule *flow, 417 const struct nft_payload *priv) 418 { 419 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 420 421 switch (priv->offset) { 422 case offsetof(struct ipv6hdr, saddr): 423 if (!nft_payload_offload_mask(reg, priv->len, 424 sizeof(struct in6_addr))) 425 return -EOPNOTSUPP; 426 427 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, src, 428 sizeof(struct in6_addr), reg); 429 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS); 430 break; 431 case offsetof(struct ipv6hdr, daddr): 432 if (!nft_payload_offload_mask(reg, priv->len, 433 sizeof(struct in6_addr))) 434 return -EOPNOTSUPP; 435 436 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, dst, 437 sizeof(struct in6_addr), reg); 438 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS); 439 break; 440 case offsetof(struct ipv6hdr, nexthdr): 441 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8))) 442 return -EOPNOTSUPP; 443 444 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto, 445 sizeof(__u8), reg); 446 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT); 447 break; 448 default: 449 return -EOPNOTSUPP; 450 } 451 452 return 0; 453 } 454 nft_payload_offload_nh(struct nft_offload_ctx * ctx,struct nft_flow_rule * flow,const struct nft_payload * priv)455 static int nft_payload_offload_nh(struct nft_offload_ctx *ctx, 456 struct nft_flow_rule *flow, 457 const struct nft_payload *priv) 458 { 459 int err; 460 461 switch (ctx->dep.l3num) { 462 case htons(ETH_P_IP): 463 err = nft_payload_offload_ip(ctx, flow, priv); 464 break; 465 case htons(ETH_P_IPV6): 466 err = nft_payload_offload_ip6(ctx, flow, priv); 467 break; 468 default: 469 return -EOPNOTSUPP; 470 } 471 472 return err; 473 } 474 nft_payload_offload_tcp(struct nft_offload_ctx * ctx,struct nft_flow_rule * flow,const struct nft_payload * priv)475 static int nft_payload_offload_tcp(struct nft_offload_ctx *ctx, 476 struct nft_flow_rule *flow, 477 const struct nft_payload *priv) 478 { 479 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 480 481 switch (priv->offset) { 482 case offsetof(struct tcphdr, source): 483 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 484 return -EOPNOTSUPP; 485 486 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src, 487 sizeof(__be16), reg); 488 break; 489 case offsetof(struct tcphdr, dest): 490 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 491 return -EOPNOTSUPP; 492 493 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst, 494 sizeof(__be16), reg); 495 break; 496 default: 497 return -EOPNOTSUPP; 498 } 499 500 return 0; 501 } 502 nft_payload_offload_udp(struct nft_offload_ctx * ctx,struct nft_flow_rule * flow,const struct nft_payload * priv)503 static int nft_payload_offload_udp(struct nft_offload_ctx *ctx, 504 struct nft_flow_rule *flow, 505 const struct nft_payload *priv) 506 { 507 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 508 509 switch (priv->offset) { 510 case offsetof(struct udphdr, source): 511 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 512 return -EOPNOTSUPP; 513 514 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src, 515 sizeof(__be16), reg); 516 break; 517 case offsetof(struct udphdr, dest): 518 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 519 return -EOPNOTSUPP; 520 521 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst, 522 sizeof(__be16), reg); 523 break; 524 default: 525 return -EOPNOTSUPP; 526 } 527 528 return 0; 529 } 530 nft_payload_offload_th(struct nft_offload_ctx * ctx,struct nft_flow_rule * flow,const struct nft_payload * priv)531 static int nft_payload_offload_th(struct nft_offload_ctx *ctx, 532 struct nft_flow_rule *flow, 533 const struct nft_payload *priv) 534 { 535 int err; 536 537 switch (ctx->dep.protonum) { 538 case IPPROTO_TCP: 539 err = nft_payload_offload_tcp(ctx, flow, priv); 540 break; 541 case IPPROTO_UDP: 542 err = nft_payload_offload_udp(ctx, flow, priv); 543 break; 544 default: 545 return -EOPNOTSUPP; 546 } 547 548 return err; 549 } 550 nft_payload_offload(struct nft_offload_ctx * ctx,struct nft_flow_rule * flow,const struct nft_expr * expr)551 static int nft_payload_offload(struct nft_offload_ctx *ctx, 552 struct nft_flow_rule *flow, 553 const struct nft_expr *expr) 554 { 555 const struct nft_payload *priv = nft_expr_priv(expr); 556 int err; 557 558 switch (priv->base) { 559 case NFT_PAYLOAD_LL_HEADER: 560 err = nft_payload_offload_ll(ctx, flow, priv); 561 break; 562 case NFT_PAYLOAD_NETWORK_HEADER: 563 err = nft_payload_offload_nh(ctx, flow, priv); 564 break; 565 case NFT_PAYLOAD_TRANSPORT_HEADER: 566 err = nft_payload_offload_th(ctx, flow, priv); 567 break; 568 default: 569 err = -EOPNOTSUPP; 570 break; 571 } 572 return err; 573 } 574 575 static const struct nft_expr_ops nft_payload_ops = { 576 .type = &nft_payload_type, 577 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)), 578 .eval = nft_payload_eval, 579 .init = nft_payload_init, 580 .dump = nft_payload_dump, 581 .reduce = nft_payload_reduce, 582 .offload = nft_payload_offload, 583 }; 584 585 const struct nft_expr_ops nft_payload_fast_ops = { 586 .type = &nft_payload_type, 587 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)), 588 .eval = nft_payload_eval, 589 .init = nft_payload_init, 590 .dump = nft_payload_dump, 591 .reduce = nft_payload_reduce, 592 .offload = nft_payload_offload, 593 }; 594 nft_payload_inner_eval(const struct nft_expr * expr,struct nft_regs * regs,const struct nft_pktinfo * pkt,struct nft_inner_tun_ctx * tun_ctx)595 void nft_payload_inner_eval(const struct nft_expr *expr, struct nft_regs *regs, 596 const struct nft_pktinfo *pkt, 597 struct nft_inner_tun_ctx *tun_ctx) 598 { 599 const struct nft_payload *priv = nft_expr_priv(expr); 600 const struct sk_buff *skb = pkt->skb; 601 u32 *dest = ®s->data[priv->dreg]; 602 int offset; 603 604 if (priv->len % NFT_REG32_SIZE) 605 dest[priv->len / NFT_REG32_SIZE] = 0; 606 607 switch (priv->base) { 608 case NFT_PAYLOAD_TUN_HEADER: 609 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_TUN)) 610 goto err; 611 612 offset = tun_ctx->inner_tunoff; 613 break; 614 case NFT_PAYLOAD_LL_HEADER: 615 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_LL)) 616 goto err; 617 618 offset = tun_ctx->inner_lloff; 619 break; 620 case NFT_PAYLOAD_NETWORK_HEADER: 621 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_NH)) 622 goto err; 623 624 offset = tun_ctx->inner_nhoff; 625 break; 626 case NFT_PAYLOAD_TRANSPORT_HEADER: 627 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_TH)) 628 goto err; 629 630 offset = tun_ctx->inner_thoff; 631 break; 632 default: 633 WARN_ON_ONCE(1); 634 goto err; 635 } 636 offset += priv->offset; 637 638 if (skb_copy_bits(skb, offset, dest, priv->len) < 0) 639 goto err; 640 641 return; 642 err: 643 regs->verdict.code = NFT_BREAK; 644 } 645 nft_payload_inner_init(const struct nft_ctx * ctx,const struct nft_expr * expr,const struct nlattr * const tb[])646 static int nft_payload_inner_init(const struct nft_ctx *ctx, 647 const struct nft_expr *expr, 648 const struct nlattr * const tb[]) 649 { 650 struct nft_payload *priv = nft_expr_priv(expr); 651 u32 base; 652 653 if (!tb[NFTA_PAYLOAD_BASE] || !tb[NFTA_PAYLOAD_OFFSET] || 654 !tb[NFTA_PAYLOAD_LEN] || !tb[NFTA_PAYLOAD_DREG]) 655 return -EINVAL; 656 657 base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); 658 switch (base) { 659 case NFT_PAYLOAD_TUN_HEADER: 660 case NFT_PAYLOAD_LL_HEADER: 661 case NFT_PAYLOAD_NETWORK_HEADER: 662 case NFT_PAYLOAD_TRANSPORT_HEADER: 663 break; 664 default: 665 return -EOPNOTSUPP; 666 } 667 668 priv->base = base; 669 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET])); 670 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); 671 672 return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG], 673 &priv->dreg, NULL, NFT_DATA_VALUE, 674 priv->len); 675 } 676 677 static const struct nft_expr_ops nft_payload_inner_ops = { 678 .type = &nft_payload_type, 679 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)), 680 .init = nft_payload_inner_init, 681 .dump = nft_payload_dump, 682 /* direct call to nft_payload_inner_eval(). */ 683 }; 684 nft_csum_replace(__sum16 * sum,__wsum fsum,__wsum tsum)685 static inline void nft_csum_replace(__sum16 *sum, __wsum fsum, __wsum tsum) 686 { 687 *sum = csum_fold(csum_add(csum_sub(~csum_unfold(*sum), fsum), tsum)); 688 if (*sum == 0) 689 *sum = CSUM_MANGLED_0; 690 } 691 nft_payload_udp_checksum(struct sk_buff * skb,unsigned int thoff)692 static bool nft_payload_udp_checksum(struct sk_buff *skb, unsigned int thoff) 693 { 694 struct udphdr *uh, _uh; 695 696 uh = skb_header_pointer(skb, thoff, sizeof(_uh), &_uh); 697 if (!uh) 698 return false; 699 700 return (__force bool)uh->check; 701 } 702 nft_payload_l4csum_offset(const struct nft_pktinfo * pkt,struct sk_buff * skb,unsigned int * l4csum_offset)703 static int nft_payload_l4csum_offset(const struct nft_pktinfo *pkt, 704 struct sk_buff *skb, 705 unsigned int *l4csum_offset) 706 { 707 if (pkt->fragoff) 708 return -1; 709 710 switch (pkt->tprot) { 711 case IPPROTO_TCP: 712 *l4csum_offset = offsetof(struct tcphdr, check); 713 break; 714 case IPPROTO_UDP: 715 if (!nft_payload_udp_checksum(skb, nft_thoff(pkt))) 716 return -1; 717 fallthrough; 718 case IPPROTO_UDPLITE: 719 *l4csum_offset = offsetof(struct udphdr, check); 720 break; 721 case IPPROTO_ICMPV6: 722 *l4csum_offset = offsetof(struct icmp6hdr, icmp6_cksum); 723 break; 724 default: 725 return -1; 726 } 727 728 *l4csum_offset += nft_thoff(pkt); 729 return 0; 730 } 731 nft_payload_csum_sctp(struct sk_buff * skb,int offset)732 static int nft_payload_csum_sctp(struct sk_buff *skb, int offset) 733 { 734 struct sctphdr *sh; 735 736 if (skb_ensure_writable(skb, offset + sizeof(*sh))) 737 return -1; 738 739 sh = (struct sctphdr *)(skb->data + offset); 740 sh->checksum = sctp_compute_cksum(skb, offset); 741 skb->ip_summed = CHECKSUM_UNNECESSARY; 742 return 0; 743 } 744 nft_payload_l4csum_update(const struct nft_pktinfo * pkt,struct sk_buff * skb,__wsum fsum,__wsum tsum)745 static int nft_payload_l4csum_update(const struct nft_pktinfo *pkt, 746 struct sk_buff *skb, 747 __wsum fsum, __wsum tsum) 748 { 749 int l4csum_offset; 750 __sum16 sum; 751 752 /* If we cannot determine layer 4 checksum offset or this packet doesn't 753 * require layer 4 checksum recalculation, skip this packet. 754 */ 755 if (nft_payload_l4csum_offset(pkt, skb, &l4csum_offset) < 0) 756 return 0; 757 758 if (skb_copy_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0) 759 return -1; 760 761 /* Checksum mangling for an arbitrary amount of bytes, based on 762 * inet_proto_csum_replace*() functions. 763 */ 764 if (skb->ip_summed != CHECKSUM_PARTIAL) { 765 nft_csum_replace(&sum, fsum, tsum); 766 if (skb->ip_summed == CHECKSUM_COMPLETE) { 767 skb->csum = ~csum_add(csum_sub(~(skb->csum), fsum), 768 tsum); 769 } 770 } else { 771 sum = ~csum_fold(csum_add(csum_sub(csum_unfold(sum), fsum), 772 tsum)); 773 } 774 775 if (skb_ensure_writable(skb, l4csum_offset + sizeof(sum)) || 776 skb_store_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0) 777 return -1; 778 779 return 0; 780 } 781 nft_payload_csum_inet(struct sk_buff * skb,const u32 * src,__wsum fsum,__wsum tsum,int csum_offset)782 static int nft_payload_csum_inet(struct sk_buff *skb, const u32 *src, 783 __wsum fsum, __wsum tsum, int csum_offset) 784 { 785 __sum16 sum; 786 787 if (skb_copy_bits(skb, csum_offset, &sum, sizeof(sum)) < 0) 788 return -1; 789 790 nft_csum_replace(&sum, fsum, tsum); 791 if (skb_ensure_writable(skb, csum_offset + sizeof(sum)) || 792 skb_store_bits(skb, csum_offset, &sum, sizeof(sum)) < 0) 793 return -1; 794 795 return 0; 796 } 797 798 struct nft_payload_set { 799 enum nft_payload_bases base:8; 800 u8 offset; 801 u8 len; 802 u8 sreg; 803 u8 csum_type; 804 u8 csum_offset; 805 u8 csum_flags; 806 }; 807 808 /* This is not struct vlan_hdr. */ 809 struct nft_payload_vlan_hdr { 810 __be16 h_vlan_proto; 811 __be16 h_vlan_TCI; 812 }; 813 814 static bool nft_payload_set_vlan(const u32 * src,struct sk_buff * skb,u8 offset,u8 len,int * vlan_hlen)815 nft_payload_set_vlan(const u32 *src, struct sk_buff *skb, u8 offset, u8 len, 816 int *vlan_hlen) 817 { 818 struct nft_payload_vlan_hdr *vlanh; 819 __be16 vlan_proto; 820 u16 vlan_tci; 821 822 if (offset >= offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto)) { 823 *vlan_hlen = VLAN_HLEN; 824 return true; 825 } 826 827 switch (offset) { 828 case offsetof(struct vlan_ethhdr, h_vlan_proto): 829 if (len == 2) { 830 vlan_proto = nft_reg_load_be16(src); 831 skb->vlan_proto = vlan_proto; 832 } else if (len == 4) { 833 vlanh = (struct nft_payload_vlan_hdr *)src; 834 __vlan_hwaccel_put_tag(skb, vlanh->h_vlan_proto, 835 ntohs(vlanh->h_vlan_TCI)); 836 } else { 837 return false; 838 } 839 break; 840 case offsetof(struct vlan_ethhdr, h_vlan_TCI): 841 if (len != 2) 842 return false; 843 844 vlan_tci = ntohs(nft_reg_load_be16(src)); 845 skb->vlan_tci = vlan_tci; 846 break; 847 default: 848 return false; 849 } 850 851 return true; 852 } 853 nft_payload_set_eval(const struct nft_expr * expr,struct nft_regs * regs,const struct nft_pktinfo * pkt)854 static void nft_payload_set_eval(const struct nft_expr *expr, 855 struct nft_regs *regs, 856 const struct nft_pktinfo *pkt) 857 { 858 const struct nft_payload_set *priv = nft_expr_priv(expr); 859 const u32 *src = ®s->data[priv->sreg]; 860 int offset, csum_offset, vlan_hlen = 0; 861 struct sk_buff *skb = pkt->skb; 862 __wsum fsum, tsum; 863 864 switch (priv->base) { 865 case NFT_PAYLOAD_LL_HEADER: 866 if (!skb_mac_header_was_set(skb)) 867 goto err; 868 869 if (skb_vlan_tag_present(skb) && 870 nft_payload_need_vlan_adjust(priv->offset, priv->len)) { 871 if (!nft_payload_set_vlan(src, skb, 872 priv->offset, priv->len, 873 &vlan_hlen)) 874 goto err; 875 876 if (!vlan_hlen) 877 return; 878 } 879 880 offset = skb_mac_header(skb) - skb->data - vlan_hlen; 881 break; 882 case NFT_PAYLOAD_NETWORK_HEADER: 883 offset = skb_network_offset(skb); 884 break; 885 case NFT_PAYLOAD_TRANSPORT_HEADER: 886 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff) 887 goto err; 888 offset = nft_thoff(pkt); 889 break; 890 case NFT_PAYLOAD_INNER_HEADER: 891 offset = nft_payload_inner_offset(pkt); 892 if (offset < 0) 893 goto err; 894 break; 895 default: 896 WARN_ON_ONCE(1); 897 goto err; 898 } 899 900 csum_offset = offset + priv->csum_offset; 901 offset += priv->offset; 902 903 if ((priv->csum_type == NFT_PAYLOAD_CSUM_INET || priv->csum_flags) && 904 ((priv->base != NFT_PAYLOAD_TRANSPORT_HEADER && 905 priv->base != NFT_PAYLOAD_INNER_HEADER) || 906 skb->ip_summed != CHECKSUM_PARTIAL)) { 907 if (offset + priv->len > skb->len) 908 goto err; 909 910 fsum = skb_checksum(skb, offset, priv->len, 0); 911 tsum = csum_partial(src, priv->len, 0); 912 913 if (priv->csum_type == NFT_PAYLOAD_CSUM_INET && 914 nft_payload_csum_inet(skb, src, fsum, tsum, csum_offset)) 915 goto err; 916 917 if (priv->csum_flags && 918 nft_payload_l4csum_update(pkt, skb, fsum, tsum) < 0) 919 goto err; 920 } 921 922 if (skb_ensure_writable(skb, max(offset + priv->len, 0)) || 923 skb_store_bits(skb, offset, src, priv->len) < 0) 924 goto err; 925 926 if (priv->csum_type == NFT_PAYLOAD_CSUM_SCTP && 927 pkt->tprot == IPPROTO_SCTP && 928 skb->ip_summed != CHECKSUM_PARTIAL) { 929 if (pkt->fragoff == 0 && 930 nft_payload_csum_sctp(skb, nft_thoff(pkt))) 931 goto err; 932 } 933 934 return; 935 err: 936 regs->verdict.code = NFT_BREAK; 937 } 938 nft_payload_set_init(const struct nft_ctx * ctx,const struct nft_expr * expr,const struct nlattr * const tb[])939 static int nft_payload_set_init(const struct nft_ctx *ctx, 940 const struct nft_expr *expr, 941 const struct nlattr * const tb[]) 942 { 943 struct nft_payload_set *priv = nft_expr_priv(expr); 944 u32 csum_offset, csum_type = NFT_PAYLOAD_CSUM_NONE; 945 int err; 946 947 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); 948 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET])); 949 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); 950 951 if (tb[NFTA_PAYLOAD_CSUM_TYPE]) 952 csum_type = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_TYPE])); 953 if (tb[NFTA_PAYLOAD_CSUM_OFFSET]) { 954 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_CSUM_OFFSET], U8_MAX, 955 &csum_offset); 956 if (err < 0) 957 return err; 958 959 priv->csum_offset = csum_offset; 960 } 961 if (tb[NFTA_PAYLOAD_CSUM_FLAGS]) { 962 u32 flags; 963 964 flags = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_FLAGS])); 965 if (flags & ~NFT_PAYLOAD_L4CSUM_PSEUDOHDR) 966 return -EINVAL; 967 968 priv->csum_flags = flags; 969 } 970 971 switch (csum_type) { 972 case NFT_PAYLOAD_CSUM_NONE: 973 case NFT_PAYLOAD_CSUM_INET: 974 break; 975 case NFT_PAYLOAD_CSUM_SCTP: 976 if (priv->base != NFT_PAYLOAD_TRANSPORT_HEADER) 977 return -EINVAL; 978 979 if (priv->csum_offset != offsetof(struct sctphdr, checksum)) 980 return -EINVAL; 981 break; 982 default: 983 return -EOPNOTSUPP; 984 } 985 priv->csum_type = csum_type; 986 987 return nft_parse_register_load(ctx, tb[NFTA_PAYLOAD_SREG], &priv->sreg, 988 priv->len); 989 } 990 nft_payload_set_dump(struct sk_buff * skb,const struct nft_expr * expr,bool reset)991 static int nft_payload_set_dump(struct sk_buff *skb, 992 const struct nft_expr *expr, bool reset) 993 { 994 const struct nft_payload_set *priv = nft_expr_priv(expr); 995 996 if (nft_dump_register(skb, NFTA_PAYLOAD_SREG, priv->sreg) || 997 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) || 998 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) || 999 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)) || 1000 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_TYPE, htonl(priv->csum_type)) || 1001 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_OFFSET, 1002 htonl(priv->csum_offset)) || 1003 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_FLAGS, htonl(priv->csum_flags))) 1004 goto nla_put_failure; 1005 return 0; 1006 1007 nla_put_failure: 1008 return -1; 1009 } 1010 nft_payload_set_reduce(struct nft_regs_track * track,const struct nft_expr * expr)1011 static bool nft_payload_set_reduce(struct nft_regs_track *track, 1012 const struct nft_expr *expr) 1013 { 1014 int i; 1015 1016 for (i = 0; i < NFT_REG32_NUM; i++) { 1017 if (!track->regs[i].selector) 1018 continue; 1019 1020 if (track->regs[i].selector->ops != &nft_payload_ops && 1021 track->regs[i].selector->ops != &nft_payload_fast_ops) 1022 continue; 1023 1024 __nft_reg_track_cancel(track, i); 1025 } 1026 1027 return false; 1028 } 1029 1030 static const struct nft_expr_ops nft_payload_set_ops = { 1031 .type = &nft_payload_type, 1032 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload_set)), 1033 .eval = nft_payload_set_eval, 1034 .init = nft_payload_set_init, 1035 .dump = nft_payload_set_dump, 1036 .reduce = nft_payload_set_reduce, 1037 }; 1038 1039 static const struct nft_expr_ops * nft_payload_select_ops(const struct nft_ctx * ctx,const struct nlattr * const tb[])1040 nft_payload_select_ops(const struct nft_ctx *ctx, 1041 const struct nlattr * const tb[]) 1042 { 1043 enum nft_payload_bases base; 1044 unsigned int offset, len; 1045 int err; 1046 1047 if (tb[NFTA_PAYLOAD_BASE] == NULL || 1048 tb[NFTA_PAYLOAD_OFFSET] == NULL || 1049 tb[NFTA_PAYLOAD_LEN] == NULL) 1050 return ERR_PTR(-EINVAL); 1051 1052 base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); 1053 switch (base) { 1054 case NFT_PAYLOAD_LL_HEADER: 1055 case NFT_PAYLOAD_NETWORK_HEADER: 1056 case NFT_PAYLOAD_TRANSPORT_HEADER: 1057 case NFT_PAYLOAD_INNER_HEADER: 1058 break; 1059 default: 1060 return ERR_PTR(-EOPNOTSUPP); 1061 } 1062 1063 if (tb[NFTA_PAYLOAD_SREG] != NULL) { 1064 if (tb[NFTA_PAYLOAD_DREG] != NULL) 1065 return ERR_PTR(-EINVAL); 1066 return &nft_payload_set_ops; 1067 } 1068 1069 if (tb[NFTA_PAYLOAD_DREG] == NULL) 1070 return ERR_PTR(-EINVAL); 1071 1072 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U8_MAX, &offset); 1073 if (err < 0) 1074 return ERR_PTR(err); 1075 1076 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_LEN], U8_MAX, &len); 1077 if (err < 0) 1078 return ERR_PTR(err); 1079 1080 if (len <= 4 && is_power_of_2(len) && IS_ALIGNED(offset, len) && 1081 base != NFT_PAYLOAD_LL_HEADER && base != NFT_PAYLOAD_INNER_HEADER) 1082 return &nft_payload_fast_ops; 1083 else 1084 return &nft_payload_ops; 1085 } 1086 1087 struct nft_expr_type nft_payload_type __read_mostly = { 1088 .name = "payload", 1089 .select_ops = nft_payload_select_ops, 1090 .inner_ops = &nft_payload_inner_ops, 1091 .policy = nft_payload_policy, 1092 .maxattr = NFTA_PAYLOAD_MAX, 1093 .owner = THIS_MODULE, 1094 }; 1095