1 /* 2 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/string.h> 35 #include <linux/etherdevice.h> 36 37 #include <linux/mlx4/cmd.h> 38 #include <linux/mlx4/qp.h> 39 #include <linux/export.h> 40 41 #include "mlx4.h" 42 mlx4_get_mgm_entry_size(struct mlx4_dev * dev)43 int mlx4_get_mgm_entry_size(struct mlx4_dev *dev) 44 { 45 return 1 << dev->oper_log_mgm_entry_size; 46 } 47 mlx4_get_qp_per_mgm(struct mlx4_dev * dev)48 int mlx4_get_qp_per_mgm(struct mlx4_dev *dev) 49 { 50 return 4 * (mlx4_get_mgm_entry_size(dev) / 16 - 2); 51 } 52 mlx4_QP_FLOW_STEERING_ATTACH(struct mlx4_dev * dev,struct mlx4_cmd_mailbox * mailbox,u32 size,u64 * reg_id)53 static int mlx4_QP_FLOW_STEERING_ATTACH(struct mlx4_dev *dev, 54 struct mlx4_cmd_mailbox *mailbox, 55 u32 size, 56 u64 *reg_id) 57 { 58 u64 imm; 59 int err = 0; 60 61 err = mlx4_cmd_imm(dev, mailbox->dma, &imm, size, 0, 62 MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A, 63 MLX4_CMD_NATIVE); 64 if (err) 65 return err; 66 *reg_id = imm; 67 68 return err; 69 } 70 mlx4_QP_FLOW_STEERING_DETACH(struct mlx4_dev * dev,u64 regid)71 static int mlx4_QP_FLOW_STEERING_DETACH(struct mlx4_dev *dev, u64 regid) 72 { 73 int err = 0; 74 75 err = mlx4_cmd(dev, regid, 0, 0, 76 MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A, 77 MLX4_CMD_NATIVE); 78 79 return err; 80 } 81 mlx4_READ_ENTRY(struct mlx4_dev * dev,int index,struct mlx4_cmd_mailbox * mailbox)82 static int mlx4_READ_ENTRY(struct mlx4_dev *dev, int index, 83 struct mlx4_cmd_mailbox *mailbox) 84 { 85 return mlx4_cmd_box(dev, 0, mailbox->dma, index, 0, MLX4_CMD_READ_MCG, 86 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); 87 } 88 mlx4_WRITE_ENTRY(struct mlx4_dev * dev,int index,struct mlx4_cmd_mailbox * mailbox)89 static int mlx4_WRITE_ENTRY(struct mlx4_dev *dev, int index, 90 struct mlx4_cmd_mailbox *mailbox) 91 { 92 return mlx4_cmd(dev, mailbox->dma, index, 0, MLX4_CMD_WRITE_MCG, 93 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); 94 } 95 mlx4_WRITE_PROMISC(struct mlx4_dev * dev,u8 port,u8 steer,struct mlx4_cmd_mailbox * mailbox)96 static int mlx4_WRITE_PROMISC(struct mlx4_dev *dev, u8 port, u8 steer, 97 struct mlx4_cmd_mailbox *mailbox) 98 { 99 u32 in_mod; 100 101 in_mod = (u32) port << 16 | steer << 1; 102 return mlx4_cmd(dev, mailbox->dma, in_mod, 0x1, 103 MLX4_CMD_WRITE_MCG, MLX4_CMD_TIME_CLASS_A, 104 MLX4_CMD_NATIVE); 105 } 106 mlx4_GID_HASH(struct mlx4_dev * dev,struct mlx4_cmd_mailbox * mailbox,u16 * hash,u8 op_mod)107 static int mlx4_GID_HASH(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 108 u16 *hash, u8 op_mod) 109 { 110 u64 imm; 111 int err; 112 113 err = mlx4_cmd_imm(dev, mailbox->dma, &imm, 0, op_mod, 114 MLX4_CMD_MGID_HASH, MLX4_CMD_TIME_CLASS_A, 115 MLX4_CMD_NATIVE); 116 117 if (!err) 118 *hash = imm; 119 120 return err; 121 } 122 get_promisc_qp(struct mlx4_dev * dev,u8 port,enum mlx4_steer_type steer,u32 qpn)123 static struct mlx4_promisc_qp *get_promisc_qp(struct mlx4_dev *dev, u8 port, 124 enum mlx4_steer_type steer, 125 u32 qpn) 126 { 127 struct mlx4_steer *s_steer; 128 struct mlx4_promisc_qp *pqp; 129 130 if (port < 1 || port > dev->caps.num_ports) 131 return NULL; 132 133 s_steer = &mlx4_priv(dev)->steer[port - 1]; 134 135 list_for_each_entry(pqp, &s_steer->promisc_qps[steer], list) { 136 if (pqp->qpn == qpn) 137 return pqp; 138 } 139 /* not found */ 140 return NULL; 141 } 142 143 /* 144 * Add new entry to steering data structure. 145 * All promisc QPs should be added as well 146 */ new_steering_entry(struct mlx4_dev * dev,u8 port,enum mlx4_steer_type steer,unsigned int index,u32 qpn)147 static int new_steering_entry(struct mlx4_dev *dev, u8 port, 148 enum mlx4_steer_type steer, 149 unsigned int index, u32 qpn) 150 { 151 struct mlx4_steer *s_steer; 152 struct mlx4_cmd_mailbox *mailbox; 153 struct mlx4_mgm *mgm; 154 u32 members_count; 155 struct mlx4_steer_index *new_entry; 156 struct mlx4_promisc_qp *pqp; 157 struct mlx4_promisc_qp *dqp = NULL; 158 u32 prot; 159 int err; 160 161 if (port < 1 || port > dev->caps.num_ports) 162 return -EINVAL; 163 164 s_steer = &mlx4_priv(dev)->steer[port - 1]; 165 new_entry = kzalloc(sizeof(*new_entry), GFP_KERNEL); 166 if (!new_entry) 167 return -ENOMEM; 168 169 INIT_LIST_HEAD(&new_entry->duplicates); 170 new_entry->index = index; 171 list_add_tail(&new_entry->list, &s_steer->steer_entries[steer]); 172 173 /* If the given qpn is also a promisc qp, 174 * it should be inserted to duplicates list 175 */ 176 pqp = get_promisc_qp(dev, port, steer, qpn); 177 if (pqp) { 178 dqp = kmalloc(sizeof(*dqp), GFP_KERNEL); 179 if (!dqp) { 180 err = -ENOMEM; 181 goto out_alloc; 182 } 183 dqp->qpn = qpn; 184 list_add_tail(&dqp->list, &new_entry->duplicates); 185 } 186 187 /* if no promisc qps for this vep, we are done */ 188 if (list_empty(&s_steer->promisc_qps[steer])) 189 return 0; 190 191 /* now need to add all the promisc qps to the new 192 * steering entry, as they should also receive the packets 193 * destined to this address */ 194 mailbox = mlx4_alloc_cmd_mailbox(dev); 195 if (IS_ERR(mailbox)) { 196 err = -ENOMEM; 197 goto out_alloc; 198 } 199 mgm = mailbox->buf; 200 201 err = mlx4_READ_ENTRY(dev, index, mailbox); 202 if (err) 203 goto out_mailbox; 204 205 members_count = be32_to_cpu(mgm->members_count) & 0xffffff; 206 prot = be32_to_cpu(mgm->members_count) >> 30; 207 list_for_each_entry(pqp, &s_steer->promisc_qps[steer], list) { 208 /* don't add already existing qpn */ 209 if (pqp->qpn == qpn) 210 continue; 211 if (members_count == dev->caps.num_qp_per_mgm) { 212 /* out of space */ 213 err = -ENOMEM; 214 goto out_mailbox; 215 } 216 217 /* add the qpn */ 218 mgm->qp[members_count++] = cpu_to_be32(pqp->qpn & MGM_QPN_MASK); 219 } 220 /* update the qps count and update the entry with all the promisc qps*/ 221 mgm->members_count = cpu_to_be32(members_count | (prot << 30)); 222 err = mlx4_WRITE_ENTRY(dev, index, mailbox); 223 224 out_mailbox: 225 mlx4_free_cmd_mailbox(dev, mailbox); 226 if (!err) 227 return 0; 228 out_alloc: 229 if (dqp) { 230 list_del(&dqp->list); 231 kfree(dqp); 232 } 233 list_del(&new_entry->list); 234 kfree(new_entry); 235 return err; 236 } 237 238 /* update the data structures with existing steering entry */ existing_steering_entry(struct mlx4_dev * dev,u8 port,enum mlx4_steer_type steer,unsigned int index,u32 qpn)239 static int existing_steering_entry(struct mlx4_dev *dev, u8 port, 240 enum mlx4_steer_type steer, 241 unsigned int index, u32 qpn) 242 { 243 struct mlx4_steer *s_steer; 244 struct mlx4_steer_index *tmp_entry, *entry = NULL; 245 struct mlx4_promisc_qp *pqp; 246 struct mlx4_promisc_qp *dqp; 247 248 if (port < 1 || port > dev->caps.num_ports) 249 return -EINVAL; 250 251 s_steer = &mlx4_priv(dev)->steer[port - 1]; 252 253 pqp = get_promisc_qp(dev, port, steer, qpn); 254 if (!pqp) 255 return 0; /* nothing to do */ 256 257 list_for_each_entry(tmp_entry, &s_steer->steer_entries[steer], list) { 258 if (tmp_entry->index == index) { 259 entry = tmp_entry; 260 break; 261 } 262 } 263 if (unlikely(!entry)) { 264 mlx4_warn(dev, "Steering entry at index %x is not registered\n", index); 265 return -EINVAL; 266 } 267 268 /* the given qpn is listed as a promisc qpn 269 * we need to add it as a duplicate to this entry 270 * for future references */ 271 list_for_each_entry(dqp, &entry->duplicates, list) { 272 if (qpn == dqp->qpn) 273 return 0; /* qp is already duplicated */ 274 } 275 276 /* add the qp as a duplicate on this index */ 277 dqp = kmalloc(sizeof(*dqp), GFP_KERNEL); 278 if (!dqp) 279 return -ENOMEM; 280 dqp->qpn = qpn; 281 list_add_tail(&dqp->list, &entry->duplicates); 282 283 return 0; 284 } 285 286 /* Check whether a qpn is a duplicate on steering entry 287 * If so, it should not be removed from mgm */ check_duplicate_entry(struct mlx4_dev * dev,u8 port,enum mlx4_steer_type steer,unsigned int index,u32 qpn)288 static bool check_duplicate_entry(struct mlx4_dev *dev, u8 port, 289 enum mlx4_steer_type steer, 290 unsigned int index, u32 qpn) 291 { 292 struct mlx4_steer *s_steer; 293 struct mlx4_steer_index *tmp_entry, *entry = NULL; 294 struct mlx4_promisc_qp *dqp, *tmp_dqp; 295 296 if (port < 1 || port > dev->caps.num_ports) 297 return false; 298 299 s_steer = &mlx4_priv(dev)->steer[port - 1]; 300 301 /* if qp is not promisc, it cannot be duplicated */ 302 if (!get_promisc_qp(dev, port, steer, qpn)) 303 return false; 304 305 /* The qp is promisc qp so it is a duplicate on this index 306 * Find the index entry, and remove the duplicate */ 307 list_for_each_entry(tmp_entry, &s_steer->steer_entries[steer], list) { 308 if (tmp_entry->index == index) { 309 entry = tmp_entry; 310 break; 311 } 312 } 313 if (unlikely(!entry)) { 314 mlx4_warn(dev, "Steering entry for index %x is not registered\n", index); 315 return false; 316 } 317 list_for_each_entry_safe(dqp, tmp_dqp, &entry->duplicates, list) { 318 if (dqp->qpn == qpn) { 319 list_del(&dqp->list); 320 kfree(dqp); 321 } 322 } 323 return true; 324 } 325 326 /* Returns true if all the QPs != tqpn contained in this entry 327 * are Promisc QPs. Returns false otherwise. 328 */ promisc_steering_entry(struct mlx4_dev * dev,u8 port,enum mlx4_steer_type steer,unsigned int index,u32 tqpn,u32 * members_count)329 static bool promisc_steering_entry(struct mlx4_dev *dev, u8 port, 330 enum mlx4_steer_type steer, 331 unsigned int index, u32 tqpn, 332 u32 *members_count) 333 { 334 struct mlx4_cmd_mailbox *mailbox; 335 struct mlx4_mgm *mgm; 336 u32 m_count; 337 bool ret = false; 338 int i; 339 340 if (port < 1 || port > dev->caps.num_ports) 341 return false; 342 343 mailbox = mlx4_alloc_cmd_mailbox(dev); 344 if (IS_ERR(mailbox)) 345 return false; 346 mgm = mailbox->buf; 347 348 if (mlx4_READ_ENTRY(dev, index, mailbox)) 349 goto out; 350 m_count = be32_to_cpu(mgm->members_count) & 0xffffff; 351 if (members_count) 352 *members_count = m_count; 353 354 for (i = 0; i < m_count; i++) { 355 u32 qpn = be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK; 356 if (!get_promisc_qp(dev, port, steer, qpn) && qpn != tqpn) { 357 /* the qp is not promisc, the entry can't be removed */ 358 goto out; 359 } 360 } 361 ret = true; 362 out: 363 mlx4_free_cmd_mailbox(dev, mailbox); 364 return ret; 365 } 366 367 /* IF a steering entry contains only promisc QPs, it can be removed. */ can_remove_steering_entry(struct mlx4_dev * dev,u8 port,enum mlx4_steer_type steer,unsigned int index,u32 tqpn)368 static bool can_remove_steering_entry(struct mlx4_dev *dev, u8 port, 369 enum mlx4_steer_type steer, 370 unsigned int index, u32 tqpn) 371 { 372 struct mlx4_steer *s_steer; 373 struct mlx4_steer_index *entry = NULL, *tmp_entry; 374 u32 members_count; 375 bool ret = false; 376 377 if (port < 1 || port > dev->caps.num_ports) 378 return false; 379 380 s_steer = &mlx4_priv(dev)->steer[port - 1]; 381 382 if (!promisc_steering_entry(dev, port, steer, index, 383 tqpn, &members_count)) 384 goto out; 385 386 /* All the qps currently registered for this entry are promiscuous, 387 * Checking for duplicates */ 388 ret = true; 389 list_for_each_entry_safe(entry, tmp_entry, &s_steer->steer_entries[steer], list) { 390 if (entry->index == index) { 391 if (list_empty(&entry->duplicates) || 392 members_count == 1) { 393 struct mlx4_promisc_qp *pqp, *tmp_pqp; 394 /* If there is only 1 entry in duplicates then 395 * this is the QP we want to delete, going over 396 * the list and deleting the entry. 397 */ 398 list_del(&entry->list); 399 list_for_each_entry_safe(pqp, tmp_pqp, 400 &entry->duplicates, 401 list) { 402 list_del(&pqp->list); 403 kfree(pqp); 404 } 405 kfree(entry); 406 } else { 407 /* This entry contains duplicates so it shouldn't be removed */ 408 ret = false; 409 goto out; 410 } 411 } 412 } 413 414 out: 415 return ret; 416 } 417 add_promisc_qp(struct mlx4_dev * dev,u8 port,enum mlx4_steer_type steer,u32 qpn)418 static int add_promisc_qp(struct mlx4_dev *dev, u8 port, 419 enum mlx4_steer_type steer, u32 qpn) 420 { 421 struct mlx4_steer *s_steer; 422 struct mlx4_cmd_mailbox *mailbox; 423 struct mlx4_mgm *mgm; 424 struct mlx4_steer_index *entry; 425 struct mlx4_promisc_qp *pqp; 426 struct mlx4_promisc_qp *dqp; 427 u32 members_count; 428 u32 prot; 429 int i; 430 bool found; 431 int err; 432 struct mlx4_priv *priv = mlx4_priv(dev); 433 434 if (port < 1 || port > dev->caps.num_ports) 435 return -EINVAL; 436 437 s_steer = &mlx4_priv(dev)->steer[port - 1]; 438 439 mutex_lock(&priv->mcg_table.mutex); 440 441 if (get_promisc_qp(dev, port, steer, qpn)) { 442 err = 0; /* Noting to do, already exists */ 443 goto out_mutex; 444 } 445 446 pqp = kmalloc(sizeof(*pqp), GFP_KERNEL); 447 if (!pqp) { 448 err = -ENOMEM; 449 goto out_mutex; 450 } 451 pqp->qpn = qpn; 452 453 mailbox = mlx4_alloc_cmd_mailbox(dev); 454 if (IS_ERR(mailbox)) { 455 err = -ENOMEM; 456 goto out_alloc; 457 } 458 mgm = mailbox->buf; 459 460 if (!(mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER)) { 461 /* The promisc QP needs to be added for each one of the steering 462 * entries. If it already exists, needs to be added as 463 * a duplicate for this entry. 464 */ 465 list_for_each_entry(entry, 466 &s_steer->steer_entries[steer], 467 list) { 468 err = mlx4_READ_ENTRY(dev, entry->index, mailbox); 469 if (err) 470 goto out_mailbox; 471 472 members_count = be32_to_cpu(mgm->members_count) & 473 0xffffff; 474 prot = be32_to_cpu(mgm->members_count) >> 30; 475 found = false; 476 for (i = 0; i < members_count; i++) { 477 if ((be32_to_cpu(mgm->qp[i]) & 478 MGM_QPN_MASK) == qpn) { 479 /* Entry already exists. 480 * Add to duplicates. 481 */ 482 dqp = kmalloc(sizeof(*dqp), GFP_KERNEL); 483 if (!dqp) { 484 err = -ENOMEM; 485 goto out_mailbox; 486 } 487 dqp->qpn = qpn; 488 list_add_tail(&dqp->list, 489 &entry->duplicates); 490 found = true; 491 } 492 } 493 if (!found) { 494 /* Need to add the qpn to mgm */ 495 if (members_count == 496 dev->caps.num_qp_per_mgm) { 497 /* entry is full */ 498 err = -ENOMEM; 499 goto out_mailbox; 500 } 501 mgm->qp[members_count++] = 502 cpu_to_be32(qpn & MGM_QPN_MASK); 503 mgm->members_count = 504 cpu_to_be32(members_count | 505 (prot << 30)); 506 err = mlx4_WRITE_ENTRY(dev, entry->index, 507 mailbox); 508 if (err) 509 goto out_mailbox; 510 } 511 } 512 } 513 514 /* add the new qpn to list of promisc qps */ 515 list_add_tail(&pqp->list, &s_steer->promisc_qps[steer]); 516 /* now need to add all the promisc qps to default entry */ 517 memset(mgm, 0, sizeof(*mgm)); 518 members_count = 0; 519 list_for_each_entry(dqp, &s_steer->promisc_qps[steer], list) { 520 if (members_count == dev->caps.num_qp_per_mgm) { 521 /* entry is full */ 522 err = -ENOMEM; 523 goto out_list; 524 } 525 mgm->qp[members_count++] = cpu_to_be32(dqp->qpn & MGM_QPN_MASK); 526 } 527 mgm->members_count = cpu_to_be32(members_count | MLX4_PROT_ETH << 30); 528 529 err = mlx4_WRITE_PROMISC(dev, port, steer, mailbox); 530 if (err) 531 goto out_list; 532 533 mlx4_free_cmd_mailbox(dev, mailbox); 534 mutex_unlock(&priv->mcg_table.mutex); 535 return 0; 536 537 out_list: 538 list_del(&pqp->list); 539 out_mailbox: 540 mlx4_free_cmd_mailbox(dev, mailbox); 541 out_alloc: 542 kfree(pqp); 543 out_mutex: 544 mutex_unlock(&priv->mcg_table.mutex); 545 return err; 546 } 547 remove_promisc_qp(struct mlx4_dev * dev,u8 port,enum mlx4_steer_type steer,u32 qpn)548 static int remove_promisc_qp(struct mlx4_dev *dev, u8 port, 549 enum mlx4_steer_type steer, u32 qpn) 550 { 551 struct mlx4_priv *priv = mlx4_priv(dev); 552 struct mlx4_steer *s_steer; 553 struct mlx4_cmd_mailbox *mailbox; 554 struct mlx4_mgm *mgm; 555 struct mlx4_steer_index *entry, *tmp_entry; 556 struct mlx4_promisc_qp *pqp; 557 struct mlx4_promisc_qp *dqp; 558 u32 members_count; 559 bool found; 560 bool back_to_list = false; 561 int i; 562 int err; 563 564 if (port < 1 || port > dev->caps.num_ports) 565 return -EINVAL; 566 567 s_steer = &mlx4_priv(dev)->steer[port - 1]; 568 mutex_lock(&priv->mcg_table.mutex); 569 570 pqp = get_promisc_qp(dev, port, steer, qpn); 571 if (unlikely(!pqp)) { 572 mlx4_warn(dev, "QP %x is not promiscuous QP\n", qpn); 573 /* nothing to do */ 574 err = 0; 575 goto out_mutex; 576 } 577 578 /*remove from list of promisc qps */ 579 list_del(&pqp->list); 580 581 /* set the default entry not to include the removed one */ 582 mailbox = mlx4_alloc_cmd_mailbox(dev); 583 if (IS_ERR(mailbox)) { 584 err = -ENOMEM; 585 back_to_list = true; 586 goto out_list; 587 } 588 mgm = mailbox->buf; 589 members_count = 0; 590 list_for_each_entry(dqp, &s_steer->promisc_qps[steer], list) 591 mgm->qp[members_count++] = cpu_to_be32(dqp->qpn & MGM_QPN_MASK); 592 mgm->members_count = cpu_to_be32(members_count | MLX4_PROT_ETH << 30); 593 594 err = mlx4_WRITE_PROMISC(dev, port, steer, mailbox); 595 if (err) 596 goto out_mailbox; 597 598 if (!(mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER)) { 599 /* Remove the QP from all the steering entries */ 600 list_for_each_entry_safe(entry, tmp_entry, 601 &s_steer->steer_entries[steer], 602 list) { 603 found = false; 604 list_for_each_entry(dqp, &entry->duplicates, list) { 605 if (dqp->qpn == qpn) { 606 found = true; 607 break; 608 } 609 } 610 if (found) { 611 /* A duplicate, no need to change the MGM, 612 * only update the duplicates list 613 */ 614 list_del(&dqp->list); 615 kfree(dqp); 616 } else { 617 int loc = -1; 618 619 err = mlx4_READ_ENTRY(dev, 620 entry->index, 621 mailbox); 622 if (err) 623 goto out_mailbox; 624 members_count = 625 be32_to_cpu(mgm->members_count) & 626 0xffffff; 627 if (!members_count) { 628 mlx4_warn(dev, "QP %06x wasn't found in entry %x mcount=0. deleting entry...\n", 629 qpn, entry->index); 630 list_del(&entry->list); 631 kfree(entry); 632 continue; 633 } 634 635 for (i = 0; i < members_count; ++i) 636 if ((be32_to_cpu(mgm->qp[i]) & 637 MGM_QPN_MASK) == qpn) { 638 loc = i; 639 break; 640 } 641 642 if (loc < 0) { 643 mlx4_err(dev, "QP %06x wasn't found in entry %d\n", 644 qpn, entry->index); 645 err = -EINVAL; 646 goto out_mailbox; 647 } 648 649 /* Copy the last QP in this MGM 650 * over removed QP 651 */ 652 mgm->qp[loc] = mgm->qp[members_count - 1]; 653 mgm->qp[members_count - 1] = 0; 654 mgm->members_count = 655 cpu_to_be32(--members_count | 656 (MLX4_PROT_ETH << 30)); 657 658 err = mlx4_WRITE_ENTRY(dev, 659 entry->index, 660 mailbox); 661 if (err) 662 goto out_mailbox; 663 } 664 } 665 } 666 667 out_mailbox: 668 mlx4_free_cmd_mailbox(dev, mailbox); 669 out_list: 670 if (back_to_list) 671 list_add_tail(&pqp->list, &s_steer->promisc_qps[steer]); 672 else 673 kfree(pqp); 674 out_mutex: 675 mutex_unlock(&priv->mcg_table.mutex); 676 return err; 677 } 678 679 /* 680 * Caller must hold MCG table semaphore. gid and mgm parameters must 681 * be properly aligned for command interface. 682 * 683 * Returns 0 unless a firmware command error occurs. 684 * 685 * If GID is found in MGM or MGM is empty, *index = *hash, *prev = -1 686 * and *mgm holds MGM entry. 687 * 688 * if GID is found in AMGM, *index = index in AMGM, *prev = index of 689 * previous entry in hash chain and *mgm holds AMGM entry. 690 * 691 * If no AMGM exists for given gid, *index = -1, *prev = index of last 692 * entry in hash chain and *mgm holds end of hash chain. 693 */ find_entry(struct mlx4_dev * dev,u8 port,u8 * gid,enum mlx4_protocol prot,struct mlx4_cmd_mailbox * mgm_mailbox,int * prev,int * index)694 static int find_entry(struct mlx4_dev *dev, u8 port, 695 u8 *gid, enum mlx4_protocol prot, 696 struct mlx4_cmd_mailbox *mgm_mailbox, 697 int *prev, int *index) 698 { 699 struct mlx4_cmd_mailbox *mailbox; 700 struct mlx4_mgm *mgm = mgm_mailbox->buf; 701 u8 *mgid; 702 int err; 703 u16 hash; 704 u8 op_mod = (prot == MLX4_PROT_ETH) ? 705 !!(dev->caps.flags & MLX4_DEV_CAP_FLAG_VEP_MC_STEER) : 0; 706 707 mailbox = mlx4_alloc_cmd_mailbox(dev); 708 if (IS_ERR(mailbox)) 709 return -ENOMEM; 710 mgid = mailbox->buf; 711 712 memcpy(mgid, gid, 16); 713 714 err = mlx4_GID_HASH(dev, mailbox, &hash, op_mod); 715 mlx4_free_cmd_mailbox(dev, mailbox); 716 if (err) 717 return err; 718 719 if (0) 720 mlx4_dbg(dev, "Hash for %pI6 is %04x\n", gid, hash); 721 722 *index = hash; 723 *prev = -1; 724 725 do { 726 err = mlx4_READ_ENTRY(dev, *index, mgm_mailbox); 727 if (err) 728 return err; 729 730 if (!(be32_to_cpu(mgm->members_count) & 0xffffff)) { 731 if (*index != hash) { 732 mlx4_err(dev, "Found zero MGID in AMGM\n"); 733 err = -EINVAL; 734 } 735 return err; 736 } 737 738 if (!memcmp(mgm->gid, gid, 16) && 739 be32_to_cpu(mgm->members_count) >> 30 == prot) 740 return err; 741 742 *prev = *index; 743 *index = be32_to_cpu(mgm->next_gid_index) >> 6; 744 } while (*index); 745 746 *index = -1; 747 return err; 748 } 749 750 static const u8 __promisc_mode[] = { 751 [MLX4_FS_REGULAR] = 0x0, 752 [MLX4_FS_ALL_DEFAULT] = 0x1, 753 [MLX4_FS_MC_DEFAULT] = 0x3, 754 [MLX4_FS_MIRROR_RX_PORT] = 0x4, 755 [MLX4_FS_MIRROR_SX_PORT] = 0x5, 756 [MLX4_FS_UC_SNIFFER] = 0x6, 757 [MLX4_FS_MC_SNIFFER] = 0x7, 758 }; 759 mlx4_map_sw_to_hw_steering_mode(struct mlx4_dev * dev,enum mlx4_net_trans_promisc_mode flow_type)760 int mlx4_map_sw_to_hw_steering_mode(struct mlx4_dev *dev, 761 enum mlx4_net_trans_promisc_mode flow_type) 762 { 763 if (flow_type >= MLX4_FS_MODE_NUM) { 764 mlx4_err(dev, "Invalid flow type. type = %d\n", flow_type); 765 return -EINVAL; 766 } 767 return __promisc_mode[flow_type]; 768 } 769 EXPORT_SYMBOL_GPL(mlx4_map_sw_to_hw_steering_mode); 770 trans_rule_ctrl_to_hw(struct mlx4_net_trans_rule * ctrl,struct mlx4_net_trans_rule_hw_ctrl * hw)771 static void trans_rule_ctrl_to_hw(struct mlx4_net_trans_rule *ctrl, 772 struct mlx4_net_trans_rule_hw_ctrl *hw) 773 { 774 u8 flags = 0; 775 776 flags = ctrl->queue_mode == MLX4_NET_TRANS_Q_LIFO ? 1 : 0; 777 flags |= ctrl->exclusive ? (1 << 2) : 0; 778 flags |= ctrl->allow_loopback ? (1 << 3) : 0; 779 780 hw->flags = flags; 781 hw->type = __promisc_mode[ctrl->promisc_mode]; 782 hw->prio = cpu_to_be16(ctrl->priority); 783 hw->port = ctrl->port; 784 hw->qpn = cpu_to_be32(ctrl->qpn); 785 } 786 787 const u16 __sw_id_hw[] = { 788 [MLX4_NET_TRANS_RULE_ID_ETH] = 0xE001, 789 [MLX4_NET_TRANS_RULE_ID_IB] = 0xE005, 790 [MLX4_NET_TRANS_RULE_ID_IPV6] = 0xE003, 791 [MLX4_NET_TRANS_RULE_ID_IPV4] = 0xE002, 792 [MLX4_NET_TRANS_RULE_ID_TCP] = 0xE004, 793 [MLX4_NET_TRANS_RULE_ID_UDP] = 0xE006, 794 [MLX4_NET_TRANS_RULE_ID_VXLAN] = 0xE008 795 }; 796 mlx4_map_sw_to_hw_steering_id(struct mlx4_dev * dev,enum mlx4_net_trans_rule_id id)797 int mlx4_map_sw_to_hw_steering_id(struct mlx4_dev *dev, 798 enum mlx4_net_trans_rule_id id) 799 { 800 if (id >= MLX4_NET_TRANS_RULE_NUM) { 801 mlx4_err(dev, "Invalid network rule id. id = %d\n", id); 802 return -EINVAL; 803 } 804 return __sw_id_hw[id]; 805 } 806 EXPORT_SYMBOL_GPL(mlx4_map_sw_to_hw_steering_id); 807 808 static const int __rule_hw_sz[] = { 809 [MLX4_NET_TRANS_RULE_ID_ETH] = 810 sizeof(struct mlx4_net_trans_rule_hw_eth), 811 [MLX4_NET_TRANS_RULE_ID_IB] = 812 sizeof(struct mlx4_net_trans_rule_hw_ib), 813 [MLX4_NET_TRANS_RULE_ID_IPV6] = 0, 814 [MLX4_NET_TRANS_RULE_ID_IPV4] = 815 sizeof(struct mlx4_net_trans_rule_hw_ipv4), 816 [MLX4_NET_TRANS_RULE_ID_TCP] = 817 sizeof(struct mlx4_net_trans_rule_hw_tcp_udp), 818 [MLX4_NET_TRANS_RULE_ID_UDP] = 819 sizeof(struct mlx4_net_trans_rule_hw_tcp_udp), 820 [MLX4_NET_TRANS_RULE_ID_VXLAN] = 821 sizeof(struct mlx4_net_trans_rule_hw_vxlan) 822 }; 823 mlx4_hw_rule_sz(struct mlx4_dev * dev,enum mlx4_net_trans_rule_id id)824 int mlx4_hw_rule_sz(struct mlx4_dev *dev, 825 enum mlx4_net_trans_rule_id id) 826 { 827 if (id >= MLX4_NET_TRANS_RULE_NUM) { 828 mlx4_err(dev, "Invalid network rule id. id = %d\n", id); 829 return -EINVAL; 830 } 831 832 return __rule_hw_sz[id]; 833 } 834 EXPORT_SYMBOL_GPL(mlx4_hw_rule_sz); 835 parse_trans_rule(struct mlx4_dev * dev,struct mlx4_spec_list * spec,struct _rule_hw * rule_hw)836 static int parse_trans_rule(struct mlx4_dev *dev, struct mlx4_spec_list *spec, 837 struct _rule_hw *rule_hw) 838 { 839 if (mlx4_hw_rule_sz(dev, spec->id) < 0) 840 return -EINVAL; 841 memset(rule_hw, 0, mlx4_hw_rule_sz(dev, spec->id)); 842 rule_hw->id = cpu_to_be16(__sw_id_hw[spec->id]); 843 rule_hw->size = mlx4_hw_rule_sz(dev, spec->id) >> 2; 844 845 switch (spec->id) { 846 case MLX4_NET_TRANS_RULE_ID_ETH: 847 memcpy(rule_hw->eth.dst_mac, spec->eth.dst_mac, ETH_ALEN); 848 memcpy(rule_hw->eth.dst_mac_msk, spec->eth.dst_mac_msk, 849 ETH_ALEN); 850 memcpy(rule_hw->eth.src_mac, spec->eth.src_mac, ETH_ALEN); 851 memcpy(rule_hw->eth.src_mac_msk, spec->eth.src_mac_msk, 852 ETH_ALEN); 853 if (spec->eth.ether_type_enable) { 854 rule_hw->eth.ether_type_enable = 1; 855 rule_hw->eth.ether_type = spec->eth.ether_type; 856 } 857 rule_hw->eth.vlan_tag = spec->eth.vlan_id; 858 rule_hw->eth.vlan_tag_msk = spec->eth.vlan_id_msk; 859 break; 860 861 case MLX4_NET_TRANS_RULE_ID_IB: 862 rule_hw->ib.l3_qpn = spec->ib.l3_qpn; 863 rule_hw->ib.qpn_mask = spec->ib.qpn_msk; 864 memcpy(&rule_hw->ib.dst_gid, &spec->ib.dst_gid, 16); 865 memcpy(&rule_hw->ib.dst_gid_msk, &spec->ib.dst_gid_msk, 16); 866 break; 867 868 case MLX4_NET_TRANS_RULE_ID_IPV6: 869 return -EOPNOTSUPP; 870 871 case MLX4_NET_TRANS_RULE_ID_IPV4: 872 rule_hw->ipv4.src_ip = spec->ipv4.src_ip; 873 rule_hw->ipv4.src_ip_msk = spec->ipv4.src_ip_msk; 874 rule_hw->ipv4.dst_ip = spec->ipv4.dst_ip; 875 rule_hw->ipv4.dst_ip_msk = spec->ipv4.dst_ip_msk; 876 break; 877 878 case MLX4_NET_TRANS_RULE_ID_TCP: 879 case MLX4_NET_TRANS_RULE_ID_UDP: 880 rule_hw->tcp_udp.dst_port = spec->tcp_udp.dst_port; 881 rule_hw->tcp_udp.dst_port_msk = spec->tcp_udp.dst_port_msk; 882 rule_hw->tcp_udp.src_port = spec->tcp_udp.src_port; 883 rule_hw->tcp_udp.src_port_msk = spec->tcp_udp.src_port_msk; 884 break; 885 886 case MLX4_NET_TRANS_RULE_ID_VXLAN: 887 rule_hw->vxlan.vni = 888 cpu_to_be32(be32_to_cpu(spec->vxlan.vni) << 8); 889 rule_hw->vxlan.vni_mask = 890 cpu_to_be32(be32_to_cpu(spec->vxlan.vni_mask) << 8); 891 break; 892 893 default: 894 return -EINVAL; 895 } 896 897 return __rule_hw_sz[spec->id]; 898 } 899 mlx4_err_rule(struct mlx4_dev * dev,char * str,struct mlx4_net_trans_rule * rule)900 static void mlx4_err_rule(struct mlx4_dev *dev, char *str, 901 struct mlx4_net_trans_rule *rule) 902 { 903 #define BUF_SIZE 256 904 struct mlx4_spec_list *cur; 905 char buf[BUF_SIZE]; 906 int len = 0; 907 908 mlx4_err(dev, "%s", str); 909 len += scnprintf(buf + len, BUF_SIZE - len, 910 "port = %d prio = 0x%x qp = 0x%x ", 911 rule->port, rule->priority, rule->qpn); 912 913 list_for_each_entry(cur, &rule->list, list) { 914 switch (cur->id) { 915 case MLX4_NET_TRANS_RULE_ID_ETH: 916 len += scnprintf(buf + len, BUF_SIZE - len, 917 "dmac = %pM ", &cur->eth.dst_mac); 918 if (cur->eth.ether_type) 919 len += scnprintf(buf + len, BUF_SIZE - len, 920 "ethertype = 0x%x ", 921 be16_to_cpu(cur->eth.ether_type)); 922 if (cur->eth.vlan_id) 923 len += scnprintf(buf + len, BUF_SIZE - len, 924 "vlan-id = %d ", 925 be16_to_cpu(cur->eth.vlan_id)); 926 break; 927 928 case MLX4_NET_TRANS_RULE_ID_IPV4: 929 if (cur->ipv4.src_ip) 930 len += scnprintf(buf + len, BUF_SIZE - len, 931 "src-ip = %pI4 ", 932 &cur->ipv4.src_ip); 933 if (cur->ipv4.dst_ip) 934 len += scnprintf(buf + len, BUF_SIZE - len, 935 "dst-ip = %pI4 ", 936 &cur->ipv4.dst_ip); 937 break; 938 939 case MLX4_NET_TRANS_RULE_ID_TCP: 940 case MLX4_NET_TRANS_RULE_ID_UDP: 941 if (cur->tcp_udp.src_port) 942 len += scnprintf(buf + len, BUF_SIZE - len, 943 "src-port = %d ", 944 be16_to_cpu(cur->tcp_udp.src_port)); 945 if (cur->tcp_udp.dst_port) 946 len += scnprintf(buf + len, BUF_SIZE - len, 947 "dst-port = %d ", 948 be16_to_cpu(cur->tcp_udp.dst_port)); 949 break; 950 951 case MLX4_NET_TRANS_RULE_ID_IB: 952 len += scnprintf(buf + len, BUF_SIZE - len, 953 "dst-gid = %pI6\n", cur->ib.dst_gid); 954 len += scnprintf(buf + len, BUF_SIZE - len, 955 "dst-gid-mask = %pI6\n", 956 cur->ib.dst_gid_msk); 957 break; 958 959 case MLX4_NET_TRANS_RULE_ID_VXLAN: 960 len += scnprintf(buf + len, BUF_SIZE - len, 961 "VNID = %d ", be32_to_cpu(cur->vxlan.vni)); 962 break; 963 case MLX4_NET_TRANS_RULE_ID_IPV6: 964 break; 965 966 default: 967 break; 968 } 969 } 970 len += scnprintf(buf + len, BUF_SIZE - len, "\n"); 971 mlx4_err(dev, "%s", buf); 972 973 if (len >= BUF_SIZE) 974 mlx4_err(dev, "Network rule error message was truncated, print buffer is too small\n"); 975 } 976 mlx4_flow_attach(struct mlx4_dev * dev,struct mlx4_net_trans_rule * rule,u64 * reg_id)977 int mlx4_flow_attach(struct mlx4_dev *dev, 978 struct mlx4_net_trans_rule *rule, u64 *reg_id) 979 { 980 struct mlx4_cmd_mailbox *mailbox; 981 struct mlx4_spec_list *cur; 982 u32 size = 0; 983 int ret; 984 985 mailbox = mlx4_alloc_cmd_mailbox(dev); 986 if (IS_ERR(mailbox)) 987 return PTR_ERR(mailbox); 988 989 if (!mlx4_qp_lookup(dev, rule->qpn)) { 990 mlx4_err_rule(dev, "QP doesn't exist\n", rule); 991 ret = -EINVAL; 992 goto out; 993 } 994 995 trans_rule_ctrl_to_hw(rule, mailbox->buf); 996 997 size += sizeof(struct mlx4_net_trans_rule_hw_ctrl); 998 999 list_for_each_entry(cur, &rule->list, list) { 1000 ret = parse_trans_rule(dev, cur, mailbox->buf + size); 1001 if (ret < 0) 1002 goto out; 1003 1004 size += ret; 1005 } 1006 1007 ret = mlx4_QP_FLOW_STEERING_ATTACH(dev, mailbox, size >> 2, reg_id); 1008 if (ret == -ENOMEM) { 1009 mlx4_err_rule(dev, 1010 "mcg table is full. Fail to register network rule\n", 1011 rule); 1012 } else if (ret) { 1013 if (ret == -ENXIO) { 1014 if (dev->caps.steering_mode != MLX4_STEERING_MODE_DEVICE_MANAGED) 1015 mlx4_err_rule(dev, 1016 "DMFS is not enabled, " 1017 "failed to register network rule.\n", 1018 rule); 1019 else 1020 mlx4_err_rule(dev, 1021 "Rule exceeds the dmfs_high_rate_mode limitations, " 1022 "failed to register network rule.\n", 1023 rule); 1024 1025 } else { 1026 mlx4_err_rule(dev, "Fail to register network rule.\n", rule); 1027 } 1028 } 1029 1030 out: 1031 mlx4_free_cmd_mailbox(dev, mailbox); 1032 1033 return ret; 1034 } 1035 EXPORT_SYMBOL_GPL(mlx4_flow_attach); 1036 mlx4_flow_detach(struct mlx4_dev * dev,u64 reg_id)1037 int mlx4_flow_detach(struct mlx4_dev *dev, u64 reg_id) 1038 { 1039 int err; 1040 1041 err = mlx4_QP_FLOW_STEERING_DETACH(dev, reg_id); 1042 if (err) 1043 mlx4_err(dev, "Fail to detach network rule. registration id = 0x%llx\n", 1044 reg_id); 1045 return err; 1046 } 1047 EXPORT_SYMBOL_GPL(mlx4_flow_detach); 1048 mlx4_tunnel_steer_add(struct mlx4_dev * dev,const unsigned char * addr,int port,int qpn,u16 prio,u64 * reg_id)1049 int mlx4_tunnel_steer_add(struct mlx4_dev *dev, const unsigned char *addr, 1050 int port, int qpn, u16 prio, u64 *reg_id) 1051 { 1052 int err; 1053 struct mlx4_spec_list spec_eth_outer = { {NULL} }; 1054 struct mlx4_spec_list spec_vxlan = { {NULL} }; 1055 struct mlx4_spec_list spec_eth_inner = { {NULL} }; 1056 1057 struct mlx4_net_trans_rule rule = { 1058 .queue_mode = MLX4_NET_TRANS_Q_FIFO, 1059 .exclusive = 0, 1060 .allow_loopback = 1, 1061 .promisc_mode = MLX4_FS_REGULAR, 1062 }; 1063 1064 __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); 1065 1066 rule.port = port; 1067 rule.qpn = qpn; 1068 rule.priority = prio; 1069 INIT_LIST_HEAD(&rule.list); 1070 1071 spec_eth_outer.id = MLX4_NET_TRANS_RULE_ID_ETH; 1072 memcpy(spec_eth_outer.eth.dst_mac, addr, ETH_ALEN); 1073 memcpy(spec_eth_outer.eth.dst_mac_msk, &mac_mask, ETH_ALEN); 1074 1075 spec_vxlan.id = MLX4_NET_TRANS_RULE_ID_VXLAN; /* any vxlan header */ 1076 spec_eth_inner.id = MLX4_NET_TRANS_RULE_ID_ETH; /* any inner eth header */ 1077 1078 list_add_tail(&spec_eth_outer.list, &rule.list); 1079 list_add_tail(&spec_vxlan.list, &rule.list); 1080 list_add_tail(&spec_eth_inner.list, &rule.list); 1081 1082 err = mlx4_flow_attach(dev, &rule, reg_id); 1083 return err; 1084 } 1085 EXPORT_SYMBOL(mlx4_tunnel_steer_add); 1086 mlx4_FLOW_STEERING_IB_UC_QP_RANGE(struct mlx4_dev * dev,u32 min_range_qpn,u32 max_range_qpn)1087 int mlx4_FLOW_STEERING_IB_UC_QP_RANGE(struct mlx4_dev *dev, u32 min_range_qpn, 1088 u32 max_range_qpn) 1089 { 1090 int err; 1091 u64 in_param; 1092 1093 in_param = ((u64) min_range_qpn) << 32; 1094 in_param |= ((u64) max_range_qpn) & 0xFFFFFFFF; 1095 1096 err = mlx4_cmd(dev, in_param, 0, 0, 1097 MLX4_FLOW_STEERING_IB_UC_QP_RANGE, 1098 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE); 1099 1100 return err; 1101 } 1102 EXPORT_SYMBOL_GPL(mlx4_FLOW_STEERING_IB_UC_QP_RANGE); 1103 mlx4_qp_attach_common(struct mlx4_dev * dev,struct mlx4_qp * qp,u8 gid[16],int block_mcast_loopback,enum mlx4_protocol prot,enum mlx4_steer_type steer)1104 int mlx4_qp_attach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], 1105 int block_mcast_loopback, enum mlx4_protocol prot, 1106 enum mlx4_steer_type steer) 1107 { 1108 struct mlx4_priv *priv = mlx4_priv(dev); 1109 struct mlx4_cmd_mailbox *mailbox; 1110 struct mlx4_mgm *mgm; 1111 u32 members_count; 1112 int index = -1, prev; 1113 int link = 0; 1114 int i; 1115 int err; 1116 u8 port = gid[5]; 1117 u8 new_entry = 0; 1118 1119 mailbox = mlx4_alloc_cmd_mailbox(dev); 1120 if (IS_ERR(mailbox)) 1121 return PTR_ERR(mailbox); 1122 mgm = mailbox->buf; 1123 1124 mutex_lock(&priv->mcg_table.mutex); 1125 err = find_entry(dev, port, gid, prot, 1126 mailbox, &prev, &index); 1127 if (err) 1128 goto out; 1129 1130 if (index != -1) { 1131 if (!(be32_to_cpu(mgm->members_count) & 0xffffff)) { 1132 new_entry = 1; 1133 memcpy(mgm->gid, gid, 16); 1134 } 1135 } else { 1136 link = 1; 1137 1138 index = mlx4_bitmap_alloc(&priv->mcg_table.bitmap); 1139 if (index == -1) { 1140 mlx4_err(dev, "No AMGM entries left\n"); 1141 err = -ENOMEM; 1142 goto out; 1143 } 1144 index += dev->caps.num_mgms; 1145 1146 new_entry = 1; 1147 memset(mgm, 0, sizeof(*mgm)); 1148 memcpy(mgm->gid, gid, 16); 1149 } 1150 1151 members_count = be32_to_cpu(mgm->members_count) & 0xffffff; 1152 if (members_count == dev->caps.num_qp_per_mgm) { 1153 mlx4_err(dev, "MGM at index %x is full\n", index); 1154 err = -ENOMEM; 1155 goto out; 1156 } 1157 1158 for (i = 0; i < members_count; ++i) 1159 if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) { 1160 mlx4_dbg(dev, "QP %06x already a member of MGM\n", qp->qpn); 1161 err = 0; 1162 goto out; 1163 } 1164 1165 if (block_mcast_loopback) 1166 mgm->qp[members_count++] = cpu_to_be32((qp->qpn & MGM_QPN_MASK) | 1167 (1U << MGM_BLCK_LB_BIT)); 1168 else 1169 mgm->qp[members_count++] = cpu_to_be32(qp->qpn & MGM_QPN_MASK); 1170 1171 mgm->members_count = cpu_to_be32(members_count | (u32) prot << 30); 1172 1173 err = mlx4_WRITE_ENTRY(dev, index, mailbox); 1174 if (err) 1175 goto out; 1176 1177 if (!link) 1178 goto out; 1179 1180 err = mlx4_READ_ENTRY(dev, prev, mailbox); 1181 if (err) 1182 goto out; 1183 1184 mgm->next_gid_index = cpu_to_be32(index << 6); 1185 1186 err = mlx4_WRITE_ENTRY(dev, prev, mailbox); 1187 if (err) 1188 goto out; 1189 1190 out: 1191 if (prot == MLX4_PROT_ETH && index != -1) { 1192 /* manage the steering entry for promisc mode */ 1193 if (new_entry) 1194 err = new_steering_entry(dev, port, steer, 1195 index, qp->qpn); 1196 else 1197 err = existing_steering_entry(dev, port, steer, 1198 index, qp->qpn); 1199 } 1200 if (err && link && index != -1) { 1201 if (index < dev->caps.num_mgms) 1202 mlx4_warn(dev, "Got AMGM index %d < %d\n", 1203 index, dev->caps.num_mgms); 1204 else 1205 mlx4_bitmap_free(&priv->mcg_table.bitmap, 1206 index - dev->caps.num_mgms, MLX4_USE_RR); 1207 } 1208 mutex_unlock(&priv->mcg_table.mutex); 1209 1210 mlx4_free_cmd_mailbox(dev, mailbox); 1211 return err; 1212 } 1213 mlx4_qp_detach_common(struct mlx4_dev * dev,struct mlx4_qp * qp,u8 gid[16],enum mlx4_protocol prot,enum mlx4_steer_type steer)1214 int mlx4_qp_detach_common(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], 1215 enum mlx4_protocol prot, enum mlx4_steer_type steer) 1216 { 1217 struct mlx4_priv *priv = mlx4_priv(dev); 1218 struct mlx4_cmd_mailbox *mailbox; 1219 struct mlx4_mgm *mgm; 1220 u32 members_count; 1221 int prev, index; 1222 int i, loc = -1; 1223 int err; 1224 u8 port = gid[5]; 1225 bool removed_entry = false; 1226 1227 mailbox = mlx4_alloc_cmd_mailbox(dev); 1228 if (IS_ERR(mailbox)) 1229 return PTR_ERR(mailbox); 1230 mgm = mailbox->buf; 1231 1232 mutex_lock(&priv->mcg_table.mutex); 1233 1234 err = find_entry(dev, port, gid, prot, 1235 mailbox, &prev, &index); 1236 if (err) 1237 goto out; 1238 1239 if (index == -1) { 1240 mlx4_err(dev, "MGID %pI6 not found\n", gid); 1241 err = -EINVAL; 1242 goto out; 1243 } 1244 1245 /* If this QP is also a promisc QP, it shouldn't be removed only if 1246 * at least one none promisc QP is also attached to this MCG 1247 */ 1248 if (prot == MLX4_PROT_ETH && 1249 check_duplicate_entry(dev, port, steer, index, qp->qpn) && 1250 !promisc_steering_entry(dev, port, steer, index, qp->qpn, NULL)) 1251 goto out; 1252 1253 members_count = be32_to_cpu(mgm->members_count) & 0xffffff; 1254 for (i = 0; i < members_count; ++i) 1255 if ((be32_to_cpu(mgm->qp[i]) & MGM_QPN_MASK) == qp->qpn) { 1256 loc = i; 1257 break; 1258 } 1259 1260 if (loc == -1) { 1261 mlx4_err(dev, "QP %06x not found in MGM\n", qp->qpn); 1262 err = -EINVAL; 1263 goto out; 1264 } 1265 1266 /* copy the last QP in this MGM over removed QP */ 1267 mgm->qp[loc] = mgm->qp[members_count - 1]; 1268 mgm->qp[members_count - 1] = 0; 1269 mgm->members_count = cpu_to_be32(--members_count | (u32) prot << 30); 1270 1271 if (prot == MLX4_PROT_ETH) 1272 removed_entry = can_remove_steering_entry(dev, port, steer, 1273 index, qp->qpn); 1274 if (members_count && (prot != MLX4_PROT_ETH || !removed_entry)) { 1275 err = mlx4_WRITE_ENTRY(dev, index, mailbox); 1276 goto out; 1277 } 1278 1279 /* We are going to delete the entry, members count should be 0 */ 1280 mgm->members_count = cpu_to_be32((u32) prot << 30); 1281 1282 if (prev == -1) { 1283 /* Remove entry from MGM */ 1284 int amgm_index = be32_to_cpu(mgm->next_gid_index) >> 6; 1285 if (amgm_index) { 1286 err = mlx4_READ_ENTRY(dev, amgm_index, mailbox); 1287 if (err) 1288 goto out; 1289 } else 1290 memset(mgm->gid, 0, 16); 1291 1292 err = mlx4_WRITE_ENTRY(dev, index, mailbox); 1293 if (err) 1294 goto out; 1295 1296 if (amgm_index) { 1297 if (amgm_index < dev->caps.num_mgms) 1298 mlx4_warn(dev, "MGM entry %d had AMGM index %d < %d\n", 1299 index, amgm_index, dev->caps.num_mgms); 1300 else 1301 mlx4_bitmap_free(&priv->mcg_table.bitmap, 1302 amgm_index - dev->caps.num_mgms, MLX4_USE_RR); 1303 } 1304 } else { 1305 /* Remove entry from AMGM */ 1306 int cur_next_index = be32_to_cpu(mgm->next_gid_index) >> 6; 1307 err = mlx4_READ_ENTRY(dev, prev, mailbox); 1308 if (err) 1309 goto out; 1310 1311 mgm->next_gid_index = cpu_to_be32(cur_next_index << 6); 1312 1313 err = mlx4_WRITE_ENTRY(dev, prev, mailbox); 1314 if (err) 1315 goto out; 1316 1317 if (index < dev->caps.num_mgms) 1318 mlx4_warn(dev, "entry %d had next AMGM index %d < %d\n", 1319 prev, index, dev->caps.num_mgms); 1320 else 1321 mlx4_bitmap_free(&priv->mcg_table.bitmap, 1322 index - dev->caps.num_mgms, MLX4_USE_RR); 1323 } 1324 1325 out: 1326 mutex_unlock(&priv->mcg_table.mutex); 1327 1328 mlx4_free_cmd_mailbox(dev, mailbox); 1329 if (err && dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) 1330 /* In case device is under an error, return success as a closing command */ 1331 err = 0; 1332 return err; 1333 } 1334 mlx4_QP_ATTACH(struct mlx4_dev * dev,struct mlx4_qp * qp,u8 gid[16],u8 attach,u8 block_loopback,enum mlx4_protocol prot)1335 static int mlx4_QP_ATTACH(struct mlx4_dev *dev, struct mlx4_qp *qp, 1336 u8 gid[16], u8 attach, u8 block_loopback, 1337 enum mlx4_protocol prot) 1338 { 1339 struct mlx4_cmd_mailbox *mailbox; 1340 int err = 0; 1341 int qpn; 1342 1343 if (!mlx4_is_mfunc(dev)) 1344 return -EBADF; 1345 1346 mailbox = mlx4_alloc_cmd_mailbox(dev); 1347 if (IS_ERR(mailbox)) 1348 return PTR_ERR(mailbox); 1349 1350 memcpy(mailbox->buf, gid, 16); 1351 qpn = qp->qpn; 1352 qpn |= (prot << 28); 1353 if (attach && block_loopback) 1354 qpn |= (1 << 31); 1355 1356 err = mlx4_cmd(dev, mailbox->dma, qpn, attach, 1357 MLX4_CMD_QP_ATTACH, MLX4_CMD_TIME_CLASS_A, 1358 MLX4_CMD_WRAPPED); 1359 1360 mlx4_free_cmd_mailbox(dev, mailbox); 1361 if (err && !attach && 1362 dev->persist->state & MLX4_DEVICE_STATE_INTERNAL_ERROR) 1363 err = 0; 1364 return err; 1365 } 1366 mlx4_trans_to_dmfs_attach(struct mlx4_dev * dev,struct mlx4_qp * qp,u8 gid[16],u8 port,int block_mcast_loopback,enum mlx4_protocol prot,u64 * reg_id)1367 int mlx4_trans_to_dmfs_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, 1368 u8 gid[16], u8 port, 1369 int block_mcast_loopback, 1370 enum mlx4_protocol prot, u64 *reg_id) 1371 { 1372 struct mlx4_spec_list spec = { {NULL} }; 1373 __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); 1374 1375 struct mlx4_net_trans_rule rule = { 1376 .queue_mode = MLX4_NET_TRANS_Q_FIFO, 1377 .exclusive = 0, 1378 .promisc_mode = MLX4_FS_REGULAR, 1379 .priority = MLX4_DOMAIN_NIC, 1380 }; 1381 1382 rule.allow_loopback = !block_mcast_loopback; 1383 rule.port = port; 1384 rule.qpn = qp->qpn; 1385 INIT_LIST_HEAD(&rule.list); 1386 1387 switch (prot) { 1388 case MLX4_PROT_ETH: 1389 spec.id = MLX4_NET_TRANS_RULE_ID_ETH; 1390 memcpy(spec.eth.dst_mac, &gid[10], ETH_ALEN); 1391 memcpy(spec.eth.dst_mac_msk, &mac_mask, ETH_ALEN); 1392 break; 1393 1394 case MLX4_PROT_IB_IPV6: 1395 spec.id = MLX4_NET_TRANS_RULE_ID_IB; 1396 memcpy(spec.ib.dst_gid, gid, 16); 1397 memset(&spec.ib.dst_gid_msk, 0xff, 16); 1398 break; 1399 default: 1400 return -EINVAL; 1401 } 1402 list_add_tail(&spec.list, &rule.list); 1403 1404 return mlx4_flow_attach(dev, &rule, reg_id); 1405 } 1406 mlx4_multicast_attach(struct mlx4_dev * dev,struct mlx4_qp * qp,u8 gid[16],u8 port,int block_mcast_loopback,enum mlx4_protocol prot,u64 * reg_id)1407 int mlx4_multicast_attach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], 1408 u8 port, int block_mcast_loopback, 1409 enum mlx4_protocol prot, u64 *reg_id) 1410 { 1411 switch (dev->caps.steering_mode) { 1412 case MLX4_STEERING_MODE_A0: 1413 if (prot == MLX4_PROT_ETH) 1414 return 0; 1415 fallthrough; 1416 1417 case MLX4_STEERING_MODE_B0: 1418 if (prot == MLX4_PROT_ETH) 1419 gid[7] |= (MLX4_MC_STEER << 1); 1420 1421 if (mlx4_is_mfunc(dev)) 1422 return mlx4_QP_ATTACH(dev, qp, gid, 1, 1423 block_mcast_loopback, prot); 1424 return mlx4_qp_attach_common(dev, qp, gid, 1425 block_mcast_loopback, prot, 1426 MLX4_MC_STEER); 1427 1428 case MLX4_STEERING_MODE_DEVICE_MANAGED: 1429 return mlx4_trans_to_dmfs_attach(dev, qp, gid, port, 1430 block_mcast_loopback, 1431 prot, reg_id); 1432 default: 1433 return -EINVAL; 1434 } 1435 } 1436 EXPORT_SYMBOL_GPL(mlx4_multicast_attach); 1437 mlx4_multicast_detach(struct mlx4_dev * dev,struct mlx4_qp * qp,u8 gid[16],enum mlx4_protocol prot,u64 reg_id)1438 int mlx4_multicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, u8 gid[16], 1439 enum mlx4_protocol prot, u64 reg_id) 1440 { 1441 switch (dev->caps.steering_mode) { 1442 case MLX4_STEERING_MODE_A0: 1443 if (prot == MLX4_PROT_ETH) 1444 return 0; 1445 fallthrough; 1446 1447 case MLX4_STEERING_MODE_B0: 1448 if (prot == MLX4_PROT_ETH) 1449 gid[7] |= (MLX4_MC_STEER << 1); 1450 1451 if (mlx4_is_mfunc(dev)) 1452 return mlx4_QP_ATTACH(dev, qp, gid, 0, 0, prot); 1453 1454 return mlx4_qp_detach_common(dev, qp, gid, prot, 1455 MLX4_MC_STEER); 1456 1457 case MLX4_STEERING_MODE_DEVICE_MANAGED: 1458 return mlx4_flow_detach(dev, reg_id); 1459 1460 default: 1461 return -EINVAL; 1462 } 1463 } 1464 EXPORT_SYMBOL_GPL(mlx4_multicast_detach); 1465 mlx4_flow_steer_promisc_add(struct mlx4_dev * dev,u8 port,u32 qpn,enum mlx4_net_trans_promisc_mode mode)1466 int mlx4_flow_steer_promisc_add(struct mlx4_dev *dev, u8 port, 1467 u32 qpn, enum mlx4_net_trans_promisc_mode mode) 1468 { 1469 struct mlx4_net_trans_rule rule = { 1470 .queue_mode = MLX4_NET_TRANS_Q_FIFO, 1471 .exclusive = 0, 1472 .allow_loopback = 1, 1473 }; 1474 1475 u64 *regid_p; 1476 1477 switch (mode) { 1478 case MLX4_FS_ALL_DEFAULT: 1479 regid_p = &dev->regid_promisc_array[port]; 1480 break; 1481 case MLX4_FS_MC_DEFAULT: 1482 regid_p = &dev->regid_allmulti_array[port]; 1483 break; 1484 default: 1485 return -1; 1486 } 1487 1488 if (*regid_p != 0) 1489 return -1; 1490 1491 rule.promisc_mode = mode; 1492 rule.port = port; 1493 rule.qpn = qpn; 1494 INIT_LIST_HEAD(&rule.list); 1495 mlx4_info(dev, "going promisc on %x\n", port); 1496 1497 return mlx4_flow_attach(dev, &rule, regid_p); 1498 } 1499 EXPORT_SYMBOL_GPL(mlx4_flow_steer_promisc_add); 1500 mlx4_flow_steer_promisc_remove(struct mlx4_dev * dev,u8 port,enum mlx4_net_trans_promisc_mode mode)1501 int mlx4_flow_steer_promisc_remove(struct mlx4_dev *dev, u8 port, 1502 enum mlx4_net_trans_promisc_mode mode) 1503 { 1504 int ret; 1505 u64 *regid_p; 1506 1507 switch (mode) { 1508 case MLX4_FS_ALL_DEFAULT: 1509 regid_p = &dev->regid_promisc_array[port]; 1510 break; 1511 case MLX4_FS_MC_DEFAULT: 1512 regid_p = &dev->regid_allmulti_array[port]; 1513 break; 1514 default: 1515 return -1; 1516 } 1517 1518 if (*regid_p == 0) 1519 return -1; 1520 1521 ret = mlx4_flow_detach(dev, *regid_p); 1522 if (ret == 0) 1523 *regid_p = 0; 1524 1525 return ret; 1526 } 1527 EXPORT_SYMBOL_GPL(mlx4_flow_steer_promisc_remove); 1528 mlx4_unicast_attach(struct mlx4_dev * dev,struct mlx4_qp * qp,u8 gid[16],int block_mcast_loopback,enum mlx4_protocol prot)1529 int mlx4_unicast_attach(struct mlx4_dev *dev, 1530 struct mlx4_qp *qp, u8 gid[16], 1531 int block_mcast_loopback, enum mlx4_protocol prot) 1532 { 1533 if (prot == MLX4_PROT_ETH) 1534 gid[7] |= (MLX4_UC_STEER << 1); 1535 1536 if (mlx4_is_mfunc(dev)) 1537 return mlx4_QP_ATTACH(dev, qp, gid, 1, 1538 block_mcast_loopback, prot); 1539 1540 return mlx4_qp_attach_common(dev, qp, gid, block_mcast_loopback, 1541 prot, MLX4_UC_STEER); 1542 } 1543 EXPORT_SYMBOL_GPL(mlx4_unicast_attach); 1544 mlx4_unicast_detach(struct mlx4_dev * dev,struct mlx4_qp * qp,u8 gid[16],enum mlx4_protocol prot)1545 int mlx4_unicast_detach(struct mlx4_dev *dev, struct mlx4_qp *qp, 1546 u8 gid[16], enum mlx4_protocol prot) 1547 { 1548 if (prot == MLX4_PROT_ETH) 1549 gid[7] |= (MLX4_UC_STEER << 1); 1550 1551 if (mlx4_is_mfunc(dev)) 1552 return mlx4_QP_ATTACH(dev, qp, gid, 0, 0, prot); 1553 1554 return mlx4_qp_detach_common(dev, qp, gid, prot, MLX4_UC_STEER); 1555 } 1556 EXPORT_SYMBOL_GPL(mlx4_unicast_detach); 1557 mlx4_PROMISC_wrapper(struct mlx4_dev * dev,int slave,struct mlx4_vhcr * vhcr,struct mlx4_cmd_mailbox * inbox,struct mlx4_cmd_mailbox * outbox,struct mlx4_cmd_info * cmd)1558 int mlx4_PROMISC_wrapper(struct mlx4_dev *dev, int slave, 1559 struct mlx4_vhcr *vhcr, 1560 struct mlx4_cmd_mailbox *inbox, 1561 struct mlx4_cmd_mailbox *outbox, 1562 struct mlx4_cmd_info *cmd) 1563 { 1564 u32 qpn = (u32) vhcr->in_param & 0xffffffff; 1565 int port = mlx4_slave_convert_port(dev, slave, vhcr->in_param >> 62); 1566 enum mlx4_steer_type steer = vhcr->in_modifier; 1567 1568 if (port < 0) 1569 return -EINVAL; 1570 1571 /* Promiscuous unicast is not allowed in mfunc */ 1572 if (mlx4_is_mfunc(dev) && steer == MLX4_UC_STEER) 1573 return 0; 1574 1575 if (vhcr->op_modifier) 1576 return add_promisc_qp(dev, port, steer, qpn); 1577 else 1578 return remove_promisc_qp(dev, port, steer, qpn); 1579 } 1580 mlx4_PROMISC(struct mlx4_dev * dev,u32 qpn,enum mlx4_steer_type steer,u8 add,u8 port)1581 static int mlx4_PROMISC(struct mlx4_dev *dev, u32 qpn, 1582 enum mlx4_steer_type steer, u8 add, u8 port) 1583 { 1584 return mlx4_cmd(dev, (u64) qpn | (u64) port << 62, (u32) steer, add, 1585 MLX4_CMD_PROMISC, MLX4_CMD_TIME_CLASS_A, 1586 MLX4_CMD_WRAPPED); 1587 } 1588 mlx4_multicast_promisc_add(struct mlx4_dev * dev,u32 qpn,u8 port)1589 int mlx4_multicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port) 1590 { 1591 if (mlx4_is_mfunc(dev)) 1592 return mlx4_PROMISC(dev, qpn, MLX4_MC_STEER, 1, port); 1593 1594 return add_promisc_qp(dev, port, MLX4_MC_STEER, qpn); 1595 } 1596 EXPORT_SYMBOL_GPL(mlx4_multicast_promisc_add); 1597 mlx4_multicast_promisc_remove(struct mlx4_dev * dev,u32 qpn,u8 port)1598 int mlx4_multicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port) 1599 { 1600 if (mlx4_is_mfunc(dev)) 1601 return mlx4_PROMISC(dev, qpn, MLX4_MC_STEER, 0, port); 1602 1603 return remove_promisc_qp(dev, port, MLX4_MC_STEER, qpn); 1604 } 1605 EXPORT_SYMBOL_GPL(mlx4_multicast_promisc_remove); 1606 mlx4_unicast_promisc_add(struct mlx4_dev * dev,u32 qpn,u8 port)1607 int mlx4_unicast_promisc_add(struct mlx4_dev *dev, u32 qpn, u8 port) 1608 { 1609 if (mlx4_is_mfunc(dev)) 1610 return mlx4_PROMISC(dev, qpn, MLX4_UC_STEER, 1, port); 1611 1612 return add_promisc_qp(dev, port, MLX4_UC_STEER, qpn); 1613 } 1614 EXPORT_SYMBOL_GPL(mlx4_unicast_promisc_add); 1615 mlx4_unicast_promisc_remove(struct mlx4_dev * dev,u32 qpn,u8 port)1616 int mlx4_unicast_promisc_remove(struct mlx4_dev *dev, u32 qpn, u8 port) 1617 { 1618 if (mlx4_is_mfunc(dev)) 1619 return mlx4_PROMISC(dev, qpn, MLX4_UC_STEER, 0, port); 1620 1621 return remove_promisc_qp(dev, port, MLX4_UC_STEER, qpn); 1622 } 1623 EXPORT_SYMBOL_GPL(mlx4_unicast_promisc_remove); 1624 mlx4_init_mcg_table(struct mlx4_dev * dev)1625 int mlx4_init_mcg_table(struct mlx4_dev *dev) 1626 { 1627 struct mlx4_priv *priv = mlx4_priv(dev); 1628 int err; 1629 1630 /* No need for mcg_table when fw managed the mcg table*/ 1631 if (dev->caps.steering_mode == 1632 MLX4_STEERING_MODE_DEVICE_MANAGED) 1633 return 0; 1634 err = mlx4_bitmap_init(&priv->mcg_table.bitmap, dev->caps.num_amgms, 1635 dev->caps.num_amgms - 1, 0, 0); 1636 if (err) 1637 return err; 1638 1639 mutex_init(&priv->mcg_table.mutex); 1640 1641 return 0; 1642 } 1643 mlx4_cleanup_mcg_table(struct mlx4_dev * dev)1644 void mlx4_cleanup_mcg_table(struct mlx4_dev *dev) 1645 { 1646 if (dev->caps.steering_mode != 1647 MLX4_STEERING_MODE_DEVICE_MANAGED) 1648 mlx4_bitmap_cleanup(&mlx4_priv(dev)->mcg_table.bitmap); 1649 } 1650