1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
3
4 #include "lib/sd.h"
5 #include "mlx5_core.h"
6 #include "lib/mlx5.h"
7 #include "fs_cmd.h"
8 #include <linux/mlx5/vport.h>
9 #include <linux/debugfs.h>
10
11 #define sd_info(__dev, format, ...) \
12 dev_info((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__)
13 #define sd_warn(__dev, format, ...) \
14 dev_warn((__dev)->device, "Socket-Direct: " format, ##__VA_ARGS__)
15
16 struct mlx5_sd {
17 u32 group_id;
18 u8 host_buses;
19 struct mlx5_devcom_comp_dev *devcom;
20 struct dentry *dfs;
21 bool primary;
22 union {
23 struct { /* primary */
24 struct mlx5_core_dev *secondaries[MLX5_SD_MAX_GROUP_SZ - 1];
25 struct mlx5_flow_table *tx_ft;
26 };
27 struct { /* secondary */
28 struct mlx5_core_dev *primary_dev;
29 u32 alias_obj_id;
30 };
31 };
32 };
33
mlx5_sd_get_host_buses(struct mlx5_core_dev * dev)34 static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev)
35 {
36 struct mlx5_sd *sd = mlx5_get_sd(dev);
37
38 if (!sd)
39 return 1;
40
41 return sd->host_buses;
42 }
43
mlx5_sd_get_primary(struct mlx5_core_dev * dev)44 static struct mlx5_core_dev *mlx5_sd_get_primary(struct mlx5_core_dev *dev)
45 {
46 struct mlx5_sd *sd = mlx5_get_sd(dev);
47
48 if (!sd)
49 return dev;
50
51 return sd->primary ? dev : sd->primary_dev;
52 }
53
54 struct mlx5_core_dev *
mlx5_sd_primary_get_peer(struct mlx5_core_dev * primary,int idx)55 mlx5_sd_primary_get_peer(struct mlx5_core_dev *primary, int idx)
56 {
57 struct mlx5_sd *sd;
58
59 if (idx == 0)
60 return primary;
61
62 if (idx >= mlx5_sd_get_host_buses(primary))
63 return NULL;
64
65 sd = mlx5_get_sd(primary);
66 return sd->secondaries[idx - 1];
67 }
68
mlx5_sd_ch_ix_get_dev_ix(struct mlx5_core_dev * dev,int ch_ix)69 int mlx5_sd_ch_ix_get_dev_ix(struct mlx5_core_dev *dev, int ch_ix)
70 {
71 return ch_ix % mlx5_sd_get_host_buses(dev);
72 }
73
mlx5_sd_ch_ix_get_vec_ix(struct mlx5_core_dev * dev,int ch_ix)74 int mlx5_sd_ch_ix_get_vec_ix(struct mlx5_core_dev *dev, int ch_ix)
75 {
76 return ch_ix / mlx5_sd_get_host_buses(dev);
77 }
78
mlx5_sd_ch_ix_get_dev(struct mlx5_core_dev * primary,int ch_ix)79 struct mlx5_core_dev *mlx5_sd_ch_ix_get_dev(struct mlx5_core_dev *primary, int ch_ix)
80 {
81 int mdev_idx = mlx5_sd_ch_ix_get_dev_ix(primary, ch_ix);
82
83 return mlx5_sd_primary_get_peer(primary, mdev_idx);
84 }
85
ft_create_alias_supported(struct mlx5_core_dev * dev)86 static bool ft_create_alias_supported(struct mlx5_core_dev *dev)
87 {
88 u64 obj_allowed = MLX5_CAP_GEN_2_64(dev, allowed_object_for_other_vhca_access);
89 u32 obj_supp = MLX5_CAP_GEN_2(dev, cross_vhca_object_to_object_supported);
90
91 if (!(obj_supp &
92 MLX5_CROSS_VHCA_OBJ_TO_OBJ_SUPPORTED_LOCAL_FLOW_TABLE_ROOT_TO_REMOTE_FLOW_TABLE))
93 return false;
94
95 if (!(obj_allowed & MLX5_ALLOWED_OBJ_FOR_OTHER_VHCA_ACCESS_FLOW_TABLE))
96 return false;
97
98 return true;
99 }
100
mlx5_sd_is_supported(struct mlx5_core_dev * dev,u8 host_buses)101 static bool mlx5_sd_is_supported(struct mlx5_core_dev *dev, u8 host_buses)
102 {
103 /* Honor the SW implementation limit */
104 if (host_buses > MLX5_SD_MAX_GROUP_SZ)
105 return false;
106
107 /* Disconnect secondaries from the network */
108 if (!MLX5_CAP_GEN(dev, eswitch_manager))
109 return false;
110 if (!MLX5_CAP_GEN(dev, silent_mode))
111 return false;
112
113 /* RX steering from primary to secondaries */
114 if (!MLX5_CAP_GEN(dev, cross_vhca_rqt))
115 return false;
116 if (host_buses > MLX5_CAP_GEN_2(dev, max_rqt_vhca_id))
117 return false;
118
119 /* TX steering from secondaries to primary */
120 if (!ft_create_alias_supported(dev))
121 return false;
122 if (!MLX5_CAP_FLOWTABLE_NIC_TX(dev, reset_root_to_default))
123 return false;
124
125 return true;
126 }
127
mlx5_query_sd(struct mlx5_core_dev * dev,bool * sdm,u8 * host_buses)128 static int mlx5_query_sd(struct mlx5_core_dev *dev, bool *sdm,
129 u8 *host_buses)
130 {
131 u32 out[MLX5_ST_SZ_DW(mpir_reg)];
132 int err;
133
134 err = mlx5_query_mpir_reg(dev, out);
135 if (err)
136 return err;
137
138 *sdm = MLX5_GET(mpir_reg, out, sdm);
139 *host_buses = MLX5_GET(mpir_reg, out, host_buses);
140
141 return 0;
142 }
143
mlx5_sd_group_id(struct mlx5_core_dev * dev,u8 sd_group)144 static u32 mlx5_sd_group_id(struct mlx5_core_dev *dev, u8 sd_group)
145 {
146 return (u32)((MLX5_CAP_GEN(dev, native_port_num) << 8) | sd_group);
147 }
148
sd_init(struct mlx5_core_dev * dev)149 static int sd_init(struct mlx5_core_dev *dev)
150 {
151 u8 host_buses, sd_group;
152 struct mlx5_sd *sd;
153 u32 group_id;
154 bool sdm;
155 int err;
156
157 /* Feature is currently implemented for PFs only */
158 if (!mlx5_core_is_pf(dev))
159 return 0;
160
161 /* Block on embedded CPU PFs */
162 if (mlx5_core_is_ecpf(dev))
163 return 0;
164
165 err = mlx5_query_nic_vport_sd_group(dev, &sd_group);
166 if (err)
167 return err;
168
169 if (!sd_group)
170 return 0;
171
172 if (!MLX5_CAP_MCAM_REG(dev, mpir))
173 return 0;
174
175 err = mlx5_query_sd(dev, &sdm, &host_buses);
176 if (err)
177 return err;
178
179 if (!sdm)
180 return 0;
181
182 group_id = mlx5_sd_group_id(dev, sd_group);
183
184 if (!mlx5_sd_is_supported(dev, host_buses)) {
185 sd_warn(dev, "can't support requested netdev combining for group id 0x%x), skipping\n",
186 group_id);
187 return 0;
188 }
189
190 sd = kzalloc(sizeof(*sd), GFP_KERNEL);
191 if (!sd)
192 return -ENOMEM;
193
194 sd->host_buses = host_buses;
195 sd->group_id = group_id;
196
197 mlx5_set_sd(dev, sd);
198
199 return 0;
200 }
201
sd_cleanup(struct mlx5_core_dev * dev)202 static void sd_cleanup(struct mlx5_core_dev *dev)
203 {
204 struct mlx5_sd *sd = mlx5_get_sd(dev);
205
206 mlx5_set_sd(dev, NULL);
207 kfree(sd);
208 }
209
sd_register(struct mlx5_core_dev * dev)210 static int sd_register(struct mlx5_core_dev *dev)
211 {
212 struct mlx5_devcom_comp_dev *devcom, *pos;
213 struct mlx5_core_dev *peer, *primary;
214 struct mlx5_sd *sd, *primary_sd;
215 int err, i;
216
217 sd = mlx5_get_sd(dev);
218 devcom = mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_SD_GROUP,
219 sd->group_id, NULL, dev);
220 if (IS_ERR(devcom))
221 return PTR_ERR(devcom);
222
223 sd->devcom = devcom;
224
225 if (mlx5_devcom_comp_get_size(devcom) != sd->host_buses)
226 return 0;
227
228 mlx5_devcom_comp_lock(devcom);
229 mlx5_devcom_comp_set_ready(devcom, true);
230 mlx5_devcom_comp_unlock(devcom);
231
232 if (!mlx5_devcom_for_each_peer_begin(devcom)) {
233 err = -ENODEV;
234 goto err_devcom_unreg;
235 }
236
237 primary = dev;
238 mlx5_devcom_for_each_peer_entry(devcom, peer, pos)
239 if (peer->pdev->bus->number < primary->pdev->bus->number)
240 primary = peer;
241
242 primary_sd = mlx5_get_sd(primary);
243 primary_sd->primary = true;
244 i = 0;
245 /* loop the secondaries */
246 mlx5_devcom_for_each_peer_entry(primary_sd->devcom, peer, pos) {
247 struct mlx5_sd *peer_sd = mlx5_get_sd(peer);
248
249 primary_sd->secondaries[i++] = peer;
250 peer_sd->primary = false;
251 peer_sd->primary_dev = primary;
252 }
253
254 mlx5_devcom_for_each_peer_end(devcom);
255 return 0;
256
257 err_devcom_unreg:
258 mlx5_devcom_comp_lock(sd->devcom);
259 mlx5_devcom_comp_set_ready(sd->devcom, false);
260 mlx5_devcom_comp_unlock(sd->devcom);
261 mlx5_devcom_unregister_component(sd->devcom);
262 return err;
263 }
264
sd_unregister(struct mlx5_core_dev * dev)265 static void sd_unregister(struct mlx5_core_dev *dev)
266 {
267 struct mlx5_sd *sd = mlx5_get_sd(dev);
268
269 mlx5_devcom_comp_lock(sd->devcom);
270 mlx5_devcom_comp_set_ready(sd->devcom, false);
271 mlx5_devcom_comp_unlock(sd->devcom);
272 mlx5_devcom_unregister_component(sd->devcom);
273 }
274
sd_cmd_set_primary(struct mlx5_core_dev * primary,u8 * alias_key)275 static int sd_cmd_set_primary(struct mlx5_core_dev *primary, u8 *alias_key)
276 {
277 struct mlx5_cmd_allow_other_vhca_access_attr allow_attr = {};
278 struct mlx5_sd *sd = mlx5_get_sd(primary);
279 struct mlx5_flow_table_attr ft_attr = {};
280 struct mlx5_flow_namespace *nic_ns;
281 struct mlx5_flow_table *ft;
282 int err;
283
284 nic_ns = mlx5_get_flow_namespace(primary, MLX5_FLOW_NAMESPACE_EGRESS);
285 if (!nic_ns)
286 return -EOPNOTSUPP;
287
288 ft = mlx5_create_flow_table(nic_ns, &ft_attr);
289 if (IS_ERR(ft)) {
290 err = PTR_ERR(ft);
291 return err;
292 }
293 sd->tx_ft = ft;
294 memcpy(allow_attr.access_key, alias_key, ACCESS_KEY_LEN);
295 allow_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
296 allow_attr.obj_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id;
297
298 err = mlx5_cmd_allow_other_vhca_access(primary, &allow_attr);
299 if (err) {
300 mlx5_core_err(primary, "Failed to allow other vhca access err=%d\n",
301 err);
302 mlx5_destroy_flow_table(ft);
303 return err;
304 }
305
306 return 0;
307 }
308
sd_cmd_unset_primary(struct mlx5_core_dev * primary)309 static void sd_cmd_unset_primary(struct mlx5_core_dev *primary)
310 {
311 struct mlx5_sd *sd = mlx5_get_sd(primary);
312
313 mlx5_destroy_flow_table(sd->tx_ft);
314 }
315
sd_secondary_create_alias_ft(struct mlx5_core_dev * secondary,struct mlx5_core_dev * primary,struct mlx5_flow_table * ft,u32 * obj_id,u8 * alias_key)316 static int sd_secondary_create_alias_ft(struct mlx5_core_dev *secondary,
317 struct mlx5_core_dev *primary,
318 struct mlx5_flow_table *ft,
319 u32 *obj_id, u8 *alias_key)
320 {
321 u32 aliased_object_id = (ft->type << FT_ID_FT_TYPE_OFFSET) | ft->id;
322 u16 vhca_id_to_be_accessed = MLX5_CAP_GEN(primary, vhca_id);
323 struct mlx5_cmd_alias_obj_create_attr alias_attr = {};
324 int ret;
325
326 memcpy(alias_attr.access_key, alias_key, ACCESS_KEY_LEN);
327 alias_attr.obj_id = aliased_object_id;
328 alias_attr.obj_type = MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS;
329 alias_attr.vhca_id = vhca_id_to_be_accessed;
330 ret = mlx5_cmd_alias_obj_create(secondary, &alias_attr, obj_id);
331 if (ret) {
332 mlx5_core_err(secondary, "Failed to create alias object err=%d\n",
333 ret);
334 return ret;
335 }
336
337 return 0;
338 }
339
sd_secondary_destroy_alias_ft(struct mlx5_core_dev * secondary)340 static void sd_secondary_destroy_alias_ft(struct mlx5_core_dev *secondary)
341 {
342 struct mlx5_sd *sd = mlx5_get_sd(secondary);
343
344 mlx5_cmd_alias_obj_destroy(secondary, sd->alias_obj_id,
345 MLX5_GENERAL_OBJECT_TYPES_FLOW_TABLE_ALIAS);
346 }
347
sd_cmd_set_secondary(struct mlx5_core_dev * secondary,struct mlx5_core_dev * primary,u8 * alias_key)348 static int sd_cmd_set_secondary(struct mlx5_core_dev *secondary,
349 struct mlx5_core_dev *primary,
350 u8 *alias_key)
351 {
352 struct mlx5_sd *primary_sd = mlx5_get_sd(primary);
353 struct mlx5_sd *sd = mlx5_get_sd(secondary);
354 int err;
355
356 err = mlx5_fs_cmd_set_l2table_entry_silent(secondary, 1);
357 if (err)
358 return err;
359
360 err = sd_secondary_create_alias_ft(secondary, primary, primary_sd->tx_ft,
361 &sd->alias_obj_id, alias_key);
362 if (err)
363 goto err_unset_silent;
364
365 err = mlx5_fs_cmd_set_tx_flow_table_root(secondary, sd->alias_obj_id, false);
366 if (err)
367 goto err_destroy_alias_ft;
368
369 return 0;
370
371 err_destroy_alias_ft:
372 sd_secondary_destroy_alias_ft(secondary);
373 err_unset_silent:
374 mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0);
375 return err;
376 }
377
sd_cmd_unset_secondary(struct mlx5_core_dev * secondary)378 static void sd_cmd_unset_secondary(struct mlx5_core_dev *secondary)
379 {
380 mlx5_fs_cmd_set_tx_flow_table_root(secondary, 0, true);
381 sd_secondary_destroy_alias_ft(secondary);
382 mlx5_fs_cmd_set_l2table_entry_silent(secondary, 0);
383 }
384
sd_print_group(struct mlx5_core_dev * primary)385 static void sd_print_group(struct mlx5_core_dev *primary)
386 {
387 struct mlx5_sd *sd = mlx5_get_sd(primary);
388 struct mlx5_core_dev *pos;
389 int i;
390
391 sd_info(primary, "group id %#x, primary %s, vhca %#x\n",
392 sd->group_id, pci_name(primary->pdev),
393 MLX5_CAP_GEN(primary, vhca_id));
394 mlx5_sd_for_each_secondary(i, primary, pos)
395 sd_info(primary, "group id %#x, secondary_%d %s, vhca %#x\n",
396 sd->group_id, i - 1, pci_name(pos->pdev),
397 MLX5_CAP_GEN(pos, vhca_id));
398 }
399
dev_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)400 static ssize_t dev_read(struct file *filp, char __user *buf, size_t count,
401 loff_t *pos)
402 {
403 struct mlx5_core_dev *dev;
404 char tbuf[32];
405 int ret;
406
407 dev = filp->private_data;
408 ret = snprintf(tbuf, sizeof(tbuf), "%s vhca %#x\n", pci_name(dev->pdev),
409 MLX5_CAP_GEN(dev, vhca_id));
410
411 return simple_read_from_buffer(buf, count, pos, tbuf, ret);
412 }
413
414 static const struct file_operations dev_fops = {
415 .owner = THIS_MODULE,
416 .open = simple_open,
417 .read = dev_read,
418 };
419
mlx5_sd_init(struct mlx5_core_dev * dev)420 int mlx5_sd_init(struct mlx5_core_dev *dev)
421 {
422 struct mlx5_core_dev *primary, *pos, *to;
423 struct mlx5_sd *sd = mlx5_get_sd(dev);
424 u8 alias_key[ACCESS_KEY_LEN];
425 int err, i;
426
427 err = sd_init(dev);
428 if (err)
429 return err;
430
431 sd = mlx5_get_sd(dev);
432 if (!sd)
433 return 0;
434
435 err = sd_register(dev);
436 if (err)
437 goto err_sd_cleanup;
438
439 if (!mlx5_devcom_comp_is_ready(sd->devcom))
440 return 0;
441
442 primary = mlx5_sd_get_primary(dev);
443
444 for (i = 0; i < ACCESS_KEY_LEN; i++)
445 alias_key[i] = get_random_u8();
446
447 err = sd_cmd_set_primary(primary, alias_key);
448 if (err)
449 goto err_sd_unregister;
450
451 sd->dfs = debugfs_create_dir("multi-pf", mlx5_debugfs_get_dev_root(primary));
452 debugfs_create_x32("group_id", 0400, sd->dfs, &sd->group_id);
453 debugfs_create_file("primary", 0400, sd->dfs, primary, &dev_fops);
454
455 mlx5_sd_for_each_secondary(i, primary, pos) {
456 char name[32];
457
458 err = sd_cmd_set_secondary(pos, primary, alias_key);
459 if (err)
460 goto err_unset_secondaries;
461
462 snprintf(name, sizeof(name), "secondary_%d", i - 1);
463 debugfs_create_file(name, 0400, sd->dfs, pos, &dev_fops);
464
465 }
466
467 sd_info(primary, "group id %#x, size %d, combined\n",
468 sd->group_id, mlx5_devcom_comp_get_size(sd->devcom));
469 sd_print_group(primary);
470
471 return 0;
472
473 err_unset_secondaries:
474 to = pos;
475 mlx5_sd_for_each_secondary_to(i, primary, to, pos)
476 sd_cmd_unset_secondary(pos);
477 sd_cmd_unset_primary(primary);
478 debugfs_remove_recursive(sd->dfs);
479 err_sd_unregister:
480 sd_unregister(dev);
481 err_sd_cleanup:
482 sd_cleanup(dev);
483 return err;
484 }
485
mlx5_sd_cleanup(struct mlx5_core_dev * dev)486 void mlx5_sd_cleanup(struct mlx5_core_dev *dev)
487 {
488 struct mlx5_sd *sd = mlx5_get_sd(dev);
489 struct mlx5_core_dev *primary, *pos;
490 int i;
491
492 if (!sd)
493 return;
494
495 if (!mlx5_devcom_comp_is_ready(sd->devcom))
496 goto out;
497
498 primary = mlx5_sd_get_primary(dev);
499 mlx5_sd_for_each_secondary(i, primary, pos)
500 sd_cmd_unset_secondary(pos);
501 sd_cmd_unset_primary(primary);
502 debugfs_remove_recursive(sd->dfs);
503
504 sd_info(primary, "group id %#x, uncombined\n", sd->group_id);
505 out:
506 sd_unregister(dev);
507 sd_cleanup(dev);
508 }
509
mlx5_sd_get_adev(struct mlx5_core_dev * dev,struct auxiliary_device * adev,int idx)510 struct auxiliary_device *mlx5_sd_get_adev(struct mlx5_core_dev *dev,
511 struct auxiliary_device *adev,
512 int idx)
513 {
514 struct mlx5_sd *sd = mlx5_get_sd(dev);
515 struct mlx5_core_dev *primary;
516
517 if (!sd)
518 return adev;
519
520 if (!mlx5_devcom_comp_is_ready(sd->devcom))
521 return NULL;
522
523 primary = mlx5_sd_get_primary(dev);
524 if (dev == primary)
525 return adev;
526
527 return &primary->priv.adev[idx]->adev;
528 }
529