1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2020 Mellanox Technologies Ltd */
3 
4 #include <linux/mlx5/driver.h>
5 #include <linux/mlx5/device.h>
6 #include "mlx5_core.h"
7 #include "dev.h"
8 #include "sf/vhca_event.h"
9 #include "sf/sf.h"
10 #include "sf/mlx5_ifc_vhca_event.h"
11 #include "ecpf.h"
12 #define CREATE_TRACE_POINTS
13 #include "diag/dev_tracepoint.h"
14 
15 struct mlx5_sf_dev_table {
16 	struct xarray devices;
17 	phys_addr_t base_address;
18 	u64 sf_bar_length;
19 	struct notifier_block nb;
20 	struct workqueue_struct *active_wq;
21 	struct work_struct work;
22 	u8 stop_active_wq:1;
23 	struct mlx5_core_dev *dev;
24 };
25 
26 struct mlx5_sf_dev_active_work_ctx {
27 	struct work_struct work;
28 	struct mlx5_vhca_state_event event;
29 	struct mlx5_sf_dev_table *table;
30 	int sf_index;
31 };
32 
mlx5_sf_dev_supported(const struct mlx5_core_dev * dev)33 static bool mlx5_sf_dev_supported(const struct mlx5_core_dev *dev)
34 {
35 	return MLX5_CAP_GEN(dev, sf) && mlx5_vhca_event_supported(dev);
36 }
37 
mlx5_sf_dev_allocated(const struct mlx5_core_dev * dev)38 bool mlx5_sf_dev_allocated(const struct mlx5_core_dev *dev)
39 {
40 	struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table;
41 
42 	return table && !xa_empty(&table->devices);
43 }
44 
sfnum_show(struct device * dev,struct device_attribute * attr,char * buf)45 static ssize_t sfnum_show(struct device *dev, struct device_attribute *attr, char *buf)
46 {
47 	struct auxiliary_device *adev = container_of(dev, struct auxiliary_device, dev);
48 	struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev);
49 
50 	return sysfs_emit(buf, "%u\n", sf_dev->sfnum);
51 }
52 static DEVICE_ATTR_RO(sfnum);
53 
54 static struct attribute *sf_device_attrs[] = {
55 	&dev_attr_sfnum.attr,
56 	NULL,
57 };
58 
59 static const struct attribute_group sf_attr_group = {
60 	.attrs = sf_device_attrs,
61 };
62 
63 static const struct attribute_group *sf_attr_groups[2] = {
64 	&sf_attr_group,
65 	NULL
66 };
67 
mlx5_sf_dev_release(struct device * device)68 static void mlx5_sf_dev_release(struct device *device)
69 {
70 	struct auxiliary_device *adev = container_of(device, struct auxiliary_device, dev);
71 	struct mlx5_sf_dev *sf_dev = container_of(adev, struct mlx5_sf_dev, adev);
72 
73 	mlx5_adev_idx_free(adev->id);
74 	kfree(sf_dev);
75 }
76 
mlx5_sf_dev_remove_aux(struct mlx5_core_dev * dev,struct mlx5_sf_dev * sf_dev)77 static void mlx5_sf_dev_remove_aux(struct mlx5_core_dev *dev,
78 				   struct mlx5_sf_dev *sf_dev)
79 {
80 	int id;
81 
82 	id = sf_dev->adev.id;
83 	trace_mlx5_sf_dev_del(dev, sf_dev, id);
84 
85 	auxiliary_device_delete(&sf_dev->adev);
86 	auxiliary_device_uninit(&sf_dev->adev);
87 }
88 
mlx5_sf_dev_add(struct mlx5_core_dev * dev,u16 sf_index,u16 fn_id,u32 sfnum)89 static void mlx5_sf_dev_add(struct mlx5_core_dev *dev, u16 sf_index, u16 fn_id, u32 sfnum)
90 {
91 	struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table;
92 	struct mlx5_sf_dev *sf_dev;
93 	struct pci_dev *pdev;
94 	int err;
95 	int id;
96 
97 	id = mlx5_adev_idx_alloc();
98 	if (id < 0) {
99 		err = id;
100 		goto add_err;
101 	}
102 
103 	sf_dev = kzalloc(sizeof(*sf_dev), GFP_KERNEL);
104 	if (!sf_dev) {
105 		mlx5_adev_idx_free(id);
106 		err = -ENOMEM;
107 		goto add_err;
108 	}
109 	pdev = dev->pdev;
110 	sf_dev->adev.id = id;
111 	sf_dev->adev.name = MLX5_SF_DEV_ID_NAME;
112 	sf_dev->adev.dev.release = mlx5_sf_dev_release;
113 	sf_dev->adev.dev.parent = &pdev->dev;
114 	sf_dev->adev.dev.groups = sf_attr_groups;
115 	sf_dev->sfnum = sfnum;
116 	sf_dev->parent_mdev = dev;
117 	sf_dev->fn_id = fn_id;
118 
119 	sf_dev->bar_base_addr = table->base_address + (sf_index * table->sf_bar_length);
120 
121 	trace_mlx5_sf_dev_add(dev, sf_dev, id);
122 
123 	err = auxiliary_device_init(&sf_dev->adev);
124 	if (err) {
125 		mlx5_adev_idx_free(id);
126 		kfree(sf_dev);
127 		goto add_err;
128 	}
129 
130 	err = auxiliary_device_add(&sf_dev->adev);
131 	if (err) {
132 		auxiliary_device_uninit(&sf_dev->adev);
133 		goto add_err;
134 	}
135 
136 	err = xa_insert(&table->devices, sf_index, sf_dev, GFP_KERNEL);
137 	if (err)
138 		goto xa_err;
139 	return;
140 
141 xa_err:
142 	mlx5_sf_dev_remove_aux(dev, sf_dev);
143 add_err:
144 	mlx5_core_err(dev, "SF DEV: fail device add for index=%d sfnum=%d err=%d\n",
145 		      sf_index, sfnum, err);
146 }
147 
mlx5_sf_dev_del(struct mlx5_core_dev * dev,struct mlx5_sf_dev * sf_dev,u16 sf_index)148 static void mlx5_sf_dev_del(struct mlx5_core_dev *dev, struct mlx5_sf_dev *sf_dev, u16 sf_index)
149 {
150 	struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table;
151 
152 	xa_erase(&table->devices, sf_index);
153 	mlx5_sf_dev_remove_aux(dev, sf_dev);
154 }
155 
156 static int
mlx5_sf_dev_state_change_handler(struct notifier_block * nb,unsigned long event_code,void * data)157 mlx5_sf_dev_state_change_handler(struct notifier_block *nb, unsigned long event_code, void *data)
158 {
159 	struct mlx5_sf_dev_table *table = container_of(nb, struct mlx5_sf_dev_table, nb);
160 	const struct mlx5_vhca_state_event *event = data;
161 	struct mlx5_sf_dev *sf_dev;
162 	u16 max_functions;
163 	u16 sf_index;
164 	u16 base_id;
165 
166 	max_functions = mlx5_sf_max_functions(table->dev);
167 	if (!max_functions)
168 		return 0;
169 
170 	base_id = mlx5_sf_start_function_id(table->dev);
171 	if (event->function_id < base_id || event->function_id >= (base_id + max_functions))
172 		return 0;
173 
174 	sf_index = event->function_id - base_id;
175 	sf_dev = xa_load(&table->devices, sf_index);
176 	switch (event->new_vhca_state) {
177 	case MLX5_VHCA_STATE_INVALID:
178 	case MLX5_VHCA_STATE_ALLOCATED:
179 		if (sf_dev)
180 			mlx5_sf_dev_del(table->dev, sf_dev, sf_index);
181 		break;
182 	case MLX5_VHCA_STATE_TEARDOWN_REQUEST:
183 		if (sf_dev)
184 			mlx5_sf_dev_del(table->dev, sf_dev, sf_index);
185 		else
186 			mlx5_core_err(table->dev,
187 				      "SF DEV: teardown state for invalid dev index=%d sfnum=0x%x\n",
188 				      sf_index, event->sw_function_id);
189 		break;
190 	case MLX5_VHCA_STATE_ACTIVE:
191 		if (!sf_dev)
192 			mlx5_sf_dev_add(table->dev, sf_index, event->function_id,
193 					event->sw_function_id);
194 		break;
195 	default:
196 		break;
197 	}
198 	return 0;
199 }
200 
mlx5_sf_dev_vhca_arm_all(struct mlx5_sf_dev_table * table)201 static int mlx5_sf_dev_vhca_arm_all(struct mlx5_sf_dev_table *table)
202 {
203 	struct mlx5_core_dev *dev = table->dev;
204 	u16 max_functions;
205 	u16 function_id;
206 	int err = 0;
207 	int i;
208 
209 	max_functions = mlx5_sf_max_functions(dev);
210 	function_id = mlx5_sf_start_function_id(dev);
211 	/* Arm the vhca context as the vhca event notifier */
212 	for (i = 0; i < max_functions; i++) {
213 		err = mlx5_vhca_event_arm(dev, function_id);
214 		if (err)
215 			return err;
216 
217 		function_id++;
218 	}
219 	return 0;
220 }
221 
mlx5_sf_dev_add_active_work(struct work_struct * _work)222 static void mlx5_sf_dev_add_active_work(struct work_struct *_work)
223 {
224 	struct mlx5_sf_dev_active_work_ctx *work_ctx;
225 
226 	work_ctx = container_of(_work, struct mlx5_sf_dev_active_work_ctx, work);
227 	if (work_ctx->table->stop_active_wq)
228 		goto out;
229 	/* Don't probe device which is already probe */
230 	if (!xa_load(&work_ctx->table->devices, work_ctx->sf_index))
231 		mlx5_sf_dev_add(work_ctx->table->dev, work_ctx->sf_index,
232 				work_ctx->event.function_id, work_ctx->event.sw_function_id);
233 	/* There is a race where SF got inactive after the query
234 	 * above. e.g.: the query returns that the state of the
235 	 * SF is active, and after that the eswitch manager set it to
236 	 * inactive.
237 	 * This case cannot be managed in SW, since the probing of the
238 	 * SF is on one system, and the inactivation is on a different
239 	 * system.
240 	 * If the inactive is done after the SF perform init_hca(),
241 	 * the SF will fully probe and then removed. If it was
242 	 * done before init_hca(), the SF probe will fail.
243 	 */
244 out:
245 	kfree(work_ctx);
246 }
247 
248 /* In case SFs are generated externally, probe active SFs */
mlx5_sf_dev_queue_active_works(struct work_struct * _work)249 static void mlx5_sf_dev_queue_active_works(struct work_struct *_work)
250 {
251 	struct mlx5_sf_dev_table *table = container_of(_work, struct mlx5_sf_dev_table, work);
252 	u32 out[MLX5_ST_SZ_DW(query_vhca_state_out)] = {};
253 	struct mlx5_sf_dev_active_work_ctx *work_ctx;
254 	struct mlx5_core_dev *dev = table->dev;
255 	u16 max_functions;
256 	u16 function_id;
257 	u16 sw_func_id;
258 	int err = 0;
259 	int wq_idx;
260 	u8 state;
261 	int i;
262 
263 	max_functions = mlx5_sf_max_functions(dev);
264 	function_id = mlx5_sf_start_function_id(dev);
265 	for (i = 0; i < max_functions; i++, function_id++) {
266 		if (table->stop_active_wq)
267 			return;
268 		err = mlx5_cmd_query_vhca_state(dev, function_id, out, sizeof(out));
269 		if (err)
270 			/* A failure of specific vhca doesn't mean others will
271 			 * fail as well.
272 			 */
273 			continue;
274 		state = MLX5_GET(query_vhca_state_out, out, vhca_state_context.vhca_state);
275 		if (state != MLX5_VHCA_STATE_ACTIVE)
276 			continue;
277 
278 		sw_func_id = MLX5_GET(query_vhca_state_out, out, vhca_state_context.sw_function_id);
279 		work_ctx = kzalloc(sizeof(*work_ctx), GFP_KERNEL);
280 		if (!work_ctx)
281 			return;
282 
283 		INIT_WORK(&work_ctx->work, &mlx5_sf_dev_add_active_work);
284 		work_ctx->event.function_id = function_id;
285 		work_ctx->event.sw_function_id = sw_func_id;
286 		work_ctx->table = table;
287 		work_ctx->sf_index = i;
288 		wq_idx = work_ctx->event.function_id % MLX5_DEV_MAX_WQS;
289 		mlx5_vhca_events_work_enqueue(dev, wq_idx, &work_ctx->work);
290 	}
291 }
292 
293 /* In case SFs are generated externally, probe active SFs */
mlx5_sf_dev_create_active_works(struct mlx5_sf_dev_table * table)294 static int mlx5_sf_dev_create_active_works(struct mlx5_sf_dev_table *table)
295 {
296 	if (MLX5_CAP_GEN(table->dev, eswitch_manager))
297 		return 0; /* the table is local */
298 
299 	/* Use a workqueue to probe active SFs, which are in large
300 	 * quantity and may take up to minutes to probe.
301 	 */
302 	table->active_wq = create_singlethread_workqueue("mlx5_active_sf");
303 	if (!table->active_wq)
304 		return -ENOMEM;
305 	INIT_WORK(&table->work, &mlx5_sf_dev_queue_active_works);
306 	queue_work(table->active_wq, &table->work);
307 	return 0;
308 }
309 
mlx5_sf_dev_destroy_active_works(struct mlx5_sf_dev_table * table)310 static void mlx5_sf_dev_destroy_active_works(struct mlx5_sf_dev_table *table)
311 {
312 	if (table->active_wq) {
313 		table->stop_active_wq = true;
314 		destroy_workqueue(table->active_wq);
315 	}
316 }
317 
mlx5_sf_dev_table_create(struct mlx5_core_dev * dev)318 void mlx5_sf_dev_table_create(struct mlx5_core_dev *dev)
319 {
320 	struct mlx5_sf_dev_table *table;
321 	int err;
322 
323 	if (!mlx5_sf_dev_supported(dev))
324 		return;
325 
326 	table = kzalloc(sizeof(*table), GFP_KERNEL);
327 	if (!table) {
328 		err = -ENOMEM;
329 		goto table_err;
330 	}
331 
332 	table->nb.notifier_call = mlx5_sf_dev_state_change_handler;
333 	table->dev = dev;
334 	table->sf_bar_length = 1 << (MLX5_CAP_GEN(dev, log_min_sf_size) + 12);
335 	table->base_address = pci_resource_start(dev->pdev, 2);
336 	xa_init(&table->devices);
337 	dev->priv.sf_dev_table = table;
338 
339 	err = mlx5_vhca_event_notifier_register(dev, &table->nb);
340 	if (err)
341 		goto vhca_err;
342 
343 	err = mlx5_sf_dev_create_active_works(table);
344 	if (err)
345 		goto add_active_err;
346 
347 	err = mlx5_sf_dev_vhca_arm_all(table);
348 	if (err)
349 		goto arm_err;
350 	return;
351 
352 arm_err:
353 	mlx5_sf_dev_destroy_active_works(table);
354 add_active_err:
355 	mlx5_vhca_event_notifier_unregister(dev, &table->nb);
356 	mlx5_vhca_event_work_queues_flush(dev);
357 vhca_err:
358 	kfree(table);
359 	dev->priv.sf_dev_table = NULL;
360 table_err:
361 	mlx5_core_err(dev, "SF DEV table create err = %d\n", err);
362 }
363 
mlx5_sf_dev_destroy_all(struct mlx5_sf_dev_table * table)364 static void mlx5_sf_dev_destroy_all(struct mlx5_sf_dev_table *table)
365 {
366 	struct mlx5_sf_dev *sf_dev;
367 	unsigned long index;
368 
369 	xa_for_each(&table->devices, index, sf_dev) {
370 		xa_erase(&table->devices, index);
371 		mlx5_sf_dev_remove_aux(table->dev, sf_dev);
372 	}
373 }
374 
mlx5_sf_dev_table_destroy(struct mlx5_core_dev * dev)375 void mlx5_sf_dev_table_destroy(struct mlx5_core_dev *dev)
376 {
377 	struct mlx5_sf_dev_table *table = dev->priv.sf_dev_table;
378 
379 	if (!table)
380 		return;
381 
382 	mlx5_sf_dev_destroy_active_works(table);
383 	mlx5_vhca_event_notifier_unregister(dev, &table->nb);
384 	mlx5_vhca_event_work_queues_flush(dev);
385 
386 	/* Now that event handler is not running, it is safe to destroy
387 	 * the sf device without race.
388 	 */
389 	mlx5_sf_dev_destroy_all(table);
390 
391 	WARN_ON(!xa_empty(&table->devices));
392 	kfree(table);
393 	dev->priv.sf_dev_table = NULL;
394 }
395