1 // SPDX-License-Identifier: BSD-3-Clause
2 /*
3 * Copyright (c) 2020, MIPI Alliance, Inc.
4 *
5 * Author: Nicolas Pitre <npitre@baylibre.com>
6 *
7 * Note: The I3C HCI v2.0 spec is still in flux. The IBI support is based on
8 * v1.x of the spec and v2.0 will likely be split out.
9 */
10
11 #include <linux/bitfield.h>
12 #include <linux/device.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/errno.h>
15 #include <linux/i3c/master.h>
16 #include <linux/io.h>
17
18 #include "hci.h"
19 #include "cmd.h"
20 #include "ibi.h"
21
22
23 /*
24 * Software Parameter Values (somewhat arb itrary for now).
25 * Some of them could be determined at run time eventually.
26 */
27
28 #define XFER_RINGS 1 /* max: 8 */
29 #define XFER_RING_ENTRIES 16 /* max: 255 */
30
31 #define IBI_RINGS 1 /* max: 8 */
32 #define IBI_STATUS_RING_ENTRIES 32 /* max: 255 */
33 #define IBI_CHUNK_CACHELINES 1 /* max: 256 bytes equivalent */
34 #define IBI_CHUNK_POOL_SIZE 128 /* max: 1023 */
35
36 /*
37 * Ring Header Preamble
38 */
39
40 #define rhs_reg_read(r) readl(hci->RHS_regs + (RHS_##r))
41 #define rhs_reg_write(r, v) writel(v, hci->RHS_regs + (RHS_##r))
42
43 #define RHS_CONTROL 0x00
44 #define PREAMBLE_SIZE GENMASK(31, 24) /* Preamble Section Size */
45 #define HEADER_SIZE GENMASK(23, 16) /* Ring Header Size */
46 #define MAX_HEADER_COUNT_CAP GENMASK(7, 4) /* HC Max Header Count */
47 #define MAX_HEADER_COUNT GENMASK(3, 0) /* Driver Max Header Count */
48
49 #define RHS_RHn_OFFSET(n) (0x04 + (n)*4)
50
51 /*
52 * Ring Header (Per-Ring Bundle)
53 */
54
55 #define rh_reg_read(r) readl(rh->regs + (RH_##r))
56 #define rh_reg_write(r, v) writel(v, rh->regs + (RH_##r))
57
58 #define RH_CR_SETUP 0x00 /* Command/Response Ring */
59 #define CR_XFER_STRUCT_SIZE GENMASK(31, 24)
60 #define CR_RESP_STRUCT_SIZE GENMASK(23, 16)
61 #define CR_RING_SIZE GENMASK(8, 0)
62
63 #define RH_IBI_SETUP 0x04
64 #define IBI_STATUS_STRUCT_SIZE GENMASK(31, 24)
65 #define IBI_STATUS_RING_SIZE GENMASK(23, 16)
66 #define IBI_DATA_CHUNK_SIZE GENMASK(12, 10)
67 #define IBI_DATA_CHUNK_COUNT GENMASK(9, 0)
68
69 #define RH_CHUNK_CONTROL 0x08
70
71 #define RH_INTR_STATUS 0x10
72 #define RH_INTR_STATUS_ENABLE 0x14
73 #define RH_INTR_SIGNAL_ENABLE 0x18
74 #define RH_INTR_FORCE 0x1c
75 #define INTR_IBI_READY BIT(12)
76 #define INTR_TRANSFER_COMPLETION BIT(11)
77 #define INTR_RING_OP BIT(10)
78 #define INTR_TRANSFER_ERR BIT(9)
79 #define INTR_WARN_INS_STOP_MODE BIT(7)
80 #define INTR_IBI_RING_FULL BIT(6)
81 #define INTR_TRANSFER_ABORT BIT(5)
82
83 #define RH_RING_STATUS 0x20
84 #define RING_STATUS_LOCKED BIT(3)
85 #define RING_STATUS_ABORTED BIT(2)
86 #define RING_STATUS_RUNNING BIT(1)
87 #define RING_STATUS_ENABLED BIT(0)
88
89 #define RH_RING_CONTROL 0x24
90 #define RING_CTRL_ABORT BIT(2)
91 #define RING_CTRL_RUN_STOP BIT(1)
92 #define RING_CTRL_ENABLE BIT(0)
93
94 #define RH_RING_OPERATION1 0x28
95 #define RING_OP1_IBI_DEQ_PTR GENMASK(23, 16)
96 #define RING_OP1_CR_SW_DEQ_PTR GENMASK(15, 8)
97 #define RING_OP1_CR_ENQ_PTR GENMASK(7, 0)
98
99 #define RH_RING_OPERATION2 0x2c
100 #define RING_OP2_IBI_ENQ_PTR GENMASK(23, 16)
101 #define RING_OP2_CR_DEQ_PTR GENMASK(7, 0)
102
103 #define RH_CMD_RING_BASE_LO 0x30
104 #define RH_CMD_RING_BASE_HI 0x34
105 #define RH_RESP_RING_BASE_LO 0x38
106 #define RH_RESP_RING_BASE_HI 0x3c
107 #define RH_IBI_STATUS_RING_BASE_LO 0x40
108 #define RH_IBI_STATUS_RING_BASE_HI 0x44
109 #define RH_IBI_DATA_RING_BASE_LO 0x48
110 #define RH_IBI_DATA_RING_BASE_HI 0x4c
111
112 #define RH_CMD_RING_SG 0x50 /* Ring Scatter Gather Support */
113 #define RH_RESP_RING_SG 0x54
114 #define RH_IBI_STATUS_RING_SG 0x58
115 #define RH_IBI_DATA_RING_SG 0x5c
116 #define RING_SG_BLP BIT(31) /* Buffer Vs. List Pointer */
117 #define RING_SG_LIST_SIZE GENMASK(15, 0)
118
119 /*
120 * Data Buffer Descriptor (in memory)
121 */
122
123 #define DATA_BUF_BLP BIT(31) /* Buffer Vs. List Pointer */
124 #define DATA_BUF_IOC BIT(30) /* Interrupt on Completion */
125 #define DATA_BUF_BLOCK_SIZE GENMASK(15, 0)
126
127
128 struct hci_rh_data {
129 void __iomem *regs;
130 void *xfer, *resp, *ibi_status, *ibi_data;
131 dma_addr_t xfer_dma, resp_dma, ibi_status_dma, ibi_data_dma;
132 unsigned int xfer_entries, ibi_status_entries, ibi_chunks_total;
133 unsigned int xfer_struct_sz, resp_struct_sz, ibi_status_sz, ibi_chunk_sz;
134 unsigned int done_ptr, ibi_chunk_ptr;
135 struct hci_xfer **src_xfers;
136 spinlock_t lock;
137 struct completion op_done;
138 };
139
140 struct hci_rings_data {
141 unsigned int total;
142 struct hci_rh_data headers[] __counted_by(total);
143 };
144
145 struct hci_dma_dev_ibi_data {
146 struct i3c_generic_ibi_pool *pool;
147 unsigned int max_len;
148 };
149
hci_dma_cleanup(struct i3c_hci * hci)150 static void hci_dma_cleanup(struct i3c_hci *hci)
151 {
152 struct hci_rings_data *rings = hci->io_data;
153 struct hci_rh_data *rh;
154 unsigned int i;
155
156 if (!rings)
157 return;
158
159 for (i = 0; i < rings->total; i++) {
160 rh = &rings->headers[i];
161
162 rh_reg_write(RING_CONTROL, 0);
163 rh_reg_write(CR_SETUP, 0);
164 rh_reg_write(IBI_SETUP, 0);
165 rh_reg_write(INTR_SIGNAL_ENABLE, 0);
166
167 if (rh->xfer)
168 dma_free_coherent(&hci->master.dev,
169 rh->xfer_struct_sz * rh->xfer_entries,
170 rh->xfer, rh->xfer_dma);
171 if (rh->resp)
172 dma_free_coherent(&hci->master.dev,
173 rh->resp_struct_sz * rh->xfer_entries,
174 rh->resp, rh->resp_dma);
175 kfree(rh->src_xfers);
176 if (rh->ibi_status)
177 dma_free_coherent(&hci->master.dev,
178 rh->ibi_status_sz * rh->ibi_status_entries,
179 rh->ibi_status, rh->ibi_status_dma);
180 if (rh->ibi_data_dma)
181 dma_unmap_single(&hci->master.dev, rh->ibi_data_dma,
182 rh->ibi_chunk_sz * rh->ibi_chunks_total,
183 DMA_FROM_DEVICE);
184 kfree(rh->ibi_data);
185 }
186
187 rhs_reg_write(CONTROL, 0);
188
189 kfree(rings);
190 hci->io_data = NULL;
191 }
192
hci_dma_init(struct i3c_hci * hci)193 static int hci_dma_init(struct i3c_hci *hci)
194 {
195 struct hci_rings_data *rings;
196 struct hci_rh_data *rh;
197 u32 regval;
198 unsigned int i, nr_rings, xfers_sz, resps_sz;
199 unsigned int ibi_status_ring_sz, ibi_data_ring_sz;
200 int ret;
201
202 regval = rhs_reg_read(CONTROL);
203 nr_rings = FIELD_GET(MAX_HEADER_COUNT_CAP, regval);
204 dev_info(&hci->master.dev, "%d DMA rings available\n", nr_rings);
205 if (unlikely(nr_rings > 8)) {
206 dev_err(&hci->master.dev, "number of rings should be <= 8\n");
207 nr_rings = 8;
208 }
209 if (nr_rings > XFER_RINGS)
210 nr_rings = XFER_RINGS;
211 rings = kzalloc(struct_size(rings, headers, nr_rings), GFP_KERNEL);
212 if (!rings)
213 return -ENOMEM;
214 hci->io_data = rings;
215 rings->total = nr_rings;
216
217 regval = FIELD_PREP(MAX_HEADER_COUNT, rings->total);
218 rhs_reg_write(CONTROL, regval);
219
220 for (i = 0; i < rings->total; i++) {
221 u32 offset = rhs_reg_read(RHn_OFFSET(i));
222
223 dev_info(&hci->master.dev, "Ring %d at offset %#x\n", i, offset);
224 ret = -EINVAL;
225 if (!offset)
226 goto err_out;
227 rh = &rings->headers[i];
228 rh->regs = hci->base_regs + offset;
229 spin_lock_init(&rh->lock);
230 init_completion(&rh->op_done);
231
232 rh->xfer_entries = XFER_RING_ENTRIES;
233
234 regval = rh_reg_read(CR_SETUP);
235 rh->xfer_struct_sz = FIELD_GET(CR_XFER_STRUCT_SIZE, regval);
236 rh->resp_struct_sz = FIELD_GET(CR_RESP_STRUCT_SIZE, regval);
237 DBG("xfer_struct_sz = %d, resp_struct_sz = %d",
238 rh->xfer_struct_sz, rh->resp_struct_sz);
239 xfers_sz = rh->xfer_struct_sz * rh->xfer_entries;
240 resps_sz = rh->resp_struct_sz * rh->xfer_entries;
241
242 rh->xfer = dma_alloc_coherent(&hci->master.dev, xfers_sz,
243 &rh->xfer_dma, GFP_KERNEL);
244 rh->resp = dma_alloc_coherent(&hci->master.dev, resps_sz,
245 &rh->resp_dma, GFP_KERNEL);
246 rh->src_xfers =
247 kmalloc_array(rh->xfer_entries, sizeof(*rh->src_xfers),
248 GFP_KERNEL);
249 ret = -ENOMEM;
250 if (!rh->xfer || !rh->resp || !rh->src_xfers)
251 goto err_out;
252
253 rh_reg_write(CMD_RING_BASE_LO, lower_32_bits(rh->xfer_dma));
254 rh_reg_write(CMD_RING_BASE_HI, upper_32_bits(rh->xfer_dma));
255 rh_reg_write(RESP_RING_BASE_LO, lower_32_bits(rh->resp_dma));
256 rh_reg_write(RESP_RING_BASE_HI, upper_32_bits(rh->resp_dma));
257
258 regval = FIELD_PREP(CR_RING_SIZE, rh->xfer_entries);
259 rh_reg_write(CR_SETUP, regval);
260
261 rh_reg_write(INTR_STATUS_ENABLE, 0xffffffff);
262 rh_reg_write(INTR_SIGNAL_ENABLE, INTR_IBI_READY |
263 INTR_TRANSFER_COMPLETION |
264 INTR_RING_OP |
265 INTR_TRANSFER_ERR |
266 INTR_WARN_INS_STOP_MODE |
267 INTR_IBI_RING_FULL |
268 INTR_TRANSFER_ABORT);
269
270 /* IBIs */
271
272 if (i >= IBI_RINGS)
273 goto ring_ready;
274
275 regval = rh_reg_read(IBI_SETUP);
276 rh->ibi_status_sz = FIELD_GET(IBI_STATUS_STRUCT_SIZE, regval);
277 rh->ibi_status_entries = IBI_STATUS_RING_ENTRIES;
278 rh->ibi_chunks_total = IBI_CHUNK_POOL_SIZE;
279
280 rh->ibi_chunk_sz = dma_get_cache_alignment();
281 rh->ibi_chunk_sz *= IBI_CHUNK_CACHELINES;
282 /*
283 * Round IBI data chunk size to number of bytes supported by
284 * the HW. Chunk size can be 2^n number of DWORDs which is the
285 * same as 2^(n+2) bytes, where n is 0..6.
286 */
287 rh->ibi_chunk_sz = umax(4, rh->ibi_chunk_sz);
288 rh->ibi_chunk_sz = roundup_pow_of_two(rh->ibi_chunk_sz);
289 if (rh->ibi_chunk_sz > 256) {
290 ret = -EINVAL;
291 goto err_out;
292 }
293
294 ibi_status_ring_sz = rh->ibi_status_sz * rh->ibi_status_entries;
295 ibi_data_ring_sz = rh->ibi_chunk_sz * rh->ibi_chunks_total;
296
297 rh->ibi_status =
298 dma_alloc_coherent(&hci->master.dev, ibi_status_ring_sz,
299 &rh->ibi_status_dma, GFP_KERNEL);
300 rh->ibi_data = kmalloc(ibi_data_ring_sz, GFP_KERNEL);
301 ret = -ENOMEM;
302 if (!rh->ibi_status || !rh->ibi_data)
303 goto err_out;
304 rh->ibi_data_dma =
305 dma_map_single(&hci->master.dev, rh->ibi_data,
306 ibi_data_ring_sz, DMA_FROM_DEVICE);
307 if (dma_mapping_error(&hci->master.dev, rh->ibi_data_dma)) {
308 rh->ibi_data_dma = 0;
309 ret = -ENOMEM;
310 goto err_out;
311 }
312
313 rh_reg_write(IBI_STATUS_RING_BASE_LO, lower_32_bits(rh->ibi_status_dma));
314 rh_reg_write(IBI_STATUS_RING_BASE_HI, upper_32_bits(rh->ibi_status_dma));
315 rh_reg_write(IBI_DATA_RING_BASE_LO, lower_32_bits(rh->ibi_data_dma));
316 rh_reg_write(IBI_DATA_RING_BASE_HI, upper_32_bits(rh->ibi_data_dma));
317
318 regval = FIELD_PREP(IBI_STATUS_RING_SIZE,
319 rh->ibi_status_entries) |
320 FIELD_PREP(IBI_DATA_CHUNK_SIZE,
321 ilog2(rh->ibi_chunk_sz) - 2) |
322 FIELD_PREP(IBI_DATA_CHUNK_COUNT,
323 rh->ibi_chunks_total);
324 rh_reg_write(IBI_SETUP, regval);
325
326 regval = rh_reg_read(INTR_SIGNAL_ENABLE);
327 regval |= INTR_IBI_READY;
328 rh_reg_write(INTR_SIGNAL_ENABLE, regval);
329
330 ring_ready:
331 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE |
332 RING_CTRL_RUN_STOP);
333 }
334
335 return 0;
336
337 err_out:
338 hci_dma_cleanup(hci);
339 return ret;
340 }
341
hci_dma_unmap_xfer(struct i3c_hci * hci,struct hci_xfer * xfer_list,unsigned int n)342 static void hci_dma_unmap_xfer(struct i3c_hci *hci,
343 struct hci_xfer *xfer_list, unsigned int n)
344 {
345 struct hci_xfer *xfer;
346 unsigned int i;
347
348 for (i = 0; i < n; i++) {
349 xfer = xfer_list + i;
350 if (!xfer->data)
351 continue;
352 dma_unmap_single(&hci->master.dev,
353 xfer->data_dma, xfer->data_len,
354 xfer->rnw ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
355 }
356 }
357
hci_dma_queue_xfer(struct i3c_hci * hci,struct hci_xfer * xfer_list,int n)358 static int hci_dma_queue_xfer(struct i3c_hci *hci,
359 struct hci_xfer *xfer_list, int n)
360 {
361 struct hci_rings_data *rings = hci->io_data;
362 struct hci_rh_data *rh;
363 unsigned int i, ring, enqueue_ptr;
364 u32 op1_val, op2_val;
365 void *buf;
366
367 /* For now we only use ring 0 */
368 ring = 0;
369 rh = &rings->headers[ring];
370
371 op1_val = rh_reg_read(RING_OPERATION1);
372 enqueue_ptr = FIELD_GET(RING_OP1_CR_ENQ_PTR, op1_val);
373 for (i = 0; i < n; i++) {
374 struct hci_xfer *xfer = xfer_list + i;
375 u32 *ring_data = rh->xfer + rh->xfer_struct_sz * enqueue_ptr;
376
377 /* store cmd descriptor */
378 *ring_data++ = xfer->cmd_desc[0];
379 *ring_data++ = xfer->cmd_desc[1];
380 if (hci->cmd == &mipi_i3c_hci_cmd_v2) {
381 *ring_data++ = xfer->cmd_desc[2];
382 *ring_data++ = xfer->cmd_desc[3];
383 }
384
385 /* first word of Data Buffer Descriptor Structure */
386 if (!xfer->data)
387 xfer->data_len = 0;
388 *ring_data++ =
389 FIELD_PREP(DATA_BUF_BLOCK_SIZE, xfer->data_len) |
390 ((i == n - 1) ? DATA_BUF_IOC : 0);
391
392 /* 2nd and 3rd words of Data Buffer Descriptor Structure */
393 if (xfer->data) {
394 buf = xfer->bounce_buf ? xfer->bounce_buf : xfer->data;
395 xfer->data_dma =
396 dma_map_single(&hci->master.dev,
397 buf,
398 xfer->data_len,
399 xfer->rnw ?
400 DMA_FROM_DEVICE :
401 DMA_TO_DEVICE);
402 if (dma_mapping_error(&hci->master.dev,
403 xfer->data_dma)) {
404 hci_dma_unmap_xfer(hci, xfer_list, i);
405 return -ENOMEM;
406 }
407 *ring_data++ = lower_32_bits(xfer->data_dma);
408 *ring_data++ = upper_32_bits(xfer->data_dma);
409 } else {
410 *ring_data++ = 0;
411 *ring_data++ = 0;
412 }
413
414 /* remember corresponding xfer struct */
415 rh->src_xfers[enqueue_ptr] = xfer;
416 /* remember corresponding ring/entry for this xfer structure */
417 xfer->ring_number = ring;
418 xfer->ring_entry = enqueue_ptr;
419
420 enqueue_ptr = (enqueue_ptr + 1) % rh->xfer_entries;
421
422 /*
423 * We may update the hardware view of the enqueue pointer
424 * only if we didn't reach its dequeue pointer.
425 */
426 op2_val = rh_reg_read(RING_OPERATION2);
427 if (enqueue_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val)) {
428 /* the ring is full */
429 hci_dma_unmap_xfer(hci, xfer_list, i + 1);
430 return -EBUSY;
431 }
432 }
433
434 /* take care to update the hardware enqueue pointer atomically */
435 spin_lock_irq(&rh->lock);
436 op1_val = rh_reg_read(RING_OPERATION1);
437 op1_val &= ~RING_OP1_CR_ENQ_PTR;
438 op1_val |= FIELD_PREP(RING_OP1_CR_ENQ_PTR, enqueue_ptr);
439 rh_reg_write(RING_OPERATION1, op1_val);
440 spin_unlock_irq(&rh->lock);
441
442 return 0;
443 }
444
hci_dma_dequeue_xfer(struct i3c_hci * hci,struct hci_xfer * xfer_list,int n)445 static bool hci_dma_dequeue_xfer(struct i3c_hci *hci,
446 struct hci_xfer *xfer_list, int n)
447 {
448 struct hci_rings_data *rings = hci->io_data;
449 struct hci_rh_data *rh = &rings->headers[xfer_list[0].ring_number];
450 unsigned int i;
451 bool did_unqueue = false;
452
453 /* stop the ring */
454 rh_reg_write(RING_CONTROL, RING_CTRL_ABORT);
455 if (wait_for_completion_timeout(&rh->op_done, HZ) == 0) {
456 /*
457 * We're deep in it if ever this condition is ever met.
458 * Hardware might still be writing to memory, etc.
459 */
460 dev_crit(&hci->master.dev, "unable to abort the ring\n");
461 WARN_ON(1);
462 }
463
464 for (i = 0; i < n; i++) {
465 struct hci_xfer *xfer = xfer_list + i;
466 int idx = xfer->ring_entry;
467
468 /*
469 * At the time the abort happened, the xfer might have
470 * completed already. If not then replace corresponding
471 * descriptor entries with a no-op.
472 */
473 if (idx >= 0) {
474 u32 *ring_data = rh->xfer + rh->xfer_struct_sz * idx;
475
476 /* store no-op cmd descriptor */
477 *ring_data++ = FIELD_PREP(CMD_0_ATTR, 0x7);
478 *ring_data++ = 0;
479 if (hci->cmd == &mipi_i3c_hci_cmd_v2) {
480 *ring_data++ = 0;
481 *ring_data++ = 0;
482 }
483
484 /* disassociate this xfer struct */
485 rh->src_xfers[idx] = NULL;
486
487 /* and unmap it */
488 hci_dma_unmap_xfer(hci, xfer, 1);
489
490 did_unqueue = true;
491 }
492 }
493
494 /* restart the ring */
495 rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE);
496
497 return did_unqueue;
498 }
499
hci_dma_xfer_done(struct i3c_hci * hci,struct hci_rh_data * rh)500 static void hci_dma_xfer_done(struct i3c_hci *hci, struct hci_rh_data *rh)
501 {
502 u32 op1_val, op2_val, resp, *ring_resp;
503 unsigned int tid, done_ptr = rh->done_ptr;
504 struct hci_xfer *xfer;
505
506 for (;;) {
507 op2_val = rh_reg_read(RING_OPERATION2);
508 if (done_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val))
509 break;
510
511 ring_resp = rh->resp + rh->resp_struct_sz * done_ptr;
512 resp = *ring_resp;
513 tid = RESP_TID(resp);
514 DBG("resp = 0x%08x", resp);
515
516 xfer = rh->src_xfers[done_ptr];
517 if (!xfer) {
518 DBG("orphaned ring entry");
519 } else {
520 hci_dma_unmap_xfer(hci, xfer, 1);
521 xfer->ring_entry = -1;
522 xfer->response = resp;
523 if (tid != xfer->cmd_tid) {
524 dev_err(&hci->master.dev,
525 "response tid=%d when expecting %d\n",
526 tid, xfer->cmd_tid);
527 /* TODO: do something about it? */
528 }
529 if (xfer->completion)
530 complete(xfer->completion);
531 }
532
533 done_ptr = (done_ptr + 1) % rh->xfer_entries;
534 rh->done_ptr = done_ptr;
535 }
536
537 /* take care to update the software dequeue pointer atomically */
538 spin_lock(&rh->lock);
539 op1_val = rh_reg_read(RING_OPERATION1);
540 op1_val &= ~RING_OP1_CR_SW_DEQ_PTR;
541 op1_val |= FIELD_PREP(RING_OP1_CR_SW_DEQ_PTR, done_ptr);
542 rh_reg_write(RING_OPERATION1, op1_val);
543 spin_unlock(&rh->lock);
544 }
545
hci_dma_request_ibi(struct i3c_hci * hci,struct i3c_dev_desc * dev,const struct i3c_ibi_setup * req)546 static int hci_dma_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev,
547 const struct i3c_ibi_setup *req)
548 {
549 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
550 struct i3c_generic_ibi_pool *pool;
551 struct hci_dma_dev_ibi_data *dev_ibi;
552
553 dev_ibi = kmalloc(sizeof(*dev_ibi), GFP_KERNEL);
554 if (!dev_ibi)
555 return -ENOMEM;
556 pool = i3c_generic_ibi_alloc_pool(dev, req);
557 if (IS_ERR(pool)) {
558 kfree(dev_ibi);
559 return PTR_ERR(pool);
560 }
561 dev_ibi->pool = pool;
562 dev_ibi->max_len = req->max_payload_len;
563 dev_data->ibi_data = dev_ibi;
564 return 0;
565 }
566
hci_dma_free_ibi(struct i3c_hci * hci,struct i3c_dev_desc * dev)567 static void hci_dma_free_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev)
568 {
569 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
570 struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data;
571
572 dev_data->ibi_data = NULL;
573 i3c_generic_ibi_free_pool(dev_ibi->pool);
574 kfree(dev_ibi);
575 }
576
hci_dma_recycle_ibi_slot(struct i3c_hci * hci,struct i3c_dev_desc * dev,struct i3c_ibi_slot * slot)577 static void hci_dma_recycle_ibi_slot(struct i3c_hci *hci,
578 struct i3c_dev_desc *dev,
579 struct i3c_ibi_slot *slot)
580 {
581 struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev);
582 struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data;
583
584 i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot);
585 }
586
hci_dma_process_ibi(struct i3c_hci * hci,struct hci_rh_data * rh)587 static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh)
588 {
589 struct i3c_dev_desc *dev;
590 struct i3c_hci_dev_data *dev_data;
591 struct hci_dma_dev_ibi_data *dev_ibi;
592 struct i3c_ibi_slot *slot;
593 u32 op1_val, op2_val, ibi_status_error;
594 unsigned int ptr, enq_ptr, deq_ptr;
595 unsigned int ibi_size, ibi_chunks, ibi_data_offset, first_part;
596 int ibi_addr, last_ptr;
597 void *ring_ibi_data;
598 dma_addr_t ring_ibi_data_dma;
599
600 op1_val = rh_reg_read(RING_OPERATION1);
601 deq_ptr = FIELD_GET(RING_OP1_IBI_DEQ_PTR, op1_val);
602
603 op2_val = rh_reg_read(RING_OPERATION2);
604 enq_ptr = FIELD_GET(RING_OP2_IBI_ENQ_PTR, op2_val);
605
606 ibi_status_error = 0;
607 ibi_addr = -1;
608 ibi_chunks = 0;
609 ibi_size = 0;
610 last_ptr = -1;
611
612 /* let's find all we can about this IBI */
613 for (ptr = deq_ptr; ptr != enq_ptr;
614 ptr = (ptr + 1) % rh->ibi_status_entries) {
615 u32 ibi_status, *ring_ibi_status;
616 unsigned int chunks;
617
618 ring_ibi_status = rh->ibi_status + rh->ibi_status_sz * ptr;
619 ibi_status = *ring_ibi_status;
620 DBG("status = %#x", ibi_status);
621
622 if (ibi_status_error) {
623 /* we no longer care */
624 } else if (ibi_status & IBI_ERROR) {
625 ibi_status_error = ibi_status;
626 } else if (ibi_addr == -1) {
627 ibi_addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status);
628 } else if (ibi_addr != FIELD_GET(IBI_TARGET_ADDR, ibi_status)) {
629 /* the address changed unexpectedly */
630 ibi_status_error = ibi_status;
631 }
632
633 chunks = FIELD_GET(IBI_CHUNKS, ibi_status);
634 ibi_chunks += chunks;
635 if (!(ibi_status & IBI_LAST_STATUS)) {
636 ibi_size += chunks * rh->ibi_chunk_sz;
637 } else {
638 ibi_size += FIELD_GET(IBI_DATA_LENGTH, ibi_status);
639 last_ptr = ptr;
640 break;
641 }
642 }
643
644 /* validate what we've got */
645
646 if (last_ptr == -1) {
647 /* this IBI sequence is not yet complete */
648 DBG("no LAST_STATUS available (e=%d d=%d)", enq_ptr, deq_ptr);
649 return;
650 }
651 deq_ptr = last_ptr + 1;
652 deq_ptr %= rh->ibi_status_entries;
653
654 if (ibi_status_error) {
655 dev_err(&hci->master.dev, "IBI error from %#x\n", ibi_addr);
656 goto done;
657 }
658
659 /* determine who this is for */
660 dev = i3c_hci_addr_to_dev(hci, ibi_addr);
661 if (!dev) {
662 dev_err(&hci->master.dev,
663 "IBI for unknown device %#x\n", ibi_addr);
664 goto done;
665 }
666
667 dev_data = i3c_dev_get_master_data(dev);
668 dev_ibi = dev_data->ibi_data;
669 if (ibi_size > dev_ibi->max_len) {
670 dev_err(&hci->master.dev, "IBI payload too big (%d > %d)\n",
671 ibi_size, dev_ibi->max_len);
672 goto done;
673 }
674
675 /*
676 * This ring model is not suitable for zero-copy processing of IBIs.
677 * We have the data chunk ring wrap-around to deal with, meaning
678 * that the payload might span multiple chunks beginning at the
679 * end of the ring and wrap to the start of the ring. Furthermore
680 * there is no guarantee that those chunks will be released in order
681 * and in a timely manner by the upper driver. So let's just copy
682 * them to a discrete buffer. In practice they're supposed to be
683 * small anyway.
684 */
685 slot = i3c_generic_ibi_get_free_slot(dev_ibi->pool);
686 if (!slot) {
687 dev_err(&hci->master.dev, "no free slot for IBI\n");
688 goto done;
689 }
690
691 /* copy first part of the payload */
692 ibi_data_offset = rh->ibi_chunk_sz * rh->ibi_chunk_ptr;
693 ring_ibi_data = rh->ibi_data + ibi_data_offset;
694 ring_ibi_data_dma = rh->ibi_data_dma + ibi_data_offset;
695 first_part = (rh->ibi_chunks_total - rh->ibi_chunk_ptr)
696 * rh->ibi_chunk_sz;
697 if (first_part > ibi_size)
698 first_part = ibi_size;
699 dma_sync_single_for_cpu(&hci->master.dev, ring_ibi_data_dma,
700 first_part, DMA_FROM_DEVICE);
701 memcpy(slot->data, ring_ibi_data, first_part);
702
703 /* copy second part if any */
704 if (ibi_size > first_part) {
705 /* we wrap back to the start and copy remaining data */
706 ring_ibi_data = rh->ibi_data;
707 ring_ibi_data_dma = rh->ibi_data_dma;
708 dma_sync_single_for_cpu(&hci->master.dev, ring_ibi_data_dma,
709 ibi_size - first_part, DMA_FROM_DEVICE);
710 memcpy(slot->data + first_part, ring_ibi_data,
711 ibi_size - first_part);
712 }
713
714 /* submit it */
715 slot->dev = dev;
716 slot->len = ibi_size;
717 i3c_master_queue_ibi(dev, slot);
718
719 done:
720 /* take care to update the ibi dequeue pointer atomically */
721 spin_lock(&rh->lock);
722 op1_val = rh_reg_read(RING_OPERATION1);
723 op1_val &= ~RING_OP1_IBI_DEQ_PTR;
724 op1_val |= FIELD_PREP(RING_OP1_IBI_DEQ_PTR, deq_ptr);
725 rh_reg_write(RING_OPERATION1, op1_val);
726 spin_unlock(&rh->lock);
727
728 /* update the chunk pointer */
729 rh->ibi_chunk_ptr += ibi_chunks;
730 rh->ibi_chunk_ptr %= rh->ibi_chunks_total;
731
732 /* and tell the hardware about freed chunks */
733 rh_reg_write(CHUNK_CONTROL, rh_reg_read(CHUNK_CONTROL) + ibi_chunks);
734 }
735
hci_dma_irq_handler(struct i3c_hci * hci,unsigned int mask)736 static bool hci_dma_irq_handler(struct i3c_hci *hci, unsigned int mask)
737 {
738 struct hci_rings_data *rings = hci->io_data;
739 unsigned int i;
740 bool handled = false;
741
742 for (i = 0; mask && i < rings->total; i++) {
743 struct hci_rh_data *rh;
744 u32 status;
745
746 if (!(mask & BIT(i)))
747 continue;
748 mask &= ~BIT(i);
749
750 rh = &rings->headers[i];
751 status = rh_reg_read(INTR_STATUS);
752 DBG("rh%d status: %#x", i, status);
753 if (!status)
754 continue;
755 rh_reg_write(INTR_STATUS, status);
756
757 if (status & INTR_IBI_READY)
758 hci_dma_process_ibi(hci, rh);
759 if (status & (INTR_TRANSFER_COMPLETION | INTR_TRANSFER_ERR))
760 hci_dma_xfer_done(hci, rh);
761 if (status & INTR_RING_OP)
762 complete(&rh->op_done);
763
764 if (status & INTR_TRANSFER_ABORT) {
765 dev_notice_ratelimited(&hci->master.dev,
766 "ring %d: Transfer Aborted\n", i);
767 mipi_i3c_hci_resume(hci);
768 }
769 if (status & INTR_WARN_INS_STOP_MODE)
770 dev_warn_ratelimited(&hci->master.dev,
771 "ring %d: Inserted Stop on Mode Change\n", i);
772 if (status & INTR_IBI_RING_FULL)
773 dev_err_ratelimited(&hci->master.dev,
774 "ring %d: IBI Ring Full Condition\n", i);
775
776 handled = true;
777 }
778
779 return handled;
780 }
781
782 const struct hci_io_ops mipi_i3c_hci_dma = {
783 .init = hci_dma_init,
784 .cleanup = hci_dma_cleanup,
785 .queue_xfer = hci_dma_queue_xfer,
786 .dequeue_xfer = hci_dma_dequeue_xfer,
787 .irq_handler = hci_dma_irq_handler,
788 .request_ibi = hci_dma_request_ibi,
789 .free_ibi = hci_dma_free_ibi,
790 .recycle_ibi_slot = hci_dma_recycle_ibi_slot,
791 };
792