1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2017 - 2018 Intel Corporation
4 * Copyright 2017 Google LLC
5 *
6 * Based on Intel IPU4 driver.
7 *
8 */
9
10 #include <linux/delay.h>
11 #include <linux/interrupt.h>
12 #include <linux/module.h>
13 #include <linux/pm_runtime.h>
14
15 #include "ipu3.h"
16 #include "ipu3-css-fw.h"
17 #include "ipu3-dmamap.h"
18 #include "ipu3-mmu.h"
19
20 #define IMGU_PCI_ID 0x1919
21 #define IMGU_PCI_BAR 0
22 #define IMGU_DMA_MASK DMA_BIT_MASK(39)
23 #define IMGU_MAX_QUEUE_DEPTH (2 + 2)
24
25 /*
26 * pre-allocated buffer size for IMGU dummy buffers. Those
27 * values should be tuned to big enough to avoid buffer
28 * re-allocation when streaming to lower streaming latency.
29 */
30 #define CSS_QUEUE_IN_BUF_SIZE 0
31 #define CSS_QUEUE_PARAMS_BUF_SIZE 0
32 #define CSS_QUEUE_OUT_BUF_SIZE (4160 * 3120 * 12 / 8)
33 #define CSS_QUEUE_VF_BUF_SIZE (1920 * 1080 * 12 / 8)
34 #define CSS_QUEUE_STAT_3A_BUF_SIZE sizeof(struct ipu3_uapi_stats_3a)
35
36 static const size_t css_queue_buf_size_map[IPU3_CSS_QUEUES] = {
37 [IPU3_CSS_QUEUE_IN] = CSS_QUEUE_IN_BUF_SIZE,
38 [IPU3_CSS_QUEUE_PARAMS] = CSS_QUEUE_PARAMS_BUF_SIZE,
39 [IPU3_CSS_QUEUE_OUT] = CSS_QUEUE_OUT_BUF_SIZE,
40 [IPU3_CSS_QUEUE_VF] = CSS_QUEUE_VF_BUF_SIZE,
41 [IPU3_CSS_QUEUE_STAT_3A] = CSS_QUEUE_STAT_3A_BUF_SIZE,
42 };
43
44 static const struct imgu_node_mapping imgu_node_map[IMGU_NODE_NUM] = {
45 [IMGU_NODE_IN] = {IPU3_CSS_QUEUE_IN, "input"},
46 [IMGU_NODE_PARAMS] = {IPU3_CSS_QUEUE_PARAMS, "parameters"},
47 [IMGU_NODE_OUT] = {IPU3_CSS_QUEUE_OUT, "output"},
48 [IMGU_NODE_VF] = {IPU3_CSS_QUEUE_VF, "viewfinder"},
49 [IMGU_NODE_STAT_3A] = {IPU3_CSS_QUEUE_STAT_3A, "3a stat"},
50 };
51
imgu_node_to_queue(unsigned int node)52 unsigned int imgu_node_to_queue(unsigned int node)
53 {
54 return imgu_node_map[node].css_queue;
55 }
56
imgu_map_node(struct imgu_device * imgu,unsigned int css_queue)57 unsigned int imgu_map_node(struct imgu_device *imgu, unsigned int css_queue)
58 {
59 unsigned int i;
60
61 for (i = 0; i < IMGU_NODE_NUM; i++)
62 if (imgu_node_map[i].css_queue == css_queue)
63 break;
64
65 return i;
66 }
67
68 /**************** Dummy buffers ****************/
69
imgu_dummybufs_cleanup(struct imgu_device * imgu,unsigned int pipe)70 static void imgu_dummybufs_cleanup(struct imgu_device *imgu, unsigned int pipe)
71 {
72 unsigned int i;
73 struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
74
75 for (i = 0; i < IPU3_CSS_QUEUES; i++)
76 imgu_dmamap_free(imgu,
77 &imgu_pipe->queues[i].dmap);
78 }
79
imgu_dummybufs_preallocate(struct imgu_device * imgu,unsigned int pipe)80 static int imgu_dummybufs_preallocate(struct imgu_device *imgu,
81 unsigned int pipe)
82 {
83 unsigned int i;
84 size_t size;
85 struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
86
87 for (i = 0; i < IPU3_CSS_QUEUES; i++) {
88 size = css_queue_buf_size_map[i];
89 /*
90 * Do not enable dummy buffers for master queue,
91 * always require that real buffers from user are
92 * available.
93 */
94 if (i == IMGU_QUEUE_MASTER || size == 0)
95 continue;
96
97 if (!imgu_dmamap_alloc(imgu,
98 &imgu_pipe->queues[i].dmap, size)) {
99 imgu_dummybufs_cleanup(imgu, pipe);
100 return -ENOMEM;
101 }
102 }
103
104 return 0;
105 }
106
imgu_dummybufs_init(struct imgu_device * imgu,unsigned int pipe)107 static int imgu_dummybufs_init(struct imgu_device *imgu, unsigned int pipe)
108 {
109 const struct v4l2_pix_format_mplane *mpix;
110 const struct v4l2_meta_format *meta;
111 unsigned int i, k, node;
112 size_t size;
113 struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
114
115 /* Allocate a dummy buffer for each queue where buffer is optional */
116 for (i = 0; i < IPU3_CSS_QUEUES; i++) {
117 node = imgu_map_node(imgu, i);
118 if (!imgu_pipe->queue_enabled[node] || i == IMGU_QUEUE_MASTER)
119 continue;
120
121 if (!imgu_pipe->nodes[IMGU_NODE_VF].enabled &&
122 i == IPU3_CSS_QUEUE_VF)
123 /*
124 * Do not enable dummy buffers for VF if it is not
125 * requested by the user.
126 */
127 continue;
128
129 meta = &imgu_pipe->nodes[node].vdev_fmt.fmt.meta;
130 mpix = &imgu_pipe->nodes[node].vdev_fmt.fmt.pix_mp;
131
132 if (node == IMGU_NODE_STAT_3A || node == IMGU_NODE_PARAMS)
133 size = meta->buffersize;
134 else
135 size = mpix->plane_fmt[0].sizeimage;
136
137 if (imgu_css_dma_buffer_resize(imgu,
138 &imgu_pipe->queues[i].dmap,
139 size)) {
140 imgu_dummybufs_cleanup(imgu, pipe);
141 return -ENOMEM;
142 }
143
144 for (k = 0; k < IMGU_MAX_QUEUE_DEPTH; k++)
145 imgu_css_buf_init(&imgu_pipe->queues[i].dummybufs[k], i,
146 imgu_pipe->queues[i].dmap.daddr);
147 }
148
149 return 0;
150 }
151
152 /* May be called from atomic context */
imgu_dummybufs_get(struct imgu_device * imgu,int queue,unsigned int pipe)153 static struct imgu_css_buffer *imgu_dummybufs_get(struct imgu_device *imgu,
154 int queue, unsigned int pipe)
155 {
156 unsigned int i;
157 struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
158
159 /* dummybufs are not allocated for master q */
160 if (queue == IPU3_CSS_QUEUE_IN)
161 return NULL;
162
163 if (WARN_ON(!imgu_pipe->queues[queue].dmap.vaddr))
164 /* Buffer should not be allocated here */
165 return NULL;
166
167 for (i = 0; i < IMGU_MAX_QUEUE_DEPTH; i++)
168 if (imgu_css_buf_state(&imgu_pipe->queues[queue].dummybufs[i]) !=
169 IPU3_CSS_BUFFER_QUEUED)
170 break;
171
172 if (i == IMGU_MAX_QUEUE_DEPTH)
173 return NULL;
174
175 imgu_css_buf_init(&imgu_pipe->queues[queue].dummybufs[i], queue,
176 imgu_pipe->queues[queue].dmap.daddr);
177
178 return &imgu_pipe->queues[queue].dummybufs[i];
179 }
180
181 /* Check if given buffer is a dummy buffer */
imgu_dummybufs_check(struct imgu_device * imgu,struct imgu_css_buffer * buf,unsigned int pipe)182 static bool imgu_dummybufs_check(struct imgu_device *imgu,
183 struct imgu_css_buffer *buf,
184 unsigned int pipe)
185 {
186 unsigned int i;
187 struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
188
189 for (i = 0; i < IMGU_MAX_QUEUE_DEPTH; i++)
190 if (buf == &imgu_pipe->queues[buf->queue].dummybufs[i])
191 break;
192
193 return i < IMGU_MAX_QUEUE_DEPTH;
194 }
195
imgu_buffer_done(struct imgu_device * imgu,struct vb2_buffer * vb,enum vb2_buffer_state state)196 static void imgu_buffer_done(struct imgu_device *imgu, struct vb2_buffer *vb,
197 enum vb2_buffer_state state)
198 {
199 mutex_lock(&imgu->lock);
200 imgu_v4l2_buffer_done(vb, state);
201 mutex_unlock(&imgu->lock);
202 }
203
imgu_queue_getbuf(struct imgu_device * imgu,unsigned int node,unsigned int pipe)204 static struct imgu_css_buffer *imgu_queue_getbuf(struct imgu_device *imgu,
205 unsigned int node,
206 unsigned int pipe)
207 {
208 struct imgu_buffer *buf;
209 struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
210
211 if (WARN_ON(node >= IMGU_NODE_NUM))
212 return NULL;
213
214 /* Find first free buffer from the node */
215 list_for_each_entry(buf, &imgu_pipe->nodes[node].buffers, vid_buf.list) {
216 if (imgu_css_buf_state(&buf->css_buf) == IPU3_CSS_BUFFER_NEW)
217 return &buf->css_buf;
218 }
219
220 /* There were no free buffers, try to return a dummy buffer */
221 return imgu_dummybufs_get(imgu, imgu_node_map[node].css_queue, pipe);
222 }
223
224 /*
225 * Queue as many buffers to CSS as possible. If all buffers don't fit into
226 * CSS buffer queues, they remain unqueued and will be queued later.
227 */
imgu_queue_buffers(struct imgu_device * imgu,bool initial,unsigned int pipe)228 int imgu_queue_buffers(struct imgu_device *imgu, bool initial, unsigned int pipe)
229 {
230 unsigned int node;
231 int r = 0;
232 struct imgu_media_pipe *imgu_pipe = &imgu->imgu_pipe[pipe];
233
234 if (!imgu_css_is_streaming(&imgu->css))
235 return 0;
236
237 dev_dbg(&imgu->pci_dev->dev, "Queue buffers to pipe %d", pipe);
238 mutex_lock(&imgu->lock);
239
240 if (!imgu_css_pipe_queue_empty(&imgu->css, pipe)) {
241 mutex_unlock(&imgu->lock);
242 return 0;
243 }
244
245 /* Buffer set is queued to FW only when input buffer is ready */
246 for (node = IMGU_NODE_NUM - 1;
247 imgu_queue_getbuf(imgu, IMGU_NODE_IN, pipe);
248 node = node ? node - 1 : IMGU_NODE_NUM - 1) {
249 if (node == IMGU_NODE_VF &&
250 !imgu_pipe->nodes[IMGU_NODE_VF].enabled) {
251 dev_warn(&imgu->pci_dev->dev,
252 "Vf not enabled, ignore queue");
253 continue;
254 } else if (node == IMGU_NODE_PARAMS &&
255 imgu_pipe->nodes[node].enabled) {
256 struct vb2_buffer *vb;
257 struct imgu_vb2_buffer *ivb;
258
259 /* No parameters for this frame */
260 if (list_empty(&imgu_pipe->nodes[node].buffers))
261 continue;
262
263 ivb = list_first_entry(&imgu_pipe->nodes[node].buffers,
264 struct imgu_vb2_buffer, list);
265 list_del(&ivb->list);
266 vb = &ivb->vbb.vb2_buf;
267 r = imgu_css_set_parameters(&imgu->css, pipe,
268 vb2_plane_vaddr(vb, 0));
269 if (r) {
270 vb2_buffer_done(vb, VB2_BUF_STATE_ERROR);
271 dev_warn(&imgu->pci_dev->dev,
272 "set parameters failed.");
273 continue;
274 }
275
276 vb2_buffer_done(vb, VB2_BUF_STATE_DONE);
277 dev_dbg(&imgu->pci_dev->dev,
278 "queue user parameters %d to css.", vb->index);
279 } else if (imgu_pipe->queue_enabled[node]) {
280 struct imgu_css_buffer *buf =
281 imgu_queue_getbuf(imgu, node, pipe);
282 struct imgu_buffer *ibuf = NULL;
283 bool dummy;
284
285 if (!buf)
286 break;
287
288 r = imgu_css_buf_queue(&imgu->css, pipe, buf);
289 if (r)
290 break;
291 dummy = imgu_dummybufs_check(imgu, buf, pipe);
292 if (!dummy)
293 ibuf = container_of(buf, struct imgu_buffer,
294 css_buf);
295 dev_dbg(&imgu->pci_dev->dev,
296 "queue %s %s buffer %u to css da: 0x%08x\n",
297 dummy ? "dummy" : "user",
298 imgu_node_map[node].name,
299 dummy ? 0 : ibuf->vid_buf.vbb.vb2_buf.index,
300 (u32)buf->daddr);
301 }
302 }
303 mutex_unlock(&imgu->lock);
304
305 if (r && r != -EBUSY)
306 goto failed;
307
308 return 0;
309
310 failed:
311 /*
312 * On error, mark all buffers as failed which are not
313 * yet queued to CSS
314 */
315 dev_err(&imgu->pci_dev->dev,
316 "failed to queue buffer to CSS on queue %i (%d)\n",
317 node, r);
318
319 if (initial)
320 /* If we were called from streamon(), no need to finish bufs */
321 return r;
322
323 for (node = 0; node < IMGU_NODE_NUM; node++) {
324 struct imgu_buffer *buf, *buf0;
325
326 if (!imgu_pipe->queue_enabled[node])
327 continue; /* Skip disabled queues */
328
329 mutex_lock(&imgu->lock);
330 list_for_each_entry_safe(buf, buf0,
331 &imgu_pipe->nodes[node].buffers,
332 vid_buf.list) {
333 if (imgu_css_buf_state(&buf->css_buf) ==
334 IPU3_CSS_BUFFER_QUEUED)
335 continue; /* Was already queued, skip */
336
337 imgu_v4l2_buffer_done(&buf->vid_buf.vbb.vb2_buf,
338 VB2_BUF_STATE_ERROR);
339 }
340 mutex_unlock(&imgu->lock);
341 }
342
343 return r;
344 }
345
imgu_powerup(struct imgu_device * imgu)346 static int imgu_powerup(struct imgu_device *imgu)
347 {
348 int r;
349 unsigned int pipe;
350 unsigned int freq = 200;
351 struct v4l2_mbus_framefmt *fmt;
352
353 /* input larger than 2048*1152, ask imgu to work on high freq */
354 for_each_set_bit(pipe, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM) {
355 fmt = &imgu->imgu_pipe[pipe].nodes[IMGU_NODE_IN].pad_fmt;
356 dev_dbg(&imgu->pci_dev->dev, "pipe %u input format = %ux%u",
357 pipe, fmt->width, fmt->height);
358 if ((fmt->width * fmt->height) >= (2048 * 1152))
359 freq = 450;
360 }
361
362 r = imgu_css_set_powerup(&imgu->pci_dev->dev, imgu->base, freq);
363 if (r)
364 return r;
365
366 imgu_mmu_resume(imgu->mmu);
367 return 0;
368 }
369
imgu_powerdown(struct imgu_device * imgu)370 static void imgu_powerdown(struct imgu_device *imgu)
371 {
372 imgu_mmu_suspend(imgu->mmu);
373 imgu_css_set_powerdown(&imgu->pci_dev->dev, imgu->base);
374 }
375
imgu_s_stream(struct imgu_device * imgu,int enable)376 int imgu_s_stream(struct imgu_device *imgu, int enable)
377 {
378 struct device *dev = &imgu->pci_dev->dev;
379 int r, pipe;
380
381 if (!enable) {
382 /* Stop streaming */
383 dev_dbg(dev, "stream off\n");
384 /* Block new buffers to be queued to CSS. */
385 atomic_set(&imgu->qbuf_barrier, 1);
386 imgu_css_stop_streaming(&imgu->css);
387 synchronize_irq(imgu->pci_dev->irq);
388 atomic_set(&imgu->qbuf_barrier, 0);
389 imgu_powerdown(imgu);
390 pm_runtime_put(&imgu->pci_dev->dev);
391
392 return 0;
393 }
394
395 /* Set Power */
396 r = pm_runtime_resume_and_get(dev);
397 if (r < 0) {
398 dev_err(dev, "failed to set imgu power\n");
399 return r;
400 }
401
402 r = imgu_powerup(imgu);
403 if (r) {
404 dev_err(dev, "failed to power up imgu\n");
405 pm_runtime_put(dev);
406 return r;
407 }
408
409 /* Start CSS streaming */
410 r = imgu_css_start_streaming(&imgu->css);
411 if (r) {
412 dev_err(dev, "failed to start css streaming (%d)", r);
413 goto fail_start_streaming;
414 }
415
416 for_each_set_bit(pipe, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM) {
417 /* Initialize dummy buffers */
418 r = imgu_dummybufs_init(imgu, pipe);
419 if (r) {
420 dev_err(dev, "failed to initialize dummy buffers (%d)", r);
421 goto fail_dummybufs;
422 }
423
424 /* Queue as many buffers from queue as possible */
425 r = imgu_queue_buffers(imgu, true, pipe);
426 if (r) {
427 dev_err(dev, "failed to queue initial buffers (%d)", r);
428 goto fail_queueing;
429 }
430 }
431
432 return 0;
433 fail_queueing:
434 for_each_set_bit(pipe, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM)
435 imgu_dummybufs_cleanup(imgu, pipe);
436 fail_dummybufs:
437 imgu_css_stop_streaming(&imgu->css);
438 fail_start_streaming:
439 pm_runtime_put(dev);
440
441 return r;
442 }
443
imgu_video_nodes_exit(struct imgu_device * imgu)444 static void imgu_video_nodes_exit(struct imgu_device *imgu)
445 {
446 int i;
447
448 for (i = 0; i < IMGU_MAX_PIPE_NUM; i++)
449 imgu_dummybufs_cleanup(imgu, i);
450
451 imgu_v4l2_unregister(imgu);
452 }
453
imgu_video_nodes_init(struct imgu_device * imgu)454 static int imgu_video_nodes_init(struct imgu_device *imgu)
455 {
456 struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES] = { NULL };
457 struct v4l2_rect *rects[IPU3_CSS_RECTS] = { NULL };
458 struct imgu_media_pipe *imgu_pipe;
459 unsigned int i, j;
460 int r;
461
462 imgu->buf_struct_size = sizeof(struct imgu_buffer);
463
464 for (j = 0; j < IMGU_MAX_PIPE_NUM; j++) {
465 imgu_pipe = &imgu->imgu_pipe[j];
466
467 for (i = 0; i < IMGU_NODE_NUM; i++) {
468 imgu_pipe->nodes[i].name = imgu_node_map[i].name;
469 imgu_pipe->nodes[i].output = i < IMGU_QUEUE_FIRST_INPUT;
470 imgu_pipe->nodes[i].enabled = false;
471
472 if (i != IMGU_NODE_PARAMS && i != IMGU_NODE_STAT_3A)
473 fmts[imgu_node_map[i].css_queue] =
474 &imgu_pipe->nodes[i].vdev_fmt.fmt.pix_mp;
475 atomic_set(&imgu_pipe->nodes[i].sequence, 0);
476 }
477 }
478
479 r = imgu_v4l2_register(imgu);
480 if (r)
481 return r;
482
483 /* Set initial formats and initialize formats of video nodes */
484 for (j = 0; j < IMGU_MAX_PIPE_NUM; j++) {
485 imgu_pipe = &imgu->imgu_pipe[j];
486
487 rects[IPU3_CSS_RECT_EFFECTIVE] = &imgu_pipe->imgu_sd.rect.eff;
488 rects[IPU3_CSS_RECT_BDS] = &imgu_pipe->imgu_sd.rect.bds;
489 imgu_css_fmt_set(&imgu->css, fmts, rects, j);
490
491 /* Pre-allocate dummy buffers */
492 r = imgu_dummybufs_preallocate(imgu, j);
493 if (r) {
494 dev_err(&imgu->pci_dev->dev,
495 "failed to pre-allocate dummy buffers (%d)", r);
496 goto out_cleanup;
497 }
498 }
499
500 return 0;
501
502 out_cleanup:
503 imgu_video_nodes_exit(imgu);
504
505 return r;
506 }
507
508 /**************** PCI interface ****************/
509
imgu_isr_threaded(int irq,void * imgu_ptr)510 static irqreturn_t imgu_isr_threaded(int irq, void *imgu_ptr)
511 {
512 struct imgu_device *imgu = imgu_ptr;
513 struct imgu_media_pipe *imgu_pipe;
514 int p;
515
516 /* Dequeue / queue buffers */
517 do {
518 u64 ns = ktime_get_ns();
519 struct imgu_css_buffer *b;
520 struct imgu_buffer *buf = NULL;
521 unsigned int node, pipe;
522 bool dummy;
523
524 do {
525 mutex_lock(&imgu->lock);
526 b = imgu_css_buf_dequeue(&imgu->css);
527 mutex_unlock(&imgu->lock);
528 } while (PTR_ERR(b) == -EAGAIN);
529
530 if (IS_ERR(b)) {
531 if (PTR_ERR(b) != -EBUSY) /* All done */
532 dev_err(&imgu->pci_dev->dev,
533 "failed to dequeue buffers (%ld)\n",
534 PTR_ERR(b));
535 break;
536 }
537
538 node = imgu_map_node(imgu, b->queue);
539 pipe = b->pipe;
540 dummy = imgu_dummybufs_check(imgu, b, pipe);
541 if (!dummy)
542 buf = container_of(b, struct imgu_buffer, css_buf);
543 dev_dbg(&imgu->pci_dev->dev,
544 "dequeue %s %s buffer %d daddr 0x%x from css\n",
545 dummy ? "dummy" : "user",
546 imgu_node_map[node].name,
547 dummy ? 0 : buf->vid_buf.vbb.vb2_buf.index,
548 (u32)b->daddr);
549
550 if (dummy)
551 /* It was a dummy buffer, skip it */
552 continue;
553
554 /* Fill vb2 buffer entries and tell it's ready */
555 imgu_pipe = &imgu->imgu_pipe[pipe];
556 if (!imgu_pipe->nodes[node].output) {
557 buf->vid_buf.vbb.vb2_buf.timestamp = ns;
558 buf->vid_buf.vbb.field = V4L2_FIELD_NONE;
559 buf->vid_buf.vbb.sequence =
560 atomic_inc_return(
561 &imgu_pipe->nodes[node].sequence);
562 dev_dbg(&imgu->pci_dev->dev, "vb2 buffer sequence %d",
563 buf->vid_buf.vbb.sequence);
564 }
565 imgu_buffer_done(imgu, &buf->vid_buf.vbb.vb2_buf,
566 imgu_css_buf_state(&buf->css_buf) ==
567 IPU3_CSS_BUFFER_DONE ?
568 VB2_BUF_STATE_DONE :
569 VB2_BUF_STATE_ERROR);
570 mutex_lock(&imgu->lock);
571 if (imgu_css_queue_empty(&imgu->css))
572 wake_up_all(&imgu->buf_drain_wq);
573 mutex_unlock(&imgu->lock);
574 } while (1);
575
576 /*
577 * Try to queue more buffers for CSS.
578 * qbuf_barrier is used to disable new buffers
579 * to be queued to CSS.
580 */
581 if (!atomic_read(&imgu->qbuf_barrier))
582 for_each_set_bit(p, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM)
583 imgu_queue_buffers(imgu, false, p);
584
585 return IRQ_HANDLED;
586 }
587
imgu_isr(int irq,void * imgu_ptr)588 static irqreturn_t imgu_isr(int irq, void *imgu_ptr)
589 {
590 struct imgu_device *imgu = imgu_ptr;
591
592 /* acknowledge interruption */
593 if (imgu_css_irq_ack(&imgu->css) < 0)
594 return IRQ_NONE;
595
596 return IRQ_WAKE_THREAD;
597 }
598
imgu_pci_config_setup(struct pci_dev * dev)599 static int imgu_pci_config_setup(struct pci_dev *dev)
600 {
601 u16 pci_command;
602 int r = pci_enable_msi(dev);
603
604 if (r) {
605 dev_err(&dev->dev, "failed to enable MSI (%d)\n", r);
606 return r;
607 }
608
609 pci_read_config_word(dev, PCI_COMMAND, &pci_command);
610 pci_command |= PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
611 PCI_COMMAND_INTX_DISABLE;
612 pci_write_config_word(dev, PCI_COMMAND, pci_command);
613
614 return 0;
615 }
616
imgu_pci_probe(struct pci_dev * pci_dev,const struct pci_device_id * id)617 static int imgu_pci_probe(struct pci_dev *pci_dev,
618 const struct pci_device_id *id)
619 {
620 struct imgu_device *imgu;
621 phys_addr_t phys;
622 unsigned long phys_len;
623 void __iomem *const *iomap;
624 int r;
625
626 imgu = devm_kzalloc(&pci_dev->dev, sizeof(*imgu), GFP_KERNEL);
627 if (!imgu)
628 return -ENOMEM;
629
630 imgu->pci_dev = pci_dev;
631
632 r = pcim_enable_device(pci_dev);
633 if (r) {
634 dev_err(&pci_dev->dev, "failed to enable device (%d)\n", r);
635 return r;
636 }
637
638 dev_info(&pci_dev->dev, "device 0x%x (rev: 0x%x)\n",
639 pci_dev->device, pci_dev->revision);
640
641 phys = pci_resource_start(pci_dev, IMGU_PCI_BAR);
642 phys_len = pci_resource_len(pci_dev, IMGU_PCI_BAR);
643
644 r = pcim_iomap_regions(pci_dev, 1 << IMGU_PCI_BAR, pci_name(pci_dev));
645 if (r) {
646 dev_err(&pci_dev->dev, "failed to remap I/O memory (%d)\n", r);
647 return r;
648 }
649 dev_info(&pci_dev->dev, "physical base address %pap, %lu bytes\n",
650 &phys, phys_len);
651
652 iomap = pcim_iomap_table(pci_dev);
653 if (!iomap) {
654 dev_err(&pci_dev->dev, "failed to iomap table\n");
655 return -ENODEV;
656 }
657
658 imgu->base = iomap[IMGU_PCI_BAR];
659
660 pci_set_drvdata(pci_dev, imgu);
661
662 pci_set_master(pci_dev);
663
664 r = dma_coerce_mask_and_coherent(&pci_dev->dev, IMGU_DMA_MASK);
665 if (r) {
666 dev_err(&pci_dev->dev, "failed to set DMA mask (%d)\n", r);
667 return -ENODEV;
668 }
669
670 r = imgu_pci_config_setup(pci_dev);
671 if (r)
672 return r;
673
674 mutex_init(&imgu->lock);
675 mutex_init(&imgu->streaming_lock);
676 atomic_set(&imgu->qbuf_barrier, 0);
677 init_waitqueue_head(&imgu->buf_drain_wq);
678
679 r = imgu_css_set_powerup(&pci_dev->dev, imgu->base, 200);
680 if (r) {
681 dev_err(&pci_dev->dev,
682 "failed to power up CSS (%d)\n", r);
683 goto out_mutex_destroy;
684 }
685
686 imgu->mmu = imgu_mmu_init(&pci_dev->dev, imgu->base);
687 if (IS_ERR(imgu->mmu)) {
688 r = PTR_ERR(imgu->mmu);
689 dev_err(&pci_dev->dev, "failed to initialize MMU (%d)\n", r);
690 goto out_css_powerdown;
691 }
692
693 r = imgu_dmamap_init(imgu);
694 if (r) {
695 dev_err(&pci_dev->dev,
696 "failed to initialize DMA mapping (%d)\n", r);
697 goto out_mmu_exit;
698 }
699
700 /* ISP programming */
701 r = imgu_css_init(&pci_dev->dev, &imgu->css, imgu->base, phys_len);
702 if (r) {
703 dev_err(&pci_dev->dev, "failed to initialize CSS (%d)\n", r);
704 goto out_dmamap_exit;
705 }
706
707 /* v4l2 sub-device registration */
708 r = imgu_video_nodes_init(imgu);
709 if (r) {
710 dev_err(&pci_dev->dev, "failed to create V4L2 devices (%d)\n",
711 r);
712 goto out_css_cleanup;
713 }
714
715 r = devm_request_threaded_irq(&pci_dev->dev, pci_dev->irq,
716 imgu_isr, imgu_isr_threaded,
717 IRQF_SHARED, IMGU_NAME, imgu);
718 if (r) {
719 dev_err(&pci_dev->dev, "failed to request IRQ (%d)\n", r);
720 goto out_video_exit;
721 }
722
723 pm_runtime_put_noidle(&pci_dev->dev);
724 pm_runtime_allow(&pci_dev->dev);
725
726 return 0;
727
728 out_video_exit:
729 imgu_video_nodes_exit(imgu);
730 out_css_cleanup:
731 imgu_css_cleanup(&imgu->css);
732 out_dmamap_exit:
733 imgu_dmamap_exit(imgu);
734 out_mmu_exit:
735 imgu_mmu_exit(imgu->mmu);
736 out_css_powerdown:
737 imgu_css_set_powerdown(&pci_dev->dev, imgu->base);
738 out_mutex_destroy:
739 mutex_destroy(&imgu->streaming_lock);
740 mutex_destroy(&imgu->lock);
741
742 return r;
743 }
744
imgu_pci_remove(struct pci_dev * pci_dev)745 static void imgu_pci_remove(struct pci_dev *pci_dev)
746 {
747 struct imgu_device *imgu = pci_get_drvdata(pci_dev);
748
749 pm_runtime_forbid(&pci_dev->dev);
750 pm_runtime_get_noresume(&pci_dev->dev);
751
752 imgu_video_nodes_exit(imgu);
753 imgu_css_cleanup(&imgu->css);
754 imgu_css_set_powerdown(&pci_dev->dev, imgu->base);
755 imgu_dmamap_exit(imgu);
756 imgu_mmu_exit(imgu->mmu);
757 mutex_destroy(&imgu->streaming_lock);
758 mutex_destroy(&imgu->lock);
759 }
760
imgu_suspend(struct device * dev)761 static int __maybe_unused imgu_suspend(struct device *dev)
762 {
763 struct pci_dev *pci_dev = to_pci_dev(dev);
764 struct imgu_device *imgu = pci_get_drvdata(pci_dev);
765
766 imgu->suspend_in_stream = imgu_css_is_streaming(&imgu->css);
767 if (!imgu->suspend_in_stream)
768 goto out;
769 /* Block new buffers to be queued to CSS. */
770 atomic_set(&imgu->qbuf_barrier, 1);
771 /*
772 * Wait for currently running irq handler to be done so that
773 * no new buffers will be queued to fw later.
774 */
775 synchronize_irq(pci_dev->irq);
776 /* Wait until all buffers in CSS are done. */
777 if (!wait_event_timeout(imgu->buf_drain_wq,
778 imgu_css_queue_empty(&imgu->css), msecs_to_jiffies(1000)))
779 dev_err(dev, "wait buffer drain timeout.\n");
780
781 imgu_css_stop_streaming(&imgu->css);
782 atomic_set(&imgu->qbuf_barrier, 0);
783 imgu_powerdown(imgu);
784 pm_runtime_force_suspend(dev);
785 out:
786 return 0;
787 }
788
imgu_resume(struct device * dev)789 static int __maybe_unused imgu_resume(struct device *dev)
790 {
791 struct imgu_device *imgu = dev_get_drvdata(dev);
792 int r = 0;
793 unsigned int pipe;
794
795 if (!imgu->suspend_in_stream)
796 goto out;
797
798 pm_runtime_force_resume(dev);
799
800 r = imgu_powerup(imgu);
801 if (r) {
802 dev_err(dev, "failed to power up imgu\n");
803 goto out;
804 }
805
806 /* Start CSS streaming */
807 r = imgu_css_start_streaming(&imgu->css);
808 if (r) {
809 dev_err(dev, "failed to resume css streaming (%d)", r);
810 goto out;
811 }
812
813 for_each_set_bit(pipe, imgu->css.enabled_pipes, IMGU_MAX_PIPE_NUM) {
814 r = imgu_queue_buffers(imgu, true, pipe);
815 if (r)
816 dev_err(dev, "failed to queue buffers to pipe %d (%d)",
817 pipe, r);
818 }
819
820 out:
821 return r;
822 }
823
824 /*
825 * PCI rpm framework checks the existence of driver rpm callbacks.
826 * Place a dummy callback here to avoid rpm going into error state.
827 */
imgu_rpm_dummy_cb(struct device * dev)828 static __maybe_unused int imgu_rpm_dummy_cb(struct device *dev)
829 {
830 return 0;
831 }
832
833 static const struct dev_pm_ops imgu_pm_ops = {
834 SET_RUNTIME_PM_OPS(&imgu_rpm_dummy_cb, &imgu_rpm_dummy_cb, NULL)
835 SET_SYSTEM_SLEEP_PM_OPS(&imgu_suspend, &imgu_resume)
836 };
837
838 static const struct pci_device_id imgu_pci_tbl[] = {
839 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, IMGU_PCI_ID) },
840 { 0, }
841 };
842
843 MODULE_DEVICE_TABLE(pci, imgu_pci_tbl);
844
845 static struct pci_driver imgu_pci_driver = {
846 .name = IMGU_NAME,
847 .id_table = imgu_pci_tbl,
848 .probe = imgu_pci_probe,
849 .remove = imgu_pci_remove,
850 .driver = {
851 .pm = &imgu_pm_ops,
852 },
853 };
854
855 module_pci_driver(imgu_pci_driver);
856
857 MODULE_AUTHOR("Tuukka Toivonen");
858 MODULE_AUTHOR("Tianshu Qiu <tian.shu.qiu@intel.com>");
859 MODULE_AUTHOR("Jian Xu Zheng");
860 MODULE_AUTHOR("Yuning Pu");
861 MODULE_AUTHOR("Yong Zhi <yong.zhi@intel.com>");
862 MODULE_LICENSE("GPL v2");
863 MODULE_DESCRIPTION("Intel ipu3_imgu PCI driver");
864 MODULE_FIRMWARE(IMGU_FW_NAME);
865 MODULE_FIRMWARE(IMGU_FW_NAME_20161208);
866 MODULE_FIRMWARE(IMGU_FW_NAME_IPU_20161208);
867