1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. 4 * Copyright (C) 2017 Linaro Ltd. 5 */ 6 7 #include <linux/delay.h> 8 #include <linux/device.h> 9 #include <linux/dma-mapping.h> 10 #include <linux/interrupt.h> 11 #include <linux/iopoll.h> 12 #include <linux/kernel.h> 13 #include <linux/slab.h> 14 15 #include "core.h" 16 #include "hfi_cmds.h" 17 #include "hfi_msgs.h" 18 #include "hfi_venus.h" 19 #include "hfi_venus_io.h" 20 #include "firmware.h" 21 22 #define HFI_MASK_QHDR_TX_TYPE 0xff000000 23 #define HFI_MASK_QHDR_RX_TYPE 0x00ff0000 24 #define HFI_MASK_QHDR_PRI_TYPE 0x0000ff00 25 #define HFI_MASK_QHDR_ID_TYPE 0x000000ff 26 27 #define HFI_HOST_TO_CTRL_CMD_Q 0 28 #define HFI_CTRL_TO_HOST_MSG_Q 1 29 #define HFI_CTRL_TO_HOST_DBG_Q 2 30 #define HFI_MASK_QHDR_STATUS 0x000000ff 31 32 #define IFACEQ_NUM 3 33 #define IFACEQ_CMD_IDX 0 34 #define IFACEQ_MSG_IDX 1 35 #define IFACEQ_DBG_IDX 2 36 #define IFACEQ_MAX_BUF_COUNT 50 37 #define IFACEQ_MAX_PARALLEL_CLNTS 16 38 #define IFACEQ_DFLT_QHDR 0x01010000 39 40 #define POLL_INTERVAL_US 50 41 42 #define IFACEQ_MAX_PKT_SIZE 1024 43 #define IFACEQ_MED_PKT_SIZE 768 44 #define IFACEQ_MIN_PKT_SIZE 8 45 #define IFACEQ_VAR_SMALL_PKT_SIZE 100 46 #define IFACEQ_VAR_LARGE_PKT_SIZE 512 47 #define IFACEQ_VAR_HUGE_PKT_SIZE (1024 * 12) 48 49 struct hfi_queue_table_header { 50 u32 version; 51 u32 size; 52 u32 qhdr0_offset; 53 u32 qhdr_size; 54 u32 num_q; 55 u32 num_active_q; 56 }; 57 58 struct hfi_queue_header { 59 u32 status; 60 u32 start_addr; 61 u32 type; 62 u32 q_size; 63 u32 pkt_size; 64 u32 pkt_drop_cnt; 65 u32 rx_wm; 66 u32 tx_wm; 67 u32 rx_req; 68 u32 tx_req; 69 u32 rx_irq_status; 70 u32 tx_irq_status; 71 u32 read_idx; 72 u32 write_idx; 73 }; 74 75 #define IFACEQ_TABLE_SIZE \ 76 (sizeof(struct hfi_queue_table_header) + \ 77 sizeof(struct hfi_queue_header) * IFACEQ_NUM) 78 79 #define IFACEQ_QUEUE_SIZE (IFACEQ_MAX_PKT_SIZE * \ 80 IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS) 81 82 #define IFACEQ_GET_QHDR_START_ADDR(ptr, i) \ 83 (void *)(((ptr) + sizeof(struct hfi_queue_table_header)) + \ 84 ((i) * sizeof(struct hfi_queue_header))) 85 86 #define QDSS_SIZE SZ_4K 87 #define SFR_SIZE SZ_4K 88 #define QUEUE_SIZE \ 89 (IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM)) 90 91 #define ALIGNED_QDSS_SIZE ALIGN(QDSS_SIZE, SZ_4K) 92 #define ALIGNED_SFR_SIZE ALIGN(SFR_SIZE, SZ_4K) 93 #define ALIGNED_QUEUE_SIZE ALIGN(QUEUE_SIZE, SZ_4K) 94 #define SHARED_QSIZE ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \ 95 ALIGNED_QDSS_SIZE, SZ_1M) 96 97 struct mem_desc { 98 dma_addr_t da; /* device address */ 99 void *kva; /* kernel virtual address */ 100 u32 size; 101 unsigned long attrs; 102 }; 103 104 struct iface_queue { 105 struct hfi_queue_header *qhdr; 106 struct mem_desc qmem; 107 }; 108 109 enum venus_state { 110 VENUS_STATE_DEINIT = 1, 111 VENUS_STATE_INIT, 112 }; 113 114 struct venus_hfi_device { 115 struct venus_core *core; 116 u32 irq_status; 117 u32 last_packet_type; 118 bool power_enabled; 119 bool suspended; 120 enum venus_state state; 121 /* serialize read / write to the shared memory */ 122 struct mutex lock; 123 struct completion pwr_collapse_prep; 124 struct completion release_resource; 125 struct mem_desc ifaceq_table; 126 struct mem_desc sfr; 127 struct iface_queue queues[IFACEQ_NUM]; 128 u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE]; 129 u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE]; 130 }; 131 132 static bool venus_pkt_debug; 133 int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL; 134 static bool venus_fw_low_power_mode = true; 135 static int venus_hw_rsp_timeout = 1000; 136 static bool venus_fw_coverage; 137 venus_set_state(struct venus_hfi_device * hdev,enum venus_state state)138 static void venus_set_state(struct venus_hfi_device *hdev, 139 enum venus_state state) 140 { 141 mutex_lock(&hdev->lock); 142 hdev->state = state; 143 mutex_unlock(&hdev->lock); 144 } 145 venus_is_valid_state(struct venus_hfi_device * hdev)146 static bool venus_is_valid_state(struct venus_hfi_device *hdev) 147 { 148 return hdev->state != VENUS_STATE_DEINIT; 149 } 150 venus_dump_packet(struct venus_hfi_device * hdev,const void * packet)151 static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet) 152 { 153 size_t pkt_size = *(u32 *)packet; 154 155 if (!venus_pkt_debug) 156 return; 157 158 print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet, 159 pkt_size, true); 160 } 161 venus_write_queue(struct venus_hfi_device * hdev,struct iface_queue * queue,void * packet,u32 * rx_req)162 static int venus_write_queue(struct venus_hfi_device *hdev, 163 struct iface_queue *queue, 164 void *packet, u32 *rx_req) 165 { 166 struct hfi_queue_header *qhdr; 167 u32 dwords, new_wr_idx; 168 u32 empty_space, rd_idx, wr_idx, qsize; 169 u32 *wr_ptr; 170 171 if (!queue->qmem.kva) 172 return -EINVAL; 173 174 qhdr = queue->qhdr; 175 if (!qhdr) 176 return -EINVAL; 177 178 venus_dump_packet(hdev, packet); 179 180 dwords = (*(u32 *)packet) >> 2; 181 if (!dwords) 182 return -EINVAL; 183 184 rd_idx = qhdr->read_idx; 185 wr_idx = qhdr->write_idx; 186 qsize = qhdr->q_size; 187 /* ensure rd/wr indices's are read from memory */ 188 rmb(); 189 190 if (wr_idx >= rd_idx) 191 empty_space = qsize - (wr_idx - rd_idx); 192 else 193 empty_space = rd_idx - wr_idx; 194 195 if (empty_space <= dwords) { 196 qhdr->tx_req = 1; 197 /* ensure tx_req is updated in memory */ 198 wmb(); 199 return -ENOSPC; 200 } 201 202 qhdr->tx_req = 0; 203 /* ensure tx_req is updated in memory */ 204 wmb(); 205 206 new_wr_idx = wr_idx + dwords; 207 wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2)); 208 209 if (wr_ptr < (u32 *)queue->qmem.kva || 210 wr_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*wr_ptr))) 211 return -EINVAL; 212 213 if (new_wr_idx < qsize) { 214 memcpy(wr_ptr, packet, dwords << 2); 215 } else { 216 size_t len; 217 218 new_wr_idx -= qsize; 219 len = (dwords - new_wr_idx) << 2; 220 memcpy(wr_ptr, packet, len); 221 memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2); 222 } 223 224 /* make sure packet is written before updating the write index */ 225 wmb(); 226 227 qhdr->write_idx = new_wr_idx; 228 *rx_req = qhdr->rx_req ? 1 : 0; 229 230 /* make sure write index is updated before an interrupt is raised */ 231 mb(); 232 233 return 0; 234 } 235 venus_read_queue(struct venus_hfi_device * hdev,struct iface_queue * queue,void * pkt,u32 * tx_req)236 static int venus_read_queue(struct venus_hfi_device *hdev, 237 struct iface_queue *queue, void *pkt, u32 *tx_req) 238 { 239 struct hfi_queue_header *qhdr; 240 u32 dwords, new_rd_idx; 241 u32 rd_idx, wr_idx, type, qsize; 242 u32 *rd_ptr; 243 u32 recv_request = 0; 244 int ret = 0; 245 246 if (!queue->qmem.kva) 247 return -EINVAL; 248 249 qhdr = queue->qhdr; 250 if (!qhdr) 251 return -EINVAL; 252 253 type = qhdr->type; 254 rd_idx = qhdr->read_idx; 255 wr_idx = qhdr->write_idx; 256 qsize = qhdr->q_size; 257 258 /* make sure data is valid before using it */ 259 rmb(); 260 261 /* 262 * Do not set receive request for debug queue, if set, Venus generates 263 * interrupt for debug messages even when there is no response message 264 * available. In general debug queue will not become full as it is being 265 * emptied out for every interrupt from Venus. Venus will anyway 266 * generates interrupt if it is full. 267 */ 268 if (type & HFI_CTRL_TO_HOST_MSG_Q) 269 recv_request = 1; 270 271 if (rd_idx == wr_idx) { 272 qhdr->rx_req = recv_request; 273 *tx_req = 0; 274 /* update rx_req field in memory */ 275 wmb(); 276 return -ENODATA; 277 } 278 279 rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2)); 280 281 if (rd_ptr < (u32 *)queue->qmem.kva || 282 rd_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*rd_ptr))) 283 return -EINVAL; 284 285 dwords = *rd_ptr >> 2; 286 if (!dwords) 287 return -EINVAL; 288 289 new_rd_idx = rd_idx + dwords; 290 if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) { 291 if (new_rd_idx < qsize) { 292 memcpy(pkt, rd_ptr, dwords << 2); 293 } else { 294 size_t len; 295 296 new_rd_idx -= qsize; 297 len = (dwords - new_rd_idx) << 2; 298 memcpy(pkt, rd_ptr, len); 299 memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2); 300 } 301 } else { 302 /* bad packet received, dropping */ 303 new_rd_idx = qhdr->write_idx; 304 ret = -EBADMSG; 305 } 306 307 /* ensure the packet is read before updating read index */ 308 rmb(); 309 310 qhdr->read_idx = new_rd_idx; 311 /* ensure updating read index */ 312 wmb(); 313 314 rd_idx = qhdr->read_idx; 315 wr_idx = qhdr->write_idx; 316 /* ensure rd/wr indices are read from memory */ 317 rmb(); 318 319 if (rd_idx != wr_idx) 320 qhdr->rx_req = 0; 321 else 322 qhdr->rx_req = recv_request; 323 324 *tx_req = qhdr->tx_req ? 1 : 0; 325 326 /* ensure rx_req is stored to memory and tx_req is loaded from memory */ 327 mb(); 328 329 venus_dump_packet(hdev, pkt); 330 331 return ret; 332 } 333 venus_alloc(struct venus_hfi_device * hdev,struct mem_desc * desc,u32 size)334 static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc, 335 u32 size) 336 { 337 struct device *dev = hdev->core->dev; 338 339 desc->attrs = DMA_ATTR_WRITE_COMBINE; 340 desc->size = ALIGN(size, SZ_4K); 341 342 desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL, 343 desc->attrs); 344 if (!desc->kva) 345 return -ENOMEM; 346 347 return 0; 348 } 349 venus_free(struct venus_hfi_device * hdev,struct mem_desc * mem)350 static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem) 351 { 352 struct device *dev = hdev->core->dev; 353 354 dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs); 355 } 356 venus_set_registers(struct venus_hfi_device * hdev)357 static void venus_set_registers(struct venus_hfi_device *hdev) 358 { 359 const struct venus_resources *res = hdev->core->res; 360 const struct reg_val *tbl = res->reg_tbl; 361 unsigned int count = res->reg_tbl_size; 362 unsigned int i; 363 364 for (i = 0; i < count; i++) 365 writel(tbl[i].value, hdev->core->base + tbl[i].reg); 366 } 367 venus_soft_int(struct venus_hfi_device * hdev)368 static void venus_soft_int(struct venus_hfi_device *hdev) 369 { 370 void __iomem *cpu_ic_base = hdev->core->cpu_ic_base; 371 u32 clear_bit; 372 373 if (IS_V6(hdev->core)) 374 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT_V6); 375 else 376 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT); 377 378 writel(clear_bit, cpu_ic_base + CPU_IC_SOFTINT); 379 } 380 venus_iface_cmdq_write_nolock(struct venus_hfi_device * hdev,void * pkt,bool sync)381 static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev, 382 void *pkt, bool sync) 383 { 384 struct device *dev = hdev->core->dev; 385 struct hfi_pkt_hdr *cmd_packet; 386 struct iface_queue *queue; 387 u32 rx_req; 388 int ret; 389 390 if (!venus_is_valid_state(hdev)) 391 return -EINVAL; 392 393 cmd_packet = (struct hfi_pkt_hdr *)pkt; 394 hdev->last_packet_type = cmd_packet->pkt_type; 395 396 queue = &hdev->queues[IFACEQ_CMD_IDX]; 397 398 ret = venus_write_queue(hdev, queue, pkt, &rx_req); 399 if (ret) { 400 dev_err(dev, "write to iface cmd queue failed (%d)\n", ret); 401 return ret; 402 } 403 404 if (sync) { 405 /* 406 * Inform video hardware to raise interrupt for synchronous 407 * commands 408 */ 409 queue = &hdev->queues[IFACEQ_MSG_IDX]; 410 queue->qhdr->rx_req = 1; 411 /* ensure rx_req is updated in memory */ 412 wmb(); 413 } 414 415 if (rx_req) 416 venus_soft_int(hdev); 417 418 return 0; 419 } 420 venus_iface_cmdq_write(struct venus_hfi_device * hdev,void * pkt,bool sync)421 static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt, bool sync) 422 { 423 int ret; 424 425 mutex_lock(&hdev->lock); 426 ret = venus_iface_cmdq_write_nolock(hdev, pkt, sync); 427 mutex_unlock(&hdev->lock); 428 429 return ret; 430 } 431 venus_hfi_core_set_resource(struct venus_core * core,u32 id,u32 size,u32 addr,void * cookie)432 static int venus_hfi_core_set_resource(struct venus_core *core, u32 id, 433 u32 size, u32 addr, void *cookie) 434 { 435 struct venus_hfi_device *hdev = to_hfi_priv(core); 436 struct hfi_sys_set_resource_pkt *pkt; 437 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 438 int ret; 439 440 if (id == VIDC_RESOURCE_NONE) 441 return 0; 442 443 pkt = (struct hfi_sys_set_resource_pkt *)packet; 444 445 ret = pkt_sys_set_resource(pkt, id, size, addr, cookie); 446 if (ret) 447 return ret; 448 449 ret = venus_iface_cmdq_write(hdev, pkt, false); 450 if (ret) 451 return ret; 452 453 return 0; 454 } 455 venus_boot_core(struct venus_hfi_device * hdev)456 static int venus_boot_core(struct venus_hfi_device *hdev) 457 { 458 struct device *dev = hdev->core->dev; 459 static const unsigned int max_tries = 100; 460 u32 ctrl_status = 0, mask_val = 0; 461 unsigned int count = 0; 462 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 463 void __iomem *wrapper_base = hdev->core->wrapper_base; 464 int ret = 0; 465 466 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) { 467 mask_val = readl(wrapper_base + WRAPPER_INTR_MASK); 468 mask_val &= ~(WRAPPER_INTR_MASK_A2HWD_BASK_V6 | 469 WRAPPER_INTR_MASK_A2HCPU_MASK); 470 } else { 471 mask_val = WRAPPER_INTR_MASK_A2HVCODEC_MASK; 472 } 473 474 writel(mask_val, wrapper_base + WRAPPER_INTR_MASK); 475 if (IS_V1(hdev->core)) 476 writel(1, cpu_cs_base + CPU_CS_SCIACMDARG3); 477 478 writel(BIT(VIDC_CTRL_INIT_CTRL_SHIFT), cpu_cs_base + VIDC_CTRL_INIT); 479 while (!ctrl_status && count < max_tries) { 480 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 481 if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) { 482 dev_err(dev, "invalid setting for UC_REGION\n"); 483 ret = -EINVAL; 484 break; 485 } 486 487 usleep_range(500, 1000); 488 count++; 489 } 490 491 if (count >= max_tries) 492 ret = -ETIMEDOUT; 493 494 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) { 495 writel(0x1, cpu_cs_base + CPU_CS_H2XSOFTINTEN_V6); 496 writel(0x0, cpu_cs_base + CPU_CS_X2RPMH_V6); 497 } 498 499 return ret; 500 } 501 venus_hwversion(struct venus_hfi_device * hdev)502 static u32 venus_hwversion(struct venus_hfi_device *hdev) 503 { 504 struct device *dev = hdev->core->dev; 505 void __iomem *wrapper_base = hdev->core->wrapper_base; 506 u32 ver; 507 u32 major, minor, step; 508 509 ver = readl(wrapper_base + WRAPPER_HW_VERSION); 510 major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK; 511 major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT; 512 minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK; 513 minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT; 514 step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK; 515 516 dev_dbg(dev, VDBGL "venus hw version %x.%x.%x\n", major, minor, step); 517 518 return major; 519 } 520 venus_run(struct venus_hfi_device * hdev)521 static int venus_run(struct venus_hfi_device *hdev) 522 { 523 struct device *dev = hdev->core->dev; 524 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 525 int ret; 526 527 /* 528 * Re-program all of the registers that get reset as a result of 529 * regulator_disable() and _enable() 530 */ 531 venus_set_registers(hdev); 532 533 writel(hdev->ifaceq_table.da, cpu_cs_base + UC_REGION_ADDR); 534 writel(SHARED_QSIZE, cpu_cs_base + UC_REGION_SIZE); 535 writel(hdev->ifaceq_table.da, cpu_cs_base + CPU_CS_SCIACMDARG2); 536 writel(0x01, cpu_cs_base + CPU_CS_SCIACMDARG1); 537 if (hdev->sfr.da) 538 writel(hdev->sfr.da, cpu_cs_base + SFR_ADDR); 539 540 ret = venus_boot_core(hdev); 541 if (ret) { 542 dev_err(dev, "failed to reset venus core\n"); 543 return ret; 544 } 545 546 venus_hwversion(hdev); 547 548 return 0; 549 } 550 venus_halt_axi(struct venus_hfi_device * hdev)551 static int venus_halt_axi(struct venus_hfi_device *hdev) 552 { 553 void __iomem *wrapper_base = hdev->core->wrapper_base; 554 void __iomem *vbif_base = hdev->core->vbif_base; 555 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 556 void __iomem *aon_base = hdev->core->aon_base; 557 struct device *dev = hdev->core->dev; 558 u32 val; 559 u32 mask_val; 560 int ret; 561 562 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) { 563 writel(0x3, cpu_cs_base + CPU_CS_X2RPMH_V6); 564 565 if (IS_IRIS2_1(hdev->core)) 566 goto skip_aon_mvp_noc; 567 568 writel(0x1, aon_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL); 569 ret = readl_poll_timeout(aon_base + AON_WRAPPER_MVP_NOC_LPI_STATUS, 570 val, 571 val & BIT(0), 572 POLL_INTERVAL_US, 573 VBIF_AXI_HALT_ACK_TIMEOUT_US); 574 if (ret) 575 return -ETIMEDOUT; 576 577 skip_aon_mvp_noc: 578 mask_val = (BIT(2) | BIT(1) | BIT(0)); 579 writel(mask_val, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6); 580 581 writel(0x00, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6); 582 ret = readl_poll_timeout(wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_STATUS_V6, 583 val, 584 val == 0, 585 POLL_INTERVAL_US, 586 VBIF_AXI_HALT_ACK_TIMEOUT_US); 587 588 if (ret) { 589 dev_err(dev, "DBLP Release: lpi_status %x\n", val); 590 return -ETIMEDOUT; 591 } 592 return 0; 593 } 594 595 if (IS_V4(hdev->core)) { 596 val = readl(wrapper_base + WRAPPER_CPU_AXI_HALT); 597 val |= WRAPPER_CPU_AXI_HALT_HALT; 598 writel(val, wrapper_base + WRAPPER_CPU_AXI_HALT); 599 600 ret = readl_poll_timeout(wrapper_base + WRAPPER_CPU_AXI_HALT_STATUS, 601 val, 602 val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE, 603 POLL_INTERVAL_US, 604 VBIF_AXI_HALT_ACK_TIMEOUT_US); 605 if (ret) { 606 dev_err(dev, "AXI bus port halt timeout\n"); 607 return ret; 608 } 609 610 return 0; 611 } 612 613 /* Halt AXI and AXI IMEM VBIF Access */ 614 val = readl(vbif_base + VBIF_AXI_HALT_CTRL0); 615 val |= VBIF_AXI_HALT_CTRL0_HALT_REQ; 616 writel(val, vbif_base + VBIF_AXI_HALT_CTRL0); 617 618 /* Request for AXI bus port halt */ 619 ret = readl_poll_timeout(vbif_base + VBIF_AXI_HALT_CTRL1, val, 620 val & VBIF_AXI_HALT_CTRL1_HALT_ACK, 621 POLL_INTERVAL_US, 622 VBIF_AXI_HALT_ACK_TIMEOUT_US); 623 if (ret) { 624 dev_err(dev, "AXI bus port halt timeout\n"); 625 return ret; 626 } 627 628 return 0; 629 } 630 venus_power_off(struct venus_hfi_device * hdev)631 static int venus_power_off(struct venus_hfi_device *hdev) 632 { 633 int ret; 634 635 if (!hdev->power_enabled) 636 return 0; 637 638 ret = venus_set_hw_state_suspend(hdev->core); 639 if (ret) 640 return ret; 641 642 ret = venus_halt_axi(hdev); 643 if (ret) 644 return ret; 645 646 hdev->power_enabled = false; 647 648 return 0; 649 } 650 venus_power_on(struct venus_hfi_device * hdev)651 static int venus_power_on(struct venus_hfi_device *hdev) 652 { 653 int ret; 654 655 if (hdev->power_enabled) 656 return 0; 657 658 ret = venus_set_hw_state_resume(hdev->core); 659 if (ret) 660 goto err; 661 662 ret = venus_run(hdev); 663 if (ret) 664 goto err_suspend; 665 666 hdev->power_enabled = true; 667 668 return 0; 669 670 err_suspend: 671 venus_set_hw_state_suspend(hdev->core); 672 err: 673 hdev->power_enabled = false; 674 return ret; 675 } 676 venus_iface_msgq_read_nolock(struct venus_hfi_device * hdev,void * pkt)677 static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev, 678 void *pkt) 679 { 680 struct iface_queue *queue; 681 u32 tx_req; 682 int ret; 683 684 if (!venus_is_valid_state(hdev)) 685 return -EINVAL; 686 687 queue = &hdev->queues[IFACEQ_MSG_IDX]; 688 689 ret = venus_read_queue(hdev, queue, pkt, &tx_req); 690 if (ret) 691 return ret; 692 693 if (tx_req) 694 venus_soft_int(hdev); 695 696 return 0; 697 } 698 venus_iface_msgq_read(struct venus_hfi_device * hdev,void * pkt)699 static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt) 700 { 701 int ret; 702 703 mutex_lock(&hdev->lock); 704 ret = venus_iface_msgq_read_nolock(hdev, pkt); 705 mutex_unlock(&hdev->lock); 706 707 return ret; 708 } 709 venus_iface_dbgq_read_nolock(struct venus_hfi_device * hdev,void * pkt)710 static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev, 711 void *pkt) 712 { 713 struct iface_queue *queue; 714 u32 tx_req; 715 int ret; 716 717 ret = venus_is_valid_state(hdev); 718 if (!ret) 719 return -EINVAL; 720 721 queue = &hdev->queues[IFACEQ_DBG_IDX]; 722 723 ret = venus_read_queue(hdev, queue, pkt, &tx_req); 724 if (ret) 725 return ret; 726 727 if (tx_req) 728 venus_soft_int(hdev); 729 730 return 0; 731 } 732 venus_iface_dbgq_read(struct venus_hfi_device * hdev,void * pkt)733 static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt) 734 { 735 int ret; 736 737 if (!pkt) 738 return -EINVAL; 739 740 mutex_lock(&hdev->lock); 741 ret = venus_iface_dbgq_read_nolock(hdev, pkt); 742 mutex_unlock(&hdev->lock); 743 744 return ret; 745 } 746 venus_set_qhdr_defaults(struct hfi_queue_header * qhdr)747 static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr) 748 { 749 qhdr->status = 1; 750 qhdr->type = IFACEQ_DFLT_QHDR; 751 qhdr->q_size = IFACEQ_QUEUE_SIZE / 4; 752 qhdr->pkt_size = 0; 753 qhdr->rx_wm = 1; 754 qhdr->tx_wm = 1; 755 qhdr->rx_req = 1; 756 qhdr->tx_req = 0; 757 qhdr->rx_irq_status = 0; 758 qhdr->tx_irq_status = 0; 759 qhdr->read_idx = 0; 760 qhdr->write_idx = 0; 761 } 762 venus_interface_queues_release(struct venus_hfi_device * hdev)763 static void venus_interface_queues_release(struct venus_hfi_device *hdev) 764 { 765 mutex_lock(&hdev->lock); 766 767 venus_free(hdev, &hdev->ifaceq_table); 768 venus_free(hdev, &hdev->sfr); 769 770 memset(hdev->queues, 0, sizeof(hdev->queues)); 771 memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table)); 772 memset(&hdev->sfr, 0, sizeof(hdev->sfr)); 773 774 mutex_unlock(&hdev->lock); 775 } 776 venus_interface_queues_init(struct venus_hfi_device * hdev)777 static int venus_interface_queues_init(struct venus_hfi_device *hdev) 778 { 779 struct hfi_queue_table_header *tbl_hdr; 780 struct iface_queue *queue; 781 struct hfi_sfr *sfr; 782 struct mem_desc desc = {0}; 783 unsigned int offset; 784 unsigned int i; 785 int ret; 786 787 ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE); 788 if (ret) 789 return ret; 790 791 hdev->ifaceq_table = desc; 792 offset = IFACEQ_TABLE_SIZE; 793 794 for (i = 0; i < IFACEQ_NUM; i++) { 795 queue = &hdev->queues[i]; 796 queue->qmem.da = desc.da + offset; 797 queue->qmem.kva = desc.kva + offset; 798 queue->qmem.size = IFACEQ_QUEUE_SIZE; 799 offset += queue->qmem.size; 800 queue->qhdr = 801 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i); 802 803 venus_set_qhdr_defaults(queue->qhdr); 804 805 queue->qhdr->start_addr = queue->qmem.da; 806 807 if (i == IFACEQ_CMD_IDX) 808 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q; 809 else if (i == IFACEQ_MSG_IDX) 810 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q; 811 else if (i == IFACEQ_DBG_IDX) 812 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q; 813 } 814 815 tbl_hdr = hdev->ifaceq_table.kva; 816 tbl_hdr->version = 0; 817 tbl_hdr->size = IFACEQ_TABLE_SIZE; 818 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header); 819 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header); 820 tbl_hdr->num_q = IFACEQ_NUM; 821 tbl_hdr->num_active_q = IFACEQ_NUM; 822 823 /* 824 * Set receive request to zero on debug queue as there is no 825 * need of interrupt from video hardware for debug messages 826 */ 827 queue = &hdev->queues[IFACEQ_DBG_IDX]; 828 queue->qhdr->rx_req = 0; 829 830 ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE); 831 if (ret) { 832 hdev->sfr.da = 0; 833 } else { 834 hdev->sfr = desc; 835 sfr = hdev->sfr.kva; 836 sfr->buf_size = ALIGNED_SFR_SIZE; 837 } 838 839 /* ensure table and queue header structs are settled in memory */ 840 wmb(); 841 842 return 0; 843 } 844 venus_sys_set_debug(struct venus_hfi_device * hdev,u32 debug)845 static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug) 846 { 847 struct hfi_sys_set_property_pkt *pkt; 848 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 849 850 pkt = (struct hfi_sys_set_property_pkt *)packet; 851 852 pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug); 853 854 return venus_iface_cmdq_write(hdev, pkt, false); 855 } 856 venus_sys_set_coverage(struct venus_hfi_device * hdev,u32 mode)857 static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode) 858 { 859 struct hfi_sys_set_property_pkt *pkt; 860 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 861 862 pkt = (struct hfi_sys_set_property_pkt *)packet; 863 864 pkt_sys_coverage_config(pkt, mode); 865 866 return venus_iface_cmdq_write(hdev, pkt, false); 867 } 868 venus_sys_set_idle_message(struct venus_hfi_device * hdev,bool enable)869 static int venus_sys_set_idle_message(struct venus_hfi_device *hdev, 870 bool enable) 871 { 872 struct hfi_sys_set_property_pkt *pkt; 873 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 874 875 if (!enable) 876 return 0; 877 878 pkt = (struct hfi_sys_set_property_pkt *)packet; 879 880 pkt_sys_idle_indicator(pkt, enable); 881 882 return venus_iface_cmdq_write(hdev, pkt, false); 883 } 884 venus_sys_set_power_control(struct venus_hfi_device * hdev,bool enable)885 static int venus_sys_set_power_control(struct venus_hfi_device *hdev, 886 bool enable) 887 { 888 struct hfi_sys_set_property_pkt *pkt; 889 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 890 891 pkt = (struct hfi_sys_set_property_pkt *)packet; 892 893 pkt_sys_power_control(pkt, enable); 894 895 return venus_iface_cmdq_write(hdev, pkt, false); 896 } 897 venus_sys_set_ubwc_config(struct venus_hfi_device * hdev)898 static int venus_sys_set_ubwc_config(struct venus_hfi_device *hdev) 899 { 900 struct hfi_sys_set_property_pkt *pkt; 901 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 902 const struct venus_resources *res = hdev->core->res; 903 int ret; 904 905 pkt = (struct hfi_sys_set_property_pkt *)packet; 906 907 pkt_sys_ubwc_config(pkt, res->ubwc_conf); 908 909 ret = venus_iface_cmdq_write(hdev, pkt, false); 910 if (ret) 911 return ret; 912 913 return 0; 914 } 915 venus_get_queue_size(struct venus_hfi_device * hdev,unsigned int index)916 static int venus_get_queue_size(struct venus_hfi_device *hdev, 917 unsigned int index) 918 { 919 struct hfi_queue_header *qhdr; 920 921 if (index >= IFACEQ_NUM) 922 return -EINVAL; 923 924 qhdr = hdev->queues[index].qhdr; 925 if (!qhdr) 926 return -EINVAL; 927 928 return abs(qhdr->read_idx - qhdr->write_idx); 929 } 930 venus_sys_set_default_properties(struct venus_hfi_device * hdev)931 static int venus_sys_set_default_properties(struct venus_hfi_device *hdev) 932 { 933 struct device *dev = hdev->core->dev; 934 const struct venus_resources *res = hdev->core->res; 935 int ret; 936 937 ret = venus_sys_set_debug(hdev, venus_fw_debug); 938 if (ret) 939 dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret); 940 941 /* HFI_PROPERTY_SYS_IDLE_INDICATOR is not supported beyond 8916 (HFI V1) */ 942 if (IS_V1(hdev->core)) { 943 ret = venus_sys_set_idle_message(hdev, false); 944 if (ret) 945 dev_warn(dev, "setting idle response ON failed (%d)\n", ret); 946 } 947 948 ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode); 949 if (ret) 950 dev_warn(dev, "setting hw power collapse ON failed (%d)\n", 951 ret); 952 953 /* For specific venus core, it is mandatory to set the UBWC configuration */ 954 if (res->ubwc_conf) { 955 ret = venus_sys_set_ubwc_config(hdev); 956 if (ret) 957 dev_warn(dev, "setting ubwc config failed (%d)\n", ret); 958 } 959 960 return ret; 961 } 962 venus_session_cmd(struct venus_inst * inst,u32 pkt_type,bool sync)963 static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type, bool sync) 964 { 965 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 966 struct hfi_session_pkt pkt; 967 968 pkt_session_cmd(&pkt, pkt_type, inst); 969 970 return venus_iface_cmdq_write(hdev, &pkt, sync); 971 } 972 venus_flush_debug_queue(struct venus_hfi_device * hdev)973 static void venus_flush_debug_queue(struct venus_hfi_device *hdev) 974 { 975 struct device *dev = hdev->core->dev; 976 void *packet = hdev->dbg_buf; 977 978 while (!venus_iface_dbgq_read(hdev, packet)) { 979 struct hfi_msg_sys_coverage_pkt *pkt = packet; 980 981 if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) { 982 struct hfi_msg_sys_debug_pkt *pkt = packet; 983 984 dev_dbg(dev, VDBGFW "%s", pkt->msg_data); 985 } 986 } 987 } 988 venus_prepare_power_collapse(struct venus_hfi_device * hdev,bool wait)989 static int venus_prepare_power_collapse(struct venus_hfi_device *hdev, 990 bool wait) 991 { 992 unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout); 993 struct hfi_sys_pc_prep_pkt pkt; 994 int ret; 995 996 init_completion(&hdev->pwr_collapse_prep); 997 998 pkt_sys_pc_prep(&pkt); 999 1000 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1001 if (ret) 1002 return ret; 1003 1004 if (!wait) 1005 return 0; 1006 1007 ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout); 1008 if (!ret) { 1009 venus_flush_debug_queue(hdev); 1010 return -ETIMEDOUT; 1011 } 1012 1013 return 0; 1014 } 1015 venus_are_queues_empty(struct venus_hfi_device * hdev)1016 static int venus_are_queues_empty(struct venus_hfi_device *hdev) 1017 { 1018 int ret1, ret2; 1019 1020 ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX); 1021 if (ret1 < 0) 1022 return ret1; 1023 1024 ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX); 1025 if (ret2 < 0) 1026 return ret2; 1027 1028 if (!ret1 && !ret2) 1029 return 1; 1030 1031 return 0; 1032 } 1033 venus_sfr_print(struct venus_hfi_device * hdev)1034 static void venus_sfr_print(struct venus_hfi_device *hdev) 1035 { 1036 struct device *dev = hdev->core->dev; 1037 struct hfi_sfr *sfr = hdev->sfr.kva; 1038 void *p; 1039 1040 if (!sfr) 1041 return; 1042 1043 p = memchr(sfr->data, '\0', sfr->buf_size); 1044 /* 1045 * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates 1046 * that Venus is in the process of crashing. 1047 */ 1048 if (!p) 1049 sfr->data[sfr->buf_size - 1] = '\0'; 1050 1051 dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data); 1052 } 1053 venus_process_msg_sys_error(struct venus_hfi_device * hdev,void * packet)1054 static void venus_process_msg_sys_error(struct venus_hfi_device *hdev, 1055 void *packet) 1056 { 1057 struct hfi_msg_event_notify_pkt *event_pkt = packet; 1058 1059 if (event_pkt->event_id != HFI_EVENT_SYS_ERROR) 1060 return; 1061 1062 venus_set_state(hdev, VENUS_STATE_DEINIT); 1063 1064 venus_sfr_print(hdev); 1065 } 1066 venus_isr_thread(struct venus_core * core)1067 static irqreturn_t venus_isr_thread(struct venus_core *core) 1068 { 1069 struct venus_hfi_device *hdev = to_hfi_priv(core); 1070 const struct venus_resources *res; 1071 void *pkt; 1072 u32 msg_ret; 1073 1074 if (!hdev) 1075 return IRQ_NONE; 1076 1077 res = hdev->core->res; 1078 pkt = hdev->pkt_buf; 1079 1080 1081 while (!venus_iface_msgq_read(hdev, pkt)) { 1082 msg_ret = hfi_process_msg_packet(core, pkt); 1083 switch (msg_ret) { 1084 case HFI_MSG_EVENT_NOTIFY: 1085 venus_process_msg_sys_error(hdev, pkt); 1086 break; 1087 case HFI_MSG_SYS_INIT: 1088 venus_hfi_core_set_resource(core, res->vmem_id, 1089 res->vmem_size, 1090 res->vmem_addr, 1091 hdev); 1092 break; 1093 case HFI_MSG_SYS_RELEASE_RESOURCE: 1094 complete(&hdev->release_resource); 1095 break; 1096 case HFI_MSG_SYS_PC_PREP: 1097 complete(&hdev->pwr_collapse_prep); 1098 break; 1099 default: 1100 break; 1101 } 1102 } 1103 1104 venus_flush_debug_queue(hdev); 1105 1106 return IRQ_HANDLED; 1107 } 1108 venus_isr(struct venus_core * core)1109 static irqreturn_t venus_isr(struct venus_core *core) 1110 { 1111 struct venus_hfi_device *hdev = to_hfi_priv(core); 1112 u32 status; 1113 void __iomem *cpu_cs_base; 1114 void __iomem *wrapper_base; 1115 1116 if (!hdev) 1117 return IRQ_NONE; 1118 1119 cpu_cs_base = hdev->core->cpu_cs_base; 1120 wrapper_base = hdev->core->wrapper_base; 1121 1122 status = readl(wrapper_base + WRAPPER_INTR_STATUS); 1123 if (IS_IRIS2(core) || IS_IRIS2_1(core)) { 1124 if (status & WRAPPER_INTR_STATUS_A2H_MASK || 1125 status & WRAPPER_INTR_STATUS_A2HWD_MASK_V6 || 1126 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK) 1127 hdev->irq_status = status; 1128 } else { 1129 if (status & WRAPPER_INTR_STATUS_A2H_MASK || 1130 status & WRAPPER_INTR_STATUS_A2HWD_MASK || 1131 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK) 1132 hdev->irq_status = status; 1133 } 1134 writel(1, cpu_cs_base + CPU_CS_A2HSOFTINTCLR); 1135 if (!(IS_IRIS2(core) || IS_IRIS2_1(core))) 1136 writel(status, wrapper_base + WRAPPER_INTR_CLEAR); 1137 1138 return IRQ_WAKE_THREAD; 1139 } 1140 venus_core_init(struct venus_core * core)1141 static int venus_core_init(struct venus_core *core) 1142 { 1143 struct venus_hfi_device *hdev = to_hfi_priv(core); 1144 struct device *dev = core->dev; 1145 struct hfi_sys_get_property_pkt version_pkt; 1146 struct hfi_sys_init_pkt pkt; 1147 int ret; 1148 1149 pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX); 1150 1151 venus_set_state(hdev, VENUS_STATE_INIT); 1152 1153 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1154 if (ret) 1155 return ret; 1156 1157 pkt_sys_image_version(&version_pkt); 1158 1159 ret = venus_iface_cmdq_write(hdev, &version_pkt, false); 1160 if (ret) 1161 dev_warn(dev, "failed to send image version pkt to fw\n"); 1162 1163 ret = venus_sys_set_default_properties(hdev); 1164 if (ret) 1165 return ret; 1166 1167 return 0; 1168 } 1169 venus_core_deinit(struct venus_core * core)1170 static int venus_core_deinit(struct venus_core *core) 1171 { 1172 struct venus_hfi_device *hdev = to_hfi_priv(core); 1173 1174 venus_set_state(hdev, VENUS_STATE_DEINIT); 1175 hdev->suspended = true; 1176 hdev->power_enabled = false; 1177 1178 return 0; 1179 } 1180 venus_core_ping(struct venus_core * core,u32 cookie)1181 static int venus_core_ping(struct venus_core *core, u32 cookie) 1182 { 1183 struct venus_hfi_device *hdev = to_hfi_priv(core); 1184 struct hfi_sys_ping_pkt pkt; 1185 1186 pkt_sys_ping(&pkt, cookie); 1187 1188 return venus_iface_cmdq_write(hdev, &pkt, false); 1189 } 1190 venus_core_trigger_ssr(struct venus_core * core,u32 trigger_type)1191 static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type) 1192 { 1193 struct venus_hfi_device *hdev = to_hfi_priv(core); 1194 struct hfi_sys_test_ssr_pkt pkt; 1195 int ret; 1196 1197 ret = pkt_sys_ssr_cmd(&pkt, trigger_type); 1198 if (ret) 1199 return ret; 1200 1201 return venus_iface_cmdq_write(hdev, &pkt, false); 1202 } 1203 venus_session_init(struct venus_inst * inst,u32 session_type,u32 codec)1204 static int venus_session_init(struct venus_inst *inst, u32 session_type, 1205 u32 codec) 1206 { 1207 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1208 struct hfi_session_init_pkt pkt; 1209 int ret; 1210 1211 ret = venus_sys_set_debug(hdev, venus_fw_debug); 1212 if (ret) 1213 goto err; 1214 1215 ret = pkt_session_init(&pkt, inst, session_type, codec); 1216 if (ret) 1217 goto err; 1218 1219 ret = venus_iface_cmdq_write(hdev, &pkt, true); 1220 if (ret) 1221 goto err; 1222 1223 return 0; 1224 1225 err: 1226 venus_flush_debug_queue(hdev); 1227 return ret; 1228 } 1229 venus_session_end(struct venus_inst * inst)1230 static int venus_session_end(struct venus_inst *inst) 1231 { 1232 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1233 struct device *dev = hdev->core->dev; 1234 1235 if (venus_fw_coverage) { 1236 if (venus_sys_set_coverage(hdev, venus_fw_coverage)) 1237 dev_warn(dev, "fw coverage msg ON failed\n"); 1238 } 1239 1240 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END, true); 1241 } 1242 venus_session_abort(struct venus_inst * inst)1243 static int venus_session_abort(struct venus_inst *inst) 1244 { 1245 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1246 1247 venus_flush_debug_queue(hdev); 1248 1249 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT, true); 1250 } 1251 venus_session_flush(struct venus_inst * inst,u32 flush_mode)1252 static int venus_session_flush(struct venus_inst *inst, u32 flush_mode) 1253 { 1254 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1255 struct hfi_session_flush_pkt pkt; 1256 int ret; 1257 1258 ret = pkt_session_flush(&pkt, inst, flush_mode); 1259 if (ret) 1260 return ret; 1261 1262 return venus_iface_cmdq_write(hdev, &pkt, true); 1263 } 1264 venus_session_start(struct venus_inst * inst)1265 static int venus_session_start(struct venus_inst *inst) 1266 { 1267 return venus_session_cmd(inst, HFI_CMD_SESSION_START, true); 1268 } 1269 venus_session_stop(struct venus_inst * inst)1270 static int venus_session_stop(struct venus_inst *inst) 1271 { 1272 return venus_session_cmd(inst, HFI_CMD_SESSION_STOP, true); 1273 } 1274 venus_session_continue(struct venus_inst * inst)1275 static int venus_session_continue(struct venus_inst *inst) 1276 { 1277 return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE, false); 1278 } 1279 venus_session_etb(struct venus_inst * inst,struct hfi_frame_data * in_frame)1280 static int venus_session_etb(struct venus_inst *inst, 1281 struct hfi_frame_data *in_frame) 1282 { 1283 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1284 u32 session_type = inst->session_type; 1285 int ret; 1286 1287 if (session_type == VIDC_SESSION_TYPE_DEC) { 1288 struct hfi_session_empty_buffer_compressed_pkt pkt; 1289 1290 ret = pkt_session_etb_decoder(&pkt, inst, in_frame); 1291 if (ret) 1292 return ret; 1293 1294 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1295 } else if (session_type == VIDC_SESSION_TYPE_ENC) { 1296 struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt; 1297 1298 ret = pkt_session_etb_encoder(&pkt, inst, in_frame); 1299 if (ret) 1300 return ret; 1301 1302 ret = venus_iface_cmdq_write(hdev, &pkt, false); 1303 } else { 1304 ret = -EINVAL; 1305 } 1306 1307 return ret; 1308 } 1309 venus_session_ftb(struct venus_inst * inst,struct hfi_frame_data * out_frame)1310 static int venus_session_ftb(struct venus_inst *inst, 1311 struct hfi_frame_data *out_frame) 1312 { 1313 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1314 struct hfi_session_fill_buffer_pkt pkt; 1315 int ret; 1316 1317 ret = pkt_session_ftb(&pkt, inst, out_frame); 1318 if (ret) 1319 return ret; 1320 1321 return venus_iface_cmdq_write(hdev, &pkt, false); 1322 } 1323 venus_session_set_buffers(struct venus_inst * inst,struct hfi_buffer_desc * bd)1324 static int venus_session_set_buffers(struct venus_inst *inst, 1325 struct hfi_buffer_desc *bd) 1326 { 1327 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1328 struct hfi_session_set_buffers_pkt *pkt; 1329 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE]; 1330 int ret; 1331 1332 if (bd->buffer_type == HFI_BUFFER_INPUT) 1333 return 0; 1334 1335 pkt = (struct hfi_session_set_buffers_pkt *)packet; 1336 1337 ret = pkt_session_set_buffers(pkt, inst, bd); 1338 if (ret) 1339 return ret; 1340 1341 return venus_iface_cmdq_write(hdev, pkt, false); 1342 } 1343 venus_session_unset_buffers(struct venus_inst * inst,struct hfi_buffer_desc * bd)1344 static int venus_session_unset_buffers(struct venus_inst *inst, 1345 struct hfi_buffer_desc *bd) 1346 { 1347 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1348 struct hfi_session_release_buffer_pkt *pkt; 1349 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE]; 1350 int ret; 1351 1352 if (bd->buffer_type == HFI_BUFFER_INPUT) 1353 return 0; 1354 1355 pkt = (struct hfi_session_release_buffer_pkt *)packet; 1356 1357 ret = pkt_session_unset_buffers(pkt, inst, bd); 1358 if (ret) 1359 return ret; 1360 1361 return venus_iface_cmdq_write(hdev, pkt, true); 1362 } 1363 venus_session_load_res(struct venus_inst * inst)1364 static int venus_session_load_res(struct venus_inst *inst) 1365 { 1366 return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES, true); 1367 } 1368 venus_session_release_res(struct venus_inst * inst)1369 static int venus_session_release_res(struct venus_inst *inst) 1370 { 1371 return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES, true); 1372 } 1373 venus_session_parse_seq_hdr(struct venus_inst * inst,u32 seq_hdr,u32 seq_hdr_len)1374 static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr, 1375 u32 seq_hdr_len) 1376 { 1377 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1378 struct hfi_session_parse_sequence_header_pkt *pkt; 1379 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 1380 int ret; 1381 1382 pkt = (struct hfi_session_parse_sequence_header_pkt *)packet; 1383 1384 ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len); 1385 if (ret) 1386 return ret; 1387 1388 ret = venus_iface_cmdq_write(hdev, pkt, false); 1389 if (ret) 1390 return ret; 1391 1392 return 0; 1393 } 1394 venus_session_get_seq_hdr(struct venus_inst * inst,u32 seq_hdr,u32 seq_hdr_len)1395 static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr, 1396 u32 seq_hdr_len) 1397 { 1398 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1399 struct hfi_session_get_sequence_header_pkt *pkt; 1400 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE]; 1401 int ret; 1402 1403 pkt = (struct hfi_session_get_sequence_header_pkt *)packet; 1404 1405 ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len); 1406 if (ret) 1407 return ret; 1408 1409 return venus_iface_cmdq_write(hdev, pkt, false); 1410 } 1411 venus_session_set_property(struct venus_inst * inst,u32 ptype,void * pdata)1412 static int venus_session_set_property(struct venus_inst *inst, u32 ptype, 1413 void *pdata) 1414 { 1415 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1416 struct hfi_session_set_property_pkt *pkt; 1417 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE]; 1418 int ret; 1419 1420 pkt = (struct hfi_session_set_property_pkt *)packet; 1421 1422 ret = pkt_session_set_property(pkt, inst, ptype, pdata); 1423 if (ret == -ENOTSUPP) 1424 return 0; 1425 if (ret) 1426 return ret; 1427 1428 return venus_iface_cmdq_write(hdev, pkt, false); 1429 } 1430 venus_session_get_property(struct venus_inst * inst,u32 ptype)1431 static int venus_session_get_property(struct venus_inst *inst, u32 ptype) 1432 { 1433 struct venus_hfi_device *hdev = to_hfi_priv(inst->core); 1434 struct hfi_session_get_property_pkt pkt; 1435 int ret; 1436 1437 ret = pkt_session_get_property(&pkt, inst, ptype); 1438 if (ret) 1439 return ret; 1440 1441 return venus_iface_cmdq_write(hdev, &pkt, true); 1442 } 1443 venus_resume(struct venus_core * core)1444 static int venus_resume(struct venus_core *core) 1445 { 1446 struct venus_hfi_device *hdev = to_hfi_priv(core); 1447 int ret = 0; 1448 1449 mutex_lock(&hdev->lock); 1450 1451 if (!hdev->suspended) 1452 goto unlock; 1453 1454 ret = venus_power_on(hdev); 1455 1456 unlock: 1457 if (!ret) 1458 hdev->suspended = false; 1459 1460 mutex_unlock(&hdev->lock); 1461 1462 return ret; 1463 } 1464 venus_suspend_1xx(struct venus_core * core)1465 static int venus_suspend_1xx(struct venus_core *core) 1466 { 1467 struct venus_hfi_device *hdev = to_hfi_priv(core); 1468 struct device *dev = core->dev; 1469 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 1470 u32 ctrl_status; 1471 int ret; 1472 1473 if (!hdev->power_enabled || hdev->suspended) 1474 return 0; 1475 1476 mutex_lock(&hdev->lock); 1477 ret = venus_is_valid_state(hdev); 1478 mutex_unlock(&hdev->lock); 1479 1480 if (!ret) { 1481 dev_err(dev, "bad state, cannot suspend\n"); 1482 return -EINVAL; 1483 } 1484 1485 ret = venus_prepare_power_collapse(hdev, true); 1486 if (ret) { 1487 dev_err(dev, "prepare for power collapse fail (%d)\n", ret); 1488 return ret; 1489 } 1490 1491 mutex_lock(&hdev->lock); 1492 1493 if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) { 1494 mutex_unlock(&hdev->lock); 1495 return -EINVAL; 1496 } 1497 1498 ret = venus_are_queues_empty(hdev); 1499 if (ret < 0 || !ret) { 1500 mutex_unlock(&hdev->lock); 1501 return -EINVAL; 1502 } 1503 1504 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 1505 if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) { 1506 mutex_unlock(&hdev->lock); 1507 return -EINVAL; 1508 } 1509 1510 ret = venus_power_off(hdev); 1511 if (ret) { 1512 mutex_unlock(&hdev->lock); 1513 return ret; 1514 } 1515 1516 hdev->suspended = true; 1517 1518 mutex_unlock(&hdev->lock); 1519 1520 return 0; 1521 } 1522 venus_cpu_and_video_core_idle(struct venus_hfi_device * hdev)1523 static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev) 1524 { 1525 void __iomem *wrapper_base = hdev->core->wrapper_base; 1526 void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base; 1527 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 1528 u32 ctrl_status, cpu_status; 1529 1530 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) 1531 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6); 1532 else 1533 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS); 1534 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 1535 1536 if (cpu_status & WRAPPER_CPU_STATUS_WFI && 1537 ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK) 1538 return true; 1539 1540 return false; 1541 } 1542 venus_cpu_idle_and_pc_ready(struct venus_hfi_device * hdev)1543 static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev) 1544 { 1545 void __iomem *wrapper_base = hdev->core->wrapper_base; 1546 void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base; 1547 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 1548 u32 ctrl_status, cpu_status; 1549 1550 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) 1551 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6); 1552 else 1553 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS); 1554 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 1555 1556 if (cpu_status & WRAPPER_CPU_STATUS_WFI && 1557 ctrl_status & CPU_CS_SCIACMDARG0_PC_READY) 1558 return true; 1559 1560 return false; 1561 } 1562 venus_suspend_3xx(struct venus_core * core)1563 static int venus_suspend_3xx(struct venus_core *core) 1564 { 1565 struct venus_hfi_device *hdev = to_hfi_priv(core); 1566 struct device *dev = core->dev; 1567 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base; 1568 u32 ctrl_status; 1569 bool val; 1570 int ret; 1571 1572 if (!hdev->power_enabled || hdev->suspended) 1573 return 0; 1574 1575 mutex_lock(&hdev->lock); 1576 ret = venus_is_valid_state(hdev); 1577 mutex_unlock(&hdev->lock); 1578 1579 if (!ret) { 1580 dev_err(dev, "bad state, cannot suspend\n"); 1581 return -EINVAL; 1582 } 1583 1584 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0); 1585 if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY) 1586 goto power_off; 1587 1588 /* 1589 * Power collapse sequence for Venus 3xx and 4xx versions: 1590 * 1. Check for ARM9 and video core to be idle by checking WFI bit 1591 * (bit 0) in CPU status register and by checking Idle (bit 30) in 1592 * Control status register for video core. 1593 * 2. Send a command to prepare for power collapse. 1594 * 3. Check for WFI and PC_READY bits. 1595 */ 1596 ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val, 1597 1500, 100 * 1500); 1598 if (ret) { 1599 dev_err(dev, "wait for cpu and video core idle fail (%d)\n", ret); 1600 return ret; 1601 } 1602 1603 ret = venus_prepare_power_collapse(hdev, false); 1604 if (ret) { 1605 dev_err(dev, "prepare for power collapse fail (%d)\n", ret); 1606 return ret; 1607 } 1608 1609 ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val, 1610 1500, 100 * 1500); 1611 if (ret) 1612 return ret; 1613 1614 power_off: 1615 mutex_lock(&hdev->lock); 1616 1617 ret = venus_power_off(hdev); 1618 if (ret) { 1619 dev_err(dev, "venus_power_off (%d)\n", ret); 1620 mutex_unlock(&hdev->lock); 1621 return ret; 1622 } 1623 1624 hdev->suspended = true; 1625 1626 mutex_unlock(&hdev->lock); 1627 1628 return 0; 1629 } 1630 venus_suspend(struct venus_core * core)1631 static int venus_suspend(struct venus_core *core) 1632 { 1633 if (IS_V3(core) || IS_V4(core) || IS_V6(core)) 1634 return venus_suspend_3xx(core); 1635 1636 return venus_suspend_1xx(core); 1637 } 1638 1639 static const struct hfi_ops venus_hfi_ops = { 1640 .core_init = venus_core_init, 1641 .core_deinit = venus_core_deinit, 1642 .core_ping = venus_core_ping, 1643 .core_trigger_ssr = venus_core_trigger_ssr, 1644 1645 .session_init = venus_session_init, 1646 .session_end = venus_session_end, 1647 .session_abort = venus_session_abort, 1648 .session_flush = venus_session_flush, 1649 .session_start = venus_session_start, 1650 .session_stop = venus_session_stop, 1651 .session_continue = venus_session_continue, 1652 .session_etb = venus_session_etb, 1653 .session_ftb = venus_session_ftb, 1654 .session_set_buffers = venus_session_set_buffers, 1655 .session_unset_buffers = venus_session_unset_buffers, 1656 .session_load_res = venus_session_load_res, 1657 .session_release_res = venus_session_release_res, 1658 .session_parse_seq_hdr = venus_session_parse_seq_hdr, 1659 .session_get_seq_hdr = venus_session_get_seq_hdr, 1660 .session_set_property = venus_session_set_property, 1661 .session_get_property = venus_session_get_property, 1662 1663 .resume = venus_resume, 1664 .suspend = venus_suspend, 1665 1666 .isr = venus_isr, 1667 .isr_thread = venus_isr_thread, 1668 }; 1669 venus_hfi_destroy(struct venus_core * core)1670 void venus_hfi_destroy(struct venus_core *core) 1671 { 1672 struct venus_hfi_device *hdev = to_hfi_priv(core); 1673 1674 core->priv = NULL; 1675 venus_interface_queues_release(hdev); 1676 mutex_destroy(&hdev->lock); 1677 kfree(hdev); 1678 core->ops = NULL; 1679 } 1680 venus_hfi_create(struct venus_core * core)1681 int venus_hfi_create(struct venus_core *core) 1682 { 1683 struct venus_hfi_device *hdev; 1684 int ret; 1685 1686 hdev = kzalloc(sizeof(*hdev), GFP_KERNEL); 1687 if (!hdev) 1688 return -ENOMEM; 1689 1690 mutex_init(&hdev->lock); 1691 1692 hdev->core = core; 1693 hdev->suspended = true; 1694 core->priv = hdev; 1695 core->ops = &venus_hfi_ops; 1696 1697 ret = venus_interface_queues_init(hdev); 1698 if (ret) 1699 goto err_kfree; 1700 1701 return 0; 1702 1703 err_kfree: 1704 kfree(hdev); 1705 core->priv = NULL; 1706 core->ops = NULL; 1707 return ret; 1708 } 1709 venus_hfi_queues_reinit(struct venus_core * core)1710 void venus_hfi_queues_reinit(struct venus_core *core) 1711 { 1712 struct venus_hfi_device *hdev = to_hfi_priv(core); 1713 struct hfi_queue_table_header *tbl_hdr; 1714 struct iface_queue *queue; 1715 struct hfi_sfr *sfr; 1716 unsigned int i; 1717 1718 mutex_lock(&hdev->lock); 1719 1720 for (i = 0; i < IFACEQ_NUM; i++) { 1721 queue = &hdev->queues[i]; 1722 queue->qhdr = 1723 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i); 1724 1725 venus_set_qhdr_defaults(queue->qhdr); 1726 1727 queue->qhdr->start_addr = queue->qmem.da; 1728 1729 if (i == IFACEQ_CMD_IDX) 1730 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q; 1731 else if (i == IFACEQ_MSG_IDX) 1732 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q; 1733 else if (i == IFACEQ_DBG_IDX) 1734 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q; 1735 } 1736 1737 tbl_hdr = hdev->ifaceq_table.kva; 1738 tbl_hdr->version = 0; 1739 tbl_hdr->size = IFACEQ_TABLE_SIZE; 1740 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header); 1741 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header); 1742 tbl_hdr->num_q = IFACEQ_NUM; 1743 tbl_hdr->num_active_q = IFACEQ_NUM; 1744 1745 /* 1746 * Set receive request to zero on debug queue as there is no 1747 * need of interrupt from video hardware for debug messages 1748 */ 1749 queue = &hdev->queues[IFACEQ_DBG_IDX]; 1750 queue->qhdr->rx_req = 0; 1751 1752 sfr = hdev->sfr.kva; 1753 sfr->buf_size = ALIGNED_SFR_SIZE; 1754 1755 /* ensure table and queue header structs are settled in memory */ 1756 wmb(); 1757 1758 mutex_unlock(&hdev->lock); 1759 } 1760