1 /* 2 * Copyright (c) 2016-2021 The Linux Foundation. All rights reserved. 3 * 4 * Permission to use, copy, modify, and/or distribute this software for 5 * any purpose with or without fee is hereby granted, provided that the 6 * above copyright notice and this permission notice appear in all 7 * copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL 10 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED 11 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE 12 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 13 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR 14 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 15 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 * PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include "hal_hw_headers.h" 20 #include "hal_api.h" 21 #include "hal_reo.h" 22 #include "target_type.h" 23 #include "qdf_module.h" 24 #include "wcss_version.h" 25 26 #ifdef QCA_WIFI_QCA8074 27 void hal_qca6290_attach(struct hal_soc *hal); 28 #endif 29 #ifdef QCA_WIFI_QCA8074 30 void hal_qca8074_attach(struct hal_soc *hal); 31 #endif 32 #if defined(QCA_WIFI_QCA8074V2) || defined(QCA_WIFI_QCA6018) || \ 33 defined(QCA_WIFI_QCA9574) 34 void hal_qca8074v2_attach(struct hal_soc *hal); 35 #endif 36 #ifdef QCA_WIFI_QCA6390 37 void hal_qca6390_attach(struct hal_soc *hal); 38 #endif 39 #ifdef QCA_WIFI_QCA6490 40 void hal_qca6490_attach(struct hal_soc *hal); 41 #endif 42 #ifdef QCA_WIFI_QCN9000 43 void hal_qcn9000_attach(struct hal_soc *hal); 44 #endif 45 #ifdef QCA_WIFI_QCN9224 46 void hal_qcn9224_attach(struct hal_soc *hal); 47 #endif 48 #ifdef QCA_WIFI_QCN6122 49 void hal_qcn6122_attach(struct hal_soc *hal); 50 #endif 51 #ifdef QCA_WIFI_QCA6750 52 void hal_qca6750_attach(struct hal_soc *hal); 53 #endif 54 #ifdef QCA_WIFI_QCA5018 55 void hal_qca5018_attach(struct hal_soc *hal); 56 #endif 57 #ifdef QCA_WIFI_WCN7850 58 void hal_wcn7850_attach(struct hal_soc *hal); 59 #endif 60 61 #ifdef ENABLE_VERBOSE_DEBUG 62 bool is_hal_verbose_debug_enabled; 63 #endif 64 65 #define HAL_REO_DESTINATION_RING_CTRL_IX_0_ADDR(x) ((x) + 0x4) 66 #define HAL_REO_DESTINATION_RING_CTRL_IX_1_ADDR(x) ((x) + 0x8) 67 #define HAL_REO_DESTINATION_RING_CTRL_IX_2_ADDR(x) ((x) + 0xc) 68 #define HAL_REO_DESTINATION_RING_CTRL_IX_3_ADDR(x) ((x) + 0x10) 69 70 #ifdef ENABLE_HAL_REG_WR_HISTORY 71 struct hal_reg_write_fail_history hal_reg_wr_hist; 72 73 void hal_reg_wr_fail_history_add(struct hal_soc *hal_soc, 74 uint32_t offset, 75 uint32_t wr_val, uint32_t rd_val) 76 { 77 struct hal_reg_write_fail_entry *record; 78 int idx; 79 80 idx = hal_history_get_next_index(&hal_soc->reg_wr_fail_hist->index, 81 HAL_REG_WRITE_HIST_SIZE); 82 83 record = &hal_soc->reg_wr_fail_hist->record[idx]; 84 85 record->timestamp = qdf_get_log_timestamp(); 86 record->reg_offset = offset; 87 record->write_val = wr_val; 88 record->read_val = rd_val; 89 } 90 91 static void hal_reg_write_fail_history_init(struct hal_soc *hal) 92 { 93 hal->reg_wr_fail_hist = &hal_reg_wr_hist; 94 95 qdf_atomic_set(&hal->reg_wr_fail_hist->index, -1); 96 } 97 #else 98 static void hal_reg_write_fail_history_init(struct hal_soc *hal) 99 { 100 } 101 #endif 102 103 /** 104 * hal_get_srng_ring_id() - get the ring id of a descriped ring 105 * @hal: hal_soc data structure 106 * @ring_type: type enum describing the ring 107 * @ring_num: which ring of the ring type 108 * @mac_id: which mac does the ring belong to (or 0 for non-lmac rings) 109 * 110 * Return: the ring id or -EINVAL if the ring does not exist. 111 */ 112 static int hal_get_srng_ring_id(struct hal_soc *hal, int ring_type, 113 int ring_num, int mac_id) 114 { 115 struct hal_hw_srng_config *ring_config = 116 HAL_SRNG_CONFIG(hal, ring_type); 117 int ring_id; 118 119 if (ring_num >= ring_config->max_rings) { 120 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_INFO, 121 "%s: ring_num exceeded maximum no. of supported rings", 122 __func__); 123 /* TODO: This is a programming error. Assert if this happens */ 124 return -EINVAL; 125 } 126 127 /* 128 * For BE, dmac_cmn_src_rxbuf_ring is set. If this is set 129 * and ring is dst and also lmac ring then provide ring id per lmac 130 */ 131 if (ring_config->lmac_ring && 132 (!hal->dmac_cmn_src_rxbuf_ring || 133 ring_config->ring_dir == HAL_SRNG_DST_RING)) { 134 ring_id = (ring_config->start_ring_id + ring_num + 135 (mac_id * HAL_MAX_RINGS_PER_LMAC)); 136 } else { 137 ring_id = ring_config->start_ring_id + ring_num; 138 } 139 140 return ring_id; 141 } 142 143 static struct hal_srng *hal_get_srng(struct hal_soc *hal, int ring_id) 144 { 145 /* TODO: Should we allocate srng structures dynamically? */ 146 return &(hal->srng_list[ring_id]); 147 } 148 149 #ifndef SHADOW_REG_CONFIG_DISABLED 150 #define HP_OFFSET_IN_REG_START 1 151 #define OFFSET_FROM_HP_TO_TP 4 152 static void hal_update_srng_hp_tp_address(struct hal_soc *hal_soc, 153 int shadow_config_index, 154 int ring_type, 155 int ring_num) 156 { 157 struct hal_srng *srng; 158 int ring_id; 159 struct hal_hw_srng_config *ring_config = 160 HAL_SRNG_CONFIG(hal_soc, ring_type); 161 162 ring_id = hal_get_srng_ring_id(hal_soc, ring_type, ring_num, 0); 163 if (ring_id < 0) 164 return; 165 166 srng = hal_get_srng(hal_soc, ring_id); 167 168 if (ring_config->ring_dir == HAL_SRNG_DST_RING) { 169 srng->u.dst_ring.tp_addr = SHADOW_REGISTER(shadow_config_index) 170 + hal_soc->dev_base_addr; 171 hal_debug("tp_addr=%pK dev base addr %pK index %u", 172 srng->u.dst_ring.tp_addr, hal_soc->dev_base_addr, 173 shadow_config_index); 174 } else { 175 srng->u.src_ring.hp_addr = SHADOW_REGISTER(shadow_config_index) 176 + hal_soc->dev_base_addr; 177 hal_debug("hp_addr=%pK dev base addr %pK index %u", 178 srng->u.src_ring.hp_addr, 179 hal_soc->dev_base_addr, shadow_config_index); 180 } 181 182 } 183 #endif 184 185 #ifdef GENERIC_SHADOW_REGISTER_ACCESS_ENABLE 186 void hal_set_one_target_reg_config(struct hal_soc *hal, 187 uint32_t target_reg_offset, 188 int list_index) 189 { 190 int i = list_index; 191 192 qdf_assert_always(i < MAX_GENERIC_SHADOW_REG); 193 hal->list_shadow_reg_config[i].target_register = 194 target_reg_offset; 195 hal->num_generic_shadow_regs_configured++; 196 } 197 198 qdf_export_symbol(hal_set_one_target_reg_config); 199 200 #define REO_R0_DESTINATION_RING_CTRL_ADDR_OFFSET 0x4 201 #define MAX_REO_REMAP_SHADOW_REGS 4 202 QDF_STATUS hal_set_shadow_regs(void *hal_soc) 203 { 204 uint32_t target_reg_offset; 205 struct hal_soc *hal = (struct hal_soc *)hal_soc; 206 int i; 207 struct hal_hw_srng_config *srng_config = 208 &hal->hw_srng_table[WBM2SW_RELEASE]; 209 uint32_t reo_reg_base; 210 211 reo_reg_base = hal_get_reo_reg_base_offset(hal_soc); 212 213 target_reg_offset = 214 HAL_REO_DESTINATION_RING_CTRL_IX_0_ADDR(reo_reg_base); 215 216 for (i = 0; i < MAX_REO_REMAP_SHADOW_REGS; i++) { 217 hal_set_one_target_reg_config(hal, target_reg_offset, i); 218 target_reg_offset += REO_R0_DESTINATION_RING_CTRL_ADDR_OFFSET; 219 } 220 221 target_reg_offset = srng_config->reg_start[HP_OFFSET_IN_REG_START]; 222 target_reg_offset += (srng_config->reg_size[HP_OFFSET_IN_REG_START] 223 * HAL_IPA_TX_COMP_RING_IDX); 224 225 hal_set_one_target_reg_config(hal, target_reg_offset, i); 226 return QDF_STATUS_SUCCESS; 227 } 228 229 qdf_export_symbol(hal_set_shadow_regs); 230 231 QDF_STATUS hal_construct_shadow_regs(void *hal_soc) 232 { 233 struct hal_soc *hal = (struct hal_soc *)hal_soc; 234 int shadow_config_index = hal->num_shadow_registers_configured; 235 int i; 236 int num_regs = hal->num_generic_shadow_regs_configured; 237 238 for (i = 0; i < num_regs; i++) { 239 qdf_assert_always(shadow_config_index < MAX_SHADOW_REGISTERS); 240 hal->shadow_config[shadow_config_index].addr = 241 hal->list_shadow_reg_config[i].target_register; 242 hal->list_shadow_reg_config[i].shadow_config_index = 243 shadow_config_index; 244 hal->list_shadow_reg_config[i].va = 245 SHADOW_REGISTER(shadow_config_index) + 246 (uintptr_t)hal->dev_base_addr; 247 hal_debug("target_reg %x, shadow register 0x%x shadow_index 0x%x", 248 hal->shadow_config[shadow_config_index].addr, 249 SHADOW_REGISTER(shadow_config_index), 250 shadow_config_index); 251 shadow_config_index++; 252 hal->num_shadow_registers_configured++; 253 } 254 return QDF_STATUS_SUCCESS; 255 } 256 257 qdf_export_symbol(hal_construct_shadow_regs); 258 #endif 259 260 #ifndef SHADOW_REG_CONFIG_DISABLED 261 262 QDF_STATUS hal_set_one_shadow_config(void *hal_soc, 263 int ring_type, 264 int ring_num) 265 { 266 uint32_t target_register; 267 struct hal_soc *hal = (struct hal_soc *)hal_soc; 268 struct hal_hw_srng_config *srng_config = &hal->hw_srng_table[ring_type]; 269 int shadow_config_index = hal->num_shadow_registers_configured; 270 271 if (shadow_config_index >= MAX_SHADOW_REGISTERS) { 272 QDF_ASSERT(0); 273 return QDF_STATUS_E_RESOURCES; 274 } 275 276 hal->num_shadow_registers_configured++; 277 278 target_register = srng_config->reg_start[HP_OFFSET_IN_REG_START]; 279 target_register += (srng_config->reg_size[HP_OFFSET_IN_REG_START] 280 *ring_num); 281 282 /* if the ring is a dst ring, we need to shadow the tail pointer */ 283 if (srng_config->ring_dir == HAL_SRNG_DST_RING) 284 target_register += OFFSET_FROM_HP_TO_TP; 285 286 hal->shadow_config[shadow_config_index].addr = target_register; 287 288 /* update hp/tp addr in the hal_soc structure*/ 289 hal_update_srng_hp_tp_address(hal_soc, shadow_config_index, ring_type, 290 ring_num); 291 292 hal_debug("target_reg %x, shadow register 0x%x shadow_index 0x%x, ring_type %d, ring num %d", 293 target_register, 294 SHADOW_REGISTER(shadow_config_index), 295 shadow_config_index, 296 ring_type, ring_num); 297 298 return QDF_STATUS_SUCCESS; 299 } 300 301 qdf_export_symbol(hal_set_one_shadow_config); 302 303 QDF_STATUS hal_construct_srng_shadow_regs(void *hal_soc) 304 { 305 int ring_type, ring_num; 306 struct hal_soc *hal = (struct hal_soc *)hal_soc; 307 308 for (ring_type = 0; ring_type < MAX_RING_TYPES; ring_type++) { 309 struct hal_hw_srng_config *srng_config = 310 &hal->hw_srng_table[ring_type]; 311 312 if (ring_type == CE_SRC || 313 ring_type == CE_DST || 314 ring_type == CE_DST_STATUS) 315 continue; 316 317 if (srng_config->lmac_ring) 318 continue; 319 320 for (ring_num = 0; ring_num < srng_config->max_rings; 321 ring_num++) 322 hal_set_one_shadow_config(hal_soc, ring_type, ring_num); 323 } 324 325 return QDF_STATUS_SUCCESS; 326 } 327 328 qdf_export_symbol(hal_construct_srng_shadow_regs); 329 #else 330 331 QDF_STATUS hal_construct_srng_shadow_regs(void *hal_soc) 332 { 333 return QDF_STATUS_SUCCESS; 334 } 335 336 qdf_export_symbol(hal_construct_srng_shadow_regs); 337 338 QDF_STATUS hal_set_one_shadow_config(void *hal_soc, int ring_type, 339 int ring_num) 340 { 341 return QDF_STATUS_SUCCESS; 342 } 343 qdf_export_symbol(hal_set_one_shadow_config); 344 #endif 345 346 void hal_get_shadow_config(void *hal_soc, 347 struct pld_shadow_reg_v2_cfg **shadow_config, 348 int *num_shadow_registers_configured) 349 { 350 struct hal_soc *hal = (struct hal_soc *)hal_soc; 351 352 *shadow_config = hal->shadow_config; 353 *num_shadow_registers_configured = 354 hal->num_shadow_registers_configured; 355 } 356 357 qdf_export_symbol(hal_get_shadow_config); 358 359 static bool hal_validate_shadow_register(struct hal_soc *hal, 360 uint32_t *destination, 361 uint32_t *shadow_address) 362 { 363 unsigned int index; 364 uint32_t *shadow_0_offset = SHADOW_REGISTER(0) + hal->dev_base_addr; 365 int destination_ba_offset = 366 ((char *)destination) - (char *)hal->dev_base_addr; 367 368 index = shadow_address - shadow_0_offset; 369 370 if (index >= MAX_SHADOW_REGISTERS) { 371 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR, 372 "%s: index %x out of bounds", __func__, index); 373 goto error; 374 } else if (hal->shadow_config[index].addr != destination_ba_offset) { 375 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR, 376 "%s: sanity check failure, expected %x, found %x", 377 __func__, destination_ba_offset, 378 hal->shadow_config[index].addr); 379 goto error; 380 } 381 return true; 382 error: 383 qdf_print("baddr %pK, desination %pK, shadow_address %pK s0offset %pK index %x", 384 hal->dev_base_addr, destination, shadow_address, 385 shadow_0_offset, index); 386 QDF_BUG(0); 387 return false; 388 } 389 390 static void hal_target_based_configure(struct hal_soc *hal) 391 { 392 /** 393 * Indicate Initialization of srngs to avoid force wake 394 * as umac power collapse is not enabled yet 395 */ 396 hal->init_phase = true; 397 398 switch (hal->target_type) { 399 #ifdef QCA_WIFI_QCA6290 400 case TARGET_TYPE_QCA6290: 401 hal->use_register_windowing = true; 402 hal_qca6290_attach(hal); 403 break; 404 #endif 405 #ifdef QCA_WIFI_QCA6390 406 case TARGET_TYPE_QCA6390: 407 hal->use_register_windowing = true; 408 hal_qca6390_attach(hal); 409 break; 410 #endif 411 #ifdef QCA_WIFI_QCA6490 412 case TARGET_TYPE_QCA6490: 413 hal->use_register_windowing = true; 414 hal_qca6490_attach(hal); 415 break; 416 #endif 417 #ifdef QCA_WIFI_QCA6750 418 case TARGET_TYPE_QCA6750: 419 hal->use_register_windowing = true; 420 hal->static_window_map = true; 421 hal_qca6750_attach(hal); 422 break; 423 #endif 424 #ifdef QCA_WIFI_WCN7850 425 case TARGET_TYPE_WCN7850: 426 hal->use_register_windowing = true; 427 hal_wcn7850_attach(hal); 428 hal->init_phase = false; 429 break; 430 #endif 431 #if defined(QCA_WIFI_QCA8074) && defined(WIFI_TARGET_TYPE_3_0) 432 case TARGET_TYPE_QCA8074: 433 hal_qca8074_attach(hal); 434 break; 435 #endif 436 437 #if defined(QCA_WIFI_QCA8074V2) 438 case TARGET_TYPE_QCA8074V2: 439 hal_qca8074v2_attach(hal); 440 break; 441 #endif 442 443 #if defined(QCA_WIFI_QCA6018) 444 case TARGET_TYPE_QCA6018: 445 hal_qca8074v2_attach(hal); 446 break; 447 #endif 448 449 #if defined(QCA_WIFI_QCA9574) 450 case TARGET_TYPE_QCA9574: 451 hal_qca8074v2_attach(hal); 452 break; 453 #endif 454 455 #if defined(QCA_WIFI_QCN6122) 456 case TARGET_TYPE_QCN6122: 457 hal->use_register_windowing = true; 458 /* 459 * Static window map is enabled for qcn9000 to use 2mb bar 460 * size and use multiple windows to write into registers. 461 */ 462 hal->static_window_map = true; 463 hal_qcn6122_attach(hal); 464 break; 465 #endif 466 467 #ifdef QCA_WIFI_QCN9000 468 case TARGET_TYPE_QCN9000: 469 hal->use_register_windowing = true; 470 /* 471 * Static window map is enabled for qcn9000 to use 2mb bar 472 * size and use multiple windows to write into registers. 473 */ 474 hal->static_window_map = true; 475 hal_qcn9000_attach(hal); 476 break; 477 #endif 478 #ifdef QCA_WIFI_QCA5018 479 case TARGET_TYPE_QCA5018: 480 hal->use_register_windowing = true; 481 hal->static_window_map = true; 482 hal_qca5018_attach(hal); 483 break; 484 #endif 485 #ifdef QCA_WIFI_QCN9224 486 case TARGET_TYPE_QCN9224: 487 hal->use_register_windowing = true; 488 hal->static_window_map = true; 489 hal_qcn9224_attach(hal); 490 break; 491 #endif 492 default: 493 break; 494 } 495 } 496 497 uint32_t hal_get_target_type(hal_soc_handle_t hal_soc_hdl) 498 { 499 struct hal_soc *hal_soc = (struct hal_soc *)hal_soc_hdl; 500 struct hif_target_info *tgt_info = 501 hif_get_target_info_handle(hal_soc->hif_handle); 502 503 return tgt_info->target_type; 504 } 505 506 qdf_export_symbol(hal_get_target_type); 507 508 #if defined(FEATURE_HAL_DELAYED_REG_WRITE) 509 /** 510 * hal_is_reg_write_tput_level_high() - throughput level for delayed reg writes 511 * @hal: hal_soc pointer 512 * 513 * Return: true if throughput is high, else false. 514 */ 515 static inline bool hal_is_reg_write_tput_level_high(struct hal_soc *hal) 516 { 517 int bw_level = hif_get_bandwidth_level(hal->hif_handle); 518 519 return (bw_level >= PLD_BUS_WIDTH_MEDIUM) ? true : false; 520 } 521 522 static inline 523 char *hal_fill_reg_write_srng_stats(struct hal_srng *srng, 524 char *buf, qdf_size_t size) 525 { 526 qdf_scnprintf(buf, size, "enq %u deq %u coal %u direct %u", 527 srng->wstats.enqueues, srng->wstats.dequeues, 528 srng->wstats.coalesces, srng->wstats.direct); 529 return buf; 530 } 531 532 /* bytes for local buffer */ 533 #define HAL_REG_WRITE_SRNG_STATS_LEN 100 534 535 void hal_dump_reg_write_srng_stats(hal_soc_handle_t hal_soc_hdl) 536 { 537 struct hal_srng *srng; 538 char buf[HAL_REG_WRITE_SRNG_STATS_LEN]; 539 struct hal_soc *hal = (struct hal_soc *)hal_soc_hdl; 540 541 srng = hal_get_srng(hal, HAL_SRNG_SW2TCL1); 542 hal_debug("SW2TCL1: %s", 543 hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf))); 544 545 srng = hal_get_srng(hal, HAL_SRNG_WBM2SW0_RELEASE); 546 hal_debug("WBM2SW0: %s", 547 hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf))); 548 549 srng = hal_get_srng(hal, HAL_SRNG_REO2SW1); 550 hal_debug("REO2SW1: %s", 551 hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf))); 552 553 srng = hal_get_srng(hal, HAL_SRNG_REO2SW2); 554 hal_debug("REO2SW2: %s", 555 hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf))); 556 557 srng = hal_get_srng(hal, HAL_SRNG_REO2SW3); 558 hal_debug("REO2SW3: %s", 559 hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf))); 560 } 561 562 void hal_dump_reg_write_stats(hal_soc_handle_t hal_soc_hdl) 563 { 564 uint32_t *hist; 565 struct hal_soc *hal = (struct hal_soc *)hal_soc_hdl; 566 567 hist = hal->stats.wstats.sched_delay; 568 hal_debug("wstats: enq %u deq %u coal %u direct %u q_depth %u max_q %u sched-delay hist %u %u %u %u", 569 qdf_atomic_read(&hal->stats.wstats.enqueues), 570 hal->stats.wstats.dequeues, 571 qdf_atomic_read(&hal->stats.wstats.coalesces), 572 qdf_atomic_read(&hal->stats.wstats.direct), 573 qdf_atomic_read(&hal->stats.wstats.q_depth), 574 hal->stats.wstats.max_q_depth, 575 hist[REG_WRITE_SCHED_DELAY_SUB_100us], 576 hist[REG_WRITE_SCHED_DELAY_SUB_1000us], 577 hist[REG_WRITE_SCHED_DELAY_SUB_5000us], 578 hist[REG_WRITE_SCHED_DELAY_GT_5000us]); 579 } 580 581 int hal_get_reg_write_pending_work(void *hal_soc) 582 { 583 struct hal_soc *hal = (struct hal_soc *)hal_soc; 584 585 return qdf_atomic_read(&hal->active_work_cnt); 586 } 587 588 #endif 589 590 #ifdef FEATURE_HAL_DELAYED_REG_WRITE 591 #ifdef MEMORY_DEBUG 592 /* 593 * Length of the queue(array) used to hold delayed register writes. 594 * Must be a multiple of 2. 595 */ 596 #define HAL_REG_WRITE_QUEUE_LEN 128 597 #else 598 #define HAL_REG_WRITE_QUEUE_LEN 32 599 #endif 600 601 /** 602 * hal_process_reg_write_q_elem() - process a regiter write queue element 603 * @hal: hal_soc pointer 604 * @q_elem: pointer to hal regiter write queue element 605 * 606 * Return: The value which was written to the address 607 */ 608 static uint32_t 609 hal_process_reg_write_q_elem(struct hal_soc *hal, 610 struct hal_reg_write_q_elem *q_elem) 611 { 612 struct hal_srng *srng = q_elem->srng; 613 uint32_t write_val; 614 615 SRNG_LOCK(&srng->lock); 616 617 srng->reg_write_in_progress = false; 618 srng->wstats.dequeues++; 619 620 if (srng->ring_dir == HAL_SRNG_SRC_RING) { 621 q_elem->dequeue_val = srng->u.src_ring.hp; 622 hal_write_address_32_mb(hal, 623 srng->u.src_ring.hp_addr, 624 srng->u.src_ring.hp, false); 625 write_val = srng->u.src_ring.hp; 626 } else { 627 q_elem->dequeue_val = srng->u.dst_ring.tp; 628 hal_write_address_32_mb(hal, 629 srng->u.dst_ring.tp_addr, 630 srng->u.dst_ring.tp, false); 631 write_val = srng->u.dst_ring.tp; 632 } 633 634 q_elem->valid = 0; 635 srng->last_dequeue_time = q_elem->dequeue_time; 636 SRNG_UNLOCK(&srng->lock); 637 638 return write_val; 639 } 640 641 /** 642 * hal_reg_write_fill_sched_delay_hist() - fill reg write delay histogram in hal 643 * @hal: hal_soc pointer 644 * @delay: delay in us 645 * 646 * Return: None 647 */ 648 static inline void hal_reg_write_fill_sched_delay_hist(struct hal_soc *hal, 649 uint64_t delay_us) 650 { 651 uint32_t *hist; 652 653 hist = hal->stats.wstats.sched_delay; 654 655 if (delay_us < 100) 656 hist[REG_WRITE_SCHED_DELAY_SUB_100us]++; 657 else if (delay_us < 1000) 658 hist[REG_WRITE_SCHED_DELAY_SUB_1000us]++; 659 else if (delay_us < 5000) 660 hist[REG_WRITE_SCHED_DELAY_SUB_5000us]++; 661 else 662 hist[REG_WRITE_SCHED_DELAY_GT_5000us]++; 663 } 664 665 #ifdef SHADOW_WRITE_DELAY 666 667 #define SHADOW_WRITE_MIN_DELTA_US 5 668 #define SHADOW_WRITE_DELAY_US 50 669 670 /* 671 * Never add those srngs which are performance relate. 672 * The delay itself will hit performance heavily. 673 */ 674 #define IS_SRNG_MATCH(s) ((s)->ring_id == HAL_SRNG_CE_1_DST_STATUS || \ 675 (s)->ring_id == HAL_SRNG_CE_1_DST) 676 677 static inline bool hal_reg_write_need_delay(struct hal_reg_write_q_elem *elem) 678 { 679 struct hal_srng *srng = elem->srng; 680 struct hal_soc *hal; 681 qdf_time_t now; 682 qdf_iomem_t real_addr; 683 684 if (qdf_unlikely(!srng)) 685 return false; 686 687 hal = srng->hal_soc; 688 if (qdf_unlikely(!hal)) 689 return false; 690 691 /* Check if it is target srng, and valid shadow reg */ 692 if (qdf_likely(!IS_SRNG_MATCH(srng))) 693 return false; 694 695 if (srng->ring_dir == HAL_SRNG_SRC_RING) 696 real_addr = SRNG_SRC_ADDR(srng, HP); 697 else 698 real_addr = SRNG_DST_ADDR(srng, TP); 699 if (!hal_validate_shadow_register(hal, real_addr, elem->addr)) 700 return false; 701 702 /* Check the time delta from last write of same srng */ 703 now = qdf_get_log_timestamp(); 704 if (qdf_log_timestamp_to_usecs(now - srng->last_dequeue_time) > 705 SHADOW_WRITE_MIN_DELTA_US) 706 return false; 707 708 /* Delay dequeue, and record */ 709 qdf_udelay(SHADOW_WRITE_DELAY_US); 710 711 srng->wstats.dequeue_delay++; 712 hal->stats.wstats.dequeue_delay++; 713 714 return true; 715 } 716 #else 717 static inline bool hal_reg_write_need_delay(struct hal_reg_write_q_elem *elem) 718 { 719 return false; 720 } 721 #endif 722 723 /** 724 * hal_reg_write_work() - Worker to process delayed writes 725 * @arg: hal_soc pointer 726 * 727 * Return: None 728 */ 729 static void hal_reg_write_work(void *arg) 730 { 731 int32_t q_depth, write_val; 732 struct hal_soc *hal = arg; 733 struct hal_reg_write_q_elem *q_elem; 734 uint64_t delta_us; 735 uint8_t ring_id; 736 uint32_t *addr; 737 uint32_t num_processed = 0; 738 739 q_elem = &hal->reg_write_queue[(hal->read_idx)]; 740 q_elem->work_scheduled_time = qdf_get_log_timestamp(); 741 q_elem->cpu_id = qdf_get_cpu(); 742 743 /* Make sure q_elem consistent in the memory for multi-cores */ 744 qdf_rmb(); 745 if (!q_elem->valid) 746 return; 747 748 q_depth = qdf_atomic_read(&hal->stats.wstats.q_depth); 749 if (q_depth > hal->stats.wstats.max_q_depth) 750 hal->stats.wstats.max_q_depth = q_depth; 751 752 if (hif_prevent_link_low_power_states(hal->hif_handle)) { 753 hal->stats.wstats.prevent_l1_fails++; 754 return; 755 } 756 757 while (true) { 758 qdf_rmb(); 759 if (!q_elem->valid) 760 break; 761 762 q_elem->dequeue_time = qdf_get_log_timestamp(); 763 ring_id = q_elem->srng->ring_id; 764 addr = q_elem->addr; 765 delta_us = qdf_log_timestamp_to_usecs(q_elem->dequeue_time - 766 q_elem->enqueue_time); 767 hal_reg_write_fill_sched_delay_hist(hal, delta_us); 768 769 hal->stats.wstats.dequeues++; 770 qdf_atomic_dec(&hal->stats.wstats.q_depth); 771 772 if (hal_reg_write_need_delay(q_elem)) 773 hal_verbose_debug("Delay reg writer for srng 0x%x, addr 0x%pK", 774 q_elem->srng->ring_id, q_elem->addr); 775 776 write_val = hal_process_reg_write_q_elem(hal, q_elem); 777 hal_verbose_debug("read_idx %u srng 0x%x, addr 0x%pK dequeue_val %u sched delay %llu us", 778 hal->read_idx, ring_id, addr, write_val, delta_us); 779 780 num_processed++; 781 hal->read_idx = (hal->read_idx + 1) & 782 (HAL_REG_WRITE_QUEUE_LEN - 1); 783 q_elem = &hal->reg_write_queue[(hal->read_idx)]; 784 } 785 786 hif_allow_link_low_power_states(hal->hif_handle); 787 /* 788 * Decrement active_work_cnt by the number of elements dequeued after 789 * hif_allow_link_low_power_states. 790 * This makes sure that hif_try_complete_tasks will wait till we make 791 * the bus access in hif_allow_link_low_power_states. This will avoid 792 * race condition between delayed register worker and bus suspend 793 * (system suspend or runtime suspend). 794 * 795 * The following decrement should be done at the end! 796 */ 797 qdf_atomic_sub(num_processed, &hal->active_work_cnt); 798 } 799 800 static void __hal_flush_reg_write_work(struct hal_soc *hal) 801 { 802 qdf_flush_work(&hal->reg_write_work); 803 qdf_disable_work(&hal->reg_write_work); 804 } 805 806 void hal_flush_reg_write_work(hal_soc_handle_t hal_handle) 807 { __hal_flush_reg_write_work((struct hal_soc *)hal_handle); 808 } 809 810 /** 811 * hal_reg_write_enqueue() - enqueue register writes into kworker 812 * @hal_soc: hal_soc pointer 813 * @srng: srng pointer 814 * @addr: iomem address of regiter 815 * @value: value to be written to iomem address 816 * 817 * This function executes from within the SRNG LOCK 818 * 819 * Return: None 820 */ 821 static void hal_reg_write_enqueue(struct hal_soc *hal_soc, 822 struct hal_srng *srng, 823 void __iomem *addr, 824 uint32_t value) 825 { 826 struct hal_reg_write_q_elem *q_elem; 827 uint32_t write_idx; 828 829 if (srng->reg_write_in_progress) { 830 hal_verbose_debug("Already in progress srng ring id 0x%x addr 0x%pK val %u", 831 srng->ring_id, addr, value); 832 qdf_atomic_inc(&hal_soc->stats.wstats.coalesces); 833 srng->wstats.coalesces++; 834 return; 835 } 836 837 write_idx = qdf_atomic_inc_return(&hal_soc->write_idx); 838 839 write_idx = write_idx & (HAL_REG_WRITE_QUEUE_LEN - 1); 840 841 q_elem = &hal_soc->reg_write_queue[write_idx]; 842 843 if (q_elem->valid) { 844 hal_err("queue full"); 845 QDF_BUG(0); 846 return; 847 } 848 849 qdf_atomic_inc(&hal_soc->stats.wstats.enqueues); 850 srng->wstats.enqueues++; 851 852 qdf_atomic_inc(&hal_soc->stats.wstats.q_depth); 853 854 q_elem->srng = srng; 855 q_elem->addr = addr; 856 q_elem->enqueue_val = value; 857 q_elem->enqueue_time = qdf_get_log_timestamp(); 858 859 /* 860 * Before the valid flag is set to true, all the other 861 * fields in the q_elem needs to be updated in memory. 862 * Else there is a chance that the dequeuing worker thread 863 * might read stale entries and process incorrect srng. 864 */ 865 qdf_wmb(); 866 q_elem->valid = true; 867 868 /* 869 * After all other fields in the q_elem has been updated 870 * in memory successfully, the valid flag needs to be updated 871 * in memory in time too. 872 * Else there is a chance that the dequeuing worker thread 873 * might read stale valid flag and the work will be bypassed 874 * for this round. And if there is no other work scheduled 875 * later, this hal register writing won't be updated any more. 876 */ 877 qdf_wmb(); 878 879 srng->reg_write_in_progress = true; 880 qdf_atomic_inc(&hal_soc->active_work_cnt); 881 882 hal_verbose_debug("write_idx %u srng ring id 0x%x addr 0x%pK val %u", 883 write_idx, srng->ring_id, addr, value); 884 885 qdf_queue_work(hal_soc->qdf_dev, hal_soc->reg_write_wq, 886 &hal_soc->reg_write_work); 887 } 888 889 /** 890 * hal_delayed_reg_write_init() - Initialization function for delayed reg writes 891 * @hal_soc: hal_soc pointer 892 * 893 * Initialize main data structures to process register writes in a delayed 894 * workqueue. 895 * 896 * Return: QDF_STATUS_SUCCESS on success else a QDF error. 897 */ 898 static QDF_STATUS hal_delayed_reg_write_init(struct hal_soc *hal) 899 { 900 hal->reg_write_wq = 901 qdf_alloc_high_prior_ordered_workqueue("hal_register_write_wq"); 902 qdf_create_work(0, &hal->reg_write_work, hal_reg_write_work, hal); 903 hal->reg_write_queue = qdf_mem_malloc(HAL_REG_WRITE_QUEUE_LEN * 904 sizeof(*hal->reg_write_queue)); 905 if (!hal->reg_write_queue) { 906 hal_err("unable to allocate memory"); 907 QDF_BUG(0); 908 return QDF_STATUS_E_NOMEM; 909 } 910 911 /* Initial value of indices */ 912 hal->read_idx = 0; 913 qdf_atomic_set(&hal->write_idx, -1); 914 return QDF_STATUS_SUCCESS; 915 } 916 917 /** 918 * hal_delayed_reg_write_deinit() - De-Initialize delayed reg write processing 919 * @hal_soc: hal_soc pointer 920 * 921 * De-initialize main data structures to process register writes in a delayed 922 * workqueue. 923 * 924 * Return: None 925 */ 926 static void hal_delayed_reg_write_deinit(struct hal_soc *hal) 927 { 928 __hal_flush_reg_write_work(hal); 929 930 qdf_flush_workqueue(0, hal->reg_write_wq); 931 qdf_destroy_workqueue(0, hal->reg_write_wq); 932 qdf_mem_free(hal->reg_write_queue); 933 } 934 935 #else 936 static inline QDF_STATUS hal_delayed_reg_write_init(struct hal_soc *hal) 937 { 938 return QDF_STATUS_SUCCESS; 939 } 940 941 static inline void hal_delayed_reg_write_deinit(struct hal_soc *hal) 942 { 943 } 944 #endif 945 946 #ifdef FEATURE_HAL_DELAYED_REG_WRITE 947 #ifdef QCA_WIFI_QCA6750 948 void hal_delayed_reg_write(struct hal_soc *hal_soc, 949 struct hal_srng *srng, 950 void __iomem *addr, 951 uint32_t value) 952 { 953 uint8_t vote_access; 954 955 switch (srng->ring_type) { 956 case CE_SRC: 957 case CE_DST: 958 case CE_DST_STATUS: 959 vote_access = hif_get_ep_vote_access(hal_soc->hif_handle, 960 HIF_EP_VOTE_NONDP_ACCESS); 961 if ((vote_access == HIF_EP_VOTE_ACCESS_DISABLE) || 962 (vote_access == HIF_EP_VOTE_INTERMEDIATE_ACCESS && 963 PLD_MHI_STATE_L0 == 964 pld_get_mhi_state(hal_soc->qdf_dev->dev))) { 965 hal_write_address_32_mb(hal_soc, addr, value, false); 966 qdf_atomic_inc(&hal_soc->stats.wstats.direct); 967 srng->wstats.direct++; 968 } else { 969 hal_reg_write_enqueue(hal_soc, srng, addr, value); 970 } 971 break; 972 default: 973 if (hif_get_ep_vote_access(hal_soc->hif_handle, 974 HIF_EP_VOTE_DP_ACCESS) == 975 HIF_EP_VOTE_ACCESS_DISABLE || 976 hal_is_reg_write_tput_level_high(hal_soc) || 977 PLD_MHI_STATE_L0 == 978 pld_get_mhi_state(hal_soc->qdf_dev->dev)) { 979 hal_write_address_32_mb(hal_soc, addr, value, false); 980 qdf_atomic_inc(&hal_soc->stats.wstats.direct); 981 srng->wstats.direct++; 982 } else { 983 hal_reg_write_enqueue(hal_soc, srng, addr, value); 984 } 985 986 break; 987 } 988 } 989 #else 990 void hal_delayed_reg_write(struct hal_soc *hal_soc, 991 struct hal_srng *srng, 992 void __iomem *addr, 993 uint32_t value) 994 { 995 if (pld_is_device_awake(hal_soc->qdf_dev->dev) || 996 hal_is_reg_write_tput_level_high(hal_soc)) { 997 qdf_atomic_inc(&hal_soc->stats.wstats.direct); 998 srng->wstats.direct++; 999 hal_write_address_32_mb(hal_soc, addr, value, false); 1000 } else { 1001 hal_reg_write_enqueue(hal_soc, srng, addr, value); 1002 } 1003 } 1004 #endif 1005 #endif 1006 1007 /** 1008 * hal_attach - Initialize HAL layer 1009 * @hif_handle: Opaque HIF handle 1010 * @qdf_dev: QDF device 1011 * 1012 * Return: Opaque HAL SOC handle 1013 * NULL on failure (if given ring is not available) 1014 * 1015 * This function should be called as part of HIF initialization (for accessing 1016 * copy engines). DP layer will get hal_soc handle using hif_get_hal_handle() 1017 * 1018 */ 1019 void *hal_attach(struct hif_opaque_softc *hif_handle, qdf_device_t qdf_dev) 1020 { 1021 struct hal_soc *hal; 1022 int i; 1023 1024 hal = qdf_mem_malloc(sizeof(*hal)); 1025 1026 if (!hal) { 1027 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR, 1028 "%s: hal_soc allocation failed", __func__); 1029 goto fail0; 1030 } 1031 hal->hif_handle = hif_handle; 1032 hal->dev_base_addr = hif_get_dev_ba(hif_handle); /* UMAC */ 1033 hal->dev_base_addr_ce = hif_get_dev_ba_ce(hif_handle); /* CE */ 1034 hal->qdf_dev = qdf_dev; 1035 hal->shadow_rdptr_mem_vaddr = (uint32_t *)qdf_mem_alloc_consistent( 1036 qdf_dev, qdf_dev->dev, sizeof(*(hal->shadow_rdptr_mem_vaddr)) * 1037 HAL_SRNG_ID_MAX, &(hal->shadow_rdptr_mem_paddr)); 1038 if (!hal->shadow_rdptr_mem_paddr) { 1039 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR, 1040 "%s: hal->shadow_rdptr_mem_paddr allocation failed", 1041 __func__); 1042 goto fail1; 1043 } 1044 qdf_mem_zero(hal->shadow_rdptr_mem_vaddr, 1045 sizeof(*(hal->shadow_rdptr_mem_vaddr)) * HAL_SRNG_ID_MAX); 1046 1047 hal->shadow_wrptr_mem_vaddr = 1048 (uint32_t *)qdf_mem_alloc_consistent(qdf_dev, qdf_dev->dev, 1049 sizeof(*(hal->shadow_wrptr_mem_vaddr)) * HAL_MAX_LMAC_RINGS, 1050 &(hal->shadow_wrptr_mem_paddr)); 1051 if (!hal->shadow_wrptr_mem_vaddr) { 1052 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR, 1053 "%s: hal->shadow_wrptr_mem_vaddr allocation failed", 1054 __func__); 1055 goto fail2; 1056 } 1057 qdf_mem_zero(hal->shadow_wrptr_mem_vaddr, 1058 sizeof(*(hal->shadow_wrptr_mem_vaddr)) * HAL_MAX_LMAC_RINGS); 1059 1060 for (i = 0; i < HAL_SRNG_ID_MAX; i++) { 1061 hal->srng_list[i].initialized = 0; 1062 hal->srng_list[i].ring_id = i; 1063 } 1064 1065 qdf_spinlock_create(&hal->register_access_lock); 1066 hal->register_window = 0; 1067 hal->target_type = hal_get_target_type(hal_soc_to_hal_soc_handle(hal)); 1068 hal->ops = qdf_mem_malloc(sizeof(*hal->ops)); 1069 1070 if (!hal->ops) { 1071 hal_err("unable to allocable memory for HAL ops"); 1072 goto fail3; 1073 } 1074 1075 hal_target_based_configure(hal); 1076 1077 hal_reg_write_fail_history_init(hal); 1078 1079 qdf_minidump_log(hal, sizeof(*hal), "hal_soc"); 1080 1081 qdf_atomic_init(&hal->active_work_cnt); 1082 hal_delayed_reg_write_init(hal); 1083 1084 return (void *)hal; 1085 fail3: 1086 qdf_mem_free_consistent(qdf_dev, qdf_dev->dev, 1087 sizeof(*hal->shadow_wrptr_mem_vaddr) * 1088 HAL_MAX_LMAC_RINGS, 1089 hal->shadow_wrptr_mem_vaddr, 1090 hal->shadow_wrptr_mem_paddr, 0); 1091 fail2: 1092 qdf_mem_free_consistent(qdf_dev, qdf_dev->dev, 1093 sizeof(*(hal->shadow_rdptr_mem_vaddr)) * HAL_SRNG_ID_MAX, 1094 hal->shadow_rdptr_mem_vaddr, hal->shadow_rdptr_mem_paddr, 0); 1095 fail1: 1096 qdf_mem_free(hal); 1097 fail0: 1098 return NULL; 1099 } 1100 qdf_export_symbol(hal_attach); 1101 1102 /** 1103 * hal_mem_info - Retrieve hal memory base address 1104 * 1105 * @hal_soc: Opaque HAL SOC handle 1106 * @mem: pointer to structure to be updated with hal mem info 1107 */ 1108 void hal_get_meminfo(hal_soc_handle_t hal_soc_hdl, struct hal_mem_info *mem) 1109 { 1110 struct hal_soc *hal = (struct hal_soc *)hal_soc_hdl; 1111 mem->dev_base_addr = (void *)hal->dev_base_addr; 1112 mem->shadow_rdptr_mem_vaddr = (void *)hal->shadow_rdptr_mem_vaddr; 1113 mem->shadow_wrptr_mem_vaddr = (void *)hal->shadow_wrptr_mem_vaddr; 1114 mem->shadow_rdptr_mem_paddr = (void *)hal->shadow_rdptr_mem_paddr; 1115 mem->shadow_wrptr_mem_paddr = (void *)hal->shadow_wrptr_mem_paddr; 1116 hif_read_phy_mem_base((void *)hal->hif_handle, 1117 (qdf_dma_addr_t *)&mem->dev_base_paddr); 1118 mem->lmac_srng_start_id = HAL_SRNG_LMAC1_ID_START; 1119 return; 1120 } 1121 qdf_export_symbol(hal_get_meminfo); 1122 1123 /** 1124 * hal_detach - Detach HAL layer 1125 * @hal_soc: HAL SOC handle 1126 * 1127 * Return: Opaque HAL SOC handle 1128 * NULL on failure (if given ring is not available) 1129 * 1130 * This function should be called as part of HIF initialization (for accessing 1131 * copy engines). DP layer will get hal_soc handle using hif_get_hal_handle() 1132 * 1133 */ 1134 extern void hal_detach(void *hal_soc) 1135 { 1136 struct hal_soc *hal = (struct hal_soc *)hal_soc; 1137 1138 hal_delayed_reg_write_deinit(hal); 1139 qdf_mem_free(hal->ops); 1140 1141 qdf_mem_free_consistent(hal->qdf_dev, hal->qdf_dev->dev, 1142 sizeof(*(hal->shadow_rdptr_mem_vaddr)) * HAL_SRNG_ID_MAX, 1143 hal->shadow_rdptr_mem_vaddr, hal->shadow_rdptr_mem_paddr, 0); 1144 qdf_mem_free_consistent(hal->qdf_dev, hal->qdf_dev->dev, 1145 sizeof(*(hal->shadow_wrptr_mem_vaddr)) * HAL_MAX_LMAC_RINGS, 1146 hal->shadow_wrptr_mem_vaddr, hal->shadow_wrptr_mem_paddr, 0); 1147 qdf_minidump_remove(hal, sizeof(*hal), "hal_soc"); 1148 qdf_mem_free(hal); 1149 1150 return; 1151 } 1152 qdf_export_symbol(hal_detach); 1153 1154 #define HAL_CE_CHANNEL_DST_DEST_CTRL_ADDR(x) ((x) + 0x000000b0) 1155 #define HAL_CE_CHANNEL_DST_DEST_CTRL_DEST_MAX_LENGTH_BMSK 0x0000ffff 1156 #define HAL_CE_CHANNEL_DST_DEST_RING_CONSUMER_PREFETCH_TIMER_ADDR(x) ((x) + 0x00000040) 1157 #define HAL_CE_CHANNEL_DST_DEST_RING_CONSUMER_PREFETCH_TIMER_RMSK 0x00000007 1158 /** 1159 * hal_ce_dst_setup - Initialize CE destination ring registers 1160 * @hal_soc: HAL SOC handle 1161 * @srng: SRNG ring pointer 1162 */ 1163 static inline void hal_ce_dst_setup(struct hal_soc *hal, struct hal_srng *srng, 1164 int ring_num) 1165 { 1166 uint32_t reg_val = 0; 1167 uint32_t reg_addr; 1168 struct hal_hw_srng_config *ring_config = 1169 HAL_SRNG_CONFIG(hal, CE_DST); 1170 1171 /* set DEST_MAX_LENGTH according to ce assignment */ 1172 reg_addr = HAL_CE_CHANNEL_DST_DEST_CTRL_ADDR( 1173 ring_config->reg_start[R0_INDEX] + 1174 (ring_num * ring_config->reg_size[R0_INDEX])); 1175 1176 reg_val = HAL_REG_READ(hal, reg_addr); 1177 reg_val &= ~HAL_CE_CHANNEL_DST_DEST_CTRL_DEST_MAX_LENGTH_BMSK; 1178 reg_val |= srng->u.dst_ring.max_buffer_length & 1179 HAL_CE_CHANNEL_DST_DEST_CTRL_DEST_MAX_LENGTH_BMSK; 1180 HAL_REG_WRITE(hal, reg_addr, reg_val); 1181 1182 if (srng->prefetch_timer) { 1183 reg_addr = HAL_CE_CHANNEL_DST_DEST_RING_CONSUMER_PREFETCH_TIMER_ADDR( 1184 ring_config->reg_start[R0_INDEX] + 1185 (ring_num * ring_config->reg_size[R0_INDEX])); 1186 1187 reg_val = HAL_REG_READ(hal, reg_addr); 1188 reg_val &= ~HAL_CE_CHANNEL_DST_DEST_RING_CONSUMER_PREFETCH_TIMER_RMSK; 1189 reg_val |= srng->prefetch_timer; 1190 HAL_REG_WRITE(hal, reg_addr, reg_val); 1191 reg_val = HAL_REG_READ(hal, reg_addr); 1192 } 1193 1194 } 1195 1196 /** 1197 * hal_reo_read_write_ctrl_ix - Read or write REO_DESTINATION_RING_CTRL_IX 1198 * @hal: HAL SOC handle 1199 * @read: boolean value to indicate if read or write 1200 * @ix0: pointer to store IX0 reg value 1201 * @ix1: pointer to store IX1 reg value 1202 * @ix2: pointer to store IX2 reg value 1203 * @ix3: pointer to store IX3 reg value 1204 */ 1205 void hal_reo_read_write_ctrl_ix(hal_soc_handle_t hal_soc_hdl, bool read, 1206 uint32_t *ix0, uint32_t *ix1, 1207 uint32_t *ix2, uint32_t *ix3) 1208 { 1209 uint32_t reg_offset; 1210 struct hal_soc *hal = (struct hal_soc *)hal_soc_hdl; 1211 uint32_t reo_reg_base; 1212 1213 reo_reg_base = hal_get_reo_reg_base_offset(hal_soc_hdl); 1214 1215 if (read) { 1216 if (ix0) { 1217 reg_offset = 1218 HAL_REO_DESTINATION_RING_CTRL_IX_0_ADDR( 1219 reo_reg_base); 1220 *ix0 = HAL_REG_READ(hal, reg_offset); 1221 } 1222 1223 if (ix1) { 1224 reg_offset = 1225 HAL_REO_DESTINATION_RING_CTRL_IX_1_ADDR( 1226 reo_reg_base); 1227 *ix1 = HAL_REG_READ(hal, reg_offset); 1228 } 1229 1230 if (ix2) { 1231 reg_offset = 1232 HAL_REO_DESTINATION_RING_CTRL_IX_2_ADDR( 1233 reo_reg_base); 1234 *ix2 = HAL_REG_READ(hal, reg_offset); 1235 } 1236 1237 if (ix3) { 1238 reg_offset = 1239 HAL_REO_DESTINATION_RING_CTRL_IX_3_ADDR( 1240 reo_reg_base); 1241 *ix3 = HAL_REG_READ(hal, reg_offset); 1242 } 1243 } else { 1244 if (ix0) { 1245 reg_offset = 1246 HAL_REO_DESTINATION_RING_CTRL_IX_0_ADDR( 1247 reo_reg_base); 1248 HAL_REG_WRITE_CONFIRM_RETRY(hal, reg_offset, 1249 *ix0, true); 1250 } 1251 1252 if (ix1) { 1253 reg_offset = 1254 HAL_REO_DESTINATION_RING_CTRL_IX_1_ADDR( 1255 reo_reg_base); 1256 HAL_REG_WRITE_CONFIRM_RETRY(hal, reg_offset, 1257 *ix1, true); 1258 } 1259 1260 if (ix2) { 1261 reg_offset = 1262 HAL_REO_DESTINATION_RING_CTRL_IX_2_ADDR( 1263 reo_reg_base); 1264 HAL_REG_WRITE_CONFIRM_RETRY(hal, reg_offset, 1265 *ix2, true); 1266 } 1267 1268 if (ix3) { 1269 reg_offset = 1270 HAL_REO_DESTINATION_RING_CTRL_IX_3_ADDR( 1271 reo_reg_base); 1272 HAL_REG_WRITE_CONFIRM_RETRY(hal, reg_offset, 1273 *ix3, true); 1274 } 1275 } 1276 } 1277 1278 qdf_export_symbol(hal_reo_read_write_ctrl_ix); 1279 1280 /** 1281 * hal_srng_dst_set_hp_paddr_confirm() - Set physical address to dest ring head 1282 * pointer and confirm that write went through by reading back the value 1283 * @srng: sring pointer 1284 * @paddr: physical address 1285 * 1286 * Return: None 1287 */ 1288 void hal_srng_dst_set_hp_paddr_confirm(struct hal_srng *srng, uint64_t paddr) 1289 { 1290 SRNG_DST_REG_WRITE_CONFIRM(srng, HP_ADDR_LSB, paddr & 0xffffffff); 1291 SRNG_DST_REG_WRITE_CONFIRM(srng, HP_ADDR_MSB, paddr >> 32); 1292 } 1293 1294 qdf_export_symbol(hal_srng_dst_set_hp_paddr_confirm); 1295 1296 /** 1297 * hal_srng_dst_init_hp() - Initialize destination ring head 1298 * pointer 1299 * @hal_soc: hal_soc handle 1300 * @srng: sring pointer 1301 * @vaddr: virtual address 1302 */ 1303 void hal_srng_dst_init_hp(struct hal_soc_handle *hal_soc, 1304 struct hal_srng *srng, 1305 uint32_t *vaddr) 1306 { 1307 uint32_t reg_offset; 1308 struct hal_soc *hal = (struct hal_soc *)hal_soc; 1309 1310 if (!srng) 1311 return; 1312 1313 srng->u.dst_ring.hp_addr = vaddr; 1314 reg_offset = SRNG_DST_ADDR(srng, HP) - hal->dev_base_addr; 1315 HAL_REG_WRITE_CONFIRM_RETRY( 1316 hal, reg_offset, srng->u.dst_ring.cached_hp, true); 1317 1318 if (vaddr) { 1319 *srng->u.dst_ring.hp_addr = srng->u.dst_ring.cached_hp; 1320 QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR, 1321 "hp_addr=%pK, cached_hp=%d, hp=%d", 1322 (void *)srng->u.dst_ring.hp_addr, 1323 srng->u.dst_ring.cached_hp, 1324 *srng->u.dst_ring.hp_addr); 1325 } 1326 } 1327 1328 qdf_export_symbol(hal_srng_dst_init_hp); 1329 1330 /** 1331 * hal_srng_hw_init - Private function to initialize SRNG HW 1332 * @hal_soc: HAL SOC handle 1333 * @srng: SRNG ring pointer 1334 */ 1335 static inline void hal_srng_hw_init(struct hal_soc *hal, 1336 struct hal_srng *srng) 1337 { 1338 if (srng->ring_dir == HAL_SRNG_SRC_RING) 1339 hal_srng_src_hw_init(hal, srng); 1340 else 1341 hal_srng_dst_hw_init(hal, srng); 1342 } 1343 1344 #ifdef CONFIG_SHADOW_V2 1345 #define ignore_shadow false 1346 #define CHECK_SHADOW_REGISTERS true 1347 #else 1348 #define ignore_shadow true 1349 #define CHECK_SHADOW_REGISTERS false 1350 #endif 1351 1352 #ifdef WLAN_FEATURE_NEAR_FULL_IRQ 1353 /** 1354 * hal_srng_is_near_full_irq_supported() - Check if near full irq is 1355 * supported on this SRNG 1356 * @hal_soc: HAL SoC handle 1357 * @ring_type: SRNG type 1358 * @ring_num: ring number 1359 * 1360 * Return: true, if near full irq is supported for this SRNG 1361 * false, if near full irq is not supported for this SRNG 1362 */ 1363 bool hal_srng_is_near_full_irq_supported(hal_soc_handle_t hal_soc, 1364 int ring_type, int ring_num) 1365 { 1366 struct hal_soc *hal = (struct hal_soc *)hal_soc; 1367 struct hal_hw_srng_config *ring_config = 1368 HAL_SRNG_CONFIG(hal, ring_type); 1369 1370 return ring_config->nf_irq_support; 1371 } 1372 1373 /** 1374 * hal_srng_set_msi2_params() - Set MSI2 params to SRNG data structure from 1375 * ring params 1376 * @srng: SRNG handle 1377 * @ring_params: ring params for this SRNG 1378 * 1379 * Return: None 1380 */ 1381 static inline void 1382 hal_srng_set_msi2_params(struct hal_srng *srng, 1383 struct hal_srng_params *ring_params) 1384 { 1385 srng->msi2_addr = ring_params->msi2_addr; 1386 srng->msi2_data = ring_params->msi2_data; 1387 } 1388 1389 /** 1390 * hal_srng_get_nf_params() - Get the near full MSI2 params from srng 1391 * @srng: SRNG handle 1392 * @ring_params: ring params for this SRNG 1393 * 1394 * Return: None 1395 */ 1396 static inline void 1397 hal_srng_get_nf_params(struct hal_srng *srng, 1398 struct hal_srng_params *ring_params) 1399 { 1400 ring_params->msi2_addr = srng->msi2_addr; 1401 ring_params->msi2_data = srng->msi2_data; 1402 } 1403 1404 /** 1405 * hal_srng_set_nf_thresholds() - Set the near full thresholds in SRNG 1406 * @srng: SRNG handle where the params are to be set 1407 * @ring_params: ring params, from where threshold is to be fetched 1408 * 1409 * Return: None 1410 */ 1411 static inline void 1412 hal_srng_set_nf_thresholds(struct hal_srng *srng, 1413 struct hal_srng_params *ring_params) 1414 { 1415 srng->u.dst_ring.nf_irq_support = ring_params->nf_irq_support; 1416 srng->u.dst_ring.high_thresh = ring_params->high_thresh; 1417 } 1418 #else 1419 static inline void 1420 hal_srng_set_msi2_params(struct hal_srng *srng, 1421 struct hal_srng_params *ring_params) 1422 { 1423 } 1424 1425 static inline void 1426 hal_srng_get_nf_params(struct hal_srng *srng, 1427 struct hal_srng_params *ring_params) 1428 { 1429 } 1430 1431 static inline void 1432 hal_srng_set_nf_thresholds(struct hal_srng *srng, 1433 struct hal_srng_params *ring_params) 1434 { 1435 } 1436 #endif 1437 1438 #if defined(CLEAR_SW2TCL_CONSUMED_DESC) 1439 /** 1440 * hal_srng_last_desc_cleared_init - Initialize SRNG last_desc_cleared ptr 1441 * 1442 * @srng: Source ring pointer 1443 * 1444 * Return: None 1445 */ 1446 static inline 1447 void hal_srng_last_desc_cleared_init(struct hal_srng *srng) 1448 { 1449 srng->last_desc_cleared = srng->ring_size - srng->entry_size; 1450 } 1451 1452 #else 1453 static inline 1454 void hal_srng_last_desc_cleared_init(struct hal_srng *srng) 1455 { 1456 } 1457 #endif /* CLEAR_SW2TCL_CONSUMED_DESC */ 1458 1459 /** 1460 * hal_srng_setup - Initialize HW SRNG ring. 1461 * @hal_soc: Opaque HAL SOC handle 1462 * @ring_type: one of the types from hal_ring_type 1463 * @ring_num: Ring number if there are multiple rings of same type (staring 1464 * from 0) 1465 * @mac_id: valid MAC Id should be passed if ring type is one of lmac rings 1466 * @ring_params: SRNG ring params in hal_srng_params structure. 1467 1468 * Callers are expected to allocate contiguous ring memory of size 1469 * 'num_entries * entry_size' bytes and pass the physical and virtual base 1470 * addresses through 'ring_base_paddr' and 'ring_base_vaddr' in 1471 * hal_srng_params structure. Ring base address should be 8 byte aligned 1472 * and size of each ring entry should be queried using the API 1473 * hal_srng_get_entrysize 1474 * 1475 * Return: Opaque pointer to ring on success 1476 * NULL on failure (if given ring is not available) 1477 */ 1478 void *hal_srng_setup(void *hal_soc, int ring_type, int ring_num, 1479 int mac_id, struct hal_srng_params *ring_params) 1480 { 1481 int ring_id; 1482 struct hal_soc *hal = (struct hal_soc *)hal_soc; 1483 struct hal_srng *srng; 1484 struct hal_hw_srng_config *ring_config = 1485 HAL_SRNG_CONFIG(hal, ring_type); 1486 void *dev_base_addr; 1487 int i; 1488 1489 ring_id = hal_get_srng_ring_id(hal_soc, ring_type, ring_num, mac_id); 1490 if (ring_id < 0) 1491 return NULL; 1492 1493 hal_verbose_debug("mac_id %d ring_id %d", mac_id, ring_id); 1494 1495 srng = hal_get_srng(hal_soc, ring_id); 1496 1497 if (srng->initialized) { 1498 hal_verbose_debug("Ring (ring_type, ring_num) already initialized"); 1499 return NULL; 1500 } 1501 1502 dev_base_addr = hal->dev_base_addr; 1503 srng->ring_id = ring_id; 1504 srng->ring_type = ring_type; 1505 srng->ring_dir = ring_config->ring_dir; 1506 srng->ring_base_paddr = ring_params->ring_base_paddr; 1507 srng->ring_base_vaddr = ring_params->ring_base_vaddr; 1508 srng->entry_size = ring_config->entry_size; 1509 srng->num_entries = ring_params->num_entries; 1510 srng->ring_size = srng->num_entries * srng->entry_size; 1511 srng->ring_size_mask = srng->ring_size - 1; 1512 srng->msi_addr = ring_params->msi_addr; 1513 srng->msi_data = ring_params->msi_data; 1514 srng->intr_timer_thres_us = ring_params->intr_timer_thres_us; 1515 srng->intr_batch_cntr_thres_entries = 1516 ring_params->intr_batch_cntr_thres_entries; 1517 srng->prefetch_timer = ring_params->prefetch_timer; 1518 srng->hal_soc = hal_soc; 1519 hal_srng_set_msi2_params(srng, ring_params); 1520 1521 for (i = 0 ; i < MAX_SRNG_REG_GROUPS; i++) { 1522 srng->hwreg_base[i] = dev_base_addr + ring_config->reg_start[i] 1523 + (ring_num * ring_config->reg_size[i]); 1524 } 1525 1526 /* Zero out the entire ring memory */ 1527 qdf_mem_zero(srng->ring_base_vaddr, (srng->entry_size * 1528 srng->num_entries) << 2); 1529 1530 srng->flags = ring_params->flags; 1531 #ifdef BIG_ENDIAN_HOST 1532 /* TODO: See if we should we get these flags from caller */ 1533 srng->flags |= HAL_SRNG_DATA_TLV_SWAP; 1534 srng->flags |= HAL_SRNG_MSI_SWAP; 1535 srng->flags |= HAL_SRNG_RING_PTR_SWAP; 1536 #endif 1537 1538 hal_srng_last_desc_cleared_init(srng); 1539 1540 if (srng->ring_dir == HAL_SRNG_SRC_RING) { 1541 srng->u.src_ring.hp = 0; 1542 srng->u.src_ring.reap_hp = srng->ring_size - 1543 srng->entry_size; 1544 srng->u.src_ring.tp_addr = 1545 &(hal->shadow_rdptr_mem_vaddr[ring_id]); 1546 srng->u.src_ring.low_threshold = 1547 ring_params->low_threshold * srng->entry_size; 1548 if (ring_config->lmac_ring) { 1549 /* For LMAC rings, head pointer updates will be done 1550 * through FW by writing to a shared memory location 1551 */ 1552 srng->u.src_ring.hp_addr = 1553 &(hal->shadow_wrptr_mem_vaddr[ring_id - 1554 HAL_SRNG_LMAC1_ID_START]); 1555 srng->flags |= HAL_SRNG_LMAC_RING; 1556 } else if (ignore_shadow || (srng->u.src_ring.hp_addr == 0)) { 1557 srng->u.src_ring.hp_addr = 1558 hal_get_window_address(hal, 1559 SRNG_SRC_ADDR(srng, HP)); 1560 1561 if (CHECK_SHADOW_REGISTERS) { 1562 QDF_TRACE(QDF_MODULE_ID_TXRX, 1563 QDF_TRACE_LEVEL_ERROR, 1564 "%s: Ring (%d, %d) missing shadow config", 1565 __func__, ring_type, ring_num); 1566 } 1567 } else { 1568 hal_validate_shadow_register(hal, 1569 SRNG_SRC_ADDR(srng, HP), 1570 srng->u.src_ring.hp_addr); 1571 } 1572 } else { 1573 /* During initialization loop count in all the descriptors 1574 * will be set to zero, and HW will set it to 1 on completing 1575 * descriptor update in first loop, and increments it by 1 on 1576 * subsequent loops (loop count wraps around after reaching 1577 * 0xffff). The 'loop_cnt' in SW ring state is the expected 1578 * loop count in descriptors updated by HW (to be processed 1579 * by SW). 1580 */ 1581 hal_srng_set_nf_thresholds(srng, ring_params); 1582 srng->u.dst_ring.loop_cnt = 1; 1583 srng->u.dst_ring.tp = 0; 1584 srng->u.dst_ring.hp_addr = 1585 &(hal->shadow_rdptr_mem_vaddr[ring_id]); 1586 if (ring_config->lmac_ring) { 1587 /* For LMAC rings, tail pointer updates will be done 1588 * through FW by writing to a shared memory location 1589 */ 1590 srng->u.dst_ring.tp_addr = 1591 &(hal->shadow_wrptr_mem_vaddr[ring_id - 1592 HAL_SRNG_LMAC1_ID_START]); 1593 srng->flags |= HAL_SRNG_LMAC_RING; 1594 } else if (ignore_shadow || srng->u.dst_ring.tp_addr == 0) { 1595 srng->u.dst_ring.tp_addr = 1596 hal_get_window_address(hal, 1597 SRNG_DST_ADDR(srng, TP)); 1598 1599 if (CHECK_SHADOW_REGISTERS) { 1600 QDF_TRACE(QDF_MODULE_ID_TXRX, 1601 QDF_TRACE_LEVEL_ERROR, 1602 "%s: Ring (%d, %d) missing shadow config", 1603 __func__, ring_type, ring_num); 1604 } 1605 } else { 1606 hal_validate_shadow_register(hal, 1607 SRNG_DST_ADDR(srng, TP), 1608 srng->u.dst_ring.tp_addr); 1609 } 1610 } 1611 1612 if (!(ring_config->lmac_ring)) { 1613 hal_srng_hw_init(hal, srng); 1614 1615 if (ring_type == CE_DST) { 1616 srng->u.dst_ring.max_buffer_length = ring_params->max_buffer_length; 1617 hal_ce_dst_setup(hal, srng, ring_num); 1618 } 1619 } 1620 1621 SRNG_LOCK_INIT(&srng->lock); 1622 1623 srng->srng_event = 0; 1624 1625 srng->initialized = true; 1626 1627 return (void *)srng; 1628 } 1629 qdf_export_symbol(hal_srng_setup); 1630 1631 /** 1632 * hal_srng_cleanup - Deinitialize HW SRNG ring. 1633 * @hal_soc: Opaque HAL SOC handle 1634 * @hal_srng: Opaque HAL SRNG pointer 1635 */ 1636 void hal_srng_cleanup(void *hal_soc, hal_ring_handle_t hal_ring_hdl) 1637 { 1638 struct hal_srng *srng = (struct hal_srng *)hal_ring_hdl; 1639 SRNG_LOCK_DESTROY(&srng->lock); 1640 srng->initialized = 0; 1641 } 1642 qdf_export_symbol(hal_srng_cleanup); 1643 1644 /** 1645 * hal_srng_get_entrysize - Returns size of ring entry in bytes 1646 * @hal_soc: Opaque HAL SOC handle 1647 * @ring_type: one of the types from hal_ring_type 1648 * 1649 */ 1650 uint32_t hal_srng_get_entrysize(void *hal_soc, int ring_type) 1651 { 1652 struct hal_soc *hal = (struct hal_soc *)hal_soc; 1653 struct hal_hw_srng_config *ring_config = 1654 HAL_SRNG_CONFIG(hal, ring_type); 1655 return ring_config->entry_size << 2; 1656 } 1657 qdf_export_symbol(hal_srng_get_entrysize); 1658 1659 /** 1660 * hal_srng_max_entries - Returns maximum possible number of ring entries 1661 * @hal_soc: Opaque HAL SOC handle 1662 * @ring_type: one of the types from hal_ring_type 1663 * 1664 * Return: Maximum number of entries for the given ring_type 1665 */ 1666 uint32_t hal_srng_max_entries(void *hal_soc, int ring_type) 1667 { 1668 struct hal_soc *hal = (struct hal_soc *)hal_soc; 1669 struct hal_hw_srng_config *ring_config = 1670 HAL_SRNG_CONFIG(hal, ring_type); 1671 1672 return ring_config->max_size / ring_config->entry_size; 1673 } 1674 qdf_export_symbol(hal_srng_max_entries); 1675 1676 enum hal_srng_dir hal_srng_get_dir(void *hal_soc, int ring_type) 1677 { 1678 struct hal_soc *hal = (struct hal_soc *)hal_soc; 1679 struct hal_hw_srng_config *ring_config = 1680 HAL_SRNG_CONFIG(hal, ring_type); 1681 1682 return ring_config->ring_dir; 1683 } 1684 1685 /** 1686 * hal_srng_dump - Dump ring status 1687 * @srng: hal srng pointer 1688 */ 1689 void hal_srng_dump(struct hal_srng *srng) 1690 { 1691 if (srng->ring_dir == HAL_SRNG_SRC_RING) { 1692 hal_debug("=== SRC RING %d ===", srng->ring_id); 1693 hal_debug("hp %u, reap_hp %u, tp %u, cached tp %u", 1694 srng->u.src_ring.hp, 1695 srng->u.src_ring.reap_hp, 1696 *srng->u.src_ring.tp_addr, 1697 srng->u.src_ring.cached_tp); 1698 } else { 1699 hal_debug("=== DST RING %d ===", srng->ring_id); 1700 hal_debug("tp %u, hp %u, cached tp %u, loop_cnt %u", 1701 srng->u.dst_ring.tp, 1702 *srng->u.dst_ring.hp_addr, 1703 srng->u.dst_ring.cached_hp, 1704 srng->u.dst_ring.loop_cnt); 1705 } 1706 } 1707 1708 /** 1709 * hal_get_srng_params - Retrieve SRNG parameters for a given ring from HAL 1710 * 1711 * @hal_soc: Opaque HAL SOC handle 1712 * @hal_ring: Ring pointer (Source or Destination ring) 1713 * @ring_params: SRNG parameters will be returned through this structure 1714 */ 1715 extern void hal_get_srng_params(hal_soc_handle_t hal_soc_hdl, 1716 hal_ring_handle_t hal_ring_hdl, 1717 struct hal_srng_params *ring_params) 1718 { 1719 struct hal_srng *srng = (struct hal_srng *)hal_ring_hdl; 1720 int i =0; 1721 ring_params->ring_id = srng->ring_id; 1722 ring_params->ring_dir = srng->ring_dir; 1723 ring_params->entry_size = srng->entry_size; 1724 1725 ring_params->ring_base_paddr = srng->ring_base_paddr; 1726 ring_params->ring_base_vaddr = srng->ring_base_vaddr; 1727 ring_params->num_entries = srng->num_entries; 1728 ring_params->msi_addr = srng->msi_addr; 1729 ring_params->msi_data = srng->msi_data; 1730 ring_params->intr_timer_thres_us = srng->intr_timer_thres_us; 1731 ring_params->intr_batch_cntr_thres_entries = 1732 srng->intr_batch_cntr_thres_entries; 1733 ring_params->low_threshold = srng->u.src_ring.low_threshold; 1734 ring_params->flags = srng->flags; 1735 ring_params->ring_id = srng->ring_id; 1736 for (i = 0 ; i < MAX_SRNG_REG_GROUPS; i++) 1737 ring_params->hwreg_base[i] = srng->hwreg_base[i]; 1738 1739 hal_srng_get_nf_params(srng, ring_params); 1740 } 1741 qdf_export_symbol(hal_get_srng_params); 1742 1743 void hal_set_low_threshold(hal_ring_handle_t hal_ring_hdl, 1744 uint32_t low_threshold) 1745 { 1746 struct hal_srng *srng = (struct hal_srng *)hal_ring_hdl; 1747 srng->u.src_ring.low_threshold = low_threshold * srng->entry_size; 1748 } 1749 qdf_export_symbol(hal_set_low_threshold); 1750 1751 #ifdef FORCE_WAKE 1752 void hal_set_init_phase(hal_soc_handle_t soc, bool init_phase) 1753 { 1754 struct hal_soc *hal_soc = (struct hal_soc *)soc; 1755 1756 hal_soc->init_phase = init_phase; 1757 } 1758 #endif /* FORCE_WAKE */ 1759