1 /* 2 * Copyright (c) 2016-2021 The Linux Foundation. All rights reserved. 3 * 4 * Permission to use, copy, modify, and/or distribute this software for 5 * any purpose with or without fee is hereby granted, provided that the 6 * above copyright notice and this permission notice appear in all 7 * copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL 10 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED 11 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE 12 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 13 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR 14 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 15 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 * PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include "hal_hw_headers.h" 20 #include "hal_api.h" 21 #include "hal_reo.h" 22 #include "target_type.h" 23 #include "qdf_module.h" 24 #include "wcss_version.h" 25 26 #ifdef QCA_WIFI_QCA8074 27 void hal_qca6290_attach(struct hal_soc *hal); 28 #endif 29 #ifdef QCA_WIFI_QCA8074 30 void hal_qca8074_attach(struct hal_soc *hal); 31 #endif 32 #if defined(QCA_WIFI_QCA8074V2) || defined(QCA_WIFI_QCA6018) || \ 33 defined(QCA_WIFI_QCA9574) 34 void hal_qca8074v2_attach(struct hal_soc *hal); 35 #endif 36 #ifdef QCA_WIFI_QCA6390 37 void hal_qca6390_attach(struct hal_soc *hal); 38 #endif 39 #ifdef QCA_WIFI_QCA6490 40 void hal_qca6490_attach(struct hal_soc *hal); 41 #endif 42 #ifdef QCA_WIFI_QCN9000 43 void hal_qcn9000_attach(struct hal_soc *hal); 44 #endif 45 #ifdef QCA_WIFI_QCN9224 46 void hal_qcn9224_attach(struct hal_soc *hal); 47 #endif 48 #ifdef QCA_WIFI_QCN6122 49 void hal_qcn6122_attach(struct hal_soc *hal); 50 #endif 51 #ifdef QCA_WIFI_QCA6750 52 void hal_qca6750_attach(struct hal_soc *hal); 53 #endif 54 #ifdef QCA_WIFI_QCA5018 55 void hal_qca5018_attach(struct hal_soc *hal); 56 #endif 57 #ifdef QCA_WIFI_WCN7850 58 void hal_wcn7850_attach(struct hal_soc *hal); 59 #endif 60 61 #ifdef ENABLE_VERBOSE_DEBUG 62 bool is_hal_verbose_debug_enabled; 63 #endif 64 65 #define HAL_REO_DESTINATION_RING_CTRL_IX_0_ADDR(x) ((x) + 0x4) 66 #define HAL_REO_DESTINATION_RING_CTRL_IX_1_ADDR(x) ((x) + 0x8) 67 #define HAL_REO_DESTINATION_RING_CTRL_IX_2_ADDR(x) ((x) + 0xc) 68 #define HAL_REO_DESTINATION_RING_CTRL_IX_3_ADDR(x) ((x) + 0x10) 69 70 #ifdef ENABLE_HAL_REG_WR_HISTORY 71 struct hal_reg_write_fail_history hal_reg_wr_hist; 72 73 void hal_reg_wr_fail_history_add(struct hal_soc *hal_soc, 74 uint32_t offset, 75 uint32_t wr_val, uint32_t rd_val) 76 { 77 struct hal_reg_write_fail_entry *record; 78 int idx; 79 80 idx = hal_history_get_next_index(&hal_soc->reg_wr_fail_hist->index, 81 HAL_REG_WRITE_HIST_SIZE); 82 83 record = &hal_soc->reg_wr_fail_hist->record[idx]; 84 85 record->timestamp = qdf_get_log_timestamp(); 86 record->reg_offset = offset; 87 record->write_val = wr_val; 88 record->read_val = rd_val; 89 } 90 91 static void hal_reg_write_fail_history_init(struct hal_soc *hal) 92 { 93 hal->reg_wr_fail_hist = &hal_reg_wr_hist; 94 95 qdf_atomic_set(&hal->reg_wr_fail_hist->index, -1); 96 } 97 #else 98 static void hal_reg_write_fail_history_init(struct hal_soc *hal) 99 { 100 } 101 #endif 102 103 /** 104 * hal_get_srng_ring_id() - get the ring id of a descriped ring 105 * @hal: hal_soc data structure 106 * @ring_type: type enum describing the ring 107 * @ring_num: which ring of the ring type 108 * @mac_id: which mac does the ring belong to (or 0 for non-lmac rings) 109 * 110 * Return: the ring id or -EINVAL if the ring does not exist. 111 */ 112 static int hal_get_srng_ring_id(struct hal_soc *hal, int ring_type, 113 int ring_num, int mac_id) 114 { 115 struct hal_hw_srng_config *ring_config = 116 HAL_SRNG_CONFIG(hal, ring_type); 117 int ring_id; 118 119 if (ring_num >= ring_config->max_rings) { 120 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_INFO, 121 "%s: ring_num exceeded maximum no. of supported rings", 122 __func__); 123 /* TODO: This is a programming error. Assert if this happens */ 124 return -EINVAL; 125 } 126 127 /* 128 * For BE, dmac_cmn_src_rxbuf_ring is set. If this is set 129 * and ring is dst and also lmac ring then provide ring id per lmac 130 */ 131 if (ring_config->lmac_ring && 132 (!hal->dmac_cmn_src_rxbuf_ring || 133 ring_config->ring_dir == HAL_SRNG_DST_RING)) { 134 ring_id = (ring_config->start_ring_id + ring_num + 135 (mac_id * HAL_MAX_RINGS_PER_LMAC)); 136 } else { 137 ring_id = ring_config->start_ring_id + ring_num; 138 } 139 140 return ring_id; 141 } 142 143 static struct hal_srng *hal_get_srng(struct hal_soc *hal, int ring_id) 144 { 145 /* TODO: Should we allocate srng structures dynamically? */ 146 return &(hal->srng_list[ring_id]); 147 } 148 149 #ifndef SHADOW_REG_CONFIG_DISABLED 150 #define HP_OFFSET_IN_REG_START 1 151 #define OFFSET_FROM_HP_TO_TP 4 152 static void hal_update_srng_hp_tp_address(struct hal_soc *hal_soc, 153 int shadow_config_index, 154 int ring_type, 155 int ring_num) 156 { 157 struct hal_srng *srng; 158 int ring_id; 159 struct hal_hw_srng_config *ring_config = 160 HAL_SRNG_CONFIG(hal_soc, ring_type); 161 162 ring_id = hal_get_srng_ring_id(hal_soc, ring_type, ring_num, 0); 163 if (ring_id < 0) 164 return; 165 166 srng = hal_get_srng(hal_soc, ring_id); 167 168 if (ring_config->ring_dir == HAL_SRNG_DST_RING) { 169 srng->u.dst_ring.tp_addr = SHADOW_REGISTER(shadow_config_index) 170 + hal_soc->dev_base_addr; 171 hal_debug("tp_addr=%pK dev base addr %pK index %u", 172 srng->u.dst_ring.tp_addr, hal_soc->dev_base_addr, 173 shadow_config_index); 174 } else { 175 srng->u.src_ring.hp_addr = SHADOW_REGISTER(shadow_config_index) 176 + hal_soc->dev_base_addr; 177 hal_debug("hp_addr=%pK dev base addr %pK index %u", 178 srng->u.src_ring.hp_addr, 179 hal_soc->dev_base_addr, shadow_config_index); 180 } 181 182 } 183 #endif 184 185 #ifdef GENERIC_SHADOW_REGISTER_ACCESS_ENABLE 186 void hal_set_one_target_reg_config(struct hal_soc *hal, 187 uint32_t target_reg_offset, 188 int list_index) 189 { 190 int i = list_index; 191 192 qdf_assert_always(i < MAX_GENERIC_SHADOW_REG); 193 hal->list_shadow_reg_config[i].target_register = 194 target_reg_offset; 195 hal->num_generic_shadow_regs_configured++; 196 } 197 198 qdf_export_symbol(hal_set_one_target_reg_config); 199 200 #define REO_R0_DESTINATION_RING_CTRL_ADDR_OFFSET 0x4 201 #define MAX_REO_REMAP_SHADOW_REGS 4 202 QDF_STATUS hal_set_shadow_regs(void *hal_soc) 203 { 204 uint32_t target_reg_offset; 205 struct hal_soc *hal = (struct hal_soc *)hal_soc; 206 int i; 207 struct hal_hw_srng_config *srng_config = 208 &hal->hw_srng_table[WBM2SW_RELEASE]; 209 uint32_t reo_reg_base; 210 211 reo_reg_base = hal_get_reo_reg_base_offset(hal_soc); 212 213 target_reg_offset = 214 HAL_REO_DESTINATION_RING_CTRL_IX_0_ADDR(reo_reg_base); 215 216 for (i = 0; i < MAX_REO_REMAP_SHADOW_REGS; i++) { 217 hal_set_one_target_reg_config(hal, target_reg_offset, i); 218 target_reg_offset += REO_R0_DESTINATION_RING_CTRL_ADDR_OFFSET; 219 } 220 221 target_reg_offset = srng_config->reg_start[HP_OFFSET_IN_REG_START]; 222 target_reg_offset += (srng_config->reg_size[HP_OFFSET_IN_REG_START] 223 * HAL_IPA_TX_COMP_RING_IDX); 224 225 hal_set_one_target_reg_config(hal, target_reg_offset, i); 226 return QDF_STATUS_SUCCESS; 227 } 228 229 qdf_export_symbol(hal_set_shadow_regs); 230 231 QDF_STATUS hal_construct_shadow_regs(void *hal_soc) 232 { 233 struct hal_soc *hal = (struct hal_soc *)hal_soc; 234 int shadow_config_index = hal->num_shadow_registers_configured; 235 int i; 236 int num_regs = hal->num_generic_shadow_regs_configured; 237 238 for (i = 0; i < num_regs; i++) { 239 qdf_assert_always(shadow_config_index < MAX_SHADOW_REGISTERS); 240 hal->shadow_config[shadow_config_index].addr = 241 hal->list_shadow_reg_config[i].target_register; 242 hal->list_shadow_reg_config[i].shadow_config_index = 243 shadow_config_index; 244 hal->list_shadow_reg_config[i].va = 245 SHADOW_REGISTER(shadow_config_index) + 246 (uintptr_t)hal->dev_base_addr; 247 hal_debug("target_reg %x, shadow register 0x%x shadow_index 0x%x", 248 hal->shadow_config[shadow_config_index].addr, 249 SHADOW_REGISTER(shadow_config_index), 250 shadow_config_index); 251 shadow_config_index++; 252 hal->num_shadow_registers_configured++; 253 } 254 return QDF_STATUS_SUCCESS; 255 } 256 257 qdf_export_symbol(hal_construct_shadow_regs); 258 #endif 259 260 #ifndef SHADOW_REG_CONFIG_DISABLED 261 262 QDF_STATUS hal_set_one_shadow_config(void *hal_soc, 263 int ring_type, 264 int ring_num) 265 { 266 uint32_t target_register; 267 struct hal_soc *hal = (struct hal_soc *)hal_soc; 268 struct hal_hw_srng_config *srng_config = &hal->hw_srng_table[ring_type]; 269 int shadow_config_index = hal->num_shadow_registers_configured; 270 271 if (shadow_config_index >= MAX_SHADOW_REGISTERS) { 272 QDF_ASSERT(0); 273 return QDF_STATUS_E_RESOURCES; 274 } 275 276 hal->num_shadow_registers_configured++; 277 278 target_register = srng_config->reg_start[HP_OFFSET_IN_REG_START]; 279 target_register += (srng_config->reg_size[HP_OFFSET_IN_REG_START] 280 *ring_num); 281 282 /* if the ring is a dst ring, we need to shadow the tail pointer */ 283 if (srng_config->ring_dir == HAL_SRNG_DST_RING) 284 target_register += OFFSET_FROM_HP_TO_TP; 285 286 hal->shadow_config[shadow_config_index].addr = target_register; 287 288 /* update hp/tp addr in the hal_soc structure*/ 289 hal_update_srng_hp_tp_address(hal_soc, shadow_config_index, ring_type, 290 ring_num); 291 292 hal_debug("target_reg %x, shadow register 0x%x shadow_index 0x%x, ring_type %d, ring num %d", 293 target_register, 294 SHADOW_REGISTER(shadow_config_index), 295 shadow_config_index, 296 ring_type, ring_num); 297 298 return QDF_STATUS_SUCCESS; 299 } 300 301 qdf_export_symbol(hal_set_one_shadow_config); 302 303 QDF_STATUS hal_construct_srng_shadow_regs(void *hal_soc) 304 { 305 int ring_type, ring_num; 306 struct hal_soc *hal = (struct hal_soc *)hal_soc; 307 308 for (ring_type = 0; ring_type < MAX_RING_TYPES; ring_type++) { 309 struct hal_hw_srng_config *srng_config = 310 &hal->hw_srng_table[ring_type]; 311 312 if (ring_type == CE_SRC || 313 ring_type == CE_DST || 314 ring_type == CE_DST_STATUS) 315 continue; 316 317 if (srng_config->lmac_ring) 318 continue; 319 320 for (ring_num = 0; ring_num < srng_config->max_rings; 321 ring_num++) 322 hal_set_one_shadow_config(hal_soc, ring_type, ring_num); 323 } 324 325 return QDF_STATUS_SUCCESS; 326 } 327 328 qdf_export_symbol(hal_construct_srng_shadow_regs); 329 #else 330 331 QDF_STATUS hal_construct_srng_shadow_regs(void *hal_soc) 332 { 333 return QDF_STATUS_SUCCESS; 334 } 335 336 qdf_export_symbol(hal_construct_srng_shadow_regs); 337 338 QDF_STATUS hal_set_one_shadow_config(void *hal_soc, int ring_type, 339 int ring_num) 340 { 341 return QDF_STATUS_SUCCESS; 342 } 343 qdf_export_symbol(hal_set_one_shadow_config); 344 #endif 345 346 void hal_get_shadow_config(void *hal_soc, 347 struct pld_shadow_reg_v2_cfg **shadow_config, 348 int *num_shadow_registers_configured) 349 { 350 struct hal_soc *hal = (struct hal_soc *)hal_soc; 351 352 *shadow_config = hal->shadow_config; 353 *num_shadow_registers_configured = 354 hal->num_shadow_registers_configured; 355 } 356 357 qdf_export_symbol(hal_get_shadow_config); 358 359 static bool hal_validate_shadow_register(struct hal_soc *hal, 360 uint32_t *destination, 361 uint32_t *shadow_address) 362 { 363 unsigned int index; 364 uint32_t *shadow_0_offset = SHADOW_REGISTER(0) + hal->dev_base_addr; 365 int destination_ba_offset = 366 ((char *)destination) - (char *)hal->dev_base_addr; 367 368 index = shadow_address - shadow_0_offset; 369 370 if (index >= MAX_SHADOW_REGISTERS) { 371 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR, 372 "%s: index %x out of bounds", __func__, index); 373 goto error; 374 } else if (hal->shadow_config[index].addr != destination_ba_offset) { 375 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR, 376 "%s: sanity check failure, expected %x, found %x", 377 __func__, destination_ba_offset, 378 hal->shadow_config[index].addr); 379 goto error; 380 } 381 return true; 382 error: 383 qdf_print("baddr %pK, desination %pK, shadow_address %pK s0offset %pK index %x", 384 hal->dev_base_addr, destination, shadow_address, 385 shadow_0_offset, index); 386 QDF_BUG(0); 387 return false; 388 } 389 390 static void hal_target_based_configure(struct hal_soc *hal) 391 { 392 /** 393 * Indicate Initialization of srngs to avoid force wake 394 * as umac power collapse is not enabled yet 395 */ 396 hal->init_phase = true; 397 398 switch (hal->target_type) { 399 #ifdef QCA_WIFI_QCA6290 400 case TARGET_TYPE_QCA6290: 401 hal->use_register_windowing = true; 402 hal_qca6290_attach(hal); 403 break; 404 #endif 405 #ifdef QCA_WIFI_QCA6390 406 case TARGET_TYPE_QCA6390: 407 hal->use_register_windowing = true; 408 hal_qca6390_attach(hal); 409 break; 410 #endif 411 #ifdef QCA_WIFI_QCA6490 412 case TARGET_TYPE_QCA6490: 413 hal->use_register_windowing = true; 414 hal_qca6490_attach(hal); 415 break; 416 #endif 417 #ifdef QCA_WIFI_QCA6750 418 case TARGET_TYPE_QCA6750: 419 hal->use_register_windowing = true; 420 hal->static_window_map = true; 421 hal_qca6750_attach(hal); 422 break; 423 #endif 424 #ifdef QCA_WIFI_WCN7850 425 case TARGET_TYPE_WCN7850: 426 hal->use_register_windowing = true; 427 hal_wcn7850_attach(hal); 428 hal->init_phase = false; 429 break; 430 #endif 431 #if defined(QCA_WIFI_QCA8074) && defined(WIFI_TARGET_TYPE_3_0) 432 case TARGET_TYPE_QCA8074: 433 hal_qca8074_attach(hal); 434 break; 435 #endif 436 437 #if defined(QCA_WIFI_QCA8074V2) 438 case TARGET_TYPE_QCA8074V2: 439 hal_qca8074v2_attach(hal); 440 break; 441 #endif 442 443 #if defined(QCA_WIFI_QCA6018) 444 case TARGET_TYPE_QCA6018: 445 hal_qca8074v2_attach(hal); 446 break; 447 #endif 448 449 #if defined(QCA_WIFI_QCA9574) 450 case TARGET_TYPE_QCA9574: 451 hal_qca8074v2_attach(hal); 452 break; 453 #endif 454 455 #if defined(QCA_WIFI_QCN6122) 456 case TARGET_TYPE_QCN6122: 457 hal->use_register_windowing = true; 458 /* 459 * Static window map is enabled for qcn9000 to use 2mb bar 460 * size and use multiple windows to write into registers. 461 */ 462 hal->static_window_map = true; 463 hal_qcn6122_attach(hal); 464 break; 465 #endif 466 467 #ifdef QCA_WIFI_QCN9000 468 case TARGET_TYPE_QCN9000: 469 hal->use_register_windowing = true; 470 /* 471 * Static window map is enabled for qcn9000 to use 2mb bar 472 * size and use multiple windows to write into registers. 473 */ 474 hal->static_window_map = true; 475 hal_qcn9000_attach(hal); 476 break; 477 #endif 478 #ifdef QCA_WIFI_QCA5018 479 case TARGET_TYPE_QCA5018: 480 hal->use_register_windowing = true; 481 hal->static_window_map = true; 482 hal_qca5018_attach(hal); 483 break; 484 #endif 485 #ifdef QCA_WIFI_QCN9224 486 case TARGET_TYPE_QCN9224: 487 hal->use_register_windowing = true; 488 hal->static_window_map = true; 489 hal_qcn9224_attach(hal); 490 break; 491 #endif 492 default: 493 break; 494 } 495 } 496 497 uint32_t hal_get_target_type(hal_soc_handle_t hal_soc_hdl) 498 { 499 struct hal_soc *hal_soc = (struct hal_soc *)hal_soc_hdl; 500 struct hif_target_info *tgt_info = 501 hif_get_target_info_handle(hal_soc->hif_handle); 502 503 return tgt_info->target_type; 504 } 505 506 qdf_export_symbol(hal_get_target_type); 507 508 #if defined(FEATURE_HAL_DELAYED_REG_WRITE) || \ 509 defined(FEATURE_HAL_DELAYED_REG_WRITE_V2) 510 /** 511 * hal_is_reg_write_tput_level_high() - throughput level for delayed reg writes 512 * @hal: hal_soc pointer 513 * 514 * Return: true if throughput is high, else false. 515 */ 516 static inline bool hal_is_reg_write_tput_level_high(struct hal_soc *hal) 517 { 518 int bw_level = hif_get_bandwidth_level(hal->hif_handle); 519 520 return (bw_level >= PLD_BUS_WIDTH_MEDIUM) ? true : false; 521 } 522 523 static inline 524 char *hal_fill_reg_write_srng_stats(struct hal_srng *srng, 525 char *buf, qdf_size_t size) 526 { 527 qdf_scnprintf(buf, size, "enq %u deq %u coal %u direct %u", 528 srng->wstats.enqueues, srng->wstats.dequeues, 529 srng->wstats.coalesces, srng->wstats.direct); 530 return buf; 531 } 532 533 /* bytes for local buffer */ 534 #define HAL_REG_WRITE_SRNG_STATS_LEN 100 535 536 void hal_dump_reg_write_srng_stats(hal_soc_handle_t hal_soc_hdl) 537 { 538 struct hal_srng *srng; 539 char buf[HAL_REG_WRITE_SRNG_STATS_LEN]; 540 struct hal_soc *hal = (struct hal_soc *)hal_soc_hdl; 541 542 srng = hal_get_srng(hal, HAL_SRNG_SW2TCL1); 543 hal_debug("SW2TCL1: %s", 544 hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf))); 545 546 srng = hal_get_srng(hal, HAL_SRNG_WBM2SW0_RELEASE); 547 hal_debug("WBM2SW0: %s", 548 hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf))); 549 550 srng = hal_get_srng(hal, HAL_SRNG_REO2SW1); 551 hal_debug("REO2SW1: %s", 552 hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf))); 553 554 srng = hal_get_srng(hal, HAL_SRNG_REO2SW2); 555 hal_debug("REO2SW2: %s", 556 hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf))); 557 558 srng = hal_get_srng(hal, HAL_SRNG_REO2SW3); 559 hal_debug("REO2SW3: %s", 560 hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf))); 561 } 562 563 #ifdef FEATURE_HAL_DELAYED_REG_WRITE_V2 564 /** 565 * hal_dump_tcl_stats() - dump the TCL reg write stats 566 * @hal: hal_soc pointer 567 * 568 * Return: None 569 */ 570 static inline void hal_dump_tcl_stats(struct hal_soc *hal) 571 { 572 struct hal_srng *srng = hal_get_srng(hal, HAL_SRNG_SW2TCL1); 573 uint32_t *hist = hal->tcl_stats.sched_delay; 574 char buf[HAL_REG_WRITE_SRNG_STATS_LEN]; 575 576 hal_debug("TCL: %s sched-delay hist %u %u %u %u", 577 hal_fill_reg_write_srng_stats(srng, buf, sizeof(buf)), 578 hist[REG_WRITE_SCHED_DELAY_SUB_100us], 579 hist[REG_WRITE_SCHED_DELAY_SUB_1000us], 580 hist[REG_WRITE_SCHED_DELAY_SUB_5000us], 581 hist[REG_WRITE_SCHED_DELAY_GT_5000us]); 582 hal_debug("wq_dly %u wq_dir %u tim_enq %u tim_dir %u enq_tim_cnt %u dir_tim_cnt %u rst_tim_cnt %u", 583 hal->tcl_stats.wq_delayed, 584 hal->tcl_stats.wq_direct, 585 hal->tcl_stats.timer_enq, 586 hal->tcl_stats.timer_direct, 587 hal->tcl_stats.enq_timer_set, 588 hal->tcl_stats.direct_timer_set, 589 hal->tcl_stats.timer_reset); 590 } 591 592 #else 593 static inline void hal_dump_tcl_stats(struct hal_soc *hal) 594 { 595 } 596 #endif 597 598 void hal_dump_reg_write_stats(hal_soc_handle_t hal_soc_hdl) 599 { 600 uint32_t *hist; 601 struct hal_soc *hal = (struct hal_soc *)hal_soc_hdl; 602 603 hist = hal->stats.wstats.sched_delay; 604 hal_debug("wstats: enq %u deq %u coal %u direct %u q_depth %u max_q %u sched-delay hist %u %u %u %u", 605 qdf_atomic_read(&hal->stats.wstats.enqueues), 606 hal->stats.wstats.dequeues, 607 qdf_atomic_read(&hal->stats.wstats.coalesces), 608 qdf_atomic_read(&hal->stats.wstats.direct), 609 qdf_atomic_read(&hal->stats.wstats.q_depth), 610 hal->stats.wstats.max_q_depth, 611 hist[REG_WRITE_SCHED_DELAY_SUB_100us], 612 hist[REG_WRITE_SCHED_DELAY_SUB_1000us], 613 hist[REG_WRITE_SCHED_DELAY_SUB_5000us], 614 hist[REG_WRITE_SCHED_DELAY_GT_5000us]); 615 616 hal_dump_tcl_stats(hal); 617 } 618 619 int hal_get_reg_write_pending_work(void *hal_soc) 620 { 621 struct hal_soc *hal = (struct hal_soc *)hal_soc; 622 623 return qdf_atomic_read(&hal->active_work_cnt); 624 } 625 626 #endif 627 628 #ifdef FEATURE_HAL_DELAYED_REG_WRITE 629 #ifdef MEMORY_DEBUG 630 /* 631 * Length of the queue(array) used to hold delayed register writes. 632 * Must be a multiple of 2. 633 */ 634 #define HAL_REG_WRITE_QUEUE_LEN 128 635 #else 636 #define HAL_REG_WRITE_QUEUE_LEN 32 637 #endif 638 639 #ifdef FEATURE_HAL_DELAYED_REG_WRITE_V2 640 /** 641 * hal_process_reg_write_q_elem() - process a regiter write queue element 642 * @hal: hal_soc pointer 643 * @q_elem: pointer to hal regiter write queue element 644 * 645 * Return: The value which was written to the address 646 */ 647 static uint32_t 648 hal_process_reg_write_q_elem(struct hal_soc *hal, 649 struct hal_reg_write_q_elem *q_elem) 650 { 651 struct hal_srng *srng = q_elem->srng; 652 uint32_t write_val; 653 654 SRNG_LOCK(&srng->lock); 655 srng->reg_write_in_progress = false; 656 srng->wstats.dequeues++; 657 658 if (srng->ring_dir == HAL_SRNG_SRC_RING) { 659 write_val = srng->u.src_ring.hp; 660 q_elem->dequeue_val = write_val; 661 q_elem->valid = 0; 662 SRNG_UNLOCK(&srng->lock); 663 hal_write_address_32_mb(hal, 664 srng->u.src_ring.hp_addr, 665 write_val, false); 666 } else { 667 write_val = srng->u.dst_ring.tp; 668 q_elem->dequeue_val = write_val; 669 q_elem->valid = 0; 670 SRNG_UNLOCK(&srng->lock); 671 hal_write_address_32_mb(hal, 672 srng->u.dst_ring.tp_addr, 673 write_val, false); 674 } 675 676 return write_val; 677 } 678 #else 679 /** 680 * hal_process_reg_write_q_elem() - process a regiter write queue element 681 * @hal: hal_soc pointer 682 * @q_elem: pointer to hal regiter write queue element 683 * 684 * Return: The value which was written to the address 685 */ 686 static uint32_t 687 hal_process_reg_write_q_elem(struct hal_soc *hal, 688 struct hal_reg_write_q_elem *q_elem) 689 { 690 struct hal_srng *srng = q_elem->srng; 691 uint32_t write_val; 692 693 SRNG_LOCK(&srng->lock); 694 695 srng->reg_write_in_progress = false; 696 srng->wstats.dequeues++; 697 698 if (srng->ring_dir == HAL_SRNG_SRC_RING) { 699 q_elem->dequeue_val = srng->u.src_ring.hp; 700 hal_write_address_32_mb(hal, 701 srng->u.src_ring.hp_addr, 702 srng->u.src_ring.hp, false); 703 write_val = srng->u.src_ring.hp; 704 } else { 705 q_elem->dequeue_val = srng->u.dst_ring.tp; 706 hal_write_address_32_mb(hal, 707 srng->u.dst_ring.tp_addr, 708 srng->u.dst_ring.tp, false); 709 write_val = srng->u.dst_ring.tp; 710 } 711 712 q_elem->valid = 0; 713 srng->last_dequeue_time = q_elem->dequeue_time; 714 SRNG_UNLOCK(&srng->lock); 715 716 return write_val; 717 } 718 #endif 719 720 /** 721 * hal_reg_write_fill_sched_delay_hist() - fill reg write delay histogram in hal 722 * @hal: hal_soc pointer 723 * @delay: delay in us 724 * 725 * Return: None 726 */ 727 static inline void hal_reg_write_fill_sched_delay_hist(struct hal_soc *hal, 728 uint64_t delay_us) 729 { 730 uint32_t *hist; 731 732 hist = hal->stats.wstats.sched_delay; 733 734 if (delay_us < 100) 735 hist[REG_WRITE_SCHED_DELAY_SUB_100us]++; 736 else if (delay_us < 1000) 737 hist[REG_WRITE_SCHED_DELAY_SUB_1000us]++; 738 else if (delay_us < 5000) 739 hist[REG_WRITE_SCHED_DELAY_SUB_5000us]++; 740 else 741 hist[REG_WRITE_SCHED_DELAY_GT_5000us]++; 742 } 743 744 #ifdef SHADOW_WRITE_DELAY 745 746 #define SHADOW_WRITE_MIN_DELTA_US 5 747 #define SHADOW_WRITE_DELAY_US 50 748 749 /* 750 * Never add those srngs which are performance relate. 751 * The delay itself will hit performance heavily. 752 */ 753 #define IS_SRNG_MATCH(s) ((s)->ring_id == HAL_SRNG_CE_1_DST_STATUS || \ 754 (s)->ring_id == HAL_SRNG_CE_1_DST) 755 756 static inline bool hal_reg_write_need_delay(struct hal_reg_write_q_elem *elem) 757 { 758 struct hal_srng *srng = elem->srng; 759 struct hal_soc *hal; 760 qdf_time_t now; 761 qdf_iomem_t real_addr; 762 763 if (qdf_unlikely(!srng)) 764 return false; 765 766 hal = srng->hal_soc; 767 if (qdf_unlikely(!hal)) 768 return false; 769 770 /* Check if it is target srng, and valid shadow reg */ 771 if (qdf_likely(!IS_SRNG_MATCH(srng))) 772 return false; 773 774 if (srng->ring_dir == HAL_SRNG_SRC_RING) 775 real_addr = SRNG_SRC_ADDR(srng, HP); 776 else 777 real_addr = SRNG_DST_ADDR(srng, TP); 778 if (!hal_validate_shadow_register(hal, real_addr, elem->addr)) 779 return false; 780 781 /* Check the time delta from last write of same srng */ 782 now = qdf_get_log_timestamp(); 783 if (qdf_log_timestamp_to_usecs(now - srng->last_dequeue_time) > 784 SHADOW_WRITE_MIN_DELTA_US) 785 return false; 786 787 /* Delay dequeue, and record */ 788 qdf_udelay(SHADOW_WRITE_DELAY_US); 789 790 srng->wstats.dequeue_delay++; 791 hal->stats.wstats.dequeue_delay++; 792 793 return true; 794 } 795 #else 796 static inline bool hal_reg_write_need_delay(struct hal_reg_write_q_elem *elem) 797 { 798 return false; 799 } 800 #endif 801 802 /** 803 * hal_reg_write_work() - Worker to process delayed writes 804 * @arg: hal_soc pointer 805 * 806 * Return: None 807 */ 808 static void hal_reg_write_work(void *arg) 809 { 810 int32_t q_depth, write_val; 811 struct hal_soc *hal = arg; 812 struct hal_reg_write_q_elem *q_elem; 813 uint64_t delta_us; 814 uint8_t ring_id; 815 uint32_t *addr; 816 uint32_t num_processed = 0; 817 818 q_elem = &hal->reg_write_queue[(hal->read_idx)]; 819 q_elem->work_scheduled_time = qdf_get_log_timestamp(); 820 821 /* Make sure q_elem consistent in the memory for multi-cores */ 822 qdf_rmb(); 823 if (!q_elem->valid) 824 return; 825 826 q_depth = qdf_atomic_read(&hal->stats.wstats.q_depth); 827 if (q_depth > hal->stats.wstats.max_q_depth) 828 hal->stats.wstats.max_q_depth = q_depth; 829 830 if (hif_prevent_link_low_power_states(hal->hif_handle)) { 831 hal->stats.wstats.prevent_l1_fails++; 832 return; 833 } 834 835 while (true) { 836 qdf_rmb(); 837 if (!q_elem->valid) 838 break; 839 840 q_elem->dequeue_time = qdf_get_log_timestamp(); 841 ring_id = q_elem->srng->ring_id; 842 addr = q_elem->addr; 843 delta_us = qdf_log_timestamp_to_usecs(q_elem->dequeue_time - 844 q_elem->enqueue_time); 845 hal_reg_write_fill_sched_delay_hist(hal, delta_us); 846 847 hal->stats.wstats.dequeues++; 848 qdf_atomic_dec(&hal->stats.wstats.q_depth); 849 850 if (hal_reg_write_need_delay(q_elem)) 851 hal_verbose_debug("Delay reg writer for srng 0x%x, addr 0x%pK", 852 q_elem->srng->ring_id, q_elem->addr); 853 854 write_val = hal_process_reg_write_q_elem(hal, q_elem); 855 hal_verbose_debug("read_idx %u srng 0x%x, addr 0x%pK dequeue_val %u sched delay %llu us", 856 hal->read_idx, ring_id, addr, write_val, delta_us); 857 858 num_processed++; 859 hal->read_idx = (hal->read_idx + 1) & 860 (HAL_REG_WRITE_QUEUE_LEN - 1); 861 q_elem = &hal->reg_write_queue[(hal->read_idx)]; 862 } 863 864 hif_allow_link_low_power_states(hal->hif_handle); 865 /* 866 * Decrement active_work_cnt by the number of elements dequeued after 867 * hif_allow_link_low_power_states. 868 * This makes sure that hif_try_complete_tasks will wait till we make 869 * the bus access in hif_allow_link_low_power_states. This will avoid 870 * race condition between delayed register worker and bus suspend 871 * (system suspend or runtime suspend). 872 * 873 * The following decrement should be done at the end! 874 */ 875 qdf_atomic_sub(num_processed, &hal->active_work_cnt); 876 } 877 878 static void __hal_flush_reg_write_work(struct hal_soc *hal) 879 { 880 qdf_cancel_work(&hal->reg_write_work); 881 882 } 883 884 void hal_flush_reg_write_work(hal_soc_handle_t hal_handle) 885 { __hal_flush_reg_write_work((struct hal_soc *)hal_handle); 886 } 887 888 /** 889 * hal_reg_write_enqueue() - enqueue register writes into kworker 890 * @hal_soc: hal_soc pointer 891 * @srng: srng pointer 892 * @addr: iomem address of regiter 893 * @value: value to be written to iomem address 894 * 895 * This function executes from within the SRNG LOCK 896 * 897 * Return: None 898 */ 899 static void hal_reg_write_enqueue(struct hal_soc *hal_soc, 900 struct hal_srng *srng, 901 void __iomem *addr, 902 uint32_t value) 903 { 904 struct hal_reg_write_q_elem *q_elem; 905 uint32_t write_idx; 906 907 if (srng->reg_write_in_progress) { 908 hal_verbose_debug("Already in progress srng ring id 0x%x addr 0x%pK val %u", 909 srng->ring_id, addr, value); 910 qdf_atomic_inc(&hal_soc->stats.wstats.coalesces); 911 srng->wstats.coalesces++; 912 return; 913 } 914 915 write_idx = qdf_atomic_inc_return(&hal_soc->write_idx); 916 917 write_idx = write_idx & (HAL_REG_WRITE_QUEUE_LEN - 1); 918 919 q_elem = &hal_soc->reg_write_queue[write_idx]; 920 921 if (q_elem->valid) { 922 hal_err("queue full"); 923 QDF_BUG(0); 924 return; 925 } 926 927 qdf_atomic_inc(&hal_soc->stats.wstats.enqueues); 928 srng->wstats.enqueues++; 929 930 qdf_atomic_inc(&hal_soc->stats.wstats.q_depth); 931 932 q_elem->srng = srng; 933 q_elem->addr = addr; 934 q_elem->enqueue_val = value; 935 q_elem->enqueue_time = qdf_get_log_timestamp(); 936 937 /* 938 * Before the valid flag is set to true, all the other 939 * fields in the q_elem needs to be updated in memory. 940 * Else there is a chance that the dequeuing worker thread 941 * might read stale entries and process incorrect srng. 942 */ 943 qdf_wmb(); 944 q_elem->valid = true; 945 946 /* 947 * After all other fields in the q_elem has been updated 948 * in memory successfully, the valid flag needs to be updated 949 * in memory in time too. 950 * Else there is a chance that the dequeuing worker thread 951 * might read stale valid flag and the work will be bypassed 952 * for this round. And if there is no other work scheduled 953 * later, this hal register writing won't be updated any more. 954 */ 955 qdf_wmb(); 956 957 srng->reg_write_in_progress = true; 958 qdf_atomic_inc(&hal_soc->active_work_cnt); 959 960 hal_verbose_debug("write_idx %u srng ring id 0x%x addr 0x%pK val %u", 961 write_idx, srng->ring_id, addr, value); 962 963 qdf_queue_work(hal_soc->qdf_dev, hal_soc->reg_write_wq, 964 &hal_soc->reg_write_work); 965 } 966 967 /** 968 * hal_delayed_reg_write_init() - Initialization function for delayed reg writes 969 * @hal_soc: hal_soc pointer 970 * 971 * Initialize main data structures to process register writes in a delayed 972 * workqueue. 973 * 974 * Return: QDF_STATUS_SUCCESS on success else a QDF error. 975 */ 976 static QDF_STATUS hal_delayed_reg_write_init(struct hal_soc *hal) 977 { 978 hal->reg_write_wq = 979 qdf_alloc_high_prior_ordered_workqueue("hal_register_write_wq"); 980 qdf_create_work(0, &hal->reg_write_work, hal_reg_write_work, hal); 981 hal->reg_write_queue = qdf_mem_malloc(HAL_REG_WRITE_QUEUE_LEN * 982 sizeof(*hal->reg_write_queue)); 983 if (!hal->reg_write_queue) { 984 hal_err("unable to allocate memory"); 985 QDF_BUG(0); 986 return QDF_STATUS_E_NOMEM; 987 } 988 989 /* Initial value of indices */ 990 hal->read_idx = 0; 991 qdf_atomic_set(&hal->write_idx, -1); 992 return QDF_STATUS_SUCCESS; 993 } 994 995 /** 996 * hal_delayed_reg_write_deinit() - De-Initialize delayed reg write processing 997 * @hal_soc: hal_soc pointer 998 * 999 * De-initialize main data structures to process register writes in a delayed 1000 * workqueue. 1001 * 1002 * Return: None 1003 */ 1004 static void hal_delayed_reg_write_deinit(struct hal_soc *hal) 1005 { 1006 __hal_flush_reg_write_work(hal); 1007 1008 qdf_flush_workqueue(0, hal->reg_write_wq); 1009 qdf_destroy_workqueue(0, hal->reg_write_wq); 1010 qdf_mem_free(hal->reg_write_queue); 1011 } 1012 1013 #else 1014 static inline QDF_STATUS hal_delayed_reg_write_init(struct hal_soc *hal) 1015 { 1016 return QDF_STATUS_SUCCESS; 1017 } 1018 1019 static inline void hal_delayed_reg_write_deinit(struct hal_soc *hal) 1020 { 1021 } 1022 #endif 1023 1024 #ifdef FEATURE_HAL_DELAYED_REG_WRITE_V2 1025 #ifdef MEMORY_DEBUG 1026 /** 1027 * hal_reg_write_get_timestamp() - Function to get the timestamp 1028 * 1029 * Return: return present simestamp 1030 */ 1031 static inline qdf_time_t hal_del_reg_write_get_ts(void) 1032 { 1033 return qdf_get_log_timestamp(); 1034 } 1035 1036 /** 1037 * hal_del_reg_write_ts_usecs() - Convert the timestamp to micro secs 1038 * @ts: timestamp value to be converted 1039 * 1040 * Return: return the timestamp in micro secs 1041 */ 1042 static inline qdf_time_t hal_del_reg_write_ts_usecs(qdf_time_t ts) 1043 { 1044 return qdf_log_timestamp_to_usecs(ts); 1045 } 1046 1047 /** 1048 * hal_tcl_write_fill_sched_delay_hist() - fill TCL reg write delay histogram 1049 * @hal: hal_soc pointer 1050 * @delay: delay in us 1051 * 1052 * Return: None 1053 */ 1054 static inline void hal_tcl_write_fill_sched_delay_hist(struct hal_soc *hal) 1055 { 1056 uint32_t *hist; 1057 uint32_t delay_us; 1058 1059 hal->tcl_stats.deq_time = hal_del_reg_write_get_ts(); 1060 delay_us = hal_del_reg_write_ts_usecs(hal->tcl_stats.deq_time - 1061 hal->tcl_stats.enq_time); 1062 1063 hist = hal->tcl_stats.sched_delay; 1064 if (delay_us < 100) 1065 hist[REG_WRITE_SCHED_DELAY_SUB_100us]++; 1066 else if (delay_us < 1000) 1067 hist[REG_WRITE_SCHED_DELAY_SUB_1000us]++; 1068 else if (delay_us < 5000) 1069 hist[REG_WRITE_SCHED_DELAY_SUB_5000us]++; 1070 else 1071 hist[REG_WRITE_SCHED_DELAY_GT_5000us]++; 1072 } 1073 1074 #else 1075 static inline qdf_time_t hal_del_reg_write_get_ts(void) 1076 { 1077 return 0; 1078 } 1079 1080 static inline qdf_time_t hal_del_reg_write_ts_usecs(qdf_time_t ts) 1081 { 1082 return 0; 1083 } 1084 1085 static inline void hal_tcl_write_fill_sched_delay_hist(struct hal_soc *hal) 1086 { 1087 } 1088 #endif 1089 1090 /** 1091 * hal_tcl_reg_write_work() - Worker to process delayed SW2TCL1 writes 1092 * @arg: hal_soc pointer 1093 * 1094 * Return: None 1095 */ 1096 static void hal_tcl_reg_write_work(void *arg) 1097 { 1098 struct hal_soc *hal = arg; 1099 struct hal_srng *srng = hal_get_srng(hal, HAL_SRNG_SW2TCL1); 1100 1101 SRNG_LOCK(&srng->lock); 1102 srng->wstats.dequeues++; 1103 hal_tcl_write_fill_sched_delay_hist(hal); 1104 1105 /* 1106 * During the tranition of low to high tput scenario, reg write moves 1107 * from delayed to direct write context, there is a little chance that 1108 * worker thread gets scheduled later than direct context write which 1109 * already wrote the latest HP value. This check can catch that case 1110 * and avoid the repetitive writing of the same HP value. 1111 */ 1112 if (srng->last_reg_wr_val != srng->u.src_ring.hp) { 1113 srng->last_reg_wr_val = srng->u.src_ring.hp; 1114 if (hal->tcl_direct) { 1115 /* 1116 * TCL reg writes have been moved to direct context and 1117 * the assumption is that PCIe bus stays in Active state 1118 * during high tput, hence its fine to write the HP 1119 * while the SRNG_LOCK is being held. 1120 */ 1121 hal->tcl_stats.wq_direct++; 1122 hal_write_address_32_mb(hal, srng->u.src_ring.hp_addr, 1123 srng->last_reg_wr_val, false); 1124 srng->reg_write_in_progress = false; 1125 SRNG_UNLOCK(&srng->lock); 1126 } else { 1127 /* 1128 * TCL reg write to happen in delayed context, 1129 * write operation might take time due to possibility of 1130 * PCIe bus stays in low power state during low tput, 1131 * Hence release the SRNG_LOCK before writing. 1132 */ 1133 hal->tcl_stats.wq_delayed++; 1134 srng->reg_write_in_progress = false; 1135 SRNG_UNLOCK(&srng->lock); 1136 hal_write_address_32_mb(hal, srng->u.src_ring.hp_addr, 1137 srng->last_reg_wr_val, false); 1138 } 1139 } else { 1140 srng->reg_write_in_progress = false; 1141 SRNG_UNLOCK(&srng->lock); 1142 } 1143 1144 /* 1145 * Decrement active_work_cnt to make sure that hif_try_complete_tasks 1146 * will wait. This will avoid race condition between delayed register 1147 * worker and bus suspend (system suspend or runtime suspend). 1148 * 1149 * The following decrement should be done at the end! 1150 */ 1151 qdf_atomic_dec(&hal->active_work_cnt); 1152 qdf_atomic_set(&hal->tcl_work_active, false); 1153 } 1154 1155 static void __hal_flush_tcl_reg_write_work(struct hal_soc *hal) 1156 { 1157 qdf_cancel_work(&hal->tcl_reg_write_work); 1158 } 1159 1160 /** 1161 * hal_tcl_reg_write_enqueue() - enqueue TCL register writes into kworker 1162 * @hal_soc: hal_soc pointer 1163 * @srng: srng pointer 1164 * @addr: iomem address of regiter 1165 * @value: value to be written to iomem address 1166 * 1167 * This function executes from within the SRNG LOCK 1168 * 1169 * Return: None 1170 */ 1171 static void hal_tcl_reg_write_enqueue(struct hal_soc *hal_soc, 1172 struct hal_srng *srng, 1173 void __iomem *addr, 1174 uint32_t value) 1175 { 1176 hal_soc->tcl_stats.enq_time = hal_del_reg_write_get_ts(); 1177 1178 if (qdf_queue_work(hal_soc->qdf_dev, hal_soc->tcl_reg_write_wq, 1179 &hal_soc->tcl_reg_write_work)) { 1180 srng->reg_write_in_progress = true; 1181 qdf_atomic_inc(&hal_soc->active_work_cnt); 1182 qdf_atomic_set(&hal_soc->tcl_work_active, true); 1183 srng->wstats.enqueues++; 1184 } else { 1185 hal_soc->tcl_stats.enq_timer_set++; 1186 qdf_timer_mod(&hal_soc->tcl_reg_write_timer, 1); 1187 } 1188 } 1189 1190 /** 1191 * hal_tcl_reg_write_timer() - timer handler to take care of pending TCL writes 1192 * @arg: srng handle 1193 * 1194 * This function handles the pending TCL reg writes missed due to the previous 1195 * scheduled worker running. 1196 * 1197 * Return: None 1198 */ 1199 static void hal_tcl_reg_write_timer(void *arg) 1200 { 1201 hal_ring_handle_t srng_hdl = arg; 1202 struct hal_srng *srng; 1203 struct hal_soc *hal; 1204 1205 srng = (struct hal_srng *)srng_hdl; 1206 hal = srng->hal_soc; 1207 1208 if (hif_pm_runtime_get(hal->hif_handle, RTPM_ID_DW_TX_HW_ENQUEUE, 1209 true)) { 1210 hal_srng_set_event(srng_hdl, HAL_SRNG_FLUSH_EVENT); 1211 hal_srng_inc_flush_cnt(srng_hdl); 1212 goto fail; 1213 } 1214 1215 SRNG_LOCK(&srng->lock); 1216 if (hal->tcl_direct) { 1217 /* 1218 * Due to the previous scheduled worker still running, 1219 * direct reg write cannot be performed, so posted the 1220 * pending writes to timer context. 1221 */ 1222 if (srng->last_reg_wr_val != srng->u.src_ring.hp) { 1223 srng->last_reg_wr_val = srng->u.src_ring.hp; 1224 srng->wstats.direct++; 1225 hal->tcl_stats.timer_direct++; 1226 hal_write_address_32_mb(hal, srng->u.src_ring.hp_addr, 1227 srng->last_reg_wr_val, false); 1228 } 1229 } else { 1230 /* 1231 * Due to the previous scheduled worker still running, 1232 * queue_work from delayed context would fail, 1233 * so retry from timer context. 1234 */ 1235 if (qdf_queue_work(hal->qdf_dev, hal->tcl_reg_write_wq, 1236 &hal->tcl_reg_write_work)) { 1237 srng->reg_write_in_progress = true; 1238 qdf_atomic_inc(&hal->active_work_cnt); 1239 qdf_atomic_set(&hal->tcl_work_active, true); 1240 srng->wstats.enqueues++; 1241 hal->tcl_stats.timer_enq++; 1242 } else { 1243 if (srng->last_reg_wr_val != srng->u.src_ring.hp) { 1244 hal->tcl_stats.timer_reset++; 1245 qdf_timer_mod(&hal->tcl_reg_write_timer, 1); 1246 } 1247 } 1248 } 1249 SRNG_UNLOCK(&srng->lock); 1250 hif_pm_runtime_put(hal->hif_handle, RTPM_ID_DW_TX_HW_ENQUEUE); 1251 1252 fail: 1253 return; 1254 } 1255 1256 /** 1257 * hal_delayed_tcl_reg_write_init() - Initialization for delayed TCL reg writes 1258 * @hal_soc: hal_soc pointer 1259 * 1260 * Initialize main data structures to process TCL register writes in a delayed 1261 * workqueue. 1262 * 1263 * Return: QDF_STATUS_SUCCESS on success else a QDF error. 1264 */ 1265 static QDF_STATUS hal_delayed_tcl_reg_write_init(struct hal_soc *hal) 1266 { 1267 struct hal_srng *srng = hal_get_srng(hal, HAL_SRNG_SW2TCL1); 1268 QDF_STATUS status; 1269 1270 hal->tcl_reg_write_wq = 1271 qdf_alloc_high_prior_ordered_workqueue("hal_tcl_reg_write_wq"); 1272 if (!hal->tcl_reg_write_wq) { 1273 hal_err("hal_tcl_reg_write_wq alloc failed"); 1274 return QDF_STATUS_E_NOMEM; 1275 } 1276 1277 status = qdf_create_work(0, &hal->tcl_reg_write_work, 1278 hal_tcl_reg_write_work, hal); 1279 if (status != QDF_STATUS_SUCCESS) { 1280 hal_err("tcl_reg_write_work create failed"); 1281 goto fail; 1282 } 1283 1284 status = qdf_timer_init(hal->qdf_dev, &hal->tcl_reg_write_timer, 1285 hal_tcl_reg_write_timer, (void *)srng, 1286 QDF_TIMER_TYPE_WAKE_APPS); 1287 if (status != QDF_STATUS_SUCCESS) { 1288 hal_err("tcl_reg_write_timer init failed"); 1289 goto fail; 1290 } 1291 1292 qdf_atomic_init(&hal->tcl_work_active); 1293 1294 return QDF_STATUS_SUCCESS; 1295 1296 fail: 1297 qdf_destroy_workqueue(0, hal->tcl_reg_write_wq); 1298 return status; 1299 } 1300 1301 /** 1302 * hal_delayed_tcl_reg_write_deinit() - De-Initialize delayed TCL reg writes 1303 * @hal_soc: hal_soc pointer 1304 * 1305 * De-initialize main data structures to process TCL register writes in a 1306 * delayed workqueue. 1307 * 1308 * Return: None 1309 */ 1310 static void hal_delayed_tcl_reg_write_deinit(struct hal_soc *hal) 1311 { 1312 qdf_timer_stop(&hal->tcl_reg_write_timer); 1313 qdf_timer_free(&hal->tcl_reg_write_timer); 1314 1315 __hal_flush_tcl_reg_write_work(hal); 1316 qdf_flush_workqueue(0, hal->tcl_reg_write_wq); 1317 qdf_destroy_workqueue(0, hal->tcl_reg_write_wq); 1318 } 1319 1320 #else 1321 static inline QDF_STATUS hal_delayed_tcl_reg_write_init(struct hal_soc *hal) 1322 { 1323 return QDF_STATUS_SUCCESS; 1324 } 1325 1326 static inline void hal_delayed_tcl_reg_write_deinit(struct hal_soc *hal) 1327 { 1328 } 1329 #endif 1330 1331 #ifdef FEATURE_HAL_DELAYED_REG_WRITE_V2 1332 #ifdef FEATURE_HAL_DELAYED_REG_WRITE 1333 static inline void hal_reg_write_enqueue_v2(struct hal_soc *hal_soc, 1334 struct hal_srng *srng, 1335 void __iomem *addr, 1336 uint32_t value) 1337 { 1338 hal_reg_write_enqueue(hal_soc, srng, addr, value); 1339 } 1340 #else 1341 static inline void hal_reg_write_enqueue_v2(struct hal_soc *hal_soc, 1342 struct hal_srng *srng, 1343 void __iomem *addr, 1344 uint32_t value) 1345 { 1346 qdf_atomic_inc(&hal_soc->stats.wstats.direct); 1347 srng->wstats.direct++; 1348 hal_write_address_32_mb(hal_soc, addr, value, false); 1349 } 1350 #endif 1351 1352 void hal_delayed_reg_write(struct hal_soc *hal_soc, 1353 struct hal_srng *srng, 1354 void __iomem *addr, 1355 uint32_t value) 1356 { 1357 switch (srng->ring_type) { 1358 case TCL_DATA: 1359 if (hal_is_reg_write_tput_level_high(hal_soc)) { 1360 hal_soc->tcl_direct = true; 1361 if (srng->reg_write_in_progress || 1362 !qdf_atomic_read(&hal_soc->tcl_work_active)) { 1363 /* 1364 * Now the delayed work have either completed 1365 * the writing or not even scheduled and would 1366 * be blocked by SRNG_LOCK, hence it is fine to 1367 * do direct write here. 1368 */ 1369 srng->last_reg_wr_val = srng->u.src_ring.hp; 1370 srng->wstats.direct++; 1371 hal_write_address_32_mb(hal_soc, addr, 1372 srng->last_reg_wr_val, 1373 false); 1374 } else { 1375 hal_soc->tcl_stats.direct_timer_set++; 1376 qdf_timer_mod(&hal_soc->tcl_reg_write_timer, 1); 1377 } 1378 } else { 1379 hal_soc->tcl_direct = false; 1380 if (srng->reg_write_in_progress) { 1381 srng->wstats.coalesces++; 1382 } else { 1383 hal_tcl_reg_write_enqueue(hal_soc, srng, 1384 addr, value); 1385 } 1386 } 1387 break; 1388 case CE_SRC: 1389 case CE_DST: 1390 case CE_DST_STATUS: 1391 hal_reg_write_enqueue_v2(hal_soc, srng, addr, value); 1392 break; 1393 default: 1394 qdf_atomic_inc(&hal_soc->stats.wstats.direct); 1395 srng->wstats.direct++; 1396 hal_write_address_32_mb(hal_soc, addr, value, false); 1397 break; 1398 } 1399 } 1400 1401 #else 1402 #ifdef FEATURE_HAL_DELAYED_REG_WRITE 1403 #ifdef QCA_WIFI_QCA6750 1404 void hal_delayed_reg_write(struct hal_soc *hal_soc, 1405 struct hal_srng *srng, 1406 void __iomem *addr, 1407 uint32_t value) 1408 { 1409 uint8_t vote_access; 1410 1411 switch (srng->ring_type) { 1412 case CE_SRC: 1413 case CE_DST: 1414 case CE_DST_STATUS: 1415 vote_access = hif_get_ep_vote_access(hal_soc->hif_handle, 1416 HIF_EP_VOTE_NONDP_ACCESS); 1417 if ((vote_access == HIF_EP_VOTE_ACCESS_DISABLE) || 1418 (vote_access == HIF_EP_VOTE_INTERMEDIATE_ACCESS && 1419 PLD_MHI_STATE_L0 == 1420 pld_get_mhi_state(hal_soc->qdf_dev->dev))) { 1421 hal_write_address_32_mb(hal_soc, addr, value, false); 1422 qdf_atomic_inc(&hal_soc->stats.wstats.direct); 1423 srng->wstats.direct++; 1424 } else { 1425 hal_reg_write_enqueue(hal_soc, srng, addr, value); 1426 } 1427 break; 1428 default: 1429 if (hif_get_ep_vote_access(hal_soc->hif_handle, 1430 HIF_EP_VOTE_DP_ACCESS) == 1431 HIF_EP_VOTE_ACCESS_DISABLE || 1432 hal_is_reg_write_tput_level_high(hal_soc) || 1433 PLD_MHI_STATE_L0 == 1434 pld_get_mhi_state(hal_soc->qdf_dev->dev)) { 1435 hal_write_address_32_mb(hal_soc, addr, value, false); 1436 qdf_atomic_inc(&hal_soc->stats.wstats.direct); 1437 srng->wstats.direct++; 1438 } else { 1439 hal_reg_write_enqueue(hal_soc, srng, addr, value); 1440 } 1441 1442 break; 1443 } 1444 } 1445 #else 1446 void hal_delayed_reg_write(struct hal_soc *hal_soc, 1447 struct hal_srng *srng, 1448 void __iomem *addr, 1449 uint32_t value) 1450 { 1451 if (pld_is_device_awake(hal_soc->qdf_dev->dev) || 1452 hal_is_reg_write_tput_level_high(hal_soc)) { 1453 qdf_atomic_inc(&hal_soc->stats.wstats.direct); 1454 srng->wstats.direct++; 1455 hal_write_address_32_mb(hal_soc, addr, value, false); 1456 } else { 1457 hal_reg_write_enqueue(hal_soc, srng, addr, value); 1458 } 1459 } 1460 #endif 1461 #endif 1462 #endif 1463 1464 /** 1465 * hal_attach - Initialize HAL layer 1466 * @hif_handle: Opaque HIF handle 1467 * @qdf_dev: QDF device 1468 * 1469 * Return: Opaque HAL SOC handle 1470 * NULL on failure (if given ring is not available) 1471 * 1472 * This function should be called as part of HIF initialization (for accessing 1473 * copy engines). DP layer will get hal_soc handle using hif_get_hal_handle() 1474 * 1475 */ 1476 void *hal_attach(struct hif_opaque_softc *hif_handle, qdf_device_t qdf_dev) 1477 { 1478 struct hal_soc *hal; 1479 int i; 1480 1481 hal = qdf_mem_malloc(sizeof(*hal)); 1482 1483 if (!hal) { 1484 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR, 1485 "%s: hal_soc allocation failed", __func__); 1486 goto fail0; 1487 } 1488 hal->hif_handle = hif_handle; 1489 hal->dev_base_addr = hif_get_dev_ba(hif_handle); /* UMAC */ 1490 hal->dev_base_addr_ce = hif_get_dev_ba_ce(hif_handle); /* CE */ 1491 hal->qdf_dev = qdf_dev; 1492 hal->shadow_rdptr_mem_vaddr = (uint32_t *)qdf_mem_alloc_consistent( 1493 qdf_dev, qdf_dev->dev, sizeof(*(hal->shadow_rdptr_mem_vaddr)) * 1494 HAL_SRNG_ID_MAX, &(hal->shadow_rdptr_mem_paddr)); 1495 if (!hal->shadow_rdptr_mem_paddr) { 1496 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR, 1497 "%s: hal->shadow_rdptr_mem_paddr allocation failed", 1498 __func__); 1499 goto fail1; 1500 } 1501 qdf_mem_zero(hal->shadow_rdptr_mem_vaddr, 1502 sizeof(*(hal->shadow_rdptr_mem_vaddr)) * HAL_SRNG_ID_MAX); 1503 1504 hal->shadow_wrptr_mem_vaddr = 1505 (uint32_t *)qdf_mem_alloc_consistent(qdf_dev, qdf_dev->dev, 1506 sizeof(*(hal->shadow_wrptr_mem_vaddr)) * HAL_MAX_LMAC_RINGS, 1507 &(hal->shadow_wrptr_mem_paddr)); 1508 if (!hal->shadow_wrptr_mem_vaddr) { 1509 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR, 1510 "%s: hal->shadow_wrptr_mem_vaddr allocation failed", 1511 __func__); 1512 goto fail2; 1513 } 1514 qdf_mem_zero(hal->shadow_wrptr_mem_vaddr, 1515 sizeof(*(hal->shadow_wrptr_mem_vaddr)) * HAL_MAX_LMAC_RINGS); 1516 1517 for (i = 0; i < HAL_SRNG_ID_MAX; i++) { 1518 hal->srng_list[i].initialized = 0; 1519 hal->srng_list[i].ring_id = i; 1520 } 1521 1522 qdf_spinlock_create(&hal->register_access_lock); 1523 hal->register_window = 0; 1524 hal->target_type = hal_get_target_type(hal_soc_to_hal_soc_handle(hal)); 1525 hal->ops = qdf_mem_malloc(sizeof(*hal->ops)); 1526 1527 if (!hal->ops) { 1528 hal_err("unable to allocable memory for HAL ops"); 1529 goto fail3; 1530 } 1531 1532 hal_target_based_configure(hal); 1533 1534 hal_reg_write_fail_history_init(hal); 1535 1536 qdf_minidump_log(hal, sizeof(*hal), "hal_soc"); 1537 1538 qdf_atomic_init(&hal->active_work_cnt); 1539 hal_delayed_reg_write_init(hal); 1540 hal_delayed_tcl_reg_write_init(hal); 1541 1542 return (void *)hal; 1543 fail3: 1544 qdf_mem_free_consistent(qdf_dev, qdf_dev->dev, 1545 sizeof(*hal->shadow_wrptr_mem_vaddr) * 1546 HAL_MAX_LMAC_RINGS, 1547 hal->shadow_wrptr_mem_vaddr, 1548 hal->shadow_wrptr_mem_paddr, 0); 1549 fail2: 1550 qdf_mem_free_consistent(qdf_dev, qdf_dev->dev, 1551 sizeof(*(hal->shadow_rdptr_mem_vaddr)) * HAL_SRNG_ID_MAX, 1552 hal->shadow_rdptr_mem_vaddr, hal->shadow_rdptr_mem_paddr, 0); 1553 fail1: 1554 qdf_mem_free(hal); 1555 fail0: 1556 return NULL; 1557 } 1558 qdf_export_symbol(hal_attach); 1559 1560 /** 1561 * hal_mem_info - Retrieve hal memory base address 1562 * 1563 * @hal_soc: Opaque HAL SOC handle 1564 * @mem: pointer to structure to be updated with hal mem info 1565 */ 1566 void hal_get_meminfo(hal_soc_handle_t hal_soc_hdl, struct hal_mem_info *mem) 1567 { 1568 struct hal_soc *hal = (struct hal_soc *)hal_soc_hdl; 1569 mem->dev_base_addr = (void *)hal->dev_base_addr; 1570 mem->shadow_rdptr_mem_vaddr = (void *)hal->shadow_rdptr_mem_vaddr; 1571 mem->shadow_wrptr_mem_vaddr = (void *)hal->shadow_wrptr_mem_vaddr; 1572 mem->shadow_rdptr_mem_paddr = (void *)hal->shadow_rdptr_mem_paddr; 1573 mem->shadow_wrptr_mem_paddr = (void *)hal->shadow_wrptr_mem_paddr; 1574 hif_read_phy_mem_base((void *)hal->hif_handle, 1575 (qdf_dma_addr_t *)&mem->dev_base_paddr); 1576 mem->lmac_srng_start_id = HAL_SRNG_LMAC1_ID_START; 1577 return; 1578 } 1579 qdf_export_symbol(hal_get_meminfo); 1580 1581 /** 1582 * hal_detach - Detach HAL layer 1583 * @hal_soc: HAL SOC handle 1584 * 1585 * Return: Opaque HAL SOC handle 1586 * NULL on failure (if given ring is not available) 1587 * 1588 * This function should be called as part of HIF initialization (for accessing 1589 * copy engines). DP layer will get hal_soc handle using hif_get_hal_handle() 1590 * 1591 */ 1592 extern void hal_detach(void *hal_soc) 1593 { 1594 struct hal_soc *hal = (struct hal_soc *)hal_soc; 1595 1596 hal_delayed_reg_write_deinit(hal); 1597 hal_delayed_tcl_reg_write_deinit(hal); 1598 qdf_mem_free(hal->ops); 1599 1600 qdf_mem_free_consistent(hal->qdf_dev, hal->qdf_dev->dev, 1601 sizeof(*(hal->shadow_rdptr_mem_vaddr)) * HAL_SRNG_ID_MAX, 1602 hal->shadow_rdptr_mem_vaddr, hal->shadow_rdptr_mem_paddr, 0); 1603 qdf_mem_free_consistent(hal->qdf_dev, hal->qdf_dev->dev, 1604 sizeof(*(hal->shadow_wrptr_mem_vaddr)) * HAL_MAX_LMAC_RINGS, 1605 hal->shadow_wrptr_mem_vaddr, hal->shadow_wrptr_mem_paddr, 0); 1606 qdf_minidump_remove(hal, sizeof(*hal), "hal_soc"); 1607 qdf_mem_free(hal); 1608 1609 return; 1610 } 1611 qdf_export_symbol(hal_detach); 1612 1613 #define HAL_CE_CHANNEL_DST_DEST_CTRL_ADDR(x) ((x) + 0x000000b0) 1614 #define HAL_CE_CHANNEL_DST_DEST_CTRL_DEST_MAX_LENGTH_BMSK 0x0000ffff 1615 #define HAL_CE_CHANNEL_DST_DEST_RING_CONSUMER_PREFETCH_TIMER_ADDR(x) ((x) + 0x00000040) 1616 #define HAL_CE_CHANNEL_DST_DEST_RING_CONSUMER_PREFETCH_TIMER_RMSK 0x00000007 1617 /** 1618 * hal_ce_dst_setup - Initialize CE destination ring registers 1619 * @hal_soc: HAL SOC handle 1620 * @srng: SRNG ring pointer 1621 */ 1622 static inline void hal_ce_dst_setup(struct hal_soc *hal, struct hal_srng *srng, 1623 int ring_num) 1624 { 1625 uint32_t reg_val = 0; 1626 uint32_t reg_addr; 1627 struct hal_hw_srng_config *ring_config = 1628 HAL_SRNG_CONFIG(hal, CE_DST); 1629 1630 /* set DEST_MAX_LENGTH according to ce assignment */ 1631 reg_addr = HAL_CE_CHANNEL_DST_DEST_CTRL_ADDR( 1632 ring_config->reg_start[R0_INDEX] + 1633 (ring_num * ring_config->reg_size[R0_INDEX])); 1634 1635 reg_val = HAL_REG_READ(hal, reg_addr); 1636 reg_val &= ~HAL_CE_CHANNEL_DST_DEST_CTRL_DEST_MAX_LENGTH_BMSK; 1637 reg_val |= srng->u.dst_ring.max_buffer_length & 1638 HAL_CE_CHANNEL_DST_DEST_CTRL_DEST_MAX_LENGTH_BMSK; 1639 HAL_REG_WRITE(hal, reg_addr, reg_val); 1640 1641 if (srng->prefetch_timer) { 1642 reg_addr = HAL_CE_CHANNEL_DST_DEST_RING_CONSUMER_PREFETCH_TIMER_ADDR( 1643 ring_config->reg_start[R0_INDEX] + 1644 (ring_num * ring_config->reg_size[R0_INDEX])); 1645 1646 reg_val = HAL_REG_READ(hal, reg_addr); 1647 reg_val &= ~HAL_CE_CHANNEL_DST_DEST_RING_CONSUMER_PREFETCH_TIMER_RMSK; 1648 reg_val |= srng->prefetch_timer; 1649 HAL_REG_WRITE(hal, reg_addr, reg_val); 1650 reg_val = HAL_REG_READ(hal, reg_addr); 1651 } 1652 1653 } 1654 1655 /** 1656 * hal_reo_read_write_ctrl_ix - Read or write REO_DESTINATION_RING_CTRL_IX 1657 * @hal: HAL SOC handle 1658 * @read: boolean value to indicate if read or write 1659 * @ix0: pointer to store IX0 reg value 1660 * @ix1: pointer to store IX1 reg value 1661 * @ix2: pointer to store IX2 reg value 1662 * @ix3: pointer to store IX3 reg value 1663 */ 1664 void hal_reo_read_write_ctrl_ix(hal_soc_handle_t hal_soc_hdl, bool read, 1665 uint32_t *ix0, uint32_t *ix1, 1666 uint32_t *ix2, uint32_t *ix3) 1667 { 1668 uint32_t reg_offset; 1669 struct hal_soc *hal = (struct hal_soc *)hal_soc_hdl; 1670 uint32_t reo_reg_base; 1671 1672 reo_reg_base = hal_get_reo_reg_base_offset(hal_soc_hdl); 1673 1674 if (read) { 1675 if (ix0) { 1676 reg_offset = 1677 HAL_REO_DESTINATION_RING_CTRL_IX_0_ADDR( 1678 reo_reg_base); 1679 *ix0 = HAL_REG_READ(hal, reg_offset); 1680 } 1681 1682 if (ix1) { 1683 reg_offset = 1684 HAL_REO_DESTINATION_RING_CTRL_IX_1_ADDR( 1685 reo_reg_base); 1686 *ix1 = HAL_REG_READ(hal, reg_offset); 1687 } 1688 1689 if (ix2) { 1690 reg_offset = 1691 HAL_REO_DESTINATION_RING_CTRL_IX_2_ADDR( 1692 reo_reg_base); 1693 *ix2 = HAL_REG_READ(hal, reg_offset); 1694 } 1695 1696 if (ix3) { 1697 reg_offset = 1698 HAL_REO_DESTINATION_RING_CTRL_IX_3_ADDR( 1699 reo_reg_base); 1700 *ix3 = HAL_REG_READ(hal, reg_offset); 1701 } 1702 } else { 1703 if (ix0) { 1704 reg_offset = 1705 HAL_REO_DESTINATION_RING_CTRL_IX_0_ADDR( 1706 reo_reg_base); 1707 HAL_REG_WRITE_CONFIRM_RETRY(hal, reg_offset, 1708 *ix0, true); 1709 } 1710 1711 if (ix1) { 1712 reg_offset = 1713 HAL_REO_DESTINATION_RING_CTRL_IX_1_ADDR( 1714 reo_reg_base); 1715 HAL_REG_WRITE_CONFIRM_RETRY(hal, reg_offset, 1716 *ix1, true); 1717 } 1718 1719 if (ix2) { 1720 reg_offset = 1721 HAL_REO_DESTINATION_RING_CTRL_IX_2_ADDR( 1722 reo_reg_base); 1723 HAL_REG_WRITE_CONFIRM_RETRY(hal, reg_offset, 1724 *ix2, true); 1725 } 1726 1727 if (ix3) { 1728 reg_offset = 1729 HAL_REO_DESTINATION_RING_CTRL_IX_3_ADDR( 1730 reo_reg_base); 1731 HAL_REG_WRITE_CONFIRM_RETRY(hal, reg_offset, 1732 *ix3, true); 1733 } 1734 } 1735 } 1736 1737 /** 1738 * hal_srng_dst_set_hp_paddr_confirm() - Set physical address to dest ring head 1739 * pointer and confirm that write went through by reading back the value 1740 * @srng: sring pointer 1741 * @paddr: physical address 1742 * 1743 * Return: None 1744 */ 1745 void hal_srng_dst_set_hp_paddr_confirm(struct hal_srng *srng, uint64_t paddr) 1746 { 1747 SRNG_DST_REG_WRITE_CONFIRM(srng, HP_ADDR_LSB, paddr & 0xffffffff); 1748 SRNG_DST_REG_WRITE_CONFIRM(srng, HP_ADDR_MSB, paddr >> 32); 1749 } 1750 1751 /** 1752 * hal_srng_dst_init_hp() - Initialize destination ring head 1753 * pointer 1754 * @hal_soc: hal_soc handle 1755 * @srng: sring pointer 1756 * @vaddr: virtual address 1757 */ 1758 void hal_srng_dst_init_hp(struct hal_soc_handle *hal_soc, 1759 struct hal_srng *srng, 1760 uint32_t *vaddr) 1761 { 1762 uint32_t reg_offset; 1763 struct hal_soc *hal = (struct hal_soc *)hal_soc; 1764 1765 if (!srng) 1766 return; 1767 1768 srng->u.dst_ring.hp_addr = vaddr; 1769 reg_offset = SRNG_DST_ADDR(srng, HP) - hal->dev_base_addr; 1770 HAL_REG_WRITE_CONFIRM_RETRY( 1771 hal, reg_offset, srng->u.dst_ring.cached_hp, true); 1772 1773 if (vaddr) { 1774 *srng->u.dst_ring.hp_addr = srng->u.dst_ring.cached_hp; 1775 QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_ERROR, 1776 "hp_addr=%pK, cached_hp=%d, hp=%d", 1777 (void *)srng->u.dst_ring.hp_addr, 1778 srng->u.dst_ring.cached_hp, 1779 *srng->u.dst_ring.hp_addr); 1780 } 1781 } 1782 1783 /** 1784 * hal_srng_hw_init - Private function to initialize SRNG HW 1785 * @hal_soc: HAL SOC handle 1786 * @srng: SRNG ring pointer 1787 */ 1788 static inline void hal_srng_hw_init(struct hal_soc *hal, 1789 struct hal_srng *srng) 1790 { 1791 if (srng->ring_dir == HAL_SRNG_SRC_RING) 1792 hal_srng_src_hw_init(hal, srng); 1793 else 1794 hal_srng_dst_hw_init(hal, srng); 1795 } 1796 1797 #ifdef CONFIG_SHADOW_V2 1798 #define ignore_shadow false 1799 #define CHECK_SHADOW_REGISTERS true 1800 #else 1801 #define ignore_shadow true 1802 #define CHECK_SHADOW_REGISTERS false 1803 #endif 1804 1805 #ifdef WLAN_FEATURE_NEAR_FULL_IRQ 1806 /** 1807 * hal_srng_is_near_full_irq_supported() - Check if near full irq is 1808 * supported on this SRNG 1809 * @hal_soc: HAL SoC handle 1810 * @ring_type: SRNG type 1811 * @ring_num: ring number 1812 * 1813 * Return: true, if near full irq is supported for this SRNG 1814 * false, if near full irq is not supported for this SRNG 1815 */ 1816 bool hal_srng_is_near_full_irq_supported(hal_soc_handle_t hal_soc, 1817 int ring_type, int ring_num) 1818 { 1819 struct hal_soc *hal = (struct hal_soc *)hal_soc; 1820 struct hal_hw_srng_config *ring_config = 1821 HAL_SRNG_CONFIG(hal, ring_type); 1822 1823 return ring_config->nf_irq_support; 1824 } 1825 1826 /** 1827 * hal_srng_set_msi2_params() - Set MSI2 params to SRNG data structure from 1828 * ring params 1829 * @srng: SRNG handle 1830 * @ring_params: ring params for this SRNG 1831 * 1832 * Return: None 1833 */ 1834 static inline void 1835 hal_srng_set_msi2_params(struct hal_srng *srng, 1836 struct hal_srng_params *ring_params) 1837 { 1838 srng->msi2_addr = ring_params->msi2_addr; 1839 srng->msi2_data = ring_params->msi2_data; 1840 } 1841 1842 /** 1843 * hal_srng_get_nf_params() - Get the near full MSI2 params from srng 1844 * @srng: SRNG handle 1845 * @ring_params: ring params for this SRNG 1846 * 1847 * Return: None 1848 */ 1849 static inline void 1850 hal_srng_get_nf_params(struct hal_srng *srng, 1851 struct hal_srng_params *ring_params) 1852 { 1853 ring_params->msi2_addr = srng->msi2_addr; 1854 ring_params->msi2_data = srng->msi2_data; 1855 } 1856 1857 /** 1858 * hal_srng_set_nf_thresholds() - Set the near full thresholds in SRNG 1859 * @srng: SRNG handle where the params are to be set 1860 * @ring_params: ring params, from where threshold is to be fetched 1861 * 1862 * Return: None 1863 */ 1864 static inline void 1865 hal_srng_set_nf_thresholds(struct hal_srng *srng, 1866 struct hal_srng_params *ring_params) 1867 { 1868 srng->u.dst_ring.nf_irq_support = ring_params->nf_irq_support; 1869 srng->u.dst_ring.high_thresh = ring_params->high_thresh; 1870 } 1871 #else 1872 static inline void 1873 hal_srng_set_msi2_params(struct hal_srng *srng, 1874 struct hal_srng_params *ring_params) 1875 { 1876 } 1877 1878 static inline void 1879 hal_srng_get_nf_params(struct hal_srng *srng, 1880 struct hal_srng_params *ring_params) 1881 { 1882 } 1883 1884 static inline void 1885 hal_srng_set_nf_thresholds(struct hal_srng *srng, 1886 struct hal_srng_params *ring_params) 1887 { 1888 } 1889 #endif 1890 1891 #if defined(CLEAR_SW2TCL_CONSUMED_DESC) 1892 /** 1893 * hal_srng_last_desc_cleared_init - Initialize SRNG last_desc_cleared ptr 1894 * 1895 * @srng: Source ring pointer 1896 * 1897 * Return: None 1898 */ 1899 static inline 1900 void hal_srng_last_desc_cleared_init(struct hal_srng *srng) 1901 { 1902 srng->last_desc_cleared = srng->ring_size - srng->entry_size; 1903 } 1904 1905 #else 1906 static inline 1907 void hal_srng_last_desc_cleared_init(struct hal_srng *srng) 1908 { 1909 } 1910 #endif /* CLEAR_SW2TCL_CONSUMED_DESC */ 1911 1912 /** 1913 * hal_srng_setup - Initialize HW SRNG ring. 1914 * @hal_soc: Opaque HAL SOC handle 1915 * @ring_type: one of the types from hal_ring_type 1916 * @ring_num: Ring number if there are multiple rings of same type (staring 1917 * from 0) 1918 * @mac_id: valid MAC Id should be passed if ring type is one of lmac rings 1919 * @ring_params: SRNG ring params in hal_srng_params structure. 1920 1921 * Callers are expected to allocate contiguous ring memory of size 1922 * 'num_entries * entry_size' bytes and pass the physical and virtual base 1923 * addresses through 'ring_base_paddr' and 'ring_base_vaddr' in 1924 * hal_srng_params structure. Ring base address should be 8 byte aligned 1925 * and size of each ring entry should be queried using the API 1926 * hal_srng_get_entrysize 1927 * 1928 * Return: Opaque pointer to ring on success 1929 * NULL on failure (if given ring is not available) 1930 */ 1931 void *hal_srng_setup(void *hal_soc, int ring_type, int ring_num, 1932 int mac_id, struct hal_srng_params *ring_params) 1933 { 1934 int ring_id; 1935 struct hal_soc *hal = (struct hal_soc *)hal_soc; 1936 struct hal_srng *srng; 1937 struct hal_hw_srng_config *ring_config = 1938 HAL_SRNG_CONFIG(hal, ring_type); 1939 void *dev_base_addr; 1940 int i; 1941 1942 ring_id = hal_get_srng_ring_id(hal_soc, ring_type, ring_num, mac_id); 1943 if (ring_id < 0) 1944 return NULL; 1945 1946 hal_verbose_debug("mac_id %d ring_id %d", mac_id, ring_id); 1947 1948 srng = hal_get_srng(hal_soc, ring_id); 1949 1950 if (srng->initialized) { 1951 hal_verbose_debug("Ring (ring_type, ring_num) already initialized"); 1952 return NULL; 1953 } 1954 1955 dev_base_addr = hal->dev_base_addr; 1956 srng->ring_id = ring_id; 1957 srng->ring_type = ring_type; 1958 srng->ring_dir = ring_config->ring_dir; 1959 srng->ring_base_paddr = ring_params->ring_base_paddr; 1960 srng->ring_base_vaddr = ring_params->ring_base_vaddr; 1961 srng->entry_size = ring_config->entry_size; 1962 srng->num_entries = ring_params->num_entries; 1963 srng->ring_size = srng->num_entries * srng->entry_size; 1964 srng->ring_size_mask = srng->ring_size - 1; 1965 srng->msi_addr = ring_params->msi_addr; 1966 srng->msi_data = ring_params->msi_data; 1967 srng->intr_timer_thres_us = ring_params->intr_timer_thres_us; 1968 srng->intr_batch_cntr_thres_entries = 1969 ring_params->intr_batch_cntr_thres_entries; 1970 srng->prefetch_timer = ring_params->prefetch_timer; 1971 srng->hal_soc = hal_soc; 1972 hal_srng_set_msi2_params(srng, ring_params); 1973 1974 for (i = 0 ; i < MAX_SRNG_REG_GROUPS; i++) { 1975 srng->hwreg_base[i] = dev_base_addr + ring_config->reg_start[i] 1976 + (ring_num * ring_config->reg_size[i]); 1977 } 1978 1979 /* Zero out the entire ring memory */ 1980 qdf_mem_zero(srng->ring_base_vaddr, (srng->entry_size * 1981 srng->num_entries) << 2); 1982 1983 srng->flags = ring_params->flags; 1984 #ifdef BIG_ENDIAN_HOST 1985 /* TODO: See if we should we get these flags from caller */ 1986 srng->flags |= HAL_SRNG_DATA_TLV_SWAP; 1987 srng->flags |= HAL_SRNG_MSI_SWAP; 1988 srng->flags |= HAL_SRNG_RING_PTR_SWAP; 1989 #endif 1990 1991 hal_srng_last_desc_cleared_init(srng); 1992 1993 if (srng->ring_dir == HAL_SRNG_SRC_RING) { 1994 srng->u.src_ring.hp = 0; 1995 srng->u.src_ring.reap_hp = srng->ring_size - 1996 srng->entry_size; 1997 srng->u.src_ring.tp_addr = 1998 &(hal->shadow_rdptr_mem_vaddr[ring_id]); 1999 srng->u.src_ring.low_threshold = 2000 ring_params->low_threshold * srng->entry_size; 2001 if (ring_config->lmac_ring) { 2002 /* For LMAC rings, head pointer updates will be done 2003 * through FW by writing to a shared memory location 2004 */ 2005 srng->u.src_ring.hp_addr = 2006 &(hal->shadow_wrptr_mem_vaddr[ring_id - 2007 HAL_SRNG_LMAC1_ID_START]); 2008 srng->flags |= HAL_SRNG_LMAC_RING; 2009 } else if (ignore_shadow || (srng->u.src_ring.hp_addr == 0)) { 2010 srng->u.src_ring.hp_addr = 2011 hal_get_window_address(hal, 2012 SRNG_SRC_ADDR(srng, HP)); 2013 2014 if (CHECK_SHADOW_REGISTERS) { 2015 QDF_TRACE(QDF_MODULE_ID_TXRX, 2016 QDF_TRACE_LEVEL_ERROR, 2017 "%s: Ring (%d, %d) missing shadow config", 2018 __func__, ring_type, ring_num); 2019 } 2020 } else { 2021 hal_validate_shadow_register(hal, 2022 SRNG_SRC_ADDR(srng, HP), 2023 srng->u.src_ring.hp_addr); 2024 } 2025 } else { 2026 /* During initialization loop count in all the descriptors 2027 * will be set to zero, and HW will set it to 1 on completing 2028 * descriptor update in first loop, and increments it by 1 on 2029 * subsequent loops (loop count wraps around after reaching 2030 * 0xffff). The 'loop_cnt' in SW ring state is the expected 2031 * loop count in descriptors updated by HW (to be processed 2032 * by SW). 2033 */ 2034 hal_srng_set_nf_thresholds(srng, ring_params); 2035 srng->u.dst_ring.loop_cnt = 1; 2036 srng->u.dst_ring.tp = 0; 2037 srng->u.dst_ring.hp_addr = 2038 &(hal->shadow_rdptr_mem_vaddr[ring_id]); 2039 if (ring_config->lmac_ring) { 2040 /* For LMAC rings, tail pointer updates will be done 2041 * through FW by writing to a shared memory location 2042 */ 2043 srng->u.dst_ring.tp_addr = 2044 &(hal->shadow_wrptr_mem_vaddr[ring_id - 2045 HAL_SRNG_LMAC1_ID_START]); 2046 srng->flags |= HAL_SRNG_LMAC_RING; 2047 } else if (ignore_shadow || srng->u.dst_ring.tp_addr == 0) { 2048 srng->u.dst_ring.tp_addr = 2049 hal_get_window_address(hal, 2050 SRNG_DST_ADDR(srng, TP)); 2051 2052 if (CHECK_SHADOW_REGISTERS) { 2053 QDF_TRACE(QDF_MODULE_ID_TXRX, 2054 QDF_TRACE_LEVEL_ERROR, 2055 "%s: Ring (%d, %d) missing shadow config", 2056 __func__, ring_type, ring_num); 2057 } 2058 } else { 2059 hal_validate_shadow_register(hal, 2060 SRNG_DST_ADDR(srng, TP), 2061 srng->u.dst_ring.tp_addr); 2062 } 2063 } 2064 2065 if (!(ring_config->lmac_ring)) { 2066 hal_srng_hw_init(hal, srng); 2067 2068 if (ring_type == CE_DST) { 2069 srng->u.dst_ring.max_buffer_length = ring_params->max_buffer_length; 2070 hal_ce_dst_setup(hal, srng, ring_num); 2071 } 2072 } 2073 2074 SRNG_LOCK_INIT(&srng->lock); 2075 2076 srng->srng_event = 0; 2077 2078 srng->initialized = true; 2079 2080 return (void *)srng; 2081 } 2082 qdf_export_symbol(hal_srng_setup); 2083 2084 /** 2085 * hal_srng_cleanup - Deinitialize HW SRNG ring. 2086 * @hal_soc: Opaque HAL SOC handle 2087 * @hal_srng: Opaque HAL SRNG pointer 2088 */ 2089 void hal_srng_cleanup(void *hal_soc, hal_ring_handle_t hal_ring_hdl) 2090 { 2091 struct hal_srng *srng = (struct hal_srng *)hal_ring_hdl; 2092 SRNG_LOCK_DESTROY(&srng->lock); 2093 srng->initialized = 0; 2094 } 2095 qdf_export_symbol(hal_srng_cleanup); 2096 2097 /** 2098 * hal_srng_get_entrysize - Returns size of ring entry in bytes 2099 * @hal_soc: Opaque HAL SOC handle 2100 * @ring_type: one of the types from hal_ring_type 2101 * 2102 */ 2103 uint32_t hal_srng_get_entrysize(void *hal_soc, int ring_type) 2104 { 2105 struct hal_soc *hal = (struct hal_soc *)hal_soc; 2106 struct hal_hw_srng_config *ring_config = 2107 HAL_SRNG_CONFIG(hal, ring_type); 2108 return ring_config->entry_size << 2; 2109 } 2110 qdf_export_symbol(hal_srng_get_entrysize); 2111 2112 /** 2113 * hal_srng_max_entries - Returns maximum possible number of ring entries 2114 * @hal_soc: Opaque HAL SOC handle 2115 * @ring_type: one of the types from hal_ring_type 2116 * 2117 * Return: Maximum number of entries for the given ring_type 2118 */ 2119 uint32_t hal_srng_max_entries(void *hal_soc, int ring_type) 2120 { 2121 struct hal_soc *hal = (struct hal_soc *)hal_soc; 2122 struct hal_hw_srng_config *ring_config = 2123 HAL_SRNG_CONFIG(hal, ring_type); 2124 2125 return ring_config->max_size / ring_config->entry_size; 2126 } 2127 qdf_export_symbol(hal_srng_max_entries); 2128 2129 enum hal_srng_dir hal_srng_get_dir(void *hal_soc, int ring_type) 2130 { 2131 struct hal_soc *hal = (struct hal_soc *)hal_soc; 2132 struct hal_hw_srng_config *ring_config = 2133 HAL_SRNG_CONFIG(hal, ring_type); 2134 2135 return ring_config->ring_dir; 2136 } 2137 2138 /** 2139 * hal_srng_dump - Dump ring status 2140 * @srng: hal srng pointer 2141 */ 2142 void hal_srng_dump(struct hal_srng *srng) 2143 { 2144 if (srng->ring_dir == HAL_SRNG_SRC_RING) { 2145 hal_debug("=== SRC RING %d ===", srng->ring_id); 2146 hal_debug("hp %u, reap_hp %u, tp %u, cached tp %u", 2147 srng->u.src_ring.hp, 2148 srng->u.src_ring.reap_hp, 2149 *srng->u.src_ring.tp_addr, 2150 srng->u.src_ring.cached_tp); 2151 } else { 2152 hal_debug("=== DST RING %d ===", srng->ring_id); 2153 hal_debug("tp %u, hp %u, cached tp %u, loop_cnt %u", 2154 srng->u.dst_ring.tp, 2155 *srng->u.dst_ring.hp_addr, 2156 srng->u.dst_ring.cached_hp, 2157 srng->u.dst_ring.loop_cnt); 2158 } 2159 } 2160 2161 /** 2162 * hal_get_srng_params - Retrieve SRNG parameters for a given ring from HAL 2163 * 2164 * @hal_soc: Opaque HAL SOC handle 2165 * @hal_ring: Ring pointer (Source or Destination ring) 2166 * @ring_params: SRNG parameters will be returned through this structure 2167 */ 2168 extern void hal_get_srng_params(hal_soc_handle_t hal_soc_hdl, 2169 hal_ring_handle_t hal_ring_hdl, 2170 struct hal_srng_params *ring_params) 2171 { 2172 struct hal_srng *srng = (struct hal_srng *)hal_ring_hdl; 2173 int i =0; 2174 ring_params->ring_id = srng->ring_id; 2175 ring_params->ring_dir = srng->ring_dir; 2176 ring_params->entry_size = srng->entry_size; 2177 2178 ring_params->ring_base_paddr = srng->ring_base_paddr; 2179 ring_params->ring_base_vaddr = srng->ring_base_vaddr; 2180 ring_params->num_entries = srng->num_entries; 2181 ring_params->msi_addr = srng->msi_addr; 2182 ring_params->msi_data = srng->msi_data; 2183 ring_params->intr_timer_thres_us = srng->intr_timer_thres_us; 2184 ring_params->intr_batch_cntr_thres_entries = 2185 srng->intr_batch_cntr_thres_entries; 2186 ring_params->low_threshold = srng->u.src_ring.low_threshold; 2187 ring_params->flags = srng->flags; 2188 ring_params->ring_id = srng->ring_id; 2189 for (i = 0 ; i < MAX_SRNG_REG_GROUPS; i++) 2190 ring_params->hwreg_base[i] = srng->hwreg_base[i]; 2191 2192 hal_srng_get_nf_params(srng, ring_params); 2193 } 2194 qdf_export_symbol(hal_get_srng_params); 2195 2196 void hal_set_low_threshold(hal_ring_handle_t hal_ring_hdl, 2197 uint32_t low_threshold) 2198 { 2199 struct hal_srng *srng = (struct hal_srng *)hal_ring_hdl; 2200 srng->u.src_ring.low_threshold = low_threshold * srng->entry_size; 2201 } 2202 qdf_export_symbol(hal_set_low_threshold); 2203 2204 2205 #ifdef FORCE_WAKE 2206 void hal_set_init_phase(hal_soc_handle_t soc, bool init_phase) 2207 { 2208 struct hal_soc *hal_soc = (struct hal_soc *)soc; 2209 2210 hal_soc->init_phase = init_phase; 2211 } 2212 #endif /* FORCE_WAKE */ 2213