1 /* 2 * Copyright (c) 2015-2021 The Linux Foundation. All rights reserved. 3 * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved. 4 * 5 * Permission to use, copy, modify, and/or distribute this software for 6 * any purpose with or without fee is hereby granted, provided that the 7 * above copyright notice and this permission notice appear in all 8 * copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL 11 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED 12 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE 13 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 14 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR 15 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 16 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 17 * PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 /** 21 * DOC: hif_napi.c 22 * 23 * HIF NAPI interface implementation 24 */ 25 26 #include <linux/string.h> /* memset */ 27 28 /* Linux headers */ 29 #include <linux/cpumask.h> 30 #include <linux/cpufreq.h> 31 #include <linux/cpu.h> 32 #include <linux/topology.h> 33 #include <linux/interrupt.h> 34 #ifdef CONFIG_SCHED_CORE_CTL 35 #include <linux/sched/core_ctl.h> 36 #endif 37 #include <pld_common.h> 38 #include <linux/pm.h> 39 40 /* Driver headers */ 41 #include <hif_napi.h> 42 #include <hif_debug.h> 43 #include <hif_io32.h> 44 #include <ce_api.h> 45 #include <ce_internal.h> 46 #include <hif_irq_affinity.h> 47 #include "qdf_cpuhp.h" 48 #include "qdf_module.h" 49 #include "qdf_net_if.h" 50 #include "qdf_dev.h" 51 #include "qdf_irq.h" 52 53 enum napi_decision_vector { 54 HIF_NAPI_NOEVENT = 0, 55 HIF_NAPI_INITED = 1, 56 HIF_NAPI_CONF_UP = 2 57 }; 58 #define ENABLE_NAPI_MASK (HIF_NAPI_INITED | HIF_NAPI_CONF_UP) 59 60 #ifdef RECEIVE_OFFLOAD 61 /** 62 * hif_rxthread_napi_poll() - dummy napi poll for rx_thread NAPI 63 * @napi: Rx_thread NAPI 64 * @budget: NAPI BUDGET 65 * 66 * Return: 0 as it is not supposed to be polled at all as it is not scheduled. 67 */ 68 static int hif_rxthread_napi_poll(struct napi_struct *napi, int budget) 69 { 70 hif_err("This napi_poll should not be polled as we don't schedule it"); 71 QDF_ASSERT(0); 72 return 0; 73 } 74 75 /** 76 * hif_init_rx_thread_napi() - Initialize dummy Rx_thread NAPI 77 * @napii: Handle to napi_info holding rx_thread napi 78 * 79 * Return: None 80 */ 81 static void hif_init_rx_thread_napi(struct qca_napi_info *napii) 82 { 83 struct qdf_net_if *nd = (struct qdf_net_if *)&napii->rx_thread_netdev; 84 85 qdf_net_if_create_dummy_if(nd); 86 netif_napi_add(&napii->rx_thread_netdev, &napii->rx_thread_napi, 87 hif_rxthread_napi_poll, 64); 88 napi_enable(&napii->rx_thread_napi); 89 } 90 91 /** 92 * hif_deinit_rx_thread_napi() - Deinitialize dummy Rx_thread NAPI 93 * @napii: Handle to napi_info holding rx_thread napi 94 * 95 * Return: None 96 */ 97 static void hif_deinit_rx_thread_napi(struct qca_napi_info *napii) 98 { 99 netif_napi_del(&napii->rx_thread_napi); 100 } 101 #else /* RECEIVE_OFFLOAD */ 102 static void hif_init_rx_thread_napi(struct qca_napi_info *napii) 103 { 104 } 105 106 static void hif_deinit_rx_thread_napi(struct qca_napi_info *napii) 107 { 108 } 109 #endif 110 111 /** 112 * hif_napi_create() - creates the NAPI structures for a given CE 113 * @hif : pointer to hif context 114 * @pipe_id: the CE id on which the instance will be created 115 * @poll : poll function to be used for this NAPI instance 116 * @budget : budget to be registered with the NAPI instance 117 * @scale : scale factor on the weight (to scaler budget to 1000) 118 * @flags : feature flags 119 * 120 * Description: 121 * Creates NAPI instances. This function is called 122 * unconditionally during initialization. It creates 123 * napi structures through the proper HTC/HIF calls. 124 * The structures are disabled on creation. 125 * Note that for each NAPI instance a separate dummy netdev is used 126 * 127 * Return: 128 * < 0: error 129 * = 0: <should never happen> 130 * > 0: id of the created object (for multi-NAPI, number of objects created) 131 */ 132 int hif_napi_create(struct hif_opaque_softc *hif_ctx, 133 int (*poll)(struct napi_struct *, int), 134 int budget, 135 int scale, 136 uint8_t flags) 137 { 138 int i; 139 struct qca_napi_data *napid; 140 struct qca_napi_info *napii; 141 struct CE_state *ce_state; 142 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 143 int rc = 0; 144 145 NAPI_DEBUG("-->(budget=%d, scale=%d)", 146 budget, scale); 147 NAPI_DEBUG("hif->napi_data.state = 0x%08x", 148 hif->napi_data.state); 149 NAPI_DEBUG("hif->napi_data.ce_map = 0x%08x", 150 hif->napi_data.ce_map); 151 152 napid = &(hif->napi_data); 153 if (0 == (napid->state & HIF_NAPI_INITED)) { 154 memset(napid, 0, sizeof(struct qca_napi_data)); 155 qdf_spinlock_create(&(napid->lock)); 156 157 napid->state |= HIF_NAPI_INITED; 158 napid->flags = flags; 159 160 rc = hif_napi_cpu_init(hif_ctx); 161 if (rc != 0 && rc != -EALREADY) { 162 hif_err("NAPI_initialization failed(rc=%d)", rc); 163 rc = napid->ce_map; 164 goto hnc_err; 165 } else 166 rc = 0; 167 168 hif_debug("NAPI structures initialized, rc=%d", rc); 169 } 170 for (i = 0; i < hif->ce_count; i++) { 171 ce_state = hif->ce_id_to_state[i]; 172 NAPI_DEBUG("ce %d: htt_rx=%d htt_tx=%d", 173 i, ce_state->htt_rx_data, 174 ce_state->htt_tx_data); 175 if (ce_srng_based(hif)) 176 continue; 177 178 if (!ce_state->htt_rx_data) 179 continue; 180 181 /* Now this is a CE where we need NAPI on */ 182 NAPI_DEBUG("Creating NAPI on pipe %d", i); 183 napii = qdf_mem_malloc(sizeof(*napii)); 184 napid->napis[i] = napii; 185 if (!napii) { 186 rc = -ENOMEM; 187 goto napii_free; 188 } 189 } 190 191 for (i = 0; i < hif->ce_count; i++) { 192 napii = napid->napis[i]; 193 if (!napii) 194 continue; 195 196 NAPI_DEBUG("initializing NAPI for pipe %d", i); 197 memset(napii, 0, sizeof(struct qca_napi_info)); 198 napii->scale = scale; 199 napii->id = NAPI_PIPE2ID(i); 200 napii->hif_ctx = hif_ctx; 201 napii->irq = pld_get_irq(hif->qdf_dev->dev, i); 202 203 if (napii->irq < 0) 204 hif_warn("bad IRQ value for CE %d: %d", i, napii->irq); 205 206 qdf_net_if_create_dummy_if((struct qdf_net_if *)&napii->netdev); 207 208 NAPI_DEBUG("adding napi=%pK to netdev=%pK (poll=%pK, bdgt=%d)", 209 &(napii->napi), &(napii->netdev), poll, budget); 210 netif_napi_add(&(napii->netdev), &(napii->napi), poll, budget); 211 212 NAPI_DEBUG("after napi_add"); 213 NAPI_DEBUG("napi=0x%pK, netdev=0x%pK", 214 &(napii->napi), &(napii->netdev)); 215 NAPI_DEBUG("napi.dev_list.prev=0x%pK, next=0x%pK", 216 napii->napi.dev_list.prev, 217 napii->napi.dev_list.next); 218 NAPI_DEBUG("dev.napi_list.prev=0x%pK, next=0x%pK", 219 napii->netdev.napi_list.prev, 220 napii->netdev.napi_list.next); 221 222 hif_init_rx_thread_napi(napii); 223 napii->lro_ctx = qdf_lro_init(); 224 NAPI_DEBUG("Registering LRO for ce_id %d NAPI callback for %d lro_ctx %pK\n", 225 i, napii->id, napii->lro_ctx); 226 227 /* It is OK to change the state variable below without 228 * protection as there should be no-one around yet 229 */ 230 napid->ce_map |= (0x01 << i); 231 hif_debug("NAPI id %d created for pipe %d", napii->id, i); 232 } 233 234 /* no ces registered with the napi */ 235 if (!ce_srng_based(hif) && napid->ce_map == 0) { 236 hif_warn("no napis created for copy engines"); 237 rc = -EFAULT; 238 goto napii_free; 239 } 240 241 NAPI_DEBUG("napi map = %x", napid->ce_map); 242 NAPI_DEBUG("NAPI ids created for all applicable pipes"); 243 return napid->ce_map; 244 245 napii_free: 246 for (i = 0; i < hif->ce_count; i++) { 247 napii = napid->napis[i]; 248 napid->napis[i] = NULL; 249 if (napii) 250 qdf_mem_free(napii); 251 } 252 253 hnc_err: 254 NAPI_DEBUG("<--napi_instances_map=%x]", napid->ce_map); 255 return rc; 256 } 257 qdf_export_symbol(hif_napi_create); 258 259 #ifdef RECEIVE_OFFLOAD 260 void hif_napi_rx_offld_flush_cb_register(struct hif_opaque_softc *hif_hdl, 261 void (offld_flush_handler)(void *)) 262 { 263 int i; 264 struct CE_state *ce_state; 265 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl); 266 struct qca_napi_data *napid; 267 struct qca_napi_info *napii; 268 269 if (!scn) { 270 hif_err("hif_state NULL!"); 271 QDF_ASSERT(0); 272 return; 273 } 274 275 napid = hif_napi_get_all(hif_hdl); 276 for (i = 0; i < scn->ce_count; i++) { 277 ce_state = scn->ce_id_to_state[i]; 278 if (ce_state && (ce_state->htt_rx_data)) { 279 napii = napid->napis[i]; 280 napii->offld_flush_cb = offld_flush_handler; 281 hif_debug("Registering offload for ce_id %d NAPI callback for %d flush_cb %pK", 282 i, napii->id, napii->offld_flush_cb); 283 } 284 } 285 } 286 287 void hif_napi_rx_offld_flush_cb_deregister(struct hif_opaque_softc *hif_hdl) 288 { 289 int i; 290 struct CE_state *ce_state; 291 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl); 292 struct qca_napi_data *napid; 293 struct qca_napi_info *napii; 294 295 if (!scn) { 296 hif_err("hif_state NULL!"); 297 QDF_ASSERT(0); 298 return; 299 } 300 301 napid = hif_napi_get_all(hif_hdl); 302 for (i = 0; i < scn->ce_count; i++) { 303 ce_state = scn->ce_id_to_state[i]; 304 if (ce_state && (ce_state->htt_rx_data)) { 305 napii = napid->napis[i]; 306 hif_debug("deRegistering offld for ce_id %d NAPI callback for %d flush_cb %pK", 307 i, napii->id, napii->offld_flush_cb); 308 /* Not required */ 309 napii->offld_flush_cb = NULL; 310 } 311 } 312 } 313 #endif /* RECEIVE_OFFLOAD */ 314 315 /** 316 * 317 * hif_napi_destroy() - destroys the NAPI structures for a given instance 318 * @hif : pointer to hif context 319 * @ce_id : the CE id whose napi instance will be destroyed 320 * @force : if set, will destroy even if entry is active (de-activates) 321 * 322 * Description: 323 * Destroy a given NAPI instance. This function is called 324 * unconditionally during cleanup. 325 * Refuses to destroy an entry of it is still enabled (unless force=1) 326 * Marks the whole napi_data invalid if all instances are destroyed. 327 * 328 * Return: 329 * -EINVAL: specific entry has not been created 330 * -EPERM : specific entry is still active 331 * 0 < : error 332 * 0 = : success 333 */ 334 int hif_napi_destroy(struct hif_opaque_softc *hif_ctx, 335 uint8_t id, 336 int force) 337 { 338 uint8_t ce = NAPI_ID2PIPE(id); 339 int rc = 0; 340 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 341 342 NAPI_DEBUG("-->(id=%d, force=%d)", id, force); 343 344 if (0 == (hif->napi_data.state & HIF_NAPI_INITED)) { 345 hif_err("NAPI not initialized or entry %d not created", id); 346 rc = -EINVAL; 347 } else if (0 == (hif->napi_data.ce_map & (0x01 << ce))) { 348 hif_err("NAPI instance %d (pipe %d) not created", id, ce); 349 if (hif->napi_data.napis[ce]) 350 hif_err("memory allocated but ce_map not set %d (pipe %d)", 351 id, ce); 352 rc = -EINVAL; 353 } else { 354 struct qca_napi_data *napid; 355 struct qca_napi_info *napii; 356 357 napid = &(hif->napi_data); 358 napii = napid->napis[ce]; 359 if (!napii) { 360 if (napid->ce_map & (0x01 << ce)) 361 hif_err("napii & ce_map out of sync(ce %d)", ce); 362 return -EINVAL; 363 } 364 365 366 if (hif->napi_data.state == HIF_NAPI_CONF_UP) { 367 if (force) { 368 napi_disable(&(napii->napi)); 369 hif_debug("NAPI entry %d force disabled", id); 370 NAPI_DEBUG("NAPI %d force disabled", id); 371 } else { 372 hif_err("Cannot destroy active NAPI %d", id); 373 rc = -EPERM; 374 } 375 } 376 if (0 == rc) { 377 NAPI_DEBUG("before napi_del"); 378 NAPI_DEBUG("napi.dlist.prv=0x%pK, next=0x%pK", 379 napii->napi.dev_list.prev, 380 napii->napi.dev_list.next); 381 NAPI_DEBUG("dev.napi_l.prv=0x%pK, next=0x%pK", 382 napii->netdev.napi_list.prev, 383 napii->netdev.napi_list.next); 384 385 qdf_lro_deinit(napii->lro_ctx); 386 netif_napi_del(&(napii->napi)); 387 hif_deinit_rx_thread_napi(napii); 388 389 napid->ce_map &= ~(0x01 << ce); 390 napid->napis[ce] = NULL; 391 napii->scale = 0; 392 qdf_mem_free(napii); 393 hif_debug("NAPI %d destroyed", id); 394 395 /* if there are no active instances and 396 * if they are all destroyed, 397 * set the whole structure to uninitialized state 398 */ 399 if (napid->ce_map == 0) { 400 rc = hif_napi_cpu_deinit(hif_ctx); 401 /* caller is tolerant to receiving !=0 rc */ 402 403 qdf_spinlock_destroy(&(napid->lock)); 404 memset(napid, 405 0, sizeof(struct qca_napi_data)); 406 hif_debug("no NAPI instances. Zapped"); 407 } 408 } 409 } 410 411 return rc; 412 } 413 qdf_export_symbol(hif_napi_destroy); 414 415 #ifdef FEATURE_LRO 416 void *hif_napi_get_lro_info(struct hif_opaque_softc *hif_hdl, int napi_id) 417 { 418 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl); 419 struct qca_napi_data *napid; 420 struct qca_napi_info *napii; 421 422 napid = &(scn->napi_data); 423 napii = napid->napis[NAPI_ID2PIPE(napi_id)]; 424 425 if (napii) 426 return napii->lro_ctx; 427 return 0; 428 } 429 #endif 430 431 /** 432 * 433 * hif_napi_get_all() - returns the address of the whole HIF NAPI structure 434 * @hif: pointer to hif context 435 * 436 * Description: 437 * Returns the address of the whole structure 438 * 439 * Return: 440 * <addr>: address of the whole HIF NAPI structure 441 */ 442 inline struct qca_napi_data *hif_napi_get_all(struct hif_opaque_softc *hif_ctx) 443 { 444 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 445 446 return &(hif->napi_data); 447 } 448 449 struct qca_napi_info *hif_get_napi(int napi_id, struct qca_napi_data *napid) 450 { 451 int id = NAPI_ID2PIPE(napi_id); 452 453 return napid->napis[id]; 454 } 455 456 /** 457 * 458 * hif_napi_event() - reacts to events that impact NAPI 459 * @hif : pointer to hif context 460 * @evnt: event that has been detected 461 * @data: more data regarding the event 462 * 463 * Description: 464 * This function handles two types of events: 465 * 1- Events that change the state of NAPI (enabled/disabled): 466 * {NAPI_EVT_INI_FILE, NAPI_EVT_CMD_STATE} 467 * The state is retrievable by "hdd_napi_enabled(-1)" 468 * - NAPI will be on if either INI file is on and it has not been disabled 469 * by a subsequent vendor CMD, 470 * or it has been enabled by a vendor CMD. 471 * 2- Events that change the CPU affinity of a NAPI instance/IRQ: 472 * {NAPI_EVT_TPUT_STATE, NAPI_EVT_CPU_STATE} 473 * - NAPI will support a throughput mode (HI/LO), kept at napid->napi_mode 474 * - NAPI will switch throughput mode based on hdd_napi_throughput_policy() 475 * - In LO tput mode, NAPI will yield control if its interrupts to the system 476 * management functions. However in HI throughput mode, NAPI will actively 477 * manage its interrupts/instances (by trying to disperse them out to 478 * separate performance cores). 479 * - CPU eligibility is kept up-to-date by NAPI_EVT_CPU_STATE events. 480 * 481 * + In some cases (roaming peer management is the only case so far), a 482 * a client can trigger a "SERIALIZE" event. Basically, this means that the 483 * users is asking NAPI to go into a truly single execution context state. 484 * So, NAPI indicates to msm-irqbalancer that it wants to be denylisted, 485 * (if called for the first time) and then moves all IRQs (for NAPI 486 * instances) to be collapsed to a single core. If called multiple times, 487 * it will just re-collapse the CPUs. This is because denylist-on() API 488 * is reference-counted, and because the API has already been called. 489 * 490 * Such a user, should call "DESERIALIZE" (NORMAL) event, to set NAPI to go 491 * to its "normal" operation. Optionally, they can give a timeout value (in 492 * multiples of BusBandwidthCheckPeriod -- 100 msecs by default). In this 493 * case, NAPI will just set the current throughput state to uninitialized 494 * and set the delay period. Once policy handler is called, it would skip 495 * applying the policy delay period times, and otherwise apply the policy. 496 * 497 * Return: 498 * < 0: some error 499 * = 0: event handled successfully 500 */ 501 int hif_napi_event(struct hif_opaque_softc *hif_ctx, enum qca_napi_event event, 502 void *data) 503 { 504 int rc = 0; 505 uint32_t prev_state; 506 int i; 507 bool state_changed; 508 struct napi_struct *napi; 509 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 510 struct qca_napi_data *napid = &(hif->napi_data); 511 enum qca_napi_tput_state tput_mode = QCA_NAPI_TPUT_UNINITIALIZED; 512 enum { 513 DENYLIST_NOT_PENDING, 514 DENYLIST_ON_PENDING, 515 DENYLIST_OFF_PENDING 516 } denylist_pending = DENYLIST_NOT_PENDING; 517 518 NAPI_DEBUG("%s: -->(event=%d, aux=%pK)", __func__, event, data); 519 520 if (ce_srng_based(hif)) 521 return hif_exec_event(hif_ctx, event, data); 522 523 if ((napid->state & HIF_NAPI_INITED) == 0) { 524 NAPI_DEBUG("%s: got event when NAPI not initialized", 525 __func__); 526 return -EINVAL; 527 } 528 qdf_spin_lock_bh(&(napid->lock)); 529 prev_state = napid->state; 530 switch (event) { 531 case NAPI_EVT_INI_FILE: 532 case NAPI_EVT_CMD_STATE: 533 case NAPI_EVT_INT_STATE: { 534 int on = (data != ((void *)0)); 535 536 hif_debug("recved evnt: STATE_CMD %d; v = %d (state=0x%0x)", 537 event, on, prev_state); 538 if (on) 539 if (prev_state & HIF_NAPI_CONF_UP) { 540 hif_debug("Duplicate NAPI conf ON msg"); 541 } else { 542 hif_debug("Setting state to ON"); 543 napid->state |= HIF_NAPI_CONF_UP; 544 } 545 else /* off request */ 546 if (prev_state & HIF_NAPI_CONF_UP) { 547 hif_debug("Setting state to OFF"); 548 napid->state &= ~HIF_NAPI_CONF_UP; 549 } else { 550 hif_debug("Duplicate NAPI conf OFF msg"); 551 } 552 break; 553 } 554 /* case NAPI_INIT_FILE/CMD_STATE */ 555 556 case NAPI_EVT_CPU_STATE: { 557 int cpu = ((unsigned long int)data >> 16); 558 int val = ((unsigned long int)data & 0x0ff); 559 560 NAPI_DEBUG("%s: evt=CPU_STATE on CPU %d value=%d", 561 __func__, cpu, val); 562 563 /* state has already been set by hnc_cpu_notify_cb */ 564 if ((val == QCA_NAPI_CPU_DOWN) && 565 (napid->napi_mode == QCA_NAPI_TPUT_HI) && /* we manage */ 566 (napid->napi_cpu[cpu].napis != 0)) { 567 NAPI_DEBUG("%s: Migrating NAPIs out of cpu %d", 568 __func__, cpu); 569 rc = hif_napi_cpu_migrate(napid, 570 cpu, 571 HNC_ACT_RELOCATE); 572 napid->napi_cpu[cpu].napis = 0; 573 } 574 /* in QCA_NAPI_TPUT_LO case, napis MUST == 0 */ 575 break; 576 } 577 578 case NAPI_EVT_TPUT_STATE: { 579 tput_mode = (enum qca_napi_tput_state)data; 580 if (tput_mode == QCA_NAPI_TPUT_LO) { 581 /* from TPUT_HI -> TPUT_LO */ 582 NAPI_DEBUG("%s: Moving to napi_tput_LO state", 583 __func__); 584 denylist_pending = DENYLIST_OFF_PENDING; 585 /* 586 * Ideally we should "collapse" interrupts here, since 587 * we are "dispersing" interrupts in the "else" case. 588 * This allows the possibility that our interrupts may 589 * still be on the perf cluster the next time we enter 590 * high tput mode. However, the irq_balancer is free 591 * to move our interrupts to power cluster once 592 * denylisting has been turned off in the "else" case. 593 */ 594 } else { 595 /* from TPUT_LO -> TPUT->HI */ 596 NAPI_DEBUG("%s: Moving to napi_tput_HI state", 597 __func__); 598 rc = hif_napi_cpu_migrate(napid, 599 HNC_ANY_CPU, 600 HNC_ACT_DISPERSE); 601 602 denylist_pending = DENYLIST_ON_PENDING; 603 } 604 napid->napi_mode = tput_mode; 605 break; 606 } 607 608 case NAPI_EVT_USR_SERIAL: { 609 unsigned long users = (unsigned long)data; 610 611 NAPI_DEBUG("%s: User forced SERIALIZATION; users=%ld", 612 __func__, users); 613 614 rc = hif_napi_cpu_migrate(napid, 615 HNC_ANY_CPU, 616 HNC_ACT_COLLAPSE); 617 if ((users == 0) && (rc == 0)) 618 denylist_pending = DENYLIST_ON_PENDING; 619 break; 620 } 621 case NAPI_EVT_USR_NORMAL: { 622 NAPI_DEBUG("%s: User forced DE-SERIALIZATION", __func__); 623 if (!napid->user_cpu_affin_mask) 624 denylist_pending = DENYLIST_OFF_PENDING; 625 /* 626 * Deserialization timeout is handled at hdd layer; 627 * just mark current mode to uninitialized to ensure 628 * it will be set when the delay is over 629 */ 630 napid->napi_mode = QCA_NAPI_TPUT_UNINITIALIZED; 631 break; 632 } 633 default: { 634 hif_err("Unknown event: %d (data=0x%0lx)", 635 event, (unsigned long) data); 636 break; 637 } /* default */ 638 }; /* switch */ 639 640 641 switch (denylist_pending) { 642 case DENYLIST_ON_PENDING: 643 /* assume the control of WLAN IRQs */ 644 hif_napi_cpu_denylist(napid, DENYLIST_ON); 645 break; 646 case DENYLIST_OFF_PENDING: 647 /* yield the control of WLAN IRQs */ 648 hif_napi_cpu_denylist(napid, DENYLIST_OFF); 649 break; 650 default: /* nothing to do */ 651 break; 652 } /* switch denylist_pending */ 653 654 /* we want to perform the comparison in lock: 655 * there is a possibility of hif_napi_event get called 656 * from two different contexts (driver unload and cpu hotplug 657 * notification) and napid->state get changed 658 * in driver unload context and can lead to race condition 659 * in cpu hotplug context. Therefore, perform the napid->state 660 * comparison before releasing lock. 661 */ 662 state_changed = (prev_state != napid->state); 663 qdf_spin_unlock_bh(&(napid->lock)); 664 665 if (state_changed) { 666 if (napid->state == ENABLE_NAPI_MASK) { 667 rc = 1; 668 for (i = 0; i < CE_COUNT_MAX; i++) { 669 struct qca_napi_info *napii = napid->napis[i]; 670 if (napii) { 671 napi = &(napii->napi); 672 NAPI_DEBUG("%s: enabling NAPI %d", 673 __func__, i); 674 napi_enable(napi); 675 } 676 } 677 } else { 678 rc = 0; 679 for (i = 0; i < CE_COUNT_MAX; i++) { 680 struct qca_napi_info *napii = napid->napis[i]; 681 if (napii) { 682 napi = &(napii->napi); 683 NAPI_DEBUG("%s: disabling NAPI %d", 684 __func__, i); 685 napi_disable(napi); 686 /* in case it is affined, remove it */ 687 qdf_dev_set_irq_affinity(napii->irq, 688 NULL); 689 } 690 } 691 } 692 } else { 693 hif_debug("no change in hif napi state (still %d)", prev_state); 694 } 695 696 NAPI_DEBUG("<--[rc=%d]", rc); 697 return rc; 698 } 699 qdf_export_symbol(hif_napi_event); 700 701 /** 702 * hif_napi_enabled() - checks whether NAPI is enabled for given ce or not 703 * @hif: hif context 704 * @ce : CE instance (or -1, to check if any CEs are enabled) 705 * 706 * Return: bool 707 */ 708 int hif_napi_enabled(struct hif_opaque_softc *hif_ctx, int ce) 709 { 710 int rc; 711 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 712 713 if (-1 == ce) 714 rc = ((hif->napi_data.state == ENABLE_NAPI_MASK)); 715 else 716 rc = ((hif->napi_data.state == ENABLE_NAPI_MASK) && 717 (hif->napi_data.ce_map & (0x01 << ce))); 718 return rc; 719 } 720 qdf_export_symbol(hif_napi_enabled); 721 722 /** 723 * hif_napi_created() - checks whether NAPI is created for given ce or not 724 * @hif: hif context 725 * @ce : CE instance 726 * 727 * Return: bool 728 */ 729 bool hif_napi_created(struct hif_opaque_softc *hif_ctx, int ce) 730 { 731 int rc; 732 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 733 734 rc = (hif->napi_data.ce_map & (0x01 << ce)); 735 736 return !!rc; 737 } 738 qdf_export_symbol(hif_napi_created); 739 740 /** 741 * hif_napi_enable_irq() - enables bus interrupts after napi_complete 742 * 743 * @hif: hif context 744 * @id : id of NAPI instance calling this (used to determine the CE) 745 * 746 * Return: void 747 */ 748 inline void hif_napi_enable_irq(struct hif_opaque_softc *hif, int id) 749 { 750 struct hif_softc *scn = HIF_GET_SOFTC(hif); 751 752 hif_irq_enable(scn, NAPI_ID2PIPE(id)); 753 } 754 755 756 /** 757 * hif_napi_schedule() - schedules napi, updates stats 758 * @scn: hif context 759 * @ce_id: index of napi instance 760 * 761 * Return: false if napi didn't enable or already scheduled, otherwise true 762 */ 763 bool hif_napi_schedule(struct hif_opaque_softc *hif_ctx, int ce_id) 764 { 765 int cpu = smp_processor_id(); 766 struct hif_softc *scn = HIF_GET_SOFTC(hif_ctx); 767 struct qca_napi_info *napii; 768 769 napii = scn->napi_data.napis[ce_id]; 770 if (qdf_unlikely(!napii)) { 771 hif_err("scheduling unallocated napi (ce:%d)", ce_id); 772 qdf_atomic_dec(&scn->active_tasklet_cnt); 773 return false; 774 } 775 776 if (test_bit(NAPI_STATE_SCHED, &napii->napi.state)) { 777 NAPI_DEBUG("napi scheduled, return"); 778 qdf_atomic_dec(&scn->active_tasklet_cnt); 779 return false; 780 } 781 782 hif_record_ce_desc_event(scn, ce_id, NAPI_SCHEDULE, 783 NULL, NULL, 0, 0); 784 napii->stats[cpu].napi_schedules++; 785 NAPI_DEBUG("scheduling napi %d (ce:%d)", napii->id, ce_id); 786 napi_schedule(&(napii->napi)); 787 788 return true; 789 } 790 qdf_export_symbol(hif_napi_schedule); 791 792 /** 793 * hif_napi_correct_cpu() - correct the interrupt affinity for napi if needed 794 * @napi_info: pointer to qca_napi_info for the napi instance 795 * 796 * Return: true => interrupt already on correct cpu, no correction needed 797 * false => interrupt on wrong cpu, correction done for cpu affinity 798 * of the interrupt 799 */ 800 static inline 801 bool hif_napi_correct_cpu(struct qca_napi_info *napi_info) 802 { 803 bool right_cpu = true; 804 int rc = 0; 805 int cpu; 806 struct qca_napi_data *napid; 807 QDF_STATUS ret; 808 809 napid = hif_napi_get_all(GET_HIF_OPAQUE_HDL(napi_info->hif_ctx)); 810 811 if (napid->flags & QCA_NAPI_FEATURE_CPU_CORRECTION) { 812 813 cpu = qdf_get_cpu(); 814 if (unlikely((hif_napi_cpu_denylist(napid, 815 DENYLIST_QUERY) > 0) && 816 cpu != napi_info->cpu)) { 817 right_cpu = false; 818 819 NAPI_DEBUG("interrupt on wrong CPU, correcting"); 820 napi_info->cpumask.bits[0] = (0x01 << napi_info->cpu); 821 822 qdf_dev_modify_irq_status(napi_info->irq, 823 QDF_IRQ_NO_BALANCING, 0); 824 ret = qdf_dev_set_irq_affinity(napi_info->irq, 825 (struct qdf_cpu_mask *) 826 &napi_info->cpumask); 827 rc = qdf_status_to_os_return(ret); 828 qdf_dev_modify_irq_status(napi_info->irq, 0, 829 QDF_IRQ_NO_BALANCING); 830 831 if (rc) 832 hif_err("Setting irq affinity hint: %d", rc); 833 else 834 napi_info->stats[cpu].cpu_corrected++; 835 } 836 } 837 return right_cpu; 838 } 839 840 #ifdef RECEIVE_OFFLOAD 841 /** 842 * hif_napi_offld_flush_cb() - Call upper layer flush callback 843 * @napi_info: Handle to hif_napi_info 844 * 845 * Return: None 846 */ 847 static void hif_napi_offld_flush_cb(struct qca_napi_info *napi_info) 848 { 849 if (napi_info->offld_flush_cb) 850 napi_info->offld_flush_cb(napi_info); 851 } 852 #else 853 static void hif_napi_offld_flush_cb(struct qca_napi_info *napi_info) 854 { 855 } 856 #endif 857 858 /** 859 * hif_napi_poll() - NAPI poll routine 860 * @napi : pointer to NAPI struct as kernel holds it 861 * @budget: 862 * 863 * This is the body of the poll function. 864 * The poll function is called by kernel. So, there is a wrapper 865 * function in HDD, which in turn calls this function. 866 * Two main reasons why the whole thing is not implemented in HDD: 867 * a) references to things like ce_service that HDD is not aware of 868 * b) proximity to the implementation of ce_tasklet, which the body 869 * of this function should be very close to. 870 * 871 * NOTE TO THE MAINTAINER: 872 * Consider this function and ce_tasklet very tightly coupled pairs. 873 * Any changes to ce_tasklet or this function may likely need to be 874 * reflected in the counterpart. 875 * 876 * Returns: 877 * int: the amount of work done in this poll (<= budget) 878 */ 879 int hif_napi_poll(struct hif_opaque_softc *hif_ctx, 880 struct napi_struct *napi, 881 int budget) 882 { 883 int rc = 0; /* default: no work done, also takes care of error */ 884 int normalized = 0; 885 int bucket; 886 int cpu = smp_processor_id(); 887 bool poll_on_right_cpu; 888 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 889 struct qca_napi_info *napi_info; 890 struct CE_state *ce_state = NULL; 891 892 if (unlikely(!hif)) { 893 hif_err("hif context is NULL"); 894 QDF_ASSERT(0); 895 goto out; 896 } 897 898 napi_info = (struct qca_napi_info *) 899 container_of(napi, struct qca_napi_info, napi); 900 901 NAPI_DEBUG("%s -->(napi(%d, irq=%d), budget=%d)", 902 __func__, napi_info->id, napi_info->irq, budget); 903 904 napi_info->stats[cpu].napi_polls++; 905 906 hif_record_ce_desc_event(hif, NAPI_ID2PIPE(napi_info->id), 907 NAPI_POLL_ENTER, NULL, NULL, cpu, 0); 908 909 rc = ce_per_engine_service(hif, NAPI_ID2PIPE(napi_info->id)); 910 NAPI_DEBUG("%s: ce_per_engine_service processed %d msgs", 911 __func__, rc); 912 913 hif_napi_offld_flush_cb(napi_info); 914 915 /* do not return 0, if there was some work done, 916 * even if it is below the scale 917 */ 918 if (rc) { 919 napi_info->stats[cpu].napi_workdone += rc; 920 normalized = (rc / napi_info->scale); 921 if (normalized == 0) 922 normalized++; 923 bucket = (normalized - 1) / 924 (QCA_NAPI_BUDGET / QCA_NAPI_NUM_BUCKETS); 925 if (bucket >= QCA_NAPI_NUM_BUCKETS) { 926 bucket = QCA_NAPI_NUM_BUCKETS - 1; 927 hif_err("Bad bucket#(%d) > QCA_NAPI_NUM_BUCKETS(%d)" 928 " normalized %d, napi budget %d", 929 bucket, QCA_NAPI_NUM_BUCKETS, 930 normalized, QCA_NAPI_BUDGET); 931 } 932 napi_info->stats[cpu].napi_budget_uses[bucket]++; 933 } else { 934 /* if ce_per engine reports 0, then poll should be terminated */ 935 NAPI_DEBUG("%s:%d: nothing processed by CE. Completing NAPI", 936 __func__, __LINE__); 937 } 938 939 ce_state = hif->ce_id_to_state[NAPI_ID2PIPE(napi_info->id)]; 940 941 /* 942 * Not using the API hif_napi_correct_cpu directly in the if statement 943 * below since the API may not get evaluated if put at the end if any 944 * prior condition would evaluate to be true. The CPU correction 945 * check should kick in every poll. 946 */ 947 #ifdef NAPI_YIELD_BUDGET_BASED 948 if (ce_state && (ce_state->force_break || 0 == rc)) { 949 #else 950 poll_on_right_cpu = hif_napi_correct_cpu(napi_info); 951 if ((ce_state) && 952 (!ce_check_rx_pending(ce_state) || (0 == rc) || 953 !poll_on_right_cpu)) { 954 #endif 955 napi_info->stats[cpu].napi_completes++; 956 #ifdef NAPI_YIELD_BUDGET_BASED 957 ce_state->force_break = 0; 958 #endif 959 960 hif_record_ce_desc_event(hif, ce_state->id, NAPI_COMPLETE, 961 NULL, NULL, 0, 0); 962 if (normalized >= budget) 963 normalized = budget - 1; 964 965 napi_complete(napi); 966 /* enable interrupts */ 967 hif_napi_enable_irq(hif_ctx, napi_info->id); 968 /* support suspend/resume */ 969 qdf_atomic_dec(&(hif->active_tasklet_cnt)); 970 971 NAPI_DEBUG("%s:%d: napi_complete + enabling the interrupts", 972 __func__, __LINE__); 973 } else { 974 /* 4.4 kernel NAPI implementation requires drivers to 975 * return full work when they ask to be re-scheduled, 976 * or napi_complete and re-start with a fresh interrupt 977 */ 978 normalized = budget; 979 } 980 981 hif_record_ce_desc_event(hif, NAPI_ID2PIPE(napi_info->id), 982 NAPI_POLL_EXIT, NULL, NULL, normalized, 0); 983 984 NAPI_DEBUG("%s <--[normalized=%d]", __func__, normalized); 985 return normalized; 986 out: 987 return rc; 988 } 989 qdf_export_symbol(hif_napi_poll); 990 991 void hif_update_napi_max_poll_time(struct CE_state *ce_state, 992 int ce_id, 993 int cpu_id) 994 { 995 struct hif_softc *hif; 996 struct qca_napi_info *napi_info; 997 unsigned long long napi_poll_time = qdf_time_sched_clock() - 998 ce_state->ce_service_start_time; 999 1000 hif = ce_state->scn; 1001 napi_info = hif->napi_data.napis[ce_id]; 1002 if (napi_poll_time > 1003 napi_info->stats[cpu_id].napi_max_poll_time) 1004 napi_info->stats[cpu_id].napi_max_poll_time = napi_poll_time; 1005 } 1006 qdf_export_symbol(hif_update_napi_max_poll_time); 1007 1008 #ifdef HIF_IRQ_AFFINITY 1009 /** 1010 * 1011 * hif_napi_update_yield_stats() - update NAPI yield related stats 1012 * @cpu_id: CPU ID for which stats needs to be updates 1013 * @ce_id: Copy Engine ID for which yield stats needs to be updates 1014 * @time_limit_reached: indicates whether the time limit was reached 1015 * @rxpkt_thresh_reached: indicates whether rx packet threshold was reached 1016 * 1017 * Return: None 1018 */ 1019 void hif_napi_update_yield_stats(struct CE_state *ce_state, 1020 bool time_limit_reached, 1021 bool rxpkt_thresh_reached) 1022 { 1023 struct hif_softc *hif; 1024 struct qca_napi_data *napi_data = NULL; 1025 int ce_id = 0; 1026 int cpu_id = 0; 1027 1028 if (unlikely(!ce_state)) { 1029 QDF_ASSERT(ce_state); 1030 return; 1031 } 1032 1033 hif = ce_state->scn; 1034 1035 if (unlikely(!hif)) { 1036 QDF_ASSERT(hif); 1037 return; 1038 } 1039 napi_data = &(hif->napi_data); 1040 if (unlikely(!napi_data)) { 1041 QDF_ASSERT(napi_data); 1042 return; 1043 } 1044 1045 ce_id = ce_state->id; 1046 cpu_id = qdf_get_cpu(); 1047 1048 if (unlikely(!napi_data->napis[ce_id])) { 1049 return; 1050 } 1051 1052 if (time_limit_reached) 1053 napi_data->napis[ce_id]->stats[cpu_id].time_limit_reached++; 1054 else 1055 napi_data->napis[ce_id]->stats[cpu_id].rxpkt_thresh_reached++; 1056 1057 hif_update_napi_max_poll_time(ce_state, ce_id, 1058 cpu_id); 1059 } 1060 1061 /** 1062 * 1063 * hif_napi_stats() - display NAPI CPU statistics 1064 * @napid: pointer to qca_napi_data 1065 * 1066 * Description: 1067 * Prints the various CPU cores on which the NAPI instances /CEs interrupts 1068 * are being executed. Can be called from outside NAPI layer. 1069 * 1070 * Return: None 1071 */ 1072 void hif_napi_stats(struct qca_napi_data *napid) 1073 { 1074 int i; 1075 struct qca_napi_cpu *cpu; 1076 1077 if (!napid) { 1078 qdf_debug("%s: napiid struct is null", __func__); 1079 return; 1080 } 1081 1082 cpu = napid->napi_cpu; 1083 qdf_debug("NAPI CPU TABLE"); 1084 qdf_debug("lilclhead=%d, bigclhead=%d", 1085 napid->lilcl_head, napid->bigcl_head); 1086 for (i = 0; i < NR_CPUS; i++) { 1087 qdf_debug("CPU[%02d]: state:%d crid=%02d clid=%02d crmk:0x%0lx thmk:0x%0lx frq:%d napi = 0x%08x lnk:%d", 1088 i, 1089 cpu[i].state, cpu[i].core_id, cpu[i].cluster_id, 1090 cpu[i].core_mask.bits[0], 1091 cpu[i].thread_mask.bits[0], 1092 cpu[i].max_freq, cpu[i].napis, 1093 cpu[i].cluster_nxt); 1094 } 1095 } 1096 1097 #ifdef FEATURE_NAPI_DEBUG 1098 /* 1099 * Local functions 1100 * - no argument checks, all internal/trusted callers 1101 */ 1102 static void hnc_dump_cpus(struct qca_napi_data *napid) 1103 { 1104 hif_napi_stats(napid); 1105 } 1106 #else 1107 static void hnc_dump_cpus(struct qca_napi_data *napid) { /* no-op */ }; 1108 #endif /* FEATURE_NAPI_DEBUG */ 1109 /** 1110 * hnc_link_clusters() - partitions to cpu table into clusters 1111 * @napid: pointer to NAPI data 1112 * 1113 * Takes in a CPU topology table and builds two linked lists 1114 * (big cluster cores, list-head at bigcl_head, and little cluster 1115 * cores, list-head at lilcl_head) out of it. 1116 * 1117 * If there are more than two clusters: 1118 * - bigcl_head and lilcl_head will be different, 1119 * - the cluster with highest cpufreq will be considered the "big" cluster. 1120 * If there are more than one with the highest frequency, the *last* of such 1121 * clusters will be designated as the "big cluster" 1122 * - the cluster with lowest cpufreq will be considered the "li'l" cluster. 1123 * If there are more than one clusters with the lowest cpu freq, the *first* 1124 * of such clusters will be designated as the "little cluster" 1125 * - We only support up to 32 clusters 1126 * Return: 0 : OK 1127 * !0: error (at least one of lil/big clusters could not be found) 1128 */ 1129 #define HNC_MIN_CLUSTER 0 1130 #define HNC_MAX_CLUSTER 1 1131 static int hnc_link_clusters(struct qca_napi_data *napid) 1132 { 1133 int rc = 0; 1134 1135 int i; 1136 int it = 0; 1137 uint32_t cl_done = 0x0; 1138 int cl, curcl, curclhead = 0; 1139 int more; 1140 unsigned int lilfrq = INT_MAX; 1141 unsigned int bigfrq = 0; 1142 unsigned int clfrq = 0; 1143 int prev = 0; 1144 struct qca_napi_cpu *cpus = napid->napi_cpu; 1145 1146 napid->lilcl_head = napid->bigcl_head = -1; 1147 1148 do { 1149 more = 0; 1150 it++; curcl = -1; 1151 for (i = 0; i < NR_CPUS; i++) { 1152 cl = cpus[i].cluster_id; 1153 NAPI_DEBUG("Processing cpu[%d], cluster=%d\n", 1154 i, cl); 1155 if ((cl < HNC_MIN_CLUSTER) || (cl > HNC_MAX_CLUSTER)) { 1156 NAPI_DEBUG("Bad cluster (%d). SKIPPED\n", cl); 1157 /* continue if ASSERTs are disabled */ 1158 continue; 1159 }; 1160 if (cpumask_weight(&(cpus[i].core_mask)) == 0) { 1161 NAPI_DEBUG("Core mask 0. SKIPPED\n"); 1162 continue; 1163 } 1164 if (cl_done & (0x01 << cl)) { 1165 NAPI_DEBUG("Cluster already processed. SKIPPED\n"); 1166 continue; 1167 } else { 1168 if (more == 0) { 1169 more = 1; 1170 curcl = cl; 1171 curclhead = i; /* row */ 1172 clfrq = cpus[i].max_freq; 1173 prev = -1; 1174 }; 1175 if ((curcl >= 0) && (curcl != cl)) { 1176 NAPI_DEBUG("Entry cl(%d) != curcl(%d). SKIPPED\n", 1177 cl, curcl); 1178 continue; 1179 } 1180 if (cpus[i].max_freq != clfrq) 1181 NAPI_DEBUG("WARN: frq(%d)!=clfrq(%d)\n", 1182 cpus[i].max_freq, clfrq); 1183 if (clfrq >= bigfrq) { 1184 bigfrq = clfrq; 1185 napid->bigcl_head = curclhead; 1186 NAPI_DEBUG("bigcl=%d\n", curclhead); 1187 } 1188 if (clfrq < lilfrq) { 1189 lilfrq = clfrq; 1190 napid->lilcl_head = curclhead; 1191 NAPI_DEBUG("lilcl=%d\n", curclhead); 1192 } 1193 if (prev != -1) 1194 cpus[prev].cluster_nxt = i; 1195 1196 prev = i; 1197 } 1198 } 1199 if (curcl >= 0) 1200 cl_done |= (0x01 << curcl); 1201 1202 } while (more); 1203 1204 if (qdf_unlikely((napid->lilcl_head < 0) && (napid->bigcl_head < 0))) 1205 rc = -EFAULT; 1206 1207 hnc_dump_cpus(napid); /* if NAPI_DEBUG */ 1208 return rc; 1209 } 1210 #undef HNC_MIN_CLUSTER 1211 #undef HNC_MAX_CLUSTER 1212 1213 /* 1214 * hotplug function group 1215 */ 1216 1217 /** 1218 * hnc_cpu_online_cb() - handles CPU hotplug "up" events 1219 * @context: the associated HIF context 1220 * @cpu: the CPU Id of the CPU the event happened on 1221 * 1222 * Return: None 1223 */ 1224 static void hnc_cpu_online_cb(void *context, uint32_t cpu) 1225 { 1226 struct hif_softc *hif = context; 1227 struct qca_napi_data *napid = &hif->napi_data; 1228 1229 if (cpu >= NR_CPUS) 1230 return; 1231 1232 NAPI_DEBUG("-->%s(act=online, cpu=%u)", __func__, cpu); 1233 1234 napid->napi_cpu[cpu].state = QCA_NAPI_CPU_UP; 1235 NAPI_DEBUG("%s: CPU %u marked %d", 1236 __func__, cpu, napid->napi_cpu[cpu].state); 1237 1238 NAPI_DEBUG("<--%s", __func__); 1239 } 1240 1241 /** 1242 * hnc_cpu_before_offline_cb() - handles CPU hotplug "prepare down" events 1243 * @context: the associated HIF context 1244 * @cpu: the CPU Id of the CPU the event happened on 1245 * 1246 * On transition to offline, we act on PREP events, because we may need to move 1247 * the irqs/NAPIs to another CPU before it is actually off-lined. 1248 * 1249 * Return: None 1250 */ 1251 static void hnc_cpu_before_offline_cb(void *context, uint32_t cpu) 1252 { 1253 struct hif_softc *hif = context; 1254 struct qca_napi_data *napid = &hif->napi_data; 1255 1256 if (cpu >= NR_CPUS) 1257 return; 1258 1259 NAPI_DEBUG("-->%s(act=before_offline, cpu=%u)", __func__, cpu); 1260 1261 napid->napi_cpu[cpu].state = QCA_NAPI_CPU_DOWN; 1262 1263 NAPI_DEBUG("%s: CPU %u marked %d; updating affinity", 1264 __func__, cpu, napid->napi_cpu[cpu].state); 1265 1266 /** 1267 * we need to move any NAPIs on this CPU out. 1268 * if we are in LO throughput mode, then this is valid 1269 * if the CPU is the the low designated CPU. 1270 */ 1271 hif_napi_event(GET_HIF_OPAQUE_HDL(hif), 1272 NAPI_EVT_CPU_STATE, 1273 (void *) 1274 ((size_t)cpu << 16 | napid->napi_cpu[cpu].state)); 1275 1276 NAPI_DEBUG("<--%s", __func__); 1277 } 1278 1279 static int hnc_hotplug_register(struct hif_softc *hif_sc) 1280 { 1281 QDF_STATUS status; 1282 1283 NAPI_DEBUG("-->%s", __func__); 1284 1285 status = qdf_cpuhp_register(&hif_sc->napi_data.cpuhp_handler, 1286 hif_sc, 1287 hnc_cpu_online_cb, 1288 hnc_cpu_before_offline_cb); 1289 1290 NAPI_DEBUG("<--%s [%d]", __func__, status); 1291 1292 return qdf_status_to_os_return(status); 1293 } 1294 1295 static void hnc_hotplug_unregister(struct hif_softc *hif_sc) 1296 { 1297 NAPI_DEBUG("-->%s", __func__); 1298 1299 if (hif_sc->napi_data.cpuhp_handler) 1300 qdf_cpuhp_unregister(&hif_sc->napi_data.cpuhp_handler); 1301 1302 NAPI_DEBUG("<--%s", __func__); 1303 } 1304 1305 /** 1306 * hnc_install_tput() - installs a callback in the throughput detector 1307 * @register: !0 => register; =0: unregister 1308 * 1309 * installs a callback to be called when wifi driver throughput (tx+rx) 1310 * crosses a threshold. Currently, we are using the same criteria as 1311 * TCP ack suppression (500 packets/100ms by default). 1312 * 1313 * Return: 0 : success 1314 * <0: failure 1315 */ 1316 1317 static int hnc_tput_hook(int install) 1318 { 1319 int rc = 0; 1320 1321 /* 1322 * Nothing, until the bw_calculation accepts registration 1323 * it is now hardcoded in the wlan_hdd_main.c::hdd_bus_bw_compute_cbk 1324 * hdd_napi_throughput_policy(...) 1325 */ 1326 return rc; 1327 } 1328 1329 /* 1330 * Implementation of hif_napi_cpu API 1331 */ 1332 1333 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) 1334 static inline void record_sibling_cpumask(struct qca_napi_cpu *cpus, int i) 1335 { 1336 cpumask_copy(&(cpus[i].thread_mask), 1337 topology_sibling_cpumask(i)); 1338 } 1339 #else 1340 static inline void record_sibling_cpumask(struct qca_napi_cpu *cpus, int i) 1341 { 1342 } 1343 #endif 1344 1345 1346 /** 1347 * hif_napi_cpu_init() - initialization of irq affinity block 1348 * @ctx: pointer to qca_napi_data 1349 * 1350 * called by hif_napi_create, after the first instance is called 1351 * - builds napi_rss_cpus table from cpu topology 1352 * - links cores of the same clusters together 1353 * - installs hot-plug notifier 1354 * - installs throughput trigger notifier (when such mechanism exists) 1355 * 1356 * Return: 0: OK 1357 * <0: error code 1358 */ 1359 int hif_napi_cpu_init(struct hif_opaque_softc *hif) 1360 { 1361 int rc = 0; 1362 int i; 1363 struct qca_napi_data *napid = &HIF_GET_SOFTC(hif)->napi_data; 1364 struct qca_napi_cpu *cpus = napid->napi_cpu; 1365 1366 NAPI_DEBUG("--> "); 1367 1368 if (cpus[0].state != QCA_NAPI_CPU_UNINITIALIZED) { 1369 NAPI_DEBUG("NAPI RSS table already initialized.\n"); 1370 rc = -EALREADY; 1371 goto lab_rss_init; 1372 } 1373 1374 /* build CPU topology table */ 1375 for_each_possible_cpu(i) { 1376 cpus[i].state = ((cpumask_test_cpu(i, cpu_online_mask) 1377 ? QCA_NAPI_CPU_UP 1378 : QCA_NAPI_CPU_DOWN)); 1379 cpus[i].core_id = topology_core_id(i); 1380 cpus[i].cluster_id = topology_physical_package_id(i); 1381 cpumask_copy(&(cpus[i].core_mask), 1382 topology_core_cpumask(i)); 1383 record_sibling_cpumask(cpus, i); 1384 cpus[i].max_freq = cpufreq_quick_get_max(i); 1385 cpus[i].napis = 0x0; 1386 cpus[i].cluster_nxt = -1; /* invalid */ 1387 } 1388 1389 /* link clusters together */ 1390 rc = hnc_link_clusters(napid); 1391 if (0 != rc) 1392 goto lab_err_topology; 1393 1394 /* install hotplug notifier */ 1395 rc = hnc_hotplug_register(HIF_GET_SOFTC(hif)); 1396 if (0 != rc) 1397 goto lab_err_hotplug; 1398 1399 /* install throughput notifier */ 1400 rc = hnc_tput_hook(1); 1401 if (0 == rc) 1402 goto lab_rss_init; 1403 1404 lab_err_hotplug: 1405 hnc_tput_hook(0); 1406 hnc_hotplug_unregister(HIF_GET_SOFTC(hif)); 1407 lab_err_topology: 1408 memset(napid->napi_cpu, 0, sizeof(struct qca_napi_cpu) * NR_CPUS); 1409 lab_rss_init: 1410 NAPI_DEBUG("<-- [rc=%d]", rc); 1411 return rc; 1412 } 1413 1414 /** 1415 * hif_napi_cpu_deinit() - clean-up of irq affinity block 1416 * 1417 * called by hif_napi_destroy, when the last instance is removed 1418 * - uninstalls throughput and hotplug notifiers 1419 * - clears cpu topology table 1420 * Return: 0: OK 1421 */ 1422 int hif_napi_cpu_deinit(struct hif_opaque_softc *hif) 1423 { 1424 int rc = 0; 1425 struct qca_napi_data *napid = &HIF_GET_SOFTC(hif)->napi_data; 1426 1427 NAPI_DEBUG("-->%s(...)", __func__); 1428 1429 /* uninstall tput notifier */ 1430 rc = hnc_tput_hook(0); 1431 1432 /* uninstall hotplug notifier */ 1433 hnc_hotplug_unregister(HIF_GET_SOFTC(hif)); 1434 1435 /* clear the topology table */ 1436 memset(napid->napi_cpu, 0, sizeof(struct qca_napi_cpu) * NR_CPUS); 1437 1438 NAPI_DEBUG("<--%s[rc=%d]", __func__, rc); 1439 1440 return rc; 1441 } 1442 1443 /** 1444 * hncm_migrate_to() - migrates a NAPI to a CPU 1445 * @napid: pointer to NAPI block 1446 * @ce_id: CE_id of the NAPI instance 1447 * @didx : index in the CPU topology table for the CPU to migrate to 1448 * 1449 * Migrates NAPI (identified by the CE_id) to the destination core 1450 * Updates the napi_map of the destination entry 1451 * 1452 * Return: 1453 * =0 : success 1454 * <0 : error 1455 */ 1456 static int hncm_migrate_to(struct qca_napi_data *napid, 1457 int napi_ce, 1458 int didx) 1459 { 1460 int rc = 0; 1461 QDF_STATUS status; 1462 1463 NAPI_DEBUG("-->%s(napi_cd=%d, didx=%d)", __func__, napi_ce, didx); 1464 1465 if (!napid->napis[napi_ce]) 1466 return -EINVAL; 1467 1468 napid->napis[napi_ce]->cpumask.bits[0] = (1 << didx); 1469 1470 qdf_dev_modify_irq_status(napid->napis[napi_ce]->irq, 1471 QDF_IRQ_NO_BALANCING, 0); 1472 status = qdf_dev_set_irq_affinity(napid->napis[napi_ce]->irq, 1473 (struct qdf_cpu_mask *) 1474 &napid->napis[napi_ce]->cpumask); 1475 rc = qdf_status_to_os_return(status); 1476 1477 /* unmark the napis bitmap in the cpu table */ 1478 napid->napi_cpu[napid->napis[napi_ce]->cpu].napis &= ~(0x01 << napi_ce); 1479 /* mark the napis bitmap for the new designated cpu */ 1480 napid->napi_cpu[didx].napis |= (0x01 << napi_ce); 1481 napid->napis[napi_ce]->cpu = didx; 1482 1483 NAPI_DEBUG("<--%s[%d]", __func__, rc); 1484 return rc; 1485 } 1486 /** 1487 * hncm_dest_cpu() - finds a destination CPU for NAPI 1488 * @napid: pointer to NAPI block 1489 * @act : RELOCATE | COLLAPSE | DISPERSE 1490 * 1491 * Finds the designated destination for the next IRQ. 1492 * RELOCATE: translated to either COLLAPSE or DISPERSE based 1493 * on napid->napi_mode (throughput state) 1494 * COLLAPSE: All have the same destination: the first online CPU in lilcl 1495 * DISPERSE: One of the CPU in bigcl, which has the smallest number of 1496 * NAPIs on it 1497 * 1498 * Return: >=0 : index in the cpu topology table 1499 * : < 0 : error 1500 */ 1501 static int hncm_dest_cpu(struct qca_napi_data *napid, int act) 1502 { 1503 int destidx = -1; 1504 int head, i; 1505 1506 NAPI_DEBUG("-->%s(act=%d)", __func__, act); 1507 if (act == HNC_ACT_RELOCATE) { 1508 if (napid->napi_mode == QCA_NAPI_TPUT_LO) 1509 act = HNC_ACT_COLLAPSE; 1510 else 1511 act = HNC_ACT_DISPERSE; 1512 NAPI_DEBUG("%s: act changed from HNC_ACT_RELOCATE to %d", 1513 __func__, act); 1514 } 1515 if (act == HNC_ACT_COLLAPSE) { 1516 head = i = napid->lilcl_head; 1517 retry_collapse: 1518 while (i >= 0) { 1519 if (napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) { 1520 destidx = i; 1521 break; 1522 } 1523 i = napid->napi_cpu[i].cluster_nxt; 1524 } 1525 if ((destidx < 0) && (head == napid->lilcl_head)) { 1526 NAPI_DEBUG("%s: COLLAPSE: no lilcl dest, try bigcl", 1527 __func__); 1528 head = i = napid->bigcl_head; 1529 goto retry_collapse; 1530 } 1531 } else { /* HNC_ACT_DISPERSE */ 1532 int smallest = 99; /* all 32 bits full */ 1533 int smallidx = -1; 1534 1535 head = i = napid->bigcl_head; 1536 retry_disperse: 1537 while (i >= 0) { 1538 if ((napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) && 1539 (hweight32(napid->napi_cpu[i].napis) <= smallest)) { 1540 smallest = napid->napi_cpu[i].napis; 1541 smallidx = i; 1542 } 1543 i = napid->napi_cpu[i].cluster_nxt; 1544 } 1545 /* Check if matches with user specified CPU mask */ 1546 smallidx = ((1 << smallidx) & napid->user_cpu_affin_mask) ? 1547 smallidx : -1; 1548 1549 if ((smallidx < 0) && (head == napid->bigcl_head)) { 1550 NAPI_DEBUG("%s: DISPERSE: no bigcl dest, try lilcl", 1551 __func__); 1552 head = i = napid->lilcl_head; 1553 goto retry_disperse; 1554 } 1555 destidx = smallidx; 1556 } 1557 NAPI_DEBUG("<--%s[dest=%d]", __func__, destidx); 1558 return destidx; 1559 } 1560 /** 1561 * hif_napi_cpu_migrate() - migrate IRQs away 1562 * @cpu: -1: all CPUs <n> specific CPU 1563 * @act: COLLAPSE | DISPERSE 1564 * 1565 * Moves IRQs/NAPIs from specific or all CPUs (specified by @cpu) to eligible 1566 * cores. Eligible cores are: 1567 * act=COLLAPSE -> the first online core of the little cluster 1568 * act=DISPERSE -> separate cores of the big cluster, so that each core will 1569 * host minimum number of NAPIs/IRQs (napid->cpus[cpu].napis) 1570 * 1571 * Note that this function is called with a spinlock acquired already. 1572 * 1573 * Return: =0: success 1574 * <0: error 1575 */ 1576 1577 int hif_napi_cpu_migrate(struct qca_napi_data *napid, int cpu, int action) 1578 { 1579 int rc = 0; 1580 struct qca_napi_cpu *cpup; 1581 int i, dind; 1582 uint32_t napis; 1583 1584 NAPI_DEBUG("-->%s(.., cpu=%d, act=%d)", 1585 __func__, cpu, action); 1586 /* the following is really: hif_napi_enabled() with less overhead */ 1587 if (napid->ce_map == 0) { 1588 NAPI_DEBUG("%s: NAPI disabled. Not migrating.", __func__); 1589 goto hncm_return; 1590 } 1591 1592 cpup = napid->napi_cpu; 1593 1594 switch (action) { 1595 case HNC_ACT_RELOCATE: 1596 case HNC_ACT_DISPERSE: 1597 case HNC_ACT_COLLAPSE: { 1598 /* first find the src napi set */ 1599 if (cpu == HNC_ANY_CPU) 1600 napis = napid->ce_map; 1601 else 1602 napis = cpup[cpu].napis; 1603 /* then clear the napi bitmap on each CPU */ 1604 for (i = 0; i < NR_CPUS; i++) 1605 cpup[i].napis = 0; 1606 /* then for each of the NAPIs to disperse: */ 1607 for (i = 0; i < CE_COUNT_MAX; i++) 1608 if (napis & (1 << i)) { 1609 /* find a destination CPU */ 1610 dind = hncm_dest_cpu(napid, action); 1611 if (dind >= 0) { 1612 NAPI_DEBUG("Migrating NAPI ce%d to %d", 1613 i, dind); 1614 rc = hncm_migrate_to(napid, i, dind); 1615 } else { 1616 NAPI_DEBUG("No dest for NAPI ce%d", i); 1617 hnc_dump_cpus(napid); 1618 rc = -1; 1619 } 1620 } 1621 break; 1622 } 1623 default: { 1624 NAPI_DEBUG("%s: bad action: %d\n", __func__, action); 1625 QDF_BUG(0); 1626 break; 1627 } 1628 } /* switch action */ 1629 1630 hncm_return: 1631 hnc_dump_cpus(napid); 1632 return rc; 1633 } 1634 1635 1636 /** 1637 * hif_napi_dl_irq() - calls irq_modify_status to enable/disable denylisting 1638 * @napid: pointer to qca_napi_data structure 1639 * @dl_flag: denylist flag to enable/disable denylisting 1640 * 1641 * The function enables/disables denylisting for all the copy engine 1642 * interrupts on which NAPI is enabled. 1643 * 1644 * Return: None 1645 */ 1646 static inline void hif_napi_dl_irq(struct qca_napi_data *napid, bool dl_flag) 1647 { 1648 int i; 1649 struct qca_napi_info *napii; 1650 1651 for (i = 0; i < CE_COUNT_MAX; i++) { 1652 /* check if NAPI is enabled on the CE */ 1653 if (!(napid->ce_map & (0x01 << i))) 1654 continue; 1655 1656 /*double check that NAPI is allocated for the CE */ 1657 napii = napid->napis[i]; 1658 if (!(napii)) 1659 continue; 1660 1661 if (dl_flag == true) 1662 qdf_dev_modify_irq_status(napii->irq, 1663 0, QDF_IRQ_NO_BALANCING); 1664 else 1665 qdf_dev_modify_irq_status(napii->irq, 1666 QDF_IRQ_NO_BALANCING, 0); 1667 hif_debug("dl_flag %d CE %d", dl_flag, i); 1668 } 1669 } 1670 1671 /** 1672 * hif_napi_cpu_denylist() - en(dis)ables denylisting for NAPI RX interrupts. 1673 * @napid: pointer to qca_napi_data structure 1674 * @op: denylist operation to perform 1675 * 1676 * The function enables/disables/queries denylisting for all CE RX 1677 * interrupts with NAPI enabled. Besides denylisting, it also enables/disables 1678 * core_ctl_set_boost. 1679 * Once denylisting is enabled, the interrupts will not be managed by the IRQ 1680 * balancer. 1681 * 1682 * Return: -EINVAL, in case IRQ_DENYLISTING and CORE_CTL_BOOST is not enabled 1683 * for DENYLIST_QUERY op - denylist refcount 1684 * for DENYLIST_ON op - return value from core_ctl_set_boost API 1685 * for DENYLIST_OFF op - return value from core_ctl_set_boost API 1686 */ 1687 int hif_napi_cpu_denylist(struct qca_napi_data *napid, 1688 enum qca_denylist_op op) 1689 { 1690 int rc = 0; 1691 static int ref_count; /* = 0 by the compiler */ 1692 uint8_t flags = napid->flags; 1693 bool dl_en = flags & QCA_NAPI_FEATURE_IRQ_BLACKLISTING; 1694 bool ccb_en = flags & QCA_NAPI_FEATURE_CORE_CTL_BOOST; 1695 1696 NAPI_DEBUG("-->%s(%d %d)", __func__, flags, op); 1697 1698 if (!(dl_en && ccb_en)) { 1699 rc = -EINVAL; 1700 goto out; 1701 } 1702 1703 switch (op) { 1704 case DENYLIST_QUERY: 1705 rc = ref_count; 1706 break; 1707 case DENYLIST_ON: 1708 ref_count++; 1709 rc = 0; 1710 if (ref_count == 1) { 1711 rc = hif_napi_core_ctl_set_boost(true); 1712 NAPI_DEBUG("boost_on() returns %d - refcnt=%d", 1713 rc, ref_count); 1714 hif_napi_dl_irq(napid, true); 1715 } 1716 break; 1717 case DENYLIST_OFF: 1718 if (ref_count) { 1719 ref_count--; 1720 rc = 0; 1721 if (ref_count == 0) { 1722 rc = hif_napi_core_ctl_set_boost(false); 1723 NAPI_DEBUG("boost_off() returns %d - refcnt=%d", 1724 rc, ref_count); 1725 hif_napi_dl_irq(napid, false); 1726 } 1727 } 1728 break; 1729 default: 1730 NAPI_DEBUG("Invalid denylist op: %d", op); 1731 rc = -EINVAL; 1732 } /* switch */ 1733 out: 1734 NAPI_DEBUG("<--%s[%d]", __func__, rc); 1735 return rc; 1736 } 1737 1738 /** 1739 * hif_napi_serialize() - [de-]serialize NAPI operations 1740 * @hif: context 1741 * @is_on: 1: serialize, 0: deserialize 1742 * 1743 * hif_napi_serialize(hif, 1) can be called multiple times. It will perform the 1744 * following steps (see hif_napi_event for code): 1745 * - put irqs of all NAPI instances on the same CPU 1746 * - only for the first serialize call: denylist 1747 * 1748 * hif_napi_serialize(hif, 0): 1749 * - start a timer (multiple of BusBandwidthTimer -- default: 100 msec) 1750 * - at the end of the timer, check the current throughput state and 1751 * implement it. 1752 */ 1753 static unsigned long napi_serialize_reqs; 1754 int hif_napi_serialize(struct hif_opaque_softc *hif, int is_on) 1755 { 1756 int rc = -EINVAL; 1757 1758 if (hif) 1759 switch (is_on) { 1760 case 0: { /* de-serialize */ 1761 rc = hif_napi_event(hif, NAPI_EVT_USR_NORMAL, 1762 (void *) 0); 1763 napi_serialize_reqs = 0; 1764 break; 1765 } /* end de-serialize */ 1766 case 1: { /* serialize */ 1767 rc = hif_napi_event(hif, NAPI_EVT_USR_SERIAL, 1768 (void *)napi_serialize_reqs++); 1769 break; 1770 } /* end serialize */ 1771 default: 1772 break; /* no-op */ 1773 } /* switch */ 1774 return rc; 1775 } 1776 1777 #endif /* ifdef HIF_IRQ_AFFINITY */ 1778