1 /* 2 * Copyright (c) 2015-2021 The Linux Foundation. All rights reserved. 3 * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved. 4 * 5 * Permission to use, copy, modify, and/or distribute this software for 6 * any purpose with or without fee is hereby granted, provided that the 7 * above copyright notice and this permission notice appear in all 8 * copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL 11 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED 12 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE 13 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 14 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR 15 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 16 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 17 * PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 /** 21 * DOC: hif_napi.c 22 * 23 * HIF NAPI interface implementation 24 */ 25 26 #include <linux/string.h> /* memset */ 27 28 /* Linux headers */ 29 #include <linux/cpumask.h> 30 #include <linux/cpufreq.h> 31 #include <linux/cpu.h> 32 #include <linux/topology.h> 33 #include <linux/interrupt.h> 34 #ifdef CONFIG_SCHED_CORE_CTL 35 #include <linux/sched/core_ctl.h> 36 #endif 37 #include <pld_common.h> 38 #include <linux/pm.h> 39 40 /* Driver headers */ 41 #include <hif_napi.h> 42 #include <hif_debug.h> 43 #include <hif_io32.h> 44 #include <ce_api.h> 45 #include <ce_internal.h> 46 #include <hif_irq_affinity.h> 47 #include "qdf_cpuhp.h" 48 #include "qdf_module.h" 49 #include "qdf_net_if.h" 50 #include "qdf_dev.h" 51 #include "qdf_irq.h" 52 53 enum napi_decision_vector { 54 HIF_NAPI_NOEVENT = 0, 55 HIF_NAPI_INITED = 1, 56 HIF_NAPI_CONF_UP = 2 57 }; 58 #define ENABLE_NAPI_MASK (HIF_NAPI_INITED | HIF_NAPI_CONF_UP) 59 60 #ifdef RECEIVE_OFFLOAD 61 /** 62 * hif_rxthread_napi_poll() - dummy napi poll for rx_thread NAPI 63 * @napi: Rx_thread NAPI 64 * @budget: NAPI BUDGET 65 * 66 * Return: 0 as it is not supposed to be polled at all as it is not scheduled. 67 */ 68 static int hif_rxthread_napi_poll(struct napi_struct *napi, int budget) 69 { 70 hif_err("This napi_poll should not be polled as we don't schedule it"); 71 QDF_ASSERT(0); 72 return 0; 73 } 74 75 /** 76 * hif_init_rx_thread_napi() - Initialize dummy Rx_thread NAPI 77 * @napii: Handle to napi_info holding rx_thread napi 78 * 79 * Return: None 80 */ 81 static void hif_init_rx_thread_napi(struct qca_napi_info *napii) 82 { 83 struct qdf_net_if *nd = (struct qdf_net_if *)&napii->rx_thread_netdev; 84 85 qdf_net_if_create_dummy_if(nd); 86 qdf_netif_napi_add(&napii->rx_thread_netdev, &napii->rx_thread_napi, 87 hif_rxthread_napi_poll, 64); 88 qdf_napi_enable(&napii->rx_thread_napi); 89 } 90 91 /** 92 * hif_deinit_rx_thread_napi() - Deinitialize dummy Rx_thread NAPI 93 * @napii: Handle to napi_info holding rx_thread napi 94 * 95 * Return: None 96 */ 97 static void hif_deinit_rx_thread_napi(struct qca_napi_info *napii) 98 { 99 qdf_netif_napi_del(&napii->rx_thread_napi); 100 } 101 #else /* RECEIVE_OFFLOAD */ 102 static void hif_init_rx_thread_napi(struct qca_napi_info *napii) 103 { 104 } 105 106 static void hif_deinit_rx_thread_napi(struct qca_napi_info *napii) 107 { 108 } 109 #endif 110 111 /** 112 * hif_napi_create() - creates the NAPI structures for a given CE 113 * @hif_ctx: pointer to hif context 114 * @poll: poll function to be used for this NAPI instance 115 * @budget: budget to be registered with the NAPI instance 116 * @scale: scale factor on the weight (to scaler budget to 1000) 117 * @flags: feature flags 118 * 119 * Description: 120 * Creates NAPI instances. This function is called 121 * unconditionally during initialization. It creates 122 * napi structures through the proper HTC/HIF calls. 123 * The structures are disabled on creation. 124 * Note that for each NAPI instance a separate dummy netdev is used 125 * 126 * Return: 127 * < 0: error 128 * = 0: <should never happen> 129 * > 0: id of the created object (for multi-NAPI, number of objects created) 130 */ 131 int hif_napi_create(struct hif_opaque_softc *hif_ctx, 132 int (*poll)(struct napi_struct *, int), 133 int budget, 134 int scale, 135 uint8_t flags) 136 { 137 int i; 138 struct qca_napi_data *napid; 139 struct qca_napi_info *napii; 140 struct CE_state *ce_state; 141 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 142 int rc = 0; 143 144 NAPI_DEBUG("-->(budget=%d, scale=%d)", 145 budget, scale); 146 NAPI_DEBUG("hif->napi_data.state = 0x%08x", 147 hif->napi_data.state); 148 NAPI_DEBUG("hif->napi_data.ce_map = 0x%08x", 149 hif->napi_data.ce_map); 150 151 napid = &(hif->napi_data); 152 if (0 == (napid->state & HIF_NAPI_INITED)) { 153 memset(napid, 0, sizeof(struct qca_napi_data)); 154 qdf_spinlock_create(&(napid->lock)); 155 156 napid->state |= HIF_NAPI_INITED; 157 napid->flags = flags; 158 159 rc = hif_napi_cpu_init(hif_ctx); 160 if (rc != 0 && rc != -EALREADY) { 161 hif_err("NAPI_initialization failed(rc=%d)", rc); 162 rc = napid->ce_map; 163 goto hnc_err; 164 } else 165 rc = 0; 166 167 hif_debug("NAPI structures initialized, rc=%d", rc); 168 } 169 for (i = 0; i < hif->ce_count; i++) { 170 ce_state = hif->ce_id_to_state[i]; 171 NAPI_DEBUG("ce %d: htt_rx=%d htt_tx=%d", 172 i, ce_state->htt_rx_data, 173 ce_state->htt_tx_data); 174 if (ce_srng_based(hif)) 175 continue; 176 177 if (!ce_state->htt_rx_data) 178 continue; 179 180 /* Now this is a CE where we need NAPI on */ 181 NAPI_DEBUG("Creating NAPI on pipe %d", i); 182 napii = qdf_mem_malloc(sizeof(*napii)); 183 napid->napis[i] = napii; 184 if (!napii) { 185 rc = -ENOMEM; 186 goto napii_free; 187 } 188 } 189 190 for (i = 0; i < hif->ce_count; i++) { 191 napii = napid->napis[i]; 192 if (!napii) 193 continue; 194 195 NAPI_DEBUG("initializing NAPI for pipe %d", i); 196 memset(napii, 0, sizeof(struct qca_napi_info)); 197 napii->scale = scale; 198 napii->id = NAPI_PIPE2ID(i); 199 napii->hif_ctx = hif_ctx; 200 napii->irq = pld_get_irq(hif->qdf_dev->dev, i); 201 202 if (napii->irq < 0) 203 hif_warn("bad IRQ value for CE %d: %d", i, napii->irq); 204 205 qdf_net_if_create_dummy_if((struct qdf_net_if *)&napii->netdev); 206 207 NAPI_DEBUG("adding napi=%pK to netdev=%pK (poll=%pK, bdgt=%d)", 208 &(napii->napi), &(napii->netdev), poll, budget); 209 qdf_netif_napi_add(&(napii->netdev), &(napii->napi), 210 poll, budget); 211 212 NAPI_DEBUG("after napi_add"); 213 NAPI_DEBUG("napi=0x%pK, netdev=0x%pK", 214 &(napii->napi), &(napii->netdev)); 215 NAPI_DEBUG("napi.dev_list.prev=0x%pK, next=0x%pK", 216 napii->napi.dev_list.prev, 217 napii->napi.dev_list.next); 218 NAPI_DEBUG("dev.napi_list.prev=0x%pK, next=0x%pK", 219 napii->netdev.napi_list.prev, 220 napii->netdev.napi_list.next); 221 222 hif_init_rx_thread_napi(napii); 223 napii->lro_ctx = qdf_lro_init(); 224 NAPI_DEBUG("Registering LRO for ce_id %d NAPI callback for %d lro_ctx %pK\n", 225 i, napii->id, napii->lro_ctx); 226 227 /* It is OK to change the state variable below without 228 * protection as there should be no-one around yet 229 */ 230 napid->ce_map |= (0x01 << i); 231 hif_debug("NAPI id %d created for pipe %d", napii->id, i); 232 } 233 234 /* no ces registered with the napi */ 235 if (!ce_srng_based(hif) && napid->ce_map == 0) { 236 hif_warn("no napis created for copy engines"); 237 rc = -EFAULT; 238 goto napii_free; 239 } 240 241 NAPI_DEBUG("napi map = %x", napid->ce_map); 242 NAPI_DEBUG("NAPI ids created for all applicable pipes"); 243 return napid->ce_map; 244 245 napii_free: 246 for (i = 0; i < hif->ce_count; i++) { 247 napii = napid->napis[i]; 248 napid->napis[i] = NULL; 249 if (napii) 250 qdf_mem_free(napii); 251 } 252 253 hnc_err: 254 NAPI_DEBUG("<--napi_instances_map=%x]", napid->ce_map); 255 return rc; 256 } 257 qdf_export_symbol(hif_napi_create); 258 259 #ifdef RECEIVE_OFFLOAD 260 void hif_napi_rx_offld_flush_cb_register(struct hif_opaque_softc *hif_hdl, 261 void (offld_flush_handler)(void *)) 262 { 263 int i; 264 struct CE_state *ce_state; 265 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl); 266 struct qca_napi_data *napid; 267 struct qca_napi_info *napii; 268 269 if (!scn) { 270 hif_err("hif_state NULL!"); 271 QDF_ASSERT(0); 272 return; 273 } 274 275 napid = hif_napi_get_all(hif_hdl); 276 for (i = 0; i < scn->ce_count; i++) { 277 ce_state = scn->ce_id_to_state[i]; 278 if (ce_state && (ce_state->htt_rx_data)) { 279 napii = napid->napis[i]; 280 napii->offld_flush_cb = offld_flush_handler; 281 hif_debug("Registering offload for ce_id %d NAPI callback for %d flush_cb %pK", 282 i, napii->id, napii->offld_flush_cb); 283 } 284 } 285 } 286 287 void hif_napi_rx_offld_flush_cb_deregister(struct hif_opaque_softc *hif_hdl) 288 { 289 int i; 290 struct CE_state *ce_state; 291 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl); 292 struct qca_napi_data *napid; 293 struct qca_napi_info *napii; 294 295 if (!scn) { 296 hif_err("hif_state NULL!"); 297 QDF_ASSERT(0); 298 return; 299 } 300 301 napid = hif_napi_get_all(hif_hdl); 302 for (i = 0; i < scn->ce_count; i++) { 303 ce_state = scn->ce_id_to_state[i]; 304 if (ce_state && (ce_state->htt_rx_data)) { 305 napii = napid->napis[i]; 306 hif_debug("deRegistering offld for ce_id %d NAPI callback for %d flush_cb %pK", 307 i, napii->id, napii->offld_flush_cb); 308 /* Not required */ 309 napii->offld_flush_cb = NULL; 310 } 311 } 312 } 313 #endif /* RECEIVE_OFFLOAD */ 314 315 /** 316 * hif_napi_destroy() - destroys the NAPI structures for a given instance 317 * @hif_ctx: pointer to hif context 318 * @id: the CE id whose napi instance will be destroyed 319 * @force: if set, will destroy even if entry is active (de-activates) 320 * 321 * Description: 322 * Destroy a given NAPI instance. This function is called 323 * unconditionally during cleanup. 324 * Refuses to destroy an entry of it is still enabled (unless force=1) 325 * Marks the whole napi_data invalid if all instances are destroyed. 326 * 327 * Return: 328 * -EINVAL: specific entry has not been created 329 * -EPERM : specific entry is still active 330 * 0 < : error 331 * 0 = : success 332 */ 333 int hif_napi_destroy(struct hif_opaque_softc *hif_ctx, 334 uint8_t id, 335 int force) 336 { 337 uint8_t ce = NAPI_ID2PIPE(id); 338 int rc = 0; 339 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 340 341 NAPI_DEBUG("-->(id=%d, force=%d)", id, force); 342 343 if (0 == (hif->napi_data.state & HIF_NAPI_INITED)) { 344 hif_err("NAPI not initialized or entry %d not created", id); 345 rc = -EINVAL; 346 } else if (0 == (hif->napi_data.ce_map & (0x01 << ce))) { 347 hif_err("NAPI instance %d (pipe %d) not created", id, ce); 348 if (hif->napi_data.napis[ce]) 349 hif_err("memory allocated but ce_map not set %d (pipe %d)", 350 id, ce); 351 rc = -EINVAL; 352 } else { 353 struct qca_napi_data *napid; 354 struct qca_napi_info *napii; 355 356 napid = &(hif->napi_data); 357 napii = napid->napis[ce]; 358 if (!napii) { 359 if (napid->ce_map & (0x01 << ce)) 360 hif_err("napii & ce_map out of sync(ce %d)", ce); 361 return -EINVAL; 362 } 363 364 365 if (hif->napi_data.state == HIF_NAPI_CONF_UP) { 366 if (force) { 367 qdf_napi_disable(&(napii->napi)); 368 hif_debug("NAPI entry %d force disabled", id); 369 NAPI_DEBUG("NAPI %d force disabled", id); 370 } else { 371 hif_err("Cannot destroy active NAPI %d", id); 372 rc = -EPERM; 373 } 374 } 375 if (0 == rc) { 376 NAPI_DEBUG("before napi_del"); 377 NAPI_DEBUG("napi.dlist.prv=0x%pK, next=0x%pK", 378 napii->napi.dev_list.prev, 379 napii->napi.dev_list.next); 380 NAPI_DEBUG("dev.napi_l.prv=0x%pK, next=0x%pK", 381 napii->netdev.napi_list.prev, 382 napii->netdev.napi_list.next); 383 384 qdf_lro_deinit(napii->lro_ctx); 385 qdf_netif_napi_del(&(napii->napi)); 386 hif_deinit_rx_thread_napi(napii); 387 388 napid->ce_map &= ~(0x01 << ce); 389 napid->napis[ce] = NULL; 390 napii->scale = 0; 391 qdf_mem_free(napii); 392 hif_debug("NAPI %d destroyed", id); 393 394 /* if there are no active instances and 395 * if they are all destroyed, 396 * set the whole structure to uninitialized state 397 */ 398 if (napid->ce_map == 0) { 399 rc = hif_napi_cpu_deinit(hif_ctx); 400 /* caller is tolerant to receiving !=0 rc */ 401 402 qdf_spinlock_destroy(&(napid->lock)); 403 memset(napid, 404 0, sizeof(struct qca_napi_data)); 405 hif_debug("no NAPI instances. Zapped"); 406 } 407 } 408 } 409 410 return rc; 411 } 412 qdf_export_symbol(hif_napi_destroy); 413 414 #ifdef FEATURE_LRO 415 void *hif_napi_get_lro_info(struct hif_opaque_softc *hif_hdl, int napi_id) 416 { 417 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl); 418 struct qca_napi_data *napid; 419 struct qca_napi_info *napii; 420 421 napid = &(scn->napi_data); 422 napii = napid->napis[NAPI_ID2PIPE(napi_id)]; 423 424 if (napii) 425 return napii->lro_ctx; 426 return 0; 427 } 428 #endif 429 430 /** 431 * hif_napi_get_all() - returns the address of the whole HIF NAPI structure 432 * @hif_ctx: pointer to hif context 433 * 434 * Description: 435 * Returns the address of the whole structure 436 * 437 * Return: 438 * <addr>: address of the whole HIF NAPI structure 439 */ 440 inline struct qca_napi_data *hif_napi_get_all(struct hif_opaque_softc *hif_ctx) 441 { 442 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 443 444 return &(hif->napi_data); 445 } 446 447 struct qca_napi_info *hif_get_napi(int napi_id, struct qca_napi_data *napid) 448 { 449 int id = NAPI_ID2PIPE(napi_id); 450 451 return napid->napis[id]; 452 } 453 454 /** 455 * hif_napi_event() - reacts to events that impact NAPI 456 * @hif_ctx: pointer to hif context 457 * @event: event that has been detected 458 * @data: more data regarding the event 459 * 460 * Description: 461 * This function handles two types of events: 462 * 1- Events that change the state of NAPI (enabled/disabled): 463 * {NAPI_EVT_INI_FILE, NAPI_EVT_CMD_STATE} 464 * The state is retrievable by "hdd_napi_enabled(-1)" 465 * - NAPI will be on if either INI file is on and it has not been disabled 466 * by a subsequent vendor CMD, 467 * or it has been enabled by a vendor CMD. 468 * 2- Events that change the CPU affinity of a NAPI instance/IRQ: 469 * {NAPI_EVT_TPUT_STATE, NAPI_EVT_CPU_STATE} 470 * - NAPI will support a throughput mode (HI/LO), kept at napid->napi_mode 471 * - NAPI will switch throughput mode based on hdd_napi_throughput_policy() 472 * - In LO tput mode, NAPI will yield control if its interrupts to the system 473 * management functions. However in HI throughput mode, NAPI will actively 474 * manage its interrupts/instances (by trying to disperse them out to 475 * separate performance cores). 476 * - CPU eligibility is kept up-to-date by NAPI_EVT_CPU_STATE events. 477 * 478 * + In some cases (roaming peer management is the only case so far), a 479 * a client can trigger a "SERIALIZE" event. Basically, this means that the 480 * users is asking NAPI to go into a truly single execution context state. 481 * So, NAPI indicates to msm-irqbalancer that it wants to be denylisted, 482 * (if called for the first time) and then moves all IRQs (for NAPI 483 * instances) to be collapsed to a single core. If called multiple times, 484 * it will just re-collapse the CPUs. This is because denylist-on() API 485 * is reference-counted, and because the API has already been called. 486 * 487 * Such a user, should call "DESERIALIZE" (NORMAL) event, to set NAPI to go 488 * to its "normal" operation. Optionally, they can give a timeout value (in 489 * multiples of BusBandwidthCheckPeriod -- 100 msecs by default). In this 490 * case, NAPI will just set the current throughput state to uninitialized 491 * and set the delay period. Once policy handler is called, it would skip 492 * applying the policy delay period times, and otherwise apply the policy. 493 * 494 * Return: 495 * < 0: some error 496 * = 0: event handled successfully 497 */ 498 int hif_napi_event(struct hif_opaque_softc *hif_ctx, enum qca_napi_event event, 499 void *data) 500 { 501 int rc = 0; 502 uint32_t prev_state; 503 int i; 504 bool state_changed; 505 struct napi_struct *napi; 506 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 507 struct qca_napi_data *napid = &(hif->napi_data); 508 enum qca_napi_tput_state tput_mode = QCA_NAPI_TPUT_UNINITIALIZED; 509 enum { 510 DENYLIST_NOT_PENDING, 511 DENYLIST_ON_PENDING, 512 DENYLIST_OFF_PENDING 513 } denylist_pending = DENYLIST_NOT_PENDING; 514 515 NAPI_DEBUG("%s: -->(event=%d, aux=%pK)", __func__, event, data); 516 517 if (ce_srng_based(hif)) 518 return hif_exec_event(hif_ctx, event, data); 519 520 if ((napid->state & HIF_NAPI_INITED) == 0) { 521 NAPI_DEBUG("%s: got event when NAPI not initialized", 522 __func__); 523 return -EINVAL; 524 } 525 qdf_spin_lock_bh(&(napid->lock)); 526 prev_state = napid->state; 527 switch (event) { 528 case NAPI_EVT_INI_FILE: 529 case NAPI_EVT_CMD_STATE: 530 case NAPI_EVT_INT_STATE: { 531 int on = (data != ((void *)0)); 532 533 hif_debug("recved evnt: STATE_CMD %d; v = %d (state=0x%0x)", 534 event, on, prev_state); 535 if (on) 536 if (prev_state & HIF_NAPI_CONF_UP) { 537 hif_debug("Duplicate NAPI conf ON msg"); 538 } else { 539 hif_debug("Setting state to ON"); 540 napid->state |= HIF_NAPI_CONF_UP; 541 } 542 else /* off request */ 543 if (prev_state & HIF_NAPI_CONF_UP) { 544 hif_debug("Setting state to OFF"); 545 napid->state &= ~HIF_NAPI_CONF_UP; 546 } else { 547 hif_debug("Duplicate NAPI conf OFF msg"); 548 } 549 break; 550 } 551 /* case NAPI_INIT_FILE/CMD_STATE */ 552 553 case NAPI_EVT_CPU_STATE: { 554 int cpu = ((unsigned long int)data >> 16); 555 int val = ((unsigned long int)data & 0x0ff); 556 557 NAPI_DEBUG("%s: evt=CPU_STATE on CPU %d value=%d", 558 __func__, cpu, val); 559 560 /* state has already been set by hnc_cpu_notify_cb */ 561 if ((val == QCA_NAPI_CPU_DOWN) && 562 (napid->napi_mode == QCA_NAPI_TPUT_HI) && /* we manage */ 563 (napid->napi_cpu[cpu].napis != 0)) { 564 NAPI_DEBUG("%s: Migrating NAPIs out of cpu %d", 565 __func__, cpu); 566 rc = hif_napi_cpu_migrate(napid, 567 cpu, 568 HNC_ACT_RELOCATE); 569 napid->napi_cpu[cpu].napis = 0; 570 } 571 /* in QCA_NAPI_TPUT_LO case, napis MUST == 0 */ 572 break; 573 } 574 575 case NAPI_EVT_TPUT_STATE: { 576 tput_mode = (enum qca_napi_tput_state)data; 577 if (tput_mode == QCA_NAPI_TPUT_LO) { 578 /* from TPUT_HI -> TPUT_LO */ 579 NAPI_DEBUG("%s: Moving to napi_tput_LO state", 580 __func__); 581 denylist_pending = DENYLIST_OFF_PENDING; 582 /* 583 * Ideally we should "collapse" interrupts here, since 584 * we are "dispersing" interrupts in the "else" case. 585 * This allows the possibility that our interrupts may 586 * still be on the perf cluster the next time we enter 587 * high tput mode. However, the irq_balancer is free 588 * to move our interrupts to power cluster once 589 * denylisting has been turned off in the "else" case. 590 */ 591 } else { 592 /* from TPUT_LO -> TPUT->HI */ 593 NAPI_DEBUG("%s: Moving to napi_tput_HI state", 594 __func__); 595 rc = hif_napi_cpu_migrate(napid, 596 HNC_ANY_CPU, 597 HNC_ACT_DISPERSE); 598 599 denylist_pending = DENYLIST_ON_PENDING; 600 } 601 napid->napi_mode = tput_mode; 602 break; 603 } 604 605 case NAPI_EVT_USR_SERIAL: { 606 unsigned long users = (unsigned long)data; 607 608 NAPI_DEBUG("%s: User forced SERIALIZATION; users=%ld", 609 __func__, users); 610 611 rc = hif_napi_cpu_migrate(napid, 612 HNC_ANY_CPU, 613 HNC_ACT_COLLAPSE); 614 if ((users == 0) && (rc == 0)) 615 denylist_pending = DENYLIST_ON_PENDING; 616 break; 617 } 618 case NAPI_EVT_USR_NORMAL: { 619 NAPI_DEBUG("%s: User forced DE-SERIALIZATION", __func__); 620 if (!napid->user_cpu_affin_mask) 621 denylist_pending = DENYLIST_OFF_PENDING; 622 /* 623 * Deserialization timeout is handled at hdd layer; 624 * just mark current mode to uninitialized to ensure 625 * it will be set when the delay is over 626 */ 627 napid->napi_mode = QCA_NAPI_TPUT_UNINITIALIZED; 628 break; 629 } 630 default: { 631 hif_err("Unknown event: %d (data=0x%0lx)", 632 event, (unsigned long) data); 633 break; 634 } /* default */ 635 }; /* switch */ 636 637 638 switch (denylist_pending) { 639 case DENYLIST_ON_PENDING: 640 /* assume the control of WLAN IRQs */ 641 hif_napi_cpu_denylist(napid, DENYLIST_ON); 642 break; 643 case DENYLIST_OFF_PENDING: 644 /* yield the control of WLAN IRQs */ 645 hif_napi_cpu_denylist(napid, DENYLIST_OFF); 646 break; 647 default: /* nothing to do */ 648 break; 649 } /* switch denylist_pending */ 650 651 /* we want to perform the comparison in lock: 652 * there is a possibility of hif_napi_event get called 653 * from two different contexts (driver unload and cpu hotplug 654 * notification) and napid->state get changed 655 * in driver unload context and can lead to race condition 656 * in cpu hotplug context. Therefore, perform the napid->state 657 * comparison before releasing lock. 658 */ 659 state_changed = (prev_state != napid->state); 660 qdf_spin_unlock_bh(&(napid->lock)); 661 662 if (state_changed) { 663 if (napid->state == ENABLE_NAPI_MASK) { 664 rc = 1; 665 for (i = 0; i < CE_COUNT_MAX; i++) { 666 struct qca_napi_info *napii = napid->napis[i]; 667 if (napii) { 668 napi = &(napii->napi); 669 NAPI_DEBUG("%s: enabling NAPI %d", 670 __func__, i); 671 qdf_napi_enable(napi); 672 } 673 } 674 } else { 675 rc = 0; 676 for (i = 0; i < CE_COUNT_MAX; i++) { 677 struct qca_napi_info *napii = napid->napis[i]; 678 if (napii) { 679 napi = &(napii->napi); 680 NAPI_DEBUG("%s: disabling NAPI %d", 681 __func__, i); 682 qdf_napi_disable(napi); 683 /* in case it is affined, remove it */ 684 qdf_dev_set_irq_affinity(napii->irq, 685 NULL); 686 } 687 } 688 } 689 } else { 690 hif_debug("no change in hif napi state (still %d)", prev_state); 691 } 692 693 NAPI_DEBUG("<--[rc=%d]", rc); 694 return rc; 695 } 696 qdf_export_symbol(hif_napi_event); 697 698 /** 699 * hif_napi_enabled() - checks whether NAPI is enabled for given ce or not 700 * @hif_ctx: hif context 701 * @ce: CE instance (or -1, to check if any CEs are enabled) 702 * 703 * Return: bool 704 */ 705 int hif_napi_enabled(struct hif_opaque_softc *hif_ctx, int ce) 706 { 707 int rc; 708 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 709 710 if (-1 == ce) 711 rc = ((hif->napi_data.state == ENABLE_NAPI_MASK)); 712 else 713 rc = ((hif->napi_data.state == ENABLE_NAPI_MASK) && 714 (hif->napi_data.ce_map & (0x01 << ce))); 715 return rc; 716 } 717 qdf_export_symbol(hif_napi_enabled); 718 719 /** 720 * hif_napi_created() - checks whether NAPI is created for given ce or not 721 * @hif_ctx: hif context 722 * @ce: CE instance 723 * 724 * Return: bool 725 */ 726 bool hif_napi_created(struct hif_opaque_softc *hif_ctx, int ce) 727 { 728 int rc; 729 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 730 731 rc = (hif->napi_data.ce_map & (0x01 << ce)); 732 733 return !!rc; 734 } 735 qdf_export_symbol(hif_napi_created); 736 737 /** 738 * hif_napi_enable_irq() - enables bus interrupts after napi_complete 739 * 740 * @hif: hif context 741 * @id: id of NAPI instance calling this (used to determine the CE) 742 * 743 * Return: void 744 */ 745 inline void hif_napi_enable_irq(struct hif_opaque_softc *hif, int id) 746 { 747 struct hif_softc *scn = HIF_GET_SOFTC(hif); 748 749 hif_irq_enable(scn, NAPI_ID2PIPE(id)); 750 } 751 752 #if defined(QCA_WIFI_WCN6450) && defined(HIF_LATENCY_PROFILE_ENABLE) 753 /* 754 * hif_napi_latency_profile_start() - update the schedule start timestamp 755 * 756 * @scn: HIF context 757 * ce_id: Copyengine id 758 * 759 * Return: None 760 */ 761 static inline void hif_napi_latency_profile_start(struct hif_softc *scn, 762 int ce_id) 763 { 764 struct qca_napi_info *napii; 765 766 napii = scn->napi_data.napis[ce_id]; 767 if (napii) 768 napii->tstamp = qdf_ktime_to_ms(qdf_ktime_get()); 769 } 770 771 /* 772 * hif_napi_latency_profile_measure() - calculate the NAPI schedule latency 773 * and update histogram 774 * 775 * @napi_info: pointer to qca_napi_info for the napi instance 776 * 777 * Return: None 778 */ 779 static void hif_napi_latency_profile_measure(struct qca_napi_info *napi_info) 780 { 781 int64_t cur_tstamp; 782 int64_t time_elapsed; 783 784 cur_tstamp = qdf_ktime_to_ms(qdf_ktime_get()); 785 786 if (cur_tstamp > napi_info->tstamp) 787 time_elapsed = (cur_tstamp - napi_info->tstamp); 788 else 789 time_elapsed = ~0x0 - (napi_info->tstamp - cur_tstamp); 790 791 napi_info->tstamp = cur_tstamp; 792 793 if (time_elapsed <= HIF_SCHED_LATENCY_BUCKET_0_2) 794 napi_info->sched_latency_stats[0]++; 795 else if (time_elapsed <= HIF_SCHED_LATENCY_BUCKET_3_10) 796 napi_info->sched_latency_stats[1]++; 797 else if (time_elapsed <= HIF_SCHED_LATENCY_BUCKET_11_20) 798 napi_info->sched_latency_stats[2]++; 799 else if (time_elapsed <= HIF_SCHED_LATENCY_BUCKET_21_50) 800 napi_info->sched_latency_stats[3]++; 801 else if (time_elapsed <= HIF_SCHED_LATENCY_BUCKET_51_100) 802 napi_info->sched_latency_stats[4]++; 803 else if (time_elapsed <= HIF_SCHED_LATENCY_BUCKET_101_250) 804 napi_info->sched_latency_stats[5]++; 805 else if (time_elapsed <= HIF_SCHED_LATENCY_BUCKET_251_500) 806 napi_info->sched_latency_stats[6]++; 807 else 808 napi_info->sched_latency_stats[7]++; 809 } 810 811 static void hif_print_napi_latency_stats(struct qca_napi_info *napii, int ce_id) 812 { 813 int i; 814 int64_t cur_tstamp; 815 816 const char time_str[HIF_SCHED_LATENCY_BUCKETS][15] = { 817 "0-2 ms", 818 "3-10 ms", 819 "11-20 ms", 820 "21-50 ms", 821 "51-100 ms", 822 "101-250 ms", 823 "251-500 ms", 824 "> 500 ms" 825 }; 826 827 cur_tstamp = qdf_ktime_to_ms(qdf_ktime_get()); 828 829 QDF_TRACE(QDF_MODULE_ID_HIF, QDF_TRACE_LEVEL_INFO_HIGH, 830 "Current timestamp: %lld", cur_tstamp); 831 832 QDF_TRACE(QDF_MODULE_ID_HIF, QDF_TRACE_LEVEL_INFO_HIGH, 833 "ce id %d Last serviced timestamp: %lld", 834 ce_id, napii->tstamp); 835 836 QDF_TRACE(QDF_MODULE_ID_HIF, QDF_TRACE_LEVEL_INFO_HIGH, 837 "Latency Bucket | Time elapsed"); 838 839 for (i = 0; i < HIF_SCHED_LATENCY_BUCKETS; i++) 840 QDF_TRACE(QDF_MODULE_ID_HIF, 841 QDF_TRACE_LEVEL_INFO_HIGH, 842 "%s | %lld", 843 time_str[i], 844 napii->sched_latency_stats[i]); 845 } 846 #else 847 static inline void 848 hif_napi_latency_profile_start(struct hif_softc *scn, int ce_id) 849 { 850 } 851 852 static inline void 853 hif_napi_latency_profile_measure(struct qca_napi_info *napi_info) 854 { 855 } 856 857 static inline void 858 hif_print_napi_latency_stats(struct qca_napi_info *napii, int ce_id) 859 { 860 } 861 #endif 862 863 #ifdef QCA_WIFI_WCN6450 864 #ifdef WLAN_FEATURE_RX_SOFTIRQ_TIME_LIMIT 865 /** 866 * hif_napi_update_service_start_time() - Update NAPI poll start time 867 * 868 * @napi_info: per NAPI instance data structure 869 * 870 * The function is called at the beginning of a NAPI poll to record the poll 871 * start time. 872 * 873 * Return: None 874 */ 875 static inline void 876 hif_napi_update_service_start_time(struct qca_napi_info *napi_info) 877 { 878 napi_info->poll_start_time = qdf_time_sched_clock(); 879 } 880 881 /** 882 * hif_napi_fill_poll_time_histogram() - fills poll time histogram for a NAPI 883 * 884 * @napi_info: per NAPI instance data structure 885 * 886 * The function is called at the end of a NAPI poll to calculate poll time 887 * buckets. 888 * 889 * Return: void 890 */ 891 static void hif_napi_fill_poll_time_histogram(struct qca_napi_info *napi_info) 892 { 893 struct qca_napi_stat *napi_stat; 894 unsigned long long poll_time_ns; 895 uint32_t poll_time_us; 896 uint32_t bucket_size_us = 500; 897 uint32_t bucket; 898 uint32_t cpu_id = qdf_get_cpu(); 899 900 poll_time_ns = qdf_time_sched_clock() - napi_info->poll_start_time; 901 poll_time_us = qdf_do_div(poll_time_ns, 1000); 902 903 napi_stat = &napi_info->stats[cpu_id]; 904 if (poll_time_ns > napi_info->stats[cpu_id].napi_max_poll_time) 905 napi_info->stats[cpu_id].napi_max_poll_time = poll_time_ns; 906 907 bucket = poll_time_us / bucket_size_us; 908 if (bucket >= QCA_NAPI_NUM_BUCKETS) 909 bucket = QCA_NAPI_NUM_BUCKETS - 1; 910 911 ++napi_stat->poll_time_buckets[bucket]; 912 } 913 914 /* 915 * hif_get_poll_times_hist_str() - Get HIF poll times histogram string 916 * @stats: NAPI stats to get poll time buckets 917 * @buf: buffer to fill histogram string 918 * @buf_len: length of the buffer 919 * 920 * Return: void 921 */ 922 static void hif_get_poll_times_hist_str(struct qca_napi_stat *stats, char *buf, 923 uint8_t buf_len) 924 { 925 int i; 926 int str_index = 0; 927 928 for (i = 0; i < QCA_NAPI_NUM_BUCKETS; i++) 929 str_index += qdf_scnprintf(buf + str_index, buf_len - str_index, 930 "%u|", stats->poll_time_buckets[i]); 931 } 932 933 void hif_print_napi_stats(struct hif_opaque_softc *hif_ctx) 934 { 935 struct hif_softc *scn = HIF_GET_SOFTC(hif_ctx); 936 struct qca_napi_info *napii; 937 struct qca_napi_stat *napi_stats; 938 int ce_id, cpu; 939 940 /* 941 * Max value of uint_32 (poll_time_bucket) = 4294967295 942 * Thus we need 10 chars + 1 space =11 chars for each bucket value. 943 * +1 space for '\0'. 944 */ 945 char hist_str[(QCA_NAPI_NUM_BUCKETS * 11) + 1] = {'\0'}; 946 947 QDF_TRACE(QDF_MODULE_ID_HIF, QDF_TRACE_LEVEL_INFO_HIGH, 948 "NAPI[#]CPU[#] |scheds |polls |comps |dones |t-lim |max(us)|hist(500us buckets)"); 949 950 for (ce_id = 0; ce_id < CE_COUNT_MAX; ce_id++) { 951 if (!hif_napi_enabled(hif_ctx, ce_id)) 952 continue; 953 954 napii = scn->napi_data.napis[ce_id]; 955 if (napii) { 956 for (cpu = 0; cpu < num_possible_cpus(); cpu++) { 957 napi_stats = &napii->stats[cpu]; 958 959 hif_get_poll_times_hist_str(napi_stats, 960 hist_str, 961 sizeof(hist_str)); 962 963 if (napi_stats->napi_schedules != 0) 964 QDF_TRACE(QDF_MODULE_ID_HIF, 965 QDF_TRACE_LEVEL_INFO_HIGH, 966 "NAPI[%d]CPU[%d]: %7u %7u %7u %7u %7u %7llu %s", 967 ce_id, cpu, 968 napi_stats->napi_schedules, 969 napi_stats->napi_polls, 970 napi_stats->napi_completes, 971 napi_stats->napi_workdone, 972 napi_stats->time_limit_reached, 973 qdf_do_div(napi_stats->napi_max_poll_time, 1000), 974 hist_str); 975 } 976 977 hif_print_napi_latency_stats(napii, ce_id); 978 } 979 } 980 } 981 #else 982 static inline void 983 hif_napi_update_service_start_time(struct qca_napi_info *napi_info) 984 { 985 } 986 987 static inline void 988 hif_napi_fill_poll_time_histogram(struct qca_napi_info *napi_info) 989 { 990 } 991 992 void hif_print_napi_stats(struct hif_opaque_softc *hif_ctx) 993 { 994 struct hif_softc *scn = HIF_GET_SOFTC(hif_ctx); 995 struct qca_napi_info *napii; 996 struct qca_napi_stat *napi_stats; 997 int ce_id, cpu; 998 999 QDF_TRACE(QDF_MODULE_ID_HIF, QDF_TRACE_LEVEL_FATAL, 1000 "NAPI[#ctx]CPU[#] |schedules |polls |completes |workdone"); 1001 1002 for (ce_id = 0; ce_id < CE_COUNT_MAX; ce_id++) { 1003 if (!hif_napi_enabled(hif_ctx, ce_id)) 1004 continue; 1005 1006 napii = scn->napi_data.napis[ce_id]; 1007 if (napii) { 1008 for (cpu = 0; cpu < num_possible_cpus(); cpu++) { 1009 napi_stats = &napii->stats[cpu]; 1010 1011 if (napi_stats->napi_schedules != 0) 1012 QDF_TRACE(QDF_MODULE_ID_HIF, 1013 QDF_TRACE_LEVEL_FATAL, 1014 "NAPI[%2d]CPU[%d]: " 1015 "%7d %7d %7d %7d ", 1016 ce_id, cpu, 1017 napi_stats->napi_schedules, 1018 napi_stats->napi_polls, 1019 napi_stats->napi_completes, 1020 napi_stats->napi_workdone); 1021 } 1022 1023 hif_print_napi_latency_stats(napii, ce_id); 1024 } 1025 } 1026 } 1027 #endif 1028 1029 #ifdef HIF_LATENCY_PROFILE_ENABLE 1030 void hif_clear_napi_stats(struct hif_opaque_softc *hif_ctx) 1031 { 1032 struct hif_softc *scn = HIF_GET_SOFTC(hif_ctx); 1033 struct qca_napi_info *napii; 1034 int ce_id; 1035 1036 for (ce_id = 0; ce_id < CE_COUNT_MAX; ce_id++) { 1037 if (!hif_napi_enabled(hif_ctx, ce_id)) 1038 continue; 1039 1040 napii = scn->napi_data.napis[ce_id]; 1041 if (napii) 1042 qdf_mem_set(napii->sched_latency_stats, 1043 sizeof(napii->sched_latency_stats), 0); 1044 } 1045 } 1046 #else 1047 inline void hif_clear_napi_stats(struct hif_opaque_softc *hif_ctx) 1048 { 1049 } 1050 #endif /* HIF_LATENCY_PROFILE_ENABLE */ 1051 1052 #else 1053 static inline void 1054 hif_napi_update_service_start_time(struct qca_napi_info *napi_info) 1055 { 1056 } 1057 1058 static inline void 1059 hif_napi_fill_poll_time_histogram(struct qca_napi_info *napi_info) 1060 { 1061 } 1062 #endif 1063 1064 /** 1065 * hif_napi_schedule() - schedules napi, updates stats 1066 * @hif_ctx: hif context 1067 * @ce_id: index of napi instance 1068 * 1069 * Return: false if napi didn't enable or already scheduled, otherwise true 1070 */ 1071 bool hif_napi_schedule(struct hif_opaque_softc *hif_ctx, int ce_id) 1072 { 1073 int cpu = smp_processor_id(); 1074 struct hif_softc *scn = HIF_GET_SOFTC(hif_ctx); 1075 struct qca_napi_info *napii; 1076 1077 napii = scn->napi_data.napis[ce_id]; 1078 if (qdf_unlikely(!napii)) { 1079 hif_err("scheduling unallocated napi (ce:%d)", ce_id); 1080 qdf_atomic_dec(&scn->active_tasklet_cnt); 1081 return false; 1082 } 1083 1084 if (test_bit(NAPI_STATE_SCHED, &napii->napi.state)) { 1085 NAPI_DEBUG("napi scheduled, return"); 1086 qdf_atomic_dec(&scn->active_tasklet_cnt); 1087 return false; 1088 } 1089 1090 hif_record_ce_desc_event(scn, ce_id, NAPI_SCHEDULE, 1091 NULL, NULL, 0, 0); 1092 napii->stats[cpu].napi_schedules++; 1093 NAPI_DEBUG("scheduling napi %d (ce:%d)", napii->id, ce_id); 1094 hif_napi_latency_profile_start(scn, ce_id); 1095 napi_schedule(&(napii->napi)); 1096 1097 return true; 1098 } 1099 qdf_export_symbol(hif_napi_schedule); 1100 1101 /** 1102 * hif_napi_correct_cpu() - correct the interrupt affinity for napi if needed 1103 * @napi_info: pointer to qca_napi_info for the napi instance 1104 * 1105 * Return: true => interrupt already on correct cpu, no correction needed 1106 * false => interrupt on wrong cpu, correction done for cpu affinity 1107 * of the interrupt 1108 */ 1109 static inline 1110 bool hif_napi_correct_cpu(struct qca_napi_info *napi_info) 1111 { 1112 bool right_cpu = true; 1113 int rc = 0; 1114 int cpu; 1115 struct qca_napi_data *napid; 1116 QDF_STATUS ret; 1117 1118 napid = hif_napi_get_all(GET_HIF_OPAQUE_HDL(napi_info->hif_ctx)); 1119 1120 if (napid->flags & QCA_NAPI_FEATURE_CPU_CORRECTION) { 1121 1122 cpu = qdf_get_cpu(); 1123 if (unlikely((hif_napi_cpu_denylist(napid, 1124 DENYLIST_QUERY) > 0) && 1125 cpu != napi_info->cpu)) { 1126 right_cpu = false; 1127 1128 NAPI_DEBUG("interrupt on wrong CPU, correcting"); 1129 napi_info->cpumask.bits[0] = (0x01 << napi_info->cpu); 1130 1131 qdf_dev_modify_irq_status(napi_info->irq, 1132 QDF_IRQ_NO_BALANCING, 0); 1133 ret = qdf_dev_set_irq_affinity(napi_info->irq, 1134 (struct qdf_cpu_mask *) 1135 &napi_info->cpumask); 1136 rc = qdf_status_to_os_return(ret); 1137 qdf_dev_modify_irq_status(napi_info->irq, 0, 1138 QDF_IRQ_NO_BALANCING); 1139 1140 if (rc) 1141 hif_err("Setting irq affinity hint: %d", rc); 1142 else 1143 napi_info->stats[cpu].cpu_corrected++; 1144 } 1145 } 1146 return right_cpu; 1147 } 1148 1149 #ifdef RECEIVE_OFFLOAD 1150 /** 1151 * hif_napi_offld_flush_cb() - Call upper layer flush callback 1152 * @napi_info: Handle to hif_napi_info 1153 * 1154 * Return: None 1155 */ 1156 static void hif_napi_offld_flush_cb(struct qca_napi_info *napi_info) 1157 { 1158 if (napi_info->offld_flush_cb) 1159 napi_info->offld_flush_cb(napi_info); 1160 } 1161 #else 1162 static void hif_napi_offld_flush_cb(struct qca_napi_info *napi_info) 1163 { 1164 } 1165 #endif 1166 1167 /** 1168 * hif_napi_poll() - NAPI poll routine 1169 * @hif_ctx: HIF context 1170 * @napi: pointer to NAPI struct as kernel holds it 1171 * @budget: 1172 * 1173 * This is the body of the poll function. 1174 * The poll function is called by kernel. So, there is a wrapper 1175 * function in HDD, which in turn calls this function. 1176 * Two main reasons why the whole thing is not implemented in HDD: 1177 * a) references to things like ce_service that HDD is not aware of 1178 * b) proximity to the implementation of ce_tasklet, which the body 1179 * of this function should be very close to. 1180 * 1181 * NOTE TO THE MAINTAINER: 1182 * Consider this function and ce_tasklet very tightly coupled pairs. 1183 * Any changes to ce_tasklet or this function may likely need to be 1184 * reflected in the counterpart. 1185 * 1186 * Returns: 1187 * int: the amount of work done in this poll (<= budget) 1188 */ 1189 int hif_napi_poll(struct hif_opaque_softc *hif_ctx, 1190 struct napi_struct *napi, 1191 int budget) 1192 { 1193 int rc = 0; /* default: no work done, also takes care of error */ 1194 int normalized = 0; 1195 int bucket; 1196 int cpu = smp_processor_id(); 1197 bool poll_on_right_cpu; 1198 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 1199 struct qca_napi_info *napi_info; 1200 struct CE_state *ce_state = NULL; 1201 1202 if (unlikely(!hif)) { 1203 hif_err("hif context is NULL"); 1204 QDF_ASSERT(0); 1205 goto out; 1206 } 1207 1208 napi_info = (struct qca_napi_info *) 1209 container_of(napi, struct qca_napi_info, napi); 1210 1211 hif_napi_update_service_start_time(napi_info); 1212 hif_napi_latency_profile_measure(napi_info); 1213 1214 NAPI_DEBUG("%s -->(napi(%d, irq=%d), budget=%d)", 1215 __func__, napi_info->id, napi_info->irq, budget); 1216 1217 napi_info->stats[cpu].napi_polls++; 1218 1219 hif_record_ce_desc_event(hif, NAPI_ID2PIPE(napi_info->id), 1220 NAPI_POLL_ENTER, NULL, NULL, cpu, 0); 1221 1222 rc = ce_per_engine_service(hif, NAPI_ID2PIPE(napi_info->id)); 1223 NAPI_DEBUG("%s: ce_per_engine_service processed %d msgs", 1224 __func__, rc); 1225 1226 hif_napi_offld_flush_cb(napi_info); 1227 1228 /* do not return 0, if there was some work done, 1229 * even if it is below the scale 1230 */ 1231 if (rc) { 1232 napi_info->stats[cpu].napi_workdone += rc; 1233 normalized = (rc / napi_info->scale); 1234 if (normalized == 0) 1235 normalized++; 1236 bucket = (normalized - 1) / 1237 (QCA_NAPI_BUDGET / QCA_NAPI_NUM_BUCKETS); 1238 if (bucket >= QCA_NAPI_NUM_BUCKETS) { 1239 bucket = QCA_NAPI_NUM_BUCKETS - 1; 1240 hif_err("Bad bucket#(%d) > QCA_NAPI_NUM_BUCKETS(%d)" 1241 " normalized %d, napi budget %d", 1242 bucket, QCA_NAPI_NUM_BUCKETS, 1243 normalized, QCA_NAPI_BUDGET); 1244 } 1245 napi_info->stats[cpu].napi_budget_uses[bucket]++; 1246 } else { 1247 /* if ce_per engine reports 0, then poll should be terminated */ 1248 NAPI_DEBUG("%s:%d: nothing processed by CE. Completing NAPI", 1249 __func__, __LINE__); 1250 } 1251 1252 ce_state = hif->ce_id_to_state[NAPI_ID2PIPE(napi_info->id)]; 1253 1254 /* 1255 * Not using the API hif_napi_correct_cpu directly in the if statement 1256 * below since the API may not get evaluated if put at the end if any 1257 * prior condition would evaluate to be true. The CPU correction 1258 * check should kick in every poll. 1259 */ 1260 #ifdef NAPI_YIELD_BUDGET_BASED 1261 if (ce_state && (ce_state->force_break || 0 == rc)) { 1262 #else 1263 poll_on_right_cpu = hif_napi_correct_cpu(napi_info); 1264 if ((ce_state) && 1265 (!ce_check_rx_pending(ce_state) || (0 == rc) || 1266 !poll_on_right_cpu)) { 1267 #endif 1268 napi_info->stats[cpu].napi_completes++; 1269 #ifdef NAPI_YIELD_BUDGET_BASED 1270 ce_state->force_break = 0; 1271 #endif 1272 1273 hif_record_ce_desc_event(hif, ce_state->id, NAPI_COMPLETE, 1274 NULL, NULL, 0, 0); 1275 if (normalized >= budget) 1276 normalized = budget - 1; 1277 1278 napi_complete(napi); 1279 /* enable interrupts */ 1280 hif_napi_enable_irq(hif_ctx, napi_info->id); 1281 /* support suspend/resume */ 1282 qdf_atomic_dec(&(hif->active_tasklet_cnt)); 1283 1284 NAPI_DEBUG("%s:%d: napi_complete + enabling the interrupts", 1285 __func__, __LINE__); 1286 } else { 1287 /* 4.4 kernel NAPI implementation requires drivers to 1288 * return full work when they ask to be re-scheduled, 1289 * or napi_complete and re-start with a fresh interrupt 1290 */ 1291 normalized = budget; 1292 } 1293 1294 hif_record_ce_desc_event(hif, NAPI_ID2PIPE(napi_info->id), 1295 NAPI_POLL_EXIT, NULL, NULL, normalized, 0); 1296 1297 hif_napi_fill_poll_time_histogram(napi_info); 1298 1299 NAPI_DEBUG("%s <--[normalized=%d]", __func__, normalized); 1300 return normalized; 1301 out: 1302 return rc; 1303 } 1304 qdf_export_symbol(hif_napi_poll); 1305 1306 void hif_update_napi_max_poll_time(struct CE_state *ce_state, 1307 int ce_id, 1308 int cpu_id) 1309 { 1310 struct hif_softc *hif; 1311 struct qca_napi_info *napi_info; 1312 unsigned long long napi_poll_time = qdf_time_sched_clock() - 1313 ce_state->ce_service_start_time; 1314 1315 hif = ce_state->scn; 1316 napi_info = hif->napi_data.napis[ce_id]; 1317 if (napi_poll_time > 1318 napi_info->stats[cpu_id].napi_max_poll_time) 1319 napi_info->stats[cpu_id].napi_max_poll_time = napi_poll_time; 1320 } 1321 qdf_export_symbol(hif_update_napi_max_poll_time); 1322 1323 #ifdef HIF_IRQ_AFFINITY 1324 /** 1325 * hif_napi_update_yield_stats() - update NAPI yield related stats 1326 * @ce_state: CE state structure 1327 * @time_limit_reached: indicates whether the time limit was reached 1328 * @rxpkt_thresh_reached: indicates whether rx packet threshold was reached 1329 * 1330 * Return: None 1331 */ 1332 void hif_napi_update_yield_stats(struct CE_state *ce_state, 1333 bool time_limit_reached, 1334 bool rxpkt_thresh_reached) 1335 { 1336 struct hif_softc *hif; 1337 struct qca_napi_data *napi_data = NULL; 1338 int ce_id = 0; 1339 int cpu_id = 0; 1340 1341 if (unlikely(!ce_state)) { 1342 QDF_ASSERT(ce_state); 1343 return; 1344 } 1345 1346 hif = ce_state->scn; 1347 1348 if (unlikely(!hif)) { 1349 QDF_ASSERT(hif); 1350 return; 1351 } 1352 napi_data = &(hif->napi_data); 1353 if (unlikely(!napi_data)) { 1354 QDF_ASSERT(napi_data); 1355 return; 1356 } 1357 1358 ce_id = ce_state->id; 1359 cpu_id = qdf_get_cpu(); 1360 1361 if (unlikely(!napi_data->napis[ce_id])) { 1362 return; 1363 } 1364 1365 if (time_limit_reached) 1366 napi_data->napis[ce_id]->stats[cpu_id].time_limit_reached++; 1367 else 1368 napi_data->napis[ce_id]->stats[cpu_id].rxpkt_thresh_reached++; 1369 1370 hif_update_napi_max_poll_time(ce_state, ce_id, 1371 cpu_id); 1372 } 1373 1374 /** 1375 * hif_napi_stats() - display NAPI CPU statistics 1376 * @napid: pointer to qca_napi_data 1377 * 1378 * Description: 1379 * Prints the various CPU cores on which the NAPI instances /CEs interrupts 1380 * are being executed. Can be called from outside NAPI layer. 1381 * 1382 * Return: None 1383 */ 1384 void hif_napi_stats(struct qca_napi_data *napid) 1385 { 1386 int i; 1387 struct qca_napi_cpu *cpu; 1388 1389 if (!napid) { 1390 qdf_debug("%s: napiid struct is null", __func__); 1391 return; 1392 } 1393 1394 cpu = napid->napi_cpu; 1395 qdf_debug("NAPI CPU TABLE"); 1396 qdf_debug("lilclhead=%d, bigclhead=%d", 1397 napid->lilcl_head, napid->bigcl_head); 1398 for (i = 0; i < NR_CPUS; i++) { 1399 qdf_debug("CPU[%02d]: state:%d crid=%02d clid=%02d crmk:0x%0lx thmk:0x%0lx frq:%d napi = 0x%08x lnk:%d", 1400 i, 1401 cpu[i].state, cpu[i].core_id, cpu[i].cluster_id, 1402 cpu[i].core_mask.bits[0], 1403 cpu[i].thread_mask.bits[0], 1404 cpu[i].max_freq, cpu[i].napis, 1405 cpu[i].cluster_nxt); 1406 } 1407 } 1408 1409 #ifdef FEATURE_NAPI_DEBUG 1410 /* 1411 * Local functions 1412 * - no argument checks, all internal/trusted callers 1413 */ 1414 static void hnc_dump_cpus(struct qca_napi_data *napid) 1415 { 1416 hif_napi_stats(napid); 1417 } 1418 #else 1419 static void hnc_dump_cpus(struct qca_napi_data *napid) { /* no-op */ }; 1420 #endif /* FEATURE_NAPI_DEBUG */ 1421 1422 #define HNC_MIN_CLUSTER 0 1423 #define HNC_MAX_CLUSTER 1 1424 1425 /** 1426 * hnc_link_clusters() - partitions to cpu table into clusters 1427 * @napid: pointer to NAPI data 1428 * 1429 * Takes in a CPU topology table and builds two linked lists 1430 * (big cluster cores, list-head at bigcl_head, and little cluster 1431 * cores, list-head at lilcl_head) out of it. 1432 * 1433 * If there are more than two clusters: 1434 * - bigcl_head and lilcl_head will be different, 1435 * - the cluster with highest cpufreq will be considered the "big" cluster. 1436 * If there are more than one with the highest frequency, the *last* of such 1437 * clusters will be designated as the "big cluster" 1438 * - the cluster with lowest cpufreq will be considered the "li'l" cluster. 1439 * If there are more than one clusters with the lowest cpu freq, the *first* 1440 * of such clusters will be designated as the "little cluster" 1441 * - We only support up to 32 clusters 1442 * Return: 0 : OK 1443 * !0: error (at least one of lil/big clusters could not be found) 1444 */ 1445 static int hnc_link_clusters(struct qca_napi_data *napid) 1446 { 1447 int rc = 0; 1448 1449 int i; 1450 int it = 0; 1451 uint32_t cl_done = 0x0; 1452 int cl, curcl, curclhead = 0; 1453 int more; 1454 unsigned int lilfrq = INT_MAX; 1455 unsigned int bigfrq = 0; 1456 unsigned int clfrq = 0; 1457 int prev = 0; 1458 struct qca_napi_cpu *cpus = napid->napi_cpu; 1459 1460 napid->lilcl_head = napid->bigcl_head = -1; 1461 1462 do { 1463 more = 0; 1464 it++; curcl = -1; 1465 for (i = 0; i < NR_CPUS; i++) { 1466 cl = cpus[i].cluster_id; 1467 NAPI_DEBUG("Processing cpu[%d], cluster=%d\n", 1468 i, cl); 1469 if ((cl < HNC_MIN_CLUSTER) || (cl > HNC_MAX_CLUSTER)) { 1470 NAPI_DEBUG("Bad cluster (%d). SKIPPED\n", cl); 1471 /* continue if ASSERTs are disabled */ 1472 continue; 1473 }; 1474 if (cpumask_weight(&(cpus[i].core_mask)) == 0) { 1475 NAPI_DEBUG("Core mask 0. SKIPPED\n"); 1476 continue; 1477 } 1478 if (cl_done & (0x01 << cl)) { 1479 NAPI_DEBUG("Cluster already processed. SKIPPED\n"); 1480 continue; 1481 } else { 1482 if (more == 0) { 1483 more = 1; 1484 curcl = cl; 1485 curclhead = i; /* row */ 1486 clfrq = cpus[i].max_freq; 1487 prev = -1; 1488 }; 1489 if ((curcl >= 0) && (curcl != cl)) { 1490 NAPI_DEBUG("Entry cl(%d) != curcl(%d). SKIPPED\n", 1491 cl, curcl); 1492 continue; 1493 } 1494 if (cpus[i].max_freq != clfrq) 1495 NAPI_DEBUG("WARN: frq(%d)!=clfrq(%d)\n", 1496 cpus[i].max_freq, clfrq); 1497 if (clfrq >= bigfrq) { 1498 bigfrq = clfrq; 1499 napid->bigcl_head = curclhead; 1500 NAPI_DEBUG("bigcl=%d\n", curclhead); 1501 } 1502 if (clfrq < lilfrq) { 1503 lilfrq = clfrq; 1504 napid->lilcl_head = curclhead; 1505 NAPI_DEBUG("lilcl=%d\n", curclhead); 1506 } 1507 if (prev != -1) 1508 cpus[prev].cluster_nxt = i; 1509 1510 prev = i; 1511 } 1512 } 1513 if (curcl >= 0) 1514 cl_done |= (0x01 << curcl); 1515 1516 } while (more); 1517 1518 if (qdf_unlikely((napid->lilcl_head < 0) && (napid->bigcl_head < 0))) 1519 rc = -EFAULT; 1520 1521 hnc_dump_cpus(napid); /* if NAPI_DEBUG */ 1522 return rc; 1523 } 1524 #undef HNC_MIN_CLUSTER 1525 #undef HNC_MAX_CLUSTER 1526 1527 /* 1528 * hotplug function group 1529 */ 1530 1531 /** 1532 * hnc_cpu_online_cb() - handles CPU hotplug "up" events 1533 * @context: the associated HIF context 1534 * @cpu: the CPU Id of the CPU the event happened on 1535 * 1536 * Return: None 1537 */ 1538 static void hnc_cpu_online_cb(void *context, uint32_t cpu) 1539 { 1540 struct hif_softc *hif = context; 1541 struct qca_napi_data *napid = &hif->napi_data; 1542 1543 if (cpu >= NR_CPUS) 1544 return; 1545 1546 NAPI_DEBUG("-->%s(act=online, cpu=%u)", __func__, cpu); 1547 1548 napid->napi_cpu[cpu].state = QCA_NAPI_CPU_UP; 1549 NAPI_DEBUG("%s: CPU %u marked %d", 1550 __func__, cpu, napid->napi_cpu[cpu].state); 1551 1552 NAPI_DEBUG("<--%s", __func__); 1553 } 1554 1555 /** 1556 * hnc_cpu_before_offline_cb() - handles CPU hotplug "prepare down" events 1557 * @context: the associated HIF context 1558 * @cpu: the CPU Id of the CPU the event happened on 1559 * 1560 * On transition to offline, we act on PREP events, because we may need to move 1561 * the irqs/NAPIs to another CPU before it is actually off-lined. 1562 * 1563 * Return: None 1564 */ 1565 static void hnc_cpu_before_offline_cb(void *context, uint32_t cpu) 1566 { 1567 struct hif_softc *hif = context; 1568 struct qca_napi_data *napid = &hif->napi_data; 1569 1570 if (cpu >= NR_CPUS) 1571 return; 1572 1573 NAPI_DEBUG("-->%s(act=before_offline, cpu=%u)", __func__, cpu); 1574 1575 napid->napi_cpu[cpu].state = QCA_NAPI_CPU_DOWN; 1576 1577 NAPI_DEBUG("%s: CPU %u marked %d; updating affinity", 1578 __func__, cpu, napid->napi_cpu[cpu].state); 1579 1580 /** 1581 * we need to move any NAPIs on this CPU out. 1582 * if we are in LO throughput mode, then this is valid 1583 * if the CPU is the the low designated CPU. 1584 */ 1585 hif_napi_event(GET_HIF_OPAQUE_HDL(hif), 1586 NAPI_EVT_CPU_STATE, 1587 (void *) 1588 ((size_t)cpu << 16 | napid->napi_cpu[cpu].state)); 1589 1590 NAPI_DEBUG("<--%s", __func__); 1591 } 1592 1593 static int hnc_hotplug_register(struct hif_softc *hif_sc) 1594 { 1595 QDF_STATUS status; 1596 1597 NAPI_DEBUG("-->%s", __func__); 1598 1599 status = qdf_cpuhp_register(&hif_sc->napi_data.cpuhp_handler, 1600 hif_sc, 1601 hnc_cpu_online_cb, 1602 hnc_cpu_before_offline_cb); 1603 1604 NAPI_DEBUG("<--%s [%d]", __func__, status); 1605 1606 return qdf_status_to_os_return(status); 1607 } 1608 1609 static void hnc_hotplug_unregister(struct hif_softc *hif_sc) 1610 { 1611 NAPI_DEBUG("-->%s", __func__); 1612 1613 if (hif_sc->napi_data.cpuhp_handler) 1614 qdf_cpuhp_unregister(&hif_sc->napi_data.cpuhp_handler); 1615 1616 NAPI_DEBUG("<--%s", __func__); 1617 } 1618 1619 /** 1620 * hnc_tput_hook() - installs a callback in the throughput detector 1621 * @install: !0 => install; =0: uninstall 1622 * 1623 * installs a callback to be called when wifi driver throughput (tx+rx) 1624 * crosses a threshold. Currently, we are using the same criteria as 1625 * TCP ack suppression (500 packets/100ms by default). 1626 * 1627 * Return: 0 : success 1628 * <0: failure 1629 */ 1630 1631 static int hnc_tput_hook(int install) 1632 { 1633 int rc = 0; 1634 1635 /* 1636 * Nothing, until the bw_calculation accepts registration 1637 * it is now hardcoded in the wlan_hdd_main.c::hdd_bus_bw_compute_cbk 1638 * hdd_napi_throughput_policy(...) 1639 */ 1640 return rc; 1641 } 1642 1643 /* 1644 * Implementation of hif_napi_cpu API 1645 */ 1646 1647 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) 1648 static inline void record_sibling_cpumask(struct qca_napi_cpu *cpus, int i) 1649 { 1650 cpumask_copy(&(cpus[i].thread_mask), 1651 topology_sibling_cpumask(i)); 1652 } 1653 #else 1654 static inline void record_sibling_cpumask(struct qca_napi_cpu *cpus, int i) 1655 { 1656 } 1657 #endif 1658 1659 1660 /** 1661 * hif_napi_cpu_init() - initialization of irq affinity block 1662 * @hif: HIF context 1663 * 1664 * called by hif_napi_create, after the first instance is called 1665 * - builds napi_rss_cpus table from cpu topology 1666 * - links cores of the same clusters together 1667 * - installs hot-plug notifier 1668 * - installs throughput trigger notifier (when such mechanism exists) 1669 * 1670 * Return: 0: OK 1671 * <0: error code 1672 */ 1673 int hif_napi_cpu_init(struct hif_opaque_softc *hif) 1674 { 1675 int rc = 0; 1676 int i; 1677 struct qca_napi_data *napid = &HIF_GET_SOFTC(hif)->napi_data; 1678 struct qca_napi_cpu *cpus = napid->napi_cpu; 1679 1680 NAPI_DEBUG("--> "); 1681 1682 if (cpus[0].state != QCA_NAPI_CPU_UNINITIALIZED) { 1683 NAPI_DEBUG("NAPI RSS table already initialized.\n"); 1684 rc = -EALREADY; 1685 goto lab_rss_init; 1686 } 1687 1688 /* build CPU topology table */ 1689 for_each_possible_cpu(i) { 1690 cpus[i].state = ((cpumask_test_cpu(i, cpu_online_mask) 1691 ? QCA_NAPI_CPU_UP 1692 : QCA_NAPI_CPU_DOWN)); 1693 cpus[i].core_id = topology_core_id(i); 1694 cpus[i].cluster_id = topology_physical_package_id(i); 1695 cpumask_copy(&(cpus[i].core_mask), 1696 topology_core_cpumask(i)); 1697 record_sibling_cpumask(cpus, i); 1698 cpus[i].max_freq = cpufreq_quick_get_max(i); 1699 cpus[i].napis = 0x0; 1700 cpus[i].cluster_nxt = -1; /* invalid */ 1701 } 1702 1703 /* link clusters together */ 1704 rc = hnc_link_clusters(napid); 1705 if (0 != rc) 1706 goto lab_err_topology; 1707 1708 /* install hotplug notifier */ 1709 rc = hnc_hotplug_register(HIF_GET_SOFTC(hif)); 1710 if (0 != rc) 1711 goto lab_err_hotplug; 1712 1713 /* install throughput notifier */ 1714 rc = hnc_tput_hook(1); 1715 if (0 == rc) 1716 goto lab_rss_init; 1717 1718 lab_err_hotplug: 1719 hnc_tput_hook(0); 1720 hnc_hotplug_unregister(HIF_GET_SOFTC(hif)); 1721 lab_err_topology: 1722 memset(napid->napi_cpu, 0, sizeof(struct qca_napi_cpu) * NR_CPUS); 1723 lab_rss_init: 1724 NAPI_DEBUG("<-- [rc=%d]", rc); 1725 return rc; 1726 } 1727 1728 /** 1729 * hif_napi_cpu_deinit() - clean-up of irq affinity block 1730 * @hif: HIF context 1731 * 1732 * called by hif_napi_destroy, when the last instance is removed 1733 * - uninstalls throughput and hotplug notifiers 1734 * - clears cpu topology table 1735 * Return: 0: OK 1736 */ 1737 int hif_napi_cpu_deinit(struct hif_opaque_softc *hif) 1738 { 1739 int rc = 0; 1740 struct qca_napi_data *napid = &HIF_GET_SOFTC(hif)->napi_data; 1741 1742 NAPI_DEBUG("-->%s(...)", __func__); 1743 1744 /* uninstall tput notifier */ 1745 rc = hnc_tput_hook(0); 1746 1747 /* uninstall hotplug notifier */ 1748 hnc_hotplug_unregister(HIF_GET_SOFTC(hif)); 1749 1750 /* clear the topology table */ 1751 memset(napid->napi_cpu, 0, sizeof(struct qca_napi_cpu) * NR_CPUS); 1752 1753 NAPI_DEBUG("<--%s[rc=%d]", __func__, rc); 1754 1755 return rc; 1756 } 1757 1758 /** 1759 * hncm_migrate_to() - migrates a NAPI to a CPU 1760 * @napid: pointer to NAPI block 1761 * @napi_ce: CE_id of the NAPI instance 1762 * @didx: index in the CPU topology table for the CPU to migrate to 1763 * 1764 * Migrates NAPI (identified by the CE_id) to the destination core 1765 * Updates the napi_map of the destination entry 1766 * 1767 * Return: 1768 * =0 : success 1769 * <0 : error 1770 */ 1771 static int hncm_migrate_to(struct qca_napi_data *napid, 1772 int napi_ce, 1773 int didx) 1774 { 1775 int rc = 0; 1776 QDF_STATUS status; 1777 1778 NAPI_DEBUG("-->%s(napi_cd=%d, didx=%d)", __func__, napi_ce, didx); 1779 1780 if (!napid->napis[napi_ce]) 1781 return -EINVAL; 1782 1783 napid->napis[napi_ce]->cpumask.bits[0] = (1 << didx); 1784 1785 qdf_dev_modify_irq_status(napid->napis[napi_ce]->irq, 1786 QDF_IRQ_NO_BALANCING, 0); 1787 status = qdf_dev_set_irq_affinity(napid->napis[napi_ce]->irq, 1788 (struct qdf_cpu_mask *) 1789 &napid->napis[napi_ce]->cpumask); 1790 rc = qdf_status_to_os_return(status); 1791 1792 /* unmark the napis bitmap in the cpu table */ 1793 napid->napi_cpu[napid->napis[napi_ce]->cpu].napis &= ~(0x01 << napi_ce); 1794 /* mark the napis bitmap for the new designated cpu */ 1795 napid->napi_cpu[didx].napis |= (0x01 << napi_ce); 1796 napid->napis[napi_ce]->cpu = didx; 1797 1798 NAPI_DEBUG("<--%s[%d]", __func__, rc); 1799 return rc; 1800 } 1801 /** 1802 * hncm_dest_cpu() - finds a destination CPU for NAPI 1803 * @napid: pointer to NAPI block 1804 * @act: RELOCATE | COLLAPSE | DISPERSE 1805 * 1806 * Finds the designated destination for the next IRQ. 1807 * RELOCATE: translated to either COLLAPSE or DISPERSE based 1808 * on napid->napi_mode (throughput state) 1809 * COLLAPSE: All have the same destination: the first online CPU in lilcl 1810 * DISPERSE: One of the CPU in bigcl, which has the smallest number of 1811 * NAPIs on it 1812 * 1813 * Return: >=0 : index in the cpu topology table 1814 * : < 0 : error 1815 */ 1816 static int hncm_dest_cpu(struct qca_napi_data *napid, int act) 1817 { 1818 int destidx = -1; 1819 int head, i; 1820 1821 NAPI_DEBUG("-->%s(act=%d)", __func__, act); 1822 if (act == HNC_ACT_RELOCATE) { 1823 if (napid->napi_mode == QCA_NAPI_TPUT_LO) 1824 act = HNC_ACT_COLLAPSE; 1825 else 1826 act = HNC_ACT_DISPERSE; 1827 NAPI_DEBUG("%s: act changed from HNC_ACT_RELOCATE to %d", 1828 __func__, act); 1829 } 1830 if (act == HNC_ACT_COLLAPSE) { 1831 head = i = napid->lilcl_head; 1832 retry_collapse: 1833 while (i >= 0) { 1834 if (napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) { 1835 destidx = i; 1836 break; 1837 } 1838 i = napid->napi_cpu[i].cluster_nxt; 1839 } 1840 if ((destidx < 0) && (head == napid->lilcl_head)) { 1841 NAPI_DEBUG("%s: COLLAPSE: no lilcl dest, try bigcl", 1842 __func__); 1843 head = i = napid->bigcl_head; 1844 goto retry_collapse; 1845 } 1846 } else { /* HNC_ACT_DISPERSE */ 1847 int smallest = 99; /* all 32 bits full */ 1848 int smallidx = -1; 1849 1850 head = i = napid->bigcl_head; 1851 retry_disperse: 1852 while (i >= 0) { 1853 if ((napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) && 1854 (hweight32(napid->napi_cpu[i].napis) <= smallest)) { 1855 smallest = napid->napi_cpu[i].napis; 1856 smallidx = i; 1857 } 1858 i = napid->napi_cpu[i].cluster_nxt; 1859 } 1860 /* Check if matches with user specified CPU mask */ 1861 smallidx = ((1 << smallidx) & napid->user_cpu_affin_mask) ? 1862 smallidx : -1; 1863 1864 if ((smallidx < 0) && (head == napid->bigcl_head)) { 1865 NAPI_DEBUG("%s: DISPERSE: no bigcl dest, try lilcl", 1866 __func__); 1867 head = i = napid->lilcl_head; 1868 goto retry_disperse; 1869 } 1870 destidx = smallidx; 1871 } 1872 NAPI_DEBUG("<--%s[dest=%d]", __func__, destidx); 1873 return destidx; 1874 } 1875 /** 1876 * hif_napi_cpu_migrate() - migrate IRQs away 1877 * @napid: pointer to NAPI block 1878 * @cpu: -1: all CPUs <n> specific CPU 1879 * @action: COLLAPSE | DISPERSE 1880 * 1881 * Moves IRQs/NAPIs from specific or all CPUs (specified by @cpu) to eligible 1882 * cores. Eligible cores are: 1883 * act=COLLAPSE -> the first online core of the little cluster 1884 * act=DISPERSE -> separate cores of the big cluster, so that each core will 1885 * host minimum number of NAPIs/IRQs (napid->cpus[cpu].napis) 1886 * 1887 * Note that this function is called with a spinlock acquired already. 1888 * 1889 * Return: =0: success 1890 * <0: error 1891 */ 1892 1893 int hif_napi_cpu_migrate(struct qca_napi_data *napid, int cpu, int action) 1894 { 1895 int rc = 0; 1896 struct qca_napi_cpu *cpup; 1897 int i, dind; 1898 uint32_t napis; 1899 1900 NAPI_DEBUG("-->%s(.., cpu=%d, act=%d)", 1901 __func__, cpu, action); 1902 /* the following is really: hif_napi_enabled() with less overhead */ 1903 if (napid->ce_map == 0) { 1904 NAPI_DEBUG("%s: NAPI disabled. Not migrating.", __func__); 1905 goto hncm_return; 1906 } 1907 1908 cpup = napid->napi_cpu; 1909 1910 switch (action) { 1911 case HNC_ACT_RELOCATE: 1912 case HNC_ACT_DISPERSE: 1913 case HNC_ACT_COLLAPSE: { 1914 /* first find the src napi set */ 1915 if (cpu == HNC_ANY_CPU) 1916 napis = napid->ce_map; 1917 else 1918 napis = cpup[cpu].napis; 1919 /* then clear the napi bitmap on each CPU */ 1920 for (i = 0; i < NR_CPUS; i++) 1921 cpup[i].napis = 0; 1922 /* then for each of the NAPIs to disperse: */ 1923 for (i = 0; i < CE_COUNT_MAX; i++) 1924 if (napis & (1 << i)) { 1925 /* find a destination CPU */ 1926 dind = hncm_dest_cpu(napid, action); 1927 if (dind >= 0) { 1928 NAPI_DEBUG("Migrating NAPI ce%d to %d", 1929 i, dind); 1930 rc = hncm_migrate_to(napid, i, dind); 1931 } else { 1932 NAPI_DEBUG("No dest for NAPI ce%d", i); 1933 hnc_dump_cpus(napid); 1934 rc = -1; 1935 } 1936 } 1937 break; 1938 } 1939 default: { 1940 NAPI_DEBUG("%s: bad action: %d\n", __func__, action); 1941 QDF_BUG(0); 1942 break; 1943 } 1944 } /* switch action */ 1945 1946 hncm_return: 1947 hnc_dump_cpus(napid); 1948 return rc; 1949 } 1950 1951 1952 /** 1953 * hif_napi_dl_irq() - calls irq_modify_status to enable/disable denylisting 1954 * @napid: pointer to qca_napi_data structure 1955 * @dl_flag: denylist flag to enable/disable denylisting 1956 * 1957 * The function enables/disables denylisting for all the copy engine 1958 * interrupts on which NAPI is enabled. 1959 * 1960 * Return: None 1961 */ 1962 static inline void hif_napi_dl_irq(struct qca_napi_data *napid, bool dl_flag) 1963 { 1964 int i; 1965 struct qca_napi_info *napii; 1966 1967 for (i = 0; i < CE_COUNT_MAX; i++) { 1968 /* check if NAPI is enabled on the CE */ 1969 if (!(napid->ce_map & (0x01 << i))) 1970 continue; 1971 1972 /*double check that NAPI is allocated for the CE */ 1973 napii = napid->napis[i]; 1974 if (!(napii)) 1975 continue; 1976 1977 if (dl_flag == true) 1978 qdf_dev_modify_irq_status(napii->irq, 1979 0, QDF_IRQ_NO_BALANCING); 1980 else 1981 qdf_dev_modify_irq_status(napii->irq, 1982 QDF_IRQ_NO_BALANCING, 0); 1983 hif_debug("dl_flag %d CE %d", dl_flag, i); 1984 } 1985 } 1986 1987 /** 1988 * hif_napi_cpu_denylist() - en(dis)ables denylisting for NAPI RX interrupts. 1989 * @napid: pointer to qca_napi_data structure 1990 * @op: denylist operation to perform 1991 * 1992 * The function enables/disables/queries denylisting for all CE RX 1993 * interrupts with NAPI enabled. Besides denylisting, it also enables/disables 1994 * core_ctl_set_boost. 1995 * Once denylisting is enabled, the interrupts will not be managed by the IRQ 1996 * balancer. 1997 * 1998 * Return: -EINVAL, in case IRQ_DENYLISTING and CORE_CTL_BOOST is not enabled 1999 * for DENYLIST_QUERY op - denylist refcount 2000 * for DENYLIST_ON op - return value from core_ctl_set_boost API 2001 * for DENYLIST_OFF op - return value from core_ctl_set_boost API 2002 */ 2003 int hif_napi_cpu_denylist(struct qca_napi_data *napid, 2004 enum qca_denylist_op op) 2005 { 2006 int rc = 0; 2007 static int ref_count; /* = 0 by the compiler */ 2008 uint8_t flags = napid->flags; 2009 bool dl_en = flags & QCA_NAPI_FEATURE_IRQ_BLACKLISTING; 2010 bool ccb_en = flags & QCA_NAPI_FEATURE_CORE_CTL_BOOST; 2011 2012 NAPI_DEBUG("-->%s(%d %d)", __func__, flags, op); 2013 2014 if (!(dl_en && ccb_en)) { 2015 rc = -EINVAL; 2016 goto out; 2017 } 2018 2019 switch (op) { 2020 case DENYLIST_QUERY: 2021 rc = ref_count; 2022 break; 2023 case DENYLIST_ON: 2024 ref_count++; 2025 rc = 0; 2026 if (ref_count == 1) { 2027 rc = hif_napi_core_ctl_set_boost(true); 2028 NAPI_DEBUG("boost_on() returns %d - refcnt=%d", 2029 rc, ref_count); 2030 hif_napi_dl_irq(napid, true); 2031 } 2032 break; 2033 case DENYLIST_OFF: 2034 if (ref_count) { 2035 ref_count--; 2036 rc = 0; 2037 if (ref_count == 0) { 2038 rc = hif_napi_core_ctl_set_boost(false); 2039 NAPI_DEBUG("boost_off() returns %d - refcnt=%d", 2040 rc, ref_count); 2041 hif_napi_dl_irq(napid, false); 2042 } 2043 } 2044 break; 2045 default: 2046 NAPI_DEBUG("Invalid denylist op: %d", op); 2047 rc = -EINVAL; 2048 } /* switch */ 2049 out: 2050 NAPI_DEBUG("<--%s[%d]", __func__, rc); 2051 return rc; 2052 } 2053 2054 static unsigned long napi_serialize_reqs; 2055 /** 2056 * hif_napi_serialize() - [de-]serialize NAPI operations 2057 * @hif: context 2058 * @is_on: 1: serialize, 0: deserialize 2059 * 2060 * hif_napi_serialize(hif, 1) can be called multiple times. It will perform the 2061 * following steps (see hif_napi_event for code): 2062 * - put irqs of all NAPI instances on the same CPU 2063 * - only for the first serialize call: denylist 2064 * 2065 * hif_napi_serialize(hif, 0): 2066 * - start a timer (multiple of BusBandwidthTimer -- default: 100 msec) 2067 * - at the end of the timer, check the current throughput state and 2068 * implement it. 2069 */ 2070 int hif_napi_serialize(struct hif_opaque_softc *hif, int is_on) 2071 { 2072 int rc = -EINVAL; 2073 2074 if (hif) 2075 switch (is_on) { 2076 case 0: { /* de-serialize */ 2077 rc = hif_napi_event(hif, NAPI_EVT_USR_NORMAL, 2078 (void *) 0); 2079 napi_serialize_reqs = 0; 2080 break; 2081 } /* end de-serialize */ 2082 case 1: { /* serialize */ 2083 rc = hif_napi_event(hif, NAPI_EVT_USR_SERIAL, 2084 (void *)napi_serialize_reqs++); 2085 break; 2086 } /* end serialize */ 2087 default: 2088 break; /* no-op */ 2089 } /* switch */ 2090 return rc; 2091 } 2092 2093 #endif /* ifdef HIF_IRQ_AFFINITY */ 2094