1 /* 2 * Copyright (c) 2015-2018 The Linux Foundation. All rights reserved. 3 * 4 * Previously licensed under the ISC license by Qualcomm Atheros, Inc. 5 * 6 * 7 * Permission to use, copy, modify, and/or distribute this software for 8 * any purpose with or without fee is hereby granted, provided that the 9 * above copyright notice and this permission notice appear in all 10 * copies. 11 * 12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL 13 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED 14 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE 15 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 16 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR 17 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 18 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 19 * PERFORMANCE OF THIS SOFTWARE. 20 */ 21 22 /* 23 * This file was originally distributed by Qualcomm Atheros, Inc. 24 * under proprietary terms before Copyright ownership was assigned 25 * to the Linux Foundation. 26 */ 27 28 /** 29 * DOC: hif_napi.c 30 * 31 * HIF NAPI interface implementation 32 */ 33 34 #include <linux/string.h> /* memset */ 35 36 /* Linux headers */ 37 #include <linux/cpumask.h> 38 #include <linux/cpufreq.h> 39 #include <linux/cpu.h> 40 #include <linux/topology.h> 41 #include <linux/interrupt.h> 42 #include <linux/irq.h> 43 #ifdef CONFIG_SCHED_CORE_CTL 44 #include <linux/sched/core_ctl.h> 45 #endif 46 #include <pld_common.h> 47 #include <linux/pm.h> 48 49 /* Driver headers */ 50 #include <hif_napi.h> 51 #include <hif_debug.h> 52 #include <hif_io32.h> 53 #include <ce_api.h> 54 #include <ce_internal.h> 55 #include <hif_irq_affinity.h> 56 #include "qdf_cpuhp.h" 57 #include "qdf_module.h" 58 59 enum napi_decision_vector { 60 HIF_NAPI_NOEVENT = 0, 61 HIF_NAPI_INITED = 1, 62 HIF_NAPI_CONF_UP = 2 63 }; 64 #define ENABLE_NAPI_MASK (HIF_NAPI_INITED | HIF_NAPI_CONF_UP) 65 66 /** 67 * hif_napi_create() - creates the NAPI structures for a given CE 68 * @hif : pointer to hif context 69 * @pipe_id: the CE id on which the instance will be created 70 * @poll : poll function to be used for this NAPI instance 71 * @budget : budget to be registered with the NAPI instance 72 * @scale : scale factor on the weight (to scaler budget to 1000) 73 * @flags : feature flags 74 * 75 * Description: 76 * Creates NAPI instances. This function is called 77 * unconditionally during initialization. It creates 78 * napi structures through the proper HTC/HIF calls. 79 * The structures are disabled on creation. 80 * Note that for each NAPI instance a separate dummy netdev is used 81 * 82 * Return: 83 * < 0: error 84 * = 0: <should never happen> 85 * > 0: id of the created object (for multi-NAPI, number of objects created) 86 */ 87 int hif_napi_create(struct hif_opaque_softc *hif_ctx, 88 int (*poll)(struct napi_struct *, int), 89 int budget, 90 int scale, 91 uint8_t flags) 92 { 93 int i; 94 struct qca_napi_data *napid; 95 struct qca_napi_info *napii; 96 struct CE_state *ce_state; 97 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 98 int rc = 0; 99 100 NAPI_DEBUG("-->(budget=%d, scale=%d)", 101 budget, scale); 102 NAPI_DEBUG("hif->napi_data.state = 0x%08x", 103 hif->napi_data.state); 104 NAPI_DEBUG("hif->napi_data.ce_map = 0x%08x", 105 hif->napi_data.ce_map); 106 107 napid = &(hif->napi_data); 108 if (0 == (napid->state & HIF_NAPI_INITED)) { 109 memset(napid, 0, sizeof(struct qca_napi_data)); 110 qdf_spinlock_create(&(napid->lock)); 111 112 napid->state |= HIF_NAPI_INITED; 113 napid->flags = flags; 114 115 rc = hif_napi_cpu_init(hif_ctx); 116 if (rc != 0 && rc != -EALREADY) { 117 HIF_ERROR("NAPI_initialization failed,. %d", rc); 118 rc = napid->ce_map; 119 goto hnc_err; 120 } else 121 rc = 0; 122 123 HIF_DBG("%s: NAPI structures initialized, rc=%d", 124 __func__, rc); 125 } 126 for (i = 0; i < hif->ce_count; i++) { 127 ce_state = hif->ce_id_to_state[i]; 128 NAPI_DEBUG("ce %d: htt_rx=%d htt_tx=%d", 129 i, ce_state->htt_rx_data, 130 ce_state->htt_tx_data); 131 if (ce_srng_based(hif)) 132 continue; 133 134 if (!ce_state->htt_rx_data) 135 continue; 136 137 /* Now this is a CE where we need NAPI on */ 138 NAPI_DEBUG("Creating NAPI on pipe %d", i); 139 napii = qdf_mem_malloc(sizeof(*napii)); 140 napid->napis[i] = napii; 141 if (!napii) { 142 NAPI_DEBUG("NAPI alloc failure %d", i); 143 rc = -ENOMEM; 144 goto napii_alloc_failure; 145 } 146 } 147 148 for (i = 0; i < hif->ce_count; i++) { 149 napii = napid->napis[i]; 150 if (!napii) 151 continue; 152 153 NAPI_DEBUG("initializing NAPI for pipe %d", i); 154 memset(napii, 0, sizeof(struct qca_napi_info)); 155 napii->scale = scale; 156 napii->id = NAPI_PIPE2ID(i); 157 napii->hif_ctx = hif_ctx; 158 napii->irq = pld_get_irq(hif->qdf_dev->dev, i); 159 160 if (napii->irq < 0) 161 HIF_WARN("%s: bad IRQ value for CE %d: %d", 162 __func__, i, napii->irq); 163 164 init_dummy_netdev(&(napii->netdev)); 165 166 NAPI_DEBUG("adding napi=%pK to netdev=%pK (poll=%pK, bdgt=%d)", 167 &(napii->napi), &(napii->netdev), poll, budget); 168 netif_napi_add(&(napii->netdev), &(napii->napi), poll, budget); 169 170 NAPI_DEBUG("after napi_add"); 171 NAPI_DEBUG("napi=0x%pK, netdev=0x%pK", 172 &(napii->napi), &(napii->netdev)); 173 NAPI_DEBUG("napi.dev_list.prev=0x%pK, next=0x%pK", 174 napii->napi.dev_list.prev, 175 napii->napi.dev_list.next); 176 NAPI_DEBUG("dev.napi_list.prev=0x%pK, next=0x%pK", 177 napii->netdev.napi_list.prev, 178 napii->netdev.napi_list.next); 179 180 napii->lro_ctx = qdf_lro_init(); 181 NAPI_DEBUG("Registering LRO for ce_id %d NAPI callback for %d lro_ctx %pK\n", 182 i, napii->id, napii->lro_ctx); 183 184 /* It is OK to change the state variable below without 185 * protection as there should be no-one around yet 186 */ 187 napid->ce_map |= (0x01 << i); 188 HIF_DBG("%s: NAPI id %d created for pipe %d", __func__, 189 napii->id, i); 190 } 191 192 /* no ces registered with the napi */ 193 if (!ce_srng_based(hif) && napid->ce_map == 0) { 194 HIF_WARN("%s: no napis created for copy engines", __func__); 195 return -EFAULT; 196 } 197 198 NAPI_DEBUG("napi map = %x", napid->ce_map); 199 NAPI_DEBUG("NAPI ids created for all applicable pipes"); 200 return napid->ce_map; 201 202 napii_alloc_failure: 203 for (i = 0; i < hif->ce_count; i++) { 204 napii = napid->napis[i]; 205 napid->napis[i] = NULL; 206 if (napii) 207 qdf_mem_free(napii); 208 } 209 210 hnc_err: 211 NAPI_DEBUG("<--napi_instances_map=%x]", napid->ce_map); 212 return rc; 213 } 214 qdf_export_symbol(hif_napi_create); 215 216 /** 217 * 218 * hif_napi_destroy() - destroys the NAPI structures for a given instance 219 * @hif : pointer to hif context 220 * @ce_id : the CE id whose napi instance will be destroyed 221 * @force : if set, will destroy even if entry is active (de-activates) 222 * 223 * Description: 224 * Destroy a given NAPI instance. This function is called 225 * unconditionally during cleanup. 226 * Refuses to destroy an entry of it is still enabled (unless force=1) 227 * Marks the whole napi_data invalid if all instances are destroyed. 228 * 229 * Return: 230 * -EINVAL: specific entry has not been created 231 * -EPERM : specific entry is still active 232 * 0 < : error 233 * 0 = : success 234 */ 235 int hif_napi_destroy(struct hif_opaque_softc *hif_ctx, 236 uint8_t id, 237 int force) 238 { 239 uint8_t ce = NAPI_ID2PIPE(id); 240 int rc = 0; 241 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 242 243 NAPI_DEBUG("-->(id=%d, force=%d)", id, force); 244 245 if (0 == (hif->napi_data.state & HIF_NAPI_INITED)) { 246 HIF_ERROR("%s: NAPI not initialized or entry %d not created", 247 __func__, id); 248 rc = -EINVAL; 249 } else if (0 == (hif->napi_data.ce_map & (0x01 << ce))) { 250 HIF_ERROR("%s: NAPI instance %d (pipe %d) not created", 251 __func__, id, ce); 252 if (hif->napi_data.napis[ce]) 253 HIF_ERROR("%s: memory allocated but ce_map not set %d (pipe %d)", 254 __func__, id, ce); 255 rc = -EINVAL; 256 } else { 257 struct qca_napi_data *napid; 258 struct qca_napi_info *napii; 259 260 napid = &(hif->napi_data); 261 napii = napid->napis[ce]; 262 if (!napii) { 263 if (napid->ce_map & (0x01 << ce)) 264 HIF_ERROR("%s: napii & ce_map out of sync(ce %d)", 265 __func__, ce); 266 return -EINVAL; 267 } 268 269 270 if (hif->napi_data.state == HIF_NAPI_CONF_UP) { 271 if (force) { 272 napi_disable(&(napii->napi)); 273 HIF_DBG("%s: NAPI entry %d force disabled", 274 __func__, id); 275 NAPI_DEBUG("NAPI %d force disabled", id); 276 } else { 277 HIF_ERROR("%s: Cannot destroy active NAPI %d", 278 __func__, id); 279 rc = -EPERM; 280 } 281 } 282 if (0 == rc) { 283 NAPI_DEBUG("before napi_del"); 284 NAPI_DEBUG("napi.dlist.prv=0x%pK, next=0x%pK", 285 napii->napi.dev_list.prev, 286 napii->napi.dev_list.next); 287 NAPI_DEBUG("dev.napi_l.prv=0x%pK, next=0x%pK", 288 napii->netdev.napi_list.prev, 289 napii->netdev.napi_list.next); 290 291 qdf_lro_deinit(napii->lro_ctx); 292 netif_napi_del(&(napii->napi)); 293 294 napid->ce_map &= ~(0x01 << ce); 295 napid->napis[ce] = NULL; 296 napii->scale = 0; 297 qdf_mem_free(napii); 298 HIF_DBG("%s: NAPI %d destroyed\n", __func__, id); 299 300 /* if there are no active instances and 301 * if they are all destroyed, 302 * set the whole structure to uninitialized state 303 */ 304 if (napid->ce_map == 0) { 305 rc = hif_napi_cpu_deinit(hif_ctx); 306 /* caller is tolerant to receiving !=0 rc */ 307 308 qdf_spinlock_destroy(&(napid->lock)); 309 memset(napid, 310 0, sizeof(struct qca_napi_data)); 311 HIF_DBG("%s: no NAPI instances. Zapped.", 312 __func__); 313 } 314 } 315 } 316 317 return rc; 318 } 319 qdf_export_symbol(hif_napi_destroy); 320 321 /** 322 * hif_napi_get_lro_info() - returns the address LRO data for napi_id 323 * @hif: pointer to hif context 324 * @napi_id: napi instance 325 * 326 * Description: 327 * Returns the address of the LRO structure 328 * 329 * Return: 330 * <addr>: address of the LRO structure 331 */ 332 void *hif_napi_get_lro_info(struct hif_opaque_softc *hif_hdl, int napi_id) 333 { 334 struct hif_softc *scn = HIF_GET_SOFTC(hif_hdl); 335 struct qca_napi_data *napid; 336 struct qca_napi_info *napii; 337 338 napid = &(scn->napi_data); 339 napii = napid->napis[NAPI_ID2PIPE(napi_id)]; 340 341 if (napii) 342 return napii->lro_ctx; 343 return 0; 344 } 345 346 /** 347 * 348 * hif_napi_get_all() - returns the address of the whole HIF NAPI structure 349 * @hif: pointer to hif context 350 * 351 * Description: 352 * Returns the address of the whole structure 353 * 354 * Return: 355 * <addr>: address of the whole HIF NAPI structure 356 */ 357 inline struct qca_napi_data *hif_napi_get_all(struct hif_opaque_softc *hif_ctx) 358 { 359 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 360 361 return &(hif->napi_data); 362 } 363 364 /** 365 * 366 * hif_napi_event() - reacts to events that impact NAPI 367 * @hif : pointer to hif context 368 * @evnt: event that has been detected 369 * @data: more data regarding the event 370 * 371 * Description: 372 * This function handles two types of events: 373 * 1- Events that change the state of NAPI (enabled/disabled): 374 * {NAPI_EVT_INI_FILE, NAPI_EVT_CMD_STATE} 375 * The state is retrievable by "hdd_napi_enabled(-1)" 376 * - NAPI will be on if either INI file is on and it has not been disabled 377 * by a subsequent vendor CMD, 378 * or it has been enabled by a vendor CMD. 379 * 2- Events that change the CPU affinity of a NAPI instance/IRQ: 380 * {NAPI_EVT_TPUT_STATE, NAPI_EVT_CPU_STATE} 381 * - NAPI will support a throughput mode (HI/LO), kept at napid->napi_mode 382 * - NAPI will switch throughput mode based on hdd_napi_throughput_policy() 383 * - In LO tput mode, NAPI will yield control if its interrupts to the system 384 * management functions. However in HI throughput mode, NAPI will actively 385 * manage its interrupts/instances (by trying to disperse them out to 386 * separate performance cores). 387 * - CPU eligibility is kept up-to-date by NAPI_EVT_CPU_STATE events. 388 * 389 * + In some cases (roaming peer management is the only case so far), a 390 * a client can trigger a "SERIALIZE" event. Basically, this means that the 391 * users is asking NAPI to go into a truly single execution context state. 392 * So, NAPI indicates to msm-irqbalancer that it wants to be blacklisted, 393 * (if called for the first time) and then moves all IRQs (for NAPI 394 * instances) to be collapsed to a single core. If called multiple times, 395 * it will just re-collapse the CPUs. This is because blacklist-on() API 396 * is reference-counted, and because the API has already been called. 397 * 398 * Such a user, should call "DESERIALIZE" (NORMAL) event, to set NAPI to go 399 * to its "normal" operation. Optionally, they can give a timeout value (in 400 * multiples of BusBandwidthCheckPeriod -- 100 msecs by default). In this 401 * case, NAPI will just set the current throughput state to uninitialized 402 * and set the delay period. Once policy handler is called, it would skip 403 * applying the policy delay period times, and otherwise apply the policy. 404 * 405 * Return: 406 * < 0: some error 407 * = 0: event handled successfully 408 */ 409 int hif_napi_event(struct hif_opaque_softc *hif_ctx, enum qca_napi_event event, 410 void *data) 411 { 412 int rc = 0; 413 uint32_t prev_state; 414 int i; 415 bool state_changed; 416 struct napi_struct *napi; 417 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 418 struct qca_napi_data *napid = &(hif->napi_data); 419 enum qca_napi_tput_state tput_mode = QCA_NAPI_TPUT_UNINITIALIZED; 420 enum { 421 BLACKLIST_NOT_PENDING, 422 BLACKLIST_ON_PENDING, 423 BLACKLIST_OFF_PENDING 424 } blacklist_pending = BLACKLIST_NOT_PENDING; 425 426 NAPI_DEBUG("%s: -->(event=%d, aux=%pK)", __func__, event, data); 427 428 if (ce_srng_based(hif)) 429 return hif_exec_event(hif_ctx, event, data); 430 431 if ((napid->state & HIF_NAPI_INITED) == 0) { 432 NAPI_DEBUG("%s: got event when NAPI not initialized", 433 __func__); 434 return -EINVAL; 435 } 436 qdf_spin_lock_bh(&(napid->lock)); 437 prev_state = napid->state; 438 switch (event) { 439 case NAPI_EVT_INI_FILE: 440 case NAPI_EVT_CMD_STATE: 441 case NAPI_EVT_INT_STATE: { 442 int on = (data != ((void *)0)); 443 444 HIF_DBG("%s: recved evnt: STATE_CMD %d; v = %d (state=0x%0x)", 445 __func__, event, 446 on, prev_state); 447 if (on) 448 if (prev_state & HIF_NAPI_CONF_UP) { 449 HIF_DBG("%s: duplicate NAPI conf ON msg", 450 __func__); 451 } else { 452 HIF_DBG("%s: setting state to ON", 453 __func__); 454 napid->state |= HIF_NAPI_CONF_UP; 455 } 456 else /* off request */ 457 if (prev_state & HIF_NAPI_CONF_UP) { 458 HIF_DBG("%s: setting state to OFF", 459 __func__); 460 napid->state &= ~HIF_NAPI_CONF_UP; 461 } else { 462 HIF_DBG("%s: duplicate NAPI conf OFF msg", 463 __func__); 464 } 465 break; 466 } 467 /* case NAPI_INIT_FILE/CMD_STATE */ 468 469 case NAPI_EVT_CPU_STATE: { 470 int cpu = ((unsigned long int)data >> 16); 471 int val = ((unsigned long int)data & 0x0ff); 472 473 NAPI_DEBUG("%s: evt=CPU_STATE on CPU %d value=%d", 474 __func__, cpu, val); 475 476 /* state has already been set by hnc_cpu_notify_cb */ 477 if ((val == QCA_NAPI_CPU_DOWN) && 478 (napid->napi_mode == QCA_NAPI_TPUT_HI) && /* we manage */ 479 (napid->napi_cpu[cpu].napis != 0)) { 480 NAPI_DEBUG("%s: Migrating NAPIs out of cpu %d", 481 __func__, cpu); 482 rc = hif_napi_cpu_migrate(napid, 483 cpu, 484 HNC_ACT_RELOCATE); 485 napid->napi_cpu[cpu].napis = 0; 486 } 487 /* in QCA_NAPI_TPUT_LO case, napis MUST == 0 */ 488 break; 489 } 490 491 case NAPI_EVT_TPUT_STATE: { 492 tput_mode = (enum qca_napi_tput_state)data; 493 if (tput_mode == QCA_NAPI_TPUT_LO) { 494 /* from TPUT_HI -> TPUT_LO */ 495 NAPI_DEBUG("%s: Moving to napi_tput_LO state", 496 __func__); 497 blacklist_pending = BLACKLIST_OFF_PENDING; 498 /* 499 * Ideally we should "collapse" interrupts here, since 500 * we are "dispersing" interrupts in the "else" case. 501 * This allows the possibility that our interrupts may 502 * still be on the perf cluster the next time we enter 503 * high tput mode. However, the irq_balancer is free 504 * to move our interrupts to power cluster once 505 * blacklisting has been turned off in the "else" case. 506 */ 507 } else { 508 /* from TPUT_LO -> TPUT->HI */ 509 NAPI_DEBUG("%s: Moving to napi_tput_HI state", 510 __func__); 511 rc = hif_napi_cpu_migrate(napid, 512 HNC_ANY_CPU, 513 HNC_ACT_DISPERSE); 514 515 blacklist_pending = BLACKLIST_ON_PENDING; 516 } 517 napid->napi_mode = tput_mode; 518 break; 519 } 520 521 case NAPI_EVT_USR_SERIAL: { 522 unsigned long users = (unsigned long)data; 523 524 NAPI_DEBUG("%s: User forced SERIALIZATION; users=%ld", 525 __func__, users); 526 527 rc = hif_napi_cpu_migrate(napid, 528 HNC_ANY_CPU, 529 HNC_ACT_COLLAPSE); 530 if ((users == 0) && (rc == 0)) 531 blacklist_pending = BLACKLIST_ON_PENDING; 532 break; 533 } 534 case NAPI_EVT_USR_NORMAL: { 535 NAPI_DEBUG("%s: User forced DE-SERIALIZATION", __func__); 536 /* 537 * Deserialization timeout is handled at hdd layer; 538 * just mark current mode to uninitialized to ensure 539 * it will be set when the delay is over 540 */ 541 napid->napi_mode = QCA_NAPI_TPUT_UNINITIALIZED; 542 break; 543 } 544 default: { 545 HIF_ERROR("%s: unknown event: %d (data=0x%0lx)", 546 __func__, event, (unsigned long) data); 547 break; 548 } /* default */ 549 }; /* switch */ 550 551 552 switch (blacklist_pending) { 553 case BLACKLIST_ON_PENDING: 554 /* assume the control of WLAN IRQs */ 555 hif_napi_cpu_blacklist(napid, BLACKLIST_ON); 556 break; 557 case BLACKLIST_OFF_PENDING: 558 /* yield the control of WLAN IRQs */ 559 hif_napi_cpu_blacklist(napid, BLACKLIST_OFF); 560 break; 561 default: /* nothing to do */ 562 break; 563 } /* switch blacklist_pending */ 564 565 /* we want to perform the comparison in lock: 566 * there is a possiblity of hif_napi_event get called 567 * from two different contexts (driver unload and cpu hotplug 568 * notification) and napid->state get changed 569 * in driver unload context and can lead to race condition 570 * in cpu hotplug context. Therefore, perform the napid->state 571 * comparison before releasing lock. 572 */ 573 state_changed = (prev_state != napid->state); 574 qdf_spin_unlock_bh(&(napid->lock)); 575 576 if (state_changed) { 577 if (napid->state == ENABLE_NAPI_MASK) { 578 rc = 1; 579 for (i = 0; i < CE_COUNT_MAX; i++) { 580 struct qca_napi_info *napii = napid->napis[i]; 581 if (napii) { 582 napi = &(napii->napi); 583 NAPI_DEBUG("%s: enabling NAPI %d", 584 __func__, i); 585 napi_enable(napi); 586 } 587 } 588 } else { 589 rc = 0; 590 for (i = 0; i < CE_COUNT_MAX; i++) { 591 struct qca_napi_info *napii = napid->napis[i]; 592 if (napii) { 593 napi = &(napii->napi); 594 NAPI_DEBUG("%s: disabling NAPI %d", 595 __func__, i); 596 napi_disable(napi); 597 /* in case it is affined, remove it */ 598 irq_set_affinity_hint(napii->irq, NULL); 599 } 600 } 601 } 602 } else { 603 HIF_DBG("%s: no change in hif napi state (still %d)", 604 __func__, prev_state); 605 } 606 607 NAPI_DEBUG("<--[rc=%d]", rc); 608 return rc; 609 } 610 qdf_export_symbol(hif_napi_event); 611 612 /** 613 * hif_napi_enabled() - checks whether NAPI is enabled for given ce or not 614 * @hif: hif context 615 * @ce : CE instance (or -1, to check if any CEs are enabled) 616 * 617 * Return: bool 618 */ 619 int hif_napi_enabled(struct hif_opaque_softc *hif_ctx, int ce) 620 { 621 int rc; 622 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 623 624 if (-1 == ce) 625 rc = ((hif->napi_data.state == ENABLE_NAPI_MASK)); 626 else 627 rc = ((hif->napi_data.state == ENABLE_NAPI_MASK) && 628 (hif->napi_data.ce_map & (0x01 << ce))); 629 return rc; 630 } 631 qdf_export_symbol(hif_napi_enabled); 632 633 /** 634 * hif_napi_enable_irq() - enables bus interrupts after napi_complete 635 * 636 * @hif: hif context 637 * @id : id of NAPI instance calling this (used to determine the CE) 638 * 639 * Return: void 640 */ 641 inline void hif_napi_enable_irq(struct hif_opaque_softc *hif, int id) 642 { 643 struct hif_softc *scn = HIF_GET_SOFTC(hif); 644 645 hif_irq_enable(scn, NAPI_ID2PIPE(id)); 646 } 647 648 649 /** 650 * hif_napi_schedule() - schedules napi, updates stats 651 * @scn: hif context 652 * @ce_id: index of napi instance 653 * 654 * Return: void 655 */ 656 int hif_napi_schedule(struct hif_opaque_softc *hif_ctx, int ce_id) 657 { 658 int cpu = smp_processor_id(); 659 struct hif_softc *scn = HIF_GET_SOFTC(hif_ctx); 660 struct qca_napi_info *napii; 661 662 hif_record_ce_desc_event(scn, ce_id, NAPI_SCHEDULE, 663 NULL, NULL, 0, 0); 664 665 napii = scn->napi_data.napis[ce_id]; 666 if (qdf_unlikely(!napii)) { 667 HIF_ERROR("%s, scheduling unallocated napi (ce:%d)", 668 __func__, ce_id); 669 qdf_atomic_dec(&scn->active_tasklet_cnt); 670 return false; 671 } 672 673 napii->stats[cpu].napi_schedules++; 674 NAPI_DEBUG("scheduling napi %d (ce:%d)", napii->id, ce_id); 675 napi_schedule(&(napii->napi)); 676 677 return true; 678 } 679 qdf_export_symbol(hif_napi_schedule); 680 681 /** 682 * hif_napi_correct_cpu() - correct the interrupt affinity for napi if needed 683 * @napi_info: pointer to qca_napi_info for the napi instance 684 * 685 * Return: true => interrupt already on correct cpu, no correction needed 686 * false => interrupt on wrong cpu, correction done for cpu affinity 687 * of the interrupt 688 */ 689 static inline 690 bool hif_napi_correct_cpu(struct qca_napi_info *napi_info) 691 { 692 bool right_cpu = true; 693 int rc = 0; 694 cpumask_t cpumask; 695 int cpu; 696 struct qca_napi_data *napid; 697 698 napid = hif_napi_get_all(GET_HIF_OPAQUE_HDL(napi_info->hif_ctx)); 699 700 if (napid->flags & QCA_NAPI_FEATURE_CPU_CORRECTION) { 701 702 cpu = qdf_get_cpu(); 703 if (unlikely((hif_napi_cpu_blacklist(napid, 704 BLACKLIST_QUERY) > 0) && 705 (cpu != napi_info->cpu))) { 706 right_cpu = false; 707 708 NAPI_DEBUG("interrupt on wrong CPU, correcting"); 709 cpumask.bits[0] = (0x01 << napi_info->cpu); 710 711 irq_modify_status(napi_info->irq, IRQ_NO_BALANCING, 0); 712 rc = irq_set_affinity_hint(napi_info->irq, 713 &cpumask); 714 irq_modify_status(napi_info->irq, 0, IRQ_NO_BALANCING); 715 716 if (rc) 717 HIF_ERROR("error setting irq affinity hint: %d", 718 rc); 719 else 720 napi_info->stats[cpu].cpu_corrected++; 721 } 722 } 723 return right_cpu; 724 } 725 726 /** 727 * hif_napi_poll() - NAPI poll routine 728 * @napi : pointer to NAPI struct as kernel holds it 729 * @budget: 730 * 731 * This is the body of the poll function. 732 * The poll function is called by kernel. So, there is a wrapper 733 * function in HDD, which in turn calls this function. 734 * Two main reasons why the whole thing is not implemented in HDD: 735 * a) references to things like ce_service that HDD is not aware of 736 * b) proximity to the implementation of ce_tasklet, which the body 737 * of this function should be very close to. 738 * 739 * NOTE TO THE MAINTAINER: 740 * Consider this function and ce_tasklet very tightly coupled pairs. 741 * Any changes to ce_tasklet or this function may likely need to be 742 * reflected in the counterpart. 743 * 744 * Returns: 745 * int: the amount of work done in this poll (<= budget) 746 */ 747 int hif_napi_poll(struct hif_opaque_softc *hif_ctx, 748 struct napi_struct *napi, 749 int budget) 750 { 751 int rc = 0; /* default: no work done, also takes care of error */ 752 int normalized = 0; 753 int bucket; 754 int cpu = smp_processor_id(); 755 bool poll_on_right_cpu; 756 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 757 struct qca_napi_info *napi_info; 758 struct CE_state *ce_state = NULL; 759 760 if (unlikely(NULL == hif)) { 761 HIF_ERROR("%s: hif context is NULL", __func__); 762 QDF_ASSERT(0); 763 goto out; 764 } 765 766 napi_info = (struct qca_napi_info *) 767 container_of(napi, struct qca_napi_info, napi); 768 769 NAPI_DEBUG("%s -->(napi(%d, irq=%d), budget=%d)", 770 __func__, napi_info->id, napi_info->irq, budget); 771 772 napi_info->stats[cpu].napi_polls++; 773 774 hif_record_ce_desc_event(hif, NAPI_ID2PIPE(napi_info->id), 775 NAPI_POLL_ENTER, NULL, NULL, cpu, 0); 776 777 rc = ce_per_engine_service(hif, NAPI_ID2PIPE(napi_info->id)); 778 NAPI_DEBUG("%s: ce_per_engine_service processed %d msgs", 779 __func__, rc); 780 781 qdf_lro_flush(napi_info->lro_ctx); 782 783 /* do not return 0, if there was some work done, 784 * even if it is below the scale 785 */ 786 if (rc) { 787 napi_info->stats[cpu].napi_workdone += rc; 788 normalized = (rc / napi_info->scale); 789 if (normalized == 0) 790 normalized++; 791 bucket = normalized / (QCA_NAPI_BUDGET / QCA_NAPI_NUM_BUCKETS); 792 if (bucket >= QCA_NAPI_NUM_BUCKETS) { 793 bucket = QCA_NAPI_NUM_BUCKETS - 1; 794 HIF_ERROR("Bad bucket#(%d) > QCA_NAPI_NUM_BUCKETS(%d)", 795 bucket, QCA_NAPI_NUM_BUCKETS); 796 } 797 napi_info->stats[cpu].napi_budget_uses[bucket]++; 798 } else { 799 /* if ce_per engine reports 0, then poll should be terminated */ 800 NAPI_DEBUG("%s:%d: nothing processed by CE. Completing NAPI", 801 __func__, __LINE__); 802 } 803 804 ce_state = hif->ce_id_to_state[NAPI_ID2PIPE(napi_info->id)]; 805 806 /* 807 * Not using the API hif_napi_correct_cpu directly in the if statement 808 * below since the API may not get evaluated if put at the end if any 809 * prior condition would evaluate to be true. The CPU correction 810 * check should kick in every poll. 811 */ 812 #ifdef NAPI_YIELD_BUDGET_BASED 813 if (ce_state && (ce_state->force_break || 0 == rc)) { 814 #else 815 poll_on_right_cpu = hif_napi_correct_cpu(napi_info); 816 if ((ce_state) && 817 (!ce_check_rx_pending(ce_state) || (0 == rc) || 818 !poll_on_right_cpu)) { 819 #endif 820 napi_info->stats[cpu].napi_completes++; 821 #ifdef NAPI_YIELD_BUDGET_BASED 822 ce_state->force_break = 0; 823 #endif 824 825 hif_record_ce_desc_event(hif, ce_state->id, NAPI_COMPLETE, 826 NULL, NULL, 0, 0); 827 if (normalized >= budget) 828 normalized = budget - 1; 829 830 /* enable interrupts */ 831 napi_complete(napi); 832 hif_napi_enable_irq(hif_ctx, napi_info->id); 833 /* support suspend/resume */ 834 qdf_atomic_dec(&(hif->active_tasklet_cnt)); 835 836 NAPI_DEBUG("%s:%d: napi_complete + enabling the interrupts", 837 __func__, __LINE__); 838 } else { 839 /* 4.4 kernel NAPI implementation requires drivers to 840 * return full work when they ask to be re-scheduled, 841 * or napi_complete and re-start with a fresh interrupt 842 */ 843 normalized = budget; 844 } 845 846 hif_record_ce_desc_event(hif, NAPI_ID2PIPE(napi_info->id), 847 NAPI_POLL_EXIT, NULL, NULL, normalized, 0); 848 849 NAPI_DEBUG("%s <--[normalized=%d]", __func__, normalized); 850 return normalized; 851 out: 852 return rc; 853 } 854 qdf_export_symbol(hif_napi_poll); 855 856 #ifdef HIF_IRQ_AFFINITY 857 /** 858 * 859 * hif_napi_update_yield_stats() - update NAPI yield related stats 860 * @cpu_id: CPU ID for which stats needs to be updates 861 * @ce_id: Copy Engine ID for which yield stats needs to be updates 862 * @time_limit_reached: indicates whether the time limit was reached 863 * @rxpkt_thresh_reached: indicates whether rx packet threshold was reached 864 * 865 * Return: None 866 */ 867 void hif_napi_update_yield_stats(struct CE_state *ce_state, 868 bool time_limit_reached, 869 bool rxpkt_thresh_reached) 870 { 871 struct hif_softc *hif; 872 struct qca_napi_data *napi_data = NULL; 873 int ce_id = 0; 874 int cpu_id = 0; 875 876 if (unlikely(NULL == ce_state)) { 877 QDF_ASSERT(NULL != ce_state); 878 return; 879 } 880 881 hif = ce_state->scn; 882 883 if (unlikely(NULL == hif)) { 884 QDF_ASSERT(NULL != hif); 885 return; 886 } 887 napi_data = &(hif->napi_data); 888 if (unlikely(NULL == napi_data)) { 889 QDF_ASSERT(NULL != napi_data); 890 return; 891 } 892 893 if (unlikely(NULL == napi_data->napis[ce_id])) 894 return; 895 896 ce_id = ce_state->id; 897 cpu_id = qdf_get_cpu(); 898 899 if (time_limit_reached) 900 napi_data->napis[ce_id]->stats[cpu_id].time_limit_reached++; 901 else 902 napi_data->napis[ce_id]->stats[cpu_id].rxpkt_thresh_reached++; 903 } 904 905 /** 906 * 907 * hif_napi_stats() - display NAPI CPU statistics 908 * @napid: pointer to qca_napi_data 909 * 910 * Description: 911 * Prints the various CPU cores on which the NAPI instances /CEs interrupts 912 * are being executed. Can be called from outside NAPI layer. 913 * 914 * Return: None 915 */ 916 void hif_napi_stats(struct qca_napi_data *napid) 917 { 918 int i; 919 struct qca_napi_cpu *cpu; 920 921 if (napid == NULL) { 922 qdf_debug("%s: napiid struct is null", __func__); 923 return; 924 } 925 926 cpu = napid->napi_cpu; 927 qdf_debug("NAPI CPU TABLE"); 928 qdf_debug("lilclhead=%d, bigclhead=%d", 929 napid->lilcl_head, napid->bigcl_head); 930 for (i = 0; i < NR_CPUS; i++) { 931 qdf_debug("CPU[%02d]: state:%d crid=%02d clid=%02d crmk:0x%0lx thmk:0x%0lx frq:%d napi = 0x%08x lnk:%d", 932 i, 933 cpu[i].state, cpu[i].core_id, cpu[i].cluster_id, 934 cpu[i].core_mask.bits[0], 935 cpu[i].thread_mask.bits[0], 936 cpu[i].max_freq, cpu[i].napis, 937 cpu[i].cluster_nxt); 938 } 939 } 940 941 #ifdef FEATURE_NAPI_DEBUG 942 /* 943 * Local functions 944 * - no argument checks, all internal/trusted callers 945 */ 946 static void hnc_dump_cpus(struct qca_napi_data *napid) 947 { 948 hif_napi_stats(napid); 949 } 950 #else 951 static void hnc_dump_cpus(struct qca_napi_data *napid) { /* no-op */ }; 952 #endif /* FEATURE_NAPI_DEBUG */ 953 /** 954 * hnc_link_clusters() - partitions to cpu table into clusters 955 * @napid: pointer to NAPI data 956 * 957 * Takes in a CPU topology table and builds two linked lists 958 * (big cluster cores, list-head at bigcl_head, and little cluster 959 * cores, list-head at lilcl_head) out of it. 960 * 961 * If there are more than two clusters: 962 * - bigcl_head and lilcl_head will be different, 963 * - the cluster with highest cpufreq will be considered the "big" cluster. 964 * If there are more than one with the highest frequency, the *last* of such 965 * clusters will be designated as the "big cluster" 966 * - the cluster with lowest cpufreq will be considered the "li'l" cluster. 967 * If there are more than one clusters with the lowest cpu freq, the *first* 968 * of such clusters will be designated as the "little cluster" 969 * - We only support up to 32 clusters 970 * Return: 0 : OK 971 * !0: error (at least one of lil/big clusters could not be found) 972 */ 973 #define HNC_MIN_CLUSTER 0 974 #define HNC_MAX_CLUSTER 31 975 static int hnc_link_clusters(struct qca_napi_data *napid) 976 { 977 int rc = 0; 978 979 int i; 980 int it = 0; 981 uint32_t cl_done = 0x0; 982 int cl, curcl, curclhead = 0; 983 int more; 984 unsigned int lilfrq = INT_MAX; 985 unsigned int bigfrq = 0; 986 unsigned int clfrq = 0; 987 int prev = 0; 988 struct qca_napi_cpu *cpus = napid->napi_cpu; 989 990 napid->lilcl_head = napid->bigcl_head = -1; 991 992 do { 993 more = 0; 994 it++; curcl = -1; 995 for (i = 0; i < NR_CPUS; i++) { 996 cl = cpus[i].cluster_id; 997 NAPI_DEBUG("Processing cpu[%d], cluster=%d\n", 998 i, cl); 999 if ((cl < HNC_MIN_CLUSTER) || (cl > HNC_MAX_CLUSTER)) { 1000 NAPI_DEBUG("Bad cluster (%d). SKIPPED\n", cl); 1001 QDF_ASSERT(0); 1002 /* continue if ASSERTs are disabled */ 1003 continue; 1004 }; 1005 if (cpumask_weight(&(cpus[i].core_mask)) == 0) { 1006 NAPI_DEBUG("Core mask 0. SKIPPED\n"); 1007 continue; 1008 } 1009 if (cl_done & (0x01 << cl)) { 1010 NAPI_DEBUG("Cluster already processed. SKIPPED\n"); 1011 continue; 1012 } else { 1013 if (more == 0) { 1014 more = 1; 1015 curcl = cl; 1016 curclhead = i; /* row */ 1017 clfrq = cpus[i].max_freq; 1018 prev = -1; 1019 }; 1020 if ((curcl >= 0) && (curcl != cl)) { 1021 NAPI_DEBUG("Entry cl(%d) != curcl(%d). SKIPPED\n", 1022 cl, curcl); 1023 continue; 1024 } 1025 if (cpus[i].max_freq != clfrq) 1026 NAPI_DEBUG("WARN: frq(%d)!=clfrq(%d)\n", 1027 cpus[i].max_freq, clfrq); 1028 if (clfrq >= bigfrq) { 1029 bigfrq = clfrq; 1030 napid->bigcl_head = curclhead; 1031 NAPI_DEBUG("bigcl=%d\n", curclhead); 1032 } 1033 if (clfrq < lilfrq) { 1034 lilfrq = clfrq; 1035 napid->lilcl_head = curclhead; 1036 NAPI_DEBUG("lilcl=%d\n", curclhead); 1037 } 1038 if (prev != -1) 1039 cpus[prev].cluster_nxt = i; 1040 1041 prev = i; 1042 } 1043 } 1044 if (curcl >= 0) 1045 cl_done |= (0x01 << curcl); 1046 1047 } while (more); 1048 1049 if (qdf_unlikely((napid->lilcl_head < 0) && (napid->bigcl_head < 0))) 1050 rc = -EFAULT; 1051 1052 hnc_dump_cpus(napid); /* if NAPI_DEBUG */ 1053 return rc; 1054 } 1055 #undef HNC_MIN_CLUSTER 1056 #undef HNC_MAX_CLUSTER 1057 1058 /* 1059 * hotplug function group 1060 */ 1061 1062 /** 1063 * hnc_cpu_online_cb() - handles CPU hotplug "up" events 1064 * @context: the associated HIF context 1065 * @cpu: the CPU Id of the CPU the event happened on 1066 * 1067 * Return: None 1068 */ 1069 static void hnc_cpu_online_cb(void *context, uint32_t cpu) 1070 { 1071 struct hif_softc *hif = context; 1072 struct qca_napi_data *napid = &hif->napi_data; 1073 1074 if (cpu >= NR_CPUS) 1075 return; 1076 1077 NAPI_DEBUG("-->%s(act=online, cpu=%u)", __func__, cpu); 1078 1079 napid->napi_cpu[cpu].state = QCA_NAPI_CPU_UP; 1080 NAPI_DEBUG("%s: CPU %u marked %d", 1081 __func__, cpu, napid->napi_cpu[cpu].state); 1082 1083 NAPI_DEBUG("<--%s", __func__); 1084 } 1085 1086 /** 1087 * hnc_cpu_before_offline_cb() - handles CPU hotplug "prepare down" events 1088 * @context: the associated HIF context 1089 * @cpu: the CPU Id of the CPU the event happened on 1090 * 1091 * On transtion to offline, we act on PREP events, because we may need to move 1092 * the irqs/NAPIs to another CPU before it is actually off-lined. 1093 * 1094 * Return: None 1095 */ 1096 static void hnc_cpu_before_offline_cb(void *context, uint32_t cpu) 1097 { 1098 struct hif_softc *hif = context; 1099 struct qca_napi_data *napid = &hif->napi_data; 1100 1101 if (cpu >= NR_CPUS) 1102 return; 1103 1104 NAPI_DEBUG("-->%s(act=before_offline, cpu=%u)", __func__, cpu); 1105 1106 napid->napi_cpu[cpu].state = QCA_NAPI_CPU_DOWN; 1107 1108 NAPI_DEBUG("%s: CPU %u marked %d; updating affinity", 1109 __func__, cpu, napid->napi_cpu[cpu].state); 1110 1111 /** 1112 * we need to move any NAPIs on this CPU out. 1113 * if we are in LO throughput mode, then this is valid 1114 * if the CPU is the the low designated CPU. 1115 */ 1116 hif_napi_event(GET_HIF_OPAQUE_HDL(hif), 1117 NAPI_EVT_CPU_STATE, 1118 (void *) 1119 ((size_t)cpu << 16 | napid->napi_cpu[cpu].state)); 1120 1121 NAPI_DEBUG("<--%s", __func__); 1122 } 1123 1124 static int hnc_hotplug_register(struct hif_softc *hif_sc) 1125 { 1126 QDF_STATUS status; 1127 1128 NAPI_DEBUG("-->%s", __func__); 1129 1130 status = qdf_cpuhp_register(&hif_sc->napi_data.cpuhp_handler, 1131 hif_sc, 1132 hnc_cpu_online_cb, 1133 hnc_cpu_before_offline_cb); 1134 1135 NAPI_DEBUG("<--%s [%d]", __func__, status); 1136 1137 return qdf_status_to_os_return(status); 1138 } 1139 1140 static void hnc_hotplug_unregister(struct hif_softc *hif_sc) 1141 { 1142 NAPI_DEBUG("-->%s", __func__); 1143 1144 if (hif_sc->napi_data.cpuhp_handler) 1145 qdf_cpuhp_unregister(&hif_sc->napi_data.cpuhp_handler); 1146 1147 NAPI_DEBUG("<--%s", __func__); 1148 } 1149 1150 /** 1151 * hnc_install_tput() - installs a callback in the throughput detector 1152 * @register: !0 => register; =0: unregister 1153 * 1154 * installs a callback to be called when wifi driver throughput (tx+rx) 1155 * crosses a threshold. Currently, we are using the same criteria as 1156 * TCP ack suppression (500 packets/100ms by default). 1157 * 1158 * Return: 0 : success 1159 * <0: failure 1160 */ 1161 1162 static int hnc_tput_hook(int install) 1163 { 1164 int rc = 0; 1165 1166 /* 1167 * Nothing, until the bw_calculation accepts registration 1168 * it is now hardcoded in the wlan_hdd_main.c::hdd_bus_bw_compute_cbk 1169 * hdd_napi_throughput_policy(...) 1170 */ 1171 return rc; 1172 } 1173 1174 /* 1175 * Implementation of hif_napi_cpu API 1176 */ 1177 1178 #if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 4, 0)) 1179 static inline void record_sibling_cpumask(struct qca_napi_cpu *cpus, int i) 1180 { 1181 cpumask_copy(&(cpus[i].thread_mask), 1182 topology_sibling_cpumask(i)); 1183 } 1184 #else 1185 static inline void record_sibling_cpumask(struct qca_napi_cpu *cpus, int i) 1186 { 1187 } 1188 #endif 1189 1190 1191 /** 1192 * hif_napi_cpu_init() - initialization of irq affinity block 1193 * @ctx: pointer to qca_napi_data 1194 * 1195 * called by hif_napi_create, after the first instance is called 1196 * - builds napi_rss_cpus table from cpu topology 1197 * - links cores of the same clusters together 1198 * - installs hot-plug notifier 1199 * - installs throughput trigger notifier (when such mechanism exists) 1200 * 1201 * Return: 0: OK 1202 * <0: error code 1203 */ 1204 int hif_napi_cpu_init(struct hif_opaque_softc *hif) 1205 { 1206 int rc = 0; 1207 int i; 1208 struct qca_napi_data *napid = &HIF_GET_SOFTC(hif)->napi_data; 1209 struct qca_napi_cpu *cpus = napid->napi_cpu; 1210 1211 NAPI_DEBUG("--> "); 1212 1213 if (cpus[0].state != QCA_NAPI_CPU_UNINITIALIZED) { 1214 NAPI_DEBUG("NAPI RSS table already initialized.\n"); 1215 rc = -EALREADY; 1216 goto lab_rss_init; 1217 } 1218 1219 /* build CPU topology table */ 1220 for_each_possible_cpu(i) { 1221 cpus[i].state = ((cpumask_test_cpu(i, cpu_online_mask) 1222 ? QCA_NAPI_CPU_UP 1223 : QCA_NAPI_CPU_DOWN)); 1224 cpus[i].core_id = topology_core_id(i); 1225 cpus[i].cluster_id = topology_physical_package_id(i); 1226 cpumask_copy(&(cpus[i].core_mask), 1227 topology_core_cpumask(i)); 1228 record_sibling_cpumask(cpus, i); 1229 cpus[i].max_freq = cpufreq_quick_get_max(i); 1230 cpus[i].napis = 0x0; 1231 cpus[i].cluster_nxt = -1; /* invalid */ 1232 } 1233 1234 /* link clusters together */ 1235 rc = hnc_link_clusters(napid); 1236 if (0 != rc) 1237 goto lab_err_topology; 1238 1239 /* install hotplug notifier */ 1240 rc = hnc_hotplug_register(HIF_GET_SOFTC(hif)); 1241 if (0 != rc) 1242 goto lab_err_hotplug; 1243 1244 /* install throughput notifier */ 1245 rc = hnc_tput_hook(1); 1246 if (0 == rc) 1247 goto lab_rss_init; 1248 1249 lab_err_hotplug: 1250 hnc_tput_hook(0); 1251 hnc_hotplug_unregister(HIF_GET_SOFTC(hif)); 1252 lab_err_topology: 1253 memset(napid->napi_cpu, 0, sizeof(struct qca_napi_cpu) * NR_CPUS); 1254 lab_rss_init: 1255 NAPI_DEBUG("<-- [rc=%d]", rc); 1256 return rc; 1257 } 1258 1259 /** 1260 * hif_napi_cpu_deinit() - clean-up of irq affinity block 1261 * 1262 * called by hif_napi_destroy, when the last instance is removed 1263 * - uninstalls throughput and hotplug notifiers 1264 * - clears cpu topology table 1265 * Return: 0: OK 1266 */ 1267 int hif_napi_cpu_deinit(struct hif_opaque_softc *hif) 1268 { 1269 int rc = 0; 1270 struct qca_napi_data *napid = &HIF_GET_SOFTC(hif)->napi_data; 1271 1272 NAPI_DEBUG("-->%s(...)", __func__); 1273 1274 /* uninstall tput notifier */ 1275 rc = hnc_tput_hook(0); 1276 1277 /* uninstall hotplug notifier */ 1278 hnc_hotplug_unregister(HIF_GET_SOFTC(hif)); 1279 1280 /* clear the topology table */ 1281 memset(napid->napi_cpu, 0, sizeof(struct qca_napi_cpu) * NR_CPUS); 1282 1283 NAPI_DEBUG("<--%s[rc=%d]", __func__, rc); 1284 1285 return rc; 1286 } 1287 1288 /** 1289 * hncm_migrate_to() - migrates a NAPI to a CPU 1290 * @napid: pointer to NAPI block 1291 * @ce_id: CE_id of the NAPI instance 1292 * @didx : index in the CPU topology table for the CPU to migrate to 1293 * 1294 * Migrates NAPI (identified by the CE_id) to the destination core 1295 * Updates the napi_map of the destination entry 1296 * 1297 * Return: 1298 * =0 : success 1299 * <0 : error 1300 */ 1301 static int hncm_migrate_to(struct qca_napi_data *napid, 1302 int napi_ce, 1303 int didx) 1304 { 1305 int rc = 0; 1306 cpumask_t cpumask; 1307 1308 NAPI_DEBUG("-->%s(napi_cd=%d, didx=%d)", __func__, napi_ce, didx); 1309 1310 cpumask.bits[0] = (1 << didx); 1311 if (!napid->napis[napi_ce]) 1312 return -EINVAL; 1313 1314 irq_modify_status(napid->napis[napi_ce]->irq, IRQ_NO_BALANCING, 0); 1315 rc = irq_set_affinity_hint(napid->napis[napi_ce]->irq, &cpumask); 1316 1317 /* unmark the napis bitmap in the cpu table */ 1318 napid->napi_cpu[napid->napis[napi_ce]->cpu].napis &= ~(0x01 << napi_ce); 1319 /* mark the napis bitmap for the new designated cpu */ 1320 napid->napi_cpu[didx].napis |= (0x01 << napi_ce); 1321 napid->napis[napi_ce]->cpu = didx; 1322 1323 NAPI_DEBUG("<--%s[%d]", __func__, rc); 1324 return rc; 1325 } 1326 /** 1327 * hncm_dest_cpu() - finds a destination CPU for NAPI 1328 * @napid: pointer to NAPI block 1329 * @act : RELOCATE | COLLAPSE | DISPERSE 1330 * 1331 * Finds the designated destionation for the next IRQ. 1332 * RELOCATE: translated to either COLLAPSE or DISPERSE based 1333 * on napid->napi_mode (throughput state) 1334 * COLLAPSE: All have the same destination: the first online CPU in lilcl 1335 * DISPERSE: One of the CPU in bigcl, which has the smallest number of 1336 * NAPIs on it 1337 * 1338 * Return: >=0 : index in the cpu topology table 1339 * : < 0 : error 1340 */ 1341 static int hncm_dest_cpu(struct qca_napi_data *napid, int act) 1342 { 1343 int destidx = -1; 1344 int head, i; 1345 1346 NAPI_DEBUG("-->%s(act=%d)", __func__, act); 1347 if (act == HNC_ACT_RELOCATE) { 1348 if (napid->napi_mode == QCA_NAPI_TPUT_LO) 1349 act = HNC_ACT_COLLAPSE; 1350 else 1351 act = HNC_ACT_DISPERSE; 1352 NAPI_DEBUG("%s: act changed from HNC_ACT_RELOCATE to %d", 1353 __func__, act); 1354 } 1355 if (act == HNC_ACT_COLLAPSE) { 1356 head = i = napid->lilcl_head; 1357 retry_collapse: 1358 while (i >= 0) { 1359 if (napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) { 1360 destidx = i; 1361 break; 1362 } 1363 i = napid->napi_cpu[i].cluster_nxt; 1364 } 1365 if ((destidx < 0) && (head == napid->lilcl_head)) { 1366 NAPI_DEBUG("%s: COLLAPSE: no lilcl dest, try bigcl", 1367 __func__); 1368 head = i = napid->bigcl_head; 1369 goto retry_collapse; 1370 } 1371 } else { /* HNC_ACT_DISPERSE */ 1372 int smallest = 99; /* all 32 bits full */ 1373 int smallidx = -1; 1374 1375 head = i = napid->bigcl_head; 1376 retry_disperse: 1377 while (i >= 0) { 1378 if ((napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) && 1379 (hweight32(napid->napi_cpu[i].napis) <= smallest)) { 1380 smallest = napid->napi_cpu[i].napis; 1381 smallidx = i; 1382 } 1383 i = napid->napi_cpu[i].cluster_nxt; 1384 } 1385 destidx = smallidx; 1386 if ((destidx < 0) && (head == napid->bigcl_head)) { 1387 NAPI_DEBUG("%s: DISPERSE: no bigcl dest, try lilcl", 1388 __func__); 1389 head = i = napid->lilcl_head; 1390 goto retry_disperse; 1391 } 1392 } 1393 NAPI_DEBUG("<--%s[dest=%d]", __func__, destidx); 1394 return destidx; 1395 } 1396 /** 1397 * hif_napi_cpu_migrate() - migrate IRQs away 1398 * @cpu: -1: all CPUs <n> specific CPU 1399 * @act: COLLAPSE | DISPERSE 1400 * 1401 * Moves IRQs/NAPIs from specific or all CPUs (specified by @cpu) to eligible 1402 * cores. Eligible cores are: 1403 * act=COLLAPSE -> the first online core of the little cluster 1404 * act=DISPERSE -> separate cores of the big cluster, so that each core will 1405 * host minimum number of NAPIs/IRQs (napid->cpus[cpu].napis) 1406 * 1407 * Note that this function is called with a spinlock acquired already. 1408 * 1409 * Return: =0: success 1410 * <0: error 1411 */ 1412 1413 int hif_napi_cpu_migrate(struct qca_napi_data *napid, int cpu, int action) 1414 { 1415 int rc = 0; 1416 struct qca_napi_cpu *cpup; 1417 int i, dind; 1418 uint32_t napis; 1419 1420 NAPI_DEBUG("-->%s(.., cpu=%d, act=%d)", 1421 __func__, cpu, action); 1422 /* the following is really: hif_napi_enabled() with less overhead */ 1423 if (napid->ce_map == 0) { 1424 NAPI_DEBUG("%s: NAPI disabled. Not migrating.", __func__); 1425 goto hncm_return; 1426 } 1427 1428 cpup = napid->napi_cpu; 1429 1430 switch (action) { 1431 case HNC_ACT_RELOCATE: 1432 case HNC_ACT_DISPERSE: 1433 case HNC_ACT_COLLAPSE: { 1434 /* first find the src napi set */ 1435 if (cpu == HNC_ANY_CPU) 1436 napis = napid->ce_map; 1437 else 1438 napis = cpup[cpu].napis; 1439 /* then clear the napi bitmap on each CPU */ 1440 for (i = 0; i < NR_CPUS; i++) 1441 cpup[i].napis = 0; 1442 /* then for each of the NAPIs to disperse: */ 1443 for (i = 0; i < CE_COUNT_MAX; i++) 1444 if (napis & (1 << i)) { 1445 /* find a destination CPU */ 1446 dind = hncm_dest_cpu(napid, action); 1447 if (dind >= 0) { 1448 NAPI_DEBUG("Migrating NAPI ce%d to %d", 1449 i, dind); 1450 rc = hncm_migrate_to(napid, i, dind); 1451 } else { 1452 NAPI_DEBUG("No dest for NAPI ce%d", i); 1453 hnc_dump_cpus(napid); 1454 rc = -1; 1455 } 1456 } 1457 break; 1458 } 1459 default: { 1460 NAPI_DEBUG("%s: bad action: %d\n", __func__, action); 1461 QDF_BUG(0); 1462 break; 1463 } 1464 } /* switch action */ 1465 1466 hncm_return: 1467 hnc_dump_cpus(napid); 1468 return rc; 1469 } 1470 1471 1472 /** 1473 * hif_napi_bl_irq() - calls irq_modify_status to enable/disable blacklisting 1474 * @napid: pointer to qca_napi_data structure 1475 * @bl_flag: blacklist flag to enable/disable blacklisting 1476 * 1477 * The function enables/disables blacklisting for all the copy engine 1478 * interrupts on which NAPI is enabled. 1479 * 1480 * Return: None 1481 */ 1482 static inline void hif_napi_bl_irq(struct qca_napi_data *napid, bool bl_flag) 1483 { 1484 int i; 1485 struct qca_napi_info *napii; 1486 1487 for (i = 0; i < CE_COUNT_MAX; i++) { 1488 /* check if NAPI is enabled on the CE */ 1489 if (!(napid->ce_map & (0x01 << i))) 1490 continue; 1491 1492 /*double check that NAPI is allocated for the CE */ 1493 napii = napid->napis[i]; 1494 if (!(napii)) 1495 continue; 1496 1497 if (bl_flag == true) 1498 irq_modify_status(napii->irq, 1499 0, IRQ_NO_BALANCING); 1500 else 1501 irq_modify_status(napii->irq, 1502 IRQ_NO_BALANCING, 0); 1503 HIF_DBG("%s: bl_flag %d CE %d", __func__, bl_flag, i); 1504 } 1505 } 1506 1507 #ifdef CONFIG_SCHED_CORE_CTL 1508 /* Enable this API only if kernel feature - CONFIG_SCHED_CORE_CTL is defined */ 1509 static inline int hif_napi_core_ctl_set_boost(bool boost) 1510 { 1511 return core_ctl_set_boost(boost); 1512 } 1513 #else 1514 static inline int hif_napi_core_ctl_set_boost(bool boost) 1515 { 1516 return 0; 1517 } 1518 #endif 1519 /** 1520 * hif_napi_cpu_blacklist() - en(dis)ables blacklisting for NAPI RX interrupts. 1521 * @napid: pointer to qca_napi_data structure 1522 * @op: blacklist operation to perform 1523 * 1524 * The function enables/disables/queries blacklisting for all CE RX 1525 * interrupts with NAPI enabled. Besides blacklisting, it also enables/disables 1526 * core_ctl_set_boost. 1527 * Once blacklisting is enabled, the interrupts will not be managed by the IRQ 1528 * balancer. 1529 * 1530 * Return: -EINVAL, in case IRQ_BLACKLISTING and CORE_CTL_BOOST is not enabled 1531 * for BLACKLIST_QUERY op - blacklist refcount 1532 * for BLACKLIST_ON op - return value from core_ctl_set_boost API 1533 * for BLACKLIST_OFF op - return value from core_ctl_set_boost API 1534 */ 1535 int hif_napi_cpu_blacklist(struct qca_napi_data *napid, 1536 enum qca_blacklist_op op) 1537 { 1538 int rc = 0; 1539 static int ref_count; /* = 0 by the compiler */ 1540 uint8_t flags = napid->flags; 1541 bool bl_en = flags & QCA_NAPI_FEATURE_IRQ_BLACKLISTING; 1542 bool ccb_en = flags & QCA_NAPI_FEATURE_CORE_CTL_BOOST; 1543 1544 NAPI_DEBUG("-->%s(%d %d)", __func__, flags, op); 1545 1546 if (!(bl_en && ccb_en)) { 1547 rc = -EINVAL; 1548 goto out; 1549 } 1550 1551 switch (op) { 1552 case BLACKLIST_QUERY: 1553 rc = ref_count; 1554 break; 1555 case BLACKLIST_ON: 1556 ref_count++; 1557 rc = 0; 1558 if (ref_count == 1) { 1559 rc = hif_napi_core_ctl_set_boost(true); 1560 NAPI_DEBUG("boost_on() returns %d - refcnt=%d", 1561 rc, ref_count); 1562 hif_napi_bl_irq(napid, true); 1563 } 1564 break; 1565 case BLACKLIST_OFF: 1566 if (ref_count) { 1567 ref_count--; 1568 rc = 0; 1569 if (ref_count == 0) { 1570 rc = hif_napi_core_ctl_set_boost(false); 1571 NAPI_DEBUG("boost_off() returns %d - refcnt=%d", 1572 rc, ref_count); 1573 hif_napi_bl_irq(napid, false); 1574 } 1575 } 1576 break; 1577 default: 1578 NAPI_DEBUG("Invalid blacklist op: %d", op); 1579 rc = -EINVAL; 1580 } /* switch */ 1581 out: 1582 NAPI_DEBUG("<--%s[%d]", __func__, rc); 1583 return rc; 1584 } 1585 1586 /** 1587 * hif_napi_serialize() - [de-]serialize NAPI operations 1588 * @hif: context 1589 * @is_on: 1: serialize, 0: deserialize 1590 * 1591 * hif_napi_serialize(hif, 1) can be called multiple times. It will perform the 1592 * following steps (see hif_napi_event for code): 1593 * - put irqs of all NAPI instances on the same CPU 1594 * - only for the first serialize call: blacklist 1595 * 1596 * hif_napi_serialize(hif, 0): 1597 * - start a timer (multiple of BusBandwidthTimer -- default: 100 msec) 1598 * - at the end of the timer, check the current throughput state and 1599 * implement it. 1600 */ 1601 static unsigned long napi_serialize_reqs; 1602 int hif_napi_serialize(struct hif_opaque_softc *hif, int is_on) 1603 { 1604 int rc = -EINVAL; 1605 1606 if (hif != NULL) 1607 switch (is_on) { 1608 case 0: { /* de-serialize */ 1609 rc = hif_napi_event(hif, NAPI_EVT_USR_NORMAL, 1610 (void *) 0); 1611 napi_serialize_reqs = 0; 1612 break; 1613 } /* end de-serialize */ 1614 case 1: { /* serialize */ 1615 rc = hif_napi_event(hif, NAPI_EVT_USR_SERIAL, 1616 (void *)napi_serialize_reqs++); 1617 break; 1618 } /* end serialize */ 1619 default: 1620 break; /* no-op */ 1621 } /* switch */ 1622 return rc; 1623 } 1624 1625 #endif /* ifdef HIF_IRQ_AFFINITY */ 1626