1 /* 2 * Copyright (c) 2015-2020 The Linux Foundation. All rights reserved. 3 * 4 * Permission to use, copy, modify, and/or distribute this software for 5 * any purpose with or without fee is hereby granted, provided that the 6 * above copyright notice and this permission notice appear in all 7 * copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL 10 * WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED 11 * WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE 12 * AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 13 * DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR 14 * PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER 15 * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 16 * PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 /** 20 * DOC: hif_irq_afinity.c 21 * 22 * This irq afinity implementation is os dependent, so this can be treated as 23 * an abstraction layer... Should this be moved into a /linux folder? 24 */ 25 26 #include <linux/string.h> /* memset */ 27 28 /* Linux headers */ 29 #include <linux/cpumask.h> 30 #include <linux/cpufreq.h> 31 #include <linux/cpu.h> 32 #include <linux/topology.h> 33 #include <linux/interrupt.h> 34 #include <linux/irq.h> 35 #include <linux/pm.h> 36 #include <hif_napi.h> 37 #include <hif_irq_affinity.h> 38 #include <hif_exec.h> 39 #include <hif_main.h> 40 41 #if defined(FEATURE_NAPI_DEBUG) && defined(HIF_IRQ_AFFINITY) 42 /* 43 * Local functions 44 * - no argument checks, all internal/trusted callers 45 */ 46 static void hnc_dump_cpus(struct qca_napi_data *napid) 47 { 48 hif_napi_stats(napid); 49 } 50 #else 51 static void hnc_dump_cpus(struct qca_napi_data *napid) { /* no-op */ }; 52 #endif /* FEATURE_NAPI_DEBUG */ 53 54 #ifdef HIF_IRQ_AFFINITY 55 /** 56 * 57 * hif_exec_event() - reacts to events that impact irq affinity 58 * @hif : pointer to hif context 59 * @evnt: event that has been detected 60 * @data: more data regarding the event 61 * 62 * Description: 63 * This function handles two types of events: 64 * 1- Events that change the state of NAPI (enabled/disabled): 65 * {NAPI_EVT_INI_FILE, NAPI_EVT_CMD_STATE} 66 * The state is retrievable by "hdd_napi_enabled(-1)" 67 * - NAPI will be on if either INI file is on and it has not been disabled 68 * by a subsequent vendor CMD, 69 * or it has been enabled by a vendor CMD. 70 * 2- Events that change the CPU affinity of a NAPI instance/IRQ: 71 * {NAPI_EVT_TPUT_STATE, NAPI_EVT_CPU_STATE} 72 * - NAPI will support a throughput mode (HI/LO), kept at napid->napi_mode 73 * - NAPI will switch throughput mode based on hdd_napi_throughput_policy() 74 * - In LO tput mode, NAPI will yield control if its interrupts to the system 75 * management functions. However in HI throughput mode, NAPI will actively 76 * manage its interrupts/instances (by trying to disperse them out to 77 * separate performance cores). 78 * - CPU eligibility is kept up-to-date by NAPI_EVT_CPU_STATE events. 79 * 80 * + In some cases (roaming peer management is the only case so far), a 81 * a client can trigger a "SERIALIZE" event. Basically, this means that the 82 * users is asking NAPI to go into a truly single execution context state. 83 * So, NAPI indicates to msm-irqbalancer that it wants to be blacklisted, 84 * (if called for the first time) and then moves all IRQs (for NAPI 85 * instances) to be collapsed to a single core. If called multiple times, 86 * it will just re-collapse the CPUs. This is because blacklist-on() API 87 * is reference-counted, and because the API has already been called. 88 * 89 * Such a user, should call "DESERIALIZE" (NORMAL) event, to set NAPI to go 90 * to its "normal" operation. Optionally, they can give a timeout value (in 91 * multiples of BusBandwidthCheckPeriod -- 100 msecs by default). In this 92 * case, NAPI will just set the current throughput state to uninitialized 93 * and set the delay period. Once policy handler is called, it would skip 94 * applying the policy delay period times, and otherwise apply the policy. 95 * 96 * Return: 97 * < 0: some error 98 * = 0: event handled successfully 99 */ 100 int hif_exec_event(struct hif_opaque_softc *hif_ctx, enum qca_napi_event event, 101 void *data) 102 { 103 int rc = 0; 104 uint32_t prev_state; 105 struct hif_softc *hif = HIF_GET_SOFTC(hif_ctx); 106 struct qca_napi_data *napid = &(hif->napi_data); 107 enum qca_napi_tput_state tput_mode = QCA_NAPI_TPUT_UNINITIALIZED; 108 enum { 109 BLACKLIST_NOT_PENDING, 110 BLACKLIST_ON_PENDING, 111 BLACKLIST_OFF_PENDING 112 } blacklist_pending = BLACKLIST_NOT_PENDING; 113 114 NAPI_DEBUG("%s: -->(event=%d, aux=%pK)", __func__, event, data); 115 116 qdf_spin_lock_bh(&(napid->lock)); 117 prev_state = napid->state; 118 switch (event) { 119 case NAPI_EVT_INI_FILE: 120 case NAPI_EVT_CMD_STATE: 121 case NAPI_EVT_INT_STATE: 122 /* deprecated */ 123 break; 124 125 case NAPI_EVT_CPU_STATE: { 126 int cpu = ((unsigned long int)data >> 16); 127 int val = ((unsigned long int)data & 0x0ff); 128 129 NAPI_DEBUG("%s: evt=CPU_STATE on CPU %d value=%d", 130 __func__, cpu, val); 131 132 /* state has already been set by hnc_cpu_notify_cb */ 133 if ((val == QCA_NAPI_CPU_DOWN) && 134 (napid->napi_mode == QCA_NAPI_TPUT_HI) && /* we manage */ 135 (napid->napi_cpu[cpu].napis != 0)) { 136 NAPI_DEBUG("%s: Migrating NAPIs out of cpu %d", 137 __func__, cpu); 138 rc = hif_exec_cpu_migrate(napid, 139 cpu, 140 HNC_ACT_RELOCATE); 141 napid->napi_cpu[cpu].napis = 0; 142 } 143 /* in QCA_NAPI_TPUT_LO case, napis MUST == 0 */ 144 break; 145 } 146 147 case NAPI_EVT_TPUT_STATE: { 148 tput_mode = (enum qca_napi_tput_state)data; 149 if (tput_mode == QCA_NAPI_TPUT_LO) { 150 /* from TPUT_HI -> TPUT_LO */ 151 NAPI_DEBUG("%s: Moving to napi_tput_LO state", 152 __func__); 153 blacklist_pending = BLACKLIST_OFF_PENDING; 154 /* 155 * Ideally we should "collapse" interrupts here, since 156 * we are "dispersing" interrupts in the "else" case. 157 * This allows the possibility that our interrupts may 158 * still be on the perf cluster the next time we enter 159 * high tput mode. However, the irq_balancer is free 160 * to move our interrupts to power cluster once 161 * blacklisting has been turned off in the "else" case. 162 */ 163 } else { 164 /* from TPUT_LO -> TPUT->HI */ 165 NAPI_DEBUG("%s: Moving to napi_tput_HI state", 166 __func__); 167 rc = hif_exec_cpu_migrate(napid, 168 HNC_ANY_CPU, 169 HNC_ACT_DISPERSE); 170 171 blacklist_pending = BLACKLIST_ON_PENDING; 172 } 173 napid->napi_mode = tput_mode; 174 break; 175 } 176 177 case NAPI_EVT_USR_SERIAL: { 178 unsigned long users = (unsigned long)data; 179 180 NAPI_DEBUG("%s: User forced SERIALIZATION; users=%ld", 181 __func__, users); 182 183 rc = hif_exec_cpu_migrate(napid, 184 HNC_ANY_CPU, 185 HNC_ACT_COLLAPSE); 186 if ((users == 0) && (rc == 0)) 187 blacklist_pending = BLACKLIST_ON_PENDING; 188 break; 189 } 190 case NAPI_EVT_USR_NORMAL: { 191 NAPI_DEBUG("%s: User forced DE-SERIALIZATION", __func__); 192 if (!napid->user_cpu_affin_mask) 193 blacklist_pending = BLACKLIST_OFF_PENDING; 194 /* 195 * Deserialization timeout is handled at hdd layer; 196 * just mark current mode to uninitialized to ensure 197 * it will be set when the delay is over 198 */ 199 napid->napi_mode = QCA_NAPI_TPUT_UNINITIALIZED; 200 break; 201 } 202 default: { 203 hif_err("Unknown event: %d (data=0x%0lx)", 204 event, (unsigned long) data); 205 break; 206 } /* default */ 207 }; /* switch */ 208 209 210 switch (blacklist_pending) { 211 case BLACKLIST_ON_PENDING: 212 /* assume the control of WLAN IRQs */ 213 hif_napi_cpu_blacklist(napid, BLACKLIST_ON); 214 break; 215 case BLACKLIST_OFF_PENDING: 216 /* yield the control of WLAN IRQs */ 217 hif_napi_cpu_blacklist(napid, BLACKLIST_OFF); 218 break; 219 default: /* nothing to do */ 220 break; 221 } /* switch blacklist_pending */ 222 223 qdf_spin_unlock_bh(&(napid->lock)); 224 225 NAPI_DEBUG("<--[rc=%d]", rc); 226 return rc; 227 } 228 229 #endif 230 231 /** 232 * hncm_migrate_to() - migrates a NAPI to a CPU 233 * @napid: pointer to NAPI block 234 * @ce_id: CE_id of the NAPI instance 235 * @didx : index in the CPU topology table for the CPU to migrate to 236 * 237 * Migrates NAPI (identified by the CE_id) to the destination core 238 * Updates the napi_map of the destination entry 239 * 240 * Return: 241 * =0 : success 242 * <0 : error 243 */ 244 static int hncm_exec_migrate_to(struct qca_napi_data *napid, uint8_t ctx_id, 245 int didx) 246 { 247 struct hif_exec_context *exec_ctx; 248 int rc = 0; 249 int status = 0; 250 int ind; 251 252 NAPI_DEBUG("-->%s(napi_cd=%d, didx=%d)", __func__, ctx_id, didx); 253 254 exec_ctx = hif_exec_get_ctx(&napid->hif_softc->osc, ctx_id); 255 if (!exec_ctx) 256 return -EINVAL; 257 258 exec_ctx->cpumask.bits[0] = (1 << didx); 259 260 for (ind = 0; ind < exec_ctx->numirq; ind++) { 261 if (exec_ctx->os_irq[ind]) { 262 irq_modify_status(exec_ctx->os_irq[ind], 263 IRQ_NO_BALANCING, 0); 264 rc = irq_set_affinity_hint(exec_ctx->os_irq[ind], 265 &exec_ctx->cpumask); 266 if (rc) 267 status = rc; 268 } 269 } 270 271 /* unmark the napis bitmap in the cpu table */ 272 napid->napi_cpu[exec_ctx->cpu].napis &= ~(0x01 << ctx_id); 273 /* mark the napis bitmap for the new designated cpu */ 274 napid->napi_cpu[didx].napis |= (0x01 << ctx_id); 275 exec_ctx->cpu = didx; 276 277 NAPI_DEBUG("<--%s[%d]", __func__, rc); 278 return status; 279 } 280 281 /** 282 * hncm_dest_cpu() - finds a destination CPU for NAPI 283 * @napid: pointer to NAPI block 284 * @act : RELOCATE | COLLAPSE | DISPERSE 285 * 286 * Finds the designated destionation for the next IRQ. 287 * RELOCATE: translated to either COLLAPSE or DISPERSE based 288 * on napid->napi_mode (throughput state) 289 * COLLAPSE: All have the same destination: the first online CPU in lilcl 290 * DISPERSE: One of the CPU in bigcl, which has the smallest number of 291 * NAPIs on it 292 * 293 * Return: >=0 : index in the cpu topology table 294 * : < 0 : error 295 */ 296 static int hncm_dest_cpu(struct qca_napi_data *napid, int act) 297 { 298 int destidx = -1; 299 int head, i; 300 301 NAPI_DEBUG("-->%s(act=%d)", __func__, act); 302 if (act == HNC_ACT_RELOCATE) { 303 if (napid->napi_mode == QCA_NAPI_TPUT_LO) 304 act = HNC_ACT_COLLAPSE; 305 else 306 act = HNC_ACT_DISPERSE; 307 NAPI_DEBUG("%s: act changed from HNC_ACT_RELOCATE to %d", 308 __func__, act); 309 } 310 if (act == HNC_ACT_COLLAPSE) { 311 head = i = napid->lilcl_head; 312 retry_collapse: 313 while (i >= 0) { 314 if (napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) { 315 destidx = i; 316 break; 317 } 318 i = napid->napi_cpu[i].cluster_nxt; 319 } 320 if ((destidx < 0) && (head == napid->lilcl_head)) { 321 NAPI_DEBUG("%s: COLLAPSE: no lilcl dest, try bigcl", 322 __func__); 323 head = i = napid->bigcl_head; 324 goto retry_collapse; 325 } 326 } else { /* HNC_ACT_DISPERSE */ 327 int smallest = 99; /* all 32 bits full */ 328 int smallidx = -1; 329 330 head = i = napid->bigcl_head; 331 retry_disperse: 332 while (i >= 0) { 333 if ((napid->napi_cpu[i].state == QCA_NAPI_CPU_UP) && 334 (hweight32(napid->napi_cpu[i].napis) <= smallest)) { 335 smallest = napid->napi_cpu[i].napis; 336 smallidx = i; 337 } 338 i = napid->napi_cpu[i].cluster_nxt; 339 } 340 destidx = smallidx; 341 if ((destidx < 0) && (head == napid->bigcl_head)) { 342 NAPI_DEBUG("%s: DISPERSE: no bigcl dest, try lilcl", 343 __func__); 344 head = i = napid->lilcl_head; 345 goto retry_disperse; 346 } 347 } 348 NAPI_DEBUG("<--%s[dest=%d]", __func__, destidx); 349 return destidx; 350 } 351 /** 352 * hif_napi_cpu_migrate() - migrate IRQs away 353 * @cpu: -1: all CPUs <n> specific CPU 354 * @act: COLLAPSE | DISPERSE 355 * 356 * Moves IRQs/NAPIs from specific or all CPUs (specified by @cpu) to eligible 357 * cores. Eligible cores are: 358 * act=COLLAPSE -> the first online core of the little cluster 359 * act=DISPERSE -> separate cores of the big cluster, so that each core will 360 * host minimum number of NAPIs/IRQs (napid->cpus[cpu].napis) 361 * 362 * Note that this function is called with a spinlock acquired already. 363 * 364 * Return: =0: success 365 * <0: error 366 */ 367 int hif_exec_cpu_migrate(struct qca_napi_data *napid, int cpu, int action) 368 { 369 int rc = 0; 370 struct qca_napi_cpu *cpup; 371 int i, dind; 372 uint32_t napis; 373 374 375 NAPI_DEBUG("-->%s(.., cpu=%d, act=%d)", 376 __func__, cpu, action); 377 378 if (napid->exec_map == 0) { 379 NAPI_DEBUG("%s: datapath contexts to disperse", __func__); 380 goto hncm_return; 381 } 382 cpup = napid->napi_cpu; 383 384 switch (action) { 385 case HNC_ACT_RELOCATE: 386 case HNC_ACT_DISPERSE: 387 case HNC_ACT_COLLAPSE: { 388 /* first find the src napi set */ 389 if (cpu == HNC_ANY_CPU) 390 napis = napid->exec_map; 391 else 392 napis = cpup[cpu].napis; 393 /* then clear the napi bitmap on each CPU */ 394 for (i = 0; i < NR_CPUS; i++) 395 cpup[i].napis = 0; 396 /* then for each of the NAPIs to disperse: */ 397 for (i = 0; i < HIF_MAX_GROUP; i++) 398 if (napis & (1 << i)) { 399 /* find a destination CPU */ 400 dind = hncm_dest_cpu(napid, action); 401 if (dind >= 0) { 402 rc = hncm_exec_migrate_to(napid, i, 403 dind); 404 } else { 405 NAPI_DEBUG("No dest for NAPI ce%d", i); 406 hnc_dump_cpus(napid); 407 rc = -1; 408 } 409 } 410 break; 411 } 412 default: { 413 NAPI_DEBUG("%s: bad action: %d\n", __func__, action); 414 QDF_BUG(0); 415 break; 416 } 417 } /* switch action */ 418 419 hncm_return: 420 hnc_dump_cpus(napid); 421 return rc; 422 } 423 424 425 /** 426 * hif_exec_bl_irq() - calls irq_modify_status to enable/disable blacklisting 427 * @napid: pointer to qca_napi_data structure 428 * @bl_flag: blacklist flag to enable/disable blacklisting 429 * 430 * The function enables/disables blacklisting for all the copy engine 431 * interrupts on which NAPI is enabled. 432 * 433 * Return: None 434 */ 435 static inline void hif_exec_bl_irq(struct qca_napi_data *napid, bool bl_flag) 436 { 437 int i, j; 438 struct hif_exec_context *exec_ctx; 439 440 for (i = 0; i < HIF_MAX_GROUP; i++) { 441 /* check if NAPI is enabled on the CE */ 442 if (!(napid->exec_map & (0x01 << i))) 443 continue; 444 445 /*double check that NAPI is allocated for the CE */ 446 exec_ctx = hif_exec_get_ctx(&napid->hif_softc->osc, i); 447 if (!(exec_ctx)) 448 continue; 449 450 if (bl_flag == true) 451 for (j = 0; j < exec_ctx->numirq; j++) 452 irq_modify_status(exec_ctx->os_irq[j], 453 0, IRQ_NO_BALANCING); 454 else 455 for (j = 0; j < exec_ctx->numirq; j++) 456 irq_modify_status(exec_ctx->os_irq[j], 457 IRQ_NO_BALANCING, 0); 458 hif_debug("bl_flag %d CE %d", bl_flag, i); 459 } 460 } 461 462 /** 463 * hif_napi_cpu_blacklist() - en(dis)ables blacklisting for NAPI RX interrupts. 464 * @napid: pointer to qca_napi_data structure 465 * @op: blacklist operation to perform 466 * 467 * The function enables/disables/queries blacklisting for all CE RX 468 * interrupts with NAPI enabled. Besides blacklisting, it also enables/disables 469 * core_ctl_set_boost. 470 * Once blacklisting is enabled, the interrupts will not be managed by the IRQ 471 * balancer. 472 * 473 * Return: -EINVAL, in case IRQ_BLACKLISTING and CORE_CTL_BOOST is not enabled 474 * for BLACKLIST_QUERY op - blacklist refcount 475 * for BLACKLIST_ON op - return value from core_ctl_set_boost API 476 * for BLACKLIST_OFF op - return value from core_ctl_set_boost API 477 */ 478 int hif_exec_cpu_blacklist(struct qca_napi_data *napid, 479 enum qca_blacklist_op op) 480 { 481 int rc = 0; 482 static int ref_count; /* = 0 by the compiler */ 483 uint8_t flags = napid->flags; 484 bool bl_en = flags & QCA_NAPI_FEATURE_IRQ_BLACKLISTING; 485 bool ccb_en = flags & QCA_NAPI_FEATURE_CORE_CTL_BOOST; 486 487 NAPI_DEBUG("-->%s(%d %d)", __func__, flags, op); 488 489 if (!(bl_en && ccb_en)) { 490 rc = -EINVAL; 491 goto out; 492 } 493 494 switch (op) { 495 case BLACKLIST_QUERY: 496 rc = ref_count; 497 break; 498 case BLACKLIST_ON: 499 ref_count++; 500 rc = 0; 501 if (ref_count == 1) { 502 rc = hif_napi_core_ctl_set_boost(true); 503 NAPI_DEBUG("boost_on() returns %d - refcnt=%d", 504 rc, ref_count); 505 hif_exec_bl_irq(napid, true); 506 } 507 break; 508 case BLACKLIST_OFF: 509 if (ref_count) 510 ref_count--; 511 rc = 0; 512 if (ref_count == 0) { 513 rc = hif_napi_core_ctl_set_boost(false); 514 NAPI_DEBUG("boost_off() returns %d - refcnt=%d", 515 rc, ref_count); 516 hif_exec_bl_irq(napid, false); 517 } 518 break; 519 default: 520 NAPI_DEBUG("Invalid blacklist op: %d", op); 521 rc = -EINVAL; 522 } /* switch */ 523 out: 524 NAPI_DEBUG("<--%s[%d]", __func__, rc); 525 return rc; 526 } 527 528