1 /* 2 * Copyright (c) 2019-2020, The Linux Foundation. All rights reserved. 3 * 4 * Permission to use, copy, modify, and/or distribute this software for any 5 * purpose with or without fee is hereby granted, provided that the above 6 * copyright notice and this permission notice appear in all copies. 7 * 8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 15 */ 16 17 #include "qdf_module.h" 18 #include "dp_types.h" 19 #include "hal_rx_flow.h" 20 21 #if defined(WLAN_SUPPORT_RX_FISA) 22 void hal_rx_dump_fse_table(struct hal_rx_fst *fst) 23 { 24 int i = 0; 25 struct rx_flow_search_entry *fse = 26 (struct rx_flow_search_entry *)fst->base_vaddr; 27 28 dp_info("Number flow table entries %d", fst->add_flow_count); 29 for (i = 0; i < fst->max_entries; i++) { 30 if (fse[i].valid) { 31 dp_info("index %d:" 32 " src_ip_127_96 0x%x" 33 " src_ip_95_640 0x%x" 34 " src_ip_63_32 0x%x" 35 " src_ip_31_0 0x%x" 36 " dest_ip_127_96 0x%x" 37 " dest_ip_95_64 0x%x" 38 " dest_ip_63_32 0x%x" 39 " dest_ip_31_0 0x%x" 40 " src_port 0x%x" 41 " dest_port 0x%x" 42 " l4_protocol 0x%x" 43 " valid 0x%x" 44 " reo_destination_indication 0x%x" 45 " msdu_drop 0x%x" 46 " reo_destination_handler 0x%x" 47 " metadata 0x%x" 48 " aggregation_count0x%x" 49 " lro_eligible 0x%x" 50 " msdu_count 0x%x" 51 " msdu_byte_count 0x%x" 52 " timestamp 0x%x" 53 " cumulative_l4_checksum 0x%x" 54 " cumulative_ip_length 0x%x" 55 " tcp_sequence_number 0x%x", 56 i, 57 fse[i].src_ip_127_96, 58 fse[i].src_ip_95_64, 59 fse[i].src_ip_63_32, 60 fse[i].src_ip_31_0, 61 fse[i].dest_ip_127_96, 62 fse[i].dest_ip_95_64, 63 fse[i].dest_ip_63_32, 64 fse[i].dest_ip_31_0, 65 fse[i].src_port, 66 fse[i].dest_port, 67 fse[i].l4_protocol, 68 fse[i].valid, 69 fse[i].reo_destination_indication, 70 fse[i].msdu_drop, 71 fse[i].reo_destination_handler, 72 fse[i].metadata, 73 fse[i].aggregation_count, 74 fse[i].lro_eligible, 75 fse[i].msdu_count, 76 fse[i].msdu_byte_count, 77 fse[i].timestamp, 78 fse[i].cumulative_l4_checksum, 79 fse[i].cumulative_ip_length, 80 fse[i].tcp_sequence_number); 81 } 82 } 83 } 84 #else 85 void hal_rx_dump_fse_table(struct hal_rx_fst *fst) 86 { 87 } 88 #endif 89 90 /** 91 * hal_rx_flow_setup_fse() - Setup a flow search entry in HW FST 92 * @fst: Pointer to the Rx Flow Search Table 93 * @table_offset: offset into the table where the flow is to be setup 94 * @flow: Flow Parameters 95 * 96 * Return: Success/Failure 97 */ 98 void * 99 hal_rx_flow_setup_fse(hal_soc_handle_t hal_soc_hdl, 100 struct hal_rx_fst *fst, uint32_t table_offset, 101 struct hal_rx_flow *flow) 102 { 103 struct hal_soc *hal_soc = (struct hal_soc *)hal_soc_hdl; 104 105 if (hal_soc->ops->hal_rx_flow_setup_fse) { 106 return hal_soc->ops->hal_rx_flow_setup_fse((uint8_t *)fst, 107 table_offset, 108 (uint8_t *)flow); 109 } 110 111 return NULL; 112 } 113 qdf_export_symbol(hal_rx_flow_setup_fse); 114 115 /** 116 * hal_rx_flow_delete_entry() - Delete a flow from the Rx Flow Search Table 117 * @fst: Pointer to the Rx Flow Search Table 118 * @hal_rx_fse: Pointer to the Rx Flow that is to be deleted from the FST 119 * 120 * Return: Success/Failure 121 */ 122 inline QDF_STATUS 123 hal_rx_flow_delete_entry(struct hal_rx_fst *fst, void *hal_rx_fse) 124 { 125 uint8_t *fse = (uint8_t *)hal_rx_fse; 126 127 if (!HAL_GET_FLD(fse, RX_FLOW_SEARCH_ENTRY_9, VALID)) 128 return QDF_STATUS_E_NOENT; 129 130 HAL_CLR_FLD(fse, RX_FLOW_SEARCH_ENTRY_9, VALID); 131 132 return QDF_STATUS_SUCCESS; 133 } 134 qdf_export_symbol(hal_rx_flow_delete_entry); 135 136 #ifndef WLAN_SUPPORT_RX_FISA 137 /** 138 * hal_rx_fst_key_configure() - Configure the Toeplitz key in the FST 139 * @fst: Pointer to the Rx Flow Search Table 140 * 141 * Return: Success/Failure 142 */ 143 static void hal_rx_fst_key_configure(struct hal_rx_fst *fst) 144 { 145 uint8_t key_bytes[HAL_FST_HASH_KEY_SIZE_BYTES]; 146 147 qdf_mem_copy(key_bytes, fst->key, HAL_FST_HASH_KEY_SIZE_BYTES); 148 149 /** 150 * The Toeplitz algorithm as per the Microsoft spec works in a 151 * “big-endian” manner, using the MSBs of the key to hash the 152 * initial bytes of the input going on to use up the lower order bits 153 * of the key to hash further bytes of the input until the LSBs of the 154 * key are used finally. 155 * 156 * So first, rightshift 320-bit input key 5 times to get 315 MS bits 157 */ 158 key_bitwise_shift_left(key_bytes, HAL_FST_HASH_KEY_SIZE_BYTES, 5); 159 key_reverse(fst->shifted_key, key_bytes, HAL_FST_HASH_KEY_SIZE_BYTES); 160 } 161 #else 162 static void hal_rx_fst_key_configure(struct hal_rx_fst *fst) 163 { 164 } 165 #endif 166 167 /** 168 * hal_rx_fst_get_base() - Retrieve the virtual base address of the Rx FST 169 * @fst: Pointer to the Rx Flow Search Table 170 * 171 * Return: Success/Failure 172 */ 173 static inline void *hal_rx_fst_get_base(struct hal_rx_fst *fst) 174 { 175 return fst->base_vaddr; 176 } 177 178 /** 179 * hal_rx_fst_get_fse_size() - Retrieve the size of each entry(flow) in Rx FST 180 * 181 * Return: size of each entry/flow in Rx FST 182 */ 183 static inline uint32_t hal_rx_fst_get_fse_size(void) 184 { 185 return HAL_RX_FST_ENTRY_SIZE; 186 } 187 188 /** 189 * hal_rx_flow_get_tuple_info() - Retrieve the 5-tuple flow info for an entry 190 * @hal_fse: Pointer to the Flow in Rx FST 191 * @tuple_info: 5-tuple info of the flow returned to the caller 192 * 193 * Return: Success/Failure 194 */ 195 QDF_STATUS hal_rx_flow_get_tuple_info(void *hal_fse, 196 struct hal_flow_tuple_info *tuple_info) 197 { 198 if (!hal_fse || !tuple_info) 199 return QDF_STATUS_E_INVAL; 200 201 if (!HAL_GET_FLD(hal_fse, RX_FLOW_SEARCH_ENTRY_9, VALID)) 202 return QDF_STATUS_E_NOENT; 203 204 tuple_info->src_ip_127_96 = 205 qdf_ntohl(HAL_GET_FLD(hal_fse, 206 RX_FLOW_SEARCH_ENTRY_0, 207 SRC_IP_127_96)); 208 tuple_info->src_ip_95_64 = 209 qdf_ntohl(HAL_GET_FLD(hal_fse, 210 RX_FLOW_SEARCH_ENTRY_1, 211 SRC_IP_95_64)); 212 tuple_info->src_ip_63_32 = 213 qdf_ntohl(HAL_GET_FLD(hal_fse, 214 RX_FLOW_SEARCH_ENTRY_2, 215 SRC_IP_63_32)); 216 tuple_info->src_ip_31_0 = 217 qdf_ntohl(HAL_GET_FLD(hal_fse, 218 RX_FLOW_SEARCH_ENTRY_3, 219 SRC_IP_31_0)); 220 tuple_info->dest_ip_127_96 = 221 qdf_ntohl(HAL_GET_FLD(hal_fse, 222 RX_FLOW_SEARCH_ENTRY_4, 223 DEST_IP_127_96)); 224 tuple_info->dest_ip_95_64 = 225 qdf_ntohl(HAL_GET_FLD(hal_fse, 226 RX_FLOW_SEARCH_ENTRY_5, 227 DEST_IP_95_64)); 228 tuple_info->dest_ip_63_32 = 229 qdf_ntohl(HAL_GET_FLD(hal_fse, 230 RX_FLOW_SEARCH_ENTRY_6, 231 DEST_IP_63_32)); 232 tuple_info->dest_ip_31_0 = 233 qdf_ntohl(HAL_GET_FLD(hal_fse, 234 RX_FLOW_SEARCH_ENTRY_7, 235 DEST_IP_31_0)); 236 tuple_info->dest_port = HAL_GET_FLD(hal_fse, 237 RX_FLOW_SEARCH_ENTRY_8, 238 DEST_PORT); 239 tuple_info->src_port = HAL_GET_FLD(hal_fse, 240 RX_FLOW_SEARCH_ENTRY_8, 241 SRC_PORT); 242 tuple_info->l4_protocol = HAL_GET_FLD(hal_fse, 243 RX_FLOW_SEARCH_ENTRY_9, 244 L4_PROTOCOL); 245 246 return QDF_STATUS_SUCCESS; 247 } 248 249 #ifndef WLAN_SUPPORT_RX_FISA 250 /** 251 * hal_flow_toeplitz_create_cache() - Calculate hashes for each possible 252 * byte value with the key taken as is 253 * 254 * @fst: FST Handle 255 * @key: Hash Key 256 * 257 * Return: Success/Failure 258 */ 259 static void hal_flow_toeplitz_create_cache(struct hal_rx_fst *fst) 260 { 261 int bit; 262 int val; 263 int i; 264 uint8_t *key = fst->shifted_key; 265 266 /* 267 * Initialise to first 32 bits of the key; shift in further key material 268 * through the loop 269 */ 270 uint32_t cur_key = (key[0] << 24) | (key[1] << 16) | (key[2] << 8) | 271 key[3]; 272 273 for (i = 0; i < HAL_FST_HASH_KEY_SIZE_BYTES; i++) { 274 uint8_t new_key_byte; 275 uint32_t shifted_key[8]; 276 277 if (i + 4 < HAL_FST_HASH_KEY_SIZE_BYTES) 278 new_key_byte = key[i + 4]; 279 else 280 new_key_byte = 0; 281 282 shifted_key[0] = cur_key; 283 284 for (bit = 1; bit < 8; bit++) { 285 /* 286 * For each iteration, shift out one more bit of the 287 * current key and shift in one more bit of the new key 288 * material 289 */ 290 shifted_key[bit] = cur_key << bit | 291 new_key_byte >> (8 - bit); 292 } 293 294 for (val = 0; val < (1 << 8); val++) { 295 uint32_t hash = 0; 296 int mask; 297 298 /* 299 * For each bit set in the input, XOR in 300 * the appropriately shifted key 301 */ 302 for (bit = 0, mask = 1 << 7; bit < 8; bit++, mask >>= 1) 303 if ((val & mask)) 304 hash ^= shifted_key[bit]; 305 306 fst->key_cache[i][val] = hash; 307 } 308 309 cur_key = cur_key << 8 | new_key_byte; 310 } 311 } 312 #else 313 static void hal_flow_toeplitz_create_cache(struct hal_rx_fst *fst) 314 { 315 } 316 #endif 317 318 /** 319 * hal_rx_fst_attach() - Initialize Rx flow search table in HW FST 320 * 321 * @qdf_dev: QDF device handle 322 * @hal_fst_base_paddr: Pointer to the physical base address of the Rx FST 323 * @max_entries: Max number of flows allowed in the FST 324 * @max_search: Number of collisions allowed in the hash-based FST 325 * @hash_key: Toeplitz key used for the hash FST 326 * 327 * Return: 328 */ 329 struct hal_rx_fst * 330 hal_rx_fst_attach(qdf_device_t qdf_dev, 331 uint64_t *hal_fst_base_paddr, uint16_t max_entries, 332 uint16_t max_search, uint8_t *hash_key) 333 { 334 struct hal_rx_fst *fst = qdf_mem_malloc(sizeof(struct hal_rx_fst)); 335 336 if (!fst) { 337 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR, 338 FL("hal fst allocation failed,")); 339 return NULL; 340 } 341 342 qdf_mem_set(fst, 0, sizeof(struct hal_rx_fst)); 343 344 fst->key = hash_key; 345 fst->max_skid_length = max_search; 346 fst->max_entries = max_entries; 347 fst->hash_mask = max_entries - 1; 348 349 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_DEBUG, 350 "HAL FST allocation %pK %d * %d\n", fst, 351 fst->max_entries, HAL_RX_FST_ENTRY_SIZE); 352 353 fst->base_vaddr = (uint8_t *)qdf_mem_alloc_consistent(qdf_dev, 354 qdf_dev->dev, 355 (fst->max_entries * HAL_RX_FST_ENTRY_SIZE), 356 &fst->base_paddr); 357 358 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_INFO, 359 "hal_rx_fst base address 0x%pK", (void *)fst->base_paddr); 360 if (!fst->base_vaddr) { 361 QDF_TRACE(QDF_MODULE_ID_TXRX, QDF_TRACE_LEVEL_ERROR, 362 FL("hal fst->base_vaddr allocation failed")); 363 qdf_mem_free(fst); 364 return NULL; 365 } 366 QDF_TRACE_HEX_DUMP(QDF_MODULE_ID_ANY, QDF_TRACE_LEVEL_DEBUG, 367 (void *)fst->key, HAL_FST_HASH_KEY_SIZE_BYTES); 368 369 qdf_mem_set((uint8_t *)fst->base_vaddr, 0, 370 (fst->max_entries * HAL_RX_FST_ENTRY_SIZE)); 371 372 hal_rx_fst_key_configure(fst); 373 hal_flow_toeplitz_create_cache(fst); 374 *hal_fst_base_paddr = (uint64_t)fst->base_paddr; 375 return fst; 376 } 377 qdf_export_symbol(hal_rx_fst_attach); 378 379 /** 380 * hal_rx_fst_detach() - De-init the Rx flow search table from HW 381 * 382 * @rx_fst: Pointer to the Rx FST 383 * @qdf_dev: QDF device handle 384 * 385 * Return: 386 */ 387 void hal_rx_fst_detach(struct hal_rx_fst *rx_fst, 388 qdf_device_t qdf_dev) 389 { 390 if (!rx_fst || !qdf_dev) 391 return; 392 393 qdf_mem_free_consistent(qdf_dev, qdf_dev->dev, 394 rx_fst->max_entries * HAL_RX_FST_ENTRY_SIZE, 395 rx_fst->base_vaddr, rx_fst->base_paddr, 0); 396 397 qdf_mem_free(rx_fst); 398 } 399 qdf_export_symbol(hal_rx_fst_detach); 400 401 #ifndef WLAN_SUPPORT_RX_FISA 402 /** 403 * hal_flow_toeplitz_hash() - Calculate Toeplitz hash by using the cached key 404 * 405 * @hal_fst: FST Handle 406 * @flow: Flow Parameters 407 * 408 * Return: Success/Failure 409 */ 410 uint32_t 411 hal_flow_toeplitz_hash(void *hal_fst, struct hal_rx_flow *flow) 412 { 413 int i, j; 414 uint32_t hash = 0; 415 struct hal_rx_fst *fst = (struct hal_rx_fst *)hal_fst; 416 uint32_t input[HAL_FST_HASH_KEY_SIZE_WORDS]; 417 uint8_t *tuple; 418 419 qdf_mem_zero(input, HAL_FST_HASH_KEY_SIZE_BYTES); 420 *(uint32_t *)&input[0] = qdf_htonl(flow->tuple_info.src_ip_127_96); 421 *(uint32_t *)&input[1] = qdf_htonl(flow->tuple_info.src_ip_95_64); 422 *(uint32_t *)&input[2] = qdf_htonl(flow->tuple_info.src_ip_63_32); 423 *(uint32_t *)&input[3] = qdf_htonl(flow->tuple_info.src_ip_31_0); 424 *(uint32_t *)&input[4] = qdf_htonl(flow->tuple_info.dest_ip_127_96); 425 *(uint32_t *)&input[5] = qdf_htonl(flow->tuple_info.dest_ip_95_64); 426 *(uint32_t *)&input[6] = qdf_htonl(flow->tuple_info.dest_ip_63_32); 427 *(uint32_t *)&input[7] = qdf_htonl(flow->tuple_info.dest_ip_31_0); 428 *(uint32_t *)&input[8] = (flow->tuple_info.dest_port << 16) | 429 (flow->tuple_info.src_port); 430 *(uint32_t *)&input[9] = flow->tuple_info.l4_protocol; 431 432 tuple = (uint8_t *)input; 433 QDF_TRACE_HEX_DUMP(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_DEBUG, 434 tuple, sizeof(input)); 435 for (i = 0, j = HAL_FST_HASH_DATA_SIZE - 1; 436 i < HAL_FST_HASH_KEY_SIZE_BYTES && j >= 0; i++, j--) { 437 hash ^= fst->key_cache[i][tuple[j]]; 438 } 439 440 QDF_TRACE(QDF_MODULE_ID_DP, QDF_TRACE_LEVEL_INFO_LOW, 441 "Hash value %u %u truncated hash %u\n", hash, 442 (hash >> 12), (hash >> 12) % (fst->max_entries)); 443 444 hash >>= 12; 445 hash &= (fst->max_entries - 1); 446 447 return hash; 448 } 449 #else 450 uint32_t 451 hal_flow_toeplitz_hash(void *hal_fst, struct hal_rx_flow *flow) 452 { 453 return 0; 454 } 455 #endif 456 qdf_export_symbol(hal_flow_toeplitz_hash); 457 458 /** 459 * hal_rx_get_hal_hash() - Retrieve hash index of a flow in the FST table 460 * 461 * @hal_fst: HAL Rx FST Handle 462 * @flow_hash: Flow hash computed from flow tuple 463 * 464 * Return: hash index truncated to the size of the hash table 465 */ 466 uint32_t hal_rx_get_hal_hash(struct hal_rx_fst *hal_fst, uint32_t flow_hash) 467 { 468 uint32_t trunc_hash = flow_hash; 469 470 /* Take care of hash wrap around scenario */ 471 if (flow_hash >= hal_fst->max_entries) 472 trunc_hash &= hal_fst->hash_mask; 473 return trunc_hash; 474 } 475 qdf_export_symbol(hal_rx_get_hal_hash); 476 477 /** 478 * hal_rx_insert_flow_entry() - Add a flow into the FST table 479 * 480 * @hal_fst: HAL Rx FST Handle 481 * @flow_hash: Flow hash computed from flow tuple 482 * @flow_tuple_info: Flow tuple used to compute the hash 483 * @flow_index: Hash index of the flow in the table when inserted successfully 484 * 485 * Return: Success if flow is inserted into the table, error otherwise 486 */ 487 QDF_STATUS 488 hal_rx_insert_flow_entry(struct hal_rx_fst *fst, uint32_t flow_hash, 489 void *flow_tuple_info, uint32_t *flow_idx) 490 { 491 int i; 492 void *hal_fse = NULL; 493 uint32_t hal_hash = 0; 494 struct hal_flow_tuple_info hal_tuple_info = { 0 }; 495 QDF_STATUS status; 496 497 for (i = 0; i < fst->max_skid_length; i++) { 498 hal_hash = hal_rx_get_hal_hash(fst, (flow_hash + i)); 499 hal_fse = (uint8_t *)fst->base_vaddr + 500 (hal_hash * HAL_RX_FST_ENTRY_SIZE); 501 status = hal_rx_flow_get_tuple_info(hal_fse, &hal_tuple_info); 502 if (status == QDF_STATUS_E_NOENT) 503 break; 504 505 /* Find the matching flow entry in HW FST */ 506 if (!qdf_mem_cmp(&hal_tuple_info, 507 flow_tuple_info, 508 sizeof(struct hal_flow_tuple_info))) { 509 dp_err("Duplicate flow entry in FST %u at skid %u ", 510 hal_hash, i); 511 return QDF_STATUS_E_EXISTS; 512 } 513 } 514 if (i == fst->max_skid_length) { 515 dp_err("Max skid length reached for hash %u", flow_hash); 516 return QDF_STATUS_E_RANGE; 517 } 518 *flow_idx = hal_hash; 519 dp_info("flow_hash = %u, skid_entry = %d, flow_addr = %pK flow_idx = %d", 520 flow_hash, i, hal_fse, *flow_idx); 521 522 return QDF_STATUS_SUCCESS; 523 } 524 qdf_export_symbol(hal_rx_insert_flow_entry); 525 526 /** 527 * hal_rx_find_flow_from_tuple() - Find a flow in the FST table 528 * 529 * @fst: HAL Rx FST Handle 530 * @flow_hash: Flow hash computed from flow tuple 531 * @flow_tuple_info: Flow tuple used to compute the hash 532 * @flow_index: Hash index of the flow in the table when found 533 * 534 * Return: Success if matching flow is found in the table, error otherwise 535 */ 536 QDF_STATUS 537 hal_rx_find_flow_from_tuple(struct hal_rx_fst *fst, uint32_t flow_hash, 538 void *flow_tuple_info, uint32_t *flow_idx) 539 { 540 int i; 541 void *hal_fse = NULL; 542 uint32_t hal_hash = 0; 543 struct hal_flow_tuple_info hal_tuple_info = { 0 }; 544 QDF_STATUS status; 545 546 for (i = 0; i < fst->max_skid_length; i++) { 547 hal_hash = hal_rx_get_hal_hash(fst, (flow_hash + i)); 548 hal_fse = (uint8_t *)fst->base_vaddr + 549 (hal_hash * HAL_RX_FST_ENTRY_SIZE); 550 status = hal_rx_flow_get_tuple_info(hal_fse, &hal_tuple_info); 551 if (status != QDF_STATUS_SUCCESS) 552 continue; 553 554 /* Find the matching flow entry in HW FST */ 555 if (!qdf_mem_cmp(&hal_tuple_info, 556 flow_tuple_info, 557 sizeof(struct hal_flow_tuple_info))) { 558 break; 559 } 560 } 561 562 if (i == fst->max_skid_length) { 563 dp_err("Max skid length reached for hash %u", flow_hash); 564 return QDF_STATUS_E_RANGE; 565 } 566 567 *flow_idx = hal_hash; 568 dp_info("flow_hash = %u, skid_entry = %d, flow_addr = %pK flow_idx = %d", 569 flow_hash, i, hal_fse, *flow_idx); 570 571 return QDF_STATUS_SUCCESS; 572 } 573 qdf_export_symbol(hal_rx_find_flow_from_tuple); 574