417  *      rvu_exact_calculate_hash - calculate hash index to mem table.
423  *	@table_depth: Depth of table.
429 	struct npc_exact_table *table = rvu->hw->table;  in rvu_exact_calculate_hash()  local
448 	hash &= table->mem_table.hash_mask;  in rvu_exact_calculate_hash()
449 	hash += table->mem_table.hash_offset;  in rvu_exact_calculate_hash()
456  *      rvu_npc_exact_alloc_mem_table_entry - find free entry in 4 way table.
458  *	@way: Indicate way to table.
459  *	@index: Hash index to 4 way table.
462  *	Searches 4 way table using hash index. Returns 0 on success.
468 	struct npc_exact_table *table;  in rvu_npc_exact_alloc_mem_table_entry()  local
471 	table = rvu->hw->table;  in rvu_npc_exact_alloc_mem_table_entry()
472 	depth = table->mem_table.depth;  in rvu_npc_exact_alloc_mem_table_entry()
475 	mutex_lock(&table->lock);  in rvu_npc_exact_alloc_mem_table_entry()
476 	for (i = 0; i <  table->mem_table.ways; i++) {  in rvu_npc_exact_alloc_mem_table_entry()
477 		if (test_bit(hash + i * depth, table->mem_table.bmap))  in rvu_npc_exact_alloc_mem_table_entry()
480 		set_bit(hash + i * depth, table->mem_table.bmap);  in rvu_npc_exact_alloc_mem_table_entry()
481 		mutex_unlock(&table->lock);  in rvu_npc_exact_alloc_mem_table_entry()
483 		dev_dbg(rvu->dev, "%s: mem table entry alloc success (way=%d index=%d)\n",  in rvu_npc_exact_alloc_mem_table_entry()
490 	mutex_unlock(&table->lock);  in rvu_npc_exact_alloc_mem_table_entry()
493 		bitmap_weight(table->mem_table.bmap, table->mem_table.depth));  in rvu_npc_exact_alloc_mem_table_entry()
504 	struct npc_exact_table *table;  in rvu_npc_exact_free_id()  local
506 	table = rvu->hw->table;  in rvu_npc_exact_free_id()
507 	mutex_lock(&table->lock);  in rvu_npc_exact_free_id()
508 	clear_bit(seq_id, table->id_bmap);  in rvu_npc_exact_free_id()
509 	mutex_unlock(&table->lock);  in rvu_npc_exact_free_id()
521 	struct npc_exact_table *table;  in rvu_npc_exact_alloc_id()  local
524 	table = rvu->hw->table;  in rvu_npc_exact_alloc_id()
526 	mutex_lock(&table->lock);  in rvu_npc_exact_alloc_id()
527 	idx = find_first_zero_bit(table->id_bmap, table->tot_ids);  in rvu_npc_exact_alloc_id()
528 	if (idx == table->tot_ids) {  in rvu_npc_exact_alloc_id()
529 		mutex_unlock(&table->lock);  in rvu_npc_exact_alloc_id()
531 			__func__, table->tot_ids);  in rvu_npc_exact_alloc_id()
537 	set_bit(idx, table->id_bmap);  in rvu_npc_exact_alloc_id()
538 	mutex_unlock(&table->lock);  in rvu_npc_exact_alloc_id()
547  *      rvu_npc_exact_alloc_cam_table_entry - find free slot in fully associative table.
549  *	@index: Index to exact CAM table.
554 	struct npc_exact_table *table;  in rvu_npc_exact_alloc_cam_table_entry()  local
557 	table = rvu->hw->table;  in rvu_npc_exact_alloc_cam_table_entry()
559 	mutex_lock(&table->lock);  in rvu_npc_exact_alloc_cam_table_entry()
560 	idx = find_first_zero_bit(table->cam_table.bmap, table->cam_table.depth);  in rvu_npc_exact_alloc_cam_table_entry()
561 	if (idx == table->cam_table.depth) {  in rvu_npc_exact_alloc_cam_table_entry()
562 		mutex_unlock(&table->lock);  in rvu_npc_exact_alloc_cam_table_entry()
563 		dev_info(rvu->dev, "%s: No space in exact cam table, weight=%u\n", __func__,  in rvu_npc_exact_alloc_cam_table_entry()
564 			 bitmap_weight(table->cam_table.bmap, table->cam_table.depth));  in rvu_npc_exact_alloc_cam_table_entry()
569 	set_bit(idx, table->cam_table.bmap);  in rvu_npc_exact_alloc_cam_table_entry()
570 	mutex_unlock(&table->lock);  in rvu_npc_exact_alloc_cam_table_entry()
573 	dev_dbg(rvu->dev, "%s: cam table entry alloc success (index=%d)\n",  in rvu_npc_exact_alloc_cam_table_entry()
579  *	rvu_exact_prepare_table_entry - Data for exact match table entry.
585  *	Return: mdata for exact match table.
663  *	rvu_exact_config_result_ctrl - Set exact table hash control
665  *	@depth: Depth of Exact match table.
667  *	Sets mask and offset for hash for mem table.
677 	rvu->hw->table->mem_table.hash_mask = (depth - 1);  in rvu_exact_config_result_ctrl()
681 	rvu->hw->table->mem_table.hash_offset = 0;  in rvu_exact_config_result_ctrl()
690  *	rvu_exact_config_table_mask - Set exact table mask.
710 	rvu->hw->table->mem_table.mask = mask;  in rvu_exact_config_table_mask()
717  *      rvu_npc_exact_get_max_entries - Get total number of entries in table.
719  *	Return: Maximum table entries possible.
723 	struct npc_exact_table *table;  in rvu_npc_exact_get_max_entries()  local
725 	table = rvu->hw->table;  in rvu_npc_exact_get_max_entries()
726 	return table->tot_ids;  in rvu_npc_exact_get_max_entries()
732  *	Return: True if exact match table is supported/enabled.
745  *	Return: Pointer to table entry.
750 	struct npc_exact_table *table = rvu->hw->table;  in __rvu_npc_exact_find_entry_by_seq_id()  local
754 	lhead = &table->lhead_gbl;  in __rvu_npc_exact_find_entry_by_seq_id()
770  *	@opc_type: OPCODE to select MEM/CAM table.
771  *	@ways: MEM table ways.
772  *	@index: Index in MEM/CAM table.
789 	struct npc_exact_table *table = rvu->hw->table;  in rvu_npc_exact_add_to_list()  local
806 	mutex_lock(&table->lock);  in rvu_npc_exact_add_to_list()
809 		lhead = &table->lhead_cam_tbl_entry;  in rvu_npc_exact_add_to_list()
810 		table->cam_tbl_entry_cnt++;  in rvu_npc_exact_add_to_list()
814 		lhead = &table->lhead_mem_tbl_entry[ways];  in rvu_npc_exact_add_to_list()
815 		table->mem_tbl_entry_cnt++;  in rvu_npc_exact_add_to_list()
819 		mutex_unlock(&table->lock);  in rvu_npc_exact_add_to_list()
829 	list_add_tail(&entry->glist, &table->lhead_gbl);  in rvu_npc_exact_add_to_list()
858 	/* Add to each table list */  in rvu_npc_exact_add_to_list()
860 	mutex_unlock(&table->lock);  in rvu_npc_exact_add_to_list()
868  *	@ways: ways for MEM table.
892  *      rvu_npc_exact_dealloc_table_entry - dealloc table entry
894  *	@opc_type: OPCODE for selection of table(MEM or CAM)
895  *	@ways: ways if opc_type is MEM table.
896  *	@index: Index of MEM or CAM table.
903 	struct npc_exact_table *table;  in rvu_npc_exact_dealloc_table_entry()  local
910 	table = rvu->hw->table;  in rvu_npc_exact_dealloc_table_entry()
911 	depth = table->mem_table.depth;  in rvu_npc_exact_dealloc_table_entry()
913 	mutex_lock(&table->lock);  in rvu_npc_exact_dealloc_table_entry()
919 		if (!test_bit(index, table->cam_table.bmap)) {  in rvu_npc_exact_dealloc_table_entry()
920 			mutex_unlock(&table->lock);  in rvu_npc_exact_dealloc_table_entry()
927 		clear_bit(index, table->cam_table.bmap);  in rvu_npc_exact_dealloc_table_entry()
933 		if (!test_bit(index + ways * depth, table->mem_table.bmap)) {  in rvu_npc_exact_dealloc_table_entry()
934 			mutex_unlock(&table->lock);  in rvu_npc_exact_dealloc_table_entry()
941 		clear_bit(index + ways * depth, table->mem_table.bmap);  in rvu_npc_exact_dealloc_table_entry()
945 		mutex_unlock(&table->lock);  in rvu_npc_exact_dealloc_table_entry()
950 	mutex_unlock(&table->lock);  in rvu_npc_exact_dealloc_table_entry()
964  *	@index: Index of MEM table or CAM table.
965  *	@ways: Ways. Only valid for MEM table.
966  *	@opc_type: OPCODE to select table (MEM or CAM)
968  *	Try allocating a slot from MEM table. If all 4 ways
970  *	32-entry CAM table for allocation.
976 	struct npc_exact_table *table;  in rvu_npc_exact_alloc_table_entry()  local
980 	table = rvu->hw->table;  in rvu_npc_exact_alloc_table_entry()
983 	hash =  rvu_exact_calculate_hash(rvu, chan, ctype, mac, table->mem_table.mask,  in rvu_npc_exact_alloc_table_entry()
984 					 table->mem_table.depth);  in rvu_npc_exact_alloc_table_entry()
988 		dev_dbg(rvu->dev, "%s: inserted in 4 ways hash table ways=%d, index=%d\n",  in rvu_npc_exact_alloc_table_entry()
993 	dev_dbg(rvu->dev, "%s: failed to insert in 4 ways hash table\n", __func__);  in rvu_npc_exact_alloc_table_entry()
995 	/* wayss is 0 for cam table */  in rvu_npc_exact_alloc_table_entry()
1000 		dev_dbg(rvu->dev, "%s: inserted in fully associative hash table index=%u\n",  in rvu_npc_exact_alloc_table_entry()
1005 	dev_err(rvu->dev, "%s: failed to insert in fully associative hash table\n", __func__);  in rvu_npc_exact_alloc_table_entry()
1021 	struct npc_exact_table *table;  in rvu_npc_exact_save_drop_rule_chan_and_mask()  local
1024 	table = rvu->hw->table;  in rvu_npc_exact_save_drop_rule_chan_and_mask()
1027 		if (!table->drop_rule_map[i].valid)  in rvu_npc_exact_save_drop_rule_chan_and_mask()
1030 		if (table->drop_rule_map[i].chan_val != (u16)chan_val)  in rvu_npc_exact_save_drop_rule_chan_and_mask()
1033 		if (table->drop_rule_map[i].chan_mask != (u16)chan_mask)  in rvu_npc_exact_save_drop_rule_chan_and_mask()
1042 	table->drop_rule_map[i].drop_rule_idx = drop_mcam_idx;  in rvu_npc_exact_save_drop_rule_chan_and_mask()
1043 	table->drop_rule_map[i].chan_val = (u16)chan_val;  in rvu_npc_exact_save_drop_rule_chan_and_mask()
1044 	table->drop_rule_map[i].chan_mask = (u16)chan_mask;  in rvu_npc_exact_save_drop_rule_chan_and_mask()
1045 	table->drop_rule_map[i].pcifunc = pcifunc;  in rvu_npc_exact_save_drop_rule_chan_and_mask()
1046 	table->drop_rule_map[i].valid = true;  in rvu_npc_exact_save_drop_rule_chan_and_mask()
1093 	struct npc_exact_table *table;  in rvu_npc_exact_drop_rule_to_pcifunc()  local
1096 	table = rvu->hw->table;  in rvu_npc_exact_drop_rule_to_pcifunc()
1099 		if (!table->drop_rule_map[i].valid)  in rvu_npc_exact_drop_rule_to_pcifunc()
1102 		if (table->drop_rule_map[i].drop_rule_idx != drop_rule_idx)  in rvu_npc_exact_drop_rule_to_pcifunc()
1105 		return table->drop_rule_map[i].pcifunc;  in rvu_npc_exact_drop_rule_to_pcifunc()
1129 	struct npc_exact_table *table;  in rvu_npc_exact_get_drop_rule_info()  local
1134 	table = rvu->hw->table;  in rvu_npc_exact_get_drop_rule_info()
1147 		if (!table->drop_rule_map[i].valid)  in rvu_npc_exact_get_drop_rule_info()
1150 		if (table->drop_rule_map[i].chan_val != (u16)chan_val)  in rvu_npc_exact_get_drop_rule_info()
1154 			*val = table->drop_rule_map[i].chan_val;  in rvu_npc_exact_get_drop_rule_info()
1156 			*mask = table->drop_rule_map[i].chan_mask;  in rvu_npc_exact_get_drop_rule_info()
1158 			*pcifunc = table->drop_rule_map[i].pcifunc;  in rvu_npc_exact_get_drop_rule_info()
1190 	struct npc_exact_table *table;  in __rvu_npc_exact_cmd_rules_cnt_update()  local
1194 	table = rvu->hw->table;  in __rvu_npc_exact_cmd_rules_cnt_update()
1195 	promisc = table->promisc_mode[drop_mcam_idx];  in __rvu_npc_exact_cmd_rules_cnt_update()
1197 	cnt = &table->cnt_cmd_rules[drop_mcam_idx];  in __rvu_npc_exact_cmd_rules_cnt_update()
1229  *      rvu_npc_exact_del_table_entry_by_id - Delete and free table entry.
1234  *	table.
1240 	struct npc_exact_table *table;  in rvu_npc_exact_del_table_entry_by_id()  local
1246 	table = rvu->hw->table;  in rvu_npc_exact_del_table_entry_by_id()
1248 	mutex_lock(&table->lock);  in rvu_npc_exact_del_table_entry_by_id()
1254 		mutex_unlock(&table->lock);  in rvu_npc_exact_del_table_entry_by_id()
1258 	cnt = (entry->opc_type == NPC_EXACT_OPC_CAM) ? &table->cam_tbl_entry_cnt :  in rvu_npc_exact_del_table_entry_by_id()
1259 				&table->mem_tbl_entry_cnt;  in rvu_npc_exact_del_table_entry_by_id()
1272 		mutex_unlock(&table->lock);  in rvu_npc_exact_del_table_entry_by_id()
1286 	mutex_unlock(&table->lock);  in rvu_npc_exact_del_table_entry_by_id()
1300  *      rvu_npc_exact_add_table_entry - Adds a table entry
1312  *	Creates a new exact match table entry in either CAM or
1313  *	MEM table.
1334 		dev_err(rvu->dev, "%s: Could not alloc in exact match table\n", __func__);  in rvu_npc_exact_add_table_entry()
1338 	/* Write mdata to table */  in rvu_npc_exact_add_table_entry()
1351 		dev_err(rvu->dev, "%s: could not add to exact match table\n", __func__);  in rvu_npc_exact_add_table_entry()
1382  *      rvu_npc_exact_update_table_entry - Update exact match table.
1390  *	Updates MAC address of an entry. If entry is in MEM table, new
1399 	struct npc_exact_table *table;  in rvu_npc_exact_update_table_entry()  local
1403 	table = rvu->hw->table;  in rvu_npc_exact_update_table_entry()
1405 	mutex_lock(&table->lock);  in rvu_npc_exact_update_table_entry()
1410 		mutex_unlock(&table->lock);  in rvu_npc_exact_update_table_entry()
1417 	/* If entry is in mem table and new hash index is different than old  in rvu_npc_exact_update_table_entry()
1422 						       new_mac, table->mem_table.mask,  in rvu_npc_exact_update_table_entry()
1423 						       table->mem_table.depth);  in rvu_npc_exact_update_table_entry()
1428 			mutex_unlock(&table->lock);  in rvu_npc_exact_update_table_entry()
1451 	mutex_unlock(&table->lock);  in rvu_npc_exact_update_table_entry()
1467 	struct npc_exact_table *table;  in rvu_npc_exact_promisc_disable()  local
1474 	table = rvu->hw->table;  in rvu_npc_exact_promisc_disable()
1485 	mutex_lock(&table->lock);  in rvu_npc_exact_promisc_disable()
1486 	promisc = &table->promisc_mode[drop_mcam_idx];  in rvu_npc_exact_promisc_disable()
1489 		mutex_unlock(&table->lock);  in rvu_npc_exact_promisc_disable()
1495 	mutex_unlock(&table->lock);  in rvu_npc_exact_promisc_disable()
1514 	struct npc_exact_table *table;  in rvu_npc_exact_promisc_enable()  local
1521 	table = rvu->hw->table;  in rvu_npc_exact_promisc_enable()
1532 	mutex_lock(&table->lock);  in rvu_npc_exact_promisc_enable()
1533 	promisc = &table->promisc_mode[drop_mcam_idx];  in rvu_npc_exact_promisc_enable()
1536 		mutex_unlock(&table->lock);  in rvu_npc_exact_promisc_enable()
1542 	mutex_unlock(&table->lock);  in rvu_npc_exact_promisc_enable()
1598 	struct npc_exact_table *table;  in rvu_npc_exact_mac_addr_update()  local
1615 	table = rvu->hw->table;  in rvu_npc_exact_mac_addr_update()
1617 	mutex_lock(&table->lock);  in rvu_npc_exact_mac_addr_update()
1623 		mutex_unlock(&table->lock);  in rvu_npc_exact_mac_addr_update()
1629 	mutex_unlock(&table->lock);  in rvu_npc_exact_mac_addr_update()
1668  *	rvu_npc_exact_mac_addr_add - Adds MAC address to exact match table.
1750 	/* If table does not have an entry; both update entry and del table entry API  in rvu_npc_exact_mac_addr_set()
1803 	struct npc_exact_table *table = rvu->hw->table;  in rvu_npc_exact_can_disable_feature()  local
1809 	mutex_lock(&table->lock);  in rvu_npc_exact_can_disable_feature()
1810 	empty = list_empty(&table->lhead_gbl);  in rvu_npc_exact_can_disable_feature()
1811 	mutex_unlock(&table->lock);  in rvu_npc_exact_can_disable_feature()
1832 	struct npc_exact_table *table = rvu->hw->table;  in rvu_npc_exact_reset()  local
1836 	mutex_lock(&table->lock);  in rvu_npc_exact_reset()
1837 	list_for_each_entry_safe(iter, tmp, &table->lhead_gbl, glist) {  in rvu_npc_exact_reset()
1845 		mutex_unlock(&table->lock);  in rvu_npc_exact_reset()
1847 		mutex_lock(&table->lock);  in rvu_npc_exact_reset()
1849 	mutex_unlock(&table->lock);  in rvu_npc_exact_reset()
1853  *      rvu_npc_exact_init - initialize exact match table
1856  *	Initialize HW and SW resources to manage 4way-2K table and fully
1857  *	associative 32-entry mcam table.
1863 	struct npc_exact_table *table;  in rvu_npc_exact_init()  local
1899 	table = kzalloc(sizeof(*table), GFP_KERNEL);  in rvu_npc_exact_init()
1900 	if (!table)  in rvu_npc_exact_init()
1903 	dev_dbg(rvu->dev, "%s: Memory allocation for table success\n", __func__);  in rvu_npc_exact_init()
1904 	rvu->hw->table = table;  in rvu_npc_exact_init()
1906 	/* Read table size, ways and depth */  in rvu_npc_exact_init()
1907 	table->mem_table.ways = FIELD_GET(GENMASK_ULL(19, 16), npc_const3);  in rvu_npc_exact_init()
1908 	table->mem_table.depth = FIELD_GET(GENMASK_ULL(15, 0), npc_const3);  in rvu_npc_exact_init()
1909 	table->cam_table.depth = FIELD_GET(GENMASK_ULL(31, 24), npc_const3);  in rvu_npc_exact_init()
1911 	dev_dbg(rvu->dev, "%s: NPC exact match 4way_2k table(ways=%d, depth=%d)\n",  in rvu_npc_exact_init()
1912 		__func__,  table->mem_table.ways, table->cam_table.depth);  in rvu_npc_exact_init()
1914 	/* Check if depth of table is not a sequre of 2  in rvu_npc_exact_init()
1917 	if ((table->mem_table.depth & (table->mem_table.depth - 1)) != 0) {  in rvu_npc_exact_init()
1919 			"%s: NPC exact match 4way_2k table depth(%d) is not square of 2\n",  in rvu_npc_exact_init()
1920 			__func__,  table->mem_table.depth);  in rvu_npc_exact_init()
1924 	table_size = table->mem_table.depth * table->mem_table.ways;  in rvu_npc_exact_init()
1926 	/* Allocate bitmap for 4way 2K table */  in rvu_npc_exact_init()
1927 	table->mem_table.bmap = devm_bitmap_zalloc(rvu->dev, table_size,  in rvu_npc_exact_init()
1929 	if (!table->mem_table.bmap)  in rvu_npc_exact_init()
1932 	dev_dbg(rvu->dev, "%s: Allocated bitmap for 4way 2K entry table\n", __func__);  in rvu_npc_exact_init()
1935 	table->cam_table.bmap = devm_bitmap_zalloc(rvu->dev, 32, GFP_KERNEL);  in rvu_npc_exact_init()
1937 	if (!table->cam_table.bmap)  in rvu_npc_exact_init()
1942 	table->tot_ids = table_size + table->cam_table.depth;  in rvu_npc_exact_init()
1943 	table->id_bmap = devm_bitmap_zalloc(rvu->dev, table->tot_ids,  in rvu_npc_exact_init()
1946 	if (!table->id_bmap)  in rvu_npc_exact_init()
1950 		__func__, table->tot_ids);  in rvu_npc_exact_init()
1954 	 * exact match table.  in rvu_npc_exact_init()
1957 		INIT_LIST_HEAD(&table->lhead_mem_tbl_entry[i]);  in rvu_npc_exact_init()
1959 	INIT_LIST_HEAD(&table->lhead_cam_tbl_entry);  in rvu_npc_exact_init()
1960 	INIT_LIST_HEAD(&table->lhead_gbl);  in rvu_npc_exact_init()
1962 	mutex_init(&table->lock);  in rvu_npc_exact_init()
1968 	rvu_exact_config_result_ctrl(rvu, table->mem_table.depth);  in rvu_npc_exact_init()
1983 	drop_mcam_idx = &table->num_drop_rules;  in rvu_npc_exact_init()
2010 		rc = rvu_npc_exact_save_drop_rule_chan_and_mask(rvu, table->num_drop_rules,  in rvu_npc_exact_init()
2020 						 &table->counter_idx[*drop_mcam_idx],  in rvu_npc_exact_init()
2034 	dev_info(rvu->dev, "initialized exact match table successfully\n");  in rvu_npc_exact_init()