49 	u16             table[] __counted_by(table_len);
75 	 * GID entry is detached from the table.
101 	/* In RoCE, adding a GID to the table requires:
165  * @table:	GID table pointer
166  * @index:	Index to check in GID table
170 static bool is_gid_index_default(const struct ib_gid_table *table,  in is_gid_index_default()  argument
173 	return index < 32 && (BIT(index) & table->default_gid_indices);  in is_gid_index_default()
241 	struct ib_gid_table *table = rdma_gid_table(device, port_num);  in free_gid_entry_locked()  local
246 	write_lock_irq(&table->rwlock);  in free_gid_entry_locked()
249 	 * The only way to avoid overwriting NULL in table is  in free_gid_entry_locked()
250 	 * by comparing if it is same entry in table or not!  in free_gid_entry_locked()
251 	 * If new entry in table is added by the time we free here,  in free_gid_entry_locked()
252 	 * don't overwrite the table entry.  in free_gid_entry_locked()
254 	if (entry == table->data_vec[entry->attr.index])  in free_gid_entry_locked()
255 		table->data_vec[entry->attr.index] = NULL;  in free_gid_entry_locked()
257 	write_unlock_irq(&table->rwlock);  in free_gid_entry_locked()
277  * free_gid_work() frees the entry from the HCA's hardware table
286 	struct ib_gid_table *table = rdma_gid_table(device, port_num);  in free_gid_work()  local
288 	mutex_lock(&table->lock);  in free_gid_work()
290 	mutex_unlock(&table->lock);  in free_gid_work()
321 static void store_gid_entry(struct ib_gid_table *table,  in store_gid_entry()  argument
330 	lockdep_assert_held(&table->lock);  in store_gid_entry()
331 	write_lock_irq(&table->rwlock);  in store_gid_entry()
332 	table->data_vec[entry->attr.index] = entry;  in store_gid_entry()
333 	write_unlock_irq(&table->rwlock);  in store_gid_entry()
374  * del_gid - Delete GID table entry
378  * @table:	GID table of the IB device for a port
383 		    struct ib_gid_table *table, int ix)  in del_gid()  argument
388 	lockdep_assert_held(&table->lock);  in del_gid()
391 		ix, table->data_vec[ix]->attr.gid.raw);  in del_gid()
393 	write_lock_irq(&table->rwlock);  in del_gid()
394 	entry = table->data_vec[ix];  in del_gid()
400 		table->data_vec[ix] = NULL;  in del_gid()
401 	write_unlock_irq(&table->rwlock);  in del_gid()
417  * add_modify_gid - Add or modify GID table entry
419  * @table:	GID table in which GID to be added or modified
426 static int add_modify_gid(struct ib_gid_table *table,  in add_modify_gid()  argument
433 	 * Invalidate any old entry in the table to make it safe to write to  in add_modify_gid()
436 	if (is_gid_entry_valid(table->data_vec[attr->index]))  in add_modify_gid()
437 		del_gid(attr->device, attr->port_num, table, attr->index);  in add_modify_gid()
442 	 * empty table entries instead of storing them.  in add_modify_gid()
457 	store_gid_entry(table, entry);  in add_modify_gid()
466 static int find_gid(struct ib_gid_table *table, const union ib_gid *gid,  in find_gid()  argument
474 	while (i < table->sz && (found < 0 || empty < 0)) {  in find_gid()
475 		struct ib_gid_table_entry *data = table->data_vec[i];  in find_gid()
489 				is_gid_index_default(table, curr_index)) {  in find_gid()
528 		    is_gid_index_default(table, curr_index) != default_gid)  in find_gid()
550 	struct ib_gid_table *table;  in __ib_cache_gid_add()  local
562 	table = rdma_gid_table(ib_dev, port);  in __ib_cache_gid_add()
564 	mutex_lock(&table->lock);  in __ib_cache_gid_add()
566 	ix = find_gid(table, gid, attr, default_gid, mask, &empty);  in __ib_cache_gid_add()
578 	ret = add_modify_gid(table, attr);  in __ib_cache_gid_add()
583 	mutex_unlock(&table->lock);  in __ib_cache_gid_add()
605 	struct ib_gid_table *table;  in _ib_cache_gid_del()  local
609 	table = rdma_gid_table(ib_dev, port);  in _ib_cache_gid_del()
611 	mutex_lock(&table->lock);  in _ib_cache_gid_del()
613 	ix = find_gid(table, gid, attr, default_gid, mask, NULL);  in _ib_cache_gid_del()
619 	del_gid(ib_dev, port, table, ix);  in _ib_cache_gid_del()
623 	mutex_unlock(&table->lock);  in _ib_cache_gid_del()
644 	struct ib_gid_table *table;  in ib_cache_gid_del_all_netdev_gids()  local
648 	table = rdma_gid_table(ib_dev, port);  in ib_cache_gid_del_all_netdev_gids()
650 	mutex_lock(&table->lock);  in ib_cache_gid_del_all_netdev_gids()
652 	for (ix = 0; ix < table->sz; ix++) {  in ib_cache_gid_del_all_netdev_gids()
653 		if (is_gid_entry_valid(table->data_vec[ix]) &&  in ib_cache_gid_del_all_netdev_gids()
654 		    table->data_vec[ix]->attr.ndev == ndev) {  in ib_cache_gid_del_all_netdev_gids()
655 			del_gid(ib_dev, port, table, ix);  in ib_cache_gid_del_all_netdev_gids()
660 	mutex_unlock(&table->lock);  in ib_cache_gid_del_all_netdev_gids()
689 	struct ib_gid_table *table;  in rdma_find_gid_by_port()  local
699 	table = rdma_gid_table(ib_dev, port);  in rdma_find_gid_by_port()
704 	read_lock_irqsave(&table->rwlock, flags);  in rdma_find_gid_by_port()
705 	local_index = find_gid(table, gid, &val, false, mask, NULL);  in rdma_find_gid_by_port()
707 		get_gid_entry(table->data_vec[local_index]);  in rdma_find_gid_by_port()
708 		attr = &table->data_vec[local_index]->attr;  in rdma_find_gid_by_port()
709 		read_unlock_irqrestore(&table->rwlock, flags);  in rdma_find_gid_by_port()
713 	read_unlock_irqrestore(&table->rwlock, flags);  in rdma_find_gid_by_port()
719  * rdma_find_gid_by_filter - Returns the GID table attribute where a
725  * @filter: The filter function is executed on any matching GID in the table.
727  *   otherwise, we continue searching the GID table. It's guaranteed that
733  * of which the filter function returns true in the port's GID table.
743 	struct ib_gid_table *table;  in rdma_find_gid_by_filter()  local
750 	table = rdma_gid_table(ib_dev, port);  in rdma_find_gid_by_filter()
752 	read_lock_irqsave(&table->rwlock, flags);  in rdma_find_gid_by_filter()
753 	for (i = 0; i < table->sz; i++) {  in rdma_find_gid_by_filter()
754 		struct ib_gid_table_entry *entry = table->data_vec[i];  in rdma_find_gid_by_filter()
768 	read_unlock_irqrestore(&table->rwlock, flags);  in rdma_find_gid_by_filter()
774 	struct ib_gid_table *table = kzalloc(sizeof(*table), GFP_KERNEL);  in alloc_gid_table()  local
776 	if (!table)  in alloc_gid_table()
779 	table->data_vec = kcalloc(sz, sizeof(*table->data_vec), GFP_KERNEL);  in alloc_gid_table()
780 	if (!table->data_vec)  in alloc_gid_table()
783 	mutex_init(&table->lock);  in alloc_gid_table()
785 	table->sz = sz;  in alloc_gid_table()
786 	rwlock_init(&table->rwlock);  in alloc_gid_table()
787 	return table;  in alloc_gid_table()
790 	kfree(table);  in alloc_gid_table()
795 			      struct ib_gid_table *table)  in release_gid_table()  argument
799 	if (!table)  in release_gid_table()
802 	for (i = 0; i < table->sz; i++) {  in release_gid_table()
803 		if (is_gid_entry_free(table->data_vec[i]))  in release_gid_table()
809 			  kref_read(&table->data_vec[i]->kref));  in release_gid_table()
812 	mutex_destroy(&table->lock);  in release_gid_table()
813 	kfree(table->data_vec);  in release_gid_table()
814 	kfree(table);  in release_gid_table()
818 				   struct ib_gid_table *table)  in cleanup_gid_table_port()  argument
822 	if (!table)  in cleanup_gid_table_port()
825 	mutex_lock(&table->lock);  in cleanup_gid_table_port()
826 	for (i = 0; i < table->sz; ++i) {  in cleanup_gid_table_port()
827 		if (is_gid_entry_valid(table->data_vec[i]))  in cleanup_gid_table_port()
828 			del_gid(ib_dev, port, table, i);  in cleanup_gid_table_port()
830 	mutex_unlock(&table->lock);  in cleanup_gid_table_port()
867 				      struct ib_gid_table *table)  in gid_table_reserve_default()  argument
876 	for (i = 0; i < num_default_gids && i < table->sz; i++)  in gid_table_reserve_default()
877 		table->default_gid_indices |= BIT(i);  in gid_table_reserve_default()
893 	struct ib_gid_table *table;  in _gid_table_setup_one()  local
897 		table = alloc_gid_table(  in _gid_table_setup_one()
899 		if (!table)  in _gid_table_setup_one()
902 		gid_table_reserve_default(ib_dev, rdma_port, table);  in _gid_table_setup_one()
903 		ib_dev->port_data[rdma_port].cache.gid = table;  in _gid_table_setup_one()
939  * @index:		Index of the GID table entry to read
944  * hold any reference to the GID table entry in the HCA or software cache.
952 	struct ib_gid_table *table;  in rdma_query_gid()  local
959 	table = rdma_gid_table(device, port_num);  in rdma_query_gid()
960 	read_lock_irqsave(&table->rwlock, flags);  in rdma_query_gid()
962 	if (index < 0 || index >= table->sz) {  in rdma_query_gid()
967 	if (!is_gid_entry_valid(table->data_vec[index])) {  in rdma_query_gid()
972 	memcpy(gid, &table->data_vec[index]->attr.gid, sizeof(*gid));  in rdma_query_gid()
976 	read_unlock_irqrestore(&table->rwlock, flags);  in rdma_query_gid()
1025 		struct ib_gid_table *table;  in rdma_find_gid()  local
1029 		table = device->port_data[p].cache.gid;  in rdma_find_gid()
1030 		read_lock_irqsave(&table->rwlock, flags);  in rdma_find_gid()
1031 		index = find_gid(table, gid, &gid_attr_val, false, mask, NULL);  in rdma_find_gid()
1035 			get_gid_entry(table->data_vec[index]);  in rdma_find_gid()
1036 			attr = &table->data_vec[index]->attr;  in rdma_find_gid()
1037 			read_unlock_irqrestore(&table->rwlock, flags);  in rdma_find_gid()
1040 		read_unlock_irqrestore(&table->rwlock, flags);  in rdma_find_gid()
1066 		*pkey = cache->table[index];  in ib_get_cached_pkey()
1108 		if ((cache->table[i] & 0x7fff) == (pkey & 0x7fff)) {  in ib_find_cached_pkey()
1109 			if (cache->table[i] & 0x8000) {  in ib_find_cached_pkey()
1152 		if (cache->table[i] == pkey) {  in ib_find_exact_cached_pkey()
1204  * @index:		Index of the GID table entry whose attributes are to
1208  * cached GID table. Caller must invoke rdma_put_gid_attr() to release
1218 	struct ib_gid_table *table;  in rdma_get_gid_attr()  local
1224 	table = rdma_gid_table(device, port_num);  in rdma_get_gid_attr()
1225 	if (index < 0 || index >= table->sz)  in rdma_get_gid_attr()
1228 	read_lock_irqsave(&table->rwlock, flags);  in rdma_get_gid_attr()
1229 	if (!is_gid_entry_valid(table->data_vec[index]))  in rdma_get_gid_attr()
1232 	get_gid_entry(table->data_vec[index]);  in rdma_get_gid_attr()
1233 	attr = &table->data_vec[index]->attr;  in rdma_get_gid_attr()
1235 	read_unlock_irqrestore(&table->rwlock, flags);  in rdma_get_gid_attr()
1241  * rdma_query_gid_table - Reads GID table entries of all the ports of a device up to max_entries.
1255 	struct ib_gid_table *table;  in rdma_query_gid_table()  local
1261 		table = rdma_gid_table(device, port_num);  in rdma_query_gid_table()
1262 		read_lock_irqsave(&table->rwlock, flags);  in rdma_query_gid_table()
1263 		for (i = 0; i < table->sz; i++) {  in rdma_query_gid_table()
1264 			if (!is_gid_entry_valid(table->data_vec[i]))  in rdma_query_gid_table()
1271 			gid_attr = &table->data_vec[i]->attr;  in rdma_query_gid_table()
1280 				lockdep_is_held(&table->rwlock));  in rdma_query_gid_table()
1287 		read_unlock_irqrestore(&table->rwlock, flags);  in rdma_query_gid_table()
1292 	read_unlock_irqrestore(&table->rwlock, flags);  in rdma_query_gid_table()
1355 	struct ib_gid_table *table;  in rdma_read_gid_attr_ndev_rcu()  local
1359 	table = rdma_gid_table(device, port_num);  in rdma_read_gid_attr_ndev_rcu()
1361 	read_lock_irqsave(&table->rwlock, flags);  in rdma_read_gid_attr_ndev_rcu()
1362 	valid = is_gid_entry_valid(table->data_vec[attr->index]);  in rdma_read_gid_attr_ndev_rcu()
1368 	read_unlock_irqrestore(&table->rwlock, flags);  in rdma_read_gid_attr_ndev_rcu()
1437 	struct ib_gid_table *table;  in config_non_roce_gid_cache()  local
1443 	table = rdma_gid_table(device, port);  in config_non_roce_gid_cache()
1445 	mutex_lock(&table->lock);  in config_non_roce_gid_cache()
1470 		add_modify_gid(table, &gid_attr);  in config_non_roce_gid_cache()
1473 	mutex_unlock(&table->lock);  in config_non_roce_gid_cache()
1510 		pkey_cache = kmalloc(struct_size(pkey_cache, table,  in ib_cache_update()
1522 					    pkey_cache->table + i);  in ib_cache_update()