100 static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter);
101 static inline void reinit_init_done(struct ibmvnic_adapter *adapter);
102 static void send_query_map(struct ibmvnic_adapter *adapter);
105 static int send_login(struct ibmvnic_adapter *adapter);
106 static void send_query_cap(struct ibmvnic_adapter *adapter);
108 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter);
112 static int init_crq_queue(struct ibmvnic_adapter *adapter);
113 static int send_query_phys_parms(struct ibmvnic_adapter *adapter);
114 static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter,
116 static void free_long_term_buff(struct ibmvnic_adapter *adapter,
118 static void ibmvnic_disable_irqs(struct ibmvnic_adapter *adapter);
119 static void flush_reset_queue(struct ibmvnic_adapter *adapter);
156 static int send_crq_init_complete(struct ibmvnic_adapter *adapter)  in send_crq_init_complete()  argument
164 	return ibmvnic_send_crq(adapter, &crq);  in send_crq_init_complete()
167 static int send_version_xchg(struct ibmvnic_adapter *adapter)  in send_version_xchg()  argument
176 	return ibmvnic_send_crq(adapter, &crq);  in send_version_xchg()
179 static void ibmvnic_clean_queue_affinity(struct ibmvnic_adapter *adapter,  in ibmvnic_clean_queue_affinity()  argument
188 		netdev_warn(adapter->netdev,  in ibmvnic_clean_queue_affinity()
193 static void ibmvnic_clean_affinity(struct ibmvnic_adapter *adapter)  in ibmvnic_clean_affinity()  argument
200 	rxqs = adapter->rx_scrq;  in ibmvnic_clean_affinity()
201 	txqs = adapter->tx_scrq;  in ibmvnic_clean_affinity()
202 	num_txqs = adapter->num_active_tx_scrqs;  in ibmvnic_clean_affinity()
203 	num_rxqs = adapter->num_active_rx_scrqs;  in ibmvnic_clean_affinity()
205 	netdev_dbg(adapter->netdev, "%s: Cleaning irq affinity hints", __func__);  in ibmvnic_clean_affinity()
208 			ibmvnic_clean_queue_affinity(adapter, txqs[i]);  in ibmvnic_clean_affinity()
212 			ibmvnic_clean_queue_affinity(adapter, rxqs[i]);  in ibmvnic_clean_affinity()
251 static void ibmvnic_set_affinity(struct ibmvnic_adapter *adapter)  in ibmvnic_set_affinity()  argument
253 	struct ibmvnic_sub_crq_queue **rxqs = adapter->rx_scrq;  in ibmvnic_set_affinity()
254 	struct ibmvnic_sub_crq_queue **txqs = adapter->tx_scrq;  in ibmvnic_set_affinity()
256 	int num_rxqs = adapter->num_active_rx_scrqs, i_rxqs = 0;  in ibmvnic_set_affinity()
257 	int num_txqs = adapter->num_active_tx_scrqs, i_txqs = 0;  in ibmvnic_set_affinity()
263 	netdev_dbg(adapter->netdev, "%s: Setting irq affinity hints", __func__);  in ibmvnic_set_affinity()
264 	if (!(adapter->rx_scrq && adapter->tx_scrq)) {  in ibmvnic_set_affinity()
265 		netdev_warn(adapter->netdev,  in ibmvnic_set_affinity()
300 		rc = __netif_set_xps_queue(adapter->netdev,  in ibmvnic_set_affinity()
304 			netdev_warn(adapter->netdev, "%s: Set XPS on queue %d failed, rc = %d.\n",  in ibmvnic_set_affinity()
310 		netdev_warn(adapter->netdev,  in ibmvnic_set_affinity()
313 		ibmvnic_clean_affinity(adapter);  in ibmvnic_set_affinity()
319 	struct ibmvnic_adapter *adapter;  in ibmvnic_cpu_online()  local
321 	adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node);  in ibmvnic_cpu_online()
322 	ibmvnic_set_affinity(adapter);  in ibmvnic_cpu_online()
328 	struct ibmvnic_adapter *adapter;  in ibmvnic_cpu_dead()  local
330 	adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node_dead);  in ibmvnic_cpu_dead()
331 	ibmvnic_set_affinity(adapter);  in ibmvnic_cpu_dead()
337 	struct ibmvnic_adapter *adapter;  in ibmvnic_cpu_down_prep()  local
339 	adapter = hlist_entry_safe(node, struct ibmvnic_adapter, node);  in ibmvnic_cpu_down_prep()
340 	ibmvnic_clean_affinity(adapter);  in ibmvnic_cpu_down_prep()
346 static int ibmvnic_cpu_notif_add(struct ibmvnic_adapter *adapter)  in ibmvnic_cpu_notif_add()  argument
350 	ret = cpuhp_state_add_instance_nocalls(ibmvnic_online, &adapter->node);  in ibmvnic_cpu_notif_add()
354 					       &adapter->node_dead);  in ibmvnic_cpu_notif_add()
357 	cpuhp_state_remove_instance_nocalls(ibmvnic_online, &adapter->node);  in ibmvnic_cpu_notif_add()
361 static void ibmvnic_cpu_notif_remove(struct ibmvnic_adapter *adapter)  in ibmvnic_cpu_notif_remove()  argument
363 	cpuhp_state_remove_instance_nocalls(ibmvnic_online, &adapter->node);  in ibmvnic_cpu_notif_remove()
365 					    &adapter->node_dead);  in ibmvnic_cpu_notif_remove()
391 static int ibmvnic_wait_for_completion(struct ibmvnic_adapter *adapter,  in ibmvnic_wait_for_completion()  argument
399 	netdev = adapter->netdev;  in ibmvnic_wait_for_completion()
403 		if (!adapter->crq.active) {  in ibmvnic_wait_for_completion()
451 static int alloc_long_term_buff(struct ibmvnic_adapter *adapter,  in alloc_long_term_buff()  argument
454 	struct device *dev = &adapter->vdev->dev;  in alloc_long_term_buff()
463 		free_long_term_buff(adapter, ltb);  in alloc_long_term_buff()
478 		ltb->map_id = find_first_zero_bit(adapter->map_ids,  in alloc_long_term_buff()
480 		bitmap_set(adapter->map_ids, ltb->map_id, 1);  in alloc_long_term_buff()
490 	mutex_lock(&adapter->fw_lock);  in alloc_long_term_buff()
491 	adapter->fw_done_rc = 0;  in alloc_long_term_buff()
492 	reinit_completion(&adapter->fw_done);  in alloc_long_term_buff()
494 	rc = send_request_map(adapter, ltb->addr, ltb->size, ltb->map_id);  in alloc_long_term_buff()
500 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in alloc_long_term_buff()
507 	if (adapter->fw_done_rc) {  in alloc_long_term_buff()
509 			adapter->fw_done_rc);  in alloc_long_term_buff()
516 	mutex_unlock(&adapter->fw_lock);  in alloc_long_term_buff()
520 static void free_long_term_buff(struct ibmvnic_adapter *adapter,  in free_long_term_buff()  argument
523 	struct device *dev = &adapter->vdev->dev;  in free_long_term_buff()
532 	if (adapter->reset_reason != VNIC_RESET_FAILOVER &&  in free_long_term_buff()
533 	    adapter->reset_reason != VNIC_RESET_MOBILITY &&  in free_long_term_buff()
534 	    adapter->reset_reason != VNIC_RESET_TIMEOUT)  in free_long_term_buff()
535 		send_request_unmap(adapter, ltb->map_id);  in free_long_term_buff()
541 	bitmap_clear(adapter->map_ids, ltb->map_id, 1);  in free_long_term_buff()
553 static void free_ltb_set(struct ibmvnic_adapter *adapter,  in free_ltb_set()  argument
559 		free_long_term_buff(adapter, &ltb_set->ltbs[i]);  in free_ltb_set()
583 static int alloc_ltb_set(struct ibmvnic_adapter *adapter,  in alloc_ltb_set()  argument
587 	struct device *dev = &adapter->vdev->dev;  in alloc_ltb_set()
626 			free_long_term_buff(adapter, &old_set.ltbs[i]);  in alloc_ltb_set()
663 		rc = alloc_long_term_buff(adapter, &new_set.ltbs[i], ltb_size);  in alloc_ltb_set()
747 static void deactivate_rx_pools(struct ibmvnic_adapter *adapter)  in deactivate_rx_pools()  argument
751 	for (i = 0; i < adapter->num_active_rx_pools; i++)  in deactivate_rx_pools()
752 		adapter->rx_pool[i].active = 0;  in deactivate_rx_pools()
755 static void replenish_rx_pool(struct ibmvnic_adapter *adapter,  in replenish_rx_pool()  argument
759 	u64 handle = adapter->rx_scrq[pool->index]->handle;  in replenish_rx_pool()
760 	struct device *dev = &adapter->vdev->dev;  in replenish_rx_pool()
778 	rx_scrq = adapter->rx_scrq[pool->index];  in replenish_rx_pool()
797 			skb = netdev_alloc_skb(adapter->netdev,  in replenish_rx_pool()
801 				adapter->replenish_no_mem++;  in replenish_rx_pool()
845 				send_subcrq_indirect(adapter, handle,  in replenish_rx_pool()
851 			adapter->replenish_add_buff_success += ind_bufp->index;  in replenish_rx_pool()
874 	adapter->replenish_add_buff_failure += ind_bufp->index;  in replenish_rx_pool()
877 	if (lpar_rc == H_CLOSED || adapter->failover_pending) {  in replenish_rx_pool()
883 		deactivate_rx_pools(adapter);  in replenish_rx_pool()
884 		netif_carrier_off(adapter->netdev);  in replenish_rx_pool()
888 static void replenish_pools(struct ibmvnic_adapter *adapter)  in replenish_pools()  argument
892 	adapter->replenish_task_cycles++;  in replenish_pools()
893 	for (i = 0; i < adapter->num_active_rx_pools; i++) {  in replenish_pools()
894 		if (adapter->rx_pool[i].active)  in replenish_pools()
895 			replenish_rx_pool(adapter, &adapter->rx_pool[i]);  in replenish_pools()
898 	netdev_dbg(adapter->netdev, "Replenished %d pools\n", i);  in replenish_pools()
901 static void release_stats_buffers(struct ibmvnic_adapter *adapter)  in release_stats_buffers()  argument
903 	kfree(adapter->tx_stats_buffers);  in release_stats_buffers()
904 	kfree(adapter->rx_stats_buffers);  in release_stats_buffers()
905 	adapter->tx_stats_buffers = NULL;  in release_stats_buffers()
906 	adapter->rx_stats_buffers = NULL;  in release_stats_buffers()
909 static int init_stats_buffers(struct ibmvnic_adapter *adapter)  in init_stats_buffers()  argument
911 	adapter->tx_stats_buffers =  in init_stats_buffers()
915 	if (!adapter->tx_stats_buffers)  in init_stats_buffers()
918 	adapter->rx_stats_buffers =  in init_stats_buffers()
922 	if (!adapter->rx_stats_buffers)  in init_stats_buffers()
928 static void release_stats_token(struct ibmvnic_adapter *adapter)  in release_stats_token()  argument
930 	struct device *dev = &adapter->vdev->dev;  in release_stats_token()
932 	if (!adapter->stats_token)  in release_stats_token()
935 	dma_unmap_single(dev, adapter->stats_token,  in release_stats_token()
938 	adapter->stats_token = 0;  in release_stats_token()
941 static int init_stats_token(struct ibmvnic_adapter *adapter)  in init_stats_token()  argument
943 	struct device *dev = &adapter->vdev->dev;  in init_stats_token()
947 	stok = dma_map_single(dev, &adapter->stats,  in init_stats_token()
956 	adapter->stats_token = stok;  in init_stats_token()
957 	netdev_dbg(adapter->netdev, "Stats token initialized (%llx)\n", stok);  in init_stats_token()
967 static void release_rx_pools(struct ibmvnic_adapter *adapter)  in release_rx_pools()  argument
972 	if (!adapter->rx_pool)  in release_rx_pools()
975 	for (i = 0; i < adapter->num_active_rx_pools; i++) {  in release_rx_pools()
976 		rx_pool = &adapter->rx_pool[i];  in release_rx_pools()
978 		netdev_dbg(adapter->netdev, "Releasing rx_pool[%d]\n", i);  in release_rx_pools()
982 		free_ltb_set(adapter, &rx_pool->ltb_set);  in release_rx_pools()
997 	kfree(adapter->rx_pool);  in release_rx_pools()
998 	adapter->rx_pool = NULL;  in release_rx_pools()
999 	adapter->num_active_rx_pools = 0;  in release_rx_pools()
1000 	adapter->prev_rx_pool_size = 0;  in release_rx_pools()
1017 static bool reuse_rx_pools(struct ibmvnic_adapter *adapter)  in reuse_rx_pools()  argument
1023 	if (!adapter->rx_pool)  in reuse_rx_pools()
1026 	old_num_pools = adapter->num_active_rx_pools;  in reuse_rx_pools()
1027 	new_num_pools = adapter->req_rx_queues;  in reuse_rx_pools()
1029 	old_pool_size = adapter->prev_rx_pool_size;  in reuse_rx_pools()
1030 	new_pool_size = adapter->req_rx_add_entries_per_subcrq;  in reuse_rx_pools()
1032 	old_buff_size = adapter->prev_rx_buf_sz;  in reuse_rx_pools()
1033 	new_buff_size = adapter->cur_rx_buf_sz;  in reuse_rx_pools()
1056 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in init_rx_pools()  local
1057 	struct device *dev = &adapter->vdev->dev;  in init_rx_pools()
1064 	pool_size = adapter->req_rx_add_entries_per_subcrq;  in init_rx_pools()
1065 	num_pools = adapter->req_rx_queues;  in init_rx_pools()
1066 	buff_size = adapter->cur_rx_buf_sz;  in init_rx_pools()
1068 	if (reuse_rx_pools(adapter)) {  in init_rx_pools()
1074 	release_rx_pools(adapter);  in init_rx_pools()
1076 	adapter->rx_pool = kcalloc(num_pools,  in init_rx_pools()
1079 	if (!adapter->rx_pool) {  in init_rx_pools()
1087 	adapter->num_active_rx_pools = num_pools;  in init_rx_pools()
1090 		rx_pool = &adapter->rx_pool[i];  in init_rx_pools()
1092 		netdev_dbg(adapter->netdev,  in init_rx_pools()
1118 	adapter->prev_rx_pool_size = pool_size;  in init_rx_pools()
1119 	adapter->prev_rx_buf_sz = adapter->cur_rx_buf_sz;  in init_rx_pools()
1123 		rx_pool = &adapter->rx_pool[i];  in init_rx_pools()
1127 		rc = alloc_ltb_set(adapter, &rx_pool->ltb_set,  in init_rx_pools()
1161 	release_rx_pools(adapter);  in init_rx_pools()
1169 static void release_vpd_data(struct ibmvnic_adapter *adapter)  in release_vpd_data()  argument
1171 	if (!adapter->vpd)  in release_vpd_data()
1174 	kfree(adapter->vpd->buff);  in release_vpd_data()
1175 	kfree(adapter->vpd);  in release_vpd_data()
1177 	adapter->vpd = NULL;  in release_vpd_data()
1180 static void release_one_tx_pool(struct ibmvnic_adapter *adapter,  in release_one_tx_pool()  argument
1185 	free_ltb_set(adapter, &tx_pool->ltb_set);  in release_one_tx_pool()
1194 static void release_tx_pools(struct ibmvnic_adapter *adapter)  in release_tx_pools()  argument
1201 	if (!adapter->tx_pool)  in release_tx_pools()
1204 	for (i = 0; i < adapter->num_active_tx_pools; i++) {  in release_tx_pools()
1205 		release_one_tx_pool(adapter, &adapter->tx_pool[i]);  in release_tx_pools()
1206 		release_one_tx_pool(adapter, &adapter->tso_pool[i]);  in release_tx_pools()
1209 	kfree(adapter->tx_pool);  in release_tx_pools()
1210 	adapter->tx_pool = NULL;  in release_tx_pools()
1211 	kfree(adapter->tso_pool);  in release_tx_pools()
1212 	adapter->tso_pool = NULL;  in release_tx_pools()
1213 	adapter->num_active_tx_pools = 0;  in release_tx_pools()
1214 	adapter->prev_tx_pool_size = 0;  in release_tx_pools()
1260 static bool reuse_tx_pools(struct ibmvnic_adapter *adapter)  in reuse_tx_pools()  argument
1266 	if (!adapter->tx_pool)  in reuse_tx_pools()
1269 	old_num_pools = adapter->num_active_tx_pools;  in reuse_tx_pools()
1270 	new_num_pools = adapter->num_active_tx_scrqs;  in reuse_tx_pools()
1271 	old_pool_size = adapter->prev_tx_pool_size;  in reuse_tx_pools()
1272 	new_pool_size = adapter->req_tx_entries_per_subcrq;  in reuse_tx_pools()
1273 	old_mtu = adapter->prev_mtu;  in reuse_tx_pools()
1274 	new_mtu = adapter->req_mtu;  in reuse_tx_pools()
1297 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in init_tx_pools()  local
1298 	struct device *dev = &adapter->vdev->dev;  in init_tx_pools()
1304 	num_pools = adapter->req_tx_queues;  in init_tx_pools()
1310 	if (reuse_tx_pools(adapter)) {  in init_tx_pools()
1316 	release_tx_pools(adapter);  in init_tx_pools()
1318 	pool_size = adapter->req_tx_entries_per_subcrq;  in init_tx_pools()
1319 	num_pools = adapter->num_active_tx_scrqs;  in init_tx_pools()
1321 	adapter->tx_pool = kcalloc(num_pools,  in init_tx_pools()
1323 	if (!adapter->tx_pool)  in init_tx_pools()
1326 	adapter->tso_pool = kcalloc(num_pools,  in init_tx_pools()
1331 	if (!adapter->tso_pool) {  in init_tx_pools()
1332 		kfree(adapter->tx_pool);  in init_tx_pools()
1333 		adapter->tx_pool = NULL;  in init_tx_pools()
1340 	adapter->num_active_tx_pools = num_pools;  in init_tx_pools()
1342 	buff_size = adapter->req_mtu + VLAN_HLEN;  in init_tx_pools()
1347 			i, adapter->req_tx_entries_per_subcrq, buff_size);  in init_tx_pools()
1349 		rc = init_one_tx_pool(netdev, &adapter->tx_pool[i],  in init_tx_pools()
1354 		rc = init_one_tx_pool(netdev, &adapter->tso_pool[i],  in init_tx_pools()
1361 	adapter->prev_tx_pool_size = pool_size;  in init_tx_pools()
1362 	adapter->prev_mtu = adapter->req_mtu;  in init_tx_pools()
1376 		tx_pool = &adapter->tx_pool[i];  in init_tx_pools()
1381 		rc = alloc_ltb_set(adapter, &tx_pool->ltb_set,  in init_tx_pools()
1392 		tso_pool = &adapter->tso_pool[i];  in init_tx_pools()
1397 		rc = alloc_ltb_set(adapter, &tso_pool->ltb_set,  in init_tx_pools()
1411 	release_tx_pools(adapter);  in init_tx_pools()
1419 static void ibmvnic_napi_enable(struct ibmvnic_adapter *adapter)  in ibmvnic_napi_enable()  argument
1423 	if (adapter->napi_enabled)  in ibmvnic_napi_enable()
1426 	for (i = 0; i < adapter->req_rx_queues; i++)  in ibmvnic_napi_enable()
1427 		napi_enable(&adapter->napi[i]);  in ibmvnic_napi_enable()
1429 	adapter->napi_enabled = true;  in ibmvnic_napi_enable()
1432 static void ibmvnic_napi_disable(struct ibmvnic_adapter *adapter)  in ibmvnic_napi_disable()  argument
1436 	if (!adapter->napi_enabled)  in ibmvnic_napi_disable()
1439 	for (i = 0; i < adapter->req_rx_queues; i++) {  in ibmvnic_napi_disable()
1440 		netdev_dbg(adapter->netdev, "Disabling napi[%d]\n", i);  in ibmvnic_napi_disable()
1441 		napi_disable(&adapter->napi[i]);  in ibmvnic_napi_disable()
1444 	adapter->napi_enabled = false;  in ibmvnic_napi_disable()
1447 static int init_napi(struct ibmvnic_adapter *adapter)  in init_napi()  argument
1451 	adapter->napi = kcalloc(adapter->req_rx_queues,  in init_napi()
1453 	if (!adapter->napi)  in init_napi()
1456 	for (i = 0; i < adapter->req_rx_queues; i++) {  in init_napi()
1457 		netdev_dbg(adapter->netdev, "Adding napi[%d]\n", i);  in init_napi()
1458 		netif_napi_add(adapter->netdev, &adapter->napi[i],  in init_napi()
1462 	adapter->num_active_rx_napi = adapter->req_rx_queues;  in init_napi()
1466 static void release_napi(struct ibmvnic_adapter *adapter)  in release_napi()  argument
1470 	if (!adapter->napi)  in release_napi()
1473 	for (i = 0; i < adapter->num_active_rx_napi; i++) {  in release_napi()
1474 		netdev_dbg(adapter->netdev, "Releasing napi[%d]\n", i);  in release_napi()
1475 		netif_napi_del(&adapter->napi[i]);  in release_napi()
1478 	kfree(adapter->napi);  in release_napi()
1479 	adapter->napi = NULL;  in release_napi()
1480 	adapter->num_active_rx_napi = 0;  in release_napi()
1481 	adapter->napi_enabled = false;  in release_napi()
1512 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_login()  local
1525 		adapter->init_done_rc = 0;  in ibmvnic_login()
1526 		reinit_completion(&adapter->init_done);  in ibmvnic_login()
1527 		rc = send_login(adapter);  in ibmvnic_login()
1531 		if (!wait_for_completion_timeout(&adapter->init_done,  in ibmvnic_login()
1534 			adapter->login_pending = false;  in ibmvnic_login()
1538 		if (adapter->init_done_rc == ABORTED) {  in ibmvnic_login()
1541 			adapter->init_done_rc = 0;  in ibmvnic_login()
1547 		} else if (adapter->init_done_rc == PARTIALSUCCESS) {  in ibmvnic_login()
1549 			release_sub_crqs(adapter, 1);  in ibmvnic_login()
1554 			adapter->init_done_rc = 0;  in ibmvnic_login()
1555 			reinit_completion(&adapter->init_done);  in ibmvnic_login()
1556 			send_query_cap(adapter);  in ibmvnic_login()
1557 			if (!wait_for_completion_timeout(&adapter->init_done,  in ibmvnic_login()
1564 			rc = init_sub_crqs(adapter);  in ibmvnic_login()
1571 			rc = init_sub_crq_irqs(adapter);  in ibmvnic_login()
1578 		} else if (adapter->init_done_rc) {  in ibmvnic_login()
1580 				    adapter->init_done_rc);  in ibmvnic_login()
1592 			adapter->init_done_rc = 0;  in ibmvnic_login()
1593 			release_sub_crqs(adapter, true);  in ibmvnic_login()
1601 				reinit_init_done(adapter);  in ibmvnic_login()
1605 				adapter->failover_pending = false;  in ibmvnic_login()
1606 				release_crq_queue(adapter);  in ibmvnic_login()
1617 				spin_lock_irqsave(&adapter->rwi_lock, flags);  in ibmvnic_login()
1618 				flush_reset_queue(adapter);  in ibmvnic_login()
1619 				spin_unlock_irqrestore(&adapter->rwi_lock,  in ibmvnic_login()
1622 				rc = init_crq_queue(adapter);  in ibmvnic_login()
1629 				rc = ibmvnic_reset_init(adapter, false);  in ibmvnic_login()
1643 	__ibmvnic_set_mac(netdev, adapter->mac_addr);  in ibmvnic_login()
1645 	netdev_dbg(netdev, "[S:%s] Login succeeded\n", adapter_state_to_string(adapter->state));  in ibmvnic_login()
1649 static void release_login_buffer(struct ibmvnic_adapter *adapter)  in release_login_buffer()  argument
1651 	if (!adapter->login_buf)  in release_login_buffer()
1654 	dma_unmap_single(&adapter->vdev->dev, adapter->login_buf_token,  in release_login_buffer()
1655 			 adapter->login_buf_sz, DMA_TO_DEVICE);  in release_login_buffer()
1656 	kfree(adapter->login_buf);  in release_login_buffer()
1657 	adapter->login_buf = NULL;  in release_login_buffer()
1660 static void release_login_rsp_buffer(struct ibmvnic_adapter *adapter)  in release_login_rsp_buffer()  argument
1662 	if (!adapter->login_rsp_buf)  in release_login_rsp_buffer()
1665 	dma_unmap_single(&adapter->vdev->dev, adapter->login_rsp_buf_token,  in release_login_rsp_buffer()
1666 			 adapter->login_rsp_buf_sz, DMA_FROM_DEVICE);  in release_login_rsp_buffer()
1667 	kfree(adapter->login_rsp_buf);  in release_login_rsp_buffer()
1668 	adapter->login_rsp_buf = NULL;  in release_login_rsp_buffer()
1671 static void release_resources(struct ibmvnic_adapter *adapter)  in release_resources()  argument
1673 	release_vpd_data(adapter);  in release_resources()
1675 	release_napi(adapter);  in release_resources()
1676 	release_login_buffer(adapter);  in release_resources()
1677 	release_login_rsp_buffer(adapter);  in release_resources()
1680 static int set_link_state(struct ibmvnic_adapter *adapter, u8 link_state)  in set_link_state()  argument
1682 	struct net_device *netdev = adapter->netdev;  in set_link_state()
1698 		reinit_completion(&adapter->init_done);  in set_link_state()
1699 		rc = ibmvnic_send_crq(adapter, &crq);  in set_link_state()
1705 		if (!wait_for_completion_timeout(&adapter->init_done,  in set_link_state()
1711 		if (adapter->init_done_rc == PARTIALSUCCESS) {  in set_link_state()
1715 		} else if (adapter->init_done_rc) {  in set_link_state()
1717 				    adapter->init_done_rc);  in set_link_state()
1718 			return adapter->init_done_rc;  in set_link_state()
1727 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in set_real_num_queues()  local
1731 		   adapter->req_tx_queues, adapter->req_rx_queues);  in set_real_num_queues()
1733 	rc = netif_set_real_num_tx_queues(netdev, adapter->req_tx_queues);  in set_real_num_queues()
1739 	rc = netif_set_real_num_rx_queues(netdev, adapter->req_rx_queues);  in set_real_num_queues()
1746 static int ibmvnic_get_vpd(struct ibmvnic_adapter *adapter)  in ibmvnic_get_vpd()  argument
1748 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_get_vpd()
1753 	if (adapter->vpd->buff)  in ibmvnic_get_vpd()
1754 		len = adapter->vpd->len;  in ibmvnic_get_vpd()
1756 	mutex_lock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1757 	adapter->fw_done_rc = 0;  in ibmvnic_get_vpd()
1758 	reinit_completion(&adapter->fw_done);  in ibmvnic_get_vpd()
1762 	rc = ibmvnic_send_crq(adapter, &crq);  in ibmvnic_get_vpd()
1764 		mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1768 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in ibmvnic_get_vpd()
1771 		mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1774 	mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1776 	if (!adapter->vpd->len)  in ibmvnic_get_vpd()
1779 	if (!adapter->vpd->buff)  in ibmvnic_get_vpd()
1780 		adapter->vpd->buff = kzalloc(adapter->vpd->len, GFP_KERNEL);  in ibmvnic_get_vpd()
1781 	else if (adapter->vpd->len != len)  in ibmvnic_get_vpd()
1782 		adapter->vpd->buff =  in ibmvnic_get_vpd()
1783 			krealloc(adapter->vpd->buff,  in ibmvnic_get_vpd()
1784 				 adapter->vpd->len, GFP_KERNEL);  in ibmvnic_get_vpd()
1786 	if (!adapter->vpd->buff) {  in ibmvnic_get_vpd()
1791 	adapter->vpd->dma_addr =  in ibmvnic_get_vpd()
1792 		dma_map_single(dev, adapter->vpd->buff, adapter->vpd->len,  in ibmvnic_get_vpd()
1794 	if (dma_mapping_error(dev, adapter->vpd->dma_addr)) {  in ibmvnic_get_vpd()
1796 		kfree(adapter->vpd->buff);  in ibmvnic_get_vpd()
1797 		adapter->vpd->buff = NULL;  in ibmvnic_get_vpd()
1801 	mutex_lock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1802 	adapter->fw_done_rc = 0;  in ibmvnic_get_vpd()
1803 	reinit_completion(&adapter->fw_done);  in ibmvnic_get_vpd()
1807 	crq.get_vpd.ioba = cpu_to_be32(adapter->vpd->dma_addr);  in ibmvnic_get_vpd()
1808 	crq.get_vpd.len = cpu_to_be32((u32)adapter->vpd->len);  in ibmvnic_get_vpd()
1809 	rc = ibmvnic_send_crq(adapter, &crq);  in ibmvnic_get_vpd()
1811 		kfree(adapter->vpd->buff);  in ibmvnic_get_vpd()
1812 		adapter->vpd->buff = NULL;  in ibmvnic_get_vpd()
1813 		mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1817 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in ibmvnic_get_vpd()
1820 		kfree(adapter->vpd->buff);  in ibmvnic_get_vpd()
1821 		adapter->vpd->buff = NULL;  in ibmvnic_get_vpd()
1822 		mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1826 	mutex_unlock(&adapter->fw_lock);  in ibmvnic_get_vpd()
1830 static int init_resources(struct ibmvnic_adapter *adapter)  in init_resources()  argument
1832 	struct net_device *netdev = adapter->netdev;  in init_resources()
1839 	adapter->vpd = kzalloc(sizeof(*adapter->vpd), GFP_KERNEL);  in init_resources()
1840 	if (!adapter->vpd)  in init_resources()
1844 	rc = ibmvnic_get_vpd(adapter);  in init_resources()
1850 	rc = init_napi(adapter);  in init_resources()
1854 	send_query_map(adapter);  in init_resources()
1866 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in __ibmvnic_open()  local
1867 	enum vnic_state prev_state = adapter->state;  in __ibmvnic_open()
1870 	adapter->state = VNIC_OPENING;  in __ibmvnic_open()
1871 	replenish_pools(adapter);  in __ibmvnic_open()
1872 	ibmvnic_napi_enable(adapter);  in __ibmvnic_open()
1877 	for (i = 0; i < adapter->req_rx_queues; i++) {  in __ibmvnic_open()
1880 			enable_irq(adapter->rx_scrq[i]->irq);  in __ibmvnic_open()
1881 		enable_scrq_irq(adapter, adapter->rx_scrq[i]);  in __ibmvnic_open()
1884 	for (i = 0; i < adapter->req_tx_queues; i++) {  in __ibmvnic_open()
1887 			enable_irq(adapter->tx_scrq[i]->irq);  in __ibmvnic_open()
1888 		enable_scrq_irq(adapter, adapter->tx_scrq[i]);  in __ibmvnic_open()
1895 		if (adapter->reset_reason != VNIC_RESET_NON_FATAL)  in __ibmvnic_open()
1899 	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_UP);  in __ibmvnic_open()
1901 		ibmvnic_napi_disable(adapter);  in __ibmvnic_open()
1902 		ibmvnic_disable_irqs(adapter);  in __ibmvnic_open()
1906 	adapter->tx_queues_active = true;  in __ibmvnic_open()
1918 		for (i = 0; i < adapter->req_rx_queues; i++)  in __ibmvnic_open()
1919 			napi_schedule(&adapter->napi[i]);  in __ibmvnic_open()
1922 	adapter->state = VNIC_OPEN;  in __ibmvnic_open()
1928 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_open()  local
1944 	if (adapter->failover_pending || (test_bit(0, &adapter->resetting))) {  in ibmvnic_open()
1946 			   adapter_state_to_string(adapter->state),  in ibmvnic_open()
1947 			   adapter->failover_pending);  in ibmvnic_open()
1948 		adapter->state = VNIC_OPEN;  in ibmvnic_open()
1953 	if (adapter->state != VNIC_CLOSED) {  in ibmvnic_open()
1958 		rc = init_resources(adapter);  in ibmvnic_open()
1973 	    (adapter->failover_pending || (test_bit(0, &adapter->resetting)))) {  in ibmvnic_open()
1974 		adapter->state = VNIC_OPEN;  in ibmvnic_open()
1979 		release_resources(adapter);  in ibmvnic_open()
1980 		release_rx_pools(adapter);  in ibmvnic_open()
1981 		release_tx_pools(adapter);  in ibmvnic_open()
1987 static void clean_rx_pools(struct ibmvnic_adapter *adapter)  in clean_rx_pools()  argument
1995 	if (!adapter->rx_pool)  in clean_rx_pools()
1998 	rx_scrqs = adapter->num_active_rx_pools;  in clean_rx_pools()
1999 	rx_entries = adapter->req_rx_add_entries_per_subcrq;  in clean_rx_pools()
2003 		rx_pool = &adapter->rx_pool[i];  in clean_rx_pools()
2007 		netdev_dbg(adapter->netdev, "Cleaning rx_pool[%d]\n", i);  in clean_rx_pools()
2018 static void clean_one_tx_pool(struct ibmvnic_adapter *adapter,  in clean_one_tx_pool()  argument
2039 static void clean_tx_pools(struct ibmvnic_adapter *adapter)  in clean_tx_pools()  argument
2044 	if (!adapter->tx_pool || !adapter->tso_pool)  in clean_tx_pools()
2047 	tx_scrqs = adapter->num_active_tx_pools;  in clean_tx_pools()
2051 		netdev_dbg(adapter->netdev, "Cleaning tx_pool[%d]\n", i);  in clean_tx_pools()
2052 		clean_one_tx_pool(adapter, &adapter->tx_pool[i]);  in clean_tx_pools()
2053 		clean_one_tx_pool(adapter, &adapter->tso_pool[i]);  in clean_tx_pools()
2057 static void ibmvnic_disable_irqs(struct ibmvnic_adapter *adapter)  in ibmvnic_disable_irqs()  argument
2059 	struct net_device *netdev = adapter->netdev;  in ibmvnic_disable_irqs()
2062 	if (adapter->tx_scrq) {  in ibmvnic_disable_irqs()
2063 		for (i = 0; i < adapter->req_tx_queues; i++)  in ibmvnic_disable_irqs()
2064 			if (adapter->tx_scrq[i]->irq) {  in ibmvnic_disable_irqs()
2067 				disable_scrq_irq(adapter, adapter->tx_scrq[i]);  in ibmvnic_disable_irqs()
2068 				disable_irq(adapter->tx_scrq[i]->irq);  in ibmvnic_disable_irqs()
2072 	if (adapter->rx_scrq) {  in ibmvnic_disable_irqs()
2073 		for (i = 0; i < adapter->req_rx_queues; i++) {  in ibmvnic_disable_irqs()
2074 			if (adapter->rx_scrq[i]->irq) {  in ibmvnic_disable_irqs()
2077 				disable_scrq_irq(adapter, adapter->rx_scrq[i]);  in ibmvnic_disable_irqs()
2078 				disable_irq(adapter->rx_scrq[i]->irq);  in ibmvnic_disable_irqs()
2086 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_cleanup()  local
2090 	adapter->tx_queues_active = false;  in ibmvnic_cleanup()
2097 	if (test_bit(0, &adapter->resetting))  in ibmvnic_cleanup()
2102 	ibmvnic_napi_disable(adapter);  in ibmvnic_cleanup()
2103 	ibmvnic_disable_irqs(adapter);  in ibmvnic_cleanup()
2108 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in __ibmvnic_close()  local
2111 	adapter->state = VNIC_CLOSING;  in __ibmvnic_close()
2112 	rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);  in __ibmvnic_close()
2113 	adapter->state = VNIC_CLOSED;  in __ibmvnic_close()
2119 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_close()  local
2123 		   adapter_state_to_string(adapter->state),  in ibmvnic_close()
2124 		   adapter->failover_pending,  in ibmvnic_close()
2125 		   adapter->force_reset_recovery);  in ibmvnic_close()
2130 	if (adapter->failover_pending) {  in ibmvnic_close()
2131 		adapter->state = VNIC_CLOSED;  in ibmvnic_close()
2137 	clean_rx_pools(adapter);  in ibmvnic_close()
2138 	clean_tx_pools(adapter);  in ibmvnic_close()
2279 static void ibmvnic_tx_scrq_clean_buffer(struct ibmvnic_adapter *adapter,  in ibmvnic_tx_scrq_clean_buffer()  argument
2301 			tx_pool = &adapter->tso_pool[queue_num];  in ibmvnic_tx_scrq_clean_buffer()
2304 			tx_pool = &adapter->tx_pool[queue_num];  in ibmvnic_tx_scrq_clean_buffer()
2311 		adapter->netdev->stats.tx_packets--;  in ibmvnic_tx_scrq_clean_buffer()
2312 		adapter->netdev->stats.tx_bytes -= tx_buff->skb->len;  in ibmvnic_tx_scrq_clean_buffer()
2313 		adapter->tx_stats_buffers[queue_num].packets--;  in ibmvnic_tx_scrq_clean_buffer()
2314 		adapter->tx_stats_buffers[queue_num].bytes -=  in ibmvnic_tx_scrq_clean_buffer()
2318 		adapter->netdev->stats.tx_dropped++;  in ibmvnic_tx_scrq_clean_buffer()
2324 	    (adapter->req_tx_entries_per_subcrq / 2) &&  in ibmvnic_tx_scrq_clean_buffer()
2325 	    __netif_subqueue_stopped(adapter->netdev, queue_num)) {  in ibmvnic_tx_scrq_clean_buffer()
2328 		if (adapter->tx_queues_active) {  in ibmvnic_tx_scrq_clean_buffer()
2329 			netif_wake_subqueue(adapter->netdev, queue_num);  in ibmvnic_tx_scrq_clean_buffer()
2330 			netdev_dbg(adapter->netdev, "Started queue %d\n",  in ibmvnic_tx_scrq_clean_buffer()
2338 static int send_subcrq_direct(struct ibmvnic_adapter *adapter,  in send_subcrq_direct()  argument
2341 	unsigned int ua = adapter->vdev->unit_address;  in send_subcrq_direct()
2342 	struct device *dev = &adapter->vdev->dev;  in send_subcrq_direct()
2358 static int ibmvnic_tx_scrq_flush(struct ibmvnic_adapter *adapter,  in ibmvnic_tx_scrq_flush()  argument
2377 		rc = send_subcrq_indirect(adapter, handle, dma_addr, entries);  in ibmvnic_tx_scrq_flush()
2379 		rc = send_subcrq_direct(adapter, handle,  in ibmvnic_tx_scrq_flush()
2383 		ibmvnic_tx_scrq_clean_buffer(adapter, tx_scrq);  in ibmvnic_tx_scrq_flush()
2391 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_xmit()  local
2393 	u8 *hdrs = (u8 *)&adapter->tx_rx_desc_req;  in ibmvnic_xmit()
2394 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_xmit()
2423 	if (!adapter->tx_queues_active) {  in ibmvnic_xmit()
2432 	tx_scrq = adapter->tx_scrq[queue_num];  in ibmvnic_xmit()
2440 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, true);  in ibmvnic_xmit()
2447 		tx_pool = &adapter->tso_pool[queue_num];  in ibmvnic_xmit()
2449 		tx_pool = &adapter->tx_pool[queue_num];  in ibmvnic_xmit()
2458 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, true);  in ibmvnic_xmit()
2541 	if (adapter->vlan_header_insertion && skb_vlan_tag_present(skb)) {  in ibmvnic_xmit()
2574 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, false);  in ibmvnic_xmit()
2588 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, true);  in ibmvnic_xmit()
2601 		lpar_rc = ibmvnic_tx_scrq_flush(adapter, tx_scrq, true);  in ibmvnic_xmit()
2608 					>= adapter->req_tx_entries_per_subcrq) {  in ibmvnic_xmit()
2630 	if (lpar_rc == H_CLOSED || adapter->failover_pending) {  in ibmvnic_xmit()
2644 	adapter->tx_send_failed += tx_send_failed;  in ibmvnic_xmit()
2645 	adapter->tx_map_failed += tx_map_failed;  in ibmvnic_xmit()
2646 	adapter->tx_stats_buffers[queue_num].packets += tx_packets;  in ibmvnic_xmit()
2647 	adapter->tx_stats_buffers[queue_num].bytes += tx_bytes;  in ibmvnic_xmit()
2648 	adapter->tx_stats_buffers[queue_num].dropped_packets += tx_dropped;  in ibmvnic_xmit()
2655 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_multi()  local
2664 		if (!adapter->promisc_supported)  in ibmvnic_set_multi()
2673 			ibmvnic_send_crq(adapter, &crq);  in ibmvnic_set_multi()
2680 			ibmvnic_send_crq(adapter, &crq);  in ibmvnic_set_multi()
2690 				ibmvnic_send_crq(adapter, &crq);  in ibmvnic_set_multi()
2698 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in __ibmvnic_set_mac()  local
2712 	mutex_lock(&adapter->fw_lock);  in __ibmvnic_set_mac()
2713 	adapter->fw_done_rc = 0;  in __ibmvnic_set_mac()
2714 	reinit_completion(&adapter->fw_done);  in __ibmvnic_set_mac()
2716 	rc = ibmvnic_send_crq(adapter, &crq);  in __ibmvnic_set_mac()
2719 		mutex_unlock(&adapter->fw_lock);  in __ibmvnic_set_mac()
2723 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in __ibmvnic_set_mac()
2725 	if (rc || adapter->fw_done_rc) {  in __ibmvnic_set_mac()
2727 		mutex_unlock(&adapter->fw_lock);  in __ibmvnic_set_mac()
2730 	mutex_unlock(&adapter->fw_lock);  in __ibmvnic_set_mac()
2733 	ether_addr_copy(adapter->mac_addr, netdev->dev_addr);  in __ibmvnic_set_mac()
2739 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_mac()  local
2747 	ether_addr_copy(adapter->mac_addr, addr->sa_data);  in ibmvnic_set_mac()
2748 	if (adapter->state != VNIC_PROBED)  in ibmvnic_set_mac()
2782 static inline void reinit_init_done(struct ibmvnic_adapter *adapter)  in reinit_init_done()  argument
2784 	reinit_completion(&adapter->init_done);  in reinit_init_done()
2785 	adapter->init_done_rc = 0;  in reinit_init_done()
2792 static int do_reset(struct ibmvnic_adapter *adapter,  in do_reset()  argument
2795 	struct net_device *netdev = adapter->netdev;  in do_reset()
2800 	netdev_dbg(adapter->netdev,  in do_reset()
2802 		   adapter_state_to_string(adapter->state),  in do_reset()
2803 		   adapter->failover_pending,  in do_reset()
2807 	adapter->reset_reason = rwi->reset_reason;  in do_reset()
2809 	if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM))  in do_reset()
2817 		adapter->failover_pending = false;  in do_reset()
2820 	reset_state = adapter->state;  in do_reset()
2829 	old_num_rx_queues = adapter->req_rx_queues;  in do_reset()
2830 	old_num_tx_queues = adapter->req_tx_queues;  in do_reset()
2831 	old_num_rx_slots = adapter->req_rx_add_entries_per_subcrq;  in do_reset()
2832 	old_num_tx_slots = adapter->req_tx_entries_per_subcrq;  in do_reset()
2837 	    adapter->reset_reason != VNIC_RESET_MOBILITY &&  in do_reset()
2838 	    adapter->reset_reason != VNIC_RESET_FAILOVER) {  in do_reset()
2839 		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {  in do_reset()
2844 			adapter->state = VNIC_CLOSING;  in do_reset()
2852 			rc = set_link_state(adapter, IBMVNIC_LOGICAL_LNK_DN);  in do_reset()
2857 			if (adapter->state == VNIC_OPEN) {  in do_reset()
2868 				adapter->state = VNIC_CLOSING;  in do_reset()
2871 			if (adapter->state != VNIC_CLOSING) {  in do_reset()
2878 			adapter->state = VNIC_CLOSED;  in do_reset()
2882 	if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {  in do_reset()
2883 		release_resources(adapter);  in do_reset()
2884 		release_sub_crqs(adapter, 1);  in do_reset()
2885 		release_crq_queue(adapter);  in do_reset()
2888 	if (adapter->reset_reason != VNIC_RESET_NON_FATAL) {  in do_reset()
2892 		adapter->state = VNIC_PROBED;  in do_reset()
2894 		reinit_init_done(adapter);  in do_reset()
2896 		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {  in do_reset()
2897 			rc = init_crq_queue(adapter);  in do_reset()
2898 		} else if (adapter->reset_reason == VNIC_RESET_MOBILITY) {  in do_reset()
2899 			rc = ibmvnic_reenable_crq_queue(adapter);  in do_reset()
2900 			release_sub_crqs(adapter, 1);  in do_reset()
2902 			rc = ibmvnic_reset_crq(adapter);  in do_reset()
2904 				rc = vio_enable_interrupts(adapter->vdev);  in do_reset()
2906 					netdev_err(adapter->netdev,  in do_reset()
2913 			netdev_err(adapter->netdev,  in do_reset()
2918 		rc = ibmvnic_reset_init(adapter, true);  in do_reset()
2934 		if (adapter->reset_reason == VNIC_RESET_CHANGE_PARAM) {  in do_reset()
2935 			rc = init_resources(adapter);  in do_reset()
2938 		} else if (adapter->req_rx_queues != old_num_rx_queues ||  in do_reset()
2939 		    adapter->req_tx_queues != old_num_tx_queues ||  in do_reset()
2940 		    adapter->req_rx_add_entries_per_subcrq !=  in do_reset()
2942 		    adapter->req_tx_entries_per_subcrq !=  in do_reset()
2944 		    !adapter->rx_pool ||  in do_reset()
2945 		    !adapter->tso_pool ||  in do_reset()
2946 		    !adapter->tx_pool) {  in do_reset()
2947 			release_napi(adapter);  in do_reset()
2948 			release_vpd_data(adapter);  in do_reset()
2950 			rc = init_resources(adapter);  in do_reset()
2971 		ibmvnic_disable_irqs(adapter);  in do_reset()
2973 	adapter->state = VNIC_CLOSED;  in do_reset()
2989 	if (adapter->reset_reason == VNIC_RESET_FAILOVER ||  in do_reset()
2990 	    adapter->reset_reason == VNIC_RESET_MOBILITY)  in do_reset()
2998 		adapter->state = reset_state;  in do_reset()
3000 	if (!(adapter->reset_reason == VNIC_RESET_CHANGE_PARAM))  in do_reset()
3003 	netdev_dbg(adapter->netdev, "[S:%s FOP:%d] Reset done, rc %d\n",  in do_reset()
3004 		   adapter_state_to_string(adapter->state),  in do_reset()
3005 		   adapter->failover_pending, rc);  in do_reset()
3009 static int do_hard_reset(struct ibmvnic_adapter *adapter,  in do_hard_reset()  argument
3012 	struct net_device *netdev = adapter->netdev;  in do_hard_reset()
3015 	netdev_dbg(adapter->netdev, "Hard resetting driver (%s)\n",  in do_hard_reset()
3019 	reset_state = adapter->state;  in do_hard_reset()
3027 	adapter->reset_reason = rwi->reset_reason;  in do_hard_reset()
3030 	release_resources(adapter);  in do_hard_reset()
3031 	release_sub_crqs(adapter, 0);  in do_hard_reset()
3032 	release_crq_queue(adapter);  in do_hard_reset()
3037 	adapter->state = VNIC_PROBED;  in do_hard_reset()
3039 	reinit_init_done(adapter);  in do_hard_reset()
3041 	rc = init_crq_queue(adapter);  in do_hard_reset()
3043 		netdev_err(adapter->netdev,  in do_hard_reset()
3048 	rc = ibmvnic_reset_init(adapter, false);  in do_hard_reset()
3062 	rc = init_resources(adapter);  in do_hard_reset()
3066 	ibmvnic_disable_irqs(adapter);  in do_hard_reset()
3067 	adapter->state = VNIC_CLOSED;  in do_hard_reset()
3082 		adapter->state = reset_state;  in do_hard_reset()
3083 	netdev_dbg(adapter->netdev, "[S:%s FOP:%d] Hard reset done, rc %d\n",  in do_hard_reset()
3084 		   adapter_state_to_string(adapter->state),  in do_hard_reset()
3085 		   adapter->failover_pending, rc);  in do_hard_reset()
3089 static struct ibmvnic_rwi *get_next_rwi(struct ibmvnic_adapter *adapter)  in get_next_rwi()  argument
3094 	spin_lock_irqsave(&adapter->rwi_lock, flags);  in get_next_rwi()
3096 	if (!list_empty(&adapter->rwi_list)) {  in get_next_rwi()
3097 		rwi = list_first_entry(&adapter->rwi_list, struct ibmvnic_rwi,  in get_next_rwi()
3104 	spin_unlock_irqrestore(&adapter->rwi_lock, flags);  in get_next_rwi()
3122 static int do_passive_init(struct ibmvnic_adapter *adapter)  in do_passive_init()  argument
3125 	struct net_device *netdev = adapter->netdev;  in do_passive_init()
3126 	struct device *dev = &adapter->vdev->dev;  in do_passive_init()
3131 	adapter->state = VNIC_PROBING;  in do_passive_init()
3132 	reinit_completion(&adapter->init_done);  in do_passive_init()
3133 	adapter->init_done_rc = 0;  in do_passive_init()
3134 	adapter->crq.active = true;  in do_passive_init()
3136 	rc = send_crq_init_complete(adapter);  in do_passive_init()
3140 	rc = send_version_xchg(adapter);  in do_passive_init()
3142 		netdev_dbg(adapter->netdev, "send_version_xchg failed, rc=%d\n", rc);  in do_passive_init()
3144 	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {  in do_passive_init()
3150 	rc = init_sub_crqs(adapter);  in do_passive_init()
3156 	rc = init_sub_crq_irqs(adapter);  in do_passive_init()
3162 	netdev->mtu = adapter->req_mtu - ETH_HLEN;  in do_passive_init()
3163 	netdev->min_mtu = adapter->min_mtu - ETH_HLEN;  in do_passive_init()
3164 	netdev->max_mtu = adapter->max_mtu - ETH_HLEN;  in do_passive_init()
3166 	adapter->state = VNIC_PROBED;  in do_passive_init()
3172 	release_sub_crqs(adapter, 1);  in do_passive_init()
3174 	adapter->state = VNIC_DOWN;  in do_passive_init()
3180 	struct ibmvnic_adapter *adapter;  in __ibmvnic_reset()  local
3192 	adapter = container_of(work, struct ibmvnic_adapter, ibmvnic_reset);  in __ibmvnic_reset()
3193 		dev = &adapter->vdev->dev;  in __ibmvnic_reset()
3204 	if (adapter->state == VNIC_PROBING &&  in __ibmvnic_reset()
3205 	    !wait_for_completion_timeout(&adapter->probe_done, timeout)) {  in __ibmvnic_reset()
3208 				   &adapter->ibmvnic_delayed_reset,  in __ibmvnic_reset()
3214 	if (adapter->state == VNIC_REMOVING)  in __ibmvnic_reset()
3242 	spin_lock(&adapter->rwi_lock);  in __ibmvnic_reset()
3243 	if (!list_empty(&adapter->rwi_list)) {  in __ibmvnic_reset()
3244 		if (test_and_set_bit_lock(0, &adapter->resetting)) {  in __ibmvnic_reset()
3246 					   &adapter->ibmvnic_delayed_reset,  in __ibmvnic_reset()
3252 	spin_unlock(&adapter->rwi_lock);  in __ibmvnic_reset()
3257 	rwi = get_next_rwi(adapter);  in __ibmvnic_reset()
3259 		spin_lock_irqsave(&adapter->state_lock, flags);  in __ibmvnic_reset()
3261 		if (adapter->state == VNIC_REMOVING ||  in __ibmvnic_reset()
3262 		    adapter->state == VNIC_REMOVED) {  in __ibmvnic_reset()
3263 			spin_unlock_irqrestore(&adapter->state_lock, flags);  in __ibmvnic_reset()
3270 			reset_state = adapter->state;  in __ibmvnic_reset()
3273 		spin_unlock_irqrestore(&adapter->state_lock, flags);  in __ibmvnic_reset()
3277 			rc = do_passive_init(adapter);  in __ibmvnic_reset()
3280 				netif_carrier_on(adapter->netdev);  in __ibmvnic_reset()
3281 		} else if (adapter->force_reset_recovery) {  in __ibmvnic_reset()
3286 			adapter->failover_pending = false;  in __ibmvnic_reset()
3289 			if (adapter->wait_for_reset) {  in __ibmvnic_reset()
3291 				adapter->force_reset_recovery = false;  in __ibmvnic_reset()
3292 				rc = do_hard_reset(adapter, rwi, reset_state);  in __ibmvnic_reset()
3295 				adapter->force_reset_recovery = false;  in __ibmvnic_reset()
3296 				rc = do_hard_reset(adapter, rwi, reset_state);  in __ibmvnic_reset()
3312 				netdev_dbg(adapter->netdev,  in __ibmvnic_reset()
3314 					   adapter_state_to_string(adapter->state),  in __ibmvnic_reset()
3320 			rc = do_reset(adapter, rwi, reset_state);  in __ibmvnic_reset()
3323 		adapter->last_reset_time = jiffies;  in __ibmvnic_reset()
3326 			netdev_dbg(adapter->netdev, "Reset failed, rc=%d\n", rc);  in __ibmvnic_reset()
3328 		rwi = get_next_rwi(adapter);  in __ibmvnic_reset()
3347 			adapter->force_reset_recovery = true;  in __ibmvnic_reset()
3350 	if (adapter->wait_for_reset) {  in __ibmvnic_reset()
3351 		adapter->reset_done_rc = rc;  in __ibmvnic_reset()
3352 		complete(&adapter->reset_done);  in __ibmvnic_reset()
3355 	clear_bit_unlock(0, &adapter->resetting);  in __ibmvnic_reset()
3357 	netdev_dbg(adapter->netdev,  in __ibmvnic_reset()
3359 		   adapter_state_to_string(adapter->state),  in __ibmvnic_reset()
3360 		   adapter->force_reset_recovery,  in __ibmvnic_reset()
3361 		   adapter->wait_for_reset);  in __ibmvnic_reset()
3366 	struct ibmvnic_adapter *adapter;  in __ibmvnic_delayed_reset()  local
3368 	adapter = container_of(work, struct ibmvnic_adapter,  in __ibmvnic_delayed_reset()
3370 	__ibmvnic_reset(&adapter->ibmvnic_reset);  in __ibmvnic_delayed_reset()
3373 static void flush_reset_queue(struct ibmvnic_adapter *adapter)  in flush_reset_queue()  argument
3377 	if (!list_empty(&adapter->rwi_list)) {  in flush_reset_queue()
3378 		list_for_each_safe(entry, tmp_entry, &adapter->rwi_list) {  in flush_reset_queue()
3385 static int ibmvnic_reset(struct ibmvnic_adapter *adapter,  in ibmvnic_reset()  argument
3388 	struct net_device *netdev = adapter->netdev;  in ibmvnic_reset()
3393 	spin_lock_irqsave(&adapter->rwi_lock, flags);  in ibmvnic_reset()
3400 	if (adapter->state == VNIC_REMOVING ||  in ibmvnic_reset()
3401 	    adapter->state == VNIC_REMOVED ||  in ibmvnic_reset()
3402 	    (adapter->failover_pending && reason != VNIC_RESET_FAILOVER)) {  in ibmvnic_reset()
3408 	list_for_each_entry(tmp, &adapter->rwi_list, list) {  in ibmvnic_reset()
3425 	if (adapter->force_reset_recovery)  in ibmvnic_reset()
3426 		flush_reset_queue(adapter);  in ibmvnic_reset()
3429 	list_add_tail(&rwi->list, &adapter->rwi_list);  in ibmvnic_reset()
3430 	netdev_dbg(adapter->netdev, "Scheduling reset (reason %s)\n",  in ibmvnic_reset()
3432 	queue_work(system_long_wq, &adapter->ibmvnic_reset);  in ibmvnic_reset()
3437 	spin_unlock_irqrestore(&adapter->rwi_lock, flags);  in ibmvnic_reset()
3447 	struct ibmvnic_adapter *adapter = netdev_priv(dev);  in ibmvnic_tx_timeout()  local
3449 	if (test_bit(0, &adapter->resetting)) {  in ibmvnic_tx_timeout()
3450 		netdev_err(adapter->netdev,  in ibmvnic_tx_timeout()
3457 	if (time_before(jiffies, (adapter->last_reset_time + dev->watchdog_timeo))) {  in ibmvnic_tx_timeout()
3461 	ibmvnic_reset(adapter, VNIC_RESET_TIMEOUT);  in ibmvnic_tx_timeout()
3464 static void remove_buff_from_pool(struct ibmvnic_adapter *adapter,  in remove_buff_from_pool()  argument
3467 	struct ibmvnic_rx_pool *pool = &adapter->rx_pool[rx_buff->pool_index];  in remove_buff_from_pool()
3480 	struct ibmvnic_adapter *adapter;  in ibmvnic_poll()  local
3486 	adapter = netdev_priv(netdev);  in ibmvnic_poll()
3487 	scrq_num = (int)(napi - adapter->napi);  in ibmvnic_poll()
3489 	rx_scrq = adapter->rx_scrq[scrq_num];  in ibmvnic_poll()
3500 		if (unlikely(test_bit(0, &adapter->resetting) &&  in ibmvnic_poll()
3501 			     adapter->reset_reason != VNIC_RESET_NON_FATAL)) {  in ibmvnic_poll()
3502 			enable_scrq_irq(adapter, rx_scrq);  in ibmvnic_poll()
3507 		if (!pending_scrq(adapter, rx_scrq))  in ibmvnic_poll()
3509 		next = ibmvnic_next_scrq(adapter, rx_scrq);  in ibmvnic_poll()
3519 			remove_buff_from_pool(adapter, rx_buff);  in ibmvnic_poll()
3524 			remove_buff_from_pool(adapter, rx_buff);  in ibmvnic_poll()
3540 		if (adapter->rx_vlan_header_insertion &&  in ibmvnic_poll()
3547 		remove_buff_from_pool(adapter, rx_buff);  in ibmvnic_poll()
3562 		adapter->rx_stats_buffers[scrq_num].packets++;  in ibmvnic_poll()
3563 		adapter->rx_stats_buffers[scrq_num].bytes += length;  in ibmvnic_poll()
3567 	if (adapter->state != VNIC_CLOSING &&  in ibmvnic_poll()
3568 	    (atomic_read(&adapter->rx_pool[scrq_num].available) <  in ibmvnic_poll()
3569 	      adapter->req_rx_add_entries_per_subcrq / 2))  in ibmvnic_poll()
3570 		replenish_rx_pool(adapter, &adapter->rx_pool[scrq_num]);  in ibmvnic_poll()
3573 			enable_scrq_irq(adapter, rx_scrq);  in ibmvnic_poll()
3574 			if (pending_scrq(adapter, rx_scrq)) {  in ibmvnic_poll()
3576 					disable_scrq_irq(adapter, rx_scrq);  in ibmvnic_poll()
3585 static int wait_for_reset(struct ibmvnic_adapter *adapter)  in wait_for_reset()  argument
3589 	adapter->fallback.mtu = adapter->req_mtu;  in wait_for_reset()
3590 	adapter->fallback.rx_queues = adapter->req_rx_queues;  in wait_for_reset()
3591 	adapter->fallback.tx_queues = adapter->req_tx_queues;  in wait_for_reset()
3592 	adapter->fallback.rx_entries = adapter->req_rx_add_entries_per_subcrq;  in wait_for_reset()
3593 	adapter->fallback.tx_entries = adapter->req_tx_entries_per_subcrq;  in wait_for_reset()
3595 	reinit_completion(&adapter->reset_done);  in wait_for_reset()
3596 	adapter->wait_for_reset = true;  in wait_for_reset()
3597 	rc = ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM);  in wait_for_reset()
3603 	rc = ibmvnic_wait_for_completion(adapter, &adapter->reset_done, 60000);  in wait_for_reset()
3610 	if (adapter->reset_done_rc) {  in wait_for_reset()
3612 		adapter->desired.mtu = adapter->fallback.mtu;  in wait_for_reset()
3613 		adapter->desired.rx_queues = adapter->fallback.rx_queues;  in wait_for_reset()
3614 		adapter->desired.tx_queues = adapter->fallback.tx_queues;  in wait_for_reset()
3615 		adapter->desired.rx_entries = adapter->fallback.rx_entries;  in wait_for_reset()
3616 		adapter->desired.tx_entries = adapter->fallback.tx_entries;  in wait_for_reset()
3618 		reinit_completion(&adapter->reset_done);  in wait_for_reset()
3619 		adapter->wait_for_reset = true;  in wait_for_reset()
3620 		rc = ibmvnic_reset(adapter, VNIC_RESET_CHANGE_PARAM);  in wait_for_reset()
3625 		rc = ibmvnic_wait_for_completion(adapter, &adapter->reset_done,  in wait_for_reset()
3633 	adapter->wait_for_reset = false;  in wait_for_reset()
3640 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_change_mtu()  local
3642 	adapter->desired.mtu = new_mtu + ETH_HLEN;  in ibmvnic_change_mtu()
3644 	return wait_for_reset(adapter);  in ibmvnic_change_mtu()
3681 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_link_ksettings()  local
3684 	rc = send_query_phys_parms(adapter);  in ibmvnic_get_link_ksettings()
3686 		adapter->speed = SPEED_UNKNOWN;  in ibmvnic_get_link_ksettings()
3687 		adapter->duplex = DUPLEX_UNKNOWN;  in ibmvnic_get_link_ksettings()
3689 	cmd->base.speed = adapter->speed;  in ibmvnic_get_link_ksettings()
3690 	cmd->base.duplex = adapter->duplex;  in ibmvnic_get_link_ksettings()
3701 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_drvinfo()  local
3705 	strscpy(info->fw_version, adapter->fw_version,  in ibmvnic_get_drvinfo()
3711 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_msglevel()  local
3713 	return adapter->msg_enable;  in ibmvnic_get_msglevel()
3718 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_msglevel()  local
3720 	adapter->msg_enable = data;  in ibmvnic_set_msglevel()
3725 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_link()  local
3730 	return adapter->logical_link_state;  in ibmvnic_get_link()
3738 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_ringparam()  local
3740 	ring->rx_max_pending = adapter->max_rx_add_entries_per_subcrq;  in ibmvnic_get_ringparam()
3741 	ring->tx_max_pending = adapter->max_tx_entries_per_subcrq;  in ibmvnic_get_ringparam()
3744 	ring->rx_pending = adapter->req_rx_add_entries_per_subcrq;  in ibmvnic_get_ringparam()
3745 	ring->tx_pending = adapter->req_tx_entries_per_subcrq;  in ibmvnic_get_ringparam()
3755 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_ringparam()  local
3757 	if (ring->rx_pending > adapter->max_rx_add_entries_per_subcrq  ||  in ibmvnic_set_ringparam()
3758 	    ring->tx_pending > adapter->max_tx_entries_per_subcrq) {  in ibmvnic_set_ringparam()
3761 			   adapter->max_rx_add_entries_per_subcrq);  in ibmvnic_set_ringparam()
3763 			   adapter->max_tx_entries_per_subcrq);  in ibmvnic_set_ringparam()
3767 	adapter->desired.rx_entries = ring->rx_pending;  in ibmvnic_set_ringparam()
3768 	adapter->desired.tx_entries = ring->tx_pending;  in ibmvnic_set_ringparam()
3770 	return wait_for_reset(adapter);  in ibmvnic_set_ringparam()
3776 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_get_channels()  local
3778 	channels->max_rx = adapter->max_rx_queues;  in ibmvnic_get_channels()
3779 	channels->max_tx = adapter->max_tx_queues;  in ibmvnic_get_channels()
3782 	channels->rx_count = adapter->req_rx_queues;  in ibmvnic_get_channels()
3783 	channels->tx_count = adapter->req_tx_queues;  in ibmvnic_get_channels()
3791 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_set_channels()  local
3793 	adapter->desired.rx_queues = channels->rx_count;  in ibmvnic_set_channels()
3794 	adapter->desired.tx_queues = channels->tx_count;  in ibmvnic_set_channels()
3796 	return wait_for_reset(adapter);  in ibmvnic_set_channels()
3801 	struct ibmvnic_adapter *adapter = netdev_priv(dev);  in ibmvnic_get_strings()  local
3810 	for (i = 0; i < adapter->req_tx_queues; i++) {  in ibmvnic_get_strings()
3821 	for (i = 0; i < adapter->req_rx_queues; i++) {  in ibmvnic_get_strings()
3835 	struct ibmvnic_adapter *adapter = netdev_priv(dev);  in ibmvnic_get_sset_count()  local
3840 		       adapter->req_tx_queues * NUM_TX_STATS +  in ibmvnic_get_sset_count()
3841 		       adapter->req_rx_queues * NUM_RX_STATS;  in ibmvnic_get_sset_count()
3850 	struct ibmvnic_adapter *adapter = netdev_priv(dev);  in ibmvnic_get_ethtool_stats()  local
3858 	crq.request_statistics.ioba = cpu_to_be32(adapter->stats_token);  in ibmvnic_get_ethtool_stats()
3863 	reinit_completion(&adapter->stats_done);  in ibmvnic_get_ethtool_stats()
3864 	rc = ibmvnic_send_crq(adapter, &crq);  in ibmvnic_get_ethtool_stats()
3867 	rc = ibmvnic_wait_for_completion(adapter, &adapter->stats_done, 10000);  in ibmvnic_get_ethtool_stats()
3873 				      (adapter, ibmvnic_stats[i].offset));  in ibmvnic_get_ethtool_stats()
3875 	for (j = 0; j < adapter->req_tx_queues; j++) {  in ibmvnic_get_ethtool_stats()
3876 		data[i] = adapter->tx_stats_buffers[j].packets;  in ibmvnic_get_ethtool_stats()
3878 		data[i] = adapter->tx_stats_buffers[j].bytes;  in ibmvnic_get_ethtool_stats()
3880 		data[i] = adapter->tx_stats_buffers[j].dropped_packets;  in ibmvnic_get_ethtool_stats()
3884 	for (j = 0; j < adapter->req_rx_queues; j++) {  in ibmvnic_get_ethtool_stats()
3885 		data[i] = adapter->rx_stats_buffers[j].packets;  in ibmvnic_get_ethtool_stats()
3887 		data[i] = adapter->rx_stats_buffers[j].bytes;  in ibmvnic_get_ethtool_stats()
3889 		data[i] = adapter->rx_stats_buffers[j].interrupts;  in ibmvnic_get_ethtool_stats()
3911 static int reset_one_sub_crq_queue(struct ibmvnic_adapter *adapter,  in reset_one_sub_crq_queue()  argument
3917 		netdev_dbg(adapter->netdev, "Invalid scrq reset.\n");  in reset_one_sub_crq_queue()
3933 		netdev_dbg(adapter->netdev, "Invalid scrq reset\n");  in reset_one_sub_crq_queue()
3937 	rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,  in reset_one_sub_crq_queue()
3942 static int reset_sub_crq_queues(struct ibmvnic_adapter *adapter)  in reset_sub_crq_queues()  argument
3946 	if (!adapter->tx_scrq || !adapter->rx_scrq)  in reset_sub_crq_queues()
3949 	ibmvnic_clean_affinity(adapter);  in reset_sub_crq_queues()
3951 	for (i = 0; i < adapter->req_tx_queues; i++) {  in reset_sub_crq_queues()
3952 		netdev_dbg(adapter->netdev, "Re-setting tx_scrq[%d]\n", i);  in reset_sub_crq_queues()
3953 		rc = reset_one_sub_crq_queue(adapter, adapter->tx_scrq[i]);  in reset_sub_crq_queues()
3958 	for (i = 0; i < adapter->req_rx_queues; i++) {  in reset_sub_crq_queues()
3959 		netdev_dbg(adapter->netdev, "Re-setting rx_scrq[%d]\n", i);  in reset_sub_crq_queues()
3960 		rc = reset_one_sub_crq_queue(adapter, adapter->rx_scrq[i]);  in reset_sub_crq_queues()
3968 static void release_sub_crq_queue(struct ibmvnic_adapter *adapter,  in release_sub_crq_queue()  argument
3972 	struct device *dev = &adapter->vdev->dev;  in release_sub_crq_queue()
3975 	netdev_dbg(adapter->netdev, "Releasing sub-CRQ\n");  in release_sub_crq_queue()
3981 						adapter->vdev->unit_address,  in release_sub_crq_queue()
3986 			netdev_err(adapter->netdev,  in release_sub_crq_queue()
4005 							*adapter)  in init_sub_crq_queue()
4007 	struct device *dev = &adapter->vdev->dev;  in init_sub_crq_queue()
4031 	rc = h_reg_sub_crq(adapter->vdev->unit_address, scrq->msg_token,  in init_sub_crq_queue()
4035 		rc = ibmvnic_reset_crq(adapter);  in init_sub_crq_queue()
4044 	scrq->adapter = adapter;  in init_sub_crq_queue()
4059 	netdev_dbg(adapter->netdev,  in init_sub_crq_queue()
4068 					adapter->vdev->unit_address,  in init_sub_crq_queue()
4084 static void release_sub_crqs(struct ibmvnic_adapter *adapter, bool do_h_free)  in release_sub_crqs()  argument
4088 	ibmvnic_clean_affinity(adapter);  in release_sub_crqs()
4089 	if (adapter->tx_scrq) {  in release_sub_crqs()
4090 		for (i = 0; i < adapter->num_active_tx_scrqs; i++) {  in release_sub_crqs()
4091 			if (!adapter->tx_scrq[i])  in release_sub_crqs()
4094 			netdev_dbg(adapter->netdev, "Releasing tx_scrq[%d]\n",  in release_sub_crqs()
4096 			ibmvnic_tx_scrq_clean_buffer(adapter, adapter->tx_scrq[i]);  in release_sub_crqs()
4097 			if (adapter->tx_scrq[i]->irq) {  in release_sub_crqs()
4098 				free_irq(adapter->tx_scrq[i]->irq,  in release_sub_crqs()
4099 					 adapter->tx_scrq[i]);  in release_sub_crqs()
4100 				irq_dispose_mapping(adapter->tx_scrq[i]->irq);  in release_sub_crqs()
4101 				adapter->tx_scrq[i]->irq = 0;  in release_sub_crqs()
4104 			release_sub_crq_queue(adapter, adapter->tx_scrq[i],  in release_sub_crqs()
4108 		kfree(adapter->tx_scrq);  in release_sub_crqs()
4109 		adapter->tx_scrq = NULL;  in release_sub_crqs()
4110 		adapter->num_active_tx_scrqs = 0;  in release_sub_crqs()
4117 	clean_tx_pools(adapter);  in release_sub_crqs()
4119 	if (adapter->rx_scrq) {  in release_sub_crqs()
4120 		for (i = 0; i < adapter->num_active_rx_scrqs; i++) {  in release_sub_crqs()
4121 			if (!adapter->rx_scrq[i])  in release_sub_crqs()
4124 			netdev_dbg(adapter->netdev, "Releasing rx_scrq[%d]\n",  in release_sub_crqs()
4126 			if (adapter->rx_scrq[i]->irq) {  in release_sub_crqs()
4127 				free_irq(adapter->rx_scrq[i]->irq,  in release_sub_crqs()
4128 					 adapter->rx_scrq[i]);  in release_sub_crqs()
4129 				irq_dispose_mapping(adapter->rx_scrq[i]->irq);  in release_sub_crqs()
4130 				adapter->rx_scrq[i]->irq = 0;  in release_sub_crqs()
4133 			release_sub_crq_queue(adapter, adapter->rx_scrq[i],  in release_sub_crqs()
4137 		kfree(adapter->rx_scrq);  in release_sub_crqs()
4138 		adapter->rx_scrq = NULL;  in release_sub_crqs()
4139 		adapter->num_active_rx_scrqs = 0;  in release_sub_crqs()
4143 static int disable_scrq_irq(struct ibmvnic_adapter *adapter,  in disable_scrq_irq()  argument
4146 	struct device *dev = &adapter->vdev->dev;  in disable_scrq_irq()
4149 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,  in disable_scrq_irq()
4181 static int enable_scrq_irq(struct ibmvnic_adapter *adapter,  in enable_scrq_irq()  argument
4184 	struct device *dev = &adapter->vdev->dev;  in enable_scrq_irq()
4192 	if (test_bit(0, &adapter->resetting) &&  in enable_scrq_irq()
4193 	    adapter->reset_reason == VNIC_RESET_MOBILITY) {  in enable_scrq_irq()
4197 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,  in enable_scrq_irq()
4205 static int ibmvnic_complete_tx(struct ibmvnic_adapter *adapter,  in ibmvnic_complete_tx()  argument
4208 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_complete_tx()
4217 	while (pending_scrq(adapter, scrq)) {  in ibmvnic_complete_tx()
4220 		next = ibmvnic_next_scrq(adapter, scrq);  in ibmvnic_complete_tx()
4224 				tx_pool = &adapter->tso_pool[pool];  in ibmvnic_complete_tx()
4227 				tx_pool = &adapter->tx_pool[pool];  in ibmvnic_complete_tx()
4244 				netdev_warn(adapter->netdev,  in ibmvnic_complete_tx()
4257 		    (adapter->req_tx_entries_per_subcrq / 2) &&  in ibmvnic_complete_tx()
4258 		    __netif_subqueue_stopped(adapter->netdev,  in ibmvnic_complete_tx()
4261 			if (adapter->tx_queues_active) {  in ibmvnic_complete_tx()
4262 				netif_wake_subqueue(adapter->netdev,  in ibmvnic_complete_tx()
4264 				netdev_dbg(adapter->netdev,  in ibmvnic_complete_tx()
4272 	enable_scrq_irq(adapter, scrq);  in ibmvnic_complete_tx()
4274 	if (pending_scrq(adapter, scrq)) {  in ibmvnic_complete_tx()
4275 		disable_scrq_irq(adapter, scrq);  in ibmvnic_complete_tx()
4279 	txq = netdev_get_tx_queue(adapter->netdev, scrq->pool_index);  in ibmvnic_complete_tx()
4288 	struct ibmvnic_adapter *adapter = scrq->adapter;  in ibmvnic_interrupt_tx()  local
4290 	disable_scrq_irq(adapter, scrq);  in ibmvnic_interrupt_tx()
4291 	ibmvnic_complete_tx(adapter, scrq);  in ibmvnic_interrupt_tx()
4299 	struct ibmvnic_adapter *adapter = scrq->adapter;  in ibmvnic_interrupt_rx()  local
4304 	if (unlikely(adapter->state != VNIC_OPEN))  in ibmvnic_interrupt_rx()
4307 	adapter->rx_stats_buffers[scrq->scrq_num].interrupts++;  in ibmvnic_interrupt_rx()
4309 	if (napi_schedule_prep(&adapter->napi[scrq->scrq_num])) {  in ibmvnic_interrupt_rx()
4310 		disable_scrq_irq(adapter, scrq);  in ibmvnic_interrupt_rx()
4311 		__napi_schedule(&adapter->napi[scrq->scrq_num]);  in ibmvnic_interrupt_rx()
4317 static int init_sub_crq_irqs(struct ibmvnic_adapter *adapter)  in init_sub_crq_irqs()  argument
4319 	struct device *dev = &adapter->vdev->dev;  in init_sub_crq_irqs()
4324 	for (i = 0; i < adapter->req_tx_queues; i++) {  in init_sub_crq_irqs()
4325 		netdev_dbg(adapter->netdev, "Initializing tx_scrq[%d] irq\n",  in init_sub_crq_irqs()
4327 		scrq = adapter->tx_scrq[i];  in init_sub_crq_irqs()
4337 			 adapter->vdev->unit_address, i);  in init_sub_crq_irqs()
4349 	for (i = 0; i < adapter->req_rx_queues; i++) {  in init_sub_crq_irqs()
4350 		netdev_dbg(adapter->netdev, "Initializing rx_scrq[%d] irq\n",  in init_sub_crq_irqs()
4352 		scrq = adapter->rx_scrq[i];  in init_sub_crq_irqs()
4360 			 adapter->vdev->unit_address, i);  in init_sub_crq_irqs()
4372 	ibmvnic_set_affinity(adapter);  in init_sub_crq_irqs()
4379 		free_irq(adapter->rx_scrq[j]->irq, adapter->rx_scrq[j]);  in init_sub_crq_irqs()
4380 		irq_dispose_mapping(adapter->rx_scrq[j]->irq);  in init_sub_crq_irqs()
4382 	i = adapter->req_tx_queues;  in init_sub_crq_irqs()
4385 		free_irq(adapter->tx_scrq[j]->irq, adapter->tx_scrq[j]);  in init_sub_crq_irqs()
4386 		irq_dispose_mapping(adapter->tx_scrq[j]->irq);  in init_sub_crq_irqs()
4388 	release_sub_crqs(adapter, 1);  in init_sub_crq_irqs()
4392 static int init_sub_crqs(struct ibmvnic_adapter *adapter)  in init_sub_crqs()  argument
4394 	struct device *dev = &adapter->vdev->dev;  in init_sub_crqs()
4401 	total_queues = adapter->req_tx_queues + adapter->req_rx_queues;  in init_sub_crqs()
4408 		allqueues[i] = init_sub_crq_queue(adapter);  in init_sub_crqs()
4418 	    adapter->min_tx_queues + adapter->min_rx_queues) {  in init_sub_crqs()
4425 		netdev_dbg(adapter->netdev, "Reducing number of queues\n");  in init_sub_crqs()
4428 			if (adapter->req_rx_queues > adapter->min_rx_queues)  in init_sub_crqs()
4429 				adapter->req_rx_queues--;  in init_sub_crqs()
4434 			if (adapter->req_tx_queues > adapter->min_tx_queues)  in init_sub_crqs()
4435 				adapter->req_tx_queues--;  in init_sub_crqs()
4442 	adapter->tx_scrq = kcalloc(adapter->req_tx_queues,  in init_sub_crqs()
4443 				   sizeof(*adapter->tx_scrq), GFP_KERNEL);  in init_sub_crqs()
4444 	if (!adapter->tx_scrq)  in init_sub_crqs()
4447 	for (i = 0; i < adapter->req_tx_queues; i++) {  in init_sub_crqs()
4448 		adapter->tx_scrq[i] = allqueues[i];  in init_sub_crqs()
4449 		adapter->tx_scrq[i]->pool_index = i;  in init_sub_crqs()
4450 		adapter->num_active_tx_scrqs++;  in init_sub_crqs()
4453 	adapter->rx_scrq = kcalloc(adapter->req_rx_queues,  in init_sub_crqs()
4454 				   sizeof(*adapter->rx_scrq), GFP_KERNEL);  in init_sub_crqs()
4455 	if (!adapter->rx_scrq)  in init_sub_crqs()
4458 	for (i = 0; i < adapter->req_rx_queues; i++) {  in init_sub_crqs()
4459 		adapter->rx_scrq[i] = allqueues[i + adapter->req_tx_queues];  in init_sub_crqs()
4460 		adapter->rx_scrq[i]->scrq_num = i;  in init_sub_crqs()
4461 		adapter->num_active_rx_scrqs++;  in init_sub_crqs()
4468 	kfree(adapter->tx_scrq);  in init_sub_crqs()
4469 	adapter->tx_scrq = NULL;  in init_sub_crqs()
4472 		release_sub_crq_queue(adapter, allqueues[i], 1);  in init_sub_crqs()
4477 static void send_request_cap(struct ibmvnic_adapter *adapter, int retry)  in send_request_cap()  argument
4479 	struct device *dev = &adapter->vdev->dev;  in send_request_cap()
4489 	if (!(adapter->netdev->flags & IFF_PROMISC) ||  in send_request_cap()
4490 	    adapter->promisc_supported)  in send_request_cap()
4499 		atomic_set(&adapter->running_cap_crqs, cap_reqs);  in send_request_cap()
4501 		if (adapter->min_tx_entries_per_subcrq > entries_page ||  in send_request_cap()
4502 		    adapter->min_rx_add_entries_per_subcrq > entries_page) {  in send_request_cap()
4507 		if (adapter->desired.mtu)  in send_request_cap()
4508 			adapter->req_mtu = adapter->desired.mtu;  in send_request_cap()
4510 			adapter->req_mtu = adapter->netdev->mtu + ETH_HLEN;  in send_request_cap()
4512 		if (!adapter->desired.tx_entries)  in send_request_cap()
4513 			adapter->desired.tx_entries =  in send_request_cap()
4514 					adapter->max_tx_entries_per_subcrq;  in send_request_cap()
4515 		if (!adapter->desired.rx_entries)  in send_request_cap()
4516 			adapter->desired.rx_entries =  in send_request_cap()
4517 					adapter->max_rx_add_entries_per_subcrq;  in send_request_cap()
4520 			      (adapter->req_mtu + IBMVNIC_BUFFER_HLEN);  in send_request_cap()
4522 		if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) *  in send_request_cap()
4523 			adapter->desired.tx_entries > IBMVNIC_LTB_SET_SIZE) {  in send_request_cap()
4524 			adapter->desired.tx_entries = max_entries;  in send_request_cap()
4527 		if ((adapter->req_mtu + IBMVNIC_BUFFER_HLEN) *  in send_request_cap()
4528 			adapter->desired.rx_entries > IBMVNIC_LTB_SET_SIZE) {  in send_request_cap()
4529 			adapter->desired.rx_entries = max_entries;  in send_request_cap()
4532 		if (adapter->desired.tx_entries)  in send_request_cap()
4533 			adapter->req_tx_entries_per_subcrq =  in send_request_cap()
4534 					adapter->desired.tx_entries;  in send_request_cap()
4536 			adapter->req_tx_entries_per_subcrq =  in send_request_cap()
4537 					adapter->max_tx_entries_per_subcrq;  in send_request_cap()
4539 		if (adapter->desired.rx_entries)  in send_request_cap()
4540 			adapter->req_rx_add_entries_per_subcrq =  in send_request_cap()
4541 					adapter->desired.rx_entries;  in send_request_cap()
4543 			adapter->req_rx_add_entries_per_subcrq =  in send_request_cap()
4544 					adapter->max_rx_add_entries_per_subcrq;  in send_request_cap()
4546 		if (adapter->desired.tx_queues)  in send_request_cap()
4547 			adapter->req_tx_queues =  in send_request_cap()
4548 					adapter->desired.tx_queues;  in send_request_cap()
4550 			adapter->req_tx_queues =  in send_request_cap()
4551 					adapter->opt_tx_comp_sub_queues;  in send_request_cap()
4553 		if (adapter->desired.rx_queues)  in send_request_cap()
4554 			adapter->req_rx_queues =  in send_request_cap()
4555 					adapter->desired.rx_queues;  in send_request_cap()
4557 			adapter->req_rx_queues =  in send_request_cap()
4558 					adapter->opt_rx_comp_queues;  in send_request_cap()
4560 		adapter->req_rx_add_queues = adapter->max_rx_add_queues;  in send_request_cap()
4562 		atomic_add(cap_reqs, &adapter->running_cap_crqs);  in send_request_cap()
4569 	crq.request_capability.number = cpu_to_be64(adapter->req_tx_queues);  in send_request_cap()
4571 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4574 	crq.request_capability.number = cpu_to_be64(adapter->req_rx_queues);  in send_request_cap()
4576 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4579 	crq.request_capability.number = cpu_to_be64(adapter->req_rx_add_queues);  in send_request_cap()
4581 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4586 	    cpu_to_be64(adapter->req_tx_entries_per_subcrq);  in send_request_cap()
4588 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4593 	    cpu_to_be64(adapter->req_rx_add_entries_per_subcrq);  in send_request_cap()
4595 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4598 	crq.request_capability.number = cpu_to_be64(adapter->req_mtu);  in send_request_cap()
4600 	ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4602 	if (adapter->netdev->flags & IFF_PROMISC) {  in send_request_cap()
4603 		if (adapter->promisc_supported) {  in send_request_cap()
4608 			ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4615 		ibmvnic_send_crq(adapter, &crq);  in send_request_cap()
4624 static int pending_scrq(struct ibmvnic_adapter *adapter,  in pending_scrq()  argument
4640 static union sub_crq *ibmvnic_next_scrq(struct ibmvnic_adapter *adapter,  in ibmvnic_next_scrq()  argument
4664 static union ibmvnic_crq *ibmvnic_next_crq(struct ibmvnic_adapter *adapter)  in ibmvnic_next_crq()  argument
4666 	struct ibmvnic_crq_queue *queue = &adapter->crq;  in ibmvnic_next_crq()
4699 static int send_subcrq_indirect(struct ibmvnic_adapter *adapter,  in send_subcrq_indirect()  argument
4702 	unsigned int ua = adapter->vdev->unit_address;  in send_subcrq_indirect()
4703 	struct device *dev = &adapter->vdev->dev;  in send_subcrq_indirect()
4718 static int ibmvnic_send_crq(struct ibmvnic_adapter *adapter,  in ibmvnic_send_crq()  argument
4721 	unsigned int ua = adapter->vdev->unit_address;  in ibmvnic_send_crq()
4722 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_send_crq()
4726 	netdev_dbg(adapter->netdev, "Sending CRQ: %016lx %016lx\n",  in ibmvnic_send_crq()
4730 	if (!adapter->crq.active &&  in ibmvnic_send_crq()
4755 static int ibmvnic_send_crq_init(struct ibmvnic_adapter *adapter)  in ibmvnic_send_crq_init()  argument
4757 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_send_crq_init()
4765 	netdev_dbg(adapter->netdev, "Sending CRQ init\n");  in ibmvnic_send_crq_init()
4768 		rc = ibmvnic_send_crq(adapter, &crq);  in ibmvnic_send_crq_init()
4790 static int vnic_client_data_len(struct ibmvnic_adapter *adapter)  in vnic_client_data_len()  argument
4801 	len += strlen(adapter->netdev->name) + 1;  in vnic_client_data_len()
4806 static void vnic_add_client_data(struct ibmvnic_adapter *adapter,  in vnic_add_client_data()  argument
4828 	len = strlen(adapter->netdev->name) + 1;  in vnic_add_client_data()
4830 	strscpy(vlcd->name, adapter->netdev->name, len);  in vnic_add_client_data()
4833 static int send_login(struct ibmvnic_adapter *adapter)  in send_login()  argument
4837 	struct device *dev = &adapter->vdev->dev;  in send_login()
4850 	if (!adapter->tx_scrq || !adapter->rx_scrq) {  in send_login()
4851 		netdev_err(adapter->netdev,  in send_login()
4856 	release_login_buffer(adapter);  in send_login()
4857 	release_login_rsp_buffer(adapter);  in send_login()
4859 	client_data_len = vnic_client_data_len(adapter);  in send_login()
4863 	    sizeof(u64) * (adapter->req_tx_queues + adapter->req_rx_queues) +  in send_login()
4878 			  sizeof(u64) * adapter->req_tx_queues +  in send_login()
4879 			  sizeof(u64) * adapter->req_rx_queues +  in send_login()
4880 			  sizeof(u64) * adapter->req_rx_queues +  in send_login()
4894 	adapter->login_buf = login_buffer;  in send_login()
4895 	adapter->login_buf_token = buffer_token;  in send_login()
4896 	adapter->login_buf_sz = buffer_size;  in send_login()
4897 	adapter->login_rsp_buf = login_rsp_buffer;  in send_login()
4898 	adapter->login_rsp_buf_token = rsp_buffer_token;  in send_login()
4899 	adapter->login_rsp_buf_sz = rsp_buffer_size;  in send_login()
4903 	login_buffer->num_txcomp_subcrqs = cpu_to_be32(adapter->req_tx_queues);  in send_login()
4906 	login_buffer->num_rxcomp_subcrqs = cpu_to_be32(adapter->req_rx_queues);  in send_login()
4909 			sizeof(u64) * adapter->req_tx_queues);  in send_login()
4917 				      sizeof(u64) * adapter->req_tx_queues);  in send_login()
4919 	for (i = 0; i < adapter->req_tx_queues; i++) {  in send_login()
4920 		if (adapter->tx_scrq[i]) {  in send_login()
4922 				cpu_to_be64(adapter->tx_scrq[i]->crq_num);  in send_login()
4926 	for (i = 0; i < adapter->req_rx_queues; i++) {  in send_login()
4927 		if (adapter->rx_scrq[i]) {  in send_login()
4929 				cpu_to_be64(adapter->rx_scrq[i]->crq_num);  in send_login()
4935 		((char *)rx_list_p + (sizeof(u64) * adapter->req_rx_queues));  in send_login()
4940 	vnic_add_client_data(adapter, vlcd);  in send_login()
4942 	netdev_dbg(adapter->netdev, "Login Buffer:\n");  in send_login()
4943 	for (i = 0; i < (adapter->login_buf_sz - 1) / 8 + 1; i++) {  in send_login()
4944 		netdev_dbg(adapter->netdev, "%016lx\n",  in send_login()
4945 			   ((unsigned long *)(adapter->login_buf))[i]);  in send_login()
4954 	adapter->login_pending = true;  in send_login()
4955 	rc = ibmvnic_send_crq(adapter, &crq);  in send_login()
4957 		adapter->login_pending = false;  in send_login()
4958 		netdev_err(adapter->netdev, "Failed to send login, rc=%d\n", rc);  in send_login()
4969 	adapter->login_rsp_buf = NULL;  in send_login()
4974 	adapter->login_buf = NULL;  in send_login()
4979 static int send_request_map(struct ibmvnic_adapter *adapter, dma_addr_t addr,  in send_request_map()  argument
4990 	return ibmvnic_send_crq(adapter, &crq);  in send_request_map()
4993 static int send_request_unmap(struct ibmvnic_adapter *adapter, u8 map_id)  in send_request_unmap()  argument
5001 	return ibmvnic_send_crq(adapter, &crq);  in send_request_unmap()
5004 static void send_query_map(struct ibmvnic_adapter *adapter)  in send_query_map()  argument
5011 	ibmvnic_send_crq(adapter, &crq);  in send_query_map()
5015 static void send_query_cap(struct ibmvnic_adapter *adapter)  in send_query_cap()  argument
5026 	atomic_set(&adapter->running_cap_crqs, cap_reqs);  in send_query_cap()
5033 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5037 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5041 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5045 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5049 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5053 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5058 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5063 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5068 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5073 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5077 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5081 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5085 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5089 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5093 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5097 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5101 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5105 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5109 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5113 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5117 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5122 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5127 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5132 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5137 	ibmvnic_send_crq(adapter, &crq);  in send_query_cap()
5146 static void send_query_ip_offload(struct ibmvnic_adapter *adapter)  in send_query_ip_offload()  argument
5149 	struct device *dev = &adapter->vdev->dev;  in send_query_ip_offload()
5152 	adapter->ip_offload_tok =  in send_query_ip_offload()
5154 			       &adapter->ip_offload_buf,  in send_query_ip_offload()
5158 	if (dma_mapping_error(dev, adapter->ip_offload_tok)) {  in send_query_ip_offload()
5169 	    cpu_to_be32(adapter->ip_offload_tok);  in send_query_ip_offload()
5171 	ibmvnic_send_crq(adapter, &crq);  in send_query_ip_offload()
5174 static void send_control_ip_offload(struct ibmvnic_adapter *adapter)  in send_control_ip_offload()  argument
5176 	struct ibmvnic_control_ip_offload_buffer *ctrl_buf = &adapter->ip_offload_ctrl;  in send_control_ip_offload()
5177 	struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf;  in send_control_ip_offload()
5178 	struct device *dev = &adapter->vdev->dev;  in send_control_ip_offload()
5182 	adapter->ip_offload_ctrl_tok =  in send_control_ip_offload()
5185 			       sizeof(adapter->ip_offload_ctrl),  in send_control_ip_offload()
5188 	if (dma_mapping_error(dev, adapter->ip_offload_ctrl_tok)) {  in send_control_ip_offload()
5193 	ctrl_buf->len = cpu_to_be32(sizeof(adapter->ip_offload_ctrl));  in send_control_ip_offload()
5208 	if (adapter->state != VNIC_PROBING) {  in send_control_ip_offload()
5209 		old_hw_features = adapter->netdev->hw_features;  in send_control_ip_offload()
5210 		adapter->netdev->hw_features = 0;  in send_control_ip_offload()
5213 	adapter->netdev->hw_features = NETIF_F_SG | NETIF_F_GSO | NETIF_F_GRO;  in send_control_ip_offload()
5216 		adapter->netdev->hw_features |= NETIF_F_IP_CSUM;  in send_control_ip_offload()
5219 		adapter->netdev->hw_features |= NETIF_F_IPV6_CSUM;  in send_control_ip_offload()
5221 	if ((adapter->netdev->features &  in send_control_ip_offload()
5223 		adapter->netdev->hw_features |= NETIF_F_RXCSUM;  in send_control_ip_offload()
5226 		adapter->netdev->hw_features |= NETIF_F_TSO;  in send_control_ip_offload()
5228 		adapter->netdev->hw_features |= NETIF_F_TSO6;  in send_control_ip_offload()
5230 	if (adapter->state == VNIC_PROBING) {  in send_control_ip_offload()
5231 		adapter->netdev->features |= adapter->netdev->hw_features;  in send_control_ip_offload()
5232 	} else if (old_hw_features != adapter->netdev->hw_features) {  in send_control_ip_offload()
5236 		adapter->netdev->features &= adapter->netdev->hw_features;  in send_control_ip_offload()
5238 		tmp = (old_hw_features ^ adapter->netdev->hw_features) &  in send_control_ip_offload()
5239 			adapter->netdev->hw_features;  in send_control_ip_offload()
5240 		adapter->netdev->features |=  in send_control_ip_offload()
5241 				tmp & adapter->netdev->wanted_features;  in send_control_ip_offload()
5248 	    cpu_to_be32(sizeof(adapter->ip_offload_ctrl));  in send_control_ip_offload()
5249 	crq.control_ip_offload.ioba = cpu_to_be32(adapter->ip_offload_ctrl_tok);  in send_control_ip_offload()
5250 	ibmvnic_send_crq(adapter, &crq);  in send_control_ip_offload()
5254 				struct ibmvnic_adapter *adapter)  in handle_vpd_size_rsp()  argument
5256 	struct device *dev = &adapter->vdev->dev;  in handle_vpd_size_rsp()
5261 		complete(&adapter->fw_done);  in handle_vpd_size_rsp()
5265 	adapter->vpd->len = be64_to_cpu(crq->get_vpd_size_rsp.len);  in handle_vpd_size_rsp()
5266 	complete(&adapter->fw_done);  in handle_vpd_size_rsp()
5270 			   struct ibmvnic_adapter *adapter)  in handle_vpd_rsp()  argument
5272 	struct device *dev = &adapter->vdev->dev;  in handle_vpd_rsp()
5276 	memset(adapter->fw_version, 0, 32);  in handle_vpd_rsp()
5278 	dma_unmap_single(dev, adapter->vpd->dma_addr, adapter->vpd->len,  in handle_vpd_rsp()
5290 	substr = strnstr(adapter->vpd->buff, "RM", adapter->vpd->len);  in handle_vpd_rsp()
5297 	if ((substr + 2) < (adapter->vpd->buff + adapter->vpd->len)) {  in handle_vpd_rsp()
5306 	    (adapter->vpd->buff + adapter->vpd->len)) {  in handle_vpd_rsp()
5307 		strscpy(adapter->fw_version, substr + 3,  in handle_vpd_rsp()
5308 			sizeof(adapter->fw_version));  in handle_vpd_rsp()
5314 	if (adapter->fw_version[0] == '\0')  in handle_vpd_rsp()
5315 		strscpy((char *)adapter->fw_version, "N/A", sizeof(adapter->fw_version));  in handle_vpd_rsp()
5316 	complete(&adapter->fw_done);  in handle_vpd_rsp()
5319 static void handle_query_ip_offload_rsp(struct ibmvnic_adapter *adapter)  in handle_query_ip_offload_rsp()  argument
5321 	struct device *dev = &adapter->vdev->dev;  in handle_query_ip_offload_rsp()
5322 	struct ibmvnic_query_ip_offload_buffer *buf = &adapter->ip_offload_buf;  in handle_query_ip_offload_rsp()
5325 	dma_unmap_single(dev, adapter->ip_offload_tok,  in handle_query_ip_offload_rsp()
5326 			 sizeof(adapter->ip_offload_buf), DMA_FROM_DEVICE);  in handle_query_ip_offload_rsp()
5328 	netdev_dbg(adapter->netdev, "Query IP Offload Buffer:\n");  in handle_query_ip_offload_rsp()
5329 	for (i = 0; i < (sizeof(adapter->ip_offload_buf) - 1) / 8 + 1; i++)  in handle_query_ip_offload_rsp()
5330 		netdev_dbg(adapter->netdev, "%016lx\n",  in handle_query_ip_offload_rsp()
5333 	netdev_dbg(adapter->netdev, "ipv4_chksum = %d\n", buf->ipv4_chksum);  in handle_query_ip_offload_rsp()
5334 	netdev_dbg(adapter->netdev, "ipv6_chksum = %d\n", buf->ipv6_chksum);  in handle_query_ip_offload_rsp()
5335 	netdev_dbg(adapter->netdev, "tcp_ipv4_chksum = %d\n",  in handle_query_ip_offload_rsp()
5337 	netdev_dbg(adapter->netdev, "tcp_ipv6_chksum = %d\n",  in handle_query_ip_offload_rsp()
5339 	netdev_dbg(adapter->netdev, "udp_ipv4_chksum = %d\n",  in handle_query_ip_offload_rsp()
5341 	netdev_dbg(adapter->netdev, "udp_ipv6_chksum = %d\n",  in handle_query_ip_offload_rsp()
5343 	netdev_dbg(adapter->netdev, "large_tx_ipv4 = %d\n",  in handle_query_ip_offload_rsp()
5345 	netdev_dbg(adapter->netdev, "large_tx_ipv6 = %d\n",  in handle_query_ip_offload_rsp()
5347 	netdev_dbg(adapter->netdev, "large_rx_ipv4 = %d\n",  in handle_query_ip_offload_rsp()
5349 	netdev_dbg(adapter->netdev, "large_rx_ipv6 = %d\n",  in handle_query_ip_offload_rsp()
5351 	netdev_dbg(adapter->netdev, "max_ipv4_hdr_sz = %d\n",  in handle_query_ip_offload_rsp()
5353 	netdev_dbg(adapter->netdev, "max_ipv6_hdr_sz = %d\n",  in handle_query_ip_offload_rsp()
5355 	netdev_dbg(adapter->netdev, "max_tcp_hdr_size = %d\n",  in handle_query_ip_offload_rsp()
5357 	netdev_dbg(adapter->netdev, "max_udp_hdr_size = %d\n",  in handle_query_ip_offload_rsp()
5359 	netdev_dbg(adapter->netdev, "max_large_tx_size = %d\n",  in handle_query_ip_offload_rsp()
5361 	netdev_dbg(adapter->netdev, "max_large_rx_size = %d\n",  in handle_query_ip_offload_rsp()
5363 	netdev_dbg(adapter->netdev, "ipv6_ext_hdr = %d\n",  in handle_query_ip_offload_rsp()
5365 	netdev_dbg(adapter->netdev, "tcp_pseudosum_req = %d\n",  in handle_query_ip_offload_rsp()
5367 	netdev_dbg(adapter->netdev, "num_ipv6_ext_hd = %d\n",  in handle_query_ip_offload_rsp()
5369 	netdev_dbg(adapter->netdev, "off_ipv6_ext_hd = %d\n",  in handle_query_ip_offload_rsp()
5372 	send_control_ip_offload(adapter);  in handle_query_ip_offload_rsp()
5398 				    struct ibmvnic_adapter *adapter)  in handle_error_indication()  argument
5400 	struct device *dev = &adapter->vdev->dev;  in handle_error_indication()
5412 		ibmvnic_reset(adapter, VNIC_RESET_FATAL);  in handle_error_indication()
5414 		ibmvnic_reset(adapter, VNIC_RESET_NON_FATAL);  in handle_error_indication()
5418 				 struct ibmvnic_adapter *adapter)  in handle_change_mac_rsp()  argument
5420 	struct net_device *netdev = adapter->netdev;  in handle_change_mac_rsp()
5421 	struct device *dev = &adapter->vdev->dev;  in handle_change_mac_rsp()
5433 	ether_addr_copy(adapter->mac_addr,  in handle_change_mac_rsp()
5436 	complete(&adapter->fw_done);  in handle_change_mac_rsp()
5441 				   struct ibmvnic_adapter *adapter)  in handle_request_cap_rsp()  argument
5443 	struct device *dev = &adapter->vdev->dev;  in handle_request_cap_rsp()
5447 	atomic_dec(&adapter->running_cap_crqs);  in handle_request_cap_rsp()
5448 	netdev_dbg(adapter->netdev, "Outstanding request-caps: %d\n",  in handle_request_cap_rsp()
5449 		   atomic_read(&adapter->running_cap_crqs));  in handle_request_cap_rsp()
5452 		req_value = &adapter->req_tx_queues;  in handle_request_cap_rsp()
5456 		req_value = &adapter->req_rx_queues;  in handle_request_cap_rsp()
5460 		req_value = &adapter->req_rx_add_queues;  in handle_request_cap_rsp()
5464 		req_value = &adapter->req_tx_entries_per_subcrq;  in handle_request_cap_rsp()
5468 		req_value = &adapter->req_rx_add_entries_per_subcrq;  in handle_request_cap_rsp()
5472 		req_value = &adapter->req_mtu;  in handle_request_cap_rsp()
5476 		req_value = &adapter->promisc;  in handle_request_cap_rsp()
5498 			*req_value = adapter->fallback.mtu;  in handle_request_cap_rsp()
5504 		send_request_cap(adapter, 1);  in handle_request_cap_rsp()
5513 	if (atomic_read(&adapter->running_cap_crqs) == 0)  in handle_request_cap_rsp()
5514 		send_query_ip_offload(adapter);  in handle_request_cap_rsp()
5518 			    struct ibmvnic_adapter *adapter)  in handle_login_rsp()  argument
5520 	struct device *dev = &adapter->vdev->dev;  in handle_login_rsp()
5521 	struct net_device *netdev = adapter->netdev;  in handle_login_rsp()
5522 	struct ibmvnic_login_rsp_buffer *login_rsp = adapter->login_rsp_buf;  in handle_login_rsp()
5523 	struct ibmvnic_login_buffer *login = adapter->login_buf;  in handle_login_rsp()
5535 	if (!adapter->login_pending) {  in handle_login_rsp()
5539 	adapter->login_pending = false;  in handle_login_rsp()
5546 		adapter->init_done_rc = login_rsp_crq->generic.rc.code;  in handle_login_rsp()
5547 		complete(&adapter->init_done);  in handle_login_rsp()
5551 	if (adapter->failover_pending) {  in handle_login_rsp()
5552 		adapter->init_done_rc = -EAGAIN;  in handle_login_rsp()
5554 		complete(&adapter->init_done);  in handle_login_rsp()
5559 	netdev->mtu = adapter->req_mtu - ETH_HLEN;  in handle_login_rsp()
5561 	netdev_dbg(adapter->netdev, "Login Response Buffer:\n");  in handle_login_rsp()
5562 	for (i = 0; i < (adapter->login_rsp_buf_sz - 1) / 8 + 1; i++) {  in handle_login_rsp()
5563 		netdev_dbg(adapter->netdev, "%016lx\n",  in handle_login_rsp()
5564 			   ((unsigned long *)(adapter->login_rsp_buf))[i]);  in handle_login_rsp()
5570 	     adapter->req_rx_add_queues !=  in handle_login_rsp()
5573 		ibmvnic_reset(adapter, VNIC_RESET_FATAL);  in handle_login_rsp()
5589 		ibmvnic_reset(adapter, VNIC_RESET_FATAL);  in handle_login_rsp()
5593 	size_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +  in handle_login_rsp()
5594 		be32_to_cpu(adapter->login_rsp_buf->off_rxadd_buff_size));  in handle_login_rsp()
5598 	adapter->cur_rx_buf_sz = be64_to_cpu(size_array[0]);  in handle_login_rsp()
5600 	num_tx_pools = be32_to_cpu(adapter->login_rsp_buf->num_txsubm_subcrqs);  in handle_login_rsp()
5601 	num_rx_pools = be32_to_cpu(adapter->login_rsp_buf->num_rxadd_subcrqs);  in handle_login_rsp()
5603 	tx_handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +  in handle_login_rsp()
5604 				  be32_to_cpu(adapter->login_rsp_buf->off_txsubm_subcrqs));  in handle_login_rsp()
5605 	rx_handle_array = (u64 *)((u8 *)(adapter->login_rsp_buf) +  in handle_login_rsp()
5606 				  be32_to_cpu(adapter->login_rsp_buf->off_rxadd_subcrqs));  in handle_login_rsp()
5609 		adapter->tx_scrq[i]->handle = tx_handle_array[i];  in handle_login_rsp()
5612 		adapter->rx_scrq[i]->handle = rx_handle_array[i];  in handle_login_rsp()
5614 	adapter->num_active_tx_scrqs = num_tx_pools;  in handle_login_rsp()
5615 	adapter->num_active_rx_scrqs = num_rx_pools;  in handle_login_rsp()
5616 	release_login_rsp_buffer(adapter);  in handle_login_rsp()
5617 	release_login_buffer(adapter);  in handle_login_rsp()
5618 	complete(&adapter->init_done);  in handle_login_rsp()
5624 				     struct ibmvnic_adapter *adapter)  in handle_request_unmap_rsp()  argument
5626 	struct device *dev = &adapter->vdev->dev;  in handle_request_unmap_rsp()
5635 				 struct ibmvnic_adapter *adapter)  in handle_query_map_rsp()  argument
5637 	struct net_device *netdev = adapter->netdev;  in handle_query_map_rsp()
5638 	struct device *dev = &adapter->vdev->dev;  in handle_query_map_rsp()
5653 				 struct ibmvnic_adapter *adapter)  in handle_query_cap_rsp()  argument
5655 	struct net_device *netdev = adapter->netdev;  in handle_query_cap_rsp()
5656 	struct device *dev = &adapter->vdev->dev;  in handle_query_cap_rsp()
5659 	atomic_dec(&adapter->running_cap_crqs);  in handle_query_cap_rsp()
5661 		   atomic_read(&adapter->running_cap_crqs));  in handle_query_cap_rsp()
5670 		adapter->min_tx_queues =  in handle_query_cap_rsp()
5673 			   adapter->min_tx_queues);  in handle_query_cap_rsp()
5676 		adapter->min_rx_queues =  in handle_query_cap_rsp()
5679 			   adapter->min_rx_queues);  in handle_query_cap_rsp()
5682 		adapter->min_rx_add_queues =  in handle_query_cap_rsp()
5685 			   adapter->min_rx_add_queues);  in handle_query_cap_rsp()
5688 		adapter->max_tx_queues =  in handle_query_cap_rsp()
5691 			   adapter->max_tx_queues);  in handle_query_cap_rsp()
5694 		adapter->max_rx_queues =  in handle_query_cap_rsp()
5697 			   adapter->max_rx_queues);  in handle_query_cap_rsp()
5700 		adapter->max_rx_add_queues =  in handle_query_cap_rsp()
5703 			   adapter->max_rx_add_queues);  in handle_query_cap_rsp()
5706 		adapter->min_tx_entries_per_subcrq =  in handle_query_cap_rsp()
5709 			   adapter->min_tx_entries_per_subcrq);  in handle_query_cap_rsp()
5712 		adapter->min_rx_add_entries_per_subcrq =  in handle_query_cap_rsp()
5715 			   adapter->min_rx_add_entries_per_subcrq);  in handle_query_cap_rsp()
5718 		adapter->max_tx_entries_per_subcrq =  in handle_query_cap_rsp()
5721 			   adapter->max_tx_entries_per_subcrq);  in handle_query_cap_rsp()
5724 		adapter->max_rx_add_entries_per_subcrq =  in handle_query_cap_rsp()
5727 			   adapter->max_rx_add_entries_per_subcrq);  in handle_query_cap_rsp()
5730 		adapter->tcp_ip_offload =  in handle_query_cap_rsp()
5733 			   adapter->tcp_ip_offload);  in handle_query_cap_rsp()
5736 		adapter->promisc_supported =  in handle_query_cap_rsp()
5739 			   adapter->promisc_supported);  in handle_query_cap_rsp()
5742 		adapter->min_mtu = be64_to_cpu(crq->query_capability.number);  in handle_query_cap_rsp()
5743 		netdev->min_mtu = adapter->min_mtu - ETH_HLEN;  in handle_query_cap_rsp()
5744 		netdev_dbg(netdev, "min_mtu = %lld\n", adapter->min_mtu);  in handle_query_cap_rsp()
5747 		adapter->max_mtu = be64_to_cpu(crq->query_capability.number);  in handle_query_cap_rsp()
5748 		netdev->max_mtu = adapter->max_mtu - ETH_HLEN;  in handle_query_cap_rsp()
5749 		netdev_dbg(netdev, "max_mtu = %lld\n", adapter->max_mtu);  in handle_query_cap_rsp()
5752 		adapter->max_multicast_filters =  in handle_query_cap_rsp()
5755 			   adapter->max_multicast_filters);  in handle_query_cap_rsp()
5758 		adapter->vlan_header_insertion =  in handle_query_cap_rsp()
5760 		if (adapter->vlan_header_insertion)  in handle_query_cap_rsp()
5763 			   adapter->vlan_header_insertion);  in handle_query_cap_rsp()
5766 		adapter->rx_vlan_header_insertion =  in handle_query_cap_rsp()
5769 			   adapter->rx_vlan_header_insertion);  in handle_query_cap_rsp()
5772 		adapter->max_tx_sg_entries =  in handle_query_cap_rsp()
5775 			   adapter->max_tx_sg_entries);  in handle_query_cap_rsp()
5778 		adapter->rx_sg_supported =  in handle_query_cap_rsp()
5781 			   adapter->rx_sg_supported);  in handle_query_cap_rsp()
5784 		adapter->opt_tx_comp_sub_queues =  in handle_query_cap_rsp()
5787 			   adapter->opt_tx_comp_sub_queues);  in handle_query_cap_rsp()
5790 		adapter->opt_rx_comp_queues =  in handle_query_cap_rsp()
5793 			   adapter->opt_rx_comp_queues);  in handle_query_cap_rsp()
5796 		adapter->opt_rx_bufadd_q_per_rx_comp_q =  in handle_query_cap_rsp()
5799 			   adapter->opt_rx_bufadd_q_per_rx_comp_q);  in handle_query_cap_rsp()
5802 		adapter->opt_tx_entries_per_subcrq =  in handle_query_cap_rsp()
5805 			   adapter->opt_tx_entries_per_subcrq);  in handle_query_cap_rsp()
5808 		adapter->opt_rxba_entries_per_subcrq =  in handle_query_cap_rsp()
5811 			   adapter->opt_rxba_entries_per_subcrq);  in handle_query_cap_rsp()
5814 		adapter->tx_rx_desc_req = crq->query_capability.number;  in handle_query_cap_rsp()
5816 			   adapter->tx_rx_desc_req);  in handle_query_cap_rsp()
5825 	if (atomic_read(&adapter->running_cap_crqs) == 0)  in handle_query_cap_rsp()
5826 		send_request_cap(adapter, 0);  in handle_query_cap_rsp()
5829 static int send_query_phys_parms(struct ibmvnic_adapter *adapter)  in send_query_phys_parms()  argument
5838 	mutex_lock(&adapter->fw_lock);  in send_query_phys_parms()
5839 	adapter->fw_done_rc = 0;  in send_query_phys_parms()
5840 	reinit_completion(&adapter->fw_done);  in send_query_phys_parms()
5842 	rc = ibmvnic_send_crq(adapter, &crq);  in send_query_phys_parms()
5844 		mutex_unlock(&adapter->fw_lock);  in send_query_phys_parms()
5848 	rc = ibmvnic_wait_for_completion(adapter, &adapter->fw_done, 10000);  in send_query_phys_parms()
5850 		mutex_unlock(&adapter->fw_lock);  in send_query_phys_parms()
5854 	mutex_unlock(&adapter->fw_lock);  in send_query_phys_parms()
5855 	return adapter->fw_done_rc ? -EIO : 0;  in send_query_phys_parms()
5859 				       struct ibmvnic_adapter *adapter)  in handle_query_phys_parms_rsp()  argument
5861 	struct net_device *netdev = adapter->netdev;  in handle_query_phys_parms_rsp()
5872 		adapter->speed = SPEED_10;  in handle_query_phys_parms_rsp()
5875 		adapter->speed = SPEED_100;  in handle_query_phys_parms_rsp()
5878 		adapter->speed = SPEED_1000;  in handle_query_phys_parms_rsp()
5881 		adapter->speed = SPEED_10000;  in handle_query_phys_parms_rsp()
5884 		adapter->speed = SPEED_25000;  in handle_query_phys_parms_rsp()
5887 		adapter->speed = SPEED_40000;  in handle_query_phys_parms_rsp()
5890 		adapter->speed = SPEED_50000;  in handle_query_phys_parms_rsp()
5893 		adapter->speed = SPEED_100000;  in handle_query_phys_parms_rsp()
5896 		adapter->speed = SPEED_200000;  in handle_query_phys_parms_rsp()
5901 		adapter->speed = SPEED_UNKNOWN;  in handle_query_phys_parms_rsp()
5904 		adapter->duplex = DUPLEX_FULL;  in handle_query_phys_parms_rsp()
5906 		adapter->duplex = DUPLEX_HALF;  in handle_query_phys_parms_rsp()
5908 		adapter->duplex = DUPLEX_UNKNOWN;  in handle_query_phys_parms_rsp()
5914 			       struct ibmvnic_adapter *adapter)  in ibmvnic_handle_crq()  argument
5917 	struct net_device *netdev = adapter->netdev;  in ibmvnic_handle_crq()
5918 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_handle_crq()
5930 			adapter->from_passive_init = true;  in ibmvnic_handle_crq()
5934 			adapter->login_pending = false;  in ibmvnic_handle_crq()
5936 			if (adapter->state == VNIC_DOWN)  in ibmvnic_handle_crq()
5937 				rc = ibmvnic_reset(adapter, VNIC_RESET_PASSIVE_INIT);  in ibmvnic_handle_crq()
5939 				rc = ibmvnic_reset(adapter, VNIC_RESET_FAILOVER);  in ibmvnic_handle_crq()
5954 				adapter->failover_pending = false;  in ibmvnic_handle_crq()
5957 			if (!completion_done(&adapter->init_done)) {  in ibmvnic_handle_crq()
5958 				if (!adapter->init_done_rc)  in ibmvnic_handle_crq()
5959 					adapter->init_done_rc = -EAGAIN;  in ibmvnic_handle_crq()
5960 				complete(&adapter->init_done);  in ibmvnic_handle_crq()
5966 			adapter->crq.active = true;  in ibmvnic_handle_crq()
5967 			send_version_xchg(adapter);  in ibmvnic_handle_crq()
5975 		adapter->crq.active = false;  in ibmvnic_handle_crq()
5979 		if (!completion_done(&adapter->fw_done)) {  in ibmvnic_handle_crq()
5980 			adapter->fw_done_rc = -EIO;  in ibmvnic_handle_crq()
5981 			complete(&adapter->fw_done);  in ibmvnic_handle_crq()
5985 		if (!completion_done(&adapter->init_done)) {  in ibmvnic_handle_crq()
5986 			adapter->init_done_rc = -EAGAIN;  in ibmvnic_handle_crq()
5987 			complete(&adapter->init_done);  in ibmvnic_handle_crq()
5990 		if (!completion_done(&adapter->stats_done))  in ibmvnic_handle_crq()
5991 			complete(&adapter->stats_done);  in ibmvnic_handle_crq()
5992 		if (test_bit(0, &adapter->resetting))  in ibmvnic_handle_crq()
5993 			adapter->force_reset_recovery = true;  in ibmvnic_handle_crq()
5996 			ibmvnic_reset(adapter, VNIC_RESET_MOBILITY);  in ibmvnic_handle_crq()
5999 			adapter->failover_pending = true;  in ibmvnic_handle_crq()
6004 			ibmvnic_reset(adapter, VNIC_RESET_FATAL);  in ibmvnic_handle_crq()
6026 		send_query_cap(adapter);  in ibmvnic_handle_crq()
6029 		handle_query_cap_rsp(crq, adapter);  in ibmvnic_handle_crq()
6032 		handle_query_map_rsp(crq, adapter);  in ibmvnic_handle_crq()
6035 		adapter->fw_done_rc = crq->request_map_rsp.rc.code;  in ibmvnic_handle_crq()
6036 		complete(&adapter->fw_done);  in ibmvnic_handle_crq()
6039 		handle_request_unmap_rsp(crq, adapter);  in ibmvnic_handle_crq()
6042 		handle_request_cap_rsp(crq, adapter);  in ibmvnic_handle_crq()
6046 		handle_login_rsp(crq, adapter);  in ibmvnic_handle_crq()
6053 		adapter->logical_link_state =  in ibmvnic_handle_crq()
6055 		adapter->init_done_rc = crq->logical_link_state_rsp.rc.code;  in ibmvnic_handle_crq()
6056 		complete(&adapter->init_done);  in ibmvnic_handle_crq()
6060 		adapter->phys_link_state =  in ibmvnic_handle_crq()
6062 		adapter->logical_link_state =  in ibmvnic_handle_crq()
6064 		if (adapter->phys_link_state && adapter->logical_link_state)  in ibmvnic_handle_crq()
6071 		adapter->fw_done_rc = handle_change_mac_rsp(crq, adapter);  in ibmvnic_handle_crq()
6075 		handle_error_indication(crq, adapter);  in ibmvnic_handle_crq()
6079 		complete(&adapter->stats_done);  in ibmvnic_handle_crq()
6083 		handle_query_ip_offload_rsp(adapter);  in ibmvnic_handle_crq()
6090 		dma_unmap_single(dev, adapter->ip_offload_ctrl_tok,  in ibmvnic_handle_crq()
6091 				 sizeof(adapter->ip_offload_ctrl),  in ibmvnic_handle_crq()
6093 		complete(&adapter->init_done);  in ibmvnic_handle_crq()
6097 		complete(&adapter->fw_done);  in ibmvnic_handle_crq()
6100 		handle_vpd_size_rsp(crq, adapter);  in ibmvnic_handle_crq()
6103 		handle_vpd_rsp(crq, adapter);  in ibmvnic_handle_crq()
6106 		adapter->fw_done_rc = handle_query_phys_parms_rsp(crq, adapter);  in ibmvnic_handle_crq()
6107 		complete(&adapter->fw_done);  in ibmvnic_handle_crq()
6117 	struct ibmvnic_adapter *adapter = instance;  in ibmvnic_interrupt()  local
6119 	tasklet_schedule(&adapter->tasklet);  in ibmvnic_interrupt()
6125 	struct ibmvnic_adapter *adapter = from_tasklet(adapter, t, tasklet);  in ibmvnic_tasklet()  local
6126 	struct ibmvnic_crq_queue *queue = &adapter->crq;  in ibmvnic_tasklet()
6133 	while ((crq = ibmvnic_next_crq(adapter)) != NULL) {  in ibmvnic_tasklet()
6140 		ibmvnic_handle_crq(crq, adapter);  in ibmvnic_tasklet()
6147 static int ibmvnic_reenable_crq_queue(struct ibmvnic_adapter *adapter)  in ibmvnic_reenable_crq_queue()  argument
6149 	struct vio_dev *vdev = adapter->vdev;  in ibmvnic_reenable_crq_queue()
6162 static int ibmvnic_reset_crq(struct ibmvnic_adapter *adapter)  in ibmvnic_reset_crq()  argument
6164 	struct ibmvnic_crq_queue *crq = &adapter->crq;  in ibmvnic_reset_crq()
6165 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_reset_crq()
6166 	struct vio_dev *vdev = adapter->vdev;  in ibmvnic_reset_crq()
6195 static void release_crq_queue(struct ibmvnic_adapter *adapter)  in release_crq_queue()  argument
6197 	struct ibmvnic_crq_queue *crq = &adapter->crq;  in release_crq_queue()
6198 	struct vio_dev *vdev = adapter->vdev;  in release_crq_queue()
6204 	netdev_dbg(adapter->netdev, "Releasing CRQ\n");  in release_crq_queue()
6205 	free_irq(vdev->irq, adapter);  in release_crq_queue()
6206 	tasklet_kill(&adapter->tasklet);  in release_crq_queue()
6218 static int init_crq_queue(struct ibmvnic_adapter *adapter)  in init_crq_queue()  argument
6220 	struct ibmvnic_crq_queue *crq = &adapter->crq;  in init_crq_queue()
6221 	struct device *dev = &adapter->vdev->dev;  in init_crq_queue()
6222 	struct vio_dev *vdev = adapter->vdev;  in init_crq_queue()
6245 		rc = ibmvnic_reset_crq(adapter);  in init_crq_queue()
6257 	tasklet_setup(&adapter->tasklet, (void *)ibmvnic_tasklet);  in init_crq_queue()
6259 	netdev_dbg(adapter->netdev, "registering irq 0x%x\n", vdev->irq);  in init_crq_queue()
6261 		 adapter->vdev->unit_address);  in init_crq_queue()
6262 	rc = request_irq(vdev->irq, ibmvnic_interrupt, 0, crq->name, adapter);  in init_crq_queue()
6279 	tasklet_schedule(&adapter->tasklet);  in init_crq_queue()
6284 	tasklet_kill(&adapter->tasklet);  in init_crq_queue()
6296 static int ibmvnic_reset_init(struct ibmvnic_adapter *adapter, bool reset)  in ibmvnic_reset_init()  argument
6298 	struct device *dev = &adapter->vdev->dev;  in ibmvnic_reset_init()
6300 	u64 old_num_rx_queues = adapter->req_rx_queues;  in ibmvnic_reset_init()
6301 	u64 old_num_tx_queues = adapter->req_tx_queues;  in ibmvnic_reset_init()
6304 	adapter->from_passive_init = false;  in ibmvnic_reset_init()
6306 	rc = ibmvnic_send_crq_init(adapter);  in ibmvnic_reset_init()
6312 	if (!wait_for_completion_timeout(&adapter->init_done, timeout)) {  in ibmvnic_reset_init()
6317 	if (adapter->init_done_rc) {  in ibmvnic_reset_init()
6318 		release_crq_queue(adapter);  in ibmvnic_reset_init()
6319 		dev_err(dev, "CRQ-init failed, %d\n", adapter->init_done_rc);  in ibmvnic_reset_init()
6320 		return adapter->init_done_rc;  in ibmvnic_reset_init()
6323 	if (adapter->from_passive_init) {  in ibmvnic_reset_init()
6324 		adapter->state = VNIC_OPEN;  in ibmvnic_reset_init()
6325 		adapter->from_passive_init = false;  in ibmvnic_reset_init()
6331 	    test_bit(0, &adapter->resetting) && !adapter->wait_for_reset &&  in ibmvnic_reset_init()
6332 	    adapter->reset_reason != VNIC_RESET_MOBILITY) {  in ibmvnic_reset_init()
6333 		if (adapter->req_rx_queues != old_num_rx_queues ||  in ibmvnic_reset_init()
6334 		    adapter->req_tx_queues != old_num_tx_queues) {  in ibmvnic_reset_init()
6335 			release_sub_crqs(adapter, 0);  in ibmvnic_reset_init()
6336 			rc = init_sub_crqs(adapter);  in ibmvnic_reset_init()
6345 			clean_tx_pools(adapter);  in ibmvnic_reset_init()
6347 			rc = reset_sub_crq_queues(adapter);  in ibmvnic_reset_init()
6350 		rc = init_sub_crqs(adapter);  in ibmvnic_reset_init()
6355 		release_crq_queue(adapter);  in ibmvnic_reset_init()
6359 	rc = init_sub_crq_irqs(adapter);  in ibmvnic_reset_init()
6362 		release_crq_queue(adapter);  in ibmvnic_reset_init()
6372 	struct ibmvnic_adapter *adapter;  in ibmvnic_probe()  local
6396 	adapter = netdev_priv(netdev);  in ibmvnic_probe()
6397 	adapter->state = VNIC_PROBING;  in ibmvnic_probe()
6399 	adapter->vdev = dev;  in ibmvnic_probe()
6400 	adapter->netdev = netdev;  in ibmvnic_probe()
6401 	adapter->login_pending = false;  in ibmvnic_probe()
6402 	memset(&adapter->map_ids, 0, sizeof(adapter->map_ids));  in ibmvnic_probe()
6404 	bitmap_set(adapter->map_ids, 0, 1);  in ibmvnic_probe()
6406 	ether_addr_copy(adapter->mac_addr, mac_addr_p);  in ibmvnic_probe()
6407 	eth_hw_addr_set(netdev, adapter->mac_addr);  in ibmvnic_probe()
6413 	INIT_WORK(&adapter->ibmvnic_reset, __ibmvnic_reset);  in ibmvnic_probe()
6414 	INIT_DELAYED_WORK(&adapter->ibmvnic_delayed_reset,  in ibmvnic_probe()
6416 	INIT_LIST_HEAD(&adapter->rwi_list);  in ibmvnic_probe()
6417 	spin_lock_init(&adapter->rwi_lock);  in ibmvnic_probe()
6418 	spin_lock_init(&adapter->state_lock);  in ibmvnic_probe()
6419 	mutex_init(&adapter->fw_lock);  in ibmvnic_probe()
6420 	init_completion(&adapter->probe_done);  in ibmvnic_probe()
6421 	init_completion(&adapter->init_done);  in ibmvnic_probe()
6422 	init_completion(&adapter->fw_done);  in ibmvnic_probe()
6423 	init_completion(&adapter->reset_done);  in ibmvnic_probe()
6424 	init_completion(&adapter->stats_done);  in ibmvnic_probe()
6425 	clear_bit(0, &adapter->resetting);  in ibmvnic_probe()
6426 	adapter->prev_rx_buf_sz = 0;  in ibmvnic_probe()
6427 	adapter->prev_mtu = 0;  in ibmvnic_probe()
6431 		reinit_init_done(adapter);  in ibmvnic_probe()
6436 		adapter->failover_pending = false;  in ibmvnic_probe()
6442 		release_crq_queue(adapter);  in ibmvnic_probe()
6454 		spin_lock_irqsave(&adapter->rwi_lock, flags);  in ibmvnic_probe()
6455 		flush_reset_queue(adapter);  in ibmvnic_probe()
6456 		spin_unlock_irqrestore(&adapter->rwi_lock, flags);  in ibmvnic_probe()
6458 		rc = init_crq_queue(adapter);  in ibmvnic_probe()
6465 		rc = ibmvnic_reset_init(adapter, false);  in ibmvnic_probe()
6476 	rc = init_stats_buffers(adapter);  in ibmvnic_probe()
6480 	rc = init_stats_token(adapter);  in ibmvnic_probe()
6491 		adapter->state = VNIC_PROBED;  in ibmvnic_probe()
6492 		netdev->mtu = adapter->req_mtu - ETH_HLEN;  in ibmvnic_probe()
6493 		netdev->min_mtu = adapter->min_mtu - ETH_HLEN;  in ibmvnic_probe()
6494 		netdev->max_mtu = adapter->max_mtu - ETH_HLEN;  in ibmvnic_probe()
6496 		adapter->state = VNIC_DOWN;  in ibmvnic_probe()
6499 	adapter->wait_for_reset = false;  in ibmvnic_probe()
6500 	adapter->last_reset_time = jiffies;  in ibmvnic_probe()
6509 	rc = ibmvnic_cpu_notif_add(adapter);  in ibmvnic_probe()
6515 	complete(&adapter->probe_done);  in ibmvnic_probe()
6526 	release_stats_token(adapter);  in ibmvnic_probe()
6529 	release_stats_buffers(adapter);  in ibmvnic_probe()
6532 	release_sub_crqs(adapter, 1);  in ibmvnic_probe()
6533 	release_crq_queue(adapter);  in ibmvnic_probe()
6538 	adapter->state = VNIC_REMOVING;  in ibmvnic_probe()
6539 	complete(&adapter->probe_done);  in ibmvnic_probe()
6540 	flush_work(&adapter->ibmvnic_reset);  in ibmvnic_probe()
6541 	flush_delayed_work(&adapter->ibmvnic_delayed_reset);  in ibmvnic_probe()
6543 	flush_reset_queue(adapter);  in ibmvnic_probe()
6545 	mutex_destroy(&adapter->fw_lock);  in ibmvnic_probe()
6554 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_remove()  local
6557 	spin_lock_irqsave(&adapter->state_lock, flags);  in ibmvnic_remove()
6566 	spin_lock(&adapter->rwi_lock);  in ibmvnic_remove()
6567 	adapter->state = VNIC_REMOVING;  in ibmvnic_remove()
6568 	spin_unlock(&adapter->rwi_lock);  in ibmvnic_remove()
6570 	spin_unlock_irqrestore(&adapter->state_lock, flags);  in ibmvnic_remove()
6572 	ibmvnic_cpu_notif_remove(adapter);  in ibmvnic_remove()
6574 	flush_work(&adapter->ibmvnic_reset);  in ibmvnic_remove()
6575 	flush_delayed_work(&adapter->ibmvnic_delayed_reset);  in ibmvnic_remove()
6580 	release_resources(adapter);  in ibmvnic_remove()
6581 	release_rx_pools(adapter);  in ibmvnic_remove()
6582 	release_tx_pools(adapter);  in ibmvnic_remove()
6583 	release_sub_crqs(adapter, 1);  in ibmvnic_remove()
6584 	release_crq_queue(adapter);  in ibmvnic_remove()
6586 	release_stats_token(adapter);  in ibmvnic_remove()
6587 	release_stats_buffers(adapter);  in ibmvnic_remove()
6589 	adapter->state = VNIC_REMOVED;  in ibmvnic_remove()
6592 	mutex_destroy(&adapter->fw_lock);  in ibmvnic_remove()
6602 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in failover_store()  local
6610 	rc = plpar_hcall(H_VIOCTL, retbuf, adapter->vdev->unit_address,  in failover_store()
6621 	rc = plpar_hcall_norets(H_VIOCTL, adapter->vdev->unit_address,  in failover_store()
6634 	ibmvnic_reset(adapter, VNIC_RESET_FAILOVER);  in failover_store()
6643 	struct ibmvnic_adapter *adapter;  in ibmvnic_get_desired_dma()  local
6654 	adapter = netdev_priv(netdev);  in ibmvnic_get_desired_dma()
6659 	for (i = 0; i < adapter->req_tx_queues + adapter->req_rx_queues; i++)  in ibmvnic_get_desired_dma()
6662 	for (i = 0; i < adapter->num_active_rx_pools; i++)  in ibmvnic_get_desired_dma()
6663 		ret += adapter->rx_pool[i].size *  in ibmvnic_get_desired_dma()
6664 		    IOMMU_PAGE_ALIGN(adapter->rx_pool[i].buff_size, tbl);  in ibmvnic_get_desired_dma()
6672 	struct ibmvnic_adapter *adapter = netdev_priv(netdev);  in ibmvnic_resume()  local
6674 	if (adapter->state != VNIC_OPEN)  in ibmvnic_resume()
6677 	tasklet_schedule(&adapter->tasklet);  in ibmvnic_resume()