Lines Matching +full:reserved +full:- +full:cpu +full:- +full:vectors
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2019 Intel Corporation. */
18 "Copyright(c) 2013 - 2019 Intel Corporation.";
27 * fm10k_init_module - Driver Registration Routine
43 return -ENOMEM; in fm10k_init_module()
58 * fm10k_exit_module - Driver Exit Cleanup Routine
77 struct page *page = bi->page; in fm10k_alloc_mapped_page()
87 rx_ring->rx_stats.alloc_failed++; in fm10k_alloc_mapped_page()
92 dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); in fm10k_alloc_mapped_page()
97 if (dma_mapping_error(rx_ring->dev, dma)) { in fm10k_alloc_mapped_page()
100 rx_ring->rx_stats.alloc_failed++; in fm10k_alloc_mapped_page()
104 bi->dma = dma; in fm10k_alloc_mapped_page()
105 bi->page = page; in fm10k_alloc_mapped_page()
106 bi->page_offset = 0; in fm10k_alloc_mapped_page()
112 * fm10k_alloc_rx_buffers - Replace used receive buffers
120 u16 i = rx_ring->next_to_use; in fm10k_alloc_rx_buffers()
127 bi = &rx_ring->rx_buffer[i]; in fm10k_alloc_rx_buffers()
128 i -= rx_ring->count; in fm10k_alloc_rx_buffers()
135 * because each write-back erases this info. in fm10k_alloc_rx_buffers()
137 rx_desc->q.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); in fm10k_alloc_rx_buffers()
144 bi = rx_ring->rx_buffer; in fm10k_alloc_rx_buffers()
145 i -= rx_ring->count; in fm10k_alloc_rx_buffers()
149 rx_desc->d.staterr = 0; in fm10k_alloc_rx_buffers()
151 cleaned_count--; in fm10k_alloc_rx_buffers()
154 i += rx_ring->count; in fm10k_alloc_rx_buffers()
156 if (rx_ring->next_to_use != i) { in fm10k_alloc_rx_buffers()
158 rx_ring->next_to_use = i; in fm10k_alloc_rx_buffers()
161 rx_ring->next_to_alloc = i; in fm10k_alloc_rx_buffers()
165 * applicable for weak-ordered memory model archs, in fm10k_alloc_rx_buffers()
166 * such as IA-64). in fm10k_alloc_rx_buffers()
171 writel(i, rx_ring->tail); in fm10k_alloc_rx_buffers()
176 * fm10k_reuse_rx_page - page flip buffer and store it back on the ring
186 u16 nta = rx_ring->next_to_alloc; in fm10k_reuse_rx_page()
188 new_buff = &rx_ring->rx_buffer[nta]; in fm10k_reuse_rx_page()
192 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; in fm10k_reuse_rx_page()
198 dma_sync_single_range_for_device(rx_ring->dev, old_buff->dma, in fm10k_reuse_rx_page()
199 old_buff->page_offset, in fm10k_reuse_rx_page()
208 /* avoid re-using remote and pfmemalloc pages */ in fm10k_can_reuse_rx_page()
218 rx_buffer->page_offset ^= FM10K_RX_BUFSZ; in fm10k_can_reuse_rx_page()
221 rx_buffer->page_offset += truesize; in fm10k_can_reuse_rx_page()
223 if (rx_buffer->page_offset > (PAGE_SIZE - FM10K_RX_BUFSZ)) in fm10k_can_reuse_rx_page()
236 * fm10k_add_rx_frag - Add contents of Rx buffer to sk_buff
242 * This function will add the data contained in rx_buffer->page to the skb.
255 struct page *page = rx_buffer->page; in fm10k_add_rx_frag()
256 unsigned char *va = page_address(page) + rx_buffer->page_offset; in fm10k_add_rx_frag()
270 /* page is reusable, we can reuse buffer as-is */ in fm10k_add_rx_frag()
280 * 60 bytes if the skb->len is less than 60 for skb_pad. in fm10k_add_rx_frag()
282 pull_len = eth_get_headlen(skb->dev, va, FM10K_RX_HDR_LEN); in fm10k_add_rx_frag()
289 size -= pull_len; in fm10k_add_rx_frag()
292 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, in fm10k_add_rx_frag()
302 unsigned int size = le16_to_cpu(rx_desc->w.length); in fm10k_fetch_rx_buffer()
306 rx_buffer = &rx_ring->rx_buffer[rx_ring->next_to_clean]; in fm10k_fetch_rx_buffer()
307 page = rx_buffer->page; in fm10k_fetch_rx_buffer()
312 rx_buffer->page_offset; in fm10k_fetch_rx_buffer()
318 skb = napi_alloc_skb(&rx_ring->q_vector->napi, in fm10k_fetch_rx_buffer()
321 rx_ring->rx_stats.alloc_failed++; in fm10k_fetch_rx_buffer()
325 /* we will be copying header into skb->data in in fm10k_fetch_rx_buffer()
329 prefetchw(skb->data); in fm10k_fetch_rx_buffer()
332 /* we are reusing so sync this buffer for CPU use */ in fm10k_fetch_rx_buffer()
333 dma_sync_single_range_for_cpu(rx_ring->dev, in fm10k_fetch_rx_buffer()
334 rx_buffer->dma, in fm10k_fetch_rx_buffer()
335 rx_buffer->page_offset, in fm10k_fetch_rx_buffer()
345 dma_unmap_page(rx_ring->dev, rx_buffer->dma, in fm10k_fetch_rx_buffer()
350 rx_buffer->page = NULL; in fm10k_fetch_rx_buffer()
362 if (!(ring->netdev->features & NETIF_F_RXCSUM)) in fm10k_rx_checksum()
371 ring->rx_stats.csum_err++; in fm10k_rx_checksum()
377 skb->encapsulation = true; in fm10k_rx_checksum()
381 skb->ip_summed = CHECKSUM_UNNECESSARY; in fm10k_rx_checksum()
383 ring->rx_stats.csum_good++; in fm10k_rx_checksum()
398 if (!(ring->netdev->features & NETIF_F_RXHASH)) in fm10k_rx_hash()
401 rss_type = le16_to_cpu(rx_desc->w.pkt_info) & FM10K_RXD_RSSTYPE_MASK; in fm10k_rx_hash()
405 skb_set_hash(skb, le32_to_cpu(rx_desc->d.rss), in fm10k_rx_hash()
414 struct net_device *dev = rx_ring->netdev; in fm10k_type_trans()
415 struct fm10k_l2_accel *l2_accel = rcu_dereference_bh(rx_ring->l2_accel); in fm10k_type_trans()
419 u16 idx = le16_to_cpu(FM10K_CB(skb)->fi.w.dglort) - 1; in fm10k_type_trans()
421 idx -= l2_accel->dglort; in fm10k_type_trans()
422 if (idx < l2_accel->size && l2_accel->macvlan[idx]) in fm10k_type_trans()
423 dev = l2_accel->macvlan[idx]; in fm10k_type_trans()
430 skb_record_rx_queue(skb, rx_ring->queue_index); in fm10k_type_trans()
432 macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true, in fm10k_type_trans()
435 skb->protocol = eth_type_trans(skb, dev); in fm10k_type_trans()
439 * fm10k_process_skb_fields - Populate skb header fields from Rx descriptor
452 unsigned int len = skb->len; in fm10k_process_skb_fields()
458 FM10K_CB(skb)->tstamp = rx_desc->q.timestamp; in fm10k_process_skb_fields()
460 FM10K_CB(skb)->fi.w.vlan = rx_desc->w.vlan; in fm10k_process_skb_fields()
462 FM10K_CB(skb)->fi.d.glort = rx_desc->d.glort; in fm10k_process_skb_fields()
464 if (rx_desc->w.vlan) { in fm10k_process_skb_fields()
465 u16 vid = le16_to_cpu(rx_desc->w.vlan); in fm10k_process_skb_fields()
467 if ((vid & VLAN_VID_MASK) != rx_ring->vid) in fm10k_process_skb_fields()
480 * fm10k_is_non_eop - process handling of non-EOP buffers
487 * that this is in fact a non-EOP buffer.
492 u32 ntc = rx_ring->next_to_clean + 1; in fm10k_is_non_eop()
495 ntc = (ntc < rx_ring->count) ? ntc : 0; in fm10k_is_non_eop()
496 rx_ring->next_to_clean = ntc; in fm10k_is_non_eop()
507 * fm10k_cleanup_headers - Correct corrupted or empty headers
527 ((rxd)->w.csum_err & cpu_to_le16(bit)) in fm10k_cleanup_headers()
529 rx_ring->rx_stats.switch_errors++; in fm10k_cleanup_headers()
531 rx_ring->rx_stats.drops++; in fm10k_cleanup_headers()
533 rx_ring->rx_stats.pp_errors++; in fm10k_cleanup_headers()
535 rx_ring->rx_stats.link_errors++; in fm10k_cleanup_headers()
537 rx_ring->rx_stats.length_errors++; in fm10k_cleanup_headers()
539 rx_ring->rx_stats.errors++; in fm10k_cleanup_headers()
551 * fm10k_receive_skb - helper function to handle rx indications
558 napi_gro_receive(&q_vector->napi, skb); in fm10k_receive_skb()
565 struct sk_buff *skb = rx_ring->skb; in fm10k_clean_rx_irq()
578 rx_desc = FM10K_RX_DESC(rx_ring, rx_ring->next_to_clean); in fm10k_clean_rx_irq()
580 if (!rx_desc->d.staterr) in fm10k_clean_rx_irq()
598 /* fetch next buffer in frame if non-eop */ in fm10k_clean_rx_irq()
621 rx_ring->skb = skb; in fm10k_clean_rx_irq()
623 u64_stats_update_begin(&rx_ring->syncp); in fm10k_clean_rx_irq()
624 rx_ring->stats.packets += total_packets; in fm10k_clean_rx_irq()
625 rx_ring->stats.bytes += total_bytes; in fm10k_clean_rx_irq()
626 u64_stats_update_end(&rx_ring->syncp); in fm10k_clean_rx_irq()
627 q_vector->rx.total_packets += total_packets; in fm10k_clean_rx_irq()
628 q_vector->rx.total_bytes += total_bytes; in fm10k_clean_rx_irq()
636 struct fm10k_intfc *interface = netdev_priv(skb->dev); in fm10k_port_is_vxlan()
638 if (interface->vxlan_port != udp_hdr(skb)->dest) in fm10k_port_is_vxlan()
665 /* verify all reserved flags are 0 */ in fm10k_gre_is_nvgre()
666 if (nvgre_hdr->flags & FM10K_NVGRE_RESERVED0_FLAGS) in fm10k_gre_is_nvgre()
670 if (nvgre_hdr->flags & NVGRE_TNI) in fm10k_gre_is_nvgre()
673 return (struct ethhdr *)(&nvgre_hdr->tni); in fm10k_gre_is_nvgre()
681 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER || in fm10k_tx_encap_offload()
682 skb->inner_protocol != htons(ETH_P_TEB)) in fm10k_tx_encap_offload()
687 l4_hdr = ip_hdr(skb)->protocol; in fm10k_tx_encap_offload()
690 l4_hdr = ipv6_hdr(skb)->nexthdr; in fm10k_tx_encap_offload()
710 switch (eth_hdr->h_proto) { in fm10k_tx_encap_offload()
712 inner_l4_hdr = inner_ip_hdr(skb)->protocol; in fm10k_tx_encap_offload()
715 inner_l4_hdr = inner_ipv6_hdr(skb)->nexthdr; in fm10k_tx_encap_offload()
735 if (skb_inner_transport_header(skb) + inner_l4_hlen - in fm10k_tx_encap_offload()
739 return eth_hdr->h_proto; in fm10k_tx_encap_offload()
745 struct sk_buff *skb = first->skb; in fm10k_tso()
750 if (skb->ip_summed != CHECKSUM_PARTIAL) in fm10k_tso()
757 if (skb->encapsulation) { in fm10k_tso()
766 hdrlen = (th - skb->data) + (((struct tcphdr *)th)->doff << 2); in fm10k_tso()
768 first->tx_flags |= FM10K_TX_FLAGS_CSUM; in fm10k_tso()
771 first->gso_segs = skb_shinfo(skb)->gso_segs; in fm10k_tso()
772 first->bytecount += (first->gso_segs - 1) * hdrlen; in fm10k_tso()
775 tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use); in fm10k_tso()
776 tx_desc->hdrlen = hdrlen; in fm10k_tso()
777 tx_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size); in fm10k_tso()
782 tx_ring->netdev->features &= ~NETIF_F_GSO_UDP_TUNNEL; in fm10k_tso()
784 netdev_err(tx_ring->netdev, in fm10k_tso()
786 return -1; in fm10k_tso()
792 struct sk_buff *skb = first->skb; in fm10k_tx_csum()
804 if (skb->ip_summed != CHECKSUM_PARTIAL) in fm10k_tx_csum()
807 if (skb->encapsulation) { in fm10k_tx_csum()
811 dev_warn(tx_ring->dev, in fm10k_tx_csum()
813 tx_ring->tx_stats.csum_err++; in fm10k_tx_csum()
827 l4_hdr = network_hdr.ipv4->protocol; in fm10k_tx_csum()
830 l4_hdr = network_hdr.ipv6->nexthdr; in fm10k_tx_csum()
831 if (likely((transport_hdr - network_hdr.raw) == in fm10k_tx_csum()
834 ipv6_skip_exthdr(skb, network_hdr.raw - skb->data + in fm10k_tx_csum()
849 if (skb->encapsulation) in fm10k_tx_csum()
854 dev_warn(tx_ring->dev, in fm10k_tx_csum()
859 tx_ring->tx_stats.csum_err++; in fm10k_tx_csum()
864 first->tx_flags |= FM10K_TX_FLAGS_CSUM; in fm10k_tx_csum()
865 tx_ring->tx_stats.csum_good++; in fm10k_tx_csum()
869 tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use); in fm10k_tx_csum()
870 tx_desc->hdrlen = 0; in fm10k_tx_csum()
871 tx_desc->mss = 0; in fm10k_tx_csum()
896 if ((++i & (FM10K_TXD_WB_FIFO_SIZE - 1)) == 0) in fm10k_tx_desc_push()
900 tx_desc->buffer_addr = cpu_to_le64(dma); in fm10k_tx_desc_push()
901 tx_desc->flags = desc_flags; in fm10k_tx_desc_push()
902 tx_desc->buflen = cpu_to_le16(size); in fm10k_tx_desc_push()
905 return i == tx_ring->count; in fm10k_tx_desc_push()
910 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); in __fm10k_maybe_stop_tx()
915 /* Check again in a case another CPU has just made room available */ in __fm10k_maybe_stop_tx()
917 return -EBUSY; in __fm10k_maybe_stop_tx()
919 /* A reprieve! - use start_queue because it doesn't call schedule */ in __fm10k_maybe_stop_tx()
920 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); in __fm10k_maybe_stop_tx()
921 ++tx_ring->tx_stats.restart_queue; in __fm10k_maybe_stop_tx()
935 struct sk_buff *skb = first->skb; in fm10k_tx_map()
942 u32 tx_flags = first->tx_flags; in fm10k_tx_map()
943 u16 i = tx_ring->next_to_use; in fm10k_tx_map()
950 tx_desc->vlan = cpu_to_le16(skb_vlan_tag_get(skb)); in fm10k_tx_map()
952 tx_desc->vlan = 0; in fm10k_tx_map()
955 data = skb->data; in fm10k_tx_map()
957 dma = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); in fm10k_tx_map()
959 data_len = skb->data_len; in fm10k_tx_map()
962 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { in fm10k_tx_map()
963 if (dma_mapping_error(tx_ring->dev, dma)) in fm10k_tx_map()
978 size -= FM10K_MAX_DATA_PER_TXD; in fm10k_tx_map()
991 data_len -= size; in fm10k_tx_map()
993 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, in fm10k_tx_map()
996 tx_buffer = &tx_ring->tx_buffer[i]; in fm10k_tx_map()
1006 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); in fm10k_tx_map()
1009 skb_tx_timestamp(first->skb); in fm10k_tx_map()
1012 * are new descriptors to fetch. (Only applicable for weak-ordered in fm10k_tx_map()
1013 * memory model archs, such as IA-64). in fm10k_tx_map()
1021 first->next_to_watch = tx_desc; in fm10k_tx_map()
1023 tx_ring->next_to_use = i; in fm10k_tx_map()
1030 writel(i, tx_ring->tail); in fm10k_tx_map()
1035 dev_err(tx_ring->dev, "TX DMA map failed\n"); in fm10k_tx_map()
1039 tx_buffer = &tx_ring->tx_buffer[i]; in fm10k_tx_map()
1044 i = tx_ring->count; in fm10k_tx_map()
1045 i--; in fm10k_tx_map()
1048 tx_ring->next_to_use = i; in fm10k_tx_map()
1065 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) { in fm10k_xmit_frame_ring()
1066 skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; in fm10k_xmit_frame_ring()
1072 tx_ring->tx_stats.tx_busy++; in fm10k_xmit_frame_ring()
1077 first = &tx_ring->tx_buffer[tx_ring->next_to_use]; in fm10k_xmit_frame_ring()
1078 first->skb = skb; in fm10k_xmit_frame_ring()
1079 first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); in fm10k_xmit_frame_ring()
1080 first->gso_segs = 1; in fm10k_xmit_frame_ring()
1083 first->tx_flags = tx_flags; in fm10k_xmit_frame_ring()
1096 dev_kfree_skb_any(first->skb); in fm10k_xmit_frame_ring()
1097 first->skb = NULL; in fm10k_xmit_frame_ring()
1104 return ring->stats.packets; in fm10k_get_tx_completed()
1108 * fm10k_get_tx_pending - how many Tx descriptors not processed
1114 struct fm10k_intfc *interface = ring->q_vector->interface; in fm10k_get_tx_pending()
1115 struct fm10k_hw *hw = &interface->hw; in fm10k_get_tx_pending()
1119 head = ring->next_to_clean; in fm10k_get_tx_pending()
1120 tail = ring->next_to_use; in fm10k_get_tx_pending()
1122 head = fm10k_read_reg(hw, FM10K_TDH(ring->reg_idx)); in fm10k_get_tx_pending()
1123 tail = fm10k_read_reg(hw, FM10K_TDT(ring->reg_idx)); in fm10k_get_tx_pending()
1126 return ((head <= tail) ? tail : tail + ring->count) - head; in fm10k_get_tx_pending()
1132 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; in fm10k_check_tx_hang()
1147 tx_ring->tx_stats.tx_done_old = tx_done; in fm10k_check_tx_hang()
1149 clear_bit(__FM10K_HANG_CHECK_ARMED, tx_ring->state); in fm10k_check_tx_hang()
1155 return test_and_set_bit(__FM10K_HANG_CHECK_ARMED, tx_ring->state); in fm10k_check_tx_hang()
1159 * fm10k_tx_timeout_reset - initiate reset due to Tx timeout
1165 if (!test_bit(__FM10K_DOWN, interface->state)) { in fm10k_tx_timeout_reset()
1166 interface->tx_timeout_count++; in fm10k_tx_timeout_reset()
1167 set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags); in fm10k_tx_timeout_reset()
1173 * fm10k_clean_tx_irq - Reclaim resources after transmit completes
1181 struct fm10k_intfc *interface = q_vector->interface; in fm10k_clean_tx_irq()
1185 unsigned int budget = q_vector->tx.work_limit; in fm10k_clean_tx_irq()
1186 unsigned int i = tx_ring->next_to_clean; in fm10k_clean_tx_irq()
1188 if (test_bit(__FM10K_DOWN, interface->state)) in fm10k_clean_tx_irq()
1191 tx_buffer = &tx_ring->tx_buffer[i]; in fm10k_clean_tx_irq()
1193 i -= tx_ring->count; in fm10k_clean_tx_irq()
1196 struct fm10k_tx_desc *eop_desc = tx_buffer->next_to_watch; in fm10k_clean_tx_irq()
1206 if (!(eop_desc->flags & FM10K_TXD_FLAG_DONE)) in fm10k_clean_tx_irq()
1210 tx_buffer->next_to_watch = NULL; in fm10k_clean_tx_irq()
1213 total_bytes += tx_buffer->bytecount; in fm10k_clean_tx_irq()
1214 total_packets += tx_buffer->gso_segs; in fm10k_clean_tx_irq()
1217 napi_consume_skb(tx_buffer->skb, napi_budget); in fm10k_clean_tx_irq()
1220 dma_unmap_single(tx_ring->dev, in fm10k_clean_tx_irq()
1226 tx_buffer->skb = NULL; in fm10k_clean_tx_irq()
1235 i -= tx_ring->count; in fm10k_clean_tx_irq()
1236 tx_buffer = tx_ring->tx_buffer; in fm10k_clean_tx_irq()
1242 dma_unmap_page(tx_ring->dev, in fm10k_clean_tx_irq()
1255 i -= tx_ring->count; in fm10k_clean_tx_irq()
1256 tx_buffer = tx_ring->tx_buffer; in fm10k_clean_tx_irq()
1264 budget--; in fm10k_clean_tx_irq()
1267 i += tx_ring->count; in fm10k_clean_tx_irq()
1268 tx_ring->next_to_clean = i; in fm10k_clean_tx_irq()
1269 u64_stats_update_begin(&tx_ring->syncp); in fm10k_clean_tx_irq()
1270 tx_ring->stats.bytes += total_bytes; in fm10k_clean_tx_irq()
1271 tx_ring->stats.packets += total_packets; in fm10k_clean_tx_irq()
1272 u64_stats_update_end(&tx_ring->syncp); in fm10k_clean_tx_irq()
1273 q_vector->tx.total_bytes += total_bytes; in fm10k_clean_tx_irq()
1274 q_vector->tx.total_packets += total_packets; in fm10k_clean_tx_irq()
1278 struct fm10k_hw *hw = &interface->hw; in fm10k_clean_tx_irq()
1280 netif_err(interface, drv, tx_ring->netdev, in fm10k_clean_tx_irq()
1286 tx_ring->queue_index, in fm10k_clean_tx_irq()
1287 fm10k_read_reg(hw, FM10K_TDH(tx_ring->reg_idx)), in fm10k_clean_tx_irq()
1288 fm10k_read_reg(hw, FM10K_TDT(tx_ring->reg_idx)), in fm10k_clean_tx_irq()
1289 tx_ring->next_to_use, i); in fm10k_clean_tx_irq()
1291 netif_stop_subqueue(tx_ring->netdev, in fm10k_clean_tx_irq()
1292 tx_ring->queue_index); in fm10k_clean_tx_irq()
1294 netif_info(interface, probe, tx_ring->netdev, in fm10k_clean_tx_irq()
1296 interface->tx_timeout_count + 1, in fm10k_clean_tx_irq()
1297 tx_ring->queue_index); in fm10k_clean_tx_irq()
1309 #define TX_WAKE_THRESHOLD min_t(u16, FM10K_MIN_TXD - 1, DESC_NEEDED * 2) in fm10k_clean_tx_irq()
1310 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && in fm10k_clean_tx_irq()
1316 if (__netif_subqueue_stopped(tx_ring->netdev, in fm10k_clean_tx_irq()
1317 tx_ring->queue_index) && in fm10k_clean_tx_irq()
1318 !test_bit(__FM10K_DOWN, interface->state)) { in fm10k_clean_tx_irq()
1319 netif_wake_subqueue(tx_ring->netdev, in fm10k_clean_tx_irq()
1320 tx_ring->queue_index); in fm10k_clean_tx_irq()
1321 ++tx_ring->tx_stats.restart_queue; in fm10k_clean_tx_irq()
1329 * fm10k_update_itr - update the dynamic ITR value based on packet size
1343 if (!ITR_IS_ADAPTIVE(ring_container->itr)) in fm10k_update_itr()
1346 packets = ring_container->total_packets; in fm10k_update_itr()
1350 avg_wire_size = ring_container->total_bytes / packets; in fm10k_update_itr()
1387 itr_round = READ_ONCE(ring_container->itr_scale) + 8; in fm10k_update_itr()
1388 avg_wire_size += BIT(itr_round) - 1; in fm10k_update_itr()
1392 ring_container->itr = avg_wire_size | FM10K_ITR_ADAPTIVE; in fm10k_update_itr()
1395 ring_container->total_bytes = 0; in fm10k_update_itr()
1396 ring_container->total_packets = 0; in fm10k_update_itr()
1401 /* Enable auto-mask and clear the current mask */ in fm10k_qv_enable()
1405 fm10k_update_itr(&q_vector->tx); in fm10k_qv_enable()
1408 fm10k_update_itr(&q_vector->rx); in fm10k_qv_enable()
1411 itr |= (q_vector->tx.itr & FM10K_ITR_MAX); in fm10k_qv_enable()
1414 itr |= (q_vector->rx.itr & FM10K_ITR_MAX) << FM10K_ITR_INTERVAL1_SHIFT; in fm10k_qv_enable()
1417 writel(itr, q_vector->itr); in fm10k_qv_enable()
1428 fm10k_for_each_ring(ring, q_vector->tx) { in fm10k_poll()
1440 if (q_vector->rx.count > 1) in fm10k_poll()
1441 per_ring_budget = max(budget / q_vector->rx.count, 1); in fm10k_poll()
1445 fm10k_for_each_ring(ring, q_vector->rx) { in fm10k_poll()
1457 /* Exit the polling mode, but don't re-enable interrupts if stack might in fm10k_poll()
1458 * poll us due to busy-polling in fm10k_poll()
1463 return min(work_done, budget - 1); in fm10k_poll()
1467 * fm10k_set_qos_queues: Allocate queues for a QOS-enabled device
1479 struct net_device *dev = interface->netdev; in fm10k_set_qos_queues()
1491 f = &interface->ring_feature[RING_F_QOS]; in fm10k_set_qos_queues()
1492 f->indices = pcs; in fm10k_set_qos_queues()
1493 f->mask = BIT(fls(pcs - 1)) - 1; in fm10k_set_qos_queues()
1496 rss_i = interface->hw.mac.max_queues / pcs; in fm10k_set_qos_queues()
1497 rss_i = BIT(fls(rss_i) - 1); in fm10k_set_qos_queues()
1500 f = &interface->ring_feature[RING_F_RSS]; in fm10k_set_qos_queues()
1501 rss_i = min_t(u16, rss_i, f->limit); in fm10k_set_qos_queues()
1502 f->indices = rss_i; in fm10k_set_qos_queues()
1503 f->mask = BIT(fls(rss_i - 1)) - 1; in fm10k_set_qos_queues()
1509 interface->num_rx_queues = rss_i * pcs; in fm10k_set_qos_queues()
1510 interface->num_tx_queues = rss_i * pcs; in fm10k_set_qos_queues()
1520 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
1528 f = &interface->ring_feature[RING_F_RSS]; in fm10k_set_rss_queues()
1529 rss_i = min_t(u16, interface->hw.mac.max_queues, f->limit); in fm10k_set_rss_queues()
1532 f->indices = rss_i; in fm10k_set_rss_queues()
1533 f->mask = BIT(fls(rss_i - 1)) - 1; in fm10k_set_rss_queues()
1535 interface->num_rx_queues = rss_i; in fm10k_set_rss_queues()
1536 interface->num_tx_queues = rss_i; in fm10k_set_rss_queues()
1563 * fm10k_reset_num_queues - Reset the number of queues to zero
1571 interface->num_tx_queues = 0; in fm10k_reset_num_queues()
1572 interface->num_rx_queues = 0; in fm10k_reset_num_queues()
1573 interface->num_q_vectors = 0; in fm10k_reset_num_queues()
1577 * fm10k_alloc_q_vector - Allocate memory for a single interrupt vector
1586 * We allocate one q_vector. If allocation fails we return -ENOMEM.
1602 return -ENOMEM; in fm10k_alloc_q_vector()
1605 netif_napi_add(interface->netdev, &q_vector->napi, fm10k_poll); in fm10k_alloc_q_vector()
1608 interface->q_vector[v_idx] = q_vector; in fm10k_alloc_q_vector()
1609 q_vector->interface = interface; in fm10k_alloc_q_vector()
1610 q_vector->v_idx = v_idx; in fm10k_alloc_q_vector()
1613 ring = q_vector->ring; in fm10k_alloc_q_vector()
1616 q_vector->tx.ring = ring; in fm10k_alloc_q_vector()
1617 q_vector->tx.work_limit = FM10K_DEFAULT_TX_WORK; in fm10k_alloc_q_vector()
1618 q_vector->tx.itr = interface->tx_itr; in fm10k_alloc_q_vector()
1619 q_vector->tx.itr_scale = interface->hw.mac.itr_scale; in fm10k_alloc_q_vector()
1620 q_vector->tx.count = txr_count; in fm10k_alloc_q_vector()
1624 ring->dev = &interface->pdev->dev; in fm10k_alloc_q_vector()
1625 ring->netdev = interface->netdev; in fm10k_alloc_q_vector()
1628 ring->q_vector = q_vector; in fm10k_alloc_q_vector()
1631 ring->count = interface->tx_ring_count; in fm10k_alloc_q_vector()
1632 ring->queue_index = txr_idx; in fm10k_alloc_q_vector()
1635 interface->tx_ring[txr_idx] = ring; in fm10k_alloc_q_vector()
1638 txr_count--; in fm10k_alloc_q_vector()
1646 q_vector->rx.ring = ring; in fm10k_alloc_q_vector()
1647 q_vector->rx.itr = interface->rx_itr; in fm10k_alloc_q_vector()
1648 q_vector->rx.itr_scale = interface->hw.mac.itr_scale; in fm10k_alloc_q_vector()
1649 q_vector->rx.count = rxr_count; in fm10k_alloc_q_vector()
1653 ring->dev = &interface->pdev->dev; in fm10k_alloc_q_vector()
1654 ring->netdev = interface->netdev; in fm10k_alloc_q_vector()
1655 rcu_assign_pointer(ring->l2_accel, interface->l2_accel); in fm10k_alloc_q_vector()
1658 ring->q_vector = q_vector; in fm10k_alloc_q_vector()
1661 ring->count = interface->rx_ring_count; in fm10k_alloc_q_vector()
1662 ring->queue_index = rxr_idx; in fm10k_alloc_q_vector()
1665 interface->rx_ring[rxr_idx] = ring; in fm10k_alloc_q_vector()
1668 rxr_count--; in fm10k_alloc_q_vector()
1681 * fm10k_free_q_vector - Free memory allocated for specific interrupt vector
1691 struct fm10k_q_vector *q_vector = interface->q_vector[v_idx]; in fm10k_free_q_vector()
1696 fm10k_for_each_ring(ring, q_vector->tx) in fm10k_free_q_vector()
1697 interface->tx_ring[ring->queue_index] = NULL; in fm10k_free_q_vector()
1699 fm10k_for_each_ring(ring, q_vector->rx) in fm10k_free_q_vector()
1700 interface->rx_ring[ring->queue_index] = NULL; in fm10k_free_q_vector()
1702 interface->q_vector[v_idx] = NULL; in fm10k_free_q_vector()
1703 netif_napi_del(&q_vector->napi); in fm10k_free_q_vector()
1708 * fm10k_alloc_q_vectors - Allocate memory for interrupt vectors
1712 * return -ENOMEM.
1716 unsigned int q_vectors = interface->num_q_vectors; in fm10k_alloc_q_vectors()
1717 unsigned int rxr_remaining = interface->num_rx_queues; in fm10k_alloc_q_vectors()
1718 unsigned int txr_remaining = interface->num_tx_queues; in fm10k_alloc_q_vectors()
1730 rxr_remaining--; in fm10k_alloc_q_vectors()
1736 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); in fm10k_alloc_q_vectors()
1737 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); in fm10k_alloc_q_vectors()
1747 rxr_remaining -= rqpv; in fm10k_alloc_q_vectors()
1748 txr_remaining -= tqpv; in fm10k_alloc_q_vectors()
1758 while (v_idx--) in fm10k_alloc_q_vectors()
1761 return -ENOMEM; in fm10k_alloc_q_vectors()
1765 * fm10k_free_q_vectors - Free memory allocated for interrupt vectors
1774 int v_idx = interface->num_q_vectors; in fm10k_free_q_vectors()
1778 while (v_idx--) in fm10k_free_q_vectors()
1783 * fm10k_reset_msix_capability - reset MSI-X capability
1786 * Reset the MSI-X capability back to its starting state
1790 pci_disable_msix(interface->pdev); in fm10k_reset_msix_capability()
1791 kfree(interface->msix_entries); in fm10k_reset_msix_capability()
1792 interface->msix_entries = NULL; in fm10k_reset_msix_capability()
1796 * fm10k_init_msix_capability - configure MSI-X capability
1804 struct fm10k_hw *hw = &interface->hw; in fm10k_init_msix_capability()
1807 /* It's easy to be greedy for MSI-X vectors, but it really in fm10k_init_msix_capability()
1808 * doesn't do us much good if we have a lot more vectors in fm10k_init_msix_capability()
1809 * than CPU's. So let's be conservative and only ask for in fm10k_init_msix_capability()
1810 * (roughly) the same number of vectors as there are CPU's. in fm10k_init_msix_capability()
1811 * the default is to use pairs of vectors in fm10k_init_msix_capability()
1813 v_budget = max(interface->num_rx_queues, interface->num_tx_queues); in fm10k_init_msix_capability()
1816 /* account for vectors not related to queues */ in fm10k_init_msix_capability()
1820 * hw.mac->max_msix_vectors vectors. With features in fm10k_init_msix_capability()
1823 * those rare cases where the cpu count also exceeds our vector limit. in fm10k_init_msix_capability()
1825 v_budget = min_t(int, v_budget, hw->mac.max_msix_vectors); in fm10k_init_msix_capability()
1827 /* A failure in MSI-X entry allocation is fatal. */ in fm10k_init_msix_capability()
1828 interface->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), in fm10k_init_msix_capability()
1830 if (!interface->msix_entries) in fm10k_init_msix_capability()
1831 return -ENOMEM; in fm10k_init_msix_capability()
1835 interface->msix_entries[vector].entry = vector; in fm10k_init_msix_capability()
1837 /* Attempt to enable MSI-X with requested value */ in fm10k_init_msix_capability()
1838 v_budget = pci_enable_msix_range(interface->pdev, in fm10k_init_msix_capability()
1839 interface->msix_entries, in fm10k_init_msix_capability()
1843 kfree(interface->msix_entries); in fm10k_init_msix_capability()
1844 interface->msix_entries = NULL; in fm10k_init_msix_capability()
1849 interface->num_q_vectors = v_budget - NON_Q_VECTORS; in fm10k_init_msix_capability()
1855 * fm10k_cache_ring_qos - Descriptor ring to register mapping for QoS
1862 struct net_device *dev = interface->netdev; in fm10k_cache_ring_qos()
1864 u16 pc_stride = interface->ring_feature[RING_F_QOS].mask + 1; in fm10k_cache_ring_qos()
1870 rss_i = interface->ring_feature[RING_F_RSS].indices; in fm10k_cache_ring_qos()
1876 interface->tx_ring[offset + i]->reg_idx = q_idx; in fm10k_cache_ring_qos()
1877 interface->tx_ring[offset + i]->qos_pc = pc; in fm10k_cache_ring_qos()
1878 interface->rx_ring[offset + i]->reg_idx = q_idx; in fm10k_cache_ring_qos()
1879 interface->rx_ring[offset + i]->qos_pc = pc; in fm10k_cache_ring_qos()
1888 * fm10k_cache_ring_rss - Descriptor ring to register mapping for RSS
1897 for (i = 0; i < interface->num_rx_queues; i++) in fm10k_cache_ring_rss()
1898 interface->rx_ring[i]->reg_idx = i; in fm10k_cache_ring_rss()
1900 for (i = 0; i < interface->num_tx_queues; i++) in fm10k_cache_ring_rss()
1901 interface->tx_ring[i]->reg_idx = i; in fm10k_cache_ring_rss()
1905 * fm10k_assign_rings - Map rings to network devices
1922 u16 i, rss_i = interface->ring_feature[RING_F_RSS].indices; in fm10k_init_reta()
1928 if (netif_is_rxfh_configured(interface->netdev)) { in fm10k_init_reta()
1929 for (i = FM10K_RETA_SIZE; i--;) { in fm10k_init_reta()
1930 reta = interface->reta[i]; in fm10k_init_reta()
1938 dev_err(&interface->pdev->dev, in fm10k_init_reta()
1952 * fm10k_init_queueing_scheme - Determine proper queueing scheme
1956 * - Hardware queue count (num_*_queues)
1957 * - defined by miscellaneous hardware support/features (RSS, etc.)
1966 /* Configure MSI-X capability */ in fm10k_init_queueing_scheme()
1969 dev_err(&interface->pdev->dev, in fm10k_init_queueing_scheme()
1970 "Unable to initialize MSI-X capability\n"); in fm10k_init_queueing_scheme()
1977 dev_err(&interface->pdev->dev, in fm10k_init_queueing_scheme()
1978 "Unable to allocate queue vectors\n"); in fm10k_init_queueing_scheme()
1998 * fm10k_clear_queueing_scheme - Clear the current queueing scheme settings
2002 * to pre-load conditions