Lines Matching +full:num +full:- +full:rx +full:- +full:queues
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
3 * Copyright 2015-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
12 tx_info = &tx_ring->tx_buffer_info[req_id]; in validate_xdp_req_id()
13 if (likely(tx_info->xdpf)) in validate_xdp_req_id()
24 struct ena_adapter *adapter = tx_ring->adapter; in ena_xdp_tx_map_frame()
31 tx_info->xdpf = xdpf; in ena_xdp_tx_map_frame()
32 data = tx_info->xdpf->data; in ena_xdp_tx_map_frame()
33 size = tx_info->xdpf->len; in ena_xdp_tx_map_frame()
35 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { in ena_xdp_tx_map_frame()
37 push_len = min_t(u32, size, tx_ring->tx_max_header_size); in ena_xdp_tx_map_frame()
39 ena_tx_ctx->push_header = data; in ena_xdp_tx_map_frame()
41 size -= push_len; in ena_xdp_tx_map_frame()
45 ena_tx_ctx->header_len = push_len; in ena_xdp_tx_map_frame()
48 dma = dma_map_single(tx_ring->dev, in ena_xdp_tx_map_frame()
52 if (unlikely(dma_mapping_error(tx_ring->dev, dma))) in ena_xdp_tx_map_frame()
55 tx_info->map_linear_data = 0; in ena_xdp_tx_map_frame()
57 ena_buf = tx_info->bufs; in ena_xdp_tx_map_frame()
58 ena_buf->paddr = dma; in ena_xdp_tx_map_frame()
59 ena_buf->len = size; in ena_xdp_tx_map_frame()
61 ena_tx_ctx->ena_bufs = ena_buf; in ena_xdp_tx_map_frame()
62 ena_tx_ctx->num_bufs = tx_info->num_of_bufs = 1; in ena_xdp_tx_map_frame()
68 ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1, in ena_xdp_tx_map_frame()
69 &tx_ring->syncp); in ena_xdp_tx_map_frame()
70 netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map xdp buff\n"); in ena_xdp_tx_map_frame()
72 return -EINVAL; in ena_xdp_tx_map_frame()
85 next_to_use = tx_ring->next_to_use; in ena_xdp_xmit_frame()
86 req_id = tx_ring->free_ids[next_to_use]; in ena_xdp_xmit_frame()
87 tx_info = &tx_ring->tx_buffer_info[req_id]; in ena_xdp_xmit_frame()
88 tx_info->num_of_bufs = 0; in ena_xdp_xmit_frame()
101 xdpf->len); in ena_xdp_xmit_frame()
116 tx_info->xdpf = NULL; in ena_xdp_xmit_frame()
129 return -EINVAL; in ena_xdp_xmit()
131 if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags)) in ena_xdp_xmit()
132 return -ENETDOWN; in ena_xdp_xmit()
135 if (!READ_ONCE(adapter->rx_ring->xdp_bpf_prog)) in ena_xdp_xmit()
136 return -ENXIO; in ena_xdp_xmit()
138 qid = smp_processor_id() % adapter->xdp_num_queues; in ena_xdp_xmit()
139 qid += adapter->xdp_first_ring; in ena_xdp_xmit()
140 tx_ring = &adapter->tx_ring[qid]; in ena_xdp_xmit()
143 spin_lock(&tx_ring->xdp_tx_lock); in ena_xdp_xmit()
155 spin_unlock(&tx_ring->xdp_tx_lock); in ena_xdp_xmit()
163 adapter->xdp_first_ring = adapter->num_io_queues; in ena_init_all_xdp_queues()
164 adapter->xdp_num_queues = adapter->num_io_queues; in ena_init_all_xdp_queues()
167 adapter->xdp_first_ring, in ena_init_all_xdp_queues()
168 adapter->xdp_num_queues); in ena_init_all_xdp_queues()
173 u32 xdp_first_ring = adapter->xdp_first_ring; in ena_setup_and_create_all_xdp_queues()
174 u32 xdp_num_queues = adapter->xdp_num_queues; in ena_setup_and_create_all_xdp_queues()
193 /* Provides a way for both kernel and bpf-prog to know
194 * more about the RX-queue a given XDP frame arrived on.
200 rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid, 0); in ena_xdp_register_rxq_info()
202 netif_dbg(rx_ring->adapter, ifup, rx_ring->netdev, "Registering RX info for queue %d", in ena_xdp_register_rxq_info()
203 rx_ring->qid); in ena_xdp_register_rxq_info()
205 netif_err(rx_ring->adapter, ifup, rx_ring->netdev, in ena_xdp_register_rxq_info()
206 "Failed to register xdp rx queue info. RX queue num %d rc: %d\n", in ena_xdp_register_rxq_info()
207 rx_ring->qid, rc); in ena_xdp_register_rxq_info()
211 rc = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, MEM_TYPE_PAGE_SHARED, NULL); in ena_xdp_register_rxq_info()
214 netif_err(rx_ring->adapter, ifup, rx_ring->netdev, in ena_xdp_register_rxq_info()
215 "Failed to register xdp rx queue info memory model. RX queue num %d rc: %d\n", in ena_xdp_register_rxq_info()
216 rx_ring->qid, rc); in ena_xdp_register_rxq_info()
217 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); in ena_xdp_register_rxq_info()
226 netif_dbg(rx_ring->adapter, ifdown, rx_ring->netdev, in ena_xdp_unregister_rxq_info()
227 "Unregistering RX info for queue %d", in ena_xdp_unregister_rxq_info()
228 rx_ring->qid); in ena_xdp_unregister_rxq_info()
229 xdp_rxq_info_unreg_mem_model(&rx_ring->xdp_rxq); in ena_xdp_unregister_rxq_info()
230 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); in ena_xdp_unregister_rxq_info()
242 rx_ring = &adapter->rx_ring[i]; in ena_xdp_exchange_program_rx_in_range()
243 old_bpf_prog = xchg(&rx_ring->xdp_bpf_prog, prog); in ena_xdp_exchange_program_rx_in_range()
246 rx_ring->rx_headroom = XDP_PACKET_HEADROOM; in ena_xdp_exchange_program_rx_in_range()
248 rx_ring->rx_headroom = NET_SKB_PAD; in ena_xdp_exchange_program_rx_in_range()
256 struct bpf_prog *old_bpf_prog = xchg(&adapter->xdp_bpf_prog, prog); in ena_xdp_exchange_program()
261 adapter->num_io_queues); in ena_xdp_exchange_program()
272 was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags); in ena_destroy_and_free_all_xdp_queues()
277 adapter->xdp_first_ring = 0; in ena_destroy_and_free_all_xdp_queues()
278 adapter->xdp_num_queues = 0; in ena_destroy_and_free_all_xdp_queues()
291 struct bpf_prog *prog = bpf->prog; in ena_xdp_set()
296 is_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags); in ena_xdp_set()
299 old_bpf_prog = adapter->xdp_bpf_prog; in ena_xdp_set()
309 netif_dbg(adapter, drv, adapter->netdev, "Set a new XDP program\n"); in ena_xdp_set()
319 netif_dbg(adapter, drv, adapter->netdev, "Removing XDP program\n"); in ena_xdp_set()
326 prev_mtu = netdev->max_mtu; in ena_xdp_set()
327 netdev->max_mtu = prog ? ENA_XDP_MAX_MTU : adapter->max_mtu; in ena_xdp_set()
330 netif_info(adapter, drv, adapter->netdev, in ena_xdp_set()
332 prev_mtu, netdev->max_mtu); in ena_xdp_set()
335 netif_err(adapter, drv, adapter->netdev, in ena_xdp_set()
337 netdev->mtu, ENA_XDP_MAX_MTU); in ena_xdp_set()
338 NL_SET_ERR_MSG_MOD(bpf->extack, in ena_xdp_set()
340 return -EINVAL; in ena_xdp_set()
342 netif_err(adapter, drv, adapter->netdev, in ena_xdp_set()
343 …"Failed to set xdp program, the Rx/Tx channel count should be at most half of the maximum allowed … in ena_xdp_set()
344 adapter->num_io_queues, adapter->max_num_io_queues); in ena_xdp_set()
345 NL_SET_ERR_MSG_MOD(bpf->extack, in ena_xdp_set()
346 …"Failed to set xdp program, there is no enough space for allocating XDP queues, Check the dmesg fo… in ena_xdp_set()
347 return -EINVAL; in ena_xdp_set()
358 switch (bpf->command) { in ena_xdp()
362 return -EINVAL; in ena_xdp()
377 next_to_clean = tx_ring->next_to_clean; in ena_clean_xdp_irq()
383 rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, in ena_clean_xdp_irq()
386 if (unlikely(rc == -EINVAL)) in ena_clean_xdp_irq()
396 tx_info = &tx_ring->tx_buffer_info[req_id]; in ena_clean_xdp_irq()
398 tx_info->last_jiffies = 0; in ena_clean_xdp_irq()
400 xdpf = tx_info->xdpf; in ena_clean_xdp_irq()
401 tx_info->xdpf = NULL; in ena_clean_xdp_irq()
406 total_done += tx_info->tx_descs; in ena_clean_xdp_irq()
407 tx_ring->free_ids[next_to_clean] = req_id; in ena_clean_xdp_irq()
409 tx_ring->ring_size); in ena_clean_xdp_irq()
411 netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev, in ena_clean_xdp_irq()
412 "tx_poll: q %d pkt #%d req_id %d\n", tx_ring->qid, tx_pkts, req_id); in ena_clean_xdp_irq()
415 tx_ring->next_to_clean = next_to_clean; in ena_clean_xdp_irq()
416 ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done); in ena_clean_xdp_irq()
418 netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev, in ena_clean_xdp_irq()
420 tx_ring->qid, tx_pkts); in ena_clean_xdp_irq()
425 /* This is the XDP napi callback. XDP queues use a separate napi callback
426 * than Rx/Tx queues.
435 tx_ring = ena_napi->tx_ring; in ena_xdp_io_poll()
437 if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) || in ena_xdp_io_poll()
438 test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) { in ena_xdp_io_poll()
448 if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags))) { in ena_xdp_io_poll()
452 ena_increase_stat(&tx_ring->tx_stats.napi_comp, 1, in ena_xdp_io_poll()
453 &tx_ring->syncp); in ena_xdp_io_poll()
463 u64_stats_update_begin(&tx_ring->syncp); in ena_xdp_io_poll()
464 tx_ring->tx_stats.tx_poll++; in ena_xdp_io_poll()
465 u64_stats_update_end(&tx_ring->syncp); in ena_xdp_io_poll()
466 tx_ring->tx_stats.last_napi_jiffies = jiffies; in ena_xdp_io_poll()