@@ -1402,6 +1402,46 @@ vmxnet3_xdp_xmit_frame(struct vmxnet3_adapter *adapter,
struct sk_buff *skb,
struct vmxnet3_tx_queue *tq);
+static int
+vmxnet3_xdp_xmit(struct net_device *dev,
+ int n, struct xdp_frame **frames, u32 flags)
+{
+ struct vmxnet3_adapter *adapter;
+ struct vmxnet3_tx_queue *tq;
+ struct netdev_queue *nq;
+ int i, err, cpu;
+ int tq_number;
+ int nxmit_byte = 0, nxmit = 0;
+
+ adapter = netdev_priv(dev);
+
+ if (unlikely(test_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state)))
+ return -ENETDOWN;
+ if (unlikely(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state)))
+ return -EINVAL;
+
+ tq_number = adapter->num_tx_queues;
+ cpu = smp_processor_id();
+ tq = &adapter->tx_queue[cpu % tq_number];
+ nq = netdev_get_tx_queue(adapter->netdev, tq->qid);
+
+ __netif_tx_lock(nq, cpu);
+ for (i = 0; i < n; i++) {
+ err = vmxnet3_xdp_xmit_frame(adapter, frames[i], NULL, tq);
+ if (err) {
+ tq->stats.xdp_xmit_err++;
+ break;
+ }
+ nxmit_byte += frames[i]->len;
+ nxmit++;
+ }
+
+ tq->stats.xdp_xmit += nxmit;
+ __netif_tx_unlock(nq);
+
+ return nxmit;
+}
+
static int
vmxnet3_xdp_xmit_back(struct vmxnet3_adapter *adapter,
struct xdp_frame *xdpf,
@@ -1513,8 +1553,10 @@ vmxnet3_run_xdp(struct vmxnet3_rx_queue *rq, struct sk_buff *skb,
int headroom = XDP_PACKET_HEADROOM;
struct xdp_frame *xdpf;
struct xdp_buff xdp;
+ struct page *page;
void *orig_data;
void *buf_hard_start;
+ int err;
u32 act;
buf_hard_start = skb->data - headroom;
@@ -1557,13 +1599,30 @@ vmxnet3_run_xdp(struct vmxnet3_rx_queue *rq, struct sk_buff *skb,
ctx->skb = NULL;
break;
case XDP_ABORTED:
- ctx->skb = NULL;
trace_xdp_exception(rq->adapter->netdev, rq->xdp_bpf_prog, act);
rq->stats.xdp_aborted++;
+ ctx->skb = NULL;
break;
- case XDP_REDIRECT: /* Not Supported. */
+ case XDP_REDIRECT:
+ page = alloc_page(GFP_ATOMIC);
+ if (!page) {
+ return XDP_DROP;
+ }
+ xdp_init_buff(&xdp, PAGE_SIZE, &rq->xdp_rxq);
+ xdp_prepare_buff(&xdp, page_address(page), headroom, skb->len, false);
+ memcpy(xdp.data, skb->data, skb->len);
+ err = xdp_do_redirect(rq->adapter->netdev, &xdp, rq->xdp_bpf_prog);
+ if (!err) {
+ rq->stats.xdp_redirects++;
+ dev_kfree_skb(ctx->skb);
+ } else {
+ __free_page(page);
+ dev_kfree_skb(ctx->skb);
+ rq->stats.xdp_drops++;
+ }
ctx->skb = NULL;
- fallthrough;
+ *need_xdp_flush = true;
+ break;
default:
bpf_warn_invalid_xdp_action(rq->adapter->netdev,
rq->xdp_bpf_prog, act);
@@ -1943,6 +2002,8 @@ vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
vmxnet3_getRxComp(rcd,
&rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);
}
+ if (need_xdp_flush)
+ xdp_do_flush_map();
return num_pkts;
}
@@ -3937,6 +3998,7 @@ vmxnet3_probe_device(struct pci_dev *pdev,
.ndo_poll_controller = vmxnet3_netpoll,
#endif
.ndo_bpf = vmxnet3_xdp,
+ .ndo_xdp_xmit = vmxnet3_xdp_xmit,
};
int err;
u32 ver;
@@ -76,6 +76,10 @@ vmxnet3_tq_driver_stats[] = {
copy_skb_header) },
{ " giant hdr", offsetof(struct vmxnet3_tq_driver_stats,
oversized_hdr) },
+ { " xdp xmit", offsetof(struct vmxnet3_tq_driver_stats,
+ xdp_xmit) },
+ { " xdp xmit err", offsetof(struct vmxnet3_tq_driver_stats,
+ xdp_xmit_err) },
};
/* per rq stats maintained by the device */
@@ -219,6 +219,9 @@ struct vmxnet3_tq_driver_stats {
u64 linearized; /* # of pkts linearized */
u64 copy_skb_header; /* # of times we have to copy skb header */
u64 oversized_hdr;
+
+ u64 xdp_xmit;
+ u64 xdp_xmit_err;
};
struct vmxnet3_tx_ctx {
The patch adds XDP_REDIRECT support for vmxnet3. A new page is allocated if XDP_REDIRECT is needed, and the packet contents are copied into the new page so that afterward, the original rx buffer can be unmapped/free without any issue. Tested the patch using two VMs, one runs dpdk pktgen, and another runs: $ ./samples/bpf/xdp_redirect_cpu -d ens160 -c 0 -e drop $ ./samples/bpf/xdp_redirect_cpu -d ens160 -c 0 -e pass $ ./samples/bpf/xdp_redirect_cpu -d ens160 -c 0 -r ens160 Results: XDP_DROP 385K pkt/s XDP_PASS 351K pkt/s TX 21K pkt/s (not sure if s.t is wrong here) Need feedbacks: a. I have to allocate a new page before calling xdp_do_redirect. Without doing so, I got several issues such as OOM, CPU soft lockup, invalid page acess. I'm still trying to fix it. b. I don't know whether thse performance number makes sense or not. Signed-off-by: William Tu <u9012063@gmail.com> --- drivers/net/vmxnet3/vmxnet3_drv.c | 68 +++++++++++++++++++++++++-- drivers/net/vmxnet3/vmxnet3_ethtool.c | 4 ++ drivers/net/vmxnet3/vmxnet3_int.h | 3 ++ 3 files changed, 72 insertions(+), 3 deletions(-)