| Message ID | 20201119083024.119566-7-bjorn.topel@gmail.com (mailing list archive) |
|---|---|
| State | New, archived |
| Delegated to: | BPF |
| Headers | show |
| Series | Introduce preferred busy-polling | expand |
On Thu, Nov 19, 2020 at 9:33 AM Björn Töpel <bjorn.topel@gmail.com> wrote: > > From: Björn Töpel <bjorn.topel@intel.com> > > Add napi_id to the xdp_rxq_info structure, and make sure the XDP > socket pick up the napi_id in the Rx path. The napi_id is used to find > the corresponding NAPI structure for socket busy polling. > > Acked-by: Ilias Apalodimas <ilias.apalodimas@linaro.org> > Acked-by: Tariq Toukan <tariqt@nvidia.com> > Signed-off-by: Björn Töpel <bjorn.topel@intel.com> > --- > drivers/net/ethernet/amazon/ena/ena_netdev.c | 2 +- > drivers/net/ethernet/broadcom/bnxt/bnxt.c | 2 +- > .../ethernet/cavium/thunder/nicvf_queues.c | 2 +- > .../net/ethernet/freescale/dpaa2/dpaa2-eth.c | 2 +- > drivers/net/ethernet/intel/i40e/i40e_txrx.c | 2 +- > drivers/net/ethernet/intel/ice/ice_base.c | 4 ++-- > drivers/net/ethernet/intel/ice/ice_txrx.c | 2 +- > drivers/net/ethernet/intel/igb/igb_main.c | 2 +- > drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 2 +- > .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 2 +- > drivers/net/ethernet/marvell/mvneta.c | 2 +- > .../net/ethernet/marvell/mvpp2/mvpp2_main.c | 4 ++-- > drivers/net/ethernet/mellanox/mlx4/en_rx.c | 2 +- > .../net/ethernet/mellanox/mlx5/core/en_main.c | 2 +- > .../ethernet/netronome/nfp/nfp_net_common.c | 2 +- > drivers/net/ethernet/qlogic/qede/qede_main.c | 2 +- > drivers/net/ethernet/sfc/rx_common.c | 2 +- > drivers/net/ethernet/socionext/netsec.c | 2 +- > drivers/net/ethernet/ti/cpsw_priv.c | 2 +- > drivers/net/hyperv/netvsc.c | 2 +- > drivers/net/tun.c | 2 +- > drivers/net/veth.c | 12 ++++++++---- > drivers/net/virtio_net.c | 2 +- > drivers/net/xen-netfront.c | 2 +- > include/net/busy_poll.h | 19 +++++++++++++++---- > include/net/xdp.h | 3 ++- > net/core/dev.c | 2 +- > net/core/xdp.c | 3 ++- > net/xdp/xsk.c | 1 + > 29 files changed, 54 insertions(+), 36 deletions(-) For Intel drivers: Acked-by: Magnus Karlsson <magnus.karlsson@intel.com> > diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c > index e8131dadc22c..6ad59f0068f6 100644 > --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c > +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c > @@ -416,7 +416,7 @@ static int ena_xdp_register_rxq_info(struct ena_ring *rx_ring) > { > int rc; > > - rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid); > + rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid, 0); > > if (rc) { > netif_err(rx_ring->adapter, ifup, rx_ring->netdev, > diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c > index 7975f59735d6..725d929eddb1 100644 > --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c > +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c > @@ -2884,7 +2884,7 @@ static int bnxt_alloc_rx_rings(struct bnxt *bp) > if (rc) > return rc; > > - rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i); > + rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0); > if (rc < 0) > return rc; > > diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c > index 7a141ce32e86..f782e6af45e9 100644 > --- a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c > +++ b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c > @@ -770,7 +770,7 @@ static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs, > rq->caching = 1; > > /* Driver have no proper error path for failed XDP RX-queue info reg */ > - WARN_ON(xdp_rxq_info_reg(&rq->xdp_rxq, nic->netdev, qidx) < 0); > + WARN_ON(xdp_rxq_info_reg(&rq->xdp_rxq, nic->netdev, qidx, 0) < 0); > > /* Send a mailbox msg to PF to config RQ */ > mbx.rq.msg = NIC_MBOX_MSG_RQ_CFG; > diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c > index cf9400a9886d..40953980e846 100644 > --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c > +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c > @@ -3334,7 +3334,7 @@ static int dpaa2_eth_setup_rx_flow(struct dpaa2_eth_priv *priv, > return 0; > > err = xdp_rxq_info_reg(&fq->channel->xdp_rxq, priv->net_dev, > - fq->flowid); > + fq->flowid, 0); > if (err) { > dev_err(dev, "xdp_rxq_info_reg failed\n"); > return err; > diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c > index c21548c71bb1..9f73cd7aee09 100644 > --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c > +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c > @@ -1447,7 +1447,7 @@ int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring) > /* XDP RX-queue info only needed for RX rings exposed to XDP */ > if (rx_ring->vsi->type == I40E_VSI_MAIN) { > err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, > - rx_ring->queue_index); > + rx_ring->queue_index, rx_ring->q_vector->napi.napi_id); > if (err < 0) > return err; > } > diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c > index fe4320e2d1f2..3124a3bf519a 100644 > --- a/drivers/net/ethernet/intel/ice/ice_base.c > +++ b/drivers/net/ethernet/intel/ice/ice_base.c > @@ -306,7 +306,7 @@ int ice_setup_rx_ctx(struct ice_ring *ring) > if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) > /* coverity[check_return] */ > xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, > - ring->q_index); > + ring->q_index, ring->q_vector->napi.napi_id); > > ring->xsk_pool = ice_xsk_pool(ring); > if (ring->xsk_pool) { > @@ -333,7 +333,7 @@ int ice_setup_rx_ctx(struct ice_ring *ring) > /* coverity[check_return] */ > xdp_rxq_info_reg(&ring->xdp_rxq, > ring->netdev, > - ring->q_index); > + ring->q_index, ring->q_vector->napi.napi_id); > > err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, > MEM_TYPE_PAGE_SHARED, > diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c > index eae75260fe20..77d5eae6b4c2 100644 > --- a/drivers/net/ethernet/intel/ice/ice_txrx.c > +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c > @@ -483,7 +483,7 @@ int ice_setup_rx_ring(struct ice_ring *rx_ring) > if (rx_ring->vsi->type == ICE_VSI_PF && > !xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) > if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, > - rx_ring->q_index)) > + rx_ring->q_index, rx_ring->q_vector->napi.napi_id)) > goto err; > return 0; > > diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c > index 5fc2c381da55..6a4ef4934fcf 100644 > --- a/drivers/net/ethernet/intel/igb/igb_main.c > +++ b/drivers/net/ethernet/intel/igb/igb_main.c > @@ -4352,7 +4352,7 @@ int igb_setup_rx_resources(struct igb_ring *rx_ring) > > /* XDP RX-queue info */ > if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, > - rx_ring->queue_index) < 0) > + rx_ring->queue_index, 0) < 0) > goto err; > > return 0; > diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c > index 45ae33e15303..50e6b8b6ba7b 100644 > --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c > +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c > @@ -6577,7 +6577,7 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, > > /* XDP RX-queue info */ > if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, > - rx_ring->queue_index) < 0) > + rx_ring->queue_index, rx_ring->q_vector->napi.napi_id) < 0) > goto err; > > rx_ring->xdp_prog = adapter->xdp_prog; > diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c > index 82fce27f682b..4061cd7db5dd 100644 > --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c > +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c > @@ -3493,7 +3493,7 @@ int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, > > /* XDP RX-queue info */ > if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, > - rx_ring->queue_index) < 0) > + rx_ring->queue_index, 0) < 0) > goto err; > > rx_ring->xdp_prog = adapter->xdp_prog; > diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c > index 183530ed4d1d..ba6dcb19bb1d 100644 > --- a/drivers/net/ethernet/marvell/mvneta.c > +++ b/drivers/net/ethernet/marvell/mvneta.c > @@ -3227,7 +3227,7 @@ static int mvneta_create_page_pool(struct mvneta_port *pp, > return err; > } > > - err = xdp_rxq_info_reg(&rxq->xdp_rxq, pp->dev, rxq->id); > + err = xdp_rxq_info_reg(&rxq->xdp_rxq, pp->dev, rxq->id, 0); > if (err < 0) > goto err_free_pp; > > diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c > index 3069e192d773..5504cbc24970 100644 > --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c > +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c > @@ -2614,11 +2614,11 @@ static int mvpp2_rxq_init(struct mvpp2_port *port, > mvpp2_rxq_status_update(port, rxq->id, 0, rxq->size); > > if (priv->percpu_pools) { > - err = xdp_rxq_info_reg(&rxq->xdp_rxq_short, port->dev, rxq->id); > + err = xdp_rxq_info_reg(&rxq->xdp_rxq_short, port->dev, rxq->id, 0); > if (err < 0) > goto err_free_dma; > > - err = xdp_rxq_info_reg(&rxq->xdp_rxq_long, port->dev, rxq->id); > + err = xdp_rxq_info_reg(&rxq->xdp_rxq_long, port->dev, rxq->id, 0); > if (err < 0) > goto err_unregister_rxq_short; > > diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c > index b0f79a5151cf..40775cb8fb2a 100644 > --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c > +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c > @@ -283,7 +283,7 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, > ring->log_stride = ffs(ring->stride) - 1; > ring->buf_size = ring->size * ring->stride + TXBB_SIZE; > > - if (xdp_rxq_info_reg(&ring->xdp_rxq, priv->dev, queue_index) < 0) > + if (xdp_rxq_info_reg(&ring->xdp_rxq, priv->dev, queue_index, 0) < 0) > goto err_ring; > > tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS * > diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c > index 527c5f12c5af..427fc376fe1a 100644 > --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c > +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c > @@ -434,7 +434,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, > rq_xdp_ix = rq->ix; > if (xsk) > rq_xdp_ix += params->num_channels * MLX5E_RQ_GROUP_XSK; > - err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix); > + err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix, 0); > if (err < 0) > goto err_rq_xdp_prog; > > diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c > index b150da43adb2..b4acf2f41e84 100644 > --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c > +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c > @@ -2533,7 +2533,7 @@ nfp_net_rx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_rx_ring *rx_ring) > > if (dp->netdev) { > err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, dp->netdev, > - rx_ring->idx); > + rx_ring->idx, rx_ring->r_vec->napi.napi_id); > if (err < 0) > return err; > } > diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c > index 05e3a3b60269..9cf960a6d007 100644 > --- a/drivers/net/ethernet/qlogic/qede/qede_main.c > +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c > @@ -1762,7 +1762,7 @@ static void qede_init_fp(struct qede_dev *edev) > > /* Driver have no error path from here */ > WARN_ON(xdp_rxq_info_reg(&fp->rxq->xdp_rxq, edev->ndev, > - fp->rxq->rxq_id) < 0); > + fp->rxq->rxq_id, 0) < 0); > > if (xdp_rxq_info_reg_mem_model(&fp->rxq->xdp_rxq, > MEM_TYPE_PAGE_ORDER0, > diff --git a/drivers/net/ethernet/sfc/rx_common.c b/drivers/net/ethernet/sfc/rx_common.c > index 19cf7cac1e6e..68fc7d317693 100644 > --- a/drivers/net/ethernet/sfc/rx_common.c > +++ b/drivers/net/ethernet/sfc/rx_common.c > @@ -262,7 +262,7 @@ void efx_init_rx_queue(struct efx_rx_queue *rx_queue) > > /* Initialise XDP queue information */ > rc = xdp_rxq_info_reg(&rx_queue->xdp_rxq_info, efx->net_dev, > - rx_queue->core_index); > + rx_queue->core_index, 0); > > if (rc) { > netif_err(efx, rx_err, efx->net_dev, > diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c > index 1503cc9ec6e2..27d3c9d9210e 100644 > --- a/drivers/net/ethernet/socionext/netsec.c > +++ b/drivers/net/ethernet/socionext/netsec.c > @@ -1304,7 +1304,7 @@ static int netsec_setup_rx_dring(struct netsec_priv *priv) > goto err_out; > } > > - err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0); > + err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0, priv->napi.napi_id); > if (err) > goto err_out; > > diff --git a/drivers/net/ethernet/ti/cpsw_priv.c b/drivers/net/ethernet/ti/cpsw_priv.c > index 31c5e36ff706..6dd73bd0f458 100644 > --- a/drivers/net/ethernet/ti/cpsw_priv.c > +++ b/drivers/net/ethernet/ti/cpsw_priv.c > @@ -1186,7 +1186,7 @@ static int cpsw_ndev_create_xdp_rxq(struct cpsw_priv *priv, int ch) > pool = cpsw->page_pool[ch]; > rxq = &priv->xdp_rxq[ch]; > > - ret = xdp_rxq_info_reg(rxq, priv->ndev, ch); > + ret = xdp_rxq_info_reg(rxq, priv->ndev, ch, 0); > if (ret) > return ret; > > diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c > index 0c3de94b5178..fa8341f8359a 100644 > --- a/drivers/net/hyperv/netvsc.c > +++ b/drivers/net/hyperv/netvsc.c > @@ -1499,7 +1499,7 @@ struct netvsc_device *netvsc_device_add(struct hv_device *device, > u64_stats_init(&nvchan->tx_stats.syncp); > u64_stats_init(&nvchan->rx_stats.syncp); > > - ret = xdp_rxq_info_reg(&nvchan->xdp_rxq, ndev, i); > + ret = xdp_rxq_info_reg(&nvchan->xdp_rxq, ndev, i, 0); > > if (ret) { > netdev_err(ndev, "xdp_rxq_info_reg fail: %d\n", ret); > diff --git a/drivers/net/tun.c b/drivers/net/tun.c > index 3d45d56172cb..8867d39db6ac 100644 > --- a/drivers/net/tun.c > +++ b/drivers/net/tun.c > @@ -780,7 +780,7 @@ static int tun_attach(struct tun_struct *tun, struct file *file, > } else { > /* Setup XDP RX-queue info, for new tfile getting attached */ > err = xdp_rxq_info_reg(&tfile->xdp_rxq, > - tun->dev, tfile->queue_index); > + tun->dev, tfile->queue_index, 0); > if (err < 0) > goto out; > err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq, > diff --git a/drivers/net/veth.c b/drivers/net/veth.c > index 8c737668008a..9bd37c7151f8 100644 > --- a/drivers/net/veth.c > +++ b/drivers/net/veth.c > @@ -884,7 +884,6 @@ static int veth_napi_add(struct net_device *dev) > for (i = 0; i < dev->real_num_rx_queues; i++) { > struct veth_rq *rq = &priv->rq[i]; > > - netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); > napi_enable(&rq->xdp_napi); > } > > @@ -926,7 +925,8 @@ static int veth_enable_xdp(struct net_device *dev) > for (i = 0; i < dev->real_num_rx_queues; i++) { > struct veth_rq *rq = &priv->rq[i]; > > - err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i); > + netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); > + err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id); > if (err < 0) > goto err_rxq_reg; > > @@ -952,8 +952,12 @@ static int veth_enable_xdp(struct net_device *dev) > err_reg_mem: > xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); > err_rxq_reg: > - for (i--; i >= 0; i--) > - xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); > + for (i--; i >= 0; i--) { > + struct veth_rq *rq = &priv->rq[i]; > + > + xdp_rxq_info_unreg(&rq->xdp_rxq); > + netif_napi_del(&rq->xdp_napi); > + } > > return err; > } > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > index 21b71148c532..052975ea0af4 100644 > --- a/drivers/net/virtio_net.c > +++ b/drivers/net/virtio_net.c > @@ -1485,7 +1485,7 @@ static int virtnet_open(struct net_device *dev) > if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL)) > schedule_delayed_work(&vi->refill, 0); > > - err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i); > + err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i, vi->rq[i].napi.napi_id); > if (err < 0) > return err; > > diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c > index 920cac4385bf..b01848ef4649 100644 > --- a/drivers/net/xen-netfront.c > +++ b/drivers/net/xen-netfront.c > @@ -2014,7 +2014,7 @@ static int xennet_create_page_pool(struct netfront_queue *queue) > } > > err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev, > - queue->id); > + queue->id, 0); > if (err) { > netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n"); > goto err_free_pp; > diff --git a/include/net/busy_poll.h b/include/net/busy_poll.h > index 2f8f51807b83..45b3e04b99d3 100644 > --- a/include/net/busy_poll.h > +++ b/include/net/busy_poll.h > @@ -135,14 +135,25 @@ static inline void sk_mark_napi_id(struct sock *sk, const struct sk_buff *skb) > sk_rx_queue_set(sk, skb); > } > > -/* variant used for unconnected sockets */ > -static inline void sk_mark_napi_id_once(struct sock *sk, > - const struct sk_buff *skb) > +static inline void __sk_mark_napi_id_once_xdp(struct sock *sk, unsigned int napi_id) > { > #ifdef CONFIG_NET_RX_BUSY_POLL > if (!READ_ONCE(sk->sk_napi_id)) > - WRITE_ONCE(sk->sk_napi_id, skb->napi_id); > + WRITE_ONCE(sk->sk_napi_id, napi_id); > #endif > } > > +/* variant used for unconnected sockets */ > +static inline void sk_mark_napi_id_once(struct sock *sk, > + const struct sk_buff *skb) > +{ > + __sk_mark_napi_id_once_xdp(sk, skb->napi_id); > +} > + > +static inline void sk_mark_napi_id_once_xdp(struct sock *sk, > + const struct xdp_buff *xdp) > +{ > + __sk_mark_napi_id_once_xdp(sk, xdp->rxq->napi_id); > +} > + > #endif /* _LINUX_NET_BUSY_POLL_H */ > diff --git a/include/net/xdp.h b/include/net/xdp.h > index 7d48b2ae217a..700ad5db7f5d 100644 > --- a/include/net/xdp.h > +++ b/include/net/xdp.h > @@ -59,6 +59,7 @@ struct xdp_rxq_info { > u32 queue_index; > u32 reg_state; > struct xdp_mem_info mem; > + unsigned int napi_id; > } ____cacheline_aligned; /* perf critical, avoid false-sharing */ > > struct xdp_txq_info { > @@ -226,7 +227,7 @@ static inline void xdp_release_frame(struct xdp_frame *xdpf) > } > > int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, > - struct net_device *dev, u32 queue_index); > + struct net_device *dev, u32 queue_index, unsigned int napi_id); > void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq); > void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq); > bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq); > diff --git a/net/core/dev.c b/net/core/dev.c > index 7a1e5936c67f..3b6b0e175fe7 100644 > --- a/net/core/dev.c > +++ b/net/core/dev.c > @@ -9810,7 +9810,7 @@ static int netif_alloc_rx_queues(struct net_device *dev) > rx[i].dev = dev; > > /* XDP RX-queue setup */ > - err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i); > + err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0); > if (err < 0) > goto err_rxq_info; > } > diff --git a/net/core/xdp.c b/net/core/xdp.c > index 3d330ebda893..17ffd33c6b18 100644 > --- a/net/core/xdp.c > +++ b/net/core/xdp.c > @@ -158,7 +158,7 @@ static void xdp_rxq_info_init(struct xdp_rxq_info *xdp_rxq) > > /* Returns 0 on success, negative on failure */ > int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, > - struct net_device *dev, u32 queue_index) > + struct net_device *dev, u32 queue_index, unsigned int napi_id) > { > if (xdp_rxq->reg_state == REG_STATE_UNUSED) { > WARN(1, "Driver promised not to register this"); > @@ -179,6 +179,7 @@ int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, > xdp_rxq_info_init(xdp_rxq); > xdp_rxq->dev = dev; > xdp_rxq->queue_index = queue_index; > + xdp_rxq->napi_id = napi_id; > > xdp_rxq->reg_state = REG_STATE_REGISTERED; > return 0; > diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c > index ecc4579e41ee..d4cb1c5c1abf 100644 > --- a/net/xdp/xsk.c > +++ b/net/xdp/xsk.c > @@ -233,6 +233,7 @@ static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, > if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) > return -EINVAL; > > + sk_mark_napi_id_once_xdp(&xs->sk, xdp); > len = xdp->data_end - xdp->data; > > return xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL ? > -- > 2.27.0 >
On Thu, Nov 19, 2020 at 09:30:20AM +0100, Björn Töpel wrote: > From: Björn Töpel <bjorn.topel@intel.com> > > Add napi_id to the xdp_rxq_info structure, and make sure the XDP > socket pick up the napi_id in the Rx path. The napi_id is used to find > the corresponding NAPI structure for socket busy polling. > > Acked-by: Ilias Apalodimas <ilias.apalodimas@linaro.org> > Acked-by: Tariq Toukan <tariqt@nvidia.com> > Signed-off-by: Björn Töpel <bjorn.topel@intel.com> For virtio: Acked-by: Michael S. Tsirkin <mst@redhat.com> > --- > drivers/net/ethernet/amazon/ena/ena_netdev.c | 2 +- > drivers/net/ethernet/broadcom/bnxt/bnxt.c | 2 +- > .../ethernet/cavium/thunder/nicvf_queues.c | 2 +- > .../net/ethernet/freescale/dpaa2/dpaa2-eth.c | 2 +- > drivers/net/ethernet/intel/i40e/i40e_txrx.c | 2 +- > drivers/net/ethernet/intel/ice/ice_base.c | 4 ++-- > drivers/net/ethernet/intel/ice/ice_txrx.c | 2 +- > drivers/net/ethernet/intel/igb/igb_main.c | 2 +- > drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 2 +- > .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 2 +- > drivers/net/ethernet/marvell/mvneta.c | 2 +- > .../net/ethernet/marvell/mvpp2/mvpp2_main.c | 4 ++-- > drivers/net/ethernet/mellanox/mlx4/en_rx.c | 2 +- > .../net/ethernet/mellanox/mlx5/core/en_main.c | 2 +- > .../ethernet/netronome/nfp/nfp_net_common.c | 2 +- > drivers/net/ethernet/qlogic/qede/qede_main.c | 2 +- > drivers/net/ethernet/sfc/rx_common.c | 2 +- > drivers/net/ethernet/socionext/netsec.c | 2 +- > drivers/net/ethernet/ti/cpsw_priv.c | 2 +- > drivers/net/hyperv/netvsc.c | 2 +- > drivers/net/tun.c | 2 +- > drivers/net/veth.c | 12 ++++++++---- > drivers/net/virtio_net.c | 2 +- > drivers/net/xen-netfront.c | 2 +- > include/net/busy_poll.h | 19 +++++++++++++++---- > include/net/xdp.h | 3 ++- > net/core/dev.c | 2 +- > net/core/xdp.c | 3 ++- > net/xdp/xsk.c | 1 + > 29 files changed, 54 insertions(+), 36 deletions(-) > > diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c > index e8131dadc22c..6ad59f0068f6 100644 > --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c > +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c > @@ -416,7 +416,7 @@ static int ena_xdp_register_rxq_info(struct ena_ring *rx_ring) > { > int rc; > > - rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid); > + rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid, 0); > > if (rc) { > netif_err(rx_ring->adapter, ifup, rx_ring->netdev, > diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c > index 7975f59735d6..725d929eddb1 100644 > --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c > +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c > @@ -2884,7 +2884,7 @@ static int bnxt_alloc_rx_rings(struct bnxt *bp) > if (rc) > return rc; > > - rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i); > + rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0); > if (rc < 0) > return rc; > > diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c > index 7a141ce32e86..f782e6af45e9 100644 > --- a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c > +++ b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c > @@ -770,7 +770,7 @@ static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs, > rq->caching = 1; > > /* Driver have no proper error path for failed XDP RX-queue info reg */ > - WARN_ON(xdp_rxq_info_reg(&rq->xdp_rxq, nic->netdev, qidx) < 0); > + WARN_ON(xdp_rxq_info_reg(&rq->xdp_rxq, nic->netdev, qidx, 0) < 0); > > /* Send a mailbox msg to PF to config RQ */ > mbx.rq.msg = NIC_MBOX_MSG_RQ_CFG; > diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c > index cf9400a9886d..40953980e846 100644 > --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c > +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c > @@ -3334,7 +3334,7 @@ static int dpaa2_eth_setup_rx_flow(struct dpaa2_eth_priv *priv, > return 0; > > err = xdp_rxq_info_reg(&fq->channel->xdp_rxq, priv->net_dev, > - fq->flowid); > + fq->flowid, 0); > if (err) { > dev_err(dev, "xdp_rxq_info_reg failed\n"); > return err; > diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c > index c21548c71bb1..9f73cd7aee09 100644 > --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c > +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c > @@ -1447,7 +1447,7 @@ int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring) > /* XDP RX-queue info only needed for RX rings exposed to XDP */ > if (rx_ring->vsi->type == I40E_VSI_MAIN) { > err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, > - rx_ring->queue_index); > + rx_ring->queue_index, rx_ring->q_vector->napi.napi_id); > if (err < 0) > return err; > } > diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c > index fe4320e2d1f2..3124a3bf519a 100644 > --- a/drivers/net/ethernet/intel/ice/ice_base.c > +++ b/drivers/net/ethernet/intel/ice/ice_base.c > @@ -306,7 +306,7 @@ int ice_setup_rx_ctx(struct ice_ring *ring) > if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) > /* coverity[check_return] */ > xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, > - ring->q_index); > + ring->q_index, ring->q_vector->napi.napi_id); > > ring->xsk_pool = ice_xsk_pool(ring); > if (ring->xsk_pool) { > @@ -333,7 +333,7 @@ int ice_setup_rx_ctx(struct ice_ring *ring) > /* coverity[check_return] */ > xdp_rxq_info_reg(&ring->xdp_rxq, > ring->netdev, > - ring->q_index); > + ring->q_index, ring->q_vector->napi.napi_id); > > err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, > MEM_TYPE_PAGE_SHARED, > diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c > index eae75260fe20..77d5eae6b4c2 100644 > --- a/drivers/net/ethernet/intel/ice/ice_txrx.c > +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c > @@ -483,7 +483,7 @@ int ice_setup_rx_ring(struct ice_ring *rx_ring) > if (rx_ring->vsi->type == ICE_VSI_PF && > !xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) > if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, > - rx_ring->q_index)) > + rx_ring->q_index, rx_ring->q_vector->napi.napi_id)) > goto err; > return 0; > > diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c > index 5fc2c381da55..6a4ef4934fcf 100644 > --- a/drivers/net/ethernet/intel/igb/igb_main.c > +++ b/drivers/net/ethernet/intel/igb/igb_main.c > @@ -4352,7 +4352,7 @@ int igb_setup_rx_resources(struct igb_ring *rx_ring) > > /* XDP RX-queue info */ > if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, > - rx_ring->queue_index) < 0) > + rx_ring->queue_index, 0) < 0) > goto err; > > return 0; > diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c > index 45ae33e15303..50e6b8b6ba7b 100644 > --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c > +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c > @@ -6577,7 +6577,7 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, > > /* XDP RX-queue info */ > if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, > - rx_ring->queue_index) < 0) > + rx_ring->queue_index, rx_ring->q_vector->napi.napi_id) < 0) > goto err; > > rx_ring->xdp_prog = adapter->xdp_prog; > diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c > index 82fce27f682b..4061cd7db5dd 100644 > --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c > +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c > @@ -3493,7 +3493,7 @@ int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, > > /* XDP RX-queue info */ > if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, > - rx_ring->queue_index) < 0) > + rx_ring->queue_index, 0) < 0) > goto err; > > rx_ring->xdp_prog = adapter->xdp_prog; > diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c > index 183530ed4d1d..ba6dcb19bb1d 100644 > --- a/drivers/net/ethernet/marvell/mvneta.c > +++ b/drivers/net/ethernet/marvell/mvneta.c > @@ -3227,7 +3227,7 @@ static int mvneta_create_page_pool(struct mvneta_port *pp, > return err; > } > > - err = xdp_rxq_info_reg(&rxq->xdp_rxq, pp->dev, rxq->id); > + err = xdp_rxq_info_reg(&rxq->xdp_rxq, pp->dev, rxq->id, 0); > if (err < 0) > goto err_free_pp; > > diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c > index 3069e192d773..5504cbc24970 100644 > --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c > +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c > @@ -2614,11 +2614,11 @@ static int mvpp2_rxq_init(struct mvpp2_port *port, > mvpp2_rxq_status_update(port, rxq->id, 0, rxq->size); > > if (priv->percpu_pools) { > - err = xdp_rxq_info_reg(&rxq->xdp_rxq_short, port->dev, rxq->id); > + err = xdp_rxq_info_reg(&rxq->xdp_rxq_short, port->dev, rxq->id, 0); > if (err < 0) > goto err_free_dma; > > - err = xdp_rxq_info_reg(&rxq->xdp_rxq_long, port->dev, rxq->id); > + err = xdp_rxq_info_reg(&rxq->xdp_rxq_long, port->dev, rxq->id, 0); > if (err < 0) > goto err_unregister_rxq_short; > > diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c > index b0f79a5151cf..40775cb8fb2a 100644 > --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c > +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c > @@ -283,7 +283,7 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, > ring->log_stride = ffs(ring->stride) - 1; > ring->buf_size = ring->size * ring->stride + TXBB_SIZE; > > - if (xdp_rxq_info_reg(&ring->xdp_rxq, priv->dev, queue_index) < 0) > + if (xdp_rxq_info_reg(&ring->xdp_rxq, priv->dev, queue_index, 0) < 0) > goto err_ring; > > tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS * > diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c > index 527c5f12c5af..427fc376fe1a 100644 > --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c > +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c > @@ -434,7 +434,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, > rq_xdp_ix = rq->ix; > if (xsk) > rq_xdp_ix += params->num_channels * MLX5E_RQ_GROUP_XSK; > - err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix); > + err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix, 0); > if (err < 0) > goto err_rq_xdp_prog; > > diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c > index b150da43adb2..b4acf2f41e84 100644 > --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c > +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c > @@ -2533,7 +2533,7 @@ nfp_net_rx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_rx_ring *rx_ring) > > if (dp->netdev) { > err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, dp->netdev, > - rx_ring->idx); > + rx_ring->idx, rx_ring->r_vec->napi.napi_id); > if (err < 0) > return err; > } > diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c > index 05e3a3b60269..9cf960a6d007 100644 > --- a/drivers/net/ethernet/qlogic/qede/qede_main.c > +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c > @@ -1762,7 +1762,7 @@ static void qede_init_fp(struct qede_dev *edev) > > /* Driver have no error path from here */ > WARN_ON(xdp_rxq_info_reg(&fp->rxq->xdp_rxq, edev->ndev, > - fp->rxq->rxq_id) < 0); > + fp->rxq->rxq_id, 0) < 0); > > if (xdp_rxq_info_reg_mem_model(&fp->rxq->xdp_rxq, > MEM_TYPE_PAGE_ORDER0, > diff --git a/drivers/net/ethernet/sfc/rx_common.c b/drivers/net/ethernet/sfc/rx_common.c > index 19cf7cac1e6e..68fc7d317693 100644 > --- a/drivers/net/ethernet/sfc/rx_common.c > +++ b/drivers/net/ethernet/sfc/rx_common.c > @@ -262,7 +262,7 @@ void efx_init_rx_queue(struct efx_rx_queue *rx_queue) > > /* Initialise XDP queue information */ > rc = xdp_rxq_info_reg(&rx_queue->xdp_rxq_info, efx->net_dev, > - rx_queue->core_index); > + rx_queue->core_index, 0); > > if (rc) { > netif_err(efx, rx_err, efx->net_dev, > diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c > index 1503cc9ec6e2..27d3c9d9210e 100644 > --- a/drivers/net/ethernet/socionext/netsec.c > +++ b/drivers/net/ethernet/socionext/netsec.c > @@ -1304,7 +1304,7 @@ static int netsec_setup_rx_dring(struct netsec_priv *priv) > goto err_out; > } > > - err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0); > + err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0, priv->napi.napi_id); > if (err) > goto err_out; > > diff --git a/drivers/net/ethernet/ti/cpsw_priv.c b/drivers/net/ethernet/ti/cpsw_priv.c > index 31c5e36ff706..6dd73bd0f458 100644 > --- a/drivers/net/ethernet/ti/cpsw_priv.c > +++ b/drivers/net/ethernet/ti/cpsw_priv.c > @@ -1186,7 +1186,7 @@ static int cpsw_ndev_create_xdp_rxq(struct cpsw_priv *priv, int ch) > pool = cpsw->page_pool[ch]; > rxq = &priv->xdp_rxq[ch]; > > - ret = xdp_rxq_info_reg(rxq, priv->ndev, ch); > + ret = xdp_rxq_info_reg(rxq, priv->ndev, ch, 0); > if (ret) > return ret; > > diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c > index 0c3de94b5178..fa8341f8359a 100644 > --- a/drivers/net/hyperv/netvsc.c > +++ b/drivers/net/hyperv/netvsc.c > @@ -1499,7 +1499,7 @@ struct netvsc_device *netvsc_device_add(struct hv_device *device, > u64_stats_init(&nvchan->tx_stats.syncp); > u64_stats_init(&nvchan->rx_stats.syncp); > > - ret = xdp_rxq_info_reg(&nvchan->xdp_rxq, ndev, i); > + ret = xdp_rxq_info_reg(&nvchan->xdp_rxq, ndev, i, 0); > > if (ret) { > netdev_err(ndev, "xdp_rxq_info_reg fail: %d\n", ret); > diff --git a/drivers/net/tun.c b/drivers/net/tun.c > index 3d45d56172cb..8867d39db6ac 100644 > --- a/drivers/net/tun.c > +++ b/drivers/net/tun.c > @@ -780,7 +780,7 @@ static int tun_attach(struct tun_struct *tun, struct file *file, > } else { > /* Setup XDP RX-queue info, for new tfile getting attached */ > err = xdp_rxq_info_reg(&tfile->xdp_rxq, > - tun->dev, tfile->queue_index); > + tun->dev, tfile->queue_index, 0); > if (err < 0) > goto out; > err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq, > diff --git a/drivers/net/veth.c b/drivers/net/veth.c > index 8c737668008a..9bd37c7151f8 100644 > --- a/drivers/net/veth.c > +++ b/drivers/net/veth.c > @@ -884,7 +884,6 @@ static int veth_napi_add(struct net_device *dev) > for (i = 0; i < dev->real_num_rx_queues; i++) { > struct veth_rq *rq = &priv->rq[i]; > > - netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); > napi_enable(&rq->xdp_napi); > } > > @@ -926,7 +925,8 @@ static int veth_enable_xdp(struct net_device *dev) > for (i = 0; i < dev->real_num_rx_queues; i++) { > struct veth_rq *rq = &priv->rq[i]; > > - err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i); > + netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); > + err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id); > if (err < 0) > goto err_rxq_reg; > > @@ -952,8 +952,12 @@ static int veth_enable_xdp(struct net_device *dev) > err_reg_mem: > xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); > err_rxq_reg: > - for (i--; i >= 0; i--) > - xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); > + for (i--; i >= 0; i--) { > + struct veth_rq *rq = &priv->rq[i]; > + > + xdp_rxq_info_unreg(&rq->xdp_rxq); > + netif_napi_del(&rq->xdp_napi); > + } > > return err; > } > diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c > index 21b71148c532..052975ea0af4 100644 > --- a/drivers/net/virtio_net.c > +++ b/drivers/net/virtio_net.c > @@ -1485,7 +1485,7 @@ static int virtnet_open(struct net_device *dev) > if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL)) > schedule_delayed_work(&vi->refill, 0); > > - err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i); > + err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i, vi->rq[i].napi.napi_id); > if (err < 0) > return err; > > diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c > index 920cac4385bf..b01848ef4649 100644 > --- a/drivers/net/xen-netfront.c > +++ b/drivers/net/xen-netfront.c > @@ -2014,7 +2014,7 @@ static int xennet_create_page_pool(struct netfront_queue *queue) > } > > err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev, > - queue->id); > + queue->id, 0); > if (err) { > netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n"); > goto err_free_pp; > diff --git a/include/net/busy_poll.h b/include/net/busy_poll.h > index 2f8f51807b83..45b3e04b99d3 100644 > --- a/include/net/busy_poll.h > +++ b/include/net/busy_poll.h > @@ -135,14 +135,25 @@ static inline void sk_mark_napi_id(struct sock *sk, const struct sk_buff *skb) > sk_rx_queue_set(sk, skb); > } > > -/* variant used for unconnected sockets */ > -static inline void sk_mark_napi_id_once(struct sock *sk, > - const struct sk_buff *skb) > +static inline void __sk_mark_napi_id_once_xdp(struct sock *sk, unsigned int napi_id) > { > #ifdef CONFIG_NET_RX_BUSY_POLL > if (!READ_ONCE(sk->sk_napi_id)) > - WRITE_ONCE(sk->sk_napi_id, skb->napi_id); > + WRITE_ONCE(sk->sk_napi_id, napi_id); > #endif > } > > +/* variant used for unconnected sockets */ > +static inline void sk_mark_napi_id_once(struct sock *sk, > + const struct sk_buff *skb) > +{ > + __sk_mark_napi_id_once_xdp(sk, skb->napi_id); > +} > + > +static inline void sk_mark_napi_id_once_xdp(struct sock *sk, > + const struct xdp_buff *xdp) > +{ > + __sk_mark_napi_id_once_xdp(sk, xdp->rxq->napi_id); > +} > + > #endif /* _LINUX_NET_BUSY_POLL_H */ > diff --git a/include/net/xdp.h b/include/net/xdp.h > index 7d48b2ae217a..700ad5db7f5d 100644 > --- a/include/net/xdp.h > +++ b/include/net/xdp.h > @@ -59,6 +59,7 @@ struct xdp_rxq_info { > u32 queue_index; > u32 reg_state; > struct xdp_mem_info mem; > + unsigned int napi_id; > } ____cacheline_aligned; /* perf critical, avoid false-sharing */ > > struct xdp_txq_info { > @@ -226,7 +227,7 @@ static inline void xdp_release_frame(struct xdp_frame *xdpf) > } > > int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, > - struct net_device *dev, u32 queue_index); > + struct net_device *dev, u32 queue_index, unsigned int napi_id); > void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq); > void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq); > bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq); > diff --git a/net/core/dev.c b/net/core/dev.c > index 7a1e5936c67f..3b6b0e175fe7 100644 > --- a/net/core/dev.c > +++ b/net/core/dev.c > @@ -9810,7 +9810,7 @@ static int netif_alloc_rx_queues(struct net_device *dev) > rx[i].dev = dev; > > /* XDP RX-queue setup */ > - err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i); > + err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0); > if (err < 0) > goto err_rxq_info; > } > diff --git a/net/core/xdp.c b/net/core/xdp.c > index 3d330ebda893..17ffd33c6b18 100644 > --- a/net/core/xdp.c > +++ b/net/core/xdp.c > @@ -158,7 +158,7 @@ static void xdp_rxq_info_init(struct xdp_rxq_info *xdp_rxq) > > /* Returns 0 on success, negative on failure */ > int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, > - struct net_device *dev, u32 queue_index) > + struct net_device *dev, u32 queue_index, unsigned int napi_id) > { > if (xdp_rxq->reg_state == REG_STATE_UNUSED) { > WARN(1, "Driver promised not to register this"); > @@ -179,6 +179,7 @@ int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, > xdp_rxq_info_init(xdp_rxq); > xdp_rxq->dev = dev; > xdp_rxq->queue_index = queue_index; > + xdp_rxq->napi_id = napi_id; > > xdp_rxq->reg_state = REG_STATE_REGISTERED; > return 0; > diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c > index ecc4579e41ee..d4cb1c5c1abf 100644 > --- a/net/xdp/xsk.c > +++ b/net/xdp/xsk.c > @@ -233,6 +233,7 @@ static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, > if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) > return -EINVAL; > > + sk_mark_napi_id_once_xdp(&xs->sk, xdp); > len = xdp->data_end - xdp->data; > > return xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL ? > -- > 2.27.0
Hi "Björn,
I love your patch! Yet something to improve:
[auto build test ERROR on 4e99d115d865d45e17e83478d757b58d8fa66d3c]
url: https://github.com/0day-ci/linux/commits/Bj-rn-T-pel/Introduce-preferred-busy-polling/20210929-234934
base: 4e99d115d865d45e17e83478d757b58d8fa66d3c
config: um-kunit_defconfig (attached as .config)
compiler: gcc-9 (Debian 9.3.0-22) 9.3.0
reproduce (this is a W=1 build):
# https://github.com/0day-ci/linux/commit/f481c00164924dd5d782a92cc67897cc7f804502
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Bj-rn-T-pel/Introduce-preferred-busy-polling/20210929-234934
git checkout f481c00164924dd5d782a92cc67897cc7f804502
# save the attached .config to linux build tree
mkdir build_dir
make W=1 O=build_dir ARCH=um SHELL=/bin/bash
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All errors (new ones prefixed by >>):
cc1: warning: arch/um/include/uapi: No such file or directory [-Wmissing-include-dirs]
In file included from fs/select.c:32:
include/net/busy_poll.h: In function 'sk_mark_napi_id_once':
>> include/net/busy_poll.h:150:36: error: 'const struct sk_buff' has no member named 'napi_id'
150 | __sk_mark_napi_id_once_xdp(sk, skb->napi_id);
| ^~
vim +150 include/net/busy_poll.h
145
146 /* variant used for unconnected sockets */
147 static inline void sk_mark_napi_id_once(struct sock *sk,
148 const struct sk_buff *skb)
149 {
> 150 __sk_mark_napi_id_once_xdp(sk, skb->napi_id);
151 }
152
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
On Wed, Sep 29, 2021 at 8:37 PM kernel test robot <lkp@intel.com> wrote: > > Hi "Björn, > > I love your patch! Yet something to improve: > > [auto build test ERROR on 4e99d115d865d45e17e83478d757b58d8fa66d3c] > > url: https://github.com/0day-ci/linux/commits/Bj-rn-T-pel/Introduce-preferred-busy-polling/20210929-234934 > base: 4e99d115d865d45e17e83478d757b58d8fa66d3c > config: um-kunit_defconfig (attached as .config) > compiler: gcc-9 (Debian 9.3.0-22) 9.3.0 > reproduce (this is a W=1 build): > # https://github.com/0day-ci/linux/commit/f481c00164924dd5d782a92cc67897cc7f804502 > git remote add linux-review https://github.com/0day-ci/linux > git fetch --no-tags linux-review Bj-rn-T-pel/Introduce-preferred-busy-polling/20210929-234934 > git checkout f481c00164924dd5d782a92cc67897cc7f804502 > # save the attached .config to linux build tree > mkdir build_dir > make W=1 O=build_dir ARCH=um SHELL=/bin/bash > > If you fix the issue, kindly add following tag as appropriate > Reported-by: kernel test robot <lkp@intel.com> > > All errors (new ones prefixed by >>): > > cc1: warning: arch/um/include/uapi: No such file or directory [-Wmissing-include-dirs] > In file included from fs/select.c:32: > include/net/busy_poll.h: In function 'sk_mark_napi_id_once': > >> include/net/busy_poll.h:150:36: error: 'const struct sk_buff' has no member named 'napi_id' > 150 | __sk_mark_napi_id_once_xdp(sk, skb->napi_id); > | ^~ > > > vim +150 include/net/busy_poll.h > > 145 > 146 /* variant used for unconnected sockets */ > 147 static inline void sk_mark_napi_id_once(struct sock *sk, > 148 const struct sk_buff *skb) > 149 { > > 150 __sk_mark_napi_id_once_xdp(sk, skb->napi_id); > 151 } > 152 It seems that the robot tested an old commit and that this was already fixed by Daniel 10 months ago. Slow mail delivery, a robot glitch, or am I missing something? commit ba0581749fec389e55c9d761f2716f8fcbefced5 Author: Daniel Borkmann <daniel@iogearbox.net> Date: Tue Dec 1 15:22:59 2020 +0100 net, xdp, xsk: fix __sk_mark_napi_id_once napi_id error Stephen reported the following build error for !CONFIG_NET_RX_BUSY_POLL built kernels: In file included from fs/select.c:32: include/net/busy_poll.h: In function 'sk_mark_napi_id_once': include/net/busy_poll.h:150:36: error: 'const struct sk_buff' has no member named 'napi_id' 150 | __sk_mark_napi_id_once_xdp(sk, skb->napi_id); | ^~ Fix it by wrapping a CONFIG_NET_RX_BUSY_POLL around the helpers. Fixes: b02e5a0ebb17 ("xsk: Propagate napi_id to XDP socket Rx path") Reported-by: Stephen Rothwell <sfr@canb.auug.org.au> Signed-off-by: Stephen Rothwell <sfr@canb.auug.org.au> Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> Cc: Björn Töpel <bjorn.topel@intel.com> Link: https://lore.kernel.org/linux-next/20201201190746.7d3357fb@canb.auug.org.au > --- > 0-DAY CI Kernel Test Service, Intel Corporation > https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
On Thu, Sep 30, 2021 at 08:04:06AM +0200, Magnus Karlsson wrote: > On Wed, Sep 29, 2021 at 8:37 PM kernel test robot <lkp@intel.com> wrote: > > > > Hi "Björn, > > > > I love your patch! Yet something to improve: > > > > [auto build test ERROR on 4e99d115d865d45e17e83478d757b58d8fa66d3c] > > > > url: https://github.com/0day-ci/linux/commits/Bj-rn-T-pel/Introduce-preferred-busy-polling/20210929-234934 > > base: 4e99d115d865d45e17e83478d757b58d8fa66d3c > > config: um-kunit_defconfig (attached as .config) > > compiler: gcc-9 (Debian 9.3.0-22) 9.3.0 > > reproduce (this is a W=1 build): > > # https://github.com/0day-ci/linux/commit/f481c00164924dd5d782a92cc67897cc7f804502 > > git remote add linux-review https://github.com/0day-ci/linux > > git fetch --no-tags linux-review Bj-rn-T-pel/Introduce-preferred-busy-polling/20210929-234934 > > git checkout f481c00164924dd5d782a92cc67897cc7f804502 > > # save the attached .config to linux build tree > > mkdir build_dir > > make W=1 O=build_dir ARCH=um SHELL=/bin/bash > > > > If you fix the issue, kindly add following tag as appropriate > > Reported-by: kernel test robot <lkp@intel.com> > > > > All errors (new ones prefixed by >>): > > > > cc1: warning: arch/um/include/uapi: No such file or directory [-Wmissing-include-dirs] > > In file included from fs/select.c:32: > > include/net/busy_poll.h: In function 'sk_mark_napi_id_once': > > >> include/net/busy_poll.h:150:36: error: 'const struct sk_buff' has no member named 'napi_id' > > 150 | __sk_mark_napi_id_once_xdp(sk, skb->napi_id); > > | ^~ > > > > > > vim +150 include/net/busy_poll.h > > > > 145 > > 146 /* variant used for unconnected sockets */ > > 147 static inline void sk_mark_napi_id_once(struct sock *sk, > > 148 const struct sk_buff *skb) > > 149 { > > > 150 __sk_mark_napi_id_once_xdp(sk, skb->napi_id); > > 151 } > > 152 > > It seems that the robot tested an old commit and that this was already > fixed by Daniel 10 months ago. Slow mail delivery, a robot glitch, or > am I missing something? sorry for the noise, we got storage crash and it seems to have old data back in test queue after recovery. We will do cleanup to avoid this kind of old commit being tested. > > commit ba0581749fec389e55c9d761f2716f8fcbefced5 > Author: Daniel Borkmann <daniel@iogearbox.net> > Date: Tue Dec 1 15:22:59 2020 +0100 > > net, xdp, xsk: fix __sk_mark_napi_id_once napi_id error > > Stephen reported the following build error for !CONFIG_NET_RX_BUSY_POLL > built kernels: > > In file included from fs/select.c:32: > include/net/busy_poll.h: In function 'sk_mark_napi_id_once': > include/net/busy_poll.h:150:36: error: 'const struct sk_buff' > has no member named 'napi_id' > 150 | __sk_mark_napi_id_once_xdp(sk, skb->napi_id); > | ^~ > > Fix it by wrapping a CONFIG_NET_RX_BUSY_POLL around the helpers. > > Fixes: b02e5a0ebb17 ("xsk: Propagate napi_id to XDP socket Rx path") > Reported-by: Stephen Rothwell <sfr@canb.auug.org.au> > Signed-off-by: Stephen Rothwell <sfr@canb.auug.org.au> > Signed-off-by: Daniel Borkmann <daniel@iogearbox.net> > Cc: Björn Töpel <bjorn.topel@intel.com> > Link: https://lore.kernel.org/linux-next/20201201190746.7d3357fb@canb.auug.org.au > > > > --- > > 0-DAY CI Kernel Test Service, Intel Corporation > > https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org > _______________________________________________ > kbuild-all mailing list -- kbuild-all@lists.01.org > To unsubscribe send an email to kbuild-all-leave@lists.01.org
Hi "Björn,
I love your patch! Yet something to improve:
[auto build test ERROR on 4e99d115d865d45e17e83478d757b58d8fa66d3c]
url: https://github.com/0day-ci/linux/commits/Bj-rn-T-pel/Introduce-preferred-busy-polling/20210929-234934
base: 4e99d115d865d45e17e83478d757b58d8fa66d3c
config: riscv-randconfig-r042-20210929 (attached as .config)
compiler: clang version 14.0.0 (https://github.com/llvm/llvm-project dc6e8dfdfe7efecfda318d43a06fae18b40eb498)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# install riscv cross compiling tool for clang build
# apt-get install binutils-riscv64-linux-gnu
# https://github.com/0day-ci/linux/commit/f481c00164924dd5d782a92cc67897cc7f804502
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Bj-rn-T-pel/Introduce-preferred-busy-polling/20210929-234934
git checkout f481c00164924dd5d782a92cc67897cc7f804502
# save the attached .config to linux build tree
mkdir build_dir
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=riscv SHELL=/bin/bash
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <lkp@intel.com>
All errors (new ones prefixed by >>):
In file included from fs/select.c:32:
In file included from include/net/busy_poll.h:15:
In file included from include/linux/netdevice.h:37:
In file included from include/linux/ethtool.h:18:
In file included from include/uapi/linux/ethtool.h:19:
In file included from include/linux/if_ether.h:19:
include/linux/skbuff.h:4622:26: error: implicit declaration of function 'skb_ext_add' [-Werror,-Wimplicit-function-declaration]
u64 *kcov_handle_ptr = skb_ext_add(skb, SKB_EXT_KCOV_HANDLE);
^
include/linux/skbuff.h:4622:43: error: use of undeclared identifier 'SKB_EXT_KCOV_HANDLE'
u64 *kcov_handle_ptr = skb_ext_add(skb, SKB_EXT_KCOV_HANDLE);
^
include/linux/skbuff.h:4631:21: error: implicit declaration of function 'skb_ext_find' [-Werror,-Wimplicit-function-declaration]
u64 *kcov_handle = skb_ext_find(skb, SKB_EXT_KCOV_HANDLE);
^
include/linux/skbuff.h:4631:39: error: use of undeclared identifier 'SKB_EXT_KCOV_HANDLE'
u64 *kcov_handle = skb_ext_find(skb, SKB_EXT_KCOV_HANDLE);
^
In file included from fs/select.c:32:
>> include/net/busy_poll.h:150:38: error: no member named 'napi_id' in 'struct sk_buff'
__sk_mark_napi_id_once_xdp(sk, skb->napi_id);
~~~ ^
5 errors generated.
vim +150 include/net/busy_poll.h
145
146 /* variant used for unconnected sockets */
147 static inline void sk_mark_napi_id_once(struct sock *sk,
148 const struct sk_buff *skb)
149 {
> 150 __sk_mark_napi_id_once_xdp(sk, skb->napi_id);
151 }
152
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-all@lists.01.org
diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c index e8131dadc22c..6ad59f0068f6 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c @@ -416,7 +416,7 @@ static int ena_xdp_register_rxq_info(struct ena_ring *rx_ring) { int rc; - rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid); + rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid, 0); if (rc) { netif_err(rx_ring->adapter, ifup, rx_ring->netdev, diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 7975f59735d6..725d929eddb1 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -2884,7 +2884,7 @@ static int bnxt_alloc_rx_rings(struct bnxt *bp) if (rc) return rc; - rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i); + rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i, 0); if (rc < 0) return rc; diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c index 7a141ce32e86..f782e6af45e9 100644 --- a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c +++ b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c @@ -770,7 +770,7 @@ static void nicvf_rcv_queue_config(struct nicvf *nic, struct queue_set *qs, rq->caching = 1; /* Driver have no proper error path for failed XDP RX-queue info reg */ - WARN_ON(xdp_rxq_info_reg(&rq->xdp_rxq, nic->netdev, qidx) < 0); + WARN_ON(xdp_rxq_info_reg(&rq->xdp_rxq, nic->netdev, qidx, 0) < 0); /* Send a mailbox msg to PF to config RQ */ mbx.rq.msg = NIC_MBOX_MSG_RQ_CFG; diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c index cf9400a9886d..40953980e846 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c @@ -3334,7 +3334,7 @@ static int dpaa2_eth_setup_rx_flow(struct dpaa2_eth_priv *priv, return 0; err = xdp_rxq_info_reg(&fq->channel->xdp_rxq, priv->net_dev, - fq->flowid); + fq->flowid, 0); if (err) { dev_err(dev, "xdp_rxq_info_reg failed\n"); return err; diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index c21548c71bb1..9f73cd7aee09 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -1447,7 +1447,7 @@ int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring) /* XDP RX-queue info only needed for RX rings exposed to XDP */ if (rx_ring->vsi->type == I40E_VSI_MAIN) { err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, - rx_ring->queue_index); + rx_ring->queue_index, rx_ring->q_vector->napi.napi_id); if (err < 0) return err; } diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c index fe4320e2d1f2..3124a3bf519a 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.c +++ b/drivers/net/ethernet/intel/ice/ice_base.c @@ -306,7 +306,7 @@ int ice_setup_rx_ctx(struct ice_ring *ring) if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) /* coverity[check_return] */ xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, - ring->q_index); + ring->q_index, ring->q_vector->napi.napi_id); ring->xsk_pool = ice_xsk_pool(ring); if (ring->xsk_pool) { @@ -333,7 +333,7 @@ int ice_setup_rx_ctx(struct ice_ring *ring) /* coverity[check_return] */ xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, - ring->q_index); + ring->q_index, ring->q_vector->napi.napi_id); err = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_PAGE_SHARED, diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index eae75260fe20..77d5eae6b4c2 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -483,7 +483,7 @@ int ice_setup_rx_ring(struct ice_ring *rx_ring) if (rx_ring->vsi->type == ICE_VSI_PF && !xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, - rx_ring->q_index)) + rx_ring->q_index, rx_ring->q_vector->napi.napi_id)) goto err; return 0; diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index 5fc2c381da55..6a4ef4934fcf 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -4352,7 +4352,7 @@ int igb_setup_rx_resources(struct igb_ring *rx_ring) /* XDP RX-queue info */ if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, - rx_ring->queue_index) < 0) + rx_ring->queue_index, 0) < 0) goto err; return 0; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 45ae33e15303..50e6b8b6ba7b 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -6577,7 +6577,7 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, /* XDP RX-queue info */ if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, - rx_ring->queue_index) < 0) + rx_ring->queue_index, rx_ring->q_vector->napi.napi_id) < 0) goto err; rx_ring->xdp_prog = adapter->xdp_prog; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 82fce27f682b..4061cd7db5dd 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -3493,7 +3493,7 @@ int ixgbevf_setup_rx_resources(struct ixgbevf_adapter *adapter, /* XDP RX-queue info */ if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, - rx_ring->queue_index) < 0) + rx_ring->queue_index, 0) < 0) goto err; rx_ring->xdp_prog = adapter->xdp_prog; diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 183530ed4d1d..ba6dcb19bb1d 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -3227,7 +3227,7 @@ static int mvneta_create_page_pool(struct mvneta_port *pp, return err; } - err = xdp_rxq_info_reg(&rxq->xdp_rxq, pp->dev, rxq->id); + err = xdp_rxq_info_reg(&rxq->xdp_rxq, pp->dev, rxq->id, 0); if (err < 0) goto err_free_pp; diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index 3069e192d773..5504cbc24970 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -2614,11 +2614,11 @@ static int mvpp2_rxq_init(struct mvpp2_port *port, mvpp2_rxq_status_update(port, rxq->id, 0, rxq->size); if (priv->percpu_pools) { - err = xdp_rxq_info_reg(&rxq->xdp_rxq_short, port->dev, rxq->id); + err = xdp_rxq_info_reg(&rxq->xdp_rxq_short, port->dev, rxq->id, 0); if (err < 0) goto err_free_dma; - err = xdp_rxq_info_reg(&rxq->xdp_rxq_long, port->dev, rxq->id); + err = xdp_rxq_info_reg(&rxq->xdp_rxq_long, port->dev, rxq->id, 0); if (err < 0) goto err_unregister_rxq_short; diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c b/drivers/net/ethernet/mellanox/mlx4/en_rx.c index b0f79a5151cf..40775cb8fb2a 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c @@ -283,7 +283,7 @@ int mlx4_en_create_rx_ring(struct mlx4_en_priv *priv, ring->log_stride = ffs(ring->stride) - 1; ring->buf_size = ring->size * ring->stride + TXBB_SIZE; - if (xdp_rxq_info_reg(&ring->xdp_rxq, priv->dev, queue_index) < 0) + if (xdp_rxq_info_reg(&ring->xdp_rxq, priv->dev, queue_index, 0) < 0) goto err_ring; tmp = size * roundup_pow_of_two(MLX4_EN_MAX_RX_FRAGS * diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 527c5f12c5af..427fc376fe1a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -434,7 +434,7 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c, rq_xdp_ix = rq->ix; if (xsk) rq_xdp_ix += params->num_channels * MLX5E_RQ_GROUP_XSK; - err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix); + err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix, 0); if (err < 0) goto err_rq_xdp_prog; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index b150da43adb2..b4acf2f41e84 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -2533,7 +2533,7 @@ nfp_net_rx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_rx_ring *rx_ring) if (dp->netdev) { err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, dp->netdev, - rx_ring->idx); + rx_ring->idx, rx_ring->r_vec->napi.napi_id); if (err < 0) return err; } diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c index 05e3a3b60269..9cf960a6d007 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_main.c +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c @@ -1762,7 +1762,7 @@ static void qede_init_fp(struct qede_dev *edev) /* Driver have no error path from here */ WARN_ON(xdp_rxq_info_reg(&fp->rxq->xdp_rxq, edev->ndev, - fp->rxq->rxq_id) < 0); + fp->rxq->rxq_id, 0) < 0); if (xdp_rxq_info_reg_mem_model(&fp->rxq->xdp_rxq, MEM_TYPE_PAGE_ORDER0, diff --git a/drivers/net/ethernet/sfc/rx_common.c b/drivers/net/ethernet/sfc/rx_common.c index 19cf7cac1e6e..68fc7d317693 100644 --- a/drivers/net/ethernet/sfc/rx_common.c +++ b/drivers/net/ethernet/sfc/rx_common.c @@ -262,7 +262,7 @@ void efx_init_rx_queue(struct efx_rx_queue *rx_queue) /* Initialise XDP queue information */ rc = xdp_rxq_info_reg(&rx_queue->xdp_rxq_info, efx->net_dev, - rx_queue->core_index); + rx_queue->core_index, 0); if (rc) { netif_err(efx, rx_err, efx->net_dev, diff --git a/drivers/net/ethernet/socionext/netsec.c b/drivers/net/ethernet/socionext/netsec.c index 1503cc9ec6e2..27d3c9d9210e 100644 --- a/drivers/net/ethernet/socionext/netsec.c +++ b/drivers/net/ethernet/socionext/netsec.c @@ -1304,7 +1304,7 @@ static int netsec_setup_rx_dring(struct netsec_priv *priv) goto err_out; } - err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0); + err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0, priv->napi.napi_id); if (err) goto err_out; diff --git a/drivers/net/ethernet/ti/cpsw_priv.c b/drivers/net/ethernet/ti/cpsw_priv.c index 31c5e36ff706..6dd73bd0f458 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.c +++ b/drivers/net/ethernet/ti/cpsw_priv.c @@ -1186,7 +1186,7 @@ static int cpsw_ndev_create_xdp_rxq(struct cpsw_priv *priv, int ch) pool = cpsw->page_pool[ch]; rxq = &priv->xdp_rxq[ch]; - ret = xdp_rxq_info_reg(rxq, priv->ndev, ch); + ret = xdp_rxq_info_reg(rxq, priv->ndev, ch, 0); if (ret) return ret; diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c index 0c3de94b5178..fa8341f8359a 100644 --- a/drivers/net/hyperv/netvsc.c +++ b/drivers/net/hyperv/netvsc.c @@ -1499,7 +1499,7 @@ struct netvsc_device *netvsc_device_add(struct hv_device *device, u64_stats_init(&nvchan->tx_stats.syncp); u64_stats_init(&nvchan->rx_stats.syncp); - ret = xdp_rxq_info_reg(&nvchan->xdp_rxq, ndev, i); + ret = xdp_rxq_info_reg(&nvchan->xdp_rxq, ndev, i, 0); if (ret) { netdev_err(ndev, "xdp_rxq_info_reg fail: %d\n", ret); diff --git a/drivers/net/tun.c b/drivers/net/tun.c index 3d45d56172cb..8867d39db6ac 100644 --- a/drivers/net/tun.c +++ b/drivers/net/tun.c @@ -780,7 +780,7 @@ static int tun_attach(struct tun_struct *tun, struct file *file, } else { /* Setup XDP RX-queue info, for new tfile getting attached */ err = xdp_rxq_info_reg(&tfile->xdp_rxq, - tun->dev, tfile->queue_index); + tun->dev, tfile->queue_index, 0); if (err < 0) goto out; err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq, diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 8c737668008a..9bd37c7151f8 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -884,7 +884,6 @@ static int veth_napi_add(struct net_device *dev) for (i = 0; i < dev->real_num_rx_queues; i++) { struct veth_rq *rq = &priv->rq[i]; - netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); napi_enable(&rq->xdp_napi); } @@ -926,7 +925,8 @@ static int veth_enable_xdp(struct net_device *dev) for (i = 0; i < dev->real_num_rx_queues; i++) { struct veth_rq *rq = &priv->rq[i]; - err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i); + netif_napi_add(dev, &rq->xdp_napi, veth_poll, NAPI_POLL_WEIGHT); + err = xdp_rxq_info_reg(&rq->xdp_rxq, dev, i, rq->xdp_napi.napi_id); if (err < 0) goto err_rxq_reg; @@ -952,8 +952,12 @@ static int veth_enable_xdp(struct net_device *dev) err_reg_mem: xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); err_rxq_reg: - for (i--; i >= 0; i--) - xdp_rxq_info_unreg(&priv->rq[i].xdp_rxq); + for (i--; i >= 0; i--) { + struct veth_rq *rq = &priv->rq[i]; + + xdp_rxq_info_unreg(&rq->xdp_rxq); + netif_napi_del(&rq->xdp_napi); + } return err; } diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 21b71148c532..052975ea0af4 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -1485,7 +1485,7 @@ static int virtnet_open(struct net_device *dev) if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL)) schedule_delayed_work(&vi->refill, 0); - err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i); + err = xdp_rxq_info_reg(&vi->rq[i].xdp_rxq, dev, i, vi->rq[i].napi.napi_id); if (err < 0) return err; diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index 920cac4385bf..b01848ef4649 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -2014,7 +2014,7 @@ static int xennet_create_page_pool(struct netfront_queue *queue) } err = xdp_rxq_info_reg(&queue->xdp_rxq, queue->info->netdev, - queue->id); + queue->id, 0); if (err) { netdev_err(queue->info->netdev, "xdp_rxq_info_reg failed\n"); goto err_free_pp; diff --git a/include/net/busy_poll.h b/include/net/busy_poll.h index 2f8f51807b83..45b3e04b99d3 100644 --- a/include/net/busy_poll.h +++ b/include/net/busy_poll.h @@ -135,14 +135,25 @@ static inline void sk_mark_napi_id(struct sock *sk, const struct sk_buff *skb) sk_rx_queue_set(sk, skb); } -/* variant used for unconnected sockets */ -static inline void sk_mark_napi_id_once(struct sock *sk, - const struct sk_buff *skb) +static inline void __sk_mark_napi_id_once_xdp(struct sock *sk, unsigned int napi_id) { #ifdef CONFIG_NET_RX_BUSY_POLL if (!READ_ONCE(sk->sk_napi_id)) - WRITE_ONCE(sk->sk_napi_id, skb->napi_id); + WRITE_ONCE(sk->sk_napi_id, napi_id); #endif } +/* variant used for unconnected sockets */ +static inline void sk_mark_napi_id_once(struct sock *sk, + const struct sk_buff *skb) +{ + __sk_mark_napi_id_once_xdp(sk, skb->napi_id); +} + +static inline void sk_mark_napi_id_once_xdp(struct sock *sk, + const struct xdp_buff *xdp) +{ + __sk_mark_napi_id_once_xdp(sk, xdp->rxq->napi_id); +} + #endif /* _LINUX_NET_BUSY_POLL_H */ diff --git a/include/net/xdp.h b/include/net/xdp.h index 7d48b2ae217a..700ad5db7f5d 100644 --- a/include/net/xdp.h +++ b/include/net/xdp.h @@ -59,6 +59,7 @@ struct xdp_rxq_info { u32 queue_index; u32 reg_state; struct xdp_mem_info mem; + unsigned int napi_id; } ____cacheline_aligned; /* perf critical, avoid false-sharing */ struct xdp_txq_info { @@ -226,7 +227,7 @@ static inline void xdp_release_frame(struct xdp_frame *xdpf) } int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, - struct net_device *dev, u32 queue_index); + struct net_device *dev, u32 queue_index, unsigned int napi_id); void xdp_rxq_info_unreg(struct xdp_rxq_info *xdp_rxq); void xdp_rxq_info_unused(struct xdp_rxq_info *xdp_rxq); bool xdp_rxq_info_is_reg(struct xdp_rxq_info *xdp_rxq); diff --git a/net/core/dev.c b/net/core/dev.c index 7a1e5936c67f..3b6b0e175fe7 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -9810,7 +9810,7 @@ static int netif_alloc_rx_queues(struct net_device *dev) rx[i].dev = dev; /* XDP RX-queue setup */ - err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i); + err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0); if (err < 0) goto err_rxq_info; } diff --git a/net/core/xdp.c b/net/core/xdp.c index 3d330ebda893..17ffd33c6b18 100644 --- a/net/core/xdp.c +++ b/net/core/xdp.c @@ -158,7 +158,7 @@ static void xdp_rxq_info_init(struct xdp_rxq_info *xdp_rxq) /* Returns 0 on success, negative on failure */ int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, - struct net_device *dev, u32 queue_index) + struct net_device *dev, u32 queue_index, unsigned int napi_id) { if (xdp_rxq->reg_state == REG_STATE_UNUSED) { WARN(1, "Driver promised not to register this"); @@ -179,6 +179,7 @@ int xdp_rxq_info_reg(struct xdp_rxq_info *xdp_rxq, xdp_rxq_info_init(xdp_rxq); xdp_rxq->dev = dev; xdp_rxq->queue_index = queue_index; + xdp_rxq->napi_id = napi_id; xdp_rxq->reg_state = REG_STATE_REGISTERED; return 0; diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index ecc4579e41ee..d4cb1c5c1abf 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -233,6 +233,7 @@ static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp, if (xs->dev != xdp->rxq->dev || xs->queue_id != xdp->rxq->queue_index) return -EINVAL; + sk_mark_napi_id_once_xdp(&xs->sk, xdp); len = xdp->data_end - xdp->data; return xdp->rxq->mem.type == MEM_TYPE_XSK_BUFF_POOL ?