@@ -2225,6 +2225,7 @@ struct net_device {
struct pcpu_lstats __percpu *lstats;
struct pcpu_sw_netstats __percpu *tstats;
struct pcpu_dstats __percpu *dstats;
+ struct xdp_drv_stats /* per-channel */ *xstats;
};
#if IS_ENABLED(CONFIG_GARP)
@@ -7,6 +7,7 @@
#define __LINUX_NET_XDP_H__
#include <linux/skbuff.h> /* skb_shared_info */
+#include <linux/u64_stats_sync.h> /* u64_stats_* */
/**
* DOC: XDP RX-queue information
@@ -292,4 +293,165 @@ void xdp_attachment_setup(struct xdp_attachment_info *info,
#define DEV_MAP_BULK_SIZE XDP_BULK_QUEUE_SIZE
+/* Suggested XDP/XSK driver stats, mirror &ifla_xdp_stats except
+ * for generic errors, refer to its documentation for the details.
+ * The intended usage is to either have them as a standalone array
+ * of xdp_drv_stats, or embed &xdp_{rx,tx}_drv_stats into a ring
+ * structure. Having separate XDP and XSK counters is recommended.
+ */
+
+struct ifla_xdp_stats;
+
+struct xdp_rx_drv_stats {
+ struct u64_stats_sync syncp;
+ u64_stats_t packets;
+ u64_stats_t bytes;
+ u64_stats_t pass;
+ u64_stats_t drop;
+ u64_stats_t redirect;
+ u64_stats_t tx;
+ u64_stats_t redirect_errors;
+ u64_stats_t tx_errors;
+ u64_stats_t aborted;
+ u64_stats_t invalid;
+};
+
+struct xdp_tx_drv_stats {
+ struct u64_stats_sync syncp;
+ u64_stats_t packets;
+ u64_stats_t bytes;
+ u64_stats_t errors;
+ u64_stats_t full;
+};
+
+struct xdp_drv_stats {
+ struct xdp_rx_drv_stats xdp_rx;
+ struct xdp_tx_drv_stats xdp_tx;
+ struct xdp_rx_drv_stats xsk_rx ____cacheline_aligned;
+ struct xdp_tx_drv_stats xsk_tx;
+} ____cacheline_aligned;
+
+/* Shortened copy of Rx stats to put on stack */
+struct xdp_rx_drv_stats_local {
+ u32 bytes;
+ u32 packets;
+ u32 pass;
+ u32 drop;
+ u32 tx;
+ u32 tx_errors;
+ u32 redirect;
+ u32 redirect_errors;
+ u32 aborted;
+ u32 invalid;
+};
+
+#define xdp_init_rx_drv_stats(rstats) u64_stats_init(&(rstats)->syncp)
+#define xdp_init_tx_drv_stats(tstats) u64_stats_init(&(tstats)->syncp)
+
+/**
+ * xdp_init_drv_stats - initialize driver XDP stats
+ * @xdp_stats: driver container if it uses generic xdp_drv_stats
+ *
+ * Initializes atomic/seqcount sync points inside the containers.
+ */
+static inline void xdp_init_drv_stats(struct xdp_drv_stats *xdp_stats)
+{
+ xdp_init_rx_drv_stats(&xdp_stats->xdp_rx);
+ xdp_init_tx_drv_stats(&xdp_stats->xdp_tx);
+ xdp_init_rx_drv_stats(&xdp_stats->xsk_rx);
+ xdp_init_tx_drv_stats(&xdp_stats->xsk_tx);
+}
+
+/**
+ * xdp_update_rx_drv_stats - update driver XDP stats
+ * @rstats: target driver container
+ * @lrstats: filled onstack structure
+ *
+ * Fetches Rx path XDP statistics from the onstack structure to the
+ * driver container, respecting atomic/seqcount synchronization.
+ * Typical usage is to call it at the end of Rx NAPI polling.
+ */
+static inline void
+xdp_update_rx_drv_stats(struct xdp_rx_drv_stats *rstats,
+ const struct xdp_rx_drv_stats_local *lrstats)
+{
+ if (!lrstats->packets)
+ return;
+
+ u64_stats_update_begin(&rstats->syncp);
+ u64_stats_add(&rstats->packets, lrstats->packets);
+ u64_stats_add(&rstats->bytes, lrstats->bytes);
+ u64_stats_add(&rstats->pass, lrstats->pass);
+ u64_stats_add(&rstats->drop, lrstats->drop);
+ u64_stats_add(&rstats->redirect, lrstats->redirect);
+ u64_stats_add(&rstats->tx, lrstats->tx);
+ u64_stats_add(&rstats->redirect_errors, lrstats->redirect_errors);
+ u64_stats_add(&rstats->tx_errors, lrstats->tx_errors);
+ u64_stats_add(&rstats->aborted, lrstats->aborted);
+ u64_stats_add(&rstats->invalid, lrstats->invalid);
+ u64_stats_update_end(&rstats->syncp);
+}
+
+/**
+ * xdp_update_tx_drv_stats - update driver XDP stats
+ * @tstats: target driver container
+ * @packets: onstack packet counter
+ * @bytes: onstack octete counter
+ *
+ * Adds onstack packet/byte Tx XDP counter values from the current session
+ * to the driver container. Typical usage is to call it on completion path /
+ * Tx NAPI polling.
+ */
+static inline void xdp_update_tx_drv_stats(struct xdp_tx_drv_stats *tstats,
+ u32 packets, u32 bytes)
+{
+ if (!packets)
+ return;
+
+ u64_stats_update_begin(&tstats->syncp);
+ u64_stats_add(&tstats->packets, packets);
+ u64_stats_add(&tstats->bytes, bytes);
+ u64_stats_update_end(&tstats->syncp);
+}
+
+/**
+ * xdp_update_tx_drv_err - update driver Tx XDP errors counter
+ * @tstats: target driver container
+ * @num: onstack error counter / number of non-xmitted frames
+ *
+ * Adds onstack error Tx XDP counter value from the current session
+ * to the driver container. Typical usage is to call it at on error
+ * path of .ndo_xdp_xmit() / XSK zerocopy xmit.
+ */
+static inline void xdp_update_tx_drv_err(struct xdp_tx_drv_stats *tstats,
+ u32 num)
+{
+ u64_stats_update_begin(&tstats->syncp);
+ u64_stats_add(&tstats->errors, num);
+ u64_stats_update_end(&tstats->syncp);
+}
+
+/**
+ * xdp_update_tx_drv_full - update driver Tx XDP ring full counter
+ * @tstats: target driver container
+ *
+ * Adds onstack error Tx XDP counter value from the current session
+ * to the driver container. Typical usage is to call it at in case
+ * of no free descs available on a ring in .ndo_xdp_xmit() / XSK
+ * zerocopy xmit.
+ */
+static inline void xdp_update_tx_drv_full(struct xdp_tx_drv_stats *tstats)
+{
+ u64_stats_update_begin(&tstats->syncp);
+ u64_stats_inc(&tstats->full);
+ u64_stats_update_end(&tstats->syncp);
+}
+
+void xdp_fetch_rx_drv_stats(struct ifla_xdp_stats *if_stats,
+ const struct xdp_rx_drv_stats *rstats);
+void xdp_fetch_tx_drv_stats(struct ifla_xdp_stats *if_stats,
+ const struct xdp_tx_drv_stats *tstats);
+int xdp_get_drv_stats_generic(const struct net_device *dev, u32 attr_id,
+ void *attr_data);
+
#endif /* __LINUX_NET_XDP_H__ */
@@ -611,3 +611,127 @@ struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf)
return nxdpf;
}
+
+/**
+ * xdp_fetch_rx_drv_stats - helper for implementing .ndo_get_xdp_stats()
+ * @if_stats: target container passed from rtnetlink core
+ * @rstats: driver container if it uses generic xdp_rx_drv_stats
+ *
+ * Fetches Rx path XDP statistics from a suggested driver structure to
+ * the one used by rtnetlink, respecting atomic/seqcount synchronization.
+ */
+void xdp_fetch_rx_drv_stats(struct ifla_xdp_stats *if_stats,
+ const struct xdp_rx_drv_stats *rstats)
+{
+ u32 start;
+
+ do {
+ start = u64_stats_fetch_begin_irq(&rstats->syncp);
+
+ if_stats->packets = u64_stats_read(&rstats->packets);
+ if_stats->bytes = u64_stats_read(&rstats->bytes);
+ if_stats->pass = u64_stats_read(&rstats->pass);
+ if_stats->drop = u64_stats_read(&rstats->drop);
+ if_stats->tx = u64_stats_read(&rstats->tx);
+ if_stats->tx_errors = u64_stats_read(&rstats->tx_errors);
+ if_stats->redirect = u64_stats_read(&rstats->redirect);
+ if_stats->redirect_errors =
+ u64_stats_read(&rstats->redirect_errors);
+ if_stats->aborted = u64_stats_read(&rstats->aborted);
+ if_stats->invalid = u64_stats_read(&rstats->invalid);
+ } while (u64_stats_fetch_retry_irq(&rstats->syncp, start));
+}
+EXPORT_SYMBOL_GPL(xdp_fetch_rx_drv_stats);
+
+/**
+ * xdp_fetch_tx_drv_stats - helper for implementing .ndo_get_xdp_stats()
+ * @if_stats: target container passed from rtnetlink core
+ * @tstats: driver container if it uses generic xdp_tx_drv_stats
+ *
+ * Fetches Tx path XDP statistics from a suggested driver structure to
+ * the one used by rtnetlink, respecting atomic/seqcount synchronization.
+ */
+void xdp_fetch_tx_drv_stats(struct ifla_xdp_stats *if_stats,
+ const struct xdp_tx_drv_stats *tstats)
+{
+ u32 start;
+
+ do {
+ start = u64_stats_fetch_begin_irq(&tstats->syncp);
+
+ if_stats->xmit_packets = u64_stats_read(&tstats->packets);
+ if_stats->xmit_bytes = u64_stats_read(&tstats->bytes);
+ if_stats->xmit_errors = u64_stats_read(&tstats->errors);
+ if_stats->xmit_full = u64_stats_read(&tstats->full);
+ } while (u64_stats_fetch_retry_irq(&tstats->syncp, start));
+}
+EXPORT_SYMBOL_GPL(xdp_fetch_tx_drv_stats);
+
+/**
+ * xdp_get_drv_stats_generic - generic implementation of .ndo_get_xdp_stats()
+ * @dev: network interface device structure
+ * @attr_id: type of statistics (XDP, XSK, ...)
+ * @attr_data: target stats container
+ *
+ * Returns 0 on success, -%EOPNOTSUPP if either driver or this function doesn't
+ * support this attr_id, -%ENODATA if the driver supports attr_id, but can't
+ * provide anything right now, and -%EINVAL if driver configuration is invalid.
+ */
+int xdp_get_drv_stats_generic(const struct net_device *dev, u32 attr_id,
+ void *attr_data)
+{
+ const bool xsk = attr_id == IFLA_XDP_XSTATS_TYPE_XSK;
+ const struct xdp_drv_stats *drv_iter = dev->xstats;
+ const struct net_device_ops *ops = dev->netdev_ops;
+ struct ifla_xdp_stats *iter = attr_data;
+ int nch;
+ u32 i;
+
+ switch (attr_id) {
+ case IFLA_XDP_XSTATS_TYPE_XDP:
+ if (unlikely(!ops->ndo_bpf))
+ return -EINVAL;
+
+ break;
+ case IFLA_XDP_XSTATS_TYPE_XSK:
+ if (!ops->ndo_xsk_wakeup)
+ return -EOPNOTSUPP;
+
+ break;
+ default:
+ return -EOPNOTSUPP;
+ }
+
+ if (unlikely(!drv_iter || !ops->ndo_get_xdp_stats_nch))
+ return -EINVAL;
+
+ nch = ops->ndo_get_xdp_stats_nch(dev, attr_id);
+ switch (nch) {
+ case 0:
+ /* Stats are shared across the netdev */
+ nch = 1;
+ break;
+ case 1 ... INT_MAX:
+ /* Stats are per-channel */
+ break;
+ default:
+ return nch;
+ }
+
+ for (i = 0; i < nch; i++) {
+ const struct xdp_rx_drv_stats *rstats;
+ const struct xdp_tx_drv_stats *tstats;
+
+ rstats = xsk ? &drv_iter->xsk_rx : &drv_iter->xdp_rx;
+ xdp_fetch_rx_drv_stats(iter, rstats);
+
+ tstats = xsk ? &drv_iter->xsk_tx : &drv_iter->xdp_tx;
+ xdp_fetch_tx_drv_stats(iter, tstats);
+
+ drv_iter++;
+ iter++;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(xdp_get_drv_stats_generic);
Add several shorthands to reduce driver boilerplates and unify storing and accessing generic XDP statistics in the drivers. If the driver has one of xdp_{rx,tx}_drv_stats embedded into a ring structure, it can reuse pretty much everything, but needs to implement its own .ndo_xdp_stats() and .ndo_xdp_stats_nch() if needed. If the driver stores a separate array of xdp_drv_stats, it can then export it as net_device::xstats, implement only .ndo_xdp_stats_nch() and wire up xdp_get_drv_stats_generic() as .ndo_xdp_stats(). Both XDP and XSK blocks of xdp_drv_stats are cacheline-aligned to avoid false-sharing, only extremely unlikely 'aborted' and 'invalid' falls out of a 64-byte CL. xdp_rx_drv_stats_local is provided to put it on stack and collect the stats on hotpath, with accessing a real container and its atomic/seqcount sync points just once when exiting Rx NAPI polling. Signed-off-by: Alexander Lobakin <alexandr.lobakin@intel.com> Reviewed-by: Jesse Brandeburg <jesse.brandeburg@intel.com> Reviewed-by: Michal Swiatkowski <michal.swiatkowski@linux.intel.com> Reviewed-by: Maciej Fijalkowski <maciej.fijalkowski@intel.com> --- include/linux/netdevice.h | 1 + include/net/xdp.h | 162 ++++++++++++++++++++++++++++++++++++++ net/core/xdp.c | 124 +++++++++++++++++++++++++++++ 3 files changed, 287 insertions(+) -- 2.33.1