[PATCH v2 net-next 02/26] xdp: provide common driver helpers for implementing XDP stats

[Alexander Lobakin]

Add several shorthands to reduce driver boilerplates and unify
storing and accessing generic XDP statistics in the drivers.
If the driver has one of xdp_{rx,tx}_drv_stats embedded into
a ring structure, it can reuse pretty much everything, but needs
to implement its own .ndo_xdp_stats() and .ndo_xdp_stats_nch()
if needed. If the driver stores a separate array of xdp_drv_stats,
it can then export it as net_device::xstats, implement only
.ndo_xdp_stats_nch() and wire up xdp_get_drv_stats_generic()
as .ndo_xdp_stats().
Both XDP and XSK blocks of xdp_drv_stats are cacheline-aligned
to avoid false-sharing, only extremely unlikely 'aborted' and
'invalid' falls out of a 64-byte CL. xdp_rx_drv_stats_local is
provided to put it on stack and collect the stats on hotpath,
with accessing a real container and its atomic/seqcount sync
points just once when exiting Rx NAPI polling.

Signed-off-by: Alexander Lobakin <alexandr.lobakin@xxxxxxxxx>
Reviewed-by: Jesse Brandeburg <jesse.brandeburg@xxxxxxxxx>
Reviewed-by: Michal Swiatkowski <michal.swiatkowski@xxxxxxxxxxxxxxx>
Reviewed-by: Maciej Fijalkowski <maciej.fijalkowski@xxxxxxxxx>
---
 include/linux/netdevice.h |   1 +
 include/net/xdp.h         | 162 ++++++++++++++++++++++++++++++++++++++
 net/core/xdp.c            | 124 +++++++++++++++++++++++++++++
 3 files changed, 287 insertions(+)

diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index 058a00c2d19a..728c650d290e 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -2225,6 +2225,7 @@ struct net_device {
 		struct pcpu_lstats __percpu		*lstats;
 		struct pcpu_sw_netstats __percpu	*tstats;
 		struct pcpu_dstats __percpu		*dstats;
+		struct xdp_drv_stats /* per-channel */	*xstats;
 	};

 #if IS_ENABLED(CONFIG_GARP)
diff --git a/include/net/xdp.h b/include/net/xdp.h
index 447f9b1578f3..e4f06a34d462 100644
--- a/include/net/xdp.h
+++ b/include/net/xdp.h
@@ -7,6 +7,7 @@
 #define __LINUX_NET_XDP_H__

 #include <linux/skbuff.h> /* skb_shared_info */
+#include <linux/u64_stats_sync.h> /* u64_stats_* */

 /**
  * DOC: XDP RX-queue information
@@ -292,4 +293,165 @@ void xdp_attachment_setup(struct xdp_attachment_info *info,

 #define DEV_MAP_BULK_SIZE XDP_BULK_QUEUE_SIZE

+/* Suggested XDP/XSK driver stats, mirror &ifla_xdp_stats except
+ * for generic errors, refer to its documentation for the details.
+ * The intended usage is to either have them as a standalone array
+ * of xdp_drv_stats, or embed &xdp_{rx,tx}_drv_stats into a ring
+ * structure. Having separate XDP and XSK counters is recommended.
+ */
+
+struct ifla_xdp_stats;
+
+struct xdp_rx_drv_stats {
+	struct u64_stats_sync syncp;
+	u64_stats_t packets;
+	u64_stats_t bytes;
+	u64_stats_t pass;
+	u64_stats_t drop;
+	u64_stats_t redirect;
+	u64_stats_t tx;
+	u64_stats_t redirect_errors;
+	u64_stats_t tx_errors;
+	u64_stats_t aborted;
+	u64_stats_t invalid;
+};
+
+struct xdp_tx_drv_stats {
+	struct u64_stats_sync syncp;
+	u64_stats_t packets;
+	u64_stats_t bytes;
+	u64_stats_t errors;
+	u64_stats_t full;
+};
+
+struct xdp_drv_stats {
+	struct xdp_rx_drv_stats xdp_rx;
+	struct xdp_tx_drv_stats xdp_tx;
+	struct xdp_rx_drv_stats xsk_rx ____cacheline_aligned;
+	struct xdp_tx_drv_stats xsk_tx;
+} ____cacheline_aligned;
+
+/* Shortened copy of Rx stats to put on stack */
+struct xdp_rx_drv_stats_local {
+	u32 bytes;
+	u32 packets;
+	u32 pass;
+	u32 drop;
+	u32 tx;
+	u32 tx_errors;
+	u32 redirect;
+	u32 redirect_errors;
+	u32 aborted;
+	u32 invalid;
+};
+
+#define xdp_init_rx_drv_stats(rstats) u64_stats_init(&(rstats)->syncp)
+#define xdp_init_tx_drv_stats(tstats) u64_stats_init(&(tstats)->syncp)
+
+/**
+ * xdp_init_drv_stats - initialize driver XDP stats
+ * @xdp_stats: driver container if it uses generic xdp_drv_stats
+ *
+ * Initializes atomic/seqcount sync points inside the containers.
+ */
+static inline void xdp_init_drv_stats(struct xdp_drv_stats *xdp_stats)
+{
+	xdp_init_rx_drv_stats(&xdp_stats->xdp_rx);
+	xdp_init_tx_drv_stats(&xdp_stats->xdp_tx);
+	xdp_init_rx_drv_stats(&xdp_stats->xsk_rx);
+	xdp_init_tx_drv_stats(&xdp_stats->xsk_tx);
+}
+
+/**
+ * xdp_update_rx_drv_stats - update driver XDP stats
+ * @rstats: target driver container
+ * @lrstats: filled onstack structure
+ *
+ * Fetches Rx path XDP statistics from the onstack structure to the
+ * driver container, respecting atomic/seqcount synchronization.
+ * Typical usage is to call it at the end of Rx NAPI polling.
+ */
+static inline void
+xdp_update_rx_drv_stats(struct xdp_rx_drv_stats *rstats,
+			const struct xdp_rx_drv_stats_local *lrstats)
+{
+	if (!lrstats->packets)
+		return;
+
+	u64_stats_update_begin(&rstats->syncp);
+	u64_stats_add(&rstats->packets, lrstats->packets);
+	u64_stats_add(&rstats->bytes, lrstats->bytes);
+	u64_stats_add(&rstats->pass, lrstats->pass);
+	u64_stats_add(&rstats->drop, lrstats->drop);
+	u64_stats_add(&rstats->redirect, lrstats->redirect);
+	u64_stats_add(&rstats->tx, lrstats->tx);
+	u64_stats_add(&rstats->redirect_errors, lrstats->redirect_errors);
+	u64_stats_add(&rstats->tx_errors, lrstats->tx_errors);
+	u64_stats_add(&rstats->aborted, lrstats->aborted);
+	u64_stats_add(&rstats->invalid, lrstats->invalid);
+	u64_stats_update_end(&rstats->syncp);
+}
+
+/**
+ * xdp_update_tx_drv_stats - update driver XDP stats
+ * @tstats: target driver container
+ * @packets: onstack packet counter
+ * @bytes: onstack octete counter
+ *
+ * Adds onstack packet/byte Tx XDP counter values from the current session
+ * to the driver container. Typical usage is to call it on completion path /
+ * Tx NAPI polling.
+ */
+static inline void xdp_update_tx_drv_stats(struct xdp_tx_drv_stats *tstats,
+					   u32 packets, u32 bytes)
+{
+	if (!packets)
+		return;
+
+	u64_stats_update_begin(&tstats->syncp);
+	u64_stats_add(&tstats->packets, packets);
+	u64_stats_add(&tstats->bytes, bytes);
+	u64_stats_update_end(&tstats->syncp);
+}
+
+/**
+ * xdp_update_tx_drv_err - update driver Tx XDP errors counter
+ * @tstats: target driver container
+ * @num: onstack error counter / number of non-xmitted frames
+ *
+ * Adds onstack error Tx XDP counter value from the current session
+ * to the driver container. Typical usage is to call it at on error
+ * path of .ndo_xdp_xmit() / XSK zerocopy xmit.
+ */
+static inline void xdp_update_tx_drv_err(struct xdp_tx_drv_stats *tstats,
+					 u32 num)
+{
+	u64_stats_update_begin(&tstats->syncp);
+	u64_stats_add(&tstats->errors, num);
+	u64_stats_update_end(&tstats->syncp);
+}
+
+/**
+ * xdp_update_tx_drv_full - update driver Tx XDP ring full counter
+ * @tstats: target driver container
+ *
+ * Adds onstack error Tx XDP counter value from the current session
+ * to the driver container. Typical usage is to call it at in case
+ * of no free descs available on a ring in .ndo_xdp_xmit() / XSK
+ * zerocopy xmit.
+ */
+static inline void xdp_update_tx_drv_full(struct xdp_tx_drv_stats *tstats)
+{
+	u64_stats_update_begin(&tstats->syncp);
+	u64_stats_inc(&tstats->full);
+	u64_stats_update_end(&tstats->syncp);
+}
+
+void xdp_fetch_rx_drv_stats(struct ifla_xdp_stats *if_stats,
+			    const struct xdp_rx_drv_stats *rstats);
+void xdp_fetch_tx_drv_stats(struct ifla_xdp_stats *if_stats,
+			    const struct xdp_tx_drv_stats *tstats);
+int xdp_get_drv_stats_generic(const struct net_device *dev, u32 attr_id,
+			      void *attr_data);
+
 #endif /* __LINUX_NET_XDP_H__ */
diff --git a/net/core/xdp.c b/net/core/xdp.c
index 5ddc29f29bad..24980207303c 100644
--- a/net/core/xdp.c
+++ b/net/core/xdp.c
@@ -611,3 +611,127 @@ struct xdp_frame *xdpf_clone(struct xdp_frame *xdpf)

 	return nxdpf;
 }
+
+/**
+ * xdp_fetch_rx_drv_stats - helper for implementing .ndo_get_xdp_stats()
+ * @if_stats: target container passed from rtnetlink core
+ * @rstats: driver container if it uses generic xdp_rx_drv_stats
+ *
+ * Fetches Rx path XDP statistics from a suggested driver structure to
+ * the one used by rtnetlink, respecting atomic/seqcount synchronization.
+ */
+void xdp_fetch_rx_drv_stats(struct ifla_xdp_stats *if_stats,
+			    const struct xdp_rx_drv_stats *rstats)
+{
+	u32 start;
+
+	do {
+		start = u64_stats_fetch_begin_irq(&rstats->syncp);
+
+		if_stats->packets = u64_stats_read(&rstats->packets);
+		if_stats->bytes = u64_stats_read(&rstats->bytes);
+		if_stats->pass = u64_stats_read(&rstats->pass);
+		if_stats->drop = u64_stats_read(&rstats->drop);
+		if_stats->tx = u64_stats_read(&rstats->tx);
+		if_stats->tx_errors = u64_stats_read(&rstats->tx_errors);
+		if_stats->redirect = u64_stats_read(&rstats->redirect);
+		if_stats->redirect_errors =
+			u64_stats_read(&rstats->redirect_errors);
+		if_stats->aborted = u64_stats_read(&rstats->aborted);
+		if_stats->invalid = u64_stats_read(&rstats->invalid);
+	} while (u64_stats_fetch_retry_irq(&rstats->syncp, start));
+}
+EXPORT_SYMBOL_GPL(xdp_fetch_rx_drv_stats);
+
+/**
+ * xdp_fetch_tx_drv_stats - helper for implementing .ndo_get_xdp_stats()
+ * @if_stats: target container passed from rtnetlink core
+ * @tstats: driver container if it uses generic xdp_tx_drv_stats
+ *
+ * Fetches Tx path XDP statistics from a suggested driver structure to
+ * the one used by rtnetlink, respecting atomic/seqcount synchronization.
+ */
+void xdp_fetch_tx_drv_stats(struct ifla_xdp_stats *if_stats,
+			    const struct xdp_tx_drv_stats *tstats)
+{
+	u32 start;
+
+	do {
+		start = u64_stats_fetch_begin_irq(&tstats->syncp);
+
+		if_stats->xmit_packets = u64_stats_read(&tstats->packets);
+		if_stats->xmit_bytes = u64_stats_read(&tstats->bytes);
+		if_stats->xmit_errors = u64_stats_read(&tstats->errors);
+		if_stats->xmit_full = u64_stats_read(&tstats->full);
+	} while (u64_stats_fetch_retry_irq(&tstats->syncp, start));
+}
+EXPORT_SYMBOL_GPL(xdp_fetch_tx_drv_stats);
+
+/**
+ * xdp_get_drv_stats_generic - generic implementation of .ndo_get_xdp_stats()
+ * @dev: network interface device structure
+ * @attr_id: type of statistics (XDP, XSK, ...)
+ * @attr_data: target stats container
+ *
+ * Returns 0 on success, -%EOPNOTSUPP if either driver or this function doesn't
+ * support this attr_id, -%ENODATA if the driver supports attr_id, but can't
+ * provide anything right now, and -%EINVAL if driver configuration is invalid.
+ */
+int xdp_get_drv_stats_generic(const struct net_device *dev, u32 attr_id,
+			      void *attr_data)
+{
+	const bool xsk = attr_id == IFLA_XDP_XSTATS_TYPE_XSK;
+	const struct xdp_drv_stats *drv_iter = dev->xstats;
+	const struct net_device_ops *ops = dev->netdev_ops;
+	struct ifla_xdp_stats *iter = attr_data;
+	int nch;
+	u32 i;
+
+	switch (attr_id) {
+	case IFLA_XDP_XSTATS_TYPE_XDP:
+		if (unlikely(!ops->ndo_bpf))
+			return -EINVAL;
+
+		break;
+	case IFLA_XDP_XSTATS_TYPE_XSK:
+		if (!ops->ndo_xsk_wakeup)
+			return -EOPNOTSUPP;
+
+		break;
+	default:
+		return -EOPNOTSUPP;
+	}
+
+	if (unlikely(!drv_iter || !ops->ndo_get_xdp_stats_nch))
+		return -EINVAL;
+
+	nch = ops->ndo_get_xdp_stats_nch(dev, attr_id);
+	switch (nch) {
+	case 0:
+		/* Stats are shared across the netdev */
+		nch = 1;
+		break;
+	case 1 ... INT_MAX:
+		/* Stats are per-channel */
+		break;
+	default:
+		return nch;
+	}
+
+	for (i = 0; i < nch; i++) {
+		const struct xdp_rx_drv_stats *rstats;
+		const struct xdp_tx_drv_stats *tstats;
+
+		rstats = xsk ? &drv_iter->xsk_rx : &drv_iter->xdp_rx;
+		xdp_fetch_rx_drv_stats(iter, rstats);
+
+		tstats = xsk ? &drv_iter->xsk_tx : &drv_iter->xdp_tx;
+		xdp_fetch_tx_drv_stats(iter, tstats);
+
+		drv_iter++;
+		iter++;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(xdp_get_drv_stats_generic);
--
2.33.1