Re: [RFC PATCH v3 12/12] selftests: add ncdevmem, netcat for devmem TCP

[Paolo Abeni]

On Sun, 2023-11-05 at 18:44 -0800, Mina Almasry wrote:
> @@ -91,6 +95,7 @@ TEST_PROGS += test_bridge_neigh_suppress.sh
>  TEST_PROGS += test_vxlan_nolocalbypass.sh
>  TEST_PROGS += test_bridge_backup_port.sh
>  TEST_PROGS += fdb_flush.sh
> +TEST_GEN_FILES += ncdevmem

I guess we want something added to TEST_PROGS, too ;)

>  TEST_FILES := settings
>  
> diff --git a/tools/testing/selftests/net/ncdevmem.c b/tools/testing/selftests/net/ncdevmem.c
> new file mode 100644
> index 000000000000..78bc3ad767ca
> --- /dev/null
> +++ b/tools/testing/selftests/net/ncdevmem.c
> @@ -0,0 +1,546 @@
> +// SPDX-License-Identifier: GPL-2.0
> +#define _GNU_SOURCE
> +#define __EXPORTED_HEADERS__
> +
> +#include <linux/uio.h>
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <unistd.h>
> +#include <stdbool.h>
> +#include <string.h>
> +#include <errno.h>
> +#define __iovec_defined
> +#include <fcntl.h>
> +#include <malloc.h>
> +
> +#include <arpa/inet.h>
> +#include <sys/socket.h>
> +#include <sys/mman.h>
> +#include <sys/ioctl.h>
> +#include <sys/syscall.h>
> +
> +#include <linux/memfd.h>
> +#include <linux/if.h>
> +#include <linux/dma-buf.h>
> +#include <linux/udmabuf.h>
> +#include <libmnl/libmnl.h>
> +#include <linux/types.h>
> +#include <linux/netlink.h>
> +#include <linux/genetlink.h>
> +#include <linux/netdev.h>
> +#include <time.h>
> +
> +#include "netdev-user.h"
> +#include <ynl.h>
> +
> +#define PAGE_SHIFT 12
> +#define TEST_PREFIX "ncdevmem"
> +#define NUM_PAGES 16000
> +
> +#ifndef MSG_SOCK_DEVMEM
> +#define MSG_SOCK_DEVMEM 0x2000000
> +#endif
> +
> +/*
> + * tcpdevmem netcat. Works similarly to netcat but does device memory TCP
> + * instead of regular TCP. Uses udmabuf to mock a dmabuf provider.
> + *
> + * Usage:
> + *
> + * * Without validation:
> + *
> + *	On server:
> + *	ncdevmem -s <server IP> -c <client IP> -f eth1 -n 0000:06:00.0 -l \
> + *		-p 5201
> + *
> + *	On client:
> + *	ncdevmem -s <server IP> -c <client IP> -f eth1 -n 0000:06:00.0 -p 5201
> + *
> + * * With Validation:
> + *	On server:
> + *	ncdevmem -s <server IP> -c <client IP> -l -f eth1 -n 0000:06:00.0 \
> + *		-p 5202 -v 1
> + *
> + *	On client:
> + *	ncdevmem -s <server IP> -c <client IP> -f eth1 -n 0000:06:00.0 -p 5202 \
> + *		-v 100000
> + *
> + * Note this is compatible with regular netcat. i.e. the sender or receiver can
> + * be replaced with regular netcat to test the RX or TX path in isolation.
> + */
> +
> +static char *server_ip = "192.168.1.4";
> +static char *client_ip = "192.168.1.2";
> +static char *port = "5201";
> +static size_t do_validation;
> +static int queue_num = 15;
> +static char *ifname = "eth1";
> +static char *nic_pci_addr = "0000:06:00.0";
> +static unsigned int iterations;
> +
> +void print_bytes(void *ptr, size_t size)
> +{
> +	unsigned char *p = ptr;
> +	int i;
> +
> +	for (i = 0; i < size; i++) {
> +		printf("%02hhX ", p[i]);
> +	}
> +	printf("\n");
> +}
> +
> +void print_nonzero_bytes(void *ptr, size_t size)
> +{
> +	unsigned char *p = ptr;
> +	unsigned int i;
> +
> +	for (i = 0; i < size; i++)
> +		putchar(p[i]);
> +	printf("\n");
> +}
> +
> +void validate_buffer(void *line, size_t size)
> +{
> +	static unsigned char seed = 1;
> +	unsigned char *ptr = line;
> +	int errors = 0;
> +	size_t i;
> +
> +	for (i = 0; i < size; i++) {
> +		if (ptr[i] != seed) {
> +			fprintf(stderr,
> +				"Failed validation: expected=%u, actual=%u, index=%lu\n",
> +				seed, ptr[i], i);
> +			errors++;
> +			if (errors > 20)
> +				exit(1);
> +		}
> +		seed++;
> +		if (seed == do_validation)
> +			seed = 0;
> +	}
> +
> +	fprintf(stdout, "Validated buffer\n");
> +}
> +
> +static void reset_flow_steering(void)
> +{
> +	char command[256];
> +
> +	memset(command, 0, sizeof(command));
> +	snprintf(command, sizeof(command), "sudo ethtool -K %s ntuple off",
> +		 "eth1");
> +	system(command);
> +
> +	memset(command, 0, sizeof(command));
> +	snprintf(command, sizeof(command), "sudo ethtool -K %s ntuple on",
> +		 "eth1");
> +	system(command);
> +}
> +
> +static void configure_flow_steering(void)
> +{
> +	char command[256];
> +
> +	memset(command, 0, sizeof(command));
> +	snprintf(command, sizeof(command),
> +		 "sudo ethtool -N %s flow-type tcp4 src-ip %s dst-ip %s src-port %s dst-port %s queue %d",
> +		 ifname, client_ip, server_ip, port, port, queue_num);
> +	system(command);
> +}
> +
> +/* Triggers a driver reset...
> + *
> + * The proper way to do this is probably 'ethtool --reset', but I don't have
> + * that supported on my current test bed. I resort to changing this
> + * configuration in the driver which also causes a driver reset...
> + */
> +static void trigger_device_reset(void)
> +{
> +	char command[256];
> +
> +	memset(command, 0, sizeof(command));
> +	snprintf(command, sizeof(command),
> +		 "sudo ethtool --set-priv-flags %s enable-header-split off",
> +		 ifname);
> +	system(command);
> +
> +	memset(command, 0, sizeof(command));
> +	snprintf(command, sizeof(command),
> +		 "sudo ethtool --set-priv-flags %s enable-header-split on",
> +		 ifname);
> +	system(command);
> +}
> +
> +static int bind_rx_queue(unsigned int ifindex, unsigned int dmabuf_fd,
> +			 __u32 *queue_idx, unsigned int n_queue_index,
> +			 struct ynl_sock **ys)
> +{
> +	struct netdev_bind_rx_req *req = NULL;
> +	struct ynl_error yerr;
> +	int ret = 0;
> +
> +	*ys = ynl_sock_create(&ynl_netdev_family, &yerr);
> +	if (!*ys) {
> +		fprintf(stderr, "YNL: %s\n", yerr.msg);
> +		return -1;
> +	}
> +
> +	if (ynl_subscribe(*ys, "mgmt"))
> +		goto err_close;
> +
> +	req = netdev_bind_rx_req_alloc();
> +	netdev_bind_rx_req_set_ifindex(req, ifindex);
> +	netdev_bind_rx_req_set_dmabuf_fd(req, dmabuf_fd);
> +	__netdev_bind_rx_req_set_queues(req, queue_idx, n_queue_index);
> +
> +	ret = netdev_bind_rx(*ys, req);
> +	if (!ret) {
> +		perror("netdev_bind_rx");
> +		goto err_close;
> +	}
> +
> +	netdev_bind_rx_req_free(req);
> +
> +	return 0;
> +
> +err_close:
> +	fprintf(stderr, "YNL failed: %s\n", (*ys)->err.msg);
> +	netdev_bind_rx_req_free(req);
> +	ynl_sock_destroy(*ys);
> +	return -1;
> +}
> +
> +static void create_udmabuf(int *devfd, int *memfd, int *buf, size_t dmabuf_size)
> +{
> +	struct udmabuf_create create;
> +	int ret;
> +
> +	*devfd = open("/dev/udmabuf", O_RDWR);
> +	if (*devfd < 0) {
> +		fprintf(stderr,
> +			"%s: [skip,no-udmabuf: Unable to access DMA "
> +			"buffer device file]\n",
> +			TEST_PREFIX);
> +		exit(70);
> +	}
> +
> +	*memfd = memfd_create("udmabuf-test", MFD_ALLOW_SEALING);
> +	if (*memfd < 0) {
> +		printf("%s: [skip,no-memfd]\n", TEST_PREFIX);
> +		exit(72);
> +	}
> +
> +	ret = fcntl(*memfd, F_ADD_SEALS, F_SEAL_SHRINK);
> +	if (ret < 0) {
> +		printf("%s: [skip,fcntl-add-seals]\n", TEST_PREFIX);
> +		exit(73);
> +	}
> +
> +	ret = ftruncate(*memfd, dmabuf_size);
> +	if (ret == -1) {
> +		printf("%s: [FAIL,memfd-truncate]\n", TEST_PREFIX);
> +		exit(74);
> +	}
> +
> +	memset(&create, 0, sizeof(create));
> +
> +	create.memfd = *memfd;
> +	create.offset = 0;
> +	create.size = dmabuf_size;
> +	*buf = ioctl(*devfd, UDMABUF_CREATE, &create);
> +	if (*buf < 0) {
> +		printf("%s: [FAIL, create udmabuf]\n", TEST_PREFIX);
> +		exit(75);
> +	}
> +}
> +
> +int do_server(void)
> +{
> +	char ctrl_data[sizeof(int) * 20000];
> +	size_t non_page_aligned_frags = 0;
> +	struct sockaddr_in client_addr;
> +	struct sockaddr_in server_sin;
> +	size_t page_aligned_frags = 0;
> +	int devfd, memfd, buf, ret;
> +	size_t total_received = 0;
> +	bool is_devmem = false;
> +	char *buf_mem = NULL;
> +	struct ynl_sock *ys;
> +	size_t dmabuf_size;
> +	char iobuf[819200];
> +	char buffer[256];
> +	int socket_fd;
> +	int client_fd;
> +	size_t i = 0;
> +	int opt = 1;
> +
> +	dmabuf_size = getpagesize() * NUM_PAGES;
> +
> +	create_udmabuf(&devfd, &memfd, &buf, dmabuf_size);
> +
> +	__u32 *queue_idx = malloc(sizeof(__u32) * 2);
> +
> +	queue_idx[0] = 14;
> +	queue_idx[1] = 15;
> +	if (bind_rx_queue(3 /* index for eth1 */, buf, queue_idx, 2, &ys)) {
                          ^^^^^^^^^^^^^^^^^^^
I guess we want to explicitly fetch the "ifname" index.

Side note: I'm wondering if we could extend some kind of virtual device
to allow single host self-tests? e.g. veth, if that would not cause
excessive bloat in the device driver?

Cheers,

Paolo