diff mbox series

[net-next,v2,2/2] selftests: Add test for timing a bind request to a port with a populated bhash entry

Message ID 20220510005316.3967597-3-joannelkoong@gmail.com (mailing list archive)
State Superseded
Delegated to: Netdev Maintainers
Headers show
Series Add a bhash2 table hashed by port + address | expand

Checks

Context Check Description
netdev/tree_selection success Clearly marked for net-next, async
netdev/apply fail Patch does not apply to net-next

Commit Message

Joanne Koong May 10, 2022, 12:53 a.m. UTC 
This test populates the bhash table for a given port with
MAX_THREADS * MAX_CONNECTIONS sockets, and then times how long
a bind request on the port takes.

When populating the bhash table, we create the sockets and then bind
the sockets to the same address and port (SO_REUSEADDR and SO_REUSEPORT
are set). When timing how long a bind on the port takes, we bind on a
different address without SO_REUSEPORT set. We do not set SO_REUSEPORT
because we are interested in the case where the bind request does not
go through the tb->fastreuseport path, which is fragile (eg
tb->fastreuseport path does not work if binding with a different uid).

To run the test locally, I did:
* ulimit -n 65535000
* ip addr add 2001:0db8:0:f101::1 dev eth0
* ./bind_bhash_test 443

Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
---
 tools/testing/selftests/net/.gitignore        |   1 +
 tools/testing/selftests/net/Makefile          |   2 +
 tools/testing/selftests/net/bind_bhash_test.c | 119 ++++++++++++++++++
 3 files changed, 122 insertions(+)
 create mode 100644 tools/testing/selftests/net/bind_bhash_test.c

Comments

Joanne Koong May 10, 2022, 1:01 a.m. UTC | #1 
On Mon, May 9, 2022 at 5:54 PM Joanne Koong <joannelkoong@gmail.com> wrote:
>
> This test populates the bhash table for a given port with
> MAX_THREADS * MAX_CONNECTIONS sockets, and then times how long
> a bind request on the port takes.
>
> When populating the bhash table, we create the sockets and then bind
> the sockets to the same address and port (SO_REUSEADDR and SO_REUSEPORT
> are set). When timing how long a bind on the port takes, we bind on a
> different address without SO_REUSEPORT set. We do not set SO_REUSEPORT
> because we are interested in the case where the bind request does not
> go through the tb->fastreuseport path, which is fragile (eg
> tb->fastreuseport path does not work if binding with a different uid).
>
> To run the test locally, I did:
> * ulimit -n 65535000
> * ip addr add 2001:0db8:0:f101::1 dev eth0
> * ./bind_bhash_test 443
>
> Signed-off-by: Joanne Koong <joannelkoong@gmail.com>
> ---
>  tools/testing/selftests/net/.gitignore        |   1 +
>  tools/testing/selftests/net/Makefile          |   2 +
>  tools/testing/selftests/net/bind_bhash_test.c | 119 ++++++++++++++++++
>  3 files changed, 122 insertions(+)
>  create mode 100644 tools/testing/selftests/net/bind_bhash_test.c
>
> diff --git a/tools/testing/selftests/net/.gitignore b/tools/testing/selftests/net/.gitignore
> index 21a411b04890..735423136bc4 100644
> --- a/tools/testing/selftests/net/.gitignore
> +++ b/tools/testing/selftests/net/.gitignore
> @@ -36,3 +36,4 @@ gro
>  ioam6_parser
>  toeplitz
>  cmsg_sender
> +bind_bhash_test
> diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile
> index af7f6e6ff182..b2cf3fa152ad 100644
> --- a/tools/testing/selftests/net/Makefile
> +++ b/tools/testing/selftests/net/Makefile
> @@ -55,6 +55,7 @@ TEST_GEN_PROGS = reuseport_bpf reuseport_bpf_cpu reuseport_bpf_numa
>  TEST_GEN_PROGS += reuseport_dualstack reuseaddr_conflict tls
>  TEST_GEN_FILES += toeplitz
>  TEST_GEN_FILES += cmsg_sender
> +TEST_GEN_FILES += bind_bhash_test
>
>  TEST_FILES := settings
>
> @@ -63,4 +64,5 @@ include ../lib.mk
>
>  $(OUTPUT)/reuseport_bpf_numa: LDLIBS += -lnuma
>  $(OUTPUT)/tcp_mmap: LDLIBS += -lpthread
> +$(OUTPUT)/bind_bhash_test: LDLIBS += -lpthread
>  $(OUTPUT)/tcp_inq: LDLIBS += -lpthread
> diff --git a/tools/testing/selftests/net/bind_bhash_test.c b/tools/testing/selftests/net/bind_bhash_test.c
> new file mode 100644
> index 000000000000..252e73754e76
> --- /dev/null
> +++ b/tools/testing/selftests/net/bind_bhash_test.c
> @@ -0,0 +1,119 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * This times how long it takes to bind to a port when the port already
> + * has multiple sockets in its bhash table.
> + *
> + * In the setup(), we populate the port's bhash table with
> + * MAX_THREADS * MAX_CONNECTIONS number of entries.
> + */
> +
> +#include <unistd.h>
> +#include <stdio.h>
> +#include <netdb.h>
> +#include <pthread.h>
> +
> +#define MAX_THREADS 600
> +#define MAX_CONNECTIONS 40
> +
> +static const char *bind_addr = "::1";
> +static const char *port;
> +
> +static int fd_array[MAX_THREADS][MAX_CONNECTIONS];
> +
> +static int bind_socket(int opt, const char *addr)
> +{
> +       struct addrinfo *res, hint = {};
> +       int sock_fd, reuse = 1, err;
> +
> +       sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
> +       if (sock_fd < 0) {
> +               perror("socket fd err");
> +               return -1;
> +       }
> +
> +       hint.ai_family = AF_INET6;
> +       hint.ai_socktype = SOCK_STREAM;
> +
> +       err = getaddrinfo(addr, port, &hint, &res);
> +       if (err) {
> +               perror("getaddrinfo failed");
> +               return -1;
> +       }
> +
> +       if (opt) {
> +               err = setsockopt(sock_fd, SOL_SOCKET, opt, &reuse, sizeof(reuse));
> +               if (err) {
> +                       perror("setsockopt failed");
> +                       return -1;
> +               }
> +       }
> +
> +       err = bind(sock_fd, res->ai_addr, res->ai_addrlen);
> +       if (err) {
> +               perror("failed to bind to port");
> +               return -1;
> +       }
> +
> +       return sock_fd;
> +}
> +
> +static void *setup(void *arg)
> +{
> +       int sock_fd, i;
> +       int *array = (int *)arg;
> +
> +       for (i = 0; i < MAX_CONNECTIONS; i++) {
> +               sock_fd = bind_socket(SO_REUSEADDR | SO_REUSEPORT, bind_addr);
> +               if (sock_fd < 0)
> +                       return NULL;
> +               array[i] = sock_fd;
> +       }
> +
> +       return NULL;
> +}
> +
> +int main(int argc, const char *argv[])
> +{
> +       int listener_fd, sock_fd, i, j;
> +       pthread_t tid[MAX_THREADS];
> +       clock_t begin, end;
> +
> +       if (argc != 2) {
> +               printf("Usage: listener <port>\n");
> +               return -1;
> +       }
> +
> +       port = argv[1];
> +
> +       listener_fd = bind_socket(SO_REUSEADDR | SO_REUSEPORT, bind_addr);
> +       if (listen(listener_fd, 100) < 0) {
> +               perror("listen failed");
> +               return -1;
> +       }
> +
> +       /* Set up threads to populate the bhash table entry for the port */
> +       for (i = 0; i < MAX_THREADS; i++)
> +               pthread_create(&tid[i], NULL, setup, fd_array[i]);
> +
> +       for (i = 0; i < MAX_THREADS; i++)
> +               pthread_join(tid[i], NULL);
> +
> +       begin = clock();
> +
> +       /* Bind to the same port on a different address */
> +       sock_fd  = bind_socket(0, "2001:0db8:0:f101::1");
> +
> +       end = clock();
> +
> +       printf("time spent = %f\n", (double)(end - begin) / CLOCKS_PER_SEC);
> +
> +       /* clean up */
> +       close(sock_fd);
> +       close(listener_fd);
> +       for (i = 0; i < MAX_THREADS; i++) {
> +               for (j = 0; i < MAX_THREADS; i++)
> +                       close(fd_array[i][j]);
> +       }
> +
> +       return 0;
> +}
Eric, this is what I used locally to test the bind request. Depending
on what you think about this bhash2 proposal, I am happy to clean up
this test in a v3 follow-up.
> --
> 2.30.2
>
Eric Dumazet May 10, 2022, 3:12 a.m. UTC | #2 
On Mon, May 9, 2022 at 6:01 PM Joanne Koong <joannelkoong@gmail.com> wrote:
>
>
> Eric, this is what I used locally to test the bind request. Depending
> on what you think about this bhash2 proposal, I am happy to clean up
> this test in a v3 follow-up.

Thanks, I will take a look at the series tomorrow.
diff mbox series

Patch

diff --git a/tools/testing/selftests/net/.gitignore b/tools/testing/selftests/net/.gitignore
index 21a411b04890..735423136bc4 100644
--- a/tools/testing/selftests/net/.gitignore
+++ b/tools/testing/selftests/net/.gitignore
@@ -36,3 +36,4 @@  gro
 ioam6_parser
 toeplitz
 cmsg_sender
+bind_bhash_test
diff --git a/tools/testing/selftests/net/Makefile b/tools/testing/selftests/net/Makefile
index af7f6e6ff182..b2cf3fa152ad 100644
--- a/tools/testing/selftests/net/Makefile
+++ b/tools/testing/selftests/net/Makefile
@@ -55,6 +55,7 @@  TEST_GEN_PROGS = reuseport_bpf reuseport_bpf_cpu reuseport_bpf_numa
 TEST_GEN_PROGS += reuseport_dualstack reuseaddr_conflict tls
 TEST_GEN_FILES += toeplitz
 TEST_GEN_FILES += cmsg_sender
+TEST_GEN_FILES += bind_bhash_test
 
 TEST_FILES := settings
 
@@ -63,4 +64,5 @@  include ../lib.mk
 
 $(OUTPUT)/reuseport_bpf_numa: LDLIBS += -lnuma
 $(OUTPUT)/tcp_mmap: LDLIBS += -lpthread
+$(OUTPUT)/bind_bhash_test: LDLIBS += -lpthread
 $(OUTPUT)/tcp_inq: LDLIBS += -lpthread
diff --git a/tools/testing/selftests/net/bind_bhash_test.c b/tools/testing/selftests/net/bind_bhash_test.c
new file mode 100644
index 000000000000..252e73754e76
--- /dev/null
+++ b/tools/testing/selftests/net/bind_bhash_test.c
@@ -0,0 +1,119 @@ 
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * This times how long it takes to bind to a port when the port already
+ * has multiple sockets in its bhash table.
+ *
+ * In the setup(), we populate the port's bhash table with
+ * MAX_THREADS * MAX_CONNECTIONS number of entries.
+ */
+
+#include <unistd.h>
+#include <stdio.h>
+#include <netdb.h>
+#include <pthread.h>
+
+#define MAX_THREADS 600
+#define MAX_CONNECTIONS 40
+
+static const char *bind_addr = "::1";
+static const char *port;
+
+static int fd_array[MAX_THREADS][MAX_CONNECTIONS];
+
+static int bind_socket(int opt, const char *addr)
+{
+	struct addrinfo *res, hint = {};
+	int sock_fd, reuse = 1, err;
+
+	sock_fd = socket(AF_INET6, SOCK_STREAM, 0);
+	if (sock_fd < 0) {
+		perror("socket fd err");
+		return -1;
+	}
+
+	hint.ai_family = AF_INET6;
+	hint.ai_socktype = SOCK_STREAM;
+
+	err = getaddrinfo(addr, port, &hint, &res);
+	if (err) {
+		perror("getaddrinfo failed");
+		return -1;
+	}
+
+	if (opt) {
+		err = setsockopt(sock_fd, SOL_SOCKET, opt, &reuse, sizeof(reuse));
+		if (err) {
+			perror("setsockopt failed");
+			return -1;
+		}
+	}
+
+	err = bind(sock_fd, res->ai_addr, res->ai_addrlen);
+	if (err) {
+		perror("failed to bind to port");
+		return -1;
+	}
+
+	return sock_fd;
+}
+
+static void *setup(void *arg)
+{
+	int sock_fd, i;
+	int *array = (int *)arg;
+
+	for (i = 0; i < MAX_CONNECTIONS; i++) {
+		sock_fd = bind_socket(SO_REUSEADDR | SO_REUSEPORT, bind_addr);
+		if (sock_fd < 0)
+			return NULL;
+		array[i] = sock_fd;
+	}
+
+	return NULL;
+}
+
+int main(int argc, const char *argv[])
+{
+	int listener_fd, sock_fd, i, j;
+	pthread_t tid[MAX_THREADS];
+	clock_t begin, end;
+
+	if (argc != 2) {
+		printf("Usage: listener <port>\n");
+		return -1;
+	}
+
+	port = argv[1];
+
+	listener_fd = bind_socket(SO_REUSEADDR | SO_REUSEPORT, bind_addr);
+	if (listen(listener_fd, 100) < 0) {
+		perror("listen failed");
+		return -1;
+	}
+
+	/* Set up threads to populate the bhash table entry for the port */
+	for (i = 0; i < MAX_THREADS; i++)
+		pthread_create(&tid[i], NULL, setup, fd_array[i]);
+
+	for (i = 0; i < MAX_THREADS; i++)
+		pthread_join(tid[i], NULL);
+
+	begin = clock();
+
+	/* Bind to the same port on a different address */
+	sock_fd  = bind_socket(0, "2001:0db8:0:f101::1");
+
+	end = clock();
+
+	printf("time spent = %f\n", (double)(end - begin) / CLOCKS_PER_SEC);
+
+	/* clean up */
+	close(sock_fd);
+	close(listener_fd);
+	for (i = 0; i < MAX_THREADS; i++) {
+		for (j = 0; i < MAX_THREADS; i++)
+			close(fd_array[i][j]);
+	}
+
+	return 0;
+}