diff mbox series

[net-next] net/smc: Reduce overflow of smc clcsock listen queue

Message ID 1641293930-110897-1-git-send-email-alibuda@linux.alibaba.com (mailing list archive)
State Superseded
Headers show
Series [net-next] net/smc: Reduce overflow of smc clcsock listen queue | expand

Commit Message

D. Wythe Jan. 4, 2022, 10:58 a.m. UTC 
From: "D. Wythe" <alibuda@linux.alibaba.com>

In nginx/wrk multithread and 10K connections benchmark, the
backend TCP connection established very slowly, and lots of TCP
connections stay in SYN_SENT state.

Server: smc_run nginx

Client: smc_run wrk -c 10000 -t 4 http://server

Socket state in client host (wrk) shows like:

ss -t  | wc -l
10000

ss -t  | grep "SYN-SENT"  | wc -l
6248

While the socket state in server host (nginx) shows like:

ss -t  | wc -l
3752

Furthermore, the netstate of server host shows like:
    145042 times the listen queue of a socket overflowed
    145042 SYNs to LISTEN sockets dropped

This issue caused by smc_listen_work(), since the smc_tcp_listen_work()
shared the same workqueue (smc_hs_wq) with smc_listen_work(), while
smc_listen_work() do blocking wait for smc connection established, which
meanwhile block the accept() from TCP listen queue.

This patch create a independent workqueue(smc_tcp_ls_wq) for
smc_tcp_listen_work(), separate it from smc_listen_work(), which is
quite acceptable considering that smc_tcp_listen_work() runs very fast.

After this patch, the smc 10K connections benchmark in my case is 5
times faster than before.

Before patch :

smc_run  ./wrk -c 10000 -t 3 -d 20  http://server
  3 threads and 10000 connections
  143300 requests in 20.04s, 94.29MB read
Requests/sec:   7150.33
Transfer/sec:      4.70MB

After patch:

smc_run  ./wrk -c 10000 -t 3 -d 20  http://server
  3 threads and 10000 connections
  902091 requests in 21.99s, 593.56MB read
Requests/sec:  41017.52
Transfer/sec:     26.99MB

Signed-off-by: D. Wythe <alibuda@linux.alibaba.com>
---
 net/smc/af_smc.c | 13 +++++++++++--
 net/smc/smc.h    |  1 +
 2 files changed, 12 insertions(+), 2 deletions(-)

Comments

Tony Lu Jan. 4, 2022, 1:01 p.m. UTC | #1 
On Tue, Jan 04, 2022 at 06:58:50PM +0800, D. Wythe wrote:
> From: "D. Wythe" <alibuda@linux.alibaba.com>
> 
> In nginx/wrk multithread and 10K connections benchmark, the
> backend TCP connection established very slowly, and lots of TCP
> connections stay in SYN_SENT state.

<snip>

> +struct workqueue_struct	*smc_tcp_ls_wq;	/* wq for tcp listen work*/
												missing a space here ^
>  struct workqueue_struct	*smc_hs_wq;	/* wq for handshake work */
>  struct workqueue_struct	*smc_close_wq;	/* wq for close work */

<snip>

>  	return (struct smc_sock *)sk;
>  }
>  
> +extern struct workqueue_struct	*smc_tcp_ls_wq;	/* wq for tcp listen work*/
												        missing a space here ^

There are missing two spaces in comments. Besides that, this patch looks
good to me, thanks.

Reviewed-by: Tony Lu <tonylu@linux.alibaba.com>

Thanks.
Tony Lu
D. Wythe Jan. 4, 2022, 1:04 p.m. UTC | #2 
Got it, i'll fix it soon.

Thanks.



在 2022/1/4 下午9:01, Tony Lu 写道:
> On Tue, Jan 04, 2022 at 06:58:50PM +0800, D. Wythe wrote:
>> From: "D. Wythe" <alibuda@linux.alibaba.com>
>>
>> In nginx/wrk multithread and 10K connections benchmark, the
>> backend TCP connection established very slowly, and lots of TCP
>> connections stay in SYN_SENT state.
> <snip>
>
>> +struct workqueue_struct	*smc_tcp_ls_wq;	/* wq for tcp listen work*/
> 												missing a space here ^
>>   struct workqueue_struct	*smc_hs_wq;	/* wq for handshake work */
>>   struct workqueue_struct	*smc_close_wq;	/* wq for close work */
> <snip>
>
>>   	return (struct smc_sock *)sk;
>>   }
>>   
>> +extern struct workqueue_struct	*smc_tcp_ls_wq;	/* wq for tcp listen work*/
> 												        missing a space here ^
>
> There are missing two spaces in comments. Besides that, this patch looks
> good to me, thanks.
>
> Reviewed-by: Tony Lu <tonylu@linux.alibaba.com>
>
> Thanks.
> Tony Lu
diff mbox series

Patch

diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c
index 0bb614e..4d0bc0d 100644
--- a/net/smc/af_smc.c
+++ b/net/smc/af_smc.c
@@ -62,6 +62,7 @@ 
 						 * creation on client
 						 */
 
+struct workqueue_struct	*smc_tcp_ls_wq;	/* wq for tcp listen work*/
 struct workqueue_struct	*smc_hs_wq;	/* wq for handshake work */
 struct workqueue_struct	*smc_close_wq;	/* wq for close work */
 
@@ -1872,7 +1873,7 @@  static void smc_clcsock_data_ready(struct sock *listen_clcsock)
 	lsmc->clcsk_data_ready(listen_clcsock);
 	if (lsmc->sk.sk_state == SMC_LISTEN) {
 		sock_hold(&lsmc->sk); /* sock_put in smc_tcp_listen_work() */
-		if (!queue_work(smc_hs_wq, &lsmc->tcp_listen_work))
+		if (!queue_work(smc_tcp_ls_wq, &lsmc->tcp_listen_work))
 			sock_put(&lsmc->sk);
 	}
 }
@@ -2610,9 +2611,14 @@  static int __init smc_init(void)
 		goto out_nl;
 
 	rc = -ENOMEM;
+
+	smc_tcp_ls_wq = alloc_workqueue("smc_tcp_ls_wq", 0, 0);
+	if (!smc_tcp_ls_wq)
+		goto out_pnet;
+
 	smc_hs_wq = alloc_workqueue("smc_hs_wq", 0, 0);
 	if (!smc_hs_wq)
-		goto out_pnet;
+		goto out_alloc_tcp_ls_wq;
 
 	smc_close_wq = alloc_workqueue("smc_close_wq", 0, 0);
 	if (!smc_close_wq)
@@ -2709,6 +2715,8 @@  static int __init smc_init(void)
 	destroy_workqueue(smc_close_wq);
 out_alloc_hs_wq:
 	destroy_workqueue(smc_hs_wq);
+out_alloc_tcp_ls_wq:
+	destroy_workqueue(smc_tcp_ls_wq);
 out_pnet:
 	smc_pnet_exit();
 out_nl:
@@ -2728,6 +2736,7 @@  static void __exit smc_exit(void)
 	smc_core_exit();
 	smc_ib_unregister_client();
 	destroy_workqueue(smc_close_wq);
+	destroy_workqueue(smc_tcp_ls_wq);
 	destroy_workqueue(smc_hs_wq);
 	proto_unregister(&smc_proto6);
 	proto_unregister(&smc_proto);
diff --git a/net/smc/smc.h b/net/smc/smc.h
index b1d6625..f3bb4be 100644
--- a/net/smc/smc.h
+++ b/net/smc/smc.h
@@ -256,6 +256,7 @@  static inline struct smc_sock *smc_sk(const struct sock *sk)
 	return (struct smc_sock *)sk;
 }
 
+extern struct workqueue_struct	*smc_tcp_ls_wq;	/* wq for tcp listen work*/
 extern struct workqueue_struct	*smc_hs_wq;	/* wq for handshake work */
 extern struct workqueue_struct	*smc_close_wq;	/* wq for close work */