[v2] io: Increase unix socket buffers size on macOS

Commit Message

Nir Soffer April 19, 2025, 11:12 p.m. UTC

On macOS we need to increase unix socket buffers size on the client and
server to get good performance. We set the socket buffers on macOS after
connecting or accepting a client connection.

Testing with qemu-nbd shows that reading an image with qemu-img convert
from qemu-nbd is *11.4 times faster* and qemu-img cpu usage is *8.3 times
lower*.

| qemu-img | qemu-nbd | time   | user   | system |
|----------|----------|--------|--------|--------|
| before   | before   | 12.957 |  2.643 |  5.777 |
| after    | before   | 12.803 |  2.632 |  5.742 |
| before   | after    |  1.139 |  0.074 |  0.905 |
| after    | after    |  1.179 |  0.077 |  0.931 |

For testing buffers size I built qemu-nbd and qemu-img with send buffer
size from 64k to 2m. In this test 256k send buffer and 1m receive buffer
are optimal.

| send buffer | recv buffer | time   | user   | system |
|-------------|-------------|--------|--------|--------|
|         64k |        256k |  2.233 |  0.290 |  1.408 |
|        128k |        512k |  1.189 |  0.103 |  0.841 |
|        256k |       1024k |  1.121 |  0.085 |  0.813 |
|        512k |       2048k |  1.172 |  0.081 |  0.953 |
|       1024k |       4096k |  1.160 |  0.072 |  0.907 |
|       2048k |       8192k |  1.309 |  0.056 |  0.960 |

Using null-co driver is useful to focus on the read part, but in the
real world we do something with the read data. I tested real world usage
with nbdcopy and blksum.

I tested computing a hash of the image using nbdcopy, using 4 NBD
connections and 256k request size. In this test 1m send buffer size and
4m receive buffer size are optimal.

| send buffer | recv buffer | time   | user   | system |
|-------------|-------------|--------|--------|--------|
|         64k |        256k |  2.832 |  4.866 |  2.550 |
|        128k |        512k |  2.429 |  4.762 |  2.037 |
|        256k |       1024k |  2.158 |  4.724 |  1.813 |
|        512k |       2048k |  1.777 |  4.632 |  1.790 |
|       1024k |       4096k |  1.657 |  4.466 |  1.812 |
|       2048k |       8192k |  1.782 |  4.570 |  1.912 |

I tested creating a hash of the image with blksum, using one NBD
connection and 256k read size. In this test 2m send buffer and 8m
receive buffer are optimal.

| send buffer | recv buffer | time   | user   | system |
|-------------|-------------|--------|--------|--------|
|         64k |        256k |  4.233 |  5.242 |  2.632 |
|        128k |        512k |  3.329 |  4.915 |  2.015 |
|        256k |       1024k |  2.071 |  4.647 |  1.474 |
|        512k |       2048k |  1.980 |  4.554 |  1.432 |
|       1024k |       4096k |  2.058 |  4.553 |  1.497 |
|       2048k |       8192k |  1.972 |  4.539 |  1.497 |

In the real world tests larger buffers are optimal, so I picked send
buffer of 1m and receive buffer of 4m.

This will improve other usage of unix domain sockets on macOS. I tested
only reading from qemu-nbd.

The same change for libnbd:
https://gitlab.com/nbdkit/libnbd/-/merge_requests/21

Signed-off-by: Nir Soffer <[e-mail]>
---
 io/channel-socket.c | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

Changes since v1:
- Add UNIX_SOCKET_*_BUFFER_SIZE macros (Philippe)
- Handle both server and client sockets
- Add qio_channel_socket_set_buffers() helper to cleaner code
- Add tests results for qemu-img convert
- Add tests results for different buffer sizes
- Link to same change in libnbd

v1 was here:
https://lists.gnu.org/archive/html/qemu-devel/2025-04/msg03081.html

Patch

diff --git a/io/channel-socket.c b/io/channel-socket.c
index 608bcf066e..635c5c973d 100644
--- a/io/channel-socket.c
+++ b/io/channel-socket.c
@@ -21,6 +21,7 @@ 
 #include "qapi/error.h"
 #include "qapi/qapi-visit-sockets.h"
 #include "qemu/module.h"
+#include "qemu/units.h"
 #include "io/channel-socket.h"
 #include "io/channel-util.h"
 #include "io/channel-watch.h"
@@ -37,6 +38,33 @@ 
 
 #define SOCKET_MAX_FDS 16
 
+/*
+ * Apple recommends sizing the receive buffer at 4 times the size of the send
+ * buffer. Testing shows that 1m send buffer and 4 MiB receive buffer gives
+ * best throuput and lowest cpu usage.
+ */
+#ifdef __APPLE__
+#define UNIX_SOCKET_SEND_BUFFER_SIZE (1 * MiB)
+#define UNIX_SOCKET_RECV_BUFFER_SIZE (4 * UNIX_SOCKET_SEND_BUFFER_SIZE)
+#endif /* __APPLE__ */
+
+static void qio_channel_socket_set_buffers(QIOChannelSocket *ioc)
+{
+#ifdef __APPLE__
+    if (ioc->localAddr.ss_family == AF_UNIX) {
+        int value;
+
+        /* This is a performance optimization; don't fail on errors. */
+
+        value = UNIX_SOCKET_SEND_BUFFER_SIZE;
+        setsockopt(ioc->fd, SOL_SOCKET, SO_SNDBUF, &value, sizeof(value));
+
+        value = UNIX_SOCKET_RECV_BUFFER_SIZE;
+        setsockopt(ioc->fd, SOL_SOCKET, SO_RCVBUF, &value, sizeof(value));
+    }
+#endif /* __APPLE__ */
+}
+
 SocketAddress *
 qio_channel_socket_get_local_address(QIOChannelSocket *ioc,
                                      Error **errp)
@@ -174,6 +202,8 @@  int qio_channel_socket_connect_sync(QIOChannelSocket *ioc,
     }
 #endif
 
+    qio_channel_socket_set_buffers(ioc);
+
     qio_channel_set_feature(QIO_CHANNEL(ioc),
                             QIO_CHANNEL_FEATURE_READ_MSG_PEEK);
 
@@ -410,6 +440,8 @@  qio_channel_socket_accept(QIOChannelSocket *ioc,
     }
 #endif /* WIN32 */
 
+    qio_channel_socket_set_buffers(cioc);
+
     qio_channel_set_feature(QIO_CHANNEL(cioc),
                             QIO_CHANNEL_FEATURE_READ_MSG_PEEK);