@@ -36,8 +36,6 @@
#include <sys/un.h>
#include <sys/wait.h>
-#if 0
-
#define NOTIF_TAG 0xfffffffULL
#define NONZC_TAG 0
#define ZC_TAG 1
@@ -49,7 +47,6 @@ enum {
MODE_MIXED = 3,
};
-static bool cfg_flush = false;
static bool cfg_cork = false;
static int cfg_mode = MODE_ZC_FIXED;
static int cfg_nr_reqs = 8;
@@ -168,21 +165,6 @@ static int io_uring_register_buffers(struct io_uring *ring,
return (ret < 0) ? -errno : ret;
}
-static int io_uring_register_notifications(struct io_uring *ring,
- unsigned nr,
- struct io_uring_notification_slot *slots)
-{
- int ret;
- struct io_uring_notification_register r = {
- .nr_slots = nr,
- .data = (unsigned long)slots,
- };
-
- ret = syscall(__NR_io_uring_register, ring->ring_fd,
- IORING_REGISTER_NOTIFIERS, &r, sizeof(r));
- return (ret < 0) ? -errno : ret;
-}
-
static int io_uring_mmap(int fd, struct io_uring_params *p,
struct io_uring_sq *sq, struct io_uring_cq *cq)
{
@@ -299,11 +281,10 @@ static inline void io_uring_prep_send(struct io_uring_sqe *sqe, int sockfd,
static inline void io_uring_prep_sendzc(struct io_uring_sqe *sqe, int sockfd,
const void *buf, size_t len, int flags,
- unsigned slot_idx, unsigned zc_flags)
+ unsigned zc_flags)
{
io_uring_prep_send(sqe, sockfd, buf, len, flags);
- sqe->opcode = (__u8) IORING_OP_SENDZC_NOTIF;
- sqe->notification_idx = slot_idx;
+ sqe->opcode = (__u8) IORING_OP_SEND_ZC;
sqe->ioprio = zc_flags;
}
@@ -376,7 +357,6 @@ static int do_setup_tx(int domain, int type, int protocol)
static void do_tx(int domain, int type, int protocol)
{
- struct io_uring_notification_slot b[1] = {{.tag = NOTIF_TAG}};
struct io_uring_sqe *sqe;
struct io_uring_cqe *cqe;
unsigned long packets = 0, bytes = 0;
@@ -392,10 +372,6 @@ static void do_tx(int domain, int type, int protocol)
if (ret)
error(1, ret, "io_uring: queue init");
- ret = io_uring_register_notifications(&ring, 1, b);
- if (ret)
- error(1, ret, "io_uring: tx ctx registration");
-
iov.iov_base = payload;
iov.iov_len = cfg_payload_len;
@@ -411,9 +387,8 @@ static void do_tx(int domain, int type, int protocol)
for (i = 0; i < cfg_nr_reqs; i++) {
unsigned zc_flags = 0;
unsigned buf_idx = 0;
- unsigned slot_idx = 0;
unsigned mode = cfg_mode;
- unsigned msg_flags = 0;
+ unsigned msg_flags = MSG_WAITALL;
if (cfg_mode == MODE_MIXED)
mode = rand() % 3;
@@ -425,13 +400,10 @@ static void do_tx(int domain, int type, int protocol)
cfg_payload_len, msg_flags);
sqe->user_data = NONZC_TAG;
} else {
- if (cfg_flush) {
- zc_flags |= IORING_RECVSEND_NOTIF_FLUSH;
- compl_cqes++;
- }
+ compl_cqes++;
io_uring_prep_sendzc(sqe, fd, payload,
cfg_payload_len,
- msg_flags, slot_idx, zc_flags);
+ msg_flags, zc_flags);
if (mode == MODE_ZC_FIXED) {
sqe->ioprio |= IORING_RECVSEND_FIXED_BUF;
sqe->buf_index = buf_idx;
@@ -444,51 +416,57 @@ static void do_tx(int domain, int type, int protocol)
if (ret != cfg_nr_reqs)
error(1, ret, "submit");
+ if (cfg_cork)
+ do_setsockopt(fd, IPPROTO_UDP, UDP_CORK, 0);
for (i = 0; i < cfg_nr_reqs; i++) {
ret = io_uring_wait_cqe(&ring, &cqe);
if (ret)
error(1, ret, "wait cqe");
- if (cqe->user_data == NOTIF_TAG) {
+ if (cqe->user_data != NONZC_TAG &&
+ cqe->user_data != ZC_TAG)
+ error(1, -EINVAL, "invalid cqe->user_data");
+
+ if (cqe->flags & IORING_CQE_F_NOTIF) {
+ if (cqe->flags & IORING_CQE_F_MORE)
+ error(1, -EINVAL, "invalid notif flags");
compl_cqes--;
i--;
- } else if (cqe->user_data != NONZC_TAG &&
- cqe->user_data != ZC_TAG) {
- error(1, cqe->res, "invalid user_data");
- } else if (cqe->res <= 0 && cqe->res != -EAGAIN) {
+ } else if (cqe->res <= 0) {
+ if (cqe->flags & IORING_CQE_F_MORE)
+ error(1, cqe->res, "more with a failed send");
error(1, cqe->res, "send failed");
} else {
- if (cqe->res > 0) {
- packets++;
- bytes += cqe->res;
- }
- /* failed requests don't flush */
- if (cfg_flush &&
- cqe->res <= 0 &&
- cqe->user_data == ZC_TAG)
- compl_cqes--;
+ if (cqe->user_data == ZC_TAG &&
+ !(cqe->flags & IORING_CQE_F_MORE))
+ error(1, cqe->res, "missing more flag");
+ packets++;
+ bytes += cqe->res;
}
io_uring_cqe_seen(&ring);
}
- if (cfg_cork)
- do_setsockopt(fd, IPPROTO_UDP, UDP_CORK, 0);
} while (gettimeofday_ms() < tstop);
- if (close(fd))
- error(1, errno, "close");
-
- fprintf(stderr, "tx=%lu (MB=%lu), tx/s=%lu (MB/s=%lu)\n",
- packets, bytes >> 20,
- packets / (cfg_runtime_ms / 1000),
- (bytes >> 20) / (cfg_runtime_ms / 1000));
-
while (compl_cqes) {
ret = io_uring_wait_cqe(&ring, &cqe);
if (ret)
error(1, ret, "wait cqe");
+ if (cqe->flags & IORING_CQE_F_MORE)
+ error(1, -EINVAL, "invalid notif flags");
+ if (!(cqe->flags & IORING_CQE_F_NOTIF))
+ error(1, -EINVAL, "missing notif flag");
+
io_uring_cqe_seen(&ring);
compl_cqes--;
}
+
+ fprintf(stderr, "tx=%lu (MB=%lu), tx/s=%lu (MB/s=%lu)\n",
+ packets, bytes >> 20,
+ packets / (cfg_runtime_ms / 1000),
+ (bytes >> 20) / (cfg_runtime_ms / 1000));
+
+ if (close(fd))
+ error(1, errno, "close");
}
static void do_test(int domain, int type, int protocol)
@@ -502,8 +480,8 @@ static void do_test(int domain, int type, int protocol)
static void usage(const char *filepath)
{
- error(1, 0, "Usage: %s [-f] [-n<N>] [-z0] [-s<payload size>] "
- "(-4|-6) [-t<time s>] -D<dst_ip> udp", filepath);
+ error(1, 0, "Usage: %s (-4|-6) (udp|tcp) -D<dst_ip> [-s<payload size>] "
+ "[-t<time s>] [-n<batch>] [-p<port>] [-m<mode>]", filepath);
}
static void parse_opts(int argc, char **argv)
@@ -521,7 +499,7 @@ static void parse_opts(int argc, char **argv)
usage(argv[0]);
cfg_payload_len = max_payload_len;
- while ((c = getopt(argc, argv, "46D:p:s:t:n:fc:m:")) != -1) {
+ while ((c = getopt(argc, argv, "46D:p:s:t:n:c:m:")) != -1) {
switch (c) {
case '4':
if (cfg_family != PF_UNSPEC)
@@ -550,9 +528,6 @@ static void parse_opts(int argc, char **argv)
case 'n':
cfg_nr_reqs = strtoul(optarg, NULL, 0);
break;
- case 'f':
- cfg_flush = 1;
- break;
case 'c':
cfg_cork = strtol(optarg, NULL, 0);
break;
@@ -585,8 +560,6 @@ static void parse_opts(int argc, char **argv)
if (cfg_payload_len > max_payload_len)
error(1, 0, "-s: payload exceeds max (%d)", max_payload_len);
- if (cfg_mode == MODE_NONZC && cfg_flush)
- error(1, 0, "-f: only zerocopy modes support notifications");
if (optind != argc - 1)
usage(argv[0]);
}
@@ -605,10 +578,3 @@ int main(int argc, char **argv)
error(1, 0, "unknown cfg_test %s", cfg_test);
return 0;
}
-
-#else
-int main(int argc, char **argv)
-{
- return 0;
-}
-#endif
@@ -25,15 +25,11 @@ readonly path_sysctl_mem="net.core.optmem_max"
# No arguments: automated test
if [[ "$#" -eq "0" ]]; then
IPs=( "4" "6" )
- protocols=( "tcp" "udp" )
for IP in "${IPs[@]}"; do
- for proto in "${protocols[@]}"; do
- for mode in $(seq 1 3); do
- $0 "$IP" "$proto" -m "$mode" -t 1 -n 32
- $0 "$IP" "$proto" -m "$mode" -t 1 -n 32 -f
- $0 "$IP" "$proto" -m "$mode" -t 1 -n 32 -c -f
- done
+ for mode in $(seq 1 3); do
+ $0 "$IP" udp -m "$mode" -t 1 -n 32
+ $0 "$IP" tcp -m "$mode" -t 1 -n 32
done
done
Enable io_uring zerocopy send tests back and fix them up to follow the new inteface. Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> --- .../selftests/net/io_uring_zerocopy_tx.c | 110 ++++++------------ .../selftests/net/io_uring_zerocopy_tx.sh | 10 +- 2 files changed, 41 insertions(+), 79 deletions(-)