From patchwork Fri Sep 2 11:12:36 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12964045 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 7CA7AC6FA83 for ; Fri, 2 Sep 2022 11:14:55 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S229496AbiIBLOy (ORCPT ); Fri, 2 Sep 2022 07:14:54 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:39978 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S232171AbiIBLOx (ORCPT ); Fri, 2 Sep 2022 07:14:53 -0400 Received: from mail-wr1-x435.google.com (mail-wr1-x435.google.com [IPv6:2a00:1450:4864:20::435]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id B6C90E0A7 for ; Fri, 2 Sep 2022 04:14:50 -0700 (PDT) Received: by mail-wr1-x435.google.com with SMTP id bp20so1385727wrb.9 for ; Fri, 02 Sep 2022 04:14:50 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date; bh=+EzCvqhuPXn6pBcKGFX9KUSNBtR+7W5a37WMLSsZlzc=; b=IFbvuHZyvT0yMn1h03g4YrVXgIqudz11VLV4oenbmXAJDxtRAC6YpGAJzB29UOm8NK c41bWYTNwe7MxseU07D/Y7muGRb7KRiXWsM0t5XJA/LQvNpF4tnbUjiF9K/35NqES9PM CVbS1VeCr98so2B2jCXBnsOQ0oBlCzzTrviWW5fMmf+61U3A+HhoaBXnRo0ahV4vCnI0 A8gqaeR08rP5kMSSUdQGbJlEoO17mO0LQx0jgWVjv8ONutyERYQuz9CgAP0dgti/etfQ SwdN80xoXwMAqmMIGM8kM+/I7REZKbXCC1TG8syIWh2a9gqf1C0pJKzMjcoT+X8M2Fni gxNQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date; bh=+EzCvqhuPXn6pBcKGFX9KUSNBtR+7W5a37WMLSsZlzc=; b=MbiVfTaIxSWWkdsV+B/VJ2+jQp3tZGwr0kRTV39a49deM9eWFjmHbja+yZrpAe6IJB sMp6rFj9ACa2joWTr8rLfLtwa/MFIP1asPzp2t8gd0eb0dMCoqqAg28FjlNX0rrWvvEM ADfkj0dNi47VzPwkLF+GmM1SZr5H1tV84JfG9xTEnHNZm2WoYUT4aA8shvw6vBEq60i9 dIHiUxq5PUzJlSvHia8PXi3IUfCxzKRkMZnyR4HZnFDfB5P6r7nLIisb4d8YC0DU0J4x /B01cB+xcZLDipGlc5gffY+Olyjg6H9oZrAJzJ9yt9nw5MCLl7M5dxyXpaIHYc8KikLA I7Yg== X-Gm-Message-State: ACgBeo3EdcCNbU+42xLNKwXdxRl3qYa9SW9jAcrpzC1BLU/ym4vQenbd rIvtyBxR7dUMU1rgcZBPPDV/Rf/+AlI= X-Google-Smtp-Source: AA6agR4+yDbt92zo3WTp4PKhcMIthZkM6U85BEkcplj9ENXcXdHHITdkyBJrEAVHR7PoEpO3QYStdg== X-Received: by 2002:a05:6000:78d:b0:226:d5c1:5ebd with SMTP id bu13-20020a056000078d00b00226d5c15ebdmr14830862wrb.565.1662117288786; Fri, 02 Sep 2022 04:14:48 -0700 (PDT) Received: from 127.0.0.1localhost (82-132-230-225.dab.02.net. [82.132.230.225]) by smtp.gmail.com with ESMTPSA id bg32-20020a05600c3ca000b003a536d5aa2esm2087379wmb.11.2022.09.02.04.14.47 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 02 Sep 2022 04:14:48 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH liburing 1/4] tests: verify that send addr is copied when async Date: Fri, 2 Sep 2022 12:12:36 +0100 Message-Id: <67fa1af95c4565ef0e9f43833b67d6c4d50d66f6.1662116617.git.asml.silence@gmail.com> X-Mailer: git-send-email 2.37.2 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org The kernel should not be touching timespec after returning from submit syscall, make sure it's not reloaded after a request goes async. Signed-off-by: Pavel Begunkov --- test/send-zerocopy.c | 69 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/test/send-zerocopy.c b/test/send-zerocopy.c index 307c114..97727e3 100644 --- a/test/send-zerocopy.c +++ b/test/send-zerocopy.c @@ -836,6 +836,69 @@ static int test_inet_send(struct io_uring *ring) return 0; } +static int test_async_addr(struct io_uring *ring) +{ + struct io_uring_sqe *sqe; + struct io_uring_cqe *cqe; + struct sockaddr_storage addr; + int sock_tx = -1, sock_rx = -1; + struct __kernel_timespec ts; + int ret; + + ret = prepare_ip(&addr, &sock_tx, &sock_rx, true, false, false, false); + if (ret) { + fprintf(stderr, "sock prep failed %d\n", ret); + return 1; + } + + sqe = io_uring_get_sqe(ring); + ts.tv_sec = 1; + ts.tv_nsec = 0; + io_uring_prep_timeout(sqe, &ts, 0, IORING_TIMEOUT_ETIME_SUCCESS); + sqe->user_data = 1; + sqe->flags |= IOSQE_IO_LINK; + + sqe = io_uring_get_sqe(ring); + io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, 1, 0, 0, 0); + sqe->user_data = 2; + io_uring_prep_sendzc_set_addr(sqe, (const struct sockaddr *)&addr, + sizeof(struct sockaddr_in6)); + + ret = io_uring_submit(ring); + assert(ret == 2); + memset(&addr, 0, sizeof(addr)); + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "io_uring_wait_cqe failed %i\n", ret); + return 1; + } + if (cqe->user_data != 1 || cqe->res != -ETIME) { + fprintf(stderr, "invalid timeout res %i %i\n", + (int)cqe->user_data, cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "io_uring_wait_cqe failed %i\n", ret); + return 1; + } + if (cqe->user_data != 2 || cqe->res != 1) { + fprintf(stderr, "invalid send %i %i\n", + (int)cqe->user_data, cqe->res); + return 1; + } + io_uring_cqe_seen(ring, cqe); + ret = recv(sock_rx, rx_buffer, 1, MSG_TRUNC); + assert(ret == 1); + + close(sock_tx); + close(sock_rx); + return 0; +} + int main(int argc, char *argv[]) { struct io_uring ring; @@ -973,6 +1036,12 @@ int main(int argc, char *argv[]) fprintf(stderr, "test_inet_send() failed\n"); return ret; } + + ret = test_async_addr(&ring); + if (ret) { + fprintf(stderr, "test_async_addr() failed\n"); + return ret; + } out: io_uring_queue_exit(&ring); close(sp[0]); From patchwork Fri Sep 2 11:12:37 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12964047 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id E60A0ECAAD5 for ; Fri, 2 Sep 2022 11:14:58 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S232685AbiIBLO6 (ORCPT ); Fri, 2 Sep 2022 07:14:58 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:40312 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S232699AbiIBLO4 (ORCPT ); Fri, 2 Sep 2022 07:14:56 -0400 Received: from mail-wr1-x433.google.com (mail-wr1-x433.google.com [IPv6:2a00:1450:4864:20::433]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id B5BF712098 for ; Fri, 2 Sep 2022 04:14:52 -0700 (PDT) Received: by mail-wr1-x433.google.com with SMTP id u17so1885720wrp.3 for ; Fri, 02 Sep 2022 04:14:52 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date; bh=ixN4fNCHp134Y/RfUCeJ08tVCq+qQTBygkI4LG6AIu0=; b=oq2DOldnf/0XASA3ucvPMRFgvbseRhCo7nO8g57byHRg6ynfNaTehpohkWvuGqnVOh gbScAlcCqbIwNSw0NvomfEtE13bLOM0Sj2atew2XofNhQYezzIaWRkX43/LoFkI6awvT tjWEXlm4Wd8kaGE92UVKevfe25GbUHdHYyBIxmX9paGWWXs+6m32LKdOD1IDm0neAuRb eVMWYO0G/tPjxWTxWoQd5yKlyKwcHQ4EJ0tx4My4v9ELLLgloGFgemz8G3svKu3nkKDm 9uKaJCl3twViLTdHkF9d80jxCG4u40yfKkdVJbsHJBYBW/8f00UMEiHGAoHTjdbaaDoO Ta2w== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date; bh=ixN4fNCHp134Y/RfUCeJ08tVCq+qQTBygkI4LG6AIu0=; b=zZgApZ2QwfYgQkQCYSn9TFuXSD2LrKzH1+xUskrK0oDV8KRuSdOuONGxG3E31vknjh YaBxNOdN5vuax+wMo/SbsiD9TqhOSR7XYawkfr+DgnZNCjOkznYxn4MOPUYKO6sLMUSG V3O5VasWyE+/m2JANH4BtjOXyjTTSo4tDirMw6mYBh/2Pk1T+DCpsoQgPW6p7kFcGe1V YboFRVAMd4QAyZhR4DnTACdd7iEFwy/6OFNyXvnIntoiQb97eumFvj6QvgVhRUC3n0QV lLpKkcJKz+ST7qtyjQviv++n5v+v6PnHGvqVD0VmYobGBfKuOMTgBUuBLOmuXQKqOujp XoGQ== X-Gm-Message-State: ACgBeo3HyC6oZqpF1k5KWq7pC8+n65ma5APowPglEuVZs2/P08alMwND x9QrKdWxQtBXQVhf1rwgpmng729q/7Y= X-Google-Smtp-Source: AA6agR7j1bz1cq6VvAV2ucOLdri4aPWGYBg6QQRBiRXovUcxlR1dTwP+NeMOIuVA+aIsR0HmP7FgXw== X-Received: by 2002:a05:6000:191:b0:226:eb62:775d with SMTP id p17-20020a056000019100b00226eb62775dmr7074080wrx.687.1662117290017; Fri, 02 Sep 2022 04:14:50 -0700 (PDT) Received: from 127.0.0.1localhost (82-132-230-225.dab.02.net. [82.132.230.225]) by smtp.gmail.com with ESMTPSA id bg32-20020a05600c3ca000b003a536d5aa2esm2087379wmb.11.2022.09.02.04.14.48 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 02 Sep 2022 04:14:49 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH liburing 2/4] zc: adjust sendzc to the simpler uapi Date: Fri, 2 Sep 2022 12:12:37 +0100 Message-Id: X-Mailer: git-send-email 2.37.2 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org We update the zerocopy API, reflect it in liburing, this includes removing the whole notification story, updating headers and fixing up tests. Signed-off-by: Pavel Begunkov --- src/include/liburing.h | 39 +- src/include/liburing/io_uring.h | 28 +- src/register.c | 20 -- test/send-zerocopy.c | 611 ++++---------------------------- 4 files changed, 91 insertions(+), 607 deletions(-) diff --git a/src/include/liburing.h b/src/include/liburing.h index 68a1646..c672440 100644 --- a/src/include/liburing.h +++ b/src/include/liburing.h @@ -199,10 +199,6 @@ int io_uring_register_sync_cancel(struct io_uring *ring, int io_uring_register_file_alloc_range(struct io_uring *ring, unsigned off, unsigned len); -int io_uring_register_notifications(struct io_uring *ring, unsigned nr, - struct io_uring_notification_slot *slots); -int io_uring_unregister_notifications(struct io_uring *ring); - /* * io_uring syscalls. */ @@ -702,44 +698,23 @@ static inline void io_uring_prep_send(struct io_uring_sqe *sqe, int sockfd, sqe->msg_flags = (__u32) flags; } -static inline void io_uring_prep_sendzc(struct io_uring_sqe *sqe, int sockfd, - const void *buf, size_t len, int flags, - unsigned slot_idx, unsigned zc_flags) +static inline void io_uring_prep_send_zc(struct io_uring_sqe *sqe, int sockfd, + const void *buf, size_t len, int flags, + unsigned zc_flags) { - io_uring_prep_rw(IORING_OP_SENDZC_NOTIF, sqe, sockfd, buf, (__u32) len, 0); + io_uring_prep_rw(IORING_OP_SEND_ZC, sqe, sockfd, buf, (__u32) len, 0); sqe->msg_flags = (__u32) flags; - sqe->notification_idx = slot_idx; sqe->ioprio = zc_flags; } -static inline void io_uring_prep_sendzc_fixed(struct io_uring_sqe *sqe, int sockfd, - const void *buf, size_t len, - int flags, unsigned slot_idx, - unsigned zc_flags, unsigned buf_idx) -{ - io_uring_prep_sendzc(sqe, sockfd, buf, len, flags, slot_idx, zc_flags); - sqe->ioprio |= IORING_RECVSEND_FIXED_BUF; - sqe->buf_index = buf_idx; -} - -static inline void io_uring_prep_sendzc_set_addr(struct io_uring_sqe *sqe, - const struct sockaddr *dest_addr, - __u16 addr_len) +static inline void io_uring_prep_send_set_addr(struct io_uring_sqe *sqe, + const struct sockaddr *dest_addr, + __u16 addr_len) { sqe->addr2 = (unsigned long)(void *)dest_addr; sqe->addr_len = addr_len; } -static inline void io_uring_prep_notif_update(struct io_uring_sqe *sqe, - __u64 new_tag, /* 0 to ignore */ - unsigned offset, unsigned nr) -{ - io_uring_prep_rw(IORING_OP_FILES_UPDATE, sqe, -1, 0, nr, - (__u64)offset); - sqe->addr = new_tag; - sqe->ioprio = IORING_RSRC_UPDATE_NOTIF; -} - static inline void io_uring_prep_recv(struct io_uring_sqe *sqe, int sockfd, void *buf, size_t len, int flags) { diff --git a/src/include/liburing/io_uring.h b/src/include/liburing/io_uring.h index 9e0b5c8..6b83177 100644 --- a/src/include/liburing/io_uring.h +++ b/src/include/liburing/io_uring.h @@ -71,8 +71,8 @@ struct io_uring_sqe { __s32 splice_fd_in; __u32 file_index; struct { - __u16 notification_idx; __u16 addr_len; + __u16 __pad3[1]; }; }; union { @@ -178,8 +178,7 @@ enum io_uring_op { IORING_OP_FALLOCATE, IORING_OP_OPENAT, IORING_OP_CLOSE, - IORING_OP_RSRC_UPDATE, - IORING_OP_FILES_UPDATE = IORING_OP_RSRC_UPDATE, + IORING_OP_FILES_UPDATE, IORING_OP_STATX, IORING_OP_READ, IORING_OP_WRITE, @@ -206,7 +205,7 @@ enum io_uring_op { IORING_OP_GETXATTR, IORING_OP_SOCKET, IORING_OP_URING_CMD, - IORING_OP_SENDZC_NOTIF, + IORING_OP_SEND_ZC, /* this goes last, obviously */ IORING_OP_LAST, @@ -228,7 +227,6 @@ enum io_uring_op { #define IORING_TIMEOUT_ETIME_SUCCESS (1U << 5) #define IORING_TIMEOUT_CLOCK_MASK (IORING_TIMEOUT_BOOTTIME | IORING_TIMEOUT_REALTIME) #define IORING_TIMEOUT_UPDATE_MASK (IORING_TIMEOUT_UPDATE | IORING_LINK_TIMEOUT_UPDATE) - /* * sqe->splice_flags * extends splice(2) flags @@ -281,29 +279,16 @@ enum io_uring_op { * * IORING_RECVSEND_FIXED_BUF Use registered buffers, the index is stored in * the buf_index field. - * - * IORING_RECVSEND_NOTIF_FLUSH Flush a notification after a successful - * successful. Only for zerocopy sends. */ #define IORING_RECVSEND_POLL_FIRST (1U << 0) #define IORING_RECV_MULTISHOT (1U << 1) #define IORING_RECVSEND_FIXED_BUF (1U << 2) -#define IORING_RECVSEND_NOTIF_FLUSH (1U << 3) /* * accept flags stored in sqe->ioprio */ #define IORING_ACCEPT_MULTISHOT (1U << 0) - -/* - * IORING_OP_RSRC_UPDATE flags - */ -enum { - IORING_RSRC_UPDATE_FILES, - IORING_RSRC_UPDATE_NOTIF, -}; - /* * IORING_OP_MSG_RING command types, stored in sqe->addr */ @@ -341,10 +326,13 @@ struct io_uring_cqe { * IORING_CQE_F_BUFFER If set, the upper 16 bits are the buffer ID * IORING_CQE_F_MORE If set, parent SQE will generate more CQE entries * IORING_CQE_F_SOCK_NONEMPTY If set, more data to read after socket recv + * IORING_CQE_F_NOTIF Set for notification CQEs. Can be used to distinct + * them from sends. */ #define IORING_CQE_F_BUFFER (1U << 0) #define IORING_CQE_F_MORE (1U << 1) #define IORING_CQE_F_SOCK_NONEMPTY (1U << 2) +#define IORING_CQE_F_NOTIF (1U << 3) enum { IORING_CQE_BUFFER_SHIFT = 16, @@ -485,10 +473,6 @@ enum { /* register a range of fixed file slots for automatic slot allocation */ IORING_REGISTER_FILE_ALLOC_RANGE = 25, - /* zerocopy notification API */ - IORING_REGISTER_NOTIFIERS = 26, - IORING_UNREGISTER_NOTIFIERS = 27, - /* this goes last */ IORING_REGISTER_LAST }; diff --git a/src/register.c b/src/register.c index 7482112..2b37e5f 100644 --- a/src/register.c +++ b/src/register.c @@ -364,23 +364,3 @@ int io_uring_register_file_alloc_range(struct io_uring *ring, IORING_REGISTER_FILE_ALLOC_RANGE, &range, 0); } - -int io_uring_register_notifications(struct io_uring *ring, unsigned nr, - struct io_uring_notification_slot *slots) -{ - struct io_uring_notification_register r = { - .nr_slots = nr, - .data = (unsigned long)slots, - }; - - return __sys_io_uring_register(ring->ring_fd, - IORING_REGISTER_NOTIFIERS, - &r, sizeof(r)); -} - -int io_uring_unregister_notifications(struct io_uring *ring) -{ - return __sys_io_uring_register(ring->ring_fd, - IORING_UNREGISTER_NOTIFIERS, - NULL, 0); -} diff --git a/test/send-zerocopy.c b/test/send-zerocopy.c index 97727e3..7b58ae7 100644 --- a/test/send-zerocopy.c +++ b/test/send-zerocopy.c @@ -44,7 +44,6 @@ #define HOST "127.0.0.1" #define HOSTV6 "::1" -#define NR_SLOTS 5 #define ZC_TAG 10000 #define BUFFER_OFFSET 41 @@ -52,15 +51,9 @@ #define ARRAY_SIZE(a) (sizeof(a)/sizeof((a)[0])) #endif -static int seqs[NR_SLOTS]; static char *tx_buffer, *rx_buffer; static struct iovec buffers_iov[3]; -static inline bool tag_userdata(__u64 user_data) -{ - return ZC_TAG <= user_data && user_data < ZC_TAG + NR_SLOTS; -} - static bool check_cq_empty(struct io_uring *ring) { struct io_uring_cqe *cqe = NULL; @@ -70,41 +63,18 @@ static bool check_cq_empty(struct io_uring *ring) return ret == -EAGAIN; } -static int register_notifications(struct io_uring *ring) -{ - struct io_uring_notification_slot slots[NR_SLOTS] = {}; - int i; - - memset(seqs, 0, sizeof(seqs)); - for (i = 0; i < NR_SLOTS; i++) - slots[i].tag = ZC_TAG + i; - return io_uring_register_notifications(ring, NR_SLOTS, slots); -} - -static int reregister_notifications(struct io_uring *ring) -{ - int ret; - - ret = io_uring_unregister_notifications(ring); - if (ret) { - fprintf(stderr, "unreg notifiers failed %i\n", ret); - return ret; - } - - return register_notifications(ring); -} - -static int do_one(struct io_uring *ring, int sock_tx, int slot_idx) +static int test_basic_send(struct io_uring *ring, int sock_tx, int sock_rx) { struct io_uring_sqe *sqe; struct io_uring_cqe *cqe; int msg_flags = 0; unsigned zc_flags = 0; + int payload_size = 100; int ret; sqe = io_uring_get_sqe(ring); - io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, 1, msg_flags, - slot_idx, zc_flags); + io_uring_prep_send_zc(sqe, sock_tx, tx_buffer, payload_size, + msg_flags, zc_flags); sqe->user_data = 1; ret = io_uring_submit(ring); @@ -112,434 +82,61 @@ static int do_one(struct io_uring *ring, int sock_tx, int slot_idx) ret = io_uring_wait_cqe(ring, &cqe); assert(!ret); assert(cqe->user_data == 1); - ret = cqe->res; + if (ret == -EINVAL) { + assert(!(cqe->flags & IORING_CQE_F_MORE)); + return T_EXIT_SKIP; + } + assert(cqe->res == payload_size); + assert(cqe->flags & IORING_CQE_F_MORE); io_uring_cqe_seen(ring, cqe); - assert(check_cq_empty(ring)); - return ret; -} - -static int test_invalid_slot(struct io_uring *ring, int sock_tx, int sock_rx) -{ - int ret; - - ret = do_one(ring, sock_tx, NR_SLOTS); - assert(ret == -EINVAL); - return 0; -} - -static int test_basic_send(struct io_uring *ring, int sock_tx, int sock_rx) -{ - struct io_uring_sqe *sqe; - struct io_uring_cqe *cqe; - int msg_flags = 0; - int slot_idx = 0; - unsigned zc_flags = 0; - int payload_size = 100; - int ret; - - sqe = io_uring_get_sqe(ring); - io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, payload_size, msg_flags, - slot_idx, zc_flags); - sqe->user_data = 1; - ret = io_uring_submit(ring); - assert(ret == 1); ret = io_uring_wait_cqe(ring, &cqe); assert(!ret); - assert(cqe->user_data == 1 && cqe->res >= 0); + assert(cqe->user_data == 1); + assert(cqe->flags & IORING_CQE_F_NOTIF); + assert(!(cqe->flags & IORING_CQE_F_MORE)); io_uring_cqe_seen(ring, cqe); assert(check_cq_empty(ring)); ret = recv(sock_rx, rx_buffer, payload_size, MSG_TRUNC); assert(ret == payload_size); - return 0; + return T_EXIT_PASS; } -static int test_send_flush(struct io_uring *ring, int sock_tx, int sock_rx) +static int test_send_faults(struct io_uring *ring, int sock_tx, int sock_rx) { struct io_uring_sqe *sqe; struct io_uring_cqe *cqe; int msg_flags = 0; - int slot_idx = 0; unsigned zc_flags = 0; int payload_size = 100; - int ret, i, j; - int req_cqes, notif_cqes; - - /* now do send+flush, do many times to verify seqs */ - for (j = 0; j < NR_SLOTS * 5; j++) { - zc_flags = IORING_RECVSEND_NOTIF_FLUSH; - slot_idx = rand() % NR_SLOTS; - sqe = io_uring_get_sqe(ring); - io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, payload_size, - msg_flags, slot_idx, zc_flags); - sqe->user_data = 1; - - ret = io_uring_submit(ring); - assert(ret == 1); - - req_cqes = notif_cqes = 1; - for (i = 0; i < 2; i ++) { - ret = io_uring_wait_cqe(ring, &cqe); - assert(!ret); - - if (cqe->user_data == 1) { - assert(req_cqes > 0); - req_cqes--; - assert(cqe->res == payload_size); - } else if (cqe->user_data == ZC_TAG + slot_idx) { - assert(notif_cqes > 0); - notif_cqes--; - assert(cqe->res == 0 && cqe->flags == seqs[slot_idx]); - seqs[slot_idx]++; - } else { - fprintf(stderr, "invalid cqe %lu %i\n", - (unsigned long)cqe->user_data, cqe->res); - return -1; - } - io_uring_cqe_seen(ring, cqe); - } - assert(check_cq_empty(ring)); - - ret = recv(sock_rx, rx_buffer, payload_size, MSG_TRUNC); - assert(ret == payload_size); - } - return 0; -} - -static int test_multireq_notif(struct io_uring *ring, int sock_tx, int sock_rx) -{ - bool slot_seen[NR_SLOTS] = {}; - struct io_uring_sqe *sqe; - struct io_uring_cqe *cqe; - int msg_flags = 0; - int slot_idx = 0; - unsigned zc_flags = 0; - int payload_size = 1; - int ret, j, i = 0; - int nr = NR_SLOTS * 21; - - while (i < nr) { - int nr_per_wave = 23; - - for (j = 0; j < nr_per_wave && i < nr; j++, i++) { - slot_idx = rand() % NR_SLOTS; - sqe = io_uring_get_sqe(ring); - io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, payload_size, - msg_flags, slot_idx, zc_flags); - sqe->user_data = i; - } - ret = io_uring_submit(ring); - assert(ret == j); - } - - for (i = 0; i < nr; i++) { - ret = io_uring_wait_cqe(ring, &cqe); - assert(!ret); - assert(cqe->user_data < nr && cqe->res == payload_size); - io_uring_cqe_seen(ring, cqe); - - ret = recv(sock_rx, rx_buffer, payload_size, MSG_TRUNC); - assert(ret == payload_size); - } - assert(check_cq_empty(ring)); - - zc_flags = IORING_RECVSEND_NOTIF_FLUSH; - for (slot_idx = 0; slot_idx < NR_SLOTS; slot_idx++) { - sqe = io_uring_get_sqe(ring); - io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, payload_size, - msg_flags, slot_idx, zc_flags); - sqe->user_data = slot_idx; - /* just to simplify cqe handling */ - sqe->flags |= IOSQE_CQE_SKIP_SUCCESS; - } - ret = io_uring_submit(ring); - assert(ret == NR_SLOTS); - - for (i = 0; i < NR_SLOTS; i++) { - int slot_idx; - - ret = io_uring_wait_cqe(ring, &cqe); - assert(!ret); - assert(tag_userdata(cqe->user_data)); - - slot_idx = cqe->user_data - ZC_TAG; - assert(!slot_seen[slot_idx]); - slot_seen[slot_idx] = true; - - assert(cqe->res == 0 && cqe->flags == seqs[slot_idx]); - seqs[slot_idx]++; - io_uring_cqe_seen(ring, cqe); - - ret = recv(sock_rx, rx_buffer, payload_size, MSG_TRUNC); - assert(ret == payload_size); - } - assert(check_cq_empty(ring)); - - for (i = 0; i < NR_SLOTS; i++) - assert(slot_seen[i]); - return 0; -} - -static int test_multi_send_flushing(struct io_uring *ring, int sock_tx, int sock_rx) -{ - struct io_uring_sqe *sqe; - struct io_uring_cqe *cqe; - unsigned zc_flags = IORING_RECVSEND_NOTIF_FLUSH; - int msg_flags = 0, slot_idx = 0; - int payload_size = 1; - int ret, j, i = 0; - int nr = NR_SLOTS * 30; - unsigned long long check = 0, expected = 0; - - while (i < nr) { - int nr_per_wave = 25; - - for (j = 0; j < nr_per_wave && i < nr; j++, i++) { - sqe = io_uring_get_sqe(ring); - io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, payload_size, - msg_flags, slot_idx, zc_flags); - sqe->user_data = 1; - sqe->flags |= IOSQE_CQE_SKIP_SUCCESS; - } - ret = io_uring_submit(ring); - assert(ret == j); - } - - for (i = 0; i < nr; i++) { - int seq; - - ret = io_uring_wait_cqe(ring, &cqe); - assert(!ret); - assert(!cqe->res); - assert(tag_userdata(cqe->user_data)); - - seq = cqe->flags; - check += seq * 100007UL; - io_uring_cqe_seen(ring, cqe); - - ret = recv(sock_rx, rx_buffer, payload_size, MSG_TRUNC); - assert(ret == payload_size); - } - assert(check_cq_empty(ring)); - - for (i = 0; i < nr; i++) - expected += (i + seqs[slot_idx]) * 100007UL; - assert(check == expected); - seqs[slot_idx] += nr; - return 0; -} - -static int do_one_fail_notif_flush(struct io_uring *ring, int off, int nr) -{ - struct io_uring_cqe *cqe; - struct io_uring_sqe *sqe; - int ret; - - /* single out-of-bounds slot */ - sqe = io_uring_get_sqe(ring); - io_uring_prep_notif_update(sqe, 0, off, nr); - sqe->user_data = 1; - ret = io_uring_submit(ring); - assert(ret == 1); - ret = io_uring_wait_cqe(ring, &cqe); - assert(!ret && cqe->user_data == 1); - ret = cqe->res; - io_uring_cqe_seen(ring, cqe); - return ret; -} - -static int test_update_flush_fail(struct io_uring *ring) -{ - int ret; - - /* single out-of-bounds slot */ - ret = do_one_fail_notif_flush(ring, NR_SLOTS, 1); - assert(ret == -EINVAL); - - /* out-of-bounds range */ - ret = do_one_fail_notif_flush(ring, 0, NR_SLOTS + 3); - assert(ret == -EINVAL); - ret = do_one_fail_notif_flush(ring, NR_SLOTS - 1, 2); - assert(ret == -EINVAL); - - /* overflow checks, note it's u32 internally */ - ret = do_one_fail_notif_flush(ring, ~(__u32)0, 1); - assert(ret == -EOVERFLOW); - ret = do_one_fail_notif_flush(ring, NR_SLOTS - 1, ~(__u32)0); - assert(ret == -EOVERFLOW); - return 0; -} - -static void do_one_consume(struct io_uring *ring, int sock_tx, int sock_rx, - int slot_idx) -{ - int ret; - - ret = do_one(ring, sock_tx, slot_idx); - assert(ret == 1); - - ret = recv(sock_rx, rx_buffer, 1, MSG_TRUNC); - assert(ret == 1); -} - -static int test_update_flush(struct io_uring *ring, int sock_tx, int sock_rx) -{ - struct io_uring_sqe *sqe; - struct io_uring_cqe *cqe; - int offset = 1, nr_to_flush = 3; - int ret, i, slot_idx; - - /* - * Flush will be skipped for unused slots, so attached at least 1 req - * to each active notifier / slot - */ - for (slot_idx = 0; slot_idx < NR_SLOTS; slot_idx++) - do_one_consume(ring, sock_tx, sock_rx, slot_idx); - - assert(check_cq_empty(ring)); - - /* flush first */ - sqe = io_uring_get_sqe(ring); - io_uring_prep_notif_update(sqe, 0, 0, 1); - sqe->user_data = 1; - sqe->flags |= IOSQE_CQE_SKIP_SUCCESS; - ret = io_uring_submit(ring); - assert(ret == 1); - - ret = io_uring_wait_cqe(ring, &cqe); - assert(!ret && !cqe->res && cqe->user_data == ZC_TAG); - assert(cqe->flags == seqs[0]); - seqs[0]++; - io_uring_cqe_seen(ring, cqe); - do_one_consume(ring, sock_tx, sock_rx, 0); - assert(check_cq_empty(ring)); + int ret, i; - /* flush last */ sqe = io_uring_get_sqe(ring); - io_uring_prep_notif_update(sqe, 0, NR_SLOTS - 1, 1); + io_uring_prep_send_zc(sqe, sock_tx, (void *)1UL, payload_size, + msg_flags, zc_flags); sqe->user_data = 1; - sqe->flags |= IOSQE_CQE_SKIP_SUCCESS; - ret = io_uring_submit(ring); - assert(ret == 1); - - ret = io_uring_wait_cqe(ring, &cqe); - assert(!ret && !cqe->res && cqe->user_data == ZC_TAG + NR_SLOTS - 1); - assert(cqe->flags == seqs[NR_SLOTS - 1]); - seqs[NR_SLOTS - 1]++; - io_uring_cqe_seen(ring, cqe); - assert(check_cq_empty(ring)); - /* we left the last slot without attached requests, flush should ignore it */ sqe = io_uring_get_sqe(ring); - io_uring_prep_notif_update(sqe, 0, NR_SLOTS - 1, 1); - sqe->user_data = 1; - ret = io_uring_submit(ring); - assert(ret == 1); - - ret = io_uring_wait_cqe(ring, &cqe); - assert(!ret && !cqe->res && cqe->user_data == 1); - io_uring_cqe_seen(ring, cqe); - assert(check_cq_empty(ring)); + io_uring_prep_send_zc(sqe, sock_tx, (void *)1UL, payload_size, + msg_flags, zc_flags); + sqe->user_data = 2; + io_uring_prep_send_set_addr(sqe, (const struct sockaddr *)1UL, + sizeof(struct sockaddr_in6)); - /* flush range */ - sqe = io_uring_get_sqe(ring); - io_uring_prep_notif_update(sqe, 0, offset, nr_to_flush); - sqe->user_data = 1; - sqe->flags |= IOSQE_CQE_SKIP_SUCCESS; ret = io_uring_submit(ring); - assert(ret == 1); - - for (i = 0; i < nr_to_flush; i++) { - int slot_idx; + assert(ret == 2); + for (i = 0; i < 2; i++) { ret = io_uring_wait_cqe(ring, &cqe); - assert(!ret && !cqe->res); - assert(ZC_TAG + offset <= cqe->user_data && - cqe->user_data < ZC_TAG + offset + nr_to_flush); - slot_idx = cqe->user_data - ZC_TAG; - assert(cqe->flags == seqs[slot_idx]); - seqs[slot_idx]++; + assert(!ret); + assert(cqe->user_data <= 2); + assert(cqe->res == -EFAULT); + assert(!(cqe->flags & IORING_CQE_F_MORE)); io_uring_cqe_seen(ring, cqe); } assert(check_cq_empty(ring)); - return 0; -} - -static int test_registration(int sock_tx, int sock_rx) -{ - struct io_uring_notification_slot slots[2] = { - {.tag = 1}, {.tag = 2}, - }; - void *invalid_slots = (void *)1UL; - struct io_uring ring; - int ret, i; - - ret = io_uring_queue_init(4, &ring, 0); - if (ret) { - fprintf(stderr, "queue init failed: %d\n", ret); - return 1; - } - - ret = io_uring_unregister_notifications(&ring); - if (ret != -ENXIO) { - fprintf(stderr, "unregister nothing: %d\n", ret); - return 1; - } - - ret = io_uring_register_notifications(&ring, 2, slots); - if (ret) { - fprintf(stderr, "io_uring_register_notifications failed: %d\n", ret); - return 1; - } - - ret = io_uring_register_notifications(&ring, 2, slots); - if (ret != -EBUSY) { - fprintf(stderr, "double register: %d\n", ret); - return 1; - } - - ret = io_uring_unregister_notifications(&ring); - if (ret) { - fprintf(stderr, "unregister failed: %d\n", ret); - return 1; - } - - ret = io_uring_register_notifications(&ring, 2, slots); - if (ret) { - fprintf(stderr, "second register failed: %d\n", ret); - return 1; - } - - ret = test_invalid_slot(&ring, sock_tx, sock_rx); - if (ret) { - fprintf(stderr, "test_invalid_slot() failed\n"); - return ret; - } - - for (i = 0; i < 2; i++) { - ret = do_one(&ring, sock_tx, 0); - assert(ret == 1); - - ret = recv(sock_rx, rx_buffer, 1, MSG_TRUNC); - assert(ret == 1); - } - - io_uring_queue_exit(&ring); - ret = io_uring_queue_init(4, &ring, 0); - if (ret) { - fprintf(stderr, "queue init failed: %d\n", ret); - return 1; - } - - ret = io_uring_register_notifications(&ring, 4, invalid_slots); - if (ret != -EFAULT) { - fprintf(stderr, "io_uring_register_notifications with invalid ptr: %d\n", ret); - return 1; - } - - io_uring_queue_exit(&ring); - return 0; + return T_EXIT_PASS; } static int prepare_ip(struct sockaddr_storage *addr, int *sock_client, int *sock_server, @@ -644,12 +241,11 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se size_t send_size, bool cork, bool mix_register, int buf_idx) { - const unsigned slot_idx = 0; const unsigned zc_flags = 0; struct io_uring_sqe *sqe; struct io_uring_cqe *cqe; int nr_reqs = cork ? 5 : 1; - int i, ret; + int i, ret, nr_cqes; size_t chunk_size = send_size / nr_reqs; size_t chunk_size_last = send_size - chunk_size * (nr_reqs - 1); char *buf = buffers_iov[buf_idx].iov_base; @@ -660,39 +256,35 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se memset(rx_buffer, 0, send_size); for (i = 0; i < nr_reqs; i++) { - bool cur_fixed_buf = fixed_buf; + bool real_fixed_buf = fixed_buf; size_t cur_size = chunk_size; int msg_flags = MSG_WAITALL; if (mix_register) - cur_fixed_buf = rand() & 1; + real_fixed_buf = rand() & 1; if (cork && i != nr_reqs - 1) - msg_flags = MSG_MORE; + msg_flags |= MSG_MORE; if (i == nr_reqs - 1) cur_size = chunk_size_last; sqe = io_uring_get_sqe(ring); - if (cur_fixed_buf) - io_uring_prep_sendzc_fixed(sqe, sock_client, - buf + i * chunk_size, - cur_size, msg_flags, slot_idx, - zc_flags, buf_idx); - else - io_uring_prep_sendzc(sqe, sock_client, - buf + i * chunk_size, - cur_size, msg_flags, slot_idx, - zc_flags); + io_uring_prep_send_zc(sqe, sock_client, buf + i * chunk_size, + cur_size, msg_flags, zc_flags); + sqe->user_data = i; + if (real_fixed_buf) { + sqe->ioprio |= IORING_RECVSEND_FIXED_BUF; + sqe->buf_index = buf_idx; + } if (addr) { sa_family_t fam = ((struct sockaddr_in *)addr)->sin_family; int addr_len = fam == AF_INET ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6); - io_uring_prep_sendzc_set_addr(sqe, (const struct sockaddr *)addr, - addr_len); + io_uring_prep_send_set_addr(sqe, (const struct sockaddr *)addr, + addr_len); } - sqe->user_data = i; } ret = io_uring_submit(ring); @@ -705,9 +297,7 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se if (p == -1) { fprintf(stderr, "fork() failed\n"); return 1; - } - - if (p == 0) { + } else if (p == 0) { size_t bytes_received = 0; while (bytes_received != send_size) { @@ -732,7 +322,8 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se exit(0); } - for (i = 0; i < nr_reqs; i++) { + nr_cqes = 2 * nr_reqs; + for (i = 0; i < nr_cqes; i++) { int expected = chunk_size; ret = io_uring_wait_cqe(ring, &cqe); @@ -744,11 +335,18 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se fprintf(stderr, "invalid user_data\n"); return 1; } - if (cqe->user_data == nr_reqs - 1) - expected = chunk_size_last; - if (cqe->res != expected) { - fprintf(stderr, "invalid cqe->res %d expected %d\n", - cqe->res, expected); + if (!(cqe->flags & IORING_CQE_F_NOTIF)) { + if (cqe->user_data == nr_reqs - 1) + expected = chunk_size_last; + if (cqe->res != expected) { + fprintf(stderr, "invalid cqe->res %d expected %d\n", + cqe->res, expected); + return 1; + } + } + if ((cqe->flags & IORING_CQE_F_MORE) == + (cqe->flags & IORING_CQE_F_NOTIF)) { + fprintf(stderr, "unexpected cflags %i\n", cqe->flags); return 1; } io_uring_cqe_seen(ring, cqe); @@ -845,6 +443,8 @@ static int test_async_addr(struct io_uring *ring) struct __kernel_timespec ts; int ret; + ts.tv_sec = 1; + ts.tv_nsec = 0; ret = prepare_ip(&addr, &sock_tx, &sock_rx, true, false, false, false); if (ret) { fprintf(stderr, "sock prep failed %d\n", ret); @@ -852,17 +452,15 @@ static int test_async_addr(struct io_uring *ring) } sqe = io_uring_get_sqe(ring); - ts.tv_sec = 1; - ts.tv_nsec = 0; io_uring_prep_timeout(sqe, &ts, 0, IORING_TIMEOUT_ETIME_SUCCESS); sqe->user_data = 1; sqe->flags |= IOSQE_IO_LINK; sqe = io_uring_get_sqe(ring); - io_uring_prep_sendzc(sqe, sock_tx, tx_buffer, 1, 0, 0, 0); + io_uring_prep_send_zc(sqe, sock_tx, tx_buffer, 1, 0, 0); sqe->user_data = 2; - io_uring_prep_sendzc_set_addr(sqe, (const struct sockaddr *)&addr, - sizeof(struct sockaddr_in6)); + io_uring_prep_send_set_addr(sqe, (const struct sockaddr *)&addr, + sizeof(struct sockaddr_in6)); ret = io_uring_submit(ring); assert(ret == 2); @@ -894,6 +492,14 @@ static int test_async_addr(struct io_uring *ring) ret = recv(sock_rx, rx_buffer, 1, MSG_TRUNC); assert(ret == 1); + ret = io_uring_wait_cqe(ring, &cqe); + if (ret) { + fprintf(stderr, "io_uring_wait_cqe failed %i\n", ret); + return 1; + } + assert(cqe->flags & IORING_CQE_F_NOTIF); + io_uring_cqe_seen(ring, cqe); + close(sock_tx); close(sock_rx); return 0; @@ -937,15 +543,6 @@ int main(int argc, char *argv[]) return T_EXIT_FAIL; } - ret = register_notifications(&ring); - if (ret == -EINVAL) { - printf("sendzc is not supported, skip\n"); - return T_EXIT_SKIP; - } else if (ret) { - fprintf(stderr, "register notif failed %i\n", ret); - return T_EXIT_FAIL; - } - srand((unsigned)time(NULL)); for (i = 0; i < len; i++) tx_buffer[i] = i; @@ -956,70 +553,24 @@ int main(int argc, char *argv[]) return T_EXIT_FAIL; } - ret = test_registration(sp[0], sp[1]); - if (ret) { - fprintf(stderr, "test_registration() failed\n"); - return ret; - } - - ret = test_invalid_slot(&ring, sp[0], sp[1]); - if (ret) { - fprintf(stderr, "test_invalid_slot() failed\n"); - return T_EXIT_FAIL; - } - ret = test_basic_send(&ring, sp[0], sp[1]); + if (ret == T_EXIT_SKIP) + return ret; if (ret) { fprintf(stderr, "test_basic_send() failed\n"); return T_EXIT_FAIL; } - ret = test_send_flush(&ring, sp[0], sp[1]); - if (ret) { - fprintf(stderr, "test_send_flush() failed\n"); - return T_EXIT_FAIL; - } - - ret = test_multireq_notif(&ring, sp[0], sp[1]); - if (ret) { - fprintf(stderr, "test_multireq_notif() failed\n"); - return T_EXIT_FAIL; - } - - ret = reregister_notifications(&ring); + ret = test_send_faults(&ring, sp[0], sp[1]); if (ret) { - fprintf(stderr, "reregister notifiers failed %i\n", ret); - return T_EXIT_FAIL; - } - /* retry a few tests after registering notifs */ - ret = test_invalid_slot(&ring, sp[0], sp[1]); - if (ret) { - fprintf(stderr, "test_invalid_slot() failed\n"); + fprintf(stderr, "test_send_faults() failed\n"); return T_EXIT_FAIL; } - ret = test_multireq_notif(&ring, sp[0], sp[1]); - if (ret) { - fprintf(stderr, "test_multireq_notif2() failed\n"); - return T_EXIT_FAIL; - } - - ret = test_multi_send_flushing(&ring, sp[0], sp[1]); - if (ret) { - fprintf(stderr, "test_multi_send_flushing() failed\n"); - return T_EXIT_FAIL; - } - - ret = test_update_flush_fail(&ring); - if (ret) { - fprintf(stderr, "test_update_flush_fail() failed\n"); - return T_EXIT_FAIL; - } - - ret = test_update_flush(&ring, sp[0], sp[1]); + ret = test_async_addr(&ring); if (ret) { - fprintf(stderr, "test_update_flush() failed\n"); - return T_EXIT_FAIL; + fprintf(stderr, "test_async_addr() failed\n"); + return ret; } ret = t_register_buffers(&ring, buffers_iov, ARRAY_SIZE(buffers_iov)); @@ -1036,12 +587,6 @@ int main(int argc, char *argv[]) fprintf(stderr, "test_inet_send() failed\n"); return ret; } - - ret = test_async_addr(&ring); - if (ret) { - fprintf(stderr, "test_async_addr() failed\n"); - return ret; - } out: io_uring_queue_exit(&ring); close(sp[0]); From patchwork Fri Sep 2 11:12:38 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12964046 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 8D7CAC38145 for ; Fri, 2 Sep 2022 11:14:58 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S232659AbiIBLO5 (ORCPT ); Fri, 2 Sep 2022 07:14:57 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:40224 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S232685AbiIBLOz (ORCPT ); Fri, 2 Sep 2022 07:14:55 -0400 Received: from mail-wr1-x430.google.com (mail-wr1-x430.google.com [IPv6:2a00:1450:4864:20::430]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id A94BF13E0B for ; Fri, 2 Sep 2022 04:14:53 -0700 (PDT) Received: by mail-wr1-x430.google.com with SMTP id k9so1924428wri.0 for ; Fri, 02 Sep 2022 04:14:53 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date; bh=oBqjmTxxDXC+6e/PM5LYg4CPi97y7DsIl/wxiWy6Ads=; b=aKZwzG7/wkju9WW933foJklI0aRS+5xOX1hX0Gqbk3dVhdz4z+HEyAYa+u4161N4TP wSGLOQEecqWzrIAeIHAS3z4wgMZiKOpMeeA++EeViqAhBn8eKP9Uh7Qfmn8+/UPm5snS XWhDAEzEuiKAJW8PXxo3v1R5d8nYL/F2mjPZcabnmA+sei3ZNPiwFIsfO6f4y33QJCI3 1ULouf4Nh3rpscu0kiBsoNfCf6MZi2+8DgVwEaiMktOTP6mfFLeUrV88AgrB+bKjlzIu PcTVWTDWvG7uScrAnZeMJVsnifh4YxJcIBlsgvGaBA0X++BccEHlTCWIcLL62h/sR/S8 Fu7g== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date; bh=oBqjmTxxDXC+6e/PM5LYg4CPi97y7DsIl/wxiWy6Ads=; b=GSA6gAypEzLSrtO0bVj+kBswaahHKYOtdwoftObMsQwtJ7p9BUEdAUDC3zPeku7oxV VlXIhbLaM986DByy2pGoJRJWxSsXECwNPveUSXqrAPUG/8q2k1PSsluQHg+Smj+z3bkA zeZLd7kCZznu9qzzthhGVq5Y2sQkZ1Y9yQ2h2VNY76pTJ8LNehYhnJ53u8Etbr3U5bvI 3yiXk721VaHiFicWSX8mHp2/VJjM7c4v3wtjD3mHj1BcLV0RivljzmgUbGFxqaXkN2sf V6sdj+UxSjJpLGQs89SUp3v6uwBkfsCt41mqpOU6U2MXzs1V1EvQSq6hE654SvzFWl4r xTMg== X-Gm-Message-State: ACgBeo0zVqxfgGMUON8e1VpjeIUDtst2YpVNwlCGw+6CM0B7HoW7xRDr ZFVE6oILXqkCRl2JS4aZy8haRfvS65I= X-Google-Smtp-Source: AA6agR6Uafa8zv7cuSO8W1TstWx8zNoBxPincOVa8Kj04kQnkKSZpYYxbtjPdKYbHFM0VoovgxOEaA== X-Received: by 2002:a5d:510e:0:b0:226:e949:8baf with SMTP id s14-20020a5d510e000000b00226e9498bafmr7704634wrt.204.1662117291221; Fri, 02 Sep 2022 04:14:51 -0700 (PDT) Received: from 127.0.0.1localhost (82-132-230-225.dab.02.net. [82.132.230.225]) by smtp.gmail.com with ESMTPSA id bg32-20020a05600c3ca000b003a536d5aa2esm2087379wmb.11.2022.09.02.04.14.50 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 02 Sep 2022 04:14:50 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH liburing 3/4] test: test iowq zc sends Date: Fri, 2 Sep 2022 12:12:38 +0100 Message-Id: <8562fc9c8860b0e6b92d20e4f773ce3bde7d9137.1662116617.git.asml.silence@gmail.com> X-Mailer: git-send-email 2.37.2 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org Add tests exercising io-wq paths with zc by setting IOSQE_ASYNC. Signed-off-by: Pavel Begunkov --- test/send-zerocopy.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/send-zerocopy.c b/test/send-zerocopy.c index 7b58ae7..8714b6f 100644 --- a/test/send-zerocopy.c +++ b/test/send-zerocopy.c @@ -239,7 +239,7 @@ static int prepare_ip(struct sockaddr_storage *addr, int *sock_client, int *sock static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_server, bool fixed_buf, struct sockaddr_storage *addr, size_t send_size, bool cork, bool mix_register, - int buf_idx) + int buf_idx, bool force_async) { const unsigned zc_flags = 0; struct io_uring_sqe *sqe; @@ -285,6 +285,8 @@ static int do_test_inet_send(struct io_uring *ring, int sock_client, int sock_se io_uring_prep_send_set_addr(sqe, (const struct sockaddr *)addr, addr_len); } + if (force_async) + sqe->flags |= IOSQE_ASYNC; } ret = io_uring_submit(ring); @@ -389,7 +391,7 @@ static int test_inet_send(struct io_uring *ring) return 1; } - for (i = 0; i < 128; i++) { + for (i = 0; i < 256; i++) { bool fixed_buf = i & 1; struct sockaddr_storage *addr_arg = (i & 2) ? &addr : NULL; size_t size = (i & 4) ? 137 : 4096; @@ -398,6 +400,7 @@ static int test_inet_send(struct io_uring *ring) bool aligned = i & 32; bool large_buf = i & 64; int buf_idx = aligned ? 0 : 1; + bool force_async = i & 128; if (!tcp || !large_buf) continue; @@ -418,7 +421,7 @@ static int test_inet_send(struct io_uring *ring) ret = do_test_inet_send(ring, sock_client, sock_server, fixed_buf, addr_arg, size, cork, mix_register, - buf_idx); + buf_idx, force_async); if (ret) { fprintf(stderr, "send failed fixed buf %i, conn %i, addr %i, " "cork %i\n", From patchwork Fri Sep 2 11:12:39 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Pavel Begunkov X-Patchwork-Id: 12964048 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 72147C54EE9 for ; Fri, 2 Sep 2022 11:15:00 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S232171AbiIBLO7 (ORCPT ); Fri, 2 Sep 2022 07:14:59 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:40352 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S232701AbiIBLO5 (ORCPT ); Fri, 2 Sep 2022 07:14:57 -0400 Received: from mail-wr1-x432.google.com (mail-wr1-x432.google.com [IPv6:2a00:1450:4864:20::432]) by lindbergh.monkeyblade.net (Postfix) with ESMTPS id 61ECC167F9 for ; Fri, 2 Sep 2022 04:14:54 -0700 (PDT) Received: by mail-wr1-x432.google.com with SMTP id e13so1895603wrm.1 for ; Fri, 02 Sep 2022 04:14:54 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date; bh=n/ep2tVyh0ibmO7Yma+OsUsgQqAGoiz9HFnun+tCOUg=; b=DZDHxIlHa5PL5ydhmmrK6gNM/WteZItotVG+obNQZUSE5WIQGGozSxlcqeExnYlj/o Y4CI9+Zs/sG3BPLh51YQNxPHRGQFIhoWUteFP4AuC98NxrfDpTiidyIxmcw17lBa9IGe kOUqDHEoXQcWB+/02GCsMsRlCiNXoZ7xwsBNn0sgRTtToQM1xPwQsMiMg6PDQl4XIzQo 4ruW0agUrkeZ+edMhKhdphLtJ7dIHSxTyY3SY6mvZnSdvHsdZPqTV7D/atBBLkkdng6h /exPjnxkNEa/Rfv7WNBWoM8gaRiAFh6EWqoaYfEY0YwJMyTo5Kzb62e9rvVUkea1WkQU CgGg== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20210112; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date; bh=n/ep2tVyh0ibmO7Yma+OsUsgQqAGoiz9HFnun+tCOUg=; b=aTHeLbBoSfn8Y4cT5X6Ak8YEjPYI5UDfwN8c2/dq3I7C1ZqCqHUU38txX3hRxegFIW Fgahrxf8pIT2Cu0HMhGVlTQQ8a3Ab6xu/o4Rbh8fgIC/iWs8EEAKPX5PZT7SLQa45pHx 4X6uWlU88JJpsMXnEnWw3Zlro0IahZHOnBwgnCK1iJ0xQTW1xY6yBAf3Df/CwmodcrbD 7CL13CyKw6Fgo5UZdL+THSBFgdwj/5nNjta0cyWmChehmefbHj9n/6iINpU7sjjpOcOJ q6blS7bdYXGMaUbfIG4zFM0PfizCdk+w/IsM5lFRUVJt164IWKIbW9e7Efhid7Ou6+Ns rXgQ== X-Gm-Message-State: ACgBeo38q1q4+pJ1kFI0fB9cXhCcoAp3yfRwjFeZXWB1+RF+5Z6PwiZL DHzEj9XCFhBsoctDdKUBbKfrhUyv/00= X-Google-Smtp-Source: AA6agR7/PCK/0Gv+ZMU8ikXWLbVW9g1P9Si74iK3IgsgUWW5IRULc5AgfUClExLD+SPU5YsjdyrfGw== X-Received: by 2002:a5d:5989:0:b0:221:7c34:3943 with SMTP id n9-20020a5d5989000000b002217c343943mr16189103wri.441.1662117292496; Fri, 02 Sep 2022 04:14:52 -0700 (PDT) Received: from 127.0.0.1localhost (82-132-230-225.dab.02.net. [82.132.230.225]) by smtp.gmail.com with ESMTPSA id bg32-20020a05600c3ca000b003a536d5aa2esm2087379wmb.11.2022.09.02.04.14.51 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Fri, 02 Sep 2022 04:14:52 -0700 (PDT) From: Pavel Begunkov To: io-uring@vger.kernel.org Cc: Jens Axboe , asml.silence@gmail.com Subject: [PATCH liburing 4/4] examples: adjust zc bench to the new uapi Date: Fri, 2 Sep 2022 12:12:39 +0100 Message-Id: <4d08a85d62101e0137cf2bd3587c45bd14667655.1662116617.git.asml.silence@gmail.com> X-Mailer: git-send-email 2.37.2 In-Reply-To: References: MIME-Version: 1.0 Precedence: bulk List-ID: X-Mailing-List: io-uring@vger.kernel.org Signed-off-by: Pavel Begunkov --- examples/send-zerocopy.c | 72 +++++++++++++--------------------------- 1 file changed, 23 insertions(+), 49 deletions(-) diff --git a/examples/send-zerocopy.c b/examples/send-zerocopy.c index e42aa71..4ed0f67 100644 --- a/examples/send-zerocopy.c +++ b/examples/send-zerocopy.c @@ -44,7 +44,6 @@ static bool cfg_reg_ringfd = true; static bool cfg_fixed_files = 1; static bool cfg_zc = 1; -static bool cfg_flush = 0; static int cfg_nr_reqs = 8; static bool cfg_fixed_buf = 1; @@ -146,13 +145,6 @@ static void do_tx(int domain, int type, int protocol) if (ret) error(1, ret, "io_uring: queue init"); - if (cfg_zc) { - struct io_uring_notification_slot b[1] = {{.tag = ZC_TAG}}; - - ret = io_uring_register_notifications(&ring, 1, b); - if (ret) - error(1, ret, "io_uring: tx ctx registration"); - } if (cfg_fixed_files) { ret = io_uring_register_files(&ring, &fd, 1); if (ret < 0) @@ -175,14 +167,8 @@ static void do_tx(int domain, int type, int protocol) do { struct io_uring_sqe *sqe; struct io_uring_cqe *cqe; - unsigned zc_flags = 0; unsigned buf_idx = 0; - unsigned slot_idx = 0; - unsigned msg_flags = 0; - - compl_cqes += cfg_flush ? cfg_nr_reqs : 0; - if (cfg_flush) - zc_flags |= IORING_RECVSEND_NOTIF_FLUSH; + unsigned msg_flags = MSG_WAITALL; for (i = 0; i < cfg_nr_reqs; i++) { sqe = io_uring_get_sqe(&ring); @@ -190,22 +176,22 @@ static void do_tx(int domain, int type, int protocol) if (!cfg_zc) io_uring_prep_send(sqe, fd, payload, cfg_payload_len, 0); - else if (cfg_fixed_buf) - io_uring_prep_sendzc_fixed(sqe, fd, payload, - cfg_payload_len, - msg_flags, slot_idx, - zc_flags, buf_idx); - else - io_uring_prep_sendzc(sqe, fd, payload, - cfg_payload_len, msg_flags, - slot_idx, zc_flags); - + else { + io_uring_prep_send_zc(sqe, fd, payload, + cfg_payload_len, msg_flags, 0); + if (cfg_fixed_buf) { + sqe->ioprio |= IORING_RECVSEND_FIXED_BUF; + sqe->buf_index = buf_idx; + } + } sqe->user_data = 1; if (cfg_fixed_files) { sqe->fd = 0; sqe->flags |= IOSQE_FIXED_FILE; } } + if (cfg_zc) + compl_cqes += cfg_nr_reqs; ret = io_uring_submit(&ring); if (ret != cfg_nr_reqs) @@ -214,27 +200,26 @@ static void do_tx(int domain, int type, int protocol) for (i = 0; i < cfg_nr_reqs; i++) { cqe = wait_cqe_fast(&ring); - if (cqe->user_data == ZC_TAG) { + if (cqe->flags & IORING_CQE_F_NOTIF) { + if (cqe->flags & IORING_CQE_F_MORE) + error(1, -EINVAL, "F_MORE notif"); compl_cqes--; i--; - } else if (cqe->user_data != 1) { - error(1, cqe->user_data, "invalid user_data"); - } else if (cqe->res > 0) { + } else if (cqe->res >= 0) { + if (!(cqe->flags & IORING_CQE_F_MORE) && cfg_zc) + error(1, -EINVAL, "no F_MORE"); packets++; bytes += cqe->res; } else if (cqe->res == -EAGAIN) { - /* request failed, don't flush */ - if (cfg_flush) + if (cfg_zc) compl_cqes--; - } else if (cqe->res == -ECONNREFUSED || - cqe->res == -ECONNRESET || - cqe->res == -EPIPE) { - fprintf(stderr, "Connection failure\n"); + } else if (cqe->res == -ECONNREFUSED || cqe->res == -EPIPE || + cqe->res == -ECONNRESET) { + fprintf(stderr, "Connection failure"); goto out_fail; } else { error(1, cqe->res, "send failed"); } - io_uring_cqe_seen(&ring, cqe); } } while (gettimeofday_ms() < tstop); @@ -255,12 +240,6 @@ out_fail: io_uring_cqe_seen(&ring, cqe); compl_cqes--; } - - if (cfg_zc) { - ret = io_uring_unregister_notifications(&ring); - if (ret) - error(1, ret, "io_uring: tx ctx unregistration"); - } io_uring_queue_exit(&ring); } @@ -276,7 +255,7 @@ static void do_test(int domain, int type, int protocol) static void usage(const char *filepath) { - error(1, 0, "Usage: %s [-f] [-n] [-z0] [-s] " + error(1, 0, "Usage: %s [-n] [-z] [-s] " "(-4|-6) [-t