From patchwork Mon May 25 22:08:17 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11569559 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 5AD7315E4 for ; Mon, 25 May 2020 22:09:44 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id 43FFF2071A for ; Mon, 25 May 2020 22:09:44 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 43FFF2071A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 5971524754C; Mon, 25 May 2020 15:09:13 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp4.ccs.ornl.gov (smtp4.ccs.ornl.gov [160.91.203.40]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id AB8AA2472ED for ; Mon, 25 May 2020 15:08:42 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp4.ccs.ornl.gov (Postfix) with ESMTP id 6EE111005FA3; Mon, 25 May 2020 18:08:27 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 6E09B495; Mon, 25 May 2020 18:08:27 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Mon, 25 May 2020 18:08:17 -0400 Message-Id: <1590444502-20533-41-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1590444502-20533-1-git-send-email-jsimmons@infradead.org> References: <1590444502-20533-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 40/45] lnet: socklnd: discard tx_iov. X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Mr NeilBrown tx_iov always points to tx_hdr, so we can discard tx_iov, and just use &tx_hdr. WC-bug-id: https://jira.whamcloud.com/browse/LU-13004 Lustre-commit: 89637985b9897 ("LU-13004 socklnd: discard tx_iov.") Signed-off-by: Mr NeilBrown Reviewed-on: https://review.whamcloud.com/37851 Reviewed-by: James Simmons Reviewed-by: Shaun Tancheff Reviewed-by: Oleg Drokin Signed-off-by: James Simmons --- net/lnet/klnds/socklnd/socklnd.h | 3 +-- net/lnet/klnds/socklnd/socklnd_cb.c | 33 ++++++++++++++------------------- net/lnet/klnds/socklnd/socklnd_lib.c | 32 +++++++++++++------------------- net/lnet/klnds/socklnd/socklnd_proto.c | 10 +++++----- 4 files changed, 33 insertions(+), 45 deletions(-) diff --git a/net/lnet/klnds/socklnd/socklnd.h b/net/lnet/klnds/socklnd/socklnd.h index f85eb3c..6c77b75 100644 --- a/net/lnet/klnds/socklnd/socklnd.h +++ b/net/lnet/klnds/socklnd/socklnd.h @@ -277,7 +277,6 @@ struct ksock_tx { /* transmit packet */ int tx_nob; /* # packet bytes */ int tx_resid; /* residual bytes */ int tx_niov; /* # packet iovec frags */ - struct kvec *tx_iov; /* packet iovec frags */ int tx_nkiov; /* # packet page frags */ unsigned short tx_zc_aborted; /* aborted ZC request */ unsigned short tx_zc_capable:1;/* payload is large enough for ZC */ @@ -675,7 +674,7 @@ int ksocknal_recv_hello(struct lnet_ni *ni, struct ksock_conn *conn, void ksocknal_lib_push_conn(struct ksock_conn *conn); int ksocknal_lib_get_conn_addrs(struct ksock_conn *conn); int ksocknal_lib_setup_sock(struct socket *so); -int ksocknal_lib_send_iov(struct ksock_conn *conn, struct ksock_tx *tx); +int ksocknal_lib_send_hdr(struct ksock_conn *conn, struct ksock_tx *tx); int ksocknal_lib_send_kiov(struct ksock_conn *conn, struct ksock_tx *tx); void ksocknal_lib_eager_ack(struct ksock_conn *conn); int ksocknal_lib_recv(struct ksock_conn *conn); diff --git a/net/lnet/klnds/socklnd/socklnd_cb.c b/net/lnet/klnds/socklnd/socklnd_cb.c index b40fbf1..c03f91c7 100644 --- a/net/lnet/klnds/socklnd/socklnd_cb.c +++ b/net/lnet/klnds/socklnd/socklnd_cb.c @@ -79,7 +79,6 @@ struct ksock_tx * tx->tx_lnetmsg = NULL; tx->tx_kiov = NULL; tx->tx_nkiov = 0; - tx->tx_iov = &tx->tx_hdr; tx->tx_niov = 1; tx->tx_nonblk = nonblk; @@ -109,16 +108,16 @@ struct ksock_tx * } static int -ksocknal_send_iov(struct ksock_conn *conn, struct ksock_tx *tx) +ksocknal_send_hdr(struct ksock_conn *conn, struct ksock_tx *tx) { - struct kvec *iov = tx->tx_iov; + struct kvec *iov = &tx->tx_hdr; int nob; int rc; LASSERT(tx->tx_niov > 0); - /* Never touch tx->tx_iov inside ksocknal_lib_send_iov() */ - rc = ksocknal_lib_send_iov(conn, tx); + /* Never touch tx->tx_hdr inside ksocknal_lib_send_hdr() */ + rc = ksocknal_lib_send_hdr(conn, tx); if (rc <= 0) /* sent nothing? */ return rc; @@ -128,19 +127,16 @@ struct ksock_tx * tx->tx_resid -= nob; /* "consume" iov */ - do { - LASSERT(tx->tx_niov > 0); + LASSERT(tx->tx_niov == 1); - if (nob < (int)iov->iov_len) { - iov->iov_base = (void *)((char *)iov->iov_base + nob); - iov->iov_len -= nob; - return rc; - } + if (nob < (int)iov->iov_len) { + iov->iov_base += nob; + iov->iov_len -= nob; + return rc; + } - nob -= iov->iov_len; - tx->tx_iov = ++iov; - tx->tx_niov--; - } while (nob); + LASSERT(nob == iov->iov_len); + tx->tx_niov--; return rc; } @@ -207,7 +203,7 @@ struct ksock_tx * ksocknal_data.ksnd_enomem_tx--; rc = -EAGAIN; } else if (tx->tx_niov) { - rc = ksocknal_send_iov(conn, tx); + rc = ksocknal_send_hdr(conn, tx); } else { rc = ksocknal_send_kiov(conn, tx); } @@ -694,7 +690,7 @@ struct ksock_conn * * We always expect at least 1 mapped fragment containing the * complete ksocknal message header. */ - LASSERT(lnet_iov_nob(tx->tx_niov, tx->tx_iov) + + LASSERT(lnet_iov_nob(tx->tx_niov, &tx->tx_hdr) + lnet_kiov_nob(tx->tx_nkiov, tx->tx_kiov) == (unsigned int)tx->tx_nob); LASSERT(tx->tx_niov >= 1); @@ -950,7 +946,6 @@ struct ksock_route * tx->tx_lnetmsg = lntmsg; tx->tx_niov = 1; - tx->tx_iov = &tx->tx_hdr; tx->tx_kiov = tx->tx_payload; tx->tx_nkiov = lnet_extract_kiov(payload_niov, tx->tx_kiov, payload_niov, payload_kiov, diff --git a/net/lnet/klnds/socklnd/socklnd_lib.c b/net/lnet/klnds/socklnd/socklnd_lib.c index 251c28f..2adc99c 100644 --- a/net/lnet/klnds/socklnd/socklnd_lib.c +++ b/net/lnet/klnds/socklnd/socklnd_lib.c @@ -73,11 +73,11 @@ } int -ksocknal_lib_send_iov(struct ksock_conn *conn, struct ksock_tx *tx) +ksocknal_lib_send_hdr(struct ksock_conn *conn, struct ksock_tx *tx) { struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; struct socket *sock = conn->ksnc_sock; - int nob, i; + int nob = 0; if (*ksocknal_tunables.ksnd_enable_csum && /* checksum enabled */ conn->ksnc_proto == &ksocknal_protocol_v2x && /* V2.x connection */ @@ -85,15 +85,15 @@ !tx->tx_msg.ksm_csum) /* not checksummed */ ksocknal_lib_csum_tx(tx); - for (nob = i = 0; i < tx->tx_niov; i++) - nob += tx->tx_iov[i].iov_len; + if (tx->tx_niov) + nob += tx->tx_hdr.iov_len; if (!list_empty(&conn->ksnc_tx_queue) || nob < tx->tx_resid) msg.msg_flags |= MSG_MORE; iov_iter_kvec(&msg.msg_iter, WRITE, - tx->tx_iov, tx->tx_niov, nob); + &tx->tx_hdr, tx->tx_niov, nob); return sock_sendmsg(sock, &msg); } @@ -208,28 +208,22 @@ static int lustre_csum(struct kvec *v, void *context) u32 csum; void *base; - LASSERT(tx->tx_iov[0].iov_base == &tx->tx_msg); + LASSERT(tx->tx_hdr.iov_base == &tx->tx_msg); LASSERT(tx->tx_conn); LASSERT(tx->tx_conn->ksnc_proto == &ksocknal_protocol_v2x); tx->tx_msg.ksm_csum = 0; - csum = crc32_le(~0, tx->tx_iov[0].iov_base, - tx->tx_iov[0].iov_len); + csum = crc32_le(~0, tx->tx_hdr.iov_base, + tx->tx_hdr.iov_len); - if (tx->tx_kiov) { - for (i = 0; i < tx->tx_nkiov; i++) { - base = kmap(tx->tx_kiov[i].bv_page) + - tx->tx_kiov[i].bv_offset; + for (i = 0; i < tx->tx_nkiov; i++) { + base = kmap(tx->tx_kiov[i].bv_page) + + tx->tx_kiov[i].bv_offset; - csum = crc32_le(csum, base, tx->tx_kiov[i].bv_len); + csum = crc32_le(csum, base, tx->tx_kiov[i].bv_len); - kunmap(tx->tx_kiov[i].bv_page); - } - } else { - for (i = 1; i < tx->tx_niov; i++) - csum = crc32_le(csum, tx->tx_iov[i].iov_base, - tx->tx_iov[i].iov_len); + kunmap(tx->tx_kiov[i].bv_page); } if (*ksocknal_tunables.ksnd_inject_csum_error) { diff --git a/net/lnet/klnds/socklnd/socklnd_proto.c b/net/lnet/klnds/socklnd/socklnd_proto.c index 183bb39..bc9f349 100644 --- a/net/lnet/klnds/socklnd/socklnd_proto.c +++ b/net/lnet/klnds/socklnd/socklnd_proto.c @@ -711,8 +711,8 @@ LASSERT(tx->tx_msg.ksm_type != KSOCK_MSG_NOOP); LASSERT(tx->tx_lnetmsg); - tx->tx_iov[0].iov_base = &tx->tx_lnetmsg->msg_hdr; - tx->tx_iov[0].iov_len = sizeof(struct lnet_hdr); + tx->tx_hdr.iov_base = &tx->tx_lnetmsg->msg_hdr; + tx->tx_hdr.iov_len = sizeof(struct lnet_hdr); tx->tx_nob = tx->tx_lnetmsg->msg_len + sizeof(struct lnet_hdr); tx->tx_resid = tx->tx_lnetmsg->msg_len + sizeof(struct lnet_hdr); @@ -721,19 +721,19 @@ static void ksocknal_pack_msg_v2(struct ksock_tx *tx) { - tx->tx_iov[0].iov_base = &tx->tx_msg; + tx->tx_hdr.iov_base = &tx->tx_msg; if (tx->tx_lnetmsg) { LASSERT(tx->tx_msg.ksm_type != KSOCK_MSG_NOOP); tx->tx_msg.ksm_u.lnetmsg.ksnm_hdr = tx->tx_lnetmsg->msg_hdr; - tx->tx_iov[0].iov_len = sizeof(struct ksock_msg); + tx->tx_hdr.iov_len = sizeof(struct ksock_msg); tx->tx_nob = sizeof(struct ksock_msg) + tx->tx_lnetmsg->msg_len; tx->tx_resid = sizeof(struct ksock_msg) + tx->tx_lnetmsg->msg_len; } else { LASSERT(tx->tx_msg.ksm_type == KSOCK_MSG_NOOP); - tx->tx_iov[0].iov_len = offsetof(struct ksock_msg, ksm_u.lnetmsg.ksnm_hdr); + tx->tx_hdr.iov_len = offsetof(struct ksock_msg, ksm_u.lnetmsg.ksnm_hdr); tx->tx_nob = offsetof(struct ksock_msg, ksm_u.lnetmsg.ksnm_hdr); tx->tx_resid = offsetof(struct ksock_msg, ksm_u.lnetmsg.ksnm_hdr); }