From patchwork Sat Jun 13 16:26:57 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602873 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 53038138C for ; Sat, 13 Jun 2020 16:27:25 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id 394AB2078A for ; Sat, 13 Jun 2020 16:27:25 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 394AB2078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 074E421C986; Sat, 13 Jun 2020 09:27:24 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 5A16521C90F for ; Sat, 13 Jun 2020 09:27:22 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id 80EBA240; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 785042A4; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:26:57 -0400 Message-Id: <1592065636-28333-2-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 01/20] lnet: fix kmalloc size in config.c X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: NeilBrown sizeof(net->net_cpts) should be sizeof(*net->net_cpts). For consistency, also change a sizeof(ni->ni_cpts[0]) to sizeof(*ni->ni_cpts). and when allocating 'array' use sizeof(*array), not sizeof(*net->net_cpts); Fixes: 47db2bfba415 ("lnet: add list of cpts to lnet_net.") Reviewed-by: James Simmons Signed-off-by: NeilBrown --- net/lnet/lnet/config.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/net/lnet/lnet/config.c b/net/lnet/lnet/config.c index 3fc0298..9f36c82 100644 --- a/net/lnet/lnet/config.c +++ b/net/lnet/lnet/config.c @@ -171,7 +171,7 @@ struct lnet_text_buf { /* tmp struct for parsing routes */ } if (!net->net_cpts) { - net->net_cpts = kmalloc_array(ncpts, sizeof(net->net_cpts), + net->net_cpts = kmalloc_array(ncpts, sizeof(*net->net_cpts), GFP_KERNEL); if (!net->net_cpts) return -ENOMEM; @@ -197,7 +197,7 @@ struct lnet_text_buf { /* tmp struct for parsing routes */ u32 *array = NULL, *loc; u32 total_entries = j + net->net_ncpts; - array = kmalloc_array(total_entries, sizeof(*net->net_cpts), + array = kmalloc_array(total_entries, sizeof(*array), GFP_KERNEL); if (!array) { rc = -ENOMEM; @@ -545,7 +545,7 @@ struct lnet_ni * } else { size_t array_size = ncpts * sizeof(ni->ni_cpts[0]); - ni->ni_cpts = kmalloc_array(ncpts, sizeof(ni->ni_cpts[0]), + ni->ni_cpts = kmalloc_array(ncpts, sizeof(*ni->ni_cpts), GFP_KERNEL); if (!ni->ni_cpts) goto failed; From patchwork Sat Jun 13 16:26:58 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602875 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id D7A6E1392 for ; Sat, 13 Jun 2020 16:27:25 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id BFC0E2078A for ; Sat, 13 Jun 2020 16:27:25 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org BFC0E2078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 4C9CA21C9CD; Sat, 13 Jun 2020 09:27:24 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 386E621C90F for ; Sat, 13 Jun 2020 09:27:23 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id 84B6524F; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 7AA4D2A8; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:26:58 -0400 Message-Id: <1592065636-28333-3-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 02/20] lnet: test against LNET_STATE_RUNNING rather than LNET_STATE_SHUTDOWN X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: NeilBrown The difference between LNET_STATE_STOPPING and LNET_STATE_SHUTDOWN is of little interest. As soon as the state isn't LNET_STATE_RUNNING, various API calls should refuse to act. So change tests for SHUTDOWN to instead compare against RUNNING. Reviewed-by: James Simmons Signed-off-by: NeilBrown --- net/lnet/lnet/api-ni.c | 2 +- net/lnet/lnet/peer.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/net/lnet/lnet/api-ni.c b/net/lnet/lnet/api-ni.c index 62b4fa7..7c91800 100644 --- a/net/lnet/lnet/api-ni.c +++ b/net/lnet/lnet/api-ni.c @@ -309,7 +309,7 @@ static int lnet_discover(struct lnet_process_id id, u32 force, * changing the value dynamically, we want to set it after * updating the peers */ - if (the_lnet.ln_state == LNET_STATE_SHUTDOWN) { + if (the_lnet.ln_state != LNET_STATE_RUNNING) { *discovery_off = value; mutex_unlock(&the_lnet.ln_api_mutex); return 0; diff --git a/net/lnet/lnet/peer.c b/net/lnet/lnet/peer.c index 5869a20..f745394 100644 --- a/net/lnet/lnet/peer.c +++ b/net/lnet/lnet/peer.c @@ -810,7 +810,7 @@ struct lnet_peer_ni * int rc; rc = -ESHUTDOWN; - if (the_lnet.ln_state == LNET_STATE_SHUTDOWN) + if (the_lnet.ln_state != LNET_STATE_RUNNING) goto done; lncpt = cfs_percpt_number(the_lnet.ln_peer_tables); @@ -1805,7 +1805,7 @@ struct lnet_peer_ni * lnet_net_lock(cpt); /* Lock has been dropped, check again for shutdown. */ - if (the_lnet.ln_state == LNET_STATE_SHUTDOWN) { + if (the_lnet.ln_state != LNET_STATE_RUNNING) { if (!IS_ERR(lpni)) lnet_peer_ni_decref_locked(lpni); lpni = ERR_PTR(-ESHUTDOWN); @@ -2816,7 +2816,7 @@ static int lnet_peer_data_present(struct lnet_peer *lp) * LNET_LOCK_EX mode is used. */ mutex_lock(&the_lnet.ln_api_mutex); - if (the_lnet.ln_state == LNET_STATE_SHUTDOWN) { + if (the_lnet.ln_state != LNET_STATE_RUNNING) { rc = -ESHUTDOWN; goto out; } From patchwork Sat Jun 13 16:26:59 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602889 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id E8880138C for ; Sat, 13 Jun 2020 16:27:54 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id D0D022078A for ; Sat, 13 Jun 2020 16:27:54 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org D0D022078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 0103F21F5C4; Sat, 13 Jun 2020 09:27:36 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 7B75F21C94C for ; Sat, 13 Jun 2020 09:27:23 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id 87110253; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 7C7592A9; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:26:59 -0400 Message-Id: <1592065636-28333-4-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 03/20] lnet: use lnet_md_free in lnet_res_container_cleanup() X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: NeilBrown When we added lnet_md_free(), we should have called it from lnet_res_container_cleanup() - but that was missed. Fix it now. Reviewed-by: James Simmons Signed-off-by: NeilBrown --- net/lnet/lnet/api-ni.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/net/lnet/lnet/api-ni.c b/net/lnet/lnet/api-ni.c index 7c91800..d6694cb 100644 --- a/net/lnet/lnet/api-ni.c +++ b/net/lnet/lnet/api-ni.c @@ -927,12 +927,10 @@ static void lnet_assert_wire_constants(void) struct list_head *e = rec->rec_active.next; list_del_init(e); - if (rec->rec_type == LNET_COOKIE_TYPE_MD) { - kfree(list_entry(e, struct lnet_libmd, md_list)); - - } else { /* NB: Active MEs should be attached on portals */ + if (rec->rec_type == LNET_COOKIE_TYPE_MD) + lnet_md_free(list_entry(e, struct lnet_libmd, md_list)); + else /* NB: Active MEs should be attached on portals */ LBUG(); - } count++; } From patchwork Sat Jun 13 16:27:00 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602893 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 73D321392 for ; Sat, 13 Jun 2020 16:28:00 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id 5BCF12078A for ; Sat, 13 Jun 2020 16:28:00 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 5BCF12078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id C67AC21F811; Sat, 13 Jun 2020 09:27:39 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id BDB0F21C986 for ; Sat, 13 Jun 2020 09:27:23 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id 88873257; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 7FF1F2AB; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:00 -0400 Message-Id: <1592065636-28333-5-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 04/20] lustre: obdclass: discard process_quota_config X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: NeilBrown This variable is never set, so it can be discarded since it is server side only. Should the server side quota layer need configuration, it should use class_modify_config(). Signed-off-by: NeilBrown --- fs/lustre/obdclass/obd_config.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/fs/lustre/obdclass/obd_config.c b/fs/lustre/obdclass/obd_config.c index b8973db..5495609 100644 --- a/fs/lustre/obdclass/obd_config.c +++ b/fs/lustre/obdclass/obd_config.c @@ -764,11 +764,6 @@ void class_del_profiles(void) } EXPORT_SYMBOL(class_del_profiles); -/* We can't call lquota_process_config directly because - * it lives in a module that must be loaded after this one. - */ -static int (*quota_process_config)(struct lustre_cfg *lcfg); - static int process_param2_config(struct lustre_cfg *lcfg) { char *param = lustre_cfg_string(lcfg, 1); @@ -949,11 +944,6 @@ int class_process_config(struct lustre_cfg *lcfg) err = 0; goto out; - } else if ((class_match_param(lustre_cfg_string(lcfg, 1), - PARAM_QUOTA, &tmp) == 0) && - quota_process_config) { - err = (*quota_process_config)(lcfg); - goto out; } break; From patchwork Sat Jun 13 16:27:01 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602891 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 48158138C for ; Sat, 13 Jun 2020 16:27:56 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id 3069F2078A for ; Sat, 13 Jun 2020 16:27:56 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 3069F2078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 943A321F65D; Sat, 13 Jun 2020 09:27:37 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id D44F121E036 for ; Sat, 13 Jun 2020 09:27:24 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id 96903276; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 82F3E2AC; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:01 -0400 Message-Id: <1592065636-28333-6-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 05/20] lnet: socklnd: remove comments about "darwin". X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: NeilBrown As lustre only runs on Linux now, comments about Darwin are not relevant. Signed-off-by: NeilBrown --- net/lnet/klnds/socklnd/socklnd_cb.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/net/lnet/klnds/socklnd/socklnd_cb.c b/net/lnet/klnds/socklnd/socklnd_cb.c index 2759455..623478c 100644 --- a/net/lnet/klnds/socklnd/socklnd_cb.c +++ b/net/lnet/klnds/socklnd/socklnd_cb.c @@ -701,10 +701,6 @@ struct ksock_conn * KSOCK_MSG_NOOP, tx->tx_nob, tx->tx_niov, tx->tx_nkiov); - /* - * FIXME: SOCK_WMEM_QUEUED and SOCK_ERROR could block in __DARWIN8__ - * but they're used inside spinlocks a lot. - */ bufnob = conn->ksnc_sock->sk->sk_wmem_queued; spin_lock_bh(&sched->kss_lock); @@ -2203,10 +2199,6 @@ void ksocknal_write_callback(struct ksock_conn *conn) /* Don't need the {get,put}connsock dance to deref ksnc_sock */ LASSERT(!conn->ksnc_closing); - /* - * SOCK_ERROR will reset error code of socket in - * some platform (like Darwin8.x) - */ error = conn->ksnc_sock->sk->sk_err; if (error) { ksocknal_conn_addref(conn); From patchwork Sat Jun 13 16:27:02 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602879 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id F238E1392 for ; Sat, 13 Jun 2020 16:27:34 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id DA5822078A for ; Sat, 13 Jun 2020 16:27:34 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org DA5822078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 08E3721C9E6; Sat, 13 Jun 2020 09:27:28 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 0C8D721C9CD for ; Sat, 13 Jun 2020 09:27:24 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id 94490272; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 85F222AE; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:02 -0400 Message-Id: <1592065636-28333-7-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 06/20] lustre: uapi: change LUSTRE_*_FL defines to enum X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Di Wang , Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Di Wang The preferred style in the case of LUSTRE_*_FL is to use enum. Fixes: d344c54f4a90c ("lustre: llite: handle ORPHAN/DEAD directories") WC-bug-id: https://jira.whamcloud.com/browse/LU-7579 Lustre-commit: 098fb363c39 ("LU-7579 osd: move ORPHAN/DEAD flag to OSD") Signed-off-by: Di Wang Reviewed-on: http://review.whamcloud.com/18024 Reviewed-by: John L. Hammond Reviewed-by: Andreas Dilger Reviewed-by: Oleg Drokin Signed-off-by: James Simmons --- include/uapi/linux/lustre/lustre_idl.h | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/include/uapi/linux/lustre/lustre_idl.h b/include/uapi/linux/lustre/lustre_idl.h index bb3f5f1..800154d 100644 --- a/include/uapi/linux/lustre/lustre_idl.h +++ b/include/uapi/linux/lustre/lustre_idl.h @@ -1537,19 +1537,23 @@ enum { #define MDS_STATUS_CONN 1 #define MDS_STATUS_LOV 2 -/* these should be identical to their EXT4_*_FL counterparts, they are - * redefined here only to avoid dragging in fs/ext4/ext4.h - */ -#define LUSTRE_SYNC_FL 0x00000008 /* Synchronous updates */ -#define LUSTRE_IMMUTABLE_FL 0x00000010 /* Immutable file */ -#define LUSTRE_APPEND_FL 0x00000020 /* writes to file may only append */ -#define LUSTRE_NODUMP_FL 0x00000040 /* do not dump file */ -#define LUSTRE_NOATIME_FL 0x00000080 /* do not update atime */ -#define LUSTRE_INDEX_FL 0x00001000 /* hash-indexed directory */ -#define LUSTRE_DIRSYNC_FL 0x00010000 /* dirsync behaviour (dir only) */ -#define LUSTRE_TOPDIR_FL 0x00020000 /* Top of directory hierarchies*/ -#define LUSTRE_INLINE_DATA_FL 0x10000000 /* Inode has inline data. */ -#define LUSTRE_PROJINHERIT_FL 0x20000000 /* Create with parents projid */ +enum { + /* these should be identical to their EXT4_*_FL counterparts, they are + * redefined here only to avoid dragging in fs/ext4/ext4.h + */ + LUSTRE_SYNC_FL = 0x00000008, /* Synchronous updates */ + LUSTRE_IMMUTABLE_FL = 0x00000010, /* Immutable file */ + LUSTRE_APPEND_FL = 0x00000020, /* writes to file may only + * append + */ + LUSTRE_NODUMP_FL = 0x00000040, /* do not dump file */ + LUSTRE_NOATIME_FL = 0x00000080, /* do not update atime */ + LUSTRE_INDEX_FL = 0x00001000, /* hash-indexed directory */ + LUSTRE_DIRSYNC_FL = 0x00010000, /* dirsync behaviour (dir only) */ + LUSTRE_TOPDIR_FL = 0x00020000, /* Top of directory hierarchies*/ + LUSTRE_INLINE_DATA_FL = 0x10000000, /* Inode has inline data. */ + LUSTRE_PROJINHERIT_FL = 0x20000000, /* Create with parents projid */ +}; /* 64 possible states */ enum md_transient_state { From patchwork Sat Jun 13 16:27:03 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602887 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 56C7D138C for ; Sat, 13 Jun 2020 16:27:50 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id 3E1842078A for ; Sat, 13 Jun 2020 16:27:50 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 3E1842078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id ADC2621E09B; Sat, 13 Jun 2020 09:27:34 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 910EC21CA87 for ; Sat, 13 Jun 2020 09:27:24 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id 9461C275; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 88D7E2AF; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:03 -0400 Message-Id: <1592065636-28333-8-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 07/20] lustre: SEL: Add flag & setstripe support X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Patrick Farrell , Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Patrick Farrell The self-extending layouts feature adds a new layout flag and also uses the stripe size field differently. This patch implements this basic functionality, to be used in subsequent patches. Cray-bug-id: LUS-2528 WC-bug-id: https://jira.whamcloud.com/browse/LU-10070 Lustre-commit: fed241911f61b ("LU-10070 lod: SEL: Add flag & setstripe support") Signed-off-by: Patrick Farrell Reviewed-on: https://review.whamcloud.com/33782 Reviewed-by: Alexey Lyashkov Reviewed-by: Patrick Farrell Reviewed-by: Oleg Drokin Signed-off-by: James Simmons --- fs/lustre/ptlrpc/wiretest.c | 1 + include/uapi/linux/lustre/lustre_user.h | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/fs/lustre/ptlrpc/wiretest.c b/fs/lustre/ptlrpc/wiretest.c index dc1d0af..719d73b 100644 --- a/fs/lustre/ptlrpc/wiretest.c +++ b/fs/lustre/ptlrpc/wiretest.c @@ -1659,6 +1659,7 @@ void lustre_assert_wire_constants(void) BUILD_BUG_ON(LCME_FL_OFFLINE != 0x00000008); BUILD_BUG_ON(LCME_FL_INIT != 0x00000010); BUILD_BUG_ON(LCME_FL_NOSYNC != 0x00000020); + BUILD_BUG_ON(LCME_FL_EXTENSION != 0x00000040); BUILD_BUG_ON(LCME_FL_NEG != 0x80000000); /* Checks for struct lov_comp_md_v1 */ diff --git a/include/uapi/linux/lustre/lustre_user.h b/include/uapi/linux/lustre/lustre_user.h index b255f14..004d373 100644 --- a/include/uapi/linux/lustre/lustre_user.h +++ b/include/uapi/linux/lustre/lustre_user.h @@ -548,17 +548,17 @@ enum lov_comp_md_entry_flags { LCME_FL_OFFLINE = 0x00000008, /* Not used */ LCME_FL_INIT = 0x00000010, /* instantiated */ LCME_FL_NOSYNC = 0x00000020, /* FLR: no sync for the mirror */ + LCME_FL_EXTENSION = 0x00000040, /* extension comp, never init */ LCME_FL_NEG = 0x80000000, /* used to indicate a negative * flag, won't be stored on disk */ }; #define LCME_KNOWN_FLAGS (LCME_FL_NEG | LCME_FL_INIT | LCME_FL_STALE | \ - LCME_FL_PREF_RW | LCME_FL_NOSYNC) -/* The flags can be set by users at mirror creation time. */ -#define LCME_USER_FLAGS (LCME_FL_PREF_RW) + LCME_FL_PREF_RW | LCME_FL_NOSYNC | \ + LCME_FL_EXTENSION) -/* The flags are for mirrors */ +/* The mirror flags sent by client */ #define LCME_MIRROR_FLAGS (LCME_FL_NOSYNC) /* the highest bit in obdo::o_layout_version is used to mark if the file is From patchwork Sat Jun 13 16:27:04 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602883 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 2F9001392 for ; Sat, 13 Jun 2020 16:27:44 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id 17D792078A for ; Sat, 13 Jun 2020 16:27:44 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 17D792078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 9C55921C9F9; Sat, 13 Jun 2020 09:27:31 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 4E28721CA87 for ; Sat, 13 Jun 2020 09:27:24 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id 94563274; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 8BAAF2D1; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:04 -0400 Message-Id: <1592065636-28333-9-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 08/20] lustre: lmv: check stripe FID sanity X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lai Siyao , Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Lai Siyao Striped directory layout may be broken, if some stripe FID is insane, return -ENODEV. WC-bug-id: https://jira.whamcloud.com/browse/LU-13437 Lustre-commit: 698a496aac51e ("LU-13437 lmv: check stripe FID sanity") Signed-off-by: Lai Siyao Reviewed-on: https://review.whamcloud.com/38560 Reviewed-by: Andreas Dilger Reviewed-by: Hongchao Zhang Reviewed-by: Oleg Drokin Signed-off-by: James Simmons --- fs/lustre/lmv/lmv_obd.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fs/lustre/lmv/lmv_obd.c b/fs/lustre/lmv/lmv_obd.c index 4583cea..203a17b 100644 --- a/fs/lustre/lmv/lmv_obd.c +++ b/fs/lustre/lmv/lmv_obd.c @@ -1539,6 +1539,10 @@ static struct lu_tgt_desc *lmv_locate_tgt_rr(struct lmv_obd *lmv, u32 *mdt) return ERR_CAST(oinfo); } + /* check stripe FID is sane */ + if (!fid_is_sane(&oinfo->lmo_fid)) + return ERR_PTR(-ENODEV); + *fid = oinfo->lmo_fid; *mds = oinfo->lmo_mds; tgt = lmv_tgt(lmv, oinfo->lmo_mds); From patchwork Sat Jun 13 16:27:05 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602897 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 444CE138C for ; Sat, 13 Jun 2020 16:28:06 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id 2C5582078A for ; Sat, 13 Jun 2020 16:28:06 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 2C5582078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id AFD9721F9F2; Sat, 13 Jun 2020 09:27:42 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 2260521E04D for ; Sat, 13 Jun 2020 09:27:25 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id 9701327A; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 8F04A2A4; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:05 -0400 Message-Id: <1592065636-28333-10-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 09/20] lustre: ptlrpc: Clear bd_registered in ptlrpc_unregister_bulk X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Chris Horn The patch for LU-12816 https://review.whamcloud.com/36309 has us clearing the bd_registered flag in ptl_send_rpc(). This flag is set in ptlrpc_register_bulk(), so it makes sense for us to clear it in ptlrpc_unregister_bulk(). When we're cleaning up in ptl_send_rpc() we can be sure the flag is cleared with the call to ptlrpc_unregister_bulk(). This commit also adds a test case for the LU-12816 bug. Fixes: 1010595a9bb6 ("lustre: ptlrpc: ptlrpc_register_bulk LBUG on ENOMEM") WC-bug-id: https://jira.whamcloud.com/browse/LU-13509 Lustre-commit: 15057a17ca1e2 ("LU-13509 ptlrpc: Clear bd_registered in ptlrpc_unregister_bulk") Signed-off-by: Chris Horn Reviewed-on: https://review.whamcloud.com/38457 Reviewed-by: Shaun Tancheff Reviewed-by: Wang Shilong Reviewed-by: Oleg Drokin Signed-off-by: James Simmons --- fs/lustre/include/obd_support.h | 1 + fs/lustre/ptlrpc/niobuf.c | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/fs/lustre/include/obd_support.h b/fs/lustre/include/obd_support.h index b706a20..d1d21e0 100644 --- a/fs/lustre/include/obd_support.h +++ b/fs/lustre/include/obd_support.h @@ -362,6 +362,7 @@ #define OBD_FAIL_PTLRPC_LONG_REQ_UNLINK 0x51b #define OBD_FAIL_PTLRPC_LONG_BOTH_UNLINK 0x51c #define OBD_FAIL_PTLRPC_BULK_ATTACH 0x521 +#define OBD_FAIL_PTLRPC_BULK_REPLY_ATTACH 0x522 #define OBD_FAIL_PTLRPC_ROUND_XID 0x530 #define OBD_FAIL_PTLRPC_CONNECT_RACE 0x531 diff --git a/fs/lustre/ptlrpc/niobuf.c b/fs/lustre/ptlrpc/niobuf.c index d62629a..331a0c8 100644 --- a/fs/lustre/ptlrpc/niobuf.c +++ b/fs/lustre/ptlrpc/niobuf.c @@ -250,6 +250,9 @@ int ptlrpc_unregister_bulk(struct ptlrpc_request *req, int async) LASSERT(!in_interrupt()); /* might sleep */ + if (desc) + desc->bd_registered = 0; + /* Let's setup deadline for reply unlink. */ if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_BULK_UNLINK) && async && req->rq_bulk_deadline == 0 && cfs_fail_val == 0) @@ -614,9 +617,16 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply) request->rq_repmsg = NULL; } - reply_me = LNetMEAttach(request->rq_reply_portal, - connection->c_peer, request->rq_xid, 0, - LNET_UNLINK, LNET_INS_AFTER); + if (request->rq_bulk && + OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_BULK_REPLY_ATTACH)) { + reply_me = ERR_PTR(-ENOMEM); + } else { + reply_me = LNetMEAttach(request->rq_reply_portal, + connection->c_peer, + request->rq_xid, 0, + LNET_UNLINK, LNET_INS_AFTER); + } + if (IS_ERR(reply_me)) { rc = PTR_ERR(reply_me); CERROR("LNetMEAttach failed: %d\n", rc); @@ -724,8 +734,6 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply) * the chance to have long unlink to sluggish net is smaller here. */ ptlrpc_unregister_bulk(request, 0); - if (request->rq_bulk) - request->rq_bulk->bd_registered = 0; out: if (rc == -ENOMEM) { /* From patchwork Sat Jun 13 16:27:06 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602901 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id F3CD9138C for ; Sat, 13 Jun 2020 16:28:11 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id DBD972078A for ; Sat, 13 Jun 2020 16:28:11 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org DBD972078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id B8C6D21EBAC; Sat, 13 Jun 2020 09:27:45 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 63EA421E04D for ; Sat, 13 Jun 2020 09:27:25 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id 9AC48498; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 93E852A3; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:06 -0400 Message-Id: <1592065636-28333-11-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 10/20] lustre: dne: directory restripe and auto split X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lai Siyao , Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Lai Siyao A specific restriper thread is created for each MDT, it does three tasks in a loop: 1. If there is directory whose total sub-files exceeds threshold (50000 by default, can be changed "lctl set_param mdt.*.dir_split_count=N"), split this directory by adding new stripes (4 stripes by default, which can be adjusted by "lctl set_param mdt.*.dir_split_delta=N"). 2. If a directory stripe LMV is marked 'MIGRATION', migrate sub file from current offset, and update offset to next file. 3. If a directory master LMV is marked 'RESTRIPING', check whether all stripe LMV 'MIGRATION' flag is cleared, if so, clear 'RESTRIPING' flag and update directory LMV. In last patch, the first part of manual directory stripe is implemented, and in this patch, sub file migrations and dir layout update is done. Directory auto-split is done in similar way, except that the first step is done by this thread too. Directory auto-split can be enabled/disabled by "lctl set_param mdt.*.enable_dir_auto_split=[0|1]", it's turned on by default. Auto split is triggered at the end of getattr(): since now the attr contains dirent count, check whether it exceeds threshold, if so, add this directory into mdr_auto_split list and wake up the dir restriper thread. Restripe migration is also triggered in getattr(): if the object is directory stripe, and LMV 'MIGRATION' flag set, add this object into mdr_restripe_migrate list and wake up the dir restriper thread. Directory layout update is similar: if current directory is striped, and LNV 'RESTRIPING' flag is set, add this directory into mdr_restripe_update list and wake up restriper thread. By default restripe migrate dirent only, and leave inode unchanged, it can be adjusted by "lctl set_param mdt.*.dir_restripe_nsonly=[0|1]". Currently DoM file inode migration is not supported, migrate dirent only for such files to avoid leaving dir migration/restripe unfinished. WC-bug-id: https://jira.whamcloud.com/browse/LU-11025 Lustre-commit: a336d7c7c1cd6 ("LU-11025 dne: directory restripe and auto split") Signed-off-by: Lai Siyao Reviewed-on: https://review.whamcloud.com/37284 Reviewed-by: Andreas Dilger Reviewed-by: Hongchao Zhang Reviewed-by: Oleg Drokin Signed-off-by: James Simmons --- fs/lustre/include/lustre_lmv.h | 20 +++++++++++++++----- fs/lustre/llite/dir.c | 25 ++----------------------- include/uapi/linux/lustre/lustre_idl.h | 18 ++++++++++++++++++ 3 files changed, 35 insertions(+), 28 deletions(-) diff --git a/fs/lustre/include/lustre_lmv.h b/fs/lustre/include/lustre_lmv.h index 0175816..afa4d60 100644 --- a/fs/lustre/include/lustre_lmv.h +++ b/fs/lustre/include/lustre_lmv.h @@ -452,32 +452,42 @@ static inline bool lmv_is_sane2(const struct lmv_mds_md_v1 *lmv) static inline bool lmv_is_splitting(const struct lmv_mds_md_v1 *lmv) { - LASSERT(lmv_is_sane2(lmv)); + if (!lmv_is_sane2(lmv)) + return false; + return lmv_hash_is_splitting(cpu_to_le32(lmv->lmv_hash_type)); } static inline bool lmv_is_merging(const struct lmv_mds_md_v1 *lmv) { - LASSERT(lmv_is_sane2(lmv)); + if (!lmv_is_sane2(lmv)) + return false; + return lmv_hash_is_merging(cpu_to_le32(lmv->lmv_hash_type)); } static inline bool lmv_is_migrating(const struct lmv_mds_md_v1 *lmv) { - LASSERT(lmv_is_sane(lmv)); + if (!lmv_is_sane(lmv)) + return false; + return lmv_hash_is_migrating(cpu_to_le32(lmv->lmv_hash_type)); } static inline bool lmv_is_restriping(const struct lmv_mds_md_v1 *lmv) { - LASSERT(lmv_is_sane2(lmv)); + if (!lmv_is_sane2(lmv)) + return false; + return lmv_hash_is_splitting(cpu_to_le32(lmv->lmv_hash_type)) || lmv_hash_is_merging(cpu_to_le32(lmv->lmv_hash_type)); } static inline bool lmv_is_layout_changing(const struct lmv_mds_md_v1 *lmv) { - LASSERT(lmv_is_sane2(lmv)); + if (!lmv_is_sane2(lmv)) + return false; + return lmv_hash_is_splitting(cpu_to_le32(lmv->lmv_hash_type)) || lmv_hash_is_merging(cpu_to_le32(lmv->lmv_hash_type)) || lmv_hash_is_migrating(cpu_to_le32(lmv->lmv_hash_type)); diff --git a/fs/lustre/llite/dir.c b/fs/lustre/llite/dir.c index 7fd52fe..0ffe134 100644 --- a/fs/lustre/llite/dir.c +++ b/fs/lustre/llite/dir.c @@ -48,6 +48,7 @@ #include #include +#include #include #include #include @@ -174,28 +175,6 @@ void ll_release_page(struct inode *inode, struct page *page, bool remove) put_page(page); } -/** - * return IF_* type for given lu_dirent entry. - * IF_* flag shld be converted to particular OS file type in - * platform llite module. - */ -static u16 ll_dirent_type_get(struct lu_dirent *ent) -{ - u16 type = 0; - struct luda_type *lt; - int len = 0; - - if (le32_to_cpu(ent->lde_attrs) & LUDA_TYPE) { - const unsigned int align = sizeof(struct luda_type) - 1; - - len = le16_to_cpu(ent->lde_namelen); - len = (len + align) & ~align; - lt = (void *)ent->lde_name + len; - type = IFTODT(le16_to_cpu(lt->lt_type)); - } - return type; -} - int ll_dir_read(struct inode *inode, u64 *ppos, struct md_op_data *op_data, struct dir_context *ctx) { @@ -246,7 +225,7 @@ int ll_dir_read(struct inode *inode, u64 *ppos, struct md_op_data *op_data, lhash = hash; fid_le_to_cpu(&fid, &ent->lde_fid); ino = cl_fid_build_ino(&fid, is_api32); - type = ll_dirent_type_get(ent); + type = IFTODT(lu_dirent_type_get(ent)); ctx->pos = lhash; /* For 'll_nfs_get_name_filldir()', it will try * to access the 'ent' through its 'lde_name', diff --git a/include/uapi/linux/lustre/lustre_idl.h b/include/uapi/linux/lustre/lustre_idl.h index 800154d..d0b43c8 100644 --- a/include/uapi/linux/lustre/lustre_idl.h +++ b/include/uapi/linux/lustre/lustre_idl.h @@ -492,6 +492,24 @@ static inline __kernel_size_t lu_dirent_calc_size(size_t namelen, __u16 attr) return (size + 7) & ~7; } +static inline __u16 lu_dirent_type_get(struct lu_dirent *ent) +{ + __u16 type = 0; + struct luda_type *lt; + int len = 0; + + if (__le32_to_cpu(ent->lde_attrs) & LUDA_TYPE) { + const unsigned int align = sizeof(struct luda_type) - 1; + + len = __le16_to_cpu(ent->lde_namelen); + len = (len + align) & ~align; + lt = (void *)ent->lde_name + len; + type = __le16_to_cpu(lt->lt_type); + } + + return type; +} + #define MDS_DIR_END_OFF 0xfffffffffffffffeULL /** From patchwork Sat Jun 13 16:27:07 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602877 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id F1E5D1392 for ; Sat, 13 Jun 2020 16:27:33 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id DA1EA2078A for ; Sat, 13 Jun 2020 16:27:33 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org DA1EA2078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 9C36A21EBC4; Sat, 13 Jun 2020 09:27:27 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id BB05021E076 for ; Sat, 13 Jun 2020 09:27:25 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id 9D0B8617; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 96DE62A8; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:07 -0400 Message-Id: <1592065636-28333-12-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 11/20] lustre: sec: documentation for client-side encryption X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Sebastien Buisson Add several documents about client-side encryption under Documentation/client_side_encryption: - threat_model.txt is the description of the threat model for Lustre client-side encryption; - key_hierarchy.txt is the description of the key hierarchy for Lustre client-side encryption; - modes_usage.txt is the description of the encryption modes and usage for Lustre client-side encryption; - access_semantics.txt is the description of the access semantics for Lustre client-side encryption. As we rely on kernel's fscrypt library for this feature, fscrypt's concepts are largely valid. These documents are inspired by fscrypt documentation in the Linux kernel tree, see Documentation/filesystems/fscrypt.rst WC-bug-id: https://jira.whamcloud.com/browse/LU-12275 Lustre-commit: f2bf564e691ea ("LU-12275 sec: documentation for client-side encryption") Signed-off-by: Sebastien Buisson Reviewed-on: https://review.whamcloud.com/38759 Reviewed-by: John L. Hammond Reviewed-by: Andreas Dilger Signed-off-by: James Simmons --- .../client_side_encryption/access_semantics.txt | 128 +++++++++++++++++ .../client_side_encryption/key_hierarchy.txt | 112 +++++++++++++++ .../lustre/client_side_encryption/modes_usage.txt | 97 +++++++++++++ .../lustre/client_side_encryption/threat_model.txt | 159 +++++++++++++++++++++ 4 files changed, 496 insertions(+) create mode 100644 Documentation/lustre/client_side_encryption/access_semantics.txt create mode 100644 Documentation/lustre/client_side_encryption/key_hierarchy.txt create mode 100644 Documentation/lustre/client_side_encryption/modes_usage.txt create mode 100644 Documentation/lustre/client_side_encryption/threat_model.txt diff --git a/Documentation/lustre/client_side_encryption/access_semantics.txt b/Documentation/lustre/client_side_encryption/access_semantics.txt new file mode 100644 index 0000000..fe2c28d --- /dev/null +++ b/Documentation/lustre/client_side_encryption/access_semantics.txt @@ -0,0 +1,128 @@ +=============================================== +Lustre client-level encryption access semantics +=============================================== + +Lustre client-level encryption relies on kernel's fscrypt, and more +precisely on v2 encryption policies. +fscrypt is a library which filesystems can hook into to support +transparent encryption of files and directories. + +As a consequence, the access semantics described here, extracted from +fscrypt's doc, apply to Lustre client-level encryption. + +Ref: +https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/fscrypt.rst + +Access +====== + +Only Lustre clients need access to encryption master keys. Keys are +added to the filesystem-level encryption keyring on the Lustre client. + +With the key +------------ + +With the encryption key, encrypted regular files, directories, and +symlinks behave very similarly to their unencrypted counterparts --- +after all, the encryption is intended to be transparent. However, +astute users may notice some differences in behavior: + +- Unencrypted files, or files encrypted with a different encryption + policy (i.e. different key, modes, or flags), cannot be renamed or + linked into an encrypted directory; see `Encryption policy + enforcement`_. Attempts to do so will fail with EXDEV. However, + encrypted files can be renamed within an encrypted directory, or + into an unencrypted directory. + + Note: "moving" an unencrypted file into an encrypted directory, e.g. + with the `mv` program, is implemented in userspace by a copy + followed by a delete. Be aware that the original unencrypted data + may remain recoverable from free space on the disk; prefer to keep + all files encrypted from the very beginning. The `shred` program + may be used to overwrite the source files but isn't guaranteed to be + effective on all filesystems and storage devices. + +- Direct I/O is not supported on encrypted files. Attempts to use + direct I/O on such files will fall back to buffered I/O. + +- The fallocate operations FALLOC_FL_COLLAPSE_RANGE, + FALLOC_FL_INSERT_RANGE, and FALLOC_FL_ZERO_RANGE are not supported + on encrypted files and will fail with EOPNOTSUPP. + +- DAX (Direct Access) is not supported on encrypted files. + +- The st_size of an encrypted symlink will not necessarily give the + length of the symlink target as required by POSIX. It will actually + give the length of the ciphertext, which will be slightly longer + than the plaintext due to NUL-padding and an extra 2-byte overhead. + +- The maximum length of an encrypted symlink is 2 bytes shorter than + the maximum length of an unencrypted symlink. For example, on an + EXT4 filesystem with a 4K block size, unencrypted symlinks can be up + to 4095 bytes long, while encrypted symlinks can only be up to 4093 + bytes long (both lengths excluding the terminating null). + +Note that mmap *is* supported. This is possible because the pagecache +for an encrypted file contains the plaintext, not the ciphertext. + +Without the key +--------------- + +Some filesystem operations may be performed on encrypted regular +files, directories, and symlinks even before their encryption key has +been added, or after their encryption key has been removed: + +- File metadata may be read, e.g. using stat(). + +- Directories may be listed, in which case the filenames will be + listed in an encoded form derived from their ciphertext. The + algorithm is subject to change, but it is guaranteed that the + presented filenames will be no longer than NAME_MAX bytes, will not + contain the ``/`` or ``\0`` characters, and will uniquely identify + directory entries. + + The ``.`` and ``..`` directory entries are special. They are always + present and are not encrypted or encoded. + +- Files may be deleted. That is, nondirectory files may be deleted + with unlink() as usual, and empty directories may be deleted with + rmdir() as usual. Therefore, ``rm`` and ``rm -r`` will work as + expected. + +- Symlink targets may be read and followed, but they will be presented + in encrypted form, similar to filenames in directories. Hence, they + are unlikely to point to anywhere useful. + +Without the key, regular files cannot be opened or truncated. +Attempts to do so will fail with ENOKEY. This implies that any +regular file operations that require a file descriptor, such as +read(), write(), mmap(), fallocate(), and ioctl(), are also forbidden. + +Also without the key, files of any type (including directories) cannot +be created or linked into an encrypted directory, nor can a name in an +encrypted directory be the source or target of a rename, nor can an +O_TMPFILE temporary file be created in an encrypted directory. All +such operations will fail with ENOKEY. + +It is not currently possible to backup and restore encrypted files +without the encryption key. This would require special APIs which +have not yet been implemented. + +Encryption policy enforcement +============================= + +After an encryption policy has been set on a directory, all regular +files, directories, and symbolic links created in that directory +(recursively) will inherit that encryption policy. Special files --- +that is, named pipes, device nodes, and UNIX domain sockets --- will +not be encrypted. + +Except for those special files, it is forbidden to have unencrypted +files, or files encrypted with a different encryption policy, in an +encrypted directory tree. Attempts to link or rename such a file into +an encrypted directory will fail with EXDEV. This is also enforced +during ->lookup() to provide limited protection against offline +attacks that try to disable or downgrade encryption in known locations +where applications may later write sensitive data. It is recommended +that systems implementing a form of "verified boot" take advantage of +this by validating all top-level encryption policies prior to access. diff --git a/Documentation/lustre/client_side_encryption/key_hierarchy.txt b/Documentation/lustre/client_side_encryption/key_hierarchy.txt new file mode 100644 index 0000000..d96e189 --- /dev/null +++ b/Documentation/lustre/client_side_encryption/key_hierarchy.txt @@ -0,0 +1,112 @@ +============================================ +Lustre client-level encryption key hierarchy +============================================ + +Lustre client-level encryption relies on kernel's fscrypt, and more +precisely on v2 encryption policies. +fscrypt is a library which filesystems can hook into to support +transparent encryption of files and directories. + +As a consequence, the following key hierarchy description, extracted +from fscrypt's, is directly applicable to Lustre client-level encryption. + +Ref: +https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/fscrypt.rst + +Master Keys +----------- + +Each encrypted directory tree is protected by a *master key*. Master +keys can be up to 64 bytes long, and must be at least as long as the +greater of the key length needed by the contents and filenames +encryption modes being used. For example, if AES-256-XTS is used for +contents encryption, the master key must be 64 bytes (512 bits). Note +that the XTS mode is defined to require a key twice as long as that +required by the underlying block cipher. + +To "unlock" an encrypted directory tree, userspace must provide the +appropriate master key. There can be any number of master keys, each +of which protects any number of directory trees on any number of +filesystems. + +Master keys should be pseudorandom, i.e. indistinguishable from random +bytestrings of the same length. This implies that users **must not** +directly use a password as a master key, zero-pad a shorter key, or +repeat a shorter key. Instead, users should generate master keys +either using a cryptographically secure random number generator, or by +using a KDF (Key Derivation Function). Note that whenever a KDF is +used to "stretch" a lower-entropy secret such as a passphrase, it is +critical that a KDF designed for this purpose be used, such as scrypt, +PBKDF2, or Argon2. + +Key derivation function +----------------------- + +With one exception, fscrypt never uses the master key(s) for +encryption directly. Instead, they are only used as input to a KDF +(Key Derivation Function) to derive the actual keys. + +For v2 encryption policies, the KDF is HKDF-SHA512. The master key is +passed as the "input keying material", no salt is used, and a distinct +"application-specific information string" is used for each distinct +key to be derived. For example, when a per-file encryption key is +derived, the application-specific information string is the file's +nonce prefixed with "fscrypt\0" and a context byte. Different context +bytes are used for other types of derived keys. + +HKDF-SHA512 is preferred KDF because HKDF is more flexible, is +nonreversible, and evenly distributes entropy from the master key. +HKDF is also standardized and widely used by other software. + +Per-file keys +------------- + +Since each master key can protect many files, it is necessary to +"tweak" the encryption of each file so that the same plaintext in two +files doesn't map to the same ciphertext, or vice versa. In most +cases, fscrypt does this by deriving per-file keys. When a new +encrypted inode (regular file, directory, or symlink) is created, +fscrypt randomly generates a 16-byte nonce and stores it in the +inode's encryption xattr. Then, it uses a KDF (as described in `Key +derivation function`_) to derive the file's key from the master key +and nonce. + +Key derivation was chosen over key wrapping because wrapped keys would +require larger xattrs which would be less likely to fit in-line in the +filesystem's inode table, and there didn't appear to be any +significant advantages to key wrapping. In particular, currently +there is no requirement to support unlocking a file with multiple +alternative master keys or to support rotating master keys. Instead, +the master keys may be wrapped in userspace, e.g. as is done by the +`fscrypt `_ tool. + +Including the inode number in the IVs was considered. However, it was +rejected as it would have prevented ext4 filesystems from being +resized, and by itself still wouldn't have been sufficient to prevent +the same key from being directly reused for both XTS and CTS-CBC. + +DIRECT_KEY and per-mode keys +---------------------------- + +The Adiantum encryption mode is suitable for both contents and +filenames encryption, and it accepts long IVs --- long enough to hold +both an 8-byte logical block number and a 16-byte per-file nonce. +Also, the overhead of each Adiantum key is greater than that of an +AES-256-XTS key. + +Therefore, to improve performance and save memory, for Adiantum a +"direct key" configuration is supported. When the user has enabled +this by setting FSCRYPT_POLICY_FLAG_DIRECT_KEY in the fscrypt policy, +per-file keys are not used. Instead, whenever any data (contents or +filenames) is encrypted, the file's 16-byte nonce is included in the +IV. Moreover, for v2 encryption policies, the encryption is done with +a per-mode key derived using the KDF. Users may use the same master +key for other v2 encryption policies. + +Key identifiers +--------------- + +For master keys used for v2 encryption policies, a unique 16-byte "key +identifier" is also derived using the KDF. This value is stored in +the clear, since it is needed to reliably identify the key itself. + diff --git a/Documentation/lustre/client_side_encryption/modes_usage.txt b/Documentation/lustre/client_side_encryption/modes_usage.txt new file mode 100644 index 0000000..4ad8cf5 --- /dev/null +++ b/Documentation/lustre/client_side_encryption/modes_usage.txt @@ -0,0 +1,97 @@ +============================================== +Lustre client-level encryption modes and usage +============================================== + +Lustre client-level encryption relies on kernel's fscrypt, and more +precisely on v2 encryption policies. +fscrypt is a library which filesystems can hook into to support +transparent encryption of files and directories. + +As a consequence, the encryption modes and usage described here, +extracted from fscrypt's doc, apply to Lustre client-level encryption. + +Ref: +https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/fscrypt.rst + +Encryption modes +---------------- + +fscrypt allows one encryption mode to be specified for file contents +and one encryption mode to be specified for filenames. Different +directory trees are permitted to use different encryption modes. +Currently, the following pairs of encryption modes are supported: + +- AES-256-XTS for contents and AES-256-CTS-CBC for filenames +- AES-128-CBC for contents and AES-128-CTS-CBC for filenames +- Adiantum for both contents and filenames + +If unsure, you should use the (AES-256-XTS, AES-256-CTS-CBC) pair. + +AES-128-CBC was added only for low-powered embedded devices with +crypto accelerators such as CAAM or CESA that do not support XTS. + +Adiantum is a (primarily) stream cipher-based mode that is fast even +on CPUs without dedicated crypto instructions. It's also a true +wide-block mode, unlike XTS. It can also eliminate the need to derive +per-file keys. However, it depends on the security of two primitives, +XChaCha12 and AES-256, rather than just one. See the paper +"Adiantum: length-preserving encryption for entry-level processors" +(https://eprint.iacr.org/2018/720.pdf) for more details. To use +Adiantum, CONFIG_CRYPTO_ADIANTUM must be enabled. Also, fast +implementations of ChaCha and NHPoly1305 should be enabled, e.g. +CONFIG_CRYPTO_CHACHA20_NEON and CONFIG_CRYPTO_NHPOLY1305_NEON for ARM. + +Contents encryption +------------------- + +For file contents, each filesystem block is encrypted independently. +Currently, only the case where the filesystem block size is equal to +the system's page size (usually 4096 bytes) is supported. + +Each block's IV is set to the logical block number within the file as +a little endian number, except that: + +- With CBC mode encryption, ESSIV is also used. Specifically, each IV + is encrypted with AES-256 where the AES-256 key is the SHA-256 hash + of the file's data encryption key. + +- In the "direct key" configuration (FSCRYPT_POLICY_FLAG_DIRECT_KEY + set in the fscrypt_policy), the file's nonce is also appended to the + IV. Currently this is only allowed with the Adiantum encryption + mode. + +Filenames encryption +-------------------- + +For filenames, each full filename is encrypted at once. Because of +the requirements to retain support for efficient directory lookups and +filenames of up to 255 bytes, the same IV is used for every filename +in a directory. + +However, each encrypted directory still uses a unique key; or +alternatively (for the "direct key" configuration) has the file's +nonce included in the IVs. Thus, IV reuse is limited to within a +single directory. + +With CTS-CBC, the IV reuse means that when the plaintext filenames +share a common prefix at least as long as the cipher block size (16 +bytes for AES), the corresponding encrypted filenames will also share +a common prefix. This is undesirable. Adiantum does not have this +weakness, as it is a wide-block encryption mode. + +All supported filenames encryption modes accept any plaintext length +>= 16 bytes; cipher block alignment is not required. However, +filenames shorter than 16 bytes are NUL-padded to 16 bytes before +being encrypted. In addition, to reduce leakage of filename lengths +via their ciphertexts, all filenames are NUL-padded to the next 4, 8, +16, or 32-byte boundary (configurable). 32 is recommended since this +provides the best confidentiality, at the cost of making directory +entries consume slightly more space. Note that since NUL (``\0``) is +not otherwise a valid character in filenames, the padding will never +produce duplicate plaintexts. + +Symbolic link targets are considered a type of filename and are +encrypted in the same way as filenames in directory entries, except +that IV reuse is not a problem as each symlink has its own inode. + + diff --git a/Documentation/lustre/client_side_encryption/threat_model.txt b/Documentation/lustre/client_side_encryption/threat_model.txt new file mode 100644 index 0000000..f9153a2 --- /dev/null +++ b/Documentation/lustre/client_side_encryption/threat_model.txt @@ -0,0 +1,159 @@ +=========================================== +Lustre client-level encryption threat model +=========================================== + +Lustre client-level encryption relies on kernel's fscrypt, and more +precisely on v2 encryption policies. +fscrypt is a library which filesystems can hook into to support +transparent encryption of files and directories. + +As a consequence, the following threat model, extracted from +fscrypt's and adapted, is applicable to Lustre client-level +encryption. + +Ref: +https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/fscrypt.rst + +Offline attacks +--------------- + +Provided that userspace chooses a strong encryption key, fscrypt +protects the confidentiality of file contents and filenames in the +event of a single point-in-time permanent offline compromise of the +block device content. fscrypt does not protect the confidentiality of +non-filename metadata, e.g. file sizes, file permissions, file +timestamps, and extended attributes. Also, the existence and location +of holes (unallocated blocks which logically contain all zeroes) in +files is not protected. + +fscrypt is not guaranteed to protect confidentiality or authenticity +if an attacker is able to manipulate the filesystem offline prior to +an authorized user later accessing the filesystem. + +For the Lustre case, block devices are Lustre targets attached to +the Lustre servers. Manipulating the filesystem offline means +accessing the filesystem on these targets while Lustre is offline. + +Online attacks +-------------- + +fscrypt (and storage encryption in general) can only provide limited +protection, if any at all, against online attacks. In detail: + +Side-channel attacks +~~~~~~~~~~~~~~~~~~~~ + +fscrypt is only resistant to side-channel attacks, such as timing or +electromagnetic attacks, to the extent that the underlying Linux +Cryptographic API algorithms are. If a vulnerable algorithm is used, +such as a table-based implementation of AES, it may be possible for an +attacker to mount a side channel attack against the online system. +Side channel attacks may also be mounted against applications +consuming decrypted data. + +Unauthorized file access on Lustre client +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +After an encryption key has been added, fscrypt does not hide the +plaintext file contents or filenames from other users on the same +system. Instead, existing access control mechanisms such as file mode +bits, POSIX ACLs, LSMs, or namespaces should be used for this purpose. + +(For the reasoning behind this, understand that while the key is +added, the confidentiality of the data, from the perspective of the +system itself, is *not* protected by the mathematical properties of +encryption but rather only by the correctness of the kernel. +Therefore, any encryption-specific access control checks would merely +be enforced by kernel *code* and therefore would be largely redundant +with the wide variety of access control mechanisms already available.) + +For the Lustre case, it means plaintext file contents or filenames are +not hidden from other users on the same Lustre client. + +Lustre client kernel memory compromise +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An attacker who compromises the system enough to read from arbitrary +memory, e.g. by mounting a physical attack or by exploiting a kernel +security vulnerability, can compromise all encryption keys that are +currently in use. + +However, fscrypt with v2 encryption policies allows encryption keys to +be removed from the kernel, which may protect them from later +compromise. Key removal can be carried out by non-root users. + +In more detail, the FS_IOC_REMOVE_ENCRYPTION_KEY ioctl will wipe a +master encryption key from kernel memory. Moreover, it will try to +evict all cached inodes which had been "unlocked" using the key, +thereby wiping their per-file keys and making them once again appear +"locked", i.e. in ciphertext or encrypted form. + +However, FS_IOC_REMOVE_ENCRYPTION_KEY has some limitations: + +- Per-file keys for in-use files will *not* be removed or wiped. + Therefore, for maximum effect, userspace should close the relevant + encrypted files and directories before removing a master key, as + well as kill any processes whose working directory is in an affected + encrypted directory. + +- The kernel cannot magically wipe copies of the master key(s) that + userspace might have as well. Therefore, userspace must wipe all + copies of the master key(s) it makes as well. Naturally, the same + also applies to all higher levels in the key hierarchy. Userspace + should also follow other security precautions such as mlock()ing + memory containing keys to prevent it from being swapped out. + +- In general, decrypted contents and filenames in the kernel VFS + caches are freed but not wiped. Therefore, portions thereof may be + recoverable from freed memory, even after the corresponding key(s) + were wiped. To partially solve this, you can set + CONFIG_PAGE_POISONING=y in your kernel config and add page_poison=1 + to your kernel command line. However, this has a performance cost. + +- Secret keys might still exist in CPU registers, in crypto + accelerator hardware (if used by the crypto API to implement any of + the algorithms), or in other places not explicitly considered here. + +Lustre server kernel memory compromise +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +An attacker on a Lustre server who compromises the system enough to +read from arbitrary memory, e.g. by mounting a physical attack or by +exploiting a kernel security vulnerability, cannot compromise Lustre +files content. Indeed, encryption keys are not forwarded to the Lustre +servers, and servers do not carry out decryption or encryption. +Moreover, RPCs received by servers contain encrypted data, which is +put as is in blocks to be stored on disk. + +Per-file key compromise +~~~~~~~~~~~~~~~~~~~~~~~ + +With one exception, fscrypt never uses the master key(s) for +encryption directly. Instead, they are only used as input to a KDF +(Key Derivation Function) to derive the actual keys. + +For v2 encryption policies, the KDF is HKDF-SHA512. The master key is +passed as the "input keying material", no salt is used, and a distinct +"application-specific information string" is used for each distinct +key to be derived. For example, when a per-file encryption key is +derived, the application-specific information string is the file's +nonce prefixed with "fscrypt\0" and a context byte. Different context +bytes are used for other types of derived keys. + +So the per-file keys used to encrypt each file individually are +obtained from a HKDF-SHA512 mechanism that is flexible, nonreversible, +and evenly distributes entropy from the master key. +HKDF is also standardized and widely used by other software. + +As a consequence, a compromise of a per-file key only impacts the +associated file, not the master key. + +Master key verification +~~~~~~~~~~~~~~~~~~~~~~~ + +For master keys used for v2 encryption policies, a unique 16-byte "key +identifier" is derived using the KDF. This value is stored in +the clear, and is used to reliably identify the key itself. + +Consequently, no malicious user can associate the wrong key with +encrypted files. From patchwork Sat Jun 13 16:27:08 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602881 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 05C42138C for ; Sat, 13 Jun 2020 16:27:42 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id E1E942078A for ; Sat, 13 Jun 2020 16:27:41 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org E1E942078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id C56AA21E06E; Sat, 13 Jun 2020 09:27:30 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 210D421E076 for ; Sat, 13 Jun 2020 09:27:26 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id 9F9D96CA; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 99F462A9; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:08 -0400 Message-Id: <1592065636-28333-13-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 12/20] lustre: sec: enable client side encryption X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Sebastien Buisson Enable client side encryption. By default it is activated, letting user specifies actual encryption policy to use on a per-directory basis. It is possible to deactivate client side encryption by using the 'noencrypt' mount option. Also add the test dummy encryption mode option to ease testing. WC-bug-id: https://jira.whamcloud.com/browse/LU-12275 Lustre-commit: 28be31137cd22 ("LU-12275 sec: enable client side encryption") Signed-off-by: Sebastien Buisson Reviewed-on: https://review.whamcloud.com/36143 Reviewed-by: John L. Hammond Reviewed-by: Andreas Dilger Reviewed-by: Oleg Drokin Signed-off-by: James Simmons --- fs/lustre/include/lustre_crypto.h | 106 ++++++++++++++++++++++++++ fs/lustre/include/obd_support.h | 6 ++ fs/lustre/llite/Makefile | 1 + fs/lustre/llite/crypto.c | 132 +++++++++++++++++++++++++++++++++ fs/lustre/llite/llite_internal.h | 26 +++++++ fs/lustre/llite/llite_lib.c | 33 +++++++++ fs/lustre/llite/super25.c | 1 + include/uapi/linux/lustre/lustre_idl.h | 12 +++ 8 files changed, 317 insertions(+) create mode 100644 fs/lustre/include/lustre_crypto.h create mode 100644 fs/lustre/llite/crypto.c diff --git a/fs/lustre/include/lustre_crypto.h b/fs/lustre/include/lustre_crypto.h new file mode 100644 index 0000000..e9091a2 --- /dev/null +++ b/fs/lustre/include/lustre_crypto.h @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; If not, see + * http://www.gnu.org/licenses/gpl-2.0.html + * + * GPL HEADER END + */ +/* + * Copyright (c) 2019, 2020, Whamcloud. + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + */ + +#ifndef _LUSTRE_CRYPTO_H_ +#define _LUSTRE_CRYPTO_H_ + +struct ll_sb_info; +bool ll_sbi_has_test_dummy_encryption(struct ll_sb_info *sbi); +bool ll_sbi_has_encrypt(struct ll_sb_info *sbi); +void ll_sbi_set_encrypt(struct ll_sb_info *sbi, bool set); + +#ifdef CONFIG_FS_ENCRYPTION +#define __FS_HAS_ENCRYPTION 1 +#include + +#define llcrypt_operations fscrypt_operations +#define llcrypt_symlink_data fscrypt_symlink_data +#define llcrypt_dummy_context_enabled(inode) \ + fscrypt_dummy_context_enabled(inode) +#define llcrypt_has_encryption_key(inode) fscrypt_has_encryption_key(inode) +#define llcrypt_encrypt_pagecache_blocks(page, len, offs, gfp_flags) \ + fscrypt_encrypt_pagecache_blocks(page, len, offs, gfp_flags) +#define llcrypt_decrypt_pagecache_blocks(page, len, offs) \ + fscrypt_decrypt_pagecache_blocks(page, len, offs) +#define llcrypt_inherit_context(parent, child, fs_data, preload) \ + fscrypt_inherit_context(parent, child, fs_data, preload) +#define llcrypt_get_encryption_info(inode) fscrypt_get_encryption_info(inode) +#define llcrypt_put_encryption_info(inode) fscrypt_put_encryption_info(inode) +#define llcrypt_free_inode(inode) fscrypt_free_inode(inode) +#define llcrypt_finalize_bounce_page(pagep) fscrypt_finalize_bounce_page(pagep) +#define llcrypt_file_open(inode, filp) fscrypt_file_open(inode, filp) +#define llcrypt_ioctl_set_policy(filp, arg) fscrypt_ioctl_set_policy(filp, arg) +#define llcrypt_ioctl_get_policy_ex(filp, arg) \ + fscrypt_ioctl_get_policy_ex(filp, arg) +#define llcrypt_ioctl_add_key(filp, arg) fscrypt_ioctl_add_key(filp, arg) +#define llcrypt_ioctl_remove_key(filp, arg) fscrypt_ioctl_remove_key(filp, arg) +#define llcrypt_ioctl_remove_key_all_users(filp, arg) \ + fscrypt_ioctl_remove_key_all_users(filp, arg) +#define llcrypt_ioctl_get_key_status(filp, arg) \ + fscrypt_ioctl_get_key_status(filp, arg) +#define llcrypt_drop_inode(inode) fscrypt_drop_inode(inode) +#define llcrypt_prepare_rename(olddir, olddentry, newdir, newdentry, flags) \ + fscrypt_prepare_rename(olddir, olddentry, newdir, newdentry, flags) +#define llcrypt_prepare_link(old_dentry, dir, dentry) \ + fscrypt_prepare_link(old_dentry, dir, dentry) +#define llcrypt_prepare_setattr(dentry, attr) \ + fscrypt_prepare_setattr(dentry, attr) +#define llcrypt_set_ops(sb, cop) fscrypt_set_ops(sb, cop) +#else /* !CONFIG_FS_ENCRYPTION */ +#undef IS_ENCRYPTED +#define IS_ENCRYPTED(x) 0 +#define llcrypt_dummy_context_enabled(inode) NULL +/* copied from include/linux/fscrypt.h */ +#define llcrypt_has_encryption_key(inode) false +#define llcrypt_encrypt_pagecache_blocks(page, len, offs, gfp_flags) \ + ERR_PTR(-EOPNOTSUPP) +#define llcrypt_decrypt_pagecache_blocks(page, len, offs) -EOPNOTSUPP +#define llcrypt_inherit_context(parent, child, fs_data, preload) -EOPNOTSUPP +#define llcrypt_get_encryption_info(inode) -EOPNOTSUPP +#define llcrypt_put_encryption_info(inode) do {} while (0) +#define llcrypt_free_inode(inode) do {} while (0) +#define llcrypt_finalize_bounce_page(pagep) do {} while (0) +static inline int llcrypt_file_open(struct inode *inode, struct file *filp) +{ + return IS_ENCRYPTED(inode) ? -EOPNOTSUPP : 0; +} +#define llcrypt_ioctl_set_policy(filp, arg) -EOPNOTSUPP +#define llcrypt_ioctl_get_policy_ex(filp, arg) -EOPNOTSUPP +#define llcrypt_ioctl_add_key(filp, arg) -EOPNOTSUPP +#define llcrypt_ioctl_remove_key(filp, arg) -EOPNOTSUPP +#define llcrypt_ioctl_remove_key_all_users(filp, arg) -EOPNOTSUPP +#define llcrypt_ioctl_get_key_status(filp, arg) -EOPNOTSUPP +#define llcrypt_drop_inode(inode) 0 +#define llcrypt_prepare_rename(olddir, olddentry, newdir, newdentry, flags) 0 +#define llcrypt_prepare_link(old_dentry, dir, dentry) 0 +#define llcrypt_prepare_setattr(dentry, attr) 0 +#define llcrypt_set_ops(sb, cop) do {} while (0) +#endif /* CONFIG_FS_ENCRYPTION */ + +#endif /* _LUSTRE_CRYPTO_H_ */ diff --git a/fs/lustre/include/obd_support.h b/fs/lustre/include/obd_support.h index d1d21e0..b5736f8 100644 --- a/fs/lustre/include/obd_support.h +++ b/fs/lustre/include/obd_support.h @@ -559,6 +559,9 @@ static inline int ll_ext_to_inode_flags(int flags) ((flags & LUSTRE_NOATIME_FL) ? S_NOATIME : 0) | ((flags & LUSTRE_APPEND_FL) ? S_APPEND : 0) | ((flags & LUSTRE_DIRSYNC_FL) ? S_DIRSYNC : 0) | +#if defined(S_ENCRYPTED) + ((flags & LUSTRE_ENCRYPT_FL) ? S_ENCRYPTED : 0) | +#endif ((flags & LUSTRE_IMMUTABLE_FL) ? S_IMMUTABLE : 0)); } @@ -568,6 +571,9 @@ static inline int ll_inode_to_ext_flags(int iflags) ((iflags & S_NOATIME) ? LUSTRE_NOATIME_FL : 0) | ((iflags & S_APPEND) ? LUSTRE_APPEND_FL : 0) | ((iflags & S_DIRSYNC) ? LUSTRE_DIRSYNC_FL : 0) | +#if defined(S_ENCRYPTED) + ((iflags & S_ENCRYPTED) ? LUSTRE_ENCRYPT_FL : 0) | +#endif ((iflags & S_IMMUTABLE) ? LUSTRE_IMMUTABLE_FL : 0)); } diff --git a/fs/lustre/llite/Makefile b/fs/lustre/llite/Makefile index c88a1b0..aa388bb6 100644 --- a/fs/lustre/llite/Makefile +++ b/fs/lustre/llite/Makefile @@ -10,3 +10,4 @@ lustre-y := dcache.o dir.o file.o llite_lib.o llite_nfs.o \ lproc_llite.o pcc.o lustre-$(CONFIG_LUSTRE_FS_POSIX_ACL) += acl.o +lustre-$(CONFIG_FS_ENCRYPTION) += crypto.o diff --git a/fs/lustre/llite/crypto.c b/fs/lustre/llite/crypto.c new file mode 100644 index 0000000..94189c9 --- /dev/null +++ b/fs/lustre/llite/crypto.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * GPL HEADER START + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 only, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License version 2 for more details (a copy is included + * in the LICENSE file that accompanied this code). + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; If not, see + * http://www.gnu.org/licenses/gpl-2.0.html + * + * GPL HEADER END + */ +/* + * Copyright (c) 2019, 2020, Whamcloud. + */ +/* + * This file is part of Lustre, http://www.lustre.org/ + */ + +#include "llite_internal.h" + +static int ll_get_context(struct inode *inode, void *ctx, size_t len) +{ + struct dentry *dentry; + + if (hlist_empty(&inode->i_dentry)) + return -ENODATA; + + hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) + break; + + return __vfs_getxattr(dentry, inode, LL_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len); +} + +static int ll_set_context(struct inode *inode, const void *ctx, size_t len, + void *fs_data) +{ + unsigned int ext_flags; + struct dentry *dentry; + struct md_op_data *op_data; + struct ptlrpc_request *req = NULL; + int rc; + + if (inode == NULL) + return 0; + + ext_flags = ll_inode_to_ext_flags(inode->i_flags) | LUSTRE_ENCRYPT_FL; + dentry = (struct dentry *)fs_data; + + /* Encrypting the root directory is not allowed */ + if (inode->i_ino == inode->i_sb->s_root->d_inode->i_ino) + return -EPERM; + + op_data = ll_prep_md_op_data(NULL, inode, NULL, NULL, 0, 0, + LUSTRE_OPC_ANY, NULL); + if (IS_ERR(op_data)) + return PTR_ERR(op_data); + + op_data->op_attr_flags = LUSTRE_ENCRYPT_FL; + op_data->op_xvalid |= OP_XVALID_FLAGS; + rc = md_setattr(ll_i2sbi(inode)->ll_md_exp, op_data, NULL, 0, &req); + ll_finish_md_op_data(op_data); + ptlrpc_req_finished(req); + if (rc) + return rc; + + rc = __vfs_setxattr(dentry, inode, LL_XATTR_NAME_ENCRYPTION_CONTEXT, + ctx, len, XATTR_CREATE); + if (rc) + return rc; + + ll_update_inode_flags(inode, ext_flags); + return 0; +} + +inline bool ll_sbi_has_test_dummy_encryption(struct ll_sb_info *sbi) +{ + return unlikely(sbi->ll_flags & LL_SBI_TEST_DUMMY_ENCRYPTION); +} + +static bool ll_dummy_context(struct inode *inode) +{ + struct ll_sb_info *sbi = ll_i2sbi(inode); + + return sbi ? ll_sbi_has_test_dummy_encryption(sbi) : false; +} + +inline bool ll_sbi_has_encrypt(struct ll_sb_info *sbi) +{ + return sbi->ll_flags & LL_SBI_ENCRYPT; +} + +inline void ll_sbi_set_encrypt(struct ll_sb_info *sbi, bool set) +{ + if (set) + sbi->ll_flags |= LL_SBI_ENCRYPT; + else + sbi->ll_flags &= + ~(LL_SBI_ENCRYPT | LL_SBI_TEST_DUMMY_ENCRYPTION); +} + +static bool ll_empty_dir(struct inode *inode) +{ + /* used by llcrypt_ioctl_set_policy(), because a policy can only be set + * on an empty dir. + */ + /* Here we choose to return true, meaning we always call .set_context. + * Then we rely on server side, with mdd_fix_attr() that calls + * mdd_dir_is_empty() when setting encryption flag on directory. + */ + return true; +} + +const struct llcrypt_operations lustre_cryptops = { + .key_prefix = "lustre:", + .get_context = ll_get_context, + .set_context = ll_set_context, + .dummy_context = ll_dummy_context, + .empty_dir = ll_empty_dir, + .max_namelen = NAME_MAX, +}; diff --git a/fs/lustre/llite/llite_internal.h b/fs/lustre/llite/llite_internal.h index be3a0b0..96585a7 100644 --- a/fs/lustre/llite/llite_internal.h +++ b/fs/lustre/llite/llite_internal.h @@ -44,9 +44,11 @@ #include #include #include +#include #include #include #include + #include "vvp_internal.h" #include "range_lock.h" #include "pcc.h" @@ -551,6 +553,8 @@ enum stats_track_type { */ #define LL_SBI_TINY_WRITE 0x2000000 /* tiny write support */ #define LL_SBI_FILE_HEAT 0x4000000 /* file heat support */ +#define LL_SBI_TEST_DUMMY_ENCRYPTION 0x8000000 /* test dummy encryption */ +#define LL_SBI_ENCRYPT 0x10000000 /* client side encryption */ #define LL_SBI_FLAGS { \ "nolck", \ "checksum", \ @@ -579,6 +583,8 @@ enum stats_track_type { "pio", \ "tiny_write", \ "file_heat", \ + "test_dummy_encryption", \ + "noencrypt", \ } /* @@ -1578,4 +1584,24 @@ static inline struct pcc_super *ll_info2pccs(struct ll_inode_info *lli) return ll_i2pccs(ll_info2i(lli)); } +#ifdef CONFIG_FS_ENCRYPTION +/* crypto.c */ +extern const struct llcrypt_operations lustre_cryptops; + +#else /* !CONFIG_FS_ENCRYPTION */ +inline bool ll_sbi_has_test_dummy_encryption(struct ll_sb_info *sbi) +{ + return false; +} + +inline bool ll_sbi_has_encrypt(struct ll_sb_info *sbi) +{ + return false; +} + +inline void ll_sbi_set_encrypt(struct ll_sb_info *sbi, bool set) +{ +} +#endif /* !CONFIG_FS_ENCRYPTION */ + #endif /* LLITE_INTERNAL_H */ diff --git a/fs/lustre/llite/llite_lib.c b/fs/lustre/llite/llite_lib.c index 05d949b..5639b16 100644 --- a/fs/lustre/llite/llite_lib.c +++ b/fs/lustre/llite/llite_lib.c @@ -155,6 +155,7 @@ static struct ll_sb_info *ll_init_sbi(void) sbi->ll_flags |= LL_SBI_AGL_ENABLED; sbi->ll_flags |= LL_SBI_FAST_READ; sbi->ll_flags |= LL_SBI_TINY_WRITE; + ll_sbi_set_encrypt(sbi, true); /* root squash */ sbi->ll_squash.rsi_uid = 0; @@ -551,6 +552,7 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt) #if THREAD_SIZE >= 8192 /*b=17630*/ sb->s_export_op = &lustre_export_operations; #endif + llcrypt_set_ops(sb, &lustre_cryptops); /* make root inode * XXX: move this to after cbd setup? @@ -923,6 +925,25 @@ static int ll_options(char *options, struct ll_sb_info *sbi) *flags |= tmp; goto next; } + tmp = ll_set_opt("test_dummy_encryption", s1, + LL_SBI_TEST_DUMMY_ENCRYPTION); + if (tmp) { +#ifdef CONFIG_FS_ENCRYPTION + *flags |= tmp; +#else + LCONSOLE_WARN("Test dummy encryption mount option ignored: encryption not supported\n"); +#endif + goto next; + } + tmp = ll_set_opt("noencrypt", s1, LL_SBI_ENCRYPT); + if (tmp) { +#ifdef CONFIG_FS_ENCRYPTION + *flags &= ~tmp; +#else + LCONSOLE_WARN("noencrypt mount option ignored: encryption not supported\n"); +#endif + goto next; + } LCONSOLE_ERROR_MSG(0x152, "Unknown option '%s', won't mount.\n", s1); return -EINVAL; @@ -1558,6 +1579,8 @@ void ll_clear_inode(struct inode *inode) * cl_object still uses inode lsm. */ cl_inode_fini(inode); + + llcrypt_put_encryption_info(inode); } static int ll_md_setattr(struct dentry *dentry, struct md_op_data *op_data) @@ -1978,6 +2001,8 @@ void ll_update_inode_flags(struct inode *inode, int ext_flags) { struct ll_inode_info *lli = ll_i2info(inode); + /* do not clear encryption flag */ + ext_flags |= ll_inode_to_ext_flags(inode->i_flags) & LUSTRE_ENCRYPT_FL; inode->i_flags = ll_ext_to_inode_flags(ext_flags); if (ext_flags & LUSTRE_PROJINHERIT_FL) set_bit(LLIF_PROJECT_INHERIT, &lli->lli_flags); @@ -2702,6 +2727,14 @@ int ll_show_options(struct seq_file *seq, struct dentry *dentry) if (sbi->ll_flags & LL_SBI_ALWAYS_PING) seq_puts(seq, ",always_ping"); + if (ll_sbi_has_test_dummy_encryption(sbi)) + seq_puts(seq, ",test_dummy_encryption"); + + if (ll_sbi_has_encrypt(sbi)) + seq_puts(seq, ",encrypt"); + else + seq_puts(seq, ",noencrypt"); + return 0; } diff --git a/fs/lustre/llite/super25.c b/fs/lustre/llite/super25.c index 006be6b..8eb3fc3 100644 --- a/fs/lustre/llite/super25.c +++ b/fs/lustre/llite/super25.c @@ -63,6 +63,7 @@ static void ll_inode_destroy_callback(struct rcu_head *head) struct inode *inode = container_of(head, struct inode, i_rcu); struct ll_inode_info *ptr = ll_i2info(inode); + llcrypt_free_inode(inode); kmem_cache_free(ll_inode_cachep, ptr); } diff --git a/include/uapi/linux/lustre/lustre_idl.h b/include/uapi/linux/lustre/lustre_idl.h index d0b43c8..d4df989 100644 --- a/include/uapi/linux/lustre/lustre_idl.h +++ b/include/uapi/linux/lustre/lustre_idl.h @@ -1108,6 +1108,8 @@ struct lov_mds_md_v1 { /* LOV EA mds/wire data (little-endian) */ #define XATTR_NAME_HSM "trusted.hsm" #define XATTR_NAME_LFSCK_NAMESPACE "trusted.lfsck_namespace" +#define LL_XATTR_NAME_ENCRYPTION_CONTEXT XATTR_SECURITY_PREFIX"c" + struct lov_mds_md_v3 { /* LOV EA mds/wire data (little-endian) */ __u32 lmm_magic; /* magic number = LOV_MAGIC_V3 */ __u32 lmm_pattern; /* LOV_PATTERN_RAID0, LOV_PATTERN_RAID1 */ @@ -1571,6 +1573,16 @@ enum { LUSTRE_TOPDIR_FL = 0x00020000, /* Top of directory hierarchies*/ LUSTRE_INLINE_DATA_FL = 0x10000000, /* Inode has inline data. */ LUSTRE_PROJINHERIT_FL = 0x20000000, /* Create with parents projid */ + + /* These flags will not be identical to any EXT4_*_FL counterparts, + * and only reserved for lustre purpose. Note: these flags might + * be conflict with some of EXT4 flags, so + * 1. these conflict flags needs to be removed when the flag is + * wired by la_flags see osd_attr_get(). + * 2. If these flags needs to be stored into inode, they will be + * stored in LMA. see LMAI_XXXX + */ + LUSTRE_ENCRYPT_FL = 0x00800000, /* encrypted file */ }; /* 64 possible states */ From patchwork Sat Jun 13 16:27:09 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602909 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id C5E56138C for ; Sat, 13 Jun 2020 16:28:22 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id ADB982078A for ; Sat, 13 Jun 2020 16:28:22 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org ADB982078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id ED24021F460; Sat, 13 Jun 2020 09:27:51 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id D129F21E0AA for ; Sat, 13 Jun 2020 09:27:26 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id A41636CC; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 9CF242AB; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:09 -0400 Message-Id: <1592065636-28333-14-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 13/20] lustre: ptlrpc: separate number MD and refrences for bulk X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Alexey Lyashkov , Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Alexey Lyashkov Introduce a bulk desc refs, it's different from MD's count ptlrpc expects to have events from all MD's even it's filled or not. So, number an MD's to post is related to the requested transfer size, not a number MD's with data. Cray-bug-id: LUS-8139 WC-bug-id: https://jira.whamcloud.com/browse/LU-10157 Lustre-commit: 8a7f2d4b11801 ("LU-10157 ptlrpc: separate number MD and refrences for bulk") Signed-off-by: Alexey Lyashkov Reviewed-on: https://review.whamcloud.com/37386 Reviewed-by: James Simmons Reviewed-by: Shaun Tancheff Reviewed-by: Oleg Drokin Signed-off-by: James Simmons --- fs/lustre/include/lustre_net.h | 4 +++- fs/lustre/ptlrpc/client.c | 2 +- fs/lustre/ptlrpc/events.c | 6 +++--- fs/lustre/ptlrpc/niobuf.c | 12 ++++++------ 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/fs/lustre/include/lustre_net.h b/fs/lustre/include/lustre_net.h index a94d826..18de7d9 100644 --- a/fs/lustre/include/lustre_net.h +++ b/fs/lustre/include/lustre_net.h @@ -1237,6 +1237,8 @@ struct ptlrpc_bulk_frag_ops { * Another user is readpage for MDT. */ struct ptlrpc_bulk_desc { + /* number MD's assigned including zero-sends */ + unsigned int bd_refs; /** completed with failure */ unsigned long bd_failure:1; /** client side */ @@ -1796,7 +1798,7 @@ static inline int ptlrpc_client_bulk_active(struct ptlrpc_request *req) return 1; spin_lock(&desc->bd_lock); - rc = desc->bd_md_count; + rc = desc->bd_refs; spin_unlock(&desc->bd_lock); return rc; } diff --git a/fs/lustre/ptlrpc/client.c b/fs/lustre/ptlrpc/client.c index 22022ff..e69c988 100644 --- a/fs/lustre/ptlrpc/client.c +++ b/fs/lustre/ptlrpc/client.c @@ -244,7 +244,7 @@ void __ptlrpc_prep_bulk_page(struct ptlrpc_bulk_desc *desc, void ptlrpc_free_bulk(struct ptlrpc_bulk_desc *desc) { LASSERT(desc->bd_iov_count != LI_POISON); /* not freed already */ - LASSERT(desc->bd_md_count == 0); /* network hands off */ + LASSERT(desc->bd_refs == 0); /* network hands off */ LASSERT((desc->bd_export != NULL) ^ (desc->bd_import != NULL)); LASSERT(desc->bd_frag_ops); diff --git a/fs/lustre/ptlrpc/events.c b/fs/lustre/ptlrpc/events.c index 5e3787c..eef40b3 100644 --- a/fs/lustre/ptlrpc/events.c +++ b/fs/lustre/ptlrpc/events.c @@ -204,8 +204,8 @@ void client_bulk_callback(struct lnet_event *ev) spin_lock(&desc->bd_lock); req = desc->bd_req; - LASSERT(desc->bd_md_count > 0); - desc->bd_md_count--; + LASSERT(desc->bd_refs > 0); + desc->bd_refs--; if (ev->type != LNET_EVENT_UNLINK && ev->status == 0) { desc->bd_nob_transferred += ev->mlength; @@ -223,7 +223,7 @@ void client_bulk_callback(struct lnet_event *ev) /* NB don't unlock till after wakeup; desc can disappear under us * otherwise */ - if (desc->bd_md_count == 0) + if (desc->bd_refs == 0) ptlrpc_client_wake_req(desc->bd_req); spin_unlock(&desc->bd_lock); diff --git a/fs/lustre/ptlrpc/niobuf.c b/fs/lustre/ptlrpc/niobuf.c index 331a0c8..3f8b2c6 100644 --- a/fs/lustre/ptlrpc/niobuf.c +++ b/fs/lustre/ptlrpc/niobuf.c @@ -169,7 +169,7 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req) desc->bd_registered = 1; desc->bd_last_mbits = mbits; - desc->bd_md_count = total_md; + desc->bd_refs = total_md; md.user_ptr = &desc->bd_cbid; md.handler = ptlrpc_handler; md.threshold = 1; /* PUT or GET */ @@ -211,9 +211,9 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req) if (rc != 0) { LASSERT(rc == -ENOMEM); spin_lock(&desc->bd_lock); - desc->bd_md_count -= total_md - posted_md; + desc->bd_refs -= total_md - posted_md; spin_unlock(&desc->bd_lock); - LASSERT(desc->bd_md_count >= 0); + LASSERT(desc->bd_refs >= 0); mdunlink_iterate_helper(desc->bd_mds, desc->bd_md_max_brw); req->rq_status = -ENOMEM; desc->bd_registered = 0; @@ -222,15 +222,15 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req) spin_lock(&desc->bd_lock); /* Holler if peer manages to touch buffers before he knows the mbits */ - if (desc->bd_md_count != total_md) + if (desc->bd_refs != total_md) CWARN("%s: Peer %s touched %d buffers while I registered\n", desc->bd_import->imp_obd->obd_name, libcfs_id2str(peer), - total_md - desc->bd_md_count); + total_md - desc->bd_refs); spin_unlock(&desc->bd_lock); CDEBUG(D_NET, "Setup %u bulk %s buffers: %u pages %u bytes, mbits x%#llx-%#llx, portal %u\n", - desc->bd_md_count, + desc->bd_refs, ptlrpc_is_bulk_op_get(desc->bd_type) ? "get-source" : "put-sink", desc->bd_iov_count, desc->bd_nob, desc->bd_last_mbits, req->rq_mbits, desc->bd_portal); From patchwork Sat Jun 13 16:27:10 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602905 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 5725E138C for ; Sat, 13 Jun 2020 16:28:17 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id 3F4C62078A for ; Sat, 13 Jun 2020 16:28:17 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 3F4C62078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 9B15D21EB68; Sat, 13 Jun 2020 09:27:48 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 7902D21E0AA for ; Sat, 13 Jun 2020 09:27:26 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id A39BE6CB; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id A043E378; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:10 -0400 Message-Id: <1592065636-28333-15-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 14/20] lustre: ptlrpc: fill md correctly. X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Alexey Lyashkov , Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Alexey Lyashkov MD fill should limit to the overall transfer size in additional to the number a fragment. Let's do this. Cray-bug-id: LUS-8139 WC-bug-id: https://jira.whamcloud.com/browse/LU-10157 Lustre-commit: e1ac9e74844dc ("LU-10157 ptlrpc: fill md correctly.") Signed-off-by: Alexey Lyashkov Reviewed-on: https://review.whamcloud.com/37387 Reviewed-by: James Simmons Reviewed-by: Shaun Tancheff Reviewed-by: Oleg Drokin Signed-off-by: James Simmons --- fs/lustre/include/lustre_net.h | 5 +++++ fs/lustre/ptlrpc/client.c | 22 ++++++++++++++++++---- fs/lustre/ptlrpc/niobuf.c | 10 +++++----- fs/lustre/ptlrpc/pers.c | 20 +++++++++++++++----- 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/fs/lustre/include/lustre_net.h b/fs/lustre/include/lustre_net.h index 18de7d9..2d58f13e 100644 --- a/fs/lustre/include/lustre_net.h +++ b/fs/lustre/include/lustre_net.h @@ -1260,6 +1260,7 @@ struct ptlrpc_bulk_desc { int bd_max_iov; /* allocated size of bd_iov */ int bd_nob; /* # bytes covered */ int bd_nob_transferred; /* # bytes GOT/PUT */ + unsigned int bd_nob_last; /* # bytes in last MD */ u64 bd_last_mbits; @@ -1267,6 +1268,10 @@ struct ptlrpc_bulk_desc { lnet_nid_t bd_sender; /* stash event::sender */ int bd_md_count; /* # valid entries in bd_mds */ int bd_md_max_brw; /* max entries in bd_mds */ + + /** array of offsets for each MD */ + unsigned int bd_mds_off[PTLRPC_BULK_OPS_COUNT]; + /** array of associated MDs */ struct lnet_handle_md bd_mds[PTLRPC_BULK_OPS_COUNT]; diff --git a/fs/lustre/ptlrpc/client.c b/fs/lustre/ptlrpc/client.c index e69c988..c9d9fe9 100644 --- a/fs/lustre/ptlrpc/client.c +++ b/fs/lustre/ptlrpc/client.c @@ -147,6 +147,12 @@ struct ptlrpc_bulk_desc *ptlrpc_new_bulk(unsigned int nfrags, LASSERT(ops->add_kiov_frag); + if (max_brw > PTLRPC_BULK_OPS_COUNT) + return NULL; + + if (nfrags > LNET_MAX_IOV * max_brw) + return NULL; + desc = kzalloc(sizeof(*desc), GFP_NOFS); if (!desc) return NULL; @@ -162,6 +168,7 @@ struct ptlrpc_bulk_desc *ptlrpc_new_bulk(unsigned int nfrags, desc->bd_portal = portal; desc->bd_type = type; desc->bd_md_count = 0; + desc->bd_nob_last = LNET_MTU; desc->bd_frag_ops = ops; LASSERT(max_brw > 0); desc->bd_md_max_brw = min(max_brw, PTLRPC_BULK_OPS_COUNT); @@ -228,6 +235,15 @@ void __ptlrpc_prep_bulk_page(struct ptlrpc_bulk_desc *desc, kiov = &desc->bd_vec[desc->bd_iov_count]; + if (((desc->bd_iov_count % LNET_MAX_IOV) == 0) || + ((desc->bd_nob_last + len) > LNET_MTU)) { + desc->bd_mds_off[desc->bd_md_count] = desc->bd_iov_count; + desc->bd_md_count++; + desc->bd_nob_last = 0; + LASSERT(desc->bd_md_count <= PTLRPC_BULK_OPS_COUNT); + } + + desc->bd_nob_last += len; desc->bd_nob += len; if (pin) @@ -3177,9 +3193,7 @@ void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req) || req->rq_mbits == 0) { req->rq_mbits = req->rq_xid; } else { - int total_md = (bd->bd_iov_count + LNET_MAX_IOV - 1) / - LNET_MAX_IOV; - req->rq_mbits -= total_md - 1; + req->rq_mbits -= bd->bd_md_count - 1; } } else { /* @@ -3194,7 +3208,7 @@ void ptlrpc_set_bulk_mbits(struct ptlrpc_request *req) * that server can infer the number of bulks that were prepared, * see LU-1431 */ - req->rq_mbits += DIV_ROUND_UP(bd->bd_iov_count, LNET_MAX_IOV) - 1; + req->rq_mbits += bd->bd_md_count - 1; /* Set rq_xid as rq_mbits to indicate the final bulk for the old * server which does not support OBD_CONNECT_BULK_MBITS. LU-6808 diff --git a/fs/lustre/ptlrpc/niobuf.c b/fs/lustre/ptlrpc/niobuf.c index 3f8b2c6..6fb79a2 100644 --- a/fs/lustre/ptlrpc/niobuf.c +++ b/fs/lustre/ptlrpc/niobuf.c @@ -131,7 +131,6 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req) /* NB no locking required until desc is on the network */ LASSERT(desc->bd_nob > 0); - LASSERT(desc->bd_md_count == 0); LASSERT(desc->bd_md_max_brw <= PTLRPC_BULK_OPS_COUNT); LASSERT(desc->bd_iov_count <= PTLRPC_MAX_BRW_PAGES); LASSERT(desc->bd_req); @@ -154,9 +153,9 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req) LASSERT(desc->bd_cbid.cbid_fn == client_bulk_callback); LASSERT(desc->bd_cbid.cbid_arg == desc); - total_md = DIV_ROUND_UP(desc->bd_iov_count, LNET_MAX_IOV); + total_md = desc->bd_md_count; /* rq_mbits is matchbits of the final bulk */ - mbits = req->rq_mbits - total_md + 1; + mbits = req->rq_mbits - desc->bd_md_count + 1; LASSERTF(mbits == (req->rq_mbits & PTLRPC_BULK_OPS_MASK), "first mbits = x%llu, last mbits = x%llu\n", @@ -174,13 +173,14 @@ static int ptlrpc_register_bulk(struct ptlrpc_request *req) md.handler = ptlrpc_handler; md.threshold = 1; /* PUT or GET */ - for (posted_md = 0; posted_md < total_md; posted_md++, mbits++) { + for (posted_md = 0; posted_md < desc->bd_md_count; + posted_md++, mbits++) { md.options = PTLRPC_MD_OPTIONS | (ptlrpc_is_bulk_op_get(desc->bd_type) ? LNET_MD_OP_GET : LNET_MD_OP_PUT); ptlrpc_fill_bulk_md(&md, desc, posted_md); - if (posted_md > 0 && posted_md + 1 == total_md && + if (posted_md > 0 && posted_md + 1 == desc->bd_md_count && OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_BULK_ATTACH)) { rc = -ENOMEM; } else { diff --git a/fs/lustre/ptlrpc/pers.c b/fs/lustre/ptlrpc/pers.c index d02b155..ecbc9d3 100644 --- a/fs/lustre/ptlrpc/pers.c +++ b/fs/lustre/ptlrpc/pers.c @@ -44,7 +44,7 @@ void ptlrpc_fill_bulk_md(struct lnet_md *md, struct ptlrpc_bulk_desc *desc, int mdidx) { - int offset = mdidx * LNET_MAX_IOV; + unsigned int start = desc->bd_mds_off[mdidx]; BUILD_BUG_ON(PTLRPC_MAX_BRW_PAGES >= LI_POISON); @@ -52,12 +52,22 @@ void ptlrpc_fill_bulk_md(struct lnet_md *md, struct ptlrpc_bulk_desc *desc, LASSERT(desc->bd_iov_count <= PTLRPC_MAX_BRW_PAGES); LASSERT(!(md->options & (LNET_MD_KIOV | LNET_MD_PHYS))); - md->length = max(0, desc->bd_iov_count - mdidx * LNET_MAX_IOV); - md->length = min_t(unsigned int, LNET_MAX_IOV, md->length); + /* just send a lnet header */ + if (mdidx >= desc->bd_md_count) { + md->options |= LNET_MD_KIOV; + md->length = 0; + md->start = NULL; + return; + } + + if (mdidx == (desc->bd_md_count - 1)) + md->length = desc->bd_iov_count - start; + else + md->length = desc->bd_mds_off[mdidx + 1] - start; md->options |= LNET_MD_KIOV; if (desc->bd_enc_vec) - md->start = &desc->bd_enc_vec[offset]; + md->start = &desc->bd_enc_vec[start]; else - md->start = &desc->bd_vec[offset]; + md->start = &desc->bd_vec[start]; } From patchwork Sat Jun 13 16:27:11 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602895 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id D5ED11392 for ; Sat, 13 Jun 2020 16:28:01 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id BE1B62078A for ; Sat, 13 Jun 2020 16:28:01 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org BE1B62078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 866E121EEC1; Sat, 13 Jun 2020 09:27:40 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 3546F21EBAC for ; Sat, 13 Jun 2020 09:27:27 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id A6A0A6CE; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id A3E9447A; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:11 -0400 Message-Id: <1592065636-28333-16-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 15/20] lustre: llite: don't check mirror info for page discard X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Bobi Jam The CIT_MISC is used for locks/pages manipulation, it will not go with full io procedure, i.e. cl_io_loop() will not be called for it. So don't check it for plain file since the mirror info is not initialized/set in this case. WC-bug-id: https://jira.whamcloud.com/browse/LU-13473 Lustre-commit: d0dd744ed6ae0 ("LU-13473 llite: don't check mirror info for page discard") Signed-off-by: Bobi Jam Reviewed-on: https://review.whamcloud.com/38307 Reviewed-by: Andreas Dilger Reviewed-by: Wang Shilong Reviewed-by: Oleg Drokin Signed-off-by: James Simmons --- fs/lustre/lov/lov_io.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/fs/lustre/lov/lov_io.c b/fs/lustre/lov/lov_io.c index 615db73..99b68d8 100644 --- a/fs/lustre/lov/lov_io.c +++ b/fs/lustre/lov/lov_io.c @@ -299,8 +299,13 @@ static int lov_io_mirror_init(struct lov_io *lio, struct lov_object *obj, int result; if (!lov_is_flr(obj)) { - LASSERT(comp->lo_preferred_mirror == 0); - lio->lis_mirror_index = comp->lo_preferred_mirror; + /* only locks/pages are manipulated for CIT_MISC op, no + * cl_io_loop() will be called, don't check/set mirror info. + */ + if (io->ci_type != CIT_MISC) { + LASSERT(comp->lo_preferred_mirror == 0); + lio->lis_mirror_index = comp->lo_preferred_mirror; + } io->ci_ndelay = 0; return 0; } From patchwork Sat Jun 13 16:27:12 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602899 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id AF4671392 for ; Sat, 13 Jun 2020 16:28:07 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id 97B432078A for ; Sat, 13 Jun 2020 16:28:07 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 97B432078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 7F52C21F1B8; Sat, 13 Jun 2020 09:27:43 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 7944521EBAC for ; Sat, 13 Jun 2020 09:27:27 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id A7CE66CF; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id A6AF747B; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:12 -0400 Message-Id: <1592065636-28333-17-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 16/20] lustre: sec: control client side encryption X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Sebastien Buisson Client enables encryption by default. However, this should be possible only if server side is encryption aware. Moreover, we want to give the ability to decide which clients can make use of encryption, by extending the nodemap mechanism with a new 'forbid_encryption' property, set to 0 by default. WC-bug-id: https://jira.whamcloud.com/browse/LU-12275 Lustre-commit: 3042bcd709ebf ("LU-12275 sec: control client side encryption") Signed-off-by: Sebastien Buisson Reviewed-on: https://review.whamcloud.com/36433 Reviewed-by: John L. Hammond Reviewed-by: Andreas Dilger Reviewed-by: Oleg Drokin Signed-off-by: James Simmons --- fs/lustre/llite/llite_internal.h | 17 +++++++++++++++++ fs/lustre/llite/llite_lib.c | 23 +++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/fs/lustre/llite/llite_internal.h b/fs/lustre/llite/llite_internal.h index 96585a7..a688bd8 100644 --- a/fs/lustre/llite/llite_internal.h +++ b/fs/lustre/llite/llite_internal.h @@ -398,6 +398,23 @@ int ll_inode_init_security(struct dentry *dentry, struct inode *inode, int ll_listsecurity(struct inode *inode, char *secctx_name, size_t secctx_name_size); +static inline bool obd_connect_has_enc(struct obd_connect_data *data) +{ +#ifdef CONFIG_FS_ENCRYPTION + return data->ocd_connect_flags & OBD_CONNECT_FLAGS2 && + data->ocd_connect_flags2 & OBD_CONNECT2_ENCRYPT; +#else + return false; +#endif +} + +static inline void obd_connect_set_enc(struct obd_connect_data *data) +{ +#ifdef HAVE_LUSTRE_CRYPTO + data->ocd_connect_flags2 |= OBD_CONNECT2_ENCRYPT; +#endif +} + /* * Locking to guarantee consistency of non-atomic updates to long long i_size, * consistency between file size and KMS. diff --git a/fs/lustre/llite/llite_lib.c b/fs/lustre/llite/llite_lib.c index 5639b16..6cd9f7e 100644 --- a/fs/lustre/llite/llite_lib.c +++ b/fs/lustre/llite/llite_lib.c @@ -303,6 +303,8 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt) data->ocd_connect_flags &= ~OBD_CONNECT_PINGLESS; obd_connect_set_secctx(data); + if (ll_sbi_has_encrypt(sbi)) + obd_connect_set_enc(data); #if defined(CONFIG_SECURITY) data->ocd_connect_flags2 |= OBD_CONNECT2_SELINUX_POLICY; @@ -424,6 +426,14 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt) if (obd_connect_has_secctx(data)) sbi->ll_flags |= LL_SBI_FILE_SECCTX; + if (ll_sbi_has_encrypt(sbi) && !obd_connect_has_enc(data)) { + if (ll_sbi_has_test_dummy_encryption(sbi)) + LCONSOLE_WARN("%s: server %s does not support encryption feature, encryption deactivated.\n", + sbi->ll_fsname, + sbi->ll_md_exp->exp_obd->obd_name); + ll_sbi_set_encrypt(sbi, false); + } + if (data->ocd_ibits_known & MDS_INODELOCK_XATTR) { if (!(data->ocd_connect_flags & OBD_CONNECT_MAX_EASIZE)) { LCONSOLE_INFO("%s: disabling xattr cache due to unknown maximum xattr size.\n", @@ -486,6 +496,9 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt) if (sbi->ll_flags & LL_SBI_ALWAYS_PING) data->ocd_connect_flags &= ~OBD_CONNECT_PINGLESS; + if (ll_sbi_has_encrypt(sbi)) + obd_connect_set_enc(data); + CDEBUG(D_RPCTRACE, "ocd_connect_flags: %#llx ocd_version: %d ocd_grant: %d\n", data->ocd_connect_flags, @@ -509,6 +522,16 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt) goto out_md_fid; } + if (ll_sbi_has_encrypt(sbi) && + !obd_connect_has_enc(&sbi->ll_dt_obd->u.lov.lov_ocd)) { + if (ll_sbi_has_test_dummy_encryption(sbi)) + LCONSOLE_WARN("%s: server %s does not support encryption feature, encryption deactivated.\n", + sbi->ll_fsname, dt); + ll_sbi_set_encrypt(sbi, false); + } else if (ll_sbi_has_test_dummy_encryption(sbi)) { + LCONSOLE_WARN("Test dummy encryption mode enabled\n"); + } + sbi->ll_dt_exp->exp_connect_data = *data; /* Don't change value if it was specified in the config log */ From patchwork Sat Jun 13 16:27:13 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602903 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id DAF2E1392 for ; Sat, 13 Jun 2020 16:28:12 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id C31182078A for ; Sat, 13 Jun 2020 16:28:12 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org C31182078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 5962B21E04D; Sat, 13 Jun 2020 09:27:46 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id CF87321EBAC for ; Sat, 13 Jun 2020 09:27:27 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id ABFBB6D0; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id A9B0047C; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:13 -0400 Message-Id: <1592065636-28333-18-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 17/20] lnet: fix uninitialize var in choose_ipv4_src() X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Mr NeilBrown choose_ip4_src() test "*ret" without initializing it - and callers do not (and should not) initialize the var. Instead of testing "*ret", test "err" - if this is non-zero (it will be -ENOENT) we want to use the address. If it is zero, then we only use the address if it is on the right subnet. Reported-by: Amir Shehata Fixes: 382731148e24 ("lnet: socklnd: use interface index to track local addr") WC-bug-id: https://jira.whamcloud.com/browse/LU-10391 Lustre-commit: 8ea895389a92d ("LU-10391 lnet: fix uninitialize var in choose_ipv4_src()") Signed-off-by: Mr NeilBrown Reviewed-on: https://review.whamcloud.com/38823 Reviewed-by: Aurelien Degremont Reviewed-by: James Simmons Reviewed-by: Amir Shehata Reviewed-by: Oleg Drokin Signed-off-by: James Simmons --- net/lnet/lnet/lib-socket.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/lnet/lnet/lib-socket.c b/net/lnet/lnet/lib-socket.c index a4db830..e082b52 100644 --- a/net/lnet/lnet/lib-socket.c +++ b/net/lnet/lnet/lib-socket.c @@ -158,7 +158,7 @@ int choose_ipv4_src(__u32 *ret, int interface, __u32 dst_ipaddr, struct net *ns) goto out; err = -ENOENT; in_dev_for_each_ifa_rcu(ifa, in_dev) { - if (*ret == 0 || + if (err == 0 || ((dst_ipaddr ^ ntohl(ifa->ifa_local)) & ntohl(ifa->ifa_mask)) == 0) { /* This address at least as good as what we From patchwork Sat Jun 13 16:27:14 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602907 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 43A1F138C for ; Sat, 13 Jun 2020 16:28:18 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id 2C1602078A for ; Sat, 13 Jun 2020 16:28:18 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 2C1602078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 2436921FB39; Sat, 13 Jun 2020 09:27:49 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 21F3221EBAC for ; Sat, 13 Jun 2020 09:27:28 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id AD9376D1; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id AC8E9487; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:14 -0400 Message-Id: <1592065636-28333-19-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 18/20] lustre: obd: Rename OS_STATE flags to OS_STATFS X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Patrick Farrell The statfs state flags are oddly named "OS_STATE_[STATE]" Rename them to "OS_STATFS_[STATE]" to make their role clearer and make them easier to find. WC-bug-id: https://jira.whamcloud.com/browse/LU-11963 Lustre-commit: 9be2db987767a ("LU-11963 obd: Rename OS_STATE flags to OS_STATFS") Signed-off-by: Patrick Farrell Reviewed-on: https://review.whamcloud.com/34289 Reviewed-by: Andreas Dilger Reviewed-by: Patrick Farrell Signed-off-by: James Simmons --- fs/lustre/include/obd_class.h | 4 ++-- fs/lustre/llite/llite_lib.c | 2 +- fs/lustre/lmv/lmv_obd.c | 2 +- fs/lustre/ptlrpc/wiretest.c | 28 ++++++++++++++-------------- include/uapi/linux/lustre/lustre_user.h | 20 ++++++++++---------- 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/fs/lustre/include/obd_class.h b/fs/lustre/include/obd_class.h index be73a165..746782b 100644 --- a/fs/lustre/include/obd_class.h +++ b/fs/lustre/include/obd_class.h @@ -942,14 +942,14 @@ static inline int obd_statfs(const struct lu_env *env, struct obd_export *exp, obd->obd_name, obd->obd_osfs_age, max_age); /* ignore cache if aggregated isn't expected */ if (obd->obd_osfs_age < max_age || - ((obd->obd_osfs.os_state & OS_STATE_SUM) && + ((obd->obd_osfs.os_state & OS_STATFS_SUM) && !(flags & OBD_STATFS_SUM))) { /* the RPC will block anyway, so avoid sending many at once */ rc = mutex_lock_interruptible(&obd->obd_dev_mutex); if (rc) return rc; if (obd->obd_osfs_age < max_age || - ((obd->obd_osfs.os_state & OS_STATE_SUM) && + ((obd->obd_osfs.os_state & OS_STATFS_SUM) && !(flags & OBD_STATFS_SUM))) { rc = OBP(obd, statfs)(env, exp, osfs, max_age, flags); } else { diff --git a/fs/lustre/llite/llite_lib.c b/fs/lustre/llite/llite_lib.c index 6cd9f7e..aad19a2 100644 --- a/fs/lustre/llite/llite_lib.c +++ b/fs/lustre/llite/llite_lib.c @@ -1924,7 +1924,7 @@ int ll_statfs_internal(struct ll_sb_info *sbi, struct obd_statfs *osfs, osfs->os_bavail, osfs->os_blocks, osfs->os_ffree, osfs->os_files); - if (osfs->os_state & OS_STATE_SUM) + if (osfs->os_state & OS_STATFS_SUM) goto out; rc = obd_statfs(NULL, sbi->ll_dt_exp, &obd_osfs, max_age, flags); diff --git a/fs/lustre/lmv/lmv_obd.c b/fs/lustre/lmv/lmv_obd.c index 203a17b..c5f21cd 100644 --- a/fs/lustre/lmv/lmv_obd.c +++ b/fs/lustre/lmv/lmv_obd.c @@ -1255,7 +1255,7 @@ static int lmv_statfs(const struct lu_env *env, struct obd_export *exp, goto out_free_temp; } - if (temp->os_state & OS_STATE_SUM || + if (temp->os_state & OS_STATFS_SUM || flags == OBD_STATFS_FOR_MDT0) { /* Reset to the last aggregated values * and don't sum with non-aggrated data. diff --git a/fs/lustre/ptlrpc/wiretest.c b/fs/lustre/ptlrpc/wiretest.c index 719d73b..74a4e940 100644 --- a/fs/lustre/ptlrpc/wiretest.c +++ b/fs/lustre/ptlrpc/wiretest.c @@ -1851,20 +1851,20 @@ void lustre_assert_wire_constants(void) (long long)(int)offsetof(struct obd_statfs, os_spare9)); LASSERTF((int)sizeof(((struct obd_statfs *)0)->os_spare9) == 4, "found %lld\n", (long long)(int)sizeof(((struct obd_statfs *)0)->os_spare9)); - LASSERTF(OS_STATE_DEGRADED == 0x00000001UL, "found %lld\n", - (long long)OS_STATE_DEGRADED); - LASSERTF(OS_STATE_READONLY == 0x00000002UL, "found %lld\n", - (long long)OS_STATE_READONLY); - LASSERTF(OS_STATE_NOPRECREATE == 0x00000004UL, "found %lld\n", - (long long)OS_STATE_NOPRECREATE); - LASSERTF(OS_STATE_ENOSPC == 0x00000020UL, "found %lld\n", - (long long)OS_STATE_ENOSPC); - LASSERTF(OS_STATE_ENOINO == 0x00000040UL, "found %lld\n", - (long long)OS_STATE_ENOINO); - LASSERTF(OS_STATE_SUM == 0x00000100UL, "found %lld\n", - (long long)OS_STATE_SUM); - LASSERTF(OS_STATE_NONROT == 0x00000200UL, "found %lld\n", - (long long)OS_STATE_NONROT); + LASSERTF(OS_STATFS_DEGRADED == 0x00000001UL, "found %lld\n", + (long long)OS_STATFS_DEGRADED); + LASSERTF(OS_STATFS_READONLY == 0x00000002UL, "found %lld\n", + (long long)OS_STATFS_READONLY); + LASSERTF(OS_STATFS_NOPRECREATE == 0x00000004UL, "found %lld\n", + (long long)OS_STATFS_NOPRECREATE); + LASSERTF(OS_STATFS_ENOSPC == 0x00000020UL, "found %lld\n", + (long long)OS_STATFS_ENOSPC); + LASSERTF(OS_STATFS_ENOINO == 0x00000040UL, "found %lld\n", + (long long)OS_STATFS_ENOINO); + LASSERTF(OS_STATFS_SUM == 0x00000100UL, "found %lld\n", + (long long)OS_STATFS_SUM); + LASSERTF(OS_STATFS_NONROT == 0x00000200UL, "found %lld\n", + (long long)OS_STATFS_NONROT); /* Checks for struct obd_ioobj */ LASSERTF((int)sizeof(struct obd_ioobj) == 24, "found %lld\n", diff --git a/include/uapi/linux/lustre/lustre_user.h b/include/uapi/linux/lustre/lustre_user.h index 004d373..121c064 100644 --- a/include/uapi/linux/lustre/lustre_user.h +++ b/include/uapi/linux/lustre/lustre_user.h @@ -105,15 +105,15 @@ #define LUSTRE_FIEMAP_FLAGS_COMPAT (FIEMAP_FLAG_SYNC | FIEMAP_FLAG_DEVICE_ORDER) enum obd_statfs_state { - OS_STATE_DEGRADED = 0x00000001, /**< RAID degraded/rebuilding */ - OS_STATE_READONLY = 0x00000002, /**< filesystem is read-only */ - OS_STATE_NOPRECREATE = 0x00000004, /**< no object precreation */ - OS_STATE_UNUSED1 = 0x00000008, /**< obsolete 1.6, was EROFS=30 */ - OS_STATE_UNUSED2 = 0x00000010, /**< obsolete 1.6, was EROFS=30 */ - OS_STATE_ENOSPC = 0x00000020, /**< not enough free space */ - OS_STATE_ENOINO = 0x00000040, /**< not enough inodes */ - OS_STATE_SUM = 0x00000100, /**< aggregated for all tagrets */ - OS_STATE_NONROT = 0x00000200, /**< non-rotational device */ + OS_STATFS_DEGRADED = 0x00000001, /**< RAID degraded/rebuilding */ + OS_STATFS_READONLY = 0x00000002, /**< filesystem is read-only */ + OS_STATFS_NOPRECREATE = 0x00000004, /**< no object precreation */ + OS_STATFS_UNUSED1 = 0x00000008, /**< obsolete 1.6, was EROFS=30 */ + OS_STATFS_UNUSED2 = 0x00000010, /**< obsolete 1.6, was EROFS=30 */ + OS_STATFS_ENOSPC = 0x00000020, /**< not enough free space */ + OS_STATFS_ENOINO = 0x00000040, /**< not enough inodes */ + OS_STATFS_SUM = 0x00000100, /**< aggregated for all tagrets */ + OS_STATFS_NONROT = 0x00000200, /**< non-rotational device */ }; struct obd_statfs { @@ -127,7 +127,7 @@ struct obd_statfs { __u32 os_bsize; /* block size in bytes for os_blocks */ __u32 os_namelen; /* maximum length of filename in bytes*/ __u64 os_maxbytes; /* maximum object size in bytes */ - __u32 os_state; /**< obd_statfs_state OS_STATE_* flag */ + __u32 os_state; /**< obd_statfs_state OS_STATFS_* flag */ __u32 os_fprecreated; /* objs available now to the caller * used in QoS code to find preferred OSTs */ From patchwork Sat Jun 13 16:27:15 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602911 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 37C91138C for ; Sat, 13 Jun 2020 16:28:27 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id 0ED282078A for ; Sat, 13 Jun 2020 16:28:27 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 0ED282078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id F222721F211; Sat, 13 Jun 2020 09:27:54 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 78DF521EBAC for ; Sat, 13 Jun 2020 09:27:28 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id B29116DF; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id AF8C7488; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:15 -0400 Message-Id: <1592065636-28333-20-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 19/20] lustre: mdc: allow setting max_mod_rpcs_in_flight larger X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Andreas Dilger Allow setting mdc.*.max_mod_rpcs_in_flight > mdc.*.max_rpcs_in_flight by increasing the latter value, rather than returning an error and telling the user to do that. This matches the similar behavior if mdc.*.max_rpcs_in_flight is reduced lower than max_mod_rpcs_in_flight. If there are multiple MDTs, the "mdc.*.max_mod_rpcs_in_flight" param may be set from e.g. the MDT0000 config log before MDT0001 is fully configured, catching MDT0001 with ocd_maxmodrpcs = 0 before the OCD from the MDT has been filled in, and incorrectly trigger an error. If seen during setup, allow ocd_maxmodrpcs = (max_rpcs_in_flight - 1), since this will be fixed up later if mdc.*.max_rpcs_in_flight is set smaller in the config log (if set larger it doesn't matter). WC-bug-id: https://jira.whamcloud.com/browse/LU-13503 Lustre-commit: 6d314902e6d19 ("LU-13503 mdc: allow setting max_mod_rpcs_in_flight larger") Signed-off-by: Andreas Dilger Signed-off-by: Jian Yu Reviewed-on: https://review.whamcloud.com/38455 Reviewed-by: John L. Hammond Reviewed-by: James Simmons Reviewed-by: Oleg Drokin Signed-off-by: James Simmons --- fs/lustre/obdclass/genops.c | 50 +++++++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 13 deletions(-) diff --git a/fs/lustre/obdclass/genops.c b/fs/lustre/obdclass/genops.c index 607f0d6..1647fe9 100644 --- a/fs/lustre/obdclass/genops.c +++ b/fs/lustre/obdclass/genops.c @@ -1324,7 +1324,6 @@ u32 obd_get_max_rpcs_in_flight(struct client_obd *cli) int obd_set_max_rpcs_in_flight(struct client_obd *cli, u32 max) { struct obd_request_slot_waiter *orsw; - const char *type_name; u32 old; int diff; int rc; @@ -1333,14 +1332,18 @@ int obd_set_max_rpcs_in_flight(struct client_obd *cli, u32 max) if (max > OBD_MAX_RIF_MAX || max < 1) return -ERANGE; - type_name = cli->cl_import->imp_obd->obd_type->typ_name; - if (!strcmp(type_name, LUSTRE_MDC_NAME)) { + CDEBUG(D_INFO, "%s: max = %hu max_mod = %u rif = %u\n", + cli->cl_import->imp_obd->obd_name, max, + cli->cl_max_mod_rpcs_in_flight, cli->cl_max_rpcs_in_flight); + + if (strcmp(cli->cl_import->imp_obd->obd_type->typ_name, + LUSTRE_MDC_NAME) == 0) { /* * adjust max_mod_rpcs_in_flight to ensure it is always * strictly lower that max_rpcs_in_flight */ if (max < 2) { - CERROR("%s: cannot set max_rpcs_in_flight to 1 because it must be higher than max_mod_rpcs_in_flight value\n", + CERROR("%s: cannot set mdc.*.max_rpcs_in_flight=1\n", cli->cl_import->imp_obd->obd_name); return -ERANGE; } @@ -1385,22 +1388,43 @@ int obd_set_max_mod_rpcs_in_flight(struct client_obd *cli, u16 max) if (max > OBD_MAX_RIF_MAX || max < 1) return -ERANGE; - /* cannot exceed or equal max_rpcs_in_flight */ + CDEBUG(D_INFO, "%s: max = %hu flags = %llx, max_mod = %u rif = %u\n", + cli->cl_import->imp_obd->obd_name, max, ocd->ocd_connect_flags, + ocd->ocd_maxmodrpcs, cli->cl_max_rpcs_in_flight); + + if (max == OBD_MAX_RIF_MAX) + max = OBD_MAX_RIF_MAX - 1; + + /* Cannot exceed or equal max_rpcs_in_flight. If we are asked to + * increase this value, also bump up max_rpcs_in_flight to match. + */ if (max >= cli->cl_max_rpcs_in_flight) { - CERROR("%s: can't set max_mod_rpcs_in_flight to a value (%hu) higher or equal to max_rpcs_in_flight value (%u)\n", - cli->cl_import->imp_obd->obd_name, - max, cli->cl_max_rpcs_in_flight); + CDEBUG(D_INFO, + "%s: increasing max_rpcs_in_flight=%hu to allow larger max_mod_rpcs_in_flight=%u\n", + cli->cl_import->imp_obd->obd_name, max + 1, max); return -ERANGE; } - /* cannot exceed max modify RPCs in flight supported by the server */ - ocd = &cli->cl_import->imp_connect_data; - if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS) + /* cannot exceed max modify RPCs in flight supported by the server, + * but verify ocd_connect_flags is at least initialized first. If + * not, allow it and fix value later in ptlrpc_connect_set_flags(). + */ + if (!ocd->ocd_connect_flags) { + maxmodrpcs = cli->cl_max_rpcs_in_flight - 1; + } else if (ocd->ocd_connect_flags & OBD_CONNECT_MULTIMODRPCS) { maxmodrpcs = ocd->ocd_maxmodrpcs; - else + if (maxmodrpcs == 0) { /* connection not finished yet */ + maxmodrpcs = cli->cl_max_rpcs_in_flight - 1; + CDEBUG(D_INFO, + "%s: partial connect, assume maxmodrpcs=%hu\n", + cli->cl_import->imp_obd->obd_name, maxmodrpcs); + } + } else { maxmodrpcs = 1; + } + if (max > maxmodrpcs) { - CERROR("%s: can't set max_mod_rpcs_in_flight to a value (%hu) higher than max_mod_rpcs_per_client value (%hu) returned by the server at connection\n", + CERROR("%s: can't set max_mod_rpcs_in_flight=%hu higher than ocd_maxmodrpcs=%hu returned by the server at connection\n", cli->cl_import->imp_obd->obd_name, max, maxmodrpcs); return -ERANGE; From patchwork Sat Jun 13 16:27:16 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11602913 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 486731392 for ; Sat, 13 Jun 2020 16:28:30 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id 30AE12078A for ; Sat, 13 Jun 2020 16:28:30 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org 30AE12078A Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 0497821F4B2; Sat, 13 Jun 2020 09:27:58 -0700 (PDT) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id CF99621EBAC for ; Sat, 13 Jun 2020 09:27:28 -0700 (PDT) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id B3F678E9; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id B2DEF47A; Sat, 13 Jun 2020 12:27:19 -0400 (EDT) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Sat, 13 Jun 2020 12:27:16 -0400 Message-Id: <1592065636-28333-21-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> References: <1592065636-28333-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 20/20] lnet: o2iblnd: 'Timed out tx' error message X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Sonia Sharma Fix the error message in kiblnd_check_txs_locked() to report the total RDMA time outstanding rather than the number of seconds past the deadline. This patch also adds time_on_activeq to struct kib_tx so the time spent by tx in internal queue and active queue can be tracked and reported. This would help in diagnosing the issue. WC-bug-id: https://jira.whamcloud.com/browse/LU-1742 Lustre-commit: 7308662efc02f ("LU-1742 o2iblnd: 'Timed out tx' error message") Signed-off-by: Sonia Sharma Reviewed-on: https://review.whamcloud.com/33235 Reviewed-by: Andreas Dilger Reviewed-by: Stephen Champion Reviewed-by: Oleg Drokin Signed-off-by: James Simmons --- net/lnet/klnds/o2iblnd/o2iblnd.h | 2 ++ net/lnet/klnds/o2iblnd/o2iblnd_cb.c | 26 ++++++++++++++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/net/lnet/klnds/o2iblnd/o2iblnd.h b/net/lnet/klnds/o2iblnd/o2iblnd.h index f60a69d..dc09e5e 100644 --- a/net/lnet/klnds/o2iblnd/o2iblnd.h +++ b/net/lnet/klnds/o2iblnd/o2iblnd.h @@ -541,6 +541,8 @@ struct kib_tx { /* transmit message */ bool tx_gaps; struct kib_fmr tx_fmr; /* FMR */ int tx_dmadir; /* dma direction */ + /* time when tx added on ibc_active_txs */ + ktime_t tx_on_activeq; }; struct kib_connvars { diff --git a/net/lnet/klnds/o2iblnd/o2iblnd_cb.c b/net/lnet/klnds/o2iblnd/o2iblnd_cb.c index 40e196d..f421cdf 100644 --- a/net/lnet/klnds/o2iblnd/o2iblnd_cb.c +++ b/net/lnet/klnds/o2iblnd/o2iblnd_cb.c @@ -821,6 +821,7 @@ static int kiblnd_map_tx(struct lnet_ni *ni, struct kib_tx *tx, */ tx->tx_sending++; list_add(&tx->tx_list, &conn->ibc_active_txs); + tx->tx_on_activeq = ktime_get(); /* I'm still holding ibc_lock! */ if (conn->ibc_state != IBLND_CONN_ESTABLISHED) { @@ -3169,6 +3170,8 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid, static int kiblnd_check_txs_locked(struct kib_conn *conn, struct list_head *txs) { + bool active_txs = strcmp(kiblnd_queue2str(conn, txs), + "active_txs") == 0; struct kib_tx *tx; list_for_each_entry(tx, txs, tx_list) { @@ -3179,13 +3182,28 @@ static int kiblnd_resolve_addr(struct rdma_cm_id *cmid, LASSERT(tx->tx_waiting || tx->tx_sending); } - if (ktime_compare(ktime_get(), tx->tx_deadline) >= 0) { - CERROR("Timed out tx: %s, %lld seconds\n", + if (ktime_compare(ktime_get(), tx->tx_deadline) < 0) + continue; + + if (!active_txs) { + CERROR("Timed out tx: %s, outstanding RDMA time: %lld sec\n", kiblnd_queue2str(conn, txs), + *kiblnd_tunables.kib_timeout + + (ktime_ms_delta(ktime_get(), + tx->tx_deadline) / MSEC_PER_SEC)); + } else { + CERROR("Timed out tx: %s, time in internal queue: %lld sec, time in active queue: %lld sec, outstanding RDMA time: %lld sec\n", + kiblnd_queue2str(conn, txs), + ktime_ms_delta(tx->tx_deadline, + tx->tx_on_activeq) / MSEC_PER_SEC, ktime_ms_delta(ktime_get(), - tx->tx_deadline) / MSEC_PER_SEC); - return 1; + tx->tx_on_activeq) / MSEC_PER_SEC, + *kiblnd_tunables.kib_timeout + + (ktime_ms_delta(ktime_get(), + tx->tx_deadline) / MSEC_PER_SEC)); } + + return 1; } return 0;