From patchwork Thu Feb 27 21:08:56 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: James Simmons X-Patchwork-Id: 11409771 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id 0DD69138D for ; Thu, 27 Feb 2020 21:21:53 +0000 (UTC) Received: from pdx1-mailman02.dreamhost.com (pdx1-mailman02.dreamhost.com [64.90.62.194]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by mail.kernel.org (Postfix) with ESMTPS id EA2A5246A1 for ; Thu, 27 Feb 2020 21:21:52 +0000 (UTC) DMARC-Filter: OpenDMARC Filter v1.3.2 mail.kernel.org EA2A5246A1 Authentication-Results: mail.kernel.org; dmarc=none (p=none dis=none) header.from=infradead.org Authentication-Results: mail.kernel.org; spf=none smtp.mailfrom=lustre-devel-bounces@lists.lustre.org Received: from pdx1-mailman02.dreamhost.com (localhost [IPv6:::1]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 0E59221FEB4; Thu, 27 Feb 2020 13:20:36 -0800 (PST) X-Original-To: lustre-devel@lists.lustre.org Delivered-To: lustre-devel-lustre.org@pdx1-mailman02.dreamhost.com Received: from smtp3.ccs.ornl.gov (smtp3.ccs.ornl.gov [160.91.203.39]) by pdx1-mailman02.dreamhost.com (Postfix) with ESMTP id 2666521FAF6 for ; Thu, 27 Feb 2020 13:18:36 -0800 (PST) Received: from star.ccs.ornl.gov (star.ccs.ornl.gov [160.91.202.134]) by smtp3.ccs.ornl.gov (Postfix) with ESMTP id 08891E3F; Thu, 27 Feb 2020 16:18:14 -0500 (EST) Received: by star.ccs.ornl.gov (Postfix, from userid 2004) id 072BA46D; Thu, 27 Feb 2020 16:18:14 -0500 (EST) From: James Simmons To: Andreas Dilger , Oleg Drokin , NeilBrown Date: Thu, 27 Feb 2020 16:08:56 -0500 Message-Id: <1582838290-17243-69-git-send-email-jsimmons@infradead.org> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1582838290-17243-1-git-send-email-jsimmons@infradead.org> References: <1582838290-17243-1-git-send-email-jsimmons@infradead.org> Subject: [lustre-devel] [PATCH 068/622] lustre: osc: depart grant shrinking from pinger X-BeenThere: lustre-devel@lists.lustre.org X-Mailman-Version: 2.1.23 Precedence: list List-Id: "For discussing Lustre software development." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Cc: Bobi Jam , Lustre Development List MIME-Version: 1.0 Errors-To: lustre-devel-bounces@lists.lustre.org Sender: "lustre-devel" From: Bobi Jam * Removing grant shrinking code outside of pinger, use a workqueue to handle grant shrinking timer. * Enable OSC grant shrinking by default. bugzilla: 19507 WC-bug-id: https://jira.whamcloud.com/browse/LU-8708 Lustre-commit: fc915a43786e ("LU-8708 osc: depart grant shrinking from pinger") Signed-off-by: Bobi Jam Reviewed-on: https://review.whamcloud.com/23202 Reviewed-by: Hongchao Zhang Reviewed-by: Andreas Dilger Reviewed-by: James Simmons Signed-off-by: James Simmons --- fs/lustre/ldlm/ldlm_lib.c | 1 + fs/lustre/llite/llite_lib.c | 2 +- fs/lustre/osc/osc_request.c | 155 ++++++++++++++++++++++++++++++-------------- 3 files changed, 110 insertions(+), 48 deletions(-) diff --git a/fs/lustre/ldlm/ldlm_lib.c b/fs/lustre/ldlm/ldlm_lib.c index 2c0fad3..838ddb3 100644 --- a/fs/lustre/ldlm/ldlm_lib.c +++ b/fs/lustre/ldlm/ldlm_lib.c @@ -349,6 +349,7 @@ int client_obd_setup(struct obd_device *obddev, struct lustre_cfg *lcfg) spin_lock_init(&cli->cl_lru_list_lock); atomic_long_set(&cli->cl_unstable_count, 0); INIT_LIST_HEAD(&cli->cl_shrink_list); + INIT_LIST_HEAD(&cli->cl_grant_chain); INIT_LIST_HEAD(&cli->cl_flight_waiters); cli->cl_rpcs_in_flight = 0; diff --git a/fs/lustre/llite/llite_lib.c b/fs/lustre/llite/llite_lib.c index 0844318..56624e8 100644 --- a/fs/lustre/llite/llite_lib.c +++ b/fs/lustre/llite/llite_lib.c @@ -399,7 +399,7 @@ static int client_common_fill_super(struct super_block *sb, char *md, char *dt) OBD_CONNECT_LAYOUTLOCK | OBD_CONNECT_PINGLESS | OBD_CONNECT_LFSCK | OBD_CONNECT_BULK_MBITS | OBD_CONNECT_SHORTIO | - OBD_CONNECT_FLAGS2; + OBD_CONNECT_FLAGS2 | OBD_CONNECT_GRANT_SHRINK; /* The client currently advertises support for OBD_CONNECT_LOCKAHEAD_OLD * so it can interoperate with an older version of lockahead which was diff --git a/fs/lustre/osc/osc_request.c b/fs/lustre/osc/osc_request.c index e341fcc..1a9ed8d 100644 --- a/fs/lustre/osc/osc_request.c +++ b/fs/lustre/osc/osc_request.c @@ -33,6 +33,7 @@ #define DEBUG_SUBSYSTEM S_OSC +#include #include #include #include @@ -721,6 +722,16 @@ static void osc_update_grant(struct client_obd *cli, struct ost_body *body) } } +/** + * grant thread data for shrinking space. + */ +struct grant_thread_data { + struct list_head gtd_clients; + struct mutex gtd_mutex; + unsigned long gtd_stopped:1; +}; +static struct grant_thread_data client_gtd; + static int osc_shrink_grant_interpret(const struct lu_env *env, struct ptlrpc_request *req, void *aa, int rc) @@ -823,6 +834,9 @@ static int osc_should_shrink_grant(struct client_obd *client) { time64_t next_shrink = client->cl_next_shrink_grant; + if (!client->cl_import) + return 0; + if ((client->cl_import->imp_connect_data.ocd_connect_flags & OBD_CONNECT_GRANT_SHRINK) == 0) return 0; @@ -843,38 +857,83 @@ static int osc_should_shrink_grant(struct client_obd *client) return 0; } -static int osc_grant_shrink_grant_cb(struct timeout_item *item, void *data) -{ - struct client_obd *client; +#define GRANT_SHRINK_RPC_BATCH 100 + +static void osc_grant_work_handler(struct work_struct *data); +static DECLARE_DELAYED_WORK(work, osc_grant_work_handler); - list_for_each_entry(client, &item->ti_obd_list, cl_grant_shrink_list) { - if (osc_should_shrink_grant(client)) - osc_shrink_grant(client); +static void osc_grant_work_handler(struct work_struct *data) +{ + struct client_obd *cli; + int rpc_sent; + bool init_next_shrink = true; + time64_t next_shrink = ktime_get_seconds() + GRANT_SHRINK_INTERVAL; + + rpc_sent = 0; + mutex_lock(&client_gtd.gtd_mutex); + list_for_each_entry(cli, &client_gtd.gtd_clients, + cl_grant_chain) { + if (++rpc_sent < GRANT_SHRINK_RPC_BATCH && + osc_should_shrink_grant(cli)) + osc_shrink_grant(cli); + + if (!init_next_shrink) { + if (cli->cl_next_shrink_grant < next_shrink && + cli->cl_next_shrink_grant > ktime_get_seconds()) + next_shrink = cli->cl_next_shrink_grant; + } else { + init_next_shrink = false; + next_shrink = cli->cl_next_shrink_grant; + } } - return 0; + mutex_unlock(&client_gtd.gtd_mutex); + + if (client_gtd.gtd_stopped == 1) + return; + + if (next_shrink > ktime_get_seconds()) + schedule_delayed_work(&work, msecs_to_jiffies( + (next_shrink - ktime_get_seconds()) * + MSEC_PER_SEC)); + else + schedule_work(&work.work); } -static int osc_add_shrink_grant(struct client_obd *client) +/** + * Start grant thread for returing grant to server for idle clients. + */ +static int osc_start_grant_work(void) { - int rc; + client_gtd.gtd_stopped = 0; + mutex_init(&client_gtd.gtd_mutex); + INIT_LIST_HEAD(&client_gtd.gtd_clients); + + schedule_work(&work.work); - rc = ptlrpc_add_timeout_client(client->cl_grant_shrink_interval, - TIMEOUT_GRANT, - osc_grant_shrink_grant_cb, NULL, - &client->cl_grant_shrink_list); - if (rc) { - CERROR("add grant client %s error %d\n", cli_name(client), rc); - return rc; - } - CDEBUG(D_CACHE, "add grant client %s\n", cli_name(client)); - osc_update_next_shrink(client); return 0; } -static int osc_del_shrink_grant(struct client_obd *client) +static void osc_stop_grant_work(void) +{ + client_gtd.gtd_stopped = 1; + cancel_delayed_work_sync(&work); +} + +static void osc_add_grant_list(struct client_obd *client) { - return ptlrpc_del_timeout_client(&client->cl_grant_shrink_list, - TIMEOUT_GRANT); + mutex_lock(&client_gtd.gtd_mutex); + list_add(&client->cl_grant_chain, &client_gtd.gtd_clients); + mutex_unlock(&client_gtd.gtd_mutex); +} + +static void osc_del_grant_list(struct client_obd *client) +{ + if (list_empty(&client->cl_grant_chain)) + return; + + mutex_lock(&client_gtd.gtd_mutex); + list_del_init(&client->cl_grant_chain); + mutex_unlock(&client_gtd.gtd_mutex); } void osc_init_grant(struct client_obd *cli, struct obd_connect_data *ocd) @@ -929,9 +988,8 @@ void osc_init_grant(struct client_obd *cli, struct obd_connect_data *ocd) cli_name(cli), cli->cl_avail_grant, cli->cl_lost_grant, cli->cl_chunkbits, cli->cl_max_extent_pages); - if (ocd->ocd_connect_flags & OBD_CONNECT_GRANT_SHRINK && - list_empty(&cli->cl_grant_shrink_list)) - osc_add_shrink_grant(cli); + if (OCD_HAS_FLAG(ocd, GRANT_SHRINK) && list_empty(&cli->cl_grant_chain)) + osc_add_grant_list(cli); } EXPORT_SYMBOL(osc_init_grant); @@ -2971,15 +3029,12 @@ int osc_disconnect(struct obd_export *exp) * osc_disconnect * del_shrink_grant * ptlrpc_connect_interrupt - * init_grant_shrink + * osc_init_grant * add this client to shrink list - * cleanup_osc - * Bang! pinger trigger the shrink. - * So the osc should be disconnected from the shrink list, after we - * are sure the import has been destroyed. BUG18662 + * cleanup_osc + * Bang! grant shrink thread trigger the shrink. BUG18662 */ - if (!obd->u.cli.cl_import) - osc_del_shrink_grant(&obd->u.cli); + osc_del_grant_list(&obd->u.cli); return rc; } EXPORT_SYMBOL(osc_disconnect); @@ -3159,8 +3214,8 @@ int osc_setup_common(struct obd_device *obd, struct lustre_cfg *lcfg) goto out_ptlrpcd_work; cli->cl_grant_shrink_interval = GRANT_SHRINK_INTERVAL; + osc_update_next_shrink(cli); - INIT_LIST_HEAD(&cli->cl_grant_shrink_list); return 0; out_ptlrpcd_work: @@ -3210,7 +3265,6 @@ int osc_setup(struct obd_device *obd, struct lustre_cfg *lcfg) atomic_add(added, &osc_pool_req_count); } - INIT_LIST_HEAD(&cli->cl_grant_shrink_list); ns_register_cancel(obd->obd_namespace, osc_cancel_weight); spin_lock(&osc_shrink_lock); @@ -3356,14 +3410,19 @@ static int __init osc_init(void) if (rc) return rc; + rc = class_register_type(&osc_obd_ops, NULL, + LUSTRE_OSC_NAME, &osc_device_type); + if (rc) + goto out_kmem; + rc = register_shrinker(&osc_cache_shrinker); if (rc) - goto err; + goto out_type; /* This is obviously too much memory, only prevent overflow here */ if (osc_reqpool_mem_max >= 1 << 12 || osc_reqpool_mem_max == 0) { rc = -EINVAL; - goto err; + goto out_shrinker; } reqpool_size = osc_reqpool_mem_max << 20; @@ -3383,29 +3442,31 @@ static int __init osc_init(void) atomic_set(&osc_pool_req_count, 0); osc_rq_pool = ptlrpc_init_rq_pool(0, OST_MAXREQSIZE, ptlrpc_add_rqs_to_pool); + if (!osc_rq_pool) { + rc = -ENOMEM; + goto out_shrinker; + } - rc = -ENOMEM; - - if (!osc_rq_pool) - goto err; - - rc = class_register_type(&osc_obd_ops, NULL, - LUSTRE_OSC_NAME, &osc_device_type); + rc = osc_start_grant_work(); if (rc) - goto err; + goto out_req_pool; return rc; -err: - if (osc_rq_pool) - ptlrpc_free_rq_pool(osc_rq_pool); +out_req_pool: + ptlrpc_free_rq_pool(osc_rq_pool); +out_type: + class_unregister_type(LUSTRE_OSC_NAME); +out_shrinker: unregister_shrinker(&osc_cache_shrinker); +out_kmem: lu_kmem_fini(osc_caches); return rc; } static void /*__exit*/ osc_exit(void) { + osc_stop_grant_work(); unregister_shrinker(&osc_cache_shrinker); class_unregister_type(LUSTRE_OSC_NAME); lu_kmem_fini(osc_caches);