[RFC,18/18] ceph: Don't use data_pages

Message ID	20230804131327.2574082-19-dhowells@redhat.com (mailing list archive)
State	New, archived
Headers	show Return-Path: <ceph-devel-owner@vger.kernel.org> From: David Howells <dhowells@redhat.com> To: Ilya Dryomov <idryomov@gmail.com>, Xiubo Li <xiubli@redhat.com> Cc: David Howells <dhowells@redhat.com>, Jeff Layton <jlayton@kernel.org>, Dongsheng Yang <dongsheng.yang@easystack.cn>, ceph-devel@vger.kernel.org, linux-fsdevel@vger.kernel.org, linux-block@vger.kernel.org, linux-kernel@vger.kernel.org Subject: [RFC PATCH 18/18] ceph: Don't use data_pages Date: Fri, 4 Aug 2023 14:13:27 +0100 Message-ID: <20230804131327.2574082-19-dhowells@redhat.com> In-Reply-To: <20230804131327.2574082-1-dhowells@redhat.com> References: <20230804131327.2574082-1-dhowells@redhat.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Precedence: bulk
Series	ceph, rbd: Collapse all the I/O types down to something iov_iter-based \| expand [RFC,00/18] ceph, rbd: Collapse all the I/O types down to something iov_iter-based [RFC,01/18] iov_iter: Add function to see if buffer is all zeros [RFC,02/18] ceph: Rename alignment to offset [RFC,03/18] ceph: Add a new data container type, ceph_databuf [RFC,04/18] ceph: Convert ceph_mds_request::r_pagelist to a databuf [RFC,05/18] rbd: Use ceph_databuf for rbd_obj_read_sync() [RFC,06/18] ceph: Change ceph_osdc_call()'s reply to a ceph_databuf [RFC,07/18] ceph: Unexport osd_req_op_cls_request_data_pages() [RFC,08/18] ceph: Remove osd_req_op_cls_response_data_pages() [RFC,09/18] ceph: Convert notify_id_pages to a ceph_databuf [RFC,10/18] rbd: Switch from using bvec_iter to iov_iter [RFC,11/18] ceph: Remove bvec and bio data container types [RFC,12/18] ceph: Convert some page arrays to ceph_databuf [RFC,13/18] ceph: Convert users of ceph_pagelist to ceph_databuf [RFC,14/18] ceph: Remove ceph_pagelist [RFC,15/18] ceph: Convert ceph_osdc_notify() reply to ceph_databuf [RFC,16/18] ceph: Remove CEPH_OS_DATA_TYPE_PAGES and its attendant helpers [RFC,17/18] ceph: Remove CEPH_MSG_DATA_PAGES and its helpers [RFC,18/18] ceph: Don't use data_pages

Message ID

20230804131327.2574082-19-dhowells@redhat.com (mailing list archive)

State

New, archived

Headers

From: David Howells <dhowells@redhat.com>
To: Ilya Dryomov <idryomov@gmail.com>, Xiubo Li <xiubli@redhat.com>
Cc: David Howells <dhowells@redhat.com>,
        Jeff Layton <jlayton@kernel.org>,
        Dongsheng Yang <dongsheng.yang@easystack.cn>,
        ceph-devel@vger.kernel.org, linux-fsdevel@vger.kernel.org,
        linux-block@vger.kernel.org, linux-kernel@vger.kernel.org
Subject: [RFC PATCH 18/18] ceph: Don't use data_pages
Date: Fri,  4 Aug 2023 14:13:27 +0100
Message-ID: <20230804131327.2574082-19-dhowells@redhat.com>
In-Reply-To: <20230804131327.2574082-1-dhowells@redhat.com>
References: <20230804131327.2574082-1-dhowells@redhat.com>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
Precedence: bulk

Series

ceph, rbd: Collapse all the I/O types down to something iov_iter-based | expand

Commit Message

David Howells Aug. 4, 2023, 1:13 p.m. UTC

---
 fs/ceph/addr.c                  | 16 +++++-----------
 fs/ceph/file.c                  | 34 +++++++++++++++------------------
 include/linux/ceph/osd_client.h |  1 +
 net/ceph/osd_client.c           | 16 ++--------------
 4 files changed, 23 insertions(+), 44 deletions(-)

Comments

Xiubo Li Aug. 28, 2023, 1:32 a.m. UTC | #1

David,

Please let me know if this is ready and I will review it carefully and 
have a test for them. Haven't got a time to go through this yet.

Thanks

- Xiubo

On 8/4/23 21:13, David Howells wrote:
> ---
>   fs/ceph/addr.c                  | 16 +++++-----------
>   fs/ceph/file.c                  | 34 +++++++++++++++------------------
>   include/linux/ceph/osd_client.h |  1 +
>   net/ceph/osd_client.c           | 16 ++--------------
>   4 files changed, 23 insertions(+), 44 deletions(-)
>
> diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c
> index 7571606cf61f..7557f4a85ef0 100644
> --- a/fs/ceph/addr.c
> +++ b/fs/ceph/addr.c
> @@ -277,11 +277,6 @@ static void finish_netfs_read(struct ceph_osd_request *req)
>   		}
>   	}
>   
> -	if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES) {
> -		ceph_put_page_vector(osd_data->pages,
> -				     calc_pages_for(osd_data->offset,
> -					osd_data->length), false);
> -	}
>   	netfs_subreq_terminated(subreq, err, false);
>   	iput(req->r_inode);
>   	ceph_dec_osd_stopping_blocker(fsc->mdsc);
> @@ -2007,7 +2002,7 @@ static int __ceph_pool_perm_get(struct ceph_inode_info *ci,
>   	struct ceph_osd_request *rd_req = NULL, *wr_req = NULL;
>   	struct rb_node **p, *parent;
>   	struct ceph_pool_perm *perm;
> -	struct page **pages;
> +	struct ceph_databuf *dbuf;
>   	size_t pool_ns_len;
>   	int err = 0, err2 = 0, have = 0;
>   
> @@ -2107,14 +2102,13 @@ static int __ceph_pool_perm_get(struct ceph_inode_info *ci,
>   		goto out_unlock;
>   
>   	/* one page should be large enough for STAT data */
> -	pages = ceph_alloc_page_vector(1, GFP_KERNEL);
> -	if (IS_ERR(pages)) {
> -		err = PTR_ERR(pages);
> +	dbuf = ceph_databuf_alloc(1, PAGE_SIZE, GFP_KERNEL);
> +	if (!dbuf) {
> +		err = -ENOMEM;
>   		goto out_unlock;
>   	}
>   
> -	osd_req_op_raw_data_in_pages(rd_req, 0, pages, PAGE_SIZE,
> -				     0, false, true);
> +	osd_req_op_raw_data_in_databuf(rd_req, 0, dbuf);
>   	ceph_osdc_start_request(&fsc->client->osdc, rd_req);
>   
>   	wr_req->r_mtime = ci->netfs.inode.i_mtime;
> diff --git a/fs/ceph/file.c b/fs/ceph/file.c
> index 5d16469a3690..caf557187ca8 100644
> --- a/fs/ceph/file.c
> +++ b/fs/ceph/file.c
> @@ -977,6 +977,7 @@ ssize_t __ceph_sync_read(struct inode *inode, loff_t *ki_pos,
>   	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
>   	struct ceph_client *cl = fsc->client;
>   	struct ceph_osd_client *osdc = &fsc->client->osdc;
> +	struct ceph_databuf *dbuf;
>   	ssize_t ret;
>   	u64 off = *ki_pos;
>   	u64 len = iov_iter_count(to);
> @@ -1041,16 +1042,14 @@ ssize_t __ceph_sync_read(struct inode *inode, loff_t *ki_pos,
>   
>   		num_pages = calc_pages_for(read_off, read_len);
>   		page_off = offset_in_page(off);
> -		pages = ceph_alloc_page_vector(num_pages, GFP_KERNEL);
> -		if (IS_ERR(pages)) {
> +		dbuf = ceph_databuf_alloc(num_pages, read_len, GFP_KERNEL);
> +		if (!dbuf) {
>   			ceph_osdc_put_request(req);
> -			ret = PTR_ERR(pages);
> +			ret = -ENOMEM;
>   			break;
>   		}
>   
> -		osd_req_op_extent_osd_data_pages(req, 0, pages, read_len,
> -						 offset_in_page(read_off),
> -						 false, false);
> +		osd_req_op_extent_osd_databuf(req, 0, dbuf);
>   
>   		op = &req->r_ops[0];
>   		if (sparse) {
> @@ -1137,7 +1136,7 @@ ssize_t __ceph_sync_read(struct inode *inode, loff_t *ki_pos,
>   				break;
>   			}
>   		}
> -		ceph_release_page_vector(pages, num_pages);
> +		ceph_databuf_release(dbuf);
>   
>   		if (ret < 0) {
>   			if (ret == -EBLOCKLISTED)
> @@ -1625,7 +1624,7 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
>   	struct ceph_client *cl = fsc->client;
>   	struct ceph_osd_client *osdc = &fsc->client->osdc;
>   	struct ceph_osd_request *req;
> -	struct page **pages;
> +	struct ceph_databuf *dbuf = NULL;
>   	u64 len;
>   	int num_pages;
>   	int written = 0;
> @@ -1691,9 +1690,9 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
>   		 * an array of pagecache pages.
>   		 */
>   		num_pages = calc_pages_for(write_pos, write_len);
> -		pages = ceph_alloc_page_vector(num_pages, GFP_KERNEL);
> -		if (IS_ERR(pages)) {
> -			ret = PTR_ERR(pages);
> +		dbuf = ceph_databuf_alloc(num_pages, num_pages * PAGE_SIZE, GFP_KERNEL);
> +		if (!dbuf) {
> +			ret = -ENOMEM;
>   			break;
>   		}
>   
> @@ -1722,7 +1721,6 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
>   					NULL, ci->i_truncate_seq,
>   					ci->i_truncate_size, false);
>   			if (IS_ERR(req)) {
> -				ceph_release_page_vector(pages, num_pages);
>   				ret = PTR_ERR(req);
>   				break;
>   			}
> @@ -1730,7 +1728,6 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
>   			/* Something is misaligned! */
>   			if (read_len != CEPH_FSCRYPT_BLOCK_SIZE) {
>   				ceph_osdc_put_request(req);
> -				ceph_release_page_vector(pages, num_pages);
>   				ret = -EIO;
>   				break;
>   			}
> @@ -1739,15 +1736,14 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
>   			op = &req->r_ops[0];
>   
>   			if (first) {
> -				osd_req_op_extent_osd_data_pages(req, 0, pages,
> -							 CEPH_FSCRYPT_BLOCK_SIZE,
> -							 offset_in_page(first_pos),
> -							 false, false);
> +				iov_iter_advance(&dbuf->iter, offset_in_page(first_pos));
> +				osd_req_op_extent_osd_databuf(req, 0, dbuf,
> +							 CEPH_FSCRYPT_BLOCK_SIZE);
> +				dbuf = NULL;
>   				/* We only expect a single extent here */
>   				ret = __ceph_alloc_sparse_ext_map(op, 1);
>   				if (ret) {
>   					ceph_osdc_put_request(req);
> -					ceph_release_page_vector(pages, num_pages);
>   					break;
>   				}
>   			}
> @@ -1766,7 +1762,6 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
>   				ret = __ceph_alloc_sparse_ext_map(op, 1);
>   				if (ret) {
>   					ceph_osdc_put_request(req);
> -					ceph_release_page_vector(pages, num_pages);
>   					break;
>   				}
>   
> @@ -1998,6 +1993,7 @@ ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
>   
>   	}
>   
> +	ceph_databuf_release(dbuf);
>   	if (ret != -EOLDSNAPC && written > 0) {
>   		ret = written;
>   		iocb->ki_pos = pos;
> diff --git a/include/linux/ceph/osd_client.h b/include/linux/ceph/osd_client.h
> index 1a1137787487..c26a7866695a 100644
> --- a/include/linux/ceph/osd_client.h
> +++ b/include/linux/ceph/osd_client.h
> @@ -110,6 +110,7 @@ struct ceph_osd_data {
>   	enum ceph_osd_data_type	type;
>   	struct ceph_databuf	*dbuf;
>   	struct iov_iter		iter;
> +	size_t			length;
>   };
>   
>   struct ceph_osd_req_op {
> diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
> index 6fb78ae14f03..95daf4cdb07b 100644
> --- a/net/ceph/osd_client.c
> +++ b/net/ceph/osd_client.c
> @@ -127,6 +127,7 @@ static void ceph_osd_iter_init(struct ceph_osd_data *osd_data,
>   {
>   	osd_data->type = CEPH_OSD_DATA_TYPE_ITER;
>   	osd_data->iter = *iter;
> +	osd_data->length = iter->count;
>   }
>   
>   /*
> @@ -239,19 +240,6 @@ void osd_req_op_cls_response_databuf(struct ceph_osd_request *osd_req,
>   }
>   EXPORT_SYMBOL(osd_req_op_cls_response_databuf);
>   
> -static u64 ceph_osd_data_length(struct ceph_osd_data *osd_data)
> -{
> -	switch (osd_data->type) {
> -	case CEPH_OSD_DATA_TYPE_NONE:
> -		return 0;
> -	case CEPH_OSD_DATA_TYPE_ITER:
> -		return iov_iter_count(&osd_data->iter);
> -	default:
> -		WARN(true, "unrecognized data type %d\n", (int)osd_data->type);
> -		return 0;
> -	}
> -}
> -
>   static void ceph_osd_data_release(struct ceph_osd_data *osd_data)
>   {
>   	ceph_osd_data_init(osd_data);
> @@ -4475,7 +4463,7 @@ static void handle_watch_notify(struct ceph_osd_client *osdc,
>   			if (data) {
>   				if (lreq->reply) {
>   					WARN_ON(data->type !=
> -							CEPH_MSG_DATA_PAGES);
> +							CEPH_MSG_DATA_DATABUF);
>   					*lreq->preply_pages = data->pages;
>   					*lreq->preply_len = data->length;
>   					data->own_pages = false;
>

diff --git a/fs/ceph/addr.c b/fs/ceph/addr.c
index 7571606cf61f..7557f4a85ef0 100644
--- a/fs/ceph/addr.c
+++ b/fs/ceph/addr.c
@@ -277,11 +277,6 @@  static void finish_netfs_read(struct ceph_osd_request *req)
 		}
 	}
 
-	if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES) {
-		ceph_put_page_vector(osd_data->pages,
-				     calc_pages_for(osd_data->offset,
-					osd_data->length), false);
-	}
 	netfs_subreq_terminated(subreq, err, false);
 	iput(req->r_inode);
 	ceph_dec_osd_stopping_blocker(fsc->mdsc);
@@ -2007,7 +2002,7 @@  static int __ceph_pool_perm_get(struct ceph_inode_info *ci,
 	struct ceph_osd_request *rd_req = NULL, *wr_req = NULL;
 	struct rb_node **p, *parent;
 	struct ceph_pool_perm *perm;
-	struct page **pages;
+	struct ceph_databuf *dbuf;
 	size_t pool_ns_len;
 	int err = 0, err2 = 0, have = 0;
 
@@ -2107,14 +2102,13 @@  static int __ceph_pool_perm_get(struct ceph_inode_info *ci,
 		goto out_unlock;
 
 	/* one page should be large enough for STAT data */
-	pages = ceph_alloc_page_vector(1, GFP_KERNEL);
-	if (IS_ERR(pages)) {
-		err = PTR_ERR(pages);
+	dbuf = ceph_databuf_alloc(1, PAGE_SIZE, GFP_KERNEL);
+	if (!dbuf) {
+		err = -ENOMEM;
 		goto out_unlock;
 	}
 
-	osd_req_op_raw_data_in_pages(rd_req, 0, pages, PAGE_SIZE,
-				     0, false, true);
+	osd_req_op_raw_data_in_databuf(rd_req, 0, dbuf);
 	ceph_osdc_start_request(&fsc->client->osdc, rd_req);
 
 	wr_req->r_mtime = ci->netfs.inode.i_mtime;
diff --git a/fs/ceph/file.c b/fs/ceph/file.c
index 5d16469a3690..caf557187ca8 100644
--- a/fs/ceph/file.c
+++ b/fs/ceph/file.c
@@ -977,6 +977,7 @@  ssize_t __ceph_sync_read(struct inode *inode, loff_t *ki_pos,
 	struct ceph_fs_client *fsc = ceph_inode_to_fs_client(inode);
 	struct ceph_client *cl = fsc->client;
 	struct ceph_osd_client *osdc = &fsc->client->osdc;
+	struct ceph_databuf *dbuf;
 	ssize_t ret;
 	u64 off = *ki_pos;
 	u64 len = iov_iter_count(to);
@@ -1041,16 +1042,14 @@  ssize_t __ceph_sync_read(struct inode *inode, loff_t *ki_pos,
 
 		num_pages = calc_pages_for(read_off, read_len);
 		page_off = offset_in_page(off);
-		pages = ceph_alloc_page_vector(num_pages, GFP_KERNEL);
-		if (IS_ERR(pages)) {
+		dbuf = ceph_databuf_alloc(num_pages, read_len, GFP_KERNEL);
+		if (!dbuf) {
 			ceph_osdc_put_request(req);
-			ret = PTR_ERR(pages);
+			ret = -ENOMEM;
 			break;
 		}
 
-		osd_req_op_extent_osd_data_pages(req, 0, pages, read_len,
-						 offset_in_page(read_off),
-						 false, false);
+		osd_req_op_extent_osd_databuf(req, 0, dbuf);
 
 		op = &req->r_ops[0];
 		if (sparse) {
@@ -1137,7 +1136,7 @@  ssize_t __ceph_sync_read(struct inode *inode, loff_t *ki_pos,
 				break;
 			}
 		}
-		ceph_release_page_vector(pages, num_pages);
+		ceph_databuf_release(dbuf);
 
 		if (ret < 0) {
 			if (ret == -EBLOCKLISTED)
@@ -1625,7 +1624,7 @@  ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
 	struct ceph_client *cl = fsc->client;
 	struct ceph_osd_client *osdc = &fsc->client->osdc;
 	struct ceph_osd_request *req;
-	struct page **pages;
+	struct ceph_databuf *dbuf = NULL;
 	u64 len;
 	int num_pages;
 	int written = 0;
@@ -1691,9 +1690,9 @@  ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
 		 * an array of pagecache pages.
 		 */
 		num_pages = calc_pages_for(write_pos, write_len);
-		pages = ceph_alloc_page_vector(num_pages, GFP_KERNEL);
-		if (IS_ERR(pages)) {
-			ret = PTR_ERR(pages);
+		dbuf = ceph_databuf_alloc(num_pages, num_pages * PAGE_SIZE, GFP_KERNEL);
+		if (!dbuf) {
+			ret = -ENOMEM;
 			break;
 		}
 
@@ -1722,7 +1721,6 @@  ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
 					NULL, ci->i_truncate_seq,
 					ci->i_truncate_size, false);
 			if (IS_ERR(req)) {
-				ceph_release_page_vector(pages, num_pages);
 				ret = PTR_ERR(req);
 				break;
 			}
@@ -1730,7 +1728,6 @@  ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
 			/* Something is misaligned! */
 			if (read_len != CEPH_FSCRYPT_BLOCK_SIZE) {
 				ceph_osdc_put_request(req);
-				ceph_release_page_vector(pages, num_pages);
 				ret = -EIO;
 				break;
 			}
@@ -1739,15 +1736,14 @@  ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
 			op = &req->r_ops[0];
 
 			if (first) {
-				osd_req_op_extent_osd_data_pages(req, 0, pages,
-							 CEPH_FSCRYPT_BLOCK_SIZE,
-							 offset_in_page(first_pos),
-							 false, false);
+				iov_iter_advance(&dbuf->iter, offset_in_page(first_pos));
+				osd_req_op_extent_osd_databuf(req, 0, dbuf,
+							 CEPH_FSCRYPT_BLOCK_SIZE);
+				dbuf = NULL;
 				/* We only expect a single extent here */
 				ret = __ceph_alloc_sparse_ext_map(op, 1);
 				if (ret) {
 					ceph_osdc_put_request(req);
-					ceph_release_page_vector(pages, num_pages);
 					break;
 				}
 			}
@@ -1766,7 +1762,6 @@  ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
 				ret = __ceph_alloc_sparse_ext_map(op, 1);
 				if (ret) {
 					ceph_osdc_put_request(req);
-					ceph_release_page_vector(pages, num_pages);
 					break;
 				}
 
@@ -1998,6 +1993,7 @@  ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
 
 	}
 
+	ceph_databuf_release(dbuf);
 	if (ret != -EOLDSNAPC && written > 0) {
 		ret = written;
 		iocb->ki_pos = pos;
diff --git a/include/linux/ceph/osd_client.h b/include/linux/ceph/osd_client.h
index 1a1137787487..c26a7866695a 100644
--- a/include/linux/ceph/osd_client.h
+++ b/include/linux/ceph/osd_client.h
@@ -110,6 +110,7 @@  struct ceph_osd_data {
 	enum ceph_osd_data_type	type;
 	struct ceph_databuf	*dbuf;
 	struct iov_iter		iter;
+	size_t			length;
 };
 
 struct ceph_osd_req_op {
diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c
index 6fb78ae14f03..95daf4cdb07b 100644
--- a/net/ceph/osd_client.c
+++ b/net/ceph/osd_client.c
@@ -127,6 +127,7 @@  static void ceph_osd_iter_init(struct ceph_osd_data *osd_data,
 {
 	osd_data->type = CEPH_OSD_DATA_TYPE_ITER;
 	osd_data->iter = *iter;
+	osd_data->length = iter->count;
 }
 
 /*
@@ -239,19 +240,6 @@  void osd_req_op_cls_response_databuf(struct ceph_osd_request *osd_req,
 }
 EXPORT_SYMBOL(osd_req_op_cls_response_databuf);
 
-static u64 ceph_osd_data_length(struct ceph_osd_data *osd_data)
-{
-	switch (osd_data->type) {
-	case CEPH_OSD_DATA_TYPE_NONE:
-		return 0;
-	case CEPH_OSD_DATA_TYPE_ITER:
-		return iov_iter_count(&osd_data->iter);
-	default:
-		WARN(true, "unrecognized data type %d\n", (int)osd_data->type);
-		return 0;
-	}
-}
-
 static void ceph_osd_data_release(struct ceph_osd_data *osd_data)
 {
 	ceph_osd_data_init(osd_data);
@@ -4475,7 +4463,7 @@  static void handle_watch_notify(struct ceph_osd_client *osdc,
 			if (data) {
 				if (lreq->reply) {
 					WARN_ON(data->type !=
-							CEPH_MSG_DATA_PAGES);
+							CEPH_MSG_DATA_DATABUF);
 					*lreq->preply_pages = data->pages;
 					*lreq->preply_len = data->length;
 					data->own_pages = false;

[RFC,18/18] ceph: Don't use data_pages

Commit Message

Comments

Patch