From patchwork Wed Jul 8 22:26:35 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Matthew Wilcox X-Patchwork-Id: 11652857 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id EC8E360D for ; Wed, 8 Jul 2020 22:26:57 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id D1A452078A for ; Wed, 8 Jul 2020 22:26:57 +0000 (UTC) Authentication-Results: mail.kernel.org; dkim=fail reason="signature verification failed" (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="J6R1jXtg" Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1725903AbgGHW04 (ORCPT ); Wed, 8 Jul 2020 18:26:56 -0400 Received: from casper.infradead.org ([90.155.50.34]:45766 "EHLO casper.infradead.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1726215AbgGHW0w (ORCPT ); Wed, 8 Jul 2020 18:26:52 -0400 DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Transfer-Encoding:MIME-Version: References:In-Reply-To:Message-Id:Date:Subject:Cc:To:From:Sender:Reply-To: Content-Type:Content-ID:Content-Description; bh=F0tAChMmj4QXzdRFsnSO800dOqSabaCWqBgONS8U2jg=; b=J6R1jXtgY3K0k0AR/u0bAhO46I 7ZqbuJtTxnkJWg13JiFg/ClCk9FUMIBIEuk9Z261nBkbkjPQn6CoFHSC78oK2fXEjqwKwiPqYDdPL 58KmGk/TugulXejmGYpU5PuzfyxRHy+hYm0RfmeRMSlTAqv304qcVrKtibghjewpDeyrt0vSBklFq e9KF38CHYRPDFhHrsQzfGVgB1mqdh2pEk69NY2cgmurC8POw+QpRlvR7B5j2om5pcvHWT2h8v4zwM 12O7KGxbC7yRWM/0QSQi6b9/pkYn4swbgVaMmkoRW3vofKtcXLr0TDoq2KxqFrG8Q2vjdEwiZOuEC F70VfWAQ==; Received: from willy by casper.infradead.org with local (Exim 4.92.3 #3 (Red Hat Linux)) id 1jtIWV-00060a-Np; Wed, 08 Jul 2020 22:26:39 +0000 From: "Matthew Wilcox (Oracle)" To: Jens Axboe Cc: "Matthew Wilcox (Oracle)" , linux-block@vger.kernel.org, linux-fsdevel@vger.kernel.org, linux-aio@kvack.org, io-uring@vger.kernel.org, linux-kernel@vger.kernel.org, Kanchan Joshi , Javier Gonzalez Subject: [PATCH 1/2] fs: Abstract calling the kiocb completion function Date: Wed, 8 Jul 2020 23:26:35 +0100 Message-Id: <20200708222637.23046-2-willy@infradead.org> X-Mailer: git-send-email 2.21.3 In-Reply-To: <20200708222637.23046-1-willy@infradead.org> References: <20200708222637.23046-1-willy@infradead.org> MIME-Version: 1.0 Sender: linux-block-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org Introduce complete_kiocb() to abstract the calls to ->ki_complete(). Signed-off-by: Matthew Wilcox (Oracle) --- crypto/af_alg.c | 2 +- drivers/block/loop.c | 2 +- drivers/usb/gadget/function/f_fs.c | 2 +- drivers/usb/gadget/legacy/inode.c | 4 ++-- fs/aio.c | 2 +- fs/block_dev.c | 2 +- fs/ceph/file.c | 2 +- fs/cifs/file.c | 4 ++-- fs/direct-io.c | 2 +- fs/fuse/file.c | 2 +- fs/io_uring.c | 2 +- fs/iomap/direct-io.c | 3 +-- fs/nfs/direct.c | 2 +- fs/overlayfs/file.c | 2 +- fs/read_write.c | 6 ++++++ include/linux/fs.h | 2 ++ 16 files changed, 24 insertions(+), 17 deletions(-) diff --git a/crypto/af_alg.c b/crypto/af_alg.c index b1cd3535c525..590dbbcd0e9f 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -1045,7 +1045,7 @@ void af_alg_async_cb(struct crypto_async_request *_req, int err) af_alg_free_resources(areq); sock_put(sk); - iocb->ki_complete(iocb, err ? err : (int)resultlen, 0); + complete_kiocb(iocb, err ? err : (int)resultlen, 0); } EXPORT_SYMBOL_GPL(af_alg_async_cb); diff --git a/drivers/block/loop.c b/drivers/block/loop.c index a943207705dd..f7a76e82c88c 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -591,7 +591,7 @@ static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, kthread_associate_blkcg(NULL); if (ret != -EIOCBQUEUED) - cmd->iocb.ki_complete(&cmd->iocb, ret, 0); + complete_kiocb(&cmd->iocb, ret, 0); return 0; } diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index 490d353d5fde..6b0c128fdb7e 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -829,7 +829,7 @@ static void ffs_user_copy_worker(struct work_struct *work) kthread_unuse_mm(io_data->mm); } - io_data->kiocb->ki_complete(io_data->kiocb, ret, ret); + complete_kiocb(io_data->kiocb, ret, ret); if (io_data->ffs->ffs_eventfd && !kiocb_has_eventfd) eventfd_signal(io_data->ffs->ffs_eventfd, 1); diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c index 9ee0bfe7bcda..d16ce03c872a 100644 --- a/drivers/usb/gadget/legacy/inode.c +++ b/drivers/usb/gadget/legacy/inode.c @@ -469,7 +469,7 @@ static void ep_user_copy_worker(struct work_struct *work) ret = -EFAULT; /* completing the iocb can drop the ctx and mm, don't touch mm after */ - iocb->ki_complete(iocb, ret, ret); + complete_kiocb(iocb, ret, ret); kfree(priv->buf); kfree(priv->to_free); @@ -498,7 +498,7 @@ static void ep_aio_complete(struct usb_ep *ep, struct usb_request *req) iocb->private = NULL; /* aio_complete() reports bytes-transferred _and_ faults */ - iocb->ki_complete(iocb, req->actual ? req->actual : req->status, + complete_kiocb(iocb, req->actual ? req->actual : req->status, req->status); } else { /* ep_copy_to_user() won't report both; we hide some faults */ diff --git a/fs/aio.c b/fs/aio.c index 91e7cc4a9f17..ca3b123d83f7 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -1513,7 +1513,7 @@ static inline void aio_rw_done(struct kiocb *req, ssize_t ret) ret = -EINTR; /*FALLTHRU*/ default: - req->ki_complete(req, ret, 0); + complete_kiocb(req, ret, 0); } } diff --git a/fs/block_dev.c b/fs/block_dev.c index 182ddf82938f..76c55cd14ad1 100644 --- a/fs/block_dev.c +++ b/fs/block_dev.c @@ -304,7 +304,7 @@ static void blkdev_bio_end_io(struct bio *bio) ret = blk_status_to_errno(dio->bio.bi_status); } - dio->iocb->ki_complete(iocb, ret, 0); + complete_kiocb(iocb, ret, 0); if (dio->multi_bio) bio_put(&dio->bio); } else { diff --git a/fs/ceph/file.c b/fs/ceph/file.c index 160644ddaeed..4e379c385202 100644 --- a/fs/ceph/file.c +++ b/fs/ceph/file.c @@ -1040,7 +1040,7 @@ static void ceph_aio_complete(struct inode *inode, ceph_put_cap_refs(ci, (aio_req->write ? CEPH_CAP_FILE_WR : CEPH_CAP_FILE_RD)); - aio_req->iocb->ki_complete(aio_req->iocb, ret, 0); + complete_kiocb(aio_req->iocb, ret, 0); ceph_free_cap_flush(aio_req->prealloc_cf); kfree(aio_req); diff --git a/fs/cifs/file.c b/fs/cifs/file.c index 9b0f8f33f832..cbf36a8a23aa 100644 --- a/fs/cifs/file.c +++ b/fs/cifs/file.c @@ -3116,7 +3116,7 @@ static void collect_uncached_write_data(struct cifs_aio_ctx *ctx) mutex_unlock(&ctx->aio_mutex); if (ctx->iocb && ctx->iocb->ki_complete) - ctx->iocb->ki_complete(ctx->iocb, ctx->rc, 0); + complete_kiocb(ctx->iocb, ctx->rc, 0); else complete(&ctx->done); } @@ -3849,7 +3849,7 @@ collect_uncached_read_data(struct cifs_aio_ctx *ctx) mutex_unlock(&ctx->aio_mutex); if (ctx->iocb && ctx->iocb->ki_complete) - ctx->iocb->ki_complete(ctx->iocb, ctx->rc, 0); + complete_kiocb(ctx->iocb, ctx->rc, 0); else complete(&ctx->done); } diff --git a/fs/direct-io.c b/fs/direct-io.c index 183299892465..b17dceaeebe5 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -308,7 +308,7 @@ static ssize_t dio_complete(struct dio *dio, ssize_t ret, unsigned int flags) if (ret > 0 && dio->op == REQ_OP_WRITE) ret = generic_write_sync(dio->iocb, ret); - dio->iocb->ki_complete(dio->iocb, ret, 0); + complete_kiocb(dio->iocb, ret, 0); } kmem_cache_free(dio_cache, dio); diff --git a/fs/fuse/file.c b/fs/fuse/file.c index e573b0cd2737..7eac77d950fb 100644 --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -655,7 +655,7 @@ static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos) spin_unlock(&fi->lock); } - io->iocb->ki_complete(io->iocb, res, 0); + complete_kiocb(io->iocb, res, 0); } kref_put(&io->refcnt, fuse_io_release); diff --git a/fs/io_uring.c b/fs/io_uring.c index 6e3169834bf7..f06915fcb6b6 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2469,7 +2469,7 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) ret = -EINTR; /* fall through */ default: - kiocb->ki_complete(kiocb, ret, 0); + complete_kiocb(kiocb, ret, 0); } } diff --git a/fs/iomap/direct-io.c b/fs/iomap/direct-io.c index ec7b78e6feca..3b7edd9c3c11 100644 --- a/fs/iomap/direct-io.c +++ b/fs/iomap/direct-io.c @@ -133,9 +133,8 @@ static ssize_t iomap_dio_complete(struct iomap_dio *dio) static void iomap_dio_complete_work(struct work_struct *work) { struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work); - struct kiocb *iocb = dio->iocb; - iocb->ki_complete(iocb, iomap_dio_complete(dio), 0); + complete_kiocb(dio->iocb, iomap_dio_complete(dio), 0); } /* diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index 3d113cf8908a..498a58b989cf 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -273,7 +273,7 @@ static void nfs_direct_complete(struct nfs_direct_req *dreq) res = (long) dreq->count; WARN_ON_ONCE(dreq->count < 0); } - dreq->iocb->ki_complete(dreq->iocb, res, 0); + complete_kiocb(dreq->iocb, res, 0); } complete(&dreq->completion); diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c index 01820e654a21..78e7439fc4e2 100644 --- a/fs/overlayfs/file.c +++ b/fs/overlayfs/file.c @@ -275,7 +275,7 @@ static void ovl_aio_rw_complete(struct kiocb *iocb, long res, long res2) struct kiocb *orig_iocb = aio_req->orig_iocb; ovl_aio_cleanup_handler(aio_req); - orig_iocb->ki_complete(orig_iocb, res, res2); + complete_kiocb(orig_iocb, res, res2); } static ssize_t ovl_read_iter(struct kiocb *iocb, struct iov_iter *iter) diff --git a/fs/read_write.c b/fs/read_write.c index bbfa9b12b15e..89151de19f77 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -363,6 +363,12 @@ SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high, } #endif +void complete_kiocb(struct kiocb *iocb, long ret, long ret2) +{ + iocb->ki_complete(iocb, ret, ret2); +} +EXPORT_SYMBOL(complete_kiocb); + int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count) { struct inode *inode; diff --git a/include/linux/fs.h b/include/linux/fs.h index da90323b9f92..846135aa328d 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -346,6 +346,8 @@ static inline bool is_sync_kiocb(struct kiocb *kiocb) return kiocb->ki_complete == NULL; } +void complete_kiocb(struct kiocb *kiocb, long ret, long ret2); + /* * "descriptor" for what we're up to with a read. * This allows us to use the same read code yet From patchwork Wed Jul 8 22:26:36 2020 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Matthew Wilcox X-Patchwork-Id: 11652855 Return-Path: Received: from mail.kernel.org (pdx-korg-mail-1.web.codeaurora.org [172.30.200.123]) by pdx-korg-patchwork-2.web.codeaurora.org (Postfix) with ESMTP id CDC1060D for ; Wed, 8 Jul 2020 22:26:56 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by mail.kernel.org (Postfix) with ESMTP id A66F22078A for ; Wed, 8 Jul 2020 22:26:56 +0000 (UTC) Authentication-Results: mail.kernel.org; dkim=fail reason="signature verification failed" (2048-bit key) header.d=infradead.org header.i=@infradead.org header.b="bo5DI3+v" Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1726196AbgGHW0u (ORCPT ); Wed, 8 Jul 2020 18:26:50 -0400 Received: from casper.infradead.org ([90.155.50.34]:45752 "EHLO casper.infradead.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1725903AbgGHW0s (ORCPT ); Wed, 8 Jul 2020 18:26:48 -0400 DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=infradead.org; s=casper.20170209; h=Content-Transfer-Encoding:MIME-Version: References:In-Reply-To:Message-Id:Date:Subject:Cc:To:From:Sender:Reply-To: Content-Type:Content-ID:Content-Description; bh=0cw2EpMcOWKnziLbFIKf3CiqiIzag4QKTjSZuHn5tn0=; b=bo5DI3+voPsNVB47TwoIoKy5Wy pibOBRoZptLBnGdAxX9XdVqoa34qqEG6DdeghuVLCV9DX7Ez+TPlzU1d7k7VPJp4FywQ5Yvr3FykV u7AjnZTbWuqaJo8J4b8PMNVjnHTmOgx8PC92oKu3g5FxLI5Nd0Uivc8YSLhxQcL/HCZs4ymJ5EsoF 8Pbqwkv8lS+RClSOiB9L843ld+pBVKd0cSsN4nmA1Hbb/bIUZ4HmkeHrg2lgLsTP72Bv2hAIVHojL yWFgzbRO8jZXsj94ZAS3yxy6dtxGPiAPQUN/fRm8UBU9CL5g4s5HNLqxgJ9lhBfb3KhKqEfqckMoS mGgrKZlQ==; Received: from willy by casper.infradead.org with local (Exim 4.92.3 #3 (Red Hat Linux)) id 1jtIWW-00060e-7n; Wed, 08 Jul 2020 22:26:40 +0000 From: "Matthew Wilcox (Oracle)" To: Jens Axboe Cc: "Matthew Wilcox (Oracle)" , linux-block@vger.kernel.org, linux-fsdevel@vger.kernel.org, linux-aio@kvack.org, io-uring@vger.kernel.org, linux-kernel@vger.kernel.org, Kanchan Joshi , Javier Gonzalez Subject: [PATCH 2/2] fs: Remove kiocb->ki_complete Date: Wed, 8 Jul 2020 23:26:36 +0100 Message-Id: <20200708222637.23046-3-willy@infradead.org> X-Mailer: git-send-email 2.21.3 In-Reply-To: <20200708222637.23046-1-willy@infradead.org> References: <20200708222637.23046-1-willy@infradead.org> MIME-Version: 1.0 Sender: linux-block-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-block@vger.kernel.org Use a few bits of ki_flags to indicate which completion function to call. Signed-off-by: Matthew Wilcox (Oracle) Reported-by: kernel test robot --- drivers/block/loop.c | 12 ++++++-- drivers/nvme/target/core.c | 10 ++++++- drivers/nvme/target/io-cmd-file.c | 10 +++---- drivers/nvme/target/nvmet.h | 2 ++ drivers/target/target_core_file.c | 20 +++++++++++-- fs/aio.c | 48 +++++++++++++++++-------------- fs/cifs/file.c | 4 +-- fs/io_uring.c | 12 ++++++-- fs/ocfs2/file.c | 7 +++-- fs/overlayfs/file.c | 15 ++++++++-- fs/read_write.c | 32 ++++++++++++++++++++- include/linux/fs.h | 21 ++++++++++++-- 12 files changed, 145 insertions(+), 48 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index f7a76e82c88c..6bd6e55f3e17 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -513,6 +513,7 @@ static void lo_rw_aio_do_completion(struct loop_cmd *cmd) blk_mq_complete_request(rq); } +static int lo_rw_aio_complete_id; static void lo_rw_aio_complete(struct kiocb *iocb, long ret, long ret2) { struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb); @@ -576,8 +577,8 @@ static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, cmd->iocb.ki_pos = pos; cmd->iocb.ki_filp = file; - cmd->iocb.ki_complete = lo_rw_aio_complete; cmd->iocb.ki_flags = IOCB_DIRECT; + kiocb_set_completion(&cmd->iocb, lo_rw_aio_complete_id); cmd->iocb.ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0); if (cmd->css) kthread_associate_blkcg(cmd->css); @@ -2362,10 +2363,14 @@ static int __init loop_init(void) range = 1UL << MINORBITS; } - err = misc_register(&loop_misc); + err = register_kiocb_completion(lo_rw_aio_complete); if (err < 0) goto err_out; + lo_rw_aio_complete_id = err; + err = misc_register(&loop_misc); + if (err < 0) + goto kiocb_out; if (register_blkdev(LOOP_MAJOR, "loop")) { err = -EIO; @@ -2386,6 +2391,8 @@ static int __init loop_init(void) misc_out: misc_deregister(&loop_misc); +kiocb_out: + unregister_kiocb_completion(lo_rw_aio_complete_id); err_out: return err; } @@ -2413,6 +2420,7 @@ static void __exit loop_exit(void) unregister_blkdev(LOOP_MAJOR, "loop"); misc_deregister(&loop_misc); + unregister_kiocb_completion(lo_rw_aio_complete_id); mutex_unlock(&loop_ctl_mutex); } diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 6816507fba58..8b622641c667 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -1563,11 +1563,16 @@ static int __init nvmet_init(void) nvmet_ana_group_enabled[NVMET_DEFAULT_ANA_GRPID] = 1; + error = register_kiocb_completion(nvmet_file_io_done); + if (error) + goto out; + nvmet_file_io_done_id = error; + buffered_io_wq = alloc_workqueue("nvmet-buffered-io-wq", WQ_MEM_RECLAIM, 0); if (!buffered_io_wq) { error = -ENOMEM; - goto out; + goto out_kiocb; } error = nvmet_init_discovery(); @@ -1583,6 +1588,8 @@ static int __init nvmet_init(void) nvmet_exit_discovery(); out_free_work_queue: destroy_workqueue(buffered_io_wq); +out_kiocb: + unregister_kiocb_completion(nvmet_file_io_done_id); out: return error; } @@ -1593,6 +1600,7 @@ static void __exit nvmet_exit(void) nvmet_exit_discovery(); ida_destroy(&cntlid_ida); destroy_workqueue(buffered_io_wq); + unregister_kiocb_completion(nvmet_file_io_done_id); BUILD_BUG_ON(sizeof(struct nvmf_disc_rsp_page_entry) != 1024); BUILD_BUG_ON(sizeof(struct nvmf_disc_rsp_page_hdr) != 1024); diff --git a/drivers/nvme/target/io-cmd-file.c b/drivers/nvme/target/io-cmd-file.c index 0abbefd9925e..5884039e28e1 100644 --- a/drivers/nvme/target/io-cmd-file.c +++ b/drivers/nvme/target/io-cmd-file.c @@ -123,7 +123,8 @@ static ssize_t nvmet_file_submit_bvec(struct nvmet_req *req, loff_t pos, return call_iter(iocb, &iter); } -static void nvmet_file_io_done(struct kiocb *iocb, long ret, long ret2) +int nvmet_file_io_done_id; +void nvmet_file_io_done(struct kiocb *iocb, long ret, long ret2) { struct nvmet_req *req = container_of(iocb, struct nvmet_req, f.iocb); u16 status = NVME_SC_SUCCESS; @@ -192,12 +193,9 @@ static bool nvmet_file_execute_io(struct nvmet_req *req, int ki_flags) goto complete; } - /* - * A NULL ki_complete ask for synchronous execution, which we want - * for the IOCB_NOWAIT case. - */ + /* No completion means synchronous execution */ if (!(ki_flags & IOCB_NOWAIT)) - req->f.iocb.ki_complete = nvmet_file_io_done; + kiocb_set_completion(&req->f.iocb, nvmet_file_io_done_id); ret = nvmet_file_submit_bvec(req, pos, bv_cnt, total_len, ki_flags); diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h index 809691291e73..d42c8b3bcdb5 100644 --- a/drivers/nvme/target/nvmet.h +++ b/drivers/nvme/target/nvmet.h @@ -340,6 +340,8 @@ struct nvmet_req { }; extern struct workqueue_struct *buffered_io_wq; +extern int nvmet_file_io_done_id; +extern void nvmet_file_io_done(struct kiocb *, long, long); static inline void nvmet_set_result(struct nvmet_req *req, u32 result) { diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c index 7143d03f0e02..acae6a159e91 100644 --- a/drivers/target/target_core_file.c +++ b/drivers/target/target_core_file.c @@ -243,6 +243,7 @@ struct target_core_file_cmd { struct kiocb iocb; }; +static int cmd_rw_aio_complete_id; static void cmd_rw_aio_complete(struct kiocb *iocb, long ret, long ret2) { struct target_core_file_cmd *cmd; @@ -296,8 +297,8 @@ fd_execute_rw_aio(struct se_cmd *cmd, struct scatterlist *sgl, u32 sgl_nents, aio_cmd->len = len; aio_cmd->iocb.ki_pos = cmd->t_task_lba * dev->dev_attrib.block_size; aio_cmd->iocb.ki_filp = file; - aio_cmd->iocb.ki_complete = cmd_rw_aio_complete; aio_cmd->iocb.ki_flags = IOCB_DIRECT; + kiocb_set_completion(&aio_cmd->iocb, cmd_rw_aio_complete_id); if (is_write && (cmd->se_cmd_flags & SCF_FUA)) aio_cmd->iocb.ki_flags |= IOCB_DSYNC; @@ -945,12 +946,27 @@ static const struct target_backend_ops fileio_ops = { static int __init fileio_module_init(void) { - return transport_backend_register(&fileio_ops); + int err; + + err = register_kiocb_completion(cmd_rw_aio_complete); + if (err < 0) + return err; + cmd_rw_aio_complete_id = err; + + err = transport_backend_register(&fileio_ops); + if (err) + goto out_kiocb; + return 0; + +out_kiocb: + unregister_kiocb_completion(cmd_rw_aio_complete_id); + return err; } static void __exit fileio_module_exit(void) { target_backend_unregister(&fileio_ops); + unregister_kiocb_completion(cmd_rw_aio_complete_id); } MODULE_DESCRIPTION("TCM FILEIO subsystem plugin"); diff --git a/fs/aio.c b/fs/aio.c index ca3b123d83f7..135f278fffd9 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -258,27 +258,6 @@ static int aio_init_fs_context(struct fs_context *fc) return 0; } -/* aio_setup - * Creates the slab caches used by the aio routines, panic on - * failure as this is done early during the boot sequence. - */ -static int __init aio_setup(void) -{ - static struct file_system_type aio_fs = { - .name = "aio", - .init_fs_context = aio_init_fs_context, - .kill_sb = kill_anon_super, - }; - aio_mnt = kern_mount(&aio_fs); - if (IS_ERR(aio_mnt)) - panic("Failed to create aio fs mount."); - - kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC); - kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC); - return 0; -} -__initcall(aio_setup); - static void put_aio_ring_file(struct kioctx *ctx) { struct file *aio_ring_file = ctx->aio_ring_file; @@ -1418,6 +1397,7 @@ static void aio_remove_iocb(struct aio_kiocb *iocb) spin_unlock_irqrestore(&ctx->ctx_lock, flags); } +static int aio_complete_rw_id; static void aio_complete_rw(struct kiocb *kiocb, long res, long res2) { struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, rw); @@ -1446,10 +1426,10 @@ static int aio_prep_rw(struct kiocb *req, const struct iocb *iocb) { int ret; - req->ki_complete = aio_complete_rw; req->private = NULL; req->ki_pos = iocb->aio_offset; req->ki_flags = iocb_flags(req->ki_filp); + kiocb_set_completion(req, aio_complete_rw_id); if (iocb->aio_flags & IOCB_FLAG_RESFD) req->ki_flags |= IOCB_EVENTFD; req->ki_hint = ki_hint_validate(file_write_hint(req->ki_filp)); @@ -2276,3 +2256,27 @@ COMPAT_SYSCALL_DEFINE6(io_pgetevents_time64, return ret; } #endif + +/* + * Creates the slab caches used by the aio routines, panic on + * failure as this is done early during the boot sequence. + */ +static int __init aio_setup(void) +{ + static struct file_system_type aio_fs = { + .name = "aio", + .init_fs_context = aio_init_fs_context, + .kill_sb = kill_anon_super, + }; + aio_mnt = kern_mount(&aio_fs); + if (IS_ERR(aio_mnt)) + panic("Failed to create aio fs mount."); + + kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC); + kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC); + aio_complete_rw_id = register_kiocb_completion(aio_complete_rw); + BUG_ON(aio_complete_rw_id < 0); + return 0; +} +__initcall(aio_setup); + diff --git a/fs/cifs/file.c b/fs/cifs/file.c index cbf36a8a23aa..b4cfd262a1a4 100644 --- a/fs/cifs/file.c +++ b/fs/cifs/file.c @@ -3115,7 +3115,7 @@ static void collect_uncached_write_data(struct cifs_aio_ctx *ctx) mutex_unlock(&ctx->aio_mutex); - if (ctx->iocb && ctx->iocb->ki_complete) + if (ctx->iocb && !is_sync_kiocb(ctx->iocb)) complete_kiocb(ctx->iocb, ctx->rc, 0); else complete(&ctx->done); @@ -3848,7 +3848,7 @@ collect_uncached_read_data(struct cifs_aio_ctx *ctx) mutex_unlock(&ctx->aio_mutex); - if (ctx->iocb && ctx->iocb->ki_complete) + if (ctx->iocb && !is_sync_kiocb(ctx->iocb)) complete_kiocb(ctx->iocb, ctx->rc, 0); else complete(&ctx->done); diff --git a/fs/io_uring.c b/fs/io_uring.c index f06915fcb6b6..e0f68aa78596 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -2237,6 +2237,7 @@ static void __io_complete_rw(struct io_kiocb *req, long res, long res2, io_complete_rw_common(&req->rw.kiocb, res, cs); } +static int io_complete_rw_id; static void io_complete_rw(struct kiocb *kiocb, long res, long res2) { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); @@ -2244,6 +2245,7 @@ static void io_complete_rw(struct kiocb *kiocb, long res, long res2) __io_complete_rw(req, res, res2, NULL); } +static int io_complete_rw_iopoll_id; static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) { struct io_kiocb *req = container_of(kiocb, struct io_kiocb, rw.kiocb); @@ -2437,13 +2439,13 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, return -EOPNOTSUPP; kiocb->ki_flags |= IOCB_HIPRI; - kiocb->ki_complete = io_complete_rw_iopoll; + kiocb_set_completion(kiocb, io_complete_rw_iopoll_id); req->iopoll_completed = 0; io_get_req_task(req); } else { if (kiocb->ki_flags & IOCB_HIPRI) return -EINVAL; - kiocb->ki_complete = io_complete_rw; + kiocb_set_completion(kiocb, io_complete_rw_id); } req->rw.addr = READ_ONCE(sqe->addr); @@ -2480,7 +2482,7 @@ static void kiocb_done(struct kiocb *kiocb, ssize_t ret, if (req->flags & REQ_F_CUR_POS) req->file->f_pos = kiocb->ki_pos; - if (ret >= 0 && kiocb->ki_complete == io_complete_rw) + if (ret >= 0 && kiocb_completion_id(kiocb) == io_complete_rw_id) __io_complete_rw(req, ret, 0, cs); else io_rw_done(kiocb, ret); @@ -8596,6 +8598,10 @@ static int __init io_uring_init(void) BUILD_BUG_ON(ARRAY_SIZE(io_op_defs) != IORING_OP_LAST); BUILD_BUG_ON(__REQ_F_LAST_BIT >= 8 * sizeof(int)); req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC); + io_complete_rw_id = register_kiocb_completion(io_complete_rw); + io_complete_rw_iopoll_id = + register_kiocb_completion(io_complete_rw_iopoll); + BUG_ON(io_complete_rw_id < 0 || io_complete_rw_iopoll_id < 0); return 0; }; __initcall(io_uring_init); diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c index 85979e2214b3..abcd5257ca34 100644 --- a/fs/ocfs2/file.c +++ b/fs/ocfs2/file.c @@ -2284,7 +2284,7 @@ static ssize_t ocfs2_file_write_iter(struct kiocb *iocb, struct ocfs2_super *osb = OCFS2_SB(inode->i_sb); int full_coherency = !(osb->s_mount_opt & OCFS2_MOUNT_COHERENCY_BUFFERED); - void *saved_ki_complete = NULL; + int saved_ki_complete = 0; int append_write = ((iocb->ki_pos + count) >= i_size_read(inode) ? 1 : 0); int direct_io = iocb->ki_flags & IOCB_DIRECT ? 1 : 0; @@ -2368,7 +2368,8 @@ static ssize_t ocfs2_file_write_iter(struct kiocb *iocb, /* * Make it a sync io if it's an unaligned aio. */ - saved_ki_complete = xchg(&iocb->ki_complete, NULL); + saved_ki_complete = kiocb_completion_id(iocb); + kiocb_set_completion(iocb, 0); } /* communicate with ocfs2_dio_end_io */ @@ -2416,7 +2417,7 @@ static ssize_t ocfs2_file_write_iter(struct kiocb *iocb, out: if (saved_ki_complete) - xchg(&iocb->ki_complete, saved_ki_complete); + kiocb_set_completion(iocb, saved_ki_complete); if (rw_level != -1) ocfs2_rw_unlock(inode, rw_level); diff --git a/fs/overlayfs/file.c b/fs/overlayfs/file.c index 78e7439fc4e2..5951c4180bc9 100644 --- a/fs/overlayfs/file.c +++ b/fs/overlayfs/file.c @@ -278,6 +278,8 @@ static void ovl_aio_rw_complete(struct kiocb *iocb, long res, long res2) complete_kiocb(orig_iocb, res, res2); } +static int ovl_aio_rw_complete_id; + static ssize_t ovl_read_iter(struct kiocb *iocb, struct iov_iter *iter) { struct file *file = iocb->ki_filp; @@ -308,7 +310,7 @@ static ssize_t ovl_read_iter(struct kiocb *iocb, struct iov_iter *iter) real.flags = 0; aio_req->orig_iocb = iocb; kiocb_clone(&aio_req->iocb, iocb, real.file); - aio_req->iocb.ki_complete = ovl_aio_rw_complete; + kiocb_set_completion(&aio_req->iocb, ovl_aio_rw_complete_id); ret = vfs_iocb_iter_read(real.file, &aio_req->iocb, iter); if (ret != -EIOCBQUEUED) ovl_aio_cleanup_handler(aio_req); @@ -368,7 +370,7 @@ static ssize_t ovl_write_iter(struct kiocb *iocb, struct iov_iter *iter) real.flags = 0; aio_req->orig_iocb = iocb; kiocb_clone(&aio_req->iocb, iocb, real.file); - aio_req->iocb.ki_complete = ovl_aio_rw_complete; + kiocb_set_completion(&aio_req->iocb, ovl_aio_rw_complete_id); ret = vfs_iocb_iter_write(real.file, &aio_req->iocb, iter); if (ret != -EIOCBQUEUED) ovl_aio_cleanup_handler(aio_req); @@ -792,16 +794,23 @@ const struct file_operations ovl_file_operations = { int __init ovl_aio_request_cache_init(void) { + ovl_aio_rw_complete_id = register_kiocb_completion(ovl_aio_rw_complete); + if (ovl_aio_rw_complete_id < 0) + return ovl_aio_rw_complete_id; + ovl_aio_request_cachep = kmem_cache_create("ovl_aio_req", sizeof(struct ovl_aio_req), 0, SLAB_HWCACHE_ALIGN, NULL); - if (!ovl_aio_request_cachep) + if (!ovl_aio_request_cachep) { + unregister_kiocb_completion(ovl_aio_rw_complete_id); return -ENOMEM; + } return 0; } void ovl_aio_request_cache_destroy(void) { + unregister_kiocb_completion(ovl_aio_rw_complete_id); kmem_cache_destroy(ovl_aio_request_cachep); } diff --git a/fs/read_write.c b/fs/read_write.c index 89151de19f77..0163cefb9bf1 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -363,9 +363,39 @@ SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high, } #endif +#define IOCB_CB_MAX ((1 << _IOCB_COMPLETION_BITS) - 1) + +typedef void ki_cmpl(struct kiocb *, long ret, long ret2); +static ki_cmpl *ki_cmpls[IOCB_CB_MAX]; + +int register_kiocb_completion(ki_cmpl cb) +{ + int i; + + for (i = 0; i < IOCB_CB_MAX; i++) { + if (ki_cmpls[i]) + continue; + ki_cmpls[i] = cb; + return i + 1; + } + + pr_err("Increase _IOCB_COMPLETION_BITS\n"); + return -EBUSY; +} +EXPORT_SYMBOL(register_kiocb_completion); + +void unregister_kiocb_completion(int id) +{ + ki_cmpls[id - 1] = NULL; +} +EXPORT_SYMBOL(unregister_kiocb_completion); + void complete_kiocb(struct kiocb *iocb, long ret, long ret2) { - iocb->ki_complete(iocb, ret, ret2); + unsigned int id = kiocb_completion_id(iocb); + + if (id > 0) + ki_cmpls[id - 1](iocb, ret, ret2); } EXPORT_SYMBOL(complete_kiocb); diff --git a/include/linux/fs.h b/include/linux/fs.h index 846135aa328d..fa6f98714994 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -320,6 +320,9 @@ enum rw_hint { #define IOCB_NOWAIT (1 << 7) /* iocb->ki_waitq is valid */ #define IOCB_WAITQ (1 << 8) +#define _IOCB_COMPLETION_BITS 4 +#define _IOCB_COMPLETION_SHIFT (32 - _IOCB_COMPLETION_BITS) +#define IOCB_COMPLETION_FNS (~0 << _IOCB_COMPLETION_SHIFT) struct kiocb { struct file *ki_filp; @@ -328,9 +331,8 @@ struct kiocb { randomized_struct_fields_start loff_t ki_pos; - void (*ki_complete)(struct kiocb *iocb, long ret, long ret2); void *private; - int ki_flags; + unsigned int ki_flags; u16 ki_hint; u16 ki_ioprio; /* See linux/ioprio.h */ union { @@ -341,12 +343,25 @@ struct kiocb { randomized_struct_fields_end }; +static inline int kiocb_completion_id(struct kiocb *kiocb) +{ + return kiocb->ki_flags >> _IOCB_COMPLETION_SHIFT; +} + +static inline void kiocb_set_completion(struct kiocb *kiocb, int id) +{ + kiocb->ki_flags = (kiocb->ki_flags & (~IOCB_COMPLETION_FNS)) | + (id << _IOCB_COMPLETION_SHIFT); +} + static inline bool is_sync_kiocb(struct kiocb *kiocb) { - return kiocb->ki_complete == NULL; + return kiocb_completion_id(kiocb) == 0; } void complete_kiocb(struct kiocb *kiocb, long ret, long ret2); +int register_kiocb_completion(void (*)(struct kiocb *, long, long)); +void unregister_kiocb_completion(int id); /* * "descriptor" for what we're up to with a read.