| Message ID | 1398363328-7100-16-git-send-email-dros@primarydata.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
On 04/24/2014 02:15 PM, Weston Andros Adamson wrote: > Support direct requests that span multiple pnfs data servers by > comparing nfs_pgio_header->verf to a cached verf in pnfs_commit_bucket. > Continue to use dreq->verf if the MDS is used / non-pNFS. > > Signed-off-by: Weston Andros Adamson <dros@primarydata.com> > --- > fs/nfs/direct.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++--- > fs/nfs/nfs4filelayout.c | 6 +++ > include/linux/nfs.h | 5 ++- > include/linux/nfs_xdr.h | 2 + > 4 files changed, 105 insertions(+), 6 deletions(-) > > diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c > index 2c0e08f..9349933 100644 > --- a/fs/nfs/direct.c > +++ b/fs/nfs/direct.c > @@ -108,6 +108,93 @@ static inline int put_dreq(struct nfs_direct_req *dreq) > return atomic_dec_and_test(&dreq->io_count); > } > > +/* > + * nfs_direct_select_verf - select the right verifier > + * @dreq - direct request possibly spanning multiple servers > + * @ds_clp - nfs_client of data server or NULL if MDS / non-pnfs > + * @ds_idx - index of data server in data server list, only valid if ds_clp set > + * > + * returns the correct verifier to use given the role of the server > + */ > +static struct nfs_writeverf * > +nfs_direct_select_verf(struct nfs_direct_req *dreq, > + struct nfs_client *ds_clp, > + int ds_idx) > +{ > + struct nfs_writeverf *verfp = &dreq->verf; > + > + if (ds_clp) { > + /* pNFS is in use, use the DS verf */ > + if (ds_idx >= 0 && ds_idx < dreq->ds_cinfo.nbuckets) The struct pnfs_ds_commit_info is empty if CONFIG_NFS_V4_1=n, so this won't compile. Anna > + verfp = &dreq->ds_cinfo.buckets[ds_idx].direct_verf; > + else > + WARN_ON_ONCE(1); > + } > + return verfp; > +} > + > + > +/* > + * nfs_direct_set_hdr_verf - set the write/commit verifier > + * @dreq - direct request possibly spanning multiple servers > + * @hdr - pageio header to validate against previously seen verfs > + * > + * Set the server's (MDS or DS) "seen" verifier > + */ > +static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq, > + struct nfs_pgio_header *hdr) > +{ > + struct nfs_writeverf *verfp; > + > + verfp = nfs_direct_select_verf(dreq, hdr->data->ds_clp, > + hdr->data->ds_idx); > + WARN_ON_ONCE(verfp->committed >= 0); > + memcpy(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); > + WARN_ON_ONCE(verfp->committed < 0); > +} > + > +/* > + * nfs_direct_cmp_hdr_verf - compare verifier for pgio header > + * @dreq - direct request possibly spanning multiple servers > + * @hdr - pageio header to validate against previously seen verf > + * > + * set the server's "seen" verf if not initialized. > + * returns result of comparison between @hdr->verf and the "seen" > + * verf of the server used by @hdr (DS or MDS) > + */ > +static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq, > + struct nfs_pgio_header *hdr) > +{ > + struct nfs_writeverf *verfp; > + > + verfp = nfs_direct_select_verf(dreq, hdr->data->ds_clp, > + hdr->data->ds_idx); > + if (verfp->committed < 0) { > + nfs_direct_set_hdr_verf(dreq, hdr); > + return 0; > + } > + return memcmp(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); > +} > + > +/* > + * nfs_direct_cmp_commit_data_verf - compare verifier for commit data > + * @dreq - direct request possibly spanning multiple servers > + * @data - commit data to validate against previously seen verf > + * > + * returns result of comparison between @data->verf and the verf of > + * the server used by @data (DS or MDS) > + */ > +static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq, > + struct nfs_commit_data *data) > +{ > + struct nfs_writeverf *verfp; > + > + verfp = nfs_direct_select_verf(dreq, data->ds_clp, > + data->ds_commit_index); > + WARN_ON_ONCE(verfp->committed < 0); > + return memcmp(verfp, &data->verf, sizeof(struct nfs_writeverf)); > +} > + > /** > * nfs_direct_IO - NFS address space operation for direct I/O > * @rw: direction (read or write) > @@ -168,6 +255,7 @@ static inline struct nfs_direct_req *nfs_direct_req_alloc(void) > kref_get(&dreq->kref); > init_completion(&dreq->completion); > INIT_LIST_HEAD(&dreq->mds_cinfo.list); > + dreq->verf.committed = NFS_INVALID_STABLE_HOW; /* not set yet */ > INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); > spin_lock_init(&dreq->lock); > > @@ -602,7 +690,7 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data) > dprintk("NFS: %5u commit failed with error %d.\n", > data->task.tk_pid, status); > dreq->flags = NFS_ODIRECT_RESCHED_WRITES; > - } else if (memcmp(&dreq->verf, &data->verf, sizeof(data->verf))) { > + } else if (nfs_direct_cmp_commit_data_verf(dreq, data)) { > dprintk("NFS: %5u commit verify failed\n", data->task.tk_pid); > dreq->flags = NFS_ODIRECT_RESCHED_WRITES; > } > @@ -811,13 +899,13 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr) > if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) > bit = NFS_IOHDR_NEED_RESCHED; > else if (dreq->flags == 0) { > - memcpy(&dreq->verf, &hdr->verf, > - sizeof(dreq->verf)); > + nfs_direct_set_hdr_verf(dreq, hdr); > bit = NFS_IOHDR_NEED_COMMIT; > dreq->flags = NFS_ODIRECT_DO_COMMIT; > } else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) { > - if (memcmp(&dreq->verf, &hdr->verf, sizeof(dreq->verf))) { > - dreq->flags = NFS_ODIRECT_RESCHED_WRITES; > + if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr)) { > + dreq->flags = > + NFS_ODIRECT_RESCHED_WRITES; > bit = NFS_IOHDR_NEED_RESCHED; > } else > bit = NFS_IOHDR_NEED_COMMIT; > diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c > index 7a665e0..0ebc521 100644 > --- a/fs/nfs/nfs4filelayout.c > +++ b/fs/nfs/nfs4filelayout.c > @@ -560,6 +560,7 @@ filelayout_read_pagelist(struct nfs_pgio_data *data) > /* No multipath support. Use first DS */ > atomic_inc(&ds->ds_clp->cl_count); > data->ds_clp = ds->ds_clp; > + data->ds_idx = idx; > fh = nfs4_fl_select_ds_fh(lseg, j); > if (fh) > data->args.fh = fh; > @@ -603,6 +604,7 @@ filelayout_write_pagelist(struct nfs_pgio_data *data, int sync) > data->pgio_done_cb = filelayout_write_done_cb; > atomic_inc(&ds->ds_clp->cl_count); > data->ds_clp = ds->ds_clp; > + data->ds_idx = idx; > fh = nfs4_fl_select_ds_fh(lseg, j); > if (fh) > data->args.fh = fh; > @@ -875,6 +877,8 @@ filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, > for (i = 0; i < size; i++) { > INIT_LIST_HEAD(&buckets[i].written); > INIT_LIST_HEAD(&buckets[i].committing); > + /* mark direct verifier as unset */ > + buckets[i].direct_verf.committed = NFS_INVALID_STABLE_HOW; > } > > spin_lock(cinfo->lock); > @@ -885,6 +889,8 @@ filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, > &buckets[i].written); > list_splice(&cinfo->ds->buckets[i].committing, > &buckets[i].committing); > + buckets[i].direct_verf.committed = > + cinfo->ds->buckets[i].direct_verf.committed; > buckets[i].wlseg = cinfo->ds->buckets[i].wlseg; > buckets[i].clseg = cinfo->ds->buckets[i].clseg; > } > diff --git a/include/linux/nfs.h b/include/linux/nfs.h > index 3e794c1..610af51 100644 > --- a/include/linux/nfs.h > +++ b/include/linux/nfs.h > @@ -46,6 +46,9 @@ static inline void nfs_copy_fh(struct nfs_fh *target, const struct nfs_fh *sourc > enum nfs3_stable_how { > NFS_UNSTABLE = 0, > NFS_DATA_SYNC = 1, > - NFS_FILE_SYNC = 2 > + NFS_FILE_SYNC = 2, > + > + /* used by direct.c to mark verf as invalid */ > + NFS_INVALID_STABLE_HOW = -1 > }; > #endif /* _LINUX_NFS_H */ > diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h > index 29828c7..bb9fb88 100644 > --- a/include/linux/nfs_xdr.h > +++ b/include/linux/nfs_xdr.h > @@ -1111,6 +1111,7 @@ struct pnfs_commit_bucket { > struct list_head committing; > struct pnfs_layout_segment *wlseg; > struct pnfs_layout_segment *clseg; > + struct nfs_writeverf direct_verf; > }; > > struct pnfs_ds_commit_info { > @@ -1294,6 +1295,7 @@ struct nfs_pgio_data { > __u64 mds_offset; /* Filelayout dense stripe */ > struct nfs_page_array pages; > struct nfs_client *ds_clp; /* pNFS data server */ > + int ds_idx; /* ds index if ds_clp is set */ > }; > > struct nfs_rw_header { -- To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On Apr 25, 2014, at 10:15 AM, Anna Schumaker <schumaker.anna@gmail.com> wrote: > On 04/24/2014 02:15 PM, Weston Andros Adamson wrote: >> Support direct requests that span multiple pnfs data servers by >> comparing nfs_pgio_header->verf to a cached verf in pnfs_commit_bucket. >> Continue to use dreq->verf if the MDS is used / non-pNFS. >> >> Signed-off-by: Weston Andros Adamson <dros@primarydata.com> >> --- >> fs/nfs/direct.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++--- >> fs/nfs/nfs4filelayout.c | 6 +++ >> include/linux/nfs.h | 5 ++- >> include/linux/nfs_xdr.h | 2 + >> 4 files changed, 105 insertions(+), 6 deletions(-) >> >> diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c >> index 2c0e08f..9349933 100644 >> --- a/fs/nfs/direct.c >> +++ b/fs/nfs/direct.c >> @@ -108,6 +108,93 @@ static inline int put_dreq(struct nfs_direct_req *dreq) >> return atomic_dec_and_test(&dreq->io_count); >> } >> >> +/* >> + * nfs_direct_select_verf - select the right verifier >> + * @dreq - direct request possibly spanning multiple servers >> + * @ds_clp - nfs_client of data server or NULL if MDS / non-pnfs >> + * @ds_idx - index of data server in data server list, only valid if ds_clp set >> + * >> + * returns the correct verifier to use given the role of the server >> + */ >> +static struct nfs_writeverf * >> +nfs_direct_select_verf(struct nfs_direct_req *dreq, >> + struct nfs_client *ds_clp, >> + int ds_idx) >> +{ >> + struct nfs_writeverf *verfp = &dreq->verf; >> + >> + if (ds_clp) { >> + /* pNFS is in use, use the DS verf */ >> + if (ds_idx >= 0 && ds_idx < dreq->ds_cinfo.nbuckets) > > The struct pnfs_ds_commit_info is empty if CONFIG_NFS_V4_1=n, so this won't compile. > > Anna Good catch, I’ll iterate through the patchset and test with v4.1 disabled. Time to add some #ifdefs -dros > >> + verfp = &dreq->ds_cinfo.buckets[ds_idx].direct_verf; >> + else >> + WARN_ON_ONCE(1); >> + } >> + return verfp; >> +} >> + >> + >> +/* >> + * nfs_direct_set_hdr_verf - set the write/commit verifier >> + * @dreq - direct request possibly spanning multiple servers >> + * @hdr - pageio header to validate against previously seen verfs >> + * >> + * Set the server's (MDS or DS) "seen" verifier >> + */ >> +static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq, >> + struct nfs_pgio_header *hdr) >> +{ >> + struct nfs_writeverf *verfp; >> + >> + verfp = nfs_direct_select_verf(dreq, hdr->data->ds_clp, >> + hdr->data->ds_idx); >> + WARN_ON_ONCE(verfp->committed >= 0); >> + memcpy(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); >> + WARN_ON_ONCE(verfp->committed < 0); >> +} >> + >> +/* >> + * nfs_direct_cmp_hdr_verf - compare verifier for pgio header >> + * @dreq - direct request possibly spanning multiple servers >> + * @hdr - pageio header to validate against previously seen verf >> + * >> + * set the server's "seen" verf if not initialized. >> + * returns result of comparison between @hdr->verf and the "seen" >> + * verf of the server used by @hdr (DS or MDS) >> + */ >> +static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq, >> + struct nfs_pgio_header *hdr) >> +{ >> + struct nfs_writeverf *verfp; >> + >> + verfp = nfs_direct_select_verf(dreq, hdr->data->ds_clp, >> + hdr->data->ds_idx); >> + if (verfp->committed < 0) { >> + nfs_direct_set_hdr_verf(dreq, hdr); >> + return 0; >> + } >> + return memcmp(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); >> +} >> + >> +/* >> + * nfs_direct_cmp_commit_data_verf - compare verifier for commit data >> + * @dreq - direct request possibly spanning multiple servers >> + * @data - commit data to validate against previously seen verf >> + * >> + * returns result of comparison between @data->verf and the verf of >> + * the server used by @data (DS or MDS) >> + */ >> +static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq, >> + struct nfs_commit_data *data) >> +{ >> + struct nfs_writeverf *verfp; >> + >> + verfp = nfs_direct_select_verf(dreq, data->ds_clp, >> + data->ds_commit_index); >> + WARN_ON_ONCE(verfp->committed < 0); >> + return memcmp(verfp, &data->verf, sizeof(struct nfs_writeverf)); >> +} >> + >> /** >> * nfs_direct_IO - NFS address space operation for direct I/O >> * @rw: direction (read or write) >> @@ -168,6 +255,7 @@ static inline struct nfs_direct_req *nfs_direct_req_alloc(void) >> kref_get(&dreq->kref); >> init_completion(&dreq->completion); >> INIT_LIST_HEAD(&dreq->mds_cinfo.list); >> + dreq->verf.committed = NFS_INVALID_STABLE_HOW; /* not set yet */ >> INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); >> spin_lock_init(&dreq->lock); >> >> @@ -602,7 +690,7 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data) >> dprintk("NFS: %5u commit failed with error %d.\n", >> data->task.tk_pid, status); >> dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >> - } else if (memcmp(&dreq->verf, &data->verf, sizeof(data->verf))) { >> + } else if (nfs_direct_cmp_commit_data_verf(dreq, data)) { >> dprintk("NFS: %5u commit verify failed\n", data->task.tk_pid); >> dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >> } >> @@ -811,13 +899,13 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr) >> if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) >> bit = NFS_IOHDR_NEED_RESCHED; >> else if (dreq->flags == 0) { >> - memcpy(&dreq->verf, &hdr->verf, >> - sizeof(dreq->verf)); >> + nfs_direct_set_hdr_verf(dreq, hdr); >> bit = NFS_IOHDR_NEED_COMMIT; >> dreq->flags = NFS_ODIRECT_DO_COMMIT; >> } else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) { >> - if (memcmp(&dreq->verf, &hdr->verf, sizeof(dreq->verf))) { >> - dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >> + if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr)) { >> + dreq->flags = >> + NFS_ODIRECT_RESCHED_WRITES; >> bit = NFS_IOHDR_NEED_RESCHED; >> } else >> bit = NFS_IOHDR_NEED_COMMIT; >> diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c >> index 7a665e0..0ebc521 100644 >> --- a/fs/nfs/nfs4filelayout.c >> +++ b/fs/nfs/nfs4filelayout.c >> @@ -560,6 +560,7 @@ filelayout_read_pagelist(struct nfs_pgio_data *data) >> /* No multipath support. Use first DS */ >> atomic_inc(&ds->ds_clp->cl_count); >> data->ds_clp = ds->ds_clp; >> + data->ds_idx = idx; >> fh = nfs4_fl_select_ds_fh(lseg, j); >> if (fh) >> data->args.fh = fh; >> @@ -603,6 +604,7 @@ filelayout_write_pagelist(struct nfs_pgio_data *data, int sync) >> data->pgio_done_cb = filelayout_write_done_cb; >> atomic_inc(&ds->ds_clp->cl_count); >> data->ds_clp = ds->ds_clp; >> + data->ds_idx = idx; >> fh = nfs4_fl_select_ds_fh(lseg, j); >> if (fh) >> data->args.fh = fh; >> @@ -875,6 +877,8 @@ filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, >> for (i = 0; i < size; i++) { >> INIT_LIST_HEAD(&buckets[i].written); >> INIT_LIST_HEAD(&buckets[i].committing); >> + /* mark direct verifier as unset */ >> + buckets[i].direct_verf.committed = NFS_INVALID_STABLE_HOW; >> } >> >> spin_lock(cinfo->lock); >> @@ -885,6 +889,8 @@ filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, >> &buckets[i].written); >> list_splice(&cinfo->ds->buckets[i].committing, >> &buckets[i].committing); >> + buckets[i].direct_verf.committed = >> + cinfo->ds->buckets[i].direct_verf.committed; >> buckets[i].wlseg = cinfo->ds->buckets[i].wlseg; >> buckets[i].clseg = cinfo->ds->buckets[i].clseg; >> } >> diff --git a/include/linux/nfs.h b/include/linux/nfs.h >> index 3e794c1..610af51 100644 >> --- a/include/linux/nfs.h >> +++ b/include/linux/nfs.h >> @@ -46,6 +46,9 @@ static inline void nfs_copy_fh(struct nfs_fh *target, const struct nfs_fh *sourc >> enum nfs3_stable_how { >> NFS_UNSTABLE = 0, >> NFS_DATA_SYNC = 1, >> - NFS_FILE_SYNC = 2 >> + NFS_FILE_SYNC = 2, >> + >> + /* used by direct.c to mark verf as invalid */ >> + NFS_INVALID_STABLE_HOW = -1 >> }; >> #endif /* _LINUX_NFS_H */ >> diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h >> index 29828c7..bb9fb88 100644 >> --- a/include/linux/nfs_xdr.h >> +++ b/include/linux/nfs_xdr.h >> @@ -1111,6 +1111,7 @@ struct pnfs_commit_bucket { >> struct list_head committing; >> struct pnfs_layout_segment *wlseg; >> struct pnfs_layout_segment *clseg; >> + struct nfs_writeverf direct_verf; >> }; >> >> struct pnfs_ds_commit_info { >> @@ -1294,6 +1295,7 @@ struct nfs_pgio_data { >> __u64 mds_offset; /* Filelayout dense stripe */ >> struct nfs_page_array pages; >> struct nfs_client *ds_clp; /* pNFS data server */ >> + int ds_idx; /* ds index if ds_clp is set */ >> }; >> >> struct nfs_rw_header { > -- To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On Apr 25, 2014, at 11:12 AM, Weston Andros Adamson <dros@primarydata.com> wrote: > On Apr 25, 2014, at 10:15 AM, Anna Schumaker <schumaker.anna@gmail.com> wrote: > >> On 04/24/2014 02:15 PM, Weston Andros Adamson wrote: >>> Support direct requests that span multiple pnfs data servers by >>> comparing nfs_pgio_header->verf to a cached verf in pnfs_commit_bucket. >>> Continue to use dreq->verf if the MDS is used / non-pNFS. >>> >>> Signed-off-by: Weston Andros Adamson <dros@primarydata.com> >>> --- >>> fs/nfs/direct.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++--- >>> fs/nfs/nfs4filelayout.c | 6 +++ >>> include/linux/nfs.h | 5 ++- >>> include/linux/nfs_xdr.h | 2 + >>> 4 files changed, 105 insertions(+), 6 deletions(-) >>> >>> diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c >>> index 2c0e08f..9349933 100644 >>> --- a/fs/nfs/direct.c >>> +++ b/fs/nfs/direct.c >>> @@ -108,6 +108,93 @@ static inline int put_dreq(struct nfs_direct_req *dreq) >>> return atomic_dec_and_test(&dreq->io_count); >>> } >>> >>> +/* >>> + * nfs_direct_select_verf - select the right verifier >>> + * @dreq - direct request possibly spanning multiple servers >>> + * @ds_clp - nfs_client of data server or NULL if MDS / non-pnfs >>> + * @ds_idx - index of data server in data server list, only valid if ds_clp set >>> + * >>> + * returns the correct verifier to use given the role of the server >>> + */ >>> +static struct nfs_writeverf * >>> +nfs_direct_select_verf(struct nfs_direct_req *dreq, >>> + struct nfs_client *ds_clp, >>> + int ds_idx) >>> +{ >>> + struct nfs_writeverf *verfp = &dreq->verf; >>> + >>> + if (ds_clp) { >>> + /* pNFS is in use, use the DS verf */ >>> + if (ds_idx >= 0 && ds_idx < dreq->ds_cinfo.nbuckets) >> >> The struct pnfs_ds_commit_info is empty if CONFIG_NFS_V4_1=n, so this won't compile. >> >> Anna > > Good catch, I’ll iterate through the patchset and test with v4.1 disabled. > > Time to add some #ifdefs > > -dros That was the only problem I found with 4.1 disabled. Fixed and pushed. -dros > >> >>> + verfp = &dreq->ds_cinfo.buckets[ds_idx].direct_verf; >>> + else >>> + WARN_ON_ONCE(1); >>> + } >>> + return verfp; >>> +} >>> + >>> + >>> +/* >>> + * nfs_direct_set_hdr_verf - set the write/commit verifier >>> + * @dreq - direct request possibly spanning multiple servers >>> + * @hdr - pageio header to validate against previously seen verfs >>> + * >>> + * Set the server's (MDS or DS) "seen" verifier >>> + */ >>> +static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq, >>> + struct nfs_pgio_header *hdr) >>> +{ >>> + struct nfs_writeverf *verfp; >>> + >>> + verfp = nfs_direct_select_verf(dreq, hdr->data->ds_clp, >>> + hdr->data->ds_idx); >>> + WARN_ON_ONCE(verfp->committed >= 0); >>> + memcpy(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); >>> + WARN_ON_ONCE(verfp->committed < 0); >>> +} >>> + >>> +/* >>> + * nfs_direct_cmp_hdr_verf - compare verifier for pgio header >>> + * @dreq - direct request possibly spanning multiple servers >>> + * @hdr - pageio header to validate against previously seen verf >>> + * >>> + * set the server's "seen" verf if not initialized. >>> + * returns result of comparison between @hdr->verf and the "seen" >>> + * verf of the server used by @hdr (DS or MDS) >>> + */ >>> +static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq, >>> + struct nfs_pgio_header *hdr) >>> +{ >>> + struct nfs_writeverf *verfp; >>> + >>> + verfp = nfs_direct_select_verf(dreq, hdr->data->ds_clp, >>> + hdr->data->ds_idx); >>> + if (verfp->committed < 0) { >>> + nfs_direct_set_hdr_verf(dreq, hdr); >>> + return 0; >>> + } >>> + return memcmp(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); >>> +} >>> + >>> +/* >>> + * nfs_direct_cmp_commit_data_verf - compare verifier for commit data >>> + * @dreq - direct request possibly spanning multiple servers >>> + * @data - commit data to validate against previously seen verf >>> + * >>> + * returns result of comparison between @data->verf and the verf of >>> + * the server used by @data (DS or MDS) >>> + */ >>> +static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq, >>> + struct nfs_commit_data *data) >>> +{ >>> + struct nfs_writeverf *verfp; >>> + >>> + verfp = nfs_direct_select_verf(dreq, data->ds_clp, >>> + data->ds_commit_index); >>> + WARN_ON_ONCE(verfp->committed < 0); >>> + return memcmp(verfp, &data->verf, sizeof(struct nfs_writeverf)); >>> +} >>> + >>> /** >>> * nfs_direct_IO - NFS address space operation for direct I/O >>> * @rw: direction (read or write) >>> @@ -168,6 +255,7 @@ static inline struct nfs_direct_req *nfs_direct_req_alloc(void) >>> kref_get(&dreq->kref); >>> init_completion(&dreq->completion); >>> INIT_LIST_HEAD(&dreq->mds_cinfo.list); >>> + dreq->verf.committed = NFS_INVALID_STABLE_HOW; /* not set yet */ >>> INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); >>> spin_lock_init(&dreq->lock); >>> >>> @@ -602,7 +690,7 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data) >>> dprintk("NFS: %5u commit failed with error %d.\n", >>> data->task.tk_pid, status); >>> dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >>> - } else if (memcmp(&dreq->verf, &data->verf, sizeof(data->verf))) { >>> + } else if (nfs_direct_cmp_commit_data_verf(dreq, data)) { >>> dprintk("NFS: %5u commit verify failed\n", data->task.tk_pid); >>> dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >>> } >>> @@ -811,13 +899,13 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr) >>> if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) >>> bit = NFS_IOHDR_NEED_RESCHED; >>> else if (dreq->flags == 0) { >>> - memcpy(&dreq->verf, &hdr->verf, >>> - sizeof(dreq->verf)); >>> + nfs_direct_set_hdr_verf(dreq, hdr); >>> bit = NFS_IOHDR_NEED_COMMIT; >>> dreq->flags = NFS_ODIRECT_DO_COMMIT; >>> } else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) { >>> - if (memcmp(&dreq->verf, &hdr->verf, sizeof(dreq->verf))) { >>> - dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >>> + if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr)) { >>> + dreq->flags = >>> + NFS_ODIRECT_RESCHED_WRITES; >>> bit = NFS_IOHDR_NEED_RESCHED; >>> } else >>> bit = NFS_IOHDR_NEED_COMMIT; >>> diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c >>> index 7a665e0..0ebc521 100644 >>> --- a/fs/nfs/nfs4filelayout.c >>> +++ b/fs/nfs/nfs4filelayout.c >>> @@ -560,6 +560,7 @@ filelayout_read_pagelist(struct nfs_pgio_data *data) >>> /* No multipath support. Use first DS */ >>> atomic_inc(&ds->ds_clp->cl_count); >>> data->ds_clp = ds->ds_clp; >>> + data->ds_idx = idx; >>> fh = nfs4_fl_select_ds_fh(lseg, j); >>> if (fh) >>> data->args.fh = fh; >>> @@ -603,6 +604,7 @@ filelayout_write_pagelist(struct nfs_pgio_data *data, int sync) >>> data->pgio_done_cb = filelayout_write_done_cb; >>> atomic_inc(&ds->ds_clp->cl_count); >>> data->ds_clp = ds->ds_clp; >>> + data->ds_idx = idx; >>> fh = nfs4_fl_select_ds_fh(lseg, j); >>> if (fh) >>> data->args.fh = fh; >>> @@ -875,6 +877,8 @@ filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, >>> for (i = 0; i < size; i++) { >>> INIT_LIST_HEAD(&buckets[i].written); >>> INIT_LIST_HEAD(&buckets[i].committing); >>> + /* mark direct verifier as unset */ >>> + buckets[i].direct_verf.committed = NFS_INVALID_STABLE_HOW; >>> } >>> >>> spin_lock(cinfo->lock); >>> @@ -885,6 +889,8 @@ filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, >>> &buckets[i].written); >>> list_splice(&cinfo->ds->buckets[i].committing, >>> &buckets[i].committing); >>> + buckets[i].direct_verf.committed = >>> + cinfo->ds->buckets[i].direct_verf.committed; >>> buckets[i].wlseg = cinfo->ds->buckets[i].wlseg; >>> buckets[i].clseg = cinfo->ds->buckets[i].clseg; >>> } >>> diff --git a/include/linux/nfs.h b/include/linux/nfs.h >>> index 3e794c1..610af51 100644 >>> --- a/include/linux/nfs.h >>> +++ b/include/linux/nfs.h >>> @@ -46,6 +46,9 @@ static inline void nfs_copy_fh(struct nfs_fh *target, const struct nfs_fh *sourc >>> enum nfs3_stable_how { >>> NFS_UNSTABLE = 0, >>> NFS_DATA_SYNC = 1, >>> - NFS_FILE_SYNC = 2 >>> + NFS_FILE_SYNC = 2, >>> + >>> + /* used by direct.c to mark verf as invalid */ >>> + NFS_INVALID_STABLE_HOW = -1 >>> }; >>> #endif /* _LINUX_NFS_H */ >>> diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h >>> index 29828c7..bb9fb88 100644 >>> --- a/include/linux/nfs_xdr.h >>> +++ b/include/linux/nfs_xdr.h >>> @@ -1111,6 +1111,7 @@ struct pnfs_commit_bucket { >>> struct list_head committing; >>> struct pnfs_layout_segment *wlseg; >>> struct pnfs_layout_segment *clseg; >>> + struct nfs_writeverf direct_verf; >>> }; >>> >>> struct pnfs_ds_commit_info { >>> @@ -1294,6 +1295,7 @@ struct nfs_pgio_data { >>> __u64 mds_offset; /* Filelayout dense stripe */ >>> struct nfs_page_array pages; >>> struct nfs_client *ds_clp; /* pNFS data server */ >>> + int ds_idx; /* ds index if ds_clp is set */ >>> }; >>> >>> struct nfs_rw_header { -- To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On 04/25/2014 11:38 AM, Weston Andros Adamson wrote: > On Apr 25, 2014, at 11:12 AM, Weston Andros Adamson <dros@primarydata.com> wrote: > >> On Apr 25, 2014, at 10:15 AM, Anna Schumaker <schumaker.anna@gmail.com> wrote: >> >>> On 04/24/2014 02:15 PM, Weston Andros Adamson wrote: >>>> Support direct requests that span multiple pnfs data servers by >>>> comparing nfs_pgio_header->verf to a cached verf in pnfs_commit_bucket. >>>> Continue to use dreq->verf if the MDS is used / non-pNFS. >>>> >>>> Signed-off-by: Weston Andros Adamson <dros@primarydata.com> >>>> --- >>>> fs/nfs/direct.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++--- >>>> fs/nfs/nfs4filelayout.c | 6 +++ >>>> include/linux/nfs.h | 5 ++- >>>> include/linux/nfs_xdr.h | 2 + >>>> 4 files changed, 105 insertions(+), 6 deletions(-) >>>> >>>> diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c >>>> index 2c0e08f..9349933 100644 >>>> --- a/fs/nfs/direct.c >>>> +++ b/fs/nfs/direct.c >>>> @@ -108,6 +108,93 @@ static inline int put_dreq(struct nfs_direct_req *dreq) >>>> return atomic_dec_and_test(&dreq->io_count); >>>> } >>>> >>>> +/* >>>> + * nfs_direct_select_verf - select the right verifier >>>> + * @dreq - direct request possibly spanning multiple servers >>>> + * @ds_clp - nfs_client of data server or NULL if MDS / non-pnfs >>>> + * @ds_idx - index of data server in data server list, only valid if ds_clp set >>>> + * >>>> + * returns the correct verifier to use given the role of the server >>>> + */ >>>> +static struct nfs_writeverf * >>>> +nfs_direct_select_verf(struct nfs_direct_req *dreq, >>>> + struct nfs_client *ds_clp, >>>> + int ds_idx) >>>> +{ >>>> + struct nfs_writeverf *verfp = &dreq->verf; >>>> + >>>> + if (ds_clp) { >>>> + /* pNFS is in use, use the DS verf */ >>>> + if (ds_idx >= 0 && ds_idx < dreq->ds_cinfo.nbuckets) >>> The struct pnfs_ds_commit_info is empty if CONFIG_NFS_V4_1=n, so this won't compile. >>> >>> Anna >> Good catch, I?ll iterate through the patchset and test with v4.1 disabled. >> >> Time to add some #ifdefs >> >> -dros > That was the only problem I found with 4.1 disabled. > > Fixed and pushed. Thanks! I'm now getting a "defined but not used" error if just v2 is enabled: NFS_V4_2=n PNFS_OBJLAYOUT=n PNFS_BLOCK=n PNFS_FILE_LAYOUT=n NFS_V4_1=n NFS_V4=n NFS_V3=n NFS_V2=y fs/nfs/direct.c:189:12: error: ‘nfs_direct_cmp_commit_data_verf’ defined but not used [-Werror=unused-function] static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq, Anna > > -dros > >>>> + verfp = &dreq->ds_cinfo.buckets[ds_idx].direct_verf; >>>> + else >>>> + WARN_ON_ONCE(1); >>>> + } >>>> + return verfp; >>>> +} >>>> + >>>> + >>>> +/* >>>> + * nfs_direct_set_hdr_verf - set the write/commit verifier >>>> + * @dreq - direct request possibly spanning multiple servers >>>> + * @hdr - pageio header to validate against previously seen verfs >>>> + * >>>> + * Set the server's (MDS or DS) "seen" verifier >>>> + */ >>>> +static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq, >>>> + struct nfs_pgio_header *hdr) >>>> +{ >>>> + struct nfs_writeverf *verfp; >>>> + >>>> + verfp = nfs_direct_select_verf(dreq, hdr->data->ds_clp, >>>> + hdr->data->ds_idx); >>>> + WARN_ON_ONCE(verfp->committed >= 0); >>>> + memcpy(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); >>>> + WARN_ON_ONCE(verfp->committed < 0); >>>> +} >>>> + >>>> +/* >>>> + * nfs_direct_cmp_hdr_verf - compare verifier for pgio header >>>> + * @dreq - direct request possibly spanning multiple servers >>>> + * @hdr - pageio header to validate against previously seen verf >>>> + * >>>> + * set the server's "seen" verf if not initialized. >>>> + * returns result of comparison between @hdr->verf and the "seen" >>>> + * verf of the server used by @hdr (DS or MDS) >>>> + */ >>>> +static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq, >>>> + struct nfs_pgio_header *hdr) >>>> +{ >>>> + struct nfs_writeverf *verfp; >>>> + >>>> + verfp = nfs_direct_select_verf(dreq, hdr->data->ds_clp, >>>> + hdr->data->ds_idx); >>>> + if (verfp->committed < 0) { >>>> + nfs_direct_set_hdr_verf(dreq, hdr); >>>> + return 0; >>>> + } >>>> + return memcmp(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); >>>> +} >>>> + >>>> +/* >>>> + * nfs_direct_cmp_commit_data_verf - compare verifier for commit data >>>> + * @dreq - direct request possibly spanning multiple servers >>>> + * @data - commit data to validate against previously seen verf >>>> + * >>>> + * returns result of comparison between @data->verf and the verf of >>>> + * the server used by @data (DS or MDS) >>>> + */ >>>> +static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq, >>>> + struct nfs_commit_data *data) >>>> +{ >>>> + struct nfs_writeverf *verfp; >>>> + >>>> + verfp = nfs_direct_select_verf(dreq, data->ds_clp, >>>> + data->ds_commit_index); >>>> + WARN_ON_ONCE(verfp->committed < 0); >>>> + return memcmp(verfp, &data->verf, sizeof(struct nfs_writeverf)); >>>> +} >>>> + >>>> /** >>>> * nfs_direct_IO - NFS address space operation for direct I/O >>>> * @rw: direction (read or write) >>>> @@ -168,6 +255,7 @@ static inline struct nfs_direct_req *nfs_direct_req_alloc(void) >>>> kref_get(&dreq->kref); >>>> init_completion(&dreq->completion); >>>> INIT_LIST_HEAD(&dreq->mds_cinfo.list); >>>> + dreq->verf.committed = NFS_INVALID_STABLE_HOW; /* not set yet */ >>>> INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); >>>> spin_lock_init(&dreq->lock); >>>> >>>> @@ -602,7 +690,7 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data) >>>> dprintk("NFS: %5u commit failed with error %d.\n", >>>> data->task.tk_pid, status); >>>> dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >>>> - } else if (memcmp(&dreq->verf, &data->verf, sizeof(data->verf))) { >>>> + } else if (nfs_direct_cmp_commit_data_verf(dreq, data)) { >>>> dprintk("NFS: %5u commit verify failed\n", data->task.tk_pid); >>>> dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >>>> } >>>> @@ -811,13 +899,13 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr) >>>> if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) >>>> bit = NFS_IOHDR_NEED_RESCHED; >>>> else if (dreq->flags == 0) { >>>> - memcpy(&dreq->verf, &hdr->verf, >>>> - sizeof(dreq->verf)); >>>> + nfs_direct_set_hdr_verf(dreq, hdr); >>>> bit = NFS_IOHDR_NEED_COMMIT; >>>> dreq->flags = NFS_ODIRECT_DO_COMMIT; >>>> } else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) { >>>> - if (memcmp(&dreq->verf, &hdr->verf, sizeof(dreq->verf))) { >>>> - dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >>>> + if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr)) { >>>> + dreq->flags = >>>> + NFS_ODIRECT_RESCHED_WRITES; >>>> bit = NFS_IOHDR_NEED_RESCHED; >>>> } else >>>> bit = NFS_IOHDR_NEED_COMMIT; >>>> diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c >>>> index 7a665e0..0ebc521 100644 >>>> --- a/fs/nfs/nfs4filelayout.c >>>> +++ b/fs/nfs/nfs4filelayout.c >>>> @@ -560,6 +560,7 @@ filelayout_read_pagelist(struct nfs_pgio_data *data) >>>> /* No multipath support. Use first DS */ >>>> atomic_inc(&ds->ds_clp->cl_count); >>>> data->ds_clp = ds->ds_clp; >>>> + data->ds_idx = idx; >>>> fh = nfs4_fl_select_ds_fh(lseg, j); >>>> if (fh) >>>> data->args.fh = fh; >>>> @@ -603,6 +604,7 @@ filelayout_write_pagelist(struct nfs_pgio_data *data, int sync) >>>> data->pgio_done_cb = filelayout_write_done_cb; >>>> atomic_inc(&ds->ds_clp->cl_count); >>>> data->ds_clp = ds->ds_clp; >>>> + data->ds_idx = idx; >>>> fh = nfs4_fl_select_ds_fh(lseg, j); >>>> if (fh) >>>> data->args.fh = fh; >>>> @@ -875,6 +877,8 @@ filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, >>>> for (i = 0; i < size; i++) { >>>> INIT_LIST_HEAD(&buckets[i].written); >>>> INIT_LIST_HEAD(&buckets[i].committing); >>>> + /* mark direct verifier as unset */ >>>> + buckets[i].direct_verf.committed = NFS_INVALID_STABLE_HOW; >>>> } >>>> >>>> spin_lock(cinfo->lock); >>>> @@ -885,6 +889,8 @@ filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, >>>> &buckets[i].written); >>>> list_splice(&cinfo->ds->buckets[i].committing, >>>> &buckets[i].committing); >>>> + buckets[i].direct_verf.committed = >>>> + cinfo->ds->buckets[i].direct_verf.committed; >>>> buckets[i].wlseg = cinfo->ds->buckets[i].wlseg; >>>> buckets[i].clseg = cinfo->ds->buckets[i].clseg; >>>> } >>>> diff --git a/include/linux/nfs.h b/include/linux/nfs.h >>>> index 3e794c1..610af51 100644 >>>> --- a/include/linux/nfs.h >>>> +++ b/include/linux/nfs.h >>>> @@ -46,6 +46,9 @@ static inline void nfs_copy_fh(struct nfs_fh *target, const struct nfs_fh *sourc >>>> enum nfs3_stable_how { >>>> NFS_UNSTABLE = 0, >>>> NFS_DATA_SYNC = 1, >>>> - NFS_FILE_SYNC = 2 >>>> + NFS_FILE_SYNC = 2, >>>> + >>>> + /* used by direct.c to mark verf as invalid */ >>>> + NFS_INVALID_STABLE_HOW = -1 >>>> }; >>>> #endif /* _LINUX_NFS_H */ >>>> diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h >>>> index 29828c7..bb9fb88 100644 >>>> --- a/include/linux/nfs_xdr.h >>>> +++ b/include/linux/nfs_xdr.h >>>> @@ -1111,6 +1111,7 @@ struct pnfs_commit_bucket { >>>> struct list_head committing; >>>> struct pnfs_layout_segment *wlseg; >>>> struct pnfs_layout_segment *clseg; >>>> + struct nfs_writeverf direct_verf; >>>> }; >>>> >>>> struct pnfs_ds_commit_info { >>>> @@ -1294,6 +1295,7 @@ struct nfs_pgio_data { >>>> __u64 mds_offset; /* Filelayout dense stripe */ >>>> struct nfs_page_array pages; >>>> struct nfs_client *ds_clp; /* pNFS data server */ >>>> + int ds_idx; /* ds index if ds_clp is set */ >>>> }; >>>> >>>> struct nfs_rw_header { -- To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On Apr 25, 2014, at 11:53 AM, Anna Schumaker <schumaker.anna@gmail.com> wrote: > On 04/25/2014 11:38 AM, Weston Andros Adamson wrote: >> On Apr 25, 2014, at 11:12 AM, Weston Andros Adamson <dros@primarydata.com> wrote: >> >>> On Apr 25, 2014, at 10:15 AM, Anna Schumaker <schumaker.anna@gmail.com> wrote: >>> >>>> On 04/24/2014 02:15 PM, Weston Andros Adamson wrote: >>>>> Support direct requests that span multiple pnfs data servers by >>>>> comparing nfs_pgio_header->verf to a cached verf in pnfs_commit_bucket. >>>>> Continue to use dreq->verf if the MDS is used / non-pNFS. >>>>> >>>>> Signed-off-by: Weston Andros Adamson <dros@primarydata.com> >>>>> --- >>>>> fs/nfs/direct.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++--- >>>>> fs/nfs/nfs4filelayout.c | 6 +++ >>>>> include/linux/nfs.h | 5 ++- >>>>> include/linux/nfs_xdr.h | 2 + >>>>> 4 files changed, 105 insertions(+), 6 deletions(-) >>>>> >>>>> diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c >>>>> index 2c0e08f..9349933 100644 >>>>> --- a/fs/nfs/direct.c >>>>> +++ b/fs/nfs/direct.c >>>>> @@ -108,6 +108,93 @@ static inline int put_dreq(struct nfs_direct_req *dreq) >>>>> return atomic_dec_and_test(&dreq->io_count); >>>>> } >>>>> >>>>> +/* >>>>> + * nfs_direct_select_verf - select the right verifier >>>>> + * @dreq - direct request possibly spanning multiple servers >>>>> + * @ds_clp - nfs_client of data server or NULL if MDS / non-pnfs >>>>> + * @ds_idx - index of data server in data server list, only valid if ds_clp set >>>>> + * >>>>> + * returns the correct verifier to use given the role of the server >>>>> + */ >>>>> +static struct nfs_writeverf * >>>>> +nfs_direct_select_verf(struct nfs_direct_req *dreq, >>>>> + struct nfs_client *ds_clp, >>>>> + int ds_idx) >>>>> +{ >>>>> + struct nfs_writeverf *verfp = &dreq->verf; >>>>> + >>>>> + if (ds_clp) { >>>>> + /* pNFS is in use, use the DS verf */ >>>>> + if (ds_idx >= 0 && ds_idx < dreq->ds_cinfo.nbuckets) >>>> The struct pnfs_ds_commit_info is empty if CONFIG_NFS_V4_1=n, so this won't compile. >>>> >>>> Anna >>> Good catch, I?ll iterate through the patchset and test with v4.1 disabled. >>> >>> Time to add some #ifdefs >>> >>> -dros >> That was the only problem I found with 4.1 disabled. >> >> Fixed and pushed. > > Thanks! I'm now getting a "defined but not used" error if just v2 is enabled: > > NFS_V4_2=n PNFS_OBJLAYOUT=n PNFS_BLOCK=n PNFS_FILE_LAYOUT=n NFS_V4_1=n NFS_V4=n NFS_V3=n NFS_V2=y > > fs/nfs/direct.c:189:12: error: ‘nfs_direct_cmp_commit_data_verf’ defined but not used [-Werror=unused-function] > static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq, > > Anna Oh, i didn’t find that because the direct commit path uses: IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4) fixed and pushed. -dros > >> >> -dros >> >>>>> + verfp = &dreq->ds_cinfo.buckets[ds_idx].direct_verf; >>>>> + else >>>>> + WARN_ON_ONCE(1); >>>>> + } >>>>> + return verfp; >>>>> +} >>>>> + >>>>> + >>>>> +/* >>>>> + * nfs_direct_set_hdr_verf - set the write/commit verifier >>>>> + * @dreq - direct request possibly spanning multiple servers >>>>> + * @hdr - pageio header to validate against previously seen verfs >>>>> + * >>>>> + * Set the server's (MDS or DS) "seen" verifier >>>>> + */ >>>>> +static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq, >>>>> + struct nfs_pgio_header *hdr) >>>>> +{ >>>>> + struct nfs_writeverf *verfp; >>>>> + >>>>> + verfp = nfs_direct_select_verf(dreq, hdr->data->ds_clp, >>>>> + hdr->data->ds_idx); >>>>> + WARN_ON_ONCE(verfp->committed >= 0); >>>>> + memcpy(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); >>>>> + WARN_ON_ONCE(verfp->committed < 0); >>>>> +} >>>>> + >>>>> +/* >>>>> + * nfs_direct_cmp_hdr_verf - compare verifier for pgio header >>>>> + * @dreq - direct request possibly spanning multiple servers >>>>> + * @hdr - pageio header to validate against previously seen verf >>>>> + * >>>>> + * set the server's "seen" verf if not initialized. >>>>> + * returns result of comparison between @hdr->verf and the "seen" >>>>> + * verf of the server used by @hdr (DS or MDS) >>>>> + */ >>>>> +static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq, >>>>> + struct nfs_pgio_header *hdr) >>>>> +{ >>>>> + struct nfs_writeverf *verfp; >>>>> + >>>>> + verfp = nfs_direct_select_verf(dreq, hdr->data->ds_clp, >>>>> + hdr->data->ds_idx); >>>>> + if (verfp->committed < 0) { >>>>> + nfs_direct_set_hdr_verf(dreq, hdr); >>>>> + return 0; >>>>> + } >>>>> + return memcmp(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); >>>>> +} >>>>> + >>>>> +/* >>>>> + * nfs_direct_cmp_commit_data_verf - compare verifier for commit data >>>>> + * @dreq - direct request possibly spanning multiple servers >>>>> + * @data - commit data to validate against previously seen verf >>>>> + * >>>>> + * returns result of comparison between @data->verf and the verf of >>>>> + * the server used by @data (DS or MDS) >>>>> + */ >>>>> +static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq, >>>>> + struct nfs_commit_data *data) >>>>> +{ >>>>> + struct nfs_writeverf *verfp; >>>>> + >>>>> + verfp = nfs_direct_select_verf(dreq, data->ds_clp, >>>>> + data->ds_commit_index); >>>>> + WARN_ON_ONCE(verfp->committed < 0); >>>>> + return memcmp(verfp, &data->verf, sizeof(struct nfs_writeverf)); >>>>> +} >>>>> + >>>>> /** >>>>> * nfs_direct_IO - NFS address space operation for direct I/O >>>>> * @rw: direction (read or write) >>>>> @@ -168,6 +255,7 @@ static inline struct nfs_direct_req *nfs_direct_req_alloc(void) >>>>> kref_get(&dreq->kref); >>>>> init_completion(&dreq->completion); >>>>> INIT_LIST_HEAD(&dreq->mds_cinfo.list); >>>>> + dreq->verf.committed = NFS_INVALID_STABLE_HOW; /* not set yet */ >>>>> INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); >>>>> spin_lock_init(&dreq->lock); >>>>> >>>>> @@ -602,7 +690,7 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data) >>>>> dprintk("NFS: %5u commit failed with error %d.\n", >>>>> data->task.tk_pid, status); >>>>> dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >>>>> - } else if (memcmp(&dreq->verf, &data->verf, sizeof(data->verf))) { >>>>> + } else if (nfs_direct_cmp_commit_data_verf(dreq, data)) { >>>>> dprintk("NFS: %5u commit verify failed\n", data->task.tk_pid); >>>>> dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >>>>> } >>>>> @@ -811,13 +899,13 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr) >>>>> if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) >>>>> bit = NFS_IOHDR_NEED_RESCHED; >>>>> else if (dreq->flags == 0) { >>>>> - memcpy(&dreq->verf, &hdr->verf, >>>>> - sizeof(dreq->verf)); >>>>> + nfs_direct_set_hdr_verf(dreq, hdr); >>>>> bit = NFS_IOHDR_NEED_COMMIT; >>>>> dreq->flags = NFS_ODIRECT_DO_COMMIT; >>>>> } else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) { >>>>> - if (memcmp(&dreq->verf, &hdr->verf, sizeof(dreq->verf))) { >>>>> - dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >>>>> + if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr)) { >>>>> + dreq->flags = >>>>> + NFS_ODIRECT_RESCHED_WRITES; >>>>> bit = NFS_IOHDR_NEED_RESCHED; >>>>> } else >>>>> bit = NFS_IOHDR_NEED_COMMIT; >>>>> diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c >>>>> index 7a665e0..0ebc521 100644 >>>>> --- a/fs/nfs/nfs4filelayout.c >>>>> +++ b/fs/nfs/nfs4filelayout.c >>>>> @@ -560,6 +560,7 @@ filelayout_read_pagelist(struct nfs_pgio_data *data) >>>>> /* No multipath support. Use first DS */ >>>>> atomic_inc(&ds->ds_clp->cl_count); >>>>> data->ds_clp = ds->ds_clp; >>>>> + data->ds_idx = idx; >>>>> fh = nfs4_fl_select_ds_fh(lseg, j); >>>>> if (fh) >>>>> data->args.fh = fh; >>>>> @@ -603,6 +604,7 @@ filelayout_write_pagelist(struct nfs_pgio_data *data, int sync) >>>>> data->pgio_done_cb = filelayout_write_done_cb; >>>>> atomic_inc(&ds->ds_clp->cl_count); >>>>> data->ds_clp = ds->ds_clp; >>>>> + data->ds_idx = idx; >>>>> fh = nfs4_fl_select_ds_fh(lseg, j); >>>>> if (fh) >>>>> data->args.fh = fh; >>>>> @@ -875,6 +877,8 @@ filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, >>>>> for (i = 0; i < size; i++) { >>>>> INIT_LIST_HEAD(&buckets[i].written); >>>>> INIT_LIST_HEAD(&buckets[i].committing); >>>>> + /* mark direct verifier as unset */ >>>>> + buckets[i].direct_verf.committed = NFS_INVALID_STABLE_HOW; >>>>> } >>>>> >>>>> spin_lock(cinfo->lock); >>>>> @@ -885,6 +889,8 @@ filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, >>>>> &buckets[i].written); >>>>> list_splice(&cinfo->ds->buckets[i].committing, >>>>> &buckets[i].committing); >>>>> + buckets[i].direct_verf.committed = >>>>> + cinfo->ds->buckets[i].direct_verf.committed; >>>>> buckets[i].wlseg = cinfo->ds->buckets[i].wlseg; >>>>> buckets[i].clseg = cinfo->ds->buckets[i].clseg; >>>>> } >>>>> diff --git a/include/linux/nfs.h b/include/linux/nfs.h >>>>> index 3e794c1..610af51 100644 >>>>> --- a/include/linux/nfs.h >>>>> +++ b/include/linux/nfs.h >>>>> @@ -46,6 +46,9 @@ static inline void nfs_copy_fh(struct nfs_fh *target, const struct nfs_fh *sourc >>>>> enum nfs3_stable_how { >>>>> NFS_UNSTABLE = 0, >>>>> NFS_DATA_SYNC = 1, >>>>> - NFS_FILE_SYNC = 2 >>>>> + NFS_FILE_SYNC = 2, >>>>> + >>>>> + /* used by direct.c to mark verf as invalid */ >>>>> + NFS_INVALID_STABLE_HOW = -1 >>>>> }; >>>>> #endif /* _LINUX_NFS_H */ >>>>> diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h >>>>> index 29828c7..bb9fb88 100644 >>>>> --- a/include/linux/nfs_xdr.h >>>>> +++ b/include/linux/nfs_xdr.h >>>>> @@ -1111,6 +1111,7 @@ struct pnfs_commit_bucket { >>>>> struct list_head committing; >>>>> struct pnfs_layout_segment *wlseg; >>>>> struct pnfs_layout_segment *clseg; >>>>> + struct nfs_writeverf direct_verf; >>>>> }; >>>>> >>>>> struct pnfs_ds_commit_info { >>>>> @@ -1294,6 +1295,7 @@ struct nfs_pgio_data { >>>>> __u64 mds_offset; /* Filelayout dense stripe */ >>>>> struct nfs_page_array pages; >>>>> struct nfs_client *ds_clp; /* pNFS data server */ >>>>> + int ds_idx; /* ds index if ds_clp is set */ >>>>> }; >>>>> >>>>> struct nfs_rw_header { -- To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
On 04/25/2014 12:04 PM, Weston Andros Adamson wrote: > On Apr 25, 2014, at 11:53 AM, Anna Schumaker <schumaker.anna@gmail.com> wrote: > >> On 04/25/2014 11:38 AM, Weston Andros Adamson wrote: >>> On Apr 25, 2014, at 11:12 AM, Weston Andros Adamson <dros@primarydata.com> wrote: >>> >>>> On Apr 25, 2014, at 10:15 AM, Anna Schumaker <schumaker.anna@gmail.com> wrote: >>>> >>>>> On 04/24/2014 02:15 PM, Weston Andros Adamson wrote: >>>>>> Support direct requests that span multiple pnfs data servers by >>>>>> comparing nfs_pgio_header->verf to a cached verf in pnfs_commit_bucket. >>>>>> Continue to use dreq->verf if the MDS is used / non-pNFS. >>>>>> >>>>>> Signed-off-by: Weston Andros Adamson <dros@primarydata.com> >>>>>> --- >>>>>> fs/nfs/direct.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++--- >>>>>> fs/nfs/nfs4filelayout.c | 6 +++ >>>>>> include/linux/nfs.h | 5 ++- >>>>>> include/linux/nfs_xdr.h | 2 + >>>>>> 4 files changed, 105 insertions(+), 6 deletions(-) >>>>>> >>>>>> diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c >>>>>> index 2c0e08f..9349933 100644 >>>>>> --- a/fs/nfs/direct.c >>>>>> +++ b/fs/nfs/direct.c >>>>>> @@ -108,6 +108,93 @@ static inline int put_dreq(struct nfs_direct_req *dreq) >>>>>> return atomic_dec_and_test(&dreq->io_count); >>>>>> } >>>>>> >>>>>> +/* >>>>>> + * nfs_direct_select_verf - select the right verifier >>>>>> + * @dreq - direct request possibly spanning multiple servers >>>>>> + * @ds_clp - nfs_client of data server or NULL if MDS / non-pnfs >>>>>> + * @ds_idx - index of data server in data server list, only valid if ds_clp set >>>>>> + * >>>>>> + * returns the correct verifier to use given the role of the server >>>>>> + */ >>>>>> +static struct nfs_writeverf * >>>>>> +nfs_direct_select_verf(struct nfs_direct_req *dreq, >>>>>> + struct nfs_client *ds_clp, >>>>>> + int ds_idx) >>>>>> +{ >>>>>> + struct nfs_writeverf *verfp = &dreq->verf; >>>>>> + >>>>>> + if (ds_clp) { >>>>>> + /* pNFS is in use, use the DS verf */ >>>>>> + if (ds_idx >= 0 && ds_idx < dreq->ds_cinfo.nbuckets) >>>>> The struct pnfs_ds_commit_info is empty if CONFIG_NFS_V4_1=n, so this won't compile. >>>>> >>>>> Anna >>>> Good catch, I?ll iterate through the patchset and test with v4.1 disabled. >>>> >>>> Time to add some #ifdefs >>>> >>>> -dros >>> That was the only problem I found with 4.1 disabled. >>> >>> Fixed and pushed. >> Thanks! I'm now getting a "defined but not used" error if just v2 is enabled: >> >> NFS_V4_2=n PNFS_OBJLAYOUT=n PNFS_BLOCK=n PNFS_FILE_LAYOUT=n NFS_V4_1=n NFS_V4=n NFS_V3=n NFS_V2=y >> >> fs/nfs/direct.c:189:12: error: ‘nfs_direct_cmp_commit_data_verf’ defined but not used [-Werror=unused-function] >> static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq, >> >> Anna > Oh, i didn’t find that because the direct commit path uses: > > IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4) > > fixed and pushed. Thanks! I haven't hit any other problems. Anna > > -dros > >>> -dros >>> >>>>>> + verfp = &dreq->ds_cinfo.buckets[ds_idx].direct_verf; >>>>>> + else >>>>>> + WARN_ON_ONCE(1); >>>>>> + } >>>>>> + return verfp; >>>>>> +} >>>>>> + >>>>>> + >>>>>> +/* >>>>>> + * nfs_direct_set_hdr_verf - set the write/commit verifier >>>>>> + * @dreq - direct request possibly spanning multiple servers >>>>>> + * @hdr - pageio header to validate against previously seen verfs >>>>>> + * >>>>>> + * Set the server's (MDS or DS) "seen" verifier >>>>>> + */ >>>>>> +static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq, >>>>>> + struct nfs_pgio_header *hdr) >>>>>> +{ >>>>>> + struct nfs_writeverf *verfp; >>>>>> + >>>>>> + verfp = nfs_direct_select_verf(dreq, hdr->data->ds_clp, >>>>>> + hdr->data->ds_idx); >>>>>> + WARN_ON_ONCE(verfp->committed >= 0); >>>>>> + memcpy(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); >>>>>> + WARN_ON_ONCE(verfp->committed < 0); >>>>>> +} >>>>>> + >>>>>> +/* >>>>>> + * nfs_direct_cmp_hdr_verf - compare verifier for pgio header >>>>>> + * @dreq - direct request possibly spanning multiple servers >>>>>> + * @hdr - pageio header to validate against previously seen verf >>>>>> + * >>>>>> + * set the server's "seen" verf if not initialized. >>>>>> + * returns result of comparison between @hdr->verf and the "seen" >>>>>> + * verf of the server used by @hdr (DS or MDS) >>>>>> + */ >>>>>> +static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq, >>>>>> + struct nfs_pgio_header *hdr) >>>>>> +{ >>>>>> + struct nfs_writeverf *verfp; >>>>>> + >>>>>> + verfp = nfs_direct_select_verf(dreq, hdr->data->ds_clp, >>>>>> + hdr->data->ds_idx); >>>>>> + if (verfp->committed < 0) { >>>>>> + nfs_direct_set_hdr_verf(dreq, hdr); >>>>>> + return 0; >>>>>> + } >>>>>> + return memcmp(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); >>>>>> +} >>>>>> + >>>>>> +/* >>>>>> + * nfs_direct_cmp_commit_data_verf - compare verifier for commit data >>>>>> + * @dreq - direct request possibly spanning multiple servers >>>>>> + * @data - commit data to validate against previously seen verf >>>>>> + * >>>>>> + * returns result of comparison between @data->verf and the verf of >>>>>> + * the server used by @data (DS or MDS) >>>>>> + */ >>>>>> +static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq, >>>>>> + struct nfs_commit_data *data) >>>>>> +{ >>>>>> + struct nfs_writeverf *verfp; >>>>>> + >>>>>> + verfp = nfs_direct_select_verf(dreq, data->ds_clp, >>>>>> + data->ds_commit_index); >>>>>> + WARN_ON_ONCE(verfp->committed < 0); >>>>>> + return memcmp(verfp, &data->verf, sizeof(struct nfs_writeverf)); >>>>>> +} >>>>>> + >>>>>> /** >>>>>> * nfs_direct_IO - NFS address space operation for direct I/O >>>>>> * @rw: direction (read or write) >>>>>> @@ -168,6 +255,7 @@ static inline struct nfs_direct_req *nfs_direct_req_alloc(void) >>>>>> kref_get(&dreq->kref); >>>>>> init_completion(&dreq->completion); >>>>>> INIT_LIST_HEAD(&dreq->mds_cinfo.list); >>>>>> + dreq->verf.committed = NFS_INVALID_STABLE_HOW; /* not set yet */ >>>>>> INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); >>>>>> spin_lock_init(&dreq->lock); >>>>>> >>>>>> @@ -602,7 +690,7 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data) >>>>>> dprintk("NFS: %5u commit failed with error %d.\n", >>>>>> data->task.tk_pid, status); >>>>>> dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >>>>>> - } else if (memcmp(&dreq->verf, &data->verf, sizeof(data->verf))) { >>>>>> + } else if (nfs_direct_cmp_commit_data_verf(dreq, data)) { >>>>>> dprintk("NFS: %5u commit verify failed\n", data->task.tk_pid); >>>>>> dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >>>>>> } >>>>>> @@ -811,13 +899,13 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr) >>>>>> if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) >>>>>> bit = NFS_IOHDR_NEED_RESCHED; >>>>>> else if (dreq->flags == 0) { >>>>>> - memcpy(&dreq->verf, &hdr->verf, >>>>>> - sizeof(dreq->verf)); >>>>>> + nfs_direct_set_hdr_verf(dreq, hdr); >>>>>> bit = NFS_IOHDR_NEED_COMMIT; >>>>>> dreq->flags = NFS_ODIRECT_DO_COMMIT; >>>>>> } else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) { >>>>>> - if (memcmp(&dreq->verf, &hdr->verf, sizeof(dreq->verf))) { >>>>>> - dreq->flags = NFS_ODIRECT_RESCHED_WRITES; >>>>>> + if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr)) { >>>>>> + dreq->flags = >>>>>> + NFS_ODIRECT_RESCHED_WRITES; >>>>>> bit = NFS_IOHDR_NEED_RESCHED; >>>>>> } else >>>>>> bit = NFS_IOHDR_NEED_COMMIT; >>>>>> diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c >>>>>> index 7a665e0..0ebc521 100644 >>>>>> --- a/fs/nfs/nfs4filelayout.c >>>>>> +++ b/fs/nfs/nfs4filelayout.c >>>>>> @@ -560,6 +560,7 @@ filelayout_read_pagelist(struct nfs_pgio_data *data) >>>>>> /* No multipath support. Use first DS */ >>>>>> atomic_inc(&ds->ds_clp->cl_count); >>>>>> data->ds_clp = ds->ds_clp; >>>>>> + data->ds_idx = idx; >>>>>> fh = nfs4_fl_select_ds_fh(lseg, j); >>>>>> if (fh) >>>>>> data->args.fh = fh; >>>>>> @@ -603,6 +604,7 @@ filelayout_write_pagelist(struct nfs_pgio_data *data, int sync) >>>>>> data->pgio_done_cb = filelayout_write_done_cb; >>>>>> atomic_inc(&ds->ds_clp->cl_count); >>>>>> data->ds_clp = ds->ds_clp; >>>>>> + data->ds_idx = idx; >>>>>> fh = nfs4_fl_select_ds_fh(lseg, j); >>>>>> if (fh) >>>>>> data->args.fh = fh; >>>>>> @@ -875,6 +877,8 @@ filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, >>>>>> for (i = 0; i < size; i++) { >>>>>> INIT_LIST_HEAD(&buckets[i].written); >>>>>> INIT_LIST_HEAD(&buckets[i].committing); >>>>>> + /* mark direct verifier as unset */ >>>>>> + buckets[i].direct_verf.committed = NFS_INVALID_STABLE_HOW; >>>>>> } >>>>>> >>>>>> spin_lock(cinfo->lock); >>>>>> @@ -885,6 +889,8 @@ filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, >>>>>> &buckets[i].written); >>>>>> list_splice(&cinfo->ds->buckets[i].committing, >>>>>> &buckets[i].committing); >>>>>> + buckets[i].direct_verf.committed = >>>>>> + cinfo->ds->buckets[i].direct_verf.committed; >>>>>> buckets[i].wlseg = cinfo->ds->buckets[i].wlseg; >>>>>> buckets[i].clseg = cinfo->ds->buckets[i].clseg; >>>>>> } >>>>>> diff --git a/include/linux/nfs.h b/include/linux/nfs.h >>>>>> index 3e794c1..610af51 100644 >>>>>> --- a/include/linux/nfs.h >>>>>> +++ b/include/linux/nfs.h >>>>>> @@ -46,6 +46,9 @@ static inline void nfs_copy_fh(struct nfs_fh *target, const struct nfs_fh *sourc >>>>>> enum nfs3_stable_how { >>>>>> NFS_UNSTABLE = 0, >>>>>> NFS_DATA_SYNC = 1, >>>>>> - NFS_FILE_SYNC = 2 >>>>>> + NFS_FILE_SYNC = 2, >>>>>> + >>>>>> + /* used by direct.c to mark verf as invalid */ >>>>>> + NFS_INVALID_STABLE_HOW = -1 >>>>>> }; >>>>>> #endif /* _LINUX_NFS_H */ >>>>>> diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h >>>>>> index 29828c7..bb9fb88 100644 >>>>>> --- a/include/linux/nfs_xdr.h >>>>>> +++ b/include/linux/nfs_xdr.h >>>>>> @@ -1111,6 +1111,7 @@ struct pnfs_commit_bucket { >>>>>> struct list_head committing; >>>>>> struct pnfs_layout_segment *wlseg; >>>>>> struct pnfs_layout_segment *clseg; >>>>>> + struct nfs_writeverf direct_verf; >>>>>> }; >>>>>> >>>>>> struct pnfs_ds_commit_info { >>>>>> @@ -1294,6 +1295,7 @@ struct nfs_pgio_data { >>>>>> __u64 mds_offset; /* Filelayout dense stripe */ >>>>>> struct nfs_page_array pages; >>>>>> struct nfs_client *ds_clp; /* pNFS data server */ >>>>>> + int ds_idx; /* ds index if ds_clp is set */ >>>>>> }; >>>>>> >>>>>> struct nfs_rw_header { -- To unsubscribe from this list: send the line "unsubscribe linux-nfs" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index 2c0e08f..9349933 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -108,6 +108,93 @@ static inline int put_dreq(struct nfs_direct_req *dreq) return atomic_dec_and_test(&dreq->io_count); } +/* + * nfs_direct_select_verf - select the right verifier + * @dreq - direct request possibly spanning multiple servers + * @ds_clp - nfs_client of data server or NULL if MDS / non-pnfs + * @ds_idx - index of data server in data server list, only valid if ds_clp set + * + * returns the correct verifier to use given the role of the server + */ +static struct nfs_writeverf * +nfs_direct_select_verf(struct nfs_direct_req *dreq, + struct nfs_client *ds_clp, + int ds_idx) +{ + struct nfs_writeverf *verfp = &dreq->verf; + + if (ds_clp) { + /* pNFS is in use, use the DS verf */ + if (ds_idx >= 0 && ds_idx < dreq->ds_cinfo.nbuckets) + verfp = &dreq->ds_cinfo.buckets[ds_idx].direct_verf; + else + WARN_ON_ONCE(1); + } + return verfp; +} + + +/* + * nfs_direct_set_hdr_verf - set the write/commit verifier + * @dreq - direct request possibly spanning multiple servers + * @hdr - pageio header to validate against previously seen verfs + * + * Set the server's (MDS or DS) "seen" verifier + */ +static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq, + struct nfs_pgio_header *hdr) +{ + struct nfs_writeverf *verfp; + + verfp = nfs_direct_select_verf(dreq, hdr->data->ds_clp, + hdr->data->ds_idx); + WARN_ON_ONCE(verfp->committed >= 0); + memcpy(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); + WARN_ON_ONCE(verfp->committed < 0); +} + +/* + * nfs_direct_cmp_hdr_verf - compare verifier for pgio header + * @dreq - direct request possibly spanning multiple servers + * @hdr - pageio header to validate against previously seen verf + * + * set the server's "seen" verf if not initialized. + * returns result of comparison between @hdr->verf and the "seen" + * verf of the server used by @hdr (DS or MDS) + */ +static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq, + struct nfs_pgio_header *hdr) +{ + struct nfs_writeverf *verfp; + + verfp = nfs_direct_select_verf(dreq, hdr->data->ds_clp, + hdr->data->ds_idx); + if (verfp->committed < 0) { + nfs_direct_set_hdr_verf(dreq, hdr); + return 0; + } + return memcmp(verfp, &hdr->verf, sizeof(struct nfs_writeverf)); +} + +/* + * nfs_direct_cmp_commit_data_verf - compare verifier for commit data + * @dreq - direct request possibly spanning multiple servers + * @data - commit data to validate against previously seen verf + * + * returns result of comparison between @data->verf and the verf of + * the server used by @data (DS or MDS) + */ +static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq, + struct nfs_commit_data *data) +{ + struct nfs_writeverf *verfp; + + verfp = nfs_direct_select_verf(dreq, data->ds_clp, + data->ds_commit_index); + WARN_ON_ONCE(verfp->committed < 0); + return memcmp(verfp, &data->verf, sizeof(struct nfs_writeverf)); +} + /** * nfs_direct_IO - NFS address space operation for direct I/O * @rw: direction (read or write) @@ -168,6 +255,7 @@ static inline struct nfs_direct_req *nfs_direct_req_alloc(void) kref_get(&dreq->kref); init_completion(&dreq->completion); INIT_LIST_HEAD(&dreq->mds_cinfo.list); + dreq->verf.committed = NFS_INVALID_STABLE_HOW; /* not set yet */ INIT_WORK(&dreq->work, nfs_direct_write_schedule_work); spin_lock_init(&dreq->lock); @@ -602,7 +690,7 @@ static void nfs_direct_commit_complete(struct nfs_commit_data *data) dprintk("NFS: %5u commit failed with error %d.\n", data->task.tk_pid, status); dreq->flags = NFS_ODIRECT_RESCHED_WRITES; - } else if (memcmp(&dreq->verf, &data->verf, sizeof(data->verf))) { + } else if (nfs_direct_cmp_commit_data_verf(dreq, data)) { dprintk("NFS: %5u commit verify failed\n", data->task.tk_pid); dreq->flags = NFS_ODIRECT_RESCHED_WRITES; } @@ -811,13 +899,13 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr) if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) bit = NFS_IOHDR_NEED_RESCHED; else if (dreq->flags == 0) { - memcpy(&dreq->verf, &hdr->verf, - sizeof(dreq->verf)); + nfs_direct_set_hdr_verf(dreq, hdr); bit = NFS_IOHDR_NEED_COMMIT; dreq->flags = NFS_ODIRECT_DO_COMMIT; } else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) { - if (memcmp(&dreq->verf, &hdr->verf, sizeof(dreq->verf))) { - dreq->flags = NFS_ODIRECT_RESCHED_WRITES; + if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr)) { + dreq->flags = + NFS_ODIRECT_RESCHED_WRITES; bit = NFS_IOHDR_NEED_RESCHED; } else bit = NFS_IOHDR_NEED_COMMIT; diff --git a/fs/nfs/nfs4filelayout.c b/fs/nfs/nfs4filelayout.c index 7a665e0..0ebc521 100644 --- a/fs/nfs/nfs4filelayout.c +++ b/fs/nfs/nfs4filelayout.c @@ -560,6 +560,7 @@ filelayout_read_pagelist(struct nfs_pgio_data *data) /* No multipath support. Use first DS */ atomic_inc(&ds->ds_clp->cl_count); data->ds_clp = ds->ds_clp; + data->ds_idx = idx; fh = nfs4_fl_select_ds_fh(lseg, j); if (fh) data->args.fh = fh; @@ -603,6 +604,7 @@ filelayout_write_pagelist(struct nfs_pgio_data *data, int sync) data->pgio_done_cb = filelayout_write_done_cb; atomic_inc(&ds->ds_clp->cl_count); data->ds_clp = ds->ds_clp; + data->ds_idx = idx; fh = nfs4_fl_select_ds_fh(lseg, j); if (fh) data->args.fh = fh; @@ -875,6 +877,8 @@ filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, for (i = 0; i < size; i++) { INIT_LIST_HEAD(&buckets[i].written); INIT_LIST_HEAD(&buckets[i].committing); + /* mark direct verifier as unset */ + buckets[i].direct_verf.committed = NFS_INVALID_STABLE_HOW; } spin_lock(cinfo->lock); @@ -885,6 +889,8 @@ filelayout_alloc_commit_info(struct pnfs_layout_segment *lseg, &buckets[i].written); list_splice(&cinfo->ds->buckets[i].committing, &buckets[i].committing); + buckets[i].direct_verf.committed = + cinfo->ds->buckets[i].direct_verf.committed; buckets[i].wlseg = cinfo->ds->buckets[i].wlseg; buckets[i].clseg = cinfo->ds->buckets[i].clseg; } diff --git a/include/linux/nfs.h b/include/linux/nfs.h index 3e794c1..610af51 100644 --- a/include/linux/nfs.h +++ b/include/linux/nfs.h @@ -46,6 +46,9 @@ static inline void nfs_copy_fh(struct nfs_fh *target, const struct nfs_fh *sourc enum nfs3_stable_how { NFS_UNSTABLE = 0, NFS_DATA_SYNC = 1, - NFS_FILE_SYNC = 2 + NFS_FILE_SYNC = 2, + + /* used by direct.c to mark verf as invalid */ + NFS_INVALID_STABLE_HOW = -1 }; #endif /* _LINUX_NFS_H */ diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 29828c7..bb9fb88 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1111,6 +1111,7 @@ struct pnfs_commit_bucket { struct list_head committing; struct pnfs_layout_segment *wlseg; struct pnfs_layout_segment *clseg; + struct nfs_writeverf direct_verf; }; struct pnfs_ds_commit_info { @@ -1294,6 +1295,7 @@ struct nfs_pgio_data { __u64 mds_offset; /* Filelayout dense stripe */ struct nfs_page_array pages; struct nfs_client *ds_clp; /* pNFS data server */ + int ds_idx; /* ds index if ds_clp is set */ }; struct nfs_rw_header {
Support direct requests that span multiple pnfs data servers by comparing nfs_pgio_header->verf to a cached verf in pnfs_commit_bucket. Continue to use dreq->verf if the MDS is used / non-pNFS. Signed-off-by: Weston Andros Adamson <dros@primarydata.com> --- fs/nfs/direct.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++--- fs/nfs/nfs4filelayout.c | 6 +++ include/linux/nfs.h | 5 ++- include/linux/nfs_xdr.h | 2 + 4 files changed, 105 insertions(+), 6 deletions(-)