@@ -29,6 +29,9 @@
#include "linux/nfsd/pnfs_osd_xdr_srv.h"
+/* TODO: put in sysfs per sb */
+const static unsigned sb_shared_num_stripes = 8;
+
static int exofs_layout_type(struct super_block *sb)
{
return LAYOUT_OSD2_OBJECTS;
@@ -94,14 +97,27 @@ void ore_layout_2_pnfs_layout(struct pnfs_osd_layout *pl,
}
}
-static void _align_io(struct ore_layout *layout, u64 *offset, u64 *length)
+static bool _align_io(struct ore_layout *layout, struct nfsd4_layout_seg *lseg,
+ bool shared)
{
u64 stripe_size = (layout->group_width - layout->parity) *
layout->stripe_unit;
u64 group_size = stripe_size * layout->group_depth;
- *offset = div64_u64(*offset, group_size) * group_size;
- *length = group_size;
+ /* TODO: Don't ignore shared flag. Single writer can get a full group */
+ if (lseg->iomode != IOMODE_READ &&
+ (layout->parity || (layout->mirrors_p1 > 1))) {
+ /* RAID writes */
+ lseg->offset = div64_u64(lseg->offset, stripe_size) *
+ stripe_size;
+ lseg->length = stripe_size * sb_shared_num_stripes;
+ return true;
+ } else {
+ /* reads or no data redundancy */
+ lseg->offset = div64_u64(lseg->offset, group_size) * group_size;
+ lseg->length = group_size;
+ return false;
+ }
}
static enum nfsstat4 exofs_layout_get(
@@ -116,15 +132,15 @@ static enum nfsstat4 exofs_layout_get(
struct pnfs_osd_layout layout;
__be32 *start;
unsigned i;
- bool in_recall;
+ bool in_recall, need_recall;
enum nfsstat4 nfserr;
EXOFS_DBGMSG("(0x%lx) REQUESTED offset=0x%llx len=0x%llx iomod=0x%x\n",
inode->i_ino, res->lg_seg.offset,
res->lg_seg.length, res->lg_seg.iomode);
- _align_io(&sbi->layout, &res->lg_seg.offset, &res->lg_seg.length);
- res->lg_seg.iomode = IOMODE_RW;
+ need_recall = _align_io(&sbi->layout, &res->lg_seg,
+ test_bit(OBJ_LAYOUT_IS_GIVEN, &oi->i_flags));
res->lg_return_on_close = true;
res->lg_lo_cookie = inode; /* Just for debug prints */
@@ -132,6 +148,26 @@ static enum nfsstat4 exofs_layout_get(
inode->i_ino, res->lg_seg.offset,
res->lg_seg.length, res->lg_seg.iomode);
+ if (need_recall) {
+ int rc = cb_layout_recall(inode, IOMODE_RW, res->lg_seg.offset,
+ res->lg_seg.length, (void *)0x17);
+ switch (rc) {
+ case 0:
+ case -EAGAIN:
+ EXOFS_DBGMSG("(0x%lx) @@@ Sharing of RAID5/1 stripe\n",
+ inode->i_ino);
+ return NFS4ERR_RECALLCONFLICT;
+ default:
+ /* This is fine for now */
+ /* TODO: Fence object off */
+ EXOFS_DBGMSG("(0x%lx) !!!cb_layout_recall => %d\n",
+ inode->i_ino, rc);
+ /*fallthrough*/
+ case -ENOENT:
+ break;
+ }
+ }
+
/* skip opaque size, will be filled-in later */
start = exp_xdr_reserve_qwords(xdr, 1);
if (!start) {
If we have file redundancy RAID1/4/5/6 then two clients cannot write to the same stripe/region. We take care of this by giving out smaller regions of the file. Before any layout_get we make sure to recall the same exact region from any client. If a recall was issued we return NFS4ERR_RECALLCONFLICT. The client will come again later for it's layout. Meanwhile the fist client can flush data and release the layout. The next time the segment might be free and the lo_get succeed. It is very possible that multiple writers will fight and some clients will starve forever. But the smaller the region, and if the clients randomize a wait, it should statistically be OK. (We could manage a fairness queue. What about a lo_availble notification) On the other hand a very small segment will hurt performance. Default size is 8 stripes. TODO: Let segment size be set in sysfs. TODO: For debugging we always give out small segments. But we should only start giving out small segments on a shared file. The first/single writer should get a large seg as before. Signed-off-by: Boaz Harrosh <bharrosh@panasas.com> --- fs/exofs/export.c | 48 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-)