diff mbox series

[v7,15/19] NFSD add COPY_NOTIFY operation

Message ID 20190916211353.18802-16-olga.kornievskaia@gmail.com (mailing list archive)
State New, archived
Headers show
Series client and server support for "inter" SSC copy | expand

Commit Message

Olga Kornievskaia Sept. 16, 2019, 9:13 p.m. UTC 
Introducing the COPY_NOTIFY operation.

Create a new unique stateid that will keep track of the copy
state and the upcoming READs that will use that stateid.
Each associated parent stateid has a list of copy
notify stateids. A copy notify structure makes a copy of
the parent stateid and a clientid and will use it to look
up the parent stateid during the READ request (suggested
by Trond Myklebust <trond.myklebust@hammerspace.com>).

At nfs4_put_stid() time, we walk the list of the associated
copy notify stateids and delete them.

Laundromat thread will traverse globally stored copy notify
stateid in idr and notice if any haven't been referenced in the
lease period, if so, it'll remove them.

Return single netaddr to advertise to the copy.

Suggested-by: Trond Myklebust <trond.myklebust@hammerspace.com>
Signed-off-by: Olga Kornievskaia <kolga@netapp.com>
Signed-off-by: Andy Adamson <andros@netapp.com>
---
 fs/nfsd/nfs4proc.c  |  44 ++++++++++++++++++--
 fs/nfsd/nfs4state.c | 118 ++++++++++++++++++++++++++++++++++++++++++++++++----
 fs/nfsd/state.h     |  28 ++++++++++++-
 fs/nfsd/xdr4.h      |   2 +-
 4 files changed, 178 insertions(+), 14 deletions(-)

Comments

J. Bruce Fields Oct. 1, 2019, 8:59 p.m. UTC | #1 
On Mon, Sep 16, 2019 at 05:13:49PM -0400, Olga Kornievskaia wrote:
> @@ -2914,7 +2983,8 @@ static bool client_has_state(struct nfs4_client *clp)
>  #endif
>  		|| !list_empty(&clp->cl_delegations)
>  		|| !list_empty(&clp->cl_sessions)
> -		|| !list_empty(&clp->async_copies);
> +		|| !list_empty(&clp->async_copies)
> +		|| client_has_copy_notifies(clp);

Sorry, remind me--how is the copy_notify stateid cleaned up?  Is it just
timed out by the laundromat thread, or is our client destroying it when
the copy is done?

I'm just wondering if this can result in NFSERR_CLID_INUSE just because
a copy was done recently.

--b.

>  }
>  
>  static __be32 copy_impl_id(struct nfs4_client *clp,
> @@ -5192,6 +5262,9 @@ static bool clients_still_reclaiming(struct nfsd_net *nn)
>  	struct list_head *pos, *next, reaplist;
>  	time_t cutoff = get_seconds() - nn->nfsd4_lease;
>  	time_t t, new_timeo = nn->nfsd4_lease;
> +	struct nfs4_cpntf_state *cps;
> +	copy_stateid_t *cps_t;
> +	int i;
>  
>  	dprintk("NFSD: laundromat service - starting\n");
>  
> @@ -5202,6 +5275,17 @@ static bool clients_still_reclaiming(struct nfsd_net *nn)
>  	dprintk("NFSD: end of grace period\n");
>  	nfsd4_end_grace(nn);
>  	INIT_LIST_HEAD(&reaplist);
> +
> +	spin_lock(&nn->s2s_cp_lock);
> +	idr_for_each_entry(&nn->s2s_cp_stateids, cps_t, i) {
> +		cps = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid);
> +		if (cps->cp_stateid.sc_type == NFS4_COPYNOTIFY_STID &&
> +				!time_after((unsigned long)cps->cpntf_time,
> +				(unsigned long)cutoff))
> +			_free_cpntf_state_locked(nn, cps);
> +	}
> +	spin_unlock(&nn->s2s_cp_lock);
> +
>  	spin_lock(&nn->client_lock);
>  	list_for_each_safe(pos, next, &nn->client_lru) {
>  		clp = list_entry(pos, struct nfs4_client, cl_lru);
> @@ -5577,6 +5661,24 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
>  out:
>  	return status;
>  }
> +static void
> +_free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps)
> +{
> +	WARN_ON_ONCE(cps->cp_stateid.sc_type != NFS4_COPYNOTIFY_STID);
> +	if (!refcount_dec_and_test(&cps->cp_stateid.sc_count))
> +		return;
> +	list_del(&cps->cp_list);
> +	idr_remove(&nn->s2s_cp_stateids,
> +		   cps->cp_stateid.stid.si_opaque.so_id);
> +	kfree(cps);
> +}
> +
> +void nfs4_put_cpntf_state(struct nfsd_net *nn, struct nfs4_cpntf_state *cps)
> +{
> +	spin_lock(&nn->s2s_cp_lock);
> +	_free_cpntf_state_locked(nn, cps);
> +	spin_unlock(&nn->s2s_cp_lock);
> +}
>  
>  /*
>   * Checks for stateid operations
> diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
> index d9e7cbd..967b937 100644
> --- a/fs/nfsd/state.h
> +++ b/fs/nfsd/state.h
> @@ -56,6 +56,14 @@
>  	stateid_opaque_t        si_opaque;
>  } stateid_t;
>  
> +typedef struct {
> +	stateid_t		stid;
> +#define NFS4_COPY_STID 1
> +#define NFS4_COPYNOTIFY_STID 2
> +	unsigned char		sc_type;
> +	refcount_t		sc_count;
> +} copy_stateid_t;
> +
>  #define STATEID_FMT	"(%08x/%08x/%08x/%08x)"
>  #define STATEID_VAL(s) \
>  	(s)->si_opaque.so_clid.cl_boot, \
> @@ -96,6 +104,7 @@ struct nfs4_stid {
>  #define NFS4_REVOKED_DELEG_STID 16
>  #define NFS4_CLOSED_DELEG_STID 32
>  #define NFS4_LAYOUT_STID 64
> +	struct list_head	sc_cp_list;
>  	unsigned char		sc_type;
>  	stateid_t		sc_stateid;
>  	spinlock_t		sc_lock;
> @@ -104,6 +113,17 @@ struct nfs4_stid {
>  	void			(*sc_free)(struct nfs4_stid *);
>  };
>  
> +/* Keep a list of stateids issued by the COPY_NOTIFY, associate it with the
> + * parent OPEN/LOCK/DELEG stateid.
> + */
> +struct nfs4_cpntf_state {
> +	copy_stateid_t		cp_stateid;
> +	struct list_head	cp_list;	/* per parent nfs4_stid */
> +	stateid_t		cp_p_stateid;	/* copy of parent's stateid */
> +	clientid_t		cp_p_clid;	/* copy of parent's clid */
> +	time_t			cpntf_time;	/* last time stateid used */
> +};
> +
>  /*
>   * Represents a delegation stateid. The nfs4_client holds references to these
>   * and they are put when it is being destroyed or when the delegation is
> @@ -624,8 +644,10 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
>  		     struct nfs4_stid **s, struct nfsd_net *nn);
>  struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
>  				  void (*sc_free)(struct nfs4_stid *));
> -int nfs4_init_cp_state(struct nfsd_net *nn, struct nfsd4_copy *copy);
> -void nfs4_free_cp_state(struct nfsd4_copy *copy);
> +int nfs4_init_copy_state(struct nfsd_net *nn, struct nfsd4_copy *copy);
> +void nfs4_free_copy_state(struct nfsd4_copy *copy);
> +struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn,
> +			struct nfs4_stid *p_stid);
>  void nfs4_unhash_stid(struct nfs4_stid *s);
>  void nfs4_put_stid(struct nfs4_stid *s);
>  void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);
> @@ -655,6 +677,8 @@ extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name
>  extern void nfs4_put_copy(struct nfsd4_copy *copy);
>  extern struct nfsd4_copy *
>  find_async_copy(struct nfs4_client *clp, stateid_t *staetid);
> +extern void nfs4_put_cpntf_state(struct nfsd_net *nn,
> +				 struct nfs4_cpntf_state *cps);
>  static inline void get_nfs4_file(struct nfs4_file *fi)
>  {
>  	refcount_inc(&fi->fi_ref);
> diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
> index 8231fe0..2937e06 100644
> --- a/fs/nfsd/xdr4.h
> +++ b/fs/nfsd/xdr4.h
> @@ -542,7 +542,7 @@ struct nfsd4_copy {
>  	struct nfsd_file        *nf_src;
>  	struct nfsd_file        *nf_dst;
>  
> -	stateid_t		cp_stateid;
> +	copy_stateid_t		cp_stateid;
>  
>  	struct list_head	copies;
>  	struct task_struct	*copy_task;
> -- 
> 1.8.3.1
Olga Kornievskaia Oct. 2, 2019, 12:14 a.m. UTC | #2 
On Tue, Oct 1, 2019 at 4:59 PM J. Bruce Fields <bfields@fieldses.org> wrote:
>
> On Mon, Sep 16, 2019 at 05:13:49PM -0400, Olga Kornievskaia wrote:
> > @@ -2914,7 +2983,8 @@ static bool client_has_state(struct nfs4_client *clp)
> >  #endif
> >               || !list_empty(&clp->cl_delegations)
> >               || !list_empty(&clp->cl_sessions)
> > -             || !list_empty(&clp->async_copies);
> > +             || !list_empty(&clp->async_copies)
> > +             || client_has_copy_notifies(clp);
>
> Sorry, remind me--how is the copy_notify stateid cleaned up?  Is it just
> timed out by the laundromat thread, or is our client destroying it when
> the copy is done?

Copy_notify stateid in most cases should be removed by
nfs4_put_stid()/ the "parent" stateid going away (close, unlock,
delegreturn) (or in unlikely case, if the client sent an
OFFLOAD_CANCEL to the stateid, say if copy was canceled). Otherwise,
copy notify stateid will time out and be removed by the laundromat
thread. So if a client didn't send a close/unlock/delegreturn and
quickly tries to delete a clientid, it will get ERR_CLID_INUSE but I
think if there was no close, the server will have an issue of having
an open stateid state.

> I'm just wondering if this can result in NFSERR_CLID_INUSE just because
> a copy was done recently.
>
> --b.
>
> >  }
> >
> >  static __be32 copy_impl_id(struct nfs4_client *clp,
> > @@ -5192,6 +5262,9 @@ static bool clients_still_reclaiming(struct nfsd_net *nn)
> >       struct list_head *pos, *next, reaplist;
> >       time_t cutoff = get_seconds() - nn->nfsd4_lease;
> >       time_t t, new_timeo = nn->nfsd4_lease;
> > +     struct nfs4_cpntf_state *cps;
> > +     copy_stateid_t *cps_t;
> > +     int i;
> >
> >       dprintk("NFSD: laundromat service - starting\n");
> >
> > @@ -5202,6 +5275,17 @@ static bool clients_still_reclaiming(struct nfsd_net *nn)
> >       dprintk("NFSD: end of grace period\n");
> >       nfsd4_end_grace(nn);
> >       INIT_LIST_HEAD(&reaplist);
> > +
> > +     spin_lock(&nn->s2s_cp_lock);
> > +     idr_for_each_entry(&nn->s2s_cp_stateids, cps_t, i) {
> > +             cps = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid);
> > +             if (cps->cp_stateid.sc_type == NFS4_COPYNOTIFY_STID &&
> > +                             !time_after((unsigned long)cps->cpntf_time,
> > +                             (unsigned long)cutoff))
> > +                     _free_cpntf_state_locked(nn, cps);
> > +     }
> > +     spin_unlock(&nn->s2s_cp_lock);
> > +
> >       spin_lock(&nn->client_lock);
> >       list_for_each_safe(pos, next, &nn->client_lru) {
> >               clp = list_entry(pos, struct nfs4_client, cl_lru);
> > @@ -5577,6 +5661,24 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
> >  out:
> >       return status;
> >  }
> > +static void
> > +_free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps)
> > +{
> > +     WARN_ON_ONCE(cps->cp_stateid.sc_type != NFS4_COPYNOTIFY_STID);
> > +     if (!refcount_dec_and_test(&cps->cp_stateid.sc_count))
> > +             return;
> > +     list_del(&cps->cp_list);
> > +     idr_remove(&nn->s2s_cp_stateids,
> > +                cps->cp_stateid.stid.si_opaque.so_id);
> > +     kfree(cps);
> > +}
> > +
> > +void nfs4_put_cpntf_state(struct nfsd_net *nn, struct nfs4_cpntf_state *cps)
> > +{
> > +     spin_lock(&nn->s2s_cp_lock);
> > +     _free_cpntf_state_locked(nn, cps);
> > +     spin_unlock(&nn->s2s_cp_lock);
> > +}
> >
> >  /*
> >   * Checks for stateid operations
> > diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
> > index d9e7cbd..967b937 100644
> > --- a/fs/nfsd/state.h
> > +++ b/fs/nfsd/state.h
> > @@ -56,6 +56,14 @@
> >       stateid_opaque_t        si_opaque;
> >  } stateid_t;
> >
> > +typedef struct {
> > +     stateid_t               stid;
> > +#define NFS4_COPY_STID 1
> > +#define NFS4_COPYNOTIFY_STID 2
> > +     unsigned char           sc_type;
> > +     refcount_t              sc_count;
> > +} copy_stateid_t;
> > +
> >  #define STATEID_FMT  "(%08x/%08x/%08x/%08x)"
> >  #define STATEID_VAL(s) \
> >       (s)->si_opaque.so_clid.cl_boot, \
> > @@ -96,6 +104,7 @@ struct nfs4_stid {
> >  #define NFS4_REVOKED_DELEG_STID 16
> >  #define NFS4_CLOSED_DELEG_STID 32
> >  #define NFS4_LAYOUT_STID 64
> > +     struct list_head        sc_cp_list;
> >       unsigned char           sc_type;
> >       stateid_t               sc_stateid;
> >       spinlock_t              sc_lock;
> > @@ -104,6 +113,17 @@ struct nfs4_stid {
> >       void                    (*sc_free)(struct nfs4_stid *);
> >  };
> >
> > +/* Keep a list of stateids issued by the COPY_NOTIFY, associate it with the
> > + * parent OPEN/LOCK/DELEG stateid.
> > + */
> > +struct nfs4_cpntf_state {
> > +     copy_stateid_t          cp_stateid;
> > +     struct list_head        cp_list;        /* per parent nfs4_stid */
> > +     stateid_t               cp_p_stateid;   /* copy of parent's stateid */
> > +     clientid_t              cp_p_clid;      /* copy of parent's clid */
> > +     time_t                  cpntf_time;     /* last time stateid used */
> > +};
> > +
> >  /*
> >   * Represents a delegation stateid. The nfs4_client holds references to these
> >   * and they are put when it is being destroyed or when the delegation is
> > @@ -624,8 +644,10 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
> >                    struct nfs4_stid **s, struct nfsd_net *nn);
> >  struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
> >                                 void (*sc_free)(struct nfs4_stid *));
> > -int nfs4_init_cp_state(struct nfsd_net *nn, struct nfsd4_copy *copy);
> > -void nfs4_free_cp_state(struct nfsd4_copy *copy);
> > +int nfs4_init_copy_state(struct nfsd_net *nn, struct nfsd4_copy *copy);
> > +void nfs4_free_copy_state(struct nfsd4_copy *copy);
> > +struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn,
> > +                     struct nfs4_stid *p_stid);
> >  void nfs4_unhash_stid(struct nfs4_stid *s);
> >  void nfs4_put_stid(struct nfs4_stid *s);
> >  void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);
> > @@ -655,6 +677,8 @@ extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name
> >  extern void nfs4_put_copy(struct nfsd4_copy *copy);
> >  extern struct nfsd4_copy *
> >  find_async_copy(struct nfs4_client *clp, stateid_t *staetid);
> > +extern void nfs4_put_cpntf_state(struct nfsd_net *nn,
> > +                              struct nfs4_cpntf_state *cps);
> >  static inline void get_nfs4_file(struct nfs4_file *fi)
> >  {
> >       refcount_inc(&fi->fi_ref);
> > diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
> > index 8231fe0..2937e06 100644
> > --- a/fs/nfsd/xdr4.h
> > +++ b/fs/nfsd/xdr4.h
> > @@ -542,7 +542,7 @@ struct nfsd4_copy {
> >       struct nfsd_file        *nf_src;
> >       struct nfsd_file        *nf_dst;
> >
> > -     stateid_t               cp_stateid;
> > +     copy_stateid_t          cp_stateid;
> >
> >       struct list_head        copies;
> >       struct task_struct      *copy_task;
> > --
> > 1.8.3.1
J. Bruce Fields Oct. 2, 2019, 1:35 a.m. UTC | #3 
On Tue, Oct 01, 2019 at 08:14:39PM -0400, Olga Kornievskaia wrote:
> On Tue, Oct 1, 2019 at 4:59 PM J. Bruce Fields <bfields@fieldses.org> wrote:
> >
> > On Mon, Sep 16, 2019 at 05:13:49PM -0400, Olga Kornievskaia wrote:
> > > @@ -2914,7 +2983,8 @@ static bool client_has_state(struct nfs4_client *clp)
> > >  #endif
> > >               || !list_empty(&clp->cl_delegations)
> > >               || !list_empty(&clp->cl_sessions)
> > > -             || !list_empty(&clp->async_copies);
> > > +             || !list_empty(&clp->async_copies)
> > > +             || client_has_copy_notifies(clp);
> >
> > Sorry, remind me--how is the copy_notify stateid cleaned up?  Is it just
> > timed out by the laundromat thread, or is our client destroying it when
> > the copy is done?
> 
> Copy_notify stateid in most cases should be removed by
> nfs4_put_stid()/ the "parent" stateid going away (close, unlock,
> delegreturn)

Oh, right, the parent stateid going away will be enough, good.

But doesn't that mean the client_has_copy_notifies() check here is
redundant?  If it's true, then the earlier client_has_openowner() check
was also true.

--b.

> (or in unlikely case, if the client sent an
> OFFLOAD_CANCEL to the stateid, say if copy was canceled). Otherwise,
> copy notify stateid will time out and be removed by the laundromat
> thread. So if a client didn't send a close/unlock/delegreturn and
> quickly tries to delete a clientid, it will get ERR_CLID_INUSE but I
> think if there was no close, the server will have an issue of having
> an open stateid state.
> 
> > I'm just wondering if this can result in NFSERR_CLID_INUSE just because
> > a copy was done recently.
> >
> > --b.
> >
> > >  }
> > >
> > >  static __be32 copy_impl_id(struct nfs4_client *clp,
> > > @@ -5192,6 +5262,9 @@ static bool clients_still_reclaiming(struct nfsd_net *nn)
> > >       struct list_head *pos, *next, reaplist;
> > >       time_t cutoff = get_seconds() - nn->nfsd4_lease;
> > >       time_t t, new_timeo = nn->nfsd4_lease;
> > > +     struct nfs4_cpntf_state *cps;
> > > +     copy_stateid_t *cps_t;
> > > +     int i;
> > >
> > >       dprintk("NFSD: laundromat service - starting\n");
> > >
> > > @@ -5202,6 +5275,17 @@ static bool clients_still_reclaiming(struct nfsd_net *nn)
> > >       dprintk("NFSD: end of grace period\n");
> > >       nfsd4_end_grace(nn);
> > >       INIT_LIST_HEAD(&reaplist);
> > > +
> > > +     spin_lock(&nn->s2s_cp_lock);
> > > +     idr_for_each_entry(&nn->s2s_cp_stateids, cps_t, i) {
> > > +             cps = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid);
> > > +             if (cps->cp_stateid.sc_type == NFS4_COPYNOTIFY_STID &&
> > > +                             !time_after((unsigned long)cps->cpntf_time,
> > > +                             (unsigned long)cutoff))
> > > +                     _free_cpntf_state_locked(nn, cps);
> > > +     }
> > > +     spin_unlock(&nn->s2s_cp_lock);
> > > +
> > >       spin_lock(&nn->client_lock);
> > >       list_for_each_safe(pos, next, &nn->client_lru) {
> > >               clp = list_entry(pos, struct nfs4_client, cl_lru);
> > > @@ -5577,6 +5661,24 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
> > >  out:
> > >       return status;
> > >  }
> > > +static void
> > > +_free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps)
> > > +{
> > > +     WARN_ON_ONCE(cps->cp_stateid.sc_type != NFS4_COPYNOTIFY_STID);
> > > +     if (!refcount_dec_and_test(&cps->cp_stateid.sc_count))
> > > +             return;
> > > +     list_del(&cps->cp_list);
> > > +     idr_remove(&nn->s2s_cp_stateids,
> > > +                cps->cp_stateid.stid.si_opaque.so_id);
> > > +     kfree(cps);
> > > +}
> > > +
> > > +void nfs4_put_cpntf_state(struct nfsd_net *nn, struct nfs4_cpntf_state *cps)
> > > +{
> > > +     spin_lock(&nn->s2s_cp_lock);
> > > +     _free_cpntf_state_locked(nn, cps);
> > > +     spin_unlock(&nn->s2s_cp_lock);
> > > +}
> > >
> > >  /*
> > >   * Checks for stateid operations
> > > diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
> > > index d9e7cbd..967b937 100644
> > > --- a/fs/nfsd/state.h
> > > +++ b/fs/nfsd/state.h
> > > @@ -56,6 +56,14 @@
> > >       stateid_opaque_t        si_opaque;
> > >  } stateid_t;
> > >
> > > +typedef struct {
> > > +     stateid_t               stid;
> > > +#define NFS4_COPY_STID 1
> > > +#define NFS4_COPYNOTIFY_STID 2
> > > +     unsigned char           sc_type;
> > > +     refcount_t              sc_count;
> > > +} copy_stateid_t;
> > > +
> > >  #define STATEID_FMT  "(%08x/%08x/%08x/%08x)"
> > >  #define STATEID_VAL(s) \
> > >       (s)->si_opaque.so_clid.cl_boot, \
> > > @@ -96,6 +104,7 @@ struct nfs4_stid {
> > >  #define NFS4_REVOKED_DELEG_STID 16
> > >  #define NFS4_CLOSED_DELEG_STID 32
> > >  #define NFS4_LAYOUT_STID 64
> > > +     struct list_head        sc_cp_list;
> > >       unsigned char           sc_type;
> > >       stateid_t               sc_stateid;
> > >       spinlock_t              sc_lock;
> > > @@ -104,6 +113,17 @@ struct nfs4_stid {
> > >       void                    (*sc_free)(struct nfs4_stid *);
> > >  };
> > >
> > > +/* Keep a list of stateids issued by the COPY_NOTIFY, associate it with the
> > > + * parent OPEN/LOCK/DELEG stateid.
> > > + */
> > > +struct nfs4_cpntf_state {
> > > +     copy_stateid_t          cp_stateid;
> > > +     struct list_head        cp_list;        /* per parent nfs4_stid */
> > > +     stateid_t               cp_p_stateid;   /* copy of parent's stateid */
> > > +     clientid_t              cp_p_clid;      /* copy of parent's clid */
> > > +     time_t                  cpntf_time;     /* last time stateid used */
> > > +};
> > > +
> > >  /*
> > >   * Represents a delegation stateid. The nfs4_client holds references to these
> > >   * and they are put when it is being destroyed or when the delegation is
> > > @@ -624,8 +644,10 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
> > >                    struct nfs4_stid **s, struct nfsd_net *nn);
> > >  struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
> > >                                 void (*sc_free)(struct nfs4_stid *));
> > > -int nfs4_init_cp_state(struct nfsd_net *nn, struct nfsd4_copy *copy);
> > > -void nfs4_free_cp_state(struct nfsd4_copy *copy);
> > > +int nfs4_init_copy_state(struct nfsd_net *nn, struct nfsd4_copy *copy);
> > > +void nfs4_free_copy_state(struct nfsd4_copy *copy);
> > > +struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn,
> > > +                     struct nfs4_stid *p_stid);
> > >  void nfs4_unhash_stid(struct nfs4_stid *s);
> > >  void nfs4_put_stid(struct nfs4_stid *s);
> > >  void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);
> > > @@ -655,6 +677,8 @@ extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name
> > >  extern void nfs4_put_copy(struct nfsd4_copy *copy);
> > >  extern struct nfsd4_copy *
> > >  find_async_copy(struct nfs4_client *clp, stateid_t *staetid);
> > > +extern void nfs4_put_cpntf_state(struct nfsd_net *nn,
> > > +                              struct nfs4_cpntf_state *cps);
> > >  static inline void get_nfs4_file(struct nfs4_file *fi)
> > >  {
> > >       refcount_inc(&fi->fi_ref);
> > > diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
> > > index 8231fe0..2937e06 100644
> > > --- a/fs/nfsd/xdr4.h
> > > +++ b/fs/nfsd/xdr4.h
> > > @@ -542,7 +542,7 @@ struct nfsd4_copy {
> > >       struct nfsd_file        *nf_src;
> > >       struct nfsd_file        *nf_dst;
> > >
> > > -     stateid_t               cp_stateid;
> > > +     copy_stateid_t          cp_stateid;
> > >
> > >       struct list_head        copies;
> > >       struct task_struct      *copy_task;
> > > --
> > > 1.8.3.1
Olga Kornievskaia Oct. 2, 2019, 3:32 p.m. UTC | #4 
On Tue, Oct 1, 2019 at 9:35 PM J. Bruce Fields <bfields@fieldses.org> wrote:
>
> On Tue, Oct 01, 2019 at 08:14:39PM -0400, Olga Kornievskaia wrote:
> > On Tue, Oct 1, 2019 at 4:59 PM J. Bruce Fields <bfields@fieldses.org> wrote:
> > >
> > > On Mon, Sep 16, 2019 at 05:13:49PM -0400, Olga Kornievskaia wrote:
> > > > @@ -2914,7 +2983,8 @@ static bool client_has_state(struct nfs4_client *clp)
> > > >  #endif
> > > >               || !list_empty(&clp->cl_delegations)
> > > >               || !list_empty(&clp->cl_sessions)
> > > > -             || !list_empty(&clp->async_copies);
> > > > +             || !list_empty(&clp->async_copies)
> > > > +             || client_has_copy_notifies(clp);
> > >
> > > Sorry, remind me--how is the copy_notify stateid cleaned up?  Is it just
> > > timed out by the laundromat thread, or is our client destroying it when
> > > the copy is done?
> >
> > Copy_notify stateid in most cases should be removed by
> > nfs4_put_stid()/ the "parent" stateid going away (close, unlock,
> > delegreturn)
>
> Oh, right, the parent stateid going away will be enough, good.
>
> But doesn't that mean the client_has_copy_notifies() check here is
> redundant?  If it's true, then the earlier client_has_openowner() check
> was also true.

Honestly, I didn't remove it because I didn't know if I was missing
some weird case where on clientid destruction we'd still have copy
notify left and thus need to delay it. But as you see nfs4_put_stid()
calls nfs4_free_cpntf_statelist() which iterates the list of the
copy_notify states and deletes them. Sounds like I should just remove
the client_has_copy_notifies() check.

>
> --b.
>
> > (or in unlikely case, if the client sent an
> > OFFLOAD_CANCEL to the stateid, say if copy was canceled). Otherwise,
> > copy notify stateid will time out and be removed by the laundromat
> > thread. So if a client didn't send a close/unlock/delegreturn and
> > quickly tries to delete a clientid, it will get ERR_CLID_INUSE but I
> > think if there was no close, the server will have an issue of having
> > an open stateid state.
> >
> > > I'm just wondering if this can result in NFSERR_CLID_INUSE just because
> > > a copy was done recently.
> > >
> > > --b.
> > >
> > > >  }
> > > >
> > > >  static __be32 copy_impl_id(struct nfs4_client *clp,
> > > > @@ -5192,6 +5262,9 @@ static bool clients_still_reclaiming(struct nfsd_net *nn)
> > > >       struct list_head *pos, *next, reaplist;
> > > >       time_t cutoff = get_seconds() - nn->nfsd4_lease;
> > > >       time_t t, new_timeo = nn->nfsd4_lease;
> > > > +     struct nfs4_cpntf_state *cps;
> > > > +     copy_stateid_t *cps_t;
> > > > +     int i;
> > > >
> > > >       dprintk("NFSD: laundromat service - starting\n");
> > > >
> > > > @@ -5202,6 +5275,17 @@ static bool clients_still_reclaiming(struct nfsd_net *nn)
> > > >       dprintk("NFSD: end of grace period\n");
> > > >       nfsd4_end_grace(nn);
> > > >       INIT_LIST_HEAD(&reaplist);
> > > > +
> > > > +     spin_lock(&nn->s2s_cp_lock);
> > > > +     idr_for_each_entry(&nn->s2s_cp_stateids, cps_t, i) {
> > > > +             cps = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid);
> > > > +             if (cps->cp_stateid.sc_type == NFS4_COPYNOTIFY_STID &&
> > > > +                             !time_after((unsigned long)cps->cpntf_time,
> > > > +                             (unsigned long)cutoff))
> > > > +                     _free_cpntf_state_locked(nn, cps);
> > > > +     }
> > > > +     spin_unlock(&nn->s2s_cp_lock);
> > > > +
> > > >       spin_lock(&nn->client_lock);
> > > >       list_for_each_safe(pos, next, &nn->client_lru) {
> > > >               clp = list_entry(pos, struct nfs4_client, cl_lru);
> > > > @@ -5577,6 +5661,24 @@ static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
> > > >  out:
> > > >       return status;
> > > >  }
> > > > +static void
> > > > +_free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps)
> > > > +{
> > > > +     WARN_ON_ONCE(cps->cp_stateid.sc_type != NFS4_COPYNOTIFY_STID);
> > > > +     if (!refcount_dec_and_test(&cps->cp_stateid.sc_count))
> > > > +             return;
> > > > +     list_del(&cps->cp_list);
> > > > +     idr_remove(&nn->s2s_cp_stateids,
> > > > +                cps->cp_stateid.stid.si_opaque.so_id);
> > > > +     kfree(cps);
> > > > +}
> > > > +
> > > > +void nfs4_put_cpntf_state(struct nfsd_net *nn, struct nfs4_cpntf_state *cps)
> > > > +{
> > > > +     spin_lock(&nn->s2s_cp_lock);
> > > > +     _free_cpntf_state_locked(nn, cps);
> > > > +     spin_unlock(&nn->s2s_cp_lock);
> > > > +}
> > > >
> > > >  /*
> > > >   * Checks for stateid operations
> > > > diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
> > > > index d9e7cbd..967b937 100644
> > > > --- a/fs/nfsd/state.h
> > > > +++ b/fs/nfsd/state.h
> > > > @@ -56,6 +56,14 @@
> > > >       stateid_opaque_t        si_opaque;
> > > >  } stateid_t;
> > > >
> > > > +typedef struct {
> > > > +     stateid_t               stid;
> > > > +#define NFS4_COPY_STID 1
> > > > +#define NFS4_COPYNOTIFY_STID 2
> > > > +     unsigned char           sc_type;
> > > > +     refcount_t              sc_count;
> > > > +} copy_stateid_t;
> > > > +
> > > >  #define STATEID_FMT  "(%08x/%08x/%08x/%08x)"
> > > >  #define STATEID_VAL(s) \
> > > >       (s)->si_opaque.so_clid.cl_boot, \
> > > > @@ -96,6 +104,7 @@ struct nfs4_stid {
> > > >  #define NFS4_REVOKED_DELEG_STID 16
> > > >  #define NFS4_CLOSED_DELEG_STID 32
> > > >  #define NFS4_LAYOUT_STID 64
> > > > +     struct list_head        sc_cp_list;
> > > >       unsigned char           sc_type;
> > > >       stateid_t               sc_stateid;
> > > >       spinlock_t              sc_lock;
> > > > @@ -104,6 +113,17 @@ struct nfs4_stid {
> > > >       void                    (*sc_free)(struct nfs4_stid *);
> > > >  };
> > > >
> > > > +/* Keep a list of stateids issued by the COPY_NOTIFY, associate it with the
> > > > + * parent OPEN/LOCK/DELEG stateid.
> > > > + */
> > > > +struct nfs4_cpntf_state {
> > > > +     copy_stateid_t          cp_stateid;
> > > > +     struct list_head        cp_list;        /* per parent nfs4_stid */
> > > > +     stateid_t               cp_p_stateid;   /* copy of parent's stateid */
> > > > +     clientid_t              cp_p_clid;      /* copy of parent's clid */
> > > > +     time_t                  cpntf_time;     /* last time stateid used */
> > > > +};
> > > > +
> > > >  /*
> > > >   * Represents a delegation stateid. The nfs4_client holds references to these
> > > >   * and they are put when it is being destroyed or when the delegation is
> > > > @@ -624,8 +644,10 @@ __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
> > > >                    struct nfs4_stid **s, struct nfsd_net *nn);
> > > >  struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
> > > >                                 void (*sc_free)(struct nfs4_stid *));
> > > > -int nfs4_init_cp_state(struct nfsd_net *nn, struct nfsd4_copy *copy);
> > > > -void nfs4_free_cp_state(struct nfsd4_copy *copy);
> > > > +int nfs4_init_copy_state(struct nfsd_net *nn, struct nfsd4_copy *copy);
> > > > +void nfs4_free_copy_state(struct nfsd4_copy *copy);
> > > > +struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn,
> > > > +                     struct nfs4_stid *p_stid);
> > > >  void nfs4_unhash_stid(struct nfs4_stid *s);
> > > >  void nfs4_put_stid(struct nfs4_stid *s);
> > > >  void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);
> > > > @@ -655,6 +677,8 @@ extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name
> > > >  extern void nfs4_put_copy(struct nfsd4_copy *copy);
> > > >  extern struct nfsd4_copy *
> > > >  find_async_copy(struct nfs4_client *clp, stateid_t *staetid);
> > > > +extern void nfs4_put_cpntf_state(struct nfsd_net *nn,
> > > > +                              struct nfs4_cpntf_state *cps);
> > > >  static inline void get_nfs4_file(struct nfs4_file *fi)
> > > >  {
> > > >       refcount_inc(&fi->fi_ref);
> > > > diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
> > > > index 8231fe0..2937e06 100644
> > > > --- a/fs/nfsd/xdr4.h
> > > > +++ b/fs/nfsd/xdr4.h
> > > > @@ -542,7 +542,7 @@ struct nfsd4_copy {
> > > >       struct nfsd_file        *nf_src;
> > > >       struct nfsd_file        *nf_dst;
> > > >
> > > > -     stateid_t               cp_stateid;
> > > > +     copy_stateid_t          cp_stateid;
> > > >
> > > >       struct list_head        copies;
> > > >       struct task_struct      *copy_task;
> > > > --
> > > > 1.8.3.1
diff mbox series

Patch

diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c
index 05519a2..75ecdbd 100644
--- a/fs/nfsd/nfs4proc.c
+++ b/fs/nfsd/nfs4proc.c
@@ -37,6 +37,7 @@ 
 #include <linux/falloc.h>
 #include <linux/slab.h>
 #include <linux/kthread.h>
+#include <linux/sunrpc/addr.h>
 
 #include "idmap.h"
 #include "cache.h"
@@ -1221,7 +1222,7 @@  static void dup_copy_fields(struct nfsd4_copy *src, struct nfsd4_copy *dst)
 
 static void cleanup_async_copy(struct nfsd4_copy *copy)
 {
-	nfs4_free_cp_state(copy);
+	nfs4_free_copy_state(copy);
 	nfsd_file_put(copy->nf_dst);
 	nfsd_file_put(copy->nf_src);
 	spin_lock(&copy->cp_clp->async_lock);
@@ -1275,7 +1276,7 @@  static int nfsd4_do_async_copy(void *data)
 		async_copy = kzalloc(sizeof(struct nfsd4_copy), GFP_KERNEL);
 		if (!async_copy)
 			goto out;
-		if (!nfs4_init_cp_state(nn, copy)) {
+		if (!nfs4_init_copy_state(nn, copy)) {
 			kfree(async_copy);
 			goto out;
 		}
@@ -1342,7 +1343,44 @@  struct nfsd4_copy *
 nfsd4_copy_notify(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate,
 		  union nfsd4_op_u *u)
 {
-	return nfserr_notsupp;
+	struct nfsd4_copy_notify *cn = &u->copy_notify;
+	__be32 status;
+	struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id);
+	struct nfs4_stid *stid;
+	struct nfs4_cpntf_state *cps;
+	struct nfs4_client *clp = cstate->clp;
+
+	status = nfs4_preprocess_stateid_op(rqstp, cstate, &cstate->current_fh,
+					&cn->cpn_src_stateid, RD_STATE, NULL,
+					&stid);
+	if (status)
+		return status;
+
+	cn->cpn_sec = nn->nfsd4_lease;
+	cn->cpn_nsec = 0;
+
+	status = nfserrno(-ENOMEM);
+	cps = nfs4_alloc_init_cpntf_state(nn, stid);
+	if (!cps)
+		goto out;
+	memcpy(&cn->cpn_cnr_stateid, &cps->cp_stateid.stid, sizeof(stateid_t));
+	memcpy(&cps->cp_p_stateid, &stid->sc_stateid, sizeof(stateid_t));
+	memcpy(&cps->cp_p_clid, &clp->cl_clientid, sizeof(clientid_t));
+
+	/* For now, only return one server address in cpn_src, the
+	 * address used by the client to connect to this server.
+	 */
+	cn->cpn_src.nl4_type = NL4_NETADDR;
+	status = nfsd4_set_netaddr((struct sockaddr *)&rqstp->rq_daddr,
+				 &cn->cpn_src.u.nl4_addr);
+	WARN_ON_ONCE(status);
+	if (status) {
+		nfs4_put_cpntf_state(nn, cps);
+		goto out;
+	}
+out:
+	nfs4_put_stid(stid);
+	return status;
 }
 
 static __be32
diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c
index 78e03af..857133f 100644
--- a/fs/nfsd/nfs4state.c
+++ b/fs/nfsd/nfs4state.c
@@ -80,6 +80,7 @@ 
 static bool check_for_locks(struct nfs4_file *fp, struct nfs4_lockowner *lowner);
 static void nfs4_free_ol_stateid(struct nfs4_stid *stid);
 void nfsd4_end_grace(struct nfsd_net *nn);
+static void _free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps);
 
 /* Locking: */
 
@@ -722,6 +723,7 @@  struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
 	/* Will be incremented before return to client: */
 	refcount_set(&stid->sc_count, 1);
 	spin_lock_init(&stid->sc_lock);
+	INIT_LIST_HEAD(&stid->sc_cp_list);
 
 	/*
 	 * It shouldn't be a problem to reuse an opaque stateid value.
@@ -741,30 +743,76 @@  struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *sla
 /*
  * Create a unique stateid_t to represent each COPY.
  */
-int nfs4_init_cp_state(struct nfsd_net *nn, struct nfsd4_copy *copy)
+static int nfs4_init_cp_state(struct nfsd_net *nn, copy_stateid_t *stid,
+			      unsigned char sc_type)
 {
 	int new_id;
 
+	stid->stid.si_opaque.so_clid.cl_boot = nn->boot_time;
+	stid->stid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
+	stid->sc_type = sc_type;
+
 	idr_preload(GFP_KERNEL);
 	spin_lock(&nn->s2s_cp_lock);
-	new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, copy, 0, 0, GFP_NOWAIT);
+	new_id = idr_alloc_cyclic(&nn->s2s_cp_stateids, stid, 0, 0, GFP_NOWAIT);
+	stid->stid.si_opaque.so_id = new_id;
 	spin_unlock(&nn->s2s_cp_lock);
 	idr_preload_end();
 	if (new_id < 0)
 		return 0;
-	copy->cp_stateid.si_opaque.so_id = new_id;
-	copy->cp_stateid.si_opaque.so_clid.cl_boot = nn->boot_time;
-	copy->cp_stateid.si_opaque.so_clid.cl_id = nn->s2s_cp_cl_id;
 	return 1;
 }
 
-void nfs4_free_cp_state(struct nfsd4_copy *copy)
+int nfs4_init_copy_state(struct nfsd_net *nn, struct nfsd4_copy *copy)
+{
+	return nfs4_init_cp_state(nn, &copy->cp_stateid, NFS4_COPY_STID);
+}
+
+struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn,
+						     struct nfs4_stid *p_stid)
+{
+	struct nfs4_cpntf_state *cps;
+
+	cps = kzalloc(sizeof(struct nfs4_cpntf_state), GFP_KERNEL);
+	if (!cps)
+		return NULL;
+	cps->cpntf_time = get_seconds();
+	refcount_set(&cps->cp_stateid.sc_count, 1);
+	if (!nfs4_init_cp_state(nn, &cps->cp_stateid, NFS4_COPYNOTIFY_STID))
+		goto out_free;
+	spin_lock(&nn->s2s_cp_lock);
+	list_add(&cps->cp_list, &p_stid->sc_cp_list);
+	spin_unlock(&nn->s2s_cp_lock);
+	return cps;
+out_free:
+	kfree(cps);
+	return NULL;
+}
+
+void nfs4_free_copy_state(struct nfsd4_copy *copy)
 {
 	struct nfsd_net *nn;
 
+	WARN_ON_ONCE(copy->cp_stateid.sc_type != NFS4_COPY_STID);
 	nn = net_generic(copy->cp_clp->net, nfsd_net_id);
 	spin_lock(&nn->s2s_cp_lock);
-	idr_remove(&nn->s2s_cp_stateids, copy->cp_stateid.si_opaque.so_id);
+	idr_remove(&nn->s2s_cp_stateids,
+		   copy->cp_stateid.stid.si_opaque.so_id);
+	spin_unlock(&nn->s2s_cp_lock);
+}
+
+static void nfs4_free_cpntf_statelist(struct net *net, struct nfs4_stid *stid)
+{
+	struct nfs4_cpntf_state *cps;
+	struct nfsd_net *nn;
+
+	nn = net_generic(net, nfsd_net_id);
+	spin_lock(&nn->s2s_cp_lock);
+	while (!list_empty(&stid->sc_cp_list)) {
+		cps = list_first_entry(&stid->sc_cp_list,
+				       struct nfs4_cpntf_state, cp_list);
+		_free_cpntf_state_locked(nn, cps);
+	}
 	spin_unlock(&nn->s2s_cp_lock);
 }
 
@@ -915,6 +963,7 @@  static void block_delegations(struct knfsd_fh *fh)
 		return;
 	}
 	idr_remove(&clp->cl_stateids, s->sc_stateid.si_opaque.so_id);
+	nfs4_free_cpntf_statelist(clp->net, s);
 	spin_unlock(&clp->cl_lock);
 	s->sc_free(s);
 	if (fp)
@@ -2906,6 +2955,26 @@  static bool client_has_openowners(struct nfs4_client *clp)
 	return false;
 }
 
+static bool client_has_copy_notifies(struct nfs4_client *clp)
+{
+	struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id);
+	copy_stateid_t *cps_t;
+	struct nfs4_cpntf_state *cps;
+	int i;
+
+	spin_lock(&nn->s2s_cp_lock);
+	idr_for_each_entry(&nn->s2s_cp_stateids, cps_t, i) {
+		cps = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid);
+		if (cps->cp_stateid.sc_type == NFS4_COPYNOTIFY_STID &&
+				same_clid(&cps->cp_p_clid, &clp->cl_clientid)) {
+			spin_unlock(&nn->s2s_cp_lock);
+			return true;
+		}
+	}
+	spin_unlock(&nn->s2s_cp_lock);
+	return false;
+}
+
 static bool client_has_state(struct nfs4_client *clp)
 {
 	return client_has_openowners(clp)
@@ -2914,7 +2983,8 @@  static bool client_has_state(struct nfs4_client *clp)
 #endif
 		|| !list_empty(&clp->cl_delegations)
 		|| !list_empty(&clp->cl_sessions)
-		|| !list_empty(&clp->async_copies);
+		|| !list_empty(&clp->async_copies)
+		|| client_has_copy_notifies(clp);
 }
 
 static __be32 copy_impl_id(struct nfs4_client *clp,
@@ -5192,6 +5262,9 @@  static bool clients_still_reclaiming(struct nfsd_net *nn)
 	struct list_head *pos, *next, reaplist;
 	time_t cutoff = get_seconds() - nn->nfsd4_lease;
 	time_t t, new_timeo = nn->nfsd4_lease;
+	struct nfs4_cpntf_state *cps;
+	copy_stateid_t *cps_t;
+	int i;
 
 	dprintk("NFSD: laundromat service - starting\n");
 
@@ -5202,6 +5275,17 @@  static bool clients_still_reclaiming(struct nfsd_net *nn)
 	dprintk("NFSD: end of grace period\n");
 	nfsd4_end_grace(nn);
 	INIT_LIST_HEAD(&reaplist);
+
+	spin_lock(&nn->s2s_cp_lock);
+	idr_for_each_entry(&nn->s2s_cp_stateids, cps_t, i) {
+		cps = container_of(cps_t, struct nfs4_cpntf_state, cp_stateid);
+		if (cps->cp_stateid.sc_type == NFS4_COPYNOTIFY_STID &&
+				!time_after((unsigned long)cps->cpntf_time,
+				(unsigned long)cutoff))
+			_free_cpntf_state_locked(nn, cps);
+	}
+	spin_unlock(&nn->s2s_cp_lock);
+
 	spin_lock(&nn->client_lock);
 	list_for_each_safe(pos, next, &nn->client_lru) {
 		clp = list_entry(pos, struct nfs4_client, cl_lru);
@@ -5577,6 +5661,24 @@  static __be32 nfsd4_validate_stateid(struct nfs4_client *cl, stateid_t *stateid)
 out:
 	return status;
 }
+static void
+_free_cpntf_state_locked(struct nfsd_net *nn, struct nfs4_cpntf_state *cps)
+{
+	WARN_ON_ONCE(cps->cp_stateid.sc_type != NFS4_COPYNOTIFY_STID);
+	if (!refcount_dec_and_test(&cps->cp_stateid.sc_count))
+		return;
+	list_del(&cps->cp_list);
+	idr_remove(&nn->s2s_cp_stateids,
+		   cps->cp_stateid.stid.si_opaque.so_id);
+	kfree(cps);
+}
+
+void nfs4_put_cpntf_state(struct nfsd_net *nn, struct nfs4_cpntf_state *cps)
+{
+	spin_lock(&nn->s2s_cp_lock);
+	_free_cpntf_state_locked(nn, cps);
+	spin_unlock(&nn->s2s_cp_lock);
+}
 
 /*
  * Checks for stateid operations
diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h
index d9e7cbd..967b937 100644
--- a/fs/nfsd/state.h
+++ b/fs/nfsd/state.h
@@ -56,6 +56,14 @@ 
 	stateid_opaque_t        si_opaque;
 } stateid_t;
 
+typedef struct {
+	stateid_t		stid;
+#define NFS4_COPY_STID 1
+#define NFS4_COPYNOTIFY_STID 2
+	unsigned char		sc_type;
+	refcount_t		sc_count;
+} copy_stateid_t;
+
 #define STATEID_FMT	"(%08x/%08x/%08x/%08x)"
 #define STATEID_VAL(s) \
 	(s)->si_opaque.so_clid.cl_boot, \
@@ -96,6 +104,7 @@  struct nfs4_stid {
 #define NFS4_REVOKED_DELEG_STID 16
 #define NFS4_CLOSED_DELEG_STID 32
 #define NFS4_LAYOUT_STID 64
+	struct list_head	sc_cp_list;
 	unsigned char		sc_type;
 	stateid_t		sc_stateid;
 	spinlock_t		sc_lock;
@@ -104,6 +113,17 @@  struct nfs4_stid {
 	void			(*sc_free)(struct nfs4_stid *);
 };
 
+/* Keep a list of stateids issued by the COPY_NOTIFY, associate it with the
+ * parent OPEN/LOCK/DELEG stateid.
+ */
+struct nfs4_cpntf_state {
+	copy_stateid_t		cp_stateid;
+	struct list_head	cp_list;	/* per parent nfs4_stid */
+	stateid_t		cp_p_stateid;	/* copy of parent's stateid */
+	clientid_t		cp_p_clid;	/* copy of parent's clid */
+	time_t			cpntf_time;	/* last time stateid used */
+};
+
 /*
  * Represents a delegation stateid. The nfs4_client holds references to these
  * and they are put when it is being destroyed or when the delegation is
@@ -624,8 +644,10 @@  __be32 nfsd4_lookup_stateid(struct nfsd4_compound_state *cstate,
 		     struct nfs4_stid **s, struct nfsd_net *nn);
 struct nfs4_stid *nfs4_alloc_stid(struct nfs4_client *cl, struct kmem_cache *slab,
 				  void (*sc_free)(struct nfs4_stid *));
-int nfs4_init_cp_state(struct nfsd_net *nn, struct nfsd4_copy *copy);
-void nfs4_free_cp_state(struct nfsd4_copy *copy);
+int nfs4_init_copy_state(struct nfsd_net *nn, struct nfsd4_copy *copy);
+void nfs4_free_copy_state(struct nfsd4_copy *copy);
+struct nfs4_cpntf_state *nfs4_alloc_init_cpntf_state(struct nfsd_net *nn,
+			struct nfs4_stid *p_stid);
 void nfs4_unhash_stid(struct nfs4_stid *s);
 void nfs4_put_stid(struct nfs4_stid *s);
 void nfs4_inc_and_copy_stateid(stateid_t *dst, struct nfs4_stid *stid);
@@ -655,6 +677,8 @@  extern struct nfs4_client_reclaim *nfs4_client_to_reclaim(struct xdr_netobj name
 extern void nfs4_put_copy(struct nfsd4_copy *copy);
 extern struct nfsd4_copy *
 find_async_copy(struct nfs4_client *clp, stateid_t *staetid);
+extern void nfs4_put_cpntf_state(struct nfsd_net *nn,
+				 struct nfs4_cpntf_state *cps);
 static inline void get_nfs4_file(struct nfs4_file *fi)
 {
 	refcount_inc(&fi->fi_ref);
diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h
index 8231fe0..2937e06 100644
--- a/fs/nfsd/xdr4.h
+++ b/fs/nfsd/xdr4.h
@@ -542,7 +542,7 @@  struct nfsd4_copy {
 	struct nfsd_file        *nf_src;
 	struct nfsd_file        *nf_dst;
 
-	stateid_t		cp_stateid;
+	copy_stateid_t		cp_stateid;
 
 	struct list_head	copies;
 	struct task_struct	*copy_task;