| Message ID | 20170330180707.11137-3-jlayton@redhat.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
On Thu, Mar 30, 2017 at 8:07 PM, Jeff Layton <jlayton@redhat.com> wrote: > When a Ceph volume hits capacity, a flag is set in the OSD map to > indicate that, and a new map is sprayed around the cluster. With cephfs > we want it to shut down any abortable requests that are in progress with > an -ENOSPC error as they'd just hang otherwise. > > Add a new ceph_osdc_abort_on_full helper function to handle this. It > will first check whether there is an out-of-space condition in the > cluster and then walk the tree and abort any request that has > r_abort_on_full set with a -ENOSPC error. Call this new function > directly whenever we get a new OSD map. > > Reviewed-by: "Yan, Zheng” <zyan@redhat.com> > Signed-off-by: Jeff Layton <jlayton@redhat.com> > --- > net/ceph/osd_client.c | 35 +++++++++++++++++++++++++++++++++++ > 1 file changed, 35 insertions(+) > > diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c > index 781048990599..4e56cd1ec265 100644 > --- a/net/ceph/osd_client.c > +++ b/net/ceph/osd_client.c > @@ -1806,6 +1806,40 @@ static void abort_request(struct ceph_osd_request *req, int err) > complete_request(req, err); > } > > +/* > + * Drop all pending requests that are stalled waiting on a full condition to > + * clear, and complete them with ENOSPC as the return code. > + */ > +static void ceph_osdc_abort_on_full(struct ceph_osd_client *osdc) > +{ > + struct rb_node *n; > + bool osdmap_full = ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL); This variable is still redundant ;) > + > + dout("enter abort_on_full\n"); > + > + if (!osdmap_full && !have_pool_full(osdc)) > + goto out; > + > + for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { > + struct ceph_osd *osd = rb_entry(n, struct ceph_osd, o_node); > + struct rb_node *m; > + > + m = rb_first(&osd->o_requests); > + while (m) { > + struct ceph_osd_request *req = rb_entry(m, > + struct ceph_osd_request, r_node); > + m = rb_next(m); > + > + if (req->r_abort_on_full && > + (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) || > + pool_full(osdc, req->r_t.target_oloc.pool))) > + abort_request(req, -ENOSPC); > + } > + } > +out: > + dout("return abort_on_full\n"); > +} > + > static void check_pool_dne(struct ceph_osd_request *req) > { > struct ceph_osd_client *osdc = req->r_osdc; > @@ -3264,6 +3298,7 @@ void ceph_osdc_handle_map(struct ceph_osd_client *osdc, struct ceph_msg *msg) > > kick_requests(osdc, &need_resend, &need_resend_linger); > > + ceph_osdc_abort_on_full(osdc); > ceph_monc_got_map(&osdc->client->monc, CEPH_SUB_OSDMAP, > osdc->osdmap->epoch); > up_write(&osdc->lock); With osdmap_full dropped, Reviewed-by: Ilya Dryomov <idryomov@gmail.com> Thanks, Ilya -- To unsubscribe from this list: send the line "unsubscribe ceph-devel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
diff --git a/net/ceph/osd_client.c b/net/ceph/osd_client.c index 781048990599..4e56cd1ec265 100644 --- a/net/ceph/osd_client.c +++ b/net/ceph/osd_client.c @@ -1806,6 +1806,40 @@ static void abort_request(struct ceph_osd_request *req, int err) complete_request(req, err); } +/* + * Drop all pending requests that are stalled waiting on a full condition to + * clear, and complete them with ENOSPC as the return code. + */ +static void ceph_osdc_abort_on_full(struct ceph_osd_client *osdc) +{ + struct rb_node *n; + bool osdmap_full = ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL); + + dout("enter abort_on_full\n"); + + if (!osdmap_full && !have_pool_full(osdc)) + goto out; + + for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { + struct ceph_osd *osd = rb_entry(n, struct ceph_osd, o_node); + struct rb_node *m; + + m = rb_first(&osd->o_requests); + while (m) { + struct ceph_osd_request *req = rb_entry(m, + struct ceph_osd_request, r_node); + m = rb_next(m); + + if (req->r_abort_on_full && + (ceph_osdmap_flag(osdc, CEPH_OSDMAP_FULL) || + pool_full(osdc, req->r_t.target_oloc.pool))) + abort_request(req, -ENOSPC); + } + } +out: + dout("return abort_on_full\n"); +} + static void check_pool_dne(struct ceph_osd_request *req) { struct ceph_osd_client *osdc = req->r_osdc; @@ -3264,6 +3298,7 @@ void ceph_osdc_handle_map(struct ceph_osd_client *osdc, struct ceph_msg *msg) kick_requests(osdc, &need_resend, &need_resend_linger); + ceph_osdc_abort_on_full(osdc); ceph_monc_got_map(&osdc->client->monc, CEPH_SUB_OSDMAP, osdc->osdmap->epoch); up_write(&osdc->lock);