diff mbox

Issue #5876 : assertion failure in rbd_img_obj_callback()

Message ID 53324F79.1080108@ieee.org (mailing list archive)
State New, archived
Headers show

Commit Message

Alex Elder March 26, 2014, 3:54 a.m. UTC
On 03/25/2014 09:45 PM, Olivier Bonvalet wrote:
> Le mardi 25 mars 2014 à 21:42 -0500, Alex Elder a écrit :
>> PS  I thought you said you were going to stop for the night!
> 
> Yes, I would love ! But my phone doesn't stop ring about ceph crash :D
> 

OK, one more thing to try and I'm going to bed.

I'm hoping that an image request spanning multiple objects
is an unusual case, enough so that the following won't
overwhelm with output.  I'd avoid putting it on a production
system (that's the case for all this testing, really) if
possible.

Basically I'm trying to catch an image object request being
either submitted more than once, or completing more than
once.  So if an image request has more than one object
request I produce some informative output.

This patch fixes two warnings in the previous debug patch,
and adds to it (so use it instead of the last one).

If you get a chance to try this I'll want to see the output.
But first, I shall sleep.

Thank you.

					-Alex


--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Comments

Olivier Bonvalet March 26, 2014, 4 a.m. UTC | #1
Le mardi 25 mars 2014 à 22:54 -0500, Alex Elder a écrit :
> On 03/25/2014 09:45 PM, Olivier Bonvalet wrote:
> > Le mardi 25 mars 2014 à 21:42 -0500, Alex Elder a écrit :
> >> PS  I thought you said you were going to stop for the night!
> > 
> > Yes, I would love ! But my phone doesn't stop ring about ceph crash :D
> > 
> 
> OK, one more thing to try and I'm going to bed.
> 
> I'm hoping that an image request spanning multiple objects
> is an unusual case, enough so that the following won't
> overwhelm with output.  I'd avoid putting it on a production
> system (that's the case for all this testing, really) if
> possible.
> 
> Basically I'm trying to catch an image object request being
> either submitted more than once, or completing more than
> once.  So if an image request has more than one object
> request I produce some informative output.
> 
> This patch fixes two warnings in the previous debug patch,
> and adds to it (so use it instead of the last one).
> 
> If you get a chance to try this I'll want to see the output.
> But first, I shall sleep.
> 
> Thank you.
> 
> 					-Alex
> 
> Index: b/drivers/block/rbd.c
> ===================================================================
> --- a/drivers/block/rbd.c
> +++ b/drivers/block/rbd.c
> @@ -1484,6 +1484,10 @@ static void rbd_img_request_complete(str
>  			xferred += obj_request->xferred;
>  		img_request->xferred = xferred;
>  	}
> +	if (img_request->obj_request_count > 1)
> +		printk("%s: img_request %p count %u result %d xferred %llu\n",
> +			__func__, img_request, img_request->obj_request_count,
> +			img_request->result, img_request->xferred);
> 
>  	if (img_request->callback)
>  		img_request->callback(img_request);
> @@ -2129,9 +2133,43 @@ static void rbd_img_obj_callback(struct
>  	rbd_assert(which != BAD_WHICH);
>  	rbd_assert(which < img_request->obj_request_count);
> 
> +	if (img_request->obj_request_count > 1)
> +		printk("%s: obj_request %p (%llu/%llu)\n", __func__,
> +			obj_request, obj_request->offset, obj_request->length);
>  	spin_lock_irq(&img_request->completion_lock);
>  	if (which > img_request->next_completion)
>  		goto out;
> +	if (which != img_request->next_completion) {
> +		printk("%s: bad image object request information:\n", __func__);
> +		printk("obj_request %p\n", obj_request);
> +		printk("    ->object_name <%s>\n", obj_request->object_name);
> +		printk("    ->offset %llu\n", obj_request->offset);
> +		printk("    ->length %llu\n", obj_request->length);
> +		printk("    ->type 0x%x\n", (u32)obj_request->type);
> +		printk("    ->flags 0x%lx\n", obj_request->flags);
> +		printk("    ->img_request %p\n", obj_request->img_request);
> +		printk("    ->which %u\n", obj_request->which);
> +		printk("    ->xferred %llu\n", obj_request->xferred);
> +		printk("    ->result %d\n", obj_request->result);
> +		printk("    ->kref %d\n",
> +			atomic_read(&obj_request->kref.refcount));
> +
> +		printk("img_request %p\n", img_request);
> +		printk("    ->snap 0x%016llx\n", img_request->snap_id);
> +		printk("    ->offset %llu\n", img_request->offset);
> +		printk("    ->length %llu\n", img_request->length);
> +		printk("    ->flags 0x%lx\n", img_request->flags);
> +		printk("    ->obj_request_count %u\n",
> +			img_request->obj_request_count);
> +		printk("    ->next_completion %u\n",
> +			img_request->next_completion);
> +		printk("    ->xferred %llu\n", img_request->xferred);
> +		printk("    ->result %d\n", img_request->result);
> +		printk("    ->obj_requests head %p\n",
> +			img_request->obj_requests.next);
> +		printk("    ->kref %d\n",
> +			atomic_read(&img_request->kref.refcount));
> +	}
>  	rbd_assert(which == img_request->next_completion);
> 
>  	for_each_obj_request_from(img_request, obj_request) {
> @@ -2697,11 +2735,21 @@ static int rbd_img_request_submit(struct
>  {
>  	struct rbd_obj_request *obj_request;
>  	struct rbd_obj_request *next_obj_request;
> +	bool verbose = false;
> 
>  	dout("%s: img %p\n", __func__, img_request);
> +	if (img_request->obj_request_count > 1) {
> +		printk("%s: img_request %p count %u (%llu/%llu)\n", __func__,
> +			img_request, img_request->offset, img_request->length);
> +	    	verbose = true;
> +	}
>  	for_each_obj_request_safe(img_request, obj_request, next_obj_request) {
>  		int ret;
> 
> +		if (verbose)
> +			printk("%s: obj_request %p (%llu/%llu)\n", __func__,
> +				obj_request, obj_request->offset,
> +				obj_request->length);
>  		ret = rbd_img_obj_request_submit(obj_request);
>  		if (ret)
>  			return ret;
> 
> 

Thanks ! I will check that tomorrow.

--
To unsubscribe from this list: send the line "unsubscribe ceph-devel" in
the body of a message to majordomo@vger.kernel.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html
diff mbox

Patch

Index: b/drivers/block/rbd.c
===================================================================
--- a/drivers/block/rbd.c
+++ b/drivers/block/rbd.c
@@ -1484,6 +1484,10 @@  static void rbd_img_request_complete(str
 			xferred += obj_request->xferred;
 		img_request->xferred = xferred;
 	}
+	if (img_request->obj_request_count > 1)
+		printk("%s: img_request %p count %u result %d xferred %llu\n",
+			__func__, img_request, img_request->obj_request_count,
+			img_request->result, img_request->xferred);

 	if (img_request->callback)
 		img_request->callback(img_request);
@@ -2129,9 +2133,43 @@  static void rbd_img_obj_callback(struct
 	rbd_assert(which != BAD_WHICH);
 	rbd_assert(which < img_request->obj_request_count);

+	if (img_request->obj_request_count > 1)
+		printk("%s: obj_request %p (%llu/%llu)\n", __func__,
+			obj_request, obj_request->offset, obj_request->length);
 	spin_lock_irq(&img_request->completion_lock);
 	if (which > img_request->next_completion)
 		goto out;
+	if (which != img_request->next_completion) {
+		printk("%s: bad image object request information:\n", __func__);
+		printk("obj_request %p\n", obj_request);
+		printk("    ->object_name <%s>\n", obj_request->object_name);
+		printk("    ->offset %llu\n", obj_request->offset);
+		printk("    ->length %llu\n", obj_request->length);
+		printk("    ->type 0x%x\n", (u32)obj_request->type);
+		printk("    ->flags 0x%lx\n", obj_request->flags);
+		printk("    ->img_request %p\n", obj_request->img_request);
+		printk("    ->which %u\n", obj_request->which);
+		printk("    ->xferred %llu\n", obj_request->xferred);
+		printk("    ->result %d\n", obj_request->result);
+		printk("    ->kref %d\n",
+			atomic_read(&obj_request->kref.refcount));
+
+		printk("img_request %p\n", img_request);
+		printk("    ->snap 0x%016llx\n", img_request->snap_id);
+		printk("    ->offset %llu\n", img_request->offset);
+		printk("    ->length %llu\n", img_request->length);
+		printk("    ->flags 0x%lx\n", img_request->flags);
+		printk("    ->obj_request_count %u\n",
+			img_request->obj_request_count);
+		printk("    ->next_completion %u\n",
+			img_request->next_completion);
+		printk("    ->xferred %llu\n", img_request->xferred);
+		printk("    ->result %d\n", img_request->result);
+		printk("    ->obj_requests head %p\n",
+			img_request->obj_requests.next);
+		printk("    ->kref %d\n",
+			atomic_read(&img_request->kref.refcount));
+	}
 	rbd_assert(which == img_request->next_completion);

 	for_each_obj_request_from(img_request, obj_request) {
@@ -2697,11 +2735,21 @@  static int rbd_img_request_submit(struct
 {
 	struct rbd_obj_request *obj_request;
 	struct rbd_obj_request *next_obj_request;
+	bool verbose = false;

 	dout("%s: img %p\n", __func__, img_request);
+	if (img_request->obj_request_count > 1) {
+		printk("%s: img_request %p count %u (%llu/%llu)\n", __func__,
+			img_request, img_request->offset, img_request->length);
+	    	verbose = true;
+	}
 	for_each_obj_request_safe(img_request, obj_request, next_obj_request) {
 		int ret;

+		if (verbose)
+			printk("%s: obj_request %p (%llu/%llu)\n", __func__,
+				obj_request, obj_request->offset,
+				obj_request->length);
 		ret = rbd_img_obj_request_submit(obj_request);
 		if (ret)
 			return ret;