| Message ID | 1359793988-6881-1-git-send-email-ivan.djelic@parrot.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
On Sat, Feb 02, 2013 at 08:33:08AM +0000, Ivan Djelic wrote: > Recent GCC versions (e.g. GCC-4.7.2) perform optimizations based on > assumptions about the implementation of memset and similar functions. > The current ARM optimized memset code does not return the value of > its first argument, as is usually expected from standard implementations. > > For instance in the following function: > > void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter) > { > memset(waiter, MUTEX_DEBUG_INIT, sizeof(*waiter)); > waiter->magic = waiter; > INIT_LIST_HEAD(&waiter->list); > } > > compiled as: > > 800554d0 <debug_mutex_lock_common>: > 800554d0: e92d4008 push {r3, lr} > 800554d4: e1a00001 mov r0, r1 > 800554d8: e3a02010 mov r2, #16 ; 0x10 > 800554dc: e3a01011 mov r1, #17 ; 0x11 > 800554e0: eb04426e bl 80165ea0 <memset> > 800554e4: e1a03000 mov r3, r0 > 800554e8: e583000c str r0, [r3, #12] > 800554ec: e5830000 str r0, [r3] > 800554f0: e5830004 str r0, [r3, #4] > 800554f4: e8bd8008 pop {r3, pc} > > GCC assumes memset returns the value of pointer 'waiter' in register r0; causing > register/memory corruptions. > > This patch fixes the return value of the assembly version of memset. > Could you please review, or suggest better alternatives ? Ping, Thanks, -- Ivan > Thanks, > > -- > Ivan > > (this is a shorter and (hopefully) clearer repost of > http://lists.infradead.org/pipermail/linux-arm-kernel/2013-January/144916.html) > > The patch adds a 'mov' instruction and merges an additional load+store into > existing load/store instructions. > For ease of review, here is a breakdown of the patch into 4 simple steps: > > Step 1 > ====== > Perform the following substitutions: > ip -> r8, then > r0 -> ip, > and insert 'mov ip, r0' as the first statement of the function. > At this point, we have a memset() implementation returning the proper result, > but corrupting r8 on some paths (the ones that were using ip). > > Step 2 > ====== > Make sure r8 is saved and restored when (! CALGN(1)+0) == 1: > > save r8: > - str lr, [sp, #-4]! > + stmfd sp!, {r8, lr} > > and restore r8 on both exit paths: > - ldmeqfd sp!, {pc} @ Now <64 bytes to go. > + ldmeqfd sp!, {r8, pc} @ Now <64 bytes to go. > (...) > tst r2, #16 > stmneia ip!, {r1, r3, r8, lr} > - ldr lr, [sp], #4 > + ldmfd sp!, {r8, lr} > > Step 3 > ====== > Make sure r8 is saved and restored when (! CALGN(1)+0) == 0: > > save r8: > - stmfd sp!, {r4-r7, lr} > + stmfd sp!, {r4-r8, lr} > > and restore r8 on both exit paths: > bgt 3b > - ldmeqfd sp!, {r4-r7, pc} > + ldmeqfd sp!, {r4-r8, pc} > (...) > tst r2, #16 > stmneia ip!, {r4-r7} > - ldmfd sp!, {r4-r7, lr} > + ldmfd sp!, {r4-r8, lr} > > Step 4 > ====== > Rewrite register list "r4-r7, r8" as "r4-r8". > > Signed-off-by: Ivan Djelic <ivan.djelic@parrot.com> > --- > arch/arm/lib/memset.S | 85 +++++++++++++++++++++++++------------------------ > 1 file changed, 44 insertions(+), 41 deletions(-) > > diff --git a/arch/arm/lib/memset.S b/arch/arm/lib/memset.S > index 650d592..eafd5ee 100644 > --- a/arch/arm/lib/memset.S > +++ b/arch/arm/lib/memset.S > @@ -19,9 +19,9 @@ > 1: subs r2, r2, #4 @ 1 do we have enough > blt 5f @ 1 bytes to align with? > cmp r3, #2 @ 1 > - strltb r1, [r0], #1 @ 1 > - strleb r1, [r0], #1 @ 1 > - strb r1, [r0], #1 @ 1 > + strltb r1, [ip], #1 @ 1 > + strleb r1, [ip], #1 @ 1 > + strb r1, [ip], #1 @ 1 > add r2, r2, r3 @ 1 (r2 = r2 - (4 - r3)) > /* > * The pointer is now aligned and the length is adjusted. Try doing the > @@ -29,10 +29,14 @@ > */ > > ENTRY(memset) > - ands r3, r0, #3 @ 1 unaligned? > +/* > + * Preserve the contents of r0 for the return value. > + */ > + mov ip, r0 > + ands r3, ip, #3 @ 1 unaligned? > bne 1b @ 1 > /* > - * we know that the pointer in r0 is aligned to a word boundary. > + * we know that the pointer in ip is aligned to a word boundary. > */ > orr r1, r1, r1, lsl #8 > orr r1, r1, r1, lsl #16 > @@ -43,29 +47,28 @@ ENTRY(memset) > #if ! CALGN(1)+0 > > /* > - * We need an extra register for this loop - save the return address and > - * use the LR > + * We need an 2 extra registers for this loop - use r8 and the LR > */ > - str lr, [sp, #-4]! > - mov ip, r1 > + stmfd sp!, {r8, lr} > + mov r8, r1 > mov lr, r1 > > 2: subs r2, r2, #64 > - stmgeia r0!, {r1, r3, ip, lr} @ 64 bytes at a time. > - stmgeia r0!, {r1, r3, ip, lr} > - stmgeia r0!, {r1, r3, ip, lr} > - stmgeia r0!, {r1, r3, ip, lr} > + stmgeia ip!, {r1, r3, r8, lr} @ 64 bytes at a time. > + stmgeia ip!, {r1, r3, r8, lr} > + stmgeia ip!, {r1, r3, r8, lr} > + stmgeia ip!, {r1, r3, r8, lr} > bgt 2b > - ldmeqfd sp!, {pc} @ Now <64 bytes to go. > + ldmeqfd sp!, {r8, pc} @ Now <64 bytes to go. > /* > * No need to correct the count; we're only testing bits from now on > */ > tst r2, #32 > - stmneia r0!, {r1, r3, ip, lr} > - stmneia r0!, {r1, r3, ip, lr} > + stmneia ip!, {r1, r3, r8, lr} > + stmneia ip!, {r1, r3, r8, lr} > tst r2, #16 > - stmneia r0!, {r1, r3, ip, lr} > - ldr lr, [sp], #4 > + stmneia ip!, {r1, r3, r8, lr} > + ldmfd sp!, {r8, lr} > > #else > > @@ -74,54 +77,54 @@ ENTRY(memset) > * whole cache lines at once. > */ > > - stmfd sp!, {r4-r7, lr} > + stmfd sp!, {r4-r8, lr} > mov r4, r1 > mov r5, r1 > mov r6, r1 > mov r7, r1 > - mov ip, r1 > + mov r8, r1 > mov lr, r1 > > cmp r2, #96 > - tstgt r0, #31 > + tstgt ip, #31 > ble 3f > > - and ip, r0, #31 > - rsb ip, ip, #32 > - sub r2, r2, ip > - movs ip, ip, lsl #(32 - 4) > - stmcsia r0!, {r4, r5, r6, r7} > - stmmiia r0!, {r4, r5} > - tst ip, #(1 << 30) > - mov ip, r1 > - strne r1, [r0], #4 > + and r8, ip, #31 > + rsb r8, r8, #32 > + sub r2, r2, r8 > + movs r8, r8, lsl #(32 - 4) > + stmcsia ip!, {r4, r5, r6, r7} > + stmmiia ip!, {r4, r5} > + tst r8, #(1 << 30) > + mov r8, r1 > + strne r1, [ip], #4 > > 3: subs r2, r2, #64 > - stmgeia r0!, {r1, r3-r7, ip, lr} > - stmgeia r0!, {r1, r3-r7, ip, lr} > + stmgeia ip!, {r1, r3-r8, lr} > + stmgeia ip!, {r1, r3-r8, lr} > bgt 3b > - ldmeqfd sp!, {r4-r7, pc} > + ldmeqfd sp!, {r4-r8, pc} > > tst r2, #32 > - stmneia r0!, {r1, r3-r7, ip, lr} > + stmneia ip!, {r1, r3-r8, lr} > tst r2, #16 > - stmneia r0!, {r4-r7} > - ldmfd sp!, {r4-r7, lr} > + stmneia ip!, {r4-r7} > + ldmfd sp!, {r4-r8, lr} > > #endif > > 4: tst r2, #8 > - stmneia r0!, {r1, r3} > + stmneia ip!, {r1, r3} > tst r2, #4 > - strne r1, [r0], #4 > + strne r1, [ip], #4 > /* > * When we get here, we've got less than 4 bytes to zero. We > * may have an unaligned pointer as well. > */ > 5: tst r2, #2 > - strneb r1, [r0], #1 > - strneb r1, [r0], #1 > + strneb r1, [ip], #1 > + strneb r1, [ip], #1 > tst r2, #1 > - strneb r1, [r0], #1 > + strneb r1, [ip], #1 > mov pc, lr > ENDPROC(memset) > -- > 1.7.10.4
On Sat, 2 Feb 2013, Ivan Djelic wrote: > Recent GCC versions (e.g. GCC-4.7.2) perform optimizations based on > assumptions about the implementation of memset and similar functions. > The current ARM optimized memset code does not return the value of > its first argument, as is usually expected from standard implementations. > > For instance in the following function: > > void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter) > { > memset(waiter, MUTEX_DEBUG_INIT, sizeof(*waiter)); > waiter->magic = waiter; > INIT_LIST_HEAD(&waiter->list); > } > > compiled as: > > 800554d0 <debug_mutex_lock_common>: > 800554d0: e92d4008 push {r3, lr} > 800554d4: e1a00001 mov r0, r1 > 800554d8: e3a02010 mov r2, #16 ; 0x10 > 800554dc: e3a01011 mov r1, #17 ; 0x11 > 800554e0: eb04426e bl 80165ea0 <memset> > 800554e4: e1a03000 mov r3, r0 > 800554e8: e583000c str r0, [r3, #12] > 800554ec: e5830000 str r0, [r3] > 800554f0: e5830004 str r0, [r3, #4] > 800554f4: e8bd8008 pop {r3, pc} > > GCC assumes memset returns the value of pointer 'waiter' in register r0; causing > register/memory corruptions. > > This patch fixes the return value of the assembly version of memset. > Could you please review, or suggest better alternatives ? > > Thanks, > > -- > Ivan > > (this is a shorter and (hopefully) clearer repost of > http://lists.infradead.org/pipermail/linux-arm-kernel/2013-January/144916.html) > > The patch adds a 'mov' instruction and merges an additional load+store into > existing load/store instructions. > For ease of review, here is a breakdown of the patch into 4 simple steps: > > Step 1 > ====== > Perform the following substitutions: > ip -> r8, then > r0 -> ip, > and insert 'mov ip, r0' as the first statement of the function. > At this point, we have a memset() implementation returning the proper result, > but corrupting r8 on some paths (the ones that were using ip). > > Step 2 > ====== > Make sure r8 is saved and restored when (! CALGN(1)+0) == 1: > > save r8: > - str lr, [sp, #-4]! > + stmfd sp!, {r8, lr} > > and restore r8 on both exit paths: > - ldmeqfd sp!, {pc} @ Now <64 bytes to go. > + ldmeqfd sp!, {r8, pc} @ Now <64 bytes to go. > (...) > tst r2, #16 > stmneia ip!, {r1, r3, r8, lr} > - ldr lr, [sp], #4 > + ldmfd sp!, {r8, lr} > > Step 3 > ====== > Make sure r8 is saved and restored when (! CALGN(1)+0) == 0: > > save r8: > - stmfd sp!, {r4-r7, lr} > + stmfd sp!, {r4-r8, lr} > > and restore r8 on both exit paths: > bgt 3b > - ldmeqfd sp!, {r4-r7, pc} > + ldmeqfd sp!, {r4-r8, pc} > (...) > tst r2, #16 > stmneia ip!, {r4-r7} > - ldmfd sp!, {r4-r7, lr} > + ldmfd sp!, {r4-r8, lr} > > Step 4 > ====== > Rewrite register list "r4-r7, r8" as "r4-r8". > > Signed-off-by: Ivan Djelic <ivan.djelic@parrot.com> Reviewed-by: Nicolas Pitre <nico@linaro.org> This is good for the stable tree as well. Please include in your official commit text the above breakdown explanation as well. > --- > arch/arm/lib/memset.S | 85 +++++++++++++++++++++++++------------------------ > 1 file changed, 44 insertions(+), 41 deletions(-) > > diff --git a/arch/arm/lib/memset.S b/arch/arm/lib/memset.S > index 650d592..eafd5ee 100644 > --- a/arch/arm/lib/memset.S > +++ b/arch/arm/lib/memset.S > @@ -19,9 +19,9 @@ > 1: subs r2, r2, #4 @ 1 do we have enough > blt 5f @ 1 bytes to align with? > cmp r3, #2 @ 1 > - strltb r1, [r0], #1 @ 1 > - strleb r1, [r0], #1 @ 1 > - strb r1, [r0], #1 @ 1 > + strltb r1, [ip], #1 @ 1 > + strleb r1, [ip], #1 @ 1 > + strb r1, [ip], #1 @ 1 > add r2, r2, r3 @ 1 (r2 = r2 - (4 - r3)) > /* > * The pointer is now aligned and the length is adjusted. Try doing the > @@ -29,10 +29,14 @@ > */ > > ENTRY(memset) > - ands r3, r0, #3 @ 1 unaligned? > +/* > + * Preserve the contents of r0 for the return value. > + */ > + mov ip, r0 > + ands r3, ip, #3 @ 1 unaligned? > bne 1b @ 1 > /* > - * we know that the pointer in r0 is aligned to a word boundary. > + * we know that the pointer in ip is aligned to a word boundary. > */ > orr r1, r1, r1, lsl #8 > orr r1, r1, r1, lsl #16 > @@ -43,29 +47,28 @@ ENTRY(memset) > #if ! CALGN(1)+0 > > /* > - * We need an extra register for this loop - save the return address and > - * use the LR > + * We need an 2 extra registers for this loop - use r8 and the LR > */ > - str lr, [sp, #-4]! > - mov ip, r1 > + stmfd sp!, {r8, lr} > + mov r8, r1 > mov lr, r1 > > 2: subs r2, r2, #64 > - stmgeia r0!, {r1, r3, ip, lr} @ 64 bytes at a time. > - stmgeia r0!, {r1, r3, ip, lr} > - stmgeia r0!, {r1, r3, ip, lr} > - stmgeia r0!, {r1, r3, ip, lr} > + stmgeia ip!, {r1, r3, r8, lr} @ 64 bytes at a time. > + stmgeia ip!, {r1, r3, r8, lr} > + stmgeia ip!, {r1, r3, r8, lr} > + stmgeia ip!, {r1, r3, r8, lr} > bgt 2b > - ldmeqfd sp!, {pc} @ Now <64 bytes to go. > + ldmeqfd sp!, {r8, pc} @ Now <64 bytes to go. > /* > * No need to correct the count; we're only testing bits from now on > */ > tst r2, #32 > - stmneia r0!, {r1, r3, ip, lr} > - stmneia r0!, {r1, r3, ip, lr} > + stmneia ip!, {r1, r3, r8, lr} > + stmneia ip!, {r1, r3, r8, lr} > tst r2, #16 > - stmneia r0!, {r1, r3, ip, lr} > - ldr lr, [sp], #4 > + stmneia ip!, {r1, r3, r8, lr} > + ldmfd sp!, {r8, lr} > > #else > > @@ -74,54 +77,54 @@ ENTRY(memset) > * whole cache lines at once. > */ > > - stmfd sp!, {r4-r7, lr} > + stmfd sp!, {r4-r8, lr} > mov r4, r1 > mov r5, r1 > mov r6, r1 > mov r7, r1 > - mov ip, r1 > + mov r8, r1 > mov lr, r1 > > cmp r2, #96 > - tstgt r0, #31 > + tstgt ip, #31 > ble 3f > > - and ip, r0, #31 > - rsb ip, ip, #32 > - sub r2, r2, ip > - movs ip, ip, lsl #(32 - 4) > - stmcsia r0!, {r4, r5, r6, r7} > - stmmiia r0!, {r4, r5} > - tst ip, #(1 << 30) > - mov ip, r1 > - strne r1, [r0], #4 > + and r8, ip, #31 > + rsb r8, r8, #32 > + sub r2, r2, r8 > + movs r8, r8, lsl #(32 - 4) > + stmcsia ip!, {r4, r5, r6, r7} > + stmmiia ip!, {r4, r5} > + tst r8, #(1 << 30) > + mov r8, r1 > + strne r1, [ip], #4 > > 3: subs r2, r2, #64 > - stmgeia r0!, {r1, r3-r7, ip, lr} > - stmgeia r0!, {r1, r3-r7, ip, lr} > + stmgeia ip!, {r1, r3-r8, lr} > + stmgeia ip!, {r1, r3-r8, lr} > bgt 3b > - ldmeqfd sp!, {r4-r7, pc} > + ldmeqfd sp!, {r4-r8, pc} > > tst r2, #32 > - stmneia r0!, {r1, r3-r7, ip, lr} > + stmneia ip!, {r1, r3-r8, lr} > tst r2, #16 > - stmneia r0!, {r4-r7} > - ldmfd sp!, {r4-r7, lr} > + stmneia ip!, {r4-r7} > + ldmfd sp!, {r4-r8, lr} > > #endif > > 4: tst r2, #8 > - stmneia r0!, {r1, r3} > + stmneia ip!, {r1, r3} > tst r2, #4 > - strne r1, [r0], #4 > + strne r1, [ip], #4 > /* > * When we get here, we've got less than 4 bytes to zero. We > * may have an unaligned pointer as well. > */ > 5: tst r2, #2 > - strneb r1, [r0], #1 > - strneb r1, [r0], #1 > + strneb r1, [ip], #1 > + strneb r1, [ip], #1 > tst r2, #1 > - strneb r1, [r0], #1 > + strneb r1, [ip], #1 > mov pc, lr > ENDPROC(memset) > -- > 1.7.10.4 > > > _______________________________________________ > linux-arm-kernel mailing list > linux-arm-kernel@lists.infradead.org > http://lists.infradead.org/mailman/listinfo/linux-arm-kernel >
On Sat, Feb 09, 2013 at 02:48:31PM +0000, Nicolas Pitre wrote: > On Sat, 2 Feb 2013, Ivan Djelic wrote: > > > Recent GCC versions (e.g. GCC-4.7.2) perform optimizations based on > > assumptions about the implementation of memset and similar functions. > > The current ARM optimized memset code does not return the value of > > its first argument, as is usually expected from standard implementations. > > > > For instance in the following function: > > > > void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter) > > { > > memset(waiter, MUTEX_DEBUG_INIT, sizeof(*waiter)); > > waiter->magic = waiter; > > INIT_LIST_HEAD(&waiter->list); > > } > > > > compiled as: > > > > 800554d0 <debug_mutex_lock_common>: > > 800554d0: e92d4008 push {r3, lr} > > 800554d4: e1a00001 mov r0, r1 > > 800554d8: e3a02010 mov r2, #16 ; 0x10 > > 800554dc: e3a01011 mov r1, #17 ; 0x11 > > 800554e0: eb04426e bl 80165ea0 <memset> > > 800554e4: e1a03000 mov r3, r0 > > 800554e8: e583000c str r0, [r3, #12] > > 800554ec: e5830000 str r0, [r3] > > 800554f0: e5830004 str r0, [r3, #4] > > 800554f4: e8bd8008 pop {r3, pc} > > > > GCC assumes memset returns the value of pointer 'waiter' in register r0; causing > > register/memory corruptions. > > > > This patch fixes the return value of the assembly version of memset. > > Could you please review, or suggest better alternatives ? > > > > Thanks, > > > > -- > > Ivan > > > > (this is a shorter and (hopefully) clearer repost of > > http://lists.infradead.org/pipermail/linux-arm-kernel/2013-January/144916.html) > > > > The patch adds a 'mov' instruction and merges an additional load+store into > > existing load/store instructions. > > For ease of review, here is a breakdown of the patch into 4 simple steps: > > > > Step 1 > > ====== > > Perform the following substitutions: > > ip -> r8, then > > r0 -> ip, > > and insert 'mov ip, r0' as the first statement of the function. > > At this point, we have a memset() implementation returning the proper result, > > but corrupting r8 on some paths (the ones that were using ip). > > > > Step 2 > > ====== > > Make sure r8 is saved and restored when (! CALGN(1)+0) == 1: > > > > save r8: > > - str lr, [sp, #-4]! > > + stmfd sp!, {r8, lr} > > > > and restore r8 on both exit paths: > > - ldmeqfd sp!, {pc} @ Now <64 bytes to go. > > + ldmeqfd sp!, {r8, pc} @ Now <64 bytes to go. > > (...) > > tst r2, #16 > > stmneia ip!, {r1, r3, r8, lr} > > - ldr lr, [sp], #4 > > + ldmfd sp!, {r8, lr} > > > > Step 3 > > ====== > > Make sure r8 is saved and restored when (! CALGN(1)+0) == 0: > > > > save r8: > > - stmfd sp!, {r4-r7, lr} > > + stmfd sp!, {r4-r8, lr} > > > > and restore r8 on both exit paths: > > bgt 3b > > - ldmeqfd sp!, {r4-r7, pc} > > + ldmeqfd sp!, {r4-r8, pc} > > (...) > > tst r2, #16 > > stmneia ip!, {r4-r7} > > - ldmfd sp!, {r4-r7, lr} > > + ldmfd sp!, {r4-r8, lr} > > > > Step 4 > > ====== > > Rewrite register list "r4-r7, r8" as "r4-r8". > > > > Signed-off-by: Ivan Djelic <ivan.djelic@parrot.com> > > Reviewed-by: Nicolas Pitre <nico@linaro.org> > > This is good for the stable tree as well. > > Please include in your official commit text the above breakdown > explanation as well. OK, will do. Thanks a lot for reviewing, -- Ivan
On 02/02/13 08:33, Ivan Djelic wrote: > Recent GCC versions (e.g. GCC-4.7.2) perform optimizations based on > assumptions about the implementation of memset and similar functions. > The current ARM optimized memset code does not return the value of > its first argument, as is usually expected from standard implementations. > > For instance in the following function: > > void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter) > { > memset(waiter, MUTEX_DEBUG_INIT, sizeof(*waiter)); > waiter->magic = waiter; > INIT_LIST_HEAD(&waiter->list); > } > > compiled as: > > 800554d0<debug_mutex_lock_common>: > 800554d0: e92d4008 push {r3, lr} > 800554d4: e1a00001 mov r0, r1 > 800554d8: e3a02010 mov r2, #16 ; 0x10 > 800554dc: e3a01011 mov r1, #17 ; 0x11 > 800554e0: eb04426e bl 80165ea0<memset> > 800554e4: e1a03000 mov r3, r0 > 800554e8: e583000c str r0, [r3, #12] > 800554ec: e5830000 str r0, [r3] > 800554f0: e5830004 str r0, [r3, #4] > 800554f4: e8bd8008 pop {r3, pc} > > GCC assumes memset returns the value of pointer 'waiter' in register r0; causing > register/memory corruptions. > @@ -43,29 +47,28 @@ ENTRY(memset) > #if ! CALGN(1)+0 > > /* > - * We need an extra register for this loop - save the return address and > - * use the LR > + * We need an 2 extra registers for this loop - use r8 and the LR > */ > - str lr, [sp, #-4]! > - mov ip, r1 > + stmfd sp!, {r8, lr} > + mov r8, r1 > mov lr, r1 Out of interest, why not save {r0, lr} and avoid having to re-write the entirety of the inner loop? > > 2: subs r2, r2, #64 > - stmgeia r0!, {r1, r3, ip, lr} @ 64 bytes at a time. > - stmgeia r0!, {r1, r3, ip, lr} > - stmgeia r0!, {r1, r3, ip, lr} > - stmgeia r0!, {r1, r3, ip, lr} > + stmgeia ip!, {r1, r3, r8, lr} @ 64 bytes at a time. > + stmgeia ip!, {r1, r3, r8, lr} > + stmgeia ip!, {r1, r3, r8, lr} > + stmgeia ip!, {r1, r3, r8, lr} > bgt 2b > - ldmeqfd sp!, {pc} @ Now<64 bytes to go. > + ldmeqfd sp!, {r8, pc} @ Now<64 bytes to go.
On Sat, Feb 02, 2013 at 08:33:08AM +0000, Ivan Djelic wrote: > Recent GCC versions (e.g. GCC-4.7.2) perform optimizations based on > assumptions about the implementation of memset and similar functions. > The current ARM optimized memset code does not return the value of > its first argument, as is usually expected from standard implementations. How does GCC do this? By strcmping the function name and assuming that memset is a libc implementation? If so, maybe passing something like -ffreestanding would make sense to turn this behaviour off in the compiler (otherwise we should also vet the rest of the standard string functions). Will
On Mon, Feb 11, 2013 at 06:41:14PM +0000, Will Deacon wrote: > On Sat, Feb 02, 2013 at 08:33:08AM +0000, Ivan Djelic wrote: > > Recent GCC versions (e.g. GCC-4.7.2) perform optimizations based on > > assumptions about the implementation of memset and similar functions. > > The current ARM optimized memset code does not return the value of > > its first argument, as is usually expected from standard implementations. > > How does GCC do this? By strcmping the function name and assuming that > memset is a libc implementation? > > If so, maybe passing something like -ffreestanding would make sense to turn > this behaviour off in the compiler (otherwise we should also vet the rest of > the standard string functions). In theory, yes; but there is actually a short list of libc functions that GCC always requires from the environment, even when -ffreestanding is used: memcpy, memmove, memset and memcmp (see [1] below). Which also means that the same corruption problem potentially exists for current ARM assembly implementations of memcpy and memmove... -- Ivan [1] http://gcc.gnu.org/onlinedocs/gcc/Standards.html GCC aims towards being usable as a conforming freestanding implementation, or as the compiler for a conforming hosted implementation. By default, it will act as the compiler for a hosted implementation, defining __STDC_HOSTED__ as 1 and presuming that when the names of ISO C functions are used, they have the semantics defined in the standard. To make it act as a conforming freestanding implementation for a freestanding environment, use the option -ffreestanding; it will then define __STDC_HOSTED__ to 0 and not make assumptions about the meanings of function names from the standard library, with exceptions noted below. To build an OS kernel, you may well still need to make your own arrangements for linking and startup. See Options Controlling C Dialect. GCC does not provide the library facilities required only of hosted implementations, nor yet all the facilities required by C99 of freestanding implementations; to use the facilities of a hosted environment, you will need to find them elsewhere (for example, in the GNU C library). See Standard Libraries. Most of the compiler support routines used by GCC are present in libgcc, but there are a few exceptions. GCC requires the freestanding environment provide memcpy, memmove, memset and memcmp. Finally, if __builtin_trap is used, and the target does not implement the trap pattern, then GCC will emit a call to abort.
On Mon, Feb 11, 2013 at 06:17:32PM +0000, Ben Dooks wrote: > On 02/02/13 08:33, Ivan Djelic wrote: > > Recent GCC versions (e.g. GCC-4.7.2) perform optimizations based on > > assumptions about the implementation of memset and similar functions. > > The current ARM optimized memset code does not return the value of > > its first argument, as is usually expected from standard implementations. > > > > For instance in the following function: > > > > void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter) > > { > > memset(waiter, MUTEX_DEBUG_INIT, sizeof(*waiter)); > > waiter->magic = waiter; > > INIT_LIST_HEAD(&waiter->list); > > } > > > > compiled as: > > > > 800554d0<debug_mutex_lock_common>: > > 800554d0: e92d4008 push {r3, lr} > > 800554d4: e1a00001 mov r0, r1 > > 800554d8: e3a02010 mov r2, #16 ; 0x10 > > 800554dc: e3a01011 mov r1, #17 ; 0x11 > > 800554e0: eb04426e bl 80165ea0<memset> > > 800554e4: e1a03000 mov r3, r0 > > 800554e8: e583000c str r0, [r3, #12] > > 800554ec: e5830000 str r0, [r3] > > 800554f0: e5830004 str r0, [r3, #4] > > 800554f4: e8bd8008 pop {r3, pc} > > > > GCC assumes memset returns the value of pointer 'waiter' in register r0; causing > > register/memory corruptions. > > > > @@ -43,29 +47,28 @@ ENTRY(memset) > > #if ! CALGN(1)+0 > > > > /* > > - * We need an extra register for this loop - save the return address and > > - * use the LR > > + * We need an 2 extra registers for this loop - use r8 and the LR > > */ > > - str lr, [sp, #-4]! > > - mov ip, r1 > > + stmfd sp!, {r8, lr} > > + mov r8, r1 > > mov lr, r1 > > Out of interest, why not save {r0, lr} and avoid having to > re-write the entirety of the inner loop? Because at the inner loop entry, r0 no longer contains the current write pointer: ip is the current pointer, possibly different from r0 because of the unaligned fixup code. The idea was to avoid introducing an extra load+store on load/store-free paths; thereby using a register other than r0 for the current pointer... BR, -- Ivan > > > > > 2: subs r2, r2, #64 > > - stmgeia r0!, {r1, r3, ip, lr} @ 64 bytes at a time. > > - stmgeia r0!, {r1, r3, ip, lr} > > - stmgeia r0!, {r1, r3, ip, lr} > > - stmgeia r0!, {r1, r3, ip, lr} > > + stmgeia ip!, {r1, r3, r8, lr} @ 64 bytes at a time. > > + stmgeia ip!, {r1, r3, r8, lr} > > + stmgeia ip!, {r1, r3, r8, lr} > > + stmgeia ip!, {r1, r3, r8, lr} > > bgt 2b > > - ldmeqfd sp!, {pc} @ Now<64 bytes to go. > > + ldmeqfd sp!, {r8, pc} @ Now<64 bytes to go. > > > -- > Ben Dooks http://www.codethink.co.uk/ > Senior Engineer Codethink - Providing Genius
On Mon, Feb 11, 2013 at 07:42:25PM +0000, Ivan Djelic wrote: > On Mon, Feb 11, 2013 at 06:41:14PM +0000, Will Deacon wrote: > > On Sat, Feb 02, 2013 at 08:33:08AM +0000, Ivan Djelic wrote: > > > Recent GCC versions (e.g. GCC-4.7.2) perform optimizations based on > > > assumptions about the implementation of memset and similar functions. > > > The current ARM optimized memset code does not return the value of > > > its first argument, as is usually expected from standard implementations. > > > > How does GCC do this? By strcmping the function name and assuming that > > memset is a libc implementation? > > > > If so, maybe passing something like -ffreestanding would make sense to turn > > this behaviour off in the compiler (otherwise we should also vet the rest of > > the standard string functions). > > In theory, yes; but there is actually a short list of libc functions that GCC > always requires from the environment, even when -ffreestanding is used: memcpy, > memmove, memset and memcmp (see [1] below). Interesting... the GCC documentation also states that ffreestanding implies fno-builtin, so memset and co shouldn't be targetted for this sort of optimisation by GCC. Have you observed this problem even when passing this option? Will
On Tue, Feb 12, 2013 at 02:00:08PM +0000, Will Deacon wrote: > On Mon, Feb 11, 2013 at 07:42:25PM +0000, Ivan Djelic wrote: > > On Mon, Feb 11, 2013 at 06:41:14PM +0000, Will Deacon wrote: > > > On Sat, Feb 02, 2013 at 08:33:08AM +0000, Ivan Djelic wrote: > > > > Recent GCC versions (e.g. GCC-4.7.2) perform optimizations based on > > > > assumptions about the implementation of memset and similar functions. > > > > The current ARM optimized memset code does not return the value of > > > > its first argument, as is usually expected from standard implementations. > > > > > > How does GCC do this? By strcmping the function name and assuming that > > > memset is a libc implementation? > > > > > > If so, maybe passing something like -ffreestanding would make sense to turn > > > this behaviour off in the compiler (otherwise we should also vet the rest of > > > the standard string functions). > > > > In theory, yes; but there is actually a short list of libc functions that GCC > > always requires from the environment, even when -ffreestanding is used: memcpy, > > memmove, memset and memcmp (see [1] below). > > Interesting... the GCC documentation also states that ffreestanding implies > fno-builtin, so memset and co shouldn't be targetted for this sort of > optimisation by GCC. Have you observed this problem even when passing this > option? Rather than wondering whether we should be using -ffreestanding or not (which, x86 people have strongly resisted) I suggest that we just fix our memset() implementation to be compliant. The reason it's not compliant is that I saw no reason for it to be compliant back in the gcc 2.7.x days, and it's persisted like that for the last 19-ish years. If GCC is now making use of the return value, then we need to fix that and undo the "optimization" in our string.h. So let's just bite the bullet, make the asm memset() compliant, and clean up string.h.
On Tue, Feb 12, 2013 at 03:58:01PM +0000, Russell King - ARM Linux wrote: > On Tue, Feb 12, 2013 at 02:00:08PM +0000, Will Deacon wrote: > > Interesting... the GCC documentation also states that ffreestanding implies > > fno-builtin, so memset and co shouldn't be targetted for this sort of > > optimisation by GCC. Have you observed this problem even when passing this > > option? > > Rather than wondering whether we should be using -ffreestanding or not > (which, x86 people have strongly resisted) I suggest that we just fix > our memset() implementation to be compliant. > > The reason it's not compliant is that I saw no reason for it to be > compliant back in the gcc 2.7.x days, and it's persisted like that for > the last 19-ish years. If GCC is now making use of the return value, > then we need to fix that and undo the "optimization" in our string.h. > > So let's just bite the bullet, make the asm memset() compliant, and > clean up string.h. That would be the ideal thing to do, because it allows the compiler to optimise around these functions, however it does mean we need to check/fix *all* of the string functions at least (if we don't pass -fno-builtin then any builtin function is up for optimisation, including strcpy etc). Will
On Tue, Feb 12, 2013 at 04:36:45PM +0000, Will Deacon wrote: > That would be the ideal thing to do, because it allows the compiler to > optimise around these functions, however it does mean we need to check/fix > *all* of the string functions at least (if we don't pass -fno-builtin then > any builtin function is up for optimisation, including strcpy etc). No we don't. memset() is the only one which I created in a non-standard way.
On Tue, Feb 12, 2013 at 04:37:26PM +0000, Russell King - ARM Linux wrote: > On Tue, Feb 12, 2013 at 04:36:45PM +0000, Will Deacon wrote: > > That would be the ideal thing to do, because it allows the compiler to > > optimise around these functions, however it does mean we need to check/fix > > *all* of the string functions at least (if we don't pass -fno-builtin then > > any builtin function is up for optimisation, including strcpy etc). > > No we don't. memset() is the only one which I created in a non-standard > way. Ok, I hadn't checked so that saves us some effort. Cheers, Will
On 12.02.2013 17:36, Will Deacon wrote: > On Tue, Feb 12, 2013 at 03:58:01PM +0000, Russell King - ARM Linux wrote: >> On Tue, Feb 12, 2013 at 02:00:08PM +0000, Will Deacon wrote: >>> Interesting... the GCC documentation also states that ffreestanding implies >>> fno-builtin, so memset and co shouldn't be targetted for this sort of >>> optimisation by GCC. Have you observed this problem even when passing this >>> option? >> >> Rather than wondering whether we should be using -ffreestanding or not >> (which, x86 people have strongly resisted) I suggest that we just fix >> our memset() implementation to be compliant. >> >> The reason it's not compliant is that I saw no reason for it to be >> compliant back in the gcc 2.7.x days, and it's persisted like that for >> the last 19-ish years. If GCC is now making use of the return value, >> then we need to fix that and undo the "optimization" in our string.h. >> >> So let's just bite the bullet, make the asm memset() compliant, and >> clean up string.h. > > That would be the ideal thing to do, because it allows the compiler to > optimise around these functions, however it does mean we need to check/fix > *all* of the string functions at least (if we don't pass -fno-builtin then > any builtin function is up for optimisation, including strcpy etc). Do we already have an agreed solution for this issue anywhere, now? It seems some people hit by this issue are about to import Ivan's v2 patch. Which doesn't seem to be the final fix, though, if I read the history of this thread correctly? Many thanks Dirk
Hi Dirk, On Tue, Mar 05, 2013 at 01:50:06PM +0000, Dirk Behme wrote: > On 12.02.2013 17:36, Will Deacon wrote: > > On Tue, Feb 12, 2013 at 03:58:01PM +0000, Russell King - ARM Linux wrote: > >> On Tue, Feb 12, 2013 at 02:00:08PM +0000, Will Deacon wrote: > >>> Interesting... the GCC documentation also states that ffreestanding implies > >>> fno-builtin, so memset and co shouldn't be targetted for this sort of > >>> optimisation by GCC. Have you observed this problem even when passing this > >>> option? > >> > >> Rather than wondering whether we should be using -ffreestanding or not > >> (which, x86 people have strongly resisted) I suggest that we just fix > >> our memset() implementation to be compliant. > >> > >> The reason it's not compliant is that I saw no reason for it to be > >> compliant back in the gcc 2.7.x days, and it's persisted like that for > >> the last 19-ish years. If GCC is now making use of the return value, > >> then we need to fix that and undo the "optimization" in our string.h. > >> > >> So let's just bite the bullet, make the asm memset() compliant, and > >> clean up string.h. > > > > That would be the ideal thing to do, because it allows the compiler to > > optimise around these functions, however it does mean we need to check/fix > > *all* of the string functions at least (if we don't pass -fno-builtin then > > any builtin function is up for optimisation, including strcpy etc). > > Do we already have an agreed solution for this issue anywhere, now? I thought that the conclusion was to go with Ivan's suggestion since memset is the only function which doesn't follow what the compiler expects. Will
On 06.03.2013 02:42, Will Deacon wrote: > Hi Dirk, > > On Tue, Mar 05, 2013 at 01:50:06PM +0000, Dirk Behme wrote: >> On 12.02.2013 17:36, Will Deacon wrote: >>> On Tue, Feb 12, 2013 at 03:58:01PM +0000, Russell King - ARM Linux wrote: >>>> On Tue, Feb 12, 2013 at 02:00:08PM +0000, Will Deacon wrote: >>>>> Interesting... the GCC documentation also states that ffreestanding implies >>>>> fno-builtin, so memset and co shouldn't be targetted for this sort of >>>>> optimisation by GCC. Have you observed this problem even when passing this >>>>> option? >>>> Rather than wondering whether we should be using -ffreestanding or not >>>> (which, x86 people have strongly resisted) I suggest that we just fix >>>> our memset() implementation to be compliant. >>>> >>>> The reason it's not compliant is that I saw no reason for it to be >>>> compliant back in the gcc 2.7.x days, and it's persisted like that for >>>> the last 19-ish years. If GCC is now making use of the return value, >>>> then we need to fix that and undo the "optimization" in our string.h. >>>> >>>> So let's just bite the bullet, make the asm memset() compliant, and >>>> clean up string.h. >>> That would be the ideal thing to do, because it allows the compiler to >>> optimise around these functions, however it does mean we need to check/fix >>> *all* of the string functions at least (if we don't pass -fno-builtin then >>> any builtin function is up for optimisation, including strcpy etc). >> Do we already have an agreed solution for this issue anywhere, now? > > I thought that the conclusion was to go with Ivan's suggestion since memset > is the only function which doesn't follow what the compiler expects. Ok, thanks! :) Could we get Ivan's v2 patch integrated or at least sent to the patch system, then, please? Many thanks, Dirk
On Tue, Mar 05, 2013 at 02:50:06PM +0100, Dirk Behme wrote: > On 12.02.2013 17:36, Will Deacon wrote: >> On Tue, Feb 12, 2013 at 03:58:01PM +0000, Russell King - ARM Linux wrote: >>> On Tue, Feb 12, 2013 at 02:00:08PM +0000, Will Deacon wrote: >>>> Interesting... the GCC documentation also states that ffreestanding implies >>>> fno-builtin, so memset and co shouldn't be targetted for this sort of >>>> optimisation by GCC. Have you observed this problem even when passing this >>>> option? >>> >>> Rather than wondering whether we should be using -ffreestanding or not >>> (which, x86 people have strongly resisted) I suggest that we just fix >>> our memset() implementation to be compliant. >>> >>> The reason it's not compliant is that I saw no reason for it to be >>> compliant back in the gcc 2.7.x days, and it's persisted like that for >>> the last 19-ish years. If GCC is now making use of the return value, >>> then we need to fix that and undo the "optimization" in our string.h. >>> >>> So let's just bite the bullet, make the asm memset() compliant, and >>> clean up string.h. >> >> That would be the ideal thing to do, because it allows the compiler to >> optimise around these functions, however it does mean we need to check/fix >> *all* of the string functions at least (if we don't pass -fno-builtin then >> any builtin function is up for optimisation, including strcpy etc). > > Do we already have an agreed solution for this issue anywhere, now? No idea. I've stated above what I think should happen. Where's the disagreement?
Am 06.03.2013 18:11, schrieb Russell King - ARM Linux: > On Tue, Mar 05, 2013 at 02:50:06PM +0100, Dirk Behme wrote: >> On 12.02.2013 17:36, Will Deacon wrote: >>> On Tue, Feb 12, 2013 at 03:58:01PM +0000, Russell King - ARM Linux wrote: >>>> On Tue, Feb 12, 2013 at 02:00:08PM +0000, Will Deacon wrote: >>>>> Interesting... the GCC documentation also states that ffreestanding implies >>>>> fno-builtin, so memset and co shouldn't be targetted for this sort of >>>>> optimisation by GCC. Have you observed this problem even when passing this >>>>> option? >>>> >>>> Rather than wondering whether we should be using -ffreestanding or not >>>> (which, x86 people have strongly resisted) I suggest that we just fix >>>> our memset() implementation to be compliant. >>>> >>>> The reason it's not compliant is that I saw no reason for it to be >>>> compliant back in the gcc 2.7.x days, and it's persisted like that for >>>> the last 19-ish years. If GCC is now making use of the return value, >>>> then we need to fix that and undo the "optimization" in our string.h. >>>> >>>> So let's just bite the bullet, make the asm memset() compliant, and >>>> clean up string.h. >>> >>> That would be the ideal thing to do, because it allows the compiler to >>> optimise around these functions, however it does mean we need to check/fix >>> *all* of the string functions at least (if we don't pass -fno-builtin then >>> any builtin function is up for optimisation, including strcpy etc). >> >> Do we already have an agreed solution for this issue anywhere, now? > > No idea. I've stated above what I think should happen. Where's the > disagreement? I'm not sure if there even is a disagreement ;) I've asked because it seems that Ivan's v2 patch [1] isn't applied anywhere, yet. This let me to the impression that this might not be the final ('agreed') fix and that something still has to be done. Will's answer to my question [2] sounds like Ivan's v2 patch [1] is the final solution, though. If this is correct, what stops us from applying [1]? Sorry if I misunderstood anything. Thanks Dirk [1] http://www.spinics.net/lists/arm-kernel/msg224019.html [2] http://www.spinics.net/lists/arm-kernel/msg228036.html
On Wed, Mar 06, 2013 at 06:38:06PM +0100, Dirk Behme wrote: > Am 06.03.2013 18:11, schrieb Russell King - ARM Linux: >> On Tue, Mar 05, 2013 at 02:50:06PM +0100, Dirk Behme wrote: >>> On 12.02.2013 17:36, Will Deacon wrote: >>>> On Tue, Feb 12, 2013 at 03:58:01PM +0000, Russell King - ARM Linux wrote: >>>>> On Tue, Feb 12, 2013 at 02:00:08PM +0000, Will Deacon wrote: >>>>>> Interesting... the GCC documentation also states that ffreestanding implies >>>>>> fno-builtin, so memset and co shouldn't be targetted for this sort of >>>>>> optimisation by GCC. Have you observed this problem even when passing this >>>>>> option? >>>>> >>>>> Rather than wondering whether we should be using -ffreestanding or not >>>>> (which, x86 people have strongly resisted) I suggest that we just fix >>>>> our memset() implementation to be compliant. >>>>> >>>>> The reason it's not compliant is that I saw no reason for it to be >>>>> compliant back in the gcc 2.7.x days, and it's persisted like that for >>>>> the last 19-ish years. If GCC is now making use of the return value, >>>>> then we need to fix that and undo the "optimization" in our string.h. >>>>> >>>>> So let's just bite the bullet, make the asm memset() compliant, and >>>>> clean up string.h. >>>> >>>> That would be the ideal thing to do, because it allows the compiler to >>>> optimise around these functions, however it does mean we need to check/fix >>>> *all* of the string functions at least (if we don't pass -fno-builtin then >>>> any builtin function is up for optimisation, including strcpy etc). >>> >>> Do we already have an agreed solution for this issue anywhere, now? >> >> No idea. I've stated above what I think should happen. Where's the >> disagreement? > > I'm not sure if there even is a disagreement ;) > > I've asked because it seems that Ivan's v2 patch [1] isn't applied > anywhere, yet. This let me to the impression that this might not be the > final ('agreed') fix and that something still has to be done. Well, I don't generally pick patches off the mailing list. It's too difficult for me to do that. Emails *very* *very* quickly get buried and forgotten, especially as I don't look at emails each and every day (this afternoon is the first time since Monday afternoon, and I'm not going to look back before Monday as there's been 500 _new_ emails which I'm still going through.) This is exactly why I have the patch system - to help capture the patches which need to be applied.
====== Perform the following substitutions: ip -> r8, then r0 -> ip, and insert 'mov ip, r0' as the first statement of the function. At this point, we have a memset() implementation returning the proper result, but corrupting r8 on some paths (the ones that were using ip). Step 2 ====== Make sure r8 is saved and restored when (! CALGN(1)+0) == 1: save r8: - str lr, [sp, #-4]! + stmfd sp!, {r8, lr} and restore r8 on both exit paths: - ldmeqfd sp!, {pc} @ Now <64 bytes to go. + ldmeqfd sp!, {r8, pc} @ Now <64 bytes to go. (...) tst r2, #16 stmneia ip!, {r1, r3, r8, lr} - ldr lr, [sp], #4 + ldmfd sp!, {r8, lr} Step 3 ====== Make sure r8 is saved and restored when (! CALGN(1)+0) == 0: save r8: - stmfd sp!, {r4-r7, lr} + stmfd sp!, {r4-r8, lr} and restore r8 on both exit paths: bgt 3b - ldmeqfd sp!, {r4-r7, pc} + ldmeqfd sp!, {r4-r8, pc} (...) tst r2, #16 stmneia ip!, {r4-r7} - ldmfd sp!, {r4-r7, lr} + ldmfd sp!, {r4-r8, lr} Step 4 ====== Rewrite register list "r4-r7, r8" as "r4-r8". Signed-off-by: Ivan Djelic <ivan.djelic@parrot.com> --- arch/arm/lib/memset.S | 85 +++++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 41 deletions(-) diff --git a/arch/arm/lib/memset.S b/arch/arm/lib/memset.S index 650d592..eafd5ee 100644 --- a/arch/arm/lib/memset.S +++ b/arch/arm/lib/memset.S @@ -19,9 +19,9 @@ 1: subs r2, r2, #4 @ 1 do we have enough blt 5f @ 1 bytes to align with? cmp r3, #2 @ 1 - strltb r1, [r0], #1 @ 1 - strleb r1, [r0], #1 @ 1 - strb r1, [r0], #1 @ 1 + strltb r1, [ip], #1 @ 1 + strleb r1, [ip], #1 @ 1 + strb r1, [ip], #1 @ 1 add r2, r2, r3 @ 1 (r2 = r2 - (4 - r3)) /* * The pointer is now aligned and the length is adjusted. Try doing the @@ -29,10 +29,14 @@ */ ENTRY(memset) - ands r3, r0, #3 @ 1 unaligned? +/* + * Preserve the contents of r0 for the return value. + */ + mov ip, r0 + ands r3, ip, #3 @ 1 unaligned? bne 1b @ 1 /* - * we know that the pointer in r0 is aligned to a word boundary. + * we know that the pointer in ip is aligned to a word boundary. */ orr r1, r1, r1, lsl #8 orr r1, r1, r1, lsl #16 @@ -43,29 +47,28 @@ ENTRY(memset) #if ! CALGN(1)+0 /* - * We need an extra register for this loop - save the return address and - * use the LR + * We need an 2 extra registers for this loop - use r8 and the LR */ - str lr, [sp, #-4]! - mov ip, r1 + stmfd sp!, {r8, lr} + mov r8, r1 mov lr, r1 2: subs r2, r2, #64 - stmgeia r0!, {r1, r3, ip, lr} @ 64 bytes at a time. - stmgeia r0!, {r1, r3, ip, lr} - stmgeia r0!, {r1, r3, ip, lr} - stmgeia r0!, {r1, r3, ip, lr} + stmgeia ip!, {r1, r3, r8, lr} @ 64 bytes at a time. + stmgeia ip!, {r1, r3, r8, lr} + stmgeia ip!, {r1, r3, r8, lr} + stmgeia ip!, {r1, r3, r8, lr} bgt 2b - ldmeqfd sp!, {pc} @ Now <64 bytes to go. + ldmeqfd sp!, {r8, pc} @ Now <64 bytes to go. /* * No need to correct the count; we're only testing bits from now on */ tst r2, #32 - stmneia r0!, {r1, r3, ip, lr} - stmneia r0!, {r1, r3, ip, lr} + stmneia ip!, {r1, r3, r8, lr} + stmneia ip!, {r1, r3, r8, lr} tst r2, #16 - stmneia r0!, {r1, r3, ip, lr} - ldr lr, [sp], #4 + stmneia ip!, {r1, r3, r8, lr} + ldmfd sp!, {r8, lr} #else @@ -74,54 +77,54 @@ ENTRY(memset) * whole cache lines at once. */ - stmfd sp!, {r4-r7, lr} + stmfd sp!, {r4-r8, lr} mov r4, r1 mov r5, r1 mov r6, r1 mov r7, r1 - mov ip, r1 + mov r8, r1 mov lr, r1 cmp r2, #96 - tstgt r0, #31 + tstgt ip, #31 ble 3f - and ip, r0, #31 - rsb ip, ip, #32 - sub r2, r2, ip - movs ip, ip, lsl #(32 - 4) - stmcsia r0!, {r4, r5, r6, r7} - stmmiia r0!, {r4, r5} - tst ip, #(1 << 30) - mov ip, r1 - strne r1, [r0], #4 + and r8, ip, #31 + rsb r8, r8, #32 + sub r2, r2, r8 + movs r8, r8, lsl #(32 - 4) + stmcsia ip!, {r4, r5, r6, r7} + stmmiia ip!, {r4, r5} + tst r8, #(1 << 30) + mov r8, r1 + strne r1, [ip], #4 3: subs r2, r2, #64 - stmgeia r0!, {r1, r3-r7, ip, lr} - stmgeia r0!, {r1, r3-r7, ip, lr} + stmgeia ip!, {r1, r3-r8, lr} + stmgeia ip!, {r1, r3-r8, lr} bgt 3b - ldmeqfd sp!, {r4-r7, pc} + ldmeqfd sp!, {r4-r8, pc} tst r2, #32 - stmneia r0!, {r1, r3-r7, ip, lr} + stmneia ip!, {r1, r3-r8, lr} tst r2, #16 - stmneia r0!, {r4-r7} - ldmfd sp!, {r4-r7, lr} + stmneia ip!, {r4-r7} + ldmfd sp!, {r4-r8, lr} #endif 4: tst r2, #8 - stmneia r0!, {r1, r3} + stmneia ip!, {r1, r3} tst r2, #4 - strne r1, [r0], #4 + strne r1, [ip], #4 /* * When we get here, we've got less than 4 bytes to zero. We * may have an unaligned pointer as well. */ 5: tst r2, #2 - strneb r1, [r0], #1 - strneb r1, [r0], #1 + strneb r1, [ip], #1 + strneb r1, [ip], #1 tst r2, #1 - strneb r1, [r0], #1 + strneb r1, [ip], #1 mov pc, lr ENDPROC(memset)
Recent GCC versions (e.g. GCC-4.7.2) perform optimizations based on assumptions about the implementation of memset and similar functions. The current ARM optimized memset code does not return the value of its first argument, as is usually expected from standard implementations. For instance in the following function: void debug_mutex_lock_common(struct mutex *lock, struct mutex_waiter *waiter) { memset(waiter, MUTEX_DEBUG_INIT, sizeof(*waiter)); waiter->magic = waiter; INIT_LIST_HEAD(&waiter->list); } compiled as: 800554d0 <debug_mutex_lock_common>: 800554d0: e92d4008 push {r3, lr} 800554d4: e1a00001 mov r0, r1 800554d8: e3a02010 mov r2, #16 ; 0x10 800554dc: e3a01011 mov r1, #17 ; 0x11 800554e0: eb04426e bl 80165ea0 <memset> 800554e4: e1a03000 mov r3, r0 800554e8: e583000c str r0, [r3, #12] 800554ec: e5830000 str r0, [r3] 800554f0: e5830004 str r0, [r3, #4] 800554f4: e8bd8008 pop {r3, pc} GCC assumes memset returns the value of pointer 'waiter' in register r0; causing register/memory corruptions. This patch fixes the return value of the assembly version of memset. Could you please review, or suggest better alternatives ? Thanks, -- Ivan (this is a shorter and (hopefully) clearer repost of http://lists.infradead.org/pipermail/linux-arm-kernel/2013-January/144916.html) The patch adds a 'mov' instruction and merges an additional load+store into existing load/store instructions. For ease of review, here is a breakdown of the patch into 4 simple steps: Step 1