| Message ID | 1415272377-5334-1-git-send-email-steve.capper@linaro.org (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
Hi Steve, On 6 November 2014 12:12, Steve Capper <steve.capper@linaro.org> wrote: > The generic this_cpu operations disable interrupts to ensure that the > requested operation is protected from pre-emption. For arm64, this is > overkill and can hurt throughput and latency. > > This patch provides arm64 specific implementations for the this_cpu > operations. Rather than disable interrupts, we use the exclusive > monitor or atomic operations as appropriate. > > The following operations are implemented: add, add_return, and, or, > read, write, xchg. We also wire up a cmpxchg implementation from > cmpxchg.h. > > Testing was performed using the percpu_test module and hackbench on a > Juno board running 3.18-rc3. > Got any numbers? > Signed-off-by: Steve Capper <steve.capper@linaro.org> > --- > > This patch applies on top of one I sent out earlier: > "arm64: xchg: Implement cmpxchg_double" > http://lists.infradead.org/pipermail/linux-arm-kernel/2014-October/294705.html > > My two least favourite things in the kernel are excessive pre-processor > use and assembler, so this patch has been fun to write ;-). > > I've tried to make the inline assembler clearer with named parameters, > it really upset checkpatch, if it upsets people then please shout and I > will change it. > > Cheers, > -- > Steve > > --- > arch/arm64/include/asm/cmpxchg.h | 6 +- > arch/arm64/include/asm/percpu.h | 231 +++++++++++++++++++++++++++++++++++++++ > 2 files changed, 235 insertions(+), 2 deletions(-) > > diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h > index 3e02245..3e51f49 100644 > --- a/arch/arm64/include/asm/cmpxchg.h > +++ b/arch/arm64/include/asm/cmpxchg.h > @@ -237,8 +237,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, > __ret; \ > }) > > -#define this_cpu_cmpxchg_8(ptr, o, n) \ > - cmpxchg(raw_cpu_ptr(&(ptr)), o, n); > +#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) > +#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) > +#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) > +#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) > > #define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \ > cmpxchg_double(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \ > diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h > index 5279e57..e751681 100644 > --- a/arch/arm64/include/asm/percpu.h > +++ b/arch/arm64/include/asm/percpu.h > @@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void) > > #endif /* CONFIG_SMP */ > > +#define PERCPU_OP(op, asm_op) \ > +static inline unsigned long __percpu_##op(void *ptr, \ > + unsigned long val, int size) \ > +{ \ > + unsigned long loop, ret; \ > + \ > + switch (size) { \ > + case 1: \ > + do { \ > + asm ("//__per_cpu_" #op "_1\n" \ > + "ldxrb %w[ret], %[ptr]\n" \ > + #asm_op " %w[ret], %w[ret], %w[val]\n" \ > + "stxrb %w[loop], %w[ret], %[ptr]\n" \ > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ > + [ptr] "+Q"(*(u8 *)ptr) \ > + : [val] "Ir" (val)); \ > + } while (loop); \ > + break; \ > + case 2: \ > + do { \ > + asm ("//__per_cpu_" #op "_2\n" \ > + "ldxrh %w[ret], %[ptr]\n" \ > + #asm_op " %w[ret], %w[ret], %w[val]\n" \ > + "stxrh %w[loop], %w[ret], %[ptr]\n" \ > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ > + [ptr] "+Q"(*(u16 *)ptr) \ > + : [val] "Ir" (val)); \ > + } while (loop); \ > + break; \ > + case 4: \ > + do { \ > + asm ("//__per_cpu_" #op "_4\n" \ > + "ldxr %w[ret], %[ptr]\n" \ > + #asm_op " %w[ret], %w[ret], %w[val]\n" \ > + "stxr %w[loop], %w[ret], %[ptr]\n" \ > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ > + [ptr] "+Q"(*(u32 *)ptr) \ > + : [val] "Ir" (val)); \ > + } while (loop); \ > + break; \ > + case 8: \ > + do { \ > + asm ("//__per_cpu_" #op "_8\n" \ > + "ldxr %[ret], %[ptr]\n" \ > + #asm_op " %[ret], %[ret], %[val]\n" \ > + "stxr %w[loop], %[ret], %[ptr]\n" \ > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ > + [ptr] "+Q"(*(u64 *)ptr) \ > + : [val] "Ir" (val)); \ > + } while (loop); \ > + break; \ > + default: \ > + BUILD_BUG(); \ > + } \ > + \ > + return ret; \ > +} > + > +PERCPU_OP(add, add) > +PERCPU_OP(and, and) > +PERCPU_OP(or, orr) > +#undef PERCPU_OP > + > +static inline unsigned long __percpu_read(void *ptr, int size) > +{ > + unsigned long ret; > + > + switch (size) { > + case 1: > + asm ("//__per_cpu_read_1\n" > + "ldrb %w[ret], %[ptr]\n" : > + [ret] "=&r"(ret), [ptr] "+Q"(*(u8 *)ptr)); > + break; > + case 2: > + asm ("//__per_cpu_read_2\n" > + "ldrh %w[ret], %[ptr]\n" : > + [ret] "=&r"(ret), [ptr] "+Q"(*(u16 *)ptr)); > + break; > + case 4: > + asm ("//__per_cpu_read_4\n" > + "ldr %w[ret], %[ptr]\n" : > + [ret] "=&r"(ret), [ptr] "+Q"(*(u32 *)ptr)); > + break; > + case 8: > + asm ("//__per_cpu_read_8\n" > + "ldr %[ret], %[ptr]\n" : > + [ret] "=&r"(ret), [ptr] "+Q"(*(u64 *)ptr)); > + break; > + default: > + BUILD_BUG(); > + } > + > + return ret; > +} > + Why are the 'ptr' references '+Q' outputs here rather than 'Q' inputs? > +static inline void __percpu_write(void *ptr, unsigned long val, int size) > +{ > + switch (size) { > + case 1: > + asm ("//__per_cpu_write_1\n" > + "strb %w[val], %[ptr]\n" : > + [ptr] "+Q"(*(u8 *)ptr) : [val] "r"(val)); > + break; > + case 2: > + asm ("//__per_cpu_write_2\n" > + "strh %w[val], %[ptr]\n" : > + [ptr] "+Q"(*(u16 *)ptr) : [val] "r"(val)); > + break; > + case 4: > + asm ("//__per_cpu_write_4\n" > + "str %w[val], %[ptr]\n" : > + [ptr] "+Q"(*(u32 *)ptr) : [val] "r"(val)); > + break; > + case 8: > + asm ("//__per_cpu_write_8\n" > + "str %[val], %[ptr]\n" : > + [ptr] "+Q"(*(u64 *)ptr) : [val] "r"(val)); > + break; > + default: > + BUILD_BUG(); > + } > +} > + ... and similarly, why are these '+Q' and not just '=Q' ? > +static inline unsigned long __percpu_xchg(void *ptr, unsigned long val, > + int size) > +{ > + unsigned long ret, loop; > + > + switch (size) { > + case 1: > + do { > + asm ("//__percpu_xchg_1\n" > + "ldxrb %w[ret], %[ptr]\n" > + "stxrb %w[loop], %w[val], %[ptr]\n" > + : [loop] "=&r"(loop), [ret] "=&r"(ret), > + [ptr] "+Q"(*(u8 *)ptr) > + : [val] "r" (val)); > + } while (loop); > + break; > + case 2: > + do { > + asm ("//__percpu_xchg_2\n" > + "ldxrh %w[ret], %[ptr]\n" > + "stxrh %w[loop], %w[val], %[ptr]\n" > + : [loop] "=&r"(loop), [ret] "=&r"(ret), > + [ptr] "+Q"(*(u16 *)ptr) > + : [val] "r" (val)); > + } while (loop); > + break; > + case 4: > + do { > + asm ("//__percpu_xchg_4\n" > + "ldxr %w[ret], %[ptr]\n" > + "stxr %w[loop], %w[val], %[ptr]\n" > + : [loop] "=&r"(loop), [ret] "=&r"(ret), > + [ptr] "+Q"(*(u32 *)ptr) > + : [val] "r" (val)); > + } while (loop); > + break; > + case 8: > + do { > + asm ("//__percpu_xchg_8\n" > + "ldxr %[ret], %[ptr]\n" > + "stxr %w[loop], %[val], %[ptr]\n" > + : [loop] "=&r"(loop), [ret] "=&r"(ret), > + [ptr] "+Q"(*(u64 *)ptr) > + : [val] "r" (val)); > + } while (loop); > + break; > + default: > + BUILD_BUG(); > + } > + > + return ret; > +} > + > +#define _percpu_add(pcp, val) \ > + __percpu_add(raw_cpu_ptr(&(pcp)), val, sizeof(pcp)) > + > +#define _percpu_add_return(pcp, val) (typeof(pcp)) (_percpu_add(pcp, val)) > + > +#define _percpu_and(pcp, val) \ > + __percpu_and(raw_cpu_ptr(&(pcp)), val, sizeof(pcp)) > + > +#define _percpu_or(pcp, val) \ > + __percpu_or(raw_cpu_ptr(&(pcp)), val, sizeof(pcp)) > + > +#define _percpu_read(pcp) (typeof(pcp)) \ > + (__percpu_read(raw_cpu_ptr(&(pcp)), sizeof(pcp))) > + > +#define _percpu_write(pcp, val) \ > + __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp)) > + > +#define _percpu_xchg(pcp, val) (typeof(pcp)) \ > + (__percpu_xchg(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp))) > + > +#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val) > +#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val) > +#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val) > +#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val) > + > +#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val) > +#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val) > +#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val) > +#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val) > + > +#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val) > +#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val) > +#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val) > +#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val) > + > +#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val) > +#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val) > +#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val) > +#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val) > + > +#define this_cpu_read_1(pcp) _percpu_read(pcp) > +#define this_cpu_read_2(pcp) _percpu_read(pcp) > +#define this_cpu_read_4(pcp) _percpu_read(pcp) > +#define this_cpu_read_8(pcp) _percpu_read(pcp) > + > +#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val) > +#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val) > +#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val) > +#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val) > + > +#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val) > +#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val) > +#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val) > +#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val) > + > #include <asm-generic/percpu.h> > > #endif /* __ASM_PERCPU_H */ > -- > 1.9.3 > > > _______________________________________________ > linux-arm-kernel mailing list > linux-arm-kernel@lists.infradead.org > http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
On Thu, Nov 06, 2014 at 12:26:46PM +0100, Ard Biesheuvel wrote: > Hi Steve, Hey Ard, > > On 6 November 2014 12:12, Steve Capper <steve.capper@linaro.org> wrote: > > The generic this_cpu operations disable interrupts to ensure that the > > requested operation is protected from pre-emption. For arm64, this is > > overkill and can hurt throughput and latency. > > > > This patch provides arm64 specific implementations for the this_cpu > > operations. Rather than disable interrupts, we use the exclusive > > monitor or atomic operations as appropriate. > > > > The following operations are implemented: add, add_return, and, or, > > read, write, xchg. We also wire up a cmpxchg implementation from > > cmpxchg.h. > > > > Testing was performed using the percpu_test module and hackbench on a > > Juno board running 3.18-rc3. > > > > Got any numbers? For `./hackbench 100 process 1000' on a Juno system running all cores I get: w/o patch | with patch ---------------+------------ mean 47.05 | 44.93 stddev 0.38 | 0.68 ---------------+------------ (times in seconds, rounded to 2d.p., six runs performed) So a just under 5% speed boost. [...] > > diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h > > index 5279e57..e751681 100644 > > --- a/arch/arm64/include/asm/percpu.h > > +++ b/arch/arm64/include/asm/percpu.h > > @@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void) [...] > > +static inline unsigned long __percpu_read(void *ptr, int size) > > +{ > > + unsigned long ret; > > + > > + switch (size) { > > + case 1: > > + asm ("//__per_cpu_read_1\n" > > + "ldrb %w[ret], %[ptr]\n" : > > + [ret] "=&r"(ret), [ptr] "+Q"(*(u8 *)ptr)); > > + break; > > + case 2: > > + asm ("//__per_cpu_read_2\n" > > + "ldrh %w[ret], %[ptr]\n" : > > + [ret] "=&r"(ret), [ptr] "+Q"(*(u16 *)ptr)); > > + break; > > + case 4: > > + asm ("//__per_cpu_read_4\n" > > + "ldr %w[ret], %[ptr]\n" : > > + [ret] "=&r"(ret), [ptr] "+Q"(*(u32 *)ptr)); > > + break; > > + case 8: > > + asm ("//__per_cpu_read_8\n" > > + "ldr %[ret], %[ptr]\n" : > > + [ret] "=&r"(ret), [ptr] "+Q"(*(u64 *)ptr)); > > + break; > > + default: > > + BUILD_BUG(); > > + } > > + > > + return ret; > > +} > > + > > Why are the 'ptr' references '+Q' outputs here rather than 'Q' inputs? > Because I fudged the constraint :-). Thanks, you're quite right it should be 'Q' constraint as we're only reading from that address. > > +static inline void __percpu_write(void *ptr, unsigned long val, int size) > > +{ > > + switch (size) { > > + case 1: > > + asm ("//__per_cpu_write_1\n" > > + "strb %w[val], %[ptr]\n" : > > + [ptr] "+Q"(*(u8 *)ptr) : [val] "r"(val)); > > + break; > > + case 2: > > + asm ("//__per_cpu_write_2\n" > > + "strh %w[val], %[ptr]\n" : > > + [ptr] "+Q"(*(u16 *)ptr) : [val] "r"(val)); > > + break; > > + case 4: > > + asm ("//__per_cpu_write_4\n" > > + "str %w[val], %[ptr]\n" : > > + [ptr] "+Q"(*(u32 *)ptr) : [val] "r"(val)); > > + break; > > + case 8: > > + asm ("//__per_cpu_write_8\n" > > + "str %[val], %[ptr]\n" : > > + [ptr] "+Q"(*(u64 *)ptr) : [val] "r"(val)); > > + break; > > + default: > > + BUILD_BUG(); > > + } > > +} > > + > > ... and similarly, why are these '+Q' and not just '=Q' ? Another mistake on my part, it should be "=Q" as we're only writing to that address. Thanks for going through this carefully Ard, I'll send out a corrected V2 once I've sanity checked it. Cheers,
Hi Steve, Thanks for looking at this! On Thu, Nov 06, 2014 at 11:12:57AM +0000, Steve Capper wrote: > The generic this_cpu operations disable interrupts to ensure that the > requested operation is protected from pre-emption. For arm64, this is > overkill and can hurt throughput and latency. > > This patch provides arm64 specific implementations for the this_cpu > operations. Rather than disable interrupts, we use the exclusive > monitor or atomic operations as appropriate. > > The following operations are implemented: add, add_return, and, or, > read, write, xchg. We also wire up a cmpxchg implementation from > cmpxchg.h. > > Testing was performed using the percpu_test module and hackbench on a > Juno board running 3.18-rc3. [...] > diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h > index 3e02245..3e51f49 100644 > --- a/arch/arm64/include/asm/cmpxchg.h > +++ b/arch/arm64/include/asm/cmpxchg.h > @@ -237,8 +237,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, > __ret; \ > }) > > -#define this_cpu_cmpxchg_8(ptr, o, n) \ > - cmpxchg(raw_cpu_ptr(&(ptr)), o, n); > +#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) > +#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) > +#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) > +#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) You can use cmpxchg_local here, as we don't require barrier semantics. > #define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \ > cmpxchg_double(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \ > diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h > index 5279e57..e751681 100644 > --- a/arch/arm64/include/asm/percpu.h > +++ b/arch/arm64/include/asm/percpu.h > @@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void) > > #endif /* CONFIG_SMP */ > > +#define PERCPU_OP(op, asm_op) \ > +static inline unsigned long __percpu_##op(void *ptr, \ > + unsigned long val, int size) \ > +{ \ > + unsigned long loop, ret; \ > + \ > + switch (size) { \ > + case 1: \ > + do { \ > + asm ("//__per_cpu_" #op "_1\n" \ > + "ldxrb %w[ret], %[ptr]\n" \ > + #asm_op " %w[ret], %w[ret], %w[val]\n" \ > + "stxrb %w[loop], %w[ret], %[ptr]\n" \ > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ > + [ptr] "+Q"(*(u8 *)ptr) \ > + : [val] "Ir" (val)); \ > + } while (loop); \ > + break; \ Curious, but do you see any difference in code generation over an explicit cbnz, like we use in the ATOMIC_OP macro? > + case 2: \ > + do { \ > + asm ("//__per_cpu_" #op "_2\n" \ > + "ldxrh %w[ret], %[ptr]\n" \ > + #asm_op " %w[ret], %w[ret], %w[val]\n" \ > + "stxrh %w[loop], %w[ret], %[ptr]\n" \ > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ > + [ptr] "+Q"(*(u16 *)ptr) \ > + : [val] "Ir" (val)); \ > + } while (loop); \ > + break; \ > + case 4: \ > + do { \ > + asm ("//__per_cpu_" #op "_4\n" \ > + "ldxr %w[ret], %[ptr]\n" \ > + #asm_op " %w[ret], %w[ret], %w[val]\n" \ > + "stxr %w[loop], %w[ret], %[ptr]\n" \ > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ > + [ptr] "+Q"(*(u32 *)ptr) \ > + : [val] "Ir" (val)); \ > + } while (loop); \ > + break; \ > + case 8: \ > + do { \ > + asm ("//__per_cpu_" #op "_8\n" \ > + "ldxr %[ret], %[ptr]\n" \ > + #asm_op " %[ret], %[ret], %[val]\n" \ > + "stxr %w[loop], %[ret], %[ptr]\n" \ > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ > + [ptr] "+Q"(*(u64 *)ptr) \ > + : [val] "Ir" (val)); \ > + } while (loop); \ > + break; \ > + default: \ > + BUILD_BUG(); \ > + } \ > + \ > + return ret; \ > +} > + > +PERCPU_OP(add, add) > +PERCPU_OP(and, and) > +PERCPU_OP(or, orr) Can you use these to generate local_t versions too? Will
On Thu, Nov 06, 2014 at 12:27:53PM +0000, Will Deacon wrote: > Hi Steve, > > Thanks for looking at this! Hey Will, No problem, it's quite beneficial for performance. > > On Thu, Nov 06, 2014 at 11:12:57AM +0000, Steve Capper wrote: > > The generic this_cpu operations disable interrupts to ensure that the > > requested operation is protected from pre-emption. For arm64, this is > > overkill and can hurt throughput and latency. > > > > This patch provides arm64 specific implementations for the this_cpu > > operations. Rather than disable interrupts, we use the exclusive > > monitor or atomic operations as appropriate. > > > > The following operations are implemented: add, add_return, and, or, > > read, write, xchg. We also wire up a cmpxchg implementation from > > cmpxchg.h. > > > > Testing was performed using the percpu_test module and hackbench on a > > Juno board running 3.18-rc3. > > [...] > > > diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h > > index 3e02245..3e51f49 100644 > > --- a/arch/arm64/include/asm/cmpxchg.h > > +++ b/arch/arm64/include/asm/cmpxchg.h > > @@ -237,8 +237,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, > > __ret; \ > > }) > > > > -#define this_cpu_cmpxchg_8(ptr, o, n) \ > > - cmpxchg(raw_cpu_ptr(&(ptr)), o, n); > > +#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) > > +#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) > > +#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) > > +#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) > > You can use cmpxchg_local here, as we don't require barrier semantics. Agreed, thanks, I will update that. > > > #define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \ > > cmpxchg_double(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \ > > diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h > > index 5279e57..e751681 100644 > > --- a/arch/arm64/include/asm/percpu.h > > +++ b/arch/arm64/include/asm/percpu.h > > @@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void) > > > > #endif /* CONFIG_SMP */ > > > > +#define PERCPU_OP(op, asm_op) \ > > +static inline unsigned long __percpu_##op(void *ptr, \ > > + unsigned long val, int size) \ > > +{ \ > > + unsigned long loop, ret; \ > > + \ > > + switch (size) { \ > > + case 1: \ > > + do { \ > > + asm ("//__per_cpu_" #op "_1\n" \ > > + "ldxrb %w[ret], %[ptr]\n" \ > > + #asm_op " %w[ret], %w[ret], %w[val]\n" \ > > + "stxrb %w[loop], %w[ret], %[ptr]\n" \ > > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ > > + [ptr] "+Q"(*(u8 *)ptr) \ > > + : [val] "Ir" (val)); \ > > + } while (loop); \ > > + break; \ > > Curious, but do you see any difference in code generation over an explicit > cbnz, like we use in the ATOMIC_OP macro? I've not noticed any substancial difference in the code paths I've inspected. Theoretically, a compiler that is extremely averse to branches could do some unrolling with this. I decided to give the compiler as much control as possible, so elected to put the minimum amount of assembler in. > > > + case 2: \ > > + do { \ > > + asm ("//__per_cpu_" #op "_2\n" \ > > + "ldxrh %w[ret], %[ptr]\n" \ > > + #asm_op " %w[ret], %w[ret], %w[val]\n" \ > > + "stxrh %w[loop], %w[ret], %[ptr]\n" \ > > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ > > + [ptr] "+Q"(*(u16 *)ptr) \ > > + : [val] "Ir" (val)); \ > > + } while (loop); \ > > + break; \ > > + case 4: \ > > + do { \ > > + asm ("//__per_cpu_" #op "_4\n" \ > > + "ldxr %w[ret], %[ptr]\n" \ > > + #asm_op " %w[ret], %w[ret], %w[val]\n" \ > > + "stxr %w[loop], %w[ret], %[ptr]\n" \ > > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ > > + [ptr] "+Q"(*(u32 *)ptr) \ > > + : [val] "Ir" (val)); \ > > + } while (loop); \ > > + break; \ > > + case 8: \ > > + do { \ > > + asm ("//__per_cpu_" #op "_8\n" \ > > + "ldxr %[ret], %[ptr]\n" \ > > + #asm_op " %[ret], %[ret], %[val]\n" \ > > + "stxr %w[loop], %[ret], %[ptr]\n" \ > > + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ > > + [ptr] "+Q"(*(u64 *)ptr) \ > > + : [val] "Ir" (val)); \ > > + } while (loop); \ > > + break; \ > > + default: \ > > + BUILD_BUG(); \ > > + } \ > > + \ > > + return ret; \ > > +} > > + > > +PERCPU_OP(add, add) > > +PERCPU_OP(and, and) > > +PERCPU_OP(or, orr) > > Can you use these to generate local_t versions too? Sure, I forgot about them, I will add them to V3. Cheers,
diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h index 3e02245..3e51f49 100644 --- a/arch/arm64/include/asm/cmpxchg.h +++ b/arch/arm64/include/asm/cmpxchg.h @@ -237,8 +237,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, __ret; \ }) -#define this_cpu_cmpxchg_8(ptr, o, n) \ - cmpxchg(raw_cpu_ptr(&(ptr)), o, n); +#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) +#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) +#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) +#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg(raw_cpu_ptr(&(ptr)), o, n) #define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \ cmpxchg_double(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \ diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h index 5279e57..e751681 100644 --- a/arch/arm64/include/asm/percpu.h +++ b/arch/arm64/include/asm/percpu.h @@ -44,6 +44,237 @@ static inline unsigned long __my_cpu_offset(void) #endif /* CONFIG_SMP */ +#define PERCPU_OP(op, asm_op) \ +static inline unsigned long __percpu_##op(void *ptr, \ + unsigned long val, int size) \ +{ \ + unsigned long loop, ret; \ + \ + switch (size) { \ + case 1: \ + do { \ + asm ("//__per_cpu_" #op "_1\n" \ + "ldxrb %w[ret], %[ptr]\n" \ + #asm_op " %w[ret], %w[ret], %w[val]\n" \ + "stxrb %w[loop], %w[ret], %[ptr]\n" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u8 *)ptr) \ + : [val] "Ir" (val)); \ + } while (loop); \ + break; \ + case 2: \ + do { \ + asm ("//__per_cpu_" #op "_2\n" \ + "ldxrh %w[ret], %[ptr]\n" \ + #asm_op " %w[ret], %w[ret], %w[val]\n" \ + "stxrh %w[loop], %w[ret], %[ptr]\n" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u16 *)ptr) \ + : [val] "Ir" (val)); \ + } while (loop); \ + break; \ + case 4: \ + do { \ + asm ("//__per_cpu_" #op "_4\n" \ + "ldxr %w[ret], %[ptr]\n" \ + #asm_op " %w[ret], %w[ret], %w[val]\n" \ + "stxr %w[loop], %w[ret], %[ptr]\n" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u32 *)ptr) \ + : [val] "Ir" (val)); \ + } while (loop); \ + break; \ + case 8: \ + do { \ + asm ("//__per_cpu_" #op "_8\n" \ + "ldxr %[ret], %[ptr]\n" \ + #asm_op " %[ret], %[ret], %[val]\n" \ + "stxr %w[loop], %[ret], %[ptr]\n" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u64 *)ptr) \ + : [val] "Ir" (val)); \ + } while (loop); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ + \ + return ret; \ +} + +PERCPU_OP(add, add) +PERCPU_OP(and, and) +PERCPU_OP(or, orr) +#undef PERCPU_OP + +static inline unsigned long __percpu_read(void *ptr, int size) +{ + unsigned long ret; + + switch (size) { + case 1: + asm ("//__per_cpu_read_1\n" + "ldrb %w[ret], %[ptr]\n" : + [ret] "=&r"(ret), [ptr] "+Q"(*(u8 *)ptr)); + break; + case 2: + asm ("//__per_cpu_read_2\n" + "ldrh %w[ret], %[ptr]\n" : + [ret] "=&r"(ret), [ptr] "+Q"(*(u16 *)ptr)); + break; + case 4: + asm ("//__per_cpu_read_4\n" + "ldr %w[ret], %[ptr]\n" : + [ret] "=&r"(ret), [ptr] "+Q"(*(u32 *)ptr)); + break; + case 8: + asm ("//__per_cpu_read_8\n" + "ldr %[ret], %[ptr]\n" : + [ret] "=&r"(ret), [ptr] "+Q"(*(u64 *)ptr)); + break; + default: + BUILD_BUG(); + } + + return ret; +} + +static inline void __percpu_write(void *ptr, unsigned long val, int size) +{ + switch (size) { + case 1: + asm ("//__per_cpu_write_1\n" + "strb %w[val], %[ptr]\n" : + [ptr] "+Q"(*(u8 *)ptr) : [val] "r"(val)); + break; + case 2: + asm ("//__per_cpu_write_2\n" + "strh %w[val], %[ptr]\n" : + [ptr] "+Q"(*(u16 *)ptr) : [val] "r"(val)); + break; + case 4: + asm ("//__per_cpu_write_4\n" + "str %w[val], %[ptr]\n" : + [ptr] "+Q"(*(u32 *)ptr) : [val] "r"(val)); + break; + case 8: + asm ("//__per_cpu_write_8\n" + "str %[val], %[ptr]\n" : + [ptr] "+Q"(*(u64 *)ptr) : [val] "r"(val)); + break; + default: + BUILD_BUG(); + } +} + +static inline unsigned long __percpu_xchg(void *ptr, unsigned long val, + int size) +{ + unsigned long ret, loop; + + switch (size) { + case 1: + do { + asm ("//__percpu_xchg_1\n" + "ldxrb %w[ret], %[ptr]\n" + "stxrb %w[loop], %w[val], %[ptr]\n" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u8 *)ptr) + : [val] "r" (val)); + } while (loop); + break; + case 2: + do { + asm ("//__percpu_xchg_2\n" + "ldxrh %w[ret], %[ptr]\n" + "stxrh %w[loop], %w[val], %[ptr]\n" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u16 *)ptr) + : [val] "r" (val)); + } while (loop); + break; + case 4: + do { + asm ("//__percpu_xchg_4\n" + "ldxr %w[ret], %[ptr]\n" + "stxr %w[loop], %w[val], %[ptr]\n" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u32 *)ptr) + : [val] "r" (val)); + } while (loop); + break; + case 8: + do { + asm ("//__percpu_xchg_8\n" + "ldxr %[ret], %[ptr]\n" + "stxr %w[loop], %[val], %[ptr]\n" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u64 *)ptr) + : [val] "r" (val)); + } while (loop); + break; + default: + BUILD_BUG(); + } + + return ret; +} + +#define _percpu_add(pcp, val) \ + __percpu_add(raw_cpu_ptr(&(pcp)), val, sizeof(pcp)) + +#define _percpu_add_return(pcp, val) (typeof(pcp)) (_percpu_add(pcp, val)) + +#define _percpu_and(pcp, val) \ + __percpu_and(raw_cpu_ptr(&(pcp)), val, sizeof(pcp)) + +#define _percpu_or(pcp, val) \ + __percpu_or(raw_cpu_ptr(&(pcp)), val, sizeof(pcp)) + +#define _percpu_read(pcp) (typeof(pcp)) \ + (__percpu_read(raw_cpu_ptr(&(pcp)), sizeof(pcp))) + +#define _percpu_write(pcp, val) \ + __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp)) + +#define _percpu_xchg(pcp, val) (typeof(pcp)) \ + (__percpu_xchg(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp))) + +#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val) +#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val) +#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val) +#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val) + +#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val) +#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val) +#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val) +#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val) + +#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val) +#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val) +#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val) +#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val) + +#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val) +#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val) +#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val) +#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val) + +#define this_cpu_read_1(pcp) _percpu_read(pcp) +#define this_cpu_read_2(pcp) _percpu_read(pcp) +#define this_cpu_read_4(pcp) _percpu_read(pcp) +#define this_cpu_read_8(pcp) _percpu_read(pcp) + +#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val) +#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val) +#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val) +#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val) + +#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val) +#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val) +#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val) +#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val) + #include <asm-generic/percpu.h> #endif /* __ASM_PERCPU_H */
The generic this_cpu operations disable interrupts to ensure that the requested operation is protected from pre-emption. For arm64, this is overkill and can hurt throughput and latency. This patch provides arm64 specific implementations for the this_cpu operations. Rather than disable interrupts, we use the exclusive monitor or atomic operations as appropriate. The following operations are implemented: add, add_return, and, or, read, write, xchg. We also wire up a cmpxchg implementation from cmpxchg.h. Testing was performed using the percpu_test module and hackbench on a Juno board running 3.18-rc3. Signed-off-by: Steve Capper <steve.capper@linaro.org> --- This patch applies on top of one I sent out earlier: "arm64: xchg: Implement cmpxchg_double" http://lists.infradead.org/pipermail/linux-arm-kernel/2014-October/294705.html My two least favourite things in the kernel are excessive pre-processor use and assembler, so this patch has been fun to write ;-). I've tried to make the inline assembler clearer with named parameters, it really upset checkpatch, if it upsets people then please shout and I will change it. Cheers,