locking/atomics: Clean up the atomic.h maze of #defines

Commit Message

Ingo Molnar May 5, 2018, 8:11 a.m. UTC

* Mark Rutland <[e-mail]> wrote:

> On Fri, May 04, 2018 at 08:01:05PM +0200, Peter Zijlstra wrote:
> > On Fri, May 04, 2018 at 06:39:32PM +0100, Mark Rutland wrote:
> > > Currently <asm-generic/atomic-instrumented.h> only instruments the fully
> > > ordered variants of atomic functions, ignoring the {relaxed,acquire,release}
> > > ordering variants.
> > > 
> > > This patch reworks the header to instrument all ordering variants of the atomic
> > > functions, so that architectures implementing these are instrumented
> > > appropriately.
> > > 
> > > To minimise repetition, a macro is used to generate each variant from a common
> > > template. The {full,relaxed,acquire,release} order variants respectively are
> > > then built using this template, where the architecture provides an
> > > implementation.
> 
> > >  include/asm-generic/atomic-instrumented.h | 1195 ++++++++++++++++++++++++-----
> > >  1 file changed, 1008 insertions(+), 187 deletions(-)
> > 
> > Is there really no way to either generate or further macro compress this?
> 
> I can definitely macro compress this somewhat, but the bulk of the
> repetition will be the ifdeffery, which can't be macro'd away IIUC.

The thing is, the existing #ifdeffery is suboptimal to begin with.

I just did the following cleanups (patch attached):

 include/linux/atomic.h | 1275 +++++++++++++++++++++---------------------------
 1 file changed, 543 insertions(+), 732 deletions(-)

The gist of the changes is the following simplification of the main construct:

Before:

 #ifndef atomic_fetch_dec_relaxed

 #ifndef atomic_fetch_dec
 #define atomic_fetch_dec(v)	        atomic_fetch_sub(1, (v))
 #define atomic_fetch_dec_relaxed(v)	atomic_fetch_sub_relaxed(1, (v))
 #define atomic_fetch_dec_acquire(v)	atomic_fetch_sub_acquire(1, (v))
 #define atomic_fetch_dec_release(v)	atomic_fetch_sub_release(1, (v))
 #else /* atomic_fetch_dec */
 #define atomic_fetch_dec_relaxed	atomic_fetch_dec
 #define atomic_fetch_dec_acquire	atomic_fetch_dec
 #define atomic_fetch_dec_release	atomic_fetch_dec
 #endif /* atomic_fetch_dec */

 #else /* atomic_fetch_dec_relaxed */

 #ifndef atomic_fetch_dec_acquire
 #define atomic_fetch_dec_acquire(...)					\
	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
 #endif

 #ifndef atomic_fetch_dec_release
 #define atomic_fetch_dec_release(...)					\
	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
 #endif

 #ifndef atomic_fetch_dec
 #define atomic_fetch_dec(...)						\
	__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
 #endif
 #endif /* atomic_fetch_dec_relaxed */

After:

 #ifndef atomic_fetch_dec_relaxed
 # ifndef atomic_fetch_dec
 #  define atomic_fetch_dec(v)			atomic_fetch_sub(1, (v))
 #  define atomic_fetch_dec_relaxed(v)		atomic_fetch_sub_relaxed(1, (v))
 #  define atomic_fetch_dec_acquire(v)		atomic_fetch_sub_acquire(1, (v))
 #  define atomic_fetch_dec_release(v)		atomic_fetch_sub_release(1, (v))
 # else
 #  define atomic_fetch_dec_relaxed		atomic_fetch_dec
 #  define atomic_fetch_dec_acquire		atomic_fetch_dec
 #  define atomic_fetch_dec_release		atomic_fetch_dec
 # endif
 #else
 # ifndef atomic_fetch_dec_acquire
 #  define atomic_fetch_dec_acquire(...)	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
 # endif
 # ifndef atomic_fetch_dec_release
 #  define atomic_fetch_dec_release(...)	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
 # endif
 # ifndef atomic_fetch_dec
 #  define atomic_fetch_dec(...)		__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
 # endif
 #endif

The new variant is readable at a glance, and the hierarchy of defines is very 
obvious as well.

And I think we could do even better - there's absolutely no reason why _every_ 
operation has to be made conditional on a finegrained level - they are overriden 
in API groups. In fact allowing individual override is arguably a fragility.

So we could do the following simplification on top of that:

 #ifndef atomic_fetch_dec_relaxed
 # ifndef atomic_fetch_dec
 #  define atomic_fetch_dec(v)			atomic_fetch_sub(1, (v))
 #  define atomic_fetch_dec_relaxed(v)		atomic_fetch_sub_relaxed(1, (v))
 #  define atomic_fetch_dec_acquire(v)		atomic_fetch_sub_acquire(1, (v))
 #  define atomic_fetch_dec_release(v)		atomic_fetch_sub_release(1, (v))
 # else
 #  define atomic_fetch_dec_relaxed		atomic_fetch_dec
 #  define atomic_fetch_dec_acquire		atomic_fetch_dec
 #  define atomic_fetch_dec_release		atomic_fetch_dec
 # endif
 #else
 # ifndef atomic_fetch_dec
 #  define atomic_fetch_dec(...)		__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
 #  define atomic_fetch_dec_acquire(...)	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
 #  define atomic_fetch_dec_release(...)	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
 # endif
 #endif

Note how the grouping of APIs based on 'atomic_fetch_dec' is already an assumption 
in the primary !atomic_fetch_dec_relaxed branch.

I much prefer such clear constructs of API mapping versus magic macros.

Thanks,

	Ingo

Comments

Peter Zijlstra May 5, 2018, 8:47 a.m. UTC | #1

On Sat, May 05, 2018 at 10:11:00AM +0200, Ingo Molnar wrote:

> Before:
> 
>  #ifndef atomic_fetch_dec_relaxed
> 
>  #ifndef atomic_fetch_dec
>  #define atomic_fetch_dec(v)	        atomic_fetch_sub(1, (v))
>  #define atomic_fetch_dec_relaxed(v)	atomic_fetch_sub_relaxed(1, (v))
>  #define atomic_fetch_dec_acquire(v)	atomic_fetch_sub_acquire(1, (v))
>  #define atomic_fetch_dec_release(v)	atomic_fetch_sub_release(1, (v))
>  #else /* atomic_fetch_dec */
>  #define atomic_fetch_dec_relaxed	atomic_fetch_dec
>  #define atomic_fetch_dec_acquire	atomic_fetch_dec
>  #define atomic_fetch_dec_release	atomic_fetch_dec
>  #endif /* atomic_fetch_dec */
> 
>  #else /* atomic_fetch_dec_relaxed */
> 
>  #ifndef atomic_fetch_dec_acquire
>  #define atomic_fetch_dec_acquire(...)					\
> 	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
>  #endif
> 
>  #ifndef atomic_fetch_dec_release
>  #define atomic_fetch_dec_release(...)					\
> 	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
>  #endif
> 
>  #ifndef atomic_fetch_dec
>  #define atomic_fetch_dec(...)						\
> 	__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
>  #endif
>  #endif /* atomic_fetch_dec_relaxed */
> 
> After:
> 
>  #ifndef atomic_fetch_dec_relaxed
>  # ifndef atomic_fetch_dec
>  #  define atomic_fetch_dec(v)			atomic_fetch_sub(1, (v))
>  #  define atomic_fetch_dec_relaxed(v)		atomic_fetch_sub_relaxed(1, (v))
>  #  define atomic_fetch_dec_acquire(v)		atomic_fetch_sub_acquire(1, (v))
>  #  define atomic_fetch_dec_release(v)		atomic_fetch_sub_release(1, (v))
>  # else
>  #  define atomic_fetch_dec_relaxed		atomic_fetch_dec
>  #  define atomic_fetch_dec_acquire		atomic_fetch_dec
>  #  define atomic_fetch_dec_release		atomic_fetch_dec
>  # endif
>  #else
>  # ifndef atomic_fetch_dec_acquire
>  #  define atomic_fetch_dec_acquire(...)	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
>  # endif
>  # ifndef atomic_fetch_dec_release
>  #  define atomic_fetch_dec_release(...)	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
>  # endif
>  # ifndef atomic_fetch_dec
>  #  define atomic_fetch_dec(...)		__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
>  # endif
>  #endif
> 
> The new variant is readable at a glance, and the hierarchy of defines is very 
> obvious as well.

It wraps and looks hideous in my normal setup. And I do detest that indent
after # thing.

> And I think we could do even better - there's absolutely no reason why _every_ 
> operation has to be made conditional on a finegrained level - they are overriden 
> in API groups. In fact allowing individual override is arguably a fragility.
> 
> So we could do the following simplification on top of that:
> 
>  #ifndef atomic_fetch_dec_relaxed
>  # ifndef atomic_fetch_dec
>  #  define atomic_fetch_dec(v)			atomic_fetch_sub(1, (v))
>  #  define atomic_fetch_dec_relaxed(v)		atomic_fetch_sub_relaxed(1, (v))
>  #  define atomic_fetch_dec_acquire(v)		atomic_fetch_sub_acquire(1, (v))
>  #  define atomic_fetch_dec_release(v)		atomic_fetch_sub_release(1, (v))
>  # else
>  #  define atomic_fetch_dec_relaxed		atomic_fetch_dec
>  #  define atomic_fetch_dec_acquire		atomic_fetch_dec
>  #  define atomic_fetch_dec_release		atomic_fetch_dec
>  # endif
>  #else
>  # ifndef atomic_fetch_dec
>  #  define atomic_fetch_dec(...)		__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
>  #  define atomic_fetch_dec_acquire(...)	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
>  #  define atomic_fetch_dec_release(...)	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
>  # endif
>  #endif

This would disallow an architecture to override just fetch_dec_release for
instance.

I don't think there currently is any architecture that does that, but the
intent was to allow it to override anything and only provide defaults where it
does not.

None of this takes away the giant trainwreck that is the annotated atomic stuff
though.

And I seriously hate this one:

  ba1c9f83f633 ("locking/atomic/x86: Un-macro-ify atomic ops implementation")

and will likely undo that the moment I need to change anything there.

So no, don't like.

Ingo Molnar May 5, 2018, 9:04 a.m. UTC | #2

* Peter Zijlstra <[e-mail]> wrote:

> > So we could do the following simplification on top of that:
> > 
> >  #ifndef atomic_fetch_dec_relaxed
> >  # ifndef atomic_fetch_dec
> >  #  define atomic_fetch_dec(v)		atomic_fetch_sub(1, (v))
> >  #  define atomic_fetch_dec_relaxed(v)	atomic_fetch_sub_relaxed(1, (v))
> >  #  define atomic_fetch_dec_acquire(v)	atomic_fetch_sub_acquire(1, (v))
> >  #  define atomic_fetch_dec_release(v)	atomic_fetch_sub_release(1, (v))
> >  # else
> >  #  define atomic_fetch_dec_relaxed		atomic_fetch_dec
> >  #  define atomic_fetch_dec_acquire		atomic_fetch_dec
> >  #  define atomic_fetch_dec_release		atomic_fetch_dec
> >  # endif
> >  #else
> >  # ifndef atomic_fetch_dec
> >  #  define atomic_fetch_dec(...)		__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
> >  #  define atomic_fetch_dec_acquire(...)	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
> >  #  define atomic_fetch_dec_release(...)	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
> >  # endif
> >  #endif
> 
> This would disallow an architecture to override just fetch_dec_release for
> instance.

Couldn't such a crazy arch just define _all_ the 3 APIs in this group?
That's really a small price and makes the place pay the complexity
price that does the weirdness...

> I don't think there currently is any architecture that does that, but the
> intent was to allow it to override anything and only provide defaults where it
> does not.

I'd argue that if a new arch only defines one of these APIs that's probably a bug. 
If they absolutely want to do it, they still can - by defining all 3 APIs.

So there's no loss in arch flexibility.

> None of this takes away the giant trainwreck that is the annotated atomic stuff
> though.
> 
> And I seriously hate this one:
> 
>   ba1c9f83f633 ("locking/atomic/x86: Un-macro-ify atomic ops implementation")
> 
> and will likely undo that the moment I need to change anything there.

If it makes the code more readable then I don't object - the problem was that the 
instrumentation indirection made all that code much harder to follow.

Thanks,

	Ingo

Dmitry Vyukov May 5, 2018, 9:05 a.m. UTC | #3

On Sat, May 5, 2018 at 10:47 AM, Peter Zijlstra <[e-mail]> wrote:
> On Sat, May 05, 2018 at 10:11:00AM +0200, Ingo Molnar wrote:
>
>> Before:
>>
>>  #ifndef atomic_fetch_dec_relaxed
>>
>>  #ifndef atomic_fetch_dec
>>  #define atomic_fetch_dec(v)          atomic_fetch_sub(1, (v))
>>  #define atomic_fetch_dec_relaxed(v)  atomic_fetch_sub_relaxed(1, (v))
>>  #define atomic_fetch_dec_acquire(v)  atomic_fetch_sub_acquire(1, (v))
>>  #define atomic_fetch_dec_release(v)  atomic_fetch_sub_release(1, (v))
>>  #else /* atomic_fetch_dec */
>>  #define atomic_fetch_dec_relaxed     atomic_fetch_dec
>>  #define atomic_fetch_dec_acquire     atomic_fetch_dec
>>  #define atomic_fetch_dec_release     atomic_fetch_dec
>>  #endif /* atomic_fetch_dec */
>>
>>  #else /* atomic_fetch_dec_relaxed */
>>
>>  #ifndef atomic_fetch_dec_acquire
>>  #define atomic_fetch_dec_acquire(...)                                        \
>>       __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
>>  #endif
>>
>>  #ifndef atomic_fetch_dec_release
>>  #define atomic_fetch_dec_release(...)                                        \
>>       __atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
>>  #endif
>>
>>  #ifndef atomic_fetch_dec
>>  #define atomic_fetch_dec(...)                                                \
>>       __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
>>  #endif
>>  #endif /* atomic_fetch_dec_relaxed */
>>
>> After:
>>
>>  #ifndef atomic_fetch_dec_relaxed
>>  # ifndef atomic_fetch_dec
>>  #  define atomic_fetch_dec(v)                        atomic_fetch_sub(1, (v))
>>  #  define atomic_fetch_dec_relaxed(v)                atomic_fetch_sub_relaxed(1, (v))
>>  #  define atomic_fetch_dec_acquire(v)                atomic_fetch_sub_acquire(1, (v))
>>  #  define atomic_fetch_dec_release(v)                atomic_fetch_sub_release(1, (v))
>>  # else
>>  #  define atomic_fetch_dec_relaxed           atomic_fetch_dec
>>  #  define atomic_fetch_dec_acquire           atomic_fetch_dec
>>  #  define atomic_fetch_dec_release           atomic_fetch_dec
>>  # endif
>>  #else
>>  # ifndef atomic_fetch_dec_acquire
>>  #  define atomic_fetch_dec_acquire(...)      __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
>>  # endif
>>  # ifndef atomic_fetch_dec_release
>>  #  define atomic_fetch_dec_release(...)      __atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
>>  # endif
>>  # ifndef atomic_fetch_dec
>>  #  define atomic_fetch_dec(...)              __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
>>  # endif
>>  #endif
>>
>> The new variant is readable at a glance, and the hierarchy of defines is very
>> obvious as well.
>
> It wraps and looks hideous in my normal setup. And I do detest that indent
> after # thing.
>
>> And I think we could do even better - there's absolutely no reason why _every_
>> operation has to be made conditional on a finegrained level - they are overriden
>> in API groups. In fact allowing individual override is arguably a fragility.
>>
>> So we could do the following simplification on top of that:
>>
>>  #ifndef atomic_fetch_dec_relaxed
>>  # ifndef atomic_fetch_dec
>>  #  define atomic_fetch_dec(v)                        atomic_fetch_sub(1, (v))
>>  #  define atomic_fetch_dec_relaxed(v)                atomic_fetch_sub_relaxed(1, (v))
>>  #  define atomic_fetch_dec_acquire(v)                atomic_fetch_sub_acquire(1, (v))
>>  #  define atomic_fetch_dec_release(v)                atomic_fetch_sub_release(1, (v))
>>  # else
>>  #  define atomic_fetch_dec_relaxed           atomic_fetch_dec
>>  #  define atomic_fetch_dec_acquire           atomic_fetch_dec
>>  #  define atomic_fetch_dec_release           atomic_fetch_dec
>>  # endif
>>  #else
>>  # ifndef atomic_fetch_dec
>>  #  define atomic_fetch_dec(...)              __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
>>  #  define atomic_fetch_dec_acquire(...)      __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
>>  #  define atomic_fetch_dec_release(...)      __atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
>>  # endif
>>  #endif
>
> This would disallow an architecture to override just fetch_dec_release for
> instance.
>
> I don't think there currently is any architecture that does that, but the
> intent was to allow it to override anything and only provide defaults where it
> does not.
>
> None of this takes away the giant trainwreck that is the annotated atomic stuff
> though.
>
> And I seriously hate this one:
>
>   ba1c9f83f633 ("locking/atomic/x86: Un-macro-ify atomic ops implementation")
>
> and will likely undo that the moment I need to change anything there.
>
> So no, don't like.

That was asked by Ingo:
https://groups.google.com/d/msg/kasan-dev/3sNHjjb4GCI/Xz1uVWaaAAAJ

I think in the end all of current options suck in one way or another,
so we are just going in circles.
We either need something different (e.g. codegen), or settle on one
option for doing it.

Ingo Molnar May 5, 2018, 9:09 a.m. UTC | #4

* Peter Zijlstra <[e-mail]> wrote:

> On Sat, May 05, 2018 at 10:11:00AM +0200, Ingo Molnar wrote:
> 
> > Before:
> > 
> >  #ifndef atomic_fetch_dec_relaxed
> > 
> >  #ifndef atomic_fetch_dec
> >  #define atomic_fetch_dec(v)	        atomic_fetch_sub(1, (v))
> >  #define atomic_fetch_dec_relaxed(v)	atomic_fetch_sub_relaxed(1, (v))
> >  #define atomic_fetch_dec_acquire(v)	atomic_fetch_sub_acquire(1, (v))
> >  #define atomic_fetch_dec_release(v)	atomic_fetch_sub_release(1, (v))
> >  #else /* atomic_fetch_dec */
> >  #define atomic_fetch_dec_relaxed	atomic_fetch_dec
> >  #define atomic_fetch_dec_acquire	atomic_fetch_dec
> >  #define atomic_fetch_dec_release	atomic_fetch_dec
> >  #endif /* atomic_fetch_dec */
> > 
> >  #else /* atomic_fetch_dec_relaxed */
> > 
> >  #ifndef atomic_fetch_dec_acquire
> >  #define atomic_fetch_dec_acquire(...)					\
> > 	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
> >  #endif
> > 
> >  #ifndef atomic_fetch_dec_release
> >  #define atomic_fetch_dec_release(...)					\
> > 	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
> >  #endif
> > 
> >  #ifndef atomic_fetch_dec
> >  #define atomic_fetch_dec(...)						\
> > 	__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
> >  #endif
> >  #endif /* atomic_fetch_dec_relaxed */
> > 
> > After:
> > 
> >  #ifndef atomic_fetch_dec_relaxed
> >  # ifndef atomic_fetch_dec
> >  #  define atomic_fetch_dec(v)			atomic_fetch_sub(1, (v))
> >  #  define atomic_fetch_dec_relaxed(v)		atomic_fetch_sub_relaxed(1, (v))
> >  #  define atomic_fetch_dec_acquire(v)		atomic_fetch_sub_acquire(1, (v))
> >  #  define atomic_fetch_dec_release(v)		atomic_fetch_sub_release(1, (v))
> >  # else
> >  #  define atomic_fetch_dec_relaxed		atomic_fetch_dec
> >  #  define atomic_fetch_dec_acquire		atomic_fetch_dec
> >  #  define atomic_fetch_dec_release		atomic_fetch_dec
> >  # endif
> >  #else
> >  # ifndef atomic_fetch_dec_acquire
> >  #  define atomic_fetch_dec_acquire(...)	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
> >  # endif
> >  # ifndef atomic_fetch_dec_release
> >  #  define atomic_fetch_dec_release(...)	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
> >  # endif
> >  # ifndef atomic_fetch_dec
> >  #  define atomic_fetch_dec(...)		__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
> >  # endif
> >  #endif
> > 
> > The new variant is readable at a glance, and the hierarchy of defines is very 
> > obvious as well.
> 
> It wraps and looks hideous in my normal setup. And I do detest that indent
> after # thing.

You should use wider terminals if you take a look at such code - there's already 
numerous areas of the kernel that are not readable on 80x25 terminals.

_Please_ try the following experiment, for me:

Enter the 21st century temporarily and widen two of your terminals from 80 cols to 
100 cols - it's only ~20% wider.

Apply the 3 patches I sent and then open the new and the old atomic.h in the two 
terminals and compare them visually.

The new structure is _much_ more compact, it is nicer looking and much more 
readable.

Thanks,

	Ingo

Peter Zijlstra May 5, 2018, 9:24 a.m. UTC | #5

On Sat, May 05, 2018 at 11:04:03AM +0200, Ingo Molnar wrote:
> 
> * Peter Zijlstra <[e-mail]> wrote:
> 
> > > So we could do the following simplification on top of that:
> > > 
> > >  #ifndef atomic_fetch_dec_relaxed
> > >  # ifndef atomic_fetch_dec
> > >  #  define atomic_fetch_dec(v)		atomic_fetch_sub(1, (v))
> > >  #  define atomic_fetch_dec_relaxed(v)	atomic_fetch_sub_relaxed(1, (v))
> > >  #  define atomic_fetch_dec_acquire(v)	atomic_fetch_sub_acquire(1, (v))
> > >  #  define atomic_fetch_dec_release(v)	atomic_fetch_sub_release(1, (v))
> > >  # else
> > >  #  define atomic_fetch_dec_relaxed		atomic_fetch_dec
> > >  #  define atomic_fetch_dec_acquire		atomic_fetch_dec
> > >  #  define atomic_fetch_dec_release		atomic_fetch_dec
> > >  # endif
> > >  #else
> > >  # ifndef atomic_fetch_dec
> > >  #  define atomic_fetch_dec(...)		__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
> > >  #  define atomic_fetch_dec_acquire(...)	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
> > >  #  define atomic_fetch_dec_release(...)	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
> > >  # endif
> > >  #endif
> > 
> > This would disallow an architecture to override just fetch_dec_release for
> > instance.
> 
> Couldn't such a crazy arch just define _all_ the 3 APIs in this group?
> That's really a small price and makes the place pay the complexity
> price that does the weirdness...

I would expect the pattern where it can do all 'release' and/or all
'acquire' variants special but cannot use the __atomic_op_*() wrappery.

> > I don't think there currently is any architecture that does that, but the
> > intent was to allow it to override anything and only provide defaults where it
> > does not.
> 
> I'd argue that if a new arch only defines one of these APIs that's probably a bug. 
> If they absolutely want to do it, they still can - by defining all 3 APIs.
> 
> So there's no loss in arch flexibility.

Ideally we'd generate the whole mess.. and then allowing these extra few
overrides is not a problem at all.

> > None of this takes away the giant trainwreck that is the annotated atomic stuff
> > though.
> > 
> > And I seriously hate this one:
> > 
> >   ba1c9f83f633 ("locking/atomic/x86: Un-macro-ify atomic ops implementation")
> > 
> > and will likely undo that the moment I need to change anything there.
> 
> If it makes the code more readable then I don't object - the problem was that the 
> instrumentation indirection made all that code much harder to follow.

Thing is, it is all the exact same loop, and bitrot mandates they drift
over time. When I cleaned up all the architectures I found plenty cases
where there were spurious differences between things.

Peter Zijlstra May 5, 2018, 9:29 a.m. UTC | #6

On Sat, May 05, 2018 at 11:09:03AM +0200, Ingo Molnar wrote:
> > >  # ifndef atomic_fetch_dec_acquire
> > >  #  define atomic_fetch_dec_acquire(...)	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
> > >  # endif
> > >  # ifndef atomic_fetch_dec_release
> > >  #  define atomic_fetch_dec_release(...)	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
> > >  # endif
> > >  # ifndef atomic_fetch_dec
> > >  #  define atomic_fetch_dec(...)		__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
> > >  # endif
> > >  #endif
> > > 
> > > The new variant is readable at a glance, and the hierarchy of defines is very 
> > > obvious as well.
> > 
> > It wraps and looks hideous in my normal setup. And I do detest that indent
> > after # thing.
> 
> You should use wider terminals if you take a look at such code - there's already 
> numerous areas of the kernel that are not readable on 80x25 terminals.
> 
> _Please_ try the following experiment, for me:
> 
> Enter the 21st century temporarily and widen two of your terminals from 80 cols to 
> 100 cols - it's only ~20% wider.

Doesn't work that way. The only way I get more columns is if I shrink my
font further. I work with tiles per monitor (left/right obv.) and use
two columns per editor. This gets me a total of 4 columns.

On my desktop that is slightly over 100 characters per column, on my
laptop that is slightly below 100 -- mostly because I'm pixel limited on
fontsize on that thing (FullHD sucks).

If it wraps it wraps.

Peter Zijlstra May 5, 2018, 9:32 a.m. UTC | #7

On Sat, May 05, 2018 at 11:05:51AM +0200, Dmitry Vyukov wrote:
> On Sat, May 5, 2018 at 10:47 AM, Peter Zijlstra <[e-mail]> wrote:
> > And I seriously hate this one:
> >
> >   ba1c9f83f633 ("locking/atomic/x86: Un-macro-ify atomic ops implementation")
> >
> > and will likely undo that the moment I need to change anything there.

> That was asked by Ingo:
> https://groups.google.com/d/msg/kasan-dev/3sNHjjb4GCI/Xz1uVWaaAAAJ
> 
> I think in the end all of current options suck in one way or another,
> so we are just going in circles.

Yeah, and I disagree with him, but didn't have the energy to fight at
that time (and still don't really, I'm just complaining).

> We either need something different (e.g. codegen), or settle on one
> option for doing it.

Codegen I think is the only sensible option at this point for all the
wrappers. The existing ones (without the annotation muck) were already
cumbersome, the annotation stuff just makes it completely horrid.

Ingo Molnar May 5, 2018, 9:38 a.m. UTC | #8

* Ingo Molnar <[e-mail]> wrote:

> * Peter Zijlstra <[e-mail]> wrote:
> 
> > > So we could do the following simplification on top of that:
> > > 
> > >  #ifndef atomic_fetch_dec_relaxed
> > >  # ifndef atomic_fetch_dec
> > >  #  define atomic_fetch_dec(v)		atomic_fetch_sub(1, (v))
> > >  #  define atomic_fetch_dec_relaxed(v)	atomic_fetch_sub_relaxed(1, (v))
> > >  #  define atomic_fetch_dec_acquire(v)	atomic_fetch_sub_acquire(1, (v))
> > >  #  define atomic_fetch_dec_release(v)	atomic_fetch_sub_release(1, (v))
> > >  # else
> > >  #  define atomic_fetch_dec_relaxed		atomic_fetch_dec
> > >  #  define atomic_fetch_dec_acquire		atomic_fetch_dec
> > >  #  define atomic_fetch_dec_release		atomic_fetch_dec
> > >  # endif
> > >  #else
> > >  # ifndef atomic_fetch_dec
> > >  #  define atomic_fetch_dec(...)		__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
> > >  #  define atomic_fetch_dec_acquire(...)	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
> > >  #  define atomic_fetch_dec_release(...)	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
> > >  # endif
> > >  #endif
> > 
> > This would disallow an architecture to override just fetch_dec_release for
> > instance.
> 
> Couldn't such a crazy arch just define _all_ the 3 APIs in this group?
> That's really a small price and makes the place pay the complexity
> price that does the weirdness...
> 
> > I don't think there currently is any architecture that does that, but the
> > intent was to allow it to override anything and only provide defaults where it
> > does not.
> 
> I'd argue that if a new arch only defines one of these APIs that's probably a bug. 
> If they absolutely want to do it, they still can - by defining all 3 APIs.
> 
> So there's no loss in arch flexibility.

BTW., PowerPC for example is already in such a situation, it does not define 
atomic_cmpxchg_release(), only the other APIs:

#define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
#define atomic_cmpxchg_relaxed(v, o, n) \
	cmpxchg_relaxed(&((v)->counter), (o), (n))
#define atomic_cmpxchg_acquire(v, o, n) \
	cmpxchg_acquire(&((v)->counter), (o), (n))

Was it really the intention on the PowerPC side that the generic code falls back 
to cmpxchg(), i.e.:

#  define atomic_cmpxchg_release(...)           __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)

Which after macro expansion becomes:

	smp_mb__before_atomic();
	atomic_cmpxchg_relaxed(v, o, n);

smp_mb__before_atomic() on PowerPC falls back to the generic __smp_mb(), which 
falls back to mb(), which on PowerPC is the 'sync' instruction.

Isn't this a inefficiency bug?

While I'm pretty clueless about PowerPC low level cmpxchg atomics, they appear to 
have the following basic structure:

full cmpxchg():

	PPC_ATOMIC_ENTRY_BARRIER # sync
	ldarx + stdcx
	PPC_ATOMIC_EXIT_BARRIER  # sync

cmpxchg_relaxed():

	ldarx + stdcx

cmpxchg_acquire():

	ldarx + stdcx
	PPC_ACQUIRE_BARRIER      # lwsync

The logical extension for cmpxchg_release() would be:

cmpxchg_release():

	PPC_RELEASE_BARRIER      # lwsync
	ldarx + stdcx

But instead we silently get the generic fallback, which does:

	smp_mb__before_atomic();
	atomic_cmpxchg_relaxed(v, o, n);

Which maps to:

	sync
	ldarx + stdcx

Note that it uses a full barrier instead of lwsync (does that stand for 
'lightweight sync'?).

Even if it turns out we need the full barrier, with the overly finegrained 
structure of the atomics this detail is totally undocumented and non-obvious.

Thanks,

	Ingo

Boqun Feng May 5, 2018, 10:16 a.m. UTC | #9

On Sat, May 05, 2018 at 11:38:29AM +0200, Ingo Molnar wrote:
> 
> * Ingo Molnar <[e-mail]> wrote:
> 
> > * Peter Zijlstra <[e-mail]> wrote:
> > 
> > > > So we could do the following simplification on top of that:
> > > > 
> > > >  #ifndef atomic_fetch_dec_relaxed
> > > >  # ifndef atomic_fetch_dec
> > > >  #  define atomic_fetch_dec(v)		atomic_fetch_sub(1, (v))
> > > >  #  define atomic_fetch_dec_relaxed(v)	atomic_fetch_sub_relaxed(1, (v))
> > > >  #  define atomic_fetch_dec_acquire(v)	atomic_fetch_sub_acquire(1, (v))
> > > >  #  define atomic_fetch_dec_release(v)	atomic_fetch_sub_release(1, (v))
> > > >  # else
> > > >  #  define atomic_fetch_dec_relaxed		atomic_fetch_dec
> > > >  #  define atomic_fetch_dec_acquire		atomic_fetch_dec
> > > >  #  define atomic_fetch_dec_release		atomic_fetch_dec
> > > >  # endif
> > > >  #else
> > > >  # ifndef atomic_fetch_dec
> > > >  #  define atomic_fetch_dec(...)		__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
> > > >  #  define atomic_fetch_dec_acquire(...)	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
> > > >  #  define atomic_fetch_dec_release(...)	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
> > > >  # endif
> > > >  #endif
> > > 
> > > This would disallow an architecture to override just fetch_dec_release for
> > > instance.
> > 
> > Couldn't such a crazy arch just define _all_ the 3 APIs in this group?
> > That's really a small price and makes the place pay the complexity
> > price that does the weirdness...
> > 
> > > I don't think there currently is any architecture that does that, but the
> > > intent was to allow it to override anything and only provide defaults where it
> > > does not.
> > 
> > I'd argue that if a new arch only defines one of these APIs that's probably a bug. 
> > If they absolutely want to do it, they still can - by defining all 3 APIs.
> > 
> > So there's no loss in arch flexibility.
> 
> BTW., PowerPC for example is already in such a situation, it does not define 
> atomic_cmpxchg_release(), only the other APIs:
> 
> #define atomic_cmpxchg(v, o, n) (cmpxchg(&((v)->counter), (o), (n)))
> #define atomic_cmpxchg_relaxed(v, o, n) \
> 	cmpxchg_relaxed(&((v)->counter), (o), (n))
> #define atomic_cmpxchg_acquire(v, o, n) \
> 	cmpxchg_acquire(&((v)->counter), (o), (n))
> 
> Was it really the intention on the PowerPC side that the generic code falls back 
> to cmpxchg(), i.e.:
> 
> #  define atomic_cmpxchg_release(...)           __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
> 

So ppc has its own definition __atomic_op_release() in
arch/powerpc/include/asm/atomic.h:

	#define __atomic_op_release(op, args...)				\
	({									\
		__asm__ __volatile__(PPC_RELEASE_BARRIER "" : : : "memory");	\
		op##_relaxed(args);						\
	})

, and PPC_RELEASE_BARRIER is lwsync, so we map to

	lwsync();
	atomic_cmpxchg_relaxed(v, o, n);

And the reason, why we don't define atomic_cmpxchg_release() but define
atomic_cmpxchg_acquire() is that, atomic_cmpxchg_*() could provide no
ordering guarantee if the cmp fails, we did this for
atomic_cmpxchg_acquire() but not for atomic_cmpxchg_release(), because
doing so may introduce a memory barrier inside a ll/sc critical section,
please see the comment before __cmpxchg_u32_acquire() in
arch/powerpc/include/asm/cmpxchg.h:

	/*
	 * cmpxchg family don't have order guarantee if cmp part fails, therefore we
	 * can avoid superfluous barriers if we use assembly code to implement
	 * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for
	 * cmpxchg_release() because that will result in putting a barrier in the
	 * middle of a ll/sc loop, which is probably a bad idea. For example, this
	 * might cause the conditional store more likely to fail.
	 */

Regards,
Boqun


> Which after macro expansion becomes:
> 
> 	smp_mb__before_atomic();
> 	atomic_cmpxchg_relaxed(v, o, n);
> 
> smp_mb__before_atomic() on PowerPC falls back to the generic __smp_mb(), which 
> falls back to mb(), which on PowerPC is the 'sync' instruction.
> 
> Isn't this a inefficiency bug?
> 
> While I'm pretty clueless about PowerPC low level cmpxchg atomics, they appear to 
> have the following basic structure:
> 
> full cmpxchg():
> 
> 	PPC_ATOMIC_ENTRY_BARRIER # sync
> 	ldarx + stdcx
> 	PPC_ATOMIC_EXIT_BARRIER  # sync
> 
> cmpxchg_relaxed():
> 
> 	ldarx + stdcx
> 
> cmpxchg_acquire():
> 
> 	ldarx + stdcx
> 	PPC_ACQUIRE_BARRIER      # lwsync
> 
> The logical extension for cmpxchg_release() would be:
> 
> cmpxchg_release():
> 
> 	PPC_RELEASE_BARRIER      # lwsync
> 	ldarx + stdcx
> 
> But instead we silently get the generic fallback, which does:
> 
> 	smp_mb__before_atomic();
> 	atomic_cmpxchg_relaxed(v, o, n);
> 
> Which maps to:
> 
> 	sync
> 	ldarx + stdcx
> 
> Note that it uses a full barrier instead of lwsync (does that stand for 
> 'lightweight sync'?).
> 
> Even if it turns out we need the full barrier, with the overly finegrained 
> structure of the atomics this detail is totally undocumented and non-obvious.
> 
> Thanks,
> 
> 	Ingo

Patch

=============================>
From 0171d4ed840d25befaedcf03e834bb76acb400c0 Mon Sep 17 00:00:00 2001
From: Ingo Molnar <[e-mail]>
Date: Sat, 5 May 2018 09:57:02 +0200
Subject: [PATCH] locking/atomics: Clean up the atomic.h maze of #defines

Use structured defines to make it all much more readable.

Before:

 #ifndef atomic_fetch_dec_relaxed

 #ifndef atomic_fetch_dec
 #define atomic_fetch_dec(v)	        atomic_fetch_sub(1, (v))
 #define atomic_fetch_dec_relaxed(v)	atomic_fetch_sub_relaxed(1, (v))
 #define atomic_fetch_dec_acquire(v)	atomic_fetch_sub_acquire(1, (v))
 #define atomic_fetch_dec_release(v)	atomic_fetch_sub_release(1, (v))
 #else /* atomic_fetch_dec */
 #define atomic_fetch_dec_relaxed	atomic_fetch_dec
 #define atomic_fetch_dec_acquire	atomic_fetch_dec
 #define atomic_fetch_dec_release	atomic_fetch_dec
 #endif /* atomic_fetch_dec */

 #else /* atomic_fetch_dec_relaxed */

 #ifndef atomic_fetch_dec_acquire
 #define atomic_fetch_dec_acquire(...)					\
	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
 #endif

 #ifndef atomic_fetch_dec_release
 #define atomic_fetch_dec_release(...)					\
	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
 #endif

 #ifndef atomic_fetch_dec
 #define atomic_fetch_dec(...)						\
	__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
 #endif
 #endif /* atomic_fetch_dec_relaxed */

After:

 #ifndef atomic_fetch_dec_relaxed
 # ifndef atomic_fetch_dec
 #  define atomic_fetch_dec(v)			atomic_fetch_sub(1, (v))
 #  define atomic_fetch_dec_relaxed(v)		atomic_fetch_sub_relaxed(1, (v))
 #  define atomic_fetch_dec_acquire(v)		atomic_fetch_sub_acquire(1, (v))
 #  define atomic_fetch_dec_release(v)		atomic_fetch_sub_release(1, (v))
 # else
 #  define atomic_fetch_dec_relaxed		atomic_fetch_dec
 #  define atomic_fetch_dec_acquire		atomic_fetch_dec
 #  define atomic_fetch_dec_release		atomic_fetch_dec
 # endif
 #else
 # ifndef atomic_fetch_dec_acquire
 #  define atomic_fetch_dec_acquire(...)	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
 # endif
 # ifndef atomic_fetch_dec_release
 #  define atomic_fetch_dec_release(...)	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
 # endif
 # ifndef atomic_fetch_dec
 #  define atomic_fetch_dec(...)		__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
 # endif
 #endif

Beyond the linecount reduction this also makes it easier to follow
the various conditions.

Also clean up a few other minor details and make the code more
consistent throughout.

No change in functionality.

Cc: Peter Zijlstra <[e-mail]>
Cc: Linus Torvalds <[e-mail]>
Cc: Andrew Morton <[e-mail]>
Cc: Thomas Gleixner <[e-mail]>
Cc: Paul E. McKenney <[e-mail]>
Cc: Will Deacon <[e-mail]>
Cc: [e-mail]
Signed-off-by: Ingo Molnar <[e-mail]>
---
 include/linux/atomic.h | 1275 +++++++++++++++++++++---------------------------
 1 file changed, 543 insertions(+), 732 deletions(-)

diff --git a/include/linux/atomic.h b/include/linux/atomic.h
index 01ce3997cb42..dc157c092ae5 100644
--- a/include/linux/atomic.h
+++ b/include/linux/atomic.h
@@ -24,11 +24,11 @@ 
  */
 
 #ifndef atomic_read_acquire
-#define  atomic_read_acquire(v)		smp_load_acquire(&(v)->counter)
+# define atomic_read_acquire(v)			smp_load_acquire(&(v)->counter)
 #endif
 
 #ifndef atomic_set_release
-#define  atomic_set_release(v, i)	smp_store_release(&(v)->counter, (i))
+# define atomic_set_release(v, i)		smp_store_release(&(v)->counter, (i))
 #endif
 
 /*
@@ -71,454 +71,351 @@ 
 })
 #endif
 
-/* atomic_add_return_relaxed */
-#ifndef atomic_add_return_relaxed
-#define  atomic_add_return_relaxed	atomic_add_return
-#define  atomic_add_return_acquire	atomic_add_return
-#define  atomic_add_return_release	atomic_add_return
-
-#else /* atomic_add_return_relaxed */
-
-#ifndef atomic_add_return_acquire
-#define  atomic_add_return_acquire(...)					\
-	__atomic_op_acquire(atomic_add_return, __VA_ARGS__)
-#endif
+/* atomic_add_return_relaxed() et al: */
 
-#ifndef atomic_add_return_release
-#define  atomic_add_return_release(...)					\
-	__atomic_op_release(atomic_add_return, __VA_ARGS__)
-#endif
-
-#ifndef atomic_add_return
-#define  atomic_add_return(...)						\
-	__atomic_op_fence(atomic_add_return, __VA_ARGS__)
-#endif
-#endif /* atomic_add_return_relaxed */
+#ifndef atomic_add_return_relaxed
+# define atomic_add_return_relaxed		atomic_add_return
+# define atomic_add_return_acquire		atomic_add_return
+# define atomic_add_return_release		atomic_add_return
+#else
+# ifndef atomic_add_return_acquire
+#  define atomic_add_return_acquire(...)	__atomic_op_acquire(atomic_add_return, __VA_ARGS__)
+# endif
+# ifndef atomic_add_return_release
+#  define atomic_add_return_release(...)	__atomic_op_release(atomic_add_return, __VA_ARGS__)
+# endif
+# ifndef atomic_add_return
+#  define atomic_add_return(...)		__atomic_op_fence(atomic_add_return, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic_inc_return_relaxed() et al: */
 
-/* atomic_inc_return_relaxed */
 #ifndef atomic_inc_return_relaxed
-#define  atomic_inc_return_relaxed	atomic_inc_return
-#define  atomic_inc_return_acquire	atomic_inc_return
-#define  atomic_inc_return_release	atomic_inc_return
-
-#else /* atomic_inc_return_relaxed */
-
-#ifndef atomic_inc_return_acquire
-#define  atomic_inc_return_acquire(...)					\
-	__atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
-#endif
-
-#ifndef atomic_inc_return_release
-#define  atomic_inc_return_release(...)					\
-	__atomic_op_release(atomic_inc_return, __VA_ARGS__)
-#endif
-
-#ifndef atomic_inc_return
-#define  atomic_inc_return(...)						\
-	__atomic_op_fence(atomic_inc_return, __VA_ARGS__)
-#endif
-#endif /* atomic_inc_return_relaxed */
+# define atomic_inc_return_relaxed		atomic_inc_return
+# define atomic_inc_return_acquire		atomic_inc_return
+# define atomic_inc_return_release		atomic_inc_return
+#else
+# ifndef atomic_inc_return_acquire
+#  define atomic_inc_return_acquire(...)	__atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
+# endif
+# ifndef atomic_inc_return_release
+#  define atomic_inc_return_release(...)	__atomic_op_release(atomic_inc_return, __VA_ARGS__)
+# endif
+# ifndef atomic_inc_return
+#  define atomic_inc_return(...)		__atomic_op_fence(atomic_inc_return, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic_sub_return_relaxed() et al: */
 
-/* atomic_sub_return_relaxed */
 #ifndef atomic_sub_return_relaxed
-#define  atomic_sub_return_relaxed	atomic_sub_return
-#define  atomic_sub_return_acquire	atomic_sub_return
-#define  atomic_sub_return_release	atomic_sub_return
-
-#else /* atomic_sub_return_relaxed */
-
-#ifndef atomic_sub_return_acquire
-#define  atomic_sub_return_acquire(...)					\
-	__atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
-#endif
-
-#ifndef atomic_sub_return_release
-#define  atomic_sub_return_release(...)					\
-	__atomic_op_release(atomic_sub_return, __VA_ARGS__)
-#endif
-
-#ifndef atomic_sub_return
-#define  atomic_sub_return(...)						\
-	__atomic_op_fence(atomic_sub_return, __VA_ARGS__)
-#endif
-#endif /* atomic_sub_return_relaxed */
+# define atomic_sub_return_relaxed		atomic_sub_return
+# define atomic_sub_return_acquire		atomic_sub_return
+# define atomic_sub_return_release		atomic_sub_return
+#else
+# ifndef atomic_sub_return_acquire
+#  define atomic_sub_return_acquire(...)	__atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
+# endif
+# ifndef atomic_sub_return_release
+#  define atomic_sub_return_release(...)	__atomic_op_release(atomic_sub_return, __VA_ARGS__)
+# endif
+# ifndef atomic_sub_return
+#  define atomic_sub_return(...)		__atomic_op_fence(atomic_sub_return, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic_dec_return_relaxed() et al: */
 
-/* atomic_dec_return_relaxed */
 #ifndef atomic_dec_return_relaxed
-#define  atomic_dec_return_relaxed	atomic_dec_return
-#define  atomic_dec_return_acquire	atomic_dec_return
-#define  atomic_dec_return_release	atomic_dec_return
-
-#else /* atomic_dec_return_relaxed */
-
-#ifndef atomic_dec_return_acquire
-#define  atomic_dec_return_acquire(...)					\
-	__atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
-#endif
+# define atomic_dec_return_relaxed		atomic_dec_return
+# define atomic_dec_return_acquire		atomic_dec_return
+# define atomic_dec_return_release		atomic_dec_return
+#else
+# ifndef atomic_dec_return_acquire
+#  define atomic_dec_return_acquire(...)	__atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
+# endif
+# ifndef atomic_dec_return_release
+#  define atomic_dec_return_release(...)	__atomic_op_release(atomic_dec_return, __VA_ARGS__)
+# endif
+# ifndef atomic_dec_return
+#  define atomic_dec_return(...)		__atomic_op_fence(atomic_dec_return, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic_fetch_add_relaxed() et al: */
 
-#ifndef atomic_dec_return_release
-#define  atomic_dec_return_release(...)					\
-	__atomic_op_release(atomic_dec_return, __VA_ARGS__)
-#endif
-
-#ifndef atomic_dec_return
-#define  atomic_dec_return(...)						\
-	__atomic_op_fence(atomic_dec_return, __VA_ARGS__)
-#endif
-#endif /* atomic_dec_return_relaxed */
-
-
-/* atomic_fetch_add_relaxed */
 #ifndef atomic_fetch_add_relaxed
-#define atomic_fetch_add_relaxed	atomic_fetch_add
-#define atomic_fetch_add_acquire	atomic_fetch_add
-#define atomic_fetch_add_release	atomic_fetch_add
-
-#else /* atomic_fetch_add_relaxed */
-
-#ifndef atomic_fetch_add_acquire
-#define atomic_fetch_add_acquire(...)					\
-	__atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)
-#endif
-
-#ifndef atomic_fetch_add_release
-#define atomic_fetch_add_release(...)					\
-	__atomic_op_release(atomic_fetch_add, __VA_ARGS__)
-#endif
+# define atomic_fetch_add_relaxed		atomic_fetch_add
+# define atomic_fetch_add_acquire		atomic_fetch_add
+# define atomic_fetch_add_release		atomic_fetch_add
+#else
+# ifndef atomic_fetch_add_acquire
+#  define atomic_fetch_add_acquire(...)		__atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_add_release
+#  define atomic_fetch_add_release(...)		__atomic_op_release(atomic_fetch_add, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_add
+#  define atomic_fetch_add(...)			__atomic_op_fence(atomic_fetch_add, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic_fetch_inc_relaxed() et al: */
 
-#ifndef atomic_fetch_add
-#define atomic_fetch_add(...)						\
-	__atomic_op_fence(atomic_fetch_add, __VA_ARGS__)
-#endif
-#endif /* atomic_fetch_add_relaxed */
-
-/* atomic_fetch_inc_relaxed */
 #ifndef atomic_fetch_inc_relaxed
+# ifndef atomic_fetch_inc
+#  define atomic_fetch_inc(v)			atomic_fetch_add(1, (v))
+#  define atomic_fetch_inc_relaxed(v)		atomic_fetch_add_relaxed(1, (v))
+#  define atomic_fetch_inc_acquire(v)		atomic_fetch_add_acquire(1, (v))
+#  define atomic_fetch_inc_release(v)		atomic_fetch_add_release(1, (v))
+# else
+#  define atomic_fetch_inc_relaxed		atomic_fetch_inc
+#  define atomic_fetch_inc_acquire		atomic_fetch_inc
+#  define atomic_fetch_inc_release		atomic_fetch_inc
+# endif
+#else
+# ifndef atomic_fetch_inc_acquire
+#  define atomic_fetch_inc_acquire(...)		__atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_inc_release
+#  define atomic_fetch_inc_release(...)		__atomic_op_release(atomic_fetch_inc, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_inc
+#  define atomic_fetch_inc(...)			__atomic_op_fence(atomic_fetch_inc, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic_fetch_sub_relaxed() et al: */
 
-#ifndef atomic_fetch_inc
-#define atomic_fetch_inc(v)	        atomic_fetch_add(1, (v))
-#define atomic_fetch_inc_relaxed(v)	atomic_fetch_add_relaxed(1, (v))
-#define atomic_fetch_inc_acquire(v)	atomic_fetch_add_acquire(1, (v))
-#define atomic_fetch_inc_release(v)	atomic_fetch_add_release(1, (v))
-#else /* atomic_fetch_inc */
-#define atomic_fetch_inc_relaxed	atomic_fetch_inc
-#define atomic_fetch_inc_acquire	atomic_fetch_inc
-#define atomic_fetch_inc_release	atomic_fetch_inc
-#endif /* atomic_fetch_inc */
-
-#else /* atomic_fetch_inc_relaxed */
-
-#ifndef atomic_fetch_inc_acquire
-#define atomic_fetch_inc_acquire(...)					\
-	__atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__)
-#endif
-
-#ifndef atomic_fetch_inc_release
-#define atomic_fetch_inc_release(...)					\
-	__atomic_op_release(atomic_fetch_inc, __VA_ARGS__)
-#endif
-
-#ifndef atomic_fetch_inc
-#define atomic_fetch_inc(...)						\
-	__atomic_op_fence(atomic_fetch_inc, __VA_ARGS__)
-#endif
-#endif /* atomic_fetch_inc_relaxed */
-
-/* atomic_fetch_sub_relaxed */
 #ifndef atomic_fetch_sub_relaxed
-#define atomic_fetch_sub_relaxed	atomic_fetch_sub
-#define atomic_fetch_sub_acquire	atomic_fetch_sub
-#define atomic_fetch_sub_release	atomic_fetch_sub
-
-#else /* atomic_fetch_sub_relaxed */
-
-#ifndef atomic_fetch_sub_acquire
-#define atomic_fetch_sub_acquire(...)					\
-	__atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)
-#endif
+# define atomic_fetch_sub_relaxed		atomic_fetch_sub
+# define atomic_fetch_sub_acquire		atomic_fetch_sub
+# define atomic_fetch_sub_release		atomic_fetch_sub
+#else
+# ifndef atomic_fetch_sub_acquire
+#  define atomic_fetch_sub_acquire(...)		__atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_sub_release
+#  define atomic_fetch_sub_release(...)		__atomic_op_release(atomic_fetch_sub, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_sub
+#  define atomic_fetch_sub(...)			__atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic_fetch_dec_relaxed() et al: */
 
-#ifndef atomic_fetch_sub_release
-#define atomic_fetch_sub_release(...)					\
-	__atomic_op_release(atomic_fetch_sub, __VA_ARGS__)
-#endif
-
-#ifndef atomic_fetch_sub
-#define atomic_fetch_sub(...)						\
-	__atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)
-#endif
-#endif /* atomic_fetch_sub_relaxed */
-
-/* atomic_fetch_dec_relaxed */
 #ifndef atomic_fetch_dec_relaxed
+# ifndef atomic_fetch_dec
+#  define atomic_fetch_dec(v)			atomic_fetch_sub(1, (v))
+#  define atomic_fetch_dec_relaxed(v)		atomic_fetch_sub_relaxed(1, (v))
+#  define atomic_fetch_dec_acquire(v)		atomic_fetch_sub_acquire(1, (v))
+#  define atomic_fetch_dec_release(v)		atomic_fetch_sub_release(1, (v))
+# else
+#  define atomic_fetch_dec_relaxed		atomic_fetch_dec
+#  define atomic_fetch_dec_acquire		atomic_fetch_dec
+#  define atomic_fetch_dec_release		atomic_fetch_dec
+# endif
+#else
+# ifndef atomic_fetch_dec_acquire
+#  define atomic_fetch_dec_acquire(...)		__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_dec_release
+#  define atomic_fetch_dec_release(...)		__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_dec
+#  define atomic_fetch_dec(...)			__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic_fetch_or_relaxed() et al: */
 
-#ifndef atomic_fetch_dec
-#define atomic_fetch_dec(v)	        atomic_fetch_sub(1, (v))
-#define atomic_fetch_dec_relaxed(v)	atomic_fetch_sub_relaxed(1, (v))
-#define atomic_fetch_dec_acquire(v)	atomic_fetch_sub_acquire(1, (v))
-#define atomic_fetch_dec_release(v)	atomic_fetch_sub_release(1, (v))
-#else /* atomic_fetch_dec */
-#define atomic_fetch_dec_relaxed	atomic_fetch_dec
-#define atomic_fetch_dec_acquire	atomic_fetch_dec
-#define atomic_fetch_dec_release	atomic_fetch_dec
-#endif /* atomic_fetch_dec */
-
-#else /* atomic_fetch_dec_relaxed */
-
-#ifndef atomic_fetch_dec_acquire
-#define atomic_fetch_dec_acquire(...)					\
-	__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
-#endif
-
-#ifndef atomic_fetch_dec_release
-#define atomic_fetch_dec_release(...)					\
-	__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
-#endif
-
-#ifndef atomic_fetch_dec
-#define atomic_fetch_dec(...)						\
-	__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
-#endif
-#endif /* atomic_fetch_dec_relaxed */
-
-/* atomic_fetch_or_relaxed */
 #ifndef atomic_fetch_or_relaxed
-#define atomic_fetch_or_relaxed	atomic_fetch_or
-#define atomic_fetch_or_acquire	atomic_fetch_or
-#define atomic_fetch_or_release	atomic_fetch_or
-
-#else /* atomic_fetch_or_relaxed */
-
-#ifndef atomic_fetch_or_acquire
-#define atomic_fetch_or_acquire(...)					\
-	__atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)
-#endif
-
-#ifndef atomic_fetch_or_release
-#define atomic_fetch_or_release(...)					\
-	__atomic_op_release(atomic_fetch_or, __VA_ARGS__)
-#endif
-
-#ifndef atomic_fetch_or
-#define atomic_fetch_or(...)						\
-	__atomic_op_fence(atomic_fetch_or, __VA_ARGS__)
-#endif
-#endif /* atomic_fetch_or_relaxed */
+# define atomic_fetch_or_relaxed		atomic_fetch_or
+# define atomic_fetch_or_acquire		atomic_fetch_or
+# define atomic_fetch_or_release		atomic_fetch_or
+#else
+# ifndef atomic_fetch_or_acquire
+#  define atomic_fetch_or_acquire(...)		__atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_or_release
+#  define atomic_fetch_or_release(...)		__atomic_op_release(atomic_fetch_or, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_or
+#  define atomic_fetch_or(...)			__atomic_op_fence(atomic_fetch_or, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic_fetch_and_relaxed() et al: */
 
-/* atomic_fetch_and_relaxed */
 #ifndef atomic_fetch_and_relaxed
-#define atomic_fetch_and_relaxed	atomic_fetch_and
-#define atomic_fetch_and_acquire	atomic_fetch_and
-#define atomic_fetch_and_release	atomic_fetch_and
-
-#else /* atomic_fetch_and_relaxed */
-
-#ifndef atomic_fetch_and_acquire
-#define atomic_fetch_and_acquire(...)					\
-	__atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)
-#endif
-
-#ifndef atomic_fetch_and_release
-#define atomic_fetch_and_release(...)					\
-	__atomic_op_release(atomic_fetch_and, __VA_ARGS__)
-#endif
-
-#ifndef atomic_fetch_and
-#define atomic_fetch_and(...)						\
-	__atomic_op_fence(atomic_fetch_and, __VA_ARGS__)
+# define atomic_fetch_and_relaxed		atomic_fetch_and
+# define atomic_fetch_and_acquire		atomic_fetch_and
+# define atomic_fetch_and_release		atomic_fetch_and
+#else
+# ifndef atomic_fetch_and_acquire
+#  define atomic_fetch_and_acquire(...)		__atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_and_release
+#  define atomic_fetch_and_release(...)		__atomic_op_release(atomic_fetch_and, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_and
+#  define atomic_fetch_and(...)			__atomic_op_fence(atomic_fetch_and, __VA_ARGS__)
+# endif
 #endif
-#endif /* atomic_fetch_and_relaxed */
 
 #ifdef atomic_andnot
-/* atomic_fetch_andnot_relaxed */
-#ifndef atomic_fetch_andnot_relaxed
-#define atomic_fetch_andnot_relaxed	atomic_fetch_andnot
-#define atomic_fetch_andnot_acquire	atomic_fetch_andnot
-#define atomic_fetch_andnot_release	atomic_fetch_andnot
-
-#else /* atomic_fetch_andnot_relaxed */
 
-#ifndef atomic_fetch_andnot_acquire
-#define atomic_fetch_andnot_acquire(...)					\
-	__atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)
-#endif
+/* atomic_fetch_andnot_relaxed() et al: */
 
-#ifndef atomic_fetch_andnot_release
-#define atomic_fetch_andnot_release(...)					\
-	__atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)
+#ifndef atomic_fetch_andnot_relaxed
+# define atomic_fetch_andnot_relaxed		atomic_fetch_andnot
+# define atomic_fetch_andnot_acquire		atomic_fetch_andnot
+# define atomic_fetch_andnot_release		atomic_fetch_andnot
+#else
+# ifndef atomic_fetch_andnot_acquire
+#  define atomic_fetch_andnot_acquire(...)	 __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_andnot_release
+#  define atomic_fetch_andnot_release(...)	 __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_andnot
+#  define atomic_fetch_andnot(...)		__atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)
+# endif
 #endif
 
-#ifndef atomic_fetch_andnot
-#define atomic_fetch_andnot(...)						\
-	__atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)
-#endif
-#endif /* atomic_fetch_andnot_relaxed */
 #endif /* atomic_andnot */
 
-/* atomic_fetch_xor_relaxed */
-#ifndef atomic_fetch_xor_relaxed
-#define atomic_fetch_xor_relaxed	atomic_fetch_xor
-#define atomic_fetch_xor_acquire	atomic_fetch_xor
-#define atomic_fetch_xor_release	atomic_fetch_xor
-
-#else /* atomic_fetch_xor_relaxed */
+/* atomic_fetch_xor_relaxed() et al: */
 
-#ifndef atomic_fetch_xor_acquire
-#define atomic_fetch_xor_acquire(...)					\
-	__atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)
-#endif
-
-#ifndef atomic_fetch_xor_release
-#define atomic_fetch_xor_release(...)					\
-	__atomic_op_release(atomic_fetch_xor, __VA_ARGS__)
+#ifndef atomic_fetch_xor_relaxed
+# define atomic_fetch_xor_relaxed		atomic_fetch_xor
+# define atomic_fetch_xor_acquire		atomic_fetch_xor
+# define atomic_fetch_xor_release		atomic_fetch_xor
+#else
+# ifndef atomic_fetch_xor_acquire
+#  define atomic_fetch_xor_acquire(...)		__atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_xor_release
+#  define atomic_fetch_xor_release(...)		__atomic_op_release(atomic_fetch_xor, __VA_ARGS__)
+# endif
+# ifndef atomic_fetch_xor
+#  define atomic_fetch_xor(...)			__atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)
+# endif
 #endif
 
-#ifndef atomic_fetch_xor
-#define atomic_fetch_xor(...)						\
-	__atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)
-#endif
-#endif /* atomic_fetch_xor_relaxed */
 
+/* atomic_xchg_relaxed() et al: */
 
-/* atomic_xchg_relaxed */
 #ifndef atomic_xchg_relaxed
-#define  atomic_xchg_relaxed		atomic_xchg
-#define  atomic_xchg_acquire		atomic_xchg
-#define  atomic_xchg_release		atomic_xchg
-
-#else /* atomic_xchg_relaxed */
-
-#ifndef atomic_xchg_acquire
-#define  atomic_xchg_acquire(...)					\
-	__atomic_op_acquire(atomic_xchg, __VA_ARGS__)
-#endif
-
-#ifndef atomic_xchg_release
-#define  atomic_xchg_release(...)					\
-	__atomic_op_release(atomic_xchg, __VA_ARGS__)
-#endif
+#define atomic_xchg_relaxed			atomic_xchg
+#define atomic_xchg_acquire			atomic_xchg
+#define atomic_xchg_release			atomic_xchg
+#else
+# ifndef atomic_xchg_acquire
+#  define atomic_xchg_acquire(...)		__atomic_op_acquire(atomic_xchg, __VA_ARGS__)
+# endif
+# ifndef atomic_xchg_release
+#  define atomic_xchg_release(...)		__atomic_op_release(atomic_xchg, __VA_ARGS__)
+# endif
+# ifndef atomic_xchg
+#  define atomic_xchg(...)			__atomic_op_fence(atomic_xchg, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic_cmpxchg_relaxed() et al: */
 
-#ifndef atomic_xchg
-#define  atomic_xchg(...)						\
-	__atomic_op_fence(atomic_xchg, __VA_ARGS__)
-#endif
-#endif /* atomic_xchg_relaxed */
-
-/* atomic_cmpxchg_relaxed */
 #ifndef atomic_cmpxchg_relaxed
-#define  atomic_cmpxchg_relaxed		atomic_cmpxchg
-#define  atomic_cmpxchg_acquire		atomic_cmpxchg
-#define  atomic_cmpxchg_release		atomic_cmpxchg
-
-#else /* atomic_cmpxchg_relaxed */
-
-#ifndef atomic_cmpxchg_acquire
-#define  atomic_cmpxchg_acquire(...)					\
-	__atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
+# define atomic_cmpxchg_relaxed			atomic_cmpxchg
+# define atomic_cmpxchg_acquire			atomic_cmpxchg
+# define atomic_cmpxchg_release			atomic_cmpxchg
+#else
+# ifndef atomic_cmpxchg_acquire
+#  define atomic_cmpxchg_acquire(...)		__atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
+# endif
+# ifndef atomic_cmpxchg_release
+#  define atomic_cmpxchg_release(...)		__atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
+# endif
+# ifndef atomic_cmpxchg
+#  define atomic_cmpxchg(...)			__atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
+# endif
 #endif
 
-#ifndef atomic_cmpxchg_release
-#define  atomic_cmpxchg_release(...)					\
-	__atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
-#endif
-
-#ifndef atomic_cmpxchg
-#define  atomic_cmpxchg(...)						\
-	__atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
-#endif
-#endif /* atomic_cmpxchg_relaxed */
-
 #ifndef atomic_try_cmpxchg
-
-#define __atomic_try_cmpxchg(type, _p, _po, _n)				\
-({									\
+# define __atomic_try_cmpxchg(type, _p, _po, _n)			\
+  ({									\
 	typeof(_po) __po = (_po);					\
 	typeof(*(_po)) __r, __o = *__po;				\
 	__r = atomic_cmpxchg##type((_p), __o, (_n));			\
 	if (unlikely(__r != __o))					\
 		*__po = __r;						\
 	likely(__r == __o);						\
-})
-
-#define atomic_try_cmpxchg(_p, _po, _n)		__atomic_try_cmpxchg(, _p, _po, _n)
-#define atomic_try_cmpxchg_relaxed(_p, _po, _n)	__atomic_try_cmpxchg(_relaxed, _p, _po, _n)
-#define atomic_try_cmpxchg_acquire(_p, _po, _n)	__atomic_try_cmpxchg(_acquire, _p, _po, _n)
-#define atomic_try_cmpxchg_release(_p, _po, _n)	__atomic_try_cmpxchg(_release, _p, _po, _n)
-
-#else /* atomic_try_cmpxchg */
-#define atomic_try_cmpxchg_relaxed	atomic_try_cmpxchg
-#define atomic_try_cmpxchg_acquire	atomic_try_cmpxchg
-#define atomic_try_cmpxchg_release	atomic_try_cmpxchg
-#endif /* atomic_try_cmpxchg */
-
-/* cmpxchg_relaxed */
-#ifndef cmpxchg_relaxed
-#define  cmpxchg_relaxed		cmpxchg
-#define  cmpxchg_acquire		cmpxchg
-#define  cmpxchg_release		cmpxchg
-
-#else /* cmpxchg_relaxed */
-
-#ifndef cmpxchg_acquire
-#define  cmpxchg_acquire(...)						\
-	__atomic_op_acquire(cmpxchg, __VA_ARGS__)
+  })
+# define atomic_try_cmpxchg(_p, _po, _n)	 __atomic_try_cmpxchg(, _p, _po, _n)
+# define atomic_try_cmpxchg_relaxed(_p, _po, _n) __atomic_try_cmpxchg(_relaxed, _p, _po, _n)
+# define atomic_try_cmpxchg_acquire(_p, _po, _n) __atomic_try_cmpxchg(_acquire, _p, _po, _n)
+# define atomic_try_cmpxchg_release(_p, _po, _n) __atomic_try_cmpxchg(_release, _p, _po, _n)
+#else
+# define atomic_try_cmpxchg_relaxed		atomic_try_cmpxchg
+# define atomic_try_cmpxchg_acquire		atomic_try_cmpxchg
+# define atomic_try_cmpxchg_release		atomic_try_cmpxchg
 #endif
 
-#ifndef cmpxchg_release
-#define  cmpxchg_release(...)						\
-	__atomic_op_release(cmpxchg, __VA_ARGS__)
-#endif
+/* cmpxchg_relaxed() et al: */
 
-#ifndef cmpxchg
-#define  cmpxchg(...)							\
-	__atomic_op_fence(cmpxchg, __VA_ARGS__)
-#endif
-#endif /* cmpxchg_relaxed */
+#ifndef cmpxchg_relaxed
+# define cmpxchg_relaxed			cmpxchg
+# define cmpxchg_acquire			cmpxchg
+# define cmpxchg_release			cmpxchg
+#else
+# ifndef cmpxchg_acquire
+#  define cmpxchg_acquire(...)			__atomic_op_acquire(cmpxchg, __VA_ARGS__)
+# endif
+# ifndef cmpxchg_release
+#  define cmpxchg_release(...)			__atomic_op_release(cmpxchg, __VA_ARGS__)
+# endif
+# ifndef cmpxchg
+#  define cmpxchg(...)				__atomic_op_fence(cmpxchg, __VA_ARGS__)
+# endif
+#endif
+
+/* cmpxchg64_relaxed() et al: */
 
-/* cmpxchg64_relaxed */
 #ifndef cmpxchg64_relaxed
-#define  cmpxchg64_relaxed		cmpxchg64
-#define  cmpxchg64_acquire		cmpxchg64
-#define  cmpxchg64_release		cmpxchg64
-
-#else /* cmpxchg64_relaxed */
-
-#ifndef cmpxchg64_acquire
-#define  cmpxchg64_acquire(...)						\
-	__atomic_op_acquire(cmpxchg64, __VA_ARGS__)
-#endif
-
-#ifndef cmpxchg64_release
-#define  cmpxchg64_release(...)						\
-	__atomic_op_release(cmpxchg64, __VA_ARGS__)
-#endif
-
-#ifndef cmpxchg64
-#define  cmpxchg64(...)							\
-	__atomic_op_fence(cmpxchg64, __VA_ARGS__)
-#endif
-#endif /* cmpxchg64_relaxed */
+# define cmpxchg64_relaxed			cmpxchg64
+# define cmpxchg64_acquire			cmpxchg64
+# define cmpxchg64_release			cmpxchg64
+#else
+# ifndef cmpxchg64_acquire
+#  define cmpxchg64_acquire(...)		__atomic_op_acquire(cmpxchg64, __VA_ARGS__)
+# endif
+# ifndef cmpxchg64_release
+#  define cmpxchg64_release(...)		__atomic_op_release(cmpxchg64, __VA_ARGS__)
+# endif
+# ifndef cmpxchg64
+#  define cmpxchg64(...)			__atomic_op_fence(cmpxchg64, __VA_ARGS__)
+# endif
+#endif
+
+/* xchg_relaxed() et al: */
 
-/* xchg_relaxed */
 #ifndef xchg_relaxed
-#define  xchg_relaxed			xchg
-#define  xchg_acquire			xchg
-#define  xchg_release			xchg
-
-#else /* xchg_relaxed */
-
-#ifndef xchg_acquire
-#define  xchg_acquire(...)		__atomic_op_acquire(xchg, __VA_ARGS__)
-#endif
-
-#ifndef xchg_release
-#define  xchg_release(...)		__atomic_op_release(xchg, __VA_ARGS__)
+# define xchg_relaxed				xchg
+# define xchg_acquire				xchg
+# define xchg_release				xchg
+#else
+# ifndef xchg_acquire
+#  define xchg_acquire(...)			__atomic_op_acquire(xchg, __VA_ARGS__)
+# endif
+# ifndef xchg_release
+#  define xchg_release(...)			__atomic_op_release(xchg, __VA_ARGS__)
+# endif
+# ifndef xchg
+#  define xchg(...)				__atomic_op_fence(xchg, __VA_ARGS__)
+# endif
 #endif
 
-#ifndef xchg
-#define  xchg(...)			__atomic_op_fence(xchg, __VA_ARGS__)
-#endif
-#endif /* xchg_relaxed */
-
 /**
  * atomic_add_unless - add unless the number is already a given value
  * @v: pointer of type atomic_t
@@ -541,7 +438,7 @@  static inline int atomic_add_unless(atomic_t *v, int a, int u)
  * Returns non-zero if @v was non-zero, and zero otherwise.
  */
 #ifndef atomic_inc_not_zero
-#define atomic_inc_not_zero(v)		atomic_add_unless((v), 1, 0)
+# define atomic_inc_not_zero(v)			atomic_add_unless((v), 1, 0)
 #endif
 
 #ifndef atomic_andnot
@@ -607,6 +504,7 @@  static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
 static inline int atomic_inc_unless_negative(atomic_t *p)
 {
 	int v, v1;
+
 	for (v = 0; v >= 0; v = v1) {
 		v1 = atomic_cmpxchg(p, v, v + 1);
 		if (likely(v1 == v))
@@ -620,6 +518,7 @@  static inline int atomic_inc_unless_negative(atomic_t *p)
 static inline int atomic_dec_unless_positive(atomic_t *p)
 {
 	int v, v1;
+
 	for (v = 0; v <= 0; v = v1) {
 		v1 = atomic_cmpxchg(p, v, v - 1);
 		if (likely(v1 == v))
@@ -640,6 +539,7 @@  static inline int atomic_dec_unless_positive(atomic_t *p)
 static inline int atomic_dec_if_positive(atomic_t *v)
 {
 	int c, old, dec;
+
 	c = atomic_read(v);
 	for (;;) {
 		dec = c - 1;
@@ -654,400 +554,311 @@  static inline int atomic_dec_if_positive(atomic_t *v)
 }
 #endif
 
-#define atomic_cond_read_relaxed(v, c)	smp_cond_load_relaxed(&(v)->counter, (c))
-#define atomic_cond_read_acquire(v, c)	smp_cond_load_acquire(&(v)->counter, (c))
+#define atomic_cond_read_relaxed(v, c)		smp_cond_load_relaxed(&(v)->counter, (c))
+#define atomic_cond_read_acquire(v, c)		smp_cond_load_acquire(&(v)->counter, (c))
 
 #ifdef CONFIG_GENERIC_ATOMIC64
 #include <asm-generic/atomic64.h>
 #endif
 
 #ifndef atomic64_read_acquire
-#define  atomic64_read_acquire(v)	smp_load_acquire(&(v)->counter)
+# define atomic64_read_acquire(v)		smp_load_acquire(&(v)->counter)
 #endif
 
 #ifndef atomic64_set_release
-#define  atomic64_set_release(v, i)	smp_store_release(&(v)->counter, (i))
-#endif
-
-/* atomic64_add_return_relaxed */
-#ifndef atomic64_add_return_relaxed
-#define  atomic64_add_return_relaxed	atomic64_add_return
-#define  atomic64_add_return_acquire	atomic64_add_return
-#define  atomic64_add_return_release	atomic64_add_return
-
-#else /* atomic64_add_return_relaxed */
-
-#ifndef atomic64_add_return_acquire
-#define  atomic64_add_return_acquire(...)				\
-	__atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
+# define atomic64_set_release(v, i)		smp_store_release(&(v)->counter, (i))
 #endif
 
-#ifndef atomic64_add_return_release
-#define  atomic64_add_return_release(...)				\
-	__atomic_op_release(atomic64_add_return, __VA_ARGS__)
-#endif
+/* atomic64_add_return_relaxed() et al: */
 
-#ifndef atomic64_add_return
-#define  atomic64_add_return(...)					\
-	__atomic_op_fence(atomic64_add_return, __VA_ARGS__)
-#endif
-#endif /* atomic64_add_return_relaxed */
+#ifndef atomic64_add_return_relaxed
+# define atomic64_add_return_relaxed		atomic64_add_return
+# define atomic64_add_return_acquire		atomic64_add_return
+# define atomic64_add_return_release		atomic64_add_return
+#else
+# ifndef atomic64_add_return_acquire
+#  define atomic64_add_return_acquire(...)	__atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
+# endif
+# ifndef atomic64_add_return_release
+#  define atomic64_add_return_release(...)	__atomic_op_release(atomic64_add_return, __VA_ARGS__)
+# endif
+# ifndef atomic64_add_return
+#  define atomic64_add_return(...)		__atomic_op_fence(atomic64_add_return, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic64_inc_return_relaxed() et al: */
 
-/* atomic64_inc_return_relaxed */
 #ifndef atomic64_inc_return_relaxed
-#define  atomic64_inc_return_relaxed	atomic64_inc_return
-#define  atomic64_inc_return_acquire	atomic64_inc_return
-#define  atomic64_inc_return_release	atomic64_inc_return
-
-#else /* atomic64_inc_return_relaxed */
-
-#ifndef atomic64_inc_return_acquire
-#define  atomic64_inc_return_acquire(...)				\
-	__atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_inc_return_release
-#define  atomic64_inc_return_release(...)				\
-	__atomic_op_release(atomic64_inc_return, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_inc_return
-#define  atomic64_inc_return(...)					\
-	__atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
-#endif
-#endif /* atomic64_inc_return_relaxed */
-
+# define atomic64_inc_return_relaxed		atomic64_inc_return
+# define atomic64_inc_return_acquire		atomic64_inc_return
+# define atomic64_inc_return_release		atomic64_inc_return
+#else
+# ifndef atomic64_inc_return_acquire
+#  define atomic64_inc_return_acquire(...)	__atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
+# endif
+# ifndef atomic64_inc_return_release
+#  define atomic64_inc_return_release(...)	__atomic_op_release(atomic64_inc_return, __VA_ARGS__)
+# endif
+# ifndef atomic64_inc_return
+#  define atomic64_inc_return(...)		__atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic64_sub_return_relaxed() et al: */
 
-/* atomic64_sub_return_relaxed */
 #ifndef atomic64_sub_return_relaxed
-#define  atomic64_sub_return_relaxed	atomic64_sub_return
-#define  atomic64_sub_return_acquire	atomic64_sub_return
-#define  atomic64_sub_return_release	atomic64_sub_return
+# define atomic64_sub_return_relaxed		atomic64_sub_return
+# define atomic64_sub_return_acquire		atomic64_sub_return
+# define atomic64_sub_return_release		atomic64_sub_return
+#else
+# ifndef atomic64_sub_return_acquire
+#  define atomic64_sub_return_acquire(...)	__atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
+# endif
+# ifndef atomic64_sub_return_release
+#  define atomic64_sub_return_release(...)	__atomic_op_release(atomic64_sub_return, __VA_ARGS__)
+# endif
+# ifndef atomic64_sub_return
+#  define atomic64_sub_return(...)		__atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic64_dec_return_relaxed() et al: */
 
-#else /* atomic64_sub_return_relaxed */
-
-#ifndef atomic64_sub_return_acquire
-#define  atomic64_sub_return_acquire(...)				\
-	__atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_sub_return_release
-#define  atomic64_sub_return_release(...)				\
-	__atomic_op_release(atomic64_sub_return, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_sub_return
-#define  atomic64_sub_return(...)					\
-	__atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
-#endif
-#endif /* atomic64_sub_return_relaxed */
-
-/* atomic64_dec_return_relaxed */
 #ifndef atomic64_dec_return_relaxed
-#define  atomic64_dec_return_relaxed	atomic64_dec_return
-#define  atomic64_dec_return_acquire	atomic64_dec_return
-#define  atomic64_dec_return_release	atomic64_dec_return
-
-#else /* atomic64_dec_return_relaxed */
-
-#ifndef atomic64_dec_return_acquire
-#define  atomic64_dec_return_acquire(...)				\
-	__atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_dec_return_release
-#define  atomic64_dec_return_release(...)				\
-	__atomic_op_release(atomic64_dec_return, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_dec_return
-#define  atomic64_dec_return(...)					\
-	__atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
-#endif
-#endif /* atomic64_dec_return_relaxed */
+# define atomic64_dec_return_relaxed		atomic64_dec_return
+# define atomic64_dec_return_acquire		atomic64_dec_return
+# define atomic64_dec_return_release		atomic64_dec_return
+#else
+# ifndef atomic64_dec_return_acquire
+#  define atomic64_dec_return_acquire(...)	__atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
+# endif
+# ifndef atomic64_dec_return_release
+#  define atomic64_dec_return_release(...)	__atomic_op_release(atomic64_dec_return, __VA_ARGS__)
+# endif
+# ifndef atomic64_dec_return
+#  define atomic64_dec_return(...)		__atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic64_fetch_add_relaxed() et al: */
 
-
-/* atomic64_fetch_add_relaxed */
 #ifndef atomic64_fetch_add_relaxed
-#define atomic64_fetch_add_relaxed	atomic64_fetch_add
-#define atomic64_fetch_add_acquire	atomic64_fetch_add
-#define atomic64_fetch_add_release	atomic64_fetch_add
-
-#else /* atomic64_fetch_add_relaxed */
-
-#ifndef atomic64_fetch_add_acquire
-#define atomic64_fetch_add_acquire(...)					\
-	__atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__)
-#endif
+# define atomic64_fetch_add_relaxed		atomic64_fetch_add
+# define atomic64_fetch_add_acquire		atomic64_fetch_add
+# define atomic64_fetch_add_release		atomic64_fetch_add
+#else
+# ifndef atomic64_fetch_add_acquire
+#  define atomic64_fetch_add_acquire(...)	__atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_add_release
+#  define atomic64_fetch_add_release(...)	__atomic_op_release(atomic64_fetch_add, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_add
+#  define atomic64_fetch_add(...)		__atomic_op_fence(atomic64_fetch_add, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic64_fetch_inc_relaxed() et al: */
 
-#ifndef atomic64_fetch_add_release
-#define atomic64_fetch_add_release(...)					\
-	__atomic_op_release(atomic64_fetch_add, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_fetch_add
-#define atomic64_fetch_add(...)						\
-	__atomic_op_fence(atomic64_fetch_add, __VA_ARGS__)
-#endif
-#endif /* atomic64_fetch_add_relaxed */
-
-/* atomic64_fetch_inc_relaxed */
 #ifndef atomic64_fetch_inc_relaxed
+# ifndef atomic64_fetch_inc
+#  define atomic64_fetch_inc(v)			atomic64_fetch_add(1, (v))
+#  define atomic64_fetch_inc_relaxed(v)		atomic64_fetch_add_relaxed(1, (v))
+#  define atomic64_fetch_inc_acquire(v)		atomic64_fetch_add_acquire(1, (v))
+#  define atomic64_fetch_inc_release(v)		atomic64_fetch_add_release(1, (v))
+# else
+#  define atomic64_fetch_inc_relaxed		atomic64_fetch_inc
+#  define atomic64_fetch_inc_acquire		atomic64_fetch_inc
+#  define atomic64_fetch_inc_release		atomic64_fetch_inc
+# endif
+#else
+# ifndef atomic64_fetch_inc_acquire
+#  define atomic64_fetch_inc_acquire(...)	__atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_inc_release
+#  define atomic64_fetch_inc_release(...)	__atomic_op_release(atomic64_fetch_inc, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_inc
+#  define atomic64_fetch_inc(...)		__atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic64_fetch_sub_relaxed() et al: */
 
-#ifndef atomic64_fetch_inc
-#define atomic64_fetch_inc(v)		atomic64_fetch_add(1, (v))
-#define atomic64_fetch_inc_relaxed(v)	atomic64_fetch_add_relaxed(1, (v))
-#define atomic64_fetch_inc_acquire(v)	atomic64_fetch_add_acquire(1, (v))
-#define atomic64_fetch_inc_release(v)	atomic64_fetch_add_release(1, (v))
-#else /* atomic64_fetch_inc */
-#define atomic64_fetch_inc_relaxed	atomic64_fetch_inc
-#define atomic64_fetch_inc_acquire	atomic64_fetch_inc
-#define atomic64_fetch_inc_release	atomic64_fetch_inc
-#endif /* atomic64_fetch_inc */
-
-#else /* atomic64_fetch_inc_relaxed */
-
-#ifndef atomic64_fetch_inc_acquire
-#define atomic64_fetch_inc_acquire(...)					\
-	__atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_fetch_inc_release
-#define atomic64_fetch_inc_release(...)					\
-	__atomic_op_release(atomic64_fetch_inc, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_fetch_inc
-#define atomic64_fetch_inc(...)						\
-	__atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__)
-#endif
-#endif /* atomic64_fetch_inc_relaxed */
-
-/* atomic64_fetch_sub_relaxed */
 #ifndef atomic64_fetch_sub_relaxed
-#define atomic64_fetch_sub_relaxed	atomic64_fetch_sub
-#define atomic64_fetch_sub_acquire	atomic64_fetch_sub
-#define atomic64_fetch_sub_release	atomic64_fetch_sub
-
-#else /* atomic64_fetch_sub_relaxed */
-
-#ifndef atomic64_fetch_sub_acquire
-#define atomic64_fetch_sub_acquire(...)					\
-	__atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_fetch_sub_release
-#define atomic64_fetch_sub_release(...)					\
-	__atomic_op_release(atomic64_fetch_sub, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_fetch_sub
-#define atomic64_fetch_sub(...)						\
-	__atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__)
-#endif
-#endif /* atomic64_fetch_sub_relaxed */
+# define atomic64_fetch_sub_relaxed		atomic64_fetch_sub
+# define atomic64_fetch_sub_acquire		atomic64_fetch_sub
+# define atomic64_fetch_sub_release		atomic64_fetch_sub
+#else
+# ifndef atomic64_fetch_sub_acquire
+#  define atomic64_fetch_sub_acquire(...)	__atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_sub_release
+#  define atomic64_fetch_sub_release(...)	__atomic_op_release(atomic64_fetch_sub, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_sub
+#  define atomic64_fetch_sub(...)		__atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic64_fetch_dec_relaxed() et al: */
 
-/* atomic64_fetch_dec_relaxed */
 #ifndef atomic64_fetch_dec_relaxed
+# ifndef atomic64_fetch_dec
+#  define atomic64_fetch_dec(v)			atomic64_fetch_sub(1, (v))
+#  define atomic64_fetch_dec_relaxed(v)		atomic64_fetch_sub_relaxed(1, (v))
+#  define atomic64_fetch_dec_acquire(v)		atomic64_fetch_sub_acquire(1, (v))
+#  define atomic64_fetch_dec_release(v)		atomic64_fetch_sub_release(1, (v))
+# else
+#  define atomic64_fetch_dec_relaxed		atomic64_fetch_dec
+#  define atomic64_fetch_dec_acquire		atomic64_fetch_dec
+#  define atomic64_fetch_dec_release		atomic64_fetch_dec
+# endif
+#else
+# ifndef atomic64_fetch_dec_acquire
+#  define atomic64_fetch_dec_acquire(...)	__atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_dec_release
+#  define atomic64_fetch_dec_release(...)	__atomic_op_release(atomic64_fetch_dec, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_dec
+#  define atomic64_fetch_dec(...)		__atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic64_fetch_or_relaxed() et al: */
 
-#ifndef atomic64_fetch_dec
-#define atomic64_fetch_dec(v)		atomic64_fetch_sub(1, (v))
-#define atomic64_fetch_dec_relaxed(v)	atomic64_fetch_sub_relaxed(1, (v))
-#define atomic64_fetch_dec_acquire(v)	atomic64_fetch_sub_acquire(1, (v))
-#define atomic64_fetch_dec_release(v)	atomic64_fetch_sub_release(1, (v))
-#else /* atomic64_fetch_dec */
-#define atomic64_fetch_dec_relaxed	atomic64_fetch_dec
-#define atomic64_fetch_dec_acquire	atomic64_fetch_dec
-#define atomic64_fetch_dec_release	atomic64_fetch_dec
-#endif /* atomic64_fetch_dec */
-
-#else /* atomic64_fetch_dec_relaxed */
-
-#ifndef atomic64_fetch_dec_acquire
-#define atomic64_fetch_dec_acquire(...)					\
-	__atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_fetch_dec_release
-#define atomic64_fetch_dec_release(...)					\
-	__atomic_op_release(atomic64_fetch_dec, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_fetch_dec
-#define atomic64_fetch_dec(...)						\
-	__atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__)
-#endif
-#endif /* atomic64_fetch_dec_relaxed */
-
-/* atomic64_fetch_or_relaxed */
 #ifndef atomic64_fetch_or_relaxed
-#define atomic64_fetch_or_relaxed	atomic64_fetch_or
-#define atomic64_fetch_or_acquire	atomic64_fetch_or
-#define atomic64_fetch_or_release	atomic64_fetch_or
-
-#else /* atomic64_fetch_or_relaxed */
-
-#ifndef atomic64_fetch_or_acquire
-#define atomic64_fetch_or_acquire(...)					\
-	__atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__)
+# define atomic64_fetch_or_relaxed		atomic64_fetch_or
+# define atomic64_fetch_or_acquire		atomic64_fetch_or
+# define atomic64_fetch_or_release		atomic64_fetch_or
+#else
+# ifndef atomic64_fetch_or_acquire
+#  define atomic64_fetch_or_acquire(...)	__atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_or_release
+#  define atomic64_fetch_or_release(...)	__atomic_op_release(atomic64_fetch_or, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_or
+#  define atomic64_fetch_or(...)		__atomic_op_fence(atomic64_fetch_or, __VA_ARGS__)
+# endif
 #endif
 
-#ifndef atomic64_fetch_or_release
-#define atomic64_fetch_or_release(...)					\
-	__atomic_op_release(atomic64_fetch_or, __VA_ARGS__)
-#endif
 
-#ifndef atomic64_fetch_or
-#define atomic64_fetch_or(...)						\
-	__atomic_op_fence(atomic64_fetch_or, __VA_ARGS__)
-#endif
-#endif /* atomic64_fetch_or_relaxed */
+/* atomic64_fetch_and_relaxed() et al: */
 
-/* atomic64_fetch_and_relaxed */
 #ifndef atomic64_fetch_and_relaxed
-#define atomic64_fetch_and_relaxed	atomic64_fetch_and
-#define atomic64_fetch_and_acquire	atomic64_fetch_and
-#define atomic64_fetch_and_release	atomic64_fetch_and
-
-#else /* atomic64_fetch_and_relaxed */
-
-#ifndef atomic64_fetch_and_acquire
-#define atomic64_fetch_and_acquire(...)					\
-	__atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__)
+# define atomic64_fetch_and_relaxed		atomic64_fetch_and
+# define atomic64_fetch_and_acquire		atomic64_fetch_and
+# define atomic64_fetch_and_release		atomic64_fetch_and
+#else
+# ifndef atomic64_fetch_and_acquire
+#  define atomic64_fetch_and_acquire(...)	__atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_and_release
+#  define atomic64_fetch_and_release(...)	__atomic_op_release(atomic64_fetch_and, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_and
+#  define atomic64_fetch_and(...)		__atomic_op_fence(atomic64_fetch_and, __VA_ARGS__)
+# endif
 #endif
 
-#ifndef atomic64_fetch_and_release
-#define atomic64_fetch_and_release(...)					\
-	__atomic_op_release(atomic64_fetch_and, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_fetch_and
-#define atomic64_fetch_and(...)						\
-	__atomic_op_fence(atomic64_fetch_and, __VA_ARGS__)
-#endif
-#endif /* atomic64_fetch_and_relaxed */
-
 #ifdef atomic64_andnot
-/* atomic64_fetch_andnot_relaxed */
-#ifndef atomic64_fetch_andnot_relaxed
-#define atomic64_fetch_andnot_relaxed	atomic64_fetch_andnot
-#define atomic64_fetch_andnot_acquire	atomic64_fetch_andnot
-#define atomic64_fetch_andnot_release	atomic64_fetch_andnot
-
-#else /* atomic64_fetch_andnot_relaxed */
 
-#ifndef atomic64_fetch_andnot_acquire
-#define atomic64_fetch_andnot_acquire(...)					\
-	__atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__)
-#endif
+/* atomic64_fetch_andnot_relaxed() et al: */
 
-#ifndef atomic64_fetch_andnot_release
-#define atomic64_fetch_andnot_release(...)					\
-	__atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__)
+#ifndef atomic64_fetch_andnot_relaxed
+# define atomic64_fetch_andnot_relaxed		atomic64_fetch_andnot
+# define atomic64_fetch_andnot_acquire		atomic64_fetch_andnot
+# define atomic64_fetch_andnot_release		atomic64_fetch_andnot
+#else
+# ifndef atomic64_fetch_andnot_acquire
+#  define atomic64_fetch_andnot_acquire(...)	__atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_andnot_release
+#  define atomic64_fetch_andnot_release(...)	__atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_andnot
+#  define atomic64_fetch_andnot(...)		__atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__)
+# endif
 #endif
 
-#ifndef atomic64_fetch_andnot
-#define atomic64_fetch_andnot(...)						\
-	__atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__)
-#endif
-#endif /* atomic64_fetch_andnot_relaxed */
 #endif /* atomic64_andnot */
 
-/* atomic64_fetch_xor_relaxed */
-#ifndef atomic64_fetch_xor_relaxed
-#define atomic64_fetch_xor_relaxed	atomic64_fetch_xor
-#define atomic64_fetch_xor_acquire	atomic64_fetch_xor
-#define atomic64_fetch_xor_release	atomic64_fetch_xor
-
-#else /* atomic64_fetch_xor_relaxed */
+/* atomic64_fetch_xor_relaxed() et al: */
 
-#ifndef atomic64_fetch_xor_acquire
-#define atomic64_fetch_xor_acquire(...)					\
-	__atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_fetch_xor_release
-#define atomic64_fetch_xor_release(...)					\
-	__atomic_op_release(atomic64_fetch_xor, __VA_ARGS__)
+#ifndef atomic64_fetch_xor_relaxed
+# define atomic64_fetch_xor_relaxed		atomic64_fetch_xor
+# define atomic64_fetch_xor_acquire		atomic64_fetch_xor
+# define atomic64_fetch_xor_release		atomic64_fetch_xor
+#else
+# ifndef atomic64_fetch_xor_acquire
+#  define atomic64_fetch_xor_acquire(...)	__atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_xor_release
+#  define atomic64_fetch_xor_release(...)	__atomic_op_release(atomic64_fetch_xor, __VA_ARGS__)
+# endif
+# ifndef atomic64_fetch_xor
+#  define atomic64_fetch_xor(...)		__atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__)
 #endif
-
-#ifndef atomic64_fetch_xor
-#define atomic64_fetch_xor(...)						\
-	__atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__)
 #endif
-#endif /* atomic64_fetch_xor_relaxed */
 
+/* atomic64_xchg_relaxed() et al: */
 
-/* atomic64_xchg_relaxed */
 #ifndef atomic64_xchg_relaxed
-#define  atomic64_xchg_relaxed		atomic64_xchg
-#define  atomic64_xchg_acquire		atomic64_xchg
-#define  atomic64_xchg_release		atomic64_xchg
-
-#else /* atomic64_xchg_relaxed */
-
-#ifndef atomic64_xchg_acquire
-#define  atomic64_xchg_acquire(...)					\
-	__atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
-#endif
+# define atomic64_xchg_relaxed			atomic64_xchg
+# define atomic64_xchg_acquire			atomic64_xchg
+# define atomic64_xchg_release			atomic64_xchg
+#else
+# ifndef atomic64_xchg_acquire
+#  define atomic64_xchg_acquire(...)		__atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
+# endif
+# ifndef atomic64_xchg_release
+#  define atomic64_xchg_release(...)		__atomic_op_release(atomic64_xchg, __VA_ARGS__)
+# endif
+# ifndef atomic64_xchg
+#  define atomic64_xchg(...)			__atomic_op_fence(atomic64_xchg, __VA_ARGS__)
+# endif
+#endif
+
+/* atomic64_cmpxchg_relaxed() et al: */
 
-#ifndef atomic64_xchg_release
-#define  atomic64_xchg_release(...)					\
-	__atomic_op_release(atomic64_xchg, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_xchg
-#define  atomic64_xchg(...)						\
-	__atomic_op_fence(atomic64_xchg, __VA_ARGS__)
-#endif
-#endif /* atomic64_xchg_relaxed */
-
-/* atomic64_cmpxchg_relaxed */
 #ifndef atomic64_cmpxchg_relaxed
-#define  atomic64_cmpxchg_relaxed	atomic64_cmpxchg
-#define  atomic64_cmpxchg_acquire	atomic64_cmpxchg
-#define  atomic64_cmpxchg_release	atomic64_cmpxchg
-
-#else /* atomic64_cmpxchg_relaxed */
-
-#ifndef atomic64_cmpxchg_acquire
-#define  atomic64_cmpxchg_acquire(...)					\
-	__atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_cmpxchg_release
-#define  atomic64_cmpxchg_release(...)					\
-	__atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
-#endif
-
-#ifndef atomic64_cmpxchg
-#define  atomic64_cmpxchg(...)						\
-	__atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
+# define atomic64_cmpxchg_relaxed		atomic64_cmpxchg
+# define atomic64_cmpxchg_acquire		atomic64_cmpxchg
+# define atomic64_cmpxchg_release		atomic64_cmpxchg
+#else
+# ifndef atomic64_cmpxchg_acquire
+#  define atomic64_cmpxchg_acquire(...)		__atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
+# endif
+# ifndef atomic64_cmpxchg_release
+#  define atomic64_cmpxchg_release(...)		__atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
+# endif
+# ifndef atomic64_cmpxchg
+#  define atomic64_cmpxchg(...)			__atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
+# endif
 #endif
-#endif /* atomic64_cmpxchg_relaxed */
 
 #ifndef atomic64_try_cmpxchg
-
-#define __atomic64_try_cmpxchg(type, _p, _po, _n)			\
-({									\
+# define __atomic64_try_cmpxchg(type, _p, _po, _n)			\
+  ({									\
 	typeof(_po) __po = (_po);					\
 	typeof(*(_po)) __r, __o = *__po;				\
 	__r = atomic64_cmpxchg##type((_p), __o, (_n));			\
 	if (unlikely(__r != __o))					\
 		*__po = __r;						\
 	likely(__r == __o);						\
-})
-
-#define atomic64_try_cmpxchg(_p, _po, _n)		__atomic64_try_cmpxchg(, _p, _po, _n)
-#define atomic64_try_cmpxchg_relaxed(_p, _po, _n)	__atomic64_try_cmpxchg(_relaxed, _p, _po, _n)
-#define atomic64_try_cmpxchg_acquire(_p, _po, _n)	__atomic64_try_cmpxchg(_acquire, _p, _po, _n)
-#define atomic64_try_cmpxchg_release(_p, _po, _n)	__atomic64_try_cmpxchg(_release, _p, _po, _n)
-
-#else /* atomic64_try_cmpxchg */
-#define atomic64_try_cmpxchg_relaxed	atomic64_try_cmpxchg
-#define atomic64_try_cmpxchg_acquire	atomic64_try_cmpxchg
-#define atomic64_try_cmpxchg_release	atomic64_try_cmpxchg
-#endif /* atomic64_try_cmpxchg */
+  })
+# define atomic64_try_cmpxchg(_p, _po, _n)	   __atomic64_try_cmpxchg(, _p, _po, _n)
+# define atomic64_try_cmpxchg_relaxed(_p, _po, _n) __atomic64_try_cmpxchg(_relaxed, _p, _po, _n)
+# define atomic64_try_cmpxchg_acquire(_p, _po, _n) __atomic64_try_cmpxchg(_acquire, _p, _po, _n)
+# define atomic64_try_cmpxchg_release(_p, _po, _n) __atomic64_try_cmpxchg(_release, _p, _po, _n)
+#else
+# define atomic64_try_cmpxchg_relaxed		atomic64_try_cmpxchg
+# define atomic64_try_cmpxchg_acquire		atomic64_try_cmpxchg
+# define atomic64_try_cmpxchg_release		atomic64_try_cmpxchg
+#endif
 
 #ifndef atomic64_andnot
 static inline void atomic64_andnot(long long i, atomic64_t *v)