x86: atomic64: Improve atomic64_xchg()

Remove the read-first logic from atomic64_xchg() and simplify
the loop.

This function was the last user of __atomic64_read() - remove it.

Also, change the 'real_val' assumption from the somewhat quirky
1ULL << 32 value to the (just as arbitrary, but simpler) value
of 0.

Reported-by: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Eric Dumazet <eric.dumazet@gmail.com>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Arnaldo Carvalho de Melo <acme@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: David Howells <dhowells@redhat.com>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Arnd Bergmann <arnd@arndb.de>
LKML-Reference: <tip-05118ab8859492ac9ddda0154cf90e37b0a4a0b0@git.kernel.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/include/asm/atomic_32.h b/arch/x86/include/asm/atomic_32.h
index aa045deb2e757de280e117f7c18465971f87c152..d7c8849b8c67462ac046843dcce4c4d215308d59 100644 (file)
--- a/arch/x86/include/asm/atomic_32.h
+++ b/arch/x86/include/asm/atomic_32.h
@@ -268,15 +268,6 @@ typedef struct {
 
 #define ATOMIC64_INIT(val)     { (val) }
 
-/**
- * atomic64_read - read atomic64 variable
- * @ptr: pointer of type atomic64_t
- *
- * Atomically reads the value of @v.
- * Doesn't imply a read memory barrier.
- */
-#define __atomic64_read(ptr)           ((ptr)->counter)
-
 extern u64 atomic64_cmpxchg(atomic64_t *ptr, u64 old_val, u64 new_val);
 
 /**

diff --git a/arch/x86/lib/atomic64_32.c b/arch/x86/lib/atomic64_32.c
index 6722a092e407e8914823909f17ff1359caac71da..a804f96e90e2fd01504eb267c5dceaa3913eecf0 100644 (file)
--- a/arch/x86/lib/atomic64_32.c
+++ b/arch/x86/lib/atomic64_32.c
@@ -33,14 +33,23 @@ EXPORT_SYMBOL(atomic64_cmpxchg);
  * Atomically xchgs the value of @ptr to @new_val and returns
  * the old value.
  */
-
 u64 atomic64_xchg(atomic64_t *ptr, u64 new_val)
 {
-       u64 old_val;
+       /*
+        * Try first with a (possibly incorrect) assumption about
+        * what we have there. We'll do two loops most likely,
+        * but we'll get an ownership MESI transaction straight away
+        * instead of a read transaction followed by a
+        * flush-for-ownership transaction:
+        */
+       u64 old_val, real_val = 0;
 
        do {
-               old_val = __atomic64_read(ptr);
-       } while (atomic64_cmpxchg(ptr, old_val, new_val) != old_val);
+               old_val = real_val;
+
+               real_val = atomic64_cmpxchg(ptr, old_val, new_val);
+
+       } while (real_val != old_val);
 
        return old_val;
 }
@@ -91,13 +100,13 @@ EXPORT_SYMBOL(atomic64_read);
 noinline u64 atomic64_add_return(u64 delta, atomic64_t *ptr)
 {
        /*
-        * Try first with a (probably incorrect) assumption about
+        * Try first with a (possibly incorrect) assumption about
         * what we have there. We'll do two loops most likely,
         * but we'll get an ownership MESI transaction straight away
         * instead of a read transaction followed by a
         * flush-for-ownership transaction:
         */
-       u64 old_val, new_val, real_val = 1ULL << 32;
+       u64 old_val, new_val, real_val = 0;
 
        do {
                old_val = real_val;