[PATCH] Optimize PDA accesses slightly

Based on a idea by Jeremy Fitzhardinge:

Replace the volatiles and memory clobbers in the PDA access with
telling gcc about access to a proxy PDA structure that doesn't
actually exist. But the dummy accesses give a defined ordering for
read/write accesses.

Also add some memory barriers to the early GS initialization to
make sure no PDA access is moved before it.

Advantage is some .text savings (probably most from better
code for accessing "current"):

   text    data     bss     dec     hex filename
4845647 1223688  615864 6685199  66020f vmlinux
4837780 1223688  615864 6677332  65e354 vmlinux-pda

1.2% smaller code

Cc:  Jeremy Fitzhardinge <jeremy@goop.org>
Signed-off-by: Andi Kleen <ak@suse.de>

diff --git a/arch/x86_64/kernel/setup64.c b/arch/x86_64/kernel/setup64.c
index e85cfbb49b6382aff5cdd56a6a31d21032a07869..491361752c701e13e9daf8767463f3711f5a6455 100644 (file)
--- a/arch/x86_64/kernel/setup64.c
+++ b/arch/x86_64/kernel/setup64.c
@@ -121,7 +121,10 @@ void pda_init(int cpu)
 
        /* Setup up data that may be needed in __get_free_pages early */
        asm volatile("movl %0,%%fs ; movl %0,%%gs" :: "r" (0)); 
+       /* Memory clobbers used to order PDA accessed */
+       mb();
        wrmsrl(MSR_GS_BASE, pda);
+       mb();
 
        pda->cpunumber = cpu; 
        pda->irqcount = -1;

diff --git a/include/asm-x86_64/pda.h b/include/asm-x86_64/pda.h
index b47c3df9ed1dd72d792ceca2e6f8f5d6338b7556..55e21da96e7a239534075b87c556f7ca50e52999 100644 (file)
--- a/include/asm-x86_64/pda.h
+++ b/include/asm-x86_64/pda.h
@@ -36,40 +36,43 @@ extern struct x8664_pda boot_cpu_pda[];
  * There is no fast way to get the base address of the PDA, all the accesses
  * have to mention %fs/%gs.  So it needs to be done this Torvaldian way.
  */ 
-#define sizeof_field(type,field)  (sizeof(((type *)0)->field))
-#define typeof_field(type,field)  typeof(((type *)0)->field)
-
 extern void __bad_pda_field(void);
 
+/* proxy_pda doesn't actually exist, but tell gcc it is accessed
+   for all PDA accesses so it gets read/write dependencies right. */
+extern struct x8664_pda _proxy_pda;
+
 #define pda_offset(field) offsetof(struct x8664_pda, field)
 
 #define pda_to_op(op,field,val) do { \
-       typedef typeof_field(struct x8664_pda, field) T__; \
-       switch (sizeof_field(struct x8664_pda, field)) {                \
+       typedef typeof(_proxy_pda.field) T__; \
+       switch (sizeof(_proxy_pda.field)) {             \
 case 2: \
-asm volatile(op "w %0,%%gs:%P1"::"ri" ((T__)val),"i"(pda_offset(field)):"memory"); break; \
+asm(op "w %1,%%gs:%P2" : "+m" (_proxy_pda.field) : \
+       "ri" ((T__)val),"i"(pda_offset(field))); break; \
 case 4: \
-asm volatile(op "l %0,%%gs:%P1"::"ri" ((T__)val),"i"(pda_offset(field)):"memory"); break; \
+asm(op "l %1,%%gs:%P2" : "+m" (_proxy_pda.field) : \
+       "ri" ((T__)val),"i"(pda_offset(field))); break; \
 case 8: \
-asm volatile(op "q %0,%%gs:%P1"::"ri" ((T__)val),"i"(pda_offset(field)):"memory"); break; \
-       default: __bad_pda_field();                                     \
+asm(op "q %1,%%gs:%P2": "+m" (_proxy_pda.field) : \
+        "ri" ((T__)val),"i"(pda_offset(field))); break; \
+default: __bad_pda_field();                                    \
        } \
        } while (0)
 
-/* 
- * AK: PDA read accesses should be neither volatile nor have an memory clobber.
- * Unfortunately removing them causes all hell to break lose currently.
- */
 #define pda_from_op(op,field) ({ \
-       typeof_field(struct x8664_pda, field) ret__; \
-       switch (sizeof_field(struct x8664_pda, field)) {                \
+       typeof(_proxy_pda.field) ret__; \
+       switch (sizeof(_proxy_pda.field)) {             \
 case 2: \
-asm volatile(op "w %%gs:%P1,%0":"=r" (ret__):"i"(pda_offset(field)):"memory"); break;\
+asm(op "w %%gs:%P1,%0":"=r" (ret__):\
+       "i" (pda_offset(field)), "m" (_proxy_pda.field)); break;\
 case 4: \
-asm volatile(op "l %%gs:%P1,%0":"=r" (ret__):"i"(pda_offset(field)):"memory"); break;\
+asm(op "l %%gs:%P1,%0":"=r" (ret__):\
+       "i" (pda_offset(field)), "m" (_proxy_pda.field)); break;\
 case 8: \
-asm volatile(op "q %%gs:%P1,%0":"=r" (ret__):"i"(pda_offset(field)):"memory"); break;\
-       default: __bad_pda_field();                                     \
+asm(op "q %%gs:%P1,%0":"=r" (ret__):\
+       "i" (pda_offset(field)), "m" (_proxy_pda.field)); break;\
+default: __bad_pda_field();                                    \
        } \
        ret__; })