[PATCH] powerpc: Fix runlatch performance issues

The runlatch SPR can take a lot of time to write. My original runlatch
code would set it on every exception entry even though most of the time
this was not required. It would also continually set it in the idle
loop, which is an issue on an SMT capable processor.

Now we cache the runlatch value in a threadinfo bit, and only check for
it in decrementer and hardware interrupt exceptions as well as the idle
loop. Boot on POWER3, POWER5 and iseries, and compile tested on pmac32.

Signed-off-by: Anton Blanchard <anton@samba.org>
Signed-off-by: Paul Mackerras <paulus@samba.org>

diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S
index 2b03a09fe5e9d5022fae5c99d11b5710eee92f13..bb845eed0e9868b36350fdd754fc34160f5789e6 100644 (file)
--- a/arch/powerpc/kernel/head_64.S
+++ b/arch/powerpc/kernel/head_64.S
@@ -319,7 +319,6 @@ exception_marker:
 label##_pSeries:                                       \
        HMT_MEDIUM;                                     \
        mtspr   SPRN_SPRG1,r13;         /* save r13 */  \
-       RUNLATCH_ON(r13);                               \
        EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common)
 
 #define STD_EXCEPTION_ISERIES(n, label, area)          \
@@ -327,7 +326,6 @@ label##_pSeries:                                    \
 label##_iSeries:                                       \
        HMT_MEDIUM;                                     \
        mtspr   SPRN_SPRG1,r13;         /* save r13 */  \
-       RUNLATCH_ON(r13);                               \
        EXCEPTION_PROLOG_ISERIES_1(area);               \
        EXCEPTION_PROLOG_ISERIES_2;                     \
        b       label##_common
@@ -337,7 +335,6 @@ label##_iSeries:                                    \
 label##_iSeries:                                                       \
        HMT_MEDIUM;                                                     \
        mtspr   SPRN_SPRG1,r13;         /* save r13 */                  \
-       RUNLATCH_ON(r13);                                               \
        EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN);                         \
        lbz     r10,PACAPROCENABLED(r13);                               \
        cmpwi   0,r10,0;                                                \
@@ -390,6 +387,7 @@ label##_common:                                             \
 label##_common:                                                \
        EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN);      \
        DISABLE_INTS;                                   \
+       bl      .ppc64_runlatch_on;                     \
        addi    r3,r1,STACK_FRAME_OVERHEAD;             \
        bl      hdlr;                                   \
        b       .ret_from_except_lite
@@ -407,7 +405,6 @@ __start_interrupts:
 _machine_check_pSeries:
        HMT_MEDIUM
        mtspr   SPRN_SPRG1,r13          /* save r13 */
-       RUNLATCH_ON(r13)
        EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
 
        . = 0x300
@@ -434,7 +431,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
 data_access_slb_pSeries:
        HMT_MEDIUM
        mtspr   SPRN_SPRG1,r13
-       RUNLATCH_ON(r13)
        mfspr   r13,SPRN_SPRG3          /* get paca address into r13 */
        std     r3,PACA_EXSLB+EX_R3(r13)
        mfspr   r3,SPRN_DAR
@@ -460,7 +456,6 @@ data_access_slb_pSeries:
 instruction_access_slb_pSeries:
        HMT_MEDIUM
        mtspr   SPRN_SPRG1,r13
-       RUNLATCH_ON(r13)
        mfspr   r13,SPRN_SPRG3          /* get paca address into r13 */
        std     r3,PACA_EXSLB+EX_R3(r13)
        mfspr   r3,SPRN_SRR0            /* SRR0 is faulting address */
@@ -491,7 +486,6 @@ instruction_access_slb_pSeries:
        .globl  system_call_pSeries
 system_call_pSeries:
        HMT_MEDIUM
-       RUNLATCH_ON(r9)
        mr      r9,r13
        mfmsr   r10
        mfspr   r13,SPRN_SPRG3
@@ -575,7 +569,6 @@ slb_miss_user_pseries:
 system_reset_fwnmi:
        HMT_MEDIUM
        mtspr   SPRN_SPRG1,r13          /* save r13 */
-       RUNLATCH_ON(r13)
        EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common)
 
        .globl machine_check_fwnmi
@@ -583,7 +576,6 @@ system_reset_fwnmi:
 machine_check_fwnmi:
        HMT_MEDIUM
        mtspr   SPRN_SPRG1,r13          /* save r13 */
-       RUNLATCH_ON(r13)
        EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
 
 #ifdef CONFIG_PPC_ISERIES
@@ -894,7 +886,6 @@ unrecov_fer:
        .align  7
        .globl data_access_common
 data_access_common:
-       RUNLATCH_ON(r10)                /* It wont fit in the 0x300 handler */
        mfspr   r10,SPRN_DAR
        std     r10,PACA_EXGEN+EX_DAR(r13)
        mfspr   r10,SPRN_DSISR
@@ -1042,6 +1033,7 @@ hardware_interrupt_common:
        EXCEPTION_PROLOG_COMMON(0x500, PACA_EXGEN)
 hardware_interrupt_entry:
        DISABLE_INTS
+       bl      .ppc64_runlatch_on
        addi    r3,r1,STACK_FRAME_OVERHEAD
        bl      .do_IRQ
        b       .ret_from_except_lite

diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c
index 57703994a0635033da0d1d3eefaf2a652bedb490..c225cf154bfeb0f415a4f46cd83cf169c0dfd184 100644 (file)
--- a/arch/powerpc/kernel/process.c
+++ b/arch/powerpc/kernel/process.c
@@ -888,3 +888,35 @@ void dump_stack(void)
        show_stack(current, NULL);
 }
 EXPORT_SYMBOL(dump_stack);
+
+#ifdef CONFIG_PPC64
+void ppc64_runlatch_on(void)
+{
+       unsigned long ctrl;
+
+       if (cpu_has_feature(CPU_FTR_CTRL) && !test_thread_flag(TIF_RUNLATCH)) {
+               HMT_medium();
+
+               ctrl = mfspr(SPRN_CTRLF);
+               ctrl |= CTRL_RUNLATCH;
+               mtspr(SPRN_CTRLT, ctrl);
+
+               set_thread_flag(TIF_RUNLATCH);
+       }
+}
+
+void ppc64_runlatch_off(void)
+{
+       unsigned long ctrl;
+
+       if (cpu_has_feature(CPU_FTR_CTRL) && test_thread_flag(TIF_RUNLATCH)) {
+               HMT_medium();
+
+               clear_thread_flag(TIF_RUNLATCH);
+
+               ctrl = mfspr(SPRN_CTRLF);
+               ctrl &= ~CTRL_RUNLATCH;
+               mtspr(SPRN_CTRLT, ctrl);
+       }
+}
+#endif

diff --git a/arch/powerpc/platforms/iseries/setup.c b/arch/powerpc/platforms/iseries/setup.c
index 3f8790146b00a7ae73e7f28b93f6c380b730a7d6..3ecc4a652d82e55425f6d3c3b8441266e389e316 100644 (file)
--- a/arch/powerpc/platforms/iseries/setup.c
+++ b/arch/powerpc/platforms/iseries/setup.c
@@ -648,6 +648,7 @@ static void yield_shared_processor(void)
         * here and let the timer_interrupt code sort out the actual time.
         */
        get_lppaca()->int_dword.fields.decr_int = 1;
+       ppc64_runlatch_on();
        process_iSeries_events();
 }
 

diff --git a/include/asm-powerpc/reg.h b/include/asm-powerpc/reg.h
index 12ecc9b9f2855403dc1cc1ab4de2adf9842ab2bf..72bfe3af0460fa65f1f1c232ac3e56fcd86bd236 100644 (file)
--- a/include/asm-powerpc/reg.h
+++ b/include/asm-powerpc/reg.h
@@ -615,27 +615,9 @@
 #define proc_trap()    asm volatile("trap")
 
 #ifdef CONFIG_PPC64
-static inline void ppc64_runlatch_on(void)
-{
-       unsigned long ctrl;
-
-       if (cpu_has_feature(CPU_FTR_CTRL)) {
-               ctrl = mfspr(SPRN_CTRLF);
-               ctrl |= CTRL_RUNLATCH;
-               mtspr(SPRN_CTRLT, ctrl);
-       }
-}
-
-static inline void ppc64_runlatch_off(void)
-{
-       unsigned long ctrl;
-
-       if (cpu_has_feature(CPU_FTR_CTRL)) {
-               ctrl = mfspr(SPRN_CTRLF);
-               ctrl &= ~CTRL_RUNLATCH;
-               mtspr(SPRN_CTRLT, ctrl);
-       }
-}
+
+extern void ppc64_runlatch_on(void);
+extern void ppc64_runlatch_off(void);
 
 extern unsigned long scom970_read(unsigned int address);
 extern void scom970_write(unsigned int address, unsigned long value);
@@ -645,15 +627,6 @@ extern void scom970_write(unsigned int address, unsigned long value);
 #define __get_SP()     ({unsigned long sp; \
                        asm volatile("mr %0,1": "=r" (sp)); sp;})
 
-#else /* __ASSEMBLY__ */
-
-#define RUNLATCH_ON(REG)                       \
-BEGIN_FTR_SECTION                              \
-       mfspr   (REG),SPRN_CTRLF;               \
-       ori     (REG),(REG),CTRL_RUNLATCH;      \
-       mtspr   SPRN_CTRLT,(REG);               \
-END_FTR_SECTION_IFSET(CPU_FTR_CTRL)
-
 #endif /* __ASSEMBLY__ */
 #endif /* __KERNEL__ */
 #endif /* _ASM_POWERPC_REG_H */

diff --git a/include/asm-powerpc/thread_info.h b/include/asm-powerpc/thread_info.h
index c044ec16a8791e89114e890625accd803d7bf1d5..237fc2b729745121069423ae767ae38e181f2de9 100644 (file)
--- a/include/asm-powerpc/thread_info.h
+++ b/include/asm-powerpc/thread_info.h
@@ -113,7 +113,7 @@ static inline struct thread_info *current_thread_info(void)
 #define TIF_POLLING_NRFLAG     4       /* true if poll_idle() is polling
                                           TIF_NEED_RESCHED */
 #define TIF_32BIT              5       /* 32 bit binary */
-/* #define SPARE               6 */
+#define TIF_RUNLATCH           6       /* Is the runlatch enabled? */
 #define TIF_ABI_PENDING                7       /* 32/64 bit switch needed */
 #define TIF_SYSCALL_AUDIT      8       /* syscall auditing active */
 #define TIF_SINGLESTEP         9       /* singlestepping active */
@@ -131,7 +131,7 @@ static inline struct thread_info *current_thread_info(void)
 #define _TIF_NEED_RESCHED      (1<<TIF_NEED_RESCHED)
 #define _TIF_POLLING_NRFLAG    (1<<TIF_POLLING_NRFLAG)
 #define _TIF_32BIT             (1<<TIF_32BIT)
-/* #define _SPARE              (1<<SPARE) */
+#define _TIF_RUNLATCH          (1<<TIF_RUNLATCH)
 #define _TIF_ABI_PENDING       (1<<TIF_ABI_PENDING)
 #define _TIF_SYSCALL_AUDIT     (1<<TIF_SYSCALL_AUDIT)
 #define _TIF_SINGLESTEP                (1<<TIF_SINGLESTEP)