x86: invalidate caches before going into suspend

When a CPU core is shut down, all of its caches need to be flushed
to prevent stale data from causing errors if the core is resumed.
Current Linux suspend code performs an assignment after the flush,
which can add dirty data back to the cache.  On some AMD platforms,
additional speculative reads have caused crashes on resume because
of this dirty data.

Relocate the cache flush to be the very last thing done before
halting.  Tie into an assembly line so the compile will not
reorder it.  Add some documentation explaining what is going
on and why we're doing this.

Signed-off-by: Mark Langsdorf <mark.langsdorf@amd.com>
Acked-by: Mark Borden <mark.borden@amd.com>
Acked-by: Michael Hohmuth <michael.hohmuth@amd.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 53bc653ed5ca52bab902dde3d9efbc0e1630eb6c..3b7a1ddcc0bce7eca8c989906fa377d4f0322f50 100644 (file)
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -95,7 +95,6 @@ static inline void play_dead(void)
 {
        /* This must be done before dead CPU ack */
        cpu_exit_clear();
-       wbinvd();
        mb();
        /* Ack it */
        __get_cpu_var(cpu_state) = CPU_DEAD;
@@ -104,8 +103,8 @@ static inline void play_dead(void)
         * With physical CPU hotplug, we should halt the cpu
         */
        local_irq_disable();
-       while (1)
-               halt();
+       /* mask all interrupts, flush any and all caches, and halt */
+       wbinvd_halt();
 }
 #else
 static inline void play_dead(void)

diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
index 3fb62a7d9a16250092a9827a66af062be4824513..71553b664e2af8601361a9af3413599b752a3e5c 100644 (file)
--- a/arch/x86/kernel/process_64.c
+++ b/arch/x86/kernel/process_64.c
@@ -93,14 +93,13 @@ DECLARE_PER_CPU(int, cpu_state);
 static inline void play_dead(void)
 {
        idle_task_exit();
-       wbinvd();
        mb();
        /* Ack it */
        __get_cpu_var(cpu_state) = CPU_DEAD;
 
        local_irq_disable();
-       while (1)
-               halt();
+       /* mask all interrupts, flush any and all caches, and halt */
+       wbinvd_halt();
 }
 #else
 static inline void play_dead(void)

diff --git a/include/asm-x86/processor.h b/include/asm-x86/processor.h
index 5f58da401b43696ee861377fb9aba53500af8b8e..4df3e2f6fb563f2039cbc90aa4016dce9cf9274c 100644 (file)
--- a/include/asm-x86/processor.h
+++ b/include/asm-x86/processor.h
@@ -728,6 +728,29 @@ extern unsigned long               boot_option_idle_override;
 extern unsigned long           idle_halt;
 extern unsigned long           idle_nomwait;
 
+/*
+ * on systems with caches, caches must be flashed as the absolute
+ * last instruction before going into a suspended halt.  Otherwise,
+ * dirty data can linger in the cache and become stale on resume,
+ * leading to strange errors.
+ *
+ * perform a variety of operations to guarantee that the compiler
+ * will not reorder instructions.  wbinvd itself is serializing
+ * so the processor will not reorder.
+ *
+ * Systems without cache can just go into halt.
+ */
+static inline void wbinvd_halt(void)
+{
+       mb();
+       /* check for clflush to determine if wbinvd is legal */
+       if (cpu_has_clflush)
+               asm volatile("cli; wbinvd; 1: hlt; jmp 1b" : : : "memory");
+       else
+               while (1)
+                       halt();
+}
+
 extern void enable_sep_cpu(void);
 extern int sysenter_setup(void);