function-graph/x86: Replace unbalanced ret with jmp

The function graph tracer replaces the return address with a hook
to trace the exit of the function call. This hook will finish by
returning to the real location the function should return to.

But the current implementation uses a ret to jump to the real
return location. This causes a imbalance between calls and ret.
That is the original function does a call, the ret goes to the
handler and then the handler does a ret without a matching call.

Although the function graph tracer itself still breaks the branch
predictor by replacing the original ret, by using a second ret and
causing an imbalance, it breaks the predictor even more.

This patch replaces the ret with a jmp to keep the calls and ret
balanced. I tested this on one box and it showed a 1.7% increase in
performance. Another box only showed a small 0.3% increase. But no
box that I tested this on showed a decrease in performance by
making this change.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Acked-by: Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
LKML-Reference: <20091013203425.042034383@goodmis.org>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
index c097e7d607c61e27c4e6df0701f044f4af067ff7..7d52e9da5e0cc0d1942b6cd9624f83f504491e77 100644 (file)
--- a/arch/x86/kernel/entry_32.S
+++ b/arch/x86/kernel/entry_32.S
@@ -1185,17 +1185,14 @@ END(ftrace_graph_caller)
 
 .globl return_to_handler
 return_to_handler:
-       pushl $0
        pushl %eax
-       pushl %ecx
        pushl %edx
        movl %ebp, %eax
        call ftrace_return_to_handler
-       movl %eax, 0xc(%esp)
+       movl %eax, %ecx
        popl %edx
-       popl %ecx
        popl %eax
-       ret
+       jmp *%ecx
 #endif
 
 .section .rodata,"a"

diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index b5c061f8f358489fee828e48b836d18c3bd6da79..bd5bbddddf91eab956106bff572e03a0ba86b9f9 100644 (file)
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -155,11 +155,11 @@ GLOBAL(return_to_handler)
 
        call ftrace_return_to_handler
 
-       movq %rax, 16(%rsp)
+       movq %rax, %rdi
        movq 8(%rsp), %rdx
        movq (%rsp), %rax
-       addq $16, %rsp
-       retq
+       addq $24, %rsp
+       jmp *%rdi
 #endif