x86-64: preserve large page mapping for 1st 2MB kernel txt with CONFIG_DEBUG_RODATA

In the first 2MB, kernel text is co-located with kernel static
page tables setup by head_64.S.  CONFIG_DEBUG_RODATA chops this
2MB large page mapping to small 4KB pages as we mark the kernel text as RO,
leaving the static page tables as RW.

With CONFIG_DEBUG_RODATA disabled, OLTP run on NHM-EP shows 1% improvement
with 2% reduction in system time and 1% improvement in iowait idle time.

To recover this, move the kernel static page tables to .data section, so that
we don't have to break the first 2MB of kernel text to small pages with
CONFIG_DEBUG_RODATA.

Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
LKML-Reference: <20091014220254.063193621@sbs-t61.sc.intel.com>
Signed-off-by: H. Peter Anvin <hpa@zytor.com>

diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 780cd928fcd5f9cf13cac5787450af0ec8b38b7e..b55ee4ff509fa95072bc564b38281a5f4c472dde 100644 (file)
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -262,11 +262,11 @@ ENTRY(secondary_startup_64)
        .quad   x86_64_start_kernel
        ENTRY(initial_gs)
        .quad   INIT_PER_CPU_VAR(irq_stack_union)
-       __FINITDATA
 
        ENTRY(stack_start)
        .quad  init_thread_union+THREAD_SIZE-8
        .word  0
+       __FINITDATA
 
 bad_address:
        jmp bad_address
@@ -340,6 +340,7 @@ ENTRY(name)
        i = i + 1 ;                                     \
        .endr
 
+       .data
        /*
         * This default setting generates an ident mapping at address 0x100000
         * and a mapping for the kernel that precisely maps virtual address

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index c20d30b440de4be26d82fad5716348285cb3b26d..7dafd4159ad657a83de564f44b0645ffb35c6bcb 100644 (file)
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -699,7 +699,7 @@ static int kernel_set_to_readonly;
 
 void set_kernel_text_rw(void)
 {
-       unsigned long start = PFN_ALIGN(_stext);
+       unsigned long start = PFN_ALIGN(_text);
        unsigned long end = PFN_ALIGN(__start_rodata);
 
        if (!kernel_set_to_readonly)
@@ -713,7 +713,7 @@ void set_kernel_text_rw(void)
 
 void set_kernel_text_ro(void)
 {
-       unsigned long start = PFN_ALIGN(_stext);
+       unsigned long start = PFN_ALIGN(_text);
        unsigned long end = PFN_ALIGN(__start_rodata);
 
        if (!kernel_set_to_readonly)
@@ -727,7 +727,7 @@ void set_kernel_text_ro(void)
 
 void mark_rodata_ro(void)
 {
-       unsigned long start = PFN_ALIGN(_stext), end = PFN_ALIGN(__end_rodata);
+       unsigned long start = PFN_ALIGN(_text), end = PFN_ALIGN(__end_rodata);
        unsigned long rodata_start =
                ((unsigned long)__start_rodata + PAGE_SIZE - 1) & PAGE_MASK;