xen: add mechanism to extend existing multicalls

Some Xen hypercalls accept an array of operations to work on.  In
general this is because its more efficient for the hypercall to the
work all at once rather than as separate hypercalls (even batched as a
multicall).

This patch adds a mechanism (xen_mc_extend_args()) to allocate more
argument space to the last-issued multicall, in order to extend its
argument list.

The user of this mechanism is xen/mmu.c, which uses it to extend the
args array of mmu_update.  This is particularly valuable when doing
the update for a large mprotect, which goes via
ptep_modify_prot_commit(), but it also manages to batch updates to
pgd/pmds as well.

Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com>
Acked-by: Linus Torvalds <torvalds@linux-foundation.org>
Acked-by: Hugh Dickins <hugh@veritas.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 846dad7d54a5ed6162c41a5fa1da9b2646b09300..f6b8225c2a0ba491c8312b596d7650cdab2b82a9 100644 (file)
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -230,18 +230,35 @@ static bool page_pinned(void *ptr)
        return PagePinned(page);
 }
 
-void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
+static void extend_mmu_update(const struct mmu_update *update)
 {
        struct multicall_space mcs;
        struct mmu_update *u;
 
-       preempt_disable();
+       mcs = xen_mc_extend_args(__HYPERVISOR_mmu_update, sizeof(*u));
+
+       if (mcs.mc != NULL)
+               mcs.mc->args[1]++;
+       else {
+               mcs = __xen_mc_entry(sizeof(*u));
+               MULTI_mmu_update(mcs.mc, mcs.args, 1, NULL, DOMID_SELF);
+       }
 
-       mcs = xen_mc_entry(sizeof(*u));
        u = mcs.args;
-       u->ptr = virt_to_machine(ptr).maddr;
-       u->val = pmd_val_ma(val);
-       MULTI_mmu_update(mcs.mc, u, 1, NULL, DOMID_SELF);
+       *u = *update;
+}
+
+void xen_set_pmd_hyper(pmd_t *ptr, pmd_t val)
+{
+       struct mmu_update u;
+
+       preempt_disable();
+
+       xen_mc_batch();
+
+       u.ptr = virt_to_machine(ptr).maddr;
+       u.val = pmd_val_ma(val);
+       extend_mmu_update(&u);
 
        xen_mc_issue(PARAVIRT_LAZY_MMU);
 
@@ -332,14 +349,13 @@ pte_t xen_ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, pte_t
 void xen_ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr,
                                 pte_t *ptep, pte_t pte)
 {
-       struct multicall_space mcs;
-       struct mmu_update *u;
+       struct mmu_update u;
 
-       mcs = xen_mc_entry(sizeof(*u));
-       u = mcs.args;
-       u->ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD;
-       u->val = pte_val_ma(pte);
-       MULTI_mmu_update(mcs.mc, u, 1, NULL, DOMID_SELF);
+       xen_mc_batch();
+
+       u.ptr = virt_to_machine(ptep).maddr | MMU_PT_UPDATE_PRESERVE_AD;
+       u.val = pte_val_ma(pte);
+       extend_mmu_update(&u);
 
        xen_mc_issue(PARAVIRT_LAZY_MMU);
 }
@@ -396,16 +412,15 @@ pmdval_t xen_pmd_val(pmd_t pmd)
 
 void xen_set_pud_hyper(pud_t *ptr, pud_t val)
 {
-       struct multicall_space mcs;
-       struct mmu_update *u;
+       struct mmu_update u;
 
        preempt_disable();
 
-       mcs = xen_mc_entry(sizeof(*u));
-       u = mcs.args;
-       u->ptr = virt_to_machine(ptr).maddr;
-       u->val = pud_val_ma(val);
-       MULTI_mmu_update(mcs.mc, u, 1, NULL, DOMID_SELF);
+       xen_mc_batch();
+
+       u.ptr = virt_to_machine(ptr).maddr;
+       u.val = pud_val_ma(val);
+       extend_mmu_update(&u);
 
        xen_mc_issue(PARAVIRT_LAZY_MMU);
 

diff --git a/arch/x86/xen/multicalls.c b/arch/x86/xen/multicalls.c
index 5791eb2e37508dc87ab6d9f74bdc53da6f24ae4b..3c63c4da7ed19cbe4ae43234fe6f6068eb638a79 100644 (file)
--- a/arch/x86/xen/multicalls.c
+++ b/arch/x86/xen/multicalls.c
@@ -29,14 +29,14 @@
 #define MC_DEBUG       1
 
 #define MC_BATCH       32
-#define MC_ARGS                (MC_BATCH * 16 / sizeof(u64))
+#define MC_ARGS                (MC_BATCH * 16)
 
 struct mc_buffer {
        struct multicall_entry entries[MC_BATCH];
 #if MC_DEBUG
        struct multicall_entry debug[MC_BATCH];
 #endif
-       u64 args[MC_ARGS];
+       unsigned char args[MC_ARGS];
        struct callback {
                void (*fn)(void *);
                void *data;
@@ -107,20 +107,48 @@ struct multicall_space __xen_mc_entry(size_t args)
 {
        struct mc_buffer *b = &__get_cpu_var(mc_buffer);
        struct multicall_space ret;
-       unsigned argspace = (args + sizeof(u64) - 1) / sizeof(u64);
+       unsigned argidx = roundup(b->argidx, sizeof(u64));
 
        BUG_ON(preemptible());
-       BUG_ON(argspace > MC_ARGS);
+       BUG_ON(b->argidx > MC_ARGS);
 
        if (b->mcidx == MC_BATCH ||
-           (b->argidx + argspace) > MC_ARGS)
+           (argidx + args) > MC_ARGS) {
                xen_mc_flush();
+               argidx = roundup(b->argidx, sizeof(u64));
+       }
 
        ret.mc = &b->entries[b->mcidx];
        b->mcidx++;
+       ret.args = &b->args[argidx];
+       b->argidx = argidx + args;
+
+       BUG_ON(b->argidx > MC_ARGS);
+       return ret;
+}
+
+struct multicall_space xen_mc_extend_args(unsigned long op, size_t size)
+{
+       struct mc_buffer *b = &__get_cpu_var(mc_buffer);
+       struct multicall_space ret = { NULL, NULL };
+
+       BUG_ON(preemptible());
+       BUG_ON(b->argidx > MC_ARGS);
+
+       if (b->mcidx == 0)
+               return ret;
+
+       if (b->entries[b->mcidx - 1].op != op)
+               return ret;
+
+       if ((b->argidx + size) > MC_ARGS)
+               return ret;
+
+       ret.mc = &b->entries[b->mcidx - 1];
        ret.args = &b->args[b->argidx];
-       b->argidx += argspace;
+       b->argidx += size;
 
+       BUG_ON(b->argidx > MC_ARGS);
        return ret;
 }
 

diff --git a/arch/x86/xen/multicalls.h b/arch/x86/xen/multicalls.h
index 8bae996d99a39c2c190de046f6be91b47060aacf..858938241616ab88ff53700a4463c698c11dad3d 100644 (file)
--- a/arch/x86/xen/multicalls.h
+++ b/arch/x86/xen/multicalls.h
@@ -45,4 +45,16 @@ static inline void xen_mc_issue(unsigned mode)
 /* Set up a callback to be called when the current batch is flushed */
 void xen_mc_callback(void (*fn)(void *), void *data);
 
+/*
+ * Try to extend the arguments of the previous multicall command.  The
+ * previous command's op must match.  If it does, then it attempts to
+ * extend the argument space allocated to the multicall entry by
+ * arg_size bytes.
+ *
+ * The returned multicall_space will return with mc pointing to the
+ * command on success, or NULL on failure, and args pointing to the
+ * newly allocated space.
+ */
+struct multicall_space xen_mc_extend_args(unsigned long op, size_t arg_size);
+
 #endif /* _XEN_MULTICALLS_H */