KVM: arm64: Introduce VHE-specific kvm_vcpu_run

So far this is mostly (see below) a copy of the legacy non-VHE switch
function, but we will start reworking these functions in separate
directions to work on VHE and non-VHE in the most optimal way in later
patches.

The only difference after this patch between the VHE and non-VHE run
functions is that we omit the branch-predictor variant-2 hardening for
QC Falkor CPUs, because this workaround is specific to a series of
non-VHE ARMv8.0 CPUs.

Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Christoffer Dall <christoffer.dall@linaro.org>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>

diff --git a/arch/arm/include/asm/kvm_asm.h b/arch/arm/include/asm/kvm_asm.h
index 36dd2962a42db2868badb568e38118884321eddb..5a953ecb0d7888404fe97af4724281a4f578dbcb 100644 (file)
--- a/arch/arm/include/asm/kvm_asm.h
+++ b/arch/arm/include/asm/kvm_asm.h
@@ -70,7 +70,10 @@ extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu);
 
 extern void __kvm_timer_set_cntvoff(u32 cntvoff_low, u32 cntvoff_high);
 
-extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
+/* no VHE on 32-bit :( */
+static inline int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) { BUG(); return 0; }
+
+extern int __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu);
 
 extern void __init_stage2_translation(void);
 

diff --git a/arch/arm/kvm/hyp/switch.c b/arch/arm/kvm/hyp/switch.c
index e86679daddff9f144ff4e999f298f21a5714b6fa..aac025783ee8a42b554094401035f1b7331f5e04 100644 (file)
--- a/arch/arm/kvm/hyp/switch.c
+++ b/arch/arm/kvm/hyp/switch.c
@@ -154,7 +154,7 @@ static bool __hyp_text __populate_fault_info(struct kvm_vcpu *vcpu)
        return true;
 }
 
-int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
+int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
 {
        struct kvm_cpu_context *host_ctxt;
        struct kvm_cpu_context *guest_ctxt;

diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 7149f15203826d6e38ecd586cfc4ef5be523baaa..d53d40704416699c81fe923c5af15ce8a79993a9 100644 (file)
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -58,7 +58,9 @@ extern void __kvm_tlb_flush_local_vmid(struct kvm_vcpu *vcpu);
 
 extern void __kvm_timer_set_cntvoff(u32 cntvoff_low, u32 cntvoff_high);
 
-extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
+extern int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu);
+
+extern int __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu);
 
 extern u64 __vgic_v3_get_ich_vtr_el2(void);
 extern u64 __vgic_v3_read_vmcr(void);

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index c30fc96992dfe7847f3c6dfb1e72fbbdd4fd9eab..87abc94fb5914a94a4363a35d7c4b4137882373d 100644 (file)
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -428,6 +428,13 @@ static inline void kvm_arm_vhe_guest_enter(void)
 static inline void kvm_arm_vhe_guest_exit(void)
 {
        local_daif_restore(DAIF_PROCCTX_NOIRQ);
+
+       /*
+        * When we exit from the guest we change a number of CPU configuration
+        * parameters, such as traps.  Make sure these changes take effect
+        * before running the host or additional guests.
+        */
+       isb();
 }
 
 static inline bool kvm_arm_harden_branch_predictor(void)

diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c
index b055111df1a17b1d4e62fd7b4a57235ce1bbd297..1b94ac6a85e6ea90a3e91e0e5d5787030a1d2e79 100644 (file)
--- a/arch/arm64/kvm/hyp/switch.c
+++ b/arch/arm64/kvm/hyp/switch.c
@@ -362,7 +362,71 @@ static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code)
        return false;
 }
 
-int __hyp_text __kvm_vcpu_run(struct kvm_vcpu *vcpu)
+/* Switch to the guest for VHE systems running in EL2 */
+int kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu)
+{
+       struct kvm_cpu_context *host_ctxt;
+       struct kvm_cpu_context *guest_ctxt;
+       bool fp_enabled;
+       u64 exit_code;
+
+       vcpu = kern_hyp_va(vcpu);
+
+       host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context);
+       host_ctxt->__hyp_running_vcpu = vcpu;
+       guest_ctxt = &vcpu->arch.ctxt;
+
+       __sysreg_save_host_state(host_ctxt);
+
+       __activate_traps(vcpu);
+       __activate_vm(vcpu);
+
+       __vgic_restore_state(vcpu);
+       __timer_enable_traps(vcpu);
+
+       /*
+        * We must restore the 32-bit state before the sysregs, thanks
+        * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
+        */
+       __sysreg32_restore_state(vcpu);
+       __sysreg_restore_guest_state(guest_ctxt);
+       __debug_switch_to_guest(vcpu);
+
+       do {
+               /* Jump in the fire! */
+               exit_code = __guest_enter(vcpu, host_ctxt);
+
+               /* And we're baaack! */
+       } while (fixup_guest_exit(vcpu, &exit_code));
+
+       fp_enabled = __fpsimd_enabled();
+
+       __sysreg_save_guest_state(guest_ctxt);
+       __sysreg32_save_state(vcpu);
+       __timer_disable_traps(vcpu);
+       __vgic_save_state(vcpu);
+
+       __deactivate_traps(vcpu);
+       __deactivate_vm(vcpu);
+
+       __sysreg_restore_host_state(host_ctxt);
+
+       if (fp_enabled) {
+               __fpsimd_save_state(&guest_ctxt->gp_regs.fp_regs);
+               __fpsimd_restore_state(&host_ctxt->gp_regs.fp_regs);
+       }
+
+       /*
+        * This must come after restoring the host sysregs, since a non-VHE
+        * system may enable SPE here and make use of the TTBRs.
+        */
+       __debug_switch_to_host(vcpu);
+
+       return exit_code;
+}
+
+/* Switch to the guest for legacy non-VHE systems */
+int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu)
 {
        struct kvm_cpu_context *host_ctxt;
        struct kvm_cpu_context *guest_ctxt;

diff --git a/virt/kvm/arm/arm.c b/virt/kvm/arm/arm.c
index 2062d93579717769fb9e2998120cb2832e246d0c..09dbee56ed8f75180d724113f3ee01f1c8558abf 100644 (file)
--- a/virt/kvm/arm/arm.c
+++ b/virt/kvm/arm/arm.c
@@ -733,13 +733,15 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run)
                 */
                trace_kvm_entry(*vcpu_pc(vcpu));
                guest_enter_irqoff();
-               if (has_vhe())
-                       kvm_arm_vhe_guest_enter();
-
-               ret = kvm_call_hyp(__kvm_vcpu_run, vcpu);
 
-               if (has_vhe())
+               if (has_vhe()) {
+                       kvm_arm_vhe_guest_enter();
+                       ret = kvm_vcpu_run_vhe(vcpu);
                        kvm_arm_vhe_guest_exit();
+               } else {
+                       ret = kvm_call_hyp(__kvm_vcpu_run_nvhe, vcpu);
+               }
+
                vcpu->mode = OUTSIDE_GUEST_MODE;
                vcpu->stat.exits++;
                /*