From patchwork Fri Jun 3 15:03:54 2011 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Christoffer Dall X-Patchwork-Id: 847432 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter1.kernel.org (8.14.4/8.14.3) with ESMTP id p53F4EAs023939 for ; Fri, 3 Jun 2011 15:04:14 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1755389Ab1FCPEJ (ORCPT ); Fri, 3 Jun 2011 11:04:09 -0400 Received: from mail-wy0-f174.google.com ([74.125.82.174]:37774 "EHLO mail-wy0-f174.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1755387Ab1FCPEI (ORCPT ); Fri, 3 Jun 2011 11:04:08 -0400 Received: by mail-wy0-f174.google.com with SMTP id 21so1457214wya.19 for ; Fri, 03 Jun 2011 08:04:07 -0700 (PDT) Received: by 10.227.54.6 with SMTP id o6mr2077153wbg.83.1307113446856; Fri, 03 Jun 2011 08:04:06 -0700 (PDT) Received: from [127.0.0.1] (host198-39-dynamic.245-95-r.retail.telecomitalia.it [95.245.39.198]) by mx.google.com with ESMTPS id o19sm1100006wbh.55.2011.06.03.08.04.02 (version=TLSv1/SSLv3 cipher=OTHER); Fri, 03 Jun 2011 08:04:05 -0700 (PDT) Subject: [PATCH v3 5/8] ARM: KVM: World-switch implementation To: catalin.marinas@arm.com, android-virt@lists.cs.columbia.edu From: Christoffer Dall Cc: s.raho@virtualopensystems.com, a.motakis@virtualopensystems.com, c.dall@virtualopensystems.com, kvm@vger.kernel.org, a.costa@virtualopensystems.com Date: Fri, 03 Jun 2011 17:03:54 +0200 Message-ID: <20110603150354.17011.24776.stgit@ubuntu> In-Reply-To: <20110603150318.17011.82777.stgit@ubuntu> References: <20110603150318.17011.82777.stgit@ubuntu> User-Agent: StGit/0.15 MIME-Version: 1.0 Sender: kvm-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: kvm@vger.kernel.org X-Greylist: IP, sender and recipient auto-whitelisted, not delayed by milter-greylist-4.2.6 (demeter1.kernel.org [140.211.167.41]); Fri, 03 Jun 2011 15:04:14 +0000 (UTC) Provides complete world-switch implementation to switch to other guests runinng in non-secure modes. Includes Hyp exception handlers that captures necessary exception information and stores the information on the VCPU and KVM structures. Only controversy may be the back-door call to __irq_svc (the host kernel's own physical IRQ handler) which is called when a physical IRQ exception is taken in Hyp mode while running in the guest. --- arch/arm/include/asm/kvm_arm.h | 24 +++ arch/arm/include/asm/kvm_host.h | 2 arch/arm/kernel/asm-offsets.c | 23 ++ arch/arm/kernel/entry-armv.S | 1 arch/arm/kvm/arm.c | 20 ++ arch/arm/kvm/arm_interrupts.S | 373 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 439 insertions(+), 4 deletions(-) -- To unsubscribe from this list: send the line "unsubscribe kvm" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html diff --git a/arch/arm/include/asm/kvm_arm.h b/arch/arm/include/asm/kvm_arm.h index 835abd1..505a1a5 100644 --- a/arch/arm/include/asm/kvm_arm.h +++ b/arch/arm/include/asm/kvm_arm.h @@ -99,5 +99,29 @@ #define VTTBR_X (5 - VTCR_GUEST_T0SZ) #endif +/* Hyp Syndrome Register (HSR) bits */ +#define HSR_EC_SHIFT (26) +#define HSR_EC (0x3fU << HSR_EC_SHIFT) +#define HSR_IL (1U << 25) +#define HSR_ISS (HSR_IL - 1) + +#define HSR_EC_UNKNOWN (0x00) +#define HSR_EC_WFI (0x01) +#define HSR_EC_CP15_32 (0x03) +#define HSR_EC_CP15_64 (0x04) +#define HSR_EC_CP14_MR (0x05) +#define HSR_EC_CP14_LS (0x06) +#define HSR_EC_CP_0_13 (0x07) +#define HSR_EC_CP10_ID (0x08) +#define HSR_EC_JAZELLE (0x09) +#define HSR_EC_BXJ (0x0A) +#define HSR_EC_CP14_64 (0x0C) +#define HSR_EC_SVC_HYP (0x11) +#define HSR_EC_HVC (0x12) +#define HSR_EC_SMC (0x13) +#define HSR_EC_IABT (0x20) +#define HSR_EC_IABT_HYP (0x21) +#define HSR_EC_DABT (0x24) +#define HSR_EC_DABT_HYP (0x25) #endif /* __KVM_ARM_H__ */ diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h index 5955ff4..7f96974 100644 --- a/arch/arm/include/asm/kvm_host.h +++ b/arch/arm/include/asm/kvm_host.h @@ -73,6 +73,8 @@ struct kvm_vcpu_arch { u64 c2_TTBR1; /* Translation Table Base Register 1 */ u32 c2_TTBCR; /* Translation Table Base Control Register */ u32 c3_DACR; /* Domain Access Control Register */ + u32 c10_PRRR; /* Primary Region Remap Register */ + u32 c10_NMRR; /* Normal Memory Remap Register */ } cp15; u32 exception_pending; /* Exception to raise after emulation */ diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 82da661..769fa97 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -118,5 +119,27 @@ int main(void) DEFINE(DMA_BIDIRECTIONAL, DMA_BIDIRECTIONAL); DEFINE(DMA_TO_DEVICE, DMA_TO_DEVICE); DEFINE(DMA_FROM_DEVICE, DMA_FROM_DEVICE); +#ifdef CONFIG_KVM + DEFINE(VCPU_KVM, offsetof(struct kvm_vcpu, kvm)); + DEFINE(VCPU_SCTLR, offsetof(struct kvm_vcpu, arch.cp15.c1_SCTLR)); + DEFINE(VCPU_CPACR, offsetof(struct kvm_vcpu, arch.cp15.c1_CPACR)); + DEFINE(VCPU_TTBR0, offsetof(struct kvm_vcpu, arch.cp15.c2_TTBR0)); + DEFINE(VCPU_TTBR1, offsetof(struct kvm_vcpu, arch.cp15.c2_TTBR1)); + DEFINE(VCPU_TTBCR, offsetof(struct kvm_vcpu, arch.cp15.c2_TTBCR)); + DEFINE(VCPU_DACR, offsetof(struct kvm_vcpu, arch.cp15.c3_DACR)); + DEFINE(VCPU_PRRR, offsetof(struct kvm_vcpu, arch.cp15.c10_PRRR)); + DEFINE(VCPU_NMRR, offsetof(struct kvm_vcpu, arch.cp15.c10_NMRR)); + DEFINE(VCPU_REGS, offsetof(struct kvm_vcpu, arch.regs)); + DEFINE(VCPU_USR_REGS, offsetof(struct kvm_vcpu, arch.regs.usr_regs)); + DEFINE(VCPU_SVC_REGS, offsetof(struct kvm_vcpu, arch.regs.svc_regs)); + DEFINE(VCPU_ABT_REGS, offsetof(struct kvm_vcpu, arch.regs.abt_regs)); + DEFINE(VCPU_UND_REGS, offsetof(struct kvm_vcpu, arch.regs.und_regs)); + DEFINE(VCPU_IRQ_REGS, offsetof(struct kvm_vcpu, arch.regs.irq_regs)); + DEFINE(VCPU_FIQ_REGS, offsetof(struct kvm_vcpu, arch.regs.fiq_regs)); + DEFINE(VCPU_PC, offsetof(struct kvm_vcpu, arch.regs.pc)); + DEFINE(VCPU_CPSR, offsetof(struct kvm_vcpu, arch.regs.cpsr)); + DEFINE(VCPU_HSR, offsetof(struct kvm_vcpu, arch.hsr)); + DEFINE(KVM_VTTBR, offsetof(struct kvm, arch.vttbr)); +#endif return 0; } diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 2b46fea..9cdc65e 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -197,6 +197,7 @@ __dabt_svc: ENDPROC(__dabt_svc) .align 5 + .globl __irq_svc __irq_svc: svc_entry diff --git a/arch/arm/kvm/arm.c b/arch/arm/kvm/arm.c index 714f415..1a9f168 100644 --- a/arch/arm/kvm/arm.c +++ b/arch/arm/kvm/arm.c @@ -245,7 +245,19 @@ int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) { - KVMARM_NOT_IMPLEMENTED(); + unsigned long cpsr; + unsigned long sctlr; + + /* Init execution CPSR */ + asm volatile ("mrs %[cpsr], cpsr": [cpsr] "=r" (cpsr)); + vcpu->arch.regs.cpsr = SVC_MODE | PSR_I_BIT | PSR_F_BIT | PSR_A_BIT | + (cpsr & PSR_E_BIT); + + /* Init SCTLR with MMU disabled */ + asm volatile ("mrc p15, 0, %[sctlr], c1, c0, 0": + [sctlr] "=r" (sctlr)); + vcpu->arch.cp15.c1_SCTLR = sctlr & ~1U; + return 0; } @@ -287,6 +299,12 @@ int kvm_arch_vcpu_runnable(struct kvm_vcpu *v) int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *run) { + unsigned long flags; + + local_irq_save(flags); + __kvm_vcpu_run(vcpu); + local_irq_restore(flags); + KVMARM_NOT_IMPLEMENTED(); return -EINVAL; } diff --git a/arch/arm/kvm/arm_interrupts.S b/arch/arm/kvm/arm_interrupts.S index e0bcf9e..6d3044c 100644 --- a/arch/arm/kvm/arm_interrupts.S +++ b/arch/arm/kvm/arm_interrupts.S @@ -21,6 +21,12 @@ #include #include +#define VCPU_USR_REG(_reg_nr) (VCPU_USR_REGS + (_reg_nr * 4)) +#define VCPU_USR_SP (VCPU_USR_REG(13)) +#define VCPU_FIQ_REG(_reg_nr) (VCPU_FIQ_REGS + (_reg_nr * 4)) +#define VCPU_FIQ_SPSR (VCPU_FIQ_REG(7)) + + @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @ Hypervisor initialization @ - should be called with: @@ -120,11 +126,269 @@ __kvm_hyp_init_end: @ Hypervisor world-switch code @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ -/* - * This is a stub +.macro store_mode_state base_reg, mode + .if \mode == usr + mrs r2, SP_usr + mov r3, lr + stmdb \base_reg!, {r2, r3} + .elseif \mode != fiq + mrs r2, SP_\mode + mrs r3, LR_\mode + mrs r4, SPSR_\mode + stmdb \base_reg!, {r2, r3, r4} + .else + mrs r2, r8_fiq + mrs r3, r9_fiq + mrs r4, r10_fiq + mrs r5, r11_fiq + mrs r6, r12_fiq + mrs r7, SP_fiq + mrs r8, LR_fiq + mrs r9, SPSR_fiq + stmdb \base_reg!, {r2-r9} + .endif +.endm + +.macro load_mode_state base_reg, mode + .if \mode == usr + ldmia \base_reg!, {r2, r3} + msr SP_usr, r2 + mov lr, r3 + .elseif \mode != fiq + ldmia \base_reg!, {r2, r3, r4} + msr SP_\mode, r2 + msr LR_\mode, r3 + msr SPSR_\mode, r4 + .else + ldmia \base_reg!, {r2-r9} + msr r8_fiq, r2 + msr r9_fiq, r3 + msr r10_fiq, r4 + msr r11_fiq, r5 + msr r12_fiq, r6 + msr SP_fiq, r7 + msr LR_fiq, r8 + msr SPSR_fiq, r9 + .endif +.endm + +/* Reads cp15 registers from hardware and stores then in memory + * @vcpu: If 0, registers are written in-order to the stack, + * otherwise to the VCPU struct pointed to by vcpup + * @vcpup: Register pointing to VCPU struct + */ +.macro read_cp15_state vcpu=0, vcpup + mrc p15, 0, r2, c1, c0, 0 @ SCTLR + mrc p15, 0, r3, c1, c0, 2 @ CPACR + mrc p15, 0, r4, c2, c0, 2 @ TTBCR + mcr p15, 0, r5, c3, c0, 0 @ DACR + mrrc p15, 0, r6, r7, c2 @ TTBR 0 + mrrc p15, 1, r8, r9, c2 @ TTBR 1 + mrc p15, 0, r10, c10, c2, 0 @ PRRR + mrc p15, 0, r11, c10, c2, 1 @ NMRR + + .if \vcpu == 0 + push {r2-r11} @ Push CP15 registers + .else + str r2, [\vcpup, #VCPU_SCTLR] + str r3, [\vcpup, #VCPU_CPACR] + str r4, [\vcpup, #VCPU_TTBCR] + str r5, [\vcpup, #VCPU_DACR] + add \vcpup, \vcpup, #VCPU_TTBR0 + strd r6, r7, [\vcpup] + add \vcpup, \vcpup, #(VCPU_TTBR1 - VCPU_TTBR0) + strd r8, r9, [\vcpup] + sub \vcpup, \vcpup, #(VCPU_TTBR1) + str r10, [\vcpup, #VCPU_PRRR] + str r11, [\vcpup, #VCPU_NMRR] + .endif +.endm + +/* Reads cp15 registers from memory and writes them to hardware + * @vcpu: If 0, registers are read in-order from the stack, + * otherwise from the VCPU struct pointed to by vcpup + * @vcpup: Register pointing to VCPU struct */ +.macro write_cp15_state vcpu=0, vcpup + .if \vcpu == 0 + pop {r2-r11} + .else + ldr r2, [\vcpup, #VCPU_SCTLR] + ldr r3, [\vcpup, #VCPU_CPACR] + ldr r4, [\vcpup, #VCPU_TTBCR] + ldr r5, [\vcpup, #VCPU_DACR] + add \vcpup, \vcpup, #VCPU_TTBR0 + ldrd r6, r7, [\vcpup] + add \vcpup, \vcpup, #(VCPU_TTBR1 - VCPU_TTBR0) + ldrd r8, r9, [\vcpup] + sub \vcpup, \vcpup, #(VCPU_TTBR1) + ldr r10, [\vcpup, #VCPU_PRRR] + ldr r11, [\vcpup, #VCPU_NMRR] + .endif + + mcr p15, 0, r2, c1, c0, 0 @ SCTLR + mcr p15, 0, r3, c1, c0, 2 @ CPACR + mcr p15, 0, r4, c2, c0, 2 @ TTBCR + mcr p15, 0, r5, c3, c0, 0 @ DACR + mcrr p15, 0, r6, r7, c2 @ TTBR 0 + mcrr p15, 1, r8, r9, c2 @ TTBR 1 + mcr p15, 0, r10, c10, c2, 0 @ PRRR + mcr p15, 0, r11, c10, c2, 1 @ NMRR +.endm + +/* Configures the HSTR (Hyp System Trap Register) on entry/return + * (hardware reset value is 0) */ +.macro set_hstr entry + mrc p15, 4, r2, c1, c1, 3 + ldr r3, =0xbe00 + .if \entry == 1 + orr r2, r2, r3 @ Trap CR{9,10,11,12,13,15} + .else + bic r2, r2, r3 @ Don't trap any CRx accesses + .endif + mcr p15, 4, r2, c1, c1, 3 +.endm + +/* Enable/Disable: stage-2 trans., trap interrupts, trap wfi/wfe, trap smc */ +.macro configure_hyp_role entry + mrc p15, 4, r2, c1, c1, 0 @ HCR + ldr r3, =HCR_GUEST_MASK + .if \entry == 1 + orr r2, r2, r3 + .else + bic r2, r2, r3 + .endif + mcr p15, 4, r2, c1, c1, 0 +.endm + +@ This must be called from Hyp mode! +@ Arguments: +@ r0: pointer to vcpu struct ENTRY(__kvm_vcpu_run) + hvc #0 @ Change to Hyp-mode + + @ Now we're in Hyp-mode and lr_usr, spsr_hyp are on the stack + mrs r2, sp_usr + push {r2} @ Push r13_usr + push {r4-r12} @ Push r4-r12 + + store_mode_state sp, svc + store_mode_state sp, abt + store_mode_state sp, und + store_mode_state sp, irq + store_mode_state sp, fiq + + @ Store hardware CP15 state and load guest state + read_cp15_state + write_cp15_state 1, r0 + + push {r0} @ Push the VCPU pointer + + @ Set up guest memory translation + ldr r1, [r0, #VCPU_KVM] @ r1 points to kvm struct + ldrd r2, r3, [r1, #KVM_VTTBR] + mcrr p15, 6, r2, r3, c2 @ Write VTTBR + + @ Configure Hyp-role + configure_hyp_role 1 + + @ Trap coprocessor CRx for all x except 2 and 14 + set_hstr 1 + + @ Load guest registers + add r0, r0, #(VCPU_USR_SP) + load_mode_state r0, usr + load_mode_state r0, svc + load_mode_state r0, abt + load_mode_state r0, und + load_mode_state r0, irq + load_mode_state r0, fiq + + @ Load return state (r0 now points to vcpu->arch.regs.pc) + ldmia r0, {r2, r3} + msr ELR_hyp, r2 + msr spsr, r3 + + @ Load remaining registers and do the switch + sub r0, r0, #(VCPU_PC - VCPU_USR_REGS) + ldmia r0, {r0-r12} + eret + +__kvm_vcpu_return: + @ Store return state + mrs r2, ELR_hyp + mrs r3, spsr + str r2, [r1, #VCPU_PC] + str r3, [r1, #VCPU_CPSR] + + @ Store guest registers + add r1, r1, #(VCPU_FIQ_SPSR + 4) + store_mode_state r1, fiq + store_mode_state r1, irq + store_mode_state r1, und + store_mode_state r1, abt + store_mode_state r1, svc + store_mode_state r1, usr + + @ Don't trap coprocessor accesses for host kernel + set_hstr 0 + + @ Reset Hyp-role + configure_hyp_role 0 + + @ Set VMID == 0 + mov r2, #0 + mov r3, #0 + mcrr p15, 6, r2, r3, c2 @ Write VTTBR + + @ Store guest CP15 state and restore host state + read_cp15_state 1, r1 + write_cp15_state + + load_mode_state sp, fiq + load_mode_state sp, irq + load_mode_state sp, und + load_mode_state sp, abt + load_mode_state sp, svc + + pop {r4-r12} @ Pop r4-r12 + pop {r2} @ Pop r13_usr + msr sp_usr, r2 + + hvc #0 + + cmp r0, #ARM_EXCEPTION_IRQ + bne return_to_ioctl + + /* + * It's time to launch the kernel IRQ handler for IRQ exceptions. This + * requires some manipulation though. + * + * - The easiest entry point to the host handler is __irq_svc. + * - The __irq_svc expects to be called from SVC mode, which has been + * switched to from vector_stub code in entry-armv.S. The __irq_svc calls + * svc_entry which uses values stored in memory and pointed to by r0 + * to return from handler. We allocate this memory on the stack, which + * will contain these values: + * 0x8: cpsr + * 0x4: return_address + * 0x0: r0 + */ + adr r1, irq_kernel_resume @ Where to resume + mrs r2, cpsr @ CPSR when we return + push {r0 - r2} + mov r0, sp + b __irq_svc + +irq_kernel_resume: + pop {r0} + add sp, sp, #8 + +return_to_ioctl: mov pc, lr + + .ltorg + __kvm_vcpu_run_end: .globl __kvm_vcpu_run_end @@ -136,6 +400,109 @@ __kvm_vcpu_run_end: .align 5 __kvm_hyp_vector: .globl __kvm_hyp_vector - nop + + @ Hyp-mode exception vector + b hyp_reset + b hyp_undef + b hyp_svc + b hyp_pabt + b hyp_dabt + b hyp_hvc + b hyp_irq + b hyp_fiq + + .align +hyp_reset: + sub pc, pc, #8 + + .align +hyp_undef: + sub pc, pc, #8 + + .align +hyp_svc: + @ Can only get here if HVC or SVC is called from Hyp, mode which means + @ we want to change mode back to SVC mode. + @ NB: Stack pointer should be where hyp_hvc handler left it! + ldr lr, [sp, #4] + msr spsr, lr + ldr lr, [sp] + add sp, sp, #8 + eret + + .align +hyp_pabt: + sub pc, pc, #8 + + .align +hyp_dabt: + sub pc, pc, #8 + + .align +hyp_hvc: + @ Getting here is either becuase of a trap from a guest or from calling + @ HVC from the host kernel, which means "switch to Hyp mode". + push {r0, r1, r2} + + @ Check syndrome register + mrc p15, 4, r0, c5, c2, 0 @ HSR + lsr r1, r0, #HSR_EC_SHIFT + cmp r1, #HSR_EC_HVC + bne guest_trap @ Not HVC instr. + + @ Let's check if the HVC came from VMID 0 and allow simple + @ switch to Hyp mode + mrrc p15, 6, r1, r2, c2 + lsr r2, r2, #16 + and r2, r2, #0xff + cmp r2, #0 + bne guest_trap @ Guest called HVC + + pop {r0, r1, r2} + + @ Store lr_usr,spsr (svc cpsr) on stack + sub sp, sp, #8 + str lr, [sp] + mrs lr, spsr + str lr, [sp, #4] + + @ Return to caller in Hyp mode + mrs lr, ELR_hyp + mov pc, lr + + @ Not HVC from VMID 0 - this requires more careful investigation + @ TODO: Not implemented +guest_trap: + ldr r1, [sp, #12] @ Load VCPU pointer + str r0, [r1, #VCPU_HSR] + add r1, r1, #VCPU_USR_REG(3) + stmia r1, {r3-r12} + sub r1, r1, #(VCPU_USR_REG(3) - VCPU_USR_REG(0)) + pop {r3, r4, r5} + add sp, sp, #4 @ We loaded the VCPU pointer above + stmia r1, {r3, r4, r5} + sub r1, r1, #VCPU_USR_REG(0) + + mov r0, #ARM_EXCEPTION_HVC + b __kvm_vcpu_return + + .align +hyp_irq: + push {r0} + ldr r0, [sp, #4] @ Load VCPU pointer + add r0, r0, #(VCPU_USR_REG(1)) + stmia r0, {r1-r12} + pop {r0, r1} @ r1 == vcpu pointer + str r0, [r1, #VCPU_USR_REG(0)] + + mov r0, #ARM_EXCEPTION_IRQ + b __kvm_vcpu_return + + .align +hyp_fiq: + sub pc, pc, #8 + + .ltorg + __kvm_hyp_vector_end: .globl __kvm_hyp_vector_end