| Message ID | 20230519005231.3027912-3-rananta@google.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | KVM: arm64: Add support for FEAT_TLBIRANGE | expand |
On Fri, 19 May 2023 01:52:27 +0100, Raghavendra Rao Ananta <rananta@google.com> wrote: > > Define __kvm_tlb_flush_vmid_range() (for VHE and nVHE) > to flush a range of stage-2 page-tables using IPA in one go. > If the system supports FEAT_TLBIRANGE, the following patches > would conviniently replace global TLBI such as vmalls12e1is > in the map, unmap, and dirty-logging paths with ripas2e1is > instead. > > Signed-off-by: Raghavendra Rao Ananta <rananta@google.com> > --- > arch/arm64/include/asm/kvm_asm.h | 3 +++ > arch/arm64/kvm/hyp/nvhe/hyp-main.c | 11 +++++++++ > arch/arm64/kvm/hyp/nvhe/tlb.c | 39 ++++++++++++++++++++++++++++++ > arch/arm64/kvm/hyp/vhe/tlb.c | 35 +++++++++++++++++++++++++++ > 4 files changed, 88 insertions(+) > > diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h > index 43c3bc0f9544d..33352d9399e32 100644 > --- a/arch/arm64/include/asm/kvm_asm.h > +++ b/arch/arm64/include/asm/kvm_asm.h > @@ -79,6 +79,7 @@ enum __kvm_host_smccc_func { > __KVM_HOST_SMCCC_FUNC___pkvm_init_vm, > __KVM_HOST_SMCCC_FUNC___pkvm_init_vcpu, > __KVM_HOST_SMCCC_FUNC___pkvm_teardown_vm, > + __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_range, nit: please keep this close to the other TLB operations. > }; > > #define DECLARE_KVM_VHE_SYM(sym) extern char sym[] > @@ -225,6 +226,8 @@ extern void __kvm_flush_vm_context(void); > extern void __kvm_flush_cpu_context(struct kvm_s2_mmu *mmu); > extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa, > int level); > +extern void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, > + phys_addr_t start, phys_addr_t end); > extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu); > > extern void __kvm_timer_set_cntvoff(u64 cntvoff); > diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c > index 728e01d4536b0..81d30737dc7c9 100644 > --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c > +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c > @@ -125,6 +125,16 @@ static void handle___kvm_tlb_flush_vmid_ipa(struct kvm_cpu_context *host_ctxt) > __kvm_tlb_flush_vmid_ipa(kern_hyp_va(mmu), ipa, level); > } > > +static void > +handle___kvm_tlb_flush_vmid_range(struct kvm_cpu_context *host_ctxt) > +{ > + DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); > + DECLARE_REG(phys_addr_t, start, host_ctxt, 2); > + DECLARE_REG(phys_addr_t, end, host_ctxt, 3); > + > + __kvm_tlb_flush_vmid_range(kern_hyp_va(mmu), start, end); > +} > + > static void handle___kvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) > { > DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); > @@ -315,6 +325,7 @@ static const hcall_t host_hcall[] = { > HANDLE_FUNC(__kvm_vcpu_run), > HANDLE_FUNC(__kvm_flush_vm_context), > HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa), > + HANDLE_FUNC(__kvm_tlb_flush_vmid_range), > HANDLE_FUNC(__kvm_tlb_flush_vmid), > HANDLE_FUNC(__kvm_flush_cpu_context), > HANDLE_FUNC(__kvm_timer_set_cntvoff), > diff --git a/arch/arm64/kvm/hyp/nvhe/tlb.c b/arch/arm64/kvm/hyp/nvhe/tlb.c > index 978179133f4b9..d4ea549c4b5c4 100644 > --- a/arch/arm64/kvm/hyp/nvhe/tlb.c > +++ b/arch/arm64/kvm/hyp/nvhe/tlb.c > @@ -130,6 +130,45 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, > __tlb_switch_to_host(&cxt); > } > > +void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, > + phys_addr_t start, phys_addr_t end) > +{ > + struct tlb_inv_context cxt; > + unsigned long pages, stride; > + > + /* > + * Since the range of addresses may not be mapped at > + * the same level, assume the worst case as PAGE_SIZE > + */ > + stride = PAGE_SIZE; > + start = round_down(start, stride); > + end = round_up(end, stride); > + pages = (end - start) >> PAGE_SHIFT; > + > + if (!system_supports_tlb_range() || pages >= MAX_TLBI_RANGE_PAGES) { > + __kvm_tlb_flush_vmid(mmu); > + return; Why do we give up on "pages >= MAX_TLBI_RANGE_PAGES"? I see no rationale for it in the patch. My understanding is that this is the maximum representable as a range, in which case this is a programming error. Or are you *on purpose* making the two equivalent? Thanks, M.
On Mon, May 29, 2023 at 6:54 AM Marc Zyngier <maz@kernel.org> wrote: > > On Fri, 19 May 2023 01:52:27 +0100, > Raghavendra Rao Ananta <rananta@google.com> wrote: > > > > Define __kvm_tlb_flush_vmid_range() (for VHE and nVHE) > > to flush a range of stage-2 page-tables using IPA in one go. > > If the system supports FEAT_TLBIRANGE, the following patches > > would conviniently replace global TLBI such as vmalls12e1is > > in the map, unmap, and dirty-logging paths with ripas2e1is > > instead. > > > > Signed-off-by: Raghavendra Rao Ananta <rananta@google.com> > > --- > > arch/arm64/include/asm/kvm_asm.h | 3 +++ > > arch/arm64/kvm/hyp/nvhe/hyp-main.c | 11 +++++++++ > > arch/arm64/kvm/hyp/nvhe/tlb.c | 39 ++++++++++++++++++++++++++++++ > > arch/arm64/kvm/hyp/vhe/tlb.c | 35 +++++++++++++++++++++++++++ > > 4 files changed, 88 insertions(+) > > > > diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h > > index 43c3bc0f9544d..33352d9399e32 100644 > > --- a/arch/arm64/include/asm/kvm_asm.h > > +++ b/arch/arm64/include/asm/kvm_asm.h > > @@ -79,6 +79,7 @@ enum __kvm_host_smccc_func { > > __KVM_HOST_SMCCC_FUNC___pkvm_init_vm, > > __KVM_HOST_SMCCC_FUNC___pkvm_init_vcpu, > > __KVM_HOST_SMCCC_FUNC___pkvm_teardown_vm, > > + __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_range, > > nit: please keep this close to the other TLB operations. > Sure, I'll reorder this. > > }; > > > > #define DECLARE_KVM_VHE_SYM(sym) extern char sym[] > > @@ -225,6 +226,8 @@ extern void __kvm_flush_vm_context(void); > > extern void __kvm_flush_cpu_context(struct kvm_s2_mmu *mmu); > > extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa, > > int level); > > +extern void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, > > + phys_addr_t start, phys_addr_t end); > > extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu); > > > > extern void __kvm_timer_set_cntvoff(u64 cntvoff); > > diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c > > index 728e01d4536b0..81d30737dc7c9 100644 > > --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c > > +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c > > @@ -125,6 +125,16 @@ static void handle___kvm_tlb_flush_vmid_ipa(struct kvm_cpu_context *host_ctxt) > > __kvm_tlb_flush_vmid_ipa(kern_hyp_va(mmu), ipa, level); > > } > > > > +static void > > +handle___kvm_tlb_flush_vmid_range(struct kvm_cpu_context *host_ctxt) > > +{ > > + DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); > > + DECLARE_REG(phys_addr_t, start, host_ctxt, 2); > > + DECLARE_REG(phys_addr_t, end, host_ctxt, 3); > > + > > + __kvm_tlb_flush_vmid_range(kern_hyp_va(mmu), start, end); > > +} > > + > > static void handle___kvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) > > { > > DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); > > @@ -315,6 +325,7 @@ static const hcall_t host_hcall[] = { > > HANDLE_FUNC(__kvm_vcpu_run), > > HANDLE_FUNC(__kvm_flush_vm_context), > > HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa), > > + HANDLE_FUNC(__kvm_tlb_flush_vmid_range), > > HANDLE_FUNC(__kvm_tlb_flush_vmid), > > HANDLE_FUNC(__kvm_flush_cpu_context), > > HANDLE_FUNC(__kvm_timer_set_cntvoff), > > diff --git a/arch/arm64/kvm/hyp/nvhe/tlb.c b/arch/arm64/kvm/hyp/nvhe/tlb.c > > index 978179133f4b9..d4ea549c4b5c4 100644 > > --- a/arch/arm64/kvm/hyp/nvhe/tlb.c > > +++ b/arch/arm64/kvm/hyp/nvhe/tlb.c > > @@ -130,6 +130,45 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, > > __tlb_switch_to_host(&cxt); > > } > > > > +void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, > > + phys_addr_t start, phys_addr_t end) > > +{ > > + struct tlb_inv_context cxt; > > + unsigned long pages, stride; > > + > > + /* > > + * Since the range of addresses may not be mapped at > > + * the same level, assume the worst case as PAGE_SIZE > > + */ > > + stride = PAGE_SIZE; > > + start = round_down(start, stride); > > + end = round_up(end, stride); > > + pages = (end - start) >> PAGE_SHIFT; > > + > > + if (!system_supports_tlb_range() || pages >= MAX_TLBI_RANGE_PAGES) { > > + __kvm_tlb_flush_vmid(mmu); > > + return; > > Why do we give up on "pages >= MAX_TLBI_RANGE_PAGES"? I see no > rationale for it in the patch. My understanding is that this is the > maximum representable as a range, in which case this is a programming > error. > > Or are you *on purpose* making the two equivalent? > Yes basically, I was trying to align the logic with what we have for __flush_tlb_range(). But, if you feel that it's mostly caused by a programming error, do we want to not do any flush at all? Thank you. Raghavendra > Thanks, > > M. > > -- > Without deviation from the norm, progress is not possible.
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 43c3bc0f9544d..33352d9399e32 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -79,6 +79,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_init_vm, __KVM_HOST_SMCCC_FUNC___pkvm_init_vcpu, __KVM_HOST_SMCCC_FUNC___pkvm_teardown_vm, + __KVM_HOST_SMCCC_FUNC___kvm_tlb_flush_vmid_range, }; #define DECLARE_KVM_VHE_SYM(sym) extern char sym[] @@ -225,6 +226,8 @@ extern void __kvm_flush_vm_context(void); extern void __kvm_flush_cpu_context(struct kvm_s2_mmu *mmu); extern void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, phys_addr_t ipa, int level); +extern void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, phys_addr_t end); extern void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu); extern void __kvm_timer_set_cntvoff(u64 cntvoff); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 728e01d4536b0..81d30737dc7c9 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -125,6 +125,16 @@ static void handle___kvm_tlb_flush_vmid_ipa(struct kvm_cpu_context *host_ctxt) __kvm_tlb_flush_vmid_ipa(kern_hyp_va(mmu), ipa, level); } +static void +handle___kvm_tlb_flush_vmid_range(struct kvm_cpu_context *host_ctxt) +{ + DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); + DECLARE_REG(phys_addr_t, start, host_ctxt, 2); + DECLARE_REG(phys_addr_t, end, host_ctxt, 3); + + __kvm_tlb_flush_vmid_range(kern_hyp_va(mmu), start, end); +} + static void handle___kvm_tlb_flush_vmid(struct kvm_cpu_context *host_ctxt) { DECLARE_REG(struct kvm_s2_mmu *, mmu, host_ctxt, 1); @@ -315,6 +325,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__kvm_vcpu_run), HANDLE_FUNC(__kvm_flush_vm_context), HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa), + HANDLE_FUNC(__kvm_tlb_flush_vmid_range), HANDLE_FUNC(__kvm_tlb_flush_vmid), HANDLE_FUNC(__kvm_flush_cpu_context), HANDLE_FUNC(__kvm_timer_set_cntvoff), diff --git a/arch/arm64/kvm/hyp/nvhe/tlb.c b/arch/arm64/kvm/hyp/nvhe/tlb.c index 978179133f4b9..d4ea549c4b5c4 100644 --- a/arch/arm64/kvm/hyp/nvhe/tlb.c +++ b/arch/arm64/kvm/hyp/nvhe/tlb.c @@ -130,6 +130,45 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, __tlb_switch_to_host(&cxt); } +void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, phys_addr_t end) +{ + struct tlb_inv_context cxt; + unsigned long pages, stride; + + /* + * Since the range of addresses may not be mapped at + * the same level, assume the worst case as PAGE_SIZE + */ + stride = PAGE_SIZE; + start = round_down(start, stride); + end = round_up(end, stride); + pages = (end - start) >> PAGE_SHIFT; + + if (!system_supports_tlb_range() || pages >= MAX_TLBI_RANGE_PAGES) { + __kvm_tlb_flush_vmid(mmu); + return; + } + + dsb(ishst); + + /* Switch to requested VMID */ + __tlb_switch_to_guest(mmu, &cxt); + + __flush_tlb_range_op(ipas2e1is, start, pages, stride, 0, 0, false); + + dsb(ish); + __tlbi(vmalle1is); + dsb(ish); + isb(); + + /* See the comment below in __kvm_tlb_flush_vmid_ipa() */ + if (icache_is_vpipt()) + icache_inval_all_pou(); + + __tlb_switch_to_host(&cxt); +} + void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu) { struct tlb_inv_context cxt; diff --git a/arch/arm64/kvm/hyp/vhe/tlb.c b/arch/arm64/kvm/hyp/vhe/tlb.c index 24cef9b87f9e9..f34d6dd9e4674 100644 --- a/arch/arm64/kvm/hyp/vhe/tlb.c +++ b/arch/arm64/kvm/hyp/vhe/tlb.c @@ -111,6 +111,41 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, __tlb_switch_to_host(&cxt); } +void __kvm_tlb_flush_vmid_range(struct kvm_s2_mmu *mmu, + phys_addr_t start, phys_addr_t end) +{ + struct tlb_inv_context cxt; + unsigned long pages, stride; + + /* + * Since the range of addresses may not be mapped at + * the same level, assume the worst case as PAGE_SIZE + */ + stride = PAGE_SIZE; + start = round_down(start, stride); + end = round_up(end, stride); + pages = (end - start) >> PAGE_SHIFT; + + if (!system_supports_tlb_range() || pages >= MAX_TLBI_RANGE_PAGES) { + __kvm_tlb_flush_vmid(mmu); + return; + } + + dsb(ishst); + + /* Switch to requested VMID */ + __tlb_switch_to_guest(mmu, &cxt); + + __flush_tlb_range_op(ipas2e1is, start, pages, stride, 0, 0, false); + + dsb(ish); + __tlbi(vmalle1is); + dsb(ish); + isb(); + + __tlb_switch_to_host(&cxt); +} + void __kvm_tlb_flush_vmid(struct kvm_s2_mmu *mmu) { struct tlb_inv_context cxt;
Define __kvm_tlb_flush_vmid_range() (for VHE and nVHE) to flush a range of stage-2 page-tables using IPA in one go. If the system supports FEAT_TLBIRANGE, the following patches would conviniently replace global TLBI such as vmalls12e1is in the map, unmap, and dirty-logging paths with ripas2e1is instead. Signed-off-by: Raghavendra Rao Ananta <rananta@google.com> --- arch/arm64/include/asm/kvm_asm.h | 3 +++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 11 +++++++++ arch/arm64/kvm/hyp/nvhe/tlb.c | 39 ++++++++++++++++++++++++++++++ arch/arm64/kvm/hyp/vhe/tlb.c | 35 +++++++++++++++++++++++++++ 4 files changed, 88 insertions(+)