| Message ID | 20231108183003.5981-6-xin3.li@intel.com (mailing list archive) |
|---|---|
| State | New |
| Headers | show |
| Series | Enable FRED with KVM VMX | expand |
On Wed, Nov 08, 2023 at 10:29:45AM -0800, Xin Li wrote: >Setup the global vmcs_config for FRED: >1) Add VM_ENTRY_LOAD_IA32_FRED to KVM_OPTIONAL_VMX_VM_ENTRY_CONTROLS to > have a FRED CPU load guest FRED MSRs from VMCS upon VM entry. >2) Add SECONDARY_VM_EXIT_SAVE_IA32_FRED to > KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS to have a FRED CPU save > guest FRED MSRs to VMCS during VM exit. >3) add SECONDARY_VM_EXIT_LOAD_IA32_FRED to > KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS to have a FRED CPU load > host FRED MSRs from VMCS during VM exit. > >Also add sanity checks to make sure FRED VM entry/exit controls can be >set on a FRED CPU. > >Tested-by: Shan Kang <shan.kang@intel.com> >Signed-off-by: Xin Li <xin3.li@intel.com> >--- > arch/x86/include/asm/vmx.h | 3 +++ > arch/x86/kvm/vmx/vmx.c | 19 ++++++++++++++++++- > arch/x86/kvm/vmx/vmx.h | 7 +++++-- > 3 files changed, 26 insertions(+), 3 deletions(-) > >diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h >index 4d4177ec802c..41796a733bc9 100644 >--- a/arch/x86/include/asm/vmx.h >+++ b/arch/x86/include/asm/vmx.h >@@ -106,6 +106,8 @@ > #define VM_EXIT_PT_CONCEAL_PIP 0x01000000 > #define VM_EXIT_CLEAR_IA32_RTIT_CTL 0x02000000 > #define VM_EXIT_ACTIVATE_SECONDARY_CONTROLS 0x80000000 >+#define SECONDARY_VM_EXIT_SAVE_IA32_FRED 0x00000001 >+#define SECONDARY_VM_EXIT_LOAD_IA32_FRED 0x00000002 > > #define VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR 0x00036dff > >@@ -119,6 +121,7 @@ > #define VM_ENTRY_LOAD_BNDCFGS 0x00010000 > #define VM_ENTRY_PT_CONCEAL_PIP 0x00020000 > #define VM_ENTRY_LOAD_IA32_RTIT_CTL 0x00040000 >+#define VM_ENTRY_LOAD_IA32_FRED 0x00800000 > > #define VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR 0x000011ff > >diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c >index df769207cbe0..9186f41974ab 100644 >--- a/arch/x86/kvm/vmx/vmx.c >+++ b/arch/x86/kvm/vmx/vmx.c >@@ -2694,10 +2694,27 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf, > _vmexit_control &= ~x_ctrl; > } > >- if (_vmexit_control & VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) >+ if (_vmexit_control & VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) { > _secondary_vmexit_control = > adjust_vmx_controls64(KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS, > MSR_IA32_VMX_EXIT_CTLS2); >+ if (cpu_feature_enabled(X86_FEATURE_FRED) && >+ !(_secondary_vmexit_control & SECONDARY_VM_EXIT_SAVE_IA32_FRED && >+ _secondary_vmexit_control & SECONDARY_VM_EXIT_LOAD_IA32_FRED)) { >+ pr_warn_once("FRED enabled but no VMX VM-Exit {SAVE,LOAD}_IA32_FRED controls: %llx\n", >+ _secondary_vmexit_control); if there is no VM_EXIT_ACTIVATE_SECONDARY_CONTROLS, shouldn't we also emit this warning? >+ if (error_on_inconsistent_vmcs_config) >+ return -EIO; >+ } >+ } >+ >+ if (cpu_feature_enabled(X86_FEATURE_FRED) && >+ !(_vmentry_control & VM_ENTRY_LOAD_IA32_FRED)) { >+ pr_warn_once("FRED enabled but no VMX VM-Entry LOAD_IA32_FRED control: %x\n", >+ _vmentry_control); Can we just hide FRED from guests like what KVM does for other features which have similar dependencies? see vmx_set_cpu_caps().
On Thu, Nov 09, 2023, Chao Gao wrote: > On Wed, Nov 08, 2023 at 10:29:45AM -0800, Xin Li wrote: > >Setup the global vmcs_config for FRED: > >1) Add VM_ENTRY_LOAD_IA32_FRED to KVM_OPTIONAL_VMX_VM_ENTRY_CONTROLS to > > have a FRED CPU load guest FRED MSRs from VMCS upon VM entry. > >2) Add SECONDARY_VM_EXIT_SAVE_IA32_FRED to > > KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS to have a FRED CPU save > > guest FRED MSRs to VMCS during VM exit. > >3) add SECONDARY_VM_EXIT_LOAD_IA32_FRED to > > KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS to have a FRED CPU load > > host FRED MSRs from VMCS during VM exit. > > > >Also add sanity checks to make sure FRED VM entry/exit controls can be > >set on a FRED CPU. > > > >Tested-by: Shan Kang <shan.kang@intel.com> > >Signed-off-by: Xin Li <xin3.li@intel.com> > >--- > > arch/x86/include/asm/vmx.h | 3 +++ > > arch/x86/kvm/vmx/vmx.c | 19 ++++++++++++++++++- > > arch/x86/kvm/vmx/vmx.h | 7 +++++-- > > 3 files changed, 26 insertions(+), 3 deletions(-) > > > >diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h > >index 4d4177ec802c..41796a733bc9 100644 > >--- a/arch/x86/include/asm/vmx.h > >+++ b/arch/x86/include/asm/vmx.h > >@@ -106,6 +106,8 @@ > > #define VM_EXIT_PT_CONCEAL_PIP 0x01000000 > > #define VM_EXIT_CLEAR_IA32_RTIT_CTL 0x02000000 > > #define VM_EXIT_ACTIVATE_SECONDARY_CONTROLS 0x80000000 > >+#define SECONDARY_VM_EXIT_SAVE_IA32_FRED 0x00000001 > >+#define SECONDARY_VM_EXIT_LOAD_IA32_FRED 0x00000002 > > > > #define VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR 0x00036dff > > > >@@ -119,6 +121,7 @@ > > #define VM_ENTRY_LOAD_BNDCFGS 0x00010000 > > #define VM_ENTRY_PT_CONCEAL_PIP 0x00020000 > > #define VM_ENTRY_LOAD_IA32_RTIT_CTL 0x00040000 > >+#define VM_ENTRY_LOAD_IA32_FRED 0x00800000 > > > > #define VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR 0x000011ff > > > >diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c > >index df769207cbe0..9186f41974ab 100644 > >--- a/arch/x86/kvm/vmx/vmx.c > >+++ b/arch/x86/kvm/vmx/vmx.c > >@@ -2694,10 +2694,27 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf, > > _vmexit_control &= ~x_ctrl; > > } > > > >- if (_vmexit_control & VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) > >+ if (_vmexit_control & VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) { > > _secondary_vmexit_control = > > adjust_vmx_controls64(KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS, > > MSR_IA32_VMX_EXIT_CTLS2); > >+ if (cpu_feature_enabled(X86_FEATURE_FRED) && > >+ !(_secondary_vmexit_control & SECONDARY_VM_EXIT_SAVE_IA32_FRED && > >+ _secondary_vmexit_control & SECONDARY_VM_EXIT_LOAD_IA32_FRED)) { > >+ pr_warn_once("FRED enabled but no VMX VM-Exit {SAVE,LOAD}_IA32_FRED controls: %llx\n", > >+ _secondary_vmexit_control); > > if there is no VM_EXIT_ACTIVATE_SECONDARY_CONTROLS, shouldn't we also emit this > warning? > > >+ if (error_on_inconsistent_vmcs_config) > >+ return -EIO; > >+ } > >+ } > >+ > >+ if (cpu_feature_enabled(X86_FEATURE_FRED) && > >+ !(_vmentry_control & VM_ENTRY_LOAD_IA32_FRED)) { > >+ pr_warn_once("FRED enabled but no VMX VM-Entry LOAD_IA32_FRED control: %x\n", > >+ _vmentry_control); > > Can we just hide FRED from guests like what KVM does for other features which > have similar dependencies? see vmx_set_cpu_caps(). Both of these warnings should simply be dropped. The error_on_inconsistent_vmcs_config stuff is for inconsistencies within the allowed VMCS fields. Having a feature that is supported in bare metal but not virtualized is perfectly legal, if uncommon. What *is* needed is for KVM to refuse to virtualize FRED if the entry/exit controls aren't consistent. E.g. if at least one control is present, and at least one control is missing. I.e. KVM needs a version of vmcs_entry_exit_pairs that can deal with SECONDAY_VM_EXIT controls. I'll circle back to this when I give the series a proper review, which is going to be 3+ weeks.
> > >+ if (cpu_feature_enabled(X86_FEATURE_FRED) && > > >+ !(_vmentry_control & VM_ENTRY_LOAD_IA32_FRED)) { > > >+ pr_warn_once("FRED enabled but no VMX VM-Entry > LOAD_IA32_FRED control: %x\n", > > >+ _vmentry_control); > > > > Can we just hide FRED from guests like what KVM does for other > > features which have similar dependencies? see vmx_set_cpu_caps(). > > Both of these warnings should simply be dropped. The > error_on_inconsistent_vmcs_config stuff is for inconsistencies within the allowed > VMCS fields. Having a feature that is supported in bare metal but not virtualized > is perfectly legal, if uncommon. I deliberately keep it, at least for now, because these checks are helpful during the development of nVMX FRED. It will be helpful for other VMMs being developed/tested on KVM. > What *is* needed is for KVM to refuse to virtualize FRED if the entry/exit controls > aren't consistent. E.g. if at least one control is present, and at least one > control is missing. I.e. KVM needs a version of vmcs_entry_exit_pairs that can > deal with SECONDAY_VM_EXIT controls. I agree there are better ways. But maybe after or before VMX FRED. > I'll circle back to this when I give the > series a proper review, which is going to be 3+ weeks. The traffic in KVM mailing list is surprisingly high recently. So that is totally expected.
On Fri, Nov 10, 2023, Xin3 Li wrote: > > > >+ if (cpu_feature_enabled(X86_FEATURE_FRED) && > > > >+ !(_vmentry_control & VM_ENTRY_LOAD_IA32_FRED)) { > > > >+ pr_warn_once("FRED enabled but no VMX VM-Entry > > LOAD_IA32_FRED control: %x\n", > > > >+ _vmentry_control); > > > > > > Can we just hide FRED from guests like what KVM does for other > > > features which have similar dependencies? see vmx_set_cpu_caps(). > > > > Both of these warnings should simply be dropped. The > > error_on_inconsistent_vmcs_config stuff is for inconsistencies within the allowed > > VMCS fields. Having a feature that is supported in bare metal but not virtualized > > is perfectly legal, if uncommon. > > I deliberately keep it, at least for now, because these checks are helpful > during the development of nVMX FRED. It will be helpful for other VMMs > being developed/tested on KVM. No, remove it. It's architecturally legal for a CPU to support a feature in bare metal but not provide virtualization support. > > What *is* needed is for KVM to refuse to virtualize FRED if the entry/exit controls > > aren't consistent. E.g. if at least one control is present, and at least one > > control is missing. I.e. KVM needs a version of vmcs_entry_exit_pairs that can > > deal with SECONDAY_VM_EXIT controls. > > I agree there are better ways. But maybe after or before VMX FRED. Uh, no. This is not optional. FRED is the first feature that uses SECONDAY_VM_EXIT controls, so FRED gets the honor of adding the necessary infrastructure support.
On 8.11.23 г. 20:29 ч., Xin Li wrote: > Setup the global vmcs_config for FRED: > 1) Add VM_ENTRY_LOAD_IA32_FRED to KVM_OPTIONAL_VMX_VM_ENTRY_CONTROLS to > have a FRED CPU load guest FRED MSRs from VMCS upon VM entry. > 2) Add SECONDARY_VM_EXIT_SAVE_IA32_FRED to > KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS to have a FRED CPU save > guest FRED MSRs to VMCS during VM exit. > 3) add SECONDARY_VM_EXIT_LOAD_IA32_FRED to > KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS to have a FRED CPU load > host FRED MSRs from VMCS during VM exit. > > Also add sanity checks to make sure FRED VM entry/exit controls can be > set on a FRED CPU. > > Tested-by: Shan Kang <shan.kang@intel.com> > Signed-off-by: Xin Li <xin3.li@intel.com> > --- > arch/x86/include/asm/vmx.h | 3 +++ > arch/x86/kvm/vmx/vmx.c | 19 ++++++++++++++++++- > arch/x86/kvm/vmx/vmx.h | 7 +++++-- > 3 files changed, 26 insertions(+), 3 deletions(-) > > diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h > index 4d4177ec802c..41796a733bc9 100644 > --- a/arch/x86/include/asm/vmx.h > +++ b/arch/x86/include/asm/vmx.h > @@ -106,6 +106,8 @@ > #define VM_EXIT_PT_CONCEAL_PIP 0x01000000 > #define VM_EXIT_CLEAR_IA32_RTIT_CTL 0x02000000 > #define VM_EXIT_ACTIVATE_SECONDARY_CONTROLS 0x80000000 > +#define SECONDARY_VM_EXIT_SAVE_IA32_FRED 0x00000001 > +#define SECONDARY_VM_EXIT_LOAD_IA32_FRED 0x00000002 > > #define VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR 0x00036dff > > @@ -119,6 +121,7 @@ > #define VM_ENTRY_LOAD_BNDCFGS 0x00010000 > #define VM_ENTRY_PT_CONCEAL_PIP 0x00020000 > #define VM_ENTRY_LOAD_IA32_RTIT_CTL 0x00040000 > +#define VM_ENTRY_LOAD_IA32_FRED 0x00800000 > > #define VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR 0x000011ff > > diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c > index df769207cbe0..9186f41974ab 100644 > --- a/arch/x86/kvm/vmx/vmx.c > +++ b/arch/x86/kvm/vmx/vmx.c > @@ -2694,10 +2694,27 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf, > _vmexit_control &= ~x_ctrl; > } > > - if (_vmexit_control & VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) > + if (_vmexit_control & VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) { > _secondary_vmexit_control = > adjust_vmx_controls64(KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS, > MSR_IA32_VMX_EXIT_CTLS2); > + if (cpu_feature_enabled(X86_FEATURE_FRED) && > + !(_secondary_vmexit_control & SECONDARY_VM_EXIT_SAVE_IA32_FRED && > + _secondary_vmexit_control & SECONDARY_VM_EXIT_LOAD_IA32_FRED)) { Can those checks actually trigger? I.e if FEATURE_FRED is set it means the CPU implements the FRED spec. According to the spec it's guaranteed that VMX_EXIT_CTLS2 will contain those bits set to 1. So aren't those checks superfluous? > + pr_warn_once("FRED enabled but no VMX VM-Exit {SAVE,LOAD}_IA32_FRED controls: %llx\n", > + _secondary_vmexit_control); > + if (error_on_inconsistent_vmcs_config) > + return -EIO; > + } > + } > + > + if (cpu_feature_enabled(X86_FEATURE_FRED) && > + !(_vmentry_control & VM_ENTRY_LOAD_IA32_FRED)) { DITTO <snip>
> > > > Can we just hide FRED from guests like what KVM does for other > > > > features which have similar dependencies? see vmx_set_cpu_caps(). > > > > > > Both of these warnings should simply be dropped. The > > > error_on_inconsistent_vmcs_config stuff is for inconsistencies within the > allowed > > > VMCS fields. Having a feature that is supported in bare metal but not > virtualized > > > is perfectly legal, if uncommon. > > > > I deliberately keep it, at least for now, because these checks are helpful > > during the development of nVMX FRED. It will be helpful for other VMMs > > being developed/tested on KVM. > > No, remove it. It's architecturally legal for a CPU to support a feature in bare > metal but not provide virtualization support. Like the stage when native Linux has FRED support while KVM not yet? > > > What *is* needed is for KVM to refuse to virtualize FRED if the entry/exit > controls > > > aren't consistent. E.g. if at least one control is present, and at least one > > > control is missing. I.e. KVM needs a version of vmcs_entry_exit_pairs that can > > > deal with SECONDAY_VM_EXIT controls. > > > > I agree there are better ways. But maybe after or before VMX FRED. > > Uh, no. This is not optional. FRED is the first feature that uses > SECONDAY_VM_EXIT > controls, so FRED gets the honor of adding the necessary infrastructure support. The 2nd VM exit controls is a must for FRED, so I should do it. I think you mean the consistency checks can be done in a better way (which is not just for FRED controls consistency checks). No? Thanks! Xin
> > - if (_vmexit_control & VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) > > + if (_vmexit_control & VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) { > > _secondary_vmexit_control = > > > adjust_vmx_controls64(KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CO > NTROLS, > > MSR_IA32_VMX_EXIT_CTLS2); > > + if (cpu_feature_enabled(X86_FEATURE_FRED) && > > + !(_secondary_vmexit_control & > SECONDARY_VM_EXIT_SAVE_IA32_FRED && > > + _secondary_vmexit_control & > SECONDARY_VM_EXIT_LOAD_IA32_FRED)) { > > Can those checks actually trigger? I.e if FEATURE_FRED is set it means > the CPU implements the FRED spec. According to the spec it's guaranteed > that VMX_EXIT_CTLS2 will contain those bits set to 1. So aren't those > checks superfluous? Such checks are for nVMX FRED. > > + if (cpu_feature_enabled(X86_FEATURE_FRED) && > > + !(_vmentry_control & VM_ENTRY_LOAD_IA32_FRED)) { > DITTO > > <snip>
diff --git a/arch/x86/include/asm/vmx.h b/arch/x86/include/asm/vmx.h index 4d4177ec802c..41796a733bc9 100644 --- a/arch/x86/include/asm/vmx.h +++ b/arch/x86/include/asm/vmx.h @@ -106,6 +106,8 @@ #define VM_EXIT_PT_CONCEAL_PIP 0x01000000 #define VM_EXIT_CLEAR_IA32_RTIT_CTL 0x02000000 #define VM_EXIT_ACTIVATE_SECONDARY_CONTROLS 0x80000000 +#define SECONDARY_VM_EXIT_SAVE_IA32_FRED 0x00000001 +#define SECONDARY_VM_EXIT_LOAD_IA32_FRED 0x00000002 #define VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR 0x00036dff @@ -119,6 +121,7 @@ #define VM_ENTRY_LOAD_BNDCFGS 0x00010000 #define VM_ENTRY_PT_CONCEAL_PIP 0x00020000 #define VM_ENTRY_LOAD_IA32_RTIT_CTL 0x00040000 +#define VM_ENTRY_LOAD_IA32_FRED 0x00800000 #define VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR 0x000011ff diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index df769207cbe0..9186f41974ab 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2694,10 +2694,27 @@ static int setup_vmcs_config(struct vmcs_config *vmcs_conf, _vmexit_control &= ~x_ctrl; } - if (_vmexit_control & VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) + if (_vmexit_control & VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) { _secondary_vmexit_control = adjust_vmx_controls64(KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS, MSR_IA32_VMX_EXIT_CTLS2); + if (cpu_feature_enabled(X86_FEATURE_FRED) && + !(_secondary_vmexit_control & SECONDARY_VM_EXIT_SAVE_IA32_FRED && + _secondary_vmexit_control & SECONDARY_VM_EXIT_LOAD_IA32_FRED)) { + pr_warn_once("FRED enabled but no VMX VM-Exit {SAVE,LOAD}_IA32_FRED controls: %llx\n", + _secondary_vmexit_control); + if (error_on_inconsistent_vmcs_config) + return -EIO; + } + } + + if (cpu_feature_enabled(X86_FEATURE_FRED) && + !(_vmentry_control & VM_ENTRY_LOAD_IA32_FRED)) { + pr_warn_once("FRED enabled but no VMX VM-Entry LOAD_IA32_FRED control: %x\n", + _vmentry_control); + if (error_on_inconsistent_vmcs_config) + return -EIO; + } rdmsrl(MSR_IA32_VMX_BASIC, basic_msr); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 99a0f6783085..f8c02bd37069 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -480,7 +480,8 @@ static inline u8 vmx_get_rvi(void) VM_ENTRY_LOAD_IA32_EFER | \ VM_ENTRY_LOAD_BNDCFGS | \ VM_ENTRY_PT_CONCEAL_PIP | \ - VM_ENTRY_LOAD_IA32_RTIT_CTL) + VM_ENTRY_LOAD_IA32_RTIT_CTL | \ + VM_ENTRY_LOAD_IA32_FRED) #define __KVM_REQUIRED_VMX_VM_EXIT_CONTROLS \ (VM_EXIT_SAVE_DEBUG_CONTROLS | \ @@ -506,7 +507,9 @@ static inline u8 vmx_get_rvi(void) VM_EXIT_ACTIVATE_SECONDARY_CONTROLS) #define KVM_REQUIRED_VMX_SECONDARY_VM_EXIT_CONTROLS (0) -#define KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS (0) +#define KVM_OPTIONAL_VMX_SECONDARY_VM_EXIT_CONTROLS \ + (SECONDARY_VM_EXIT_SAVE_IA32_FRED | \ + SECONDARY_VM_EXIT_LOAD_IA32_FRED) #define KVM_REQUIRED_VMX_PIN_BASED_VM_EXEC_CONTROL \ (PIN_BASED_EXT_INTR_MASK | \