@@ -278,6 +278,12 @@ static inline int kvm_arch_vm_ioctl_check_extension(struct kvm *kvm, long ext)
return 0;
}
+static inline int kvm_arm_vcpu_sve_config(struct kvm_vcpu *vcpu,
+ struct kvm_sve_vls *userp)
+{
+ return -EINVAL;
+}
+
int kvm_perf_init(void);
int kvm_perf_teardown(void);
@@ -302,6 +308,7 @@ static inline bool kvm_arm_handle_step_debug(struct kvm_vcpu *vcpu,
return false;
}
+static inline void kvm_arm_vcpu_free_sve(struct kvm_vcpu *vcpu) {}
int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu,
struct kvm_device_attr *attr);
int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
@@ -365,6 +372,11 @@ static inline int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long *type)
return 0;
}
+static inline int kvm_arm_setup_vm(struct kvm *kvm, unsigned long *type)
+{
+ return 0;
+}
+
/* Forbid "ordinary" vcpu ioctls if this returns true: */
#define vcpu_needs_configuration(vcpu) false
@@ -106,6 +106,9 @@ struct kvm_vcpu_init {
__u32 features[7];
};
+struct kvm_sve_vls {
+};
+
struct kvm_sregs {
};
@@ -52,9 +52,17 @@
DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
+#ifdef CONFIG_ARM64_SVE
+bool kvm_sve_supported(void);
+#else
+static inline bool kvm_sve_supported(void) { return false; }
+#endif
+
int __attribute_const__ kvm_target_cpu(void);
int kvm_reset_vcpu(struct kvm_vcpu *vcpu);
int kvm_arch_vm_ioctl_check_extension(struct kvm *kvm, long ext);
+int kvm_arm_vcpu_sve_config(struct kvm_vcpu *vcpu,
+ struct kvm_sve_vls __user *userp);
void __extended_idmap_trampoline(phys_addr_t boot_pgd, phys_addr_t idmap_start);
struct kvm_arch {
@@ -81,10 +89,15 @@ struct kvm_arch {
/* Mandated version of PSCI */
u32 psci_version;
+
+ /* KVM_ARM64_VM_* flags */
+ int flags;
};
#define KVM_NR_MEM_OBJS 40
+#define KVM_ARM64_VM_SVE_API (1 << 0)
+
/*
* We don't want allocation failures within the mmu code, so we preallocate
* enough memory for a single page fault in a cache.
@@ -317,6 +330,7 @@ struct kvm_vcpu_arch {
#define KVM_ARM64_HOST_SVE_IN_USE (1 << 3) /* backup for host TIF_SVE */
#define KVM_ARM64_HOST_SVE_ENABLED (1 << 4) /* SVE enabled for EL0 */
#define KVM_ARM64_GUEST_HAS_SVE (1 << 5) /* SVE exposed to guest */
+#define KVM_ARM64_VCPU_SVE_CONFIGURED (1 << 6) /* SVE configuration done */
#define vcpu_has_sve(vcpu) (system_supports_sve() && \
((vcpu)->arch.flags & KVM_ARM64_GUEST_HAS_SVE))
@@ -458,6 +472,7 @@ void kvm_arm_setup_debug(struct kvm_vcpu *vcpu);
void kvm_arm_clear_debug(struct kvm_vcpu *vcpu);
void kvm_arm_reset_debug_ptr(struct kvm_vcpu *vcpu);
bool kvm_arm_handle_step_debug(struct kvm_vcpu *vcpu, struct kvm_run *run);
+void kvm_arm_vcpu_free_sve(struct kvm_vcpu *vcpu);
int kvm_arm_vcpu_arch_set_attr(struct kvm_vcpu *vcpu,
struct kvm_device_attr *attr);
int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu,
@@ -535,8 +550,16 @@ struct kvm *kvm_arch_alloc_vm(void);
void kvm_arch_free_vm(struct kvm *kvm);
int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long *type);
+int kvm_arm_setup_vm(struct kvm *kvm, unsigned long *type);
+
+#define vcpu_using_sve_api(vcpu) \
+ (!!((vcpu)->kvm->arch.flags & KVM_ARM64_VM_SVE_API))
+
+#define vcpu_sve_config_done(vcpu) \
+ (!!((vcpu)->arch.flags & KVM_ARM64_VCPU_SVE_CONFIGURED))
/* Forbid "ordinary" vcpu ioctls if this returns true: */
-#define vcpu_needs_configuration(vcpu) false
+#define vcpu_needs_configuration(vcpu) \
+ (vcpu_using_sve_api(vcpu) && !vcpu_sve_config_done(vcpu))
#endif /* __ARM64_KVM_HOST_H__ */
@@ -32,6 +32,7 @@
#define KVM_NR_SPSR 5
#ifndef __ASSEMBLY__
+#include <linux/kernel.h>
#include <linux/psci.h>
#include <linux/types.h>
#include <asm/ptrace.h>
@@ -108,6 +109,19 @@ struct kvm_vcpu_init {
__u32 features[7];
};
+/* Vector length set for KVM_ARM_SVE_CONFIG */
+struct kvm_sve_vls {
+ __u16 cmd;
+ __u16 max_vq;
+ __u16 _reserved[2];
+ __u64 required_vqs[__KERNEL_DIV_ROUND_UP(SVE_VQ_MAX - SVE_VQ_MIN + 1, 64)];
+};
+
+/* values for cmd: */
+#define KVM_ARM_SVE_CONFIG_QUERY 0 /* query what the host can support */
+#define KVM_ARM_SVE_CONFIG_SET 1 /* enable SVE for vcpu and set VLs */
+#define KVM_ARM_SVE_CONFIG_GET 2 /* read the set of VLs for a vcpu */
+
struct kvm_sregs {
};
@@ -27,6 +27,9 @@
#include <linux/module.h>
#include <linux/vmalloc.h>
#include <linux/fs.h>
+#include <linux/slab.h>
+#include <linux/string.h>
+#include <linux/types.h>
#include <kvm/arm_psci.h>
#include <asm/cputype.h>
#include <linux/uaccess.h>
@@ -57,6 +60,11 @@ int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu)
return 0;
}
+void kvm_arm_vcpu_free_sve(struct kvm_vcpu *vcpu)
+{
+ kfree(vcpu->arch.sve_state);
+}
+
static u64 core_reg_offset_from_id(u64 id)
{
return id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_CORE);
@@ -647,6 +655,161 @@ int kvm_vcpu_preferred_target(struct kvm_vcpu_init *init)
return 0;
}
+#define VQS_PER_U64 64
+#define vq_word(vqs, vq) (&(vqs)[((vq) - SVE_VQ_MIN) / VQS_PER_U64])
+#define vq_mask(vq) ((u64)1 << (((vq) - SVE_VQ_MIN) % VQS_PER_U64))
+
+static void set_vq(u64 *vqs, unsigned int vq)
+{
+ *vq_word(vqs, vq) |= vq_mask(vq);
+}
+
+static bool vq_set(const u64 *vqs, unsigned int vq)
+{
+ return *vq_word(vqs, vq) & vq_mask(vq);
+}
+
+static int kvm_vcpu_set_sve_vls(struct kvm_vcpu *vcpu, struct kvm_sve_vls *vls,
+ struct kvm_sve_vls __user *userp)
+{
+ unsigned int vq, max_vq;
+ int ret;
+
+ if (vcpu->arch.has_run_once || vcpu_sve_config_done(vcpu))
+ return -EBADFD; /* too late, or already configured */
+
+ BUG_ON(vcpu->arch.sve_max_vl || vcpu->arch.sve_state);
+
+ /* max_vq == 0 disables SVE for this vcpu */
+ if (!vls->max_vq) {
+ vcpu->arch.flags |= KVM_ARM64_VCPU_SVE_CONFIGURED;
+ return 0;
+ }
+
+ /* Otherwise, try to enable SVE with the requested set of VLs: */
+
+ if (vls->max_vq < SVE_VQ_MIN || vls->max_vq > SVE_VQ_MAX)
+ return -EINVAL;
+
+ max_vq = 0;
+ for (vq = SVE_VQ_MIN; vq <= vls->max_vq; ++vq) {
+ bool available = sve_vq_available(vq);
+ bool required = vq_set(vls->required_vqs, vq);
+
+ if (required != available)
+ break;
+
+ if (required)
+ max_vq = vq;
+ }
+
+ if (max_vq < SVE_VQ_MIN)
+ return -EINVAL;
+
+ vls->max_vq = max_vq;
+ ret = put_user(vls->max_vq, &userp->max_vq);
+ if (ret)
+ return ret;
+
+ /*
+ * kvm_reset_vcpu() may already have run in KVM_VCPU_INIT, so we
+ * rely on kzalloc() being sufficient to reset the guest SVE
+ * state here for a new vcpu.
+ *
+ * Subsequent resets after vcpu initialisation are handled by
+ * kvm_reset_sve().
+ */
+ vcpu->arch.sve_state = kzalloc(SVE_SIG_REGS_SIZE(vls->max_vq),
+ GFP_KERNEL);
+ if (!vcpu->arch.sve_state)
+ return -ENOMEM;
+
+ vcpu->arch.flags |= KVM_ARM64_GUEST_HAS_SVE |
+ KVM_ARM64_VCPU_SVE_CONFIGURED;
+ vcpu->arch.sve_max_vl = sve_vl_from_vq(vls->max_vq);
+
+ return 0;
+}
+
+static int __kvm_vcpu_query_sve_vls(struct kvm_sve_vls *vls,
+ unsigned int max_vq, struct kvm_sve_vls __user *userp)
+{
+ unsigned int vq, max_available_vq;
+
+ memset(&vls->required_vqs, 0, sizeof(vls->required_vqs));
+
+ BUG_ON(max_vq < SVE_VQ_MIN || max_vq > SVE_VQ_MAX);
+
+ max_available_vq = 0;
+ for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq)
+ if (sve_vq_available(vq)) {
+ set_vq(vls->required_vqs, vq);
+ max_available_vq = vq;
+ }
+
+ if (WARN_ON(max_available_vq < SVE_VQ_MIN))
+ return -EIO;
+
+ vls->max_vq = max_available_vq;
+ if (copy_to_user(userp, vls, sizeof(*vls)))
+ return -EFAULT;
+
+ return 0;
+}
+
+static int kvm_vcpu_query_sve_vls(struct kvm_vcpu *vcpu, struct kvm_sve_vls *vls,
+ struct kvm_sve_vls __user *userp)
+{
+ BUG_ON(!sve_vl_valid(sve_max_vl));
+
+ return __kvm_vcpu_query_sve_vls(vls,
+ sve_vq_from_vl(sve_max_vl), userp);
+}
+
+static int kvm_vcpu_get_sve_vls(struct kvm_vcpu *vcpu, struct kvm_sve_vls *vls,
+ struct kvm_sve_vls __user *userp)
+{
+ if (!vcpu_sve_config_done(vcpu))
+ return -EBADFD; /* not configured yet */
+
+ BUG_ON(!sve_vl_valid(vcpu->arch.sve_max_vl));
+
+ return __kvm_vcpu_query_sve_vls(vls,
+ sve_vq_from_vl(vcpu->arch.sve_max_vl), userp);
+}
+
+int kvm_arm_vcpu_sve_config(struct kvm_vcpu *vcpu,
+ struct kvm_sve_vls __user *userp)
+{
+ struct kvm_sve_vls vls;
+
+ if (!kvm_sve_supported() || !vcpu_using_sve_api(vcpu))
+ return -EINVAL;
+
+ if (copy_from_user(&vls, userp, sizeof(vls)))
+ return -EFAULT;
+
+ /*
+ * For forwards compatibility, flush any set bits in _reserved[]
+ * to tell userspace that we didn't look at them:
+ */
+ memset(&vls._reserved, 0, sizeof vls._reserved);
+
+ switch (vls.cmd) {
+ case KVM_ARM_SVE_CONFIG_QUERY:
+ return kvm_vcpu_query_sve_vls(vcpu, &vls, userp);
+
+ case KVM_ARM_SVE_CONFIG_SET:
+ return kvm_vcpu_set_sve_vls(vcpu, &vls, userp);
+
+ case KVM_ARM_SVE_CONFIG_GET:
+ return kvm_vcpu_get_sve_vls(vcpu, &vls, userp);
+
+ default:
+ return -EINVAL;
+ }
+}
+
int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
{
return -EINVAL;
@@ -23,6 +23,7 @@
#include <linux/kvm_host.h>
#include <linux/kvm.h>
#include <linux/hw_breakpoint.h>
+#include <linux/string.h>
#include <kvm/arm_arch_timer.h>
@@ -58,6 +59,28 @@ static bool cpu_has_32bit_el1(void)
return !!(pfr0 & 0x20);
}
+#ifdef CONFIG_ARM64_SVE
+bool kvm_sve_supported(void)
+{
+ if (!system_supports_sve())
+ return false;
+
+ /*
+ * For now, consider the hardware broken if implementation
+ * differences between CPUs in the system result in the set of
+ * vector lengths safely virtualisable for guests being less
+ * than the set provided to userspace:
+ */
+ if (sve_max_virtualisable_vl != sve_max_vl) {
+ printk_once(KERN_ERR "kvm [%i]: Hardware SVE implementations mismatched: suppressing SVE for guests.\n",
+ task_pid_nr(current));
+ return false;
+ }
+
+ return true;
+}
+#endif
+
/**
* kvm_arch_vm_ioctl_check_extension
*
@@ -90,6 +113,8 @@ int kvm_arch_vm_ioctl_check_extension(struct kvm *kvm, long ext)
break;
case KVM_CAP_ARM_VM_IPA_SIZE:
r = kvm_ipa_limit;
+ case KVM_CAP_ARM_SVE:
+ r = kvm_sve_supported();
break;
default:
r = 0;
@@ -98,6 +123,21 @@ int kvm_arch_vm_ioctl_check_extension(struct kvm *kvm, long ext)
return r;
}
+int kvm_reset_sve(struct kvm_vcpu *vcpu)
+{
+ if (!vcpu_has_sve(vcpu))
+ return 0;
+
+ if (WARN_ON(!vcpu->arch.sve_state ||
+ !sve_vl_valid(vcpu->arch.sve_max_vl)))
+ return -EIO;
+
+ memset(vcpu->arch.sve_state, 0,
+ SVE_SIG_REGS_SIZE(sve_vq_from_vl(vcpu->arch.sve_max_vl)));
+
+ return 0;
+}
+
/**
* kvm_reset_vcpu - sets core registers and sys_regs to reset value
* @vcpu: The VCPU pointer
@@ -109,6 +149,7 @@ int kvm_arch_vm_ioctl_check_extension(struct kvm *kvm, long ext)
int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
{
const struct kvm_regs *cpu_reset;
+ int ret;
switch (vcpu->arch.target) {
default:
@@ -126,6 +167,10 @@ int kvm_reset_vcpu(struct kvm_vcpu *vcpu)
/* Reset core registers */
memcpy(vcpu_gp_regs(vcpu), cpu_reset, sizeof(*cpu_reset));
+ ret = kvm_reset_sve(vcpu);
+ if (ret)
+ return ret;
+
/* Reset system registers */
kvm_reset_sys_regs(vcpu);
@@ -233,3 +278,19 @@ int kvm_arm_setup_stage2(struct kvm *kvm, unsigned long *type)
kvm->arch.vtcr = vtcr;
return 0;
}
+
+/*
+ * Additional machine-type dependent setup: mark this VM as using the
+ * SVE KVM API if requested.
+ */
+int kvm_arm_setup_vm(struct kvm *kvm, unsigned long *type)
+{
+ unsigned long flag = *type & KVM_VM_TYPE_ARM_SVE;
+
+ if (flag) {
+ kvm->arch.flags |= KVM_ARM64_VM_SVE_API;
+ *type &= ~flag;
+ }
+
+ return 0;
+}
@@ -766,6 +766,15 @@ struct kvm_ppc_resize_hpt {
#define KVM_VM_TYPE_ARM_IPA_SIZE_MASK 0xffULL
#define KVM_VM_TYPE_ARM_IPA_SIZE(x) \
((x) & KVM_VM_TYPE_ARM_IPA_SIZE_MASK)
+
+/*
+ * On arm64, enables the SVE API extensions for KVM.
+ * This enables the KVM_ARM_SVE_CONFIG ioctl. With this flag set,
+ * KVM_ARM_SVE_CONFIG_SET must be used to configure each newly-created
+ * vcpu before further ioctls will work on it.
+ */
+#define KVM_VM_TYPE_ARM_SVE (1 << 8)
+
/*
* ioctls for /dev/kvm fds:
*/
@@ -975,6 +984,7 @@ struct kvm_ppc_resize_hpt {
#define KVM_CAP_HYPERV_ENLIGHTENED_VMCS 163
#define KVM_CAP_EXCEPTION_PAYLOAD 164
#define KVM_CAP_ARM_VM_IPA_SIZE 165
+#define KVM_CAP_ARM_SVE 166
#ifdef KVM_CAP_IRQ_ROUTING
@@ -1422,6 +1432,9 @@ struct kvm_enc_region {
#define KVM_GET_NESTED_STATE _IOWR(KVMIO, 0xbe, struct kvm_nested_state)
#define KVM_SET_NESTED_STATE _IOW(KVMIO, 0xbf, struct kvm_nested_state)
+/* Available with KVM_CAP_ARM_SVE */
+#define KVM_ARM_SVE_CONFIG _IOWR(KVMIO, 0xc0, struct kvm_sve_vls)
+
/* Secure Encrypted Virtualization command */
enum sev_cmd_id {
/* Guest initialization commands */
@@ -1090,7 +1090,8 @@ long kvm_arch_vcpu_ioctl(struct file *filp,
struct kvm_device_attr attr;
long r;
- /* Early configuration ioctls will be handled here */
+ if (ioctl == KVM_ARM_SVE_CONFIG)
+ return kvm_arm_vcpu_sve_config(vcpu, argp);
/* Other ioctls require configuration to have been done first: */
if (vcpu_needs_configuration(vcpu))
This patch adds the necessary API extensions to allow userspace to detect SVE support for guests and enable it. A new capability KVM_CAP_ARM_SVE is defined to allow userspace to detect the availability of the KVM SVE API extensions in the usual way. In addition, userspace must opt into these extensions by passing the new KVM_VM_TYPE_ARM_SVE flag to KVM_CREATE_VM. Userspace needs to enable or disable SVE explicitly per vcpu and configure the set of SVE vector lengths available to the guest before non-trivial ioctls (including KVM_ARM_VCPU_INIT and KVM_RUN) are allowed on the vcpu. For these purposes, a new arm64-specific vcpu ioctl KVM_ARM_SVE_CONFIG is added, with the following subcommands (in rough order of expected use): KVM_ARM_SVE_CONFIG_QUERY: report the set of vector lengths supported by this host. The resulting set can be supplied directly to KVM_ARM_SVE_CONFIG_SET in order to obtain the maximal possible set, or used to inform userspace's decision on the appropriate set of vector lengths (possibly taking into account the configuration of other nodes in the cluster so that the VM can migrate freely). KVM_ARM_SVE_CONFIG_SET: enable or disable SVE for this vcpu and configure the set of vector lengths it offers to the guest (in the enabled case). This can only be done once, before the vcpu is run. KVM_ARM_SVE_CONFIG_GET: report the set of vector lengths available to the guest on this vcpu (for use when snapshotting or migrating a VM). Signed-off-by: Dave Martin <Dave.Martin@arm.com> --- Changes since RFC v2: * Removed the arch vcpu ioctl hook in favour or kvm_arm_vcpu_sve_config() (with a suitable dummy version for arch/arm. The ioctl is not relevant for arm, so a dummy struct kvm_sve_vls definition is added for the purpose of compiling related declarations). * Add a kvm-arm specific helper for freeing the SVE state, instead of using kvm_arm_arch_vcpu_uninit() (which is otherwise unneeded and is dropped from the series). * Drop the homebrew "kvm_err_once" implementation for platforms with mismatched vector length support. Adding a common implementation is also probably not worthwhile just for this, so a bare printk_once(KERN_ERR) is used instead. * Migrate to a more robust protocol for avoiding drift in the set of registers available etc. when other vcpu ioctls cross over KVM_ARM_SVE_CONFIG_SET. Instead, KVM_ARM_SVE_CONFIG_SET is considered an early configuration ioctl and must be issued before KVM_ARM_VCPU_INIT and other non-trivial vcpu ioctls. Existing software would not know to do this, so an explicit opt-in is required via a new KVM_VM_TYPE_ARM_SVE flag passed to KVM_CREATE_VM. --- arch/arm/include/asm/kvm_host.h | 12 +++ arch/arm/include/uapi/asm/kvm.h | 3 + arch/arm64/include/asm/kvm_host.h | 25 +++++- arch/arm64/include/uapi/asm/kvm.h | 14 ++++ arch/arm64/kvm/guest.c | 163 ++++++++++++++++++++++++++++++++++++++ arch/arm64/kvm/reset.c | 61 ++++++++++++++ include/uapi/linux/kvm.h | 13 +++ virt/kvm/arm/arm.c | 3 +- 8 files changed, 292 insertions(+), 2 deletions(-)