@@ -492,23 +492,22 @@ static unsigned long get_num_contig_pages(unsigned long idx,
return pages;
}
-static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
+int sev_launch_update_shared_gfn_handler(struct kvm *kvm,
+ struct kvm_gfn_range *range, struct kvm_sev_cmd *argp)
{
unsigned long vaddr, vaddr_end, next_vaddr, npages, pages, size, i;
struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
- struct kvm_sev_launch_update_data params;
struct sev_data_launch_update_data data;
struct page **inpages;
int ret;
- if (!sev_guest(kvm))
- return -ENOTTY;
-
- if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params)))
- return -EFAULT;
+ vaddr = gfn_to_hva_memslot(range->slot, range->start);
+ if (kvm_is_error_hva(vaddr)) {
+ pr_err("vaddr is erroneous 0x%lx\n", vaddr);
+ return -EINVAL;
+ }
- vaddr = params.uaddr;
- size = params.len;
+ size = (range->end - range->start) << PAGE_SHIFT;
vaddr_end = vaddr + size;
/* Lock the user memory. */
@@ -560,6 +559,88 @@ static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
return ret;
}
+int sev_launch_update_priv_gfn_handler(struct kvm *kvm,
+ struct kvm_gfn_range *range, struct kvm_sev_cmd *argp)
+{
+ struct sev_data_launch_update_data data;
+ struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
+ gfn_t gfn;
+ kvm_pfn_t pfn;
+ struct kvm_memory_slot *memslot = range->slot;
+ int ret = 0;
+
+ data.reserved = 0;
+ data.handle = sev->handle;
+
+ for (gfn = range->start; gfn < range->end; gfn++) {
+ int order;
+ void *kvaddr;
+
+ ret = kvm_private_mem_get_pfn(memslot,
+ gfn, &pfn, &order);
+ if (ret)
+ return ret;
+
+ kvaddr = pfn_to_kaddr(pfn);
+ if (!virt_addr_valid(kvaddr)) {
+ pr_err("Invalid kvaddr 0x%lx\n", (uint64_t)kvaddr);
+ ret = -EINVAL;
+ goto e_ret;
+ }
+
+ ret = kvm_read_guest_page(kvm, gfn, kvaddr, 0, PAGE_SIZE);
+ if (ret) {
+ pr_err("guest read failed 0x%lx\n", ret);
+ goto e_ret;
+ }
+
+ if (!this_cpu_has(X86_FEATURE_SME_COHERENT))
+ clflush_cache_range(kvaddr, PAGE_SIZE);
+
+ data.len = PAGE_SIZE;
+ data.address = __sme_set(pfn << PAGE_SHIFT);
+ ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, &data, &argp->error);
+ if (ret)
+ goto e_ret;
+
+ kvm_private_mem_put_pfn(memslot, pfn);
+ }
+ kvm_vm_set_region_attr(kvm, range->start, range->end,
+ true /* priv_attr */);
+
+ return ret;
+
+e_ret:
+ kvm_private_mem_put_pfn(memslot, pfn);
+ return ret;
+}
+
+int sev_launch_update_gfn_handler(struct kvm *kvm,
+ struct kvm_gfn_range *range, void *data)
+{
+ struct kvm_sev_cmd *argp = (struct kvm_sev_cmd *)data;
+
+ if (kvm_slot_can_be_private(range->slot))
+ return sev_launch_update_priv_gfn_handler(kvm, range, argp);
+
+ return sev_launch_update_shared_gfn_handler(kvm, range, argp);
+}
+
+static int sev_launch_update_data(struct kvm *kvm,
+ struct kvm_sev_cmd *argp)
+{
+ struct kvm_sev_launch_update_data params;
+
+ if (!sev_guest(kvm))
+ return -ENOTTY;
+
+ if (copy_from_user(¶ms, (void __user *)(uintptr_t)argp->data, sizeof(params)))
+ return -EFAULT;
+
+ return kvm_vm_do_hva_range_op(kvm, params.uaddr, params.uaddr + params.len,
+ sev_launch_update_gfn_handler, argp);
+}
+
static int sev_es_sync_vmsa(struct vcpu_svm *svm)
{
struct sev_es_save_area *save = svm->sev_es.vmsa;
@@ -980,6 +980,8 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align,
void kvm_exit(void);
void kvm_get_kvm(struct kvm *kvm);
+int kvm_vm_set_region_attr(struct kvm *kvm, unsigned long gfn_start,
+ unsigned long gfn_end, bool priv_attr);
bool kvm_get_kvm_safe(struct kvm *kvm);
void kvm_put_kvm(struct kvm *kvm);
bool file_is_kvm(struct file *file);
@@ -981,7 +981,7 @@ static int kvm_vm_populate_private_mem(struct kvm *kvm, unsigned long gfn_start,
}
mutex_lock(&kvm->slots_lock);
- for (gfn = gfn_start; gfn <= gfn_end; gfn++) {
+ for (gfn = gfn_start; gfn < gfn_end; gfn++) {
int order;
void *kvaddr;
@@ -1012,12 +1012,29 @@ static int kvm_vm_populate_private_mem(struct kvm *kvm, unsigned long gfn_start,
}
#endif
+int kvm_vm_set_region_attr(struct kvm *kvm, unsigned long gfn_start,
+ unsigned long gfn_end, bool priv_attr)
+{
+ int r;
+ void *entry;
+ unsigned long index;
+
+ entry = priv_attr ? xa_mk_value(KVM_MEM_ATTR_PRIVATE) : NULL;
+
+ for (index = gfn_start; index < gfn_end; index++) {
+ r = xa_err(xa_store(&kvm->mem_attr_array, index, entry,
+ GFP_KERNEL_ACCOUNT));
+ if (r)
+ break;
+ }
+
+ return r;
+}
+
static int kvm_vm_ioctl_set_encrypted_region(struct kvm *kvm, unsigned int ioctl,
struct kvm_enc_region *region)
{
unsigned long start, end;
- unsigned long index;
- void *entry;
int r;
if (region->size == 0 || region->addr + region->size < region->addr)
@@ -1026,22 +1043,14 @@ static int kvm_vm_ioctl_set_encrypted_region(struct kvm *kvm, unsigned int ioctl
return -EINVAL;
start = region->addr >> PAGE_SHIFT;
- end = (region->addr + region->size - 1) >> PAGE_SHIFT;
-
- entry = ioctl == KVM_MEMORY_ENCRYPT_REG_REGION ?
- xa_mk_value(KVM_MEM_ATTR_PRIVATE) : NULL;
-
- for (index = start; index <= end; index++) {
- r = xa_err(xa_store(&kvm->mem_attr_array, index, entry,
- GFP_KERNEL_ACCOUNT));
- if (r)
- break;
- }
+ end = (region->addr + region->size) >> PAGE_SHIFT;
+ r = kvm_vm_set_region_attr(kvm, start, end,
+ (ioctl == KVM_MEMORY_ENCRYPT_REG_REGION));
kvm_zap_gfn_range(kvm, start, end + 1);
#ifdef CONFIG_HAVE_KVM_PRIVATE_MEM_TESTING
- if (!kvm->vm_entry_attempted && (ioctl == KVM_MEMORY_ENCRYPT_REG_REGION))
+ if (!r && !kvm->vm_entry_attempted && (ioctl == KVM_MEMORY_ENCRYPT_REG_REGION))
r = kvm_vm_populate_private_mem(kvm, start, end);
#endif
This change adds handling of HVA ranges to copy contents to private memory while doing sev launch update data. mem_attr array is updated during LAUNCH_UPDATE_DATA to ensure that encrypted memory is marked as private. Signed-off-by: Vishal Annapurve <vannapurve@google.com> --- arch/x86/kvm/svm/sev.c | 99 ++++++++++++++++++++++++++++++++++++---- include/linux/kvm_host.h | 2 + virt/kvm/kvm_main.c | 39 ++++++++++------ 3 files changed, 116 insertions(+), 24 deletions(-)