diff mbox series

[kvm-unit-tests,v2,5/7] lib: s390x: Use a new asce for each PV guest

Message ID 20221020090009.2189-6-frankja@linux.ibm.com (mailing list archive)
State New, archived
Headers show
Series s390x: PV fixups | expand

Commit Message

Janosch Frank Oct. 20, 2022, 9 a.m. UTC
Every PV guest needs its own ASCE so let's copy the topmost table
designated by CR1 to create a new ASCE for the PV guest. Before and
after SIE we now need to switch ASCEs to and from the PV guest / test
ASCE. The SIE assembly function does that automatically.

Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
---
 lib/s390x/asm-offsets.c |  2 ++
 lib/s390x/sie.c         |  2 ++
 lib/s390x/sie.h         |  2 ++
 lib/s390x/uv.c          | 24 +++++++++++++++++++++++-
 lib/s390x/uv.h          |  5 ++---
 s390x/cpu.S             |  6 ++++++
 6 files changed, 37 insertions(+), 4 deletions(-)

Comments

Claudio Imbrenda Oct. 20, 2022, 9:25 a.m. UTC | #1
On Thu, 20 Oct 2022 09:00:07 +0000
Janosch Frank <frankja@linux.ibm.com> wrote:

> Every PV guest needs its own ASCE so let's copy the topmost table
> designated by CR1 to create a new ASCE for the PV guest. Before and
> after SIE we now need to switch ASCEs to and from the PV guest / test
> ASCE. The SIE assembly function does that automatically.
> 
> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> ---
>  lib/s390x/asm-offsets.c |  2 ++
>  lib/s390x/sie.c         |  2 ++
>  lib/s390x/sie.h         |  2 ++
>  lib/s390x/uv.c          | 24 +++++++++++++++++++++++-
>  lib/s390x/uv.h          |  5 ++---
>  s390x/cpu.S             |  6 ++++++
>  6 files changed, 37 insertions(+), 4 deletions(-)
> 
> diff --git a/lib/s390x/asm-offsets.c b/lib/s390x/asm-offsets.c
> index fbea3278..f612f327 100644
> --- a/lib/s390x/asm-offsets.c
> +++ b/lib/s390x/asm-offsets.c
> @@ -75,9 +75,11 @@ int main(void)
>  	OFFSET(SIE_SAVEAREA_HOST_GRS, vm_save_area, host.grs[0]);
>  	OFFSET(SIE_SAVEAREA_HOST_FPRS, vm_save_area, host.fprs[0]);
>  	OFFSET(SIE_SAVEAREA_HOST_FPC, vm_save_area, host.fpc);
> +	OFFSET(SIE_SAVEAREA_HOST_ASCE, vm_save_area, host.asce);
>  	OFFSET(SIE_SAVEAREA_GUEST_GRS, vm_save_area, guest.grs[0]);
>  	OFFSET(SIE_SAVEAREA_GUEST_FPRS, vm_save_area, guest.fprs[0]);
>  	OFFSET(SIE_SAVEAREA_GUEST_FPC, vm_save_area, guest.fpc);
> +	OFFSET(SIE_SAVEAREA_GUEST_ASCE, vm_save_area, guest.asce);
>  	OFFSET(STACK_FRAME_INT_BACKCHAIN, stack_frame_int, back_chain);
>  	OFFSET(STACK_FRAME_INT_FPC, stack_frame_int, fpc);
>  	OFFSET(STACK_FRAME_INT_FPRS, stack_frame_int, fprs);
> diff --git a/lib/s390x/sie.c b/lib/s390x/sie.c
> index 3fee3def..6efad965 100644
> --- a/lib/s390x/sie.c
> +++ b/lib/s390x/sie.c
> @@ -85,6 +85,8 @@ void sie_guest_create(struct vm *vm, uint64_t guest_mem, uint64_t guest_mem_len)
>  
>  	/* Guest memory chunks are always 1MB */
>  	assert(!(guest_mem_len & ~HPAGE_MASK));
> +	/* For non-PV guests we re-use the host's ASCE for ease of use */
> +	vm->save_area.guest.asce = stctg(1);
>  	/* Currently MSO/MSL is the easiest option */
>  	vm->sblk->mso = (uint64_t)guest_mem;
>  	vm->sblk->msl = (uint64_t)guest_mem + ((guest_mem_len - 1) & HPAGE_MASK);
> diff --git a/lib/s390x/sie.h b/lib/s390x/sie.h
> index 320c4218..3e3605c9 100644
> --- a/lib/s390x/sie.h
> +++ b/lib/s390x/sie.h
> @@ -205,12 +205,14 @@ union {
>  struct vm_uv {
>  	uint64_t vm_handle;
>  	uint64_t vcpu_handle;
> +	uint64_t asce;
>  	void *conf_base_stor;
>  	void *conf_var_stor;
>  	void *cpu_stor;
>  };
>  
>  struct vm_save_regs {
> +	uint64_t asce;
>  	uint64_t grs[16];
>  	uint64_t fprs[16];
>  	uint32_t fpc;
> diff --git a/lib/s390x/uv.c b/lib/s390x/uv.c
> index 3b4cafa9..0b6eb843 100644
> --- a/lib/s390x/uv.c
> +++ b/lib/s390x/uv.c
> @@ -90,6 +90,25 @@ void uv_init(void)
>  	initialized = true;
>  }
>  
> +/*
> + * Create a new ASCE for the UV config because they can't be shared
> + * for security reasons. We just simply copy the top most table into a
> + * fresh set of allocated pages and use those pages as the asce.
> + */
> +static uint64_t create_asce(void)
> +{
> +	void *pgd_new, *pgd_old;
> +	uint64_t asce = stctg(1);
> +
> +	pgd_new = memalign_pages_flags(PAGE_SIZE, PAGE_SIZE * 4, 0);

here you can use memalign_pages, since you are not using the flags

> +	pgd_old = (void *)(asce & PAGE_MASK);
> +
> +	memcpy(pgd_new, pgd_old, PAGE_SIZE * 4);
> +
> +	asce = __pa(pgd_new) | ASCE_DT_REGION1 | REGION_TABLE_LENGTH | ASCE_P;

why not taking the flags from the old ASCE? what if we choose to use a
different type of table?

something like this:

asce = _pa(pgd_new) | ASCE_P | (asce & ~PAGE_MASK);

> +	return asce;
> +}
> +
>  void uv_create_guest(struct vm *vm)
>  {
>  	struct uv_cb_cgc uvcb_cgc = {
> @@ -125,7 +144,8 @@ void uv_create_guest(struct vm *vm)
>  	vm->uv.cpu_stor = memalign_pages_flags(PAGE_SIZE, uvcb_qui.cpu_stor_len, 0);
>  	uvcb_csc.stor_origin = (uint64_t)vm->uv.cpu_stor;
>  
> -	uvcb_cgc.guest_asce = (uint64_t)stctg(1);
> +	uvcb_cgc.guest_asce = create_asce();
> +	vm->save_area.guest.asce = uvcb_cgc.guest_asce;
>  	uvcb_cgc.guest_sca = (uint64_t)vm->sca;
>  
>  	cc = uv_call(0, (uint64_t)&uvcb_cgc);
> @@ -166,6 +186,8 @@ void uv_destroy_guest(struct vm *vm)
>  	assert(cc == 0);
>  	free_pages(vm->uv.conf_base_stor);
>  	free_pages(vm->uv.conf_var_stor);
> +
> +	free_pages((void *)(vm->uv.asce & PAGE_MASK));
>  }
>  
>  int uv_unpack(struct vm *vm, uint64_t addr, uint64_t len, uint64_t tweak)
> diff --git a/lib/s390x/uv.h b/lib/s390x/uv.h
> index 44264861..5fe29bda 100644
> --- a/lib/s390x/uv.h
> +++ b/lib/s390x/uv.h
> @@ -28,9 +28,8 @@ static inline void uv_setup_asces(void)
>  	/* We need to have a valid primary ASCE to run guests. */
>  	setup_vm();
>  
> -	/* Set P bit in ASCE as it is required for PV guests */
> -	asce = stctg(1) | ASCE_P;
> -	lctlg(1, asce);
> +	/* Grab the ASCE which setup_vm() just set up */
> +	asce = stctg(1);
>  
>  	/* Copy ASCE into home space CR */
>  	lctlg(13, asce);
> diff --git a/s390x/cpu.S b/s390x/cpu.S
> index 82b5e25d..45bd551a 100644
> --- a/s390x/cpu.S
> +++ b/s390x/cpu.S
> @@ -76,6 +76,9 @@ sie64a:
>  	.endr
>  	stfpc	SIE_SAVEAREA_HOST_FPC(%r3)
>  
> +	stctg	%c1, %c1, SIE_SAVEAREA_HOST_ASCE(%r3)
> +	lctlg	%c1, %c1, SIE_SAVEAREA_GUEST_ASCE(%r3)
> +
>  	# Store scb and save_area pointer into stack frame
>  	stg	%r2,__SF_SIE_CONTROL(%r15)	# save control block pointer
>  	stg	%r3,__SF_SIE_SAVEAREA(%r15)	# save guest register save area
> @@ -102,6 +105,9 @@ sie_exit:
>  	# Load guest register save area
>  	lg	%r14,__SF_SIE_SAVEAREA(%r15)
>  
> +	# Restore the host asce
> +	lctlg	%c1, %c1, SIE_SAVEAREA_HOST_ASCE(%r14)
> +
>  	# Store guest's gprs, fprs and fpc
>  	stmg	%r0,%r13,SIE_SAVEAREA_GUEST_GRS(%r14)	# save guest gprs 0-13
>  	.irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
Janosch Frank Oct. 20, 2022, 11:27 a.m. UTC | #2
On 10/20/22 11:25, Claudio Imbrenda wrote:
> On Thu, 20 Oct 2022 09:00:07 +0000
> Janosch Frank <frankja@linux.ibm.com> wrote:
> 
>> Every PV guest needs its own ASCE so let's copy the topmost table
>> designated by CR1 to create a new ASCE for the PV guest. Before and
>> after SIE we now need to switch ASCEs to and from the PV guest / test
>> ASCE. The SIE assembly function does that automatically.
>>
>> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
>> ---
>>   lib/s390x/asm-offsets.c |  2 ++
>>   lib/s390x/sie.c         |  2 ++
>>   lib/s390x/sie.h         |  2 ++
>>   lib/s390x/uv.c          | 24 +++++++++++++++++++++++-
>>   lib/s390x/uv.h          |  5 ++---
>>   s390x/cpu.S             |  6 ++++++
>>   6 files changed, 37 insertions(+), 4 deletions(-)
>>
>> diff --git a/lib/s390x/asm-offsets.c b/lib/s390x/asm-offsets.c
>> index fbea3278..f612f327 100644
>> --- a/lib/s390x/asm-offsets.c
>> +++ b/lib/s390x/asm-offsets.c
>> @@ -75,9 +75,11 @@ int main(void)
>>   	OFFSET(SIE_SAVEAREA_HOST_GRS, vm_save_area, host.grs[0]);
>>   	OFFSET(SIE_SAVEAREA_HOST_FPRS, vm_save_area, host.fprs[0]);
>>   	OFFSET(SIE_SAVEAREA_HOST_FPC, vm_save_area, host.fpc);
>> +	OFFSET(SIE_SAVEAREA_HOST_ASCE, vm_save_area, host.asce);
>>   	OFFSET(SIE_SAVEAREA_GUEST_GRS, vm_save_area, guest.grs[0]);
>>   	OFFSET(SIE_SAVEAREA_GUEST_FPRS, vm_save_area, guest.fprs[0]);
>>   	OFFSET(SIE_SAVEAREA_GUEST_FPC, vm_save_area, guest.fpc);
>> +	OFFSET(SIE_SAVEAREA_GUEST_ASCE, vm_save_area, guest.asce);
>>   	OFFSET(STACK_FRAME_INT_BACKCHAIN, stack_frame_int, back_chain);
>>   	OFFSET(STACK_FRAME_INT_FPC, stack_frame_int, fpc);
>>   	OFFSET(STACK_FRAME_INT_FPRS, stack_frame_int, fprs);
>> diff --git a/lib/s390x/sie.c b/lib/s390x/sie.c
>> index 3fee3def..6efad965 100644
>> --- a/lib/s390x/sie.c
>> +++ b/lib/s390x/sie.c
>> @@ -85,6 +85,8 @@ void sie_guest_create(struct vm *vm, uint64_t guest_mem, uint64_t guest_mem_len)
>>   
>>   	/* Guest memory chunks are always 1MB */
>>   	assert(!(guest_mem_len & ~HPAGE_MASK));
>> +	/* For non-PV guests we re-use the host's ASCE for ease of use */
>> +	vm->save_area.guest.asce = stctg(1);
>>   	/* Currently MSO/MSL is the easiest option */
>>   	vm->sblk->mso = (uint64_t)guest_mem;
>>   	vm->sblk->msl = (uint64_t)guest_mem + ((guest_mem_len - 1) & HPAGE_MASK);
>> diff --git a/lib/s390x/sie.h b/lib/s390x/sie.h
>> index 320c4218..3e3605c9 100644
>> --- a/lib/s390x/sie.h
>> +++ b/lib/s390x/sie.h
>> @@ -205,12 +205,14 @@ union {
>>   struct vm_uv {
>>   	uint64_t vm_handle;
>>   	uint64_t vcpu_handle;
>> +	uint64_t asce;
>>   	void *conf_base_stor;
>>   	void *conf_var_stor;
>>   	void *cpu_stor;
>>   };
>>   
>>   struct vm_save_regs {
>> +	uint64_t asce;
>>   	uint64_t grs[16];
>>   	uint64_t fprs[16];
>>   	uint32_t fpc;
>> diff --git a/lib/s390x/uv.c b/lib/s390x/uv.c
>> index 3b4cafa9..0b6eb843 100644
>> --- a/lib/s390x/uv.c
>> +++ b/lib/s390x/uv.c
>> @@ -90,6 +90,25 @@ void uv_init(void)
>>   	initialized = true;
>>   }
>>   
>> +/*
>> + * Create a new ASCE for the UV config because they can't be shared
>> + * for security reasons. We just simply copy the top most table into a
>> + * fresh set of allocated pages and use those pages as the asce.
>> + */
>> +static uint64_t create_asce(void)
>> +{
>> +	void *pgd_new, *pgd_old;
>> +	uint64_t asce = stctg(1);
>> +
>> +	pgd_new = memalign_pages_flags(PAGE_SIZE, PAGE_SIZE * 4, 0);
> 
> here you can use memalign_pages, since you are not using the flags

Sure

> 
>> +	pgd_old = (void *)(asce & PAGE_MASK);
>> +
>> +	memcpy(pgd_new, pgd_old, PAGE_SIZE * 4);
>> +
>> +	asce = __pa(pgd_new) | ASCE_DT_REGION1 | REGION_TABLE_LENGTH | ASCE_P;
> 
> why not taking the flags from the old ASCE? what if we choose to use a
> different type of table?
> 
> something like this:
> 
> asce = _pa(pgd_new) | ASCE_P | (asce & ~PAGE_MASK);

I should at least preserve DT and TL but I'd opt to not copy over the 
other bits. If someone wants to do funky ASCE stuff they now have the 
possibility to simply change vm->save_area.guest.asce
Claudio Imbrenda Oct. 20, 2022, 11:46 a.m. UTC | #3
On Thu, 20 Oct 2022 13:27:22 +0200
Janosch Frank <frankja@linux.ibm.com> wrote:

> On 10/20/22 11:25, Claudio Imbrenda wrote:
> > On Thu, 20 Oct 2022 09:00:07 +0000
> > Janosch Frank <frankja@linux.ibm.com> wrote:
> >   
> >> Every PV guest needs its own ASCE so let's copy the topmost table
> >> designated by CR1 to create a new ASCE for the PV guest. Before and
> >> after SIE we now need to switch ASCEs to and from the PV guest / test
> >> ASCE. The SIE assembly function does that automatically.
> >>
> >> Signed-off-by: Janosch Frank <frankja@linux.ibm.com>
> >> ---
> >>   lib/s390x/asm-offsets.c |  2 ++
> >>   lib/s390x/sie.c         |  2 ++
> >>   lib/s390x/sie.h         |  2 ++
> >>   lib/s390x/uv.c          | 24 +++++++++++++++++++++++-
> >>   lib/s390x/uv.h          |  5 ++---
> >>   s390x/cpu.S             |  6 ++++++
> >>   6 files changed, 37 insertions(+), 4 deletions(-)
> >>
> >> diff --git a/lib/s390x/asm-offsets.c b/lib/s390x/asm-offsets.c
> >> index fbea3278..f612f327 100644
> >> --- a/lib/s390x/asm-offsets.c
> >> +++ b/lib/s390x/asm-offsets.c
> >> @@ -75,9 +75,11 @@ int main(void)
> >>   	OFFSET(SIE_SAVEAREA_HOST_GRS, vm_save_area, host.grs[0]);
> >>   	OFFSET(SIE_SAVEAREA_HOST_FPRS, vm_save_area, host.fprs[0]);
> >>   	OFFSET(SIE_SAVEAREA_HOST_FPC, vm_save_area, host.fpc);
> >> +	OFFSET(SIE_SAVEAREA_HOST_ASCE, vm_save_area, host.asce);
> >>   	OFFSET(SIE_SAVEAREA_GUEST_GRS, vm_save_area, guest.grs[0]);
> >>   	OFFSET(SIE_SAVEAREA_GUEST_FPRS, vm_save_area, guest.fprs[0]);
> >>   	OFFSET(SIE_SAVEAREA_GUEST_FPC, vm_save_area, guest.fpc);
> >> +	OFFSET(SIE_SAVEAREA_GUEST_ASCE, vm_save_area, guest.asce);
> >>   	OFFSET(STACK_FRAME_INT_BACKCHAIN, stack_frame_int, back_chain);
> >>   	OFFSET(STACK_FRAME_INT_FPC, stack_frame_int, fpc);
> >>   	OFFSET(STACK_FRAME_INT_FPRS, stack_frame_int, fprs);
> >> diff --git a/lib/s390x/sie.c b/lib/s390x/sie.c
> >> index 3fee3def..6efad965 100644
> >> --- a/lib/s390x/sie.c
> >> +++ b/lib/s390x/sie.c
> >> @@ -85,6 +85,8 @@ void sie_guest_create(struct vm *vm, uint64_t guest_mem, uint64_t guest_mem_len)
> >>   
> >>   	/* Guest memory chunks are always 1MB */
> >>   	assert(!(guest_mem_len & ~HPAGE_MASK));
> >> +	/* For non-PV guests we re-use the host's ASCE for ease of use */
> >> +	vm->save_area.guest.asce = stctg(1);
> >>   	/* Currently MSO/MSL is the easiest option */
> >>   	vm->sblk->mso = (uint64_t)guest_mem;
> >>   	vm->sblk->msl = (uint64_t)guest_mem + ((guest_mem_len - 1) & HPAGE_MASK);
> >> diff --git a/lib/s390x/sie.h b/lib/s390x/sie.h
> >> index 320c4218..3e3605c9 100644
> >> --- a/lib/s390x/sie.h
> >> +++ b/lib/s390x/sie.h
> >> @@ -205,12 +205,14 @@ union {
> >>   struct vm_uv {
> >>   	uint64_t vm_handle;
> >>   	uint64_t vcpu_handle;
> >> +	uint64_t asce;
> >>   	void *conf_base_stor;
> >>   	void *conf_var_stor;
> >>   	void *cpu_stor;
> >>   };
> >>   
> >>   struct vm_save_regs {
> >> +	uint64_t asce;
> >>   	uint64_t grs[16];
> >>   	uint64_t fprs[16];
> >>   	uint32_t fpc;
> >> diff --git a/lib/s390x/uv.c b/lib/s390x/uv.c
> >> index 3b4cafa9..0b6eb843 100644
> >> --- a/lib/s390x/uv.c
> >> +++ b/lib/s390x/uv.c
> >> @@ -90,6 +90,25 @@ void uv_init(void)
> >>   	initialized = true;
> >>   }
> >>   
> >> +/*
> >> + * Create a new ASCE for the UV config because they can't be shared
> >> + * for security reasons. We just simply copy the top most table into a
> >> + * fresh set of allocated pages and use those pages as the asce.
> >> + */
> >> +static uint64_t create_asce(void)
> >> +{
> >> +	void *pgd_new, *pgd_old;
> >> +	uint64_t asce = stctg(1);
> >> +
> >> +	pgd_new = memalign_pages_flags(PAGE_SIZE, PAGE_SIZE * 4, 0);  
> > 
> > here you can use memalign_pages, since you are not using the flags  
> 
> Sure
> 
> >   
> >> +	pgd_old = (void *)(asce & PAGE_MASK);
> >> +
> >> +	memcpy(pgd_new, pgd_old, PAGE_SIZE * 4);
> >> +
> >> +	asce = __pa(pgd_new) | ASCE_DT_REGION1 | REGION_TABLE_LENGTH | ASCE_P;  
> > 
> > why not taking the flags from the old ASCE? what if we choose to use a
> > different type of table?
> > 
> > something like this:
> > 
> > asce = _pa(pgd_new) | ASCE_P | (asce & ~PAGE_MASK);  
> 
> I should at least preserve DT and TL but I'd opt to not copy over the 
> other bits. If someone wants to do funky ASCE stuff they now have the 
> possibility to simply change vm->save_area.guest.asce

that's ok
diff mbox series

Patch

diff --git a/lib/s390x/asm-offsets.c b/lib/s390x/asm-offsets.c
index fbea3278..f612f327 100644
--- a/lib/s390x/asm-offsets.c
+++ b/lib/s390x/asm-offsets.c
@@ -75,9 +75,11 @@  int main(void)
 	OFFSET(SIE_SAVEAREA_HOST_GRS, vm_save_area, host.grs[0]);
 	OFFSET(SIE_SAVEAREA_HOST_FPRS, vm_save_area, host.fprs[0]);
 	OFFSET(SIE_SAVEAREA_HOST_FPC, vm_save_area, host.fpc);
+	OFFSET(SIE_SAVEAREA_HOST_ASCE, vm_save_area, host.asce);
 	OFFSET(SIE_SAVEAREA_GUEST_GRS, vm_save_area, guest.grs[0]);
 	OFFSET(SIE_SAVEAREA_GUEST_FPRS, vm_save_area, guest.fprs[0]);
 	OFFSET(SIE_SAVEAREA_GUEST_FPC, vm_save_area, guest.fpc);
+	OFFSET(SIE_SAVEAREA_GUEST_ASCE, vm_save_area, guest.asce);
 	OFFSET(STACK_FRAME_INT_BACKCHAIN, stack_frame_int, back_chain);
 	OFFSET(STACK_FRAME_INT_FPC, stack_frame_int, fpc);
 	OFFSET(STACK_FRAME_INT_FPRS, stack_frame_int, fprs);
diff --git a/lib/s390x/sie.c b/lib/s390x/sie.c
index 3fee3def..6efad965 100644
--- a/lib/s390x/sie.c
+++ b/lib/s390x/sie.c
@@ -85,6 +85,8 @@  void sie_guest_create(struct vm *vm, uint64_t guest_mem, uint64_t guest_mem_len)
 
 	/* Guest memory chunks are always 1MB */
 	assert(!(guest_mem_len & ~HPAGE_MASK));
+	/* For non-PV guests we re-use the host's ASCE for ease of use */
+	vm->save_area.guest.asce = stctg(1);
 	/* Currently MSO/MSL is the easiest option */
 	vm->sblk->mso = (uint64_t)guest_mem;
 	vm->sblk->msl = (uint64_t)guest_mem + ((guest_mem_len - 1) & HPAGE_MASK);
diff --git a/lib/s390x/sie.h b/lib/s390x/sie.h
index 320c4218..3e3605c9 100644
--- a/lib/s390x/sie.h
+++ b/lib/s390x/sie.h
@@ -205,12 +205,14 @@  union {
 struct vm_uv {
 	uint64_t vm_handle;
 	uint64_t vcpu_handle;
+	uint64_t asce;
 	void *conf_base_stor;
 	void *conf_var_stor;
 	void *cpu_stor;
 };
 
 struct vm_save_regs {
+	uint64_t asce;
 	uint64_t grs[16];
 	uint64_t fprs[16];
 	uint32_t fpc;
diff --git a/lib/s390x/uv.c b/lib/s390x/uv.c
index 3b4cafa9..0b6eb843 100644
--- a/lib/s390x/uv.c
+++ b/lib/s390x/uv.c
@@ -90,6 +90,25 @@  void uv_init(void)
 	initialized = true;
 }
 
+/*
+ * Create a new ASCE for the UV config because they can't be shared
+ * for security reasons. We just simply copy the top most table into a
+ * fresh set of allocated pages and use those pages as the asce.
+ */
+static uint64_t create_asce(void)
+{
+	void *pgd_new, *pgd_old;
+	uint64_t asce = stctg(1);
+
+	pgd_new = memalign_pages_flags(PAGE_SIZE, PAGE_SIZE * 4, 0);
+	pgd_old = (void *)(asce & PAGE_MASK);
+
+	memcpy(pgd_new, pgd_old, PAGE_SIZE * 4);
+
+	asce = __pa(pgd_new) | ASCE_DT_REGION1 | REGION_TABLE_LENGTH | ASCE_P;
+	return asce;
+}
+
 void uv_create_guest(struct vm *vm)
 {
 	struct uv_cb_cgc uvcb_cgc = {
@@ -125,7 +144,8 @@  void uv_create_guest(struct vm *vm)
 	vm->uv.cpu_stor = memalign_pages_flags(PAGE_SIZE, uvcb_qui.cpu_stor_len, 0);
 	uvcb_csc.stor_origin = (uint64_t)vm->uv.cpu_stor;
 
-	uvcb_cgc.guest_asce = (uint64_t)stctg(1);
+	uvcb_cgc.guest_asce = create_asce();
+	vm->save_area.guest.asce = uvcb_cgc.guest_asce;
 	uvcb_cgc.guest_sca = (uint64_t)vm->sca;
 
 	cc = uv_call(0, (uint64_t)&uvcb_cgc);
@@ -166,6 +186,8 @@  void uv_destroy_guest(struct vm *vm)
 	assert(cc == 0);
 	free_pages(vm->uv.conf_base_stor);
 	free_pages(vm->uv.conf_var_stor);
+
+	free_pages((void *)(vm->uv.asce & PAGE_MASK));
 }
 
 int uv_unpack(struct vm *vm, uint64_t addr, uint64_t len, uint64_t tweak)
diff --git a/lib/s390x/uv.h b/lib/s390x/uv.h
index 44264861..5fe29bda 100644
--- a/lib/s390x/uv.h
+++ b/lib/s390x/uv.h
@@ -28,9 +28,8 @@  static inline void uv_setup_asces(void)
 	/* We need to have a valid primary ASCE to run guests. */
 	setup_vm();
 
-	/* Set P bit in ASCE as it is required for PV guests */
-	asce = stctg(1) | ASCE_P;
-	lctlg(1, asce);
+	/* Grab the ASCE which setup_vm() just set up */
+	asce = stctg(1);
 
 	/* Copy ASCE into home space CR */
 	lctlg(13, asce);
diff --git a/s390x/cpu.S b/s390x/cpu.S
index 82b5e25d..45bd551a 100644
--- a/s390x/cpu.S
+++ b/s390x/cpu.S
@@ -76,6 +76,9 @@  sie64a:
 	.endr
 	stfpc	SIE_SAVEAREA_HOST_FPC(%r3)
 
+	stctg	%c1, %c1, SIE_SAVEAREA_HOST_ASCE(%r3)
+	lctlg	%c1, %c1, SIE_SAVEAREA_GUEST_ASCE(%r3)
+
 	# Store scb and save_area pointer into stack frame
 	stg	%r2,__SF_SIE_CONTROL(%r15)	# save control block pointer
 	stg	%r3,__SF_SIE_SAVEAREA(%r15)	# save guest register save area
@@ -102,6 +105,9 @@  sie_exit:
 	# Load guest register save area
 	lg	%r14,__SF_SIE_SAVEAREA(%r15)
 
+	# Restore the host asce
+	lctlg	%c1, %c1, SIE_SAVEAREA_HOST_ASCE(%r14)
+
 	# Store guest's gprs, fprs and fpc
 	stmg	%r0,%r13,SIE_SAVEAREA_GUEST_GRS(%r14)	# save guest gprs 0-13
 	.irp i, 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15