diff mbox series

[v2,1/2] KVM: s390: add stat counter for shadow gmap events

Message ID 20230510121822.546629-2-nrb@linux.ibm.com (mailing list archive)
State New, archived
Headers show
Series KVM: s390: add counters for vsie performance | expand

Commit Message

Nico Boehr May 10, 2023, 12:18 p.m. UTC 
The shadow gmap tracks memory of nested guests (guest-3). In certain
scenarios, the shadow gmap needs to be rebuilt, which is a costly operation
since it involves a SIE exit into guest-1 for every entry in the respective
shadow level.

Add kvm stat counters when new shadow structures are created at various
levels. Also add a counter gmap_shadow_acquire when a completely fresh
shadow gmap is created.

Note that when several levels are shadowed at once, counters on all
affected levels will be increased.

Also note that not all page table levels need to be present and a ASCE
can directly point to e.g. a segment table. In this case, a new segment
table will always be equivalent to a new shadow gmap and hence will be
counted as gmap_shadow_acquire and not as gmap_shadow_segment.

Signed-off-by: Nico Boehr <nrb@linux.ibm.com>
---
 arch/s390/include/asm/kvm_host.h | 6 ++++++
 arch/s390/kvm/gaccess.c          | 7 +++++++
 arch/s390/kvm/kvm-s390.c         | 8 +++++++-
 arch/s390/kvm/vsie.c             | 1 +
 4 files changed, 21 insertions(+), 1 deletion(-)

Comments

David Hildenbrand July 27, 2023, 7:37 a.m. UTC | #1 
On 10.05.23 14:18, Nico Boehr wrote:
> The shadow gmap tracks memory of nested guests (guest-3). In certain
> scenarios, the shadow gmap needs to be rebuilt, which is a costly operation
> since it involves a SIE exit into guest-1 for every entry in the respective
> shadow level.
> 
> Add kvm stat counters when new shadow structures are created at various
> levels. Also add a counter gmap_shadow_acquire when a completely fresh
> shadow gmap is created.
> 
> Note that when several levels are shadowed at once, counters on all
> affected levels will be increased.
> 
> Also note that not all page table levels need to be present and a ASCE
> can directly point to e.g. a segment table. In this case, a new segment
> table will always be equivalent to a new shadow gmap and hence will be
> counted as gmap_shadow_acquire and not as gmap_shadow_segment.
> 
> Signed-off-by: Nico Boehr <nrb@linux.ibm.com>
> ---
>   arch/s390/include/asm/kvm_host.h | 6 ++++++
>   arch/s390/kvm/gaccess.c          | 7 +++++++
>   arch/s390/kvm/kvm-s390.c         | 8 +++++++-
>   arch/s390/kvm/vsie.c             | 1 +
>   4 files changed, 21 insertions(+), 1 deletion(-)
> 
> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> index 2bbc3d54959d..d35e03e82d3d 100644
> --- a/arch/s390/include/asm/kvm_host.h
> +++ b/arch/s390/include/asm/kvm_host.h
> @@ -777,6 +777,12 @@ struct kvm_vm_stat {
>   	u64 inject_service_signal;
>   	u64 inject_virtio;
>   	u64 aen_forward;
> +	u64 gmap_shadow_acquire;
> +	u64 gmap_shadow_r1_te;
> +	u64 gmap_shadow_r2_te;
> +	u64 gmap_shadow_r3_te;
> +	u64 gmap_shadow_sg_te;
> +	u64 gmap_shadow_pg_te;

Is "te" supposed to stand for "table entry" ?

If so, I'd suggest to just call this gmap_shadow_pg_entry etc.

>   };
>   
>   struct kvm_arch_memory_slot {
> diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
> index 3eb85f254881..6f4292ad0e4a 100644
> --- a/arch/s390/kvm/gaccess.c
> +++ b/arch/s390/kvm/gaccess.c
> @@ -1382,6 +1382,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
>   				  unsigned long *pgt, int *dat_protection,
>   				  int *fake)
>   {
> +	struct kvm *kvm;
>   	struct gmap *parent;
>   	union asce asce;
>   	union vaddress vaddr;
> @@ -1390,6 +1391,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
>   
>   	*fake = 0;
>   	*dat_protection = 0;
> +	kvm = sg->private;
>   	parent = sg->parent;
>   	vaddr.addr = saddr;
>   	asce.val = sg->orig_asce;
> @@ -1450,6 +1452,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
>   		rc = gmap_shadow_r2t(sg, saddr, rfte.val, *fake);
>   		if (rc)
>   			return rc;
> +		kvm->stat.gmap_shadow_r1_te++;
>   	}
>   		fallthrough;
>   	case ASCE_TYPE_REGION2: {
> @@ -1478,6 +1481,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
>   		rc = gmap_shadow_r3t(sg, saddr, rste.val, *fake);
>   		if (rc)
>   			return rc;
> +		kvm->stat.gmap_shadow_r2_te++;
>   	}
>   		fallthrough;
>   	case ASCE_TYPE_REGION3: {
> @@ -1515,6 +1519,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
>   		rc = gmap_shadow_sgt(sg, saddr, rtte.val, *fake);
>   		if (rc)
>   			return rc;
> +		kvm->stat.gmap_shadow_r3_te++;
>   	}
>   		fallthrough;
>   	case ASCE_TYPE_SEGMENT: {
> @@ -1548,6 +1553,7 @@ static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
>   		rc = gmap_shadow_pgt(sg, saddr, ste.val, *fake);
>   		if (rc)
>   			return rc;
> +		kvm->stat.gmap_shadow_sg_te++;
>   	}
>   	}
>   	/* Return the parent address of the page table */
> @@ -1618,6 +1624,7 @@ int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg,
>   	pte.p |= dat_protection;
>   	if (!rc)
>   		rc = gmap_shadow_page(sg, saddr, __pte(pte.val));
> +	vcpu->kvm->stat.gmap_shadow_pg_te++;
>   	ipte_unlock(vcpu->kvm);
>   	mmap_read_unlock(sg->mm);
>   	return rc;
> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
> index 17b81659cdb2..ded4149e145b 100644
> --- a/arch/s390/kvm/kvm-s390.c
> +++ b/arch/s390/kvm/kvm-s390.c
> @@ -66,7 +66,13 @@ const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
>   	STATS_DESC_COUNTER(VM, inject_pfault_done),
>   	STATS_DESC_COUNTER(VM, inject_service_signal),
>   	STATS_DESC_COUNTER(VM, inject_virtio),
> -	STATS_DESC_COUNTER(VM, aen_forward)
> +	STATS_DESC_COUNTER(VM, aen_forward),
> +	STATS_DESC_COUNTER(VM, gmap_shadow_acquire),
> +	STATS_DESC_COUNTER(VM, gmap_shadow_r1_te),
> +	STATS_DESC_COUNTER(VM, gmap_shadow_r2_te),
> +	STATS_DESC_COUNTER(VM, gmap_shadow_r3_te),
> +	STATS_DESC_COUNTER(VM, gmap_shadow_sg_te),
> +	STATS_DESC_COUNTER(VM, gmap_shadow_pg_te),
>   };
>   
>   const struct kvm_stats_header kvm_vm_stats_header = {
> diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
> index 8d6b765abf29..beb3be037722 100644
> --- a/arch/s390/kvm/vsie.c
> +++ b/arch/s390/kvm/vsie.c
> @@ -1221,6 +1221,7 @@ static int acquire_gmap_shadow(struct kvm_vcpu *vcpu,
>   	if (IS_ERR(gmap))
>   		return PTR_ERR(gmap);
>   	gmap->private = vcpu->kvm;
> +	vcpu->kvm->stat.gmap_shadow_acquire++;


Do you rather want to have events for

gmap_shadow_reuse (if gmap_shadow_valid() succeeded in that function)
gmap_shadow_create (if we have to create a new one via gmap_shadow)

?
Nico Boehr Aug. 1, 2023, 11:48 a.m. UTC | #2 
Quoting David Hildenbrand (2023-07-27 09:37:21)
[...]
> > diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
> > index 2bbc3d54959d..d35e03e82d3d 100644
> > --- a/arch/s390/include/asm/kvm_host.h
> > +++ b/arch/s390/include/asm/kvm_host.h
> > @@ -777,6 +777,12 @@ struct kvm_vm_stat {
> >       u64 inject_service_signal;
> >       u64 inject_virtio;
> >       u64 aen_forward;
> > +     u64 gmap_shadow_acquire;
> > +     u64 gmap_shadow_r1_te;
> > +     u64 gmap_shadow_r2_te;
> > +     u64 gmap_shadow_r3_te;
> > +     u64 gmap_shadow_sg_te;
> > +     u64 gmap_shadow_pg_te;
> 
> Is "te" supposed to stand for "table entry" ?

Yes.

> If so, I'd suggest to just call this gmap_shadow_pg_entry etc.

Janosch, since you suggested the current naming, are you OK with _entry?

[...]
> > diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
> > index 8d6b765abf29..beb3be037722 100644
> > --- a/arch/s390/kvm/vsie.c
> > +++ b/arch/s390/kvm/vsie.c
> > @@ -1221,6 +1221,7 @@ static int acquire_gmap_shadow(struct kvm_vcpu *vcpu,
> >       if (IS_ERR(gmap))
> >               return PTR_ERR(gmap);
> >       gmap->private = vcpu->kvm;
> > +     vcpu->kvm->stat.gmap_shadow_acquire++;
> 
> 
> Do you rather want to have events for
> 
> gmap_shadow_reuse (if gmap_shadow_valid() succeeded in that function)
> gmap_shadow_create (if we have to create a new one via gmap_shadow)
> 
> ?

Yeah, good suggestion. I'll add that.
Janosch Frank Aug. 1, 2023, 1:34 p.m. UTC | #3 
On 8/1/23 13:48, Nico Boehr wrote:
> Quoting David Hildenbrand (2023-07-27 09:37:21)
> [...]
>>> diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
>>> index 2bbc3d54959d..d35e03e82d3d 100644
>>> --- a/arch/s390/include/asm/kvm_host.h
>>> +++ b/arch/s390/include/asm/kvm_host.h
>>> @@ -777,6 +777,12 @@ struct kvm_vm_stat {
>>>        u64 inject_service_signal;
>>>        u64 inject_virtio;
>>>        u64 aen_forward;
>>> +     u64 gmap_shadow_acquire;
>>> +     u64 gmap_shadow_r1_te;
>>> +     u64 gmap_shadow_r2_te;
>>> +     u64 gmap_shadow_r3_te;
>>> +     u64 gmap_shadow_sg_te;
>>> +     u64 gmap_shadow_pg_te;
>>
>> Is "te" supposed to stand for "table entry" ?
> 
> Yes.
> 
>> If so, I'd suggest to just call this gmap_shadow_pg_entry etc.
> 
> Janosch, since you suggested the current naming, are you OK with _entry?

Sure

> 
> [...]
>>> diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
>>> index 8d6b765abf29..beb3be037722 100644
>>> --- a/arch/s390/kvm/vsie.c
>>> +++ b/arch/s390/kvm/vsie.c
>>> @@ -1221,6 +1221,7 @@ static int acquire_gmap_shadow(struct kvm_vcpu *vcpu,
>>>        if (IS_ERR(gmap))
>>>                return PTR_ERR(gmap);
>>>        gmap->private = vcpu->kvm;
>>> +     vcpu->kvm->stat.gmap_shadow_acquire++;
>>
>>
>> Do you rather want to have events for
>>
>> gmap_shadow_reuse (if gmap_shadow_valid() succeeded in that function)
>> gmap_shadow_create (if we have to create a new one via gmap_shadow)
>>
>> ?
> 
> Yeah, good suggestion. I'll add that.
diff mbox series

Patch

diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index 2bbc3d54959d..d35e03e82d3d 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -777,6 +777,12 @@  struct kvm_vm_stat {
 	u64 inject_service_signal;
 	u64 inject_virtio;
 	u64 aen_forward;
+	u64 gmap_shadow_acquire;
+	u64 gmap_shadow_r1_te;
+	u64 gmap_shadow_r2_te;
+	u64 gmap_shadow_r3_te;
+	u64 gmap_shadow_sg_te;
+	u64 gmap_shadow_pg_te;
 };
 
 struct kvm_arch_memory_slot {
diff --git a/arch/s390/kvm/gaccess.c b/arch/s390/kvm/gaccess.c
index 3eb85f254881..6f4292ad0e4a 100644
--- a/arch/s390/kvm/gaccess.c
+++ b/arch/s390/kvm/gaccess.c
@@ -1382,6 +1382,7 @@  static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
 				  unsigned long *pgt, int *dat_protection,
 				  int *fake)
 {
+	struct kvm *kvm;
 	struct gmap *parent;
 	union asce asce;
 	union vaddress vaddr;
@@ -1390,6 +1391,7 @@  static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
 
 	*fake = 0;
 	*dat_protection = 0;
+	kvm = sg->private;
 	parent = sg->parent;
 	vaddr.addr = saddr;
 	asce.val = sg->orig_asce;
@@ -1450,6 +1452,7 @@  static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
 		rc = gmap_shadow_r2t(sg, saddr, rfte.val, *fake);
 		if (rc)
 			return rc;
+		kvm->stat.gmap_shadow_r1_te++;
 	}
 		fallthrough;
 	case ASCE_TYPE_REGION2: {
@@ -1478,6 +1481,7 @@  static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
 		rc = gmap_shadow_r3t(sg, saddr, rste.val, *fake);
 		if (rc)
 			return rc;
+		kvm->stat.gmap_shadow_r2_te++;
 	}
 		fallthrough;
 	case ASCE_TYPE_REGION3: {
@@ -1515,6 +1519,7 @@  static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
 		rc = gmap_shadow_sgt(sg, saddr, rtte.val, *fake);
 		if (rc)
 			return rc;
+		kvm->stat.gmap_shadow_r3_te++;
 	}
 		fallthrough;
 	case ASCE_TYPE_SEGMENT: {
@@ -1548,6 +1553,7 @@  static int kvm_s390_shadow_tables(struct gmap *sg, unsigned long saddr,
 		rc = gmap_shadow_pgt(sg, saddr, ste.val, *fake);
 		if (rc)
 			return rc;
+		kvm->stat.gmap_shadow_sg_te++;
 	}
 	}
 	/* Return the parent address of the page table */
@@ -1618,6 +1624,7 @@  int kvm_s390_shadow_fault(struct kvm_vcpu *vcpu, struct gmap *sg,
 	pte.p |= dat_protection;
 	if (!rc)
 		rc = gmap_shadow_page(sg, saddr, __pte(pte.val));
+	vcpu->kvm->stat.gmap_shadow_pg_te++;
 	ipte_unlock(vcpu->kvm);
 	mmap_read_unlock(sg->mm);
 	return rc;
diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index 17b81659cdb2..ded4149e145b 100644
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -66,7 +66,13 @@  const struct _kvm_stats_desc kvm_vm_stats_desc[] = {
 	STATS_DESC_COUNTER(VM, inject_pfault_done),
 	STATS_DESC_COUNTER(VM, inject_service_signal),
 	STATS_DESC_COUNTER(VM, inject_virtio),
-	STATS_DESC_COUNTER(VM, aen_forward)
+	STATS_DESC_COUNTER(VM, aen_forward),
+	STATS_DESC_COUNTER(VM, gmap_shadow_acquire),
+	STATS_DESC_COUNTER(VM, gmap_shadow_r1_te),
+	STATS_DESC_COUNTER(VM, gmap_shadow_r2_te),
+	STATS_DESC_COUNTER(VM, gmap_shadow_r3_te),
+	STATS_DESC_COUNTER(VM, gmap_shadow_sg_te),
+	STATS_DESC_COUNTER(VM, gmap_shadow_pg_te),
 };
 
 const struct kvm_stats_header kvm_vm_stats_header = {
diff --git a/arch/s390/kvm/vsie.c b/arch/s390/kvm/vsie.c
index 8d6b765abf29..beb3be037722 100644
--- a/arch/s390/kvm/vsie.c
+++ b/arch/s390/kvm/vsie.c
@@ -1221,6 +1221,7 @@  static int acquire_gmap_shadow(struct kvm_vcpu *vcpu,
 	if (IS_ERR(gmap))
 		return PTR_ERR(gmap);
 	gmap->private = vcpu->kvm;
+	vcpu->kvm->stat.gmap_shadow_acquire++;
 	WRITE_ONCE(vsie_page->gmap, gmap);
 	return 0;
 }