| Message ID | 20181128101943.155542-11-mimu@linux.ibm.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | KVM: s390: make use of the GIB | expand |
On 28/11/2018 11:19, Michael Mueller wrote: > The patch implements a handler for GIB alert interruptions > on the host. Its task is to alert storage backed guests that > interrupts are pending for them. > > A GIB alert interrupt statistic counter is added as well: > > $ cat /proc/interrupts > CPU0 CPU1 > ... > GAL: 0 0 [I/O] GIB Alert > ... > > Signed-off-by: Michael Mueller <mimu@linux.ibm.com> > --- > arch/s390/include/asm/irq.h | 1 + > arch/s390/include/asm/isc.h | 1 + > arch/s390/kernel/irq.c | 1 + > arch/s390/kvm/interrupt.c | 55 +++++++++++++++++++++++++++++++++++++-------- > arch/s390/kvm/kvm-s390.c | 4 ++++ > arch/s390/kvm/kvm-s390.h | 5 +++++ > 6 files changed, 58 insertions(+), 9 deletions(-) > > diff --git a/arch/s390/include/asm/irq.h b/arch/s390/include/asm/irq.h > index 2f7f27e5493f..afaf5e3c57fd 100644 > --- a/arch/s390/include/asm/irq.h > +++ b/arch/s390/include/asm/irq.h > @@ -62,6 +62,7 @@ enum interruption_class { > IRQIO_MSI, > IRQIO_VIR, > IRQIO_VAI, > + IRQIO_GAL, > NMI_NMI, > CPU_RST, > NR_ARCH_IRQS > diff --git a/arch/s390/include/asm/isc.h b/arch/s390/include/asm/isc.h > index 6cb9e2ed05b6..b2cc1ec78d06 100644 > --- a/arch/s390/include/asm/isc.h > +++ b/arch/s390/include/asm/isc.h > @@ -21,6 +21,7 @@ > /* Adapter interrupts. */ > #define QDIO_AIRQ_ISC IO_SCH_ISC /* I/O subchannel in qdio mode */ > #define PCI_ISC 2 /* PCI I/O subchannels */ > +#define GAL_ISC 5 /* GIB alert */ > #define AP_ISC 6 /* adjunct processor (crypto) devices */ > > /* Functions for registration of I/O interruption subclasses */ > diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c > index 0e8d68bac82c..0cd5a5f96729 100644 > --- a/arch/s390/kernel/irq.c > +++ b/arch/s390/kernel/irq.c > @@ -88,6 +88,7 @@ static const struct irq_class irqclass_sub_desc[] = { > {.irq = IRQIO_MSI, .name = "MSI", .desc = "[I/O] MSI Interrupt" }, > {.irq = IRQIO_VIR, .name = "VIR", .desc = "[I/O] Virtual I/O Devices"}, > {.irq = IRQIO_VAI, .name = "VAI", .desc = "[I/O] Virtual I/O Devices AI"}, > + {.irq = IRQIO_GAL, .name = "GAL", .desc = "[I/O] GIB Alert"}, > {.irq = NMI_NMI, .name = "NMI", .desc = "[NMI] Machine Check"}, > {.irq = CPU_RST, .name = "RST", .desc = "[CPU] CPU Restart"}, > }; > diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c > index dd80bdc056a5..dae78d91fa53 100644 > --- a/arch/s390/kvm/interrupt.c > +++ b/arch/s390/kvm/interrupt.c > @@ -23,6 +23,7 @@ > #include <asm/gmap.h> > #include <asm/switch_to.h> > #include <asm/nmi.h> > +#include <asm/airq.h> > #include "kvm-s390.h" > #include "gaccess.h" > #include "trace-s390.h" > @@ -2904,7 +2905,7 @@ static void nullify_gisa(struct kvm_s390_gisa *gisa) > #define NONE_GISA_ADDR 0x00000001UL > #define GISA_ADDR_MASK 0xfffff000UL > > -static void __maybe_unused process_gib_alert_list(void) > +static void process_gib_alert_list(void) > { > u32 final, next_alert, origin = 0UL; > struct kvm_s390_gisa *gisa; > @@ -2953,6 +2954,9 @@ static void __maybe_unused process_gib_alert_list(void) > void kvm_s390_gisa_clear(struct kvm *kvm) > { > if (kvm->arch.gisa) { > + kvm->arch.gisa->iam = 0; > + while (in_alert_list(kvm->arch.gisa)) > + process_gib_alert_list(); > nullify_gisa(kvm->arch.gisa); > VM_EVENT(kvm, 3, "gisa 0x%pK cleared", kvm->arch.gisa); > } > @@ -2964,9 +2968,9 @@ void kvm_s390_gisa_init(struct kvm *kvm) > kvm->arch.gisa = &kvm->arch.sie_page2->gisa; > kvm->arch.iam = 0; > spin_lock_init(&kvm->arch.iam_ref_lock); > - VM_EVENT(kvm, 3, "gisa 0x%pK initialized", kvm->arch.gisa); > - kvm_s390_gisa_clear(kvm); > + nullify_gisa(kvm->arch.gisa); > kvm->arch.gib_in_use = !!gib; > + VM_EVENT(kvm, 3, "gisa 0x%pK initialized", kvm->arch.gisa); > } > } > > @@ -2974,6 +2978,10 @@ void kvm_s390_gisa_destroy(struct kvm *kvm) > { > if (!kvm->arch.gisa) > return; > + > + kvm->arch.gisa->iam = 0; > + while (in_alert_list(kvm->arch.gisa)) > + process_gib_alert_list(); > kvm->arch.gisa = NULL; > kvm->arch.iam = 0; > } > @@ -3019,36 +3027,65 @@ int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc) > } > EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister); > > +static void gib_alert_irq_handler(struct airq_struct *airq) > +{ > + inc_irq_stat(IRQIO_GAL); > + process_gib_alert_list(); > +} > + > +static struct airq_struct gib_alert_irq = { > + .handler = gib_alert_irq_handler, > + .lsi_ptr = &gib_alert_irq.lsi_mask, > +}; > + > void kvm_s390_gib_destroy(void) > { > if (!gib) > return; > chsc_sgib(0); > + unregister_adapter_interrupt(&gib_alert_irq); > free_page((unsigned long)gib); > gib = NULL; > } > > int kvm_s390_gib_init(u8 nisc) > { > + int rc = 0; > + > if (!css_general_characteristics.aiv) { > KVM_EVENT(3, "%s", "gib not initialized, no AIV facility"); > - return 0; > + goto out; > } > > gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL | GFP_DMA); > if (!gib) { > KVM_EVENT(3, "gib 0x%pK memory allocation failed", gib); > - return -ENOMEM; > + rc = -ENOMEM; > + goto out; > + } > + > + gib_alert_irq.isc = nisc; > + if (register_adapter_interrupt(&gib_alert_irq)) { > + KVM_EVENT(3, "gib 0x%pK GAI registration failed", gib); > + rc = -EIO; > + goto out_free; > } > > gib->nisc = nisc; > if (chsc_sgib((u32)(u64)gib)) { > KVM_EVENT(3, "gib 0x%pK AIV association failed", gib); > - free_page((unsigned long)gib); > - gib = NULL; > - return -EIO; > + rc = -EIO; > + goto out_unreg; > } > > KVM_EVENT(3, "gib 0x%pK (nisc=%d) initialized", gib, gib->nisc); > - return 0; > + return rc; > + > +out_unreg: > + unregister_adapter_interrupt(&gib_alert_irq); > +out_free: > + free_page((unsigned long)gib); > + gib = NULL; > +out: > + return rc; > } > diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c > index e00eae7ec0b8..283c51362ca8 100644 > --- a/arch/s390/kvm/kvm-s390.c > +++ b/arch/s390/kvm/kvm-s390.c > @@ -3531,6 +3531,10 @@ static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) > vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15; > > atomic_dec(&vcpu->kvm->arch.vcpus_in_sie); > + if (vcpu->kvm->arch.gib_in_use && > + !in_alert_list(vcpu->kvm->arch.gisa) && > + !atomic_read(&vcpu->kvm->arch.vcpus_in_sie)) > + vcpu->kvm->arch.gisa->iam = vcpu->kvm->arch.iam; Here, AFAIU, if the IAM is set, we must take care that not bit from IPM for which IAM is set is also set otherwise we won't get an interruption.
On 28.11.18 17:24, Pierre Morel wrote: > On 28/11/2018 11:19, Michael Mueller wrote: >> The patch implements a handler for GIB alert interruptions >> on the host. Its task is to alert storage backed guests that >> interrupts are pending for them. >> >> A GIB alert interrupt statistic counter is added as well: >> >> $ cat /proc/interrupts >> CPU0 CPU1 >> ... >> GAL: 0 0 [I/O] GIB Alert >> ... >> >> Signed-off-by: Michael Mueller <mimu@linux.ibm.com> >> --- ... >> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c >> index e00eae7ec0b8..283c51362ca8 100644 >> --- a/arch/s390/kvm/kvm-s390.c >> +++ b/arch/s390/kvm/kvm-s390.c >> @@ -3531,6 +3531,10 @@ static int vcpu_post_run(struct kvm_vcpu *vcpu, >> int exit_reason) >> vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15; >> atomic_dec(&vcpu->kvm->arch.vcpus_in_sie); >> + if (vcpu->kvm->arch.gib_in_use && >> + !in_alert_list(vcpu->kvm->arch.gisa) && >> + !atomic_read(&vcpu->kvm->arch.vcpus_in_sie)) >> + vcpu->kvm->arch.gisa->iam = vcpu->kvm->arch.iam; > > > Here, AFAIU, if the IAM is set, we must take care that not bit from IPM > for which IAM is set is also set otherwise we won't get an interruption. My view is that we have a short time window between SIE exit and the restoration of the IAM in the GISA, when an interruption might be indicated by setting the respective ISC bit in the IPM. We will not get an GAL interruption in the host for that. I think until here we are on the same page, right? In that situation the host needs to take action. And it is taken, see the trace. These interruptions are delivered outside of SIE because they are observed by: kvm_s390_deliver_pending_interrupts() deliverable_irqs(vcpu) IPM != 0 __deliver_io() 00 01543426770:725504 4 - 01 00000000e280ea16 inject: I/O (AI/gisa) isc 3 00 01543426770:727023 4 - 02 00000000676fd4ca 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 00 01543426770:727197 4 - 02 00000000676fd4ca 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 00 01543426770:727353 4 - 02 00000000676fd4ca 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 00 01543426770:727506 4 - 02 00000000676fd4ca 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 00 01543426770:731878 4 - 02 00000000676fd4ca 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 00 01543426770:731987 4 - 01 00000000e280ea16 inject: I/O (AI/gisa) isc 3 > >
On 28/11/2018 18:51, Michael Mueller wrote: > > > On 28.11.18 17:24, Pierre Morel wrote: >> On 28/11/2018 11:19, Michael Mueller wrote: >>> The patch implements a handler for GIB alert interruptions >>> on the host. Its task is to alert storage backed guests that >>> interrupts are pending for them. >>> >>> A GIB alert interrupt statistic counter is added as well: >>> >>> $ cat /proc/interrupts >>> CPU0 CPU1 >>> ... >>> GAL: 0 0 [I/O] GIB Alert >>> ... >>> >>> Signed-off-by: Michael Mueller <mimu@linux.ibm.com> >>> --- > > ... > >>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c >>> index e00eae7ec0b8..283c51362ca8 100644 >>> --- a/arch/s390/kvm/kvm-s390.c >>> +++ b/arch/s390/kvm/kvm-s390.c >>> @@ -3531,6 +3531,10 @@ static int vcpu_post_run(struct kvm_vcpu >>> *vcpu, int exit_reason) >>> vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15; >>> atomic_dec(&vcpu->kvm->arch.vcpus_in_sie); >>> + if (vcpu->kvm->arch.gib_in_use && >>> + !in_alert_list(vcpu->kvm->arch.gisa) && >>> + !atomic_read(&vcpu->kvm->arch.vcpus_in_sie)) >>> + vcpu->kvm->arch.gisa->iam = vcpu->kvm->arch.iam; >> >> >> Here, AFAIU, if the IAM is set, we must take care that not bit from IPM >> for which IAM is set is also set otherwise we won't get an interruption. > > My view is that we have a short time window between SIE exit and > the restoration of the IAM in the GISA, when an interruption might > be indicated by setting the respective ISC bit in the IPM. We will > not get an GAL interruption in the host for that. I think until > here we are on the same page, right? > > In that situation the host needs to take action. > > And it is taken, see the trace. These interruptions are delivered > outside of SIE because they are observed by: > > kvm_s390_deliver_pending_interrupts() > deliverable_irqs(vcpu) > IPM != 0 > __deliver_io() If I don't make mistake, this code is only executed during vcpu_pre_run(). What I understand is that the next time the vcpu is scheduled, for example due to another interrupt, the IPM interrupts are delivered through this channel. In the case the vcpu does not get scheduled they are not delivered. > > 00 01543426770:725504 4 - 01 00000000e280ea16 inject: I/O (AI/gisa) isc 3 > 00 01543426770:727023 4 - 02 00000000676fd4ca > 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 > 00 01543426770:727197 4 - 02 00000000676fd4ca > 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 > 00 01543426770:727353 4 - 02 00000000676fd4ca > 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 > 00 01543426770:727506 4 - 02 00000000676fd4ca > 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 > 00 01543426770:731878 4 - 02 00000000676fd4ca > 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 > 00 01543426770:731987 4 - 01 00000000e280ea16 inject: I/O (AI/gisa) isc 3 > > >> >> > AFAIU you do not need to deliver IRQ which IPM is set in GISA during vcpu_pre_run(), they will be delivered to the guest on SIE entry. I tested this Code example in: Message-Id: <1541009577-29656-8-git-send-email-pmorel@linux.ibm.com> Back to vcpu_post_run(), I am not sure that this is the better place to reset the IAM. Shouldn't we set the IAM only if the last active vcpu issues a halt? Is there other SIE exit reasons to activate the alert list? Regards, Pierre
On 29.11.18 11:10, Pierre Morel wrote: > On 28/11/2018 18:51, Michael Mueller wrote: >> >> >> On 28.11.18 17:24, Pierre Morel wrote: >>> On 28/11/2018 11:19, Michael Mueller wrote: >>>> The patch implements a handler for GIB alert interruptions >>>> on the host. Its task is to alert storage backed guests that >>>> interrupts are pending for them. >>>> >>>> A GIB alert interrupt statistic counter is added as well: >>>> >>>> $ cat /proc/interrupts >>>> CPU0 CPU1 >>>> ... >>>> GAL: 0 0 [I/O] GIB Alert >>>> ... >>>> >>>> Signed-off-by: Michael Mueller <mimu@linux.ibm.com> >>>> --- >> >> ... >> >>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c >>>> index e00eae7ec0b8..283c51362ca8 100644 >>>> --- a/arch/s390/kvm/kvm-s390.c >>>> +++ b/arch/s390/kvm/kvm-s390.c >>>> @@ -3531,6 +3531,10 @@ static int vcpu_post_run(struct kvm_vcpu >>>> *vcpu, int exit_reason) >>>> vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15; >>>> atomic_dec(&vcpu->kvm->arch.vcpus_in_sie); >>>> + if (vcpu->kvm->arch.gib_in_use && >>>> + !in_alert_list(vcpu->kvm->arch.gisa) && >>>> + !atomic_read(&vcpu->kvm->arch.vcpus_in_sie)) >>>> + vcpu->kvm->arch.gisa->iam = vcpu->kvm->arch.iam; >>> >>> >>> Here, AFAIU, if the IAM is set, we must take care that not bit from IPM >>> for which IAM is set is also set otherwise we won't get an interruption. >> >> My view is that we have a short time window between SIE exit and >> the restoration of the IAM in the GISA, when an interruption might >> be indicated by setting the respective ISC bit in the IPM. We will >> not get an GAL interruption in the host for that. I think until >> here we are on the same page, right? >> >> In that situation the host needs to take action. >> >> And it is taken, see the trace. These interruptions are delivered >> outside of SIE because they are observed by: >> >> kvm_s390_deliver_pending_interrupts() >> deliverable_irqs(vcpu) >> IPM != 0 >> __deliver_io() > > If I don't make mistake, this code is only executed during vcpu_pre_run(). > > What I understand is that the next time the vcpu is scheduled, for > example due to another interrupt, the IPM interrupts are delivered > through this channel. > > In the case the vcpu does not get scheduled they are not delivered. Idle vcpus will always get scheduled by the hrtimer function kvm_s390_idle_wakeup() which calls kvm_s390_vcpu_wakeup() which then go through kvm_s390_deliver_pending_interrupts() and deliver the interruption. That's what we see in the trace below. (Well that's an assumption but I know that only 5 interruptions were generated by the device) The question is if the interruption can't be delivered more early instead of waiting for one of the ckc_timers to expire. > > > >> >> 00 01543426770:725504 4 - 01 00000000e280ea16 inject: I/O (AI/gisa) >> isc 3 >> 00 01543426770:727023 4 - 02 00000000676fd4ca >> 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 >> 00 01543426770:727197 4 - 02 00000000676fd4ca >> 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 >> 00 01543426770:727353 4 - 02 00000000676fd4ca >> 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 >> 00 01543426770:727506 4 - 02 00000000676fd4ca >> 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 >> 00 01543426770:731878 4 - 02 00000000676fd4ca >> 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 >> 00 01543426770:731987 4 - 01 00000000e280ea16 inject: I/O (AI/gisa) >> isc 3 >> >> >>> >>> >> > > AFAIU you do not need to deliver IRQ which IPM is set in GISA during > vcpu_pre_run(), they will be delivered to the guest on SIE entry. I know that is the patch that I drew back, but for other reasons. > > I tested this Code example in: > Message-Id: <1541009577-29656-8-git-send-email-pmorel@linux.ibm.com> > > > Back to vcpu_post_run(), I am not sure that this is the better place to > reset the IAM. What better place you have in mind? > Shouldn't we set the IAM only if the last active vcpu issues a halt? Can you be little more specific with "vcpu issues a halt"? My view is the none of the vcpus of a guest are in SIE context. Currently I identify this situation with the vcpus_in_sie metric. > > Is there other SIE exit reasons to activate the alert list? I don't think there is. Have you read something? > > Regards, > Pierre > >
On 29/11/2018 14:10, Michael Mueller wrote: > > > On 29.11.18 11:10, Pierre Morel wrote: >> On 28/11/2018 18:51, Michael Mueller wrote: >>> >>> >>> On 28.11.18 17:24, Pierre Morel wrote: >>>> On 28/11/2018 11:19, Michael Mueller wrote: >>>>> The patch implements a handler for GIB alert interruptions >>>>> on the host. Its task is to alert storage backed guests that >>>>> interrupts are pending for them. >>>>> >>>>> A GIB alert interrupt statistic counter is added as well: >>>>> >>>>> $ cat /proc/interrupts >>>>> CPU0 CPU1 >>>>> ... >>>>> GAL: 0 0 [I/O] GIB Alert >>>>> ... >>>>> >>>>> Signed-off-by: Michael Mueller <mimu@linux.ibm.com> >>>>> --- >>> >>> ... >>> >>>>> diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c >>>>> index e00eae7ec0b8..283c51362ca8 100644 >>>>> --- a/arch/s390/kvm/kvm-s390.c >>>>> +++ b/arch/s390/kvm/kvm-s390.c >>>>> @@ -3531,6 +3531,10 @@ static int vcpu_post_run(struct kvm_vcpu >>>>> *vcpu, int exit_reason) >>>>> vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15; >>>>> atomic_dec(&vcpu->kvm->arch.vcpus_in_sie); >>>>> + if (vcpu->kvm->arch.gib_in_use && >>>>> + !in_alert_list(vcpu->kvm->arch.gisa) && >>>>> + !atomic_read(&vcpu->kvm->arch.vcpus_in_sie)) >>>>> + vcpu->kvm->arch.gisa->iam = vcpu->kvm->arch.iam; >>>> >>>> >>>> Here, AFAIU, if the IAM is set, we must take care that not bit from IPM >>>> for which IAM is set is also set otherwise we won't get an >>>> interruption. >>> >>> My view is that we have a short time window between SIE exit and >>> the restoration of the IAM in the GISA, when an interruption might >>> be indicated by setting the respective ISC bit in the IPM. We will >>> not get an GAL interruption in the host for that. I think until >>> here we are on the same page, right? >>> >>> In that situation the host needs to take action. >>> >>> And it is taken, see the trace. These interruptions are delivered >>> outside of SIE because they are observed by: >>> >>> kvm_s390_deliver_pending_interrupts() >>> deliverable_irqs(vcpu) >>> IPM != 0 >>> __deliver_io() >> >> If I don't make mistake, this code is only executed during >> vcpu_pre_run(). >> >> What I understand is that the next time the vcpu is scheduled, for >> example due to another interrupt, the IPM interrupts are delivered >> through this channel. >> >> In the case the vcpu does not get scheduled they are not delivered. > > Idle vcpus will always get scheduled by the hrtimer function > kvm_s390_idle_wakeup() which calls kvm_s390_vcpu_wakeup() which then > go through kvm_s390_deliver_pending_interrupts() and deliver the > interruption. > > That's what we see in the trace below. (Well that's an assumption but I > know that only 5 interruptions were generated by the device) > > The question is if the interruption can't be delivered more early > instead of waiting for one of the ckc_timers to expire. Yes, what if the timer takes a very long time. Think of NO_HZ_IDLE configuration > >> >> >> >>> >>> 00 01543426770:725504 4 - 01 00000000e280ea16 inject: I/O (AI/gisa) >>> isc 3 >>> 00 01543426770:727023 4 - 02 00000000676fd4ca >>> 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 >>> 00 01543426770:727197 4 - 02 00000000676fd4ca >>> 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 >>> 00 01543426770:727353 4 - 02 00000000676fd4ca >>> 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 >>> 00 01543426770:727506 4 - 02 00000000676fd4ca >>> 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 >>> 00 01543426770:731878 4 - 02 00000000676fd4ca >>> 00[0706c00180000000-00000000008e9428]: deliver: I/O (AI/gisa) isc 6 >>> 00 01543426770:731987 4 - 01 00000000e280ea16 inject: I/O (AI/gisa) >>> isc 3 >>> >>> >>>> >>>> >>> >> >> AFAIU you do not need to deliver IRQ which IPM is set in GISA during >> vcpu_pre_run(), they will be delivered to the guest on SIE entry. > > I know that is the patch that I drew back, but for other reasons. Yes but this is not what is done here. Here you deliver the interrupt on the next schedule but you do not provoque the schedule, just wait for the next interrupt. If it comes. contined here under.... > >> >> I tested this Code example in: >> Message-Id: <1541009577-29656-8-git-send-email-pmorel@linux.ibm.com> >> >> >> Back to vcpu_post_run(), I am not sure that this is the better place >> to reset the IAM. > > What better place you have in mind? ...continued from above yes, when the vCPU issues a WAIT (sorry not HALT), in post_run(), kvm_handle_sie_intercept() > >> Shouldn't we set the IAM only if the last active vcpu issues a halt? > > Can you be little more specific with "vcpu issues a halt"? My view > is the none of the vcpus of a guest are in SIE context. Currently > I identify this situation with the vcpus_in_sie metric. in __set_cpu_idle() seems to me to excatly follow the documentation. There would be two metrics: - 1: are vCPU running - 2: is one vCPU in wait state You need to setup the IAM only in the case (NO vCPU running && at least one CPU wait) > >> >> Is there other SIE exit reasons to activate the alert list? > > I don't think there is. Have you read something? No, I think there is no other reason to activate the alert list than when the condition above becomes true. In all other SIE exit reason the vCPU will be rescheduled after the event has been processed. STOP is special, but we surely do not want to send an IRQ to a stopped vCPU. > >> >> Regards, >> Pierre >> >> >
diff --git a/arch/s390/include/asm/irq.h b/arch/s390/include/asm/irq.h index 2f7f27e5493f..afaf5e3c57fd 100644 --- a/arch/s390/include/asm/irq.h +++ b/arch/s390/include/asm/irq.h @@ -62,6 +62,7 @@ enum interruption_class { IRQIO_MSI, IRQIO_VIR, IRQIO_VAI, + IRQIO_GAL, NMI_NMI, CPU_RST, NR_ARCH_IRQS diff --git a/arch/s390/include/asm/isc.h b/arch/s390/include/asm/isc.h index 6cb9e2ed05b6..b2cc1ec78d06 100644 --- a/arch/s390/include/asm/isc.h +++ b/arch/s390/include/asm/isc.h @@ -21,6 +21,7 @@ /* Adapter interrupts. */ #define QDIO_AIRQ_ISC IO_SCH_ISC /* I/O subchannel in qdio mode */ #define PCI_ISC 2 /* PCI I/O subchannels */ +#define GAL_ISC 5 /* GIB alert */ #define AP_ISC 6 /* adjunct processor (crypto) devices */ /* Functions for registration of I/O interruption subclasses */ diff --git a/arch/s390/kernel/irq.c b/arch/s390/kernel/irq.c index 0e8d68bac82c..0cd5a5f96729 100644 --- a/arch/s390/kernel/irq.c +++ b/arch/s390/kernel/irq.c @@ -88,6 +88,7 @@ static const struct irq_class irqclass_sub_desc[] = { {.irq = IRQIO_MSI, .name = "MSI", .desc = "[I/O] MSI Interrupt" }, {.irq = IRQIO_VIR, .name = "VIR", .desc = "[I/O] Virtual I/O Devices"}, {.irq = IRQIO_VAI, .name = "VAI", .desc = "[I/O] Virtual I/O Devices AI"}, + {.irq = IRQIO_GAL, .name = "GAL", .desc = "[I/O] GIB Alert"}, {.irq = NMI_NMI, .name = "NMI", .desc = "[NMI] Machine Check"}, {.irq = CPU_RST, .name = "RST", .desc = "[CPU] CPU Restart"}, }; diff --git a/arch/s390/kvm/interrupt.c b/arch/s390/kvm/interrupt.c index dd80bdc056a5..dae78d91fa53 100644 --- a/arch/s390/kvm/interrupt.c +++ b/arch/s390/kvm/interrupt.c @@ -23,6 +23,7 @@ #include <asm/gmap.h> #include <asm/switch_to.h> #include <asm/nmi.h> +#include <asm/airq.h> #include "kvm-s390.h" #include "gaccess.h" #include "trace-s390.h" @@ -2904,7 +2905,7 @@ static void nullify_gisa(struct kvm_s390_gisa *gisa) #define NONE_GISA_ADDR 0x00000001UL #define GISA_ADDR_MASK 0xfffff000UL -static void __maybe_unused process_gib_alert_list(void) +static void process_gib_alert_list(void) { u32 final, next_alert, origin = 0UL; struct kvm_s390_gisa *gisa; @@ -2953,6 +2954,9 @@ static void __maybe_unused process_gib_alert_list(void) void kvm_s390_gisa_clear(struct kvm *kvm) { if (kvm->arch.gisa) { + kvm->arch.gisa->iam = 0; + while (in_alert_list(kvm->arch.gisa)) + process_gib_alert_list(); nullify_gisa(kvm->arch.gisa); VM_EVENT(kvm, 3, "gisa 0x%pK cleared", kvm->arch.gisa); } @@ -2964,9 +2968,9 @@ void kvm_s390_gisa_init(struct kvm *kvm) kvm->arch.gisa = &kvm->arch.sie_page2->gisa; kvm->arch.iam = 0; spin_lock_init(&kvm->arch.iam_ref_lock); - VM_EVENT(kvm, 3, "gisa 0x%pK initialized", kvm->arch.gisa); - kvm_s390_gisa_clear(kvm); + nullify_gisa(kvm->arch.gisa); kvm->arch.gib_in_use = !!gib; + VM_EVENT(kvm, 3, "gisa 0x%pK initialized", kvm->arch.gisa); } } @@ -2974,6 +2978,10 @@ void kvm_s390_gisa_destroy(struct kvm *kvm) { if (!kvm->arch.gisa) return; + + kvm->arch.gisa->iam = 0; + while (in_alert_list(kvm->arch.gisa)) + process_gib_alert_list(); kvm->arch.gisa = NULL; kvm->arch.iam = 0; } @@ -3019,36 +3027,65 @@ int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc) } EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister); +static void gib_alert_irq_handler(struct airq_struct *airq) +{ + inc_irq_stat(IRQIO_GAL); + process_gib_alert_list(); +} + +static struct airq_struct gib_alert_irq = { + .handler = gib_alert_irq_handler, + .lsi_ptr = &gib_alert_irq.lsi_mask, +}; + void kvm_s390_gib_destroy(void) { if (!gib) return; chsc_sgib(0); + unregister_adapter_interrupt(&gib_alert_irq); free_page((unsigned long)gib); gib = NULL; } int kvm_s390_gib_init(u8 nisc) { + int rc = 0; + if (!css_general_characteristics.aiv) { KVM_EVENT(3, "%s", "gib not initialized, no AIV facility"); - return 0; + goto out; } gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL | GFP_DMA); if (!gib) { KVM_EVENT(3, "gib 0x%pK memory allocation failed", gib); - return -ENOMEM; + rc = -ENOMEM; + goto out; + } + + gib_alert_irq.isc = nisc; + if (register_adapter_interrupt(&gib_alert_irq)) { + KVM_EVENT(3, "gib 0x%pK GAI registration failed", gib); + rc = -EIO; + goto out_free; } gib->nisc = nisc; if (chsc_sgib((u32)(u64)gib)) { KVM_EVENT(3, "gib 0x%pK AIV association failed", gib); - free_page((unsigned long)gib); - gib = NULL; - return -EIO; + rc = -EIO; + goto out_unreg; } KVM_EVENT(3, "gib 0x%pK (nisc=%d) initialized", gib, gib->nisc); - return 0; + return rc; + +out_unreg: + unregister_adapter_interrupt(&gib_alert_irq); +out_free: + free_page((unsigned long)gib); + gib = NULL; +out: + return rc; } diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index e00eae7ec0b8..283c51362ca8 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3531,6 +3531,10 @@ static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15; atomic_dec(&vcpu->kvm->arch.vcpus_in_sie); + if (vcpu->kvm->arch.gib_in_use && + !in_alert_list(vcpu->kvm->arch.gisa) && + !atomic_read(&vcpu->kvm->arch.vcpus_in_sie)) + vcpu->kvm->arch.gisa->iam = vcpu->kvm->arch.iam; if (exit_reason == -EINTR) { VCPU_EVENT(vcpu, 3, "%s", "machine check"); diff --git a/arch/s390/kvm/kvm-s390.h b/arch/s390/kvm/kvm-s390.h index 1a79105b0e9f..1c93ceaeb9e4 100644 --- a/arch/s390/kvm/kvm-s390.h +++ b/arch/s390/kvm/kvm-s390.h @@ -196,6 +196,11 @@ static inline int kvm_s390_user_cpu_state_ctrl(struct kvm *kvm) return kvm->arch.user_cpu_state_ctrl != 0; } +static inline bool in_alert_list(struct kvm_s390_gisa *gisa) +{ + return (u32)(u64)gisa != gisa->next_alert; +} + /* implemented in interrupt.c */ int kvm_s390_handle_wait(struct kvm_vcpu *vcpu); void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu);
The patch implements a handler for GIB alert interruptions on the host. Its task is to alert storage backed guests that interrupts are pending for them. A GIB alert interrupt statistic counter is added as well: $ cat /proc/interrupts CPU0 CPU1 ... GAL: 0 0 [I/O] GIB Alert ... Signed-off-by: Michael Mueller <mimu@linux.ibm.com> --- arch/s390/include/asm/irq.h | 1 + arch/s390/include/asm/isc.h | 1 + arch/s390/kernel/irq.c | 1 + arch/s390/kvm/interrupt.c | 55 +++++++++++++++++++++++++++++++++++++-------- arch/s390/kvm/kvm-s390.c | 4 ++++ arch/s390/kvm/kvm-s390.h | 5 +++++ 6 files changed, 58 insertions(+), 9 deletions(-)