[BUG] win2008 guest cannot get ip through sriov

Commit Message

Jan Beulich June 2, 2016, 12:08 p.m. UTC

>>> On 02.06.16 at 13:03, <[e-mail]> wrote:
> On Thu, Jun 02, 2016 at 04:38:47AM -0600, Jan Beulich wrote:
>> >>> On 02.06.16 at 12:22, <[e-mail]> wrote:
>> > On Thu, Jun 02, 2016 at 07:31:06AM +0000, Xu, Quan wrote:
>> >> On May 27, 2016 10:06 PM, Jan Beulich <[e-mail]> wrote:
>> >> > >>> On 27.05.16 at 15:34, <[e-mail]> wrote:
>> >> > > On Fri, May 27, 2016 at 06:16:30AM -0600, Jan Beulich wrote:
>> >> > >> >>> On 27.05.16 at 12:39, <[e-mail]> wrote:
>> >> > >> > Is this a regression? Does it work on previous versions of Xen?
>> >> > >>
>> >> > >> I think this is what was already reported by other Intel people, see
>> >> > >> e.g. Quan's most recent reply:
>> >> > >> http://lists.xenproject.org/archives/html/xen-devel/2016-05/msg01896.
>> >> > >> html It is not clear where the problem is, and not seeing the issue
>> >> > >> myself makes it hard to analyze. In any event this quite likely is a
>> >> > >> regression.
>> >> > >>
>> >> > >
>> >> > > My reading of that email thread and all relevant links (including the
>> >> > > KVM bug report) is that there is a regression vf driver, but not in Xen.
>> >> > 
>> >> > Just from reading that I would tend to agree. But the report here is about
>> >> > Win2K8.
>> >> 
>> >> Do you know which commit is a regression one? I try to find out the 
>> > regression commit.  That may be helpful to find out the root cause.
>> >> 
>> >> Btw, some feedback from QA team, rhel 6.4 VM  doesn't work, but rhel 7.2 VM 
> does.
>> > 
>> > Isn't this at least an indication that the guest could be buggy here?
>> > It could also be both the hypervsior and guest have bugs. But we're just
>> > not sure at this point.
>> 
>> Indeed, and (with the many fixes that went in already) I really
>> suspect a combination of both, or some of the involved hypervisor
>> changes having unmasked some guest issue. Regardless, I'm
>> afraid this ought to be treated as a blocker for the release at
>> least until we understand what the issue is. But otoh making it a
>> blocker probably makes sense only if we can expect progress
>> (which we haven't really made for quite long a time).
>> 
> 
> This issue is on my list, but the information gathered so far isn't
> convincing enough to make it a blocker.
> 
> And yes, we need meaningful progress to make it a blocker. To make it
> so, commitment from various parties is needed. Let's start with setting
> out things to look at, who is going to investigate what, and a possible
> timeline for each item.
> 
> Jan, can you come up with a list of what sort of information you need?

Well, I had hoped to avoid that. But now that you ask for it,
providing an initial debugging patch seems better than a
description which may get misunderstood. Attached both a
hypervisor and a qemu patch. Their plus debug key M and i
output is what I'd like to start with.

Jan

> And then maybe Quan and Pengtao can give an estimation on how long it
> takes to gather all necessary information and move on to next stage.
> 
> Wei.
> 
>> Jan
>>

--- a/hw/xen/xen_pt.h
+++ b/hw/xen/xen_pt.h
@@ -10,6 +10,7 @@ void xen_pt_log(const PCIDevice *d, cons
 
 #define XEN_PT_ERR(d, _f, _a...) xen_pt_log(d, "%s: Error: "_f, __func__, ##_a)
 
+#define XEN_PT_LOGGING_ENABLED//temp
 #ifdef XEN_PT_LOGGING_ENABLED
 #  define XEN_PT_LOG(d, _f, _a...)  xen_pt_log(d, "%s: " _f, __func__, ##_a)
 #  define XEN_PT_WARN(d, _f, _a...) \
--- a/hw/xen/xen_pt_config_init.c
+++ b/hw/xen/xen_pt_config_init.c
@@ -1474,6 +1474,8 @@ static int xen_pt_msixctrl_reg_write(Xen
     writable_mask = reg->emu_mask & ~reg->ro_mask & valid_mask;
     *data = XEN_PT_MERGE_VALUE(*val, *data, writable_mask);
 
+XEN_PT_LOG(&s->dev, "MSI-X ctrl %04x (%04x/%04x)\n",//temp
+           *val, *data, XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask));//temp
     /* create value for writing to I/O device register */
     *val = XEN_PT_MERGE_VALUE(*val, dev_value, throughable_mask);
 
@@ -2027,7 +2029,7 @@ int xen_pt_config_init(XenPCIPassthrough
                                                   reg_grp_offset,
                                                   &reg_grp_entry->size);
             if (rc < 0) {
-                XEN_PT_LOG(&s->dev, "Failed to initialize %d/%ld, type=0x%x, rc:%d\n",
+                XEN_PT_LOG(&s->dev, "Failed to initialize %d/%zd, type=0x%x, rc:%d\n",
                            i, ARRAY_SIZE(xen_pt_emu_reg_grps),
                            xen_pt_emu_reg_grps[i].grp_type, rc);
                 xen_pt_config_delete(s);
@@ -2044,7 +2046,7 @@ int xen_pt_config_init(XenPCIPassthrough
                     /* initialize capability register */
                     rc = xen_pt_config_reg_init(s, reg_grp_entry, regs);
                     if (rc < 0) {
-                        XEN_PT_LOG(&s->dev, "Failed to initialize %d/%ld reg 0x%x in grp_type=0x%x (%d/%ld), rc=%d\n",
+                        XEN_PT_LOG(&s->dev, "Failed to initialize %d/%zd reg 0x%x in grp_type=0x%x (%d/%zd), rc=%d\n",
                                    j, ARRAY_SIZE(xen_pt_emu_reg_grps[i].emu_regs),
                                    regs->offset, xen_pt_emu_reg_grps[i].grp_type,
                                    i, ARRAY_SIZE(xen_pt_emu_reg_grps), rc);
--- a/hw/xen/xen_pt_msi.c
+++ b/hw/xen/xen_pt_msi.c
@@ -444,6 +444,7 @@ static void pci_msix_write(void *opaque,
     entry = &msix->msix_entry[entry_nr];
     offset = addr % PCI_MSIX_ENTRY_SIZE;
 
+XEN_PT_LOG(&s->dev, "[%"PRIx64"]=%08"PRIx64" (%u,%x)\n", addr, val, entry_nr, offset);//temp
     if (offset != PCI_MSIX_ENTRY_VECTOR_CTRL) {
         if (get_entry_value(entry, offset) == val
             && entry->pirq != XEN_PT_UNASSIGNED_PIRQ) {

Comments

Quan Xu June 2, 2016, 1:05 p.m. UTC | #1

On June 02, 2016 8:09 PM, Jan Beulich <[e-mail]> wrote:
> >>> On 02.06.16 at 13:03, <[e-mail]> wrote:
> > On Thu, Jun 02, 2016 at 04:38:47AM -0600, Jan Beulich wrote:
> >> >>> On 02.06.16 at 12:22, <[e-mail]> wrote:
> >> > On Thu, Jun 02, 2016 at 07:31:06AM +0000, Xu, Quan wrote:
> >> >> On May 27, 2016 10:06 PM, Jan Beulich <[e-mail]> wrote:
> >> >> > >>> On 27.05.16 at 15:34, <[e-mail]> wrote:
> >> >> > > On Fri, May 27, 2016 at 06:16:30AM -0600, Jan Beulich wrote:
> >> >> > >> >>> On 27.05.16 at 12:39, <[e-mail]> wrote:
> >> >> > >> > Is this a regression? Does it work on previous versions of Xen?
> >> >> > >>
> >> >> > >> I think this is what was already reported by other Intel
> >> >> > >> people, see e.g. Quan's most recent reply:
> >> >> > >> http://lists.xenproject.org/archives/html/xen-devel/2016-
> 05/msg01896.
> >> >> > >> html It is not clear where the problem is, and not seeing the
> >> >> > >> issue myself makes it hard to analyze. In any event this
> >> >> > >> quite likely is a regression.
> >> >> > >>
> >> >> > >
> >> >> > > My reading of that email thread and all relevant links
> >> >> > > (including the KVM bug report) is that there is a regression vf driver,
> but not in Xen.
> >> >> >
> >> >> > Just from reading that I would tend to agree. But the report
> >> >> > here is about Win2K8.
> >> >>
> >> >> Do you know which commit is a regression one? I try to find out
> >> >> the
> >> > regression commit.  That may be helpful to find out the root cause.
> >> >>
> >> >> Btw, some feedback from QA team, rhel 6.4 VM  doesn't work, but
> >> >> rhel 7.2 VM
> > does.
> >> >
> >> > Isn't this at least an indication that the guest could be buggy here?
> >> > It could also be both the hypervsior and guest have bugs. But we're
> >> > just not sure at this point.
> >>
> >> Indeed, and (with the many fixes that went in already) I really
> >> suspect a combination of both, or some of the involved hypervisor
> >> changes having unmasked some guest issue. Regardless, I'm afraid this
> >> ought to be treated as a blocker for the release at least until we
> >> understand what the issue is. But otoh making it a blocker probably
> >> makes sense only if we can expect progress (which we haven't really
> >> made for quite long a time).
> >>
> >
> > This issue is on my list, but the information gathered so far isn't
> > convincing enough to make it a blocker.
> >
> > And yes, we need meaningful progress to make it a blocker. To make it
> > so, commitment from various parties is needed. Let's start with
> > setting out things to look at, who is going to investigate what, and a
> > possible timeline for each item.
> >
> > Jan, can you come up with a list of what sort of information you need?
> 
> Well, I had hoped to avoid that. But now that you ask for it, providing an initial
> debugging patch seems better than a description which may get
> misunderstood. Attached both a hypervisor and a qemu patch. Their plus
> debug key M and i output is what I'd like to start with.
> 


     I will try these 2 patches.

btw. I read the internal Bugzilla carefully. I found that vf is working for win2k8  at  '2014-12-01 14:32:09 EST', but the bug still exist on ' 2015-02-11 15:54:05 EST '.
then, I grepped the commit logs, the below 4 MSI-X related commits are may the root cause.


From 6fb3a07bc0ad656b5f76eb9fc961bcd1d3cace58 Mon Sep 17 00:00:00 2001
From: Jan Beulich <[e-mail]>
Date: Fri, 12 Dec 2014 10:24:13 +0000
Subject: [PATCH 13/44] domctl: fix IRQ permission granting/revocation


From 1965728cd5a1635859158f5800d844fc16774668 Mon Sep 17 00:00:00 2001
From: Konrad Rzeszutek Wilk <[e-mail]>
Date: Mon, 12 Jan 2015 11:29:33 -0500
Subject: [PATCH 42/44] Revert "dpci: add 'masked' as a gate for
 hvm_dirq_assist to process"


From 72f3c1e26e96686a41d2de1663e578538659f99a Mon Sep 17 00:00:00 2001
From: Konrad Rzeszutek Wilk <[e-mail]>
Date: Mon, 12 Jan 2015 11:30:00 -0500
Subject: [PATCH 43/44] Revert "dpci: replace tasklet with softirq"

rom a8ac2290ed95dbbc0dc1bdde86fc3a49fe784b28 Mon Sep 17 00:00:00 2001
From: Konrad Rzeszutek Wilk <[e-mail]>
Date: Mon, 12 Jan 2015 11:30:05 -0500
Subject: [PATCH 44/44] Revert "dpci: move from an hvm_irq_dpci (and struct
 domain) to an hvm_dirq_dpci model"

Quan


> Jan
> 
> > And then maybe Quan and Pengtao can give an estimation on how long it
> > takes to gather all necessary information and move on to next stage.
> >
> > Wei.
> >
> >> Jan
> >>
> 
>

Jan Beulich June 2, 2016, 1:30 p.m. UTC | #2

>>> On 02.06.16 at 15:05, <[e-mail]> wrote:
> On June 02, 2016 8:09 PM, Jan Beulich <[e-mail]> wrote:
>> >>> On 02.06.16 at 13:03, <[e-mail]> wrote:
>> > On Thu, Jun 02, 2016 at 04:38:47AM -0600, Jan Beulich wrote:
>> >> >>> On 02.06.16 at 12:22, <[e-mail]> wrote:
>> >> > On Thu, Jun 02, 2016 at 07:31:06AM +0000, Xu, Quan wrote:
>> >> >> On May 27, 2016 10:06 PM, Jan Beulich <[e-mail]> wrote:
>> >> >> > >>> On 27.05.16 at 15:34, <[e-mail]> wrote:
>> >> >> > > On Fri, May 27, 2016 at 06:16:30AM -0600, Jan Beulich wrote:
>> >> >> > >> >>> On 27.05.16 at 12:39, <[e-mail]> wrote:
>> >> >> > >> > Is this a regression? Does it work on previous versions of Xen?
>> >> >> > >>
>> >> >> > >> I think this is what was already reported by other Intel
>> >> >> > >> people, see e.g. Quan's most recent reply:
>> >> >> > >> http://lists.xenproject.org/archives/html/xen-devel/2016- 
>> 05/msg01896.
>> >> >> > >> html It is not clear where the problem is, and not seeing the
>> >> >> > >> issue myself makes it hard to analyze. In any event this
>> >> >> > >> quite likely is a regression.
>> >> >> > >>
>> >> >> > >
>> >> >> > > My reading of that email thread and all relevant links
>> >> >> > > (including the KVM bug report) is that there is a regression vf driver,
>> but not in Xen.
>> >> >> >
>> >> >> > Just from reading that I would tend to agree. But the report
>> >> >> > here is about Win2K8.
>> >> >>
>> >> >> Do you know which commit is a regression one? I try to find out
>> >> >> the
>> >> > regression commit.  That may be helpful to find out the root cause.
>> >> >>
>> >> >> Btw, some feedback from QA team, rhel 6.4 VM  doesn't work, but
>> >> >> rhel 7.2 VM
>> > does.
>> >> >
>> >> > Isn't this at least an indication that the guest could be buggy here?
>> >> > It could also be both the hypervsior and guest have bugs. But we're
>> >> > just not sure at this point.
>> >>
>> >> Indeed, and (with the many fixes that went in already) I really
>> >> suspect a combination of both, or some of the involved hypervisor
>> >> changes having unmasked some guest issue. Regardless, I'm afraid this
>> >> ought to be treated as a blocker for the release at least until we
>> >> understand what the issue is. But otoh making it a blocker probably
>> >> makes sense only if we can expect progress (which we haven't really
>> >> made for quite long a time).
>> >>
>> >
>> > This issue is on my list, but the information gathered so far isn't
>> > convincing enough to make it a blocker.
>> >
>> > And yes, we need meaningful progress to make it a blocker. To make it
>> > so, commitment from various parties is needed. Let's start with
>> > setting out things to look at, who is going to investigate what, and a
>> > possible timeline for each item.
>> >
>> > Jan, can you come up with a list of what sort of information you need?
>> 
>> Well, I had hoped to avoid that. But now that you ask for it, providing an 
> initial
>> debugging patch seems better than a description which may get
>> misunderstood. Attached both a hypervisor and a qemu patch. Their plus
>> debug key M and i output is what I'd like to start with.
>> 
> 
> 
>      I will try these 2 patches.
> 
> btw. I read the internal Bugzilla carefully. I found that vf is working for 
> win2k8  at  '2014-12-01 14:32:09 EST', but the bug still exist on ' 2015-02-11 
> 15:54:05 EST '.
> then, I grepped the commit logs, the below 4 MSI-X related commits are may 
> the root cause.

DYM "vif is not working", or is the use of "still" wrong (or at least
misleading in this context)? Because if the two dates frame the
introduction of the bug, then it must be something completely
different from what I so far have been thinking of.

> From 6fb3a07bc0ad656b5f76eb9fc961bcd1d3cace58 Mon Sep 17 00:00:00 2001
> From: Jan Beulich <[e-mail]>
> Date: Fri, 12 Dec 2014 10:24:13 +0000
> Subject: [PATCH 13/44] domctl: fix IRQ permission granting/revocation
> 
> 
> From 1965728cd5a1635859158f5800d844fc16774668 Mon Sep 17 00:00:00 2001
> From: Konrad Rzeszutek Wilk <[e-mail]>
> Date: Mon, 12 Jan 2015 11:29:33 -0500
> Subject: [PATCH 42/44] Revert "dpci: add 'masked' as a gate for
>  hvm_dirq_assist to process"
> 
> 
> From 72f3c1e26e96686a41d2de1663e578538659f99a Mon Sep 17 00:00:00 2001
> From: Konrad Rzeszutek Wilk <[e-mail]>
> Date: Mon, 12 Jan 2015 11:30:00 -0500
> Subject: [PATCH 43/44] Revert "dpci: replace tasklet with softirq"
> 
> rom a8ac2290ed95dbbc0dc1bdde86fc3a49fe784b28 Mon Sep 17 00:00:00 2001
> From: Konrad Rzeszutek Wilk <[e-mail]>
> Date: Mon, 12 Jan 2015 11:30:05 -0500
> Subject: [PATCH 44/44] Revert "dpci: move from an hvm_irq_dpci (and struct
>  domain) to an hvm_dirq_dpci model"

These all pre-date 4.5 - are you saying 4.5.0 (and note, I'm not
asking about 4.5.1 or newer) is also broken? In particular the
reverts above were done on the 4.5 release branch only, i.e.
didn't ever occur on staging (and hence a staging tree from
around that time should then be fine). Yet for understanding
when the issue got introduced we'd need a range of staging
commits.

All in all I'm afraid I'm now more confused than I was before.

Jan

Quan Xu June 2, 2016, 2:02 p.m. UTC | #3

On June 02, 2016 9:30 PM, Jan Beulich <[e-mail]> wrote:
> >>> On 02.06.16 at 15:05, <[e-mail]> wrote:
> > On June 02, 2016 8:09 PM, Jan Beulich <[e-mail]> wrote:
> >> >>> On 02.06.16 at 13:03, <[e-mail]> wrote:
> >> > On Thu, Jun 02, 2016 at 04:38:47AM -0600, Jan Beulich wrote:
> >> >> >>> On 02.06.16 at 12:22, <[e-mail]> wrote:
> >> >> > On Thu, Jun 02, 2016 at 07:31:06AM +0000, Xu, Quan wrote:
> >> >> >> On May 27, 2016 10:06 PM, Jan Beulich <[e-mail]>
> wrote:
> >> >> >> > >>> On 27.05.16 at 15:34, <[e-mail]> wrote:
> >> >> >> > > On Fri, May 27, 2016 at 06:16:30AM -0600, Jan Beulich wrote:
> >> >> >> > >> >>> On 27.05.16 at 12:39, <[e-mail]> wrote:
> >> >> >> > >> > Is this a regression? Does it work on previous versions of Xen?
> >> >> >> > >>
> >> >> >> > >> I think this is what was already reported by other Intel
> >> >> >> > >> people, see e.g. Quan's most recent reply:
> >> >> >> > >> http://lists.xenproject.org/archives/html/xen-devel/2016-
> >> 05/msg01896.
> >> >> >> > >> html It is not clear where the problem is, and not seeing
> >> >> >> > >> the issue myself makes it hard to analyze. In any event
> >> >> >> > >> this quite likely is a regression.
> >> >> >> > >>
> >> >> >> > >
> >> >> >> > > My reading of that email thread and all relevant links
> >> >> >> > > (including the KVM bug report) is that there is a
> >> >> >> > > regression vf driver,
> >> but not in Xen.
> >> >> >> >
> >> >> >> > Just from reading that I would tend to agree. But the report
> >> >> >> > here is about Win2K8.
> >> >> >>
> >> >> >> Do you know which commit is a regression one? I try to find out
> >> >> >> the
> >> >> > regression commit.  That may be helpful to find out the root cause.
> >> >> >>
> >> >> >> Btw, some feedback from QA team, rhel 6.4 VM  doesn't work, but
> >> >> >> rhel 7.2 VM
> >> > does.
> >> >> >
> >> >> > Isn't this at least an indication that the guest could be buggy here?
> >> >> > It could also be both the hypervsior and guest have bugs. But
> >> >> > we're just not sure at this point.
> >> >>
> >> >> Indeed, and (with the many fixes that went in already) I really
> >> >> suspect a combination of both, or some of the involved hypervisor
> >> >> changes having unmasked some guest issue. Regardless, I'm afraid
> >> >> this ought to be treated as a blocker for the release at least
> >> >> until we understand what the issue is. But otoh making it a
> >> >> blocker probably makes sense only if we can expect progress (which
> >> >> we haven't really made for quite long a time).
> >> >>
> >> >
> >> > This issue is on my list, but the information gathered so far isn't
> >> > convincing enough to make it a blocker.
> >> >
> >> > And yes, we need meaningful progress to make it a blocker. To make
> >> > it so, commitment from various parties is needed. Let's start with
> >> > setting out things to look at, who is going to investigate what,
> >> > and a possible timeline for each item.
> >> >
> >> > Jan, can you come up with a list of what sort of information you need?
> >>
> >> Well, I had hoped to avoid that. But now that you ask for it,
> >> providing an
> > initial
> >> debugging patch seems better than a description which may get
> >> misunderstood. Attached both a hypervisor and a qemu patch. Their
> >> plus debug key M and i output is what I'd like to start with.
> >>
> >
> >
> >      I will try these 2 patches.
> >
> > btw. I read the internal Bugzilla carefully. I found that vf is
> > working for
> > win2k8  at  '2014-12-01 14:32:09 EST', but the bug still exist on '
> > 2015-02-11
> > 15:54:05 EST '.
> > then, I grepped the commit logs, the below 4 MSI-X related commits are
> > may the root cause.
> 
> DYM "vif is not working", or is the use of "still" wrong (or at least misleading in
> this context)? Because if the two dates frame the introduction of the bug,
> then it must be something completely different from what I so far have been
> thinking of.
> 
> > From 6fb3a07bc0ad656b5f76eb9fc961bcd1d3cace58 Mon Sep 17
> 00:00:00 2001
> > From: Jan Beulich <[e-mail]>
> > Date: Fri, 12 Dec 2014 10:24:13 +0000
> > Subject: [PATCH 13/44] domctl: fix IRQ permission granting/revocation
> >
> >
> > From 1965728cd5a1635859158f5800d844fc16774668 Mon Sep 17
> 00:00:00 2001
> > From: Konrad Rzeszutek Wilk <[e-mail]>
> > Date: Mon, 12 Jan 2015 11:29:33 -0500
> > Subject: [PATCH 42/44] Revert "dpci: add 'masked' as a gate for
> > hvm_dirq_assist to process"
> >
> >
> > From 72f3c1e26e96686a41d2de1663e578538659f99a Mon Sep 17
> 00:00:00 2001
> > From: Konrad Rzeszutek Wilk <[e-mail]>
> > Date: Mon, 12 Jan 2015 11:30:00 -0500
> > Subject: [PATCH 43/44] Revert "dpci: replace tasklet with softirq"
> >
> > rom a8ac2290ed95dbbc0dc1bdde86fc3a49fe784b28 Mon Sep 17
> 00:00:00 2001
> > From: Konrad Rzeszutek Wilk <[e-mail]>
> > Date: Mon, 12 Jan 2015 11:30:05 -0500
> > Subject: [PATCH 44/44] Revert "dpci: move from an hvm_irq_dpci (and
> > struct
> >  domain) to an hvm_dirq_dpci model"
> 
> These all pre-date 4.5 - are you saying 4.5.0 (and note, I'm not asking about
> 4.5.1 or newer) is also broken? In particular the reverts above were done on
> the 4.5 release branch only, i.e.
> didn't ever occur on staging (and hence a staging tree from around that time
> should then be fine). Yet for understanding when the issue got introduced
> we'd need a range of staging commits.
> 
> All in all I'm afraid I'm now more confused than I was before.
> 


Sorry, this may be noise to you. Ignore them.


Quan

Quan Xu June 7, 2016, 5:52 a.m. UTC | #4

On June 02, 2016 8:09 PM, Jan Beulich <[e-mail]> wrote:
> Attached both a hypervisor and a qemu patch. Their plus
> debug key M and i output is what I'd like to start with.

Applied these two patches, the attach files are logs.

-vf PT is working for SLES 12. The logs:
   qemu-dm-sles.log -- qemu log, vf PT for sles 12.
   xen-sles.log -- xen log, vf PT for sles 12.

-vf PT is not working for win2008: the logs:
   qemu-dm-win2k8.log -- qemu log, vf PT for win2008.
   xen-win2k8.log -- xen log, vf PT for win2008.


btw, when I updated the NIC driver for win2008,  actually, it is working.  Xudong, could you help me verify it again?

((
    xen commit: bbfd2d6ccb31a3aeea49c8f9c7884792ddc26e3b
    NIC: Intel Corporation I350 Gigabit Network Connection
     new win2k8 NIC driver ,https://downloadcenter.intel.com/download/18720/Network-Adapter-Driver-for-Windows-Server-2008-Final-Release?product=59679
))

qemu-dm-TestDom.log -- qemu log, pf PT for win2008.


... taken together, IMO, from qemu-dm-win2k8.log(also I have read xen/QEMU code), the win2k8 vf nic driver is not loading. (any idea, feel free to let me know. I will help you try as much as I can..).

Thoughts?

Quan

Quan Xu June 7, 2016, 6:07 a.m. UTC | #5

On June 07, 2016 1:52 PM, Xu, Quan <[e-mail]> wrote:
> On June 02, 2016 8:09 PM, Jan Beulich <[e-mail]> wrote:
>

Sorry, this below part is about NIC PF pass-through for win2008..
 
> btw, when I updated the NIC driver for win2008,  actually, it is working.
> Xudong, could you help me verify it again?
> 
> ((
>     xen commit: bbfd2d6ccb31a3aeea49c8f9c7884792ddc26e3b
>     NIC: Intel Corporation I350 Gigabit Network Connection
>      new win2k8 NIC
> driver ,https://downloadcenter.intel.com/download/18720/Network-
> Adapter-Driver-for-Windows-Server-2008-Final-Release?product=59679
> ))
> 
> qemu-dm-TestDom.log -- qemu log, pf PT for win2008.
> 
> 
... :(:(

Quan

Jan Beulich June 7, 2016, 7:43 a.m. UTC | #6

>>> On 07.06.16 at 08:07, <[e-mail]> wrote:
> On June 07, 2016 1:52 PM, Xu, Quan <[e-mail]> wrote:
>> On June 02, 2016 8:09 PM, Jan Beulich <[e-mail]> wrote:
>>
> 
> Sorry, this below part is about NIC PF pass-through for win2008..
>  
>> btw, when I updated the NIC driver for win2008,  actually, it is working.
>> Xudong, could you help me verify it again?
>> 
>> ((
>>     xen commit: bbfd2d6ccb31a3aeea49c8f9c7884792ddc26e3b
>>     NIC: Intel Corporation I350 Gigabit Network Connection
>>      new win2k8 NIC
>> driver ,https://downloadcenter.intel.com/download/18720/Network- 
>> Adapter-Driver-for-Windows-Server-2008-Final-Release?product=59679
>> ))
>> 
>> qemu-dm-TestDom.log -- qemu log, pf PT for win2008.

Okay, but could you please make explicit what works and what
doesn't? I.e. do I take this to mean a passed through PF does
work (at least with the newer driver, maybe also with an older
one), while a passed through VF doesn't work (regardless of
driver version)? Especially with you (earlier) suspecting a driver
issue, precise information here is quite relevant.

Thanks, Jan

Jan Beulich June 7, 2016, 7:49 a.m. UTC | #7

>>> On 07.06.16 at 07:52, <[e-mail]> wrote:
> -vf PT is not working for win2008: the logs:
>    qemu-dm-win2k8.log -- qemu log, vf PT for win2008.
>    xen-win2k8.log -- xen log, vf PT for win2008.

Hmm, that's very little output. In particular neither qemu nor Xen
see _any_ writes to the MSI-X table (without which interrupts
obviously can't get enabled for that device).

Albeit - even in the SLES case only qemu sees such writes, so I'll
have to check if I made a mistake with the debugging patch.

Jan

Quan Xu June 7, 2016, 8:03 a.m. UTC | #8

On June 07, 2016 3:50 PM, Jan Beulich <[e-mail]> wrote:
> >>> On 07.06.16 at 07:52, <[e-mail]> wrote:
> > -vf PT is not working for win2008: the logs:
> >    qemu-dm-win2k8.log -- qemu log, vf PT for win2008.
> >    xen-win2k8.log -- xen log, vf PT for win2008.
> 
> Hmm, that's very little output. In particular neither qemu nor Xen see _any_
> writes to the MSI-X table (without which interrupts obviously can't get
> enabled for that device).
> 
> Albeit - even in the SLES case only qemu sees such writes, so I'll have to check
> if I made a mistake with the debugging patch.
> 

I added more print-out when I debugged it.. I'll call Jianzhong, Chang for more information..
I __guess__ he makes vf work for win2008.

Quan

Jan Beulich June 7, 2016, 8:08 a.m. UTC | #9

>>> On 07.06.16 at 09:49, <[e-mail]> wrote:
>>>> On 07.06.16 at 07:52, <[e-mail]> wrote:
>> -vf PT is not working for win2008: the logs:
>>    qemu-dm-win2k8.log -- qemu log, vf PT for win2008.
>>    xen-win2k8.log -- xen log, vf PT for win2008.
> 
> Hmm, that's very little output. In particular neither qemu nor Xen
> see _any_ writes to the MSI-X table (without which interrupts
> obviously can't get enabled for that device).
> 
> Albeit - even in the SLES case only qemu sees such writes, so I'll
> have to check if I made a mistake with the debugging patch.

With the exact same patches I do see MSI-X table writes logged
with both SLES 11 and SLES 12  guests. What's going on here?

Jan

Quan Xu June 7, 2016, 8:19 a.m. UTC | #10

On June 07, 2016 4:09 PM, Jan Beulich <[e-mail]> wrote:
> >>> On 07.06.16 at 09:49, <[e-mail]> wrote:
> >>>> On 07.06.16 at 07:52, <[e-mail]> wrote:
> >> -vf PT is not working for win2008: the logs:
> >>    qemu-dm-win2k8.log -- qemu log, vf PT for win2008.
> >>    xen-win2k8.log -- xen log, vf PT for win2008.
> >
> > Hmm, that's very little output. In particular neither qemu nor Xen see
> > _any_ writes to the MSI-X table (without which interrupts obviously
> > can't get enabled for that device).
> >
> > Albeit - even in the SLES case only qemu sees such writes, so I'll
> > have to check if I made a mistake with the debugging patch.
> 
> With the exact same patches I do see MSI-X table writes logged with both SLES
> 11 and SLES 12  guests. What's going on here?
> 

The internet cable is not plugged in for that NIC.

Quan

Quan Xu June 8, 2016, 9:12 a.m. UTC | #11

On June 07, 2016 3:50 PM, Jan Beulich <[e-mail]> wrote:
> >>> On 07.06.16 at 07:52, <[e-mail]> wrote:
> > -vf PT is not working for win2008: the logs:
> >    qemu-dm-win2k8.log -- qemu log, vf PT for win2008.
> >    xen-win2k8.log -- xen log, vf PT for win2008.
> 
> Hmm, that's very little output. In particular neither qemu nor Xen see _any_
> writes to the MSI-X table (without which interrupts obviously can't get
> enabled for that device).
> 
> Albeit - even in the SLES case only qemu sees such writes, so I'll have to check
> if I made a mistake with the debugging patch.
> 

Jan,  do you have any other suggestions on how could I dig into this issue?

Quan

Jan Beulich June 8, 2016, 9:25 a.m. UTC | #12

>>> On 08.06.16 at 11:12, <[e-mail]> wrote:
> On June 07, 2016 3:50 PM, Jan Beulich <[e-mail]> wrote:
>> >>> On 07.06.16 at 07:52, <[e-mail]> wrote:
>> > -vf PT is not working for win2008: the logs:
>> >    qemu-dm-win2k8.log -- qemu log, vf PT for win2008.
>> >    xen-win2k8.log -- xen log, vf PT for win2008.
>> 
>> Hmm, that's very little output. In particular neither qemu nor Xen see _any_
>> writes to the MSI-X table (without which interrupts obviously can't get
>> enabled for that device).
>> 
>> Albeit - even in the SLES case only qemu sees such writes, so I'll have to 
> check
>> if I made a mistake with the debugging patch.
>> 
> 
> Jan,  do you have any other suggestions on how could I dig into this issue?

Well - first of all, provide a log with the cable plugged in.

Jan

Hao, Xudong Oct. 27, 2017, 7:27 a.m. UTC | #13

This bug exist much long time, there are many discussion last year but not a solution then. I call out it now because there is a fix in qemu upstream:
commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2
Author: Roger Pau Monne <[e-mail]>
Date:   Thu Aug 24 16:07:03 2017 +0100

    xen/pt: allow QEMU to request MSI unmasking at bind time

The fix is not in qemu-xen tree yet, when will qemu-xen sync this fix? Is it possible to catch Xen 4.10's qemu-xen?

BTW, mail report bug is direct but not easy to track, I took much time to search this BUG report mail. @Lars, is there plan to introduce any bug system for Xen?


Thanks,
-Xudong


> -----Original Message-----
> From: Xu, Quan
> Sent: Wednesday, June 8, 2016 5:12 PM
> To: Jan Beulich <[e-mail]>; Hao, Xudong <[e-mail]>
> Cc: Wei Liu <[e-mail]>; Zhang, PengtaoX
> <[e-mail]>; Xen-devel <[e-mail]>
> Subject: RE: [Xen-devel] [BUG] win2008 guest cannot get ip through sriov
> 
> On June 07, 2016 3:50 PM, Jan Beulich <[e-mail]> wrote:
> > >>> On 07.06.16 at 07:52, <[e-mail]> wrote:
> > > -vf PT is not working for win2008: the logs:
> > >    qemu-dm-win2k8.log -- qemu log, vf PT for win2008.
> > >    xen-win2k8.log -- xen log, vf PT for win2008.
> >
> > Hmm, that's very little output. In particular neither qemu nor Xen see
> > _any_ writes to the MSI-X table (without which interrupts obviously
> > can't get enabled for that device).
> >
> > Albeit - even in the SLES case only qemu sees such writes, so I'll
> > have to check if I made a mistake with the debugging patch.
> >
> 
> Jan,  do you have any other suggestions on how could I dig into this issue?
> 
> Quan

Jan Beulich Oct. 27, 2017, 8:37 a.m. UTC | #14

>>> On 27.10.17 at 09:27, <[e-mail]> wrote:
> This bug exist much long time, there are many discussion last year but not a 
> solution then. I call out it now because there is a fix in qemu upstream:
> commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2
> Author: Roger Pau Monne <[e-mail]>
> Date:   Thu Aug 24 16:07:03 2017 +0100
> 
>     xen/pt: allow QEMU to request MSI unmasking at bind time
> 
> The fix is not in qemu-xen tree yet, when will qemu-xen sync this fix? Is it 
> possible to catch Xen 4.10's qemu-xen?

Well, the question shouldn't have been directed at Quan or me -
at this point it would need to be sorted out between the qemu
maintainers (Stefano and Anthony) and the release manager
(Julien).

Jan

Hao, Xudong Oct. 27, 2017, 8:58 a.m. UTC | #15

> -----Original Message-----
> From: Jan Beulich [mailto:[e-mail]]
> Sent: Friday, October 27, 2017 4:38 PM
> To: Julien Grall <[e-mail]>; Anthony PERARD
> <[e-mail]>; Hao, Xudong <[e-mail]>;
> [e-mail]
> Cc: Lars Kurth <[e-mail]>; Wei Liu <[e-mail]>; Quan Xu
> <[e-mail]>; Kang, Luwei <[e-mail]>; Zhang,
> PengtaoX <[e-mail]>; Xen-devel <xen-
> [e-mail]>
> Subject: RE: [Xen-devel] [BUG] win2008 guest cannot get ip through sriov
> 
> >>> On 27.10.17 at 09:27, <[e-mail]> wrote:
> > This bug exist much long time, there are many discussion last year but
> > not a solution then. I call out it now because there is a fix in qemu upstream:
> > commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2
> > Author: Roger Pau Monne <[e-mail]>
> > Date:   Thu Aug 24 16:07:03 2017 +0100
> >
> >     xen/pt: allow QEMU to request MSI unmasking at bind time
> >
> > The fix is not in qemu-xen tree yet, when will qemu-xen sync this fix?
> > Is it possible to catch Xen 4.10's qemu-xen?
> 
> Well, the question shouldn't have been directed at Quan or me - at this point it
> would need to be sorted out between the qemu maintainers (Stefano and
> Anthony) and the release manager (Julien).
> 
Yes, Thanks to point out it. I just replied this mail (default is you two), and CC the two qemu maintainers and Julien.


Thanks,
-Xudong

> Jan

Julien Grall Oct. 27, 2017, 10:41 a.m. UTC | #16

Hi,

On 27/10/17 08:27, Hao, Xudong wrote:
> This bug exist much long time, there are many discussion last year but not a solution then. I call out it now because there is a fix in qemu upstream:
> commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2
> Author: Roger Pau Monne <[e-mail]>
> Date:   Thu Aug 24 16:07:03 2017 +0100
> 
>      xen/pt: allow QEMU to request MSI unmasking at bind time
> 
> The fix is not in qemu-xen tree yet, when will qemu-xen sync this fix? Is it possible to catch Xen 4.10's qemu-xen?

I will let Stefano and Anthony providing feedback before giving a 
release-ack here.

> 
> BTW, mail report bug is direct but not easy to track, I took much time to search this BUG report mail. @Lars, is there plan to introduce any bug system for Xen?

We recently introduced Jira ([1]) to track features and bugs.

[1] https://xenproject.atlassian.net/projects/XEN/issues.

Cheers,

Stefano Stabellini Oct. 27, 2017, 8:16 p.m. UTC | #17

On Fri, 27 Oct 2017, Julien Grall wrote:
> On 27/10/17 08:27, Hao, Xudong wrote:
> > This bug exist much long time, there are many discussion last year but not a
> > solution then. I call out it now because there is a fix in qemu upstream:
> > commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2
> > Author: Roger Pau Monne <[e-mail]>
> > Date:   Thu Aug 24 16:07:03 2017 +0100
> > 
> >      xen/pt: allow QEMU to request MSI unmasking at bind time
> > 
> > The fix is not in qemu-xen tree yet, when will qemu-xen sync this fix? Is it
> > possible to catch Xen 4.10's qemu-xen?
> 
> I will let Stefano and Anthony providing feedback before giving a release-ack
> here.

Yes, I think we should backport the commit as it fixes a genuine bug.
The backport is not risk-free but it only affects PCI Passthrough. Also
the commit has been in QEMU for 2 months now.


> > 
> > BTW, mail report bug is direct but not easy to track, I took much time to
> > search this BUG report mail. @Lars, is there plan to introduce any bug
> > system for Xen?
> 
> We recently introduced Jira ([1]) to track features and bugs.
> 
> [1] https://xenproject.atlassian.net/projects/XEN/issues.
> 
> Cheers,
> 
> -- 
> Julien Grall
>

Hao, Xudong Oct. 30, 2017, 1:04 a.m. UTC | #18

> -----Original Message-----

> From: Julien Grall [mailto:[e-mail]]

> Sent: Friday, October 27, 2017 6:41 PM

> To: Hao, Xudong <[e-mail]>; Jan Beulich <[e-mail]>;

> Quan Xu <[e-mail]>

> Cc: Lars Kurth <[e-mail]>; [e-mail]; Wei Liu

> <[e-mail]>; Zhang, PengtaoX <[e-mail]>; Kang,

> Luwei <[e-mail]>; Julien Grall <[e-mail]>; Anthony

> PERARD <[e-mail]>; Xen-devel <xen-

> [e-mail]>

> Subject: Re: [Xen-devel] [BUG] win2008 guest cannot get ip through sriov

> 

> Hi,

> 

> On 27/10/17 08:27, Hao, Xudong wrote:

> > This bug exist much long time, there are many discussion last year but not a

> solution then. I call out it now because there is a fix in qemu upstream:

> > commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2

> > Author: Roger Pau Monne <[e-mail]>

> > Date:   Thu Aug 24 16:07:03 2017 +0100

> >

> >      xen/pt: allow QEMU to request MSI unmasking at bind time

> >

> > The fix is not in qemu-xen tree yet, when will qemu-xen sync this fix? Is it

> possible to catch Xen 4.10's qemu-xen?

> 

> I will let Stefano and Anthony providing feedback before giving a release-ack

> here.

> 

> >

> > BTW, mail report bug is direct but not easy to track, I took much time to

> search this BUG report mail. @Lars, is there plan to introduce any bug system

> for Xen?

> 

> We recently introduced Jira ([1]) to track features and bugs.

> 

> [1] https://xenproject.atlassian.net/projects/XEN/issues.

> 


Happy to see that bugs is tracked on it as well.


Thanks,
-Xudong

Julien Grall Nov. 2, 2017, 1:49 p.m. UTC | #19

Hi,

On 27/10/17 21:16, Stefano Stabellini wrote:
> On Fri, 27 Oct 2017, Julien Grall wrote:
>> On 27/10/17 08:27, Hao, Xudong wrote:
>>> This bug exist much long time, there are many discussion last year but not a
>>> solution then. I call out it now because there is a fix in qemu upstream:
>>> commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2
>>> Author: Roger Pau Monne <[e-mail]>
>>> Date:   Thu Aug 24 16:07:03 2017 +0100
>>>
>>>       xen/pt: allow QEMU to request MSI unmasking at bind time
>>>
>>> The fix is not in qemu-xen tree yet, when will qemu-xen sync this fix? Is it
>>> possible to catch Xen 4.10's qemu-xen?
>>
>> I will let Stefano and Anthony providing feedback before giving a release-ack
>> here.
> 
> Yes, I think we should backport the commit as it fixes a genuine bug.
> The backport is not risk-free but it only affects PCI Passthrough. Also
> the commit has been in QEMU for 2 months now.

Does anyone actually tested it with QEMU Xen tree?

Cheers,

Hao, Xudong Nov. 3, 2017, 1:10 a.m. UTC | #20

> -----Original Message-----

> From: Julien Grall [mailto:[e-mail]]

> Sent: Thursday, November 2, 2017 9:50 PM

> To: Stefano Stabellini <[e-mail]>

> Cc: Hao, Xudong <[e-mail]>; Jan Beulich <[e-mail]>;

> Quan Xu <[e-mail]>; Lars Kurth <[e-mail]>; Wei Liu

> <[e-mail]>; Zhang, PengtaoX <[e-mail]>; Kang,

> Luwei <[e-mail]>; Julien Grall <[e-mail]>; Anthony

> PERARD <[e-mail]>; Xen-devel <xen-

> [e-mail]>

> Subject: Re: [Xen-devel] [BUG] win2008 guest cannot get ip through sriov

> 

> Hi,

> 

> On 27/10/17 21:16, Stefano Stabellini wrote:

> > On Fri, 27 Oct 2017, Julien Grall wrote:

> >> On 27/10/17 08:27, Hao, Xudong wrote:

> >>> This bug exist much long time, there are many discussion last year

> >>> but not a solution then. I call out it now because there is a fix in qemu

> upstream:

> >>> commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2

> >>> Author: Roger Pau Monne <[e-mail]>

> >>> Date:   Thu Aug 24 16:07:03 2017 +0100

> >>>

> >>>       xen/pt: allow QEMU to request MSI unmasking at bind time

> >>>

> >>> The fix is not in qemu-xen tree yet, when will qemu-xen sync this

> >>> fix? Is it possible to catch Xen 4.10's qemu-xen?

> >>

> >> I will let Stefano and Anthony providing feedback before giving a

> >> release-ack here.

> >

> > Yes, I think we should backport the commit as it fixes a genuine bug.

> > The backport is not risk-free but it only affects PCI Passthrough.

> > Also the commit has been in QEMU for 2 months now.

> 

> Does anyone actually tested it with QEMU Xen tree?

> 


Qemu Xen tree is default which is in Xen source code configuration file Config.mk, I tested it with it.
QEMU_UPSTREAM_URL ?= http://xenbits.xen.org/git-http/qemu-xen.git


Thanks,
-Xudong

Roger Pau Monné Nov. 3, 2017, 11:23 a.m. UTC | #21

On Fri, Nov 03, 2017 at 01:10:26AM +0000, Hao, Xudong wrote:
> 
> > -----Original Message-----
> > From: Julien Grall [mailto:[e-mail]]
> > Sent: Thursday, November 2, 2017 9:50 PM
> > To: Stefano Stabellini <[e-mail]>
> > Cc: Hao, Xudong <[e-mail]>; Jan Beulich <[e-mail]>;
> > Quan Xu <[e-mail]>; Lars Kurth <[e-mail]>; Wei Liu
> > <[e-mail]>; Zhang, PengtaoX <[e-mail]>; Kang,
> > Luwei <[e-mail]>; Julien Grall <[e-mail]>; Anthony
> > PERARD <[e-mail]>; Xen-devel <xen-
> > [e-mail]>
> > Subject: Re: [Xen-devel] [BUG] win2008 guest cannot get ip through sriov
> > 
> > Hi,
> > 
> > On 27/10/17 21:16, Stefano Stabellini wrote:
> > > On Fri, 27 Oct 2017, Julien Grall wrote:
> > >> On 27/10/17 08:27, Hao, Xudong wrote:
> > >>> This bug exist much long time, there are many discussion last year
> > >>> but not a solution then. I call out it now because there is a fix in qemu
> > upstream:
> > >>> commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2
> > >>> Author: Roger Pau Monne <[e-mail]>
> > >>> Date:   Thu Aug 24 16:07:03 2017 +0100
> > >>>
> > >>>       xen/pt: allow QEMU to request MSI unmasking at bind time
> > >>>
> > >>> The fix is not in qemu-xen tree yet, when will qemu-xen sync this
> > >>> fix? Is it possible to catch Xen 4.10's qemu-xen?
> > >>
> > >> I will let Stefano and Anthony providing feedback before giving a
> > >> release-ack here.
> > >
> > > Yes, I think we should backport the commit as it fixes a genuine bug.
> > > The backport is not risk-free but it only affects PCI Passthrough.
> > > Also the commit has been in QEMU for 2 months now.
> > 
> > Does anyone actually tested it with QEMU Xen tree?
> > 
> 
> Qemu Xen tree is default which is in Xen source code configuration file Config.mk, I tested it with it.
> QEMU_UPSTREAM_URL ?= http://xenbits.xen.org/git-http/qemu-xen.git

Can you please make sure you have QEMU commit a80363: xen/pt: allow
QEMU to request MSI unmasking at bind time. AFAICT this is not yet in
the qemu-xen tree.

Roger.

Hao, Xudong Nov. 6, 2017, 1:04 a.m. UTC | #22

> -----Original Message-----
> From: Roger Pau Monné [mailto:[e-mail]]
> Sent: Friday, November 3, 2017 7:23 PM
> To: Hao, Xudong <[e-mail]>
> Cc: Julien Grall <[e-mail]>; Stefano Stabellini
> <[e-mail]>; Lars Kurth <[e-mail]>; Quan Xu
> <[e-mail]>; Kang, Luwei <[e-mail]>; Zhang,
> PengtaoX <[e-mail]>; Julien Grall <[e-mail]>;
> Jan Beulich <[e-mail]>; Xen-devel <[e-mail]>;
> Anthony PERARD <[e-mail]>; Wei Liu <[e-mail]>
> Subject: Re: [Xen-devel] [BUG] win2008 guest cannot get ip through sriov
> 
> On Fri, Nov 03, 2017 at 01:10:26AM +0000, Hao, Xudong wrote:
> >
> > > -----Original Message-----
> > > From: Julien Grall [mailto:[e-mail]]
> > > Sent: Thursday, November 2, 2017 9:50 PM
> > > To: Stefano Stabellini <[e-mail]>
> > > Cc: Hao, Xudong <[e-mail]>; Jan Beulich
> > > <[e-mail]>; Quan Xu <[e-mail]>; Lars Kurth
> > > <[e-mail]>; Wei Liu <[e-mail]>; Zhang,
> > > PengtaoX <[e-mail]>; Kang, Luwei
> > > <[e-mail]>; Julien Grall <[e-mail]>; Anthony
> > > PERARD <[e-mail]>; Xen-devel <xen-
> > > [e-mail]>
> > > Subject: Re: [Xen-devel] [BUG] win2008 guest cannot get ip through
> > > sriov
> > >
> > > Hi,
> > >
> > > On 27/10/17 21:16, Stefano Stabellini wrote:
> > > > On Fri, 27 Oct 2017, Julien Grall wrote:
> > > >> On 27/10/17 08:27, Hao, Xudong wrote:
> > > >>> This bug exist much long time, there are many discussion last
> > > >>> year but not a solution then. I call out it now because there is
> > > >>> a fix in qemu
> > > upstream:
> > > >>> commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2
> > > >>> Author: Roger Pau Monne <[e-mail]>
> > > >>> Date:   Thu Aug 24 16:07:03 2017 +0100
> > > >>>
> > > >>>       xen/pt: allow QEMU to request MSI unmasking at bind time
> > > >>>
> > > >>> The fix is not in qemu-xen tree yet, when will qemu-xen sync
> > > >>> this fix? Is it possible to catch Xen 4.10's qemu-xen?
> > > >>
> > > >> I will let Stefano and Anthony providing feedback before giving a
> > > >> release-ack here.
> > > >
> > > > Yes, I think we should backport the commit as it fixes a genuine bug.
> > > > The backport is not risk-free but it only affects PCI Passthrough.
> > > > Also the commit has been in QEMU for 2 months now.
> > >
> > > Does anyone actually tested it with QEMU Xen tree?
> > >
> >
> > Qemu Xen tree is default which is in Xen source code configuration file
> Config.mk, I tested it with it.
> > QEMU_UPSTREAM_URL ?= http://xenbits.xen.org/git-http/qemu-xen.git
> 
> Can you please make sure you have QEMU commit a80363: xen/pt: allow QEMU
> to request MSI unmasking at bind time. AFAICT this is not yet in the qemu-xen
> tree.
> 

Roger, 
Maybe I misunderstood of your question and my last mail confused you. Qemu-xen didn't have commit a80363, so I report this issue to ask for sync up with qemu upstream. Last mail I mean I usually used Qemu Xen tree to do test, and found out this issue.


Thanks,
-Xudong

Hao, Xudong Nov. 6, 2017, 1:07 a.m. UTC | #23

> -----Original Message-----
> From: Roger Pau Monné [mailto:[e-mail]]
> Sent: Friday, November 3, 2017 7:23 PM
> To: Hao, Xudong <[e-mail]>
> Cc: Julien Grall <[e-mail]>; Stefano Stabellini
> <[e-mail]>; Lars Kurth <[e-mail]>; Quan Xu
> <[e-mail]>; Kang, Luwei <[e-mail]>; Zhang,
> PengtaoX <[e-mail]>; Julien Grall <[e-mail]>;
> Jan Beulich <[e-mail]>; Xen-devel <[e-mail]>;
> Anthony PERARD <[e-mail]>; Wei Liu <[e-mail]>
> Subject: Re: [Xen-devel] [BUG] win2008 guest cannot get ip through sriov
> 
> On Fri, Nov 03, 2017 at 01:10:26AM +0000, Hao, Xudong wrote:
> >
> > > -----Original Message-----
> > > From: Julien Grall [mailto:[e-mail]]
> > > Sent: Thursday, November 2, 2017 9:50 PM
> > > To: Stefano Stabellini <[e-mail]>
> > > Cc: Hao, Xudong <[e-mail]>; Jan Beulich
> > > <[e-mail]>; Quan Xu <[e-mail]>; Lars Kurth
> > > <[e-mail]>; Wei Liu <[e-mail]>; Zhang,
> > > PengtaoX <[e-mail]>; Kang, Luwei
> > > <[e-mail]>; Julien Grall <[e-mail]>; Anthony
> > > PERARD <[e-mail]>; Xen-devel <xen-
> > > [e-mail]>
> > > Subject: Re: [Xen-devel] [BUG] win2008 guest cannot get ip through
> > > sriov
> > >
> > > Hi,
> > >
> > > On 27/10/17 21:16, Stefano Stabellini wrote:
> > > > On Fri, 27 Oct 2017, Julien Grall wrote:
> > > >> On 27/10/17 08:27, Hao, Xudong wrote:
> > > >>> This bug exist much long time, there are many discussion last
> > > >>> year but not a solution then. I call out it now because there is
> > > >>> a fix in qemu
> > > upstream:
> > > >>> commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2
> > > >>> Author: Roger Pau Monne <[e-mail]>
> > > >>> Date:   Thu Aug 24 16:07:03 2017 +0100
> > > >>>
> > > >>>       xen/pt: allow QEMU to request MSI unmasking at bind time
> > > >>>
> > > >>> The fix is not in qemu-xen tree yet, when will qemu-xen sync
> > > >>> this fix? Is it possible to catch Xen 4.10's qemu-xen?
> > > >>
> > > >> I will let Stefano and Anthony providing feedback before giving a
> > > >> release-ack here.
> > > >
> > > > Yes, I think we should backport the commit as it fixes a genuine bug.
> > > > The backport is not risk-free but it only affects PCI Passthrough.
> > > > Also the commit has been in QEMU for 2 months now.
> > >
> > > Does anyone actually tested it with QEMU Xen tree?
> > >
> >
> > Qemu Xen tree is default which is in Xen source code configuration file
> Config.mk, I tested it with it.
> > QEMU_UPSTREAM_URL ?= http://xenbits.xen.org/git-http/qemu-xen.git
> 
> Can you please make sure you have QEMU commit a80363: xen/pt: allow QEMU
> to request MSI unmasking at bind time. AFAICT this is not yet in the qemu-xen
> tree.
> 

Roger, 
Maybe last mail confused you. Qemu-xen didn't have commit a80363, so I report this issue to ask for sync up with qemu upstream. Last mail I mean I usually used Qemu Xen tree to do test, and found out this issue.


Thanks,
-Xudong

Roger Pau Monné Nov. 7, 2017, 9:33 a.m. UTC | #24

On Mon, Nov 06, 2017 at 01:04:56AM +0000, Hao, Xudong wrote:
> > -----Original Message-----
> > From: Roger Pau Monné [mailto:[e-mail]]
> > Sent: Friday, November 3, 2017 7:23 PM
> > To: Hao, Xudong <[e-mail]>
> > Cc: Julien Grall <[e-mail]>; Stefano Stabellini
> > <[e-mail]>; Lars Kurth <[e-mail]>; Quan Xu
> > <[e-mail]>; Kang, Luwei <[e-mail]>; Zhang,
> > PengtaoX <[e-mail]>; Julien Grall <[e-mail]>;
> > Jan Beulich <[e-mail]>; Xen-devel <[e-mail]>;
> > Anthony PERARD <[e-mail]>; Wei Liu <[e-mail]>
> > Subject: Re: [Xen-devel] [BUG] win2008 guest cannot get ip through sriov
> > 
> > On Fri, Nov 03, 2017 at 01:10:26AM +0000, Hao, Xudong wrote:
> > >
> > > > -----Original Message-----
> > > > From: Julien Grall [mailto:[e-mail]]
> > > > Sent: Thursday, November 2, 2017 9:50 PM
> > > > To: Stefano Stabellini <[e-mail]>
> > > > Cc: Hao, Xudong <[e-mail]>; Jan Beulich
> > > > <[e-mail]>; Quan Xu <[e-mail]>; Lars Kurth
> > > > <[e-mail]>; Wei Liu <[e-mail]>; Zhang,
> > > > PengtaoX <[e-mail]>; Kang, Luwei
> > > > <[e-mail]>; Julien Grall <[e-mail]>; Anthony
> > > > PERARD <[e-mail]>; Xen-devel <xen-
> > > > [e-mail]>
> > > > Subject: Re: [Xen-devel] [BUG] win2008 guest cannot get ip through
> > > > sriov
> > > >
> > > > Hi,
> > > >
> > > > On 27/10/17 21:16, Stefano Stabellini wrote:
> > > > > On Fri, 27 Oct 2017, Julien Grall wrote:
> > > > >> On 27/10/17 08:27, Hao, Xudong wrote:
> > > > >>> This bug exist much long time, there are many discussion last
> > > > >>> year but not a solution then. I call out it now because there is
> > > > >>> a fix in qemu
> > > > upstream:
> > > > >>> commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2
> > > > >>> Author: Roger Pau Monne <[e-mail]>
> > > > >>> Date:   Thu Aug 24 16:07:03 2017 +0100
> > > > >>>
> > > > >>>       xen/pt: allow QEMU to request MSI unmasking at bind time
> > > > >>>
> > > > >>> The fix is not in qemu-xen tree yet, when will qemu-xen sync
> > > > >>> this fix? Is it possible to catch Xen 4.10's qemu-xen?
> > > > >>
> > > > >> I will let Stefano and Anthony providing feedback before giving a
> > > > >> release-ack here.
> > > > >
> > > > > Yes, I think we should backport the commit as it fixes a genuine bug.
> > > > > The backport is not risk-free but it only affects PCI Passthrough.
> > > > > Also the commit has been in QEMU for 2 months now.
> > > >
> > > > Does anyone actually tested it with QEMU Xen tree?
> > > >
> > >
> > > Qemu Xen tree is default which is in Xen source code configuration file
> > Config.mk, I tested it with it.
> > > QEMU_UPSTREAM_URL ?= http://xenbits.xen.org/git-http/qemu-xen.git
> > 
> > Can you please make sure you have QEMU commit a80363: xen/pt: allow QEMU
> > to request MSI unmasking at bind time. AFAICT this is not yet in the qemu-xen
> > tree.
> > 
> 
> Roger, 
> Maybe I misunderstood of your question and my last mail confused you. Qemu-xen didn't have commit a80363, so I report this issue to ask for sync up with qemu upstream. Last mail I mean I usually used Qemu Xen tree to do test, and found out this issue.

Before requesting the backport, have you tested whether it
fixes your issues?

Roger.

Anthony PERARD Nov. 8, 2017, 12:13 p.m. UTC | #25

On Thu, Nov 02, 2017 at 01:49:54PM +0000, Julien Grall wrote:
> On 27/10/17 21:16, Stefano Stabellini wrote:
> > On Fri, 27 Oct 2017, Julien Grall wrote:
> > > On 27/10/17 08:27, Hao, Xudong wrote:
> > > > This bug exist much long time, there are many discussion last year but not a
> > > > solution then. I call out it now because there is a fix in qemu upstream:
> > > > commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2
> > > > Author: Roger Pau Monne <[e-mail]>
> > > > Date:   Thu Aug 24 16:07:03 2017 +0100
> > > > 
> > > >       xen/pt: allow QEMU to request MSI unmasking at bind time
> > > > 
> > > > The fix is not in qemu-xen tree yet, when will qemu-xen sync this fix? Is it
> > > > possible to catch Xen 4.10's qemu-xen?
> > > 
> > > I will let Stefano and Anthony providing feedback before giving a release-ack
> > > here.
> > 
> > Yes, I think we should backport the commit as it fixes a genuine bug.
> > The backport is not risk-free but it only affects PCI Passthrough. Also
> > the commit has been in QEMU for 2 months now.
> 
> Does anyone actually tested it with QEMU Xen tree?

Yes.  I've tested qemu-xen with this patch and PCI passthrough still
works.

Can I get a release-ack?

Thanks,

Julien Grall Nov. 8, 2017, 12:17 p.m. UTC | #26

Hi Anthony,

On 08/11/17 12:13, Anthony PERARD wrote:
> On Thu, Nov 02, 2017 at 01:49:54PM +0000, Julien Grall wrote:
>> On 27/10/17 21:16, Stefano Stabellini wrote:
>>> On Fri, 27 Oct 2017, Julien Grall wrote:
>>>> On 27/10/17 08:27, Hao, Xudong wrote:
>>>>> This bug exist much long time, there are many discussion last year but not a
>>>>> solution then. I call out it now because there is a fix in qemu upstream:
>>>>> commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2
>>>>> Author: Roger Pau Monne <[e-mail]>
>>>>> Date:   Thu Aug 24 16:07:03 2017 +0100
>>>>>
>>>>>        xen/pt: allow QEMU to request MSI unmasking at bind time
>>>>>
>>>>> The fix is not in qemu-xen tree yet, when will qemu-xen sync this fix? Is it
>>>>> possible to catch Xen 4.10's qemu-xen?
>>>>
>>>> I will let Stefano and Anthony providing feedback before giving a release-ack
>>>> here.
>>>
>>> Yes, I think we should backport the commit as it fixes a genuine bug.
>>> The backport is not risk-free but it only affects PCI Passthrough. Also
>>> the commit has been in QEMU for 2 months now.
>>
>> Does anyone actually tested it with QEMU Xen tree?
> 
> Yes.  I've tested qemu-xen with this patch and PCI passthrough still
> works.
> 
> Can I get a release-ack?

I'd still like an answer on Roger's question whether this patch fixes 
the issue they met.

Cheers,

Hao, Xudong Nov. 9, 2017, 12:22 a.m. UTC | #27

> -----Original Message-----
> From: Roger Pau Monné [mailto:[e-mail]]
> Sent: Tuesday, November 7, 2017 5:33 PM
> To: Hao, Xudong <[e-mail]>
> Cc: Julien Grall <[e-mail]>; Stefano Stabellini
> <[e-mail]>; Lars Kurth <[e-mail]>; Quan Xu
> <[e-mail]>; Kang, Luwei <[e-mail]>; Zhang,
> PengtaoX <[e-mail]>; Julien Grall <[e-mail]>;
> Jan Beulich <[e-mail]>; Xen-devel <[e-mail]>;
> Anthony PERARD <[e-mail]>; Wei Liu <[e-mail]>
> Subject: Re: [Xen-devel] [BUG] win2008 guest cannot get ip through sriov
> 
> On Mon, Nov 06, 2017 at 01:04:56AM +0000, Hao, Xudong wrote:
> > > -----Original Message-----
> > > From: Roger Pau Monné [mailto:[e-mail]]
> > > Sent: Friday, November 3, 2017 7:23 PM
> > > To: Hao, Xudong <[e-mail]>
> > > Cc: Julien Grall <[e-mail]>; Stefano Stabellini
> > > <[e-mail]>; Lars Kurth <[e-mail]>; Quan
> > > Xu <[e-mail]>; Kang, Luwei <[e-mail]>; Zhang,
> > > PengtaoX <[e-mail]>; Julien Grall
> > > <[e-mail]>; Jan Beulich <[e-mail]>; Xen-devel
> > > <[e-mail]>; Anthony PERARD
> > > <[e-mail]>; Wei Liu <[e-mail]>
> > > Subject: Re: [Xen-devel] [BUG] win2008 guest cannot get ip through
> > > sriov
> > >
> > > On Fri, Nov 03, 2017 at 01:10:26AM +0000, Hao, Xudong wrote:
> > > >
> > > > > -----Original Message-----
> > > > > From: Julien Grall [mailto:[e-mail]]
> > > > > Sent: Thursday, November 2, 2017 9:50 PM
> > > > > To: Stefano Stabellini <[e-mail]>
> > > > > Cc: Hao, Xudong <[e-mail]>; Jan Beulich
> > > > > <[e-mail]>; Quan Xu <[e-mail]>; Lars Kurth
> > > > > <[e-mail]>; Wei Liu <[e-mail]>; Zhang,
> > > > > PengtaoX <[e-mail]>; Kang, Luwei
> > > > > <[e-mail]>; Julien Grall <[e-mail]>;
> > > > > Anthony PERARD <[e-mail]>; Xen-devel <xen-
> > > > > [e-mail]>
> > > > > Subject: Re: [Xen-devel] [BUG] win2008 guest cannot get ip
> > > > > through sriov
> > > > >
> > > > > Hi,
> > > > >
> > > > > On 27/10/17 21:16, Stefano Stabellini wrote:
> > > > > > On Fri, 27 Oct 2017, Julien Grall wrote:
> > > > > >> On 27/10/17 08:27, Hao, Xudong wrote:
> > > > > >>> This bug exist much long time, there are many discussion
> > > > > >>> last year but not a solution then. I call out it now because
> > > > > >>> there is a fix in qemu
> > > > > upstream:
> > > > > >>> commit a8036336609d2e184fc3543a4c439c0ba7d7f3a2
> > > > > >>> Author: Roger Pau Monne <[e-mail]>
> > > > > >>> Date:   Thu Aug 24 16:07:03 2017 +0100
> > > > > >>>
> > > > > >>>       xen/pt: allow QEMU to request MSI unmasking at bind
> > > > > >>> time
> > > > > >>>
> > > > > >>> The fix is not in qemu-xen tree yet, when will qemu-xen sync
> > > > > >>> this fix? Is it possible to catch Xen 4.10's qemu-xen?
> > > > > >>
> > > > > >> I will let Stefano and Anthony providing feedback before
> > > > > >> giving a release-ack here.
> > > > > >
> > > > > > Yes, I think we should backport the commit as it fixes a genuine bug.
> > > > > > The backport is not risk-free but it only affects PCI Passthrough.
> > > > > > Also the commit has been in QEMU for 2 months now.
> > > > >
> > > > > Does anyone actually tested it with QEMU Xen tree?
> > > > >
> > > >
> > > > Qemu Xen tree is default which is in Xen source code configuration
> > > > file
> > > Config.mk, I tested it with it.
> > > > QEMU_UPSTREAM_URL ?= http://xenbits.xen.org/git-http/qemu-xen.git
> > >
> > > Can you please make sure you have QEMU commit a80363: xen/pt: allow
> > > QEMU to request MSI unmasking at bind time. AFAICT this is not yet
> > > in the qemu-xen tree.
> > >
> >
> > Roger,
> > Maybe I misunderstood of your question and my last mail confused you.
> Qemu-xen didn't have commit a80363, so I report this issue to ask for sync up
> with qemu upstream. Last mail I mean I usually used Qemu Xen tree to do test,
> and found out this issue.
> 
> Before requesting the backport, have you tested whether it fixes your issues?
> 

Yes, Luwei (Cced) tested it with pass result.


Thanks,
-Xudong

Roger Pau Monné Nov. 9, 2017, 9:27 a.m. UTC | #28

On Thu, Nov 09, 2017 at 12:22:49AM +0000, Hao, Xudong wrote:
> > Qemu-xen didn't have commit a80363, so I report this issue to ask for sync up
> > with qemu upstream. Last mail I mean I usually used Qemu Xen tree to do test,
> > and found out this issue.
> > 
> > Before requesting the backport, have you tested whether it fixes your issues?
> > 
> 
> Yes, Luwei (Cced) tested it with pass result.

In which case requesting a backport would be in place on the grounds
that's a bug fix.

The patch in question fixes a bug mostly seen when doing
PCI-passthrough of devices that support MSI-X interrupts to Windows
guest (and Windows attempts to setup the interrupts using MSI-X). It
doesn't manifest on Linux or FreeBSD because these OSes use a
different dance to setup MSI-X interrupts, and thus are not affected.
It's likely that other OSes with different MSI-X implementations are
able to trigger that bug. The result of not having the patch is that
interrupts of passed through devices will stay masked, thus
preventing the device from working (unless polling mode only is
used).

Pros:
 - It fixes a bug.
 - The patch is not very big or intrusive.

Cons:
 - It hasn't been tested as it's not in qemu-xen.
 - Only Windows is affected by the bug AFAIK.

IMHO, iff the backport is accepted it should be performed ASAP, so
that the patch can be properly tested before the release.

Thanks, Roger.

Julien Grall Nov. 9, 2017, 2:36 p.m. UTC | #29

Hi Roger,

On 09/11/17 09:27, Roger Pau Monné wrote:
> On Thu, Nov 09, 2017 at 12:22:49AM +0000, Hao, Xudong wrote:
>>> Qemu-xen didn't have commit a80363, so I report this issue to ask for sync up
>>> with qemu upstream. Last mail I mean I usually used Qemu Xen tree to do test,
>>> and found out this issue.
>>>
>>> Before requesting the backport, have you tested whether it fixes your issues?
>>>
>>
>> Yes, Luwei (Cced) tested it with pass result.
> 
> In which case requesting a backport would be in place on the grounds
> that's a bug fix.
> 
> The patch in question fixes a bug mostly seen when doing
> PCI-passthrough of devices that support MSI-X interrupts to Windows
> guest (and Windows attempts to setup the interrupts using MSI-X). It
> doesn't manifest on Linux or FreeBSD because these OSes use a
> different dance to setup MSI-X interrupts, and thus are not affected.
> It's likely that other OSes with different MSI-X implementations are
> able to trigger that bug. The result of not having the patch is that
> interrupts of passed through devices will stay masked, thus
> preventing the device from working (unless polling mode only is
> used).
> 
> Pros:
>   - It fixes a bug.
>   - The patch is not very big or intrusive.
> 
> Cons:
>   - It hasn't been tested as it's not in qemu-xen.
>   - Only Windows is affected by the bug AFAIK.
> 
> IMHO, iff the backport is accepted it should be performed ASAP, so
> that the patch can be properly tested before the release.

Thank you for the detailed explanation :). On a previous e-mail Stefano 
were happy with the backport. So:

Release-acked-by: Julien Grall <[e-mail]>

Cheers,

Julien Grall Nov. 9, 2017, 2:37 p.m. UTC | #30

On 09/11/17 14:36, Julien Grall wrote:
> Hi Roger,
> 
> On 09/11/17 09:27, Roger Pau Monné wrote:
>> On Thu, Nov 09, 2017 at 12:22:49AM +0000, Hao, Xudong wrote:
>>>> Qemu-xen didn't have commit a80363, so I report this issue to ask 
>>>> for sync up
>>>> with qemu upstream. Last mail I mean I usually used Qemu Xen tree to 
>>>> do test,
>>>> and found out this issue.
>>>>
>>>> Before requesting the backport, have you tested whether it fixes 
>>>> your issues?
>>>>
>>>
>>> Yes, Luwei (Cced) tested it with pass result.
>>
>> In which case requesting a backport would be in place on the grounds
>> that's a bug fix.
>>
>> The patch in question fixes a bug mostly seen when doing
>> PCI-passthrough of devices that support MSI-X interrupts to Windows
>> guest (and Windows attempts to setup the interrupts using MSI-X). It
>> doesn't manifest on Linux or FreeBSD because these OSes use a
>> different dance to setup MSI-X interrupts, and thus are not affected.
>> It's likely that other OSes with different MSI-X implementations are
>> able to trigger that bug. The result of not having the patch is that
>> interrupts of passed through devices will stay masked, thus
>> preventing the device from working (unless polling mode only is
>> used).
>>
>> Pros:
>>   - It fixes a bug.
>>   - The patch is not very big or intrusive.
>>
>> Cons:
>>   - It hasn't been tested as it's not in qemu-xen.
>>   - Only Windows is affected by the bug AFAIK.
>>
>> IMHO, iff the backport is accepted it should be performed ASAP, so
>> that the patch can be properly tested before the release.
> 
> Thank you for the detailed explanation :). On a previous e-mail Stefano 
> were happy with the backport. So:
> 
> Release-acked-by: Julien Grall <[e-mail]>

Note that I think we would need to update xen also to point to the 
commit with this backport included.

Cheers,

Patch

--- a/xen/arch/x86/hvm/vmsi.c
+++ b/xen/arch/x86/hvm/vmsi.c
@@ -276,6 +276,7 @@  static int msixtbl_write(struct vcpu *v,
     if ( !entry )
         goto out;
     nr_entry = (address - entry->gtable) / PCI_MSIX_ENTRY_SIZE;
+printk("%pv: write MSI-X#%u: [%lx]=%0*lx\n", v, nr_entry, address, (int)len * 2, val);//temp
 
     offset = address & (PCI_MSIX_ENTRY_SIZE - 1);
     if ( offset != PCI_MSIX_ENTRY_VECTOR_CTRL_OFFSET )
@@ -321,7 +322,17 @@  static int msixtbl_write(struct vcpu *v,
 
     ASSERT(msi_desc == desc->msi_desc);
    
+{//temp
+ bool_t h = msi_desc->msi_attrib.host_masked;
+ bool_t g = msi_desc->msi_attrib.guest_masked;
+ bool_t ha = entry->pdev->msix->host_maskall;
+ bool_t ga = entry->pdev->msix->guest_maskall;
     guest_mask_msi_irq(desc, !!(val & PCI_MSIX_VECTOR_BITMASK));
+ printk("%pv: MSI-X#%u %d(%d) / %d(%d) -> %d(%d) / %d(%d)\n",
+        v, nr_entry, h, ha, g, ga,
+        msi_desc->msi_attrib.host_masked, entry->pdev->msix->host_maskall,
+        msi_desc->msi_attrib.guest_masked, entry->pdev->msix->guest_maskall);
+}
 
 unlock:
     spin_unlock_irqrestore(&desc->lock, flags);
@@ -330,6 +341,7 @@  unlock:
 
 out:
     rcu_read_unlock(&msixtbl_rcu_lock);
+printk("%pv: write MSI-X [%lx] -> %d\n", v, address, r);//temp
     return r;
 }
 
--- a/xen/arch/x86/msi.c
+++ b/xen/arch/x86/msi.c
@@ -438,6 +438,10 @@  static bool_t msi_set_mask_bit(struct ir
             if ( likely(control & PCI_MSIX_FLAGS_ENABLE) )
                 break;
 
+if(pdev->info.is_virtfn) {//temp
+ printk("%04x:%02x:%02x.%o#%u: %d/%d >> %d/%d [%p]\n", seg, bus, slot, func, entry->msi_attrib.entry_nr,
+        entry->msi_attrib.host_masked, entry->msi_attrib.guest_masked, host, guest, __builtin_return_address(0));
+}
             entry->msi_attrib.host_masked = host;
             entry->msi_attrib.guest_masked = guest;
 
@@ -1305,6 +1309,11 @@  int pci_msi_conf_write_intercept(struct
             pdev->msix->guest_maskall = !!(*data & PCI_MSIX_FLAGS_MASKALL);
             if ( pdev->msix->host_maskall )
                 *data |= PCI_MSIX_FLAGS_MASKALL;
+if(pdev->info.is_virtfn) {//temp
+ printk("%04x:%02x:%02x.%o: ctrl -> %04x (d%d:%lx,d%d)\n", seg, bus, slot, func,
+        (uint16_t)*data, current->domain->domain_id, guest_cpu_user_regs()->eip,
+        pdev->domain ? pdev->domain->domain_id : -1);
+}
 
             return 1;
         }