[v3,4/5] s390x/pci: Add routine to get the vfio dma available count

Commit Message

Matthew Rosato Sept. 15, 2020, 7:14 p.m. UTC

Create new files for separating out vfio-specific work for s390
pci. Add the first such routine, which issues VFIO_IOMMU_GET_INFO
ioctl to collect the current dma available count.

Signed-off-by: Matthew Rosato <[e-mail]>
---
 hw/s390x/meson.build     |  1 +
 hw/s390x/s390-pci-vfio.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++++
 hw/s390x/s390-pci-vfio.h | 17 +++++++++++++++
 3 files changed, 72 insertions(+)
 create mode 100644 hw/s390x/s390-pci-vfio.c
 create mode 100644 hw/s390x/s390-pci-vfio.h

Comments

Philippe Mathieu-Daudé Sept. 16, 2020, 7:21 a.m. UTC | #1

On 9/15/20 9:14 PM, Matthew Rosato wrote:
> Create new files for separating out vfio-specific work for s390
> pci. Add the first such routine, which issues VFIO_IOMMU_GET_INFO
> ioctl to collect the current dma available count.
> 
> Signed-off-by: Matthew Rosato <[e-mail]>
> ---
>  hw/s390x/meson.build     |  1 +
>  hw/s390x/s390-pci-vfio.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++++
>  hw/s390x/s390-pci-vfio.h | 17 +++++++++++++++
>  3 files changed, 72 insertions(+)
>  create mode 100644 hw/s390x/s390-pci-vfio.c
>  create mode 100644 hw/s390x/s390-pci-vfio.h
> 
> diff --git a/hw/s390x/meson.build b/hw/s390x/meson.build
> index b63782d..ed2f66b 100644
> --- a/hw/s390x/meson.build
> +++ b/hw/s390x/meson.build
> @@ -10,6 +10,7 @@ s390x_ss.add(files(
>    's390-ccw.c',
>    's390-pci-bus.c',
>    's390-pci-inst.c',
> +  's390-pci-vfio.c',
>    's390-skeys.c',
>    's390-stattrib.c',
>    's390-virtio-hcall.c',
> diff --git a/hw/s390x/s390-pci-vfio.c b/hw/s390x/s390-pci-vfio.c
> new file mode 100644
> index 0000000..75e3ac1
> --- /dev/null
> +++ b/hw/s390x/s390-pci-vfio.c
> @@ -0,0 +1,54 @@
> +/*
> + * s390 vfio-pci interfaces
> + *
> + * Copyright 2020 IBM Corp.
> + * Author(s): Matthew Rosato <[e-mail]>
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or (at
> + * your option) any later version. See the COPYING file in the top-level
> + * directory.
> + */
> +
> +#include <sys/ioctl.h>
> +
> +#include "qemu/osdep.h"
> +#include "s390-pci-vfio.h"
> +#include "hw/vfio/vfio-common.h"
> +
> +/*
> + * Get the current DMA available count from vfio.  Returns true if vfio is
> + * limiting DMA requests, false otherwise.  The current available count read
> + * from vfio is returned in avail.
> + */
> +bool s390_pci_update_dma_avail(int fd, unsigned int *avail)
> +{
> +    g_autofree struct vfio_iommu_type1_info *info;
> +    uint32_t argsz;
> +    int ret;
> +
> +    assert(avail);
> +
> +    argsz = sizeof(struct vfio_iommu_type1_info);
> +    info = g_malloc0(argsz);
> +    info->argsz = argsz;
> +    /*
> +     * If the specified argsz is not large enough to contain all
> +     * capabilities it will be updated upon return.  In this case
> +     * use the updated value to get the entire capability chain.
> +     */
> +    ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
> +    if (argsz != info->argsz) {
> +        argsz = info->argsz;
> +        info = g_realloc(info, argsz);

Do we need to bzero [sizeof(struct vfio_iommu_type1_info)..argsz[?

> +        info->argsz = argsz;
> +        ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
> +    }
> +
> +    if (ret) {
> +        return false;
> +    }
> +
> +    /* If the capability exists, update with the current value */
> +    return vfio_get_info_dma_avail(info, avail);
> +}
> +
> diff --git a/hw/s390x/s390-pci-vfio.h b/hw/s390x/s390-pci-vfio.h
> new file mode 100644
> index 0000000..2a5a261
> --- /dev/null
> +++ b/hw/s390x/s390-pci-vfio.h
> @@ -0,0 +1,17 @@
> +/*
> + * s390 vfio-pci interfaces
> + *
> + * Copyright 2020 IBM Corp.
> + * Author(s): Matthew Rosato <[e-mail]>
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or (at
> + * your option) any later version. See the COPYING file in the top-level
> + * directory.
> + */
> +
> +#ifndef HW_S390_PCI_VFIO_H
> +#define HW_S390_PCI_VFIO_H
> +
> +bool s390_pci_update_dma_avail(int fd, unsigned int *avail);
> +
> +#endif
>

Cornelia Huck Sept. 16, 2020, 10:27 a.m. UTC | #2

On Wed, 16 Sep 2020 09:21:39 +0200
Philippe Mathieu-Daudé <[e-mail]> wrote:

> On 9/15/20 9:14 PM, Matthew Rosato wrote:
> > Create new files for separating out vfio-specific work for s390
> > pci. Add the first such routine, which issues VFIO_IOMMU_GET_INFO
> > ioctl to collect the current dma available count.
> > 
> > Signed-off-by: Matthew Rosato <[e-mail]>
> > ---
> >  hw/s390x/meson.build     |  1 +
> >  hw/s390x/s390-pci-vfio.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++++
> >  hw/s390x/s390-pci-vfio.h | 17 +++++++++++++++
> >  3 files changed, 72 insertions(+)
> >  create mode 100644 hw/s390x/s390-pci-vfio.c
> >  create mode 100644 hw/s390x/s390-pci-vfio.h
> > 

(...)

> > diff --git a/hw/s390x/s390-pci-vfio.c b/hw/s390x/s390-pci-vfio.c
> > new file mode 100644
> > index 0000000..75e3ac1
> > --- /dev/null
> > +++ b/hw/s390x/s390-pci-vfio.c
> > @@ -0,0 +1,54 @@
> > +/*
> > + * s390 vfio-pci interfaces
> > + *
> > + * Copyright 2020 IBM Corp.
> > + * Author(s): Matthew Rosato <[e-mail]>
> > + *
> > + * This work is licensed under the terms of the GNU GPL, version 2 or (at
> > + * your option) any later version. See the COPYING file in the top-level
> > + * directory.
> > + */
> > +
> > +#include <sys/ioctl.h>
> > +
> > +#include "qemu/osdep.h"
> > +#include "s390-pci-vfio.h"
> > +#include "hw/vfio/vfio-common.h"
> > +
> > +/*
> > + * Get the current DMA available count from vfio.  Returns true if vfio is
> > + * limiting DMA requests, false otherwise.  The current available count read
> > + * from vfio is returned in avail.
> > + */
> > +bool s390_pci_update_dma_avail(int fd, unsigned int *avail)
> > +{
> > +    g_autofree struct vfio_iommu_type1_info *info;
> > +    uint32_t argsz;
> > +    int ret;
> > +
> > +    assert(avail);
> > +
> > +    argsz = sizeof(struct vfio_iommu_type1_info);
> > +    info = g_malloc0(argsz);
> > +    info->argsz = argsz;
> > +    /*
> > +     * If the specified argsz is not large enough to contain all
> > +     * capabilities it will be updated upon return.  In this case
> > +     * use the updated value to get the entire capability chain.
> > +     */
> > +    ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
> > +    if (argsz != info->argsz) {
> > +        argsz = info->argsz;
> > +        info = g_realloc(info, argsz);  
> 
> Do we need to bzero [sizeof(struct vfio_iommu_type1_info)..argsz[?

If we do, I think we need to do the equivalent in
vfio_get_region_info() as well?

(Also, shouldn't we check ret before looking at info->argsz?)

> 
> > +        info->argsz = argsz;
> > +        ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
> > +    }
> > +
> > +    if (ret) {
> > +        return false;
> > +    }
> > +
> > +    /* If the capability exists, update with the current value */
> > +    return vfio_get_info_dma_avail(info, avail);
> > +}
> > +

(...)

Matthew Rosato Sept. 16, 2020, 12:55 p.m. UTC | #3

On 9/16/20 6:27 AM, Cornelia Huck wrote:
> On Wed, 16 Sep 2020 09:21:39 +0200
> Philippe Mathieu-Daudé <[e-mail]> wrote:
> 
>> On 9/15/20 9:14 PM, Matthew Rosato wrote:
>>> Create new files for separating out vfio-specific work for s390
>>> pci. Add the first such routine, which issues VFIO_IOMMU_GET_INFO
>>> ioctl to collect the current dma available count.
>>>
>>> Signed-off-by: Matthew Rosato <[e-mail]>
>>> ---
>>>   hw/s390x/meson.build     |  1 +
>>>   hw/s390x/s390-pci-vfio.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++++
>>>   hw/s390x/s390-pci-vfio.h | 17 +++++++++++++++
>>>   3 files changed, 72 insertions(+)
>>>   create mode 100644 hw/s390x/s390-pci-vfio.c
>>>   create mode 100644 hw/s390x/s390-pci-vfio.h
>>>
> 
> (...)
> 
>>> diff --git a/hw/s390x/s390-pci-vfio.c b/hw/s390x/s390-pci-vfio.c
>>> new file mode 100644
>>> index 0000000..75e3ac1
>>> --- /dev/null
>>> +++ b/hw/s390x/s390-pci-vfio.c
>>> @@ -0,0 +1,54 @@
>>> +/*
>>> + * s390 vfio-pci interfaces
>>> + *
>>> + * Copyright 2020 IBM Corp.
>>> + * Author(s): Matthew Rosato <[e-mail]>
>>> + *
>>> + * This work is licensed under the terms of the GNU GPL, version 2 or (at
>>> + * your option) any later version. See the COPYING file in the top-level
>>> + * directory.
>>> + */
>>> +
>>> +#include <sys/ioctl.h>
>>> +
>>> +#include "qemu/osdep.h"
>>> +#include "s390-pci-vfio.h"
>>> +#include "hw/vfio/vfio-common.h"
>>> +
>>> +/*
>>> + * Get the current DMA available count from vfio.  Returns true if vfio is
>>> + * limiting DMA requests, false otherwise.  The current available count read
>>> + * from vfio is returned in avail.
>>> + */
>>> +bool s390_pci_update_dma_avail(int fd, unsigned int *avail)
>>> +{
>>> +    g_autofree struct vfio_iommu_type1_info *info;
>>> +    uint32_t argsz;
>>> +    int ret;
>>> +
>>> +    assert(avail);
>>> +
>>> +    argsz = sizeof(struct vfio_iommu_type1_info);
>>> +    info = g_malloc0(argsz);
>>> +    info->argsz = argsz;
>>> +    /*
>>> +     * If the specified argsz is not large enough to contain all
>>> +     * capabilities it will be updated upon return.  In this case
>>> +     * use the updated value to get the entire capability chain.
>>> +     */
>>> +    ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
>>> +    if (argsz != info->argsz) {
>>> +        argsz = info->argsz;
>>> +        info = g_realloc(info, argsz);
>>
>> Do we need to bzero [sizeof(struct vfio_iommu_type1_info)..argsz[?
> 
> If we do, I think we need to do the equivalent in
> vfio_get_region_info() as well?
> 

I agree that it would need to be in both places or neither -- I would 
expect the re-driven ioctl to overwrite the prior contents of info 
(unless we get a bad ret, but in this case we don't care what is in info)?

Perhaps the fundamental difference between this code and 
vfio_get_region_info is that the latter checks for only a growing argsz 
and retries, whereas this code checks for != so it's technically 
possible for a smaller argsz to trigger the retry here, and we wouldn't 
know for sure that all bytes from the first ioctl call were overwritten.

What if I adjust this code to look like vfio_get_region_info:

retry:
	info->argsz = argsz;

	if (ioctl(fd, VFIO_IOMMU_GET_INFO, info)) {
		// no need to g_free() bc of g_autofree
		return false;	
	}

	if (info->argsz > argsz) {
		argsz = info->argsz;
		info = g_realloc(info, argsz);
		goto retry;
	}

	/* If the capability exists, update with the current value */
	return vfio_get_info_dma_avail(info, avail);

Now we would only trigger when we are told by the host that the buffer 
must be larger.

> (Also, shouldn't we check ret before looking at info->argsz?)
> 

Yes, you are correct.  The above proposal would fix that issue too.

>>
>>> +        info->argsz = argsz;
>>> +        ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
>>> +    }
>>> +
>>> +    if (ret) {
>>> +        return false;
>>> +    }
>>> +
>>> +    /* If the capability exists, update with the current value */
>>> +    return vfio_get_info_dma_avail(info, avail);
>>> +}
>>> +
> 
> (...)
> 
>

Cornelia Huck Sept. 17, 2020, 9:59 a.m. UTC | #4

On Wed, 16 Sep 2020 08:55:00 -0400
Matthew Rosato <[e-mail]> wrote:

> On 9/16/20 6:27 AM, Cornelia Huck wrote:
> > On Wed, 16 Sep 2020 09:21:39 +0200
> > Philippe Mathieu-Daudé <[e-mail]> wrote:
> >   
> >> On 9/15/20 9:14 PM, Matthew Rosato wrote:  
> >>> Create new files for separating out vfio-specific work for s390
> >>> pci. Add the first such routine, which issues VFIO_IOMMU_GET_INFO
> >>> ioctl to collect the current dma available count.
> >>>
> >>> Signed-off-by: Matthew Rosato <[e-mail]>
> >>> ---
> >>>   hw/s390x/meson.build     |  1 +
> >>>   hw/s390x/s390-pci-vfio.c | 54 ++++++++++++++++++++++++++++++++++++++++++++++++
> >>>   hw/s390x/s390-pci-vfio.h | 17 +++++++++++++++
> >>>   3 files changed, 72 insertions(+)
> >>>   create mode 100644 hw/s390x/s390-pci-vfio.c
> >>>   create mode 100644 hw/s390x/s390-pci-vfio.h
> >>>  
> > 
> > (...)
> >   
> >>> diff --git a/hw/s390x/s390-pci-vfio.c b/hw/s390x/s390-pci-vfio.c
> >>> new file mode 100644
> >>> index 0000000..75e3ac1
> >>> --- /dev/null
> >>> +++ b/hw/s390x/s390-pci-vfio.c
> >>> @@ -0,0 +1,54 @@
> >>> +/*
> >>> + * s390 vfio-pci interfaces
> >>> + *
> >>> + * Copyright 2020 IBM Corp.
> >>> + * Author(s): Matthew Rosato <[e-mail]>
> >>> + *
> >>> + * This work is licensed under the terms of the GNU GPL, version 2 or (at
> >>> + * your option) any later version. See the COPYING file in the top-level
> >>> + * directory.
> >>> + */
> >>> +
> >>> +#include <sys/ioctl.h>
> >>> +
> >>> +#include "qemu/osdep.h"
> >>> +#include "s390-pci-vfio.h"
> >>> +#include "hw/vfio/vfio-common.h"
> >>> +
> >>> +/*
> >>> + * Get the current DMA available count from vfio.  Returns true if vfio is
> >>> + * limiting DMA requests, false otherwise.  The current available count read
> >>> + * from vfio is returned in avail.
> >>> + */
> >>> +bool s390_pci_update_dma_avail(int fd, unsigned int *avail)
> >>> +{
> >>> +    g_autofree struct vfio_iommu_type1_info *info;
> >>> +    uint32_t argsz;
> >>> +    int ret;
> >>> +
> >>> +    assert(avail);
> >>> +
> >>> +    argsz = sizeof(struct vfio_iommu_type1_info);
> >>> +    info = g_malloc0(argsz);
> >>> +    info->argsz = argsz;
> >>> +    /*
> >>> +     * If the specified argsz is not large enough to contain all
> >>> +     * capabilities it will be updated upon return.  In this case
> >>> +     * use the updated value to get the entire capability chain.
> >>> +     */
> >>> +    ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
> >>> +    if (argsz != info->argsz) {
> >>> +        argsz = info->argsz;
> >>> +        info = g_realloc(info, argsz);  
> >>
> >> Do we need to bzero [sizeof(struct vfio_iommu_type1_info)..argsz[?  
> > 
> > If we do, I think we need to do the equivalent in
> > vfio_get_region_info() as well?
> >   
> 
> I agree that it would need to be in both places or neither -- I would 
> expect the re-driven ioctl to overwrite the prior contents of info 
> (unless we get a bad ret, but in this case we don't care what is in info)?
> 
> Perhaps the fundamental difference between this code and 
> vfio_get_region_info is that the latter checks for only a growing argsz 
> and retries, whereas this code checks for != so it's technically 
> possible for a smaller argsz to trigger the retry here, and we wouldn't 
> know for sure that all bytes from the first ioctl call were overwritten.

Nod. Relying on overwriting should be fine.

> 
> What if I adjust this code to look like vfio_get_region_info:
> 
> retry:
> 	info->argsz = argsz;
> 
> 	if (ioctl(fd, VFIO_IOMMU_GET_INFO, info)) {
> 		// no need to g_free() bc of g_autofree
> 		return false;	
> 	}
> 
> 	if (info->argsz > argsz) {
> 		argsz = info->argsz;
> 		info = g_realloc(info, argsz);
> 		goto retry;
> 	}
> 
> 	/* If the capability exists, update with the current value */
> 	return vfio_get_info_dma_avail(info, avail);
> 
> Now we would only trigger when we are told by the host that the buffer 
> must be larger.

I think that makes sense.

> 
> > (Also, shouldn't we check ret before looking at info->argsz?)
> >   
> 
> Yes, you are correct.  The above proposal would fix that issue too.
> 
> >>  
> >>> +        info->argsz = argsz;
> >>> +        ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
> >>> +    }
> >>> +
> >>> +    if (ret) {
> >>> +        return false;
> >>> +    }
> >>> +
> >>> +    /* If the capability exists, update with the current value */
> >>> +    return vfio_get_info_dma_avail(info, avail);
> >>> +}
> >>> +  
> > 
> > (...)
> > 
> >   
>

Patch

diff --git a/hw/s390x/meson.build b/hw/s390x/meson.build
index b63782d..ed2f66b 100644
--- a/hw/s390x/meson.build
+++ b/hw/s390x/meson.build
@@ -10,6 +10,7 @@  s390x_ss.add(files(
   's390-ccw.c',
   's390-pci-bus.c',
   's390-pci-inst.c',
+  's390-pci-vfio.c',
   's390-skeys.c',
   's390-stattrib.c',
   's390-virtio-hcall.c',
diff --git a/hw/s390x/s390-pci-vfio.c b/hw/s390x/s390-pci-vfio.c
new file mode 100644
index 0000000..75e3ac1
--- /dev/null
+++ b/hw/s390x/s390-pci-vfio.c
@@ -0,0 +1,54 @@ 
+/*
+ * s390 vfio-pci interfaces
+ *
+ * Copyright 2020 IBM Corp.
+ * Author(s): Matthew Rosato <[e-mail]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#include <sys/ioctl.h>
+
+#include "qemu/osdep.h"
+#include "s390-pci-vfio.h"
+#include "hw/vfio/vfio-common.h"
+
+/*
+ * Get the current DMA available count from vfio.  Returns true if vfio is
+ * limiting DMA requests, false otherwise.  The current available count read
+ * from vfio is returned in avail.
+ */
+bool s390_pci_update_dma_avail(int fd, unsigned int *avail)
+{
+    g_autofree struct vfio_iommu_type1_info *info;
+    uint32_t argsz;
+    int ret;
+
+    assert(avail);
+
+    argsz = sizeof(struct vfio_iommu_type1_info);
+    info = g_malloc0(argsz);
+    info->argsz = argsz;
+    /*
+     * If the specified argsz is not large enough to contain all
+     * capabilities it will be updated upon return.  In this case
+     * use the updated value to get the entire capability chain.
+     */
+    ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
+    if (argsz != info->argsz) {
+        argsz = info->argsz;
+        info = g_realloc(info, argsz);
+        info->argsz = argsz;
+        ret = ioctl(fd, VFIO_IOMMU_GET_INFO, info);
+    }
+
+    if (ret) {
+        return false;
+    }
+
+    /* If the capability exists, update with the current value */
+    return vfio_get_info_dma_avail(info, avail);
+}
+
diff --git a/hw/s390x/s390-pci-vfio.h b/hw/s390x/s390-pci-vfio.h
new file mode 100644
index 0000000..2a5a261
--- /dev/null
+++ b/hw/s390x/s390-pci-vfio.h
@@ -0,0 +1,17 @@ 
+/*
+ * s390 vfio-pci interfaces
+ *
+ * Copyright 2020 IBM Corp.
+ * Author(s): Matthew Rosato <[e-mail]>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at
+ * your option) any later version. See the COPYING file in the top-level
+ * directory.
+ */
+
+#ifndef HW_S390_PCI_VFIO_H
+#define HW_S390_PCI_VFIO_H
+
+bool s390_pci_update_dma_avail(int fd, unsigned int *avail);
+
+#endif