[v6,1/4] vhost-user: add new vhost user messages to support virtio config space

Commit Message

Liu, Changpeng Dec. 5, 2017, 6:27 a.m. UTC

Add VHOST_USER_GET_CONFIG/VHOST_USER_SET_CONFIG messages which can be
used for live migration of vhost user devices, also vhost user devices
can benefit from the messages to get/set virtio config space from/to the
I/O target. For the purpose to support virtio config space change,
VHOST_USER_SET_CONFIG_FD message is added as the event notifier
in case virtio config space change in the I/O target.

Signed-off-by: Changpeng Liu <[e-mail]>
---
 docs/interop/vhost-user.txt       |  44 ++++++++++++++++
 hw/virtio/vhost-user.c            | 107 ++++++++++++++++++++++++++++++++++++++
 hw/virtio/vhost.c                 |  64 +++++++++++++++++++++++
 include/hw/virtio/vhost-backend.h |  14 +++++
 include/hw/virtio/vhost.h         |  16 ++++++
 5 files changed, 245 insertions(+)

Comments

Stefan Hajnoczi Dec. 11, 2017, 1:39 p.m. UTC | #1

On Tue, Dec 05, 2017 at 02:27:16PM +0800, Changpeng Liu wrote:
> +* VHOST_USER_SET_CONFIG
> +      Id: 25
> +      Equivalent ioctl: N/A
> +      Master payload: virtio device config space
> +
> +      Submitted by the vhost-user master when the Guest changes the virtio
> +      device configuration space and also can be used for live migration
> +      on the destination host. The vhost-user slave must check the flags
> +      filed, and slaves MUST NOT accept SET_CONFIG for read-only

s/filed/field/

> +static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *config,
> +                                 uint32_t offset, uint32_t size, uint32_t flags)
> +{
> +    uint8_t *p;
> +    bool reply_supported = virtio_has_feature(dev->protocol_features,
> +                                              VHOST_USER_PROTOCOL_F_REPLY_ACK);
> +
> +    VhostUserMsg msg = {
> +        msg.request = VHOST_USER_SET_CONFIG,
> +        msg.flags = VHOST_USER_VERSION,
> +        msg.size = VHOST_USER_CONFIG_HDR_SIZE + size,
> +    };
> +
> +    if (reply_supported) {
> +        msg.flags |= VHOST_USER_NEED_REPLY_MASK;
> +    }
> +
> +    msg.payload.config.offset = offset,
> +    msg.payload.config.size = size,
> +    msg.payload.config.flags = flags,
> +    p = msg.payload.config.region;
> +    memcpy(p, config + offset, size);

This function can be made more general by changing the semantics of the
config argument:

  memcpy(p, config, size);

Now the caller can pass just a single field instead of a whole 256-byte
config buffer.  It might be clearer to name the argument "data" or
"region" instead of "config" though.

> @@ -1505,6 +1508,67 @@ void vhost_ack_features(struct vhost_dev *hdev, const int *feature_bits,
>      }
>  }
>  
> +int vhost_dev_get_config(struct vhost_dev *hdev, uint8_t *config,
> +                         uint32_t config_len)
> +{
> +    assert(hdev->vhost_ops);
> +
> +    if (hdev->vhost_ops->vhost_get_config) {
> +        return hdev->vhost_ops->vhost_get_config(hdev, config, config_len);
> +    }
> +
> +    return 0;
> +}
> +
> +int vhost_dev_set_config(struct vhost_dev *hdev, const uint8_t *config,
> +                         uint32_t offset, uint32_t size, uint32_t flags)
> +{
> +    assert(hdev->vhost_ops);
> +
> +    if (hdev->vhost_ops->vhost_set_config) {
> +        return hdev->vhost_ops->vhost_set_config(hdev, config, offset,
> +                                                 size, flags);
> +    }
> +
> +    return 0;
> +}

Both vhost_dev_get_config() and vhost_dev_set_config() cannot fail
silently.  The device will not work properly if the configuration space
feature is not supported.

Please make these functions return an error if the callback is NULL.

The vhost-blk code should also check that
hdev->vhost_ops->vhost_set_config != NULL during realize.  This way
users see the error when adding the device instead of at runtime when
the function gets called.

Liu, Changpeng Dec. 12, 2017, 1:25 a.m. UTC | #2

> -----Original Message-----
> From: Stefan Hajnoczi [mailto:[e-mail]]
> Sent: Monday, December 11, 2017 9:39 PM
> To: Liu, Changpeng <[e-mail]>
> Cc: [e-mail]; [e-mail]; [e-mail];
> [e-mail]; [e-mail]; Harris, James R
> <[e-mail]>
> Subject: Re: [PATCH v6 1/4] vhost-user: add new vhost user messages to support
> virtio config space
> 
> On Tue, Dec 05, 2017 at 02:27:16PM +0800, Changpeng Liu wrote:
> > +* VHOST_USER_SET_CONFIG
> > +      Id: 25
> > +      Equivalent ioctl: N/A
> > +      Master payload: virtio device config space
> > +
> > +      Submitted by the vhost-user master when the Guest changes the virtio
> > +      device configuration space and also can be used for live migration
> > +      on the destination host. The vhost-user slave must check the flags
> > +      filed, and slaves MUST NOT accept SET_CONFIG for read-only
> 
> s/filed/field/
Thanks.
> 
> > +static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *config,
> > +                                 uint32_t offset, uint32_t size, uint32_t flags)
> > +{
> > +    uint8_t *p;
> > +    bool reply_supported = virtio_has_feature(dev->protocol_features,
> > +                                              VHOST_USER_PROTOCOL_F_REPLY_ACK);
> > +
> > +    VhostUserMsg msg = {
> > +        msg.request = VHOST_USER_SET_CONFIG,
> > +        msg.flags = VHOST_USER_VERSION,
> > +        msg.size = VHOST_USER_CONFIG_HDR_SIZE + size,
> > +    };
> > +
> > +    if (reply_supported) {
> > +        msg.flags |= VHOST_USER_NEED_REPLY_MASK;
> > +    }
> > +
> > +    msg.payload.config.offset = offset,
> > +    msg.payload.config.size = size,
> > +    msg.payload.config.flags = flags,
> > +    p = msg.payload.config.region;
> > +    memcpy(p, config + offset, size);
> 
> This function can be made more general by changing the semantics of the
> config argument:
> 
>   memcpy(p, config, size);
> 
> Now the caller can pass just a single field instead of a whole 256-byte
> config buffer.  It might be clearer to name the argument "data" or
> "region" instead of "config" though.
Good suggestion, will change it.
> 
> > @@ -1505,6 +1508,67 @@ void vhost_ack_features(struct vhost_dev *hdev,
> const int *feature_bits,
> >      }
> >  }
> >
> > +int vhost_dev_get_config(struct vhost_dev *hdev, uint8_t *config,
> > +                         uint32_t config_len)
> > +{
> > +    assert(hdev->vhost_ops);
> > +
> > +    if (hdev->vhost_ops->vhost_get_config) {
> > +        return hdev->vhost_ops->vhost_get_config(hdev, config, config_len);
> > +    }
> > +
> > +    return 0;
> > +}
> > +
> > +int vhost_dev_set_config(struct vhost_dev *hdev, const uint8_t *config,
> > +                         uint32_t offset, uint32_t size, uint32_t flags)
> > +{
> > +    assert(hdev->vhost_ops);
> > +
> > +    if (hdev->vhost_ops->vhost_set_config) {
> > +        return hdev->vhost_ops->vhost_set_config(hdev, config, offset,
> > +                                                 size, flags);
> > +    }
> > +
> > +    return 0;
> > +}
> 
> Both vhost_dev_get_config() and vhost_dev_set_config() cannot fail
> silently.  The device will not work properly if the configuration space
> feature is not supported.
> 
> Please make these functions return an error if the callback is NULL.
Ok, will add callback check.
> 
> The vhost-blk code should also check that
> hdev->vhost_ops->vhost_set_config != NULL during realize.  This way
> users see the error when adding the device instead of at runtime when
> the function gets called.
For vhost-blk, get_config is mandatory, but for set_config, it should depend
on the feature bit: VIRTIO_BLK_F_CONFIG_WCE is enabled or not. Of course,
migration should be another case. So running time error process should be
okay.

Stefan Hajnoczi Dec. 12, 2017, 10:20 a.m. UTC | #3

On Tue, Dec 12, 2017 at 01:25:30AM +0000, Liu, Changpeng wrote:
> > The vhost-blk code should also check that
> > hdev->vhost_ops->vhost_set_config != NULL during realize.  This way
> > users see the error when adding the device instead of at runtime when
> > the function gets called.
> For vhost-blk, get_config is mandatory, but for set_config, it should depend
> on the feature bit: VIRTIO_BLK_F_CONFIG_WCE is enabled or not. Of course,
> migration should be another case. So running time error process should be
> okay.

Okay.

Stefan

Patch

diff --git a/docs/interop/vhost-user.txt b/docs/interop/vhost-user.txt
index 954771d..1f9aab0 100644
--- a/docs/interop/vhost-user.txt
+++ b/docs/interop/vhost-user.txt
@@ -116,6 +116,19 @@  Depending on the request type, payload can be:
     - 3: IOTLB invalidate
     - 4: IOTLB access fail
 
+ * Virtio device config space
+   -----------------------------------
+   | offset | size | flags | payload |
+   -----------------------------------
+
+   Offset: a 32-bit offset of virtio device's configuration space
+   Size: a 32-bit configuration space access size in bytes
+   Flags: a 32-bit value:
+    - 0: Vhost master messages used for writeable fields
+    - 1: Vhost master messages used for live migration
+   Payload: Size bytes array holding the contents of the virtio
+       device's configuration space
+
 In QEMU the vhost-user message is implemented with the following struct:
 
 typedef struct VhostUserMsg {
@@ -129,6 +142,7 @@  typedef struct VhostUserMsg {
         VhostUserMemory memory;
         VhostUserLog log;
         struct vhost_iotlb_msg iotlb;
+        VhostUserConfig config;
     };
 } QEMU_PACKED VhostUserMsg;
 
@@ -596,6 +610,36 @@  Master message types
       and expect this message once (per VQ) during device configuration
       (ie. before the master starts the VQ).
 
+ * VHOST_USER_GET_CONFIG
+      Id: 24
+      Equivalent ioctl: N/A
+      Master payload: virtio device config space
+
+      Submitted by the vhost-user master to fetch the contents of the virtio
+      device configuration space. The vhost-user master may cache the contents
+      to avoid repeated VHOST_USER_GET_CONFIG calls.
+
+* VHOST_USER_SET_CONFIG
+      Id: 25
+      Equivalent ioctl: N/A
+      Master payload: virtio device config space
+
+      Submitted by the vhost-user master when the Guest changes the virtio
+      device configuration space and also can be used for live migration
+      on the destination host. The vhost-user slave must check the flags
+      filed, and slaves MUST NOT accept SET_CONFIG for read-only
+      configuration space fields unless the live migration bit is set.
+
+* VHOST_USER_SET_CONFIG_FD
+      Id: 26
+      Equivalent ioctl: N/A
+      Master payload: N/A
+
+      Sets the notifier file descriptor, which is passed as ancillary data.
+      The vhost-user slave uses the file descriptor to notify the vhost-user
+      master of changes to the virtio configuration space. The vhost-user
+      master can read the virtio configuration space to get the latest update.
+
 Slave message types
 -------------------
 
diff --git a/hw/virtio/vhost-user.c b/hw/virtio/vhost-user.c
index 093675e..bdb236c 100644
--- a/hw/virtio/vhost-user.c
+++ b/hw/virtio/vhost-user.c
@@ -26,6 +26,11 @@ 
 #define VHOST_MEMORY_MAX_NREGIONS    8
 #define VHOST_USER_F_PROTOCOL_FEATURES 30
 
+/*
+ * Maximum size of virtio device config space
+ */
+#define VHOST_USER_MAX_CONFIG_SIZE 256
+
 enum VhostUserProtocolFeature {
     VHOST_USER_PROTOCOL_F_MQ = 0,
     VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
@@ -65,6 +70,9 @@  typedef enum VhostUserRequest {
     VHOST_USER_SET_SLAVE_REQ_FD = 21,
     VHOST_USER_IOTLB_MSG = 22,
     VHOST_USER_SET_VRING_ENDIAN = 23,
+    VHOST_USER_GET_CONFIG = 24,
+    VHOST_USER_SET_CONFIG = 25,
+    VHOST_USER_SET_CONFIG_FD = 26,
     VHOST_USER_MAX
 } VhostUserRequest;
 
@@ -92,6 +100,18 @@  typedef struct VhostUserLog {
     uint64_t mmap_offset;
 } VhostUserLog;
 
+typedef struct VhostUserConfig {
+    uint32_t offset;
+    uint32_t size;
+    uint32_t flags;
+    uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
+} VhostUserConfig;
+
+static VhostUserConfig c __attribute__ ((unused));
+#define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
+                                   + sizeof(c.size) \
+                                   + sizeof(c.flags))
+
 typedef struct VhostUserMsg {
     VhostUserRequest request;
 
@@ -109,6 +129,7 @@  typedef struct VhostUserMsg {
         VhostUserMemory memory;
         VhostUserLog log;
         struct vhost_iotlb_msg iotlb;
+        VhostUserConfig config;
     } payload;
 } QEMU_PACKED VhostUserMsg;
 
@@ -922,6 +943,89 @@  static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled)
     /* No-op as the receive channel is not dedicated to IOTLB messages. */
 }
 
+static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config,
+                                 uint32_t config_len)
+{
+    VhostUserMsg msg = {
+        msg.request = VHOST_USER_GET_CONFIG,
+        msg.flags = VHOST_USER_VERSION,
+        msg.size = VHOST_USER_CONFIG_HDR_SIZE + config_len,
+    };
+
+    msg.payload.config.offset = 0;
+    msg.payload.config.size = config_len;
+    if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
+        return -1;
+    }
+
+    if (vhost_user_read(dev, &msg) < 0) {
+        return -1;
+    }
+
+    if (msg.request != VHOST_USER_GET_CONFIG) {
+        error_report("Received unexpected msg type. Expected %d received %d",
+                     VHOST_USER_GET_CONFIG, msg.request);
+        return -1;
+    }
+
+    if (msg.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) {
+        error_report("Received bad msg size.");
+        return -1;
+    }
+
+    memcpy(config, msg.payload.config.region, config_len);
+
+    return 0;
+}
+
+static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *config,
+                                 uint32_t offset, uint32_t size, uint32_t flags)
+{
+    uint8_t *p;
+    bool reply_supported = virtio_has_feature(dev->protocol_features,
+                                              VHOST_USER_PROTOCOL_F_REPLY_ACK);
+
+    VhostUserMsg msg = {
+        msg.request = VHOST_USER_SET_CONFIG,
+        msg.flags = VHOST_USER_VERSION,
+        msg.size = VHOST_USER_CONFIG_HDR_SIZE + size,
+    };
+
+    if (reply_supported) {
+        msg.flags |= VHOST_USER_NEED_REPLY_MASK;
+    }
+
+    msg.payload.config.offset = offset,
+    msg.payload.config.size = size,
+    msg.payload.config.flags = flags,
+    p = msg.payload.config.region;
+    memcpy(p, config + offset, size);
+
+    if (vhost_user_write(dev, &msg, NULL, 0) < 0) {
+        return -1;
+    }
+
+    if (reply_supported) {
+        return process_message_reply(dev, &msg);
+    }
+
+    return 0;
+}
+
+static int vhost_user_set_config_fd(struct vhost_dev *dev, int fd)
+{
+    VhostUserMsg msg = {
+       .request = VHOST_USER_SET_CONFIG_FD,
+       .flags = VHOST_USER_VERSION,
+    };
+
+    if (vhost_user_write(dev, &msg, &fd, 1) < 0) {
+        return -1;
+    }
+
+    return 0;
+}
+
 const VhostOps user_ops = {
         .backend_type = VHOST_BACKEND_TYPE_USER,
         .vhost_backend_init = vhost_user_init,
@@ -948,4 +1052,7 @@  const VhostOps user_ops = {
         .vhost_net_set_mtu = vhost_user_net_set_mtu,
         .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback,
         .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg,
+        .vhost_get_config = vhost_user_get_config,
+        .vhost_set_config = vhost_user_set_config,
+        .vhost_set_config_fd = vhost_user_set_config_fd,
 };
diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c
index e4290ce..35996a8 100644
--- a/hw/virtio/vhost.c
+++ b/hw/virtio/vhost.c
@@ -1358,6 +1358,9 @@  void vhost_dev_cleanup(struct vhost_dev *hdev)
     for (i = 0; i < hdev->nvqs; ++i) {
         vhost_virtqueue_cleanup(hdev->vqs + i);
     }
+    if (hdev->config_ops) {
+        event_notifier_cleanup(&hdev->config_notifier);
+    }
     if (hdev->mem) {
         /* those are only safe after successful init */
         memory_listener_unregister(&hdev->memory_listener);
@@ -1505,6 +1508,67 @@  void vhost_ack_features(struct vhost_dev *hdev, const int *feature_bits,
     }
 }
 
+int vhost_dev_get_config(struct vhost_dev *hdev, uint8_t *config,
+                         uint32_t config_len)
+{
+    assert(hdev->vhost_ops);
+
+    if (hdev->vhost_ops->vhost_get_config) {
+        return hdev->vhost_ops->vhost_get_config(hdev, config, config_len);
+    }
+
+    return 0;
+}
+
+int vhost_dev_set_config(struct vhost_dev *hdev, const uint8_t *config,
+                         uint32_t offset, uint32_t size, uint32_t flags)
+{
+    assert(hdev->vhost_ops);
+
+    if (hdev->vhost_ops->vhost_set_config) {
+        return hdev->vhost_ops->vhost_set_config(hdev, config, offset,
+                                                 size, flags);
+    }
+
+    return 0;
+}
+
+static void vhost_dev_config_notifier_read(EventNotifier *n)
+{
+    struct vhost_dev *hdev = container_of(n, struct vhost_dev,
+                                         config_notifier);
+
+    if (event_notifier_test_and_clear(n)) {
+        if (hdev->config_ops) {
+            hdev->config_ops->vhost_dev_config_notifier(hdev);
+        }
+    }
+}
+
+int vhost_dev_set_config_notifier(struct vhost_dev *hdev,
+                                  const VhostDevConfigOps *ops)
+{
+    int r, fd;
+
+    assert(hdev->vhost_ops);
+
+    r = event_notifier_init(&hdev->config_notifier, 0);
+    if (r < 0) {
+        return r;
+    }
+
+    hdev->config_ops = ops;
+    event_notifier_set_handler(&hdev->config_notifier,
+                               vhost_dev_config_notifier_read);
+
+    if (hdev->vhost_ops->vhost_set_config_fd) {
+        fd  = event_notifier_get_fd(&hdev->config_notifier);
+        return hdev->vhost_ops->vhost_set_config_fd(hdev, fd);
+    }
+
+    return 0;
+}
+
 /* Host notifiers must be enabled at this point. */
 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev)
 {
diff --git a/include/hw/virtio/vhost-backend.h b/include/hw/virtio/vhost-backend.h
index a7a5f22..070dd03 100644
--- a/include/hw/virtio/vhost-backend.h
+++ b/include/hw/virtio/vhost-backend.h
@@ -20,6 +20,11 @@  typedef enum VhostBackendType {
     VHOST_BACKEND_TYPE_MAX = 3,
 } VhostBackendType;
 
+typedef enum VhostSetConfigType {
+    VHOST_SET_CONFIG_TYPE_MASTER = 0,
+    VHOST_SET_CONFIG_TYPE_MIGRATION = 1,
+} VhostSetConfigType;
+
 struct vhost_dev;
 struct vhost_log;
 struct vhost_memory;
@@ -84,6 +89,12 @@  typedef void (*vhost_set_iotlb_callback_op)(struct vhost_dev *dev,
                                            int enabled);
 typedef int (*vhost_send_device_iotlb_msg_op)(struct vhost_dev *dev,
                                               struct vhost_iotlb_msg *imsg);
+typedef int (*vhost_set_config_op)(struct vhost_dev *dev, const uint8_t *config,
+                                   uint32_t offset, uint32_t size,
+                                   uint32_t flags);
+typedef int (*vhost_get_config_op)(struct vhost_dev *dev, uint8_t *config,
+                                   uint32_t config_len);
+typedef int (*vhost_set_config_fd_op)(struct vhost_dev *dev, int fd);
 
 typedef struct VhostOps {
     VhostBackendType backend_type;
@@ -118,6 +129,9 @@  typedef struct VhostOps {
     vhost_vsock_set_running_op vhost_vsock_set_running;
     vhost_set_iotlb_callback_op vhost_set_iotlb_callback;
     vhost_send_device_iotlb_msg_op vhost_send_device_iotlb_msg;
+    vhost_get_config_op vhost_get_config;
+    vhost_set_config_op vhost_set_config;
+    vhost_set_config_fd_op vhost_set_config_fd;
 } VhostOps;
 
 extern const VhostOps user_ops;
diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
index 467dc77..719491e 100644
--- a/include/hw/virtio/vhost.h
+++ b/include/hw/virtio/vhost.h
@@ -46,6 +46,12 @@  struct vhost_iommu {
     QLIST_ENTRY(vhost_iommu) iommu_next;
 };
 
+typedef struct VhostDevConfigOps {
+    /* Vhost device config space changed callback
+     */
+    void (*vhost_dev_config_notifier)(struct vhost_dev *dev);
+} VhostDevConfigOps;
+
 struct vhost_memory;
 struct vhost_dev {
     VirtIODevice *vdev;
@@ -76,6 +82,8 @@  struct vhost_dev {
     QLIST_ENTRY(vhost_dev) entry;
     QLIST_HEAD(, vhost_iommu) iommu_list;
     IOMMUNotifier n;
+    EventNotifier config_notifier;
+    const VhostDevConfigOps *config_ops;
 };
 
 int vhost_dev_init(struct vhost_dev *hdev, void *opaque,
@@ -106,4 +114,12 @@  int vhost_net_set_backend(struct vhost_dev *hdev,
                           struct vhost_vring_file *file);
 
 int vhost_device_iotlb_miss(struct vhost_dev *dev, uint64_t iova, int write);
+int vhost_dev_get_config(struct vhost_dev *dev, uint8_t *config,
+                         uint32_t config_len);
+int vhost_dev_set_config(struct vhost_dev *dev, const uint8_t *config,
+                         uint32_t offset, uint32_t size, uint32_t flags);
+/* notifier callback in case vhost device config space changed
+ */
+int vhost_dev_set_config_notifier(struct vhost_dev *dev,
+                                  const VhostDevConfigOps *ops);
 #endif