| Message ID | 63ef44664629ffd07d3bffe60351e808965dd13e.1642626515.git.jag.raman@oracle.com (mailing list archive) |
|---|---|
| State | New, archived |
| Headers | show |
| Series | vfio-user server in QEMU | expand |
On Wed, Jan 19, 2022 at 04:42:01PM -0500, Jagannathan Raman wrote: > Setup a handler to run vfio-user context. The context is driven by > messages to the file descriptor associated with it - get the fd for > the context and hook up the handler with it > > Signed-off-by: Elena Ufimtseva <elena.ufimtseva@oracle.com> > Signed-off-by: John G Johnson <john.g.johnson@oracle.com> > Signed-off-by: Jagannathan Raman <jag.raman@oracle.com> > --- > qapi/misc.json | 23 ++++++++++ > hw/remote/vfio-user-obj.c | 90 ++++++++++++++++++++++++++++++++++++++- > 2 files changed, 112 insertions(+), 1 deletion(-) > > diff --git a/qapi/misc.json b/qapi/misc.json > index e8054f415b..f0791d3311 100644 > --- a/qapi/misc.json > +++ b/qapi/misc.json > @@ -527,3 +527,26 @@ > 'data': { '*option': 'str' }, > 'returns': ['CommandLineOptionInfo'], > 'allow-preconfig': true } > + > +## > +# @VFU_CLIENT_HANGUP: > +# > +# Emitted when the client of a TYPE_VFIO_USER_SERVER closes the > +# communication channel > +# > +# @device: ID of attached PCI device > +# > +# @path: path of the socket This assumes a UNIX domain socket path was given. It doesn't work well with file descriptor passing. The x-vfio-user-server is an object with a unique QEMU Object Model path (the last path component is its id). You can get the id like this: object_get_canonical_path_component(OBJECT(o)) I suggest dropping @path and including the server object's id instead. > +# > +# Since: 6.3 > +# > +# Example: > +# > +# <- { "event": "VFU_CLIENT_HANGUP", > +# "data": { "device": "lsi1", > +# "path": "/tmp/vfu1-sock" }, > +# "timestamp": { "seconds": 1265044230, "microseconds": 450486 } } > +# > +## > +{ 'event': 'VFU_CLIENT_HANGUP', > + 'data': { 'device': 'str', 'path': 'str' } } > diff --git a/hw/remote/vfio-user-obj.c b/hw/remote/vfio-user-obj.c > index 10db78eb8d..91d49a221f 100644 > --- a/hw/remote/vfio-user-obj.c > +++ b/hw/remote/vfio-user-obj.c > @@ -27,6 +27,9 @@ > * > * device - id of a device on the server, a required option. PCI devices > * alone are supported presently. > + * > + * notes - x-vfio-user-server could block IO and monitor during the > + * initialization phase. > */ > > #include "qemu/osdep.h" > @@ -41,11 +44,14 @@ > #include "hw/remote/machine.h" > #include "qapi/error.h" > #include "qapi/qapi-visit-sockets.h" > +#include "qapi/qapi-events-misc.h" > #include "qemu/notify.h" > +#include "qemu/thread.h" > #include "sysemu/sysemu.h" > #include "libvfio-user.h" > #include "hw/qdev-core.h" > #include "hw/pci/pci.h" > +#include "qemu/timer.h" > > #define TYPE_VFU_OBJECT "x-vfio-user-server" > OBJECT_DECLARE_TYPE(VfuObject, VfuObjectClass, VFU_OBJECT) > @@ -95,6 +101,8 @@ struct VfuObject { > PCIDevice *pci_dev; > > Error *unplug_blocker; > + > + int vfu_poll_fd; > }; > > static void vfu_object_init_ctx(VfuObject *o, Error **errp); > @@ -144,6 +152,68 @@ static void vfu_object_set_device(Object *obj, const char *str, Error **errp) > vfu_object_init_ctx(o, errp); > } > > +static void vfu_object_ctx_run(void *opaque) > +{ > + VfuObject *o = opaque; > + int ret = -1; > + > + while (ret != 0) { > + ret = vfu_run_ctx(o->vfu_ctx); > + if (ret < 0) { > + if (errno == EINTR) { > + continue; > + } else if (errno == ENOTCONN) { > + qapi_event_send_vfu_client_hangup(o->device, > + o->socket->u.q_unix.path); > + qemu_set_fd_handler(o->vfu_poll_fd, NULL, NULL, NULL); Do we also stop monitoring o->vfu_poll_fd when object-del is used to delete the x-vfio-user-server object?
> On Jan 25, 2022, at 10:10 AM, Stefan Hajnoczi <stefanha@redhat.com> wrote: > > On Wed, Jan 19, 2022 at 04:42:01PM -0500, Jagannathan Raman wrote: >> Setup a handler to run vfio-user context. The context is driven by >> messages to the file descriptor associated with it - get the fd for >> the context and hook up the handler with it >> >> Signed-off-by: Elena Ufimtseva <elena.ufimtseva@oracle.com> >> Signed-off-by: John G Johnson <john.g.johnson@oracle.com> >> Signed-off-by: Jagannathan Raman <jag.raman@oracle.com> >> --- >> qapi/misc.json | 23 ++++++++++ >> hw/remote/vfio-user-obj.c | 90 ++++++++++++++++++++++++++++++++++++++- >> 2 files changed, 112 insertions(+), 1 deletion(-) >> >> diff --git a/qapi/misc.json b/qapi/misc.json >> index e8054f415b..f0791d3311 100644 >> --- a/qapi/misc.json >> +++ b/qapi/misc.json >> @@ -527,3 +527,26 @@ >> 'data': { '*option': 'str' }, >> 'returns': ['CommandLineOptionInfo'], >> 'allow-preconfig': true } >> + >> +## >> +# @VFU_CLIENT_HANGUP: >> +# >> +# Emitted when the client of a TYPE_VFIO_USER_SERVER closes the >> +# communication channel >> +# >> +# @device: ID of attached PCI device >> +# >> +# @path: path of the socket > > This assumes a UNIX domain socket path was given. It doesn't work well > with file descriptor passing. The x-vfio-user-server is an object with > a unique QEMU Object Model path (the last path component is its id). You > can get the id like this: > > object_get_canonical_path_component(OBJECT(o)) I was also wondering how to get the object ID. Thank you for the pointer! > > I suggest dropping @path and including the server object's id instead. OK, will do. > >> +# >> +# Since: 6.3 >> +# >> +# Example: >> +# >> +# <- { "event": "VFU_CLIENT_HANGUP", >> +# "data": { "device": "lsi1", >> +# "path": "/tmp/vfu1-sock" }, >> +# "timestamp": { "seconds": 1265044230, "microseconds": 450486 } } >> +# >> +## >> +{ 'event': 'VFU_CLIENT_HANGUP', >> + 'data': { 'device': 'str', 'path': 'str' } } >> diff --git a/hw/remote/vfio-user-obj.c b/hw/remote/vfio-user-obj.c >> index 10db78eb8d..91d49a221f 100644 >> --- a/hw/remote/vfio-user-obj.c >> +++ b/hw/remote/vfio-user-obj.c >> @@ -27,6 +27,9 @@ >> * >> * device - id of a device on the server, a required option. PCI devices >> * alone are supported presently. >> + * >> + * notes - x-vfio-user-server could block IO and monitor during the >> + * initialization phase. >> */ >> >> #include "qemu/osdep.h" >> @@ -41,11 +44,14 @@ >> #include "hw/remote/machine.h" >> #include "qapi/error.h" >> #include "qapi/qapi-visit-sockets.h" >> +#include "qapi/qapi-events-misc.h" >> #include "qemu/notify.h" >> +#include "qemu/thread.h" >> #include "sysemu/sysemu.h" >> #include "libvfio-user.h" >> #include "hw/qdev-core.h" >> #include "hw/pci/pci.h" >> +#include "qemu/timer.h" >> >> #define TYPE_VFU_OBJECT "x-vfio-user-server" >> OBJECT_DECLARE_TYPE(VfuObject, VfuObjectClass, VFU_OBJECT) >> @@ -95,6 +101,8 @@ struct VfuObject { >> PCIDevice *pci_dev; >> >> Error *unplug_blocker; >> + >> + int vfu_poll_fd; >> }; >> >> static void vfu_object_init_ctx(VfuObject *o, Error **errp); >> @@ -144,6 +152,68 @@ static void vfu_object_set_device(Object *obj, const char *str, Error **errp) >> vfu_object_init_ctx(o, errp); >> } >> >> +static void vfu_object_ctx_run(void *opaque) >> +{ >> + VfuObject *o = opaque; >> + int ret = -1; >> + >> + while (ret != 0) { >> + ret = vfu_run_ctx(o->vfu_ctx); >> + if (ret < 0) { >> + if (errno == EINTR) { >> + continue; >> + } else if (errno == ENOTCONN) { >> + qapi_event_send_vfu_client_hangup(o->device, >> + o->socket->u.q_unix.path); >> + qemu_set_fd_handler(o->vfu_poll_fd, NULL, NULL, NULL); > > Do we also stop monitoring o->vfu_poll_fd when object-del is used to > delete the x-vfio-user-server object? Yes, we should to stop monitoring the o->vfu_poll_fd during object-del. Will do so. -- Jag
diff --git a/qapi/misc.json b/qapi/misc.json index e8054f415b..f0791d3311 100644 --- a/qapi/misc.json +++ b/qapi/misc.json @@ -527,3 +527,26 @@ 'data': { '*option': 'str' }, 'returns': ['CommandLineOptionInfo'], 'allow-preconfig': true } + +## +# @VFU_CLIENT_HANGUP: +# +# Emitted when the client of a TYPE_VFIO_USER_SERVER closes the +# communication channel +# +# @device: ID of attached PCI device +# +# @path: path of the socket +# +# Since: 6.3 +# +# Example: +# +# <- { "event": "VFU_CLIENT_HANGUP", +# "data": { "device": "lsi1", +# "path": "/tmp/vfu1-sock" }, +# "timestamp": { "seconds": 1265044230, "microseconds": 450486 } } +# +## +{ 'event': 'VFU_CLIENT_HANGUP', + 'data': { 'device': 'str', 'path': 'str' } } diff --git a/hw/remote/vfio-user-obj.c b/hw/remote/vfio-user-obj.c index 10db78eb8d..91d49a221f 100644 --- a/hw/remote/vfio-user-obj.c +++ b/hw/remote/vfio-user-obj.c @@ -27,6 +27,9 @@ * * device - id of a device on the server, a required option. PCI devices * alone are supported presently. + * + * notes - x-vfio-user-server could block IO and monitor during the + * initialization phase. */ #include "qemu/osdep.h" @@ -41,11 +44,14 @@ #include "hw/remote/machine.h" #include "qapi/error.h" #include "qapi/qapi-visit-sockets.h" +#include "qapi/qapi-events-misc.h" #include "qemu/notify.h" +#include "qemu/thread.h" #include "sysemu/sysemu.h" #include "libvfio-user.h" #include "hw/qdev-core.h" #include "hw/pci/pci.h" +#include "qemu/timer.h" #define TYPE_VFU_OBJECT "x-vfio-user-server" OBJECT_DECLARE_TYPE(VfuObject, VfuObjectClass, VFU_OBJECT) @@ -95,6 +101,8 @@ struct VfuObject { PCIDevice *pci_dev; Error *unplug_blocker; + + int vfu_poll_fd; }; static void vfu_object_init_ctx(VfuObject *o, Error **errp); @@ -144,6 +152,68 @@ static void vfu_object_set_device(Object *obj, const char *str, Error **errp) vfu_object_init_ctx(o, errp); } +static void vfu_object_ctx_run(void *opaque) +{ + VfuObject *o = opaque; + int ret = -1; + + while (ret != 0) { + ret = vfu_run_ctx(o->vfu_ctx); + if (ret < 0) { + if (errno == EINTR) { + continue; + } else if (errno == ENOTCONN) { + qapi_event_send_vfu_client_hangup(o->device, + o->socket->u.q_unix.path); + qemu_set_fd_handler(o->vfu_poll_fd, NULL, NULL, NULL); + o->vfu_poll_fd = -1; + object_unparent(OBJECT(o)); + break; + } else { + VFU_OBJECT_ERROR(o, "vfu: Failed to run device %s - %s", + o->device, strerror(errno)); + break; + } + } + } +} + +static void vfu_object_attach_ctx(void *opaque) +{ + VfuObject *o = opaque; + GPollFD pfds[1]; + int ret; + + qemu_set_fd_handler(o->vfu_poll_fd, NULL, NULL, NULL); + + pfds[0].fd = o->vfu_poll_fd; + pfds[0].events = G_IO_IN | G_IO_HUP | G_IO_ERR; + +retry_attach: + ret = vfu_attach_ctx(o->vfu_ctx); + if (ret < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) { + /** + * vfu_object_attach_ctx can block QEMU's main loop + * during attach - the monitor and other IO + * could be unresponsive during this time. + */ + qemu_poll_ns(pfds, 1, 500 * (int64_t)SCALE_MS); + goto retry_attach; + } else if (ret < 0) { + VFU_OBJECT_ERROR(o, "vfu: Failed to attach device %s to context - %s", + o->device, strerror(errno)); + return; + } + + o->vfu_poll_fd = vfu_get_poll_fd(o->vfu_ctx); + if (o->vfu_poll_fd < 0) { + VFU_OBJECT_ERROR(o, "vfu: Failed to get poll fd %s", o->device); + return; + } + + qemu_set_fd_handler(o->vfu_poll_fd, vfu_object_ctx_run, NULL, o); +} + /* * TYPE_VFU_OBJECT depends on the availability of the 'socket' and 'device' * properties. It also depends on devices instantiated in QEMU. These @@ -182,7 +252,8 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp) return; } - o->vfu_ctx = vfu_create_ctx(VFU_TRANS_SOCK, o->socket->u.q_unix.path, 0, + o->vfu_ctx = vfu_create_ctx(VFU_TRANS_SOCK, o->socket->u.q_unix.path, + LIBVFIO_USER_FLAG_ATTACH_NB, o, VFU_DEV_TYPE_PCI); if (o->vfu_ctx == NULL) { error_setg(errp, "vfu: Failed to create context - %s", strerror(errno)); @@ -220,6 +291,21 @@ static void vfu_object_init_ctx(VfuObject *o, Error **errp) goto fail; } + ret = vfu_realize_ctx(o->vfu_ctx); + if (ret < 0) { + error_setg(errp, "vfu: Failed to realize device %s- %s", + o->device, strerror(errno)); + goto fail; + } + + o->vfu_poll_fd = vfu_get_poll_fd(o->vfu_ctx); + if (o->vfu_poll_fd < 0) { + error_setg(errp, "vfu: Failed to get poll fd %s", o->device); + goto fail; + } + + qemu_set_fd_handler(o->vfu_poll_fd, vfu_object_attach_ctx, NULL, o); + return; fail: @@ -250,6 +336,8 @@ static void vfu_object_init(Object *obj) TYPE_VFU_OBJECT, TYPE_REMOTE_MACHINE); return; } + + o->vfu_poll_fd = -1; } static void vfu_object_finalize(Object *obj)