diff mbox series

[RFC,v3,36/45] multi-process: Use separate MMIO communication channel

Message ID 6c736da21b554548b7ba2af04fb5eee1cf806883.1567534653.git.jag.raman@oracle.com (mailing list archive)
State New, archived
Headers show
Series Initial support of multi-process qemu | expand

Commit Message

Jag Raman Sept. 3, 2019, 8:38 p.m. UTC
Using a separate communication channel for MMIO helps
with improving Performance

Signed-off-by: Elena Ufimtseva <elena.ufimtseva@oracle.com>
Signed-off-by: Jagannathan Raman <jag.raman@oracle.com>
Signed-off-by: John G Johnson <john.g.johnson@oracle.com>
---
 New patch in v3

 hw/proxy/qemu-proxy.c         | 38 ++++++++++++++++++++++++++------------
 include/hw/proxy/qemu-proxy.h |  1 +
 include/io/proxy-link.h       |  7 +++++++
 io/proxy-link.c               |  2 ++
 qdev-monitor.c                |  1 +
 remote/remote-main.c          | 20 ++++++++++++++------
 6 files changed, 51 insertions(+), 18 deletions(-)
diff mbox series

Patch

diff --git a/hw/proxy/qemu-proxy.c b/hw/proxy/qemu-proxy.c
index 699a0b1..db9a208 100644
--- a/hw/proxy/qemu-proxy.c
+++ b/hw/proxy/qemu-proxy.c
@@ -205,20 +205,22 @@  static int make_argv(char *command_str, char **argv, int argc)
 int remote_spawn(PCIProxyDev *pdev, const char *command, Error **errp)
 {
     pid_t rpid;
-    int fd[2] = {-1, -1};
+    int fd[2], mmio[2];
     Error *local_error = NULL;
     char *argv[64];
     int argc = 0, _argc;
     char *sfd;
     char *exec_dir;
     int rc = -EINVAL;
+    struct timeval timeout = {.tv_sec = 10, .tv_usec = 0};
 
     if (pdev->managed) {
         /* Child is forked by external program (such as libvirt). */
         return rc;
     }
 
-    if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd)) {
+    if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) ||
+        socketpair(AF_UNIX, SOCK_STREAM, 0, mmio)) {
         error_setg(errp, "Unable to create unix socket.");
         return rc;
     }
@@ -226,6 +228,8 @@  int remote_spawn(PCIProxyDev *pdev, const char *command, Error **errp)
     argc = add_argv(exec_dir, argv, argc);
     sfd = g_strdup_printf("%d", fd[1]);
     argc = add_argv(sfd, argv, argc);
+    sfd = g_strdup_printf("%d", mmio[1]);
+    argc = add_argv(sfd, argv, argc);
     _argc = argc;
     argc = make_argv((char *)command, argv, argc);
 
@@ -235,22 +239,32 @@  int remote_spawn(PCIProxyDev *pdev, const char *command, Error **errp)
     if (rpid == -1) {
         error_setg(errp, "Unable to spawn emulation program.");
         close(fd[0]);
+        close(mmio[0]);
         goto fail;
     }
 
     if (rpid == 0) {
         close(fd[0]);
+        close(mmio[0]);
         execvp(argv[0], (char *const *)argv);
         exit(1);
     }
     pdev->remote_pid = rpid;
     pdev->rsocket = fd[1];
     pdev->socket = fd[0];
+    pdev->mmio_sock = mmio[0];
+
+    if (setsockopt(mmio[0], SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout,
+                   sizeof(timeout)) < 0) {
+        error_setg(errp, "Unable to set timeout for socket");
+        goto fail;
+    }
 
     rc = 0;
 
 fail:
     close(fd[1]);
+    close(mmio[1]);
 
     for (int i = 0; i < _argc; i++) {
         g_free(argv[i]);
@@ -466,6 +480,9 @@  static void init_proxy(PCIDevice *dev, char *command, bool need_spawn, Error **e
     proxy_link_init_channel(pdev->proxy_link, &pdev->proxy_link->com,
                             pdev->socket);
 
+    proxy_link_init_channel(pdev->proxy_link, &pdev->proxy_link->mmio,
+                            pdev->mmio_sock);
+
     configure_memory_sync(pdev->sync, pdev->proxy_link);
 }
 
@@ -514,8 +531,7 @@  static void send_bar_access_msg(PCIProxyDev *dev, MemoryRegion *mr,
                                 unsigned size, bool memory)
 {
     ProxyLinkState *proxy_link;
-    ProcMsg msg;
-    int wait;
+    ProcMsg msg, ret;
 
     proxy_link = dev->proxy_link;
 
@@ -531,11 +547,7 @@  static void send_bar_access_msg(PCIProxyDev *dev, MemoryRegion *mr,
         msg.cmd = BAR_WRITE;
         msg.data1.bar_access.val = *val;
     } else {
-        wait = GET_REMOTE_WAIT;
-
         msg.cmd = BAR_READ;
-        msg.num_fds = 1;
-        msg.fds[0] = wait;
     }
 
     if (dev->dev_id) {
@@ -546,17 +558,19 @@  static void send_bar_access_msg(PCIProxyDev *dev, MemoryRegion *mr,
         msg.size_id = 0;
     }
 
-    proxy_proc_send(proxy_link, &msg, proxy_link->com);
+    proxy_proc_send(proxy_link, &msg, proxy_link->mmio);
 
-    if (!write) {
-        *val = wait_for_remote(wait);
-        PUT_REMOTE_WAIT(wait);
+    if (write) {
+        return;
     }
 
     if (msg.id) {
         free(msg.id);
     }
 
+    proxy_proc_recv(proxy_link, &ret, proxy_link->mmio);
+
+    *val = ret.data1.mmio_ret.val;
 }
 
 void proxy_default_bar_write(PCIProxyDev *dev, MemoryRegion *mr, hwaddr addr,
diff --git a/include/hw/proxy/qemu-proxy.h b/include/hw/proxy/qemu-proxy.h
index fb408cf..a03c6cc 100644
--- a/include/hw/proxy/qemu-proxy.h
+++ b/include/hw/proxy/qemu-proxy.h
@@ -59,6 +59,7 @@  typedef struct PCIProxyDev {
     pid_t remote_pid;
     int rsocket;
     int socket;
+    int mmio_sock;
 
     char *rid;
 
diff --git a/include/io/proxy-link.h b/include/io/proxy-link.h
index ae98eac..32b2c1a 100644
--- a/include/io/proxy-link.h
+++ b/include/io/proxy-link.h
@@ -75,6 +75,7 @@  typedef enum {
     DEVICE_ADD,
     DEVICE_DEL,
     PROXY_PING,
+    MMIO_RETURN,
     MAX,
 } proc_cmd_t;
 
@@ -108,6 +109,10 @@  typedef struct {
 } set_irqfd_msg_t;
 
 typedef struct {
+    uint64_t val;
+} mmio_ret_msg_t;
+
+typedef struct {
     proc_cmd_t cmd;
     int bytestream;
     size_t size;
@@ -118,6 +123,7 @@  typedef struct {
         sync_sysmem_msg_t sync_sysmem;
         bar_access_msg_t bar_access;
         set_irqfd_msg_t set_irqfd;
+        mmio_ret_msg_t mmio_ret;
     } data1;
 
     int fds[REMOTE_MAX_FDS];
@@ -170,6 +176,7 @@  struct ProxyLinkState {
     GMainLoop *loop;
 
     ProcChannel *com;
+    ProcChannel *mmio;
 
     proxy_link_callback callback;
 };
diff --git a/io/proxy-link.c b/io/proxy-link.c
index 6f60117..3ca0185 100644
--- a/io/proxy-link.c
+++ b/io/proxy-link.c
@@ -74,6 +74,7 @@  void proxy_link_finalize(ProxyLinkState *s)
     g_main_loop_quit(s->loop);
 
     proxy_link_destroy_channel(s->com);
+    proxy_link_destroy_channel(s->mmio);
 
     object_unref(OBJECT(s));
 }
@@ -356,6 +357,7 @@  void start_handler(ProxyLinkState *s)
 {
 
     g_assert(g_source_attach(&s->com->gsrc, s->ctx));
+    g_assert(g_source_attach(&s->mmio->gsrc, s->ctx));
 
     g_main_loop_run(s->loop);
 }
diff --git a/qdev-monitor.c b/qdev-monitor.c
index 69e467e..f1065af 100644
--- a/qdev-monitor.c
+++ b/qdev-monitor.c
@@ -710,6 +710,7 @@  DeviceState *qdev_proxy_add(const char *rid, const char *id, char *bus,
     if (old_pdev) {
         pdev->rsocket = old_pdev->rsocket;
         pdev->socket = old_pdev->socket;
+        pdev->mmio_sock = old_pdev->mmio_sock;
         pdev->remote_pid = old_pdev->remote_pid;
     } else {
         pdev->rsocket = managed ? rsocket : -1;
diff --git a/remote/remote-main.c b/remote/remote-main.c
index f0a4de9..2a9ebae 100644
--- a/remote/remote-main.c
+++ b/remote/remote-main.c
@@ -180,8 +180,8 @@  static void process_bar_write(ProcMsg *msg, Error **errp)
 static void process_bar_read(ProcMsg *msg, Error **errp)
 {
     bar_access_msg_t *bar_access = &msg->data1.bar_access;
+    ProcMsg ret = { 0 };
     AddressSpace *as;
-    int wait = msg->fds[0];
     MemTxResult res;
     uint64_t val = 0;
 
@@ -215,9 +215,10 @@  static void process_bar_read(ProcMsg *msg, Error **errp)
     }
 
 fail:
-    notify_proxy(wait, val);
-
-    PUT_REMOTE_WAIT(wait);
+    ret.cmd = MMIO_RETURN;
+    ret.data1.mmio_ret.val = val;
+    ret.size = sizeof(ret.data1);
+    proxy_proc_send(proxy_link, &ret, proxy_link->mmio);
 }
 
 static void process_device_add_msg(ProcMsg *msg)
@@ -581,10 +582,17 @@  int main(int argc, char *argv[], char **envp)
         printf("Failed to parse fd for remote process.\n");
         return -EINVAL;
     }
-
     proxy_link_init_channel(proxy_link, &proxy_link->com, fd);
 
-    parse_cmdline(argc - 2, argv + 2, NULL);
+    fd = qemu_parse_fd(argv[2]);
+    if (fd == -1) {
+        printf("Failed to parse fd for remote process.\n");
+        return -EINVAL;
+    }
+    proxy_link_init_channel(proxy_link, &proxy_link->mmio, fd);
+
+    parse_cmdline(argc - 3, argv + 3, NULL);
+
     qemu_mutex_init(&remote_ds_lock);
     QLIST_INIT(&pci_devs_head);