@@ -1489,6 +1489,7 @@ if have_system
'net',
'softmmu',
'ui',
+ 'hw/remote',
]
endif
trace_events_subdirs += [
new file mode 100644
@@ -0,0 +1 @@
+#include "trace/trace-hw_remote.h"
new file mode 100644
@@ -0,0 +1,63 @@
+/*
+ * Communication channel between QEMU and remote device process
+ *
+ * Copyright © 2018, 2020 Oracle and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef MPQEMU_LINK_H
+#define MPQEMU_LINK_H
+
+#include "qom/object.h"
+#include "qemu/thread.h"
+#include "io/channel.h"
+
+#define REMOTE_MAX_FDS 8
+
+#define MPQEMU_MSG_HDR_SIZE offsetof(MPQemuMsg, data.u64)
+
+/**
+ * MPQemuCmd:
+ *
+ * MPQemuCmd enum type to specify the command to be executed on the remote
+ * device.
+ *
+ * This uses a private protocol between QEMU and the remote process. vfio-user
+ * protocol would supersede this in the future.
+ *
+ */
+typedef enum {
+ MPQEMU_CMD_MAX,
+} MPQemuCmd;
+
+/**
+ * MPQemuMsg:
+ * @cmd: The remote command
+ * @size: Size of the data to be shared
+ * @data: Structured data
+ * @fds: File descriptors to be shared with remote device
+ *
+ * MPQemuMsg Format of the message sent to the remote device from QEMU.
+ *
+ */
+typedef struct {
+ int cmd;
+ size_t size;
+
+ union {
+ uint64_t u64;
+ } data;
+
+ int fds[REMOTE_MAX_FDS];
+ int num_fds;
+} MPQemuMsg;
+
+bool mpqemu_msg_send(MPQemuMsg *msg, QIOChannel *ioc, Error **errp);
+bool mpqemu_msg_recv(MPQemuMsg *msg, QIOChannel *ioc, Error **errp);
+
+bool mpqemu_msg_valid(MPQemuMsg *msg);
+
+#endif
@@ -57,4 +57,10 @@ IOThread *iothread_create(const char *id, Error **errp);
void iothread_stop(IOThread *iothread);
void iothread_destroy(IOThread *iothread);
+/*
+ * Returns true if executing withing IOThread context,
+ * false otherwise.
+ */
+bool qemu_in_iothread(void);
+
#endif /* IOTHREAD_H */
new file mode 100644
@@ -0,0 +1,205 @@
+/*
+ * Communication channel between QEMU and remote device process
+ *
+ * Copyright © 2018, 2020 Oracle and/or its affiliates.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "qemu/osdep.h"
+#include "qemu-common.h"
+
+#include "qemu/module.h"
+#include "hw/remote/mpqemu-link.h"
+#include "qapi/error.h"
+#include "qemu/iov.h"
+#include "qemu/error-report.h"
+#include "qemu/main-loop.h"
+#include "io/channel.h"
+#include "sysemu/iothread.h"
+#include "trace.h"
+
+/*
+ * Send message over the ioc QIOChannel.
+ * This function is safe to call from:
+ * - main loop in co-routine context. Will block the main loop if not in
+ * co-routine context;
+ * - vCPU thread with no co-routine context and if the channel is not part
+ * of the main loop handling;
+ * - IOThread within co-routine context, outside of co-routine context
+ * will block IOThread;
+ * Returns true if no errors were encountered, false otherwise.
+ */
+bool mpqemu_msg_send(MPQemuMsg *msg, QIOChannel *ioc, Error **errp)
+{
+ ERRP_GUARD();
+ bool iolock = qemu_mutex_iothread_locked();
+ bool iothread = qemu_in_iothread();
+ struct iovec send[2] = {0};
+ int *fds = NULL;
+ size_t nfds = 0;
+ bool ret = false;
+
+ send[0].iov_base = msg;
+ send[0].iov_len = MPQEMU_MSG_HDR_SIZE;
+
+ send[1].iov_base = (void *)&msg->data;
+ send[1].iov_len = msg->size;
+
+ if (msg->num_fds) {
+ nfds = msg->num_fds;
+ fds = msg->fds;
+ }
+
+ /*
+ * Dont use in IOThread out of co-routine context as
+ * it will block IOThread.
+ */
+ assert(qemu_in_coroutine() || !iothread);
+
+ /*
+ * Skip unlocking/locking iothread lock when the IOThread is running
+ * in co-routine context. Co-routine context is asserted above
+ * for IOThread case.
+ * Also skip lock handling while in a co-routine in the main context.
+ */
+ if (iolock && !iothread && !qemu_in_coroutine()) {
+ qemu_mutex_unlock_iothread();
+ }
+
+ if (!qio_channel_writev_full_all(ioc, send, G_N_ELEMENTS(send),
+ fds, nfds, errp)) {
+ ret = true;
+ } else {
+ trace_mpqemu_send_io_error(msg->cmd, msg->size, nfds);
+ }
+
+ if (iolock && !iothread && !qemu_in_coroutine()) {
+ /* See above comment why skip locking here. */
+ qemu_mutex_lock_iothread();
+ }
+
+ return ret;
+}
+
+/*
+ * Read message from the ioc QIOChannel.
+ * This function is safe to call from:
+ * - From main loop in co-routine context. Will block the main loop if not in
+ * co-routine context;
+ * - From vCPU thread with no co-routine context and if the channel is not part
+ * of the main loop handling;
+ * - From IOThread within co-routine context, outside of co-routine context
+ * will block IOThread;
+ */
+static ssize_t mpqemu_read(QIOChannel *ioc, void *buf, size_t len, int **fds,
+ size_t *nfds, Error **errp)
+{
+ ERRP_GUARD();
+ struct iovec iov = { .iov_base = buf, .iov_len = len };
+ bool iolock = qemu_mutex_iothread_locked();
+ bool iothread = qemu_in_iothread();
+ int ret = -1;
+
+ /*
+ * Dont use in IOThread out of co-routine context as
+ * it will block IOThread.
+ */
+ assert(qemu_in_coroutine() || !iothread);
+
+ if (iolock && !iothread && !qemu_in_coroutine()) {
+ qemu_mutex_unlock_iothread();
+ }
+
+ ret = qio_channel_readv_full_all(ioc, &iov, 1, fds, nfds, errp);
+
+ if (iolock && !iothread && !qemu_in_coroutine()) {
+ qemu_mutex_lock_iothread();
+ }
+
+ return (ret <= 0) ? ret : iov.iov_len;
+}
+
+bool mpqemu_msg_recv(MPQemuMsg *msg, QIOChannel *ioc, Error **errp)
+{
+ ERRP_GUARD();
+ g_autofree int *fds = NULL;
+ size_t nfds = 0;
+ ssize_t len;
+ bool ret = false;
+
+ len = mpqemu_read(ioc, msg, MPQEMU_MSG_HDR_SIZE, &fds, &nfds, errp);
+ if (len <= 0) {
+ goto fail;
+ } else if (len != MPQEMU_MSG_HDR_SIZE) {
+ error_setg(errp, "Message header corrupted");
+ goto fail;
+ }
+
+ if (msg->size > sizeof(msg->data)) {
+ error_setg(errp, "Invalid size for message");
+ goto fail;
+ }
+
+ if (!msg->size) {
+ goto copy_fds;
+ }
+
+ len = mpqemu_read(ioc, &msg->data, msg->size, NULL, NULL, errp);
+ if (len <= 0) {
+ goto fail;
+ }
+ if (len != msg->size) {
+ error_setg(errp, "Unable to read full message");
+ goto fail;
+ }
+
+copy_fds:
+ msg->num_fds = nfds;
+ if (nfds > G_N_ELEMENTS(msg->fds)) {
+ error_setg(errp,
+ "Overflow error: received %zu fds, more than max of %d fds",
+ nfds, REMOTE_MAX_FDS);
+ goto fail;
+ }
+ if (nfds) {
+ memcpy(msg->fds, fds, nfds * sizeof(int));
+ }
+
+ ret = true;
+
+fail:
+ if (*errp) {
+ trace_mpqemu_recv_io_error(msg->cmd, msg->size, nfds);
+ }
+ while (*errp && nfds) {
+ close(fds[nfds - 1]);
+ nfds--;
+ }
+
+ return ret;
+}
+
+bool mpqemu_msg_valid(MPQemuMsg *msg)
+{
+ if (msg->cmd >= MPQEMU_CMD_MAX && msg->cmd < 0) {
+ return false;
+ }
+
+ /* Verify FDs. */
+ if (msg->num_fds >= REMOTE_MAX_FDS) {
+ return false;
+ }
+
+ if (msg->num_fds > 0) {
+ for (int i = 0; i < msg->num_fds; i++) {
+ if (fcntl(msg->fds[i], F_GETFL) == -1) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
@@ -375,3 +375,9 @@ IOThread *iothread_by_id(const char *id)
{
return IOTHREAD(object_resolve_path_type(id, TYPE_IOTHREAD, NULL));
}
+
+bool qemu_in_iothread(void)
+{
+ return qemu_get_current_aio_context() == qemu_get_aio_context() ?
+ false : true;
+}
@@ -3168,6 +3168,8 @@ F: hw/pci-host/remote.c
F: include/hw/pci-host/remote.h
F: hw/remote/machine.c
F: include/hw/remote/machine.h
+F: hw/remote/mpqemu-link.c
+F: include/hw/remote/mpqemu-link.h
Build and test automation
-------------------------
@@ -1,5 +1,6 @@
remote_ss = ss.source_set()
remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('machine.c'))
+remote_ss.add(when: 'CONFIG_MULTIPROCESS', if_true: files('mpqemu-link.c'))
softmmu_ss.add_all(when: 'CONFIG_MULTIPROCESS', if_true: remote_ss)
new file mode 100644
@@ -0,0 +1,4 @@
+# multi-process trace events
+
+mpqemu_send_io_error(int cmd, int size, int nfds) "send command %d size %d, %d file descriptors to remote process"
+mpqemu_recv_io_error(int cmd, int size, int nfds) "failed to receive %d size %d, %d file descriptors to remote process"