[2/2] nbd: add support to replace running daemon

Commit Message

Mike Christie Nov. 16, 2019, 5:50 a.m. UTC

This patch allows userspace to switch daemons while IO is running or
to recovery from a crashed daemon. With the patch when using AF_UNIX
sockets and a local running daemon, the new daemon can perform a
NBD_CMD_RECONFIGURE nl command with the NBD_ATTR_SWAP_SOCKETS attr
set to 0. The kernel will then switch over to using the new socket
similar to how we handle failing over from a dead socket.

Signed-off-by: Mike Christie <[e-mail]>
---
 drivers/block/nbd.c              | 44 +++++++++++++++++++++++++++++---
 include/uapi/linux/nbd-netlink.h |  1 +
 2 files changed, 41 insertions(+), 4 deletions(-)

Patch

diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index 06b63c11ae50..6aa396bea401 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -1076,7 +1076,14 @@  static int nbd_add_socket(struct nbd_device *nbd, unsigned long arg,
 	return 0;
 }
 
-static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)
+static bool nbd_requeue_req(struct request *req, void *data, bool reserved)
+{
+	nbd_requeue_cmd(blk_mq_rq_to_pdu(req));
+	return true;
+}
+
+static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg,
+				u8 swap_socks)
 {
 	struct nbd_config *config = nbd->config;
 	struct socket *sock, *old;
@@ -1094,6 +1101,16 @@  static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)
 		return -ENOMEM;
 	}
 
+	if (swap_socks) {
+		dev_dbg(disk_to_dev(nbd->disk), "stopping queue\n");
+
+		blk_mq_quiesce_queue(nbd->disk->queue);
+		send_disconnects(nbd);
+		sock_shutdown(nbd);
+		flush_workqueue(nbd->recv_workq);
+		blk_mq_tagset_busy_iter(&nbd->tag_set, nbd_requeue_req, NULL);
+	}
+
 	for (i = 0; i < config->num_connections; i++) {
 		struct nbd_sock *nsock = config->socks[i];
 
@@ -1130,11 +1147,20 @@  static int nbd_reconnect_socket(struct nbd_device *nbd, unsigned long arg)
 
 		atomic_inc(&config->live_connections);
 		wake_up(&config->conn_wait);
-		return 0;
+		err = 0;
+		goto start_queue;
 	}
 	sockfd_put(sock);
 	kfree(args);
-	return -ENOSPC;
+	err = -ENOSPC;
+
+start_queue:
+	if (swap_socks) {
+		nbd->task_recv = current;
+		blk_mq_unquiesce_queue(nbd->disk->queue);
+		dev_dbg(disk_to_dev(nbd->disk), "queue restarted\n");
+	}
+	return err;
 }
 
 static void nbd_bdev_reset(struct block_device *bdev)
@@ -2027,6 +2053,7 @@  static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
 {
 	struct nbd_device *nbd = NULL;
 	struct nbd_config *config;
+	u8 swap_socks = 0;
 	int index;
 	int ret = 0;
 	bool put_dev = false;
@@ -2107,6 +2134,15 @@  static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
 		}
 	}
 
+	if (info->attrs[NBD_ATTR_SWAP_SOCKETS]) {
+		swap_socks = nla_get_u8(info->attrs[NBD_ATTR_SWAP_SOCKETS]);
+		if (swap_socks != 1 && swap_socks != 0) {
+			printk(KERN_ERR "nbd: NBD_SOCK_SWAP must be 0 or 1\n");
+			ret = -EINVAL;
+			goto out;
+		}
+	}
+
 	if (info->attrs[NBD_ATTR_SOCKETS]) {
 		struct nlattr *attr;
 		int rem, fd;
@@ -2132,7 +2168,7 @@  static int nbd_genl_reconfigure(struct sk_buff *skb, struct genl_info *info)
 			if (!socks[NBD_SOCK_FD])
 				continue;
 			fd = (int)nla_get_u32(socks[NBD_SOCK_FD]);
-			ret = nbd_reconnect_socket(nbd, fd);
+			ret = nbd_reconnect_socket(nbd, fd, swap_socks);
 			if (ret) {
 				if (ret == -ENOSPC)
 					ret = 0;
diff --git a/include/uapi/linux/nbd-netlink.h b/include/uapi/linux/nbd-netlink.h
index c5d0ef7aa7d5..0e406445f06c 100644
--- a/include/uapi/linux/nbd-netlink.h
+++ b/include/uapi/linux/nbd-netlink.h
@@ -35,6 +35,7 @@  enum {
 	NBD_ATTR_SOCKETS,
 	NBD_ATTR_DEAD_CONN_TIMEOUT,
 	NBD_ATTR_DEVICE_LIST,
+	NBD_ATTR_SWAP_SOCKETS,
 	__NBD_ATTR_MAX,
 };
 #define NBD_ATTR_MAX (__NBD_ATTR_MAX - 1)