diff mbox series

[rdma-next] RDMA/mana_ib: handle net event for pointing to the current netdev

Message ID 1740782519-13485-1-git-send-email-longli@linuxonhyperv.com (mailing list archive)
State New
Headers show
Series [rdma-next] RDMA/mana_ib: handle net event for pointing to the current netdev | expand

Commit Message

Long Li Feb. 28, 2025, 10:41 p.m. UTC 
From: Long Li <longli@microsoft.com>

When running under Hyper-V, the master device to the RDMA device is always
bonded to this RDMA device if it's present in the kernel. This is not
user-configurable.

The master device can be unbind/bind from the kernel. During those events,
the RDMA device should set to the current netdev to relect the change of
master device from those events.

Signed-off-by: Long Li <longli@microsoft.com>
---
 drivers/infiniband/hw/mana/device.c  | 35 ++++++++++++++++++++++++++++
 drivers/infiniband/hw/mana/mana_ib.h |  1 +
 2 files changed, 36 insertions(+)

Comments

Long Li Feb. 28, 2025, 10:51 p.m. UTC | #1 
> Subject: [Patch rdma-next] RDMA/mana_ib: handle net event for
> pointing to the current netdev
> 
> From: Long Li <longli@microsoft.com>
> 
> When running under Hyper-V, the master device to the RDMA device is always
> bonded to this RDMA device if it's present in the kernel. This is not user-
> configurable.
> 
> The master device can be unbind/bind from the kernel. During those events,
> the RDMA device should set to the current netdev to relect the change of
> master device from those events.
> 
> Signed-off-by: Long Li <longli@microsoft.com>

This is not the correct patch. Please discard this patch. 

> ---
>  drivers/infiniband/hw/mana/device.c  | 35
> ++++++++++++++++++++++++++++
> drivers/infiniband/hw/mana/mana_ib.h |  1 +
>  2 files changed, 36 insertions(+)
> 
> diff --git a/drivers/infiniband/hw/mana/device.c
> b/drivers/infiniband/hw/mana/device.c
> index 3416a85f8738..3e4f069c2258 100644
> --- a/drivers/infiniband/hw/mana/device.c
> +++ b/drivers/infiniband/hw/mana/device.c
> @@ -51,6 +51,37 @@ static const struct ib_device_ops mana_ib_dev_ops = {
>  			   ib_ind_table),
>  };
> 
> +static int mana_ib_netdev_event(struct notifier_block *this,
> +				unsigned long event, void *ptr)
> +{
> +	struct mana_ib_dev *dev = container_of(this, struct mana_ib_dev,
> nb);
> +	struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
> +	struct gdma_context *gc = dev->gdma_dev->gdma_context;
> +	struct mana_context *mc = gc->mana.driver_data;
> +	struct net_device *ndev;
> +
> +	if (event_dev != mc->ports[0])
> +		return NOTIFY_DONE;
> +
> +	switch (event) {
> +	case NETDEV_CHANGEUPPER:
> +		rcu_read_lock();
> +		ndev = mana_get_primary_netdev_rcu(mc, 0);
> +		rcu_read_unlock();
> +
> +		/*
> +		 * RDMA core will setup GID based on updated netdev.
> +		 * It's not possible to race with the core as rtnl lock is being
> +		 * held.
> +		 */
> +		ib_device_set_netdev(&dev->ib_dev, ndev, 1);
> +
> +		return NOTIFY_OK;
> +	default:
> +		return NOTIFY_DONE;
> +	}
> +}
> +
>  static int mana_ib_probe(struct auxiliary_device *adev,
>  			 const struct auxiliary_device_id *id)  { @@ -109,6
> +140,9 @@ static int mana_ib_probe(struct auxiliary_device *adev,
>  	}
>  	dev->gdma_dev = &mdev->gdma_context->mana_ib;
> 
> +	dev->nb.notifier_call = mana_ib_netdev_event;
> +	register_netdevice_notifier(&dev->nb);
> +
>  	ret = mana_ib_gd_query_adapter_caps(dev);
>  	if (ret) {
>  		ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d",
> @@ -159,6 +193,7 @@ static void mana_ib_remove(struct auxiliary_device
> *adev)  {
>  	struct mana_ib_dev *dev = dev_get_drvdata(&adev->dev);
> 
> +	unregister_netdevice_notifier(&dev->nb);
>  	ib_unregister_device(&dev->ib_dev);
>  	xa_destroy(&dev->qp_table_wq);
>  	mana_ib_gd_destroy_rnic_adapter(dev);
> diff --git a/drivers/infiniband/hw/mana/mana_ib.h
> b/drivers/infiniband/hw/mana/mana_ib.h
> index b53a5b4de908..d88187072899 100644
> --- a/drivers/infiniband/hw/mana/mana_ib.h
> +++ b/drivers/infiniband/hw/mana/mana_ib.h
> @@ -64,6 +64,7 @@ struct mana_ib_dev {
>  	struct gdma_queue **eqs;
>  	struct xarray qp_table_wq;
>  	struct mana_ib_adapter_caps adapter_caps;
> +	struct notifier_block nb;
>  };
> 
>  struct mana_ib_wq {
> --
> 2.34.1
Jason Gunthorpe March 1, 2025, 12:26 a.m. UTC | #2 
On Fri, Feb 28, 2025 at 02:41:59PM -0800, longli@linuxonhyperv.com wrote:
> +	struct mana_ib_dev *dev = container_of(this, struct mana_ib_dev, nb);
> +	struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
> +	struct gdma_context *gc = dev->gdma_dev->gdma_context;
> +	struct mana_context *mc = gc->mana.driver_data;
> +	struct net_device *ndev;
> +
> +	if (event_dev != mc->ports[0])
> +		return NOTIFY_DONE;
> +
> +	switch (event) {
> +	case NETDEV_CHANGEUPPER:
> +		rcu_read_lock();
> +		ndev = mana_get_primary_netdev_rcu(mc, 0);
> +		rcu_read_unlock();

That locking sure looks weird/wrong.

Jason
diff mbox series

Patch

diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/mana/device.c
index 3416a85f8738..3e4f069c2258 100644
--- a/drivers/infiniband/hw/mana/device.c
+++ b/drivers/infiniband/hw/mana/device.c
@@ -51,6 +51,37 @@  static const struct ib_device_ops mana_ib_dev_ops = {
 			   ib_ind_table),
 };
 
+static int mana_ib_netdev_event(struct notifier_block *this,
+				unsigned long event, void *ptr)
+{
+	struct mana_ib_dev *dev = container_of(this, struct mana_ib_dev, nb);
+	struct net_device *event_dev = netdev_notifier_info_to_dev(ptr);
+	struct gdma_context *gc = dev->gdma_dev->gdma_context;
+	struct mana_context *mc = gc->mana.driver_data;
+	struct net_device *ndev;
+
+	if (event_dev != mc->ports[0])
+		return NOTIFY_DONE;
+
+	switch (event) {
+	case NETDEV_CHANGEUPPER:
+		rcu_read_lock();
+		ndev = mana_get_primary_netdev_rcu(mc, 0);
+		rcu_read_unlock();
+
+		/*
+		 * RDMA core will setup GID based on updated netdev.
+		 * It's not possible to race with the core as rtnl lock is being
+		 * held.
+		 */
+		ib_device_set_netdev(&dev->ib_dev, ndev, 1);
+
+		return NOTIFY_OK;
+	default:
+		return NOTIFY_DONE;
+	}
+}
+
 static int mana_ib_probe(struct auxiliary_device *adev,
 			 const struct auxiliary_device_id *id)
 {
@@ -109,6 +140,9 @@  static int mana_ib_probe(struct auxiliary_device *adev,
 	}
 	dev->gdma_dev = &mdev->gdma_context->mana_ib;
 
+	dev->nb.notifier_call = mana_ib_netdev_event;
+	register_netdevice_notifier(&dev->nb);
+
 	ret = mana_ib_gd_query_adapter_caps(dev);
 	if (ret) {
 		ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d",
@@ -159,6 +193,7 @@  static void mana_ib_remove(struct auxiliary_device *adev)
 {
 	struct mana_ib_dev *dev = dev_get_drvdata(&adev->dev);
 
+	unregister_netdevice_notifier(&dev->nb);
 	ib_unregister_device(&dev->ib_dev);
 	xa_destroy(&dev->qp_table_wq);
 	mana_ib_gd_destroy_rnic_adapter(dev);
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index b53a5b4de908..d88187072899 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -64,6 +64,7 @@  struct mana_ib_dev {
 	struct gdma_queue **eqs;
 	struct xarray qp_table_wq;
 	struct mana_ib_adapter_caps adapter_caps;
+	struct notifier_block nb;
 };
 
 struct mana_ib_wq {