Message ID | 20240401090531.574575-3-parav@nvidia.com (mailing list archive) |
---|---|
State | Superseded |
Delegated to: | Netdev Maintainers |
Headers | show |
Series | devlink: Add port function attribute for IO EQs | expand |
On Mon, Apr 1, 2024 at 2:36 PM Parav Pandit <parav@nvidia.com> wrote: > > Implement get and set for the maximum IO event queues for SF and VF. > This enables administrator on the hypervisor to control the maximum > IO event queues which are typically used to derive the maximum and > default number of net device channels or rdma device completion vectors. > > Signed-off-by: Parav Pandit <parav@nvidia.com> > Reviewed-by: Jiri Pirko <jiri@nvidia.com> > --- > .../mellanox/mlx5/core/esw/devlink_port.c | 2 + > .../net/ethernet/mellanox/mlx5/core/eswitch.h | 7 ++ > .../mellanox/mlx5/core/eswitch_offloads.c | 94 +++++++++++++++++++ > 3 files changed, 103 insertions(+) > > diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c > index d8e739cbcbce..76d1ed93c773 100644 > --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c > +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c > @@ -98,6 +98,8 @@ static const struct devlink_port_ops mlx5_esw_pf_vf_dl_port_ops = { > .port_fn_ipsec_packet_get = mlx5_devlink_port_fn_ipsec_packet_get, > .port_fn_ipsec_packet_set = mlx5_devlink_port_fn_ipsec_packet_set, > #endif /* CONFIG_XFRM_OFFLOAD */ > + .port_fn_max_io_eqs_get = mlx5_devlink_port_fn_max_io_eqs_get, > + .port_fn_max_io_eqs_set = mlx5_devlink_port_fn_max_io_eqs_set, > }; > > static void mlx5_esw_offloads_sf_devlink_port_attrs_set(struct mlx5_eswitch *esw, > diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h > index 349e28a6dd8d..50ce1ea20dd4 100644 > --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h > +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h > @@ -573,6 +573,13 @@ int mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port *port, bool *is_en > int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port, bool enable, > struct netlink_ext_ack *extack); > #endif /* CONFIG_XFRM_OFFLOAD */ > +int mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port *port, > + u32 *max_io_eqs, > + struct netlink_ext_ack *extack); > +int mlx5_devlink_port_fn_max_io_eqs_set(struct devlink_port *port, > + u32 max_io_eqs, > + struct netlink_ext_ack *extack); > + > void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type); > > int __mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw, > diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c > index baaae628b0a0..9d9a06a25cac 100644 > --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c > +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c > @@ -66,6 +66,8 @@ > > #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) > > +#define MLX5_ESW_MAX_CTRL_EQS 4 > + > static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = { > .max_fte = MLX5_ESW_VPORT_TBL_SIZE, > .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS, > @@ -4557,3 +4559,95 @@ int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port, > return err; > } > #endif /* CONFIG_XFRM_OFFLOAD */ > + > +int mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port *port, u32 *max_io_eqs, > + struct netlink_ext_ack *extack) > +{ > + struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); > + struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); > + int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); > + u16 vport_num = vport->vport; > + void *query_ctx; > + void *hca_caps; > + u32 max_eqs; > + int err; > + > + if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { > + NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); > + return -EOPNOTSUPP; > + } > + > + query_ctx = kzalloc(query_out_sz, GFP_KERNEL); > + if (!query_ctx) > + return -ENOMEM; > + > + mutex_lock(&esw->state_lock); > + err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, > + MLX5_CAP_GENERAL); > + if (err) { > + NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); > + goto out; > + } > + > + hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); > + max_eqs = MLX5_GET(cmd_hca_cap, hca_caps, max_num_eqs); > + if (max_eqs < MLX5_ESW_MAX_CTRL_EQS) > + *max_io_eqs = 0; > + else > + *max_io_eqs = max_eqs - MLX5_ESW_MAX_CTRL_EQS; > +out: [Kalesh]: Missing " kfree(query_ctx);" here? > + mutex_unlock(&esw->state_lock); > + return 0; [Kalesh] "return err;" to propagate the error back to the caller? > +} > + > +int mlx5_devlink_port_fn_max_io_eqs_set(struct devlink_port *port, u32 max_io_eqs, > + struct netlink_ext_ack *extack) > +{ > + struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); > + struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); > + int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); > + u16 vport_num = vport->vport; > + u16 max_eqs = max_io_eqs + MLX5_ESW_MAX_CTRL_EQS; > + void *query_ctx; > + void *hca_caps; > + int err; > + > + if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { > + NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); > + return -EOPNOTSUPP; > + } > + > + if (max_io_eqs + MLX5_ESW_MAX_CTRL_EQS > USHRT_MAX) { > + NL_SET_ERR_MSG_MOD(extack, "Supplied value out of range"); > + return -EINVAL; > + } > + > + mutex_lock(&esw->state_lock); > + > + query_ctx = kzalloc(query_out_sz, GFP_KERNEL); > + if (!query_ctx) { > + err = -ENOMEM; > + goto out; > + } > + > + err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, > + MLX5_CAP_GENERAL); > + if (err) { > + NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); > + goto out_free; > + } > + > + hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); > + MLX5_SET(cmd_hca_cap, hca_caps, max_num_eqs, max_eqs); > + > + err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num, > + MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); > + if (err) > + NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap"); > + > +out_free: > + kfree(query_ctx); > +out: > + mutex_unlock(&esw->state_lock); > + return err; > +} > -- > 2.26.2 > >
> From: Kalesh Anakkur Purayil <kalesh-anakkur.purayil@broadcom.com> > Sent: Monday, April 1, 2024 2:47 PM > > On Mon, Apr 1, 2024 at 2:36 PM Parav Pandit <parav@nvidia.com> wrote: > > > > Implement get and set for the maximum IO event queues for SF and VF. > > This enables administrator on the hypervisor to control the maximum > > IO event queues which are typically used to derive the maximum and > > default number of net device channels or rdma device completion vectors. > > > > Signed-off-by: Parav Pandit <parav@nvidia.com> > > Reviewed-by: Jiri Pirko <jiri@nvidia.com> > > --- > > .../mellanox/mlx5/core/esw/devlink_port.c | 2 + > > .../net/ethernet/mellanox/mlx5/core/eswitch.h | 7 ++ > > .../mellanox/mlx5/core/eswitch_offloads.c | 94 +++++++++++++++++++ > > 3 files changed, 103 insertions(+) > > > > diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c > b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c > > index d8e739cbcbce..76d1ed93c773 100644 > > --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c > > +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c > > @@ -98,6 +98,8 @@ static const struct devlink_port_ops > mlx5_esw_pf_vf_dl_port_ops = { > > .port_fn_ipsec_packet_get = mlx5_devlink_port_fn_ipsec_packet_get, > > .port_fn_ipsec_packet_set = mlx5_devlink_port_fn_ipsec_packet_set, > > #endif /* CONFIG_XFRM_OFFLOAD */ > > + .port_fn_max_io_eqs_get = mlx5_devlink_port_fn_max_io_eqs_get, > > + .port_fn_max_io_eqs_set = mlx5_devlink_port_fn_max_io_eqs_set, > > }; > > > > static void mlx5_esw_offloads_sf_devlink_port_attrs_set(struct > mlx5_eswitch *esw, > > diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h > b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h > > index 349e28a6dd8d..50ce1ea20dd4 100644 > > --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h > > +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h > > @@ -573,6 +573,13 @@ int mlx5_devlink_port_fn_ipsec_packet_get(struct > devlink_port *port, bool *is_en > > int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port, bool > enable, > > struct netlink_ext_ack *extack); > > #endif /* CONFIG_XFRM_OFFLOAD */ > > +int mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port *port, > > + u32 *max_io_eqs, > > + struct netlink_ext_ack *extack); > > +int mlx5_devlink_port_fn_max_io_eqs_set(struct devlink_port *port, > > + u32 max_io_eqs, > > + struct netlink_ext_ack *extack); > > + > > void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 > rep_type); > > > > int __mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw, > > diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c > b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c > > index baaae628b0a0..9d9a06a25cac 100644 > > --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c > > +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c > > @@ -66,6 +66,8 @@ > > > > #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) > > > > +#define MLX5_ESW_MAX_CTRL_EQS 4 > > + > > static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = { > > .max_fte = MLX5_ESW_VPORT_TBL_SIZE, > > .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS, > > @@ -4557,3 +4559,95 @@ int > mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port, > > return err; > > } > > #endif /* CONFIG_XFRM_OFFLOAD */ > > + > > +int mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port *port, u32 > *max_io_eqs, > > + struct netlink_ext_ack *extack) > > +{ > > + struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port- > >devlink); > > + struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); > > + int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); > > + u16 vport_num = vport->vport; > > + void *query_ctx; > > + void *hca_caps; > > + u32 max_eqs; > > + int err; > > + > > + if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { > > + NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA > management"); > > + return -EOPNOTSUPP; > > + } > > + > > + query_ctx = kzalloc(query_out_sz, GFP_KERNEL); > > + if (!query_ctx) > > + return -ENOMEM; > > + > > + mutex_lock(&esw->state_lock); > > + err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, > query_ctx, > > + MLX5_CAP_GENERAL); > > + if (err) { > > + NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); > > + goto out; > > + } > > + > > + hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); > > + max_eqs = MLX5_GET(cmd_hca_cap, hca_caps, max_num_eqs); > > + if (max_eqs < MLX5_ESW_MAX_CTRL_EQS) > > + *max_io_eqs = 0; > > + else > > + *max_io_eqs = max_eqs - MLX5_ESW_MAX_CTRL_EQS; > > +out: > [Kalesh]: Missing " kfree(query_ctx);" here? > > + mutex_unlock(&esw->state_lock); > > + return 0; > [Kalesh] "return err;" to propagate the error back to the caller? > > +} Ack. Thanks Kalesh. Fixing both the comments in v1.
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c index d8e739cbcbce..76d1ed93c773 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c @@ -98,6 +98,8 @@ static const struct devlink_port_ops mlx5_esw_pf_vf_dl_port_ops = { .port_fn_ipsec_packet_get = mlx5_devlink_port_fn_ipsec_packet_get, .port_fn_ipsec_packet_set = mlx5_devlink_port_fn_ipsec_packet_set, #endif /* CONFIG_XFRM_OFFLOAD */ + .port_fn_max_io_eqs_get = mlx5_devlink_port_fn_max_io_eqs_get, + .port_fn_max_io_eqs_set = mlx5_devlink_port_fn_max_io_eqs_set, }; static void mlx5_esw_offloads_sf_devlink_port_attrs_set(struct mlx5_eswitch *esw, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 349e28a6dd8d..50ce1ea20dd4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -573,6 +573,13 @@ int mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port *port, bool *is_en int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port, bool enable, struct netlink_ext_ack *extack); #endif /* CONFIG_XFRM_OFFLOAD */ +int mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port *port, + u32 *max_io_eqs, + struct netlink_ext_ack *extack); +int mlx5_devlink_port_fn_max_io_eqs_set(struct devlink_port *port, + u32 max_io_eqs, + struct netlink_ext_ack *extack); + void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type); int __mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index baaae628b0a0..9d9a06a25cac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -66,6 +66,8 @@ #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) +#define MLX5_ESW_MAX_CTRL_EQS 4 + static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = { .max_fte = MLX5_ESW_VPORT_TBL_SIZE, .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS, @@ -4557,3 +4559,95 @@ int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port, return err; } #endif /* CONFIG_XFRM_OFFLOAD */ + +int mlx5_devlink_port_fn_max_io_eqs_get(struct devlink_port *port, u32 *max_io_eqs, + struct netlink_ext_ack *extack) +{ + struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); + struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); + int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); + u16 vport_num = vport->vport; + void *query_ctx; + void *hca_caps; + u32 max_eqs; + int err; + + if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { + NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); + return -EOPNOTSUPP; + } + + query_ctx = kzalloc(query_out_sz, GFP_KERNEL); + if (!query_ctx) + return -ENOMEM; + + mutex_lock(&esw->state_lock); + err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, + MLX5_CAP_GENERAL); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); + goto out; + } + + hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); + max_eqs = MLX5_GET(cmd_hca_cap, hca_caps, max_num_eqs); + if (max_eqs < MLX5_ESW_MAX_CTRL_EQS) + *max_io_eqs = 0; + else + *max_io_eqs = max_eqs - MLX5_ESW_MAX_CTRL_EQS; +out: + mutex_unlock(&esw->state_lock); + return 0; +} + +int mlx5_devlink_port_fn_max_io_eqs_set(struct devlink_port *port, u32 max_io_eqs, + struct netlink_ext_ack *extack) +{ + struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); + struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); + int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); + u16 vport_num = vport->vport; + u16 max_eqs = max_io_eqs + MLX5_ESW_MAX_CTRL_EQS; + void *query_ctx; + void *hca_caps; + int err; + + if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { + NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); + return -EOPNOTSUPP; + } + + if (max_io_eqs + MLX5_ESW_MAX_CTRL_EQS > USHRT_MAX) { + NL_SET_ERR_MSG_MOD(extack, "Supplied value out of range"); + return -EINVAL; + } + + mutex_lock(&esw->state_lock); + + query_ctx = kzalloc(query_out_sz, GFP_KERNEL); + if (!query_ctx) { + err = -ENOMEM; + goto out; + } + + err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, + MLX5_CAP_GENERAL); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); + goto out_free; + } + + hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); + MLX5_SET(cmd_hca_cap, hca_caps, max_num_eqs, max_eqs); + + err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num, + MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); + if (err) + NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap"); + +out_free: + kfree(query_ctx); +out: + mutex_unlock(&esw->state_lock); + return err; +}