| Message ID | 1714795819-12543-2-git-send-email-selvin.xavier@broadcom.com (mailing list archive) |
|---|---|
| State | Superseded |
| Headers | show |
| Series | RDMA/bnxt_re: MSN table capability check for latest adapters | expand |
On Fri, May 03, 2024 at 09:10:18PM -0700, Selvin Xavier wrote: > FW reports the HW capability to use PSN table or MSN table and > driver/library need to select it based on this capability. > Use the new capability instead of the older capability check for HW > retransmission while handling the MSN/PSN table. > Also, Updated the FW interface structures to handle the new fields. > > Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com> > --- > drivers/infiniband/hw/bnxt_re/qplib_fp.c | 12 ++++++------ > drivers/infiniband/hw/bnxt_re/qplib_fp.h | 2 +- > drivers/infiniband/hw/bnxt_re/qplib_res.h | 6 ++++++ > drivers/infiniband/hw/bnxt_re/qplib_sp.c | 1 + > drivers/infiniband/hw/bnxt_re/qplib_sp.h | 1 + > drivers/infiniband/hw/bnxt_re/roce_hsi.h | 30 +++++++++++++++++++++++++++++- > 6 files changed, 44 insertions(+), 8 deletions(-) I have same comment as I gave in rdma-core PR. It seems like your change will cause to old devices to behave as they don't have PSN table anymore. Thanks > > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c > index 439d0c7..3c961a8 100644 > --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c > +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c > @@ -984,7 +984,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) > u16 nsge; > > if (res->dattr) > - qp->dev_cap_flags = res->dattr->dev_cap_flags; > + qp->is_host_msn_tbl = _is_host_msn_table(res->dattr->dev_cap_flags2); > > sq->dbinfo.flags = 0; > bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req, > @@ -1002,7 +1002,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) > sizeof(struct sq_psn_search_ext) : > sizeof(struct sq_psn_search); > > - if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { > + if (qp->is_host_msn_tbl) { > psn_sz = sizeof(struct sq_msn_search); > qp->msn = 0; > } > @@ -1015,7 +1015,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) > hwq_attr.aux_stride = psn_sz; > hwq_attr.aux_depth = bnxt_qplib_set_sq_size(sq, qp->wqe_mode); > /* Update msn tbl size */ > - if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) { > + if (qp->is_host_msn_tbl && psn_sz) { > hwq_attr.aux_depth = roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, qp->wqe_mode)); > qp->msn_tbl_sz = hwq_attr.aux_depth; > qp->msn = 0; > @@ -1636,7 +1636,7 @@ static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp, > if (!swq->psn_search) > return; > /* Handle MSN differently on cap flags */ > - if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { > + if (qp->is_host_msn_tbl) { > bnxt_qplib_fill_msn_search(qp, wqe, swq); > return; > } > @@ -1818,7 +1818,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, > } > > swq = bnxt_qplib_get_swqe(sq, &wqe_idx); > - bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags)); > + bnxt_qplib_pull_psn_buff(qp, sq, swq, qp->is_host_msn_tbl); > > idx = 0; > swq->slot_idx = hwq->prod; > @@ -2008,7 +2008,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, > rc = -EINVAL; > goto done; > } > - if (!BNXT_RE_HW_RETX(qp->dev_cap_flags) || msn_update) { > + if (!qp->is_host_msn_tbl || msn_update) { > swq->next_psn = sq->psn & BTH_PSN_MASK; > bnxt_qplib_fill_psn_search(qp, wqe, swq); > } > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h > index 7fd4506..5b8d097 100644 > --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h > +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h > @@ -340,7 +340,7 @@ struct bnxt_qplib_qp { > struct list_head rq_flush; > u32 msn; > u32 msn_tbl_sz; > - u16 dev_cap_flags; > + u16 is_host_msn_tbl; > }; > > #define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE sizeof(struct cq_base) > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h > index 61628f7..a0f78cd 100644 > --- a/drivers/infiniband/hw/bnxt_re/qplib_res.h > +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h > @@ -554,6 +554,12 @@ static inline bool _is_hw_retx_supported(u16 dev_cap_flags) > > #define BNXT_RE_HW_RETX(a) _is_hw_retx_supported((a)) > > +static inline bool _is_host_msn_table(u16 dev_cap_ext_flags2) > +{ > + return (dev_cap_ext_flags2 & CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_MASK) == > + CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_MSN_TABLE; > +} > + > static inline u8 bnxt_qplib_dbr_pacing_en(struct bnxt_qplib_chip_ctx *cctx) > { > return cctx->modes.dbr_pacing; > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c > index 8beeedd..9328db9 100644 > --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c > +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c > @@ -156,6 +156,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, > (0x01 << RCFW_DBR_BASE_PAGE_SHIFT); > attr->max_sgid = BNXT_QPLIB_NUM_GIDS_SUPPORTED; > attr->dev_cap_flags = le16_to_cpu(sb->dev_cap_flags); > + attr->dev_cap_flags2 = le16_to_cpu(sb->dev_cap_ext_flags_2); > > bnxt_qplib_query_version(rcfw, attr->fw_ver); > > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h > index d33c78b..16a67d7 100644 > --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h > +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h > @@ -72,6 +72,7 @@ struct bnxt_qplib_dev_attr { > u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ]; > bool is_atomic; > u16 dev_cap_flags; > + u16 dev_cap_flags2; > u32 max_dpi; > }; > > diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h > index 605c946..0425309 100644 > --- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h > +++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h > @@ -2157,8 +2157,36 @@ struct creq_query_func_resp_sb { > __le32 tqm_alloc_reqs[12]; > __le32 max_dpi; > u8 max_sge_var_wqe; > - u8 reserved_8; > + u8 dev_cap_ext_flags; > + #define CREQ_QUERY_FUNC_RESP_SB_ATOMIC_OPS_NOT_SUPPORTED 0x1UL > + #define CREQ_QUERY_FUNC_RESP_SB_DRV_VERSION_RGTR_SUPPORTED 0x2UL > + #define CREQ_QUERY_FUNC_RESP_SB_CREATE_QP_BATCH_SUPPORTED 0x4UL > + #define CREQ_QUERY_FUNC_RESP_SB_DESTROY_QP_BATCH_SUPPORTED 0x8UL > + #define CREQ_QUERY_FUNC_RESP_SB_ROCE_STATS_EXT_CTX_SUPPORTED 0x10UL > + #define CREQ_QUERY_FUNC_RESP_SB_CREATE_SRQ_SGE_SUPPORTED 0x20UL > + #define CREQ_QUERY_FUNC_RESP_SB_FIXED_SIZE_WQE_DISABLED 0x40UL > + #define CREQ_QUERY_FUNC_RESP_SB_DCN_SUPPORTED 0x80UL > __le16 max_inline_data_var_wqe; > + __le32 start_qid; > + u8 max_msn_table_size; > + u8 reserved8_1; > + __le16 dev_cap_ext_flags_2; > + #define CREQ_QUERY_FUNC_RESP_SB_OPTIMIZE_MODIFY_QP_SUPPORTED 0x1UL > + #define CREQ_QUERY_FUNC_RESP_SB_CHANGE_UDP_SRC_PORT_WQE_SUPPORTED 0x2UL > + #define CREQ_QUERY_FUNC_RESP_SB_CQ_COALESCING_SUPPORTED 0x4UL > + #define CREQ_QUERY_FUNC_RESP_SB_MEMORY_REGION_RO_SUPPORTED 0x8UL > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_MASK 0x30UL > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_SFT 4 > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_PSN_TABLE (0x0UL << 4) > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_MSN_TABLE (0x1UL << 4) > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_IQM_MSN_TABLE (0x2UL << 4) > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_LAST \ > + CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_IQM_MSN_TABLE > + __le16 max_xp_qp_size; > + __le16 create_qp_batch_size; > + __le16 destroy_qp_batch_size; > + __le16 reserved16; > + __le64 reserved64; > }; > > /* cmdq_set_func_resources (size:448b/56B) */ > -- > 2.5.5 >
On Sun, May 5, 2024 at 7:16 PM Leon Romanovsky <leon@kernel.org> wrote: > > On Fri, May 03, 2024 at 09:10:18PM -0700, Selvin Xavier wrote: > > FW reports the HW capability to use PSN table or MSN table and > > driver/library need to select it based on this capability. > > Use the new capability instead of the older capability check for HW > > retransmission while handling the MSN/PSN table. > > Also, Updated the FW interface structures to handle the new fields. > > > > Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com> > > --- > > drivers/infiniband/hw/bnxt_re/qplib_fp.c | 12 ++++++------ > > drivers/infiniband/hw/bnxt_re/qplib_fp.h | 2 +- > > drivers/infiniband/hw/bnxt_re/qplib_res.h | 6 ++++++ > > drivers/infiniband/hw/bnxt_re/qplib_sp.c | 1 + > > drivers/infiniband/hw/bnxt_re/qplib_sp.h | 1 + > > drivers/infiniband/hw/bnxt_re/roce_hsi.h | 30 +++++++++++++++++++++++++++++- > > 6 files changed, 44 insertions(+), 8 deletions(-) > > I have same comment as I gave in rdma-core PR. > It seems like your change will cause to old devices to behave as they > don't have PSN table anymore. Hi Leon, I replied to your comment in the rdma-core PR. For older devices, the PSN table is created. For new devices, MSN table will be created. This patch is changing the capability check from BNXT_RE_HW_RETX to _is_host_msn_table. Both capabilities are reported by FW. Driver is deciding which table to be created and it's better to decide it using _is_host_msn_table capability. For older adapters, this capability will be 0 which means PSN table. Thanks > > Thanks > > > > > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c > > index 439d0c7..3c961a8 100644 > > --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c > > +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c > > @@ -984,7 +984,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) > > u16 nsge; > > > > if (res->dattr) > > - qp->dev_cap_flags = res->dattr->dev_cap_flags; > > + qp->is_host_msn_tbl = _is_host_msn_table(res->dattr->dev_cap_flags2); > > > > sq->dbinfo.flags = 0; > > bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req, > > @@ -1002,7 +1002,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) > > sizeof(struct sq_psn_search_ext) : > > sizeof(struct sq_psn_search); > > > > - if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { > > + if (qp->is_host_msn_tbl) { > > psn_sz = sizeof(struct sq_msn_search); > > qp->msn = 0; > > } > > @@ -1015,7 +1015,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) > > hwq_attr.aux_stride = psn_sz; > > hwq_attr.aux_depth = bnxt_qplib_set_sq_size(sq, qp->wqe_mode); > > /* Update msn tbl size */ > > - if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) { > > + if (qp->is_host_msn_tbl && psn_sz) { > > hwq_attr.aux_depth = roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, qp->wqe_mode)); > > qp->msn_tbl_sz = hwq_attr.aux_depth; > > qp->msn = 0; > > @@ -1636,7 +1636,7 @@ static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp, > > if (!swq->psn_search) > > return; > > /* Handle MSN differently on cap flags */ > > - if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { > > + if (qp->is_host_msn_tbl) { > > bnxt_qplib_fill_msn_search(qp, wqe, swq); > > return; > > } > > @@ -1818,7 +1818,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, > > } > > > > swq = bnxt_qplib_get_swqe(sq, &wqe_idx); > > - bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags)); > > + bnxt_qplib_pull_psn_buff(qp, sq, swq, qp->is_host_msn_tbl); > > > > idx = 0; > > swq->slot_idx = hwq->prod; > > @@ -2008,7 +2008,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, > > rc = -EINVAL; > > goto done; > > } > > - if (!BNXT_RE_HW_RETX(qp->dev_cap_flags) || msn_update) { > > + if (!qp->is_host_msn_tbl || msn_update) { > > swq->next_psn = sq->psn & BTH_PSN_MASK; > > bnxt_qplib_fill_psn_search(qp, wqe, swq); > > } > > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h > > index 7fd4506..5b8d097 100644 > > --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h > > +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h > > @@ -340,7 +340,7 @@ struct bnxt_qplib_qp { > > struct list_head rq_flush; > > u32 msn; > > u32 msn_tbl_sz; > > - u16 dev_cap_flags; > > + u16 is_host_msn_tbl; > > }; > > > > #define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE sizeof(struct cq_base) > > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h > > index 61628f7..a0f78cd 100644 > > --- a/drivers/infiniband/hw/bnxt_re/qplib_res.h > > +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h > > @@ -554,6 +554,12 @@ static inline bool _is_hw_retx_supported(u16 dev_cap_flags) > > > > #define BNXT_RE_HW_RETX(a) _is_hw_retx_supported((a)) > > > > +static inline bool _is_host_msn_table(u16 dev_cap_ext_flags2) > > +{ > > + return (dev_cap_ext_flags2 & CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_MASK) == > > + CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_MSN_TABLE; > > +} > > + > > static inline u8 bnxt_qplib_dbr_pacing_en(struct bnxt_qplib_chip_ctx *cctx) > > { > > return cctx->modes.dbr_pacing; > > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c > > index 8beeedd..9328db9 100644 > > --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c > > +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c > > @@ -156,6 +156,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, > > (0x01 << RCFW_DBR_BASE_PAGE_SHIFT); > > attr->max_sgid = BNXT_QPLIB_NUM_GIDS_SUPPORTED; > > attr->dev_cap_flags = le16_to_cpu(sb->dev_cap_flags); > > + attr->dev_cap_flags2 = le16_to_cpu(sb->dev_cap_ext_flags_2); > > > > bnxt_qplib_query_version(rcfw, attr->fw_ver); > > > > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h > > index d33c78b..16a67d7 100644 > > --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h > > +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h > > @@ -72,6 +72,7 @@ struct bnxt_qplib_dev_attr { > > u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ]; > > bool is_atomic; > > u16 dev_cap_flags; > > + u16 dev_cap_flags2; > > u32 max_dpi; > > }; > > > > diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h > > index 605c946..0425309 100644 > > --- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h > > +++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h > > @@ -2157,8 +2157,36 @@ struct creq_query_func_resp_sb { > > __le32 tqm_alloc_reqs[12]; > > __le32 max_dpi; > > u8 max_sge_var_wqe; > > - u8 reserved_8; > > + u8 dev_cap_ext_flags; > > + #define CREQ_QUERY_FUNC_RESP_SB_ATOMIC_OPS_NOT_SUPPORTED 0x1UL > > + #define CREQ_QUERY_FUNC_RESP_SB_DRV_VERSION_RGTR_SUPPORTED 0x2UL > > + #define CREQ_QUERY_FUNC_RESP_SB_CREATE_QP_BATCH_SUPPORTED 0x4UL > > + #define CREQ_QUERY_FUNC_RESP_SB_DESTROY_QP_BATCH_SUPPORTED 0x8UL > > + #define CREQ_QUERY_FUNC_RESP_SB_ROCE_STATS_EXT_CTX_SUPPORTED 0x10UL > > + #define CREQ_QUERY_FUNC_RESP_SB_CREATE_SRQ_SGE_SUPPORTED 0x20UL > > + #define CREQ_QUERY_FUNC_RESP_SB_FIXED_SIZE_WQE_DISABLED 0x40UL > > + #define CREQ_QUERY_FUNC_RESP_SB_DCN_SUPPORTED 0x80UL > > __le16 max_inline_data_var_wqe; > > + __le32 start_qid; > > + u8 max_msn_table_size; > > + u8 reserved8_1; > > + __le16 dev_cap_ext_flags_2; > > + #define CREQ_QUERY_FUNC_RESP_SB_OPTIMIZE_MODIFY_QP_SUPPORTED 0x1UL > > + #define CREQ_QUERY_FUNC_RESP_SB_CHANGE_UDP_SRC_PORT_WQE_SUPPORTED 0x2UL > > + #define CREQ_QUERY_FUNC_RESP_SB_CQ_COALESCING_SUPPORTED 0x4UL > > + #define CREQ_QUERY_FUNC_RESP_SB_MEMORY_REGION_RO_SUPPORTED 0x8UL > > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_MASK 0x30UL > > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_SFT 4 > > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_PSN_TABLE (0x0UL << 4) > > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_MSN_TABLE (0x1UL << 4) > > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_IQM_MSN_TABLE (0x2UL << 4) > > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_LAST \ > > + CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_IQM_MSN_TABLE > > + __le16 max_xp_qp_size; > > + __le16 create_qp_batch_size; > > + __le16 destroy_qp_batch_size; > > + __le16 reserved16; > > + __le64 reserved64; > > }; > > > > /* cmdq_set_func_resources (size:448b/56B) */ > > -- > > 2.5.5 > > > > >
On Mon, May 06, 2024 at 07:30:12PM +0530, Selvin Xavier wrote: > On Sun, May 5, 2024 at 7:16 PM Leon Romanovsky <leon@kernel.org> wrote: > > > > On Fri, May 03, 2024 at 09:10:18PM -0700, Selvin Xavier wrote: > > > FW reports the HW capability to use PSN table or MSN table and > > > driver/library need to select it based on this capability. > > > Use the new capability instead of the older capability check for HW > > > retransmission while handling the MSN/PSN table. > > > Also, Updated the FW interface structures to handle the new fields. > > > > > > Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com> > > > --- > > > drivers/infiniband/hw/bnxt_re/qplib_fp.c | 12 ++++++------ > > > drivers/infiniband/hw/bnxt_re/qplib_fp.h | 2 +- > > > drivers/infiniband/hw/bnxt_re/qplib_res.h | 6 ++++++ > > > drivers/infiniband/hw/bnxt_re/qplib_sp.c | 1 + > > > drivers/infiniband/hw/bnxt_re/qplib_sp.h | 1 + > > > drivers/infiniband/hw/bnxt_re/roce_hsi.h | 30 +++++++++++++++++++++++++++++- > > > 6 files changed, 44 insertions(+), 8 deletions(-) > > > > I have same comment as I gave in rdma-core PR. > > It seems like your change will cause to old devices to behave as they > > don't have PSN table anymore. > Hi Leon, > > I replied to your comment in the rdma-core PR. For older devices, the > PSN table is created. For new devices, MSN table will be created. > This patch is changing the capability check from BNXT_RE_HW_RETX to > _is_host_msn_table. Both capabilities are reported by FW. > Driver is deciding which table to be created and it's better to decide > it using _is_host_msn_table capability. For older adapters, this > capability will be 0 which means PSN table. Thanks. > > Thanks > > > > Thanks > > > > > > > > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c > > > index 439d0c7..3c961a8 100644 > > > --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c > > > +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c > > > @@ -984,7 +984,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) > > > u16 nsge; > > > > > > if (res->dattr) > > > - qp->dev_cap_flags = res->dattr->dev_cap_flags; > > > + qp->is_host_msn_tbl = _is_host_msn_table(res->dattr->dev_cap_flags2); > > > > > > sq->dbinfo.flags = 0; > > > bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req, > > > @@ -1002,7 +1002,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) > > > sizeof(struct sq_psn_search_ext) : > > > sizeof(struct sq_psn_search); > > > > > > - if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { > > > + if (qp->is_host_msn_tbl) { > > > psn_sz = sizeof(struct sq_msn_search); > > > qp->msn = 0; > > > } > > > @@ -1015,7 +1015,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) > > > hwq_attr.aux_stride = psn_sz; > > > hwq_attr.aux_depth = bnxt_qplib_set_sq_size(sq, qp->wqe_mode); > > > /* Update msn tbl size */ > > > - if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) { > > > + if (qp->is_host_msn_tbl && psn_sz) { > > > hwq_attr.aux_depth = roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, qp->wqe_mode)); > > > qp->msn_tbl_sz = hwq_attr.aux_depth; > > > qp->msn = 0; > > > @@ -1636,7 +1636,7 @@ static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp, > > > if (!swq->psn_search) > > > return; > > > /* Handle MSN differently on cap flags */ > > > - if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { > > > + if (qp->is_host_msn_tbl) { > > > bnxt_qplib_fill_msn_search(qp, wqe, swq); > > > return; > > > } > > > @@ -1818,7 +1818,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, > > > } > > > > > > swq = bnxt_qplib_get_swqe(sq, &wqe_idx); > > > - bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags)); > > > + bnxt_qplib_pull_psn_buff(qp, sq, swq, qp->is_host_msn_tbl); > > > > > > idx = 0; > > > swq->slot_idx = hwq->prod; > > > @@ -2008,7 +2008,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, > > > rc = -EINVAL; > > > goto done; > > > } > > > - if (!BNXT_RE_HW_RETX(qp->dev_cap_flags) || msn_update) { > > > + if (!qp->is_host_msn_tbl || msn_update) { > > > swq->next_psn = sq->psn & BTH_PSN_MASK; > > > bnxt_qplib_fill_psn_search(qp, wqe, swq); > > > } > > > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h > > > index 7fd4506..5b8d097 100644 > > > --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h > > > +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h > > > @@ -340,7 +340,7 @@ struct bnxt_qplib_qp { > > > struct list_head rq_flush; > > > u32 msn; > > > u32 msn_tbl_sz; > > > - u16 dev_cap_flags; > > > + u16 is_host_msn_tbl; > > > }; > > > > > > #define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE sizeof(struct cq_base) > > > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h > > > index 61628f7..a0f78cd 100644 > > > --- a/drivers/infiniband/hw/bnxt_re/qplib_res.h > > > +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h > > > @@ -554,6 +554,12 @@ static inline bool _is_hw_retx_supported(u16 dev_cap_flags) > > > > > > #define BNXT_RE_HW_RETX(a) _is_hw_retx_supported((a)) > > > > > > +static inline bool _is_host_msn_table(u16 dev_cap_ext_flags2) > > > +{ > > > + return (dev_cap_ext_flags2 & CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_MASK) == > > > + CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_MSN_TABLE; > > > +} > > > + > > > static inline u8 bnxt_qplib_dbr_pacing_en(struct bnxt_qplib_chip_ctx *cctx) > > > { > > > return cctx->modes.dbr_pacing; > > > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c > > > index 8beeedd..9328db9 100644 > > > --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c > > > +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c > > > @@ -156,6 +156,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, > > > (0x01 << RCFW_DBR_BASE_PAGE_SHIFT); > > > attr->max_sgid = BNXT_QPLIB_NUM_GIDS_SUPPORTED; > > > attr->dev_cap_flags = le16_to_cpu(sb->dev_cap_flags); > > > + attr->dev_cap_flags2 = le16_to_cpu(sb->dev_cap_ext_flags_2); > > > > > > bnxt_qplib_query_version(rcfw, attr->fw_ver); > > > > > > diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h > > > index d33c78b..16a67d7 100644 > > > --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h > > > +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h > > > @@ -72,6 +72,7 @@ struct bnxt_qplib_dev_attr { > > > u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ]; > > > bool is_atomic; > > > u16 dev_cap_flags; > > > + u16 dev_cap_flags2; > > > u32 max_dpi; > > > }; > > > > > > diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h > > > index 605c946..0425309 100644 > > > --- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h > > > +++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h > > > @@ -2157,8 +2157,36 @@ struct creq_query_func_resp_sb { > > > __le32 tqm_alloc_reqs[12]; > > > __le32 max_dpi; > > > u8 max_sge_var_wqe; > > > - u8 reserved_8; > > > + u8 dev_cap_ext_flags; > > > + #define CREQ_QUERY_FUNC_RESP_SB_ATOMIC_OPS_NOT_SUPPORTED 0x1UL > > > + #define CREQ_QUERY_FUNC_RESP_SB_DRV_VERSION_RGTR_SUPPORTED 0x2UL > > > + #define CREQ_QUERY_FUNC_RESP_SB_CREATE_QP_BATCH_SUPPORTED 0x4UL > > > + #define CREQ_QUERY_FUNC_RESP_SB_DESTROY_QP_BATCH_SUPPORTED 0x8UL > > > + #define CREQ_QUERY_FUNC_RESP_SB_ROCE_STATS_EXT_CTX_SUPPORTED 0x10UL > > > + #define CREQ_QUERY_FUNC_RESP_SB_CREATE_SRQ_SGE_SUPPORTED 0x20UL > > > + #define CREQ_QUERY_FUNC_RESP_SB_FIXED_SIZE_WQE_DISABLED 0x40UL > > > + #define CREQ_QUERY_FUNC_RESP_SB_DCN_SUPPORTED 0x80UL > > > __le16 max_inline_data_var_wqe; > > > + __le32 start_qid; > > > + u8 max_msn_table_size; > > > + u8 reserved8_1; > > > + __le16 dev_cap_ext_flags_2; > > > + #define CREQ_QUERY_FUNC_RESP_SB_OPTIMIZE_MODIFY_QP_SUPPORTED 0x1UL > > > + #define CREQ_QUERY_FUNC_RESP_SB_CHANGE_UDP_SRC_PORT_WQE_SUPPORTED 0x2UL > > > + #define CREQ_QUERY_FUNC_RESP_SB_CQ_COALESCING_SUPPORTED 0x4UL > > > + #define CREQ_QUERY_FUNC_RESP_SB_MEMORY_REGION_RO_SUPPORTED 0x8UL > > > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_MASK 0x30UL > > > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_SFT 4 > > > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_PSN_TABLE (0x0UL << 4) > > > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_MSN_TABLE (0x1UL << 4) > > > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_IQM_MSN_TABLE (0x2UL << 4) > > > + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_LAST \ > > > + CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_IQM_MSN_TABLE > > > + __le16 max_xp_qp_size; > > > + __le16 create_qp_batch_size; > > > + __le16 destroy_qp_batch_size; > > > + __le16 reserved16; > > > + __le64 reserved64; > > > }; > > > > > > /* cmdq_set_func_resources (size:448b/56B) */ > > > -- > > > 2.5.5 > > > > > > > > >
diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.c b/drivers/infiniband/hw/bnxt_re/qplib_fp.c index 439d0c7..3c961a8 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.c @@ -984,7 +984,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) u16 nsge; if (res->dattr) - qp->dev_cap_flags = res->dattr->dev_cap_flags; + qp->is_host_msn_tbl = _is_host_msn_table(res->dattr->dev_cap_flags2); sq->dbinfo.flags = 0; bnxt_qplib_rcfw_cmd_prep((struct cmdq_base *)&req, @@ -1002,7 +1002,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) sizeof(struct sq_psn_search_ext) : sizeof(struct sq_psn_search); - if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { + if (qp->is_host_msn_tbl) { psn_sz = sizeof(struct sq_msn_search); qp->msn = 0; } @@ -1015,7 +1015,7 @@ int bnxt_qplib_create_qp(struct bnxt_qplib_res *res, struct bnxt_qplib_qp *qp) hwq_attr.aux_stride = psn_sz; hwq_attr.aux_depth = bnxt_qplib_set_sq_size(sq, qp->wqe_mode); /* Update msn tbl size */ - if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) { + if (qp->is_host_msn_tbl && psn_sz) { hwq_attr.aux_depth = roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, qp->wqe_mode)); qp->msn_tbl_sz = hwq_attr.aux_depth; qp->msn = 0; @@ -1636,7 +1636,7 @@ static void bnxt_qplib_fill_psn_search(struct bnxt_qplib_qp *qp, if (!swq->psn_search) return; /* Handle MSN differently on cap flags */ - if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { + if (qp->is_host_msn_tbl) { bnxt_qplib_fill_msn_search(qp, wqe, swq); return; } @@ -1818,7 +1818,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, } swq = bnxt_qplib_get_swqe(sq, &wqe_idx); - bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags)); + bnxt_qplib_pull_psn_buff(qp, sq, swq, qp->is_host_msn_tbl); idx = 0; swq->slot_idx = hwq->prod; @@ -2008,7 +2008,7 @@ int bnxt_qplib_post_send(struct bnxt_qplib_qp *qp, rc = -EINVAL; goto done; } - if (!BNXT_RE_HW_RETX(qp->dev_cap_flags) || msn_update) { + if (!qp->is_host_msn_tbl || msn_update) { swq->next_psn = sq->psn & BTH_PSN_MASK; bnxt_qplib_fill_psn_search(qp, wqe, swq); } diff --git a/drivers/infiniband/hw/bnxt_re/qplib_fp.h b/drivers/infiniband/hw/bnxt_re/qplib_fp.h index 7fd4506..5b8d097 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_fp.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_fp.h @@ -340,7 +340,7 @@ struct bnxt_qplib_qp { struct list_head rq_flush; u32 msn; u32 msn_tbl_sz; - u16 dev_cap_flags; + u16 is_host_msn_tbl; }; #define BNXT_QPLIB_MAX_CQE_ENTRY_SIZE sizeof(struct cq_base) diff --git a/drivers/infiniband/hw/bnxt_re/qplib_res.h b/drivers/infiniband/hw/bnxt_re/qplib_res.h index 61628f7..a0f78cd 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_res.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_res.h @@ -554,6 +554,12 @@ static inline bool _is_hw_retx_supported(u16 dev_cap_flags) #define BNXT_RE_HW_RETX(a) _is_hw_retx_supported((a)) +static inline bool _is_host_msn_table(u16 dev_cap_ext_flags2) +{ + return (dev_cap_ext_flags2 & CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_MASK) == + CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_MSN_TABLE; +} + static inline u8 bnxt_qplib_dbr_pacing_en(struct bnxt_qplib_chip_ctx *cctx) { return cctx->modes.dbr_pacing; diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.c b/drivers/infiniband/hw/bnxt_re/qplib_sp.c index 8beeedd..9328db9 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.c +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.c @@ -156,6 +156,7 @@ int bnxt_qplib_get_dev_attr(struct bnxt_qplib_rcfw *rcfw, (0x01 << RCFW_DBR_BASE_PAGE_SHIFT); attr->max_sgid = BNXT_QPLIB_NUM_GIDS_SUPPORTED; attr->dev_cap_flags = le16_to_cpu(sb->dev_cap_flags); + attr->dev_cap_flags2 = le16_to_cpu(sb->dev_cap_ext_flags_2); bnxt_qplib_query_version(rcfw, attr->fw_ver); diff --git a/drivers/infiniband/hw/bnxt_re/qplib_sp.h b/drivers/infiniband/hw/bnxt_re/qplib_sp.h index d33c78b..16a67d7 100644 --- a/drivers/infiniband/hw/bnxt_re/qplib_sp.h +++ b/drivers/infiniband/hw/bnxt_re/qplib_sp.h @@ -72,6 +72,7 @@ struct bnxt_qplib_dev_attr { u8 tqm_alloc_reqs[MAX_TQM_ALLOC_REQ]; bool is_atomic; u16 dev_cap_flags; + u16 dev_cap_flags2; u32 max_dpi; }; diff --git a/drivers/infiniband/hw/bnxt_re/roce_hsi.h b/drivers/infiniband/hw/bnxt_re/roce_hsi.h index 605c946..0425309 100644 --- a/drivers/infiniband/hw/bnxt_re/roce_hsi.h +++ b/drivers/infiniband/hw/bnxt_re/roce_hsi.h @@ -2157,8 +2157,36 @@ struct creq_query_func_resp_sb { __le32 tqm_alloc_reqs[12]; __le32 max_dpi; u8 max_sge_var_wqe; - u8 reserved_8; + u8 dev_cap_ext_flags; + #define CREQ_QUERY_FUNC_RESP_SB_ATOMIC_OPS_NOT_SUPPORTED 0x1UL + #define CREQ_QUERY_FUNC_RESP_SB_DRV_VERSION_RGTR_SUPPORTED 0x2UL + #define CREQ_QUERY_FUNC_RESP_SB_CREATE_QP_BATCH_SUPPORTED 0x4UL + #define CREQ_QUERY_FUNC_RESP_SB_DESTROY_QP_BATCH_SUPPORTED 0x8UL + #define CREQ_QUERY_FUNC_RESP_SB_ROCE_STATS_EXT_CTX_SUPPORTED 0x10UL + #define CREQ_QUERY_FUNC_RESP_SB_CREATE_SRQ_SGE_SUPPORTED 0x20UL + #define CREQ_QUERY_FUNC_RESP_SB_FIXED_SIZE_WQE_DISABLED 0x40UL + #define CREQ_QUERY_FUNC_RESP_SB_DCN_SUPPORTED 0x80UL __le16 max_inline_data_var_wqe; + __le32 start_qid; + u8 max_msn_table_size; + u8 reserved8_1; + __le16 dev_cap_ext_flags_2; + #define CREQ_QUERY_FUNC_RESP_SB_OPTIMIZE_MODIFY_QP_SUPPORTED 0x1UL + #define CREQ_QUERY_FUNC_RESP_SB_CHANGE_UDP_SRC_PORT_WQE_SUPPORTED 0x2UL + #define CREQ_QUERY_FUNC_RESP_SB_CQ_COALESCING_SUPPORTED 0x4UL + #define CREQ_QUERY_FUNC_RESP_SB_MEMORY_REGION_RO_SUPPORTED 0x8UL + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_MASK 0x30UL + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_SFT 4 + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_PSN_TABLE (0x0UL << 4) + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_HOST_MSN_TABLE (0x1UL << 4) + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_IQM_MSN_TABLE (0x2UL << 4) + #define CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_LAST \ + CREQ_QUERY_FUNC_RESP_SB_REQ_RETRANSMISSION_SUPPORT_IQM_MSN_TABLE + __le16 max_xp_qp_size; + __le16 create_qp_batch_size; + __le16 destroy_qp_batch_size; + __le16 reserved16; + __le64 reserved64; }; /* cmdq_set_func_resources (size:448b/56B) */
FW reports the HW capability to use PSN table or MSN table and driver/library need to select it based on this capability. Use the new capability instead of the older capability check for HW retransmission while handling the MSN/PSN table. Also, Updated the FW interface structures to handle the new fields. Signed-off-by: Selvin Xavier <selvin.xavier@broadcom.com> --- drivers/infiniband/hw/bnxt_re/qplib_fp.c | 12 ++++++------ drivers/infiniband/hw/bnxt_re/qplib_fp.h | 2 +- drivers/infiniband/hw/bnxt_re/qplib_res.h | 6 ++++++ drivers/infiniband/hw/bnxt_re/qplib_sp.c | 1 + drivers/infiniband/hw/bnxt_re/qplib_sp.h | 1 + drivers/infiniband/hw/bnxt_re/roce_hsi.h | 30 +++++++++++++++++++++++++++++- 6 files changed, 44 insertions(+), 8 deletions(-)