From patchwork Mon Jan 11 12:21:46 2010 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Or Gerlitz X-Patchwork-Id: 72135 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by demeter.kernel.org (8.14.3/8.14.2) with ESMTP id o0BCLbFV027273 for ; Mon, 11 Jan 2010 12:21:37 GMT Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751888Ab0AKMVh (ORCPT ); Mon, 11 Jan 2010 07:21:37 -0500 Received: (majordomo@vger.kernel.org) by vger.kernel.org id S1752827Ab0AKMVg (ORCPT ); Mon, 11 Jan 2010 07:21:36 -0500 Received: from fwil.voltaire.com ([193.47.165.2]:39360 "EHLO exil.voltaire.com" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1751888Ab0AKMVg (ORCPT ); Mon, 11 Jan 2010 07:21:36 -0500 Received: from zuben.voltaire.com ([172.25.5.15]) by exil.voltaire.com with Microsoft SMTPSVC(6.0.3790.3959); Mon, 11 Jan 2010 14:21:33 +0200 Message-ID: <4B4B17DA.2070106@Voltaire.com> Date: Mon, 11 Jan 2010 14:21:46 +0200 From: Or Gerlitz User-Agent: Thunderbird 2.0.0.19 (X11/20081209) MIME-Version: 1.0 To: Jack Morgenstein CC: =?ISO-8859-1?Q?H=E5kon_Bugge?= , linux-rdma Subject: Re: RDMA Read sge errors References: <2AAC14CC-DACE-4CD8-833A-5E7406593732@Sun.com> In-Reply-To: <2AAC14CC-DACE-4CD8-833A-5E7406593732@Sun.com> X-OriginalArrivalTime: 11 Jan 2010 12:21:33.0120 (UTC) FILETIME=[9C86D400:01CA92B8] Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org Index: ofed_kernel/drivers/infiniband/hw/mlx4/main.c =================================================================== --- ofed_kernel.orig/drivers/infiniband/hw/mlx4/main.c +++ ofed_kernel/drivers/infiniband/hw/mlx4/main.c @@ -122,7 +122,7 @@ static int mlx4_ib_query_device(struct i props->max_mr_size = ~0ull; props->page_size_cap = dev->dev->caps.page_size_cap; props->max_qp = dev->dev->caps.num_qps - dev->dev->caps.reserved_qps; - props->max_qp_wr = dev->dev->caps.max_wqes; + props->max_qp_wr = dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE; props->max_sge = min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg); props->max_cq = dev->dev->caps.num_cqs - dev->dev->caps.reserved_cqs; Index: ofed_kernel/drivers/infiniband/hw/mlx4/mlx4_ib.h =================================================================== --- ofed_kernel.orig/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ ofed_kernel/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -44,6 +44,13 @@ #include #include +enum { + MLX4_IB_SQ_MIN_WQE_SHIFT = 6 +}; + +#define MLX4_IB_SQ_HEADROOM(shift) ((2048 >> (shift)) + 1) +#define MLX4_IB_SQ_MAX_SPARE (MLX4_IB_SQ_HEADROOM(MLX4_IB_SQ_MIN_WQE_SHIFT)) + struct mlx4_ib_ucontext { struct ib_ucontext ibucontext; struct mlx4_uar uar; Index: ofed_kernel/drivers/infiniband/hw/mlx4/qp.c =================================================================== --- ofed_kernel.orig/drivers/infiniband/hw/mlx4/qp.c +++ ofed_kernel/drivers/infiniband/hw/mlx4/qp.c @@ -289,8 +289,9 @@ static int set_rq_size(struct mlx4_ib_de int is_user, int has_srq, struct mlx4_ib_qp *qp) { /* Sanity check RQ size before proceeding */ - if (cap->max_recv_wr > dev->dev->caps.max_wqes || - cap->max_recv_sge > dev->dev->caps.max_rq_sg) + if (cap->max_recv_wr > dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE || + cap->max_recv_sge > + min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg)) return -EINVAL; if (has_srq) { @@ -309,8 +310,19 @@ static int set_rq_size(struct mlx4_ib_de qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg)); } - cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt; - cap->max_recv_sge = qp->rq.max_gs; + /* leave userspace return values as they were, so as not to break ABI */ + if (is_user) { + cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt; + cap->max_recv_sge = qp->rq.max_gs; + } else { + cap->max_recv_wr = qp->rq.max_post = + min(dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE, qp->rq.wqe_cnt); + cap->max_recv_sge = min(qp->rq.max_gs, + min(dev->dev->caps.max_sq_sg, + dev->dev->caps.max_rq_sg)); + } + /* We don't support inline sends for kernel QPs (yet) */ + return 0; } @@ -321,8 +333,9 @@ static int set_kernel_sq_size(struct mlx int s; /* Sanity check SQ size before proceeding */ - if (cap->max_send_wr > dev->dev->caps.max_wqes || - cap->max_send_sge > dev->dev->caps.max_sq_sg || + if (cap->max_send_wr > (dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE) || + cap->max_send_sge > + min(dev->dev->caps.max_sq_sg, dev->dev->caps.max_rq_sg) || cap->max_inline_data + send_wqe_overhead(type, qp->flags) + sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz) return -EINVAL;