From patchwork Thu Jun 25 15:39:23 2015 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Steve Wise X-Patchwork-Id: 6676271 Return-Path: X-Original-To: patchwork-linux-rdma@patchwork.kernel.org Delivered-To: patchwork-parsemail@patchwork1.web.kernel.org Received: from mail.kernel.org (mail.kernel.org [198.145.29.136]) by patchwork1.web.kernel.org (Postfix) with ESMTP id DB2FF9F39B for ; Thu, 25 Jun 2015 15:39:26 +0000 (UTC) Received: from mail.kernel.org (localhost [127.0.0.1]) by mail.kernel.org (Postfix) with ESMTP id C4FDC206E8 for ; Thu, 25 Jun 2015 15:39:25 +0000 (UTC) Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.kernel.org (Postfix) with ESMTP id BB3E5206DC for ; Thu, 25 Jun 2015 15:39:24 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751469AbbFYPjY (ORCPT ); Thu, 25 Jun 2015 11:39:24 -0400 Received: from smtp.opengridcomputing.com ([72.48.136.20]:41172 "EHLO smtp.opengridcomputing.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752753AbbFYPjX (ORCPT ); Thu, 25 Jun 2015 11:39:23 -0400 Received: from build.ogc.int (build.ogc.int [10.10.0.2]) by smtp.opengridcomputing.com (Postfix) with ESMTP id 1B74929E7B; Thu, 25 Jun 2015 10:39:23 -0500 (CDT) Subject: [PATCH RFC 2/2] RDMA/isert: Support iWARP transport From: Steve Wise To: linux-rdma@vger.kernel.org Cc: sagig@mellanox.com, orgerlitz@mellanox.com, raid@mellanox.com Date: Thu, 25 Jun 2015 10:39:23 -0500 Message-ID: <20150625153922.13272.41789.stgit@build.ogc.int> In-Reply-To: <20150625153754.13272.432.stgit@build.ogc.int> References: <20150625153754.13272.432.stgit@build.ogc.int> User-Agent: StGit/0.17-dirty MIME-Version: 1.0 Sender: linux-rdma-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-rdma@vger.kernel.org X-Spam-Status: No, score=-8.3 required=5.0 tests=BAYES_00, RCVD_IN_DNSWL_HI, RP_MATCHES_RCVD, UNPARSEABLE_RELAY autolearn=ham version=3.3.1 X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on mail.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP Memory regions that are the target of an iWARP RDMA READ RESPONSE need REMOTE_WRITE access rights. So enable REMOTE_WRITE for iWARP devices. iWARP RDMA READ target sge depth is 1. So save the max_read_sge in the target device structure and use that when creating RDMA_READ work requests. Signed-off-by: Steve Wise Tested-by: Vasu Dev --- drivers/infiniband/ulp/isert/ib_isert.c | 55 ++++++++++++++++++++++++++----- drivers/infiniband/ulp/isert/ib_isert.h | 1 + 2 files changed, 48 insertions(+), 8 deletions(-) -- To unsubscribe from this list: send the line "unsubscribe linux-rdma" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c index 9e7b492..b5cde5d 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.c +++ b/drivers/infiniband/ulp/isert/ib_isert.c @@ -165,6 +165,11 @@ isert_create_qp(struct isert_conn *isert_conn, attr.cap.max_send_sge = max(2, device->dev_attr.max_sge - 2); isert_conn->max_sge = attr.cap.max_send_sge; + if (rdma_cap_read_multi_sge(device->ib_device, cma_id->port_num)) + isert_conn->max_read_sge = isert_conn->max_sge; + else + isert_conn->max_read_sge = 1; + attr.cap.max_recv_sge = 1; attr.sq_sig_type = IB_SIGNAL_REQ_WR; attr.qp_type = IB_QPT_RC; @@ -348,6 +353,17 @@ out_cq: return ret; } +static int any_port_is_iwarp(struct isert_device *device) +{ + int i; + + for (i = rdma_start_port(device->ib_device); + i <= rdma_end_port(device->ib_device); i++) + if (rdma_protocol_iwarp(device->ib_device, i)) + return 1; + return 0; +} + static int isert_create_device_ib_res(struct isert_device *device) { @@ -383,7 +399,17 @@ isert_create_device_ib_res(struct isert_device *device) goto out_cq; } - device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE); + /* + * IWARP transports need REMOTE_WRITE for MRs used as the target of + * an RDMA_READ. Since the DMA MR is used for all ports, then if + * any port is running IWARP, add REMOTE_WRITE. + */ + if (any_port_is_iwarp(device)) + device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE | + IB_ACCESS_REMOTE_WRITE); + else + device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(device->mr)) { ret = PTR_ERR(device->mr); isert_err("failed to create dma mr, device %p, ret=%d\n", @@ -2375,7 +2401,7 @@ isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) static int isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, struct ib_sge *ib_sge, struct ib_send_wr *send_wr, - u32 data_left, u32 offset) + u32 data_left, u32 offset, u32 max_sge) { struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; struct scatterlist *sg_start, *tmp_sg; @@ -2386,7 +2412,7 @@ isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, sg_off = offset / PAGE_SIZE; sg_start = &cmd->se_cmd.t_data_sg[sg_off]; - sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); + sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, max_sge); page_off = offset % PAGE_SIZE; send_wr->sg_list = ib_sge; @@ -2430,8 +2456,9 @@ isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, struct isert_data_buf *data = &wr->data; struct ib_send_wr *send_wr; struct ib_sge *ib_sge; - u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; + u32 offset, data_len, data_left, rdma_max_len, va_offset = 0; int ret = 0, i, ib_sge_cnt; + u32 max_sge; isert_cmd->tx_desc.isert_cmd = isert_cmd; @@ -2453,7 +2480,12 @@ isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, } wr->ib_sge = ib_sge; - wr->send_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); + if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) + max_sge = isert_conn->max_sge; + else + max_sge = isert_conn->max_read_sge; + + wr->send_wr_num = DIV_ROUND_UP(data->nents, max_sge); wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, GFP_KERNEL); if (!wr->send_wr) { @@ -2463,11 +2495,11 @@ isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, } wr->isert_cmd = isert_cmd; - rdma_write_max = isert_conn->max_sge * PAGE_SIZE; + rdma_max_len = max_sge * PAGE_SIZE; for (i = 0; i < wr->send_wr_num; i++) { send_wr = &isert_cmd->rdma_wr.send_wr[i]; - data_len = min(data_left, rdma_write_max); + data_len = min(data_left, rdma_max_len); send_wr->send_flags = 0; if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { @@ -2489,7 +2521,7 @@ isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, } ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, - send_wr, data_len, offset); + send_wr, data_len, offset, max_sge); ib_sge += ib_sge_cnt; offset += data_len; @@ -2618,6 +2650,13 @@ isert_fast_reg_mr(struct isert_conn *isert_conn, fr_wr.wr.fast_reg.rkey = mr->rkey; fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE; + /* + * IWARP transports need REMOTE_WRITE for MRs used as the target of + * an RDMA_READ. + */ + if (rdma_protocol_iwarp(ib_dev, isert_conn->cm_id->port_num)) + fr_wr.wr.fast_reg.access_flags |= IB_ACCESS_REMOTE_WRITE; + if (!wr) wr = &fr_wr; else diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h index 9ec23a7..47cf11b 100644 --- a/drivers/infiniband/ulp/isert/ib_isert.h +++ b/drivers/infiniband/ulp/isert/ib_isert.h @@ -153,6 +153,7 @@ struct isert_conn { u32 initiator_depth; bool pi_support; u32 max_sge; + u32 max_read_sge; char *login_buf; char *login_req_buf; char *login_rsp_buf;