From patchwork Tue Jul 25 20:57:49 2017 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Logan Gunthorpe X-Patchwork-Id: 9863603 X-Patchwork-Delegate: bhelgaas@google.com Return-Path: Received: from mail.wl.linuxfoundation.org (pdx-wl-mail.web.codeaurora.org [172.30.200.125]) by pdx-korg-patchwork.web.codeaurora.org (Postfix) with ESMTP id E01ED603F9 for ; Tue, 25 Jul 2017 21:00:07 +0000 (UTC) Received: from mail.wl.linuxfoundation.org (localhost [127.0.0.1]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id C9BB528700 for ; Tue, 25 Jul 2017 21:00:07 +0000 (UTC) Received: by mail.wl.linuxfoundation.org (Postfix, from userid 486) id BE26428712; Tue, 25 Jul 2017 21:00:07 +0000 (UTC) X-Spam-Checker-Version: SpamAssassin 3.3.1 (2010-03-16) on pdx-wl-mail.web.codeaurora.org X-Spam-Level: X-Spam-Status: No, score=-6.9 required=2.0 tests=BAYES_00,RCVD_IN_DNSWL_HI autolearn=unavailable version=3.3.1 Received: from vger.kernel.org (vger.kernel.org [209.132.180.67]) by mail.wl.linuxfoundation.org (Postfix) with ESMTP id C135B28700 for ; Tue, 25 Jul 2017 21:00:06 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751551AbdGYU7N (ORCPT ); Tue, 25 Jul 2017 16:59:13 -0400 Received: from ale.deltatee.com ([207.54.116.67]:36738 "EHLO ale.deltatee.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752849AbdGYU6N (ORCPT ); Tue, 25 Jul 2017 16:58:13 -0400 Received: from cgy1-donard.priv.deltatee.com ([172.16.1.31]) by ale.deltatee.com with esmtps (TLS1.2:ECDHE_RSA_AES_256_GCM_SHA384:256) (Exim 4.84_2) (envelope-from ) id 1da6tu-0006jC-R8; Tue, 25 Jul 2017 14:58:04 -0600 Received: from gunthorp by cgy1-donard.priv.deltatee.com with local (Exim 4.89) (envelope-from ) id 1da6tw-0001Ft-At; Tue, 25 Jul 2017 14:57:56 -0600 From: Logan Gunthorpe To: linux-ntb@googlegroups.com, linux-pci@vger.kernel.org, linux-kernel@vger.kernel.org Cc: Jon Mason , Dave Jiang , Allen Hubbe , Bjorn Helgaas , Greg Kroah-Hartman , Kurt Schwemmer , Stephen Bates , Serge Semin , Logan Gunthorpe Date: Tue, 25 Jul 2017 14:57:49 -0600 Message-Id: <20170725205753.4735-13-logang@deltatee.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20170725205753.4735-1-logang@deltatee.com> References: <20170725205753.4735-1-logang@deltatee.com> X-SA-Exim-Connect-IP: 172.16.1.31 X-SA-Exim-Rcpt-To: linux-ntb@googlegroups.com, linux-pci@vger.kernel.org, linux-kernel@vger.kernel.org, jdmason@kudzu.us, dave.jiang@intel.com, Allen.Hubbe@emc.com, bhelgaas@google.com, gregkh@linuxfoundation.org, kurt.schwemmer@microsemi.com, sbates@raithlin.com, fancer.lancer@gmail.com, logang@deltatee.com X-SA-Exim-Mail-From: gunthorp@deltatee.com Subject: [PATCH v3 12/16] switchtec_ntb: add link management X-SA-Exim-Version: 4.2.1 (built Mon, 26 Dec 2011 16:24:06 +0000) X-SA-Exim-Scanned: Yes (on ale.deltatee.com) Sender: linux-pci-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-pci@vger.kernel.org X-Virus-Scanned: ClamAV using ClamSMTP switchtec_ntb checks for a link by looking at the shared memory window. If the magic number is correct and the otherside indicates their link is enabled then we take the link to be up. Whenever we change our local link status we send a msg to the otherside to check whether it's up and change their status. The current status is maintained in a flag so ntb_is_link_up can return quickly. We utilize Switchtec's link status notifier to also check link changes when the switch notices a port changes state. Signed-off-by: Logan Gunthorpe Reviewed-by: Stephen Bates Reviewed-by: Kurt Schwemmer --- drivers/ntb/hw/mscc/switchtec_ntb.c | 130 +++++++++++++++++++++++++++++++++++- 1 file changed, 129 insertions(+), 1 deletion(-) diff --git a/drivers/ntb/hw/mscc/switchtec_ntb.c b/drivers/ntb/hw/mscc/switchtec_ntb.c index f87c0eef59ef..34d69fa8612d 100644 --- a/drivers/ntb/hw/mscc/switchtec_ntb.c +++ b/drivers/ntb/hw/mscc/switchtec_ntb.c @@ -64,6 +64,7 @@ static inline void _iowrite64(u64 val, void __iomem *mmio) struct shared_mw { u32 magic; + u32 link_sta; u32 partition_id; u64 mw_sizes[MAX_MWS]; }; @@ -104,8 +105,17 @@ struct switchtec_ntb { int peer_nr_direct_mw; int peer_nr_lut_mw; int peer_direct_mw_to_bar[MAX_DIRECT_MW]; + + bool link_is_up; + enum ntb_speed link_speed; + enum ntb_width link_width; }; +static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb) +{ + return container_of(ntb, struct switchtec_ntb, ntb); +} + static int switchtec_ntb_part_op(struct switchtec_ntb *sndev, struct ntb_ctrl_regs __iomem *ctl, u32 op, int wait_status) @@ -163,6 +173,17 @@ static int switchtec_ntb_part_op(struct switchtec_ntb *sndev, return -EIO; } +static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx, + u32 val) +{ + if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_self_dbmsg->omsg)) + return -EINVAL; + + iowrite32(val, &sndev->mmio_self_dbmsg->omsg[idx].msg); + + return 0; +} + static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx) { return 0; @@ -194,22 +215,124 @@ static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, return 0; } +static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev, + int partition, + enum ntb_speed *speed, + enum ntb_width *width) +{ + struct switchtec_dev *stdev = sndev->stdev; + + u32 pff = ioread32(&stdev->mmio_part_cfg[partition].vep_pff_inst_id); + u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]); + + if (speed) + *speed = (linksta >> 16) & 0xF; + + if (width) + *width = (linksta >> 20) & 0x3F; +} + +static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev) +{ + enum ntb_speed self_speed, peer_speed; + enum ntb_width self_width, peer_width; + + if (!sndev->link_is_up) { + sndev->link_speed = NTB_SPEED_NONE; + sndev->link_width = NTB_WIDTH_NONE; + return; + } + + switchtec_ntb_part_link_speed(sndev, sndev->self_partition, + &self_speed, &self_width); + switchtec_ntb_part_link_speed(sndev, sndev->peer_partition, + &peer_speed, &peer_width); + + sndev->link_speed = min(self_speed, peer_speed); + sndev->link_width = min(self_width, peer_width); +} + +enum { + LINK_MESSAGE = 0, + MSG_LINK_UP = 1, + MSG_LINK_DOWN = 2, + MSG_CHECK_LINK = 3, +}; + +static void switchtec_ntb_check_link(struct switchtec_ntb *sndev) +{ + int link_sta; + int old = sndev->link_is_up; + + link_sta = sndev->self_shared->link_sta; + if (link_sta) { + u64 peer = ioread64(&sndev->peer_shared->magic); + + if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC) + link_sta = peer >> 32; + else + link_sta = 0; + } + + sndev->link_is_up = link_sta; + switchtec_ntb_set_link_speed(sndev); + + if (link_sta != old) { + switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK); + ntb_link_event(&sndev->ntb); + dev_info(&sndev->stdev->dev, "ntb link %s", + link_sta ? "up" : "down"); + } +} + +static void switchtec_ntb_link_notification(struct switchtec_dev *stdev) +{ + struct switchtec_ntb *sndev = stdev->sndev; + + switchtec_ntb_check_link(sndev); +} + static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed, enum ntb_width *width) { - return 0; + struct switchtec_ntb *sndev = ntb_sndev(ntb); + + if (speed) + *speed = sndev->link_speed; + if (width) + *width = sndev->link_width; + + return sndev->link_is_up; } static int switchtec_ntb_link_enable(struct ntb_dev *ntb, enum ntb_speed max_speed, enum ntb_width max_width) { + struct switchtec_ntb *sndev = ntb_sndev(ntb); + + dev_dbg(&sndev->stdev->dev, "enabling link"); + + sndev->self_shared->link_sta = 1; + switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP); + + switchtec_ntb_check_link(sndev); + return 0; } static int switchtec_ntb_link_disable(struct ntb_dev *ntb) { + struct switchtec_ntb *sndev = ntb_sndev(ntb); + + dev_dbg(&sndev->stdev->dev, "disabling link"); + + sndev->self_shared->link_sta = 0; + switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP); + + switchtec_ntb_check_link(sndev); + return 0; } @@ -572,6 +695,9 @@ static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev) dev_dbg(&sndev->stdev->dev, "message: %d %08x\n", i, (u32)msg); iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status); + + if (i == LINK_MESSAGE) + switchtec_ntb_check_link(sndev); } } @@ -674,6 +800,7 @@ static int switchtec_ntb_add(struct device *dev, goto deinit_and_exit; stdev->sndev = sndev; + stdev->link_notifier = switchtec_ntb_link_notification; dev_info(dev, "NTB device registered"); return 0; @@ -697,6 +824,7 @@ void switchtec_ntb_remove(struct device *dev, if (!sndev) return; + stdev->link_notifier = NULL; stdev->sndev = NULL; ntb_unregister_device(&sndev->ntb); switchtec_ntb_deinit_db_msg_irq(sndev);